diff options
author | sos <sos@FreeBSD.org> | 2008-04-17 12:29:35 +0000 |
---|---|---|
committer | sos <sos@FreeBSD.org> | 2008-04-17 12:29:35 +0000 |
commit | b11f9dd52aac5645107b2d8cb24361e9495b17d6 (patch) | |
tree | b12f34c4c9498ff9f554b3fab04f8ac66236b802 /sys/dev/ata | |
parent | 3f4fde5950af141f4f874f9172842398de81e11d (diff) | |
download | FreeBSD-src-b11f9dd52aac5645107b2d8cb24361e9495b17d6.zip FreeBSD-src-b11f9dd52aac5645107b2d8cb24361e9495b17d6.tar.gz |
Go back to preallocating everything possible on init.
This avoids calling busdma in the request processing path which caused a traumatic performance degradation.
Allocation has be postponed to after we know how many devices we possible can have on portmulitpliers to save some space.
Diffstat (limited to 'sys/dev/ata')
-rw-r--r-- | sys/dev/ata/ata-all.c | 65 | ||||
-rw-r--r-- | sys/dev/ata/ata-all.h | 45 | ||||
-rw-r--r-- | sys/dev/ata/ata-chipset.c | 56 | ||||
-rw-r--r-- | sys/dev/ata/ata-disk.c | 9 | ||||
-rw-r--r-- | sys/dev/ata/ata-dma.c | 198 | ||||
-rw-r--r-- | sys/dev/ata/ata-lowlevel.c | 8 | ||||
-rw-r--r-- | sys/dev/ata/ata-pci.c | 16 | ||||
-rw-r--r-- | sys/dev/ata/ata-queue.c | 10 | ||||
-rw-r--r-- | sys/dev/ata/ata-raid.c | 6 | ||||
-rw-r--r-- | sys/dev/ata/atapi-cam.c | 3 | ||||
-rw-r--r-- | sys/dev/ata/atapi-cd.c | 9 | ||||
-rw-r--r-- | sys/dev/ata/atapi-fd.c | 3 | ||||
-rw-r--r-- | sys/dev/ata/atapi-tape.c | 3 |
13 files changed, 243 insertions, 188 deletions
diff --git a/sys/dev/ata/ata-all.c b/sys/dev/ata/ata-all.c index 0110c23..87f4927 100644 --- a/sys/dev/ata/ata-all.c +++ b/sys/dev/ata/ata-all.c @@ -71,12 +71,12 @@ MALLOC_DEFINE(M_ATA, "ata_generic", "ATA driver generic layer"); int (*ata_raid_ioctl_func)(u_long cmd, caddr_t data) = NULL; struct intr_config_hook *ata_delayed_attach = NULL; devclass_t ata_devclass; +uma_zone_t ata_request_zone; uma_zone_t ata_composite_zone; int ata_wc = 1; int ata_setmax = 0; /* local vars */ -static uma_zone_t ata_request_zone; static int ata_dma = 1; static int atapi_dma = 1; @@ -451,10 +451,11 @@ ata_device_ioctl(device_t dev, u_long cmd, caddr_t data) if (!(buf = malloc(ioc_request->count, M_ATA, M_NOWAIT))) { return ENOMEM; } - if (!(request = ata_alloc_request(dev))) { + if (!(request = ata_alloc_request())) { free(buf, M_ATA); return ENOMEM; } + request->dev = atadev->dev; if (ioc_request->flags & ATA_CMD_WRITE) { error = copyin(ioc_request->data, buf, ioc_request->count); if (error) { @@ -587,8 +588,9 @@ ata_getparam(struct ata_device *atadev, int init) return ENXIO; while (retries-- > 0 && error) { - if (!(request = ata_alloc_request(atadev->dev))) + if (!(request = ata_alloc_request())) break; + request->dev = atadev->dev; request->timeout = 1; request->retries = 0; request->u.ata.command = command; @@ -682,7 +684,8 @@ ata_identify(device_t dev) } devices[i]->unit = i; #ifdef ATA_STATIC_ID - unit = (device_get_unit(dev) << 1) + i; + if (ch->devices & ((ATA_ATA_MASTER << i))) + unit = (device_get_unit(dev) << 1) + i; #endif if (!(childdevs[i] = ata_add_child(dev, devices[i], unit))) { free(devices[i], M_ATA); @@ -1002,60 +1005,6 @@ bpack(int8_t *src, int8_t *dst, int len) dst[j] = 0x00; } -struct ata_request * -ata_alloc_request(device_t dev) -{ - struct ata_channel *ch = device_get_softc(device_get_parent(dev)); - struct ata_request *request; - - if ((request = uma_zalloc(ata_request_zone, M_NOWAIT | M_ZERO))) { - if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, PAGE_SIZE, - ch->dma.max_address, BUS_SPACE_MAXADDR, - NULL, NULL, PAGE_SIZE, 1, PAGE_SIZE, - 0, NULL, NULL, &request->dma.sg_tag)) { - device_printf(ch->dev, "FAILURE - create sg_tag\n"); - uma_zfree(ata_request_zone, request); - return NULL; - } - if (bus_dmamem_alloc(request->dma.sg_tag, (void **)&request->dma.sg, 0, - &request->dma.sg_map)) { - device_printf(ch->dev, "FAILURE - alloc sg_map\n"); - bus_dma_tag_destroy(request->dma.sg_tag); - uma_zfree(ata_request_zone, request); - return NULL; - } - if (bus_dma_tag_create(ch->dma.dmatag, - ch->dma.alignment, ch->dma.boundary, - ch->dma.max_address, BUS_SPACE_MAXADDR, - NULL, NULL, ch->dma.max_iosize, - ATA_DMA_ENTRIES, ch->dma.segsize, - BUS_DMA_ALLOCNOW, NULL, NULL, - &request->dma.data_tag)) { - device_printf(ch->dev, "FAILURE - create data_tag\n"); - bus_dmamem_free(request->dma.sg_tag, request->dma.sg, - request->dma.sg_map); - bus_dma_tag_destroy(request->dma.sg_tag); - uma_zfree(ata_request_zone, request); - return NULL; - } - request->dev = dev; - } - else - device_printf(dev, "FAILURE - ata_alloc_request\n"); - return request; -} - -void -ata_free_request(struct ata_request *request) -{ - if (!(request->flags & ATA_R_DANGER2)) { - if (request->dma.data_tag) - bus_dma_tag_destroy(request->dma.data_tag); - if (request->dma.sg_tag) - bus_dma_tag_destroy(request->dma.sg_tag); - uma_zfree(ata_request_zone, request); - } -} /* * module handeling diff --git a/sys/dev/ata/ata-all.h b/sys/dev/ata/ata-all.h index 4ad8701..4bccc4d 100644 --- a/sys/dev/ata/ata-all.h +++ b/sys/dev/ata/ata-all.h @@ -226,10 +226,9 @@ #define ATA_AHCI_CL_SIZE 32 #define ATA_AHCI_CL_OFFSET 0 -#define ATA_AHCI_FB_OFFSET 1024 -#define ATA_AHCI_CT_OFFSET 1024+4096 -#define ATA_AHCI_CT_SG_OFFSET 128 -#define ATA_AHCI_CT_SIZE 256 +#define ATA_AHCI_FB_OFFSET (ATA_AHCI_CL_SIZE * 32) +#define ATA_AHCI_CT_OFFSET (ATA_AHCI_FB_OFFSET + 4096) +#define ATA_AHCI_CT_SIZE (1024 + 128) struct ata_ahci_dma_prd { u_int64_t dba; @@ -243,7 +242,8 @@ struct ata_ahci_cmd_tab { u_int8_t cfis[64]; u_int8_t acmd[32]; u_int8_t reserved[32]; - struct ata_ahci_dma_prd prd_tab[16]; +#define ATA_AHCI_DMA_ENTRIES 64 + struct ata_ahci_dma_prd prd_tab[ATA_AHCI_DMA_ENTRIES]; } __packed; struct ata_ahci_cmd_list { @@ -373,18 +373,9 @@ struct ata_request { #define ATA_R_DANGER1 0x20000000 #define ATA_R_DANGER2 0x40000000 + struct ata_dmaslot *dma; /* DMA slot of this request */ u_int8_t status; /* ATA status */ u_int8_t error; /* ATA error */ - struct { - u_int8_t status; /* DMA status */ - bus_dma_tag_t sg_tag; /* SG list DMA tag */ - bus_dmamap_t sg_map; /* SG list DMA map */ - void *sg; /* DMA transfer table */ - bus_addr_t sg_bus; /* bus address of dmatab */ - bus_dma_tag_t data_tag; /* data DMA tag */ - bus_dmamap_t data_map; /* data DMA map */ - u_int32_t cur_iosize; /* DMA data current IO size */ - } dma; u_int32_t donecount; /* bytes transferred */ int result; /* result error code */ void (*callback)(struct ata_request *request); @@ -447,6 +438,16 @@ struct ata_dmasetprd_args { int error; }; +struct ata_dmaslot { + u_int8_t status; /* DMA status */ + bus_dma_tag_t sg_tag; /* SG list DMA tag */ + bus_dmamap_t sg_map; /* SG list DMA map */ + void *sg; /* DMA transfer table */ + bus_addr_t sg_bus; /* bus address of dmatab */ + bus_dma_tag_t data_tag; /* data DMA tag */ + bus_dmamap_t data_map; /* data DMA map */ +}; + /* structure holding DMA related information */ struct ata_dma { bus_dma_tag_t dmatag; /* parent DMA tag */ @@ -454,6 +455,10 @@ struct ata_dma { bus_dmamap_t work_map; /* workspace DMA map */ u_int8_t *work; /* workspace */ bus_addr_t work_bus; /* bus address of dmatab */ + +#define ATA_DMA_SLOTS 32 + int dma_slots; /* DMA slots allocated */ + struct ata_dmaslot slot[ATA_DMA_SLOTS]; u_int32_t alignment; /* DMA SG list alignment */ u_int32_t boundary; /* DMA SG list boundary */ u_int32_t segsize; /* DMA SG list segment size */ @@ -499,7 +504,7 @@ struct ata_channel { struct resource *r_irq; /* interrupt of this channel */ void *ih; /* interrupt handle */ struct ata_lowlevel hw; /* lowlevel HW functions */ - struct ata_dma dma; /* DMA data / functions */ + struct ata_dma dma; /* DMA data / functions */ int flags; /* channel flags */ #define ATA_NO_SLAVE 0x01 #define ATA_USE_16BIT 0x02 @@ -581,8 +586,12 @@ void ata_generic_reset(device_t dev); int ata_generic_command(struct ata_request *request); /* macros for alloc/free of struct ata_request */ -struct ata_request *ata_alloc_request(device_t dev); -void ata_free_request(struct ata_request *request); +extern uma_zone_t ata_request_zone; +#define ata_alloc_request() uma_zalloc(ata_request_zone, M_NOWAIT | M_ZERO) +#define ata_free_request(request) { \ + if (!(request->flags & ATA_R_DANGER2)) \ + uma_zfree(ata_request_zone, request); \ + } /* macros for alloc/free of struct ata_composite */ extern uma_zone_t ata_composite_zone; diff --git a/sys/dev/ata/ata-chipset.c b/sys/dev/ata/ata-chipset.c index 44634d8..11455fc 100644 --- a/sys/dev/ata/ata-chipset.c +++ b/sys/dev/ata/ata-chipset.c @@ -447,7 +447,7 @@ ata_pm_identify(device_t dev) case 0x37261095: /* Some of these bogusly reports 6 ports */ pm_ports = 5; - device_printf(dev, "SiI-3726-R%x Portmultiplier with %d ports\n", + device_printf(dev, "SiI 3726 r%x Portmultiplier with %d ports\n", pm_revision, pm_ports); break; @@ -456,6 +456,9 @@ ata_pm_identify(device_t dev) pm_chipid, pm_revision, pm_ports); } + /* inform dma.alloc() about needed DMA slots */ + ch->dma.dma_slots = pm_ports; + /* reset all ports and register if anything connected */ for (port=0; port < pm_ports; port++) { u_int32_t signature, status; @@ -1166,8 +1169,7 @@ ata_ahci_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) } } - /* we only have space for 16 entries in a slot */ - KASSERT(nsegs <= 16, ("too many DMA segment entries\n")); + KASSERT(nsegs <= ATA_AHCI_DMA_ENTRIES, ("too many DMA segment entries\n")); args->nsegs = nsegs; } @@ -2102,6 +2104,7 @@ ata_intel_ident(device_t dev) { ATA_I63XXESB2_R2, 0, AHCI, 0x00, ATA_SA300, "63XXESB2" }, { ATA_I82801HB_S1, 0, AHCI, 0x00, ATA_SA300, "ICH8" }, { ATA_I82801HB_S2, 0, AHCI, 0x00, ATA_SA300, "ICH8" }, + { ATA_I82801HB_R1, 2, AHCI, 0x00, ATA_SA300, "ICH9R" }, { ATA_I82801HB_R1, 0, AHCI, 0x00, ATA_SA300, "ICH8" }, { ATA_I82801HB_AH4, 0, AHCI, 0x00, ATA_SA300, "ICH8" }, { ATA_I82801HB_AH6, 0, AHCI, 0x00, ATA_SA300, "ICH8" }, @@ -3020,8 +3023,8 @@ ata_marvell_edma_begin_transaction(struct ata_request *request) quadp = (u_int32_t *)bytep; /* fill in this request */ - quadp[0] = (long)request->dma.sg_bus & 0xffffffff; - quadp[1] = (u_int64_t)request->dma.sg_bus >> 32; + quadp[0] = (long)request->dma->sg_bus & 0xffffffff; + quadp[1] = (u_int64_t)request->dma->sg_bus >> 32; wordp[4] = (request->flags & ATA_R_READ ? 0x01 : 0x00) | (request->tag<<1); i = 10; @@ -3767,11 +3770,11 @@ ata_promise_dmastart(struct ata_request *request) ATA_INB(ctlr->r_res1, 0x11) | (ch->unit ? 0x08 : 0x02)); ATA_OUTL(ctlr->r_res1, ch->unit ? 0x24 : 0x20, ((request->flags & ATA_R_READ) ? 0x05000000 : 0x06000000) | - (request->dma.cur_iosize >> 1)); + (request->bytecount >> 1)); } ATA_IDX_OUTB(ch, ATA_BMSTAT_PORT, (ATA_IDX_INB(ch, ATA_BMSTAT_PORT) | (ATA_BMSTAT_INTERRUPT | ATA_BMSTAT_ERROR))); - ATA_IDX_OUTL(ch, ATA_BMDTP_PORT, request->dma.sg_bus); + ATA_IDX_OUTL(ch, ATA_BMDTP_PORT, request->dma->sg_bus); ATA_IDX_OUTB(ch, ATA_BMCMD_PORT, ((request->flags & ATA_R_READ) ? ATA_BMCMD_WRITE_READ : 0) | ATA_BMCMD_START_STOP); @@ -4095,7 +4098,7 @@ ata_promise_mio_command(struct ata_request *request) wordp[0] = htole32(0x00 | ((ch->unit + 1) << 16) | (0x00 << 24)); break; } - wordp[1] = htole32(request->dma.sg_bus); + wordp[1] = htole32(request->dma->sg_bus); wordp[2] = 0; ata_promise_apkt((u_int8_t*)wordp, request); @@ -4438,7 +4441,7 @@ ata_promise_sx4_command(struct ata_request *request) device_t gparent = GRANDPARENT(request->dev); struct ata_pci_controller *ctlr = device_get_softc(gparent); struct ata_channel *ch = device_get_softc(request->parent); - struct ata_dma_prdentry *prd = request->dma.sg; + struct ata_dma_prdentry *prd = request->dma->sg; caddr_t window = rman_get_virtual(ctlr->r_res1); u_int32_t *wordp; int i, idx, length = 0; @@ -4896,20 +4899,20 @@ ata_sii_ident(device_t dev) { struct ata_pci_controller *ctlr = device_get_softc(dev); static struct ata_chip_id ids[] = - {{ ATA_SII3114, 0x00, SIIMEMIO, SII4CH, ATA_SA150, "SiI 3114" }, - { ATA_SII3512, 0x02, SIIMEMIO, 0, ATA_SA150, "SiI 3512" }, - { ATA_SII3112, 0x02, SIIMEMIO, 0, ATA_SA150, "SiI 3112" }, - { ATA_SII3112_1, 0x02, SIIMEMIO, 0, ATA_SA150, "SiI 3112" }, - { ATA_SII3512, 0x00, SIIMEMIO, SIIBUG, ATA_SA150, "SiI 3512" }, - { ATA_SII3112, 0x00, SIIMEMIO, SIIBUG, ATA_SA150, "SiI 3112" }, - { ATA_SII3112_1, 0x00, SIIMEMIO, SIIBUG, ATA_SA150, "SiI 3112" }, - { ATA_SII3124, 0x00, SIIPRBIO, SII4CH, ATA_SA300, "SiI 3124" }, - { ATA_SII3132, 0x00, SIIPRBIO, 0, ATA_SA300, "SiI 3132" }, - { ATA_SII0680, 0x00, SIIMEMIO, SIISETCLK, ATA_UDMA6, "SiI 0680" }, - { ATA_CMD649, 0x00, 0, SIIINTR, ATA_UDMA5, "CMD 649" }, - { ATA_CMD648, 0x00, 0, SIIINTR, ATA_UDMA4, "CMD 648" }, - { ATA_CMD646, 0x07, 0, 0, ATA_UDMA2, "CMD 646U2" }, - { ATA_CMD646, 0x00, 0, 0, ATA_WDMA2, "CMD 646" }, + {{ ATA_SII3114, 0x00, SIIMEMIO, SII4CH, ATA_SA150, "3114" }, + { ATA_SII3512, 0x02, SIIMEMIO, 0, ATA_SA150, "3512" }, + { ATA_SII3112, 0x02, SIIMEMIO, 0, ATA_SA150, "3112" }, + { ATA_SII3112_1, 0x02, SIIMEMIO, 0, ATA_SA150, "3112" }, + { ATA_SII3512, 0x00, SIIMEMIO, SIIBUG, ATA_SA150, "3512" }, + { ATA_SII3112, 0x00, SIIMEMIO, SIIBUG, ATA_SA150, "3112" }, + { ATA_SII3112_1, 0x00, SIIMEMIO, SIIBUG, ATA_SA150, "3112" }, + { ATA_SII3124, 0x00, SIIPRBIO, SII4CH, ATA_SA300, "3124" }, + { ATA_SII3132, 0x00, SIIPRBIO, 0, ATA_SA300, "3132" }, + { ATA_SII0680, 0x00, SIIMEMIO, SIISETCLK, ATA_UDMA6, "680" }, + { ATA_CMD649, 0x00, 0, SIIINTR, ATA_UDMA5, "(CMD) 649" }, + { ATA_CMD648, 0x00, 0, SIIINTR, ATA_UDMA4, "(CMD) 648" }, + { ATA_CMD646, 0x07, 0, 0, ATA_UDMA2, "(CMD) 646U2" }, + { ATA_CMD646, 0x00, 0, 0, ATA_WDMA2, "(CMD) 646" }, { 0, 0, 0, 0, 0, 0}}; if (!(ctlr->chip = ata_match_chip(dev, ids))) @@ -5245,13 +5248,14 @@ struct ata_siiprb_dma_prdentry { u_int32_t control; } __packed; +#define ATA_SIIPRB_DMA_ENTRIES 125 struct ata_siiprb_ata_command { - struct ata_siiprb_dma_prdentry prd[126]; + struct ata_siiprb_dma_prdentry prd[ATA_SIIPRB_DMA_ENTRIES]; } __packed; struct ata_siiprb_atapi_command { u_int8_t ccb[16]; - struct ata_siiprb_dma_prdentry prd[125]; + struct ata_siiprb_dma_prdentry prd[ATA_SIIPRB_DMA_ENTRIES]; } __packed; struct ata_siiprb_command { @@ -5660,7 +5664,7 @@ ata_siiprb_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) prd[i].count = htole32(segs[i].ds_len); } prd[i - 1].control = htole32(ATA_DMA_EOT); - KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries\n")); + KASSERT(nsegs <= ATA_SIIPRB_DMA_ENTRIES,("too many DMA segment entries\n")); args->nsegs = nsegs; } diff --git a/sys/dev/ata/ata-disk.c b/sys/dev/ata/ata-disk.c index 668ae72..6bf9fd9 100644 --- a/sys/dev/ata/ata-disk.c +++ b/sys/dev/ata/ata-disk.c @@ -224,10 +224,11 @@ ad_spindown(void *priv) return; device_printf(dev, "Idle, spin down\n"); atadev->spindown_state = 1; - if (!(request = ata_alloc_request(dev))) { + if (!(request = ata_alloc_request())) { device_printf(dev, "FAILURE - out of memory in ad_spindown\n"); return; } + request->dev = dev; request->flags = ATA_R_CONTROL; request->timeout = 5; request->retries = 1; @@ -248,13 +249,14 @@ ad_strategy(struct bio *bp) callout_reset(&atadev->spindown_timer, hz * atadev->spindown, ad_spindown, dev); - if (!(request = ata_alloc_request(dev))) { + if (!(request = ata_alloc_request())) { device_printf(dev, "FAILURE - out of memory in start\n"); biofinish(bp, NULL, ENOMEM); return; } /* setup request */ + request->dev = dev; request->bio = bp; request->callback = ad_done; if (atadev->spindown_state) { @@ -448,10 +450,11 @@ ad_set_geometry(device_t dev) adp->total_secs / (adp->heads * adp->sectors), adp->heads, adp->sectors); - if (!(request = ata_alloc_request(dev))) + if (!(request = ata_alloc_request())) return; /* get the max native size the device supports */ + request->dev = dev; request->u.ata.command = ATA_READ_NATIVE_MAX_ADDRESS; request->u.ata.lba = 0; request->u.ata.count = 0; diff --git a/sys/dev/ata/ata-dma.c b/sys/dev/ata/ata-dma.c index 860e1ae..59f9279 100644 --- a/sys/dev/ata/ata-dma.c +++ b/sys/dev/ata/ata-dma.c @@ -45,6 +45,8 @@ __FBSDID("$FreeBSD$"); #include <dev/ata/ata-pci.h> /* prototypes */ +static void ata_dmafini(device_t dev); +static void ata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); static void ata_dmaalloc(device_t dev); static void ata_dmafree(device_t dev); static void ata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); @@ -67,6 +69,7 @@ void ata_dmainit(device_t dev) { struct ata_channel *ch = device_get_softc(dev); + struct ata_dc_cb_args dcba; ch->dma.alloc = ata_dmaalloc; ch->dma.free = ata_dmafree; @@ -78,6 +81,59 @@ ata_dmainit(device_t dev) ch->dma.segsize = 63536; ch->dma.max_iosize = 128 * DEV_BSIZE; ch->dma.max_address = BUS_SPACE_MAXADDR_32BIT; + ch->dma.dma_slots = 1; + + if (bus_dma_tag_create(bus_get_dma_tag(dev), ch->dma.alignment, 0, + ch->dma.max_address, BUS_SPACE_MAXADDR, + NULL, NULL, ch->dma.max_iosize, + ATA_DMA_ENTRIES, ch->dma.segsize, + 0, NULL, NULL, &ch->dma.dmatag)) + goto error; + + if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, 64 * 1024, + ch->dma.max_address, BUS_SPACE_MAXADDR, + NULL, NULL, MAXWSPCSZ, 1, MAXWSPCSZ, + 0, NULL, NULL, &ch->dma.work_tag)) + goto error; + + if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 0, + &ch->dma.work_map)) + goto error; + + if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work, + MAXWSPCSZ, ata_dmasetupc_cb, &dcba, 0) || + dcba.error) { + bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); + goto error; + } + ch->dma.work_bus = dcba.maddr; + return; + +error: + device_printf(dev, "WARNING - DMA initialization failed, disabling DMA\n"); + ata_dmafini(dev); +} + +void +ata_dmafini(device_t dev) +{ + struct ata_channel *ch = device_get_softc(dev); + + if (ch->dma.work_bus) { + bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map); + bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); + ch->dma.work_bus = 0; + ch->dma.work_map = NULL; + ch->dma.work = NULL; + } + if (ch->dma.work_tag) { + bus_dma_tag_destroy(ch->dma.work_tag); + ch->dma.work_tag = NULL; + } + if (ch->dma.dmatag) { + bus_dma_tag_destroy(ch->dma.dmatag); + ch->dma.dmatag = NULL; + } } static void @@ -94,6 +150,7 @@ ata_dmaalloc(device_t dev) { struct ata_channel *ch = device_get_softc(dev); struct ata_dc_cb_args dcba; + int i; if (bus_dma_tag_create(bus_get_dma_tag(dev), ch->dma.alignment, 0, ch->dma.max_address, BUS_SPACE_MAXADDR, @@ -120,6 +177,48 @@ ata_dmaalloc(device_t dev) } ch->dma.work_bus = dcba.maddr; + /* alloc and setup needed dma slots */ + bzero(ch->dma.slot, sizeof(struct ata_dmaslot) * ATA_DMA_SLOTS); + for (i = 0; i < ch->dma.dma_slots; i++) { + struct ata_dmaslot *slot = &ch->dma.slot[i]; + + if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, PAGE_SIZE, + ch->dma.max_address, BUS_SPACE_MAXADDR, + NULL, NULL, PAGE_SIZE, 1, PAGE_SIZE, + 0, NULL, NULL, &slot->sg_tag)) { + device_printf(ch->dev, "FAILURE - create sg_tag\n"); + goto error; + } + + if (bus_dmamem_alloc(slot->sg_tag, (void **)&slot->sg, + 0, &slot->sg_map)) { + device_printf(ch->dev, "FAILURE - alloc sg_map\n"); + goto error; + } + + if (bus_dmamap_load(slot->sg_tag, slot->sg_map, slot->sg, MAXTABSZ, + ata_dmasetupc_cb, &dcba, 0) || dcba.error) { + device_printf(ch->dev, "FAILURE - load sg\n"); + goto error; + } + slot->sg_bus = dcba.maddr; + + if (bus_dma_tag_create(ch->dma.dmatag, + ch->dma.alignment, ch->dma.boundary, + ch->dma.max_address, BUS_SPACE_MAXADDR, + NULL, NULL, ch->dma.max_iosize, + ATA_DMA_ENTRIES, ch->dma.segsize, + BUS_DMA_ALLOCNOW, NULL, NULL, &slot->data_tag)) { + device_printf(ch->dev, "FAILURE - create data_tag\n"); + goto error; + } + + if (bus_dmamap_create(slot->data_tag, 0, &slot->data_map)) { + device_printf(ch->dev, "FAILURE - create data_map\n"); + goto error; + } + } + return; error: @@ -131,21 +230,34 @@ static void ata_dmafree(device_t dev) { struct ata_channel *ch = device_get_softc(dev); + int i; - if (ch->dma.work_bus) { - bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map); - bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); - ch->dma.work_bus = 0; - ch->dma.work_map = NULL; - ch->dma.work = NULL; - } - if (ch->dma.work_tag) { - bus_dma_tag_destroy(ch->dma.work_tag); - ch->dma.work_tag = NULL; - } - if (ch->dma.dmatag) { - bus_dma_tag_destroy(ch->dma.dmatag); - ch->dma.dmatag = NULL; + /* free all dma slots */ + for (i = 0; i < ATA_DMA_SLOTS; i++) { + struct ata_dmaslot *slot = &ch->dma.slot[i]; + + if (slot->sg_bus) { + bus_dmamap_unload(slot->sg_tag, slot->sg_map); + slot->sg_bus = 0; + } + if (slot->sg_map) { + bus_dmamem_free(slot->sg_tag, slot->sg, slot->sg_map); + bus_dmamap_destroy(slot->sg_tag, slot->sg_map); + slot->sg = NULL; + slot->sg_map = NULL; + } + if (slot->data_map) { + bus_dmamap_destroy(slot->data_tag, slot->data_map); + slot->data_map = NULL; + } + if (slot->sg_tag) { + bus_dma_tag_destroy(slot->sg_tag); + slot->sg_tag = NULL; + } + if (slot->data_tag) { + bus_dma_tag_destroy(slot->data_tag); + slot->data_tag = NULL; + } } } @@ -172,13 +284,13 @@ static int ata_dmaload(struct ata_request *request, void *addr, int *entries) { struct ata_channel *ch = device_get_softc(request->parent); - struct ata_dc_cb_args dcba; + struct ata_device *atadev = device_get_softc(request->dev); struct ata_dmasetprd_args dspa; int error; ATA_DEBUG_RQ(request, "dmaload"); - if (request->dma.cur_iosize) { + if (request->dma) { device_printf(request->dev, "FAILURE - already active DMA on this device\n"); return EIO; @@ -201,27 +313,15 @@ ata_dmaload(struct ata_request *request, void *addr, int *entries) return EIO; } - if (bus_dmamap_load(request->dma.sg_tag, request->dma.sg_map, - request->dma.sg, MAXTABSZ, ata_dmasetupc_cb, &dcba, 0)|| - dcba.error) { - bus_dmamem_free(request->dma.sg_tag, - request->dma.sg, request->dma.sg_map); - device_printf(request->dev, "FAILURE - load sg\n"); - goto error; - } - request->dma.sg_bus = dcba.maddr; - - if (bus_dmamap_create(request->dma.data_tag, 0, &request->dma.data_map)) { - device_printf(request->dev, "FAILURE - create data_map\n"); - goto error; - } + /* set our slot, unit for simplicity XXX SOS NCQ will change that */ + request->dma = &ch->dma.slot[atadev->unit]; if (addr) dspa.dmatab = addr; else - dspa.dmatab = request->dma.sg; + dspa.dmatab = request->dma->sg; - if ((error = bus_dmamap_load(request->dma.data_tag, request->dma.data_map, + if ((error = bus_dmamap_load(request->dma->data_tag, request->dma->data_map, request->data, request->bytecount, ch->dma.setprd, &dspa, BUS_DMA_NOWAIT)) || (error = dspa.error)) { @@ -232,15 +332,11 @@ ata_dmaload(struct ata_request *request, void *addr, int *entries) if (entries) *entries = dspa.nsegs; - bus_dmamap_sync(request->dma.sg_tag, request->dma.sg_map, + bus_dmamap_sync(request->dma->sg_tag, request->dma->sg_map, BUS_DMASYNC_PREWRITE); - - bus_dmamap_sync(request->dma.data_tag, request->dma.data_map, + bus_dmamap_sync(request->dma->data_tag, request->dma->data_map, (request->flags & ATA_R_READ) ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); - - request->dma.cur_iosize = request->bytecount; - return 0; error: @@ -253,31 +349,15 @@ ata_dmaunload(struct ata_request *request) { ATA_DEBUG_RQ(request, "dmaunload"); - if (request->dma.cur_iosize) { - bus_dmamap_sync(request->dma.sg_tag, request->dma.sg_map, + if (request->dma) { + bus_dmamap_sync(request->dma->sg_tag, request->dma->sg_map, BUS_DMASYNC_POSTWRITE); - - bus_dmamap_sync(request->dma.data_tag, request->dma.data_map, + bus_dmamap_sync(request->dma->data_tag, request->dma->data_map, (request->flags & ATA_R_READ) ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(request->dma.data_tag, request->dma.data_map); - request->dma.cur_iosize = 0; + bus_dmamap_unload(request->dma->data_tag, request->dma->data_map); + request->dma = NULL; } - - if (request->dma.data_map) { - bus_dmamap_destroy(request->dma.data_tag, request->dma.data_map); - request->dma.data_map = NULL; - } - - if (request->dma.sg_bus) { - bus_dmamap_unload(request->dma.sg_tag, request->dma.sg_map); - bus_dmamem_free(request->dma.sg_tag, request->dma.sg, - request->dma.sg_map); - request->dma.sg = NULL; - request->dma.sg_bus = 0; - request->dma.sg_map = NULL; - } - return 0; } diff --git a/sys/dev/ata/ata-lowlevel.c b/sys/dev/ata/ata-lowlevel.c index 5d552d4..6dd7662 100644 --- a/sys/dev/ata/ata-lowlevel.c +++ b/sys/dev/ata/ata-lowlevel.c @@ -313,12 +313,12 @@ ata_end_transaction(struct ata_request *request) /* stop DMA engine and get status */ if (ch->dma.stop) - request->dma.status = ch->dma.stop(request); + request->dma->status = ch->dma.stop(request); /* did we get error or data */ if (request->status & ATA_S_ERROR) request->error = ATA_IDX_INB(ch, ATA_ERROR); - else if (request->dma.status & ATA_BMSTAT_ERROR) + else if (request->dma->status & ATA_BMSTAT_ERROR) request->status |= ATA_S_ERROR; else if (!(request->flags & ATA_R_TIMEOUT)) request->donecount = request->bytecount; @@ -425,12 +425,12 @@ ata_end_transaction(struct ata_request *request) /* stop DMA engine and get status */ if (ch->dma.stop) - request->dma.status = ch->dma.stop(request); + request->dma->status = ch->dma.stop(request); /* did we get error or data */ if (request->status & (ATA_S_ERROR | ATA_S_DWF)) request->error = ATA_IDX_INB(ch, ATA_ERROR); - else if (request->dma.status & ATA_BMSTAT_ERROR) + else if (request->dma->status & ATA_BMSTAT_ERROR) request->status |= ATA_S_ERROR; else if (!(request->flags & ATA_R_TIMEOUT)) request->donecount = request->bytecount; diff --git a/sys/dev/ata/ata-pci.c b/sys/dev/ata/ata-pci.c index ab5342b..cb964fb 100644 --- a/sys/dev/ata/ata-pci.c +++ b/sys/dev/ata/ata-pci.c @@ -464,7 +464,7 @@ ata_pci_dmastart(struct ata_request *request) ATA_IDX_OUTB(ch, ATA_BMSTAT_PORT, (ATA_IDX_INB(ch, ATA_BMSTAT_PORT) | (ATA_BMSTAT_INTERRUPT | ATA_BMSTAT_ERROR))); - ATA_IDX_OUTL(ch, ATA_BMDTP_PORT, request->dma.sg_bus); + ATA_IDX_OUTL(ch, ATA_BMDTP_PORT, request->dma->sg_bus); ch->dma.flags |= ATA_DMA_ACTIVE; ATA_IDX_OUTB(ch, ATA_BMCMD_PORT, (ATA_IDX_INB(ch, ATA_BMCMD_PORT) & ~ATA_BMCMD_WRITE_READ) | @@ -609,17 +609,17 @@ ata_pcichannel_attach(device_t dev) struct ata_channel *ch = device_get_softc(dev); int error; - if (ctlr->dmainit) { + if (ctlr->dmainit) ctlr->dmainit(dev); - ch->dma.alloc(dev); - } - if ((error = ctlr->allocate(dev))) { - ch->dma.free(dev); + if ((error = ctlr->allocate(dev))) return error; - } - return ata_attach(dev); + if ((error = ata_attach(dev))) + return error; + + ch->dma.alloc(dev); + return 0; } static int diff --git a/sys/dev/ata/ata-queue.c b/sys/dev/ata/ata-queue.c index e20f0d1..5152faa 100644 --- a/sys/dev/ata/ata-queue.c +++ b/sys/dev/ata/ata-queue.c @@ -119,10 +119,11 @@ int ata_controlcmd(device_t dev, u_int8_t command, u_int16_t feature, u_int64_t lba, u_int16_t count) { - struct ata_request *request = ata_alloc_request(dev); + struct ata_request *request = ata_alloc_request(); int error = ENOMEM; if (request) { + request->dev = dev; request->u.ata.command = command; request->u.ata.lba = lba; request->u.ata.count = count; @@ -141,11 +142,12 @@ int ata_atapicmd(device_t dev, u_int8_t *ccb, caddr_t data, int count, int flags, int timeout) { - struct ata_request *request = ata_alloc_request(dev); + struct ata_request *request = ata_alloc_request(); struct ata_device *atadev = device_get_softc(dev); int error = ENOMEM; if (request) { + request->dev = dev; if ((atadev->param.config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) bcopy(ccb, request->u.atapi.ccb, 12); else @@ -356,8 +358,8 @@ ata_completed(void *context, int dummy) "\4MEDIA_CHANGE_REQEST" "\3ABORTED\2NO_MEDIA\1ILLEGAL_LENGTH"); if ((request->flags & ATA_R_DMA) && - (request->dma.status & ATA_BMSTAT_ERROR)) - printf(" dma=0x%02x", request->dma.status); + (request->dma->status & ATA_BMSTAT_ERROR)) + printf(" dma=0x%02x", request->dma->status); if (!(request->flags & (ATA_R_ATAPI | ATA_R_CONTROL))) printf(" LBA=%ju", request->u.ata.lba); printf("\n"); diff --git a/sys/dev/ata/ata-raid.c b/sys/dev/ata/ata-raid.c index d7dfac0..40a880c 100644 --- a/sys/dev/ata/ata-raid.c +++ b/sys/dev/ata/ata-raid.c @@ -4027,10 +4027,11 @@ ata_raid_init_request(device_t dev, struct ar_softc *rdp, struct bio *bio) { struct ata_request *request; - if (!(request = ata_alloc_request(dev))) { + if (!(request = ata_alloc_request())) { printf("FAILURE - out of memory in ata_raid_init_request\n"); return NULL; } + request->dev = dev; request->timeout = 5; request->retries = 2; request->callback = ata_raid_done; @@ -4098,12 +4099,13 @@ ata_raid_rw(device_t dev, u_int64_t lba, void *data, u_int bcount, int flags) return ENOMEM; } - if (!(request = ata_alloc_request(dev))) { + if (!(request = ata_alloc_request())) { device_printf(dev, "FAILURE - out of memory in ata_raid_rw\n"); return ENOMEM; } /* setup request */ + request->dev = dev; request->timeout = 10; request->retries = 0; request->data = data; diff --git a/sys/dev/ata/atapi-cam.c b/sys/dev/ata/atapi-cam.c index 6e4a2ab..d8f0b86 100644 --- a/sys/dev/ata/atapi-cam.c +++ b/sys/dev/ata/atapi-cam.c @@ -531,7 +531,7 @@ atapi_action(struct cam_sim *sim, union ccb *ccb) printf("cannot allocate ATAPI/CAM hcb\n"); goto action_oom; } - if ((request = ata_alloc_request(softc->atadev[tid]->dev)) == NULL) { + if ((request = ata_alloc_request()) == NULL) { printf("cannot allocate ATAPI/CAM request\n"); goto action_oom; } @@ -618,6 +618,7 @@ atapi_action(struct cam_sim *sim, union ccb *ccb) goto action_oom; } } + request->dev = softc->atadev[tid]->dev; request->driver = hcb; request->data = buf; request->bytecount = len; diff --git a/sys/dev/ata/atapi-cd.c b/sys/dev/ata/atapi-cd.c index f9fcf97..7bf3b9f 100644 --- a/sys/dev/ata/atapi-cd.c +++ b/sys/dev/ata/atapi-cd.c @@ -691,11 +691,12 @@ acd_geom_access(struct g_provider *pp, int dr, int dw, int de) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; int timeout = 60, track; - if (!(request = ata_alloc_request(dev))) + if (!(request = ata_alloc_request())) return ENOMEM; /* wait if drive is not finished loading the medium */ while (timeout--) { + request->dev = dev; bcopy(ccb, request->u.atapi.ccb, 16); request->flags = ATA_R_ATAPI; request->timeout = 5; @@ -855,10 +856,11 @@ acd_strategy(struct bio *bp) ccb[7] = count>>8; ccb[8] = count; - if (!(request = ata_alloc_request(dev))) { + if (!(request = ata_alloc_request())) { g_io_deliver(bp, ENOMEM); return; } + request->dev = dev; request->bio = bp; bcopy(ccb, request->u.atapi.ccb, (atadev->param.config & ATA_PROTO_MASK) == @@ -1218,9 +1220,10 @@ acd_get_progress(device_t dev, int *finished) struct ata_request *request; int8_t dummy[8]; - if (!(request = ata_alloc_request(dev))) + if (!(request = ata_alloc_request())) return ENOMEM; + request->dev = dev; bcopy(ccb, request->u.atapi.ccb, 16); request->data = dummy; request->bytecount = sizeof(dummy); diff --git a/sys/dev/ata/atapi-fd.c b/sys/dev/ata/atapi-fd.c index 45a1b17..9a26399 100644 --- a/sys/dev/ata/atapi-fd.c +++ b/sys/dev/ata/atapi-fd.c @@ -233,10 +233,11 @@ afd_strategy(struct bio *bp) ccb[7] = count>>8; ccb[8] = count; - if (!(request = ata_alloc_request(dev))) { + if (!(request = ata_alloc_request())) { biofinish(bp, NULL, ENOMEM); return; } + request->dev = dev; request->bio = bp; bcopy(ccb, request->u.atapi.ccb, (atadev->param.config & ATA_PROTO_MASK) == diff --git a/sys/dev/ata/atapi-tape.c b/sys/dev/ata/atapi-tape.c index aa65226..3304907 100644 --- a/sys/dev/ata/atapi-tape.c +++ b/sys/dev/ata/atapi-tape.c @@ -421,10 +421,11 @@ ast_strategy(struct bio *bp) ccb[3] = blkcount >> 8; ccb[4] = blkcount; - if (!(request = ata_alloc_request(dev))) { + if (!(request = ata_alloc_request())) { biofinish(bp, NULL, ENOMEM); return; } + request->dev = dev; request->driver = bp; bcopy(ccb, request->u.atapi.ccb, (atadev->param.config & ATA_PROTO_MASK) == |