Go back to preallocating everything possible on init.

This avoids calling busdma in the request processing path which caused a traumatic performance degradation.
Allocation has be postponed to after we know how many devices we possible can have on portmulitpliers to save some space.

13 files changed, 243 insertions, 188 deletions

diff --git a/sys/dev/ata/ata-all.c b/sys/dev/ata/ata-all.c
index 0110c23..87f4927 100644
--- a/sys/dev/ata/ata-all.c
+++ b/sys/dev/ata/ata-all.c
@@ -71,12 +71,12 @@ MALLOC_DEFINE(M_ATA, "ata_generic", "ATA driver generic layer");
 int (*ata_raid_ioctl_func)(u_long cmd, caddr_t data) = NULL;
 struct intr_config_hook *ata_delayed_attach = NULL;
 devclass_t ata_devclass;
+uma_zone_t ata_request_zone;
 uma_zone_t ata_composite_zone;
 int ata_wc = 1;
 int ata_setmax = 0;
 
 /* local vars */
-static uma_zone_t ata_request_zone;
 static int ata_dma = 1;
 static int atapi_dma = 1;
 
@@ -451,10 +451,11 @@ ata_device_ioctl(device_t dev, u_long cmd, caddr_t data)
 	if (!(buf = malloc(ioc_request->count, M_ATA, M_NOWAIT))) {
 	    return ENOMEM;
 	}
-	if (!(request = ata_alloc_request(dev))) {
+	if (!(request = ata_alloc_request())) {
 	    free(buf, M_ATA);
 	    return  ENOMEM;
 	}
+	request->dev = atadev->dev;
 	if (ioc_request->flags & ATA_CMD_WRITE) {
 	    error = copyin(ioc_request->data, buf, ioc_request->count);
 	    if (error) {
@@ -587,8 +588,9 @@ ata_getparam(struct ata_device *atadev, int init)
 	return ENXIO;
 
     while (retries-- > 0 && error) {
-	if (!(request = ata_alloc_request(atadev->dev)))
+	if (!(request = ata_alloc_request()))
 	    break;
+	request->dev = atadev->dev;
 	request->timeout = 1;
 	request->retries = 0;
 	request->u.ata.command = command;
@@ -682,7 +684,8 @@ ata_identify(device_t dev)
 	    }
 	    devices[i]->unit = i;
 #ifdef ATA_STATIC_ID
-	    unit = (device_get_unit(dev) << 1) + i;
+	    if (ch->devices & ((ATA_ATA_MASTER << i)))
+		unit = (device_get_unit(dev) << 1) + i;
 #endif
 	    if (!(childdevs[i] = ata_add_child(dev, devices[i], unit))) {
 		free(devices[i], M_ATA);
@@ -1002,60 +1005,6 @@ bpack(int8_t *src, int8_t *dst, int len)
 	dst[j] = 0x00;
 }
 
-struct ata_request *
-ata_alloc_request(device_t dev)
-{
-    struct ata_channel *ch = device_get_softc(device_get_parent(dev));
-    struct ata_request *request;
-
-    if ((request = uma_zalloc(ata_request_zone, M_NOWAIT | M_ZERO))) {
-	if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, PAGE_SIZE,
-			       ch->dma.max_address, BUS_SPACE_MAXADDR,
-			       NULL, NULL, PAGE_SIZE, 1, PAGE_SIZE,
-			       0, NULL, NULL, &request->dma.sg_tag)) {
-            device_printf(ch->dev, "FAILURE - create sg_tag\n");
-	    uma_zfree(ata_request_zone, request);
-            return NULL;
-	}
-	if (bus_dmamem_alloc(request->dma.sg_tag, (void **)&request->dma.sg, 0,
-			     &request->dma.sg_map)) {
-	    device_printf(ch->dev, "FAILURE - alloc sg_map\n");
-	    bus_dma_tag_destroy(request->dma.sg_tag);
-	    uma_zfree(ata_request_zone, request);
-	    return NULL;
-        }
-	if (bus_dma_tag_create(ch->dma.dmatag,
-			       ch->dma.alignment, ch->dma.boundary,
-                               ch->dma.max_address, BUS_SPACE_MAXADDR,
-                               NULL, NULL, ch->dma.max_iosize,
-                               ATA_DMA_ENTRIES, ch->dma.segsize,
-                               BUS_DMA_ALLOCNOW, NULL, NULL,
-                               &request->dma.data_tag)) {
-	    device_printf(ch->dev, "FAILURE - create data_tag\n");
-	    bus_dmamem_free(request->dma.sg_tag, request->dma.sg,
-			    request->dma.sg_map);
-	    bus_dma_tag_destroy(request->dma.sg_tag);
-	    uma_zfree(ata_request_zone, request);
-	    return NULL;
-	}
-	request->dev = dev;
-    }
-    else
-	device_printf(dev, "FAILURE - ata_alloc_request\n");
-    return request;
-}
-
-void
-ata_free_request(struct ata_request *request)
-{
-    if (!(request->flags & ATA_R_DANGER2)) {
-	if (request->dma.data_tag)
-	    bus_dma_tag_destroy(request->dma.data_tag);
-	if (request->dma.sg_tag)
-	    bus_dma_tag_destroy(request->dma.sg_tag);
-	uma_zfree(ata_request_zone, request);
-    }
-}
 
 /*
  * module handeling
diff --git a/sys/dev/ata/ata-all.h b/sys/dev/ata/ata-all.h
index 4ad8701..4bccc4d 100644
--- a/sys/dev/ata/ata-all.h
+++ b/sys/dev/ata/ata-all.h
@@ -226,10 +226,9 @@
 
 #define ATA_AHCI_CL_SIZE                32
 #define ATA_AHCI_CL_OFFSET              0
-#define ATA_AHCI_FB_OFFSET              1024
-#define ATA_AHCI_CT_OFFSET              1024+4096
-#define ATA_AHCI_CT_SG_OFFSET           128
-#define ATA_AHCI_CT_SIZE                256
+#define ATA_AHCI_FB_OFFSET              (ATA_AHCI_CL_SIZE * 32)
+#define ATA_AHCI_CT_OFFSET              (ATA_AHCI_FB_OFFSET + 4096)
+#define ATA_AHCI_CT_SIZE                (1024 + 128)
 
 struct ata_ahci_dma_prd {
     u_int64_t                   dba;
@@ -243,7 +242,8 @@ struct ata_ahci_cmd_tab {
     u_int8_t                    cfis[64];
     u_int8_t                    acmd[32];
     u_int8_t                    reserved[32];
-    struct ata_ahci_dma_prd     prd_tab[16];
+#define ATA_AHCI_DMA_ENTRIES            64
+    struct ata_ahci_dma_prd     prd_tab[ATA_AHCI_DMA_ENTRIES];
 } __packed;
 
 struct ata_ahci_cmd_list {
@@ -373,18 +373,9 @@ struct ata_request {
 #define         ATA_R_DANGER1           0x20000000
 #define         ATA_R_DANGER2           0x40000000
 
+    struct ata_dmaslot          *dma;           /* DMA slot of this request */
     u_int8_t                    status;         /* ATA status */
     u_int8_t                    error;          /* ATA error */
-    struct {
-	u_int8_t                status;         /* DMA status */
-	bus_dma_tag_t           sg_tag;         /* SG list DMA tag */
-	bus_dmamap_t            sg_map;         /* SG list DMA map */
-	void                    *sg;            /* DMA transfer table */
-	bus_addr_t              sg_bus;         /* bus address of dmatab */
-	bus_dma_tag_t           data_tag;       /* data DMA tag */
-	bus_dmamap_t            data_map;       /* data DMA map */
-	u_int32_t               cur_iosize;     /* DMA data current IO size */
-    } dma;
     u_int32_t                   donecount;      /* bytes transferred */
     int                         result;         /* result error code */
     void                        (*callback)(struct ata_request *request);
@@ -447,6 +438,16 @@ struct ata_dmasetprd_args {
     int error;
 };
 
+struct ata_dmaslot {
+    u_int8_t                    status;         /* DMA status */
+    bus_dma_tag_t               sg_tag;         /* SG list DMA tag */
+    bus_dmamap_t                sg_map;         /* SG list DMA map */
+    void                        *sg;            /* DMA transfer table */
+    bus_addr_t                  sg_bus;         /* bus address of dmatab */
+    bus_dma_tag_t               data_tag;       /* data DMA tag */
+    bus_dmamap_t                data_map;       /* data DMA map */
+};
+
 /* structure holding DMA related information */
 struct ata_dma {
     bus_dma_tag_t               dmatag;         /* parent DMA tag */
@@ -454,6 +455,10 @@ struct ata_dma {
     bus_dmamap_t                work_map;       /* workspace DMA map */
     u_int8_t                    *work;          /* workspace */
     bus_addr_t                  work_bus;       /* bus address of dmatab */
+
+#define ATA_DMA_SLOTS			32
+    int				dma_slots;	/* DMA slots allocated */
+    struct ata_dmaslot		slot[ATA_DMA_SLOTS];
     u_int32_t                   alignment;      /* DMA SG list alignment */
     u_int32_t                   boundary;       /* DMA SG list boundary */
     u_int32_t                   segsize;        /* DMA SG list segment size */
@@ -499,7 +504,7 @@ struct ata_channel {
     struct resource             *r_irq;         /* interrupt of this channel */
     void                        *ih;            /* interrupt handle */
     struct ata_lowlevel         hw;             /* lowlevel HW functions */
-    struct ata_dma              dma;           /* DMA data / functions */
+    struct ata_dma              dma;            /* DMA data / functions */
     int                         flags;          /* channel flags */
 #define         ATA_NO_SLAVE            0x01
 #define         ATA_USE_16BIT           0x02
@@ -581,8 +586,12 @@ void ata_generic_reset(device_t dev);
 int ata_generic_command(struct ata_request *request);
 
 /* macros for alloc/free of struct ata_request */
-struct ata_request *ata_alloc_request(device_t dev);
-void ata_free_request(struct ata_request *request);
+extern uma_zone_t ata_request_zone;
+#define ata_alloc_request() uma_zalloc(ata_request_zone, M_NOWAIT | M_ZERO)
+#define ata_free_request(request) { \
+	if (!(request->flags & ATA_R_DANGER2)) \
+	    uma_zfree(ata_request_zone, request); \
+	}
 
 /* macros for alloc/free of struct ata_composite */
 extern uma_zone_t ata_composite_zone;
diff --git a/sys/dev/ata/ata-chipset.c b/sys/dev/ata/ata-chipset.c
index 44634d8..11455fc 100644
--- a/sys/dev/ata/ata-chipset.c
+++ b/sys/dev/ata/ata-chipset.c
@@ -447,7 +447,7 @@ ata_pm_identify(device_t dev)
     case 0x37261095:
 	/* Some of these bogusly reports 6 ports */
 	pm_ports = 5;
-	device_printf(dev, "SiI-3726-R%x Portmultiplier with %d ports\n",
+	device_printf(dev, "SiI 3726 r%x Portmultiplier with %d ports\n",
 		      pm_revision, pm_ports);
 	break;
 
@@ -456,6 +456,9 @@ ata_pm_identify(device_t dev)
 		      pm_chipid, pm_revision, pm_ports);
     }
 
+    /* inform dma.alloc() about needed DMA slots */
+    ch->dma.dma_slots = pm_ports;
+
     /* reset all ports and register if anything connected */
     for (port=0; port < pm_ports; port++) {
 	u_int32_t signature, status;
@@ -1166,8 +1169,7 @@ ata_ahci_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
 	}
     }
 
-    /* we only have space for 16 entries in a slot */
-    KASSERT(nsegs <= 16, ("too many DMA segment entries\n"));
+    KASSERT(nsegs <= ATA_AHCI_DMA_ENTRIES, ("too many DMA segment entries\n"));
     args->nsegs = nsegs;
 }
 
@@ -2102,6 +2104,7 @@ ata_intel_ident(device_t dev)
      { ATA_I63XXESB2_R2, 0, AHCI, 0x00, ATA_SA300, "63XXESB2" },
      { ATA_I82801HB_S1,  0, AHCI, 0x00, ATA_SA300, "ICH8" },
      { ATA_I82801HB_S2,  0, AHCI, 0x00, ATA_SA300, "ICH8" },
+     { ATA_I82801HB_R1,  2, AHCI, 0x00, ATA_SA300, "ICH9R" },
      { ATA_I82801HB_R1,  0, AHCI, 0x00, ATA_SA300, "ICH8" },
      { ATA_I82801HB_AH4, 0, AHCI, 0x00, ATA_SA300, "ICH8" },
      { ATA_I82801HB_AH6, 0, AHCI, 0x00, ATA_SA300, "ICH8" },
@@ -3020,8 +3023,8 @@ ata_marvell_edma_begin_transaction(struct ata_request *request)
     quadp = (u_int32_t *)bytep;
 
     /* fill in this request */
-    quadp[0] = (long)request->dma.sg_bus & 0xffffffff;
-    quadp[1] = (u_int64_t)request->dma.sg_bus >> 32;
+    quadp[0] = (long)request->dma->sg_bus & 0xffffffff;
+    quadp[1] = (u_int64_t)request->dma->sg_bus >> 32;
     wordp[4] = (request->flags & ATA_R_READ ? 0x01 : 0x00) | (request->tag<<1);
 
     i = 10;
@@ -3767,11 +3770,11 @@ ata_promise_dmastart(struct ata_request *request)
 		 ATA_INB(ctlr->r_res1, 0x11) | (ch->unit ? 0x08 : 0x02));
 	ATA_OUTL(ctlr->r_res1, ch->unit ? 0x24 : 0x20,
 		 ((request->flags & ATA_R_READ) ? 0x05000000 : 0x06000000) |
-		 (request->dma.cur_iosize >> 1));
+		 (request->bytecount >> 1));
     }
     ATA_IDX_OUTB(ch, ATA_BMSTAT_PORT, (ATA_IDX_INB(ch, ATA_BMSTAT_PORT) |
 		 (ATA_BMSTAT_INTERRUPT | ATA_BMSTAT_ERROR)));
-    ATA_IDX_OUTL(ch, ATA_BMDTP_PORT, request->dma.sg_bus);
+    ATA_IDX_OUTL(ch, ATA_BMDTP_PORT, request->dma->sg_bus);
     ATA_IDX_OUTB(ch, ATA_BMCMD_PORT,
 		 ((request->flags & ATA_R_READ) ? ATA_BMCMD_WRITE_READ : 0) |
 		 ATA_BMCMD_START_STOP);
@@ -4095,7 +4098,7 @@ ata_promise_mio_command(struct ata_request *request)
 	wordp[0] = htole32(0x00 | ((ch->unit + 1) << 16) | (0x00 << 24));
 	break;
     }
-    wordp[1] = htole32(request->dma.sg_bus);
+    wordp[1] = htole32(request->dma->sg_bus);
     wordp[2] = 0;
     ata_promise_apkt((u_int8_t*)wordp, request);
 
@@ -4438,7 +4441,7 @@ ata_promise_sx4_command(struct ata_request *request)
     device_t gparent = GRANDPARENT(request->dev);
     struct ata_pci_controller *ctlr = device_get_softc(gparent);
     struct ata_channel *ch = device_get_softc(request->parent);
-    struct ata_dma_prdentry *prd = request->dma.sg;
+    struct ata_dma_prdentry *prd = request->dma->sg;
     caddr_t window = rman_get_virtual(ctlr->r_res1);
     u_int32_t *wordp;
     int i, idx, length = 0;
@@ -4896,20 +4899,20 @@ ata_sii_ident(device_t dev)
 {
     struct ata_pci_controller *ctlr = device_get_softc(dev);
     static struct ata_chip_id ids[] =
-    {{ ATA_SII3114,   0x00, SIIMEMIO, SII4CH,    ATA_SA150, "SiI 3114" },
-     { ATA_SII3512,   0x02, SIIMEMIO, 0,         ATA_SA150, "SiI 3512" },
-     { ATA_SII3112,   0x02, SIIMEMIO, 0,         ATA_SA150, "SiI 3112" },
-     { ATA_SII3112_1, 0x02, SIIMEMIO, 0,         ATA_SA150, "SiI 3112" },
-     { ATA_SII3512,   0x00, SIIMEMIO, SIIBUG,    ATA_SA150, "SiI 3512" },
-     { ATA_SII3112,   0x00, SIIMEMIO, SIIBUG,    ATA_SA150, "SiI 3112" },
-     { ATA_SII3112_1, 0x00, SIIMEMIO, SIIBUG,    ATA_SA150, "SiI 3112" },
-     { ATA_SII3124,   0x00, SIIPRBIO, SII4CH,    ATA_SA300, "SiI 3124" },
-     { ATA_SII3132,   0x00, SIIPRBIO, 0,         ATA_SA300, "SiI 3132" },
-     { ATA_SII0680,   0x00, SIIMEMIO, SIISETCLK, ATA_UDMA6, "SiI 0680" },
-     { ATA_CMD649,    0x00, 0,        SIIINTR,   ATA_UDMA5, "CMD 649" },
-     { ATA_CMD648,    0x00, 0,        SIIINTR,   ATA_UDMA4, "CMD 648" },
-     { ATA_CMD646,    0x07, 0,        0,         ATA_UDMA2, "CMD 646U2" },
-     { ATA_CMD646,    0x00, 0,        0,         ATA_WDMA2, "CMD 646" },
+    {{ ATA_SII3114,   0x00, SIIMEMIO, SII4CH,    ATA_SA150, "3114" },
+     { ATA_SII3512,   0x02, SIIMEMIO, 0,         ATA_SA150, "3512" },
+     { ATA_SII3112,   0x02, SIIMEMIO, 0,         ATA_SA150, "3112" },
+     { ATA_SII3112_1, 0x02, SIIMEMIO, 0,         ATA_SA150, "3112" },
+     { ATA_SII3512,   0x00, SIIMEMIO, SIIBUG,    ATA_SA150, "3512" },
+     { ATA_SII3112,   0x00, SIIMEMIO, SIIBUG,    ATA_SA150, "3112" },
+     { ATA_SII3112_1, 0x00, SIIMEMIO, SIIBUG,    ATA_SA150, "3112" },
+     { ATA_SII3124,   0x00, SIIPRBIO, SII4CH,    ATA_SA300, "3124" },
+     { ATA_SII3132,   0x00, SIIPRBIO, 0,         ATA_SA300, "3132" },
+     { ATA_SII0680,   0x00, SIIMEMIO, SIISETCLK, ATA_UDMA6, "680" },
+     { ATA_CMD649,    0x00, 0,        SIIINTR,   ATA_UDMA5, "(CMD) 649" },
+     { ATA_CMD648,    0x00, 0,        SIIINTR,   ATA_UDMA4, "(CMD) 648" },
+     { ATA_CMD646,    0x07, 0,        0,         ATA_UDMA2, "(CMD) 646U2" },
+     { ATA_CMD646,    0x00, 0,        0,         ATA_WDMA2, "(CMD) 646" },
      { 0, 0, 0, 0, 0, 0}};
 
     if (!(ctlr->chip = ata_match_chip(dev, ids)))
@@ -5245,13 +5248,14 @@ struct ata_siiprb_dma_prdentry {
     u_int32_t control;
 } __packed;
 
+#define ATA_SIIPRB_DMA_ENTRIES		125
 struct ata_siiprb_ata_command {
-    struct ata_siiprb_dma_prdentry prd[126];
+    struct ata_siiprb_dma_prdentry prd[ATA_SIIPRB_DMA_ENTRIES];
 } __packed;
 
 struct ata_siiprb_atapi_command {
     u_int8_t ccb[16];
-    struct ata_siiprb_dma_prdentry prd[125];
+    struct ata_siiprb_dma_prdentry prd[ATA_SIIPRB_DMA_ENTRIES];
 } __packed;
 
 struct ata_siiprb_command {
@@ -5660,7 +5664,7 @@ ata_siiprb_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
 	prd[i].count = htole32(segs[i].ds_len);
     }
     prd[i - 1].control = htole32(ATA_DMA_EOT);
-    KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries\n"));
+    KASSERT(nsegs <= ATA_SIIPRB_DMA_ENTRIES,("too many DMA segment entries\n"));
     args->nsegs = nsegs;
 }
 
diff --git a/sys/dev/ata/ata-disk.c b/sys/dev/ata/ata-disk.c
index 668ae72..6bf9fd9 100644
--- a/sys/dev/ata/ata-disk.c
+++ b/sys/dev/ata/ata-disk.c
@@ -224,10 +224,11 @@ ad_spindown(void *priv)
 	return;
     device_printf(dev, "Idle, spin down\n");
     atadev->spindown_state = 1;
-    if (!(request = ata_alloc_request(dev))) {
+    if (!(request = ata_alloc_request())) {
 	device_printf(dev, "FAILURE - out of memory in ad_spindown\n");
 	return;
     }
+    request->dev = dev;
     request->flags = ATA_R_CONTROL;
     request->timeout = 5;
     request->retries = 1;
@@ -248,13 +249,14 @@ ad_strategy(struct bio *bp)
 	callout_reset(&atadev->spindown_timer, hz * atadev->spindown,
 		      ad_spindown, dev);
 
-    if (!(request = ata_alloc_request(dev))) {
+    if (!(request = ata_alloc_request())) {
 	device_printf(dev, "FAILURE - out of memory in start\n");
 	biofinish(bp, NULL, ENOMEM);
 	return;
     }
 
     /* setup request */
+    request->dev = dev;
     request->bio = bp;
     request->callback = ad_done;
     if (atadev->spindown_state) {
@@ -448,10 +450,11 @@ ad_set_geometry(device_t dev)
 		      adp->total_secs / (adp->heads * adp->sectors),
 		      adp->heads, adp->sectors);
 
-    if (!(request = ata_alloc_request(dev)))
+    if (!(request = ata_alloc_request()))
 	return;
 
     /* get the max native size the device supports */
+    request->dev = dev;
     request->u.ata.command = ATA_READ_NATIVE_MAX_ADDRESS;
     request->u.ata.lba = 0;
     request->u.ata.count = 0;
diff --git a/sys/dev/ata/ata-dma.c b/sys/dev/ata/ata-dma.c
index 860e1ae..59f9279 100644
--- a/sys/dev/ata/ata-dma.c
+++ b/sys/dev/ata/ata-dma.c
@@ -45,6 +45,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/ata/ata-pci.h>
 
 /* prototypes */
+static void ata_dmafini(device_t dev);
+static void ata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
 static void ata_dmaalloc(device_t dev);
 static void ata_dmafree(device_t dev);
 static void ata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
@@ -67,6 +69,7 @@ void
 ata_dmainit(device_t dev)
 {
     struct ata_channel *ch = device_get_softc(dev);
+    struct ata_dc_cb_args dcba;
 
     ch->dma.alloc = ata_dmaalloc;
     ch->dma.free = ata_dmafree;
@@ -78,6 +81,59 @@ ata_dmainit(device_t dev)
     ch->dma.segsize = 63536;
     ch->dma.max_iosize = 128 * DEV_BSIZE;
     ch->dma.max_address = BUS_SPACE_MAXADDR_32BIT;
+    ch->dma.dma_slots = 1;
+
+    if (bus_dma_tag_create(bus_get_dma_tag(dev), ch->dma.alignment, 0,
+			   ch->dma.max_address, BUS_SPACE_MAXADDR,
+			   NULL, NULL, ch->dma.max_iosize,
+			   ATA_DMA_ENTRIES, ch->dma.segsize,
+			   0, NULL, NULL, &ch->dma.dmatag))
+	goto error;
+
+    if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, 64 * 1024,
+			   ch->dma.max_address, BUS_SPACE_MAXADDR,
+			   NULL, NULL, MAXWSPCSZ, 1, MAXWSPCSZ,
+			   0, NULL, NULL, &ch->dma.work_tag))
+	goto error;
+
+    if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 0,
+			 &ch->dma.work_map))
+	goto error;
+
+    if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work,
+			MAXWSPCSZ, ata_dmasetupc_cb, &dcba, 0) ||
+			dcba.error) {
+	bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
+	goto error;
+    }
+    ch->dma.work_bus = dcba.maddr;
+    return;
+
+error:
+    device_printf(dev, "WARNING - DMA initialization failed, disabling DMA\n");
+    ata_dmafini(dev);
+}
+
+void 
+ata_dmafini(device_t dev)
+{
+    struct ata_channel *ch = device_get_softc(dev);
+
+    if (ch->dma.work_bus) {
+	bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map);
+	bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
+	ch->dma.work_bus = 0;
+	ch->dma.work_map = NULL;
+	ch->dma.work = NULL;
+    }
+    if (ch->dma.work_tag) {
+	bus_dma_tag_destroy(ch->dma.work_tag);
+	ch->dma.work_tag = NULL;
+    }
+    if (ch->dma.dmatag) {
+	bus_dma_tag_destroy(ch->dma.dmatag);
+	ch->dma.dmatag = NULL;
+    }
 }
 
 static void
@@ -94,6 +150,7 @@ ata_dmaalloc(device_t dev)
 {
     struct ata_channel *ch = device_get_softc(dev);
     struct ata_dc_cb_args dcba;
+    int i;
 
     if (bus_dma_tag_create(bus_get_dma_tag(dev), ch->dma.alignment, 0,
 			   ch->dma.max_address, BUS_SPACE_MAXADDR,
@@ -120,6 +177,48 @@ ata_dmaalloc(device_t dev)
     }
     ch->dma.work_bus = dcba.maddr;
 
+    /* alloc and setup needed dma slots */
+    bzero(ch->dma.slot, sizeof(struct ata_dmaslot) * ATA_DMA_SLOTS);
+    for (i = 0; i < ch->dma.dma_slots; i++) {
+	struct ata_dmaslot *slot = &ch->dma.slot[i];
+
+	if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, PAGE_SIZE,
+			       ch->dma.max_address, BUS_SPACE_MAXADDR,
+			       NULL, NULL, PAGE_SIZE, 1, PAGE_SIZE,
+			       0, NULL, NULL, &slot->sg_tag)) {
+            device_printf(ch->dev, "FAILURE - create sg_tag\n");
+            goto error;
+	}
+
+	if (bus_dmamem_alloc(slot->sg_tag, (void **)&slot->sg,
+			     0, &slot->sg_map)) {
+	    device_printf(ch->dev, "FAILURE - alloc sg_map\n");
+	    goto error;
+        }
+
+	if (bus_dmamap_load(slot->sg_tag, slot->sg_map, slot->sg, MAXTABSZ,
+			    ata_dmasetupc_cb, &dcba, 0) || dcba.error) {
+	    device_printf(ch->dev, "FAILURE - load sg\n");
+	    goto error;
+	}
+	slot->sg_bus = dcba.maddr;
+
+	if (bus_dma_tag_create(ch->dma.dmatag,
+			       ch->dma.alignment, ch->dma.boundary,
+                               ch->dma.max_address, BUS_SPACE_MAXADDR,
+                               NULL, NULL, ch->dma.max_iosize,
+                               ATA_DMA_ENTRIES, ch->dma.segsize,
+                               BUS_DMA_ALLOCNOW, NULL, NULL, &slot->data_tag)) {
+	    device_printf(ch->dev, "FAILURE - create data_tag\n");
+	    goto error;
+	}
+
+	if (bus_dmamap_create(slot->data_tag, 0, &slot->data_map)) {
+	    device_printf(ch->dev, "FAILURE - create data_map\n");
+	    goto error;
+        }
+    }
+
     return;
 
 error:
@@ -131,21 +230,34 @@ static void
 ata_dmafree(device_t dev)
 {
     struct ata_channel *ch = device_get_softc(dev);
+    int i;
 
-    if (ch->dma.work_bus) {
-	bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map);
-	bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
-	ch->dma.work_bus = 0;
-	ch->dma.work_map = NULL;
-	ch->dma.work = NULL;
-    }
-    if (ch->dma.work_tag) {
-	bus_dma_tag_destroy(ch->dma.work_tag);
-	ch->dma.work_tag = NULL;
-    }
-    if (ch->dma.dmatag) {
-	bus_dma_tag_destroy(ch->dma.dmatag);
-	ch->dma.dmatag = NULL;
+    /* free all dma slots */
+    for (i = 0; i < ATA_DMA_SLOTS; i++) {
+	struct ata_dmaslot *slot = &ch->dma.slot[i];
+
+	if (slot->sg_bus) {
+            bus_dmamap_unload(slot->sg_tag, slot->sg_map);
+            slot->sg_bus = 0;
+	}
+	if (slot->sg_map) {
+            bus_dmamem_free(slot->sg_tag, slot->sg, slot->sg_map);
+            bus_dmamap_destroy(slot->sg_tag, slot->sg_map);
+            slot->sg = NULL;
+            slot->sg_map = NULL;
+	}
+	if (slot->data_map) {
+            bus_dmamap_destroy(slot->data_tag, slot->data_map);
+            slot->data_map = NULL;
+	}
+	if (slot->sg_tag) {
+            bus_dma_tag_destroy(slot->sg_tag);
+            slot->sg_tag = NULL;
+	}
+	if (slot->data_tag) {
+            bus_dma_tag_destroy(slot->data_tag);
+            slot->data_tag = NULL;
+	}
     }
 }
 
@@ -172,13 +284,13 @@ static int
 ata_dmaload(struct ata_request *request, void *addr, int *entries)
 {
     struct ata_channel *ch = device_get_softc(request->parent);
-    struct ata_dc_cb_args dcba;
+    struct ata_device *atadev = device_get_softc(request->dev);
     struct ata_dmasetprd_args dspa;
     int error;
 
     ATA_DEBUG_RQ(request, "dmaload");
 
-    if (request->dma.cur_iosize) {
+    if (request->dma) {
 	device_printf(request->dev,
 		      "FAILURE - already active DMA on this device\n");
 	return EIO;
@@ -201,27 +313,15 @@ ata_dmaload(struct ata_request *request, void *addr, int *entries)
 	return EIO;
     }
 
-    if (bus_dmamap_load(request->dma.sg_tag, request->dma.sg_map,
-			request->dma.sg, MAXTABSZ, ata_dmasetupc_cb, &dcba, 0)||
-			dcba.error) {
-	bus_dmamem_free(request->dma.sg_tag,
-			request->dma.sg, request->dma.sg_map);
-	device_printf(request->dev, "FAILURE - load sg\n");
-	goto error;
-    }
-    request->dma.sg_bus = dcba.maddr;
-
-    if (bus_dmamap_create(request->dma.data_tag, 0, &request->dma.data_map)) {
-	device_printf(request->dev, "FAILURE - create data_map\n");
-	goto error;
-    }
+    /* set our slot, unit for simplicity XXX SOS NCQ will change that */
+    request->dma = &ch->dma.slot[atadev->unit];
 
     if (addr)
 	dspa.dmatab = addr;
     else
-	dspa.dmatab = request->dma.sg;
+	dspa.dmatab = request->dma->sg;
 
-    if ((error = bus_dmamap_load(request->dma.data_tag, request->dma.data_map,
+    if ((error = bus_dmamap_load(request->dma->data_tag, request->dma->data_map,
 				 request->data, request->bytecount,
 				 ch->dma.setprd, &dspa, BUS_DMA_NOWAIT)) ||
 				 (error = dspa.error)) {
@@ -232,15 +332,11 @@ ata_dmaload(struct ata_request *request, void *addr, int *entries)
     if (entries)
 	*entries = dspa.nsegs;
 
-    bus_dmamap_sync(request->dma.sg_tag, request->dma.sg_map,
+    bus_dmamap_sync(request->dma->sg_tag, request->dma->sg_map,
 		    BUS_DMASYNC_PREWRITE);
-
-    bus_dmamap_sync(request->dma.data_tag, request->dma.data_map,
+    bus_dmamap_sync(request->dma->data_tag, request->dma->data_map,
 		    (request->flags & ATA_R_READ) ?
 		    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
-
-    request->dma.cur_iosize = request->bytecount;
-
     return 0;
 
 error:
@@ -253,31 +349,15 @@ ata_dmaunload(struct ata_request *request)
 {
     ATA_DEBUG_RQ(request, "dmaunload");
 
-    if (request->dma.cur_iosize) {
-	bus_dmamap_sync(request->dma.sg_tag, request->dma.sg_map,
+    if (request->dma) {
+	bus_dmamap_sync(request->dma->sg_tag, request->dma->sg_map,
 			BUS_DMASYNC_POSTWRITE);
-
-	bus_dmamap_sync(request->dma.data_tag, request->dma.data_map,
+	bus_dmamap_sync(request->dma->data_tag, request->dma->data_map,
 			(request->flags & ATA_R_READ) ?
 			BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
-	bus_dmamap_unload(request->dma.data_tag, request->dma.data_map);
 
-	request->dma.cur_iosize = 0;
+	bus_dmamap_unload(request->dma->data_tag, request->dma->data_map);
+        request->dma = NULL;
     }
-
-    if (request->dma.data_map) {
-	bus_dmamap_destroy(request->dma.data_tag, request->dma.data_map);
-	request->dma.data_map = NULL;
-    }
-
-    if (request->dma.sg_bus) {
-	bus_dmamap_unload(request->dma.sg_tag, request->dma.sg_map);
-	bus_dmamem_free(request->dma.sg_tag, request->dma.sg,
-			request->dma.sg_map);
-	request->dma.sg = NULL;
-	request->dma.sg_bus = 0;
-	request->dma.sg_map = NULL;
-    }
-
     return 0;
 }
diff --git a/sys/dev/ata/ata-lowlevel.c b/sys/dev/ata/ata-lowlevel.c
index 5d552d4..6dd7662 100644
--- a/sys/dev/ata/ata-lowlevel.c
+++ b/sys/dev/ata/ata-lowlevel.c
@@ -313,12 +313,12 @@ ata_end_transaction(struct ata_request *request)
 
 	/* stop DMA engine and get status */
 	if (ch->dma.stop)
-	    request->dma.status = ch->dma.stop(request);
+	    request->dma->status = ch->dma.stop(request);
 
 	/* did we get error or data */
 	if (request->status & ATA_S_ERROR)
 	    request->error = ATA_IDX_INB(ch, ATA_ERROR);
-	else if (request->dma.status & ATA_BMSTAT_ERROR)
+	else if (request->dma->status & ATA_BMSTAT_ERROR)
 	    request->status |= ATA_S_ERROR;
 	else if (!(request->flags & ATA_R_TIMEOUT))
 	    request->donecount = request->bytecount;
@@ -425,12 +425,12 @@ ata_end_transaction(struct ata_request *request)
 
 	/* stop DMA engine and get status */
 	if (ch->dma.stop)
-	    request->dma.status = ch->dma.stop(request);
+	    request->dma->status = ch->dma.stop(request);
 
 	/* did we get error or data */
 	if (request->status & (ATA_S_ERROR | ATA_S_DWF))
 	    request->error = ATA_IDX_INB(ch, ATA_ERROR);
-	else if (request->dma.status & ATA_BMSTAT_ERROR)
+	else if (request->dma->status & ATA_BMSTAT_ERROR)
 	    request->status |= ATA_S_ERROR;
 	else if (!(request->flags & ATA_R_TIMEOUT))
 	    request->donecount = request->bytecount;
diff --git a/sys/dev/ata/ata-pci.c b/sys/dev/ata/ata-pci.c
index ab5342b..cb964fb 100644
--- a/sys/dev/ata/ata-pci.c
+++ b/sys/dev/ata/ata-pci.c
@@ -464,7 +464,7 @@ ata_pci_dmastart(struct ata_request *request)
 
     ATA_IDX_OUTB(ch, ATA_BMSTAT_PORT, (ATA_IDX_INB(ch, ATA_BMSTAT_PORT) | 
 		 (ATA_BMSTAT_INTERRUPT | ATA_BMSTAT_ERROR)));
-    ATA_IDX_OUTL(ch, ATA_BMDTP_PORT, request->dma.sg_bus);
+    ATA_IDX_OUTL(ch, ATA_BMDTP_PORT, request->dma->sg_bus);
     ch->dma.flags |= ATA_DMA_ACTIVE;
     ATA_IDX_OUTB(ch, ATA_BMCMD_PORT,
 		 (ATA_IDX_INB(ch, ATA_BMCMD_PORT) & ~ATA_BMCMD_WRITE_READ) |
@@ -609,17 +609,17 @@ ata_pcichannel_attach(device_t dev)
     struct ata_channel *ch = device_get_softc(dev);
     int error;
 
-    if (ctlr->dmainit) {
+    if (ctlr->dmainit)
 	ctlr->dmainit(dev);
-	ch->dma.alloc(dev);
-    }
 
-    if ((error = ctlr->allocate(dev))) {
-	ch->dma.free(dev);
+    if ((error = ctlr->allocate(dev)))
 	return error;
-    }
 
-    return ata_attach(dev);
+    if ((error = ata_attach(dev)))
+	return error;
+
+    ch->dma.alloc(dev);
+    return 0;
 }
 
 static int
diff --git a/sys/dev/ata/ata-queue.c b/sys/dev/ata/ata-queue.c
index e20f0d1..5152faa 100644
--- a/sys/dev/ata/ata-queue.c
+++ b/sys/dev/ata/ata-queue.c
@@ -119,10 +119,11 @@ int
 ata_controlcmd(device_t dev, u_int8_t command, u_int16_t feature,
 	       u_int64_t lba, u_int16_t count)
 {
-    struct ata_request *request = ata_alloc_request(dev);
+    struct ata_request *request = ata_alloc_request();
     int error = ENOMEM;
 
     if (request) {
+	request->dev = dev;
 	request->u.ata.command = command;
 	request->u.ata.lba = lba;
 	request->u.ata.count = count;
@@ -141,11 +142,12 @@ int
 ata_atapicmd(device_t dev, u_int8_t *ccb, caddr_t data,
 	     int count, int flags, int timeout)
 {
-    struct ata_request *request = ata_alloc_request(dev);
+    struct ata_request *request = ata_alloc_request();
     struct ata_device *atadev = device_get_softc(dev);
     int error = ENOMEM;
 
     if (request) {
+	request->dev = dev;
 	if ((atadev->param.config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12)
 	    bcopy(ccb, request->u.atapi.ccb, 12);
 	else
@@ -356,8 +358,8 @@ ata_completed(void *context, int dummy)
 			      "\4MEDIA_CHANGE_REQEST"
 			      "\3ABORTED\2NO_MEDIA\1ILLEGAL_LENGTH");
 		if ((request->flags & ATA_R_DMA) &&
-		    (request->dma.status & ATA_BMSTAT_ERROR))
-		    printf(" dma=0x%02x", request->dma.status);
+		    (request->dma->status & ATA_BMSTAT_ERROR))
+		    printf(" dma=0x%02x", request->dma->status);
 		if (!(request->flags & (ATA_R_ATAPI | ATA_R_CONTROL)))
 		    printf(" LBA=%ju", request->u.ata.lba);
 		printf("\n");
diff --git a/sys/dev/ata/ata-raid.c b/sys/dev/ata/ata-raid.c
index d7dfac0..40a880c 100644
--- a/sys/dev/ata/ata-raid.c
+++ b/sys/dev/ata/ata-raid.c
@@ -4027,10 +4027,11 @@ ata_raid_init_request(device_t dev, struct ar_softc *rdp, struct bio *bio)
 {
     struct ata_request *request;
 
-    if (!(request = ata_alloc_request(dev))) {
+    if (!(request = ata_alloc_request())) {
 	printf("FAILURE - out of memory in ata_raid_init_request\n");
 	return NULL;
     }
+    request->dev = dev;
     request->timeout = 5;
     request->retries = 2;
     request->callback = ata_raid_done;
@@ -4098,12 +4099,13 @@ ata_raid_rw(device_t dev, u_int64_t lba, void *data, u_int bcount, int flags)
 	return ENOMEM;
     }
 	
-    if (!(request = ata_alloc_request(dev))) {
+    if (!(request = ata_alloc_request())) {
 	device_printf(dev, "FAILURE - out of memory in ata_raid_rw\n");
 	return ENOMEM;
     }
 
     /* setup request */
+    request->dev = dev;
     request->timeout = 10;
     request->retries = 0;
     request->data = data;
diff --git a/sys/dev/ata/atapi-cam.c b/sys/dev/ata/atapi-cam.c
index 6e4a2ab..d8f0b86 100644
--- a/sys/dev/ata/atapi-cam.c
+++ b/sys/dev/ata/atapi-cam.c
@@ -531,7 +531,7 @@ atapi_action(struct cam_sim *sim, union ccb *ccb)
 	    printf("cannot allocate ATAPI/CAM hcb\n");
 	    goto action_oom;
 	}
-	if ((request = ata_alloc_request(softc->atadev[tid]->dev)) == NULL) {
+	if ((request = ata_alloc_request()) == NULL) {
 	    printf("cannot allocate ATAPI/CAM request\n");
 	    goto action_oom;
 	}
@@ -618,6 +618,7 @@ atapi_action(struct cam_sim *sim, union ccb *ccb)
 		goto action_oom;
 	    }
 	}
+	request->dev = softc->atadev[tid]->dev;
 	request->driver = hcb;
 	request->data = buf;
 	request->bytecount = len;
diff --git a/sys/dev/ata/atapi-cd.c b/sys/dev/ata/atapi-cd.c
index f9fcf97..7bf3b9f 100644
--- a/sys/dev/ata/atapi-cd.c
+++ b/sys/dev/ata/atapi-cd.c
@@ -691,11 +691,12 @@ acd_geom_access(struct g_provider *pp, int dr, int dw, int de)
 		       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
     int timeout = 60, track;
 
-    if (!(request = ata_alloc_request(dev)))
+    if (!(request = ata_alloc_request()))
 	return ENOMEM;
 
     /* wait if drive is not finished loading the medium */
     while (timeout--) {
+	request->dev = dev;
 	bcopy(ccb, request->u.atapi.ccb, 16);
 	request->flags = ATA_R_ATAPI;
 	request->timeout = 5;
@@ -855,10 +856,11 @@ acd_strategy(struct bio *bp)
     ccb[7] = count>>8;
     ccb[8] = count;
 
-    if (!(request = ata_alloc_request(dev))) {
+    if (!(request = ata_alloc_request())) {
 	g_io_deliver(bp, ENOMEM);
 	return;
     }
+    request->dev = dev;
     request->bio = bp;
     bcopy(ccb, request->u.atapi.ccb,
 	  (atadev->param.config & ATA_PROTO_MASK) == 
@@ -1218,9 +1220,10 @@ acd_get_progress(device_t dev, int *finished)
     struct ata_request *request;
     int8_t dummy[8];
 
-    if (!(request = ata_alloc_request(dev)))
+    if (!(request = ata_alloc_request()))
 	return ENOMEM;
 
+    request->dev = dev;
     bcopy(ccb, request->u.atapi.ccb, 16);
     request->data = dummy;
     request->bytecount = sizeof(dummy);
diff --git a/sys/dev/ata/atapi-fd.c b/sys/dev/ata/atapi-fd.c
index 45a1b17..9a26399 100644
--- a/sys/dev/ata/atapi-fd.c
+++ b/sys/dev/ata/atapi-fd.c
@@ -233,10 +233,11 @@ afd_strategy(struct bio *bp)
     ccb[7] = count>>8;
     ccb[8] = count;
 
-    if (!(request = ata_alloc_request(dev))) {
+    if (!(request = ata_alloc_request())) {
 	biofinish(bp, NULL, ENOMEM);
 	return;
     }
+    request->dev = dev;
     request->bio = bp;
     bcopy(ccb, request->u.atapi.ccb,
 	  (atadev->param.config & ATA_PROTO_MASK) == 
diff --git a/sys/dev/ata/atapi-tape.c b/sys/dev/ata/atapi-tape.c
index aa65226..3304907 100644
--- a/sys/dev/ata/atapi-tape.c
+++ b/sys/dev/ata/atapi-tape.c
@@ -421,10 +421,11 @@ ast_strategy(struct bio *bp)
     ccb[3] = blkcount >> 8;
     ccb[4] = blkcount;
 
-    if (!(request = ata_alloc_request(dev))) {
+    if (!(request = ata_alloc_request())) {
 	biofinish(bp, NULL, ENOMEM);
 	return;
     }
+    request->dev = dev;
     request->driver = bp;
     bcopy(ccb, request->u.atapi.ccb,
 	  (atadev->param.config & ATA_PROTO_MASK) ==