Use UMA instead of plain malloc for getting ATA request storage.

This gives +10% performance on simple tests, so definitly worth it.
A few percent more could be had by not using M_ZERO'd alloc's, but
we then need to clear fields all over the place to be safe, and
that was deemed not worth the trouble (and it makes life dangerous).

16 files changed, 32 insertions, 36 deletions

diff --git a/sys/dev/ata/ata-all.c b/sys/dev/ata/ata-all.c
index f8bccf1..a194694 100644
--- a/sys/dev/ata/ata-all.c
+++ b/sys/dev/ata/ata-all.c
@@ -40,9 +40,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/bio.h>
 #include <sys/malloc.h>
-#include <sys/sema.h>
 #include <sys/sysctl.h>
+#include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
@@ -77,6 +78,7 @@ static void ata_init(void);
 MALLOC_DEFINE(M_ATA, "ATA generic", "ATA driver generic layer");
 struct intr_config_hook *ata_delayed_attach = NULL;
 devclass_t ata_devclass;
+uma_zone_t ata_zone;
 int ata_wc = 1;	 
 
 /* local vars */
@@ -997,10 +999,13 @@ ata_init(void)
 	free(ata_delayed_attach, M_TEMP);
     }
 
-    /* Register a handler to flush write caches on shutdown */
+    /* register handler to flush write caches on shutdown */
     if ((EVENTHANDLER_REGISTER(shutdown_post_sync, ata_shutdown,
 			       NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
 	printf("ata: shutdown event registration failed!\n");
 
+    /* init our UMA zone for ATA requests */
+    ata_zone = uma_zcreate("ata_request", sizeof(struct ata_request),
+			   NULL, NULL, NULL, NULL, 0, 0);
 }
 SYSINIT(atadev, SI_SUB_DRIVERS, SI_ORDER_SECOND, ata_init, NULL)
diff --git a/sys/dev/ata/ata-all.h b/sys/dev/ata/ata-all.h
index 821d17c..70a26a9 100644
--- a/sys/dev/ata/ata-all.h
+++ b/sys/dev/ata/ata-all.h
@@ -365,7 +365,6 @@ struct atapi_sense {
 
 /* externs */
 extern devclass_t ata_devclass;
-extern struct intr_config_hook *ata_delayed_attach;
 extern int ata_wc;
  
 /* public prototypes */
@@ -391,8 +390,6 @@ int ata_limit_mode(struct ata_device *atadev, int mode, int maxmode);
 /* ata-queue.c: */
 int ata_reinit(struct ata_channel *ch);
 void ata_start(struct ata_channel *ch);
-struct ata_request *ata_alloc_request(void);
-void ata_free_request(struct ata_request *request);
 int ata_controlcmd(struct ata_device *atadev, u_int8_t command, u_int16_t feature, u_int64_t lba, u_int16_t count);
 int ata_atapicmd(struct ata_device *atadev, u_int8_t *ccb, caddr_t data, int count, int flags, int timeout);
 void ata_queue_request(struct ata_request *request);
@@ -411,6 +408,11 @@ void atapi_cam_attach_bus(struct ata_channel *ch);
 void atapi_cam_detach_bus(struct ata_channel *ch);
 void atapi_cam_reinit_bus(struct ata_channel *ch);
 
+/* macros for alloc/free of ata_requests */
+extern uma_zone_t ata_zone;
+#define ata_alloc_request() uma_zalloc(ata_zone, M_NOWAIT | M_ZERO)
+#define ata_free_request(request) uma_zfree(ata_zone, request)
+
 /* macros for locking a channel */
 #define ATA_LOCK_CH(ch, value) \
 	atomic_cmpset_acq_int(&(ch)->state, ATA_IDLE, (value))
diff --git a/sys/dev/ata/ata-card.c b/sys/dev/ata/ata-card.c
index 08b172b..d3f9ca4 100644
--- a/sys/dev/ata/ata-card.c
+++ b/sys/dev/ata/ata-card.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
diff --git a/sys/dev/ata/ata-cbus.c b/sys/dev/ata/ata-cbus.c
index 8f065c1..008883a 100644
--- a/sys/dev/ata/ata-cbus.c
+++ b/sys/dev/ata/ata-cbus.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
diff --git a/sys/dev/ata/ata-chipset.c b/sys/dev/ata/ata-chipset.c
index 5e42085..d75bfbd 100644
--- a/sys/dev/ata/ata-chipset.c
+++ b/sys/dev/ata/ata-chipset.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
diff --git a/sys/dev/ata/ata-disk.c b/sys/dev/ata/ata-disk.c
index 9953268..50147a8 100644
--- a/sys/dev/ata/ata-disk.c
+++ b/sys/dev/ata/ata-disk.c
@@ -43,8 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
-#include <vm/vm.h>
-#include <vm/pmap.h>
+#include <vm/uma.h>
 #include <machine/md_var.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
diff --git a/sys/dev/ata/ata-dma.c b/sys/dev/ata/ata-dma.c
index f22cb8a..26e9c2d 100644
--- a/sys/dev/ata/ata-dma.c
+++ b/sys/dev/ata/ata-dma.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <sys/bus.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
diff --git a/sys/dev/ata/ata-isa.c b/sys/dev/ata/ata-isa.c
index dcbd8f1..6f42b01 100644
--- a/sys/dev/ata/ata-isa.c
+++ b/sys/dev/ata/ata-isa.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
diff --git a/sys/dev/ata/ata-lowlevel.c b/sys/dev/ata/ata-lowlevel.c
index 39eeb9a..67ed3d1 100644
--- a/sys/dev/ata/ata-lowlevel.c
+++ b/sys/dev/ata/ata-lowlevel.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 #include <dev/ata/ata-all.h>
diff --git a/sys/dev/ata/ata-pci.c b/sys/dev/ata/ata-pci.c
index 79a8eda..d9206be 100644
--- a/sys/dev/ata/ata-pci.c
+++ b/sys/dev/ata/ata-pci.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
diff --git a/sys/dev/ata/ata-queue.c b/sys/dev/ata/ata-queue.c
index aa47437..ca43afd 100644
--- a/sys/dev/ata/ata-queue.c
+++ b/sys/dev/ata/ata-queue.c
@@ -34,11 +34,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/ata.h>
 #include <sys/kernel.h>
-#include <sys/malloc.h>
 #include <sys/bus.h>
 #include <sys/conf.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 #include <dev/ata/ata-all.h>
@@ -48,36 +48,13 @@ static void ata_completed(void *, int);
 static void ata_timeout(struct ata_request *);
 static char *ata_skey2str(u_int8_t);
 
-/* local vars */
-static MALLOC_DEFINE(M_ATA_REQ, "ATA request", "ATA request");
-
-/*
- * ATA request related functions
- */
-struct ata_request *
-ata_alloc_request(void)
-{
-    struct ata_request *request;
-
-    request = malloc(sizeof(struct ata_request), M_ATA_REQ, M_NOWAIT | M_ZERO);
-    if (!request)
-	printf("FAILURE - malloc ATA request failed\n");
-    sema_init(&request->done, 0, "ATA request done");
-    return request;
-}
-
-void
-ata_free_request(struct ata_request *request)
-{
-    sema_destroy(&request->done);
-    free(request, M_ATA_REQ);
-}
-
 void
 ata_queue_request(struct ata_request *request)
 {
-    /* mark request as virgin (it might be a retry) */
+    /* mark request as virgin */
     request->result = request->status = request->error = 0;
+    if (!request->callback)
+	sema_init(&request->done, 0, "ATA request done");
 
     if (request->device->channel->flags & ATA_IMMEDIATE_MODE) {
 
@@ -122,6 +99,8 @@ ata_queue_request(struct ata_request *request)
 	    sema_wait(&request->done);
 	}
     }
+    if (!request->callback)
+	sema_destroy(&request->done);
 }
 
 int
diff --git a/sys/dev/ata/ata-raid.c b/sys/dev/ata/ata-raid.c
index 323c0ca..4ca8b2c 100644
--- a/sys/dev/ata/ata-raid.c
+++ b/sys/dev/ata/ata-raid.c
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kthread.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 #include <geom/geom_disk.h>
diff --git a/sys/dev/ata/atapi-cam.c b/sys/dev/ata/atapi-cam.c
index 5e26c22..d695b47 100644
--- a/sys/dev/ata/atapi-cam.c
+++ b/sys/dev/ata/atapi-cam.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sema.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 
 #include <cam/cam.h>
diff --git a/sys/dev/ata/atapi-cd.c b/sys/dev/ata/atapi-cd.c
index 5d897f9..9dea17d 100644
--- a/sys/dev/ata/atapi-cd.c
+++ b/sys/dev/ata/atapi-cd.c
@@ -45,8 +45,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/fcntl.h>
 #include <sys/conf.h>
 #include <sys/ctype.h>
-#include <sys/taskqueue.h>
 #include <sys/sema.h>
+#include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <geom/geom.h>
 #include <dev/ata/ata-all.h>
@@ -451,7 +452,6 @@ acd_describe(struct acd_softc *cdp)
 	       (cdp->device->unit == ATA_MASTER) ? "master" : "slave",
 	       ata_mode2str(cdp->device->mode) );
     }
-
 }
 
 static __inline void 
diff --git a/sys/dev/ata/atapi-fd.c b/sys/dev/ata/atapi-fd.c
index 4bcafc6..4a7de17 100644
--- a/sys/dev/ata/atapi-fd.c
+++ b/sys/dev/ata/atapi-fd.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/cdio.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <geom/geom_disk.h>
 #include <dev/ata/ata-all.h>
diff --git a/sys/dev/ata/atapi-tape.c b/sys/dev/ata/atapi-tape.c
index aa01594..b8ef904 100644
--- a/sys/dev/ata/atapi-tape.c
+++ b/sys/dev/ata/atapi-tape.c
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/devicestat.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <dev/ata/ata-all.h>
 #include <dev/ata/atapi-tape.h>