Bring aac out from under Giant:

- the mutex aac_io_lock protects the main codepaths which handle queues and
  hardware registers.  Only one acquire/release is done in the top-half and
  the taskqueue.  This mutex also applies to the userland command path and
  CAM data path.
- Move the taskqueue to the new Giant-free version.
- Register the disk device with DISKFLAG_NOGIANT so the top-half processing
  runs without Giant.
- Move the dynamic command allocator to the worker thread to avoid locking
  issues with bus_dmamem_alloc().

This gives about 20% improvement in most of my benchmarks.

4 files changed, 46 insertions, 18 deletions

diff --git a/sys/dev/aac/aac.c b/sys/dev/aac/aac.c
index 904c096..a14f79c 100644
--- a/sys/dev/aac/aac.c
+++ b/sys/dev/aac/aac.c
@@ -270,16 +270,16 @@ aac_attach(struct aac_softc *sc)
 	aac_describe_controller(sc);
 
 	/*
-	 * Register to probe our containers later.
+	 * Initialize locks
 	 */
+	AAC_LOCK_INIT(&sc->aac_aifq_lock, "AAC AIF lock");
 	TAILQ_INIT(&sc->aac_container_tqh);
 	AAC_LOCK_INIT(&sc->aac_container_lock, "AAC container lock");
+	AAC_LOCK_INIT(&sc->aac_io_lock, "AAC I/O lock");
 
 	/*
-	 * Lock for the AIF queue
+	 * Register to probe our containers later.
 	 */
-	AAC_LOCK_INIT(&sc->aac_aifq_lock, "AAC AIF lock");
-
 	sc->aac_ich.ich_func = aac_startup;
 	sc->aac_ich.ich_arg = sc;
 	if (config_intrhook_establish(&sc->aac_ich) != 0) {
@@ -660,7 +660,7 @@ aac_intr(void *arg)
 	/* It's not ok to return here because of races with the previous step */
 	if (reason & AAC_DB_RESPONSE_READY)
 		/* handle completion processing */
-		taskqueue_enqueue(taskqueue_swi_giant, &sc->aac_task_complete);
+		taskqueue_enqueue(taskqueue_swi, &sc->aac_task_complete);
 
 	/* controller wants to talk to the log */
 	if (reason & AAC_DB_PRINTF) {
@@ -777,7 +777,6 @@ aac_command_thread(struct aac_softc *sc)
 			tsleep(sc->aifthread, PRIBIO, "aifthd",
 			       AAC_PERIODIC_INTERVAL * hz);
 
-		/* While we're here, check to see if any commands are stuck */
 		if ((sc->aifflags & AAC_AIFFLAGS_PENDING) == 0)
 			aac_timeout(sc);
 
@@ -787,8 +786,18 @@ aac_command_thread(struct aac_softc *sc)
 			aac_print_printf(sc);
 		}
 
-		while (sc->aifflags & AAC_AIFFLAGS_AIF) {
+		/* See if any FIBs need to be allocated */
+		if ((sc->aifflags & AAC_AIFFLAGS_ALLOCFIBS) != 0) {
+			mtx_lock(&Giant);
+			AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
+			aac_alloc_commands(sc);
+			sc->aifflags &= ~AAC_AIFFLAGS_ALLOCFIBS;
+			AAC_LOCK_RELEASE(&sc->aac_io_lock);
+			mtx_unlock(&Giant);
+		}
 
+		/* While we're here, check to see if any commands are stuck */
+		while (sc->aifflags & AAC_AIFFLAGS_AIF) {
 			if (aac_dequeue_fib(sc, AAC_HOST_NORM_CMD_QUEUE,
 					    &fib_size, &fib)) {
 				sc->aifflags &= ~AAC_AIFFLAGS_AIF;
@@ -857,6 +866,8 @@ aac_complete(void *context, int pending)
 
 	sc = (struct aac_softc *)context;
 
+	AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
+
 	/* pull completed commands off the queue */
 	for (;;) {
 		/* look for completed FIBs on our queue */
@@ -886,6 +897,8 @@ aac_complete(void *context, int pending)
 
 	/* see if we can start some more I/O */
 	aac_startio(sc);
+
+	AAC_LOCK_RELEASE(&sc->aac_io_lock);
 }
 
 /*
@@ -1035,19 +1048,20 @@ aac_bio_complete(struct aac_command *cm)
 static int
 aac_wait_command(struct aac_command *cm, int timeout)
 {
-	int s, error = 0;
+	struct aac_softc *sc;
+	int error = 0;
 
 	debug_called(2);
 
+	sc = cm->cm_sc;
+
 	/* Put the command on the ready queue and get things going */
 	cm->cm_queue = AAC_ADAP_NORM_CMD_QUEUE;
 	aac_enqueue_ready(cm);
-	aac_startio(cm->cm_sc);
-	s = splbio();
+	aac_startio(sc);
 	while (!(cm->cm_flags & AAC_CMD_COMPLETED) && (error != EWOULDBLOCK)) {
-		error = tsleep(cm, PRIBIO, "aacwait", 0);
+		error = msleep(cm, &sc->aac_io_lock, PRIBIO, "aacwait", 0);
 	}
-	splx(s);
 	return(error);
 }
 
@@ -1066,9 +1080,9 @@ aac_alloc_command(struct aac_softc *sc, struct aac_command **cmp)
 	debug_called(3);
 
 	if ((cm = aac_dequeue_free(sc)) == NULL) {
-		if ((aac_alloc_commands(sc) != 0) ||
-		    (cm = aac_dequeue_free(sc)) == NULL)
-			return (ENOMEM);
+		sc->aifflags |= AAC_AIFFLAGS_ALLOCFIBS;
+		wakeup(sc->aifthread);
+		return (EBUSY);
 	}
 
 	*cmp = cm;
@@ -2364,6 +2378,7 @@ aac_ioctl_sendfib(struct aac_softc *sc, caddr_t ufib)
 	/*
 	 * Get a command
 	 */
+	AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
 	if (aac_alloc_command(sc, &cm)) {
 		error = EBUSY;
 		goto out;
@@ -2410,6 +2425,8 @@ out:
 	if (cm != NULL) {
 		aac_release_command(cm);
 	}
+
+	AAC_LOCK_RELEASE(&sc->aac_io_lock);
 	return(error);
 }
 
diff --git a/sys/dev/aac/aac_cam.c b/sys/dev/aac/aac_cam.c
index e1874be..b7e22b1 100644
--- a/sys/dev/aac/aac_cam.c
+++ b/sys/dev/aac/aac_cam.c
@@ -286,7 +286,9 @@ aac_cam_action(struct cam_sim *sim, union ccb *ccb)
 
 	/* Async ops that require communcation with the controller */
 
+	AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
 	if (aac_alloc_command(sc, &cm)) {
+		AAC_LOCK_RELEASE(&sc->aac_io_lock);
 		xpt_freeze_simq(sim, 1);
 		ccb->ccb_h.status = CAM_REQUEUE_REQ;
 		xpt_done(ccb);
@@ -402,6 +404,8 @@ aac_cam_action(struct cam_sim *sim, union ccb *ccb)
 	aac_enqueue_ready(cm);
 	aac_startio(cm->cm_sc);
 
+	AAC_LOCK_RELEASE(&sc->aac_io_lock);
+
 	return;
 }
 
diff --git a/sys/dev/aac/aac_disk.c b/sys/dev/aac/aac_disk.c
index 1160439..d178e42 100644
--- a/sys/dev/aac/aac_disk.c
+++ b/sys/dev/aac/aac_disk.c
@@ -176,10 +176,13 @@ aac_disk_strategy(struct bio *bp)
 	}
 
 	/* perform accounting */
-	devstat_start_transaction(&sc->ad_stats);
 
 	/* pass the bio to the controller - it can work out who we are */
+	AAC_LOCK_ACQUIRE(&sc->ad_controller->aac_io_lock);
+	devstat_start_transaction(&sc->ad_stats);
 	aac_submit_bio(bp);
+	AAC_LOCK_RELEASE(&sc->ad_controller->aac_io_lock);
+
 	return;
 }
 
@@ -369,7 +372,7 @@ aac_disk_attach(device_t dev)
 	sc->ad_disk.d_mediasize = (off_t)sc->ad_size * AAC_BLOCK_SIZE;
 	sc->ad_disk.d_fwsectors = sc->ad_sectors;
 	sc->ad_disk.d_fwheads = sc->ad_heads;
-	disk_create(sc->unit, &sc->ad_disk, 0, NULL, NULL);
+	disk_create(sc->unit, &sc->ad_disk, DISKFLAG_NOGIANT, NULL, NULL);
 
 	return (0);
 }
diff --git a/sys/dev/aac/aacvar.h b/sys/dev/aac/aacvar.h
index 31a845c..f704ef1 100644
--- a/sys/dev/aac/aacvar.h
+++ b/sys/dev/aac/aacvar.h
@@ -355,6 +355,8 @@ struct aac_softc
 #define AAC_SYNC_LOCK_FORCE	(1 << 0)
 	aac_lock_t		aac_sync_lock;
 
+	aac_lock_t		aac_io_lock;
+
 	/* delayed activity infrastructure */
 #if __FreeBSD_version >= 500005
 	struct task		aac_task_complete;	/* deferred-completion
@@ -376,7 +378,9 @@ struct aac_softc
 #define	AAC_AIFFLAGS_EXIT	(1 << 2)
 #define AAC_AIFFLAGS_EXITED	(1 << 3)
 #define AAC_AIFFLAGS_PRINTF	(1 << 4)
-#define AAC_AIFFLAGS_PENDING	(AAC_AIFFLAGS_AIF | AAC_AIFFLAGS_PRINTF)
+#define	AAC_AIFFLAGS_ALLOCFIBS	(1 << 5)
+#define AAC_AIFFLAGS_PENDING	(AAC_AIFFLAGS_AIF | AAC_AIFFLAGS_PRINTF | \
+				 AAC_AIFFLAGS_ALLOCFIBS)
 	u_int32_t		quirks;
 #define AAC_QUIRK_PERC2QC	(1 << 0)
 #define	AAC_QUIRK_NOCAM		(1 << 1)	/* No SCSI passthrough */