Use the biotask functionality in GEOM to put finished requests on

instead of taskqueue_swi. This shaves from 1 to 10% of the overhead.

Overhaul the locking once more, there was a few possible races that
are now closed.

7 files changed, 36 insertions, 32 deletions

diff --git a/sys/dev/ata/ata-all.h b/sys/dev/ata/ata-all.h
index 495f5ec..2f93a68 100644
--- a/sys/dev/ata/ata-all.h
+++ b/sys/dev/ata/ata-all.h
@@ -195,7 +195,6 @@ struct ata_request {
 #define		ATA_R_ORDERED		0x0100
 #define		ATA_R_IMMEDIATE		0x0200
 #define		ATA_R_REQUEUE		0x0400
-#define		ATA_R_SKIPSTART		0x0800
 
 #define		ATA_R_DEBUG		0x1000
 
@@ -206,6 +205,7 @@ struct ata_request {
     struct callout_handle	timeout_handle; /* handle for untimeout */
     int				result;		/* result error code */
     struct task			task;		/* task management */
+    struct bio			*bio;		/* bio for this request */
     TAILQ_ENTRY(ata_request)	sequence;	/* sequence management */
     TAILQ_ENTRY(ata_request)	chain;		/* list management */
 };
diff --git a/sys/dev/ata/ata-disk.c b/sys/dev/ata/ata-disk.c
index 6d39171..25b9ef9 100644
--- a/sys/dev/ata/ata-disk.c
+++ b/sys/dev/ata/ata-disk.c
@@ -253,7 +253,7 @@ ad_start(struct ata_device *atadev)
 
     /* setup request */
     request->device = atadev;
-    request->driver = bp;
+    request->bio = bp;
     request->callback = ad_done;
     request->timeout = 5;
     request->retries = 2;
@@ -306,14 +306,13 @@ ad_start(struct ata_device *atadev)
 	biofinish(bp, NULL, EIO);
 	return;
     }
-    request->flags |= ATA_R_SKIPSTART;
     ata_queue_request(request);
 }
 
 static void
 ad_done(struct ata_request *request)
 {
-    struct bio *bp = request->driver;
+    struct bio *bp = request->bio;
 
     /* finish up transfer */
     if ((bp->bio_error = request->result))
diff --git a/sys/dev/ata/ata-lowlevel.c b/sys/dev/ata/ata-lowlevel.c
index 67ed3d1..740f5b7 100644
--- a/sys/dev/ata/ata-lowlevel.c
+++ b/sys/dev/ata/ata-lowlevel.c
@@ -359,6 +359,9 @@ ata_interrupt(void *data)
 		    min((request->bytecount - request->donecount),
 			request->transfersize);
 
+		/* clear interrupt seen flag as we need to wait again */
+		request->flags &= ~ATA_R_INTR_SEEN;
+
 		/* if data write command, output the data */
 		if (request->flags & ATA_R_WRITE) {
 
@@ -510,19 +513,15 @@ ata_interrupt(void *data)
 	break;
     }
 
-    /* if we timed out, we hold on to the channel, ata_reinit() will unlock */
-    if (request->flags & ATA_R_TIMEOUT) {
-	ata_finish(request);
-	return;
+    /* if we timed out the unlocking of the ATA channel is done later */
+    if (!(request->flags & ATA_R_TIMEOUT)) {
+	ch->running = NULL;
+	ATA_UNLOCK_CH(ch);
+	ch->locking(ch, ATA_LF_UNLOCK);
     }
 
     /* schedule completition for this request */
     ata_finish(request);
-
-    /* unlock the ATA channel for new work */
-    ch->running = NULL;
-    ATA_UNLOCK_CH(ch);
-    ch->locking(ch, ATA_LF_UNLOCK);
 }
 
 /* must be called with ATA channel locked */
diff --git a/sys/dev/ata/ata-queue.c b/sys/dev/ata/ata-queue.c
index 24c9b50..5a28351 100644
--- a/sys/dev/ata/ata-queue.c
+++ b/sys/dev/ata/ata-queue.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/ata.h>
 #include <sys/kernel.h>
+#include <sys/bio.h>
 #include <sys/bus.h>
 #include <sys/conf.h>
 #include <sys/sema.h>
@@ -87,9 +88,7 @@ ata_queue_request(struct ata_request *request)
 
 	ATA_DEBUG_RQ(request, "queued");
 
-	/* should we skip start to avoid lock recursion ? */
-	if (!(request->flags & ATA_R_SKIPSTART))
-	    ata_start(request->device->channel);
+	ata_start(request->device->channel);
     }
 
     /* if this is a requeued request callback/sleep has been setup */
@@ -162,13 +161,14 @@ ata_start(struct ata_channel *ch)
 	return;
 
     /* lock the ATA HW for this request */
+    mtx_lock(&ch->queue_mtx);
     ch->locking(ch, ATA_LF_LOCK);
     if (!ATA_LOCK_CH(ch, ATA_ACTIVE)) {
+	mtx_unlock(&ch->queue_mtx);
 	return;
     }
 
     /* if we dont have any work, ask the subdriver(s) */
-    mtx_lock(&ch->queue_mtx);
     if (TAILQ_EMPTY(&ch->ata_queue)) {
 	mtx_unlock(&ch->queue_mtx);
 	if (ch->device[MASTER].start)
@@ -189,17 +189,20 @@ ata_start(struct ata_channel *ch)
 		timeout((timeout_t*)ata_timeout, request, request->timeout*hz);
 	}
 
-	/* kick HW into action */
+	/* kick HW into action and wait for interrupt if it flies*/
 	if (ch->hw.transaction(request) == ATA_OP_CONTINUES)
 	    return;
-
-	ata_finish(request);
     }
-    else
-	mtx_unlock(&ch->queue_mtx);
 
+    /* unlock ATA channel HW */
     ATA_UNLOCK_CH(ch);
     ch->locking(ch, ATA_LF_UNLOCK);
+
+    /* if we have a request here it failed and should be completed */
+    if (request)
+	ata_finish(request);
+    else
+	mtx_unlock(&ch->queue_mtx);
 }
 
 void
@@ -212,14 +215,18 @@ ata_finish(struct ata_request *request)
 	ata_completed(request, 0);
     }
     else {
-	TASK_INIT(&request->task, 0, ata_completed, request);
-	taskqueue_enqueue(taskqueue_swi, &request->task);
+	if (request->bio)
+	    bio_taskqueue(request->bio, (bio_task_t *)ata_completed, request);
+	else {
+	    TASK_INIT(&request->task, 0, ata_completed, request);
+	    taskqueue_enqueue(taskqueue_swi, &request->task);
+	}
     }
 }
 
 /* current command finished, clean up and return result */
 static void
-ata_completed(void *context, int pending)
+ata_completed(void *context, int dummy)
 {
     struct ata_request *request = (struct ata_request *)context;
     struct ata_channel *channel = request->device->channel;
@@ -231,7 +238,7 @@ ata_completed(void *context, int pending)
 
 	/* if retries still permit, reinject this request */
 	if (request->retries-- > 0) {
-	    request->flags &= ~(ATA_R_TIMEOUT | ATA_R_SKIPSTART);
+	    request->flags &= ~ATA_R_TIMEOUT;
 	    request->flags |= (ATA_R_IMMEDIATE | ATA_R_REQUEUE);
 	    ata_queue_request(request);
 	    return;
@@ -269,7 +276,6 @@ ata_completed(void *context, int pending)
 		if (!(request->flags & (ATA_R_ATAPI | ATA_R_CONTROL)))
 		    printf(" LBA=%llu", (unsigned long long)request->u.ata.lba);
 		printf("\n");
-		request->flags &= ~ATA_R_SKIPSTART;
 		request->flags |= (ATA_R_IMMEDIATE | ATA_R_REQUEUE);
 		ata_queue_request(request);
 		return;
diff --git a/sys/dev/ata/atapi-cd.c b/sys/dev/ata/atapi-cd.c
index 3896917..3886564 100644
--- a/sys/dev/ata/atapi-cd.c
+++ b/sys/dev/ata/atapi-cd.c
@@ -1153,7 +1153,7 @@ acd_start(struct ata_device *atadev)
     request->timeout = (ccb[0] == ATAPI_WRITE_BIG) ? 60 : 30;
     request->retries = 2;
     request->callback = acd_done;
-    request->flags = ATA_R_SKIPSTART | ATA_R_ATAPI;
+    request->flags = ATA_R_ATAPI;
     if (request->device->mode >= ATA_DMA)
 	request->flags |= ATA_R_DMA;
     switch (bp->bio_cmd) {
diff --git a/sys/dev/ata/atapi-fd.c b/sys/dev/ata/atapi-fd.c
index 4a7de17..be31d10 100644
--- a/sys/dev/ata/atapi-fd.c
+++ b/sys/dev/ata/atapi-fd.c
@@ -361,10 +361,10 @@ afd_start(struct ata_device *atadev)
     request->callback = afd_done;
     switch (bp->bio_cmd) {
     case BIO_READ:
-	request->flags |= (ATA_R_SKIPSTART | ATA_R_ATAPI | ATA_R_READ);
+	request->flags |= (ATA_R_ATAPI | ATA_R_READ);
 	break;
     case BIO_WRITE:
-	request->flags |= (ATA_R_SKIPSTART | ATA_R_ATAPI | ATA_R_WRITE);
+	request->flags |= (ATA_R_ATAPI | ATA_R_WRITE);
 	break;
     default:
 	ata_prtdev(atadev, "unknown BIO operation\n");
diff --git a/sys/dev/ata/atapi-tape.c b/sys/dev/ata/atapi-tape.c
index b8ef904..cd45cd0 100644
--- a/sys/dev/ata/atapi-tape.c
+++ b/sys/dev/ata/atapi-tape.c
@@ -527,10 +527,10 @@ ast_start(struct ata_device *atadev)
     request->callback = ast_done;
     switch (bp->bio_cmd) {
     case BIO_READ:
-	request->flags |= (ATA_R_SKIPSTART | ATA_R_ATAPI | ATA_R_READ);
+	request->flags |= (ATA_R_ATAPI | ATA_R_READ);
 	break;
     case BIO_WRITE:
-	request->flags |= (ATA_R_SKIPSTART | ATA_R_ATAPI | ATA_R_WRITE);
+	request->flags |= (ATA_R_ATAPI | ATA_R_WRITE);
 	break;
     default:
 	ata_prtdev(atadev, "unknown BIO operation\n");