From 3678d31f1762251640dd504e43143335331df1d0 Mon Sep 17 00:00:00 2001 From: sos Date: Wed, 28 Jan 2004 20:38:51 +0000 Subject: Use the biotask functionality in GEOM to put finished requests on instead of taskqueue_swi. This shaves from 1 to 10% of the overhead. Overhaul the locking once more, there was a few possible races that are now closed. --- sys/dev/ata/ata-all.h | 2 +- sys/dev/ata/ata-disk.c | 5 ++--- sys/dev/ata/ata-lowlevel.c | 17 ++++++++--------- sys/dev/ata/ata-queue.c | 34 ++++++++++++++++++++-------------- sys/dev/ata/atapi-cd.c | 2 +- sys/dev/ata/atapi-fd.c | 4 ++-- sys/dev/ata/atapi-tape.c | 4 ++-- 7 files changed, 36 insertions(+), 32 deletions(-) (limited to 'sys/dev') diff --git a/sys/dev/ata/ata-all.h b/sys/dev/ata/ata-all.h index 495f5ec..2f93a68 100644 --- a/sys/dev/ata/ata-all.h +++ b/sys/dev/ata/ata-all.h @@ -195,7 +195,6 @@ struct ata_request { #define ATA_R_ORDERED 0x0100 #define ATA_R_IMMEDIATE 0x0200 #define ATA_R_REQUEUE 0x0400 -#define ATA_R_SKIPSTART 0x0800 #define ATA_R_DEBUG 0x1000 @@ -206,6 +205,7 @@ struct ata_request { struct callout_handle timeout_handle; /* handle for untimeout */ int result; /* result error code */ struct task task; /* task management */ + struct bio *bio; /* bio for this request */ TAILQ_ENTRY(ata_request) sequence; /* sequence management */ TAILQ_ENTRY(ata_request) chain; /* list management */ }; diff --git a/sys/dev/ata/ata-disk.c b/sys/dev/ata/ata-disk.c index 6d39171..25b9ef9 100644 --- a/sys/dev/ata/ata-disk.c +++ b/sys/dev/ata/ata-disk.c @@ -253,7 +253,7 @@ ad_start(struct ata_device *atadev) /* setup request */ request->device = atadev; - request->driver = bp; + request->bio = bp; request->callback = ad_done; request->timeout = 5; request->retries = 2; @@ -306,14 +306,13 @@ ad_start(struct ata_device *atadev) biofinish(bp, NULL, EIO); return; } - request->flags |= ATA_R_SKIPSTART; ata_queue_request(request); } static void ad_done(struct ata_request *request) { - struct bio *bp = request->driver; + struct bio *bp = request->bio; /* finish up transfer */ if ((bp->bio_error = request->result)) diff --git a/sys/dev/ata/ata-lowlevel.c b/sys/dev/ata/ata-lowlevel.c index 67ed3d1..740f5b7 100644 --- a/sys/dev/ata/ata-lowlevel.c +++ b/sys/dev/ata/ata-lowlevel.c @@ -359,6 +359,9 @@ ata_interrupt(void *data) min((request->bytecount - request->donecount), request->transfersize); + /* clear interrupt seen flag as we need to wait again */ + request->flags &= ~ATA_R_INTR_SEEN; + /* if data write command, output the data */ if (request->flags & ATA_R_WRITE) { @@ -510,19 +513,15 @@ ata_interrupt(void *data) break; } - /* if we timed out, we hold on to the channel, ata_reinit() will unlock */ - if (request->flags & ATA_R_TIMEOUT) { - ata_finish(request); - return; + /* if we timed out the unlocking of the ATA channel is done later */ + if (!(request->flags & ATA_R_TIMEOUT)) { + ch->running = NULL; + ATA_UNLOCK_CH(ch); + ch->locking(ch, ATA_LF_UNLOCK); } /* schedule completition for this request */ ata_finish(request); - - /* unlock the ATA channel for new work */ - ch->running = NULL; - ATA_UNLOCK_CH(ch); - ch->locking(ch, ATA_LF_UNLOCK); } /* must be called with ATA channel locked */ diff --git a/sys/dev/ata/ata-queue.c b/sys/dev/ata/ata-queue.c index 24c9b50..5a28351 100644 --- a/sys/dev/ata/ata-queue.c +++ b/sys/dev/ata/ata-queue.c @@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -87,9 +88,7 @@ ata_queue_request(struct ata_request *request) ATA_DEBUG_RQ(request, "queued"); - /* should we skip start to avoid lock recursion ? */ - if (!(request->flags & ATA_R_SKIPSTART)) - ata_start(request->device->channel); + ata_start(request->device->channel); } /* if this is a requeued request callback/sleep has been setup */ @@ -162,13 +161,14 @@ ata_start(struct ata_channel *ch) return; /* lock the ATA HW for this request */ + mtx_lock(&ch->queue_mtx); ch->locking(ch, ATA_LF_LOCK); if (!ATA_LOCK_CH(ch, ATA_ACTIVE)) { + mtx_unlock(&ch->queue_mtx); return; } /* if we dont have any work, ask the subdriver(s) */ - mtx_lock(&ch->queue_mtx); if (TAILQ_EMPTY(&ch->ata_queue)) { mtx_unlock(&ch->queue_mtx); if (ch->device[MASTER].start) @@ -189,17 +189,20 @@ ata_start(struct ata_channel *ch) timeout((timeout_t*)ata_timeout, request, request->timeout*hz); } - /* kick HW into action */ + /* kick HW into action and wait for interrupt if it flies*/ if (ch->hw.transaction(request) == ATA_OP_CONTINUES) return; - - ata_finish(request); } - else - mtx_unlock(&ch->queue_mtx); + /* unlock ATA channel HW */ ATA_UNLOCK_CH(ch); ch->locking(ch, ATA_LF_UNLOCK); + + /* if we have a request here it failed and should be completed */ + if (request) + ata_finish(request); + else + mtx_unlock(&ch->queue_mtx); } void @@ -212,14 +215,18 @@ ata_finish(struct ata_request *request) ata_completed(request, 0); } else { - TASK_INIT(&request->task, 0, ata_completed, request); - taskqueue_enqueue(taskqueue_swi, &request->task); + if (request->bio) + bio_taskqueue(request->bio, (bio_task_t *)ata_completed, request); + else { + TASK_INIT(&request->task, 0, ata_completed, request); + taskqueue_enqueue(taskqueue_swi, &request->task); + } } } /* current command finished, clean up and return result */ static void -ata_completed(void *context, int pending) +ata_completed(void *context, int dummy) { struct ata_request *request = (struct ata_request *)context; struct ata_channel *channel = request->device->channel; @@ -231,7 +238,7 @@ ata_completed(void *context, int pending) /* if retries still permit, reinject this request */ if (request->retries-- > 0) { - request->flags &= ~(ATA_R_TIMEOUT | ATA_R_SKIPSTART); + request->flags &= ~ATA_R_TIMEOUT; request->flags |= (ATA_R_IMMEDIATE | ATA_R_REQUEUE); ata_queue_request(request); return; @@ -269,7 +276,6 @@ ata_completed(void *context, int pending) if (!(request->flags & (ATA_R_ATAPI | ATA_R_CONTROL))) printf(" LBA=%llu", (unsigned long long)request->u.ata.lba); printf("\n"); - request->flags &= ~ATA_R_SKIPSTART; request->flags |= (ATA_R_IMMEDIATE | ATA_R_REQUEUE); ata_queue_request(request); return; diff --git a/sys/dev/ata/atapi-cd.c b/sys/dev/ata/atapi-cd.c index 3896917..3886564 100644 --- a/sys/dev/ata/atapi-cd.c +++ b/sys/dev/ata/atapi-cd.c @@ -1153,7 +1153,7 @@ acd_start(struct ata_device *atadev) request->timeout = (ccb[0] == ATAPI_WRITE_BIG) ? 60 : 30; request->retries = 2; request->callback = acd_done; - request->flags = ATA_R_SKIPSTART | ATA_R_ATAPI; + request->flags = ATA_R_ATAPI; if (request->device->mode >= ATA_DMA) request->flags |= ATA_R_DMA; switch (bp->bio_cmd) { diff --git a/sys/dev/ata/atapi-fd.c b/sys/dev/ata/atapi-fd.c index 4a7de17..be31d10 100644 --- a/sys/dev/ata/atapi-fd.c +++ b/sys/dev/ata/atapi-fd.c @@ -361,10 +361,10 @@ afd_start(struct ata_device *atadev) request->callback = afd_done; switch (bp->bio_cmd) { case BIO_READ: - request->flags |= (ATA_R_SKIPSTART | ATA_R_ATAPI | ATA_R_READ); + request->flags |= (ATA_R_ATAPI | ATA_R_READ); break; case BIO_WRITE: - request->flags |= (ATA_R_SKIPSTART | ATA_R_ATAPI | ATA_R_WRITE); + request->flags |= (ATA_R_ATAPI | ATA_R_WRITE); break; default: ata_prtdev(atadev, "unknown BIO operation\n"); diff --git a/sys/dev/ata/atapi-tape.c b/sys/dev/ata/atapi-tape.c index b8ef904..cd45cd0 100644 --- a/sys/dev/ata/atapi-tape.c +++ b/sys/dev/ata/atapi-tape.c @@ -527,10 +527,10 @@ ast_start(struct ata_device *atadev) request->callback = ast_done; switch (bp->bio_cmd) { case BIO_READ: - request->flags |= (ATA_R_SKIPSTART | ATA_R_ATAPI | ATA_R_READ); + request->flags |= (ATA_R_ATAPI | ATA_R_READ); break; case BIO_WRITE: - request->flags |= (ATA_R_SKIPSTART | ATA_R_ATAPI | ATA_R_WRITE); + request->flags |= (ATA_R_ATAPI | ATA_R_WRITE); break; default: ata_prtdev(atadev, "unknown BIO operation\n"); -- cgit v1.1