Add new support for locking an ATA channel and use that throughout

the ATA/ATAPI driver. This solves the concurrency problem with
the new GEOM code, and also cuts a good deal of the patch size
in the upcoming MFC.

8 files changed, 70 insertions, 49 deletions

diff --git a/sys/dev/ata/ata-all.c b/sys/dev/ata/ata-all.c
index 2c63f77..870c4ea 100644
--- a/sys/dev/ata/ata-all.c
+++ b/sys/dev/ata/ata-all.c
@@ -218,11 +218,9 @@ ata_detach(device_t dev)
 	return ENXIO;
 
     /* make sure channel is not busy */
-    s = splbio();
-    while (!atomic_cmpset_int(&ch->active, ATA_IDLE, ATA_CONTROL))
-	tsleep((caddr_t)&s, PRIBIO, "atarel", hz/4);
-    splx(s);
+    ATA_SLEEPLOCK_CH(ch, ATA_CONTROL);
 
+    s = splbio();
 #ifdef DEV_ATADISK
     if (ch->devices & ATA_ATA_MASTER && ch->device[MASTER].driver)
 	ad_detach(&ch->device[MASTER], 1);
@@ -235,6 +233,7 @@ ata_detach(device_t dev)
     if (ch->devices & ATA_ATAPI_SLAVE && ch->device[SLAVE].driver)
 	atapi_detach(&ch->device[SLAVE]);
 #endif
+    splx(s);
 
     if (ch->device[MASTER].param) {
 	free(ch->device[MASTER].param, M_ATA);
@@ -260,7 +259,7 @@ ata_detach(device_t dev)
     ch->r_altio = NULL;
     ch->r_bmio = NULL;
     ch->r_irq = NULL;
-    ch->active = ATA_IDLE;
+    ATA_UNLOCK_CH(ch);
     return 0;
 }
 
@@ -276,7 +275,7 @@ ataioctl(dev_t dev, u_long cmd, caddr_t addr, int32_t flag, struct thread *td)
     struct ata_cmd *iocmd = (struct ata_cmd *)addr;
     struct ata_channel *ch;
     device_t device = devclass_get_device(ata_devclass, iocmd->channel);
-    int error, s;
+    int error;
 
     if (cmd != IOCATA)
 	return ENOTTY;
@@ -300,12 +299,8 @@ ataioctl(dev_t dev, u_long cmd, caddr_t addr, int32_t flag, struct thread *td)
 	case ATAREINIT:
 	    if (!device || !(ch = device_get_softc(device)))
 		return ENXIO;
-
-	    s = splbio();
-	    while (!atomic_cmpset_int(&ch->active, ATA_IDLE, ATA_ACTIVE))
-		tsleep((caddr_t)&s, PRIBIO, "atarin", hz/4);
+	    ATA_SLEEPLOCK_CH(ch, ATA_ACTIVE);
 	    error = ata_reinit(ch);
-	    splx(s);
 	    return error;
 
 #ifdef DEV_ATADISK
@@ -607,10 +602,12 @@ ata_start(struct ata_channel *ch)
 #if defined(DEV_ATAPICD) || defined(DEV_ATAPIFD) || defined(DEV_ATAPIST)
     struct atapi_request *atapi_request;
 #endif
+    int s;
 
-    if (!atomic_cmpset_int(&ch->active, ATA_IDLE, ATA_ACTIVE))
+    if (!ATA_LOCK_CH(ch, ATA_ACTIVE))
 	return;
 
+    s = splbio();
 #ifdef DEV_ATADISK
     /* find & call the responsible driver if anything on the ATA queue */
     if (TAILQ_EMPTY(&ch->ata_queue)) {
@@ -623,8 +620,10 @@ ata_start(struct ata_channel *ch)
 	TAILQ_REMOVE(&ch->ata_queue, ad_request, chain);
 	ch->active = ATA_ACTIVE_ATA;
 	ch->running = ad_request;
-	if (ad_transfer(ad_request) == ATA_OP_CONTINUES)
+	if (ad_transfer(ad_request) == ATA_OP_CONTINUES) {
+	    splx(s);
 	    return;
+	}
     }
 
 #endif
@@ -640,10 +639,13 @@ ata_start(struct ata_channel *ch)
 	TAILQ_REMOVE(&ch->atapi_queue, atapi_request, chain);
 	ch->active = ATA_ACTIVE_ATAPI;
 	ch->running = atapi_request;
-	if (atapi_transfer(atapi_request) == ATA_OP_CONTINUES)
+	if (atapi_transfer(atapi_request) == ATA_OP_CONTINUES) {
+	    splx(s);
 	    return;
+	}
     }
 #endif
+    splx(s);
     ch->active = ATA_IDLE;
 }
 
@@ -782,7 +784,7 @@ ata_reinit(struct ata_channel *ch)
 
     if (!ch->r_io || !ch->r_altio || !ch->r_irq)
 	return ENXIO;
-    ch->active = ATA_CONTROL;
+    ATA_FORCELOCK_CH(ch, ATA_CONTROL);
     ch->running = NULL;
     devices = ch->devices;
     ata_printf(ch, -1, "resetting devices .. ");
@@ -859,7 +861,7 @@ ata_reinit(struct ata_channel *ch)
     }
 #endif
     printf("done\n");
-    ch->active = ATA_IDLE;
+    ATA_UNLOCK_CH(ch);
     ata_start(ch);
     return 0;
 }
@@ -1037,7 +1039,7 @@ ata_command(struct ata_device *atadev, u_int8_t command,
 	if (atadev->channel->flags & ATA_QUEUED)
 	    ATA_OUTB(atadev->channel->r_altio, ATA_ALTSTAT, ATA_A_4BIT);
 
-	if (tsleep((caddr_t)atadev->channel, PRIBIO, "atacmd", 10 * hz) != 0) {
+	if (tsleep((caddr_t)atadev->channel, PRIBIO, "atacmd", 10 * hz)) {
 	    ata_prtdev(atadev, "timeout waiting for interrupt\n");
 	    atadev->channel->active &= ~ATA_WAIT_INTR;
 	    error = -1;
@@ -1086,10 +1088,6 @@ static void
 ata_change_mode(struct ata_device *atadev, int mode)
 {
     int umode, wmode, pmode;
-    int s = splbio();
-
-    while (!atomic_cmpset_int(&atadev->channel->active, ATA_IDLE, ATA_ACTIVE))
-	tsleep((caddr_t)&s, PRIBIO, "atachm", hz/4);
 
     umode = ata_umode(atadev->param);
     wmode = ata_wmode(atadev->param);
@@ -1111,11 +1109,11 @@ ata_change_mode(struct ata_device *atadev, int mode)
 	umode = -1;
 	wmode = -1;
     }
-    ata_dmainit(atadev->channel, atadev->unit, pmode, wmode, umode);
 
-    atadev->channel->active = ATA_IDLE;
+    ATA_SLEEPLOCK_CH(atadev->channel, ATA_ACTIVE);
+    ata_dmainit(atadev->channel, atadev->unit, pmode, wmode, umode);
+    ATA_UNLOCK_CH(atadev->channel);
     ata_start(atadev->channel);
-    splx(s);
 }
 
 int
diff --git a/sys/dev/ata/ata-all.h b/sys/dev/ata/ata-all.h
index 3f31e47..ee2bf2c 100644
--- a/sys/dev/ata/ata-all.h
+++ b/sys/dev/ata/ata-all.h
@@ -266,6 +266,20 @@ void ata_dmastart(struct ata_channel *, int, struct ata_dmaentry *, int);
 int ata_dmastatus(struct ata_channel *);
 int ata_dmadone(struct ata_channel *);
 
+/* macros for locking a channel */
+#define ATA_LOCK_CH(ch, value)\
+	atomic_cmpset_int(&(ch)->active, ATA_IDLE, (value))
+
+#define ATA_SLEEPLOCK_CH(ch, value)\
+	while (!atomic_cmpset_int(&(ch)->active, ATA_IDLE, (value)))\
+	    tsleep((caddr_t)&(ch), PRIBIO, "atalck", 1);
+
+#define ATA_FORCELOCK_CH(ch, value)\
+	(ch)->active = value;
+
+#define ATA_UNLOCK_CH(ch)\
+	(ch)->active = ATA_IDLE
+
 /* macros to hide busspace uglyness */
 #define ATA_INB(res, offset) \
 	bus_space_read_1(rman_get_bustag((res)), \
diff --git a/sys/dev/ata/ata-disk.c b/sys/dev/ata/ata-disk.c
index b97b009..dfe55c2 100644
--- a/sys/dev/ata/ata-disk.c
+++ b/sys/dev/ata/ata-disk.c
@@ -121,6 +121,7 @@ ad_attach(struct ata_device *atadev)
     adp->heads = atadev->param->heads;
     adp->sectors = atadev->param->sectors;
     adp->total_secs = atadev->param->cylinders * adp->heads * adp->sectors;	
+    bioq_init(&adp->queue);
 
     /* does this device need oldstyle CHS addressing */
     if (!ad_version(atadev->param->version_major) || 
@@ -137,6 +138,7 @@ ad_attach(struct ata_device *atadev)
 	atadev->param->lba_size48 > 268435455)
 	adp->total_secs = atadev->param->lba_size48;
     
+    ATA_SLEEPLOCK_CH(atadev->channel, ATA_CONTROL);
     /* use multiple sectors/interrupt if device supports it */
     adp->transfersize = DEV_BSIZE;
     if (ad_version(atadev->param->version_major)) {
@@ -185,6 +187,8 @@ ad_attach(struct ata_device *atadev)
 	    ata_prtdev(atadev, "disabling service interrupt failed\n");
     }
 
+    ATA_UNLOCK_CH(atadev->channel);
+
     devstat_add_entry(&adp->stats, "ad", adp->lun, DEV_BSIZE,
 		      DEVSTAT_NO_ORDERED_TAGS,
 		      DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_IDE,
@@ -194,7 +198,6 @@ ad_attach(struct ata_device *atadev)
     dev->si_drv1 = adp;
     dev->si_iosize_max = 256 * DEV_BSIZE;
     adp->dev = dev;
-    bioq_init(&adp->queue);
 
     /* construct the disklabel */
     bzero(&adp->disk.d_label, sizeof(struct disklabel));
@@ -267,8 +270,10 @@ adclose(dev_t dev, int flags, int fmt, struct thread *td)
 {
     struct ad_softc *adp = dev->si_drv1;
 
+    ATA_SLEEPLOCK_CH(adp->device->channel, ATA_CONTROL);
     if (ata_command(adp->device, ATA_C_FLUSHCACHE, 0, 0, 0, ATA_WAIT_READY))
 	ata_prtdev(adp->device, "flushing cache on close failed\n");
+    ATA_UNLOCK_CH(adp->device->channel);
     return 0;
 }
 
@@ -284,8 +289,8 @@ adstrategy(struct bio *bp)
     }
     s = splbio();
     bioqdisksort(&adp->queue, bp);
-    ata_start(adp->device->channel);
     splx(s);
+    ata_start(adp->device->channel);
 }
 
 int
@@ -776,14 +781,10 @@ ad_service(struct ad_softc *adp, int change)
 static void
 ad_free(struct ad_request *request)
 {
-    int s = splbio();
-
     if (request->dmatab)
 	free(request->dmatab, M_DEVBUF);
     request->softc->tags[request->tag] = NULL;
     free(request, M_AD);
-
-    splx(s);
 }
 
 static void
@@ -849,7 +850,6 @@ static void
 ad_timeout(struct ad_request *request)
 {
     struct ad_softc *adp = request->softc;
-    int s = splbio();
 
     adp->device->channel->running = NULL;
     ata_prtdev(adp->device, "%s command timeout tag=%d serv=%d - resetting\n",
@@ -869,7 +869,10 @@ ad_timeout(struct ad_request *request)
 
     /* if retries still permit, reinject this request */
     if (request->retries++ < AD_MAX_RETRIES) {
+	int s = splbio();
+
 	TAILQ_INSERT_HEAD(&adp->device->channel->ata_queue, request, chain);
+	splx(s);
     }
     else {
 	/* retries all used up, return error */
@@ -879,7 +882,6 @@ ad_timeout(struct ad_request *request)
 	ad_free(request);
     }
     ata_reinit(adp->device->channel);
-    splx(s);
 }
 
 void
diff --git a/sys/dev/ata/ata-raid.c b/sys/dev/ata/ata-raid.c
index a850c08..7ea0e26 100644
--- a/sys/dev/ata/ata-raid.c
+++ b/sys/dev/ata/ata-raid.c
@@ -179,7 +179,7 @@ ata_raid_attach()
 	    continue;
    
 	ar_config_changed(rdp, 0);
-	dev = disk_create(rdp->lun, &rdp->disk, 0, &ar_cdevsw,&ardisk_cdevsw);
+	dev = disk_create(rdp->lun, &rdp->disk, 0, &ar_cdevsw, &ardisk_cdevsw);
 	dev->si_drv1 = rdp;
 	dev->si_iosize_max = 256 * DEV_BSIZE;
 	rdp->dev = dev;
@@ -553,9 +553,8 @@ ar_config_changed(struct ar_softc *rdp, int writeback)
 static int
 ar_rebuild(struct ar_softc *rdp)
 {
+    int disk, s, count = 0, error = 0;
     caddr_t buffer;
-    int count = 0, error = 0;
-    int disk;
 
     if ((rdp->flags & (AR_F_READY|AR_F_DEGRADED)) != (AR_F_READY|AR_F_DEGRADED))
 	return EEXIST;
@@ -578,9 +577,11 @@ ar_rebuild(struct ar_softc *rdp)
 	return ENODEV;
 
     /* setup start conditions */
+    s = splbio();
     rdp->lock_start = 0;
     rdp->lock_end = rdp->lock_start + 256;
     rdp->flags |= AR_F_REBUILDING;
+    splx(s);
     buffer = malloc(256 * DEV_BSIZE, M_AR, M_NOWAIT | M_ZERO);
 
     /* now go copy entire disk(s) */
@@ -619,11 +620,12 @@ ar_rebuild(struct ar_softc *rdp)
 	    free(buffer, M_AR);
 	    return error;
 	}
+	s = splbio();
 	rdp->lock_start = rdp->lock_end;
 	rdp->lock_end = rdp->lock_start + size;
+	splx(s);
 	wakeup(rdp);
     }
-    rdp->lock_start = 0xffffffff;
     free(buffer, M_AR);
     for (disk = 0; disk < rdp->total_disks; disk++) {
 	if ((rdp->disks[disk].flags&(AR_DF_PRESENT|AR_DF_ONLINE|AR_DF_SPARE))==
@@ -632,7 +634,11 @@ ar_rebuild(struct ar_softc *rdp)
 	    rdp->disks[disk].flags |= (AR_DF_ASSIGNED | AR_DF_ONLINE);
 	}
     }
+    s = splbio();
+    rdp->lock_start = 0xffffffff;
+    rdp->lock_end = 0xffffffff;
     rdp->flags &= ~AR_F_REBUILDING;
+    splx(s);
     ar_config_changed(rdp, 1);
     return 0;
 }
diff --git a/sys/dev/ata/atapi-all.c b/sys/dev/ata/atapi-all.c
index 794db5d..033c242 100644
--- a/sys/dev/ata/atapi-all.c
+++ b/sys/dev/ata/atapi-all.c
@@ -72,6 +72,7 @@ atapi_attach(struct ata_device *atadev)
 		   ata_pmode(atadev->param), ata_wmode(atadev->param),
 		   ata_umode(atadev->param), atadev->param->support_dma);
 
+    ATA_SLEEPLOCK_CH(atadev->channel, ATA_CONTROL);
     if (atapi_dma && !(atadev->param->drq_type == ATAPI_DRQT_INTR)) {
 	ata_dmainit(atadev->channel, atadev->unit,
 		    (ata_pmode(atadev->param) < 0) ? 
@@ -84,6 +85,7 @@ atapi_attach(struct ata_device *atadev)
 	ata_dmainit(atadev->channel, atadev->unit,
 		    ata_pmode(atadev->param) < 0 ? 0 : ata_pmode(atadev->param),
 		    -1, -1);
+    ATA_UNLOCK_CH(atadev->channel);
 
     if (!(atadev->result = malloc(sizeof(struct atapi_reqsense), M_ATAPI,
 				  M_NOWAIT | M_ZERO)))
@@ -190,28 +192,25 @@ atapi_queue_cmd(struct ata_device *atadev, int8_t *ccb, caddr_t data,
 	    atadev->mode = ATA_PIO;
     }
 
-    s = splbio();
-
-    /* append onto controller queue and try to start controller */
 #ifdef ATAPI_DEBUG
     ata_prtdev(atadev, "queueing %s ", atapi_cmd2str(request->ccb[0]));
     atapi_dump("ccb = ", &request->ccb[0], sizeof(request->ccb));
 #endif
+    /* append onto controller queue and try to start controller */
+    s = splbio();
     if (flags & ATPR_F_AT_HEAD)
 	TAILQ_INSERT_HEAD(&atadev->channel->atapi_queue, request, chain);
     else
 	TAILQ_INSERT_TAIL(&atadev->channel->atapi_queue, request, chain);
+    splx(s);
     ata_start(atadev->channel);
 
     /* if callback used, then just return, gets called from interrupt context */
-    if (callback) {
-	splx(s);
+    if (callback)
 	return 0;
-    }
 
     /* wait for request to complete */
     tsleep((caddr_t)request, PRIBIO, "atprq", 0);
-    splx(s);
     error = request->error;
     if (error)
 	 bcopy(&request->sense, atadev->result, sizeof(struct atapi_reqsense));
@@ -605,7 +604,6 @@ static void
 atapi_timeout(struct atapi_request *request)
 {
     struct ata_device *atadev = request->device;
-    int s = splbio();
 
     atadev->channel->running = NULL;
     ata_prtdev(atadev, "%s command timeout - resetting\n", 
@@ -623,15 +621,18 @@ atapi_timeout(struct atapi_request *request)
     }
 
     /* if retries still permit, reinject this request */
-    if (request->retries++ < ATAPI_MAX_RETRIES)
+    if (request->retries++ < ATAPI_MAX_RETRIES) {
+	int s = splbio();
+
 	TAILQ_INSERT_HEAD(&atadev->channel->atapi_queue, request, chain);
+	splx(s);
+    }
     else {
 	/* retries all used up, return error */
 	request->error = EIO;
 	wakeup((caddr_t)request);
     } 
     ata_reinit(atadev->channel);
-    splx(s);
 }
 
 static char *
diff --git a/sys/dev/ata/atapi-cd.c b/sys/dev/ata/atapi-cd.c
index b9dc7e5..bb19b7d 100644
--- a/sys/dev/ata/atapi-cd.c
+++ b/sys/dev/ata/atapi-cd.c
@@ -1071,8 +1071,8 @@ acdstrategy(struct bio *bp)
 
     s = splbio();
     bioqdisksort(&cdp->queue, bp);
-    ata_start(cdp->device->channel);
     splx(s);
+    ata_start(cdp->device->channel);
 }
 
 void 
diff --git a/sys/dev/ata/atapi-fd.c b/sys/dev/ata/atapi-fd.c
index 303d10c..0ebac0e 100644
--- a/sys/dev/ata/atapi-fd.c
+++ b/sys/dev/ata/atapi-fd.c
@@ -295,8 +295,8 @@ afdstrategy(struct bio *bp)
 
     s = splbio();
     bioqdisksort(&fdp->queue, bp);
-    ata_start(fdp->device->channel);
     splx(s);
+    ata_start(fdp->device->channel);
 }
 
 void 
diff --git a/sys/dev/ata/atapi-tape.c b/sys/dev/ata/atapi-tape.c
index 03130ad..494404a 100644
--- a/sys/dev/ata/atapi-tape.c
+++ b/sys/dev/ata/atapi-tape.c
@@ -448,8 +448,8 @@ aststrategy(struct bio *bp)
 
     s = splbio();
     bioq_insert_tail(&stp->queue, bp);
-    ata_start(stp->device->channel);
     splx(s);
+    ata_start(stp->device->channel);
 }
 
 void