Get rid of the gv_bioq hack in most parts of the I/O path and

use the standard bioq structures.

5 files changed, 89 insertions, 81 deletions

diff --git a/sys/geom/vinum/geom_vinum_drive.c b/sys/geom/vinum/geom_vinum_drive.c
index b2841a8..98b7c57 100644
--- a/sys/geom/vinum/geom_vinum_drive.c
+++ b/sys/geom/vinum/geom_vinum_drive.c
@@ -77,7 +77,8 @@ gv_config_new_drive(struct gv_drive *d)
 	LIST_INSERT_HEAD(&d->freelist, fl, freelist);
 	d->freelist_entries = 1;
 
-	TAILQ_INIT(&d->bqueue);
+	d->bqueue = g_malloc(sizeof(struct bio_queue_head), M_WAITOK | M_ZERO);
+	bioq_init(d->bqueue);
 	mtx_init(&d->bqueue_mtx, "gv_drive", NULL, MTX_DEF);
 	kthread_create(gv_drive_worker, d, NULL, 0, 0, "gv_d %s", d->name);
 	d->flags |= GV_DRIVE_THREAD_ACTIVE;
@@ -235,15 +236,12 @@ static void
 gv_drive_done(struct bio *bp)
 {
 	struct gv_drive *d;
-	struct gv_bioq *bq;
 
 	/* Put the BIO on the worker queue again. */
 	d = bp->bio_from->geom->softc;
 	bp->bio_cflags |= GV_BIO_DONE;
-	bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO);
-	bq->bp = bp;
 	mtx_lock(&d->bqueue_mtx);
-	TAILQ_INSERT_TAIL(&d->bqueue, bq, queue);
+	bioq_insert_tail(d->bqueue, bp);
 	wakeup(d);
 	mtx_unlock(&d->bqueue_mtx);
 }
@@ -254,7 +252,6 @@ gv_drive_start(struct bio *bp)
 {
 	struct gv_drive *d;
 	struct gv_sd *s;
-	struct gv_bioq *bq;
 
 	switch (bp->bio_cmd) {
 	case BIO_READ:
@@ -279,10 +276,8 @@ gv_drive_start(struct bio *bp)
 	 * Put the BIO on the worker queue, where the worker thread will pick
 	 * it up.
 	 */
-	bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO);
-	bq->bp = bp;
 	mtx_lock(&d->bqueue_mtx);
-	TAILQ_INSERT_TAIL(&d->bqueue, bq, queue);
+	bioq_disksort(d->bqueue, bp);
 	wakeup(d);
 	mtx_unlock(&d->bqueue_mtx);
 
@@ -296,7 +291,6 @@ gv_drive_worker(void *arg)
 	struct g_provider *pp;
 	struct gv_drive *d;
 	struct gv_sd *s;
-	struct gv_bioq *bq, *bq2;
 	int error;
 
 	d = arg;
@@ -308,16 +302,13 @@ gv_drive_worker(void *arg)
 			break;
 
 		/* Take the first BIO from out queue. */
-		bq = TAILQ_FIRST(&d->bqueue);
-		if (bq == NULL) {
+		bp = bioq_takefirst(d->bqueue);
+		if (bp == NULL) {
 			msleep(d, &d->bqueue_mtx, PRIBIO, "-", hz/10);
 			continue;
  		}
-		TAILQ_REMOVE(&d->bqueue, bq, queue);
 		mtx_unlock(&d->bqueue_mtx);
  
-		bp = bq->bp;
-		g_free(bq);
 		pp = bp->bio_to;
 		gp = pp->geom;
 
@@ -371,11 +362,8 @@ gv_drive_worker(void *arg)
 		mtx_lock(&d->bqueue_mtx);
 	}
 
-	TAILQ_FOREACH_SAFE(bq, &d->bqueue, queue, bq2) {
-		TAILQ_REMOVE(&d->bqueue, bq, queue);
+	while ((bp = bioq_takefirst(d->bqueue)) != NULL) {
 		mtx_unlock(&d->bqueue_mtx);
-		bp = bq->bp;
-		g_free(bq);
 		if (bp->bio_cflags & GV_BIO_DONE) 
 			g_std_done(bp);
 		else
@@ -504,15 +492,19 @@ gv_drive_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 			LIST_INSERT_HEAD(&d->freelist, fl, freelist);
 			d->freelist_entries = 1;
 
-			TAILQ_INIT(&d->bqueue);
-
 			/* Save it into the main configuration. */
 			LIST_INSERT_HEAD(&sc->drives, d, drive);
 		}
 
 		/*
-		 * Create a bio queue mutex and a worker thread, if necessary.
+		 * Create bio queue, queue mutex and a worker thread, if
+		 * necessary.
 		 */
+		if (d->bqueue == NULL) {
+			d->bqueue = g_malloc(sizeof(struct bio_queue_head),
+			    M_WAITOK | M_ZERO);
+			bioq_init(d->bqueue);
+		}
 		if (mtx_initialized(&d->bqueue_mtx) == 0)
 			mtx_init(&d->bqueue_mtx, "gv_drive", NULL, MTX_DEF);
 
diff --git a/sys/geom/vinum/geom_vinum_plex.c b/sys/geom/vinum/geom_vinum_plex.c
index 6f68dd9..bdcce21 100644
--- a/sys/geom/vinum/geom_vinum_plex.c
+++ b/sys/geom/vinum/geom_vinum_plex.c
@@ -88,14 +88,11 @@ void
 gv_plex_done(struct bio *bp)
 {
 	struct gv_plex *p;
-	struct gv_bioq *bq;
 
 	p = bp->bio_from->geom->softc;
 	bp->bio_cflags |= GV_BIO_DONE;
-	bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO);
-	bq->bp = bp;
 	mtx_lock(&p->bqueue_mtx);
-	TAILQ_INSERT_TAIL(&p->bqueue, bq, queue);
+	bioq_insert_tail(p->bqueue, bp);
 	wakeup(p);
 	mtx_unlock(&p->bqueue_mtx);
 }
@@ -236,7 +233,6 @@ static void
 gv_plex_start(struct bio *bp)
 {
 	struct gv_plex *p;
-	struct gv_bioq *bq;
 
 	switch(bp->bio_cmd) {
 	case BIO_READ:
@@ -260,10 +256,8 @@ gv_plex_start(struct bio *bp)
 		return;
 	}
 
-	bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO);
-	bq->bp = bp;
 	mtx_lock(&p->bqueue_mtx);
-	TAILQ_INSERT_TAIL(&p->bqueue, bq, queue);
+	bioq_disksort(p->bqueue, bp);
 	wakeup(p);
 	mtx_unlock(&p->bqueue_mtx);
 }
@@ -274,7 +268,6 @@ gv_plex_worker(void *arg)
 	struct bio *bp;
 	struct gv_plex *p;
 	struct gv_sd *s;
-	struct gv_bioq *bq;
 
 	p = arg;
 	KASSERT(p != NULL, ("NULL p"));
@@ -286,20 +279,15 @@ gv_plex_worker(void *arg)
 			break;
 
 		/* Take the first BIO from our queue. */
-		bq = TAILQ_FIRST(&p->bqueue);
-		if (bq == NULL) {
+		bp = bioq_takefirst(p->bqueue);
+		if (bp == NULL) {
 			msleep(p, &p->bqueue_mtx, PRIBIO, "-", hz/10);
 			continue;
 		}
-		TAILQ_REMOVE(&p->bqueue, bq, queue);
 		mtx_unlock(&p->bqueue_mtx);
 
-		bp = bq->bp;
-
 		/* A completed request. */
 		if (bp->bio_cflags & GV_BIO_DONE) {
-			g_free(bq);
-
 			if (bp->bio_cflags & GV_BIO_SYNCREQ ||
 			    bp->bio_cflags & GV_BIO_REBUILD) {
 				s = bp->bio_to->private;
@@ -327,19 +315,16 @@ gv_plex_worker(void *arg)
 			if (gv_stripe_active(p, bp)) {
 				/* Park the bio on the waiting queue. */
 				mtx_lock(&p->bqueue_mtx);
-				TAILQ_INSERT_TAIL(&p->wqueue, bq, queue);
+				bioq_disksort(p->wqueue, bp);
 				mtx_unlock(&p->bqueue_mtx);
 			} else {
-				g_free(bq);
 				bp->bio_cflags &= ~GV_BIO_ONHOLD;
 				g_io_request(bp, bp->bio_caller2);
 			}
 
 		/* A normal request to this plex. */
-		} else {
-			g_free(bq);
+		} else
 			gv_plex_normal_request(p, bp);
-		}
 
 		mtx_lock(&p->bqueue_mtx);
 	}
@@ -380,7 +365,7 @@ gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 static int
 gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 {
-	struct bio *cbp, *pbp;
+	struct bio *pbp;
 	int err, finished, i;
 
 	err = 0;
@@ -393,12 +378,12 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 		finished = 0;
 
 	} else if (wp->parity != NULL) {
-		cbp = wp->parity;
+		pbp = wp->parity;
 		wp->parity = NULL;
 
 		/* Check if the parity is correct. */
 		for (i = 0; i < wp->length; i++) {
-			if (bp->bio_data[i] != cbp->bio_data[i]) {
+			if (bp->bio_data[i] != pbp->bio_data[i]) {
 				err = 1;
 				break;
 			}
@@ -410,7 +395,7 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 
 			/* ... but we rebuild it. */
 			if (bp->bio_parent->bio_cflags & GV_BIO_PARITY) {
-				g_io_request(cbp, cbp->bio_caller2);
+				g_io_request(pbp, pbp->bio_caller2);
 				finished = 0;
 			}
 		}
@@ -421,7 +406,7 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 		 */
 		if (finished) {
 			bp->bio_parent->bio_inbed++;
-			g_destroy_bio(cbp);
+			g_destroy_bio(pbp);
 		}
 
 	}
@@ -459,7 +444,11 @@ gv_plex_completed_request(struct gv_plex *p, struct bio *bp)
 				TAILQ_REMOVE(&p->packets, wp, list);
 				/* Bring the waiting bios back into the game. */
 				mtx_lock(&p->bqueue_mtx);
-				TAILQ_CONCAT(&p->bqueue, &p->wqueue, queue);
+				pbp = bioq_takefirst(p->wqueue);
+				while (pbp != NULL) {
+					bioq_disksort(p->bqueue, pbp);
+					pbp = bioq_takefirst(p->wqueue);
+				}
 				mtx_unlock(&p->bqueue_mtx);
 			}
 			g_free(wp);
@@ -499,7 +488,11 @@ gv_plex_completed_request(struct gv_plex *p, struct bio *bp)
 				TAILQ_REMOVE(&p->packets, wp, list);
 				/* Bring the waiting bios back into the game. */
 				mtx_lock(&p->bqueue_mtx);
-				TAILQ_CONCAT(&p->bqueue, &p->wqueue, queue);
+				pbp = bioq_takefirst(p->wqueue);
+				while (pbp != NULL) {
+					bioq_disksort(p->bqueue, pbp);
+					pbp = bioq_takefirst(p->wqueue);
+				}
 				mtx_unlock(&p->bqueue_mtx);
 				g_free(wp);
 			}
@@ -662,10 +655,8 @@ gv_plex_normal_request(struct gv_plex *p, struct bio *bp)
 		    gv_stripe_active(p, pbp)) {
 			/* Park the bio on the waiting queue. */
 			pbp->bio_cflags |= GV_BIO_ONHOLD;
-			bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
-			bq->bp = pbp;
 			mtx_lock(&p->bqueue_mtx);
-			TAILQ_INSERT_TAIL(&p->wqueue, bq, queue);
+			bioq_disksort(p->wqueue, pbp);
 			mtx_unlock(&p->bqueue_mtx);
 		} else
 			g_io_request(pbp, pbp->bio_caller2);
@@ -776,8 +767,19 @@ gv_plex_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 			gv_update_vol_size(p->vol_sc, p->size);
 
 		/*
-		 * If necessary, create a bio queue mutex and a worker thread.
+		 * If necessary, create bio queues, queue mutex and a worker
+		 * thread.
 		 */
+		if (p->bqueue == NULL) {
+			p->bqueue = g_malloc(sizeof(struct bio_queue_head),
+			    M_WAITOK | M_ZERO);
+			bioq_init(p->bqueue);
+		}
+		if (p->wqueue == NULL) {
+			p->wqueue = g_malloc(sizeof(struct bio_queue_head),
+			    M_WAITOK | M_ZERO);
+			bioq_init(p->wqueue);
+		}
 		if (mtx_initialized(&p->bqueue_mtx) == 0)
 			mtx_init(&p->bqueue_mtx, "gv_plex", NULL, MTX_DEF);
 		if (!(p->flags & GV_PLEX_THREAD_ACTIVE)) {
@@ -798,8 +800,12 @@ gv_plex_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 		p->geom = gp;
 
 		TAILQ_INIT(&p->packets);
-		TAILQ_INIT(&p->bqueue);
-		TAILQ_INIT(&p->wqueue);
+		p->bqueue = g_malloc(sizeof(struct bio_queue_head),
+		    M_WAITOK | M_ZERO);
+		bioq_init(p->bqueue);
+		p->wqueue = g_malloc(sizeof(struct bio_queue_head),
+		    M_WAITOK | M_ZERO);
+		bioq_init(p->wqueue);
 		mtx_init(&p->bqueue_mtx, "gv_plex", NULL, MTX_DEF);
 		kthread_create(gv_plex_worker, p, NULL, 0, 0, "gv_p %s",
 		    p->name);
diff --git a/sys/geom/vinum/geom_vinum_subr.c b/sys/geom/vinum/geom_vinum_subr.c
index ff91196..ce79a18 100644
--- a/sys/geom/vinum/geom_vinum_subr.c
+++ b/sys/geom/vinum/geom_vinum_subr.c
@@ -792,6 +792,8 @@ gv_kill_drive_thread(struct gv_drive *d)
 		d->flags &= ~GV_DRIVE_THREAD_ACTIVE;
 		d->flags &= ~GV_DRIVE_THREAD_DIE;
 		d->flags &= ~GV_DRIVE_THREAD_DEAD;
+		g_free(d->bqueue);
+		d->bqueue = NULL;
 		mtx_destroy(&d->bqueue_mtx);
 	}
 }
@@ -807,6 +809,10 @@ gv_kill_plex_thread(struct gv_plex *p)
 		p->flags &= ~GV_PLEX_THREAD_ACTIVE;
 		p->flags &= ~GV_PLEX_THREAD_DIE;
 		p->flags &= ~GV_PLEX_THREAD_DEAD;
+		g_free(p->bqueue);
+		g_free(p->wqueue);
+		p->bqueue = NULL;
+		p->wqueue = NULL;
 		mtx_destroy(&p->bqueue_mtx);
 	}
 }
@@ -822,6 +828,8 @@ gv_kill_vol_thread(struct gv_volume *v)
 		v->flags &= ~GV_VOL_THREAD_ACTIVE;
 		v->flags &= ~GV_VOL_THREAD_DIE;
 		v->flags &= ~GV_VOL_THREAD_DEAD;
+		g_free(v->bqueue);
+		v->bqueue = NULL;
 		mtx_destroy(&v->bqueue_mtx);
 	}
 }
diff --git a/sys/geom/vinum/geom_vinum_var.h b/sys/geom/vinum/geom_vinum_var.h
index 3de2b44..8f746f0 100644
--- a/sys/geom/vinum/geom_vinum_var.h
+++ b/sys/geom/vinum/geom_vinum_var.h
@@ -198,7 +198,11 @@ struct gv_drive {
 	LIST_HEAD(,gv_sd)	subdisks;	/* Subdisks on this drive. */
 	LIST_ENTRY(gv_drive)	drive;		/* Entry in the vinum config. */
 
-	TAILQ_HEAD(,gv_bioq)	bqueue;		/* BIO queue of this drive. */
+#ifdef _KERNEL
+	struct bio_queue_head	*bqueue;	/* BIO queue of this drive. */
+#else
+	char			*padding;
+#endif
 	struct mtx		bqueue_mtx;	/* Mtx. to protect the queue. */
 
 	struct g_geom	*geom;			/* The geom of this drive. */
@@ -277,8 +281,12 @@ struct gv_plex {
 	off_t	synced;			/* Count of synced bytes. */
 
 	struct mtx		bqueue_mtx; /* Lock for the BIO queue. */
-	TAILQ_HEAD(,gv_bioq)	bqueue;	/* BIO queue. */
-	TAILQ_HEAD(,gv_bioq)	wqueue;	/* Waiting BIO queue. */
+#ifdef _KERNEL
+	struct bio_queue_head	*bqueue; /* BIO queue. */
+	struct bio_queue_head	*wqueue; /* Waiting BIO queue. */
+#else
+	char			*bpad, *wpad;
+#endif
 	TAILQ_HEAD(,gv_raid5_packet)	packets; /* RAID5 sub-requests. */
 
 	LIST_HEAD(,gv_sd)   subdisks;	/* List of attached subdisks. */
@@ -307,7 +315,11 @@ struct gv_volume {
 #define	GV_VOL_THREAD_DEAD	0x04	/* The thread has died. */
 
 	struct mtx		bqueue_mtx; /* Lock for the BIO queue. */
-	TAILQ_HEAD(,gv_bioq)	bqueue;	/* BIO queue. */
+#ifdef _KERNEL
+	struct bio_queue_head	*bqueue; /* BIO queue. */
+#else
+	char			*padding;
+#endif
 
 	LIST_HEAD(,gv_plex)   plexes;	/* List of attached plexes. */
 	LIST_ENTRY(gv_volume) volume;	/* Entry in vinum config. */
diff --git a/sys/geom/vinum/geom_vinum_volume.c b/sys/geom/vinum/geom_vinum_volume.c
index 510d3f0..afc1f2a 100644
--- a/sys/geom/vinum/geom_vinum_volume.c
+++ b/sys/geom/vinum/geom_vinum_volume.c
@@ -79,14 +79,11 @@ static void
 gv_volume_done(struct bio *bp)
 {
 	struct gv_volume *v;
-	struct gv_bioq *bq;
 
 	v = bp->bio_from->geom->softc;
 	bp->bio_cflags |= GV_BIO_DONE;
-	bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO);
-	bq->bp = bp;
 	mtx_lock(&v->bqueue_mtx);
-	TAILQ_INSERT_TAIL(&v->bqueue, bq, queue);
+	bioq_insert_tail(v->bqueue, bp);
 	wakeup(v);
 	mtx_unlock(&v->bqueue_mtx);
 }
@@ -95,7 +92,6 @@ static void
 gv_volume_start(struct bio *bp)
 {
 	struct gv_volume *v;
-	struct gv_bioq *bq;
 
 	switch(bp->bio_cmd) {
 	case BIO_READ:
@@ -114,10 +110,8 @@ gv_volume_start(struct bio *bp)
 		return;
 	}
 
-	bq = g_malloc(sizeof(*bq), M_NOWAIT | M_ZERO);
-	bq->bp = bp;
 	mtx_lock(&v->bqueue_mtx);
-	TAILQ_INSERT_TAIL(&v->bqueue, bq, queue);
+	bioq_disksort(v->bqueue, bp);
 	wakeup(v);
 	mtx_unlock(&v->bqueue_mtx);
 }
@@ -127,7 +121,6 @@ gv_vol_worker(void *arg)
 {
 	struct bio *bp;
 	struct gv_volume *v;
-	struct gv_bioq *bq;
 
 	v = arg;
 	KASSERT(v != NULL, ("NULL v"));
@@ -138,17 +131,13 @@ gv_vol_worker(void *arg)
 			break;
 
 		/* Take the first BIO from our queue. */
-		bq = TAILQ_FIRST(&v->bqueue);
-		if (bq == NULL) {
+		bp = bioq_takefirst(v->bqueue);
+		if (bp == NULL) {
 			msleep(v, &v->bqueue_mtx, PRIBIO, "-", hz/10);
 			continue;
 		}
-		TAILQ_REMOVE(&v->bqueue, bq, queue);
 		mtx_unlock(&v->bqueue_mtx);
 
-		bp = bq->bp;
-		g_free(bq);
-
 		if (bp->bio_cflags & GV_BIO_DONE)
 			gv_vol_completed_request(v, bp);
 		else
@@ -169,7 +158,6 @@ gv_vol_completed_request(struct gv_volume *v, struct bio *bp)
 	struct bio *pbp;
 	struct g_geom *gp;
 	struct g_consumer *cp, *cp2;
-	struct gv_bioq *bq;
 
 	pbp = bp->bio_parent;
 
@@ -196,10 +184,8 @@ gv_vol_completed_request(struct gv_volume *v, struct bio *bp)
 
 		g_destroy_bio(bp);
 		pbp->bio_children--;
-		bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
-		bq->bp = pbp;
 		mtx_lock(&v->bqueue_mtx);
-		TAILQ_INSERT_TAIL(&v->bqueue, bq, queue);
+		bioq_disksort(v->bqueue, pbp);
 		mtx_unlock(&v->bqueue_mtx);
 		return;
 
@@ -370,11 +356,15 @@ gv_volume_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 		gp->access = gv_volume_access;
 		gp->softc = v;
 		first++;
-		TAILQ_INIT(&v->bqueue);
 	} else
 		gp = v->geom;
 
-	/* Create bio queue mutex and worker thread, if necessary. */
+	/* Create bio queue, queue mutex, and worker thread, if necessary. */
+	if (v->bqueue == NULL) {
+		v->bqueue = g_malloc(sizeof(struct bio_queue_head),
+		    M_WAITOK | M_ZERO);
+		bioq_init(v->bqueue);
+	}
 	if (mtx_initialized(&v->bqueue_mtx) == 0)
 		mtx_init(&v->bqueue_mtx, "gv_plex", NULL, MTX_DEF);