MFC r275335:

  Cleanup and performance improvement of the virtio_blk driver

    - Add support for GEOM direct completion. Depending on the benchmark,
      this tends to give a ~30% improvement w.r.t IOPs and BW.
    - Remove an invariants check in the strategy routine. This assertion
      is caught later on by an existing panic.
    - Rename and resort various related functions to make more sense.

1 files changed, 422 insertions, 417 deletions

diff --git a/sys/dev/virtio/block/virtio_blk.c b/sys/dev/virtio/block/virtio_blk.c
index a089e3f..7818217 100644
--- a/sys/dev/virtio/block/virtio_blk.c
+++ b/sys/dev/virtio/block/virtio_blk.c
@@ -58,7 +58,6 @@ struct vtblk_request {
 	struct virtio_blk_outhdr	 vbr_hdr;
 	struct bio			*vbr_bp;
 	uint8_t				 vbr_ack;
-
 	TAILQ_ENTRY(vtblk_request)	 vbr_link;
 };
 
@@ -132,53 +131,60 @@ static int	vtblk_dump(void *, void *, vm_offset_t, off_t, size_t);
 static void	vtblk_strategy(struct bio *);
 
 static void	vtblk_negotiate_features(struct vtblk_softc *);
+static void	vtblk_setup_features(struct vtblk_softc *);
 static int	vtblk_maximum_segments(struct vtblk_softc *,
 		    struct virtio_blk_config *);
 static int	vtblk_alloc_virtqueue(struct vtblk_softc *);
 static void	vtblk_resize_disk(struct vtblk_softc *, uint64_t);
-static void	vtblk_set_write_cache(struct vtblk_softc *, int);
-static int	vtblk_write_cache_enabled(struct vtblk_softc *sc,
-		    struct virtio_blk_config *);
-static int	vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS);
 static void	vtblk_alloc_disk(struct vtblk_softc *,
 		    struct virtio_blk_config *);
 static void	vtblk_create_disk(struct vtblk_softc *);
 
-static int	vtblk_quiesce(struct vtblk_softc *);
-static void	vtblk_startio(struct vtblk_softc *);
-static struct vtblk_request * vtblk_bio_request(struct vtblk_softc *);
-static int	vtblk_execute_request(struct vtblk_softc *,
+static int	vtblk_request_prealloc(struct vtblk_softc *);
+static void	vtblk_request_free(struct vtblk_softc *);
+static struct vtblk_request *
+		vtblk_request_dequeue(struct vtblk_softc *);
+static void	vtblk_request_enqueue(struct vtblk_softc *,
+		    struct vtblk_request *);
+static struct vtblk_request *
+		vtblk_request_next_ready(struct vtblk_softc *);
+static void	vtblk_request_requeue_ready(struct vtblk_softc *,
 		    struct vtblk_request *);
+static struct vtblk_request *
+		vtblk_request_next(struct vtblk_softc *);
+static struct vtblk_request *
+		vtblk_request_bio(struct vtblk_softc *);
+static int	vtblk_request_execute(struct vtblk_softc *,
+		    struct vtblk_request *);
+static int	vtblk_request_error(struct vtblk_request *);
 
-static void	vtblk_vq_intr(void *);
+static void	vtblk_queue_completed(struct vtblk_softc *,
+		    struct bio_queue *);
+static void	vtblk_done_completed(struct vtblk_softc *,
+		    struct bio_queue *);
+static void	vtblk_drain_vq(struct vtblk_softc *, int);
+static void	vtblk_drain(struct vtblk_softc *);
 
-static void	vtblk_stop(struct vtblk_softc *);
+static void	vtblk_startio(struct vtblk_softc *);
+static void	vtblk_bio_done(struct vtblk_softc *, struct bio *, int);
 
 static void	vtblk_read_config(struct vtblk_softc *,
 		    struct virtio_blk_config *);
-static void	vtblk_get_ident(struct vtblk_softc *);
-static void	vtblk_prepare_dump(struct vtblk_softc *);
-static int	vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
-static int	vtblk_flush_dump(struct vtblk_softc *);
+static void	vtblk_ident(struct vtblk_softc *);
 static int	vtblk_poll_request(struct vtblk_softc *,
 		    struct vtblk_request *);
+static int	vtblk_quiesce(struct vtblk_softc *);
+static void	vtblk_vq_intr(void *);
+static void	vtblk_stop(struct vtblk_softc *);
 
-static void	vtblk_finish_completed(struct vtblk_softc *);
-static void	vtblk_drain_vq(struct vtblk_softc *, int);
-static void	vtblk_drain(struct vtblk_softc *);
-
-static int	vtblk_alloc_requests(struct vtblk_softc *);
-static void	vtblk_free_requests(struct vtblk_softc *);
-static struct vtblk_request * vtblk_dequeue_request(struct vtblk_softc *);
-static void	vtblk_enqueue_request(struct vtblk_softc *,
-		    struct vtblk_request *);
-
-static struct vtblk_request * vtblk_dequeue_ready(struct vtblk_softc *);
-static void	vtblk_enqueue_ready(struct vtblk_softc *,
-		    struct vtblk_request *);
+static void	vtblk_dump_prepare(struct vtblk_softc *);
+static int	vtblk_dump_write(struct vtblk_softc *, void *, off_t, size_t);
+static int	vtblk_dump_flush(struct vtblk_softc *);
 
-static int	vtblk_request_error(struct vtblk_request *);
-static void	vtblk_finish_bio(struct bio *, int);
+static void	vtblk_set_write_cache(struct vtblk_softc *, int);
+static int	vtblk_write_cache_enabled(struct vtblk_softc *sc,
+		    struct virtio_blk_config *);
+static int	vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS);
 
 static void	vtblk_setup_sysctl(struct vtblk_softc *);
 static int	vtblk_tunable_int(struct vtblk_softc *, const char *, int);
@@ -288,30 +294,18 @@ vtblk_attach(device_t dev)
 	struct virtio_blk_config blkcfg;
 	int error;
 
+	virtio_set_feature_desc(dev, vtblk_feature_desc);
+
 	sc = device_get_softc(dev);
 	sc->vtblk_dev = dev;
-
 	VTBLK_LOCK_INIT(sc, device_get_nameunit(dev));
-
 	bioq_init(&sc->vtblk_bioq);
 	TAILQ_INIT(&sc->vtblk_req_free);
 	TAILQ_INIT(&sc->vtblk_req_ready);
 
-	virtio_set_feature_desc(dev, vtblk_feature_desc);
-	vtblk_negotiate_features(sc);
-
-	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
-		sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
-	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
-		sc->vtblk_flags |= VTBLK_FLAG_READONLY;
-	if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
-		sc->vtblk_flags |= VTBLK_FLAG_BARRIER;
-	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
-		sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG;
-
 	vtblk_setup_sysctl(sc);
+	vtblk_setup_features(sc);
 
-	/* Get local copy of config. */
 	vtblk_read_config(sc, &blkcfg);
 
 	/*
@@ -350,7 +344,7 @@ vtblk_attach(device_t dev)
 		goto fail;
 	}
 
-	error = vtblk_alloc_requests(sc);
+	error = vtblk_request_prealloc(sc);
 	if (error) {
 		device_printf(dev, "cannot preallocate requests\n");
 		goto fail;
@@ -517,14 +511,14 @@ vtblk_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset,
 	VTBLK_LOCK(sc);
 
 	if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
-		vtblk_prepare_dump(sc);
+		vtblk_dump_prepare(sc);
 		sc->vtblk_flags |= VTBLK_FLAG_DUMPING;
 	}
 
 	if (length > 0)
-		error = vtblk_write_dump(sc, virtual, offset, length);
+		error = vtblk_dump_write(sc, virtual, offset, length);
 	else if (virtual == NULL && offset == 0)
-		error = vtblk_flush_dump(sc);
+		error = vtblk_dump_flush(sc);
 	else {
 		error = EINVAL;
 		sc->vtblk_flags &= ~VTBLK_FLAG_DUMPING;
@@ -541,7 +535,7 @@ vtblk_strategy(struct bio *bp)
 	struct vtblk_softc *sc;
 
 	if ((sc = bp->bio_disk->d_drv1) == NULL) {
-		vtblk_finish_bio(bp, EINVAL);
+		vtblk_bio_done(NULL, bp, EINVAL);
 		return;
 	}
 
@@ -551,37 +545,21 @@ vtblk_strategy(struct bio *bp)
 	 */
 	if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
 	    (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
-		vtblk_finish_bio(bp, EROFS);
+		vtblk_bio_done(sc, bp, EROFS);
 		return;
 	}
 
-#ifdef INVARIANTS
-	/*
-	 * Prevent read/write buffers spanning too many segments from
-	 * getting into the queue. This should only trip if d_maxsize
-	 * was incorrectly set.
-	 */
-	if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
-		int nsegs, max_nsegs;
-
-		nsegs = sglist_count(bp->bio_data, bp->bio_bcount);
-		max_nsegs = sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS;
+	VTBLK_LOCK(sc);
 
-		KASSERT(nsegs <= max_nsegs,
-		    ("%s: bio %p spanned too many segments: %d, max: %d",
-		    __func__, bp, nsegs, max_nsegs));
+	if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
+		VTBLK_UNLOCK(sc);
+		vtblk_bio_done(sc, bp, ENXIO);
+		return;
 	}
-#endif
 
-	VTBLK_LOCK(sc);
-	if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
-		vtblk_finish_bio(bp, ENXIO);
-	else {
-		bioq_insert_tail(&sc->vtblk_bioq, bp);
+	bioq_insert_tail(&sc->vtblk_bioq, bp);
+	vtblk_startio(sc);
 
-		if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0)
-			vtblk_startio(sc);
-	}
 	VTBLK_UNLOCK(sc);
 }
 
@@ -597,6 +575,25 @@ vtblk_negotiate_features(struct vtblk_softc *sc)
 	sc->vtblk_features = virtio_negotiate_features(dev, features);
 }
 
+static void
+vtblk_setup_features(struct vtblk_softc *sc)
+{
+	device_t dev;
+
+	dev = sc->vtblk_dev;
+
+	vtblk_negotiate_features(sc);
+
+	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
+		sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
+		sc->vtblk_flags |= VTBLK_FLAG_READONLY;
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
+		sc->vtblk_flags |= VTBLK_FLAG_BARRIER;
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
+		sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG;
+}
+
 static int
 vtblk_maximum_segments(struct vtblk_softc *sc,
     struct virtio_blk_config *blkcfg)
@@ -658,59 +655,6 @@ vtblk_resize_disk(struct vtblk_softc *sc, uint64_t new_capacity)
 }
 
 static void
-vtblk_set_write_cache(struct vtblk_softc *sc, int wc)
-{
-
-	/* Set either writeback (1) or writethrough (0) mode. */
-	virtio_write_dev_config_1(sc->vtblk_dev,
-	    offsetof(struct virtio_blk_config, writeback), wc);
-}
-
-static int
-vtblk_write_cache_enabled(struct vtblk_softc *sc,
-    struct virtio_blk_config *blkcfg)
-{
-	int wc;
-
-	if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
-		wc = vtblk_tunable_int(sc, "writecache_mode",
-		    vtblk_writecache_mode);
-		if (wc >= 0 && wc < VTBLK_CACHE_MAX)
-			vtblk_set_write_cache(sc, wc);
-		else
-			wc = blkcfg->writeback;
-	} else
-		wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);
-
-	return (wc);
-}
-
-static int
-vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)
-{
-	struct vtblk_softc *sc;
-	int wc, error;
-
-	sc = oidp->oid_arg1;
-	wc = sc->vtblk_write_cache;
-
-	error = sysctl_handle_int(oidp, &wc, 0, req);
-	if (error || req->newptr == NULL)
-		return (error);
-	if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
-		return (EPERM);
-	if (wc < 0 || wc >= VTBLK_CACHE_MAX)
-		return (EINVAL);
-
-	VTBLK_LOCK(sc);
-	sc->vtblk_write_cache = wc;
-	vtblk_set_write_cache(sc, sc->vtblk_write_cache);
-	VTBLK_UNLOCK(sc);
-
-	return (0);
-}
-
-static void
 vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
 {
 	device_t dev;
@@ -726,7 +670,8 @@ vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
 	dp->d_name = VTBLK_DISK_NAME;
 	dp->d_unit = device_get_unit(dev);
 	dp->d_drv1 = sc;
-	dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO;
+	dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO |
+	    DISKFLAG_DIRECT_COMPLETION;
 	dp->d_hba_vendor = virtio_get_vendor(dev);
 	dp->d_hba_device = virtio_get_device(dev);
 	dp->d_hba_subvendor = virtio_get_subvendor(dev);
@@ -787,11 +732,7 @@ vtblk_create_disk(struct vtblk_softc *sc)
 
 	dp = sc->vtblk_disk;
 
-	/*
-	 * Retrieving the identification string must be done after
-	 * the virtqueue interrupt is setup otherwise it will hang.
-	 */
-	vtblk_get_ident(sc);
+	vtblk_ident(sc);
 
 	device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n",
 	    (uintmax_t) dp->d_mediasize >> 20,
@@ -802,57 +743,107 @@ vtblk_create_disk(struct vtblk_softc *sc)
 }
 
 static int
-vtblk_quiesce(struct vtblk_softc *sc)
+vtblk_request_prealloc(struct vtblk_softc *sc)
 {
-	int error;
+	struct vtblk_request *req;
+	int i, nreqs;
 
-	error = 0;
+	nreqs = virtqueue_size(sc->vtblk_vq);
 
-	VTBLK_LOCK_ASSERT(sc);
+	/*
+	 * Preallocate sufficient requests to keep the virtqueue full. Each
+	 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
+	 * the number allocated when indirect descriptors are not available.
+	 */
+	if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
+		nreqs /= VTBLK_MIN_SEGMENTS;
 
-	while (!virtqueue_empty(sc->vtblk_vq)) {
-		if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq",
-		    VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) {
-			error = EBUSY;
-			break;
-		}
+	for (i = 0; i < nreqs; i++) {
+		req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT);
+		if (req == NULL)
+			return (ENOMEM);
+
+		MPASS(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)) == 1);
+		MPASS(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)) == 1);
+
+		sc->vtblk_request_count++;
+		vtblk_request_enqueue(sc, req);
 	}
 
-	return (error);
+	return (0);
 }
 
 static void
-vtblk_startio(struct vtblk_softc *sc)
+vtblk_request_free(struct vtblk_softc *sc)
 {
-	struct virtqueue *vq;
 	struct vtblk_request *req;
-	int enq;
 
-	vq = sc->vtblk_vq;
-	enq = 0;
+	MPASS(TAILQ_EMPTY(&sc->vtblk_req_ready));
 
-	VTBLK_LOCK_ASSERT(sc);
+	while ((req = vtblk_request_dequeue(sc)) != NULL) {
+		sc->vtblk_request_count--;
+		free(req, M_DEVBUF);
+	}
 
-	while (!virtqueue_full(vq)) {
-		if ((req = vtblk_dequeue_ready(sc)) == NULL)
-			req = vtblk_bio_request(sc);
-		if (req == NULL)
-			break;
+	KASSERT(sc->vtblk_request_count == 0,
+	    ("%s: leaked %d requests", __func__, sc->vtblk_request_count));
+}
 
-		if (vtblk_execute_request(sc, req) != 0) {
-			vtblk_enqueue_ready(sc, req);
-			break;
-		}
+static struct vtblk_request *
+vtblk_request_dequeue(struct vtblk_softc *sc)
+{
+	struct vtblk_request *req;
 
-		enq++;
+	req = TAILQ_FIRST(&sc->vtblk_req_free);
+	if (req != NULL) {
+		TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
+		bzero(req, sizeof(struct vtblk_request));
 	}
 
-	if (enq > 0)
-		virtqueue_notify(vq);
+	return (req);
+}
+
+static void
+vtblk_request_enqueue(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+
+	TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
 }
 
 static struct vtblk_request *
-vtblk_bio_request(struct vtblk_softc *sc)
+vtblk_request_next_ready(struct vtblk_softc *sc)
+{
+	struct vtblk_request *req;
+
+	req = TAILQ_FIRST(&sc->vtblk_req_ready);
+	if (req != NULL)
+		TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
+
+	return (req);
+}
+
+static void
+vtblk_request_requeue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
+{
+
+	/* NOTE: Currently, there will be at most one request in the queue. */
+	TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
+}
+
+static struct vtblk_request *
+vtblk_request_next(struct vtblk_softc *sc)
+{
+	struct vtblk_request *req;
+
+	req = vtblk_request_next_ready(sc);
+	if (req != NULL)
+		return (req);
+
+	return (vtblk_request_bio(sc));
+}
+
+static struct vtblk_request *
+vtblk_request_bio(struct vtblk_softc *sc)
 {
 	struct bio_queue_head *bioq;
 	struct vtblk_request *req;
@@ -863,7 +854,7 @@ vtblk_bio_request(struct vtblk_softc *sc)
 	if (bioq_first(bioq) == NULL)
 		return (NULL);
 
-	req = vtblk_dequeue_request(sc);
+	req = vtblk_request_dequeue(sc);
 	if (req == NULL)
 		return (NULL);
 
@@ -888,11 +879,14 @@ vtblk_bio_request(struct vtblk_softc *sc)
 		panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);
 	}
 
+	if (bp->bio_flags & BIO_ORDERED)
+		req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
+
 	return (req);
 }
 
 static int
-vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
+vtblk_request_execute(struct vtblk_softc *sc, struct vtblk_request *req)
 {
 	struct virtqueue *vq;
 	struct sglist *sg;
@@ -905,26 +899,20 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
 	ordered = 0;
 	writable = 0;
 
-	VTBLK_LOCK_ASSERT(sc);
-
 	/*
-	 * Wait until the ordered request completes before
-	 * executing subsequent requests.
+	 * Some hosts (such as bhyve) do not implement the barrier feature,
+	 * so we emulate it in the driver by allowing the barrier request
+	 * to be the only one in flight.
 	 */
-	if (sc->vtblk_req_ordered != NULL)
-		return (EBUSY);
-
-	if (bp->bio_flags & BIO_ORDERED) {
-		if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) {
-			/*
-			 * This request will be executed once all
-			 * the in-flight requests are completed.
-			 */
+	if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) {
+		if (sc->vtblk_req_ordered != NULL)
+			return (EBUSY);
+		if (bp->bio_flags & BIO_ORDERED) {
 			if (!virtqueue_empty(vq))
 				return (EBUSY);
 			ordered = 1;
-		} else
-			req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
+			req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER;
+		}
 	}
 
 	sglist_reset(sg);
@@ -933,7 +921,7 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
 	if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
 		error = sglist_append_bio(sg, bp);
 		if (error || sg->sg_nseg == sg->sg_maxseg) {
-			panic("%s: data buffer too big bio:%p error:%d",
+			panic("%s: bio %p data buffer too big %d",
 			    __func__, bp, error);
 		}
 
@@ -953,44 +941,156 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
 	return (error);
 }
 
+static int
+vtblk_request_error(struct vtblk_request *req)
+{
+	int error;
+
+	switch (req->vbr_ack) {
+	case VIRTIO_BLK_S_OK:
+		error = 0;
+		break;
+	case VIRTIO_BLK_S_UNSUPP:
+		error = ENOTSUP;
+		break;
+	default:
+		error = EIO;
+		break;
+	}
+
+	return (error);
+}
+
 static void
-vtblk_vq_intr(void *xsc)
+vtblk_queue_completed(struct vtblk_softc *sc, struct bio_queue *queue)
+{
+	struct vtblk_request *req;
+	struct bio *bp;
+
+	while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) {
+		if (sc->vtblk_req_ordered != NULL) {
+			MPASS(sc->vtblk_req_ordered == req);
+			sc->vtblk_req_ordered = NULL;
+		}
+
+		bp = req->vbr_bp;
+		bp->bio_error = vtblk_request_error(req);
+		TAILQ_INSERT_TAIL(queue, bp, bio_queue);
+
+		vtblk_request_enqueue(sc, req);
+	}
+}
+
+static void
+vtblk_done_completed(struct vtblk_softc *sc, struct bio_queue *queue)
+{
+	struct bio *bp, *tmp;
+
+	TAILQ_FOREACH_SAFE(bp, queue, bio_queue, tmp) {
+		if (bp->bio_error != 0)
+			disk_err(bp, "hard error", -1, 1);
+		vtblk_bio_done(sc, bp, bp->bio_error);
+	}
+}
+
+static void
+vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
 {
-	struct vtblk_softc *sc;
 	struct virtqueue *vq;
+	struct vtblk_request *req;
+	int last;
 
-	sc = xsc;
 	vq = sc->vtblk_vq;
+	last = 0;
 
-again:
-	VTBLK_LOCK(sc);
-	if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
-		VTBLK_UNLOCK(sc);
-		return;
+	while ((req = virtqueue_drain(vq, &last)) != NULL) {
+		if (!skip_done)
+			vtblk_bio_done(sc, req->vbr_bp, ENXIO);
+
+		vtblk_request_enqueue(sc, req);
 	}
 
-	vtblk_finish_completed(sc);
+	sc->vtblk_req_ordered = NULL;
+	KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
+}
 
-	if ((sc->vtblk_flags & VTBLK_FLAG_SUSPEND) == 0)
-		vtblk_startio(sc);
-	else
-		wakeup(&sc->vtblk_vq);
+static void
+vtblk_drain(struct vtblk_softc *sc)
+{
+	struct bio_queue queue;
+	struct bio_queue_head *bioq;
+	struct vtblk_request *req;
+	struct bio *bp;
 
-	if (virtqueue_enable_intr(vq) != 0) {
-		virtqueue_disable_intr(vq);
-		VTBLK_UNLOCK(sc);
-		goto again;
+	bioq = &sc->vtblk_bioq;
+	TAILQ_INIT(&queue);
+
+	if (sc->vtblk_vq != NULL) {
+		vtblk_queue_completed(sc, &queue);
+		vtblk_done_completed(sc, &queue);
+
+		vtblk_drain_vq(sc, 0);
 	}
 
-	VTBLK_UNLOCK(sc);
+	while ((req = vtblk_request_next_ready(sc)) != NULL) {
+		vtblk_bio_done(sc, req->vbr_bp, ENXIO);
+		vtblk_request_enqueue(sc, req);
+	}
+
+	while (bioq_first(bioq) != NULL) {
+		bp = bioq_takefirst(bioq);
+		vtblk_bio_done(sc, bp, ENXIO);
+	}
+
+	vtblk_request_free(sc);
 }
 
 static void
-vtblk_stop(struct vtblk_softc *sc)
+vtblk_startio(struct vtblk_softc *sc)
 {
+	struct virtqueue *vq;
+	struct vtblk_request *req;
+	int enq;
 
-	virtqueue_disable_intr(sc->vtblk_vq);
-	virtio_stop(sc->vtblk_dev);
+	VTBLK_LOCK_ASSERT(sc);
+	vq = sc->vtblk_vq;
+	enq = 0;
+
+	if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
+		return;
+
+	while (!virtqueue_full(vq)) {
+		req = vtblk_request_next(sc);
+		if (req == NULL)
+			break;
+
+		if (vtblk_request_execute(sc, req) != 0) {
+			vtblk_request_requeue_ready(sc, req);
+			break;
+		}
+
+		enq++;
+	}
+
+	if (enq > 0)
+		virtqueue_notify(vq);
+}
+
+static void
+vtblk_bio_done(struct vtblk_softc *sc, struct bio *bp, int error)
+{
+
+	/* Because of GEOM direct dispatch, we cannot hold any locks. */
+	if (sc != NULL)
+		VTBLK_LOCK_ASSERT_NOTOWNED(sc);
+
+	if (error) {
+		bp->bio_resid = bp->bio_bcount;
+		bp->bio_error = error;
+		bp->bio_flags |= BIO_ERROR;
+	}
+
+	biodone(bp);
 }
 
 #define VTBLK_GET_CONFIG(_dev, _feature, _field, _cfg)			\
@@ -1025,7 +1125,7 @@ vtblk_read_config(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
 #undef VTBLK_GET_CONFIG
 
 static void
-vtblk_get_ident(struct vtblk_softc *sc)
+vtblk_ident(struct vtblk_softc *sc)
 {
 	struct bio buf;
 	struct disk *dp;
@@ -1038,7 +1138,7 @@ vtblk_get_ident(struct vtblk_softc *sc)
 	if (vtblk_tunable_int(sc, "no_ident", vtblk_no_ident) != 0)
 		return;
 
-	req = vtblk_dequeue_request(sc);
+	req = vtblk_request_dequeue(sc);
 	if (req == NULL)
 		return;
 
@@ -1058,7 +1158,7 @@ vtblk_get_ident(struct vtblk_softc *sc)
 	error = vtblk_poll_request(sc, req);
 	VTBLK_UNLOCK(sc);
 
-	vtblk_enqueue_request(sc, req);
+	vtblk_request_enqueue(sc, req);
 
 	if (error) {
 		device_printf(sc->vtblk_dev,
@@ -1066,77 +1166,6 @@ vtblk_get_ident(struct vtblk_softc *sc)
 	}
 }
 
-static void
-vtblk_prepare_dump(struct vtblk_softc *sc)
-{
-	device_t dev;
-	struct virtqueue *vq;
-
-	dev = sc->vtblk_dev;
-	vq = sc->vtblk_vq;
-
-	vtblk_stop(sc);
-
-	/*
-	 * Drain all requests caught in-flight in the virtqueue,
-	 * skipping biodone(). When dumping, only one request is
-	 * outstanding at a time, and we just poll the virtqueue
-	 * for the response.
-	 */
-	vtblk_drain_vq(sc, 1);
-
-	if (virtio_reinit(dev, sc->vtblk_features) != 0) {
-		panic("%s: cannot reinit VirtIO block device during dump",
-		    device_get_nameunit(dev));
-	}
-
-	virtqueue_disable_intr(vq);
-	virtio_reinit_complete(dev);
-}
-
-static int
-vtblk_write_dump(struct vtblk_softc *sc, void *virtual, off_t offset,
-    size_t length)
-{
-	struct bio buf;
-	struct vtblk_request *req;
-
-	req = &sc->vtblk_dump_request;
-	req->vbr_ack = -1;
-	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
-	req->vbr_hdr.ioprio = 1;
-	req->vbr_hdr.sector = offset / 512;
-
-	req->vbr_bp = &buf;
-	bzero(&buf, sizeof(struct bio));
-
-	buf.bio_cmd = BIO_WRITE;
-	buf.bio_data = virtual;
-	buf.bio_bcount = length;
-
-	return (vtblk_poll_request(sc, req));
-}
-
-static int
-vtblk_flush_dump(struct vtblk_softc *sc)
-{
-	struct bio buf;
-	struct vtblk_request *req;
-
-	req = &sc->vtblk_dump_request;
-	req->vbr_ack = -1;
-	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
-	req->vbr_hdr.ioprio = 1;
-	req->vbr_hdr.sector = 0;
-
-	req->vbr_bp = &buf;
-	bzero(&buf, sizeof(struct bio));
-
-	buf.bio_cmd = BIO_FLUSH;
-
-	return (vtblk_poll_request(sc, req));
-}
-
 static int
 vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
 {
@@ -1148,7 +1177,7 @@ vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
 	if (!virtqueue_empty(vq))
 		return (EBUSY);
 
-	error = vtblk_execute_request(sc, req);
+	error = vtblk_request_execute(sc, req);
 	if (error)
 		return (error);
 
@@ -1164,212 +1193,188 @@ vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
 	return (error);
 }
 
-static void
-vtblk_finish_completed(struct vtblk_softc *sc)
+static int
+vtblk_quiesce(struct vtblk_softc *sc)
 {
-	struct vtblk_request *req;
-	struct bio *bp;
 	int error;
 
-	while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) {
-		bp = req->vbr_bp;
+	VTBLK_LOCK_ASSERT(sc);
+	error = 0;
 
-		if (sc->vtblk_req_ordered != NULL) {
-			/* This should be the only outstanding request. */
-			MPASS(sc->vtblk_req_ordered == req);
-			sc->vtblk_req_ordered = NULL;
+	while (!virtqueue_empty(sc->vtblk_vq)) {
+		if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq",
+		    VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) {
+			error = EBUSY;
+			break;
 		}
-
-		error = vtblk_request_error(req);
-		if (error)
-			disk_err(bp, "hard error", -1, 1);
-
-		vtblk_finish_bio(bp, error);
-		vtblk_enqueue_request(sc, req);
 	}
+
+	return (error);
 }
 
 static void
-vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
+vtblk_vq_intr(void *xsc)
 {
+	struct vtblk_softc *sc;
 	struct virtqueue *vq;
-	struct vtblk_request *req;
-	int last;
+	struct bio_queue queue;
 
+	sc = xsc;
 	vq = sc->vtblk_vq;
-	last = 0;
-
-	while ((req = virtqueue_drain(vq, &last)) != NULL) {
-		if (!skip_done)
-			vtblk_finish_bio(req->vbr_bp, ENXIO);
-
-		vtblk_enqueue_request(sc, req);
-	}
+	TAILQ_INIT(&queue);
 
-	sc->vtblk_req_ordered = NULL;
-	KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
-}
-
-static void
-vtblk_drain(struct vtblk_softc *sc)
-{
-	struct bio_queue_head *bioq;
-	struct vtblk_request *req;
-	struct bio *bp;
+	VTBLK_LOCK(sc);
 
-	bioq = &sc->vtblk_bioq;
+again:
+	if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
+		goto out;
 
-	if (sc->vtblk_vq != NULL) {
-		vtblk_finish_completed(sc);
-		vtblk_drain_vq(sc, 0);
-	}
+	vtblk_queue_completed(sc, &queue);
+	vtblk_startio(sc);
 
-	while ((req = vtblk_dequeue_ready(sc)) != NULL) {
-		vtblk_finish_bio(req->vbr_bp, ENXIO);
-		vtblk_enqueue_request(sc, req);
+	if (virtqueue_enable_intr(vq) != 0) {
+		virtqueue_disable_intr(vq);
+		goto again;
 	}
 
-	while (bioq_first(bioq) != NULL) {
-		bp = bioq_takefirst(bioq);
-		vtblk_finish_bio(bp, ENXIO);
-	}
+	if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
+		wakeup(&sc->vtblk_vq);
 
-	vtblk_free_requests(sc);
+out:
+	VTBLK_UNLOCK(sc);
+	vtblk_done_completed(sc, &queue);
 }
 
-#ifdef INVARIANTS
 static void
-vtblk_request_invariants(struct vtblk_request *req)
+vtblk_stop(struct vtblk_softc *sc)
 {
-	int hdr_nsegs, ack_nsegs;
 
-	hdr_nsegs = sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr));
-	ack_nsegs = sglist_count(&req->vbr_ack, sizeof(req->vbr_ack));
-
-	KASSERT(hdr_nsegs == 1, ("request header crossed page boundary"));
-	KASSERT(ack_nsegs == 1, ("request ack crossed page boundary"));
+	virtqueue_disable_intr(sc->vtblk_vq);
+	virtio_stop(sc->vtblk_dev);
 }
-#endif
 
-static int
-vtblk_alloc_requests(struct vtblk_softc *sc)
+static void
+vtblk_dump_prepare(struct vtblk_softc *sc)
 {
-	struct vtblk_request *req;
-	int i, nreqs;
+	device_t dev;
+	struct virtqueue *vq;
 
-	nreqs = virtqueue_size(sc->vtblk_vq);
+	dev = sc->vtblk_dev;
+	vq = sc->vtblk_vq;
+
+	vtblk_stop(sc);
 
 	/*
-	 * Preallocate sufficient requests to keep the virtqueue full. Each
-	 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
-	 * the number allocated when indirect descriptors are not available.
+	 * Drain all requests caught in-flight in the virtqueue,
+	 * skipping biodone(). When dumping, only one request is
+	 * outstanding at a time, and we just poll the virtqueue
+	 * for the response.
 	 */
-	if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
-		nreqs /= VTBLK_MIN_SEGMENTS;
-
-	for (i = 0; i < nreqs; i++) {
-		req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT);
-		if (req == NULL)
-			return (ENOMEM);
-
-#ifdef INVARIANTS
-		vtblk_request_invariants(req);
-#endif
+	vtblk_drain_vq(sc, 1);
 
-		sc->vtblk_request_count++;
-		vtblk_enqueue_request(sc, req);
+	if (virtio_reinit(dev, sc->vtblk_features) != 0) {
+		panic("%s: cannot reinit VirtIO block device during dump",
+		    device_get_nameunit(dev));
 	}
 
-	return (0);
+	virtqueue_disable_intr(vq);
+	virtio_reinit_complete(dev);
 }
 
-static void
-vtblk_free_requests(struct vtblk_softc *sc)
+static int
+vtblk_dump_write(struct vtblk_softc *sc, void *virtual, off_t offset,
+    size_t length)
 {
+	struct bio buf;
 	struct vtblk_request *req;
 
-	KASSERT(TAILQ_EMPTY(&sc->vtblk_req_ready),
-	    ("%s: ready requests left on queue", __func__));
-
-	while ((req = vtblk_dequeue_request(sc)) != NULL) {
-		sc->vtblk_request_count--;
-		free(req, M_DEVBUF);
-	}
-
-	KASSERT(sc->vtblk_request_count == 0,
-	    ("%s: leaked %d requests", __func__, sc->vtblk_request_count));
-}
+	req = &sc->vtblk_dump_request;
+	req->vbr_ack = -1;
+	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
+	req->vbr_hdr.ioprio = 1;
+	req->vbr_hdr.sector = offset / 512;
 
-static struct vtblk_request *
-vtblk_dequeue_request(struct vtblk_softc *sc)
-{
-	struct vtblk_request *req;
+	req->vbr_bp = &buf;
+	bzero(&buf, sizeof(struct bio));
 
-	req = TAILQ_FIRST(&sc->vtblk_req_free);
-	if (req != NULL)
-		TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
+	buf.bio_cmd = BIO_WRITE;
+	buf.bio_data = virtual;
+	buf.bio_bcount = length;
 
-	return (req);
+	return (vtblk_poll_request(sc, req));
 }
 
-static void
-vtblk_enqueue_request(struct vtblk_softc *sc, struct vtblk_request *req)
+static int
+vtblk_dump_flush(struct vtblk_softc *sc)
 {
+	struct bio buf;
+	struct vtblk_request *req;
 
-	bzero(req, sizeof(struct vtblk_request));
-	TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
-}
+	req = &sc->vtblk_dump_request;
+	req->vbr_ack = -1;
+	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
+	req->vbr_hdr.ioprio = 1;
+	req->vbr_hdr.sector = 0;
 
-static struct vtblk_request *
-vtblk_dequeue_ready(struct vtblk_softc *sc)
-{
-	struct vtblk_request *req;
+	req->vbr_bp = &buf;
+	bzero(&buf, sizeof(struct bio));
 
-	req = TAILQ_FIRST(&sc->vtblk_req_ready);
-	if (req != NULL)
-		TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
+	buf.bio_cmd = BIO_FLUSH;
 
-	return (req);
+	return (vtblk_poll_request(sc, req));
 }
 
 static void
-vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
+vtblk_set_write_cache(struct vtblk_softc *sc, int wc)
 {
 
-	TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
+	/* Set either writeback (1) or writethrough (0) mode. */
+	virtio_write_dev_config_1(sc->vtblk_dev,
+	    offsetof(struct virtio_blk_config, writeback), wc);
 }
 
 static int
-vtblk_request_error(struct vtblk_request *req)
+vtblk_write_cache_enabled(struct vtblk_softc *sc,
+    struct virtio_blk_config *blkcfg)
 {
-	int error;
+	int wc;
 
-	switch (req->vbr_ack) {
-	case VIRTIO_BLK_S_OK:
-		error = 0;
-		break;
-	case VIRTIO_BLK_S_UNSUPP:
-		error = ENOTSUP;
-		break;
-	default:
-		error = EIO;
-		break;
-	}
+	if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
+		wc = vtblk_tunable_int(sc, "writecache_mode",
+		    vtblk_writecache_mode);
+		if (wc >= 0 && wc < VTBLK_CACHE_MAX)
+			vtblk_set_write_cache(sc, wc);
+		else
+			wc = blkcfg->writeback;
+	} else
+		wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);
 
-	return (error);
+	return (wc);
 }
 
-static void
-vtblk_finish_bio(struct bio *bp, int error)
+static int
+vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)
 {
+	struct vtblk_softc *sc;
+	int wc, error;
 
-	if (error) {
-		bp->bio_resid = bp->bio_bcount;
-		bp->bio_error = error;
-		bp->bio_flags |= BIO_ERROR;
-	}
+	sc = oidp->oid_arg1;
+	wc = sc->vtblk_write_cache;
 
-	biodone(bp);
+	error = sysctl_handle_int(oidp, &wc, 0, req);
+	if (error || req->newptr == NULL)
+		return (error);
+	if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
+		return (EPERM);
+	if (wc < 0 || wc >= VTBLK_CACHE_MAX)
+		return (EINVAL);
+
+	VTBLK_LOCK(sc);
+	sc->vtblk_write_cache = wc;
+	vtblk_set_write_cache(sc, sc->vtblk_write_cache);
+	VTBLK_UNLOCK(sc);
+
+	return (0);
 }
 
 static void