MFC Alexander Motin's GEOM direct dispatch work:

r256603:
Introduce new function devstat_end_transaction_bio_bt(), adding new argument
to specify present time.  Use this function to move binuptime() out of lock,
substantially reducing lock congestion when slow timecounter is used.

r256606:
Move g_io_deliver() out of the lock, as required for direct dispatch.
Move g_destroy_bio() out too to reduce lock scope even more.

r256607:
Fix passing uninitialized bio_resid argument to g_trace().

r256610:
Add unmapped I/O support to GEOM RAID.

r256830:
Restore BIO_UNMAPPED and BIO_TRANSIENT_MAPPING in biodonne() when unmapping
temporary mapped buffer.  That fixes double unmap if biodone() called twice
for the same BIO (but with different done methods).

r256880:
Merge GEOM direct dispatch changes from the projects/camlock branch.

When safety requirements are met, it allows to avoid passing I/O requests
to GEOM g_up/g_down thread, executing them directly in the caller context.
That allows to avoid CPU bottlenecks in g_up/g_down threads, plus avoid
several context switches per I/O.

r259247:
Fix bug introduced at r256607.  We have to recalculate bp_resid here since
sizes of original and completed requests may differ due to end of media.

Testing of the stable/10 merge was done by Netflix, but all of the credit
goes to Alexander and iX Systems.

Submitted by:   mav
Sponsored by:   iX Systems

1 files changed, 57 insertions, 16 deletions

diff --git a/sys/geom/geom_disk.c b/sys/geom/geom_disk.c
index 732b35d..e0b9776 100644
--- a/sys/geom/geom_disk.c
+++ b/sys/geom/geom_disk.c
@@ -66,6 +66,7 @@ struct g_disk_softc {
 	struct sysctl_oid	*sysctl_tree;
 	char			led[64];
 	uint32_t		state;
+	struct mtx		 start_mtx;
 };
 
 static g_access_t g_disk_access;
@@ -229,6 +230,7 @@ g_disk_setstate(struct bio *bp, struct g_disk_softc *sc)
 static void
 g_disk_done(struct bio *bp)
 {
+	struct bintime now;
 	struct bio *bp2;
 	struct g_disk_softc *sc;
 
@@ -237,19 +239,40 @@ g_disk_done(struct bio *bp)
 	bp2 = bp->bio_parent;
 	sc = bp2->bio_to->private;
 	bp->bio_completed = bp->bio_length - bp->bio_resid;
+	binuptime(&now);
 	mtx_lock(&sc->done_mtx);
 	if (bp2->bio_error == 0)
 		bp2->bio_error = bp->bio_error;
 	bp2->bio_completed += bp->bio_completed;
 	if ((bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) != 0)
-		devstat_end_transaction_bio(sc->dp->d_devstat, bp);
-	g_destroy_bio(bp);
+		devstat_end_transaction_bio_bt(sc->dp->d_devstat, bp, &now);
 	bp2->bio_inbed++;
 	if (bp2->bio_children == bp2->bio_inbed) {
+		mtx_unlock(&sc->done_mtx);
 		bp2->bio_resid = bp2->bio_bcount - bp2->bio_completed;
 		g_io_deliver(bp2, bp2->bio_error);
+	} else
+		mtx_unlock(&sc->done_mtx);
+	g_destroy_bio(bp);
+}
+
+static void
+g_disk_done_single(struct bio *bp)
+{
+	struct bintime now;
+	struct g_disk_softc *sc;
+
+	bp->bio_completed = bp->bio_length - bp->bio_resid;
+	bp->bio_done = (void *)bp->bio_to;
+	bp->bio_to = LIST_FIRST(&bp->bio_disk->d_geom->provider);
+	if ((bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) != 0) {
+		binuptime(&now);
+		sc = bp->bio_to->private;
+		mtx_lock(&sc->done_mtx);
+		devstat_end_transaction_bio_bt(sc->dp->d_devstat, bp, &now);
+		mtx_unlock(&sc->done_mtx);
 	}
-	mtx_unlock(&sc->done_mtx);
+	g_io_deliver(bp, bp->bio_error);
 }
 
 static int
@@ -277,7 +300,7 @@ g_disk_start(struct bio *bp)
 	struct disk *dp;
 	struct g_disk_softc *sc;
 	int error;
-	off_t off;
+	off_t d_maxsize, off;
 
 	sc = bp->bio_to->private;
 	if (sc == NULL || (dp = sc->dp) == NULL || dp->d_destroyed) {
@@ -294,6 +317,22 @@ g_disk_start(struct bio *bp)
 		/* fall-through */
 	case BIO_READ:
 	case BIO_WRITE:
+		d_maxsize = (bp->bio_cmd == BIO_DELETE) ?
+		    dp->d_delmaxsize : dp->d_maxsize;
+		if (bp->bio_length <= d_maxsize) {
+			bp->bio_disk = dp;
+			bp->bio_to = (void *)bp->bio_done;
+			bp->bio_done = g_disk_done_single;
+			bp->bio_pblkno = bp->bio_offset / dp->d_sectorsize;
+			bp->bio_bcount = bp->bio_length;
+			mtx_lock(&sc->start_mtx);
+			devstat_start_transaction_bio(dp->d_devstat, bp);
+			mtx_unlock(&sc->start_mtx);
+			g_disk_lock_giant(dp);
+			dp->d_strategy(bp);
+			g_disk_unlock_giant(dp);
+			break;
+		}
 		off = 0;
 		bp3 = NULL;
 		bp2 = g_clone_bio(bp);
@@ -302,10 +341,6 @@ g_disk_start(struct bio *bp)
 			break;
 		}
 		do {
-			off_t d_maxsize;
-
-			d_maxsize = (bp->bio_cmd == BIO_DELETE) ?
-			    dp->d_delmaxsize : dp->d_maxsize;
 			bp2->bio_offset += off;
 			bp2->bio_length -= off;
 			if ((bp->bio_flags & BIO_UNMAPPED) == 0) {
@@ -346,7 +381,9 @@ g_disk_start(struct bio *bp)
 			bp2->bio_pblkno = bp2->bio_offset / dp->d_sectorsize;
 			bp2->bio_bcount = bp2->bio_length;
 			bp2->bio_disk = dp;
+			mtx_lock(&sc->start_mtx); 
 			devstat_start_transaction_bio(dp->d_devstat, bp2);
+			mtx_unlock(&sc->start_mtx); 
 			g_disk_lock_giant(dp);
 			dp->d_strategy(bp2);
 			g_disk_unlock_giant(dp);
@@ -402,15 +439,11 @@ g_disk_start(struct bio *bp)
 			error = EOPNOTSUPP;
 			break;
 		}
-		bp2 = g_clone_bio(bp);
-		if (bp2 == NULL) {
-			g_io_deliver(bp, ENOMEM);
-			return;
-		}
-		bp2->bio_done = g_disk_done;
-		bp2->bio_disk = dp;
+		bp->bio_disk = dp;
+		bp->bio_to = (void *)bp->bio_done;
+		bp->bio_done = g_disk_done_single;
 		g_disk_lock_giant(dp);
-		dp->d_strategy(bp2);
+		dp->d_strategy(bp);
 		g_disk_unlock_giant(dp);
 		break;
 	default:
@@ -515,17 +548,24 @@ g_disk_create(void *arg, int flag)
 	g_topology_assert();
 	dp = arg;
 	sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
+	mtx_init(&sc->start_mtx, "g_disk_start", NULL, MTX_DEF);
 	mtx_init(&sc->done_mtx, "g_disk_done", NULL, MTX_DEF);
 	sc->dp = dp;
 	gp = g_new_geomf(&g_disk_class, "%s%d", dp->d_name, dp->d_unit);
 	gp->softc = sc;
 	pp = g_new_providerf(gp, "%s", gp->name);
+	devstat_remove_entry(pp->stat);
+	pp->stat = NULL;
+	dp->d_devstat->id = pp;
 	pp->mediasize = dp->d_mediasize;
 	pp->sectorsize = dp->d_sectorsize;
 	pp->stripeoffset = dp->d_stripeoffset;
 	pp->stripesize = dp->d_stripesize;
 	if ((dp->d_flags & DISKFLAG_UNMAPPED_BIO) != 0)
 		pp->flags |= G_PF_ACCEPT_UNMAPPED;
+	if ((dp->d_flags & DISKFLAG_DIRECT_COMPLETION) != 0)
+		pp->flags |= G_PF_DIRECT_SEND;
+	pp->flags |= G_PF_DIRECT_RECEIVE;
 	if (bootverbose)
 		printf("GEOM: new disk %s\n", gp->name);
 	sysctl_ctx_init(&sc->sysctl_ctx);
@@ -574,6 +614,7 @@ g_disk_providergone(struct g_provider *pp)
 	pp->private = NULL;
 	pp->geom->softc = NULL;
 	mtx_destroy(&sc->done_mtx);
+	mtx_destroy(&sc->start_mtx);
 	g_free(sc);
 }