MFC Alexander Motin's GEOM direct dispatch work:

r256603:
Introduce new function devstat_end_transaction_bio_bt(), adding new argument
to specify present time.  Use this function to move binuptime() out of lock,
substantially reducing lock congestion when slow timecounter is used.

r256606:
Move g_io_deliver() out of the lock, as required for direct dispatch.
Move g_destroy_bio() out too to reduce lock scope even more.

r256607:
Fix passing uninitialized bio_resid argument to g_trace().

r256610:
Add unmapped I/O support to GEOM RAID.

r256830:
Restore BIO_UNMAPPED and BIO_TRANSIENT_MAPPING in biodonne() when unmapping
temporary mapped buffer.  That fixes double unmap if biodone() called twice
for the same BIO (but with different done methods).

r256880:
Merge GEOM direct dispatch changes from the projects/camlock branch.

When safety requirements are met, it allows to avoid passing I/O requests
to GEOM g_up/g_down thread, executing them directly in the caller context.
That allows to avoid CPU bottlenecks in g_up/g_down threads, plus avoid
several context switches per I/O.

r259247:
Fix bug introduced at r256607.  We have to recalculate bp_resid here since
sizes of original and completed requests may differ due to end of media.

Testing of the stable/10 merge was done by Netflix, but all of the credit
goes to Alexander and iX Systems.

Submitted by:   mav
Sponsored by:   iX Systems

1 files changed, 17 insertions, 10 deletions

diff --git a/sys/geom/raid/tr_concat.c b/sys/geom/raid/tr_concat.c
index 60db472..44951d4 100644
--- a/sys/geom/raid/tr_concat.c
+++ b/sys/geom/raid/tr_concat.c
@@ -74,7 +74,8 @@ static struct g_raid_tr_class g_raid_tr_concat_class = {
 	g_raid_tr_concat_methods,
 	sizeof(struct g_raid_tr_concat_object),
 	.trc_enable = 1,
-	.trc_priority = 50
+	.trc_priority = 50,
+	.trc_accept_unmapped = 1
 };
 
 static int
@@ -227,7 +228,10 @@ g_raid_tr_iostart_concat(struct g_raid_tr_object *tr, struct bio *bp)
 
 	offset = bp->bio_offset;
 	remain = bp->bio_length;
-	addr = bp->bio_data;
+	if ((bp->bio_flags & BIO_UNMAPPED) != 0)
+		addr = NULL;
+	else
+		addr = bp->bio_data;
 	no = 0;
 	while (no < vol->v_disks_count &&
 	    offset >= vol->v_subdisks[no].sd_size) {
@@ -244,8 +248,16 @@ g_raid_tr_iostart_concat(struct g_raid_tr_object *tr, struct bio *bp)
 		if (cbp == NULL)
 			goto failure;
 		cbp->bio_offset = offset;
-		cbp->bio_data = addr;
 		cbp->bio_length = length;
+		if ((bp->bio_flags & BIO_UNMAPPED) != 0 &&
+		    bp->bio_cmd != BIO_DELETE) {
+			cbp->bio_ma_offset += (uintptr_t)addr;
+			cbp->bio_ma += cbp->bio_ma_offset / PAGE_SIZE;
+			cbp->bio_ma_offset %= PAGE_SIZE;
+			cbp->bio_ma_n = round_page(cbp->bio_ma_offset +
+			    cbp->bio_length) / PAGE_SIZE;
+		} else
+			cbp->bio_data = addr;
 		cbp->bio_caller1 = sd;
 		bioq_insert_tail(&queue, cbp);
 		remain -= length;
@@ -257,20 +269,15 @@ g_raid_tr_iostart_concat(struct g_raid_tr_object *tr, struct bio *bp)
 		    ("Request ends after volume end (%ju, %ju)",
 			bp->bio_offset, bp->bio_length));
 	} while (remain > 0);
-	for (cbp = bioq_first(&queue); cbp != NULL;
-	    cbp = bioq_first(&queue)) {
-		bioq_remove(&queue, cbp);
+	while ((cbp = bioq_takefirst(&queue)) != NULL) {
 		sd = cbp->bio_caller1;
 		cbp->bio_caller1 = NULL;
 		g_raid_subdisk_iostart(sd, cbp);
 	}
 	return;
 failure:
-	for (cbp = bioq_first(&queue); cbp != NULL;
-	    cbp = bioq_first(&queue)) {
-		bioq_remove(&queue, cbp);
+	while ((cbp = bioq_takefirst(&queue)) != NULL)
 		g_destroy_bio(cbp);
-	}
 	if (bp->bio_error == 0)
 		bp->bio_error = ENOMEM;
 	g_raid_iodone(bp, bp->bio_error);