Correct bioq_disksort so that bioq_insert_tail() offers barrier semantic.

Add the BIO_ORDERED flag for struct bio and update bio clients to use it.

The barrier semantics of bioq_insert_tail() were broken in two ways:

 o In bioq_disksort(), an added bio could be inserted at the head of
   the queue, even when a barrier was present, if the sort key for
   the new entry was less than that of the last queued barrier bio.

 o The last_offset used to generate the sort key for newly queued bios
   did not stay at the position of the barrier until either the
   barrier was de-queued, or a new barrier (which updates last_offset)
   was queued.  When a barrier is in effect, we know that the disk
   will pass through the barrier position just before the
   "blocked bios" are released, so using the barrier's offset for
   last_offset is the optimal choice.

sys/geom/sched/subr_disk.c:
sys/kern/subr_disk.c:
	o Update last_offset in bioq_insert_tail().

	o Only update last_offset in bioq_remove() if the removed bio is
	  at the head of the queue (typically due to a call via
	  bioq_takefirst()) and no barrier is active.

	o In bioq_disksort(), if we have a barrier (insert_point is non-NULL),
	  set prev to the barrier and cur to it's next element.  Now that
	  last_offset is kept at the barrier position, this change isn't
	  strictly necessary, but since we have to take a decision branch
	  anyway, it does avoid one, no-op, loop iteration in the while
	  loop that immediately follows.

	o In bioq_disksort(), bypass the normal sort for bios with the
	  BIO_ORDERED attribute and instead insert them into the queue
	  with bioq_insert_tail().  bioq_insert_tail() not only gives
	  the desired command order during insertion, but also provides
	  barrier semantics so that commands disksorted in the future
	  cannot pass the just enqueued transaction.

sys/sys/bio.h:
	Add BIO_ORDERED as bit 4 of the bio_flags field in struct bio.

sys/cam/ata/ata_da.c:
sys/cam/scsi/scsi_da.c
	Use an ordered command for SCSI/ATA-NCQ commands issued in
	response to bios with the BIO_ORDERED flag set.

sys/cam/scsi/scsi_da.c
	Use an ordered tag when issuing a synchronize cache command.

	Wrap some lines to 80 columns.

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c
sys/geom/geom_io.c
	Mark bios with the BIO_FLUSH command as BIO_ORDERED.

Sponsored by:	Spectra Logic Corporation
MFC after:	1 month

1 files changed, 27 insertions, 10 deletions

diff --git a/sys/geom/sched/subr_disk.c b/sys/geom/sched/subr_disk.c
index 008eaab..db2a9ef 100644
--- a/sys/geom/sched/subr_disk.c
+++ b/sys/geom/sched/subr_disk.c
@@ -86,7 +86,7 @@ __FBSDID("$FreeBSD$");
  * bioq_remove()	remove a generic element from the queue, act as
  *		bioq_takefirst() if invoked on the head of the queue.
  *
- * The semantic of these methods is the same of the operations
+ * The semantic of these methods is the same as the operations
  * on the underlying TAILQ, but with additional guarantees on
  * subsequent bioq_disksort() calls. E.g. bioq_insert_tail()
  * can be useful for making sure that all previous ops are flushed
@@ -115,10 +115,10 @@ void
 gs_bioq_remove(struct bio_queue_head *head, struct bio *bp)
 {
 
-	if (bp == TAILQ_FIRST(&head->queue))
-		head->last_offset = bp->bio_offset + bp->bio_length;
-
-	if (bp == head->insert_point)
+	if (head->insert_point == NULL) {
+		if (bp == TAILQ_FIRST(&head->queue))
+			head->last_offset = bp->bio_offset + bp->bio_length;
+	} else if (bp == head->insert_point)
 		head->insert_point = NULL;
 
 	TAILQ_REMOVE(&head->queue, bp, bio_queue);
@@ -137,7 +137,8 @@ void
 gs_bioq_insert_head(struct bio_queue_head *head, struct bio *bp)
 {
 
-	head->last_offset = bp->bio_offset;
+	if (head->insert_point == NULL)
+		head->last_offset = bp->bio_offset;
 	TAILQ_INSERT_HEAD(&head->queue, bp, bio_queue);
 }
 
@@ -147,6 +148,7 @@ gs_bioq_insert_tail(struct bio_queue_head *head, struct bio *bp)
 
 	TAILQ_INSERT_TAIL(&head->queue, bp, bio_queue);
 	head->insert_point = bp;
+	head->last_offset = bp->bio_offset;
 }
 
 struct bio *
@@ -189,13 +191,28 @@ gs_bioq_bio_key(struct bio_queue_head *head, struct bio *bp)
 void
 gs_bioq_disksort(struct bio_queue_head *head, struct bio *bp)
 {
-	struct bio *cur, *prev = NULL;
-	uoff_t key = gs_bioq_bio_key(head, bp);
+	struct bio *cur, *prev;
+	uoff_t key;
+
+	if ((bp->bio_flags & BIO_ORDERED) != 0) {
+		/*
+		 * Ordered transactions can only be dispatched
+		 * after any currently queued transactions.  They
+		 * also have barrier semantics - no transactions
+		 * queued in the future can pass them.
+		 */
+		gs_bioq_insert_tail(head, bp);
+		return;
+	}
 
+	prev = NULL;
+	key = gs_bioq_bio_key(head, bp);
 	cur = TAILQ_FIRST(&head->queue);
 
-	if (head->insert_point)
-		cur = head->insert_point;
+	if (head->insert_point) {
+		prev = head->insert_point;
+		cur = TAILQ_NEXT(head->insert_point, bio_queue);
+	}
 
 	while (cur != NULL && key >= gs_bioq_bio_key(head, cur)) {
 		prev = cur;