MFC r267537:

Add support for VERIFY(10/12/16) and COMPARE AND WRITE SCSI commands.

Make data_submit backends method support not only read and write requests,
but also two new ones: verify and compare.  Verify just checks readability
of the data in specified location without transferring them outside.
Compare reads the specified data and compares them to received data,
returning error if they are different.

VERIFY(10/12/16) commands request either verify or compare from backend,
depending on BYTCHK CDB field.  COMPARE AND WRITE command executed in two
stages: first it requests compare, and then, if succeesed, requests write.
Atomicity of operation is guarantied by CTL request ordering code.

Sponsored by:	iXsystems, Inc.

10 files changed, 444 insertions, 62 deletions

diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index 215097b..74f2d02 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -4996,6 +4996,30 @@ bailout:
 
 /*
  * This gets called by a backend driver when it is done with a
+ * data_submit method.
+ */
+void
+ctl_data_submit_done(union ctl_io *io)
+{
+	/*
+	 * If the IO_CONT flag is set, we need to call the supplied
+	 * function to continue processing the I/O, instead of completing
+	 * the I/O just yet.
+	 *
+	 * If there is an error, though, we don't want to keep processing.
+	 * Instead, just send status back to the initiator.
+	 */
+	if ((io->io_hdr.flags & CTL_FLAG_IO_CONT)
+	 && (((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)
+	  || ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS))) {
+		io->scsiio.io_cont(io);
+		return;
+	}
+	ctl_done(io);
+}
+
+/*
+ * This gets called by a backend driver when it is done with a
  * configuration write.
  */
 void
@@ -8582,7 +8606,7 @@ int
 ctl_read_write(struct ctl_scsiio *ctsio)
 {
 	struct ctl_lun *lun;
-	struct ctl_lba_len lbalen;
+	struct ctl_lba_len_flags *lbalen;
 	uint64_t lba;
 	uint32_t num_blocks;
 	int reladdr, fua, dpo, ebp;
@@ -8793,10 +8817,11 @@ ctl_read_write(struct ctl_scsiio *ctsio)
 		return (CTL_RETVAL_COMPLETE);
 	}
 
-	lbalen.lba = lba;
-	lbalen.len = num_blocks;
-	memcpy(ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes, &lbalen,
-	       sizeof(lbalen));
+	lbalen = (struct ctl_lba_len_flags *)
+	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
+	lbalen->lba = lba;
+	lbalen->len = num_blocks;
+	lbalen->flags = isread ? CTL_LLF_READ : CTL_LLF_WRITE;
 
 	ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
 	ctsio->kern_rel_offset = 0;
@@ -8808,6 +8833,228 @@ ctl_read_write(struct ctl_scsiio *ctsio)
 	return (retval);
 }
 
+static int
+ctl_cnw_cont(union ctl_io *io)
+{
+	struct ctl_scsiio *ctsio;
+	struct ctl_lun *lun;
+	struct ctl_lba_len_flags *lbalen;
+	int retval;
+
+	ctsio = &io->scsiio;
+	ctsio->io_hdr.status = CTL_STATUS_NONE;
+	ctsio->io_hdr.flags &= ~CTL_FLAG_IO_CONT;
+	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+	lbalen = (struct ctl_lba_len_flags *)
+	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
+	lbalen->flags = CTL_LLF_WRITE;
+
+	CTL_DEBUG_PRINT(("ctl_cnw_cont: calling data_submit()\n"));
+	retval = lun->backend->data_submit((union ctl_io *)ctsio);
+	return (retval);
+}
+
+int
+ctl_cnw(struct ctl_scsiio *ctsio)
+{
+	struct ctl_lun *lun;
+	struct ctl_lba_len_flags *lbalen;
+	uint64_t lba;
+	uint32_t num_blocks;
+	int fua, dpo;
+	int retval;
+
+	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+
+	CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
+
+	fua = 0;
+	dpo = 0;
+
+	retval = CTL_RETVAL_COMPLETE;
+
+	switch (ctsio->cdb[0]) {
+	case COMPARE_AND_WRITE: {
+		struct scsi_compare_and_write *cdb;
+
+		cdb = (struct scsi_compare_and_write *)ctsio->cdb;
+
+		if (cdb->byte2 & SRW10_FUA)
+			fua = 1;
+		if (cdb->byte2 & SRW10_DPO)
+			dpo = 1;
+		lba = scsi_8btou64(cdb->addr);
+		num_blocks = cdb->length;
+		break;
+	}
+	default:
+		/*
+		 * We got a command we don't support.  This shouldn't
+		 * happen, commands should be filtered out above us.
+		 */
+		ctl_set_invalid_opcode(ctsio);
+		ctl_done((union ctl_io *)ctsio);
+
+		return (CTL_RETVAL_COMPLETE);
+		break; /* NOTREACHED */
+	}
+
+	/*
+	 * XXX KDM what do we do with the DPO and FUA bits?  FUA might be
+	 * interesting for us, but if RAIDCore is in write-back mode,
+	 * getting it to do write-through for a particular transaction may
+	 * not be possible.
+	 */
+
+	/*
+	 * The first check is to make sure we're in bounds, the second
+	 * check is to catch wrap-around problems.  If the lba + num blocks
+	 * is less than the lba, then we've wrapped around and the block
+	 * range is invalid anyway.
+	 */
+	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
+	 || ((lba + num_blocks) < lba)) {
+		ctl_set_lba_out_of_range(ctsio);
+		ctl_done((union ctl_io *)ctsio);
+		return (CTL_RETVAL_COMPLETE);
+	}
+
+	/*
+	 * According to SBC-3, a transfer length of 0 is not an error.
+	 */
+	if (num_blocks == 0) {
+		ctl_set_success(ctsio);
+		ctl_done((union ctl_io *)ctsio);
+		return (CTL_RETVAL_COMPLETE);
+	}
+
+	ctsio->kern_total_len = 2 * num_blocks * lun->be_lun->blocksize;
+	ctsio->kern_rel_offset = 0;
+
+	/*
+	 * Set the IO_CONT flag, so that if this I/O gets passed to
+	 * ctl_data_submit_done(), it'll get passed back to
+	 * ctl_ctl_cnw_cont() for further processing.
+	 */
+	ctsio->io_hdr.flags |= CTL_FLAG_IO_CONT;
+	ctsio->io_cont = ctl_cnw_cont;
+
+	lbalen = (struct ctl_lba_len_flags *)
+	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
+	lbalen->lba = lba;
+	lbalen->len = num_blocks;
+	lbalen->flags = CTL_LLF_COMPARE;
+
+	CTL_DEBUG_PRINT(("ctl_cnw: calling data_submit()\n"));
+	retval = lun->backend->data_submit((union ctl_io *)ctsio);
+	return (retval);
+}
+
+int
+ctl_verify(struct ctl_scsiio *ctsio)
+{
+	struct ctl_lun *lun;
+	struct ctl_lba_len_flags *lbalen;
+	uint64_t lba;
+	uint32_t num_blocks;
+	int bytchk, dpo;
+	int retval;
+
+	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+
+	CTL_DEBUG_PRINT(("ctl_verify: command: %#x\n", ctsio->cdb[0]));
+
+	bytchk = 0;
+	dpo = 0;
+	retval = CTL_RETVAL_COMPLETE;
+
+	switch (ctsio->cdb[0]) {
+	case VERIFY_10: {
+		struct scsi_verify_10 *cdb;
+
+		cdb = (struct scsi_verify_10 *)ctsio->cdb;
+		if (cdb->byte2 & SVFY_BYTCHK)
+			bytchk = 1;
+		if (cdb->byte2 & SVFY_DPO)
+			dpo = 1;
+		lba = scsi_4btoul(cdb->addr);
+		num_blocks = scsi_2btoul(cdb->length);
+		break;
+	}
+	case VERIFY_12: {
+		struct scsi_verify_12 *cdb;
+
+		cdb = (struct scsi_verify_12 *)ctsio->cdb;
+		if (cdb->byte2 & SVFY_BYTCHK)
+			bytchk = 1;
+		if (cdb->byte2 & SVFY_DPO)
+			dpo = 1;
+		lba = scsi_4btoul(cdb->addr);
+		num_blocks = scsi_4btoul(cdb->length);
+		break;
+	}
+	case VERIFY_16: {
+		struct scsi_rw_16 *cdb;
+
+		cdb = (struct scsi_rw_16 *)ctsio->cdb;
+		if (cdb->byte2 & SVFY_BYTCHK)
+			bytchk = 1;
+		if (cdb->byte2 & SVFY_DPO)
+			dpo = 1;
+		lba = scsi_8btou64(cdb->addr);
+		num_blocks = scsi_4btoul(cdb->length);
+		break;
+	}
+	default:
+		/*
+		 * We got a command we don't support.  This shouldn't
+		 * happen, commands should be filtered out above us.
+		 */
+		ctl_set_invalid_opcode(ctsio);
+		ctl_done((union ctl_io *)ctsio);
+		return (CTL_RETVAL_COMPLETE);
+	}
+
+	/*
+	 * The first check is to make sure we're in bounds, the second
+	 * check is to catch wrap-around problems.  If the lba + num blocks
+	 * is less than the lba, then we've wrapped around and the block
+	 * range is invalid anyway.
+	 */
+	if (((lba + num_blocks) > (lun->be_lun->maxlba + 1))
+	 || ((lba + num_blocks) < lba)) {
+		ctl_set_lba_out_of_range(ctsio);
+		ctl_done((union ctl_io *)ctsio);
+		return (CTL_RETVAL_COMPLETE);
+	}
+
+	/*
+	 * According to SBC-3, a transfer length of 0 is not an error.
+	 */
+	if (num_blocks == 0) {
+		ctl_set_success(ctsio);
+		ctl_done((union ctl_io *)ctsio);
+		return (CTL_RETVAL_COMPLETE);
+	}
+
+	lbalen = (struct ctl_lba_len_flags *)
+	    &ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
+	lbalen->lba = lba;
+	lbalen->len = num_blocks;
+	if (bytchk) {
+		lbalen->flags = CTL_LLF_COMPARE;
+		ctsio->kern_total_len = num_blocks * lun->be_lun->blocksize;
+	} else {
+		lbalen->flags = CTL_LLF_VERIFY;
+		ctsio->kern_total_len = 0;
+	}
+	ctsio->kern_rel_offset = 0;
+
+	CTL_DEBUG_PRINT(("ctl_verify: calling data_submit()\n"));
+	retval = lun->backend->data_submit((union ctl_io *)ctsio);
+	return (retval);
+}
+
 int
 ctl_report_luns(struct ctl_scsiio *ctsio)
 {
@@ -9526,6 +9773,7 @@ ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
 
 	bl_ptr->page_code = SVPD_BLOCK_LIMITS;
 	scsi_ulto2b(sizeof(*bl_ptr), bl_ptr->page_length);
+	bl_ptr->max_cmp_write_len = 0xff;
 	scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
 	scsi_ulto4b(MAXPHYS / bs, bl_ptr->opt_txfer_len);
 	if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
@@ -9937,6 +10185,15 @@ ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len)
 		return (1);
 
 	switch (io->scsiio.cdb[0]) {
+	case COMPARE_AND_WRITE: {
+		struct scsi_compare_and_write *cdb;
+
+		cdb = (struct scsi_compare_and_write *)io->scsiio.cdb;
+
+		*lba = scsi_8btou64(cdb->addr);
+		*len = cdb->length;
+		break;
+	}
 	case READ_6:
 	case WRITE_6: {
 		struct scsi_rw_6 *cdb;
@@ -10025,6 +10282,33 @@ ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint32_t *len)
 		*len = scsi_4btoul(cdb->length);
 		break;
 	}
+	case VERIFY_10: {
+		struct scsi_verify_10 *cdb;
+
+		cdb = (struct scsi_verify_10 *)io->scsiio.cdb;
+
+		*lba = scsi_4btoul(cdb->addr);
+		*len = scsi_2btoul(cdb->length);
+		break;
+	}
+	case VERIFY_12: {
+		struct scsi_verify_12 *cdb;
+
+		cdb = (struct scsi_verify_12 *)io->scsiio.cdb;
+
+		*lba = scsi_4btoul(cdb->addr);
+		*len = scsi_4btoul(cdb->length);
+		break;
+	}
+	case VERIFY_16: {
+		struct scsi_verify_16 *cdb;
+
+		cdb = (struct scsi_verify_16 *)io->scsiio.cdb;
+
+		*lba = scsi_8btou64(cdb->addr);
+		*len = scsi_4btoul(cdb->length);
+		break;
+	}
 	default:
 		return (1);
 		break; /* NOTREACHED */
@@ -12753,7 +13037,7 @@ ctl_process_done(union ctl_io *io, int have_lock)
 		switch (io->io_hdr.io_type) {
 		case CTL_IO_SCSI: {
 			int isread;
-			struct ctl_lba_len *lbalen;
+			struct ctl_lba_len_flags *lbalen;
 
 			isread = 0;
 			switch (io->scsiio.cdb[0]) {
@@ -12770,7 +13054,7 @@ ctl_process_done(union ctl_io *io, int have_lock)
 			case WRITE_VERIFY_10:
 			case WRITE_VERIFY_12:
 			case WRITE_VERIFY_16:
-				lbalen = (struct ctl_lba_len *)
+				lbalen = (struct ctl_lba_len_flags *)
 				    &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
 
 				if (isread) {
diff --git a/sys/cam/ctl/ctl.h b/sys/cam/ctl/ctl.h
index 07c82ba..035147f 100644
--- a/sys/cam/ctl/ctl.h
+++ b/sys/cam/ctl/ctl.h
@@ -195,6 +195,7 @@ int ctl_debugconf_sp_select_handler(struct ctl_scsiio *ctsio,
 int ctl_config_move_done(union ctl_io *io);
 void ctl_datamove(union ctl_io *io);
 void ctl_done(union ctl_io *io);
+void ctl_data_submit_done(union ctl_io *io);
 void ctl_config_write_done(union ctl_io *io);
 #if 0
 int ctl_thread(void *arg);
diff --git a/sys/cam/ctl/ctl_backend_block.c b/sys/cam/ctl/ctl_backend_block.c
index 4a50a50..c2a8e2a 100644
--- a/sys/cam/ctl/ctl_backend_block.c
+++ b/sys/cam/ctl/ctl_backend_block.c
@@ -94,9 +94,11 @@ __FBSDID("$FreeBSD$");
  * The idea here is that we'll allocate enough S/G space to hold a 1MB
  * I/O.  If we get an I/O larger than that, we'll split it.
  */
-#define	CTLBLK_MAX_IO_SIZE	(1024 * 1024)
+#define	CTLBLK_HALF_IO_SIZE	(512 * 1024)
+#define	CTLBLK_MAX_IO_SIZE	(CTLBLK_HALF_IO_SIZE * 2)
 #define	CTLBLK_MAX_SEG		MAXPHYS
-#define	CTLBLK_MAX_SEGS		MAX(CTLBLK_MAX_IO_SIZE / CTLBLK_MAX_SEG, 1)
+#define	CTLBLK_HALF_SEGS	MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1)
+#define	CTLBLK_MAX_SEGS		(CTLBLK_HALF_SEGS * 2)
 
 #ifdef CTLBLK_DEBUG
 #define DPRINTF(fmt, args...) \
@@ -107,6 +109,8 @@ __FBSDID("$FreeBSD$");
 
 #define PRIV(io)	\
     ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND])
+#define ARGS(io)	\
+    ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN])
 
 SDT_PROVIDER_DEFINE(cbb);
 
@@ -314,6 +318,13 @@ ctl_free_beio(struct ctl_be_block_io *beio)
 
 		uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr);
 		beio->sg_segs[i].addr = NULL;
+
+		/* For compare we had two equal S/G lists. */
+		if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) {
+			uma_zfree(beio->lun->lun_zone,
+			    beio->sg_segs[i + CTLBLK_HALF_SEGS].addr);
+			beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL;
+		}
 	}
 
 	if (duplicate_free > 0) {
@@ -348,7 +359,7 @@ ctl_complete_beio(struct ctl_be_block_io *beio)
 		beio->beio_cont(beio);
 	} else {
 		ctl_free_beio(beio);
-		ctl_done(io);
+		ctl_data_submit_done(io);
 	}
 }
 
@@ -357,9 +368,11 @@ ctl_be_block_move_done(union ctl_io *io)
 {
 	struct ctl_be_block_io *beio;
 	struct ctl_be_block_lun *be_lun;
+	struct ctl_lba_len_flags *lbalen;
 #ifdef CTL_TIME_IO
 	struct bintime cur_bt;
-#endif  
+#endif
+	int i;
 
 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 	be_lun = beio->lun;
@@ -372,16 +385,37 @@ ctl_be_block_move_done(union ctl_io *io)
 	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
 	io->io_hdr.num_dmas++;
 #endif  
+	io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
 
 	/*
 	 * We set status at this point for read commands, and write
 	 * commands with errors.
 	 */
-	if ((beio->bio_cmd == BIO_READ)
-	 && (io->io_hdr.port_status == 0)
-	 && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
-	 && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
-		ctl_set_success(&io->scsiio);
+	if ((io->io_hdr.port_status == 0) &&
+	    ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0) &&
+	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
+		lbalen = ARGS(beio->io);
+		if (lbalen->flags & CTL_LLF_READ) {
+			ctl_set_success(&io->scsiio);
+		} else if (lbalen->flags & CTL_LLF_COMPARE) {
+			/* We have two data blocks ready for comparison. */
+			for (i = 0; i < beio->num_segs; i++) {
+				if (memcmp(beio->sg_segs[i].addr,
+				    beio->sg_segs[i + CTLBLK_HALF_SEGS].addr,
+				    beio->sg_segs[i].len) != 0)
+					break;
+			}
+			if (i < beio->num_segs)
+				ctl_set_sense(&io->scsiio,
+				    /*current_error*/ 1,
+				    /*sense_key*/ SSD_KEY_MISCOMPARE,
+				    /*asc*/ 0x1D,
+				    /*ascq*/ 0x00,
+				    SSD_ELEM_NONE);
+			else
+				ctl_set_success(&io->scsiio);
+		}
+	}
 	else if ((io->io_hdr.port_status != 0)
 	      && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
 	      && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
@@ -493,12 +527,13 @@ ctl_be_block_biodone(struct bio *bio)
 	}
 
 	/*
-	 * If this is a write, a flush or a delete, we're all done.
+	 * If this is a write, a flush, a delete or verify, we're all done.
 	 * If this is a read, we can now send the data to the user.
 	 */
 	if ((beio->bio_cmd == BIO_WRITE)
 	 || (beio->bio_cmd == BIO_FLUSH)
-	 || (beio->bio_cmd == BIO_DELETE)) {
+	 || (beio->bio_cmd == BIO_DELETE)
+	 || (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 		ctl_set_success(&io->scsiio);
 		ctl_complete_beio(beio);
 	} else {
@@ -574,18 +609,14 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 	io = beio->io;
 	flags = beio->bio_flags;
 
+	bzero(&xuio, sizeof(xuio));
 	if (beio->bio_cmd == BIO_READ) {
 		SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0);
+		xuio.uio_rw = UIO_READ;
 	} else {
 		SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0);
-	}
-
-	bzero(&xuio, sizeof(xuio));
-	if (beio->bio_cmd == BIO_READ)
-		xuio.uio_rw = UIO_READ;
-	else
 		xuio.uio_rw = UIO_WRITE;
-
+	}
 	xuio.uio_offset = beio->io_offset;
 	xuio.uio_resid = beio->io_len;
 	xuio.uio_segflg = UIO_SYSSPACE;
@@ -628,6 +659,7 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 				 (IO_DIRECT|IO_SYNC) : 0, file_data->cred);
 
 		VOP_UNLOCK(be_lun->vn, 0);
+		SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
 	} else {
 		struct mount *mountpoint;
 		int lock_flags;
@@ -669,6 +701,7 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 		VOP_UNLOCK(be_lun->vn, 0);
 
 		vn_finished_write(mountpoint);
+		SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0);
         }
 
 	/*
@@ -695,12 +728,10 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 	 * If this is a write, we're all done.
 	 * If this is a read, we can now send the data to the user.
 	 */
-	if (beio->bio_cmd == BIO_WRITE) {
+	if (ARGS(io)->flags & (CTL_LLF_WRITE | CTL_LLF_VERIFY)) {
 		ctl_set_success(&io->scsiio);
-		SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0);
 		ctl_complete_beio(beio);
 	} else {
-		SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
 #ifdef CTL_TIME_IO
         	getbintime(&io->io_hdr.dma_start_bt);
 #endif  
@@ -937,7 +968,7 @@ ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun,
 
 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 	softc = be_lun->softc;
-	lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
+	lbalen = ARGS(beio->io);
 
 	if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP) ||
 	    (lbalen->flags & SWS_UNMAP && be_lun->unmap == NULL)) {
@@ -1157,11 +1188,10 @@ ctl_be_block_next(struct ctl_be_block_io *beio)
 	ctl_free_beio(beio);
 	if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)
 	  && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
-		ctl_done(io);
+		ctl_data_submit_done(io);
 		return;
 	}
 
-	io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
 	io->io_hdr.status &= ~CTL_STATUS_MASK;
 	io->io_hdr.status |= CTL_STATUS_NONE;
 
@@ -1183,7 +1213,7 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 {
 	struct ctl_be_block_io *beio;
 	struct ctl_be_block_softc *softc;
-	struct ctl_lba_len *lbalen;
+	struct ctl_lba_len_flags *lbalen;
 	struct ctl_ptr_len_flags *bptrlen;
 	uint64_t len_left, lbas;
 	int i;
@@ -1192,10 +1222,11 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 
 	DPRINTF("entered\n");
 
-	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
-		SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0);
-	} else {
+	lbalen = ARGS(io);
+	if (lbalen->flags & CTL_LLF_WRITE) {
 		SDT_PROBE(cbb, kernel, write, start, 0, 0, 0, 0, 0);
+	} else {
+		SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0);
 	}
 
 	beio = ctl_alloc_beio(softc);
@@ -1233,24 +1264,22 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 		break;
 	}
 
-	/*
-	 * This path handles read and write only.  The config write path
-	 * handles flush operations.
-	 */
-	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
-		beio->bio_cmd = BIO_READ;
-		beio->ds_trans_type = DEVSTAT_READ;
-	} else {
+	if (lbalen->flags & CTL_LLF_WRITE) {
 		beio->bio_cmd = BIO_WRITE;
 		beio->ds_trans_type = DEVSTAT_WRITE;
+	} else {
+		beio->bio_cmd = BIO_READ;
+		beio->ds_trans_type = DEVSTAT_READ;
 	}
 
-	lbalen = (struct ctl_lba_len *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
 	DPRINTF("%s at LBA %jx len %u @%ju\n",
 	       (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE",
 	       (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len);
-	lbas = MIN(lbalen->len - bptrlen->len,
-	    CTLBLK_MAX_IO_SIZE / be_lun->blocksize);
+	if (lbalen->flags & CTL_LLF_COMPARE)
+		lbas = CTLBLK_HALF_IO_SIZE;
+	else
+		lbas = CTLBLK_MAX_IO_SIZE;
+	lbas = MIN(lbalen->len - bptrlen->len, lbas / be_lun->blocksize);
 	beio->io_offset = (lbalen->lba + bptrlen->len) * be_lun->blocksize;
 	beio->io_len = lbas * be_lun->blocksize;
 	bptrlen->len += lbas;
@@ -1268,13 +1297,25 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 		DPRINTF("segment %d addr %p len %zd\n", i,
 			beio->sg_segs[i].addr, beio->sg_segs[i].len);
 
+		/* Set up second segment for compare operation. */
+		if (lbalen->flags & CTL_LLF_COMPARE) {
+			beio->sg_segs[i + CTLBLK_HALF_SEGS].len =
+			    beio->sg_segs[i].len;
+			beio->sg_segs[i + CTLBLK_HALF_SEGS].addr =
+			    uma_zalloc(be_lun->lun_zone, M_WAITOK);
+		}
+
 		beio->num_segs++;
 		len_left -= beio->sg_segs[i].len;
 	}
 	if (bptrlen->len < lbalen->len)
 		beio->beio_cont = ctl_be_block_next;
 	io->scsiio.be_move_done = ctl_be_block_move_done;
-	io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
+	/* For compare we have separate S/G lists for read and datamove. */
+	if (lbalen->flags & CTL_LLF_COMPARE)
+		io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS];
+	else
+		io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
 	io->scsiio.kern_data_len = beio->io_len;
 	io->scsiio.kern_data_resid = 0;
 	io->scsiio.kern_sg_entries = beio->num_segs;
diff --git a/sys/cam/ctl/ctl_backend_ramdisk.c b/sys/cam/ctl/ctl_backend_ramdisk.c
index ac03f3a..885e9a6 100644
--- a/sys/cam/ctl/ctl_backend_ramdisk.c
+++ b/sys/cam/ctl/ctl_backend_ramdisk.c
@@ -267,7 +267,7 @@ ctl_backend_ramdisk_move_done(union ctl_io *io)
 					 /*retry_count*/
 					 io->io_hdr.port_status);
 	}
-	ctl_done(io);
+	ctl_data_submit_done(io);
 	return(0);
 }
 
@@ -275,11 +275,16 @@ static int
 ctl_backend_ramdisk_submit(union ctl_io *io)
 {
 	struct ctl_be_lun *ctl_be_lun;
-	struct ctl_lba_len *lbalen;
+	struct ctl_lba_len_flags *lbalen;
 
 	ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
 		CTL_PRIV_BACKEND_LUN].ptr;
-	lbalen = (struct ctl_lba_len *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
+	lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
+	if (lbalen->flags & CTL_LLF_VERIFY) {
+		ctl_set_success(&io->scsiio);
+		ctl_data_submit_done(io);
+		return (CTL_RETVAL_COMPLETE);
+	}
 	io->io_hdr.ctl_private[CTL_PRIV_BACKEND].integer =
 	    lbalen->len * ctl_be_lun->blocksize;
 	ctl_backend_ramdisk_continue(io);
diff --git a/sys/cam/ctl/ctl_cmd_table.c b/sys/cam/ctl/ctl_cmd_table.c
index 145ddb4..7f3c999 100644
--- a/sys/cam/ctl/ctl_cmd_table.c
+++ b/sys/cam/ctl/ctl_cmd_table.c
@@ -273,7 +273,10 @@ struct ctl_cmd_entry ctl_cmd_table[] =
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE},
 
 /* 2F VERIFY(10) */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+			      CTL_FLAG_DATA_OUT |
+			      CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
+ CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE},
 
 /* 30 SEARCH DATA HIGH(10) */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
@@ -591,8 +594,9 @@ struct ctl_cmd_entry ctl_cmd_table[] =
                                   CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE},
 
-/* 89 */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+/* 89 COMPARE AND WRITE */
+{ctl_cnw, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+ CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE},
 
 /* 8A WRITE(16) */
 {ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
@@ -612,7 +616,10 @@ struct ctl_cmd_entry ctl_cmd_table[] =
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE},
 
 /* 8F VERIFY(16) */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+			      CTL_FLAG_DATA_OUT |
+			      CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
+ CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE},
 
 /* 90 PRE-FETCH(16) */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
@@ -737,7 +744,10 @@ struct ctl_cmd_entry ctl_cmd_table[] =
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE},
 
 /* AF VERIFY(12) */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+			      CTL_FLAG_DATA_OUT |
+			      CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
+ CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE},
 
 /* B0 SEARCH DATA HIGH(12) */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
diff --git a/sys/cam/ctl/ctl_io.h b/sys/cam/ctl/ctl_io.h
index e018b3f..e453d21 100644
--- a/sys/cam/ctl/ctl_io.h
+++ b/sys/cam/ctl/ctl_io.h
@@ -138,6 +138,10 @@ struct ctl_lba_len_flags {
 	uint64_t lba;
 	uint32_t len;
 	uint32_t flags;
+#define CTL_LLF_READ	0x10000000
+#define CTL_LLF_WRITE	0x20000000
+#define CTL_LLF_VERIFY	0x40000000
+#define CTL_LLF_COMPARE	0x80000000
 };
 
 struct ctl_ptr_len_flags {
diff --git a/sys/cam/ctl/ctl_private.h b/sys/cam/ctl/ctl_private.h
index dbf6a05..ff04baf 100644
--- a/sys/cam/ctl/ctl_private.h
+++ b/sys/cam/ctl/ctl_private.h
@@ -477,9 +477,11 @@ int ctl_mode_sense(struct ctl_scsiio *ctsio);
 int ctl_read_capacity(struct ctl_scsiio *ctsio);
 int ctl_service_action_in(struct ctl_scsiio *ctsio);
 int ctl_read_write(struct ctl_scsiio *ctsio);
+int ctl_cnw(struct ctl_scsiio *ctsio);
 int ctl_report_luns(struct ctl_scsiio *ctsio);
 int ctl_request_sense(struct ctl_scsiio *ctsio);
 int ctl_tur(struct ctl_scsiio *ctsio);
+int ctl_verify(struct ctl_scsiio *ctsio);
 int ctl_inquiry(struct ctl_scsiio *ctsio);
 int ctl_persistent_reserve_in(struct ctl_scsiio *ctsio);
 int ctl_persistent_reserve_out(struct ctl_scsiio *ctsio);
diff --git a/sys/cam/scsi/scsi_all.c b/sys/cam/scsi/scsi_all.c
index ff9dc17..aab857c 100644
--- a/sys/cam/scsi/scsi_all.c
+++ b/sys/cam/scsi/scsi_all.c
@@ -471,7 +471,8 @@ static struct op_table_entry scsi_op_codes[] = {
 	 */
 	/* 88  MM  O O   O     READ(16) */
 	{ 0x88,	D | T | W | O | B, "READ(16)" },
-	/* 89 */
+	/* 89  O               COMPARE AND WRITE*/
+	{ 0x89,	D, "COMPARE AND WRITE" },
 	/* 8A  OM  O O   O     WRITE(16) */
 	{ 0x8A,	D | T | W | O | B, "WRITE(16)" },
 	/* 8B  O               ORWRITE */
diff --git a/sys/cam/scsi/scsi_all.h b/sys/cam/scsi/scsi_all.h
index 0fa121f..be681fb 100644
--- a/sys/cam/scsi/scsi_all.h
+++ b/sys/cam/scsi/scsi_all.h
@@ -1041,8 +1041,10 @@ struct ata_pass_16 {
 #define	PERSISTENT_RES_OUT	0x5F
 #define	ATA_PASS_16		0x85
 #define	READ_16			0x88
+#define	COMPARE_AND_WRITE	0x89
 #define	WRITE_16		0x8A
 #define	WRITE_VERIFY_16		0x8E
+#define	VERIFY_16		0x8F
 #define	SYNCHRONIZE_CACHE_16	0x91
 #define	WRITE_SAME_16		0x93
 #define	SERVICE_ACTION_IN	0x9E
@@ -1054,6 +1056,7 @@ struct ata_pass_16 {
 #define	READ_12			0xA8
 #define	WRITE_12		0xAA
 #define	WRITE_VERIFY_12		0xAE
+#define	VERIFY_12		0xAF
 #define	READ_ELEMENT_STATUS	0xB8
 #define	READ_CD			0xBE
 
diff --git a/sys/cam/scsi/scsi_da.h b/sys/cam/scsi/scsi_da.h
index 4fbd725..a27b173 100644
--- a/sys/cam/scsi/scsi_da.h
+++ b/sys/cam/scsi/scsi_da.h
@@ -222,18 +222,49 @@ struct scsi_read_format_capacities
 	uint8_t	reserved1[3];
 };
 
-struct scsi_verify
+struct scsi_verify_10
 {
-	uint8_t	opcode;		/* VERIFY */
+	uint8_t	opcode;		/* VERIFY(10) */
 	uint8_t	byte2;
 #define	SVFY_LUN_MASK	0xE0
 #define	SVFY_RELADR	0x01
-#define	SVFY_BYTECHK	0x02
+#define	SVFY_BYTCHK	0x02
 #define	SVFY_DPO	0x10
 	uint8_t	addr[4];	/* LBA to begin verification at */
-	uint8_t	reserved0[1];
-	uint8_t	len[2];		/* number of blocks to verify */
-	uint8_t	reserved1[3];
+	uint8_t	group;
+	uint8_t	length[2];		/* number of blocks to verify */
+	uint8_t	control;
+};
+
+struct scsi_verify_12
+{
+	uint8_t	opcode;		/* VERIFY(12) */
+	uint8_t	byte2;
+	uint8_t	addr[4];	/* LBA to begin verification at */
+	uint8_t	length[4];		/* number of blocks to verify */
+	uint8_t	group;
+	uint8_t	control;
+};
+
+struct scsi_verify_16
+{
+	uint8_t	opcode;		/* VERIFY(16) */
+	uint8_t	byte2;
+	uint8_t	addr[8];	/* LBA to begin verification at */
+	uint8_t	length[4];		/* number of blocks to verify */
+	uint8_t	group;
+	uint8_t	control;
+};
+
+struct scsi_compare_and_write
+{
+	uint8_t	opcode;		/* COMPARE AND WRITE */
+	uint8_t	byte2;
+	uint8_t	addr[8];	/* LBA to begin verification at */
+	uint8_t	reserved[3];
+	uint8_t	length;		/* number of blocks */
+	uint8_t	group;
+	uint8_t	control;
 };
 
 struct scsi_write_and_verify