Split cfiscsi_datamove() in two; no functional changes.

Approved by:	re (glebius)
Sponsored by:	FreeBSD Foundation

1 files changed, 203 insertions, 182 deletions

diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c
index eb66d8c..46eaf08 100644
--- a/sys/cam/ctl/ctl_frontend_iscsi.c
+++ b/sys/cam/ctl/ctl_frontend_iscsi.c
@@ -2239,20 +2239,16 @@ cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id)
 }
 
 static void
-cfiscsi_datamove(union ctl_io *io)
+cfiscsi_datamove_in(union ctl_io *io)
 {
 	struct cfiscsi_session *cs;
 	struct icl_pdu *request, *response;
 	const struct iscsi_bhs_scsi_command *bhssc;
 	struct iscsi_bhs_data_in *bhsdi;
-	struct iscsi_bhs_r2t *bhsr2t;
-	struct cfiscsi_data_wait *cdw;
 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
 	size_t copy_len, len, off;
 	const char *addr;
 	int ctl_sg_count, error, i;
-	uint32_t target_transfer_tag;
-	bool done;
 
 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
 	cs = PDU_SESSION(request);
@@ -2278,215 +2274,240 @@ cfiscsi_datamove(union ctl_io *io)
 	 */
 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
 
-	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
 #if 0
-		if (ctl_sg_count > 1)
-			CFISCSI_SESSION_DEBUG(cs, "ctl_sg_count = %d", ctl_sg_count);
+	if (ctl_sg_count > 1)
+		CFISCSI_SESSION_DEBUG(cs, "ctl_sg_count = %d", ctl_sg_count);
 #endif
 
-		/*
-		 * This is the offset within the current SCSI command;
-		 * i.e. for the first call of datamove(), it will be 0,
-		 * and for subsequent ones it will be the sum of lengths
-		 * of previous ones.
-		 */
-		off = htonl(io->scsiio.kern_rel_offset);
-		if (off > 1)
-			CFISCSI_SESSION_DEBUG(cs, "off = %zd", off);
-
-		i = 0;
-		addr = NULL;
-		len = 0;
-		response = NULL;
-		bhsdi = NULL;
-		for (;;) {
-			KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
+	/*
+	 * This is the offset within the current SCSI command;
+	 * i.e. for the first call of datamove(), it will be 0,
+	 * and for subsequent ones it will be the sum of lengths
+	 * of previous ones.
+	 */
+	off = htonl(io->scsiio.kern_rel_offset);
+	if (off > 1)
+		CFISCSI_SESSION_DEBUG(cs, "off = %zd", off);
+
+	i = 0;
+	addr = NULL;
+	len = 0;
+	response = NULL;
+	bhsdi = NULL;
+	for (;;) {
+		KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
+		if (response == NULL) {
+			response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 			if (response == NULL) {
-				response =
-				    cfiscsi_pdu_new_response(request, M_NOWAIT);
-				if (response == NULL) {
-					CFISCSI_SESSION_WARN(cs, "failed to "
-					    "allocate memory; dropping connection");
-					icl_pdu_free(request);
-					cfiscsi_session_terminate(cs);
-					return;
-				}
-				bhsdi = (struct iscsi_bhs_data_in *)
-				    response->ip_bhs;
-				bhsdi->bhsdi_opcode =
-				    ISCSI_BHS_OPCODE_SCSI_DATA_IN;
-				bhsdi->bhsdi_initiator_task_tag =
-				    bhssc->bhssc_initiator_task_tag;
-				bhsdi->bhsdi_datasn =
-				    htonl(PDU_EXPDATASN(request));
-				PDU_EXPDATASN(request)++;
-				bhsdi->bhsdi_buffer_offset = htonl(off);
-			}
-
-			if (len == 0) {
-				addr = ctl_sglist[i].addr;
-				len = ctl_sglist[i].len;
-				KASSERT(len > 0, ("len <= 0"));
-			}
-
-			copy_len = len;
-			if (response->ip_data_len + copy_len >
-			    cs->cs_max_data_segment_length)
-				copy_len = cs->cs_max_data_segment_length -
-				    response->ip_data_len;
-			KASSERT(copy_len <= len, ("copy_len > len"));
-			error = icl_pdu_append_data(response, addr, copy_len, M_NOWAIT);
-			if (error != 0) {
 				CFISCSI_SESSION_WARN(cs, "failed to "
 				    "allocate memory; dropping connection");
 				icl_pdu_free(request);
-				icl_pdu_free(response);
 				cfiscsi_session_terminate(cs);
 				return;
 			}
-			addr += copy_len;
-			len -= copy_len;
-			off += copy_len;
-			io->scsiio.ext_data_filled += copy_len;
+			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
+			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
+			bhsdi->bhsdi_initiator_task_tag =
+			    bhssc->bhssc_initiator_task_tag;
+			bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
+			PDU_EXPDATASN(request)++;
+			bhsdi->bhsdi_buffer_offset = htonl(off);
+		}
 
-			if (len == 0) {
-				/*
-				 * End of scatter-gather segment;
-				 * proceed to the next one...
-				 */
-				if (i == ctl_sg_count - 1) {
-					/*
-					 * ... unless this was the last one.
-					 */
-					break;
-				}
-				i++;
-			}
+		if (len == 0) {
+			addr = ctl_sglist[i].addr;
+			len = ctl_sglist[i].len;
+			KASSERT(len > 0, ("len <= 0"));
+		}
 
-			if (response->ip_data_len ==
-			    cs->cs_max_data_segment_length) {
+		copy_len = len;
+		if (response->ip_data_len + copy_len >
+		    cs->cs_max_data_segment_length)
+			copy_len = cs->cs_max_data_segment_length -
+			    response->ip_data_len;
+		KASSERT(copy_len <= len, ("copy_len > len"));
+		error = icl_pdu_append_data(response, addr, copy_len, M_NOWAIT);
+		if (error != 0) {
+			CFISCSI_SESSION_WARN(cs, "failed to "
+			    "allocate memory; dropping connection");
+			icl_pdu_free(request);
+			icl_pdu_free(response);
+			cfiscsi_session_terminate(cs);
+			return;
+		}
+		addr += copy_len;
+		len -= copy_len;
+		off += copy_len;
+		io->scsiio.ext_data_filled += copy_len;
+
+		if (len == 0) {
+			/*
+			 * End of scatter-gather segment;
+			 * proceed to the next one...
+			 */
+			if (i == ctl_sg_count - 1) {
 				/*
-				 * Can't stuff more data into the current PDU;
-				 * queue it.  Note that's not enough to check
-				 * for kern_data_resid == 0  instead; there
-				 * may be several Data-In PDUs for the final
-				 * call to cfiscsi_datamove(), and we want
-				 * to set the F flag only on the last of them.
+				 * ... unless this was the last one.
 				 */
-				if (off == io->scsiio.kern_total_len)
-					bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
-				KASSERT(response->ip_data_len > 0,
-				    ("sending empty Data-In"));
-				cfiscsi_pdu_queue(response);
-				response = NULL;
-				bhsdi = NULL;
+				break;
 			}
+			i++;
 		}
-		KASSERT(i == ctl_sg_count - 1, ("missed SG segment"));
-		KASSERT(len == 0, ("missed data from SG segment"));
-		if (response != NULL) {
-			if (off == io->scsiio.kern_total_len) {
+
+		if (response->ip_data_len == cs->cs_max_data_segment_length) {
+			/*
+			 * Can't stuff more data into the current PDU;
+			 * queue it.  Note that's not enough to check
+			 * for kern_data_resid == 0  instead; there
+			 * may be several Data-In PDUs for the final
+			 * call to cfiscsi_datamove(), and we want
+			 * to set the F flag only on the last of them.
+			 */
+			if (off == io->scsiio.kern_total_len)
 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
-			} else {
-				CFISCSI_SESSION_DEBUG(cs, "not setting the F flag; "
-				    "have %zd, need %zd", off,
-				    (size_t)io->scsiio.kern_total_len);
-			}
 			KASSERT(response->ip_data_len > 0,
 			    ("sending empty Data-In"));
 			cfiscsi_pdu_queue(response);
+			response = NULL;
+			bhsdi = NULL;
+		}
+	}
+	KASSERT(i == ctl_sg_count - 1, ("missed SG segment"));
+	KASSERT(len == 0, ("missed data from SG segment"));
+	if (response != NULL) {
+		if (off == io->scsiio.kern_total_len) {
+			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
+		} else {
+			CFISCSI_SESSION_DEBUG(cs, "not setting the F flag; "
+			    "have %zd, need %zd", off,
+			    (size_t)io->scsiio.kern_total_len);
 		}
+		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
+		cfiscsi_pdu_queue(response);
+	}
 
-		io->scsiio.be_move_done(io);
-	} else {
-		CFISCSI_SESSION_LOCK(cs);
-		target_transfer_tag = cs->cs_target_transfer_tag;
-		cs->cs_target_transfer_tag++;
-		CFISCSI_SESSION_UNLOCK(cs);
+	io->scsiio.be_move_done(io);
+}
+
+static void
+cfiscsi_datamove_out(union ctl_io *io)
+{
+	struct cfiscsi_session *cs;
+	struct icl_pdu *request, *response;
+	const struct iscsi_bhs_scsi_command *bhssc;
+	struct iscsi_bhs_r2t *bhsr2t;
+	struct cfiscsi_data_wait *cdw;
+	uint32_t target_transfer_tag;
+	bool done;
+
+	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
+	cs = PDU_SESSION(request);
+
+	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
+	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
+	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
+	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
+
+	/*
+	 * We need to record it so that we can properly report
+	 * underflow/underflow.
+	 */
+	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
+
+	CFISCSI_SESSION_LOCK(cs);
+	target_transfer_tag = cs->cs_target_transfer_tag;
+	cs->cs_target_transfer_tag++;
+	CFISCSI_SESSION_UNLOCK(cs);
 
 #if 0
-		CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
-		    "task tag 0x%x, target transfer tag 0x%x",
-		    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
+	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
+	    "task tag 0x%x, target transfer tag 0x%x",
+	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
 #endif
-		cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
-		if (cdw == NULL) {
-			CFISCSI_SESSION_WARN(cs, "failed to "
-			    "allocate memory; dropping connection");
-			icl_pdu_free(request);
-			cfiscsi_session_terminate(cs);
+	cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
+	if (cdw == NULL) {
+		CFISCSI_SESSION_WARN(cs, "failed to "
+		    "allocate memory; dropping connection");
+		icl_pdu_free(request);
+		cfiscsi_session_terminate(cs);
+	}
+	cdw->cdw_ctl_io = io;
+	cdw->cdw_target_transfer_tag = htonl(target_transfer_tag);
+	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
+
+	if (cs->cs_immediate_data && icl_pdu_data_segment_length(request) > 0) {
+		done = cfiscsi_handle_data_segment(request, cdw);
+		if (done) {
+			uma_zfree(cfiscsi_data_wait_zone, cdw);
+			io->scsiio.be_move_done(io);
+			return;
 		}
-		cdw->cdw_ctl_io = io;
-		cdw->cdw_target_transfer_tag = htonl(target_transfer_tag);
-		cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
-
-		if (cs->cs_immediate_data &&
-		    icl_pdu_data_segment_length(request) > 0) {
-			done = cfiscsi_handle_data_segment(request, cdw);
-			if (done) {
-				uma_zfree(cfiscsi_data_wait_zone, cdw);
-				io->scsiio.be_move_done(io);
-				return;
-			}
 
 #if 0
-			if (io->scsiio.ext_data_filled != 0)
-				CFISCSI_SESSION_DEBUG(cs, "got %zd bytes of immediate data, need %zd",
-				    io->scsiio.ext_data_filled, io->scsiio.kern_data_len);
+		if (io->scsiio.ext_data_filled != 0)
+			CFISCSI_SESSION_DEBUG(cs, "got %zd bytes of immediate data, need %zd",
+			    io->scsiio.ext_data_filled, io->scsiio.kern_data_len);
 #endif
-		}
+	}
 
-		CFISCSI_SESSION_LOCK(cs);
-		TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
-		CFISCSI_SESSION_UNLOCK(cs);
+	CFISCSI_SESSION_LOCK(cs);
+	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
+	CFISCSI_SESSION_UNLOCK(cs);
 
-		/*
-		 * XXX: We should limit the number of outstanding R2T PDUs
-		 * 	per task to MaxOutstandingR2T.
-		 */
-		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
-		if (response == NULL) {
-			CFISCSI_SESSION_WARN(cs, "failed to "
-			    "allocate memory; dropping connection");
-			icl_pdu_free(request);
-			cfiscsi_session_terminate(cs);
-		}
-		bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
-		bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
-		bhsr2t->bhsr2t_flags = 0x80;
-		bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
-		bhsr2t->bhsr2t_initiator_task_tag =
-		    bhssc->bhssc_initiator_task_tag;
-		bhsr2t->bhsr2t_target_transfer_tag =
-		    htonl(target_transfer_tag);
-		/*
-		 * XXX: Here we assume that cfiscsi_datamove() won't ever
-		 *	be running concurrently on several CPUs for a given
-		 *	command.
-		 */
-		bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
-		PDU_R2TSN(request)++;
-		/*
-		 * This is the offset within the current SCSI command;
-		 * i.e. for the first call of datamove(), it will be 0,
-		 * and for subsequent ones it will be the sum of lengths
-		 * of previous ones.
-		 *
-		 * The ext_data_filled is to account for unsolicited
-		 * (immediate) data that might have already arrived.
-		 */
-		bhsr2t->bhsr2t_buffer_offset =
-		    htonl(io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled);
-		/*
-		 * This is the total length (sum of S/G lengths) this call
-		 * to cfiscsi_datamove() is supposed to handle.
-		 *
-		 * XXX: Limit it to MaxBurstLength.
-		 */
-		bhsr2t->bhsr2t_desired_data_transfer_length =
-		    htonl(io->scsiio.kern_data_len - io->scsiio.ext_data_filled);
-		cfiscsi_pdu_queue(response);
+	/*
+	 * XXX: We should limit the number of outstanding R2T PDUs
+	 * 	per task to MaxOutstandingR2T.
+	 */
+	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
+	if (response == NULL) {
+		CFISCSI_SESSION_WARN(cs, "failed to "
+		    "allocate memory; dropping connection");
+		icl_pdu_free(request);
+		cfiscsi_session_terminate(cs);
 	}
+	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
+	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
+	bhsr2t->bhsr2t_flags = 0x80;
+	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
+	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
+	bhsr2t->bhsr2t_target_transfer_tag = htonl(target_transfer_tag);
+	/*
+	 * XXX: Here we assume that cfiscsi_datamove() won't ever
+	 *	be running concurrently on several CPUs for a given
+	 *	command.
+	 */
+	bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
+	PDU_R2TSN(request)++;
+	/*
+	 * This is the offset within the current SCSI command;
+	 * i.e. for the first call of datamove(), it will be 0,
+	 * and for subsequent ones it will be the sum of lengths
+	 * of previous ones.
+	 *
+	 * The ext_data_filled is to account for unsolicited
+	 * (immediate) data that might have already arrived.
+	 */
+	bhsr2t->bhsr2t_buffer_offset =
+	    htonl(io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled);
+	/*
+	 * This is the total length (sum of S/G lengths) this call
+	 * to cfiscsi_datamove() is supposed to handle.
+	 *
+	 * XXX: Limit it to MaxBurstLength.
+	 */
+	bhsr2t->bhsr2t_desired_data_transfer_length =
+	    htonl(io->scsiio.kern_data_len - io->scsiio.ext_data_filled);
+	cfiscsi_pdu_queue(response);
+}
+
+static void
+cfiscsi_datamove(union ctl_io *io)
+{
+
+	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
+		cfiscsi_datamove_in(io);
+	else
+		cfiscsi_datamove_out(io);
 }
 
 static void