Recover from some major omissions/problems with the 24XX port.

First, we were never correctly checking for a 24XX Status Type 0
response- that cased us to fall through to evaluate status for
commands as if this were a 2100/2200/2300 Status Type 0 response.
This is *close*, but not quite the same. This has been reported
to be apparent with some wierd lun configuration problems with
some arrays. It became glaringly apparent on sparc64 where none
of the correct byte swap things were done.

Fixing this omission then caused a whole universe shifting debug
cycle of endian issues for the 2400. The manual for 24XX f/w turns
out to be wrong about the endianness of a couple of entities. The
lun and cdb fields for the type 7 request are *not* unconditionally
big endian- they happen to be opposite of whatever the endian of
the current machine type is. Same with the sense data for the
24XX type 0 response.

While we're at it investigate and resolve some NVRAM endian
issues.

Approved by:	re (ken)
MFC after:	3 days

4 files changed, 57 insertions, 21 deletions

diff --git a/sys/dev/isp/isp.c b/sys/dev/isp/isp.c
index b099a5a..03c7072 100644
--- a/sys/dev/isp/isp.c
+++ b/sys/dev/isp/isp.c
@@ -4262,7 +4262,7 @@ isp_start(XS_T *xs)
 		return (i);
 	}
 	XS_SETERR(xs, HBA_NOERROR);
-	isp_prt(isp, ISP_LOGDEBUG2,
+	isp_prt(isp, ISP_LOGDEBUG0,
 	    "START cmd for %d.%d.%d cmd 0x%x datalen %ld",
 	    XS_CHANNEL(xs), XS_TGT(xs), XS_LUN(xs), XS_CDBP(xs)[0],
 	    (long) XS_XFRLEN(xs));
@@ -4739,7 +4739,7 @@ again:
 		isp_get_hdr(isp, hp, &sp->req_header);
 		etype = sp->req_header.rqs_entry_type;
 
-		if (IS_24XX(isp) && etype == RQSTYPE_T7RQS) {
+		if (IS_24XX(isp) && etype == RQSTYPE_RESPONSE) {
 			isp24xx_statusreq_t *sp2 = (isp24xx_statusreq_t *)qe;
 			isp_get_24xx_response(isp,
 			    (isp24xx_statusreq_t *)hp, sp2);
@@ -4889,7 +4889,7 @@ again:
 		rlen = 0;
 		snsp = NULL;
 		slen = 0;
-		if (IS_24XX(isp) && (scsi_status & RQCS_RV) != 0) {
+		if (IS_24XX(isp) && (scsi_status & (RQCS_RV|RQCS_SV)) != 0) {
 			resp = ((isp24xx_statusreq_t *)sp)->req_rsp_sense;
 			rlen = ((isp24xx_statusreq_t *)sp)->req_response_len;
 		} else if (IS_FC(isp) && (scsi_status & RQCS_RV) != 0) {
@@ -4927,8 +4927,8 @@ again:
 			if (resp && rlen >= 4 &&
 			    resp[FCP_RSPNS_CODE_OFFSET] != 0) {
 				isp_prt(isp, ISP_LOGWARN,
-				    "%d.%d FCP RESPONSE: 0x%x",
-				    XS_TGT(xs), XS_LUN(xs),
+				    "%d.%d.%d FCP RESPONSE: 0x%x",
+				    XS_CHANNEL(xs), XS_TGT(xs), XS_LUN(xs),
 				    resp[FCP_RSPNS_CODE_OFFSET]);
 				XS_SETERR(xs, HBA_BOTCH);
 			}
@@ -5009,13 +5009,14 @@ again:
 
 		/*
 		 * Free any DMA resources. As a side effect, this may
-		 * also do any cache flushing necessary for data coherence.			 */
+		 * also do any cache flushing necessary for data coherence.
+		 */
 		if (XS_XFRLEN(xs)) {
 			ISP_DMAFREE(isp, xs, sp->req_handle);
 		}
 
 		if (((isp->isp_dblev & (ISP_LOGDEBUG2|ISP_LOGDEBUG3))) ||
-		    ((isp->isp_dblev & ISP_LOGDEBUG1) && ((!XS_NOERR(xs)) ||
+		    ((isp->isp_dblev & ISP_LOGDEBUG0) && ((!XS_NOERR(xs)) ||
 		    (*XS_STSP(xs) != SCSI_GOOD)))) {
 			char skey;
 			if (req_state_flags & RQSF_GOT_SENSE) {
@@ -5897,6 +5898,7 @@ static void
 isp_parse_status_24xx(ispsoftc_t *isp, isp24xx_statusreq_t *sp,
     XS_T *xs, long *rp)
 {
+	int ru_marked, sv_marked;
 	switch (sp->req_completion_status) {
 	case RQCS_COMPLETE:
 		if (XS_NOERR(xs)) {
@@ -5943,7 +5945,8 @@ isp_parse_status_24xx(ispsoftc_t *isp, isp24xx_statusreq_t *sp,
 
 	case RQCS_DATA_OVERRUN:
 		XS_RESID(xs) = sp->req_resid;
-		isp_prt(isp, ISP_LOGERR, "data overrun for command on %d.%d.%d",
+		isp_prt(isp, ISP_LOGERR,
+		    "data overrun for command on %d.%d.%d",
 		    XS_CHANNEL(xs), XS_TGT(xs), XS_LUN(xs));
 		if (XS_NOERR(xs)) {
 			XS_SETERR(xs, HBA_DATAOVR);
@@ -5968,8 +5971,27 @@ isp_parse_status_24xx(ispsoftc_t *isp, isp24xx_statusreq_t *sp,
 		return;
 
 	case RQCS_DATA_UNDERRUN:
-
+		ru_marked = (sp->req_scsi_status & RQCS_RU) != 0;
+		/*
+		 * We can get an underrun w/o things being marked 
+		 * if we got a non-zero status.
+		 */
+		sv_marked = (sp->req_scsi_status & (RQCS_SV|RQCS_RV)) != 0;
+		if ((ru_marked == 0 && sv_marked == 0) ||
+		    (sp->req_resid > XS_XFRLEN(xs))) {
+			isp_prt(isp, ISP_LOGWARN, bun, XS_TGT(xs),
+			    XS_LUN(xs), XS_XFRLEN(xs), sp->req_resid,
+			    (ru_marked)? "marked" : "not marked");
+			if (XS_NOERR(xs)) {
+				XS_SETERR(xs, HBA_BOTCH);
+			}
+			return;
+		}
 		XS_RESID(xs) = sp->req_resid;
+		isp_prt(isp, ISP_LOGDEBUG0,
+		    "%d.%d.%d data underrun (%d) for command 0x%x",
+		    XS_CHANNEL(xs), XS_TGT(xs), XS_LUN(xs),
+		    sp->req_resid, XS_CDBP(xs)[0] & 0xff);
 		if (XS_NOERR(xs)) {
 			XS_SETERR(xs, HBA_NOERROR);
 		}
@@ -7412,13 +7434,16 @@ isp_read_nvram_2400(ispsoftc_t *isp)
 	}
 	if (nvram_data[0] != 'I' || nvram_data[1] != 'S' ||
 	    nvram_data[2] != 'P') {
-		isp_prt(isp, ISP_LOGWARN, "invalid NVRAM header");
+		isp_prt(isp, ISP_LOGWARN, "invalid NVRAM header (%x %x %x)",
+		    nvram_data[0], nvram_data[1], nvram_data[2]);
 		retval = -1;
 		goto out;
 	}
 	dptr = (uint32_t *) nvram_data;
 	for (csum = 0, lwrds = 0; lwrds < ISP2400_NVRAM_SIZE >> 2; lwrds++) {
-		csum += dptr[lwrds];
+		uint32_t tmp;
+		ISP_IOXGET_32(isp, &dptr[lwrds], tmp);
+		csum += tmp;
 	}
 	if (csum != 0) {
 		isp_prt(isp, ISP_LOGWARN, "invalid NVRAM checksum");
@@ -7517,8 +7542,8 @@ isp_rd_2400_nvram(ispsoftc_t *isp, uint32_t addr, uint32_t *rp)
 		}
 	}
 	if (tmp & (1U << 31)) {
-		tmp = ISP_READ(isp, BIU2400_FLASH_DATA);
-		*rp = tmp;
+		*rp = ISP_READ(isp, BIU2400_FLASH_DATA);
+		ISP_SWIZZLE_NVRAM_LONG(isp, rp);
 	} else {
 		*rp = 0xffffffff;
 	}
diff --git a/sys/dev/isp/isp_freebsd.h b/sys/dev/isp/isp_freebsd.h
index c493ecd..7f5bc15 100644
--- a/sys/dev/isp/isp_freebsd.h
+++ b/sys/dev/isp/isp_freebsd.h
@@ -409,6 +409,7 @@ default:							\
 #define	ISP_IOXGET_32(isp, s, d)	d = bswap32(*((uint32_t *)s))
 #endif
 #define	ISP_SWIZZLE_NVRAM_WORD(isp, rp)	*rp = bswap16(*rp)
+#define	ISP_SWIZZLE_NVRAM_LONG(isp, rp)	*rp = bswap32(*rp)
 
 #define	ISP_IOZGET_8(isp, s, d)		d = (*((uint8_t *)s))
 #define	ISP_IOZGET_16(isp, s, d)	d = (*((uint16_t *)s))
@@ -426,6 +427,7 @@ default:							\
 #define	ISP_IOXGET_16(isp, s, d)	d = *(s)
 #define	ISP_IOXGET_32(isp, s, d)	d = *(s)
 #define	ISP_SWIZZLE_NVRAM_WORD(isp, rp)
+#define	ISP_SWIZZLE_NVRAM_LONG(isp, rp)
 
 #define	ISP_IOZPUT_8(isp, s, d)		*(d) = s
 #define	ISP_IOZPUT_16(isp, s, d)	*(d) = bswap16(s)
@@ -437,6 +439,9 @@ default:							\
 
 #endif
 
+#define	ISP_SWAP16(isp, s)	bswap16(s)
+#define	ISP_SWAP32(isp, s)	bswap32(s)
+
 /*
  * Includes of common header files
  */
diff --git a/sys/dev/isp/isp_library.c b/sys/dev/isp/isp_library.c
index 2f628bf..1d6e6ee 100644
--- a/sys/dev/isp/isp_library.c
+++ b/sys/dev/isp/isp_library.c
@@ -593,7 +593,7 @@ isp_put_request_t7(ispsoftc_t *isp, ispreqt7_t *src, ispreqt7_t *dst)
 	a = (uint32_t *) src->req_lun;
 	b = (uint32_t *) dst->req_lun;
 	for (i = 0; i < (ASIZE(src->req_lun) >> 2); i++ ) {
-		ISP_IOZPUT_32(isp, *a++, b++);
+		*b++ = ISP_SWAP32(isp, *a++);
 	}
 	ISP_IOXPUT_8(isp, src->req_alen_datadir, &dst->req_alen_datadir);
 	ISP_IOXPUT_8(isp, src->req_task_management, &dst->req_task_management);
@@ -601,8 +601,8 @@ isp_put_request_t7(ispsoftc_t *isp, ispreqt7_t *src, ispreqt7_t *dst)
 	ISP_IOXPUT_8(isp, src->req_crn, &dst->req_crn);
 	a = (uint32_t *) src->req_cdb;
 	b = (uint32_t *) dst->req_cdb;
-	for (i = 0; i < (ASIZE(src->req_cdb) >> 2); i++ ) {
-		ISP_IOZPUT_32(isp, *a++, b++);
+	for (i = 0; i < (ASIZE(src->req_cdb) >> 2); i++) {
+		*b++ = ISP_SWAP32(isp, *a++);
 	}
 	ISP_IOXPUT_32(isp, src->req_dl, &dst->req_dl);
 	ISP_IOXPUT_16(isp, src->req_tidlo, &dst->req_tidlo);
@@ -695,6 +695,8 @@ isp_get_24xx_response(ispsoftc_t *isp, isp24xx_statusreq_t *src,
     isp24xx_statusreq_t *dst)
 {
 	int i;
+	uint32_t *s, *d;
+
 	isp_get_hdr(isp, &src->req_header, &dst->req_header);
 	ISP_IOXGET_32(isp, &src->req_handle, dst->req_handle);
 	ISP_IOXGET_16(isp, &src->req_completion_status,
@@ -708,9 +710,10 @@ isp_get_24xx_response(ispsoftc_t *isp, isp24xx_statusreq_t *src,
 	ISP_IOXGET_32(isp, &src->req_fcp_residual, dst->req_fcp_residual);
 	ISP_IOXGET_32(isp, &src->req_sense_len, dst->req_sense_len);
 	ISP_IOXGET_32(isp, &src->req_response_len, dst->req_response_len);
-	for (i = 0; i < 28; i++) {
-		ISP_IOXGET_8(isp, &src->req_rsp_sense[i],
-		    dst->req_rsp_sense[i]);
+	s = (uint32_t *)src->req_rsp_sense;
+	d = (uint32_t *)dst->req_rsp_sense;
+	for (i = 0; i < (ASIZE(src->req_rsp_sense) >> 2); i++) {
+		d[i] = ISP_SWAP32(isp, s[i]);
 	}
 }
 
@@ -723,14 +726,14 @@ isp_get_24xx_abrt(ispsoftc_t *isp, isp24xx_abrt_t *src, isp24xx_abrt_t *dst)
 	ISP_IOXGET_16(isp, &src->abrt_nphdl, dst->abrt_nphdl);
 	ISP_IOXGET_16(isp, &src->abrt_options, dst->abrt_options);
 	ISP_IOXGET_32(isp, &src->abrt_cmd_handle, dst->abrt_cmd_handle);
-	for (i = 0; i < ASIZE(&src->abrt_reserved); i++) {
+	for (i = 0; i < ASIZE(src->abrt_reserved); i++) {
 		ISP_IOXGET_8(isp, &src->abrt_reserved[i],
 		    dst->abrt_reserved[i]);
 	}
 	ISP_IOXGET_16(isp, &src->abrt_tidlo, dst->abrt_tidlo);
 	ISP_IOXGET_8(isp, &src->abrt_tidhi, dst->abrt_tidhi);
 	ISP_IOXGET_8(isp, &src->abrt_vpidx, dst->abrt_vpidx);
-	for (i = 0; i < ASIZE(&src->abrt_reserved1); i++) {
+	for (i = 0; i < ASIZE(src->abrt_reserved1); i++) {
 		ISP_IOXGET_8(isp, &src->abrt_reserved1[i],
 		    dst->abrt_reserved1[i]);
 	}
diff --git a/sys/dev/isp/ispvar.h b/sys/dev/isp/ispvar.h
index da803fd..96c8b06 100644
--- a/sys/dev/isp/ispvar.h
+++ b/sys/dev/isp/ispvar.h
@@ -1015,6 +1015,9 @@ int isp_async(ispsoftc_t *, ispasync_t, void *);
  *	ISP_IOXGET_32(ispsoftc_t *, uint32_t *srcptr, uint32_t dstrval)
  *
  *	ISP_SWIZZLE_NVRAM_WORD(ispsoftc_t *, uint16_t *)
+ *	ISP_SWIZZLE_NVRAM_LONG(ispsoftc_t *, uint32_t *)
+ *	ISP_SWAP16(ispsoftc_t *, uint16_t srcval)
+ *	ISP_SWAP32(ispsoftc_t *, uint32_t srcval)
  */
 
 #endif	/* _ISPVAR_H */