MFC r256705:

Optimize isp(4) to reduce CPU usage, especially in target mode:
 - Remove two excessive and slow register reads from isp_intr().  Instead
of rereading value every time, assume that registers contain what we have
written there.
 - Avoid sequential search through 4096 array elements when looking for
command tag.  Use hash of lists to store active tags separately from free
ones and so greatly speedup the searches.

4 files changed, 72 insertions, 67 deletions

diff --git a/sys/dev/isp/isp.c b/sys/dev/isp/isp.c
index 6a3897a..5f273d2 100644
--- a/sys/dev/isp/isp.c
+++ b/sys/dev/isp/isp.c
@@ -1422,7 +1422,7 @@ isp_scsi_init(ispsoftc_t *isp)
 		if (mbs.param[0] != MBOX_COMMAND_COMPLETE) {
 			return;
 		}
-		isp->isp_residx = mbs.param[5];
+		isp->isp_residx = isp->isp_resodx = mbs.param[5];
 
 		MBSINIT(&mbs, MBOX_INIT_REQ_QUEUE_A64, MBLOGALL, 0);
 		mbs.param[1] = RQUEST_QUEUE_LEN(isp);
@@ -1446,7 +1446,7 @@ isp_scsi_init(ispsoftc_t *isp)
 		if (mbs.param[0] != MBOX_COMMAND_COMPLETE) {
 			return;
 		}
-		isp->isp_residx = mbs.param[5];
+		isp->isp_residx = isp->isp_resodx = mbs.param[5];
 
 		MBSINIT(&mbs, MBOX_INIT_REQ_QUEUE, MBLOGALL, 0);
 		mbs.param[1] = RQUEST_QUEUE_LEN(isp);
@@ -1918,6 +1918,7 @@ isp_fibre_init(ispsoftc_t *isp)
 	isp->isp_reqidx = 0;
 	isp->isp_reqodx = 0;
 	isp->isp_residx = 0;
+	isp->isp_resodx = 0;
 
 	/*
 	 * Whatever happens, we're now committed to being here.
@@ -2208,6 +2209,8 @@ isp_fibre_init_2400(ispsoftc_t *isp)
 	isp->isp_reqidx = 0;
 	isp->isp_reqodx = 0;
 	isp->isp_residx = 0;
+	isp->isp_resodx = 0;
+	isp->isp_atioodx = 0;
 
 	/*
 	 * Whatever happens, we're now committed to being here.
@@ -5015,7 +5018,6 @@ isp_intr(ispsoftc_t *isp, uint32_t isr, uint16_t sema, uint16_t mbox)
 	int etype, last_etype = 0;
 
 again:
-	optr = isp->isp_residx;
 	/*
 	 * Is this a mailbox related interrupt?
 	 * The mailbox semaphore will be nonzero if so.
@@ -5067,7 +5069,9 @@ again:
 		/*
 		 * Thank you very much!  *Burrrp*!
 		 */
-		ISP_WRITE(isp, isp->isp_respoutrp, ISP_READ(isp, isp->isp_respinrp));
+		isp->isp_residx = ISP_READ(isp, isp->isp_respinrp);
+		isp->isp_resodx = isp->isp_residx;
+		ISP_WRITE(isp, isp->isp_respoutrp, isp->isp_resodx);
 		if (IS_24XX(isp)) {
 			ISP_DISABLE_INTS(isp);
 		}
@@ -5080,7 +5084,7 @@ again:
 	 */
 	if (IS_24XX(isp)) {
 		iptr = ISP_READ(isp, BIU2400_ATIO_RSPINP);
-		optr = ISP_READ(isp, BIU2400_ATIO_RSPOUTP);
+		optr = isp->isp_atioodx;
 
 		while (optr != iptr) {
 			uint8_t qe[QENTRY_LEN];
@@ -5105,7 +5109,7 @@ again:
 			optr = ISP_NXT_QENTRY(oop, RESULT_QUEUE_LEN(isp));
 			ISP_WRITE(isp, BIU2400_ATIO_RSPOUTP, optr);
 		}
-		optr = isp->isp_residx;
+		isp->isp_atioodx = optr;
 	}
 #endif
 
@@ -5114,18 +5118,19 @@ again:
 	 *
 	 * If we're a 2300 or 2400, we can ask what hardware what it thinks.
 	 */
+#if 0
 	if (IS_23XX(isp) || IS_24XX(isp)) {
 		optr = ISP_READ(isp, isp->isp_respoutrp);
 		/*
 		 * Debug: to be taken out eventually
 		 */
-		if (isp->isp_residx != optr) {
-			isp_prt(isp, ISP_LOGINFO, "isp_intr: hard optr=%x, soft optr %x", optr, isp->isp_residx);
-			isp->isp_residx = optr;
+		if (isp->isp_resodx != optr) {
+			isp_prt(isp, ISP_LOGINFO, "isp_intr: hard optr=%x, soft optr %x", optr, isp->isp_resodx);
+			isp->isp_resodx = optr;
 		}
-	} else {
-		optr = isp->isp_residx;
-	}
+	} else
+#endif
+		optr = isp->isp_resodx;
 
 	/*
 	 * You *must* read the Response Queue In Pointer
@@ -5147,8 +5152,6 @@ again:
 	} else {
 		iptr = ISP_READ(isp, isp->isp_respinrp);
 	}
-	isp->isp_resodx = iptr;
-
 
 	if (optr == iptr && sema == 0) {
 		/*
@@ -5182,7 +5185,7 @@ again:
 			isp_prt(isp, ISP_LOGDEBUG1, "bogus intr- isr %x (%x) iptr %x optr %x", isr, junk, iptr, optr);
 		}
 	}
-	isp->isp_resodx = iptr;
+	isp->isp_residx = iptr;
 
 	while (optr != iptr) {
 		uint8_t qe[QENTRY_LEN];
@@ -5552,13 +5555,9 @@ again:
 	 */
 	if (nlooked) {
 		ISP_WRITE(isp, isp->isp_respoutrp, optr);
-		/*
-		 * While we're at it, read the requst queue out pointer.
-		 */
-		isp->isp_reqodx = ISP_READ(isp, isp->isp_rqstoutrp);
-		if (isp->isp_rscchiwater < ndone) {
+		isp->isp_resodx = optr;
+		if (isp->isp_rscchiwater < ndone)
 			isp->isp_rscchiwater = ndone;
-		}
 	}
 
 out:
@@ -5570,7 +5569,6 @@ out:
 		ISP_WRITE(isp, BIU_SEMA, 0);
 	}
 
-	isp->isp_residx = optr;
 	for (i = 0; i < ndone; i++) {
 		xs = complist[i];
 		if (xs) {
diff --git a/sys/dev/isp/isp_freebsd.c b/sys/dev/isp/isp_freebsd.c
index 2d0d64d..a8b118e 100644
--- a/sys/dev/isp/isp_freebsd.c
+++ b/sys/dev/isp/isp_freebsd.c
@@ -804,6 +804,7 @@ static ISP_INLINE tstate_t *get_lun_statep_from_tag(ispsoftc_t *, int, uint32_t)
 static ISP_INLINE void rls_lun_statep(ispsoftc_t *, tstate_t *);
 static ISP_INLINE inot_private_data_t *get_ntp_from_tagdata(ispsoftc_t *, uint32_t, uint32_t, tstate_t **);
 static ISP_INLINE atio_private_data_t *isp_get_atpd(ispsoftc_t *, tstate_t *, uint32_t);
+static ISP_INLINE atio_private_data_t *isp_find_atpd(ispsoftc_t *, tstate_t *, uint32_t);
 static ISP_INLINE void isp_put_atpd(ispsoftc_t *, tstate_t *, atio_private_data_t *);
 static ISP_INLINE inot_private_data_t *isp_get_ntpd(ispsoftc_t *, tstate_t *);
 static ISP_INLINE inot_private_data_t *isp_find_ntpd(ispsoftc_t *, tstate_t *, uint32_t, uint32_t);
@@ -937,8 +938,8 @@ get_lun_statep_from_tag(ispsoftc_t *isp, int bus, uint32_t tagval)
 		for (i = 0; i < LUN_HASH_SIZE; i++) {
 			ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
 			SLIST_FOREACH(tptr, lhp, next) {
-				atp = isp_get_atpd(isp, tptr, tagval);
-				if (atp && atp->tag == tagval) {
+				atp = isp_find_atpd(isp, tptr, tagval);
+				if (atp) {
 					tptr->hold++;
 					return (tptr);
 				}
@@ -1034,17 +1035,23 @@ isp_get_atpd(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag)
 {
 	atio_private_data_t *atp;
 
-	if (tag == 0) {
-		atp = tptr->atfree;
-		if (atp) {
-			tptr->atfree = atp->next;
-		}
-		return (atp);
+	atp = LIST_FIRST(&tptr->atfree);
+	if (atp) {
+		LIST_REMOVE(atp, next);
+		atp->tag = tag;
+		LIST_INSERT_HEAD(&tptr->atused[ATPDPHASH(tag)], atp, next);
 	}
-	for (atp = tptr->atpool; atp < &tptr->atpool[ATPDPSIZE]; atp++) {
-		if (atp->tag == tag) {
+	return (atp);
+}
+
+static ISP_INLINE atio_private_data_t *
+isp_find_atpd(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag)
+{
+	atio_private_data_t *atp;
+
+	LIST_FOREACH(atp, &tptr->atused[ATPDPHASH(tag)], next) {
+		if (atp->tag == tag)
 			return (atp);
-		}
 	}
 	return (NULL);
 }
@@ -1055,9 +1062,9 @@ isp_put_atpd(ispsoftc_t *isp, tstate_t *tptr, atio_private_data_t *atp)
 	if (atp->ests) {
 		isp_put_ecmd(isp, atp->ests);
 	}
+	LIST_REMOVE(atp, next);
 	memset(atp, 0, sizeof (*atp));
-	atp->next = tptr->atfree;
-	tptr->atfree = atp;
+	LIST_INSERT_HEAD(&tptr->atfree, atp, next);
 }
 
 static void
@@ -1067,11 +1074,8 @@ isp_dump_atpd(ispsoftc_t *isp, tstate_t *tptr)
 	const char *states[8] = { "Free", "ATIO", "CAM", "CTIO", "LAST_CTIO", "PDON", "?6", "7" };
 
 	for (atp = tptr->atpool; atp < &tptr->atpool[ATPDPSIZE]; atp++) {
-		if (atp->tag == 0) {
-			continue;
-		}
 		xpt_print(tptr->owner, "ATP: [0x%x] origdlen %u bytes_xfrd %u lun %u nphdl 0x%04x s_id 0x%06x d_id 0x%06x oxid 0x%04x state %s\n",
-                    atp->tag, atp->orig_datalen, atp->bytes_xfered, atp->lun, atp->nphdl, atp->sid, atp->portid, atp->oxid, states[atp->state & 0x7]);
+		    atp->tag, atp->orig_datalen, atp->bytes_xfered, atp->lun, atp->nphdl, atp->sid, atp->portid, atp->oxid, states[atp->state & 0x7]);
 	}
 }
 
@@ -1137,11 +1141,13 @@ create_lun_state(ispsoftc_t *isp, int bus, struct cam_path *path, tstate_t **rsl
 	SLIST_INIT(&tptr->atios);
 	SLIST_INIT(&tptr->inots);
 	TAILQ_INIT(&tptr->waitq);
-	for (i = 0; i < ATPDPSIZE-1; i++) {
-		tptr->atpool[i].next = &tptr->atpool[i+1];
+	LIST_INIT(&tptr->atfree);
+	for (i = ATPDPSIZE-1; i >= 0; i--)
+		LIST_INSERT_HEAD(&tptr->atfree, &tptr->atpool[i], next);
+	for (i = 0; i < ATPDPHASHSIZE; i++)
+		LIST_INIT(&tptr->atused[i]);
+	for (i = 0; i < ATPDPSIZE-1; i++)
 		tptr->ntpool[i].next = &tptr->ntpool[i+1];
-	}
-	tptr->atfree = tptr->atpool;
 	tptr->ntfree = tptr->ntpool;
 	tptr->hold = 1;
 	ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(xpt_path_lun_id(tptr->owner))], lhp);
@@ -1620,7 +1626,7 @@ isp_target_start_ctio(ispsoftc_t *isp, union ccb *ccb, enum Start_Ctio_How how)
 			}
 		}
 
-		atp = isp_get_atpd(isp, tptr, cso->tag_id);
+		atp = isp_find_atpd(isp, tptr, cso->tag_id);
 		if (atp == NULL) {
 			isp_prt(isp, ISP_LOGERR, "%s: [0x%x] cannot find private data adjunct in %s", __func__, cso->tag_id, __func__);
 			isp_dump_atpd(isp, tptr);
@@ -2252,7 +2258,7 @@ isp_handle_platform_atio(ispsoftc_t *isp, at_entry_t *aep)
 		}
 	}
 
-	atp = isp_get_atpd(isp, tptr, 0);
+	atp = isp_get_atpd(isp, tptr, aep->at_handle);
 	atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios);
 	if (atiop == NULL || atp == NULL) {
 		/*
@@ -2272,7 +2278,6 @@ isp_handle_platform_atio(ispsoftc_t *isp, at_entry_t *aep)
 		rls_lun_statep(isp, tptr);
 		return;
 	}
-	atp->tag = aep->at_handle;
 	atp->rxid = aep->at_tag_val;
 	atp->state = ATPD_STATE_ATIO;
 	SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
@@ -2401,12 +2406,11 @@ isp_handle_platform_atio2(ispsoftc_t *isp, at2_entry_t *aep)
 		goto noresrc;
 	}
 
-	atp = isp_get_atpd(isp, tptr, 0);
+	atp = isp_get_atpd(isp, tptr, aep->at_rxid);
 	if (atp == NULL) {
 		goto noresrc;
 	}
 
-	atp->tag = aep->at_rxid;
 	atp->state = ATPD_STATE_ATIO;
 	SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
 	tptr->atio_count--;
@@ -2638,12 +2642,7 @@ isp_handle_platform_atio7(ispsoftc_t *isp, at7_entry_t *aep)
 		goto noresrc;
 	}
 
-	atp = isp_get_atpd(isp, tptr, 0);
-	if (atp == NULL) {
-		isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atps", aep->at_rxid);
-		goto noresrc;
-	}
-	oatp = isp_get_atpd(isp, tptr, aep->at_rxid);
+	oatp = isp_find_atpd(isp, tptr, aep->at_rxid);
 	if (oatp) {
 		isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] tag wraparound in isp_handle_platforms_atio7 (N-Port Handle 0x%04x S_ID 0x%04x OX_ID 0x%04x) oatp state %d",
 		    aep->at_rxid, nphdl, sid, aep->at_hdr.ox_id, oatp->state);
@@ -2652,8 +2651,12 @@ isp_handle_platform_atio7(ispsoftc_t *isp, at7_entry_t *aep)
 		 */
 		goto noresrc;
 	}
+	atp = isp_get_atpd(isp, tptr, aep->at_rxid);
+	if (atp == NULL) {
+		isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atps", aep->at_rxid);
+		goto noresrc;
+	}
 	atp->word3 = lp->prli_word3;
-	atp->tag = aep->at_rxid;
 	atp->state = ATPD_STATE_ATIO;
 	SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
 	tptr->atio_count--;
@@ -2846,7 +2849,7 @@ isp_handle_srr_notify(ispsoftc_t *isp, void *inot_raw)
 		isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
 		return;
 	}
-	atp = isp_get_atpd(isp, tptr, tag);
+	atp = isp_find_atpd(isp, tptr, tag);
 	if (atp == NULL) {
 		rls_lun_statep(isp, tptr);
 		isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x in SRR Notify", __func__, tag);
@@ -2905,11 +2908,11 @@ isp_handle_platform_ctio(ispsoftc_t *isp, void *arg)
 	}
 
 	if (IS_24XX(isp)) {
-		atp = isp_get_atpd(isp, tptr, ((ct7_entry_t *)arg)->ct_rxid);
+		atp = isp_find_atpd(isp, tptr, ((ct7_entry_t *)arg)->ct_rxid);
 	} else if (IS_FC(isp)) {
-		atp = isp_get_atpd(isp, tptr, ((ct2_entry_t *)arg)->ct_rxid);
+		atp = isp_find_atpd(isp, tptr, ((ct2_entry_t *)arg)->ct_rxid);
 	} else {
-		atp = isp_get_atpd(isp, tptr, ((ct_entry_t *)arg)->ct_fwhandle);
+		atp = isp_find_atpd(isp, tptr, ((ct_entry_t *)arg)->ct_fwhandle);
 	}
 	if (atp == NULL) {
 		rls_lun_statep(isp, tptr);
@@ -3093,7 +3096,7 @@ isp_handle_platform_notify_fc(ispsoftc_t *isp, in_fcentry_t *inp)
 				return;
 			}
 		}
-		atp = isp_get_atpd(isp, tptr, inp->in_seqid);
+		atp = isp_find_atpd(isp, tptr, inp->in_seqid);
 
 		if (atp) {
 			inot = (struct ccb_immediate_notify *) SLIST_FIRST(&tptr->inots);
@@ -3516,7 +3519,7 @@ isp_target_mark_aborted(ispsoftc_t *isp, union ccb *ccb)
 		}
 	}
 
-	atp = isp_get_atpd(isp, tptr, accb->atio.tag_id);
+	atp = isp_find_atpd(isp, tptr, accb->atio.tag_id);
 	if (atp == NULL) {
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 	} else {
@@ -5005,7 +5008,7 @@ isp_action(struct cam_sim *sim, union ccb *ccb)
 
 		if (ccb->ccb_h.func_code == XPT_ACCEPT_TARGET_IO) {
 			if (ccb->atio.tag_id) {
-				atio_private_data_t *atp = isp_get_atpd(isp, tptr, ccb->atio.tag_id);
+				atio_private_data_t *atp = isp_find_atpd(isp, tptr, ccb->atio.tag_id);
 				if (atp) {
 					isp_put_atpd(isp, tptr, atp);
 				}
diff --git a/sys/dev/isp/isp_freebsd.h b/sys/dev/isp/isp_freebsd.h
index d74d859..5fcae7f 100644
--- a/sys/dev/isp/isp_freebsd.h
+++ b/sys/dev/isp/isp_freebsd.h
@@ -95,11 +95,13 @@ void		isp_put_ecmd(struct ispsoftc *, isp_ecmd_t *);
 
 #define	ISP_TARGET_FUNCTIONS	1
 #define	ATPDPSIZE	4096
+#define	ATPDPHASHSIZE	16
+#define	ATPDPHASH(x)	((((x) >> 24) ^ ((x) >> 16) ^ ((x) >> 8) ^ (x)) &  \
+			    ((ATPDPHASHSIZE) - 1))
 
 #include <dev/isp/isp_target.h>
-
-typedef struct {
-	void *		next;
+typedef struct atio_private_data {
+	LIST_ENTRY(atio_private_data)	next;
 	uint32_t	orig_datalen;
 	uint32_t	bytes_xfered;
 	uint32_t	bytes_in_transit;
@@ -173,7 +175,8 @@ typedef struct tstate {
 	inot_private_data_t *	restart_queue;
 	inot_private_data_t *	ntfree;
 	inot_private_data_t	ntpool[ATPDPSIZE];
-	atio_private_data_t *	atfree;
+	LIST_HEAD(, atio_private_data)	atfree;
+	LIST_HEAD(, atio_private_data)	atused[ATPDPHASHSIZE];
 	atio_private_data_t	atpool[ATPDPSIZE];
 } tstate_t;
 
diff --git a/sys/dev/isp/ispvar.h b/sys/dev/isp/ispvar.h
index 2023f61..7c8b251 100644
--- a/sys/dev/isp/ispvar.h
+++ b/sys/dev/isp/ispvar.h
@@ -614,8 +614,9 @@ struct ispsoftc {
 	volatile mbreg_t	isp_curmbx;	/* currently active mailbox command */
 	volatile uint32_t	isp_reqodx;	/* index of last ISP pickup */
 	volatile uint32_t	isp_reqidx;	/* index of next request */
-	volatile uint32_t	isp_residx;	/* index of next result */
+	volatile uint32_t	isp_residx;	/* index of last ISP write */
 	volatile uint32_t	isp_resodx;	/* index of next result */
+	volatile uint32_t	isp_atioodx;	/* index of next ATIO */
 	volatile uint32_t	isp_obits;	/* mailbox command output */
 	volatile uint32_t	isp_serno;	/* rolling serial number */
 	volatile uint16_t	isp_mboxtmp[MAX_MAILBOX];