MFC r315579, r315670: Add initial support for multiple MSI-X vectors.

For 24xx and above use 2 vectors (default and response queue).
For 26xx and above use 3 vectors (default, response and ATIO queues).
Due to global lock interrupt hardlers never run simultaneously now, but
at least this allows to save one regitster read per interrupt.

7 files changed, 121 insertions, 64 deletions

diff --git a/sys/dev/isp/isp.c b/sys/dev/isp/isp.c
index 03900dc..3fa49d1 100644
--- a/sys/dev/isp/isp.c
+++ b/sys/dev/isp/isp.c
@@ -2089,7 +2089,7 @@ isp_fibre_init_2400(ispsoftc_t *isp)
 	}
 
 	if (IS_26XX(isp)) {
-		/* We don't support MSI-X yet, so set this unconditionally. */
+		/* Use handshake to reduce global lock congestion. */
 		icbp->icb_fwoptions2 |= ICB2400_OPT2_ENA_IHR;
 		icbp->icb_fwoptions2 |= ICB2400_OPT2_ENA_IHA;
 	}
@@ -2187,6 +2187,12 @@ isp_fibre_init_2400(ispsoftc_t *isp)
 	    DMA_WD1(isp->isp_atioq_dma), DMA_WD0(isp->isp_atioq_dma));
 #endif
 
+	if (ISP_CAP_MSIX(isp) && isp->isp_nirq >= 2) {
+		icbp->icb_msixresp = 1;
+		if (IS_26XX(isp) && isp->isp_nirq >= 3)
+			icbp->icb_msixatio = 2;
+	}
+
 	isp_prt(isp, ISP_LOGDEBUG0, "isp_fibre_init_2400: fwopt1 0x%x fwopt2 0x%x fwopt3 0x%x", icbp->icb_fwoptions1, icbp->icb_fwoptions2, icbp->icb_fwoptions3);
 
 	isp_prt(isp, ISP_LOGDEBUG0, "isp_fibre_init_2400: rqst %04x%04x%04x%04x rsp %04x%04x%04x%04x", DMA_WD3(isp->isp_rquest_dma), DMA_WD2(isp->isp_rquest_dma),
diff --git a/sys/dev/isp/isp_freebsd.c b/sys/dev/isp/isp_freebsd.c
index 2b0a325..7f65e03 100644
--- a/sys/dev/isp/isp_freebsd.c
+++ b/sys/dev/isp/isp_freebsd.c
@@ -4160,6 +4160,34 @@ isp_platform_intr(void *arg)
 }
 
 void
+isp_platform_intr_resp(void *arg)
+{
+	ispsoftc_t *isp = arg;
+
+	ISP_LOCK(isp);
+	isp_intr_respq(isp);
+	ISP_UNLOCK(isp);
+
+	/* We have handshake enabled, so explicitly complete interrupt */
+	ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_RISC_INT);
+}
+
+void
+isp_platform_intr_atio(void *arg)
+{
+	ispsoftc_t *isp = arg;
+
+	ISP_LOCK(isp);
+#ifdef	ISP_TARGET_MODE
+	isp_intr_atioq(isp);
+#endif
+	ISP_UNLOCK(isp);
+
+	/* We have handshake enabled, so explicitly complete interrupt */
+	ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_RISC_INT);
+}
+
+void
 isp_common_dmateardown(ispsoftc_t *isp, struct ccb_scsiio *csio, uint32_t hdl)
 {
 	if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
diff --git a/sys/dev/isp/isp_freebsd.h b/sys/dev/isp/isp_freebsd.h
index c99d557..10a818e 100644
--- a/sys/dev/isp/isp_freebsd.h
+++ b/sys/dev/isp/isp_freebsd.h
@@ -722,6 +722,8 @@ void isp_mbox_release(ispsoftc_t *);
 int isp_fc_scratch_acquire(ispsoftc_t *, int);
 int isp_mstohz(int);
 void isp_platform_intr(void *);
+void isp_platform_intr_resp(void *);
+void isp_platform_intr_atio(void *);
 void isp_common_dmateardown(ispsoftc_t *, struct ccb_scsiio *, uint32_t);
 void isp_fcp_reset_crn(ispsoftc_t *, int, uint32_t, int);
 int isp_fcp_next_crn(ispsoftc_t *, uint8_t *, XS_T *);
@@ -734,8 +736,6 @@ int isp_fcp_next_crn(ispsoftc_t *, uint8_t *, XS_T *);
 	bus_dma_tag_create(a, b, c, d, e, f, g, h, i, j, k, \
 	busdma_lock_mutex, &isp->isp_osinfo.lock, z)
 
-#define	isp_setup_intr	bus_setup_intr
-
 #define	isp_sim_alloc(a, b, c, d, e, f, g, h)	\
 	cam_sim_alloc(a, b, c, d, e, &(d)->isp_osinfo.lock, f, g, h)
 
diff --git a/sys/dev/isp/isp_pci.c b/sys/dev/isp/isp_pci.c
index 887dd42..6210e6d 100644
--- a/sys/dev/isp/isp_pci.c
+++ b/sys/dev/isp/isp_pci.c
@@ -364,15 +364,17 @@ struct isp_pcisoftc {
 	struct resource *		regs;
 	struct resource *		regs1;
 	struct resource *		regs2;
-	void *				irq;
-	int				iqd;
+	struct {
+		int				iqd;
+		struct resource *		irq;
+		void *				ih;
+	} irq[ISP_MAX_IRQS];
 	int				rtp;
 	int				rgd;
 	int				rtp1;
 	int				rgd1;
 	int				rtp2;
 	int				rgd2;
-	void *				ih;
 	int16_t				pci_poff[_NREG_BLKS];
 	bus_dma_tag_t			dmat;
 	int				msicount;
@@ -691,8 +693,8 @@ isp_pci_attach(device_t dev)
 	isp_get_generic_options(dev, isp);
 
 	linesz = PCI_DFLT_LNSZ;
-	pcs->irq = pcs->regs = pcs->regs2 = NULL;
-	pcs->rgd = pcs->rtp = pcs->iqd = 0;
+	pcs->regs = pcs->regs2 = NULL;
+	pcs->rgd = pcs->rtp = 0;
 
 	pcs->pci_dev = dev;
 	pcs->pci_poff[BIU_BLOCK >> _BLK_REG_SHFT] = BIU_REGS_OFF;
@@ -932,41 +934,6 @@ isp_pci_attach(device_t dev)
 	data &= ~1;
 	pci_write_config(dev, PCIR_ROMADDR, data, 4);
 
-	if (IS_26XX(isp)) {
-		/* 26XX chips support only MSI-X, so start from them. */
-		pcs->msicount = imin(pci_msix_count(dev), 1);
-		if (pcs->msicount > 0 &&
-		    (i = pci_alloc_msix(dev, &pcs->msicount)) == 0) {
-			pcs->iqd = 1;
-		} else {
-			pcs->msicount = 0;
-		}
-	}
-	if (pcs->msicount == 0 && (IS_24XX(isp) || IS_2322(isp))) {
-		/*
-		 * Older chips support both MSI and MSI-X, but I have
-		 * feeling that older firmware may not support MSI-X,
-		 * but we have no way to check the firmware flag here.
-		 */
-		pcs->msicount = imin(pci_msi_count(dev), 1);
-		if (pcs->msicount > 0 &&
-		    pci_alloc_msi(dev, &pcs->msicount) == 0) {
-			pcs->iqd = 1;
-		} else {
-			pcs->msicount = 0;
-		}
-	}
-	pcs->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &pcs->iqd, RF_ACTIVE | RF_SHAREABLE);
-	if (pcs->irq == NULL) {
-		device_printf(dev, "could not allocate interrupt\n");
-		goto bad;
-	}
-
-	if (isp_setup_intr(dev, pcs->irq, ISP_IFLAGS, NULL, isp_platform_intr, isp, &pcs->ih)) {
-		device_printf(dev, "could not setup interrupt\n");
-		goto bad;
-	}
-
 	/*
 	 * Last minute checks...
 	 */
@@ -992,11 +959,10 @@ isp_pci_attach(device_t dev)
 	return (0);
 
 bad:
-	if (pcs->ih) {
-		(void) bus_teardown_intr(dev, pcs->irq, pcs->ih);
-	}
-	if (pcs->irq) {
-		(void) bus_release_resource(dev, SYS_RES_IRQ, pcs->iqd, pcs->irq);
+	for (i = 0; i < isp->isp_nirq; i++) {
+		(void) bus_teardown_intr(dev, pcs->irq[i].irq, pcs->irq[i].ih);
+		(void) bus_release_resource(dev, SYS_RES_IRQ, pcs->irq[i].iqd,
+		    pcs->irq[0].irq);
 	}
 	if (pcs->msicount) {
 		pci_release_msi(dev);
@@ -1024,7 +990,7 @@ isp_pci_detach(device_t dev)
 {
 	struct isp_pcisoftc *pcs = device_get_softc(dev);
 	ispsoftc_t *isp = &pcs->pci_isp;
-	int status;
+	int i, status;
 
 	status = isp_detach(isp);
 	if (status)
@@ -1032,9 +998,11 @@ isp_pci_detach(device_t dev)
 	ISP_LOCK(isp);
 	isp_shutdown(isp);
 	ISP_UNLOCK(isp);
-	if (pcs->ih)
-		(void) bus_teardown_intr(dev, pcs->irq, pcs->ih);
-	(void) bus_release_resource(dev, SYS_RES_IRQ, pcs->iqd, pcs->irq);
+	for (i = 0; i < isp->isp_nirq; i++) {
+		(void) bus_teardown_intr(dev, pcs->irq[i].irq, pcs->irq[i].ih);
+		(void) bus_release_resource(dev, SYS_RES_IRQ, pcs->irq[i].iqd,
+		    pcs->irq[i].irq);
+	}
 	if (pcs->msicount)
 		pci_release_msi(dev);
 	(void) bus_release_resource(dev, pcs->rtp, pcs->rgd, pcs->regs);
@@ -2077,8 +2045,59 @@ isp_pci_dmasetup(ispsoftc_t *isp, struct ccb_scsiio *csio, void *ff)
 static int
 isp_pci_irqsetup(ispsoftc_t *isp)
 {
+	device_t dev = isp->isp_osinfo.dev;
+	struct isp_pcisoftc *pcs = device_get_softc(dev);
+	driver_intr_t *f;
+	int i, max_irq;
 
-	return (0);
+	/* Allocate IRQs only once. */
+	if (isp->isp_nirq > 0)
+		return (0);
+
+	ISP_UNLOCK(isp);
+	if (ISP_CAP_MSIX(isp)) {
+		max_irq = min(ISP_MAX_IRQS, IS_26XX(isp) ? 3 : 2);
+		pcs->msicount = imin(pci_msix_count(dev), max_irq);
+		if (pcs->msicount > 0 &&
+		    pci_alloc_msix(dev, &pcs->msicount) != 0)
+			pcs->msicount = 0;
+	}
+	if (pcs->msicount == 0) {
+		pcs->msicount = imin(pci_msi_count(dev), 1);
+		if (pcs->msicount > 0 &&
+		    pci_alloc_msi(dev, &pcs->msicount) != 0)
+			pcs->msicount = 0;
+	}
+	for (i = 0; i < MAX(1, pcs->msicount); i++) {
+		pcs->irq[i].iqd = i + (pcs->msicount > 0);
+		pcs->irq[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+		    &pcs->irq[i].iqd, RF_ACTIVE | RF_SHAREABLE);
+		if (pcs->irq[i].irq == NULL) {
+			device_printf(dev, "could not allocate interrupt\n");
+			break;
+		}
+		if (i == 0)
+			f = isp_platform_intr;
+		else if (i == 1)
+			f = isp_platform_intr_resp;
+		else
+			f = isp_platform_intr_atio;
+		if (bus_setup_intr(dev, pcs->irq[i].irq, ISP_IFLAGS, NULL,
+		    f, isp, &pcs->irq[i].ih)) {
+			device_printf(dev, "could not setup interrupt\n");
+			(void) bus_release_resource(dev, SYS_RES_IRQ,
+			    pcs->irq[i].iqd, pcs->irq[i].irq);
+			break;
+		}
+		if (pcs->msicount > 1) {
+			bus_describe_intr(dev, pcs->irq[i].irq, pcs->irq[i].ih,
+			    "%d", i);
+		}
+		isp->isp_nirq = i + 1;
+	}
+	ISP_LOCK(isp);
+
+	return (isp->isp_nirq == 0);
 }
 
 static void
diff --git a/sys/dev/isp/isp_sbus.c b/sys/dev/isp/isp_sbus.c
index e161045..1a5300f 100644
--- a/sys/dev/isp/isp_sbus.c
+++ b/sys/dev/isp/isp_sbus.c
@@ -139,7 +139,6 @@ isp_sbus_attach(device_t dev)
 	struct isp_sbussoftc *sbs = device_get_softc(dev);
 	ispsoftc_t *isp = &sbs->sbus_isp;
 	int tval, isp_debug, role, ispburst, default_id;
-	int ints_setup = 0;
 
 	sbs->sbus_dev = dev;
 	sbs->sbus_mdvec = mdvec;
@@ -262,12 +261,14 @@ isp_sbus_attach(device_t dev)
 		goto bad;
 	}
 
-	if (isp_setup_intr(dev, sbs->irq, ISP_IFLAGS, NULL, isp_platform_intr,
+	if (bus_setup_intr(dev, sbs->irq, ISP_IFLAGS, NULL, isp_platform_intr,
 	    isp, &sbs->ih)) {
 		device_printf(dev, "could not setup interrupt\n");
+		(void) bus_release_resource(dev, SYS_RES_IRQ,
+		    sbs->iqd, sbs->irq);
 		goto bad;
 	}
-	ints_setup++;
+	isp->isp_nirq = 1;
 
 	/*
 	 * Set up logging levels.
@@ -299,13 +300,10 @@ isp_sbus_attach(device_t dev)
 	return (0);
 
 bad:
-
-	if (sbs && ints_setup) {
+	if (isp->isp_nirq > 0) {
 		(void) bus_teardown_intr(dev, sbs->irq, sbs->ih);
-	}
-
-	if (sbs && sbs->irq) {
-		bus_release_resource(dev, SYS_RES_IRQ, sbs->iqd, sbs->irq);
+		(void) bus_release_resource(dev, SYS_RES_IRQ, sbs->iqd,
+		    sbs->irq);
 	}
 
 	if (sbs->regs) {
@@ -329,9 +327,11 @@ isp_sbus_detach(device_t dev)
 	ISP_LOCK(isp);
 	isp_shutdown(isp);
 	ISP_UNLOCK(isp);
-	if (sbs->ih)
+	if (isp->isp_nirq > 0) {
 		(void) bus_teardown_intr(dev, sbs->irq, sbs->ih);
-	(void) bus_release_resource(dev, SYS_RES_IRQ, sbs->iqd, sbs->irq);
+		(void) bus_release_resource(dev, SYS_RES_IRQ, sbs->iqd,
+		    sbs->irq);
+	}
 	(void) bus_release_resource(dev, SYS_RES_MEMORY, sbs->rgd, sbs->regs);
 	isp_sbus_mbxdmafree(isp);
 	mtx_destroy(&isp->isp_osinfo.lock);
diff --git a/sys/dev/isp/ispmbox.h b/sys/dev/isp/ispmbox.h
index 8bd4d1a..e929e86 100644
--- a/sys/dev/isp/ispmbox.h
+++ b/sys/dev/isp/ispmbox.h
@@ -895,6 +895,8 @@ typedef struct {
 	(IS_24XX(isp)? (isp->isp_fwattr & ISP2400_FW_ATTR_MULTIID) : 0)
 #define	ISP_GET_VPIDX(isp, tag) \
 	(ISP_CAP_MULTI_ID(isp) ? tag : 0)
+#define	ISP_CAP_MSIX(isp)	\
+	(IS_24XX(isp)? (isp->isp_fwattr & ISP2400_FW_ATTR_MSIX) : 0)
 #define	ISP_CAP_VP0(isp)	\
 	(IS_24XX(isp)? (isp->isp_fwattr & ISP2400_FW_ATTR_VP0) : 0)
 
diff --git a/sys/dev/isp/ispvar.h b/sys/dev/isp/ispvar.h
index 35b91be..002280b 100644
--- a/sys/dev/isp/ispvar.h
+++ b/sys/dev/isp/ispvar.h
@@ -80,6 +80,7 @@ struct ispmdvec {
 #endif
 #define	ISP_MAX_TARGETS(isp)	(IS_FC(isp)? MAX_FC_TARG : MAX_TARGETS)
 #define	ISP_MAX_LUNS(isp)	(isp)->isp_maxluns
+#define	ISP_MAX_IRQS		3
 
 /*
  * Macros to access ISP registers through bus specific layers-
@@ -526,6 +527,7 @@ struct ispsoftc {
 	uint16_t		isp_maxcmds;	/* max possible I/O cmds */
 	uint8_t			isp_type;	/* HBA Chip Type */
 	uint8_t			isp_revision;	/* HBA Chip H/W Revision */
+	uint8_t			isp_nirq;	/* number of IRQs */
 	uint16_t		isp_nchan;	/* number of channels */
 	uint32_t		isp_maxluns;	/* maximum luns supported */