Prefer MSI-X to MSI on controllers that support MSI-X. All

recent PCIe controllers(RTL8102E or later and RTL8168/8111C or
later) supports either 2 or 4 MSI-X messages. Unfortunately vendor
did not publicly release RSS related information yet. However
switching to MSI-X is one-step forward to support RSS.

2 files changed, 54 insertions, 11 deletions

diff --git a/sys/dev/re/if_re.c b/sys/dev/re/if_re.c
index b87ff5d..01661eb 100644
--- a/sys/dev/re/if_re.c
+++ b/sys/dev/re/if_re.c
@@ -159,6 +159,8 @@ MODULE_DEPEND(re, miibus, 1, 1, 1);
 /* Tunables. */
 static int msi_disable = 0;
 TUNABLE_INT("hw.re.msi_disable", &msi_disable);
+static int msix_disable = 0;
+TUNABLE_INT("hw.re.msix_disable", &msix_disable);
 static int prefer_iomap = 0;
 TUNABLE_INT("hw.re.prefer_iomap", &prefer_iomap);
 
@@ -1176,7 +1178,7 @@ re_attach(device_t dev)
 	int			hwrev;
 	u_int16_t		devid, re_did = 0;
 	int			error = 0, i, phy, rid;
-	int			msic, reg;
+	int			msic, msixc, reg;
 	uint8_t			cfg;
 
 	sc = device_get_softc(dev);
@@ -1226,18 +1228,51 @@ re_attach(device_t dev)
 	sc->rl_btag = rman_get_bustag(sc->rl_res);
 	sc->rl_bhandle = rman_get_bushandle(sc->rl_res);
 
-	msic = 0;
-	if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
+	msic = pci_msi_count(dev);
+	msixc = pci_msix_count(dev);
+	if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0)
 		sc->rl_flags |= RL_FLAG_PCIE;
-		msic = pci_msi_count(dev);
-		if (bootverbose)
-			device_printf(dev, "MSI count : %d\n", msic);
+	if (bootverbose) {
+		device_printf(dev, "MSI count : %d\n", msic);
+		device_printf(dev, "MSI-X count : %d\n", msixc);
+	}
+	if (msix_disable > 0)
+		msixc = 0;
+	if (msi_disable > 0)
+		msic = 0;
+	/* Prefer MSI-X to MSI. */
+	if (msixc > 0) {
+		msixc = 1;
+		rid = PCIR_BAR(4);
+		sc->rl_res_pba = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+		    &rid, RF_ACTIVE);
+		if (sc->rl_res_pba == NULL) {
+			device_printf(sc->rl_dev,
+			    "could not allocate MSI-X PBA resource\n");
+		}
+		if (sc->rl_res_pba != NULL &&
+		    pci_alloc_msix(dev, &msixc) == 0) {
+			if (msixc == 1) {
+				device_printf(dev, "Using %d MSI-X message\n",
+				    msixc);
+				sc->rl_flags |= RL_FLAG_MSIX;
+			} else
+				pci_release_msi(dev);
+		}
+		if ((sc->rl_flags & RL_FLAG_MSIX) == 0) {
+			if (sc->rl_res_pba != NULL)
+				bus_release_resource(dev, SYS_RES_MEMORY, rid,
+				    sc->rl_res_pba);
+			sc->rl_res_pba = NULL;
+			msixc = 0;
+		}
 	}
-	if (msic > 0 && msi_disable == 0) {
+	/* Prefer MSI to INTx. */
+	if (msixc == 0 && msic > 0) {
 		msic = 1;
 		if (pci_alloc_msi(dev, &msic) == 0) {
 			if (msic == RL_MSI_MESSAGES) {
-				device_printf(dev, "Using %d MSI messages\n",
+				device_printf(dev, "Using %d MSI message\n",
 				    msic);
 				sc->rl_flags |= RL_FLAG_MSI;
 				/* Explicitly set MSI enable bit. */
@@ -1249,10 +1284,12 @@ re_attach(device_t dev)
 			} else
 				pci_release_msi(dev);
 		}
+		if ((sc->rl_flags & RL_FLAG_MSI) == 0)
+			msic = 0;
 	}
 
 	/* Allocate interrupt */
-	if ((sc->rl_flags & RL_FLAG_MSI) == 0) {
+	if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0) {
 		rid = 0;
 		sc->rl_irq[0] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 		    RF_SHAREABLE | RF_ACTIVE);
@@ -1539,7 +1576,7 @@ re_attach(device_t dev)
 #endif
 
 	/* Hook interrupt last to avoid having to lock softc */
-	if ((sc->rl_flags & RL_FLAG_MSI) == 0)
+	if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0)
 		error = bus_setup_intr(dev, sc->rl_irq[0],
 		    INTR_TYPE_NET | INTR_MPSAFE, re_intr, NULL, sc,
 		    &sc->rl_intrhand[0]);
@@ -1631,7 +1668,7 @@ re_detach(device_t dev)
 	}
 	if (ifp != NULL)
 		if_free(ifp);
-	if ((sc->rl_flags & RL_FLAG_MSI) == 0) {
+	if ((sc->rl_flags & (RL_FLAG_MSI | RL_FLAG_MSIX)) == 0) {
 		if (sc->rl_irq[0] != NULL) {
 			bus_release_resource(dev, SYS_RES_IRQ, 0,
 			    sc->rl_irq[0]);
@@ -1647,6 +1684,10 @@ re_detach(device_t dev)
 		}
 		pci_release_msi(dev);
 	}
+	if (sc->rl_res_pba) {
+		rid = PCIR_BAR(4);
+		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->rl_res_pba);
+	}
 	if (sc->rl_res)
 		bus_release_resource(dev, sc->rl_res_type, sc->rl_res_id,
 		    sc->rl_res);
diff --git a/sys/pci/if_rlreg.h b/sys/pci/if_rlreg.h
index 5a0c5e6..b31cf0c 100644
--- a/sys/pci/if_rlreg.h
+++ b/sys/pci/if_rlreg.h
@@ -864,6 +864,7 @@ struct rl_softc {
 	struct resource		*rl_res;
 	int			rl_res_id;
 	int			rl_res_type;
+	struct resource		*rl_res_pba;
 	struct resource		*rl_irq[RL_MSI_MESSAGES];
 	void			*rl_intrhand[RL_MSI_MESSAGES];
 	device_t		rl_miibus;
@@ -908,6 +909,7 @@ struct rl_softc {
 #define	RL_FLAG_FASTETHER	0x0100
 #define	RL_FLAG_CMDSTOP		0x0200
 #define	RL_FLAG_MACRESET	0x0400
+#define	RL_FLAG_MSIX		0x0800
 #define	RL_FLAG_WOLRXENB	0x1000
 #define	RL_FLAG_MACSLEEP	0x2000
 #define	RL_FLAG_PCIE		0x4000