Add a method to get the PCI Routing ID for a device

Reviewed by:	kib
Sponsored by:	Sandvine, Inc

9 files changed, 334 insertions, 15 deletions

diff --git a/sys/conf/files b/sys/conf/files
index a13ef38..7af6079 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1951,6 +1951,7 @@ dev/pci/pci_pci.c		optional pci
 dev/pci/pci_subr.c		optional pci
 dev/pci/pci_user.c		optional pci
 dev/pci/pcib_if.m		standard
+dev/pci/pcib_support.c		standard
 dev/pci/vga_pci.c		optional pci
 dev/pcn/if_pcn.c		optional pcn pci
 dev/pdq/if_fea.c		optional fea eisa
diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index 41bb4b1..172ce81 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -124,6 +124,8 @@ static void		pci_resume_msix(device_t dev);
 static int		pci_remap_intr_method(device_t bus, device_t dev,
 			    u_int irq);
 
+static uint16_t		pci_get_rid_method(device_t dev, device_t child);
+
 static device_method_t pci_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		pci_probe),
@@ -182,6 +184,7 @@ static device_method_t pci_methods[] = {
 	DEVMETHOD(pci_release_msi,	pci_release_msi_method),
 	DEVMETHOD(pci_msi_count,	pci_msi_count_method),
 	DEVMETHOD(pci_msix_count,	pci_msix_count_method),
+	DEVMETHOD(pci_get_rid,		pci_get_rid_method),
 
 	DEVMETHOD_END
 };
@@ -351,6 +354,11 @@ SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
     "Ignore firmware-assigned bus numbers.");
 #endif
 
+static int pci_enable_ari = 1;
+TUNABLE_INT("hw.pci.enable_ari", &pci_enable_ari);
+SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
+    0, "Enable support for PCIe Alternative RID Interpretation");
+
 static int
 pci_has_quirk(uint32_t devid, int quirk)
 {
@@ -3451,6 +3459,19 @@ pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
 #endif
 }
 
+static struct pci_devinfo *
+pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
+    int slot, int func, size_t dinfo_size)
+{
+	struct pci_devinfo *dinfo;
+
+	dinfo = pci_read_device(pcib, domain, busno, slot, func, dinfo_size);
+	if (dinfo != NULL)
+		pci_add_child(dev, dinfo);
+
+	return (dinfo);
+}
+
 void
 pci_add_children(device_t dev, int domain, int busno, size_t dinfo_size)
 {
@@ -3460,6 +3481,24 @@ pci_add_children(device_t dev, int domain, int busno, size_t dinfo_size)
 	int maxslots;
 	int s, f, pcifunchigh;
 	uint8_t hdrtype;
+	int first_func;
+
+	/*
+	 * Try to detect a device at slot 0, function 0.  If it exists, try to
+	 * enable ARI.  We must enable ARI before detecting the rest of the
+	 * functions on this bus as ARI changes the set of slots and functions
+	 * that are legal on this bus.
+	 */
+	dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0,
+	    dinfo_size);
+	if (dinfo != NULL && pci_enable_ari)
+		PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);
+
+	/*
+	 * Start looking for new devices on slot 0 at function 1 because we
+	 * just identified the device at slot 0, function 0.
+	 */
+	first_func = 1;
 
 	KASSERT(dinfo_size >= sizeof(struct pci_devinfo),
 	    ("dinfo_size too small"));
@@ -3472,14 +3511,13 @@ pci_add_children(device_t dev, int domain, int busno, size_t dinfo_size)
 		if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
 			continue;
 		if (hdrtype & PCIM_MFDEV)
-			pcifunchigh = PCI_FUNCMAX;
-		for (f = 0; f <= pcifunchigh; f++) {
-			dinfo = pci_read_device(pcib, domain, busno, s, f,
+			pcifunchigh = PCIB_MAXFUNCS(pcib);
+		for (f = first_func; f <= pcifunchigh; f++)
+			pci_identify_function(pcib, dev, domain, busno, s, f,
 			    dinfo_size);
-			if (dinfo != NULL) {
-				pci_add_child(dev, dinfo);
-			}
-		}
+
+		/* For slots after slot 0 we need to check for function 0. */
+		first_func = 0;
 	}
 #undef REG
 }
@@ -5055,3 +5093,10 @@ pci_restore_state(device_t dev)
 	dinfo = device_get_ivars(dev);
 	pci_cfg_restore(dev, dinfo);
 }
+
+static uint16_t
+pci_get_rid_method(device_t dev, device_t child)
+{
+
+	return (PCIB_GET_RID(device_get_parent(dev), child));
+}
diff --git a/sys/dev/pci/pci_if.m b/sys/dev/pci/pci_if.m
index e35d79e..0f19358 100644
--- a/sys/dev/pci/pci_if.m
+++ b/sys/dev/pci/pci_if.m
@@ -159,3 +159,9 @@ METHOD int msix_count {
 	device_t	dev;
 	device_t	child;
 } DEFAULT null_msi_count;
+
+METHOD uint16_t get_rid {
+	device_t dev;
+	device_t child;
+};
+
diff --git a/sys/dev/pci/pci_pci.c b/sys/dev/pci/pci_pci.c
index 684e6b7..35ff48d 100644
--- a/sys/dev/pci/pci_pci.c
+++ b/sys/dev/pci/pci_pci.c
@@ -56,6 +56,14 @@ static int		pcib_suspend(device_t dev);
 static int		pcib_resume(device_t dev);
 static int		pcib_power_for_sleep(device_t pcib, device_t dev,
 			    int *pstate);
+static uint32_t		pcib_read_config(device_t dev, u_int b, u_int s, 
+    u_int f, u_int reg, int width);
+static void		pcib_write_config(device_t dev, u_int b, u_int s,
+    u_int f, u_int reg, uint32_t val, int width);
+static int		pcib_ari_maxslots(device_t dev);
+static int		pcib_ari_maxfuncs(device_t dev);
+static int		pcib_try_enable_ari(device_t pcib, device_t dev);
+
 
 static device_method_t pcib_methods[] = {
     /* Device interface */
@@ -83,7 +91,8 @@ static device_method_t pcib_methods[] = {
     DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 
     /* pcib interface */
-    DEVMETHOD(pcib_maxslots,		pcib_maxslots),
+    DEVMETHOD(pcib_maxslots,		pcib_ari_maxslots),
+    DEVMETHOD(pcib_maxfuncs,		pcib_ari_maxfuncs),
     DEVMETHOD(pcib_read_config,		pcib_read_config),
     DEVMETHOD(pcib_write_config,	pcib_write_config),
     DEVMETHOD(pcib_route_interrupt,	pcib_route_interrupt),
@@ -93,6 +102,7 @@ static device_method_t pcib_methods[] = {
     DEVMETHOD(pcib_release_msix,	pcib_release_msix),
     DEVMETHOD(pcib_map_msi,		pcib_map_msi),
     DEVMETHOD(pcib_power_for_sleep,	pcib_power_for_sleep),
+    DEVMETHOD(pcib_try_enable_ari,	pcib_try_enable_ari),
 
     DEVMETHOD_END
 };
@@ -1802,27 +1812,103 @@ pcib_alloc_resource(device_t dev, device_t child, int type, int *rid,
 #endif
 
 /*
+ * If ARI is enabled on this downstream port, translate the function number
+ * to the non-ARI slot/function.  The downstream port will convert it back in
+ * hardware.  If ARI is not enabled slot and func are not modified.
+ */
+static __inline void
+pcib_xlate_ari(device_t pcib, int bus, int *slot, int *func)
+{
+	struct pcib_softc *sc;
+	int ari_func;
+
+	sc = device_get_softc(pcib);
+	ari_func = *func;
+
+	if (sc->flags & PCIB_ENABLE_ARI) {
+		KASSERT(*slot == 0,
+		    ("Non-zero slot number with ARI enabled!"));
+		*slot = PCIE_ARI_SLOT(ari_func);
+		*func = PCIE_ARI_FUNC(ari_func);
+	}
+}
+
+
+static void
+pcib_enable_ari(struct pcib_softc *sc, uint32_t pcie_pos)
+{
+	uint32_t ctl2;
+
+	ctl2 = pci_read_config(sc->dev, pcie_pos + PCIER_DEVICE_CTL2, 4);
+	ctl2 |= PCIEM_CTL2_ARI;
+	pci_write_config(sc->dev, pcie_pos + PCIER_DEVICE_CTL2, ctl2, 4);
+
+	sc->flags |= PCIB_ENABLE_ARI;
+
+	if (bootverbose)
+		device_printf(sc->dev, "Enable ARI\n");
+}
+
+/*
  * PCIB interface.
  */
 int
 pcib_maxslots(device_t dev)
 {
-    return(PCI_SLOTMAX);
+	return (PCI_SLOTMAX);
+}
+
+int
+pcib_maxfuncs(device_t dev)
+{
+	return (PCI_FUNCMAX);
+}
+
+static int
+pcib_ari_maxslots(device_t dev)
+{
+	struct pcib_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (sc->flags & PCIB_ENABLE_ARI)
+		return (PCIE_ARI_SLOTMAX);
+	else
+		return (PCI_SLOTMAX);
+}
+
+static int
+pcib_ari_maxfuncs(device_t dev)
+{
+	struct pcib_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (sc->flags & PCIB_ENABLE_ARI)
+		return (PCIE_ARI_FUNCMAX);
+	else
+		return (PCI_FUNCMAX);
 }
 
 /*
  * Since we are a child of a PCI bus, its parent must support the pcib interface.
  */
-uint32_t
+static uint32_t
 pcib_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width)
 {
-    return(PCIB_READ_CONFIG(device_get_parent(device_get_parent(dev)), b, s, f, reg, width));
+
+	pcib_xlate_ari(dev, b, &s, &f);
+	return(PCIB_READ_CONFIG(device_get_parent(device_get_parent(dev)), b, s,
+	    f, reg, width));
 }
 
-void
+static void
 pcib_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, uint32_t val, int width)
 {
-    PCIB_WRITE_CONFIG(device_get_parent(device_get_parent(dev)), b, s, f, reg, val, width);
+
+	pcib_xlate_ari(dev, b, &s, &f);
+	PCIB_WRITE_CONFIG(device_get_parent(device_get_parent(dev)), b, s, f,
+	    reg, val, width);
 }
 
 /*
@@ -1934,3 +2020,61 @@ pcib_power_for_sleep(device_t pcib, device_t dev, int *pstate)
 	bus = device_get_parent(pcib);
 	return (PCIB_POWER_FOR_SLEEP(bus, dev, pstate));
 }
+
+
+/*
+ * Check that the downstream port (pcib) and the endpoint device (dev) both
+ * support ARI.  If so, enable it and return 0, otherwise return an error.
+ */
+static int
+pcib_try_enable_ari(device_t pcib, device_t dev)
+{
+	struct pcib_softc *sc;
+	int error;
+	uint32_t cap2;
+	int ari_cap_off;
+	uint32_t ari_ver;
+	uint32_t pcie_pos;
+
+	sc = device_get_softc(pcib);
+
+	/*
+	 * ARI is controlled in a register in the PCIe capability structure.
+	 * If the downstream port does not have the PCIe capability structure
+	 * then it does not support ARI.
+	 */
+	error = pci_find_cap(pcib, PCIY_EXPRESS, &pcie_pos);
+	if (error != 0)
+		return (ENODEV);
+
+	/* Check that the PCIe port advertises ARI support. */
+	cap2 = pci_read_config(pcib, pcie_pos + PCIER_DEVICE_CAP2, 4);
+	if (!(cap2 & PCIEM_CAP2_ARI))
+		return (ENODEV);
+
+	/*
+	 * Check that the endpoint device advertises ARI support via the ARI
+	 * extended capability structure.
+	 */
+	error = pci_find_extcap(dev, PCIZ_ARI, &ari_cap_off);
+	if (error != 0)
+		return (ENODEV);
+
+	/*
+	 * Finally, check that the endpoint device supports the same version
+	 * of ARI that we do.
+	 */
+	ari_ver = pci_read_config(dev, ari_cap_off, 4);
+	if (PCI_EXTCAP_VER(ari_ver) != PCIB_SUPPORTED_ARI_VER) {
+		if (bootverbose)
+			device_printf(pcib,
+			    "Unsupported version of ARI (%d) detected\n",
+			    PCI_EXTCAP_VER(ari_ver));
+
+		return (ENXIO);
+	}
+
+	pcib_enable_ari(sc, pcie_pos);
+
+	return (0);
+}
diff --git a/sys/dev/pci/pcib_if.m b/sys/dev/pci/pcib_if.m
index b946c0f..54c00cf 100644
--- a/sys/dev/pci/pcib_if.m
+++ b/sys/dev/pci/pcib_if.m
@@ -47,6 +47,14 @@ METHOD int maxslots {
 };
 
 #
+#
+# Return the number of functions on the attached PCI bus.
+#
+METHOD int maxfuncs {
+	device_t	dev;
+} default pcib_maxfuncs;
+
+#
 # Read configuration space on the PCI bus. The bus, slot and func
 # arguments determine the device which is being read and the reg
 # argument is a byte offset into configuration space for that
@@ -154,3 +162,21 @@ METHOD int power_for_sleep {
 	device_t	dev;
 	int		*pstate;
 };
+
+#
+# Return the PCI Routing Identifier (RID) for the device.
+#
+METHOD uint16_t get_rid {
+	device_t pcib;
+	device_t dev;
+};
+
+
+#
+# Enable Alternative RID Interpretation if both the downstream port (pcib)
+# and the endpoint device (dev) both support it.
+#
+METHOD int try_enable_ari {
+	device_t	pcib;
+	device_t	dev;
+};
diff --git a/sys/dev/pci/pcib_private.h b/sys/dev/pci/pcib_private.h
index e74f149..31ff2a5 100644
--- a/sys/dev/pci/pcib_private.h
+++ b/sys/dev/pci/pcib_private.h
@@ -105,6 +105,7 @@ struct pcib_softc
 #define	PCIB_SUBTRACTIVE	0x1
 #define	PCIB_DISABLE_MSI	0x2
 #define	PCIB_DISABLE_MSIX	0x4
+#define	PCIB_ENABLE_ARI		0x8
     uint16_t	command;	/* command register */
     u_int	domain;		/* domain number */
     u_int	pribus;		/* primary bus number */
@@ -126,6 +127,8 @@ struct pcib_softc
     uint8_t	seclat;		/* secondary bus latency timer */
 };
 
+#define	PCIB_SUPPORTED_ARI_VER	1
+
 typedef uint32_t pci_read_config_fn(int b, int s, int f, int reg, int width);
 
 int		host_pcib_get_busno(pci_read_config_fn read_config, int bus,
@@ -159,13 +162,14 @@ int		pcib_release_resource(device_t dev, device_t child, int type, int rid,
     struct resource *r);
 #endif
 int		pcib_maxslots(device_t dev);
-uint32_t	pcib_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width);
-void		pcib_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, uint32_t val, int width);
+int		pcib_maxfuncs(device_t dev);
 int		pcib_route_interrupt(device_t pcib, device_t dev, int pin);
 int		pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs);
 int		pcib_release_msi(device_t pcib, device_t dev, int count, int *irqs);
 int		pcib_alloc_msix(device_t pcib, device_t dev, int *irq);
 int		pcib_release_msix(device_t pcib, device_t dev, int irq);
 int		pcib_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data);
+uint16_t	pcib_get_rid(device_t pcib, device_t dev);
+
 
 #endif
diff --git a/sys/dev/pci/pcib_support.c b/sys/dev/pci/pcib_support.c
new file mode 100644
index 0000000..48b94c9
--- /dev/null
+++ b/sys/dev/pci/pcib_support.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) Sandvine Inc.  All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Support functions for the PCI:PCI bridge driver.  This has to be in a
+ * separate file because kernel configurations end up referencing the functions
+ * here even when pci support is compiled out of the kernel.
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcib_private.h>
+
+#include "pcib_if.h"
+
+uint16_t
+pcib_get_rid(device_t pcib, device_t dev)
+{
+	uint8_t bus, slot, func;
+
+	bus = pci_get_bus(dev);
+	slot = pci_get_slot(dev);
+	func = pci_get_function(dev);
+
+	return (PCI_RID(bus, slot, func));
+}
+
diff --git a/sys/dev/pci/pcireg.h b/sys/dev/pci/pcireg.h
index afd140f..86375ca 100644
--- a/sys/dev/pci/pcireg.h
+++ b/sys/dev/pci/pcireg.h
@@ -48,6 +48,29 @@
 #define	PCIE_REGMAX	4095	/* highest supported config register addr. */
 #define	PCI_MAXHDRTYPE	2
 
+#define PCIE_ARI_SLOTMAX 0
+#define PCIE_ARI_FUNCMAX 255
+
+#define PCIE_ARI_SLOT(func) (((func) >> 3) & PCI_SLOTMAX)
+#define PCIE_ARI_FUNC(func) ((func) & PCI_FUNCMAX)
+
+#define	PCI_RID_BUS_SHIFT	8
+#define	PCI_RID_SLOT_SHIFT	3
+#define	PCI_RID_FUNC_SHIFT	0
+
+#define PCI_RID(bus, slot, func) \
+    ((((bus) & PCI_BUSMAX) << PCI_RID_BUS_SHIFT) | \
+    (((slot) & PCI_SLOTMAX) << PCI_RID_SLOT_SHIFT) | \
+    (((func) & PCI_FUNCMAX) << PCI_RID_FUNC_SHIFT))
+
+#define PCI_ARI_RID(bus, func) \
+    ((((bus) & PCI_BUSMAX) << PCI_RID_BUS_SHIFT) | \
+    (((func) & PCIE_ARI_FUNCMAX) << PCI_RID_FUNC_SHIFT))
+
+#define PCI_RID2BUS(rid) (((rid) >> PCI_RID_BUS_SHIFT) & PCI_BUSMAX)
+#define PCI_RID2SLOT(rid) (((rid) >> PCI_RID_SLOT_SHIFT) & PCI_SLOTMAX)
+#define PCI_RID2FUNC(rid) (((rid) >> PCI_RID_FUNC_SHIFT) & PCI_FUNCMAX)
+
 /* PCI config header registers for all devices */
 
 #define	PCIR_DEVVENDOR	0x00
@@ -774,6 +797,7 @@
 #define	PCIEM_ROOT_STA_PME_STATUS	0x00010000
 #define	PCIEM_ROOT_STA_PME_PEND		0x00020000
 #define	PCIER_DEVICE_CAP2	0x24
+#define	PCIEM_CAP2_ARI		0x20
 #define	PCIER_DEVICE_CTL2	0x28
 #define	PCIEM_CTL2_COMP_TIMEOUT_VAL	0x000f
 #define	PCIEM_CTL2_COMP_TIMEOUT_DIS	0x0010
@@ -894,3 +918,4 @@
 /* Serial Number definitions */
 #define	PCIR_SERIAL_LOW		0x04
 #define	PCIR_SERIAL_HIGH	0x08
+
diff --git a/sys/dev/pci/pcivar.h b/sys/dev/pci/pcivar.h
index 06e5771..0157ee7 100644
--- a/sys/dev/pci/pcivar.h
+++ b/sys/dev/pci/pcivar.h
@@ -482,6 +482,12 @@ pci_msix_count(device_t dev)
     return (PCI_MSIX_COUNT(device_get_parent(dev), dev));
 }
 
+static __inline uint16_t
+pci_get_rid(device_t dev)
+{
+	return (PCI_GET_RID(device_get_parent(dev), dev));
+}
+
 device_t pci_find_bsf(uint8_t, uint8_t, uint8_t);
 device_t pci_find_dbsf(uint32_t, uint8_t, uint8_t, uint8_t);
 device_t pci_find_device(uint16_t, uint16_t);