Add FDT bus capabilities to ThunderX PCI driver

New ThunderX firmware incorporates modified DTB that presents
different device hierarchy. In the new device tree, MDIO
devices are below two additional buses that oddly hang on
PCI bridge.

Obtained from: Semihalf
Sponsored by:  Cavium
Differential Revision: https://reviews.freebsd.org/D5069

4 files changed, 185 insertions, 15 deletions

diff --git a/sys/arm64/cavium/thunder_pcie.c b/sys/arm64/cavium/thunder_pcie.c
index b047639..ca2fd6f 100644
--- a/sys/arm64/cavium/thunder_pcie.c
+++ b/sys/arm64/cavium/thunder_pcie.c
@@ -28,6 +28,7 @@
  */
 
 /* PCIe root complex driver for Cavium Thunder SOC */
+#include "opt_platform.h"
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
@@ -89,15 +90,11 @@ SYSCTL_INT(_hw, OID_AUTO, thunder_pcie_max_vfs, CTLFLAG_RWTUN,
     &thunder_pcie_max_vfs, 0, "Max VFs supported by ThunderX internal PCIe");
 
 /* Forward prototypes */
-static struct resource *thunder_pcie_alloc_resource(device_t,
-    device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int);
 static int thunder_pcie_identify_pcib(device_t);
 static int thunder_pcie_maxslots(device_t);
 static uint32_t thunder_pcie_read_config(device_t, u_int, u_int, u_int, u_int,
     int);
 static int thunder_pcie_read_ivar(device_t, device_t, int, uintptr_t *);
-static int thunder_pcie_release_resource(device_t, device_t, int, int,
-    struct resource *);
 static void thunder_pcie_write_config(device_t, u_int, u_int,
     u_int, u_int, uint32_t, int);
 static int thunder_pcie_write_ivar(device_t, device_t, int, uintptr_t);
@@ -262,7 +259,7 @@ thunder_pcie_write_ivar(device_t dev, device_t child, int index,
 	return (ENOENT);
 }
 
-static int
+int
 thunder_pcie_release_resource(device_t dev, device_t child, int type, int rid,
     struct resource *res)
 {
@@ -274,7 +271,7 @@ thunder_pcie_release_resource(device_t dev, device_t child, int type, int rid,
 	return (rman_release_resource(res));
 }
 
-static struct resource *
+struct resource *
 thunder_pcie_alloc_resource(device_t dev, device_t child, int type, int *rid,
     rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
 {
diff --git a/sys/arm64/cavium/thunder_pcie_common.c b/sys/arm64/cavium/thunder_pcie_common.c
index aca1c72..58239f1 100644
--- a/sys/arm64/cavium/thunder_pcie_common.c
+++ b/sys/arm64/cavium/thunder_pcie_common.c
@@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/malloc.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
@@ -44,6 +45,8 @@ __FBSDID("$FreeBSD$");
 
 #include "thunder_pcie_common.h"
 
+MALLOC_DEFINE(M_THUNDER_PCIE, "Thunder PCIe driver", "Thunder PCIe driver memory");
+
 uint32_t
 range_addr_is_pci(struct pcie_range *ranges, uint64_t addr, uint64_t size)
 {
diff --git a/sys/arm64/cavium/thunder_pcie_common.h b/sys/arm64/cavium/thunder_pcie_common.h
index 180cbda..548a435 100644
--- a/sys/arm64/cavium/thunder_pcie_common.h
+++ b/sys/arm64/cavium/thunder_pcie_common.h
@@ -34,6 +34,8 @@
 
 DECLARE_CLASS(thunder_pcie_driver);
 
+MALLOC_DECLARE(M_THUNDER_PCIE);
+
 struct pcie_range {
 	uint64_t	pci_base;
 	uint64_t	phys_base;
@@ -58,6 +60,11 @@ int thunder_common_map_msi(device_t, device_t, int, uint64_t *, uint32_t *);
 int thunder_common_release_msi(device_t, device_t, int, int *);
 int thunder_common_release_msix(device_t, device_t, int);
 
+struct resource *thunder_pcie_alloc_resource(device_t,
+    device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int);
+int thunder_pcie_release_resource(device_t, device_t, int, int,
+    struct resource *);
+
 int thunder_pcie_attach(device_t);
 
 #endif /* _CAVIUM_THUNDER_PCIE_COMMON_H_ */
diff --git a/sys/arm64/cavium/thunder_pcie_fdt.c b/sys/arm64/cavium/thunder_pcie_fdt.c
index 79eb4ed..bde4537 100644
--- a/sys/arm64/cavium/thunder_pcie_fdt.c
+++ b/sys/arm64/cavium/thunder_pcie_fdt.c
@@ -60,10 +60,32 @@ __FBSDID("$FreeBSD$");
 static int thunder_pcie_fdt_probe(device_t);
 static int thunder_pcie_fdt_attach(device_t);
 
+static struct resource * thunder_pcie_ofw_bus_alloc_res(device_t, device_t,
+    int, int *, rman_res_t, rman_res_t, rman_res_t, u_int);
+static int thunder_pcie_ofw_bus_rel_res(device_t, device_t, int, int,
+    struct resource *);
+
+static const struct ofw_bus_devinfo *thunder_pcie_ofw_get_devinfo(device_t,
+    device_t);
+
 static device_method_t thunder_pcie_fdt_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		thunder_pcie_fdt_probe),
 	DEVMETHOD(device_attach,	thunder_pcie_fdt_attach),
+
+	/* Bus interface */
+	DEVMETHOD(bus_alloc_resource,		thunder_pcie_ofw_bus_alloc_res),
+	DEVMETHOD(bus_release_resource,		thunder_pcie_ofw_bus_rel_res),
+	DEVMETHOD(bus_activate_resource,	bus_generic_activate_resource),
+
+	/* ofw_bus interface */
+	DEVMETHOD(ofw_bus_get_devinfo,	thunder_pcie_ofw_get_devinfo),
+	DEVMETHOD(ofw_bus_get_compat,	ofw_bus_gen_get_compat),
+	DEVMETHOD(ofw_bus_get_model,	ofw_bus_gen_get_model),
+	DEVMETHOD(ofw_bus_get_name,	ofw_bus_gen_get_name),
+	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
+	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
+
 	/* End */
 	DEVMETHOD_END
 };
@@ -79,6 +101,7 @@ DRIVER_MODULE(thunder_pcib, ofwbus, thunder_pcie_fdt_driver,
     thunder_pcie_fdt_devclass, 0, 0);
 
 static int thunder_pcie_fdt_ranges(device_t);
+static int thunder_pcie_ofw_bus_attach(device_t);
 
 static int
 thunder_pcie_fdt_probe(device_t dev)
@@ -99,14 +122,32 @@ thunder_pcie_fdt_probe(device_t dev)
 static int
 thunder_pcie_fdt_attach(device_t dev)
 {
+	int err;
 
 	/* Retrieve 'ranges' property from FDT */
 	if (thunder_pcie_fdt_ranges(dev) != 0)
 		return (ENXIO);
 
+	err = thunder_pcie_ofw_bus_attach(dev);
+	if (err != 0)
+		return (err);
+
 	return (thunder_pcie_attach(dev));
 }
 
+static __inline void
+get_addr_size_cells(phandle_t node, pcell_t *addr_cells, pcell_t *size_cells)
+{
+
+	*addr_cells = 2;
+	/* Find address cells if present */
+	OF_getencprop(node, "#address-cells", addr_cells, sizeof(*addr_cells));
+
+	*size_cells = 2;
+	/* Find size cells if present */
+	OF_getencprop(node, "#size-cells", size_cells, sizeof(*size_cells));
+}
+
 static int
 thunder_pcie_fdt_ranges(device_t dev)
 {
@@ -122,15 +163,7 @@ thunder_pcie_fdt_ranges(device_t dev)
 	sc = device_get_softc(dev);
 	node = ofw_bus_get_node(dev);
 
-	/* Find address cells if present */
-	if (OF_getencprop(node, "#address-cells", &pci_addr_cells,
-	    sizeof(pci_addr_cells)) < sizeof(pci_addr_cells))
-		pci_addr_cells = 2;
-
-	/* Find size cells if present */
-	if (OF_getencprop(node, "#size-cells", &size_cells,
-	    sizeof(size_cells)) < sizeof(size_cells))
-		size_cells = 1;
+	get_addr_size_cells(node, &pci_addr_cells, &size_cells);
 
 	/* Find parent address cells if present */
 	if (OF_getencprop(OF_parent(node), "#address-cells",
@@ -217,3 +250,133 @@ out:
 	free(ranges_buf, M_OFWPROP);
 	return (rv);
 }
+
+/* OFW bus interface */
+struct thunder_pcie_ofw_devinfo {
+	struct ofw_bus_devinfo	di_dinfo;
+	struct resource_list	di_rl;
+};
+
+static const struct ofw_bus_devinfo *
+thunder_pcie_ofw_get_devinfo(device_t bus __unused, device_t child)
+{
+	struct thunder_pcie_ofw_devinfo *di;
+
+	di = device_get_ivars(child);
+	return (&di->di_dinfo);
+}
+
+static struct resource *
+thunder_pcie_ofw_bus_alloc_res(device_t bus, device_t child, int type, int *rid,
+    rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
+{
+	struct thunder_pcie_softc *sc;
+	struct thunder_pcie_ofw_devinfo *di;
+	struct resource_list_entry *rle;
+	int i;
+
+	/* For PCIe devices that do not have FDT nodes, use PCIB method */
+	if (ofw_bus_get_node(child) == 0) {
+		return (thunder_pcie_alloc_resource(bus, child, type, rid,
+		    start, end, count, flags));
+	}
+
+	sc = device_get_softc(bus);
+
+	if ((start == 0UL) && (end == ~0UL)) {
+		if ((di = device_get_ivars(child)) == NULL)
+			return (NULL);
+		if (type == SYS_RES_IOPORT)
+		    type = SYS_RES_MEMORY;
+
+		/* Find defaults for this rid */
+		rle = resource_list_find(&di->di_rl, type, *rid);
+		if (rle == NULL)
+			return (NULL);
+
+		start = rle->start;
+		end = rle->end;
+		count = rle->count;
+	}
+
+	if (type == SYS_RES_MEMORY) {
+		/* Remap through ranges property */
+		for (i = 0; i < RANGES_TUPLES_MAX; i++) {
+			if (start >= sc->ranges[i].phys_base && end <
+			    sc->ranges[i].pci_base + sc->ranges[i].size) {
+				start -= sc->ranges[i].phys_base;
+				start += sc->ranges[i].pci_base;
+				end -= sc->ranges[i].phys_base;
+				end += sc->ranges[i].pci_base;
+				break;
+			}
+		}
+
+		if (i == RANGES_TUPLES_MAX) {
+			device_printf(bus, "Could not map resource "
+			    "%#lx-%#lx\n", start, end);
+			return (NULL);
+		}
+	}
+
+	return (bus_generic_alloc_resource(bus, child, type, rid, start, end,
+	    count, flags));
+}
+
+static int
+thunder_pcie_ofw_bus_rel_res(device_t bus, device_t child, int type, int rid,
+    struct resource *res)
+{
+
+	/* For PCIe devices that do not have FDT nodes, use PCIB method */
+	if (ofw_bus_get_node(child) == 0) {
+		return (thunder_pcie_release_resource(bus,
+		    child, type, rid, res));
+	}
+
+	return (bus_generic_release_resource(bus, child, type, rid, res));
+}
+
+/* Helper functions */
+
+static int
+thunder_pcie_ofw_bus_attach(device_t dev)
+{
+	struct thunder_pcie_ofw_devinfo *di;
+	device_t child;
+	phandle_t parent, node;
+	pcell_t addr_cells, size_cells;
+
+	parent = ofw_bus_get_node(dev);
+	if (parent > 0) {
+		get_addr_size_cells(parent, &addr_cells, &size_cells);
+		/* Iterate through all bus subordinates */
+		for (node = OF_child(parent); node > 0; node = OF_peer(node)) {
+			/* Allocate and populate devinfo. */
+			di = malloc(sizeof(*di), M_THUNDER_PCIE, M_WAITOK | M_ZERO);
+			if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node) != 0) {
+				free(di, M_THUNDER_PCIE);
+				continue;
+			}
+
+			/* Initialize and populate resource list. */
+			resource_list_init(&di->di_rl);
+			ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells,
+			    &di->di_rl);
+			ofw_bus_intr_to_rl(dev, node, &di->di_rl, NULL);
+
+			/* Add newbus device for this FDT node */
+			child = device_add_child(dev, NULL, -1);
+			if (child == NULL) {
+				resource_list_free(&di->di_rl);
+				ofw_bus_gen_destroy_devinfo(&di->di_dinfo);
+				free(di, M_THUNDER_PCIE);
+				continue;
+			}
+
+			device_set_ivars(child, di);
+		}
+	}
+
+	return (0);
+}