MFC r270212,r270332:

This helps reduce the diff in pci(4) between head and stable/10 to help pave
the way for bringing in IOV/nv(9) more cleanly

Differential Revision: https://reviews.freebsd.org/D4728
Relnotes: yes
Reviewed by: hselasky (ofed piece), royger (overall change)
Sponsored by: EMC / Isilon Storage Division

r270212 (by royger):

pci: make MSI(-X) enable and disable methods of the PCI bus

Make the functions pci_disable_msi, pci_enable_msi and pci_enable_msix
methods of the newbus PCI bus. This code should not include any
functional change.

Sponsored by: Citrix Systems R&D
Reviewed by: imp, jhb
Differential Revision: https://reviews.freebsd.org/D354

dev/pci/pci.c:
 - Convert the mentioned functions to newbus methods.
 - Fix the callers of the converted functions.

sys/dev/pci/pci_private.h:
dev/pci/pci_if.m:
 - Declare the new methods.

dev/pci/pcivar.h:
 - Add helpers to call the newbus methods.

ofed/include/linux/pci.h:
 - Add define to prevent the ofed version of pci_enable_msix from
   clashing with the FreeBSD native version.

r270332 (by royger):

pci: add a new pci_child_added newbus method.

This is needed so when running under Xen the calls to pci_child_added
can be intercepted and a custom Xen method can be used to register
those devices with Xen. This should not include any functional
change, since the Xen implementation will be added in a following
patch and the native implementation is a noop.

Sponsored by: Citrix Systems R&D
Reviewed by: jhb

dev/pci/pci.c:
dev/pci/pci_if.m:
dev/pci/pci_private.h:
dev/pci/pcivar.h:
 - Add the pci_child_added newbus method.

4 files changed, 85 insertions, 22 deletions

diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index fb9a202..a9c33ae 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -110,11 +110,6 @@ static int		pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
 			    int reg, uint32_t data);
 #endif
 static void		pci_read_vpd(device_t pcib, pcicfgregs *cfg);
-static void		pci_disable_msi(device_t dev);
-static void		pci_enable_msi(device_t dev, uint64_t address,
-			    uint16_t data);
-static void		pci_enable_msix(device_t dev, u_int index,
-			    uint64_t address, uint32_t data);
 static void		pci_mask_msix(device_t dev, u_int index);
 static void		pci_unmask_msix(device_t dev, u_int index);
 static int		pci_msi_blacklisted(void);
@@ -180,11 +175,15 @@ static device_method_t pci_methods[] = {
 	DEVMETHOD(pci_find_htcap,	pci_find_htcap_method),
 	DEVMETHOD(pci_alloc_msi,	pci_alloc_msi_method),
 	DEVMETHOD(pci_alloc_msix,	pci_alloc_msix_method),
+	DEVMETHOD(pci_enable_msi,	pci_enable_msi_method),
+	DEVMETHOD(pci_enable_msix,	pci_enable_msix_method),
+	DEVMETHOD(pci_disable_msi,	pci_disable_msi_method),
 	DEVMETHOD(pci_remap_msix,	pci_remap_msix_method),
 	DEVMETHOD(pci_release_msi,	pci_release_msi_method),
 	DEVMETHOD(pci_msi_count,	pci_msi_count_method),
 	DEVMETHOD(pci_msix_count,	pci_msix_count_method),
 	DEVMETHOD(pci_get_rid,		pci_get_rid_method),
+	DEVMETHOD(pci_child_added,	pci_child_added_method),
 
 	DEVMETHOD_END
 };
@@ -1374,9 +1373,10 @@ pci_find_extcap_method(device_t dev, device_t child, int capability,
  * Support for MSI-X message interrupts.
  */
 void
-pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data)
+pci_enable_msix_method(device_t dev, device_t child, u_int index,
+    uint64_t address, uint32_t data)
 {
-	struct pci_devinfo *dinfo = device_get_ivars(dev);
+	struct pci_devinfo *dinfo = device_get_ivars(child);
 	struct pcicfg_msix *msix = &dinfo->cfg.msix;
 	uint32_t offset;
 
@@ -1387,7 +1387,7 @@ pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data)
 	bus_write_4(msix->msix_table_res, offset + 8, data);
 
 	/* Enable MSI -> HT mapping. */
-	pci_ht_map_msi(dev, address);
+	pci_ht_map_msi(child, address);
 }
 
 void
@@ -1956,45 +1956,46 @@ pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
  * Support for MSI message signalled interrupts.
  */
 void
-pci_enable_msi(device_t dev, uint64_t address, uint16_t data)
+pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
+    uint16_t data)
 {
-	struct pci_devinfo *dinfo = device_get_ivars(dev);
+	struct pci_devinfo *dinfo = device_get_ivars(child);
 	struct pcicfg_msi *msi = &dinfo->cfg.msi;
 
 	/* Write data and address values. */
-	pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
+	pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
 	    address & 0xffffffff, 4);
 	if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
-		pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR_HIGH,
+		pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
 		    address >> 32, 4);
-		pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA_64BIT,
+		pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
 		    data, 2);
 	} else
-		pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA, data,
+		pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
 		    2);
 
 	/* Enable MSI in the control register. */
 	msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE;
-	pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
-	    2);
+	pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
+	    msi->msi_ctrl, 2);
 
 	/* Enable MSI -> HT mapping. */
-	pci_ht_map_msi(dev, address);
+	pci_ht_map_msi(child, address);
 }
 
 void
-pci_disable_msi(device_t dev)
+pci_disable_msi_method(device_t dev, device_t child)
 {
-	struct pci_devinfo *dinfo = device_get_ivars(dev);
+	struct pci_devinfo *dinfo = device_get_ivars(child);
 	struct pcicfg_msi *msi = &dinfo->cfg.msi;
 
 	/* Disable MSI -> HT mapping. */
-	pci_ht_map_msi(dev, 0);
+	pci_ht_map_msi(child, 0);
 
 	/* Disable MSI in the control register. */
 	msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
-	pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
-	    2);
+	pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
+	    msi->msi_ctrl, 2);
 }
 
 /*
@@ -3604,6 +3605,13 @@ pci_add_child(device_t bus, struct pci_devinfo *dinfo)
 	pci_cfg_restore(dinfo->cfg.dev, dinfo);
 	pci_print_verbose(dinfo);
 	pci_add_resources(bus, dinfo->cfg.dev, 0, 0);
+	pci_child_added(dinfo->cfg.dev);
+}
+
+void
+pci_child_added_method(device_t dev, device_t child)
+{
+
 }
 
 static int
diff --git a/sys/dev/pci/pci_if.m b/sys/dev/pci/pci_if.m
index 82864eb..227d362 100644
--- a/sys/dev/pci/pci_if.m
+++ b/sys/dev/pci/pci_if.m
@@ -138,6 +138,26 @@ METHOD int alloc_msix {
 	int		*count;
 };
 
+METHOD void enable_msi {
+	device_t	dev;
+	device_t	child;
+	uint64_t	address;
+	uint16_t	data;
+};
+
+METHOD void enable_msix {
+	device_t	dev;
+	device_t	child;
+	u_int		index;
+	uint64_t	address;
+	uint32_t	data;
+};
+
+METHOD void disable_msi {
+	device_t	dev;
+	device_t	child;
+};
+
 METHOD int remap_msix {
 	device_t	dev;
 	device_t	child;
@@ -165,3 +185,7 @@ METHOD uint16_t get_rid {
 	device_t	child;
 };
 
+METHOD void child_added {
+	device_t	dev;
+	device_t	child;
+};
diff --git a/sys/dev/pci/pci_private.h b/sys/dev/pci/pci_private.h
index 0223ee8..5a90ce9 100644
--- a/sys/dev/pci/pci_private.h
+++ b/sys/dev/pci/pci_private.h
@@ -90,6 +90,11 @@ int		pci_find_htcap_method(device_t dev, device_t child,
 		    int capability, int *capreg);
 int		pci_alloc_msi_method(device_t dev, device_t child, int *count);
 int		pci_alloc_msix_method(device_t dev, device_t child, int *count);
+void		pci_enable_msi_method(device_t dev, device_t child,
+		    uint64_t address, uint16_t data);
+void		pci_enable_msix_method(device_t dev, device_t child,
+		    u_int index, uint64_t address, uint32_t data);
+void		pci_disable_msi_method(device_t dev, device_t child);
 int		pci_remap_msix_method(device_t dev, device_t child,
 		    int count, const u_int *vectors);
 int		pci_release_msi_method(device_t dev, device_t child);
@@ -120,6 +125,7 @@ int		pci_assign_interrupt_method(device_t dev, device_t child);
 int		pci_resume(device_t dev);
 int		pci_suspend(device_t dev);
 bus_dma_tag_t pci_get_dma_tag(device_t bus, device_t dev);
+void		pci_child_added_method(device_t dev, device_t child);
 
 /** Restore the config register state.  The state must be previously
  * saved with pci_cfg_save.  However, the pci bus driver takes care of
diff --git a/sys/dev/pci/pcivar.h b/sys/dev/pci/pcivar.h
index 4b570d0..2bb26bd 100644
--- a/sys/dev/pci/pcivar.h
+++ b/sys/dev/pci/pcivar.h
@@ -458,6 +458,24 @@ pci_alloc_msix(device_t dev, int *count)
     return (PCI_ALLOC_MSIX(device_get_parent(dev), dev, count));
 }
 
+static __inline void
+pci_enable_msi(device_t dev, uint64_t address, uint16_t data)
+{
+    PCI_ENABLE_MSI(device_get_parent(dev), dev, address, data);
+}
+
+static __inline void
+pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data)
+{
+    PCI_ENABLE_MSIX(device_get_parent(dev), dev, index, address, data);
+}
+
+static __inline void
+pci_disable_msi(device_t dev)
+{
+    PCI_DISABLE_MSI(device_get_parent(dev), dev);
+}
+
 static __inline int
 pci_remap_msix(device_t dev, int count, const u_int *vectors)
 {
@@ -488,6 +506,13 @@ pci_get_rid(device_t dev)
 	return (PCI_GET_RID(device_get_parent(dev), dev));
 }
 
+static __inline void
+pci_child_added(device_t dev)
+{
+
+    return (PCI_CHILD_ADDED(device_get_parent(dev), dev));
+}
+
 device_t pci_find_bsf(uint8_t, uint8_t, uint8_t);
 device_t pci_find_dbsf(uint32_t, uint8_t, uint8_t, uint8_t);
 device_t pci_find_device(uint16_t, uint16_t);