Convert pci_delete_child() to a bus_child_deleted() method.

Instead of providing a wrapper around device_delete_child() that the PCI
bus and child bus drivers must call explicitly, move the bulk of the logic
from pci_delete_child() into a bus_child_deleted() method
(pci_child_deleted()).  This allows PCI devices to be safely deleted via
device_delete_child().
- Add a bus_child_deleted method to the ACPI PCI bus which clears the
  device_t associated with the corresponding ACPI handle in addition to
  the normal PCI bus cleanup.
- Change cardbus_detach_card to call device_delete_children() and move
  CardBus-specific delete logic into a new cardbus_child_deleted() method.
- Use device_delete_child() instead of pci_delete_child() in the SRIOV code.
- Add a bus_child_deleted method to the OpenFirmware PCI bus drivers which
  frees the OpenFirmware device info for each PCI device.

Reviewed by:	imp
Tested on:	amd64 (CardBus and PCI-e hotplug)
Differential Revision:	https://reviews.freebsd.org/D5831

7 files changed, 65 insertions, 28 deletions

diff --git a/sys/dev/acpica/acpi_pci.c b/sys/dev/acpica/acpi_pci.c
index 9509f82..09cecc3 100644
--- a/sys/dev/acpica/acpi_pci.c
+++ b/sys/dev/acpica/acpi_pci.c
@@ -70,6 +70,7 @@ CTASSERT(ACPI_STATE_D2 == PCI_POWERSTATE_D2);
 CTASSERT(ACPI_STATE_D3 == PCI_POWERSTATE_D3);
 
 static int	acpi_pci_attach(device_t dev);
+static void	acpi_pci_child_deleted(device_t dev, device_t child);
 static int	acpi_pci_child_location_str_method(device_t cbdev,
 		    device_t child, char *buf, size_t buflen);
 static int	acpi_pci_probe(device_t dev);
@@ -97,6 +98,7 @@ static device_method_t acpi_pci_methods[] = {
 	/* Bus interface */
 	DEVMETHOD(bus_read_ivar,	acpi_pci_read_ivar),
 	DEVMETHOD(bus_write_ivar,	acpi_pci_write_ivar),
+	DEVMETHOD(bus_child_deleted,	acpi_pci_child_deleted),
 	DEVMETHOD(bus_child_location_str, acpi_pci_child_location_str_method),
 	DEVMETHOD(bus_get_dma_tag,	acpi_pci_get_dma_tag),
 	DEVMETHOD(bus_get_domain,	acpi_get_domain),
@@ -153,6 +155,16 @@ acpi_pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
     return (pci_write_ivar(dev, child, which, value));
 }
 
+static void
+acpi_pci_child_deleted(device_t dev, device_t child)
+{
+	struct acpi_pci_devinfo *dinfo = device_get_ivars(child);
+
+	if (acpi_get_device(dinfo->ap_handle) == child)
+		AcpiDetachData(dinfo->ap_handle, acpi_fake_objhandler);
+	pci_child_deleted(dev, child);
+}
+
 static int
 acpi_pci_child_location_str_method(device_t cbdev, device_t child, char *buf,
     size_t buflen)
diff --git a/sys/dev/cardbus/cardbus.c b/sys/dev/cardbus/cardbus.c
index b33f462a..e785b0f 100644
--- a/sys/dev/cardbus/cardbus.c
+++ b/sys/dev/cardbus/cardbus.c
@@ -226,31 +226,30 @@ cardbus_attach_card(device_t cbdev)
 	return (ENOENT);
 }
 
+static void
+cardbus_child_deleted(device_t cbdev, device_t child)
+{
+	struct cardbus_devinfo *dinfo = device_get_ivars(child);
+
+	if (dinfo->pci.cfg.dev != child)
+		device_printf(cbdev, "devinfo dev mismatch\n");
+	cardbus_device_destroy(dinfo);
+	pci_child_deleted(cbdev, child);
+}
+
 static int
 cardbus_detach_card(device_t cbdev)
 {
-	int numdevs;
-	device_t *devlist;
-	int tmp;
 	int err = 0;
 
-	if (device_get_children(cbdev, &devlist, &numdevs) != 0)
-		return (ENOENT);
-	if (numdevs == 0) {
-		free(devlist, M_TEMP);
-		return (ENOENT);
-	}
-
-	for (tmp = 0; tmp < numdevs; tmp++) {
-		struct cardbus_devinfo *dinfo = device_get_ivars(devlist[tmp]);
+	err = bus_generic_detach(cbdev);
+	if (err)
+		return (err);
+	err = device_delete_children(cbdev);
+	if (err)
+		return (err);
 
-		if (dinfo->pci.cfg.dev != devlist[tmp])
-			device_printf(cbdev, "devinfo dev mismatch\n");
-		cardbus_device_destroy(dinfo);
-		pci_delete_child(cbdev, devlist[tmp]);
-	}
 	POWER_DISABLE_SOCKET(device_get_parent(cbdev), cbdev);
-	free(devlist, M_TEMP);
 	return (err);
 }
 
@@ -335,6 +334,7 @@ static device_method_t cardbus_methods[] = {
 	DEVMETHOD(device_resume,	cardbus_resume),
 
 	/* Bus interface */
+	DEVMETHOD(bus_child_deleted,	cardbus_child_deleted),
 	DEVMETHOD(bus_get_dma_tag,	bus_generic_get_dma_tag),
 	DEVMETHOD(bus_read_ivar,	cardbus_read_ivar),
 	DEVMETHOD(bus_driver_added,	cardbus_driver_added),
diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index 8343181..fbe9abd 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -161,6 +161,7 @@ static device_method_t pci_methods[] = {
 	DEVMETHOD(bus_release_resource,	pci_release_resource),
 	DEVMETHOD(bus_activate_resource, pci_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
+	DEVMETHOD(bus_child_deleted,	pci_child_deleted),
 	DEVMETHOD(bus_child_detached,	pci_child_detached),
 	DEVMETHOD(bus_child_pnpinfo_str, pci_child_pnpinfo_str_method),
 	DEVMETHOD(bus_child_location_str, pci_child_location_str_method),
@@ -5194,7 +5195,7 @@ pci_deactivate_resource(device_t dev, device_t child, int type,
 }
 
 void
-pci_delete_child(device_t dev, device_t child)
+pci_child_deleted(device_t dev, device_t child)
 {
 	struct resource_list_entry *rle;
 	struct resource_list *rl;
@@ -5203,12 +5204,13 @@ pci_delete_child(device_t dev, device_t child)
 	dinfo = device_get_ivars(child);
 	rl = &dinfo->resources;
 
-	if (device_is_attached(child))
-		device_detach(child);
-
 	/* Turn off access to resources we're about to free */
-	pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
-	    PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
+	if (bus_child_present(child) != 0) {
+		pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
+		    PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
+
+		pci_disable_busmaster(child);
+	}
 
 	/* Free all allocated resources */
 	STAILQ_FOREACH(rle, rl, link) {
@@ -5229,7 +5231,6 @@ pci_delete_child(device_t dev, device_t child)
 	}
 	resource_list_free(rl);
 
-	device_delete_child(dev, child);
 	pci_freecfg(dinfo);
 }
 
diff --git a/sys/dev/pci/pci_iov.c b/sys/dev/pci/pci_iov.c
index 8189ffb..d216557 100644
--- a/sys/dev/pci/pci_iov.c
+++ b/sys/dev/pci/pci_iov.c
@@ -640,7 +640,7 @@ pci_iov_enumerate_vfs(struct pci_devinfo *dinfo, const nvlist_t *config,
 		error = PCI_IOV_ADD_VF(dev, i, driver_config);
 		if (error != 0) {
 			device_printf(dev, "Failed to add VF %d\n", i);
-			pci_delete_child(bus, vf);
+			device_delete_child(bus, vf);
 		}
 	}
 
@@ -833,7 +833,7 @@ pci_iov_delete(struct cdev *cdev)
 		vf = devlist[i];
 
 		if (pci_iov_is_child_vf(iov, vf))
-			pci_delete_child(bus, vf);
+			device_delete_child(bus, vf);
 	}
 	PCI_IOV_UNINIT(dev);
 
diff --git a/sys/dev/pci/pci_private.h b/sys/dev/pci/pci_private.h
index 9b018ab..e11dcfa 100644
--- a/sys/dev/pci/pci_private.h
+++ b/sys/dev/pci/pci_private.h
@@ -57,7 +57,6 @@ void		pci_add_resources(device_t bus, device_t dev, int force,
 		    uint32_t prefetchmask);
 void		pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov);
 int		pci_attach_common(device_t dev);
-void		pci_delete_child(device_t dev, device_t child);
 void		pci_driver_added(device_t dev, driver_t *driver);
 int		pci_ea_is_enabled(device_t dev, int rid);
 int		pci_print_child(device_t dev, device_t child);
@@ -122,6 +121,7 @@ struct pci_devinfo *pci_read_device(device_t pcib, int d, int b, int s, int f,
 		    size_t size);
 void		pci_print_verbose(struct pci_devinfo *dinfo);
 int		pci_freecfg(struct pci_devinfo *dinfo);
+void		pci_child_deleted(device_t dev, device_t child);
 void		pci_child_detached(device_t dev, device_t child);
 int		pci_child_location_str_method(device_t cbdev, device_t child,
 		    char *buf, size_t buflen);
diff --git a/sys/powerpc/ofw/ofw_pcibus.c b/sys/powerpc/ofw/ofw_pcibus.c
index cde3c74..4ce6e73 100644
--- a/sys/powerpc/ofw/ofw_pcibus.c
+++ b/sys/powerpc/ofw/ofw_pcibus.c
@@ -61,6 +61,7 @@ static device_probe_t ofw_pcibus_probe;
 static device_attach_t ofw_pcibus_attach;
 static pci_assign_interrupt_t ofw_pcibus_assign_interrupt;
 static ofw_bus_get_devinfo_t ofw_pcibus_get_devinfo;
+static bus_child_deleted_t ofw_pcibus_child_deleted;
 static int ofw_pcibus_child_pnpinfo_str_method(device_t cbdev, device_t child,
     char *buf, size_t buflen);
 
@@ -73,6 +74,7 @@ static device_method_t ofw_pcibus_methods[] = {
 	DEVMETHOD(device_attach,	ofw_pcibus_attach),
 
 	/* Bus interface */
+	DEVMETHOD(bus_child_deleted,	ofw_pcibus_child_deleted),
 	DEVMETHOD(bus_child_pnpinfo_str, ofw_pcibus_child_pnpinfo_str_method),
 
 	/* PCI interface */
@@ -269,6 +271,16 @@ ofw_pcibus_enum_bus(device_t dev, u_int domain, u_int busno)
 	}
 }
 
+static void
+ofw_pcibus_child_deleted(device_t dev, device_t child)
+{
+	struct ofw_pcibus_devinfo *dinfo;
+
+	dinfo = device_get_ivars(dev);
+	ofw_bus_gen_destroy_devinfo(&dinfo->opd_obdinfo);
+	pci_child_deleted(dev, child);
+}
+
 static int
 ofw_pcibus_child_pnpinfo_str_method(device_t cbdev, device_t child, char *buf,
     size_t buflen)
diff --git a/sys/sparc64/pci/ofw_pcibus.c b/sys/sparc64/pci/ofw_pcibus.c
index 92b9f76..08ffa5d 100644
--- a/sys/sparc64/pci/ofw_pcibus.c
+++ b/sys/sparc64/pci/ofw_pcibus.c
@@ -65,6 +65,7 @@ static void ofw_pcibus_setup_device(device_t bridge, uint32_t clock,
     u_int busno, u_int slot, u_int func);
 
 /* Methods */
+static bus_child_deleted_t ofw_pcibus_child_deleted;
 static bus_child_pnpinfo_str_t ofw_pcibus_pnpinfo_str;
 static device_attach_t ofw_pcibus_attach;
 static device_probe_t ofw_pcibus_probe;
@@ -77,6 +78,7 @@ static device_method_t ofw_pcibus_methods[] = {
 	DEVMETHOD(device_attach,	ofw_pcibus_attach),
 
 	/* Bus interface */
+	DEVMETHOD(bus_child_deleted,	ofw_pcibus_child_deleted),
 	DEVMETHOD(bus_child_pnpinfo_str, ofw_pcibus_pnpinfo_str),
 
 	/* PCI interface */
@@ -327,6 +329,16 @@ ofw_pcibus_get_devinfo(device_t bus, device_t dev)
 	return (&dinfo->opd_obdinfo);
 }
 
+static void
+ofw_pcibus_child_deleted(device_t dev, device_t child)
+{
+	struct ofw_pcibus_devinfo *dinfo;
+
+	dinfo = device_get_ivars(dev);
+	ofw_bus_gen_destroy_devinfo(&dinfo->opd_obdinfo);
+	pci_child_deleted(dev, child);
+}
+
 static int
 ofw_pcibus_pnpinfo_str(device_t dev, device_t child, char *buf,
     size_t buflen)