MFC r304433

PCIe HotPlug: Detect bridges that are not really HotPlug capable

Some devices report that they have an MRL when they actually
do not.  Since they always report that the MRL is open, child
devices would be ignored.  Try to detect these devices and
ignore their claim of HotPlug support.  Specifically,
if there is an open MRL but the Data Link Layer is active,
the MRL is not real.

Revert r303645 to re-enable HotPlug support for slots with
power controllers, since it works correctly in my testing.

Start the DLL state-change timer if Presence /or/ MRL state changes,
along with other conditions.  Previously, we started the timer iff
Presence changed.  If there is an MRL, it must be closed for power
to be turned on, so Presence is unlikely to change on an MRL-close event.

Add a printf() of interesting registers on HotPlug interrupts and
commands (one from erj@).  These were very useful for debugging.
Guard them with bootverbose, since they're spam in normal operation.

In collaboration with:	jhb
Relnotes:	yes (re-enable HotPlug support for slots with power controllers)
Approved by:	re (gjb)
Sponsored by:	Dell Inc.
Differential Revision:	https://reviews.freebsd.org/D7509

1 files changed, 34 insertions, 15 deletions

diff --git a/sys/dev/pci/pci_pci.c b/sys/dev/pci/pci_pci.c
index 6e8d076..dedc55a 100644
--- a/sys/dev/pci/pci_pci.c
+++ b/sys/dev/pci/pci_pci.c
@@ -918,6 +918,7 @@ static void
 pcib_probe_hotplug(struct pcib_softc *sc)
 {
 	device_t dev;
+	uint16_t link_sta, slot_sta;
 
 	if (!pci_enable_pcie_hp)
 		return;
@@ -932,15 +933,29 @@ pcib_probe_hotplug(struct pcib_softc *sc)
 	sc->pcie_link_cap = pcie_read_config(dev, PCIER_LINK_CAP, 4);
 	sc->pcie_slot_cap = pcie_read_config(dev, PCIER_SLOT_CAP, 4);
 
+	if ((sc->pcie_slot_cap & PCIEM_SLOT_CAP_HPC) == 0)
+		return;
+
 	/*
-	 * XXX: Handling of slots with a power controller needs to be
-	 * reexamined.  Ignore hotplug on such slots for now.
+	 * Some devices report that they have an MRL when they actually
+	 * do not.  Since they always report that the MRL is open, child
+	 * devices would be ignored.  Try to detect these devices and
+	 * ignore their claim of HotPlug support.
+	 *
+	 * If there is an open MRL but the Data Link Layer is active,
+	 * the MRL is not real.
 	 */
-	if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_PCP)
-		return;
-	
-	if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_HPC)
-		sc->flags |= PCIB_HOTPLUG;
+	if ((sc->pcie_slot_cap & PCIEM_SLOT_CAP_MRLSP) != 0 &&
+	    (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) != 0) {
+		link_sta = pcie_read_config(dev, PCIER_LINK_STA, 2);
+		slot_sta = pcie_read_config(dev, PCIER_SLOT_STA, 2);
+		if ((slot_sta & PCIEM_SLOT_STA_MRLSS) != 0 &&
+		    (link_sta & PCIEM_LINK_STA_DL_ACTIVE) != 0) {
+			return;
+		}
+	}
+
+	sc->flags |= PCIB_HOTPLUG;
 }
 
 /*
@@ -966,6 +981,8 @@ pcib_pcie_hotplug_command(struct pcib_softc *sc, uint16_t val, uint16_t mask)
 	new = (ctl & ~mask) | val;
 	if (new == ctl)
 		return;
+	if (bootverbose)
+		device_printf(dev, "HotPlug command: %04x -> %04x\n", ctl, new);
 	pcie_write_config(dev, PCIER_SLOT_CTL, new, 2);
 	if (!(sc->pcie_slot_cap & PCIEM_SLOT_CAP_NCCS) &&
 	    (ctl & new) & PCIEM_SLOT_CTL_CCIE) {
@@ -1028,9 +1045,6 @@ pcib_hotplug_inserted(struct pcib_softc *sc)
 static int
 pcib_hotplug_present(struct pcib_softc *sc)
 {
-	device_t dev;
-
-	dev = sc->dev;
 
 	/* Card must be inserted. */
 	if (!pcib_hotplug_inserted(sc))
@@ -1059,7 +1073,7 @@ pcib_pcie_hotplug_update(struct pcib_softc *sc, uint16_t val, uint16_t mask,
 {
 	bool card_inserted, ei_engaged;
 
-	/* Clear DETACHING if Present Detect has cleared. */
+	/* Clear DETACHING if Presence Detect has cleared. */
 	if ((sc->pcie_slot_sta & (PCIEM_SLOT_STA_PDC | PCIEM_SLOT_STA_PDS)) ==
 	    PCIEM_SLOT_STA_PDC)
 		sc->flags &= ~PCIB_DETACHING;
@@ -1101,14 +1115,15 @@ pcib_pcie_hotplug_update(struct pcib_softc *sc, uint16_t val, uint16_t mask,
 
 	/*
 	 * Start a timer to see if the Data Link Layer times out.
-	 * Note that we only start the timer if Presence Detect
+	 * Note that we only start the timer if Presence Detect or MRL Sensor
 	 * changed on this interrupt.  Stop any scheduled timer if
 	 * the Data Link Layer is active.
 	 */
 	if (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) {
 		if (card_inserted &&
 		    !(sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE) &&
-		    sc->pcie_slot_sta & PCIEM_SLOT_STA_PDC) {
+		    sc->pcie_slot_sta &
+		    (PCIEM_SLOT_STA_MRLSC | PCIEM_SLOT_STA_PDC)) {
 			if (cold)
 				device_printf(sc->dev,
 				    "Data Link Layer inactive\n");
@@ -1144,6 +1159,10 @@ pcib_pcie_intr(void *arg)
 	/* Clear the events just reported. */
 	pcie_write_config(dev, PCIER_SLOT_STA, sc->pcie_slot_sta, 2);
 
+	if (bootverbose)
+		device_printf(dev, "HotPlug interrupt: %#x\n",
+		    sc->pcie_slot_sta);
+
 	if (sc->pcie_slot_sta & PCIEM_SLOT_STA_ABP) {
 		if (sc->flags & PCIB_DETACH_PENDING) {	
 			device_printf(dev,
@@ -1165,7 +1184,7 @@ pcib_pcie_intr(void *arg)
 		    sc->pcie_slot_sta & PCIEM_SLOT_STA_MRLSS ? "open" :
 		    "closed");
 	if (bootverbose && sc->pcie_slot_sta & PCIEM_SLOT_STA_PDC)
-		device_printf(dev, "Present Detect Changed to %s\n",
+		device_printf(dev, "Presence Detect Changed to %s\n",
 		    sc->pcie_slot_sta & PCIEM_SLOT_STA_PDS ? "card present" :
 		    "empty");
 	if (sc->pcie_slot_sta & PCIEM_SLOT_STA_CC)
@@ -1234,7 +1253,7 @@ pcib_pcie_cc_timeout(void *arg)
 	sta = pcie_read_config(dev, PCIER_SLOT_STA, 2);
 	if (!(sta & PCIEM_SLOT_STA_CC)) {
 		device_printf(dev,
-		    "Hotplug Command Timed Out - forcing detach\n");
+		    "HotPlug Command Timed Out - forcing detach\n");
 		sc->flags &= ~(PCIB_HOTPLUG_CMD_PENDING | PCIB_DETACH_PENDING);
 		sc->flags |= PCIB_DETACHING;
 		pcib_pcie_hotplug_update(sc, 0, 0, true);