Rework controller reset procedure. Previously driver saved

BGE_PCI_PCISTATE register before issuing global reset. After
issuing reset, it reads BGE_PCI_PCISTATE register again and
compares the saved register value and current value. It was used to
know whether the global reset operation was completed or not.
Unfortunately, this logic caused several issues on recent BCM5717/
5718/5719 and BCM5720 controllers. It seems APE firmware accesses
some registers while global reset is in progress such that reading
BGE_PCI_PCISTATE register after reset does not yield old pre-reset
state value. This resulted in consuming too much time in global
reset and sometimes it couldn't successfully complete reset.

The BGE_MISCCFG_RESET_CORE_CLOCKS of BGE_MISC_CFG register is
self-clearing bit so driver is able to know the reset completion.
But the core-lock reset will disable indirect/flat/standard access
modes such that driver cannot poll BGE_MISCCFG_RESET_CORE_CLOCKS
bit of BGE_MISC_CFG register. So just wait enough time for
core-clock reset to complete.
Data sheet says driver should wait 100us for PCI/PCI-X devices and
100ms for PCIe devices. I chose 1ms for PCI/PCI-X since this value
was used for many years in bge(4). For PCIe devices, use 100ms as
recommended by data sheet.

bge_chipinit() also cleared BGE_MAC_MODE register which shall clear
firmware configured mode information. I think this will result in
losing ASF/IPMI link in device attachment. Let bge_reset() honor
firmware configured BGE_MAC_MODE register and don't announce driver
is UP in bge_reset(). Firmware should have control over driver until
it's fully initialized by driver.

While I'm here, enable workaround for PCI-X BCM5704 A0 in
bge_reset(). This will prevent internal arbitration logic from
switching to the other DMA engine after a retry cycle.

2 files changed, 24 insertions, 34 deletions

diff --git a/sys/dev/bge/if_bge.c b/sys/dev/bge/if_bge.c
index 2c4ad77..24e29ff 100644
--- a/sys/dev/bge/if_bge.c
+++ b/sys/dev/bge/if_bge.c
@@ -1433,10 +1433,6 @@ bge_chipinit(struct bge_softc *sc)
 		misc_ctl |= BGE_PCIMISCCTL_TAGGED_STATUS;
 	pci_write_config(sc->bge_dev, BGE_PCI_MISC_CTL, misc_ctl, 4);
 
-	/* Clear the MAC control register */
-	CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
-	DELAY(40);
-
 	/*
 	 * Clear the MAC statistics block in the NIC's
 	 * internal memory.
@@ -3560,13 +3556,16 @@ static int
 bge_reset(struct bge_softc *sc)
 {
 	device_t dev;
-	uint32_t cachesize, command, pcistate, reset, val;
+	uint32_t cachesize, command, mac_mode, mac_mode_mask, reset, val;
 	void (*write_op)(struct bge_softc *, int, int);
 	uint16_t devctl;
 	int i;
 
 	dev = sc->bge_dev;
 
+	mac_mode_mask = BGE_MACMODE_HALF_DUPLEX | BGE_MACMODE_PORTMODE;
+	mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) & mac_mode_mask;
+
 	if (BGE_IS_575X_PLUS(sc) && !BGE_IS_5714_FAMILY(sc) &&
 	    (sc->bge_asicrev != BGE_ASICREV_BCM5906)) {
 		if (sc->bge_flags & BGE_FLAG_PCIE)
@@ -3579,7 +3578,6 @@ bge_reset(struct bge_softc *sc)
 	/* Save some important PCI state. */
 	cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
 	command = pci_read_config(dev, BGE_PCI_CMD, 4);
-	pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
 
 	pci_write_config(dev, BGE_PCI_MISC_CTL,
 	    BGE_PCIMISCCTL_INDIRECT_ACCESS | BGE_PCIMISCCTL_MASK_PCI_INTR |
@@ -3636,7 +3634,10 @@ bge_reset(struct bge_softc *sc)
 	/* Issue global reset */
 	write_op(sc, BGE_MISC_CFG, reset);
 
-	DELAY(1000);
+	if (sc->bge_flags & BGE_FLAG_PCIE)
+		DELAY(100 * 1000);
+	else
+		DELAY(1000);
 
 	/* XXX: Broadcom Linux driver. */
 	if (sc->bge_flags & BGE_FLAG_PCIE) {
@@ -3664,9 +3665,13 @@ bge_reset(struct bge_softc *sc)
 	pci_write_config(dev, BGE_PCI_MISC_CTL,
 	    BGE_PCIMISCCTL_INDIRECT_ACCESS | BGE_PCIMISCCTL_MASK_PCI_INTR |
 	    BGE_HIF_SWAP_OPTIONS | BGE_PCIMISCCTL_PCISTATE_RW, 4);
+	val = BGE_PCISTATE_ROM_ENABLE | BGE_PCISTATE_ROM_RETRY_ENABLE;
+	if (sc->bge_chipid == BGE_CHIPID_BCM5704_A0 &&
+	    (sc->bge_flags & BGE_FLAG_PCIX) != 0)
+		val |= BGE_PCISTATE_RETRY_SAME_DMA;
+	pci_write_config(dev, BGE_PCI_PCISTATE, val, 4);
 	pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
 	pci_write_config(dev, BGE_PCI_CMD, command, 4);
-	write_op(sc, BGE_MISC_CFG, BGE_32BITTIME_66MHZ);
 	/*
 	 * Disable PCI-X relaxed ordering to ensure status block update
 	 * comes first then packet buffer DMA. Otherwise driver may
@@ -3705,6 +3710,14 @@ bge_reset(struct bge_softc *sc)
 	} else
 		CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
 
+	/* Fix up byte swapping. */
+	CSR_WRITE_4(sc, BGE_MODE_CTL, bge_dma_swap_options(sc));
+
+	val = CSR_READ_4(sc, BGE_MAC_MODE);
+	val = (val & ~mac_mode_mask) | mac_mode;
+	CSR_WRITE_4(sc, BGE_MAC_MODE, val);
+	DELAY(40);
+
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
 		for (i = 0; i < BGE_TIMEOUT; i++) {
 			val = CSR_READ_4(sc, BGE_VCPU_STATUS);
@@ -3740,30 +3753,6 @@ bge_reset(struct bge_softc *sc)
 	}
 
 	/*
-	 * XXX Wait for the value of the PCISTATE register to
-	 * return to its original pre-reset state. This is a
-	 * fairly good indicator of reset completion. If we don't
-	 * wait for the reset to fully complete, trying to read
-	 * from the device's non-PCI registers may yield garbage
-	 * results.
-	 */
-	for (i = 0; i < BGE_TIMEOUT; i++) {
-		if (pci_read_config(dev, BGE_PCI_PCISTATE, 4) == pcistate)
-			break;
-		DELAY(10);
-	}
-
-	/* Fix up byte swapping. */
-	CSR_WRITE_4(sc, BGE_MODE_CTL, bge_dma_swap_options(sc));
-
-	/* Tell the ASF firmware we are up */
-	if (sc->bge_asf_mode & ASF_STACKUP)
-		BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
-
-	CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
-	DELAY(40);
-
-	/*
 	 * The 5704 in TBI mode apparently needs some special
 	 * adjustment to insure the SERDES drive level is set
 	 * to 1.2V.
diff --git a/sys/dev/bge/if_bgereg.h b/sys/dev/bge/if_bgereg.h
index 88627c2..b1031c0 100644
--- a/sys/dev/bge/if_bgereg.h
+++ b/sys/dev/bge/if_bgereg.h
@@ -430,10 +430,11 @@
 #define	BGE_PCISTATE_PCI_BUSMODE	0x00000004 /* 1 = PCI, 0 = PCI-X */
 #define	BGE_PCISTATE_PCI_BUSSPEED	0x00000008 /* 1 = 66/133, 0 = 33/66 */
 #define	BGE_PCISTATE_32BIT_BUS		0x00000010 /* 1 = 32bit, 0 = 64bit */
-#define	BGE_PCISTATE_WANT_EXPROM	0x00000020
-#define	BGE_PCISTATE_EXPROM_RETRY	0x00000040
+#define	BGE_PCISTATE_ROM_ENABLE		0x00000020
+#define	BGE_PCISTATE_ROM_RETRY_ENABLE	0x00000040
 #define	BGE_PCISTATE_FLATVIEW_MODE	0x00000100
 #define	BGE_PCISTATE_PCI_TGT_RETRY_MAX	0x00000E00
+#define	BGE_PCISTATE_RETRY_SAME_DMA	0x00002000
 
 /*
  * PCI Clock Control register -- note, this register is read only