Get rid of the redundant 'gone' field, and overload suspend instead.

Check for suspend before the device polling, rather than after it.
Check to see if the current thread owns the lock in ioctl and return
EBUSY if it does.

This advances the locking to the point that I can eject my fxp card 10
times in a row, but I agree with Jeff Hsu that we need to get the
network layer locking finished before chasing more of the races here
(actually, he doesn't think this set is worth it even).  There's a
number of races between FXP_LOCK in detach and all other users of
FXP_LOCK, and this gets back to the 'device with sleepers being
forcibly detached' problem as well...

2 files changed, 12 insertions, 13 deletions

diff --git a/sys/dev/fxp/if_fxp.c b/sys/dev/fxp/if_fxp.c
index aa16761..0fc487e 100644
--- a/sys/dev/fxp/if_fxp.c
+++ b/sys/dev/fxp/if_fxp.c
@@ -881,7 +881,7 @@ fxp_detach(device_t dev)
 	FXP_LOCK(sc);
 	s = splimp();
 
-	sc->gone = 1;
+	sc->suspend = 1;	/* Do same thing as we do for suspend */
 	/*
 	 * Close down routes etc.
 	 */
@@ -1499,10 +1499,12 @@ fxp_intr(void *xsc)
 	struct ifnet *ifp = &sc->sc_if;
 	u_int8_t statack;
 
-	if (sc->gone)
+	FXP_LOCK(sc);
+	if (sc->suspended) {
+		FXP_UNLOCK(sc);
 		return;
+	}
 
-	FXP_LOCK(sc);
 #ifdef DEVICE_POLLING
 	if (ifp->if_flags & IFF_POLLING) {
 		FXP_UNLOCK(sc);
@@ -1516,12 +1518,6 @@ fxp_intr(void *xsc)
 		return;
 	}
 #endif
-
-	if (sc->suspended) {
-		FXP_UNLOCK(sc);
-		return;
-	}
-
 	while ((statack = CSR_READ_1(sc, FXP_CSR_SCB_STATACK)) != 0) {
 		/*
 		 * It should not be possible to have all bits set; the
@@ -2362,8 +2358,12 @@ fxp_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 	struct mii_data *mii;
 	int s, error = 0;
 
-	if (sc->gone)
-		return (ENODEV);
+	/*
+	 * Detaching causes us to call ioctl with the mutex owned.  Preclude
+	 * that by saying we're busy if the lock is already held.
+	 */
+	if (mtx_owned(&sc->sc_mtx))
+		return (EBUSY);
 
 	FXP_LOCK(sc);
 	s = splimp();
diff --git a/sys/dev/fxp/if_fxpvar.h b/sys/dev/fxp/if_fxpvar.h
index b73cf50..dd52c07 100644
--- a/sys/dev/fxp/if_fxpvar.h
+++ b/sys/dev/fxp/if_fxpvar.h
@@ -185,11 +185,10 @@ struct fxp_softc {
 	int tunable_int_delay;		/* interrupt delay value for ucode */
 	int tunable_bundle_max;		/* max # frames per interrupt (ucode) */
 	int eeprom_size;		/* size of serial EEPROM */
-	int suspended;			/* 0 = normal  1 = suspended (APM) */
+	int suspended;			/* 0 = normal  1 = suspended or dead */
 	int cu_resume_bug;
 	int revision;
 	int flags;
-	int gone;
 	u_int32_t saved_maps[5];	/* pci data */
 	u_int32_t saved_biosaddr;
 	u_int8_t saved_intline;