Make ex(4) MPSAFE:

- Add a mutex to the softc to protect the softc and device hardware.
- Use a private watchdog timer.
- Setup interrupt handler after ether_ifattach().
- Use bus_foo() rather than bus_space_foo() and remove bus space tag and
  handle from softc.

Tested by:	imp

4 files changed, 101 insertions, 90 deletions

diff --git a/sys/dev/ex/if_ex.c b/sys/dev/ex/if_ex.c
index cd5b975..16553c0 100644
--- a/sys/dev/ex/if_ex.c
+++ b/sys/dev/ex/if_ex.c
@@ -98,9 +98,11 @@ u_char plus_ee2irqmap[] =
 
 /* Network Interface Functions */
 static void	ex_init(void *);
+static void	ex_init_locked(struct ex_softc *);
 static void	ex_start(struct ifnet *);
+static void	ex_start_locked(struct ifnet *);
 static int	ex_ioctl(struct ifnet *, u_long, caddr_t);
-static void	ex_watchdog(struct ifnet *);
+static void	ex_watchdog(void *);
 
 /* ifmedia Functions	*/
 static int	ex_ifmedia_upd(struct ifnet *);
@@ -158,8 +160,6 @@ ex_alloc_resources(device_t dev)
 		error = ENOMEM;
 		goto bad;
 	}
-	sc->bst = rman_get_bustag(sc->ioport);
-	sc->bsh = rman_get_bushandle(sc->ioport);
 
 	sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irq_rid,
 					RF_ACTIVE);
@@ -208,6 +208,7 @@ ex_attach(device_t dev)
 	struct ex_softc *	sc = device_get_softc(dev);
 	struct ifnet *		ifp;
 	struct ifmedia *	ifm;
+	int			error;
 	uint16_t		temp;
 
 	ifp = sc->ifp = if_alloc(IFT_ETHER);
@@ -232,15 +233,16 @@ ex_attach(device_t dev)
 	ifp->if_softc = sc;
 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 	ifp->if_mtu = ETHERMTU;
-	ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST |
-	    IFF_NEEDSGIANT;
+	ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST;
 	ifp->if_start = ex_start;
 	ifp->if_ioctl = ex_ioctl;
-	ifp->if_watchdog = ex_watchdog;
 	ifp->if_init = ex_init;
-	ifp->if_snd.ifq_maxlen = IFQ_MAXLEN;
+	IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN);
 
 	ifmedia_init(&sc->ifmedia, 0, ex_ifmedia_upd, ex_ifmedia_sts);
+	mtx_init(&sc->lock, device_get_nameunit(dev), MTX_NETWORK_LOCK,
+	    MTX_DEF);
+	callout_init_mtx(&sc->timer, &sc->lock, 0);
 
 	temp = ex_eeprom_read(sc, EE_W5);
 	if (temp & EE_W5_PORT_TPE)
@@ -255,7 +257,7 @@ ex_attach(device_t dev)
 	ifmedia_set(&sc->ifmedia, ex_get_media(sc));
 
 	ifm = &sc->ifmedia;
-	ifm->ifm_media = ifm->ifm_cur->ifm_media;
+	ifm->ifm_media = ifm->ifm_cur->ifm_media;	
 	ex_ifmedia_upd(ifp);
 
 	/*
@@ -263,6 +265,15 @@ ex_attach(device_t dev)
 	 */
 	ether_ifattach(ifp, sc->enaddr);
 
+	error = bus_setup_intr(dev, sc->irq, INTR_TYPE_NET | INTR_MPSAFE,
+				NULL, ex_intr, (void *)sc, &sc->ih);
+	if (error) {
+		device_printf(dev, "bus_setup_intr() failed!\n");
+		ether_ifdetach(ifp);
+		mtx_destroy(&sc->lock);
+		return (error);
+	}
+
 	return(0);
 }
 
@@ -275,12 +286,15 @@ ex_detach(device_t dev)
 	sc = device_get_softc(dev);
 	ifp = sc->ifp;
 
+	EX_LOCK(sc);
         ex_stop(sc);
+	EX_UNLOCK(sc);
 
-        ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 	ether_ifdetach(ifp);
+	callout_drain(&sc->timer);
 
 	ex_release_resources(dev);
+	mtx_destroy(&sc->lock);
 
 	return (0);
 }
@@ -289,15 +303,22 @@ static void
 ex_init(void *xsc)
 {
 	struct ex_softc *	sc = (struct ex_softc *) xsc;
+
+	EX_LOCK(sc);
+	ex_init_locked(sc);
+	EX_UNLOCK(sc);
+}
+
+static void
+ex_init_locked(struct ex_softc *sc)
+{
 	struct ifnet *		ifp = sc->ifp;
-	int			s;
 	int			i;
 	unsigned short		temp_reg;
 
 	DODEBUG(Start_End, printf("%s: ex_init: start\n", ifp->if_xname););
 
-	s = splimp();
-	ifp->if_timer = 0;
+	sc->tx_timeout = 0;
 
 	/*
 	 * Load the ethernet address into the card.
@@ -359,6 +380,7 @@ ex_init(void *xsc)
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 	DODEBUG(Status, printf("OIDLE init\n"););
+	callout_reset(&sc->timer, hz, ex_watchdog, sc);
 	
 	ex_setmulti(sc);
 	
@@ -369,26 +391,32 @@ ex_init(void *xsc)
 	DELAY(2);
 	CSR_WRITE_1(sc, CMD_REG, Rcv_Enable_CMD);
 
-	ex_start(ifp);
-	splx(s);
+	ex_start_locked(ifp);
 
 	DODEBUG(Start_End, printf("%s: ex_init: finish\n", ifp->if_xname););
 }
 
-
 static void
 ex_start(struct ifnet *ifp)
 {
 	struct ex_softc *	sc = ifp->if_softc;
-	int			i, s, len, data_len, avail, dest, next;
+
+	EX_LOCK(sc);
+	ex_start_locked(ifp);
+	EX_UNLOCK(sc);
+}
+
+static void
+ex_start_locked(struct ifnet *ifp)
+{
+	struct ex_softc *	sc = ifp->if_softc;
+	int			i, len, data_len, avail, dest, next;
 	unsigned char		tmp16[2];
 	struct mbuf *		opkt;
 	struct mbuf *		m;
 
 	DODEBUG(Start_End, printf("ex_start%d: start\n", unit););
 
-	s = splimp();
-
 	/*
 	 * Main loop: send outgoing packets to network card until there are no
 	 * more packets left, or the card cannot accept any more yet.
@@ -536,7 +564,7 @@ ex_start(struct ifnet *ifp)
      	 
 			BPF_MTAP(ifp, opkt);
 
-			ifp->if_timer = 2;
+			sc->tx_timeout = 2;
 			ifp->if_opackets++;
 			m_freem(opkt);
 		} else {
@@ -545,8 +573,6 @@ ex_start(struct ifnet *ifp)
 		}
 	}
 
-	splx(s);
-
 	DODEBUG(Start_End, printf("ex_start%d: finish\n", unit););
 }
 
@@ -556,6 +582,7 @@ ex_stop(struct ex_softc *sc)
 	
 	DODEBUG(Start_End, printf("ex_stop%d: start\n", unit););
 
+	EX_ASSERT_LOCKED(sc);
 	/*
 	 * Disable card operation:
 	 * - Disable the interrupt line.
@@ -573,6 +600,9 @@ ex_stop(struct ex_softc *sc)
 	CSR_WRITE_1(sc, STATUS_REG, All_Int);
 	CSR_WRITE_1(sc, CMD_REG, Reset_CMD);
 	DELAY(200);
+	sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+	sc->tx_timeout = 0;
+	callout_stop(&sc->timer);
 
 	DODEBUG(Start_End, printf("ex_stop%d: finish\n", unit););
 
@@ -589,6 +619,7 @@ ex_intr(void *arg)
 
 	DODEBUG(Start_End, printf("ex_intr%d: start\n", unit););
 
+	EX_LOCK(sc);
 	send_pkts = 0;
 	while (loops-- > 0 &&
 	    (int_status = CSR_READ_1(sc, STATUS_REG)) & (Tx_Int | Rx_Int)) {
@@ -612,7 +643,8 @@ ex_intr(void *arg)
 	 * be sent, attempt to send more packets to the network card.
 	 */
 	if (send_pkts && (ifp->if_snd.ifq_head != NULL))
-		ex_start(ifp);
+		ex_start_locked(ifp);
+	EX_UNLOCK(sc);
 
 	DODEBUG(Start_End, printf("ex_intr%d: finish\n", unit););
 
@@ -634,7 +666,7 @@ ex_tx_intr(struct ex_softc *sc)
 	 * - Update statistics.
 	 */
 
-	ifp->if_timer = 0;
+	sc->tx_timeout = 0;
 
 	while (sc->tx_head != sc->tx_tail) {
 		CSR_WRITE_2(sc, HOST_ADDR_REG, sc->tx_head);
@@ -751,7 +783,9 @@ ex_rx_intr(struct ex_softc *sc)
 		} /* QQQ */
 	}
 #endif
+				EX_UNLOCK(sc);
 				(*ifp->if_input)(ifp, ipkt);
+				EX_LOCK(sc);
 				ifp->if_ipackets++;
 			}
 		} else {
@@ -777,13 +811,10 @@ ex_ioctl(register struct ifnet *ifp, u_long cmd, caddr_t data)
 {
 	struct ex_softc *	sc = ifp->if_softc;
 	struct ifreq *		ifr = (struct ifreq *)data;
-	int			s;
 	int			error = 0;
 
 	DODEBUG(Start_End, printf("%s: ex_ioctl: start ", ifp->if_xname););
 
-	s = splimp();
-
 	switch(cmd) {
 		case SIOCSIFADDR:
 		case SIOCGIFADDR:
@@ -793,14 +824,14 @@ ex_ioctl(register struct ifnet *ifp, u_long cmd, caddr_t data)
 
 		case SIOCSIFFLAGS:
 			DODEBUG(Start_End, printf("SIOCSIFFLAGS"););
+			EX_LOCK(sc);
 			if ((ifp->if_flags & IFF_UP) == 0 &&
 			    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
-
-				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 				ex_stop(sc);
 			} else {
-      				ex_init(sc);
+      				ex_init_locked(sc);
 			}
+			EX_UNLOCK(sc);
 			break;
 		case SIOCADDMULTI:
 		case SIOCDELMULTI:
@@ -816,8 +847,6 @@ ex_ioctl(register struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = EINVAL;
 	}
 
-	splx(s);
-
 	DODEBUG(Start_End, printf("\n%s: ex_ioctl: finish\n", ifp->if_xname););
 
 	return(error);
@@ -923,16 +952,12 @@ ex_setmulti(struct ex_softc *sc)
 static void
 ex_reset(struct ex_softc *sc)
 {
-	int s;
 
 	DODEBUG(Start_End, printf("ex_reset%d: start\n", unit););
-  
-	s = splimp();
 
+	EX_ASSERT_LOCKED(sc);
 	ex_stop(sc);
-	ex_init(sc);
-
-	splx(s);
+	ex_init_locked(sc);
 
 	DODEBUG(Start_End, printf("ex_reset%d: finish\n", unit););
 
@@ -940,23 +965,26 @@ ex_reset(struct ex_softc *sc)
 }
 
 static void
-ex_watchdog(struct ifnet *ifp)
+ex_watchdog(void *arg)
 {
-	struct ex_softc *	sc = ifp->if_softc;
+	struct ex_softc *	sc = arg;
+	struct ifnet *ifp = sc->ifp;
 
-	DODEBUG(Start_End, printf("%s: ex_watchdog: start\n", ifp->if_xname););
+	if (sc->tx_timeout && --sc->tx_timeout == 0) {
+		DODEBUG(Start_End, if_printf(ifp, "ex_watchdog: start\n"););
 
-	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
-	DODEBUG(Status, printf("OIDLE watchdog\n"););
+		DODEBUG(Status, printf("OIDLE watchdog\n"););
 
-	ifp->if_oerrors++;
-	ex_reset(sc);
-	ex_start(ifp);
+		ifp->if_oerrors++;
+		ex_reset(sc);
+		ex_start_locked(ifp);
 
-	DODEBUG(Start_End, printf("%s: ex_watchdog: finish\n", ifp->if_xname););
+		DODEBUG(Start_End, if_printf(ifp, "ex_watchdog: finish\n"););
+	}
 
-	return;
+	callout_reset(&sc->timer, hz, ex_watchdog, sc);
 }
 
 static int
@@ -1001,8 +1029,10 @@ ex_ifmedia_sts(ifp, ifmr)
 {
 	struct ex_softc *       sc = ifp->if_softc;
 
+	EX_LOCK(sc);
 	ifmr->ifm_active = ex_get_media(sc);
 	ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
+	EX_UNLOCK(sc);
 
 	return;
 }
diff --git a/sys/dev/ex/if_ex_isa.c b/sys/dev/ex/if_ex_isa.c
index 84b71a2..31b00a3 100644
--- a/sys/dev/ex/if_ex_isa.c
+++ b/sys/dev/ex/if_ex_isa.c
@@ -126,7 +126,6 @@ ex_isa_identify(driver_t *driver, device_t parent)
 	int		tmp;
 	const char *	desc;
 	struct ex_softc sc;
-	struct resource *res;
 	int		rid;
 
 	if (bootverbose)
@@ -134,16 +133,15 @@ ex_isa_identify(driver_t *driver, device_t parent)
 
 	for (ioport = 0x200; ioport < 0x3a0; ioport += 0x10) {
 		rid = 0;
-		res = bus_alloc_resource(parent, SYS_RES_IOPORT, &rid,
+		sc.ioport = bus_alloc_resource(parent, SYS_RES_IOPORT, &rid,
 		    ioport, ioport, 0x10, RF_ACTIVE);
-		if (res == NULL)
+		if (sc.ioport == NULL)
 			continue;
-		sc.bst = rman_get_bustag(res);
-		sc.bsh = rman_get_bushandle(res);
 
 		/* No board found at address */
 		if (!ex_look_for_card(&sc)) {
-			bus_release_resource(parent, SYS_RES_IOPORT, rid, res);
+			bus_release_resource(parent, SYS_RES_IOPORT, rid,
+			    sc.ioport);
 			continue;
 		}
 
@@ -157,7 +155,8 @@ ex_isa_identify(driver_t *driver, device_t parent)
 			DELAY(500);
 			if (bootverbose)
 				printf("ex: card at 0x%03lx in PnP mode!\n", (unsigned long)ioport);
-			bus_release_resource(parent, SYS_RES_IOPORT, rid, res);
+			bus_release_resource(parent, SYS_RES_IOPORT, rid,
+			    sc.ioport);
 			continue;
 		}
 
@@ -179,7 +178,7 @@ ex_isa_identify(driver_t *driver, device_t parent)
 			desc = "Intel Pro/10";
 		}
 
-		bus_release_resource(parent, SYS_RES_IOPORT, rid, res);
+		bus_release_resource(parent, SYS_RES_IOPORT, rid, sc.ioport);
 		child = BUS_ADD_CHILD(parent, ISA_ORDER_SPECULATIVE, "ex", -1);
 		device_set_desc_copy(child, desc);
 		device_set_driver(child, driver);
@@ -313,13 +312,6 @@ ex_isa_attach(device_t dev)
 		goto bad;
 	}
 
-	error = bus_setup_intr(dev, sc->irq, INTR_TYPE_NET,
-				NULL, ex_intr, (void *)sc, &sc->ih);
-	if (error) {
-		device_printf(dev, "bus_setup_intr() failed!\n");
-		goto bad;
-	}
-
 	return(0);
 bad:
 	ex_release_resources(dev);
diff --git a/sys/dev/ex/if_ex_pccard.c b/sys/dev/ex/if_ex_pccard.c
index 1d16463..98d1b5d 100644
--- a/sys/dev/ex/if_ex_pccard.c
+++ b/sys/dev/ex/if_ex_pccard.c
@@ -164,13 +164,6 @@ ex_pccard_attach(device_t dev)
 		goto bad;
 	}
 
-	error = bus_setup_intr(dev, sc->irq, INTR_TYPE_NET,
-	    NULL, ex_intr, (void *)sc, &sc->ih);
-	if (error) {
-		device_printf(dev, "bus_setup_intr() failed!\n");
-		goto bad;
-	}
-
 	return(0);
 bad:
 	ex_release_resources(dev);
diff --git a/sys/dev/ex/if_exvar.h b/sys/dev/ex/if_exvar.h
index 9df9886..9278f6d 100644
--- a/sys/dev/ex/if_exvar.h
+++ b/sys/dev/ex/if_exvar.h
@@ -41,8 +41,6 @@ struct ex_softc {
 	int		irq_rid;
 	void *		ih;
 
-	bus_space_tag_t	bst;
-	bus_space_handle_t bsh;
 	u_short		irq_no;		/* IRQ number. */
 
 	char *		irq2ee;		/* irq <-> internal		*/
@@ -67,6 +65,9 @@ struct ex_softc {
 
 	u_int		tx_last;	/* Pointer to beginning of last	*/
 					/* frame in the chain.		*/
+	struct mtx	lock;
+	struct callout	timer;
+	int		tx_timeout;
 };
 
 extern devclass_t ex_devclass;
@@ -89,30 +90,25 @@ int		ex_card_type(u_char *);
 
 void		ex_stop(struct ex_softc *);
 
-#define CSR_READ_1(sc, off) (bus_space_read_1((sc)->bst, (sc)->bsh, off))
-#define CSR_READ_2(sc, off) (bus_space_read_2((sc)->bst, (sc)->bsh, off))
+#define CSR_READ_1(sc, off) (bus_read_1((sc)->ioport, off))
+#define CSR_READ_2(sc, off) (bus_read_2((sc)->ioport, off))
 #define CSR_WRITE_1(sc, off, val) \
-	bus_space_write_1((sc)->bst, (sc)->bsh, off, val)
+	bus_write_1((sc)->ioport, off, val)
 #define CSR_WRITE_2(sc, off, val) \
-	bus_space_write_2((sc)->bst, (sc)->bsh, off, val)
+	bus_write_2((sc)->ioport, off, val)
 #define CSR_WRITE_MULTI_1(sc, off, addr, count) \
-	bus_space_write_multi_1((sc)->bst, (sc)->bsh, off, addr, count)
+	bus_write_multi_1((sc)->ioport, off, addr, count)
 #define CSR_WRITE_MULTI_2(sc, off, addr, count) \
-	bus_space_write_multi_2((sc)->bst, (sc)->bsh, off, addr, count)
+	bus_write_multi_2((sc)->ioport, off, addr, count)
 #define CSR_WRITE_MULTI_4(sc, off, addr, count) \
-	bus_space_write_multi_4((sc)->bst, (sc)->bsh, off, addr, count)
+	bus_write_multi_4((sc)->ioport, off, addr, count)
 #define CSR_READ_MULTI_1(sc, off, addr, count) \
-	bus_space_read_multi_1((sc)->bst, (sc)->bsh, off, addr, count)
+	bus_read_multi_1((sc)->ioport, off, addr, count)
 #define CSR_READ_MULTI_2(sc, off, addr, count) \
-	bus_space_read_multi_2((sc)->bst, (sc)->bsh, off, addr, count)
+	bus_read_multi_2((sc)->ioport, off, addr, count)
 #define CSR_READ_MULTI_4(sc, off, addr, count) \
-	bus_space_read_multi_4((sc)->bst, (sc)->bsh, off, addr, count)
-
-#define EX_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
-#define	EX_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_mtx)
-#define EX_LOCK_INIT(_sc) \
-	mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \
-	    MTX_NETWORK_LOCK, MTX_DEF)
-#define EX_LOCK_DESTROY(_sc)	mtx_destroy(&_sc->sc_mtx);
-#define EX_ASSERT_LOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_OWNED);
-#define EX_ASSERT_UNLOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_NOTOWNED);
+	bus_read_multi_4((sc)->ioport, off, addr, count)
+
+#define	EX_LOCK(sc)		mtx_lock(&(sc)->lock)
+#define	EX_UNLOCK(sc)		mtx_unlock(&(sc)->lock)
+#define	EX_ASSERT_LOCKED(sc)	mtx_assert(&(sc)->lock, MA_OWNED)