Rework global locks for interface list and index management, correcting

several critical bugs, including race conditions and lock order issues:

Replace the single rwlock, ifnet_lock, with two locks, an rwlock and an
sxlock.  Either can be held to stablize the lists and indexes, but both
are required to write.  This allows the list to be held stable in both
network interrupt contexts and sleepable user threads across sleeping
memory allocations or device driver interactions.  As before, writes to
the interface list must occur from sleepable contexts.

Reviewed by:	bz, julian
MFC after:	3 days

16 files changed, 133 insertions, 97 deletions

diff --git a/sys/compat/linux/linux_ioctl.c b/sys/compat/linux/linux_ioctl.c
index 11dc6d5..03ac334 100644
--- a/sys/compat/linux/linux_ioctl.c
+++ b/sys/compat/linux/linux_ioctl.c
@@ -2061,22 +2061,20 @@ linux_ifname(struct ifnet *ifp, char *buffer, size_t buflen)
 	struct ifnet *ifscan;
 	int ethno;
 
+	IFNET_RLOCK_ASSERT();
+
 	/* Short-circuit non ethernet interfaces */
 	if (!IFP_IS_ETH(ifp))
 		return (strlcpy(buffer, ifp->if_xname, buflen));
 
 	/* Determine the (relative) unit number for ethernet interfaces */
 	ethno = 0;
-	IFNET_RLOCK();
 	TAILQ_FOREACH(ifscan, &V_ifnet, if_link) {
-		if (ifscan == ifp) {
-			IFNET_RUNLOCK();
+		if (ifscan == ifp)
 			return (snprintf(buffer, buflen, "eth%d", ethno));
-		}
 		if (IFP_IS_ETH(ifscan))
 			ethno++;
 	}
-	IFNET_RUNLOCK();
 
 	return (0);
 }
@@ -2177,7 +2175,7 @@ again:
 	valid_len = 0;
 
 	/* Return all AF_INET addresses of all interfaces */
-	IFNET_RLOCK();		/* could sleep XXX */
+	IFNET_RLOCK();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		int addrs = 0;
 
diff --git a/sys/contrib/altq/altq/altq_subr.c b/sys/contrib/altq/altq/altq_subr.c
index 32107e5..edacc0d 100644
--- a/sys/contrib/altq/altq/altq_subr.c
+++ b/sys/contrib/altq/altq/altq_subr.c
@@ -462,8 +462,8 @@ tbr_timeout(arg)
 	s = splimp();
 #endif
 #if defined(__FreeBSD__) && (__FreeBSD_version >= 500000)
-	IFNET_RLOCK();
-	VNET_LIST_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
+	VNET_LIST_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 #endif
@@ -480,8 +480,8 @@ tbr_timeout(arg)
 #if defined(__FreeBSD__) && (__FreeBSD_version >= 500000)
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
-	IFNET_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
+	IFNET_RUNLOCK_NOSLEEP();
 #endif
 	splx(s);
 	if (active > 0)
diff --git a/sys/kern/kern_uuid.c b/sys/kern/kern_uuid.c
index 5b61ca4..8c083f4 100644
--- a/sys/kern/kern_uuid.c
+++ b/sys/kern/kern_uuid.c
@@ -95,7 +95,7 @@ uuid_node(uint16_t *node)
 	int i;
 
 	CURVNET_SET(TD_TO_VNET(curthread));
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		/* Walk the address list */
 		IF_ADDR_LOCK(ifp);
@@ -106,14 +106,14 @@ uuid_node(uint16_t *node)
 				/* Got a MAC address. */
 				bcopy(LLADDR(sdl), node, UUID_NODE_LEN);
 				IF_ADDR_UNLOCK(ifp);
-				IFNET_RUNLOCK();
+				IFNET_RUNLOCK_NOSLEEP();
 				CURVNET_RESTORE();
 				return;
 			}
 		}
 		IF_ADDR_UNLOCK(ifp);
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	for (i = 0; i < (UUID_NODE_LEN>>1); i++)
 		node[i] = (uint16_t)arc4random();
diff --git a/sys/net/bridgestp.c b/sys/net/bridgestp.c
index 3e65113..2993838 100644
--- a/sys/net/bridgestp.c
+++ b/sys/net/bridgestp.c
@@ -2019,7 +2019,7 @@ bstp_reinit(struct bstp_state *bs)
 	 * not need to be part of the bridge, it just needs to be a unique
 	 * value.
 	 */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (ifp->if_type != IFT_ETHER)
 			continue;
@@ -2036,7 +2036,7 @@ bstp_reinit(struct bstp_state *bs)
 			continue;
 		}
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	if (LIST_EMPTY(&bs->bs_bplist) || mif == NULL) {
 		/* Set the bridge and root id (lower bits) to zero */
diff --git a/sys/net/if.c b/sys/net/if.c
index 5d10898..fe497d6 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -154,14 +154,26 @@ VNET_DEFINE(struct ifgrouphead, ifg_head);
 VNET_DEFINE(int, if_index);
 static VNET_DEFINE(int, if_indexlim) = 8;
 
-/* Table of ifnet by index.  Locked with ifnet_lock. */
+/* Table of ifnet by index. */
 static VNET_DEFINE(struct ifindex_entry *, ifindex_table);
 
 #define	V_if_indexlim		VNET(if_indexlim)
 #define	V_ifindex_table		VNET(ifindex_table)
 
 int	ifqmaxlen = IFQ_MAXLEN;
-struct rwlock ifnet_lock;
+
+/*
+ * The global network interface list (V_ifnet) and related state (such as
+ * if_index, if_indexlim, and ifindex_table) are protected by an sxlock and
+ * an rwlock.  Either may be acquired shared to stablize the list, but both
+ * must be acquired writable to modify the list.  This model allows us to
+ * both stablize the interface list during interrupt thread processing, but
+ * also to stablize it over long-running ioctls, without introducing priority
+ * inversions and deadlocks.
+ */
+struct rwlock ifnet_rwlock;
+struct sx ifnet_sxlock;
+
 static	if_com_alloc_t *if_com_alloc[256];
 static	if_com_free_t *if_com_free[256];
 
@@ -188,9 +200,9 @@ ifnet_byindex(u_short idx)
 {
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	ifp = ifnet_byindex_locked(idx);
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifp);
 }
 
@@ -199,19 +211,19 @@ ifnet_byindex_ref(u_short idx)
 {
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	ifp = ifnet_byindex_locked(idx);
 	if (ifp == NULL || (ifp->if_flags & IFF_DYING)) {
-		IFNET_RUNLOCK();
+		IFNET_RUNLOCK_NOSLEEP();
 		return (NULL);
 	}
 	if_ref(ifp);
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifp);
 }
 
 static void
-ifnet_setbyindex(u_short idx, struct ifnet *ifp)
+ifnet_setbyindex_locked(u_short idx, struct ifnet *ifp)
 {
 
 	IFNET_WLOCK_ASSERT();
@@ -219,16 +231,25 @@ ifnet_setbyindex(u_short idx, struct ifnet *ifp)
 	V_ifindex_table[idx].ife_ifnet = ifp;
 }
 
+static void
+ifnet_setbyindex(u_short idx, struct ifnet *ifp)
+{
+
+	IFNET_WLOCK();
+	ifnet_setbyindex_locked(idx, ifp);
+	IFNET_WUNLOCK();
+}
+
 struct ifaddr *
 ifaddr_byindex(u_short idx)
 {
 	struct ifaddr *ifa;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	ifa = ifnet_byindex_locked(idx)->if_addr;
 	if (ifa != NULL)
 		ifa_ref(ifa);
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifa);
 }
 
@@ -361,9 +382,7 @@ if_alloc(u_char type)
 	ifq_init(&ifp->if_snd, ifp);
 
 	refcount_init(&ifp->if_refcount, 1);	/* Index reference. */
-	IFNET_WLOCK();
 	ifnet_setbyindex(ifp->if_index, ifp);
-	IFNET_WUNLOCK();
 	return (ifp);
 }
 
@@ -383,7 +402,7 @@ if_free_internal(struct ifnet *ifp)
 	KASSERT(ifp == ifnet_byindex_locked(ifp->if_index),
 	    ("%s: freeing unallocated ifnet", ifp->if_xname));
 
-	ifnet_setbyindex(ifp->if_index, NULL);
+	ifnet_setbyindex_locked(ifp->if_index, NULL);
 	while (V_if_index > 0 && ifnet_byindex_locked(V_if_index) == NULL)
 		V_if_index--;
 	IFNET_WUNLOCK();
@@ -855,11 +874,14 @@ if_vmove(struct ifnet *ifp, struct vnet *new_vnet)
 	if_detach_internal(ifp, 1);
 
 	/*
-	 * Unlink the ifnet from ifindex_table[] in current vnet,
-	 * and shrink the if_index for that vnet if possible.
+	 * Unlink the ifnet from ifindex_table[] in current vnet, and shrink
+	 * the if_index for that vnet if possible.
+	 *
+	 * NOTE: IFNET_WLOCK/IFNET_WUNLOCK() are assumed to be unvirtualized,
+	 * or we'd lock on one vnet and unlock on another.
 	 */
 	IFNET_WLOCK();
-	ifnet_setbyindex(ifp->if_index, NULL);
+	ifnet_setbyindex_locked(ifp->if_index, NULL);
 	while (V_if_index > 0 && ifnet_byindex_locked(V_if_index) == NULL)
 		V_if_index--;
 	IFNET_WUNLOCK();
@@ -886,7 +908,7 @@ if_vmove(struct ifnet *ifp, struct vnet *new_vnet)
 		V_if_index = ifp->if_index;
 	if (V_if_index >= V_if_indexlim)
 		if_grow();
-	ifnet_setbyindex(ifp->if_index, ifp);
+	ifnet_setbyindex_locked(ifp->if_index, ifp);
 	IFNET_WUNLOCK();
 
 	if_attach_internal(ifp, 1);
@@ -1368,7 +1390,7 @@ ifa_ifwithaddr_internal(struct sockaddr *addr, int getref)
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@@ -1395,7 +1417,7 @@ ifa_ifwithaddr_internal(struct sockaddr *addr, int getref)
 	}
 	ifa = NULL;
 done:
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifa);
 }
 
@@ -1423,7 +1445,7 @@ ifa_ifwithbroadaddr(struct sockaddr *addr)
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@@ -1442,7 +1464,7 @@ ifa_ifwithbroadaddr(struct sockaddr *addr)
 	}
 	ifa = NULL;
 done:
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifa);
 }
 
@@ -1456,7 +1478,7 @@ ifa_ifwithdstaddr(struct sockaddr *addr)
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if ((ifp->if_flags & IFF_POINTOPOINT) == 0)
 			continue;
@@ -1475,7 +1497,7 @@ ifa_ifwithdstaddr(struct sockaddr *addr)
 	}
 	ifa = NULL;
 done:
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifa);
 }
 
@@ -1508,7 +1530,7 @@ ifa_ifwithnet(struct sockaddr *addr)
 	 * we find one, as we release the IF_ADDR_LOCK() that kept it stable
 	 * when we move onto the next interface.
 	 */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@@ -1584,7 +1606,7 @@ next:				continue;
 	ifa = ifa_maybe;
 	ifa_maybe = NULL;
 done:
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	if (ifa_maybe != NULL)
 		ifa_free(ifa_maybe);
 	return (ifa);
@@ -1863,7 +1885,7 @@ if_slowtimo(void *arg)
 	int s = splimp();
 
 	VNET_LIST_RLOCK_NOSLEEP();
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 		TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
@@ -1874,7 +1896,7 @@ if_slowtimo(void *arg)
 		}
 		CURVNET_RESTORE();
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	VNET_LIST_RUNLOCK_NOSLEEP();
 	splx(s);
 	timeout(if_slowtimo, (void *)0, hz / IFNET_SLOWHZ);
@@ -1889,7 +1911,7 @@ ifunit_ref(const char *name)
 {
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0 &&
 		    !(ifp->if_flags & IFF_DYING))
@@ -1897,7 +1919,7 @@ ifunit_ref(const char *name)
 	}
 	if (ifp != NULL)
 		if_ref(ifp);
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifp);
 }
 
@@ -1906,12 +1928,12 @@ ifunit(const char *name)
 {
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0)
 			break;
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifp);
 }
 
@@ -2521,7 +2543,7 @@ again:
 	max_len = 0;
 	valid_len = 0;
 
-	IFNET_RLOCK();		/* could sleep XXX */
+	IFNET_RLOCK();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		int addrs;
 
@@ -2846,13 +2868,13 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa)
 #ifdef INVARIANTS
 	struct ifnet *oifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(oifp, &V_ifnet, if_link)
 		if (ifp == oifp)
 			break;
 	if (ifp != oifp)
 		ifp = NULL;
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	KASSERT(ifp != NULL, ("%s: ifnet went away", __func__));
 #endif
@@ -2895,7 +2917,7 @@ if_delmulti_ifma(struct ifmultiaddr *ifma)
 	} else {
 		struct ifnet *oifp;
 
-		IFNET_RLOCK();
+		IFNET_RLOCK_NOSLEEP();
 		TAILQ_FOREACH(oifp, &V_ifnet, if_link)
 			if (ifp == oifp)
 				break;
@@ -2903,7 +2925,7 @@ if_delmulti_ifma(struct ifmultiaddr *ifma)
 			printf("%s: ifnet %p disappeared\n", __func__, ifp);
 			ifp = NULL;
 		}
-		IFNET_RUNLOCK();
+		IFNET_RUNLOCK_NOSLEEP();
 	}
 #endif
 	/*
diff --git a/sys/net/if_llatbl.c b/sys/net/if_llatbl.c
index a1b574f..a3617f5 100644
--- a/sys/net/if_llatbl.c
+++ b/sys/net/if_llatbl.c
@@ -200,14 +200,14 @@ lltable_prefix_free(int af, struct sockaddr *prefix, struct sockaddr *mask)
 {
 	struct lltable *llt;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	SLIST_FOREACH(llt, &lltables, llt_link) {
 		if (llt->llt_af != af)
 			continue;
 
 		llt->llt_prefix_free(llt, prefix, mask);
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 }
 
 
@@ -300,13 +300,13 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 	}
 
 	/* XXX linked list may be too expensive */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	SLIST_FOREACH(llt, &lltables, llt_link) {
 		if (llt->llt_af == dst->sa_family &&
 		    llt->llt_ifp == ifp)
 			break;
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	KASSERT(llt != NULL, ("Yep, ugly hacks are bad\n"));
 
 	if (flags && LLE_CREATE)
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index aa83161..0bfd9f1 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -85,6 +85,7 @@ struct	vnet;
 #include <sys/lock.h>		/* XXX */
 #include <sys/mutex.h>		/* XXX */
 #include <sys/rwlock.h>		/* XXX */
+#include <sys/sx.h>		/* XXX */
 #include <sys/event.h>		/* XXX */
 #include <sys/_task.h>
 
@@ -754,14 +755,39 @@ struct ifmultiaddr {
 
 #ifdef _KERNEL
 
-extern	struct rwlock ifnet_lock;
-#define	IFNET_LOCK_INIT() \
-   rw_init_flags(&ifnet_lock, "ifnet",  RW_RECURSE)
-#define	IFNET_WLOCK()		rw_wlock(&ifnet_lock)
-#define	IFNET_WUNLOCK()		rw_wunlock(&ifnet_lock)
-#define	IFNET_WLOCK_ASSERT()	rw_assert(&ifnet_lock, RA_LOCKED)
-#define	IFNET_RLOCK()		rw_rlock(&ifnet_lock)
-#define	IFNET_RUNLOCK()		rw_runlock(&ifnet_lock)	
+extern	struct rwlock ifnet_rwlock;
+extern	struct sx ifnet_sxlock;
+
+#define	IFNET_LOCK_INIT() do {						\
+	rw_init_flags(&ifnet_rwlock, "ifnet_rw",  RW_RECURSE);		\
+	sx_init_flags(&ifnet_sxlock, "ifnet_sx",  SX_RECURSE);		\
+} while(0)
+
+#define	IFNET_WLOCK() do {						\
+	sx_xlock(&ifnet_sxlock);					\
+	rw_wlock(&ifnet_rwlock);					\
+} while (0)
+
+#define	IFNET_WUNLOCK() do {						\
+	rw_wunlock(&ifnet_rwlock);					\
+	sx_xunlock(&ifnet_sxlock);					\
+} while (0)
+
+/*
+ * To assert the ifnet lock, you must know not only whether it's for read or
+ * write, but also whether it was acquired with sleep support or not.
+ */
+#define	IFNET_RLOCK_ASSERT()		sx_assert(&ifnet_sxlock, SA_SLOCKED)
+#define	IFNET_RLOCK_NOSLEEP_ASSERT()	rw_assert(&ifnet_rwlock, RA_RLOCKED)
+#define	IFNET_WLOCK_ASSERT() do {					\
+	sx_assert(&ifnet_sxlock, SA_XLOCKED);				\
+	rw_assert(&ifnet_rwlock, RA_WLOCKED);				\
+} while (0)
+
+#define	IFNET_RLOCK()		sx_slock(&ifnet_sxlock)
+#define	IFNET_RLOCK_NOSLEEP()	rw_rlock(&ifnet_rwlock)
+#define	IFNET_RUNLOCK()		sx_sunlock(&ifnet_sxlock)
+#define	IFNET_RUNLOCK_NOSLEEP()	rw_runlock(&ifnet_rwlock)
 
 /*
  * Look up an ifnet given its index; the _ref variant also acquires a
diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c
index 04c2c0d..126d83f 100644
--- a/sys/net/if_vlan.c
+++ b/sys/net/if_vlan.c
@@ -580,7 +580,7 @@ vlan_clone_match_ethertag(struct if_clone *ifc, const char *name, int *tag)
 	int t = 0;
 
 	/* Check for <etherif>.<vlan> style interface names. */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (ifp->if_type != IFT_ETHER)
 			continue;
@@ -598,7 +598,7 @@ vlan_clone_match_ethertag(struct if_clone *ifc, const char *name, int *tag)
 			*tag = t;
 		break;
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	return (ifp);
 }
diff --git a/sys/netgraph/ng_gif.c b/sys/netgraph/ng_gif.c
index ccff05a..1790cbd 100644
--- a/sys/netgraph/ng_gif.c
+++ b/sys/netgraph/ng_gif.c
@@ -560,7 +560,7 @@ ng_gif_mod_event(module_t mod, int event, void *data)
 		ng_gif_input_orphan_p = ng_gif_input_orphan;
 
 		/* Create nodes for any already-existing gif interfaces */
-		VNET_LIST_RLOCK_NOSLEEP();
+		VNET_LIST_RLOCK();
 		IFNET_RLOCK();
 		VNET_FOREACH(vnet_iter) {
 			CURVNET_SET_QUIET(vnet_iter); /* XXX revisit quiet */
@@ -571,7 +571,7 @@ ng_gif_mod_event(module_t mod, int event, void *data)
 			CURVNET_RESTORE();
 		}
 		IFNET_RUNLOCK();
-		VNET_LIST_RUNLOCK_NOSLEEP();
+		VNET_LIST_RUNLOCK();
 		break;
 
 	case MOD_UNLOAD:
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index 15d38b1..8c9984c 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -1407,12 +1407,7 @@ in_lltable_dump(struct lltable *llt, struct sysctl_req *wr)
 	} arpc;
 	int error, i;
 
-	/* XXXXX
-	 * current IFNET_RLOCK() is mapped to IFNET_WLOCK()
-	 * so it is okay to use this ASSERT, change it when
-	 * IFNET lock is finalized
-	 */
-	IFNET_WLOCK_ASSERT();
+	IFNET_RLOCK_ASSERT();
 
 	error = 0;
 	for (i = 0; i < LLTBL_HASHTBL_SIZE; i++) {
diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index 7fe63b6..6f13c69 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -1705,7 +1705,7 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp,
 		}
 	}
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	for (ifp = TAILQ_FIRST(&V_ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) {
 		addrsofif = 0;
 		IF_ADDR_LOCK(ifp);
@@ -1762,13 +1762,13 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp,
 		IF_ADDR_UNLOCK(ifp);
 		if (iffound) {
 			*ifpp = ifp;
-			IFNET_RUNLOCK();
+			IFNET_RUNLOCK_NOSLEEP();
 			return (addrsofif);
 		}
 
 		addrs += addrsofif;
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	return (addrs);
 }
@@ -1789,7 +1789,7 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6,
 	if (ifp0 == NULL && !(niflags & NI_NODEADDR_FLAG_ALL))
 		return (0);	/* needless to copy */
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
   again:
 
 	for (; ifp; ifp = TAILQ_NEXT(ifp, if_list)) {
@@ -1854,7 +1854,7 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6,
 				 * Set the truncate flag and return.
 				 */
 				nni6->ni_flags |= NI_NODEADDR_FLAG_TRUNCATE;
-				IFNET_RUNLOCK();
+				IFNET_RUNLOCK_NOSLEEP();
 				return (copied);
 			}
 
@@ -1907,7 +1907,7 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6,
 		goto again;
 	}
 
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	return (copied);
 }
diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index c4333ed..7d9f7e7 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -2234,7 +2234,7 @@ in6_setmaxmtu(void)
 	unsigned long maxmtu = 0;
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	for (ifp = TAILQ_FIRST(&V_ifnet); ifp;
 	    ifp = TAILQ_NEXT(ifp, if_list)) {
 		/* this function can be called during ifnet initialization */
@@ -2244,7 +2244,7 @@ in6_setmaxmtu(void)
 		    IN6_LINKMTU(ifp) > maxmtu)
 			maxmtu = IN6_LINKMTU(ifp);
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	if (maxmtu)	     /* update only when maxmtu is positive */
 		V_in6_maxmtu = maxmtu;
 }
@@ -2495,12 +2495,7 @@ in6_lltable_dump(struct lltable *llt, struct sysctl_req *wr)
 	} ndpc;
 	int i, error;
 
-	/* XXXXX
-	 * current IFNET_RLOCK() is mapped to IFNET_WLOCK()
-	 * so it is okay to use this ASSERT, change it when
-	 * IFNET lock is finalized
-	 */
-	IFNET_WLOCK_ASSERT();
+	IFNET_RLOCK_ASSERT();
 
 	error = 0;
 	for (i = 0; i < LLTBL_HASHTBL_SIZE; i++) {
diff --git a/sys/netinet6/in6_ifattach.c b/sys/netinet6/in6_ifattach.c
index 1fc54c6..6f9f11d 100644
--- a/sys/netinet6/in6_ifattach.c
+++ b/sys/netinet6/in6_ifattach.c
@@ -398,7 +398,7 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp,
 	}
 
 	/* next, try to get it from some other hardware interface */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	for (ifp = V_ifnet.tqh_first; ifp; ifp = ifp->if_list.tqe_next) {
 		if (ifp == ifp0)
 			continue;
@@ -413,11 +413,11 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp,
 			nd6log((LOG_DEBUG,
 			    "%s: borrow interface identifier from %s\n",
 			    if_name(ifp0), if_name(ifp)));
-			IFNET_RUNLOCK();
+			IFNET_RUNLOCK_NOSLEEP();
 			goto success;
 		}
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	/* last resort: get from random number source */
 	if (get_rand_ifid(ifp, in6) == 0) {
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 48635c5..acaa87e 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -1662,7 +1662,7 @@ nd6_slowtimo(void *arg)
 
 	callout_reset(&V_nd6_slowtimo_ch, ND6_SLOWTIMER_INTERVAL * hz,
 	    nd6_slowtimo, curvnet);
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	for (ifp = TAILQ_FIRST(&V_ifnet); ifp;
 	    ifp = TAILQ_NEXT(ifp, if_list)) {
 		nd6if = ND_IFINFO(ifp);
@@ -1678,7 +1678,7 @@ nd6_slowtimo(void *arg)
 			nd6if->reachable = ND_COMPUTE_RTIME(nd6if->basereachable);
 		}
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	CURVNET_RESTORE();
 }
 
diff --git a/sys/netipsec/xform_ipip.c b/sys/netipsec/xform_ipip.c
index c9669b1..78ab097 100644
--- a/sys/netipsec/xform_ipip.c
+++ b/sys/netipsec/xform_ipip.c
@@ -303,7 +303,7 @@ _ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp)
 	if ((m->m_pkthdr.rcvif == NULL ||
 	    !(m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK)) &&
 	    V_ipip_allow != 2) {
-	    	IFNET_RLOCK();
+	    	IFNET_RLOCK_NOSLEEP();
 		TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 			TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 #ifdef INET
@@ -318,7 +318,7 @@ _ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp)
 					    ipo->ip_src.s_addr)	{
 						V_ipipstat.ipips_spoof++;
 						m_freem(m);
-						IFNET_RUNLOCK();
+						IFNET_RUNLOCK_NOSLEEP();
 						return;
 					}
 				}
@@ -335,7 +335,7 @@ _ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp)
 					if (IN6_ARE_ADDR_EQUAL(&sin6->sin6_addr, &ip6->ip6_src)) {
 						V_ipipstat.ipips_spoof++;
 						m_freem(m);
-						IFNET_RUNLOCK();
+						IFNET_RUNLOCK_NOSLEEP();
 						return;
 					}
 
@@ -343,7 +343,7 @@ _ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp)
 #endif /* INET6 */
 			}
 		}
-		IFNET_RUNLOCK();
+		IFNET_RUNLOCK_NOSLEEP();
 	}
 
 	/* Statistics */
diff --git a/sys/nfsclient/bootp_subr.c b/sys/nfsclient/bootp_subr.c
index e1096be..e57b32a 100644
--- a/sys/nfsclient/bootp_subr.c
+++ b/sys/nfsclient/bootp_subr.c
@@ -389,7 +389,7 @@ bootpboot_p_iflist(void)
 	struct ifaddr *ifa;
 
 	printf("Interface list:\n");
-	IFNET_RLOCK(); /* could sleep, but okay for debugging XXX */
+	IFNET_RLOCK();
 	for (ifp = TAILQ_FIRST(&V_ifnet);
 	     ifp != NULL;
 	     ifp = TAILQ_NEXT(ifp, if_link)) {