Reimplement and/or implement vnet list locking by replacing a mostly

unused custom mutex/condvar-based sleep locks with two locks: an
rwlock (for non-sleeping use) and sxlock (for sleeping use).  Either
acquired for read is sufficient to stabilize the vnet list, but both
must be acquired for write to modify the list.

Replace previous no-op read locking macros, used in various places
in the stack, with actual locking to prevent race conditions.  Callers
must declare when they may perform unbounded sleeps or not when
selecting how to lock.

Refactor vnet sysinits so that the vnet list and locks are initialized
before kernel modules are linked, as the kernel linker will use them
for modules loaded by the boot loader.

Update various consumers of these KPIs based on whether they may sleep
or not.

Reviewed by:	bz
Approved by:	re (kib)

6 files changed, 18 insertions, 18 deletions

diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c
index 37cc1b3..6a7cb83 100644
--- a/sys/netinet/igmp.c
+++ b/sys/netinet/igmp.c
@@ -1616,13 +1616,13 @@ igmp_fasttimo(void)
 {
 	VNET_ITERATOR_DECL(vnet_iter);
 
-	VNET_LIST_RLOCK();
+	VNET_LIST_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 		igmp_fasttimo_vnet();
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 }
 
 /*
@@ -2159,13 +2159,13 @@ igmp_slowtimo(void)
 {
 	VNET_ITERATOR_DECL(vnet_iter);
 
-	VNET_LIST_RLOCK();
+	VNET_LIST_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 		igmp_slowtimo_vnet();
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 }
 
 /*
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 94906ad..5d080ba 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -1570,7 +1570,7 @@ ipport_tick(void *xtp)
 {
 	VNET_ITERATOR_DECL(vnet_iter);
 
-	VNET_LIST_RLOCK();
+	VNET_LIST_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);	/* XXX appease INVARIANTS here */
 		if (V_ipport_tcpallocs <=
@@ -1582,7 +1582,7 @@ ipport_tick(void *xtp)
 		V_ipport_tcplastcount = V_ipport_tcpallocs;
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 	callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
 }
 
diff --git a/sys/netinet/in_rmx.c b/sys/netinet/in_rmx.c
index 2ef4ed6..d3f9563 100644
--- a/sys/netinet/in_rmx.c
+++ b/sys/netinet/in_rmx.c
@@ -319,7 +319,7 @@ in_rtqdrain(void)
 	struct rtqk_arg arg;
 	int 	fibnum;
 
-	VNET_LIST_RLOCK();
+	VNET_LIST_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 
@@ -336,7 +336,7 @@ in_rtqdrain(void)
 		}
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 }
 
 static int _in_rt_was_here;
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index 8e84707..61fc835 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -1193,8 +1193,8 @@ ip_slowtimo(void)
 	struct ipq *fp;
 	int i;
 
+	VNET_LIST_RLOCK_NOSLEEP();
 	IPQ_LOCK();
-	VNET_LIST_RLOCK();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 		for (i = 0; i < IPREASS_NHASH; i++) {
@@ -1228,8 +1228,8 @@ ip_slowtimo(void)
 		}
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
 	IPQ_UNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 }
 
 /*
@@ -1241,8 +1241,8 @@ ip_drain(void)
 	VNET_ITERATOR_DECL(vnet_iter);
 	int     i;
 
+	VNET_LIST_RLOCK_NOSLEEP();
 	IPQ_LOCK();
-	VNET_LIST_RLOCK();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 		for (i = 0; i < IPREASS_NHASH; i++) {
@@ -1254,8 +1254,8 @@ ip_drain(void)
 		}
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
 	IPQ_UNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 	in_rtqdrain();
 }
 
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index df61a00..8ebc889 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -940,7 +940,7 @@ tcp_drain(void)
 	if (!do_tcpdrain)
 		return;
 
-	VNET_LIST_RLOCK();
+	VNET_LIST_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 		struct inpcb *inpb;
@@ -976,7 +976,7 @@ tcp_drain(void)
 		INP_INFO_RUNLOCK(&V_tcbinfo);
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 }
 
 /*
@@ -1576,7 +1576,7 @@ tcp_isn_tick(void *xtp)
 	VNET_ITERATOR_DECL(vnet_iter);
 	u_int32_t projected_offset;
 
-	VNET_LIST_RLOCK();
+	VNET_LIST_RLOCK_NOSLEEP();
 	ISN_LOCK();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS */
@@ -1590,7 +1590,7 @@ tcp_isn_tick(void *xtp)
 		CURVNET_RESTORE();
 	}
 	ISN_UNLOCK();
-	VNET_LIST_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 	callout_reset(&isn_callout, hz/100, tcp_isn_tick, NULL);
 }
 
diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index 7006b70..3050324 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -127,7 +127,7 @@ tcp_slowtimo(void)
 {
 	VNET_ITERATOR_DECL(vnet_iter);
 
-	VNET_LIST_RLOCK();
+	VNET_LIST_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 		tcp_maxidle = tcp_keepcnt * tcp_keepintvl;
@@ -136,7 +136,7 @@ tcp_slowtimo(void)
 		INP_INFO_WUNLOCK(&V_tcbinfo);
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
 }
 
 int	tcp_syn_backoff[TCP_MAXRXTSHIFT + 1] =