First pass at separating per-vnet initializer functions

        from existing functions for initializing global state.

        At this stage, the new per-vnet initializer functions are
	directly called from the existing global initialization code,
	which should in most cases result in compiler inlining those
	new functions, hence yielding a near-zero functional change.

        Modify the existing initializer functions which are invoked via
        protosw, like ip_init() et. al., to allow them to be invoked
	multiple times, i.e. per each vnet.  Global state, if any,
	is initialized only if such functions are called within the
	context of vnet0, which will be determined via the
	IS_DEFAULT_VNET(curvnet) check (currently always true).

        While here, V_irtualize a few remaining global UMA zones
        used by net/netinet/netipsec networking code.  While it is
        not yet clear to me or anybody else whether this is the right
        thing to do, at this stage this makes the code more readable,
        and makes it easier to track uncollected UMA-zone-backed
        objects on vnet removal.  In the long run, it's quite possible
        that some form of shared use of UMA zone pools among multiple
        vnets should be considered.

	Bump __FreeBSD_version due to changes in layout of structs
	vnet_ipfw, vnet_inet and vnet_net.

Approved by:	julian (mentor)

4 files changed, 40 insertions, 15 deletions

diff --git a/sys/netinet6/frag6.c b/sys/netinet6/frag6.c
index e2eda6f..d0f333d 100644
--- a/sys/netinet6/frag6.c
+++ b/sys/netinet6/frag6.c
@@ -109,14 +109,16 @@ frag6_init(void)
 {
 	INIT_VNET_INET6(curvnet);
 
+	V_ip6q.ip6q_next = V_ip6q.ip6q_prev = &V_ip6q;
 	V_ip6_maxfragpackets = nmbclusters / 4;
 	V_ip6_maxfrags = nmbclusters / 4;
-	EVENTHANDLER_REGISTER(nmbclusters_change,
-	    frag6_change, NULL, EVENTHANDLER_PRI_ANY);
 
-	IP6Q_LOCK_INIT();
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
 
-	V_ip6q.ip6q_next = V_ip6q.ip6q_prev = &V_ip6q;
+	IP6Q_LOCK_INIT();
+	EVENTHANDLER_REGISTER(nmbclusters_change,
+	    frag6_change, NULL, EVENTHANDLER_PRI_ANY);
 }
 
 /*
diff --git a/sys/netinet6/in6_src.c b/sys/netinet6/in6_src.c
index 8030416..7ae30387 100644
--- a/sys/netinet6/in6_src.c
+++ b/sys/netinet6/in6_src.c
@@ -920,8 +920,6 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct ucred *cred)
 void
 addrsel_policy_init(void)
 {
-	ADDRSEL_LOCK_INIT();
-	ADDRSEL_SXLOCK_INIT();
 	INIT_VNET_INET6(curvnet);
 
 	V_ip6_prefer_tempaddr = 0;
@@ -931,6 +929,12 @@ addrsel_policy_init(void)
 	/* initialize the "last resort" policy */
 	bzero(&V_defaultaddrpolicy, sizeof(V_defaultaddrpolicy));
 	V_defaultaddrpolicy.label = ADDR_LABEL_NOTAPP;
+
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
+	ADDRSEL_LOCK_INIT();
+	ADDRSEL_SXLOCK_INIT();
 }
 
 static struct in6_addrpolicy *
diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
index f094683..f0bb9d5 100644
--- a/sys/netinet6/ip6_input.c
+++ b/sys/netinet6/ip6_input.c
@@ -234,6 +234,17 @@ ip6_init(void)
 					/* 40 1K datagrams */
 	V_dad_init = 0;
 
+	scope6_init();
+	addrsel_policy_init();
+	nd6_init();
+	frag6_init();
+
+	V_ip6_desync_factor = arc4random() % MAX_TEMP_DESYNC_FACTOR;
+
+	/* Skip global initialization stuff for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
 #ifdef DIAGNOSTIC
 	if (sizeof(struct protosw) != sizeof(struct ip6protosw))
 		panic("sizeof(protosw) != sizeof(ip6protosw)");
@@ -265,18 +276,13 @@ ip6_init(void)
 		printf("%s: WARNING: unable to register pfil hook, "
 			"error %d\n", __func__, i);
 
-	ip6intrq.ifq_maxlen = V_ip6qmaxlen;
+	ip6intrq.ifq_maxlen = V_ip6qmaxlen; /* XXX */
 	mtx_init(&ip6intrq.ifq_mtx, "ip6_inq", NULL, MTX_DEF);
 	netisr_register(NETISR_IPV6, ip6_input, &ip6intrq, 0);
-	scope6_init();
-	addrsel_policy_init();
-	nd6_init();
-	frag6_init();
-	V_ip6_desync_factor = arc4random() % MAX_TEMP_DESYNC_FACTOR;
 }
 
-static void
-ip6_init2(void *dummy)
+static int
+ip6_init2_vnet(const void *unused __unused)
 {
 	INIT_VNET_INET6(curvnet);
 
@@ -290,6 +296,15 @@ ip6_init2(void *dummy)
 		      (V_ip6_temp_preferred_lifetime - V_ip6_desync_factor -
 		       V_ip6_temp_regen_advance) * hz,
 		      in6_tmpaddrtimer, NULL);
+
+	return (0);
+}
+
+static void
+ip6_init2(void *dummy)
+{
+
+	ip6_init2_vnet(NULL);
 }
 
 /* cheat */
diff --git a/sys/netinet6/scope6.c b/sys/netinet6/scope6.c
index 9611deb..b866cd9 100644
--- a/sys/netinet6/scope6.c
+++ b/sys/netinet6/scope6.c
@@ -83,8 +83,12 @@ scope6_init(void)
 #else
 	V_ip6_use_defzone = 0;
 #endif
-	SCOPE6_LOCK_INIT();
 	bzero(&V_sid_default, sizeof(V_sid_default));
+
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
+	SCOPE6_LOCK_INIT();
 }
 
 struct scope6_id *