Introduce vnet module registration / initialization framework with

dependency tracking and ordering enforcement.

With this change, per-vnet initialization functions introduced with
r190787 are no longer directly called from traditional initialization
functions (which cc in most cases inlined to pre-r190787 code), but are
instead registered via the vnet framework first, and are invoked only
after all prerequisite modules have been initialized.  In the long run,
this framework should allow us to both initialize and dismantle
multiple vnet instances in a correct order.

The problem this change aims to solve is how to replay the
initialization sequence of various network stack components, which
have been traditionally triggered via different mechanisms (SYSINIT,
protosw).  Note that this initialization sequence was and still can be
subtly different depending on whether certain pieces of code have been
statically compiled into the kernel, loaded as modules by boot
loader, or kldloaded at run time.

The approach is simple - we record the initialization sequence
established by the traditional mechanisms whenever vnet_mod_register()
is called for a particular vnet module.  The vnet_mod_register_multi()
variant allows a single initializer function to be registered multiple
times but with different arguments - currently this is only used in
kern/uipc_domain.c by net_add_domain() with different struct domain *
as arguments, which allows for protosw-registered initialization
routines to be invoked in a correct order by the new vnet
initialization framework.

For the purpose of identifying vnet modules, each vnet module has to
have a unique ID, which is statically assigned in sys/vimage.h.
Dynamic assignment of vnet module IDs is not supported yet.

A vnet module may specify a single prerequisite module at registration
time by filling in the vmi_dependson field of its vnet_modinfo struct
with the ID of the module it depends on.  Unless specified otherwise,
all vnet modules depend on VNET_MOD_NET (container for ifnet list head,
rt_tables etc.), which thus has to and will always be initialized
first.  The framework will panic if it detects any unresolved
dependencies before completing system initialization.  Detection of
unresolved dependencies for vnet modules registered after boot
(kldloaded modules) is not provided.

Note that the fact that each module can specify only a single
prerequisite may become problematic in the long run.  In particular,
INET6 depends on INET being already instantiated, due to TCP / UDP
structures residing in INET container.  IPSEC also depends on INET,
which will in turn additionally complicate making INET6-only kernel
configs a reality.

The entire registration framework can be compiled out by turning on the
VIMAGE_GLOBALS kernel config option.

Reviewed by:	bz
Approved by:	julian (mentor)

16 files changed, 390 insertions, 28 deletions

diff --git a/sys/kern/kern_vimage.c b/sys/kern/kern_vimage.c
index 0cba35a..c426ee6 100644
--- a/sys/kern/kern_vimage.c
+++ b/sys/kern/kern_vimage.c
@@ -44,19 +44,136 @@ __FBSDID("$FreeBSD$");
 MALLOC_DEFINE(M_VIMAGE, "vimage", "vimage resource container");
 
 static TAILQ_HEAD(vnet_modlink_head, vnet_modlink) vnet_modlink_head;
+static TAILQ_HEAD(vnet_modpending_head, vnet_modlink) vnet_modpending_head;
+static void vnet_mod_complete_registration(struct vnet_modlink *);
+static int vnet_mod_constructor(struct vnet_modlink *);
 
 void
 vnet_mod_register(const struct vnet_modinfo *vmi)
 {
+
+	vnet_mod_register_multi(vmi, NULL, NULL);
+}
+
+void
+vnet_mod_register_multi(const struct vnet_modinfo *vmi, void *iarg,
+    char *iname)
+{
 	struct vnet_modlink *vml, *vml_iter;
 	
-	/* Do not register the same module instance more than once. */
+	/* Do not register the same {module, iarg} pair more than once. */
 	TAILQ_FOREACH(vml_iter, &vnet_modlink_head, vml_mod_le)
-		if (vml_iter->vml_modinfo == vmi)
-			panic("%s: %s", __func__, vmi->vmi_name);
+		if (vml_iter->vml_modinfo == vmi && vml_iter->vml_iarg == iarg)
+			break;
+	if (vml_iter != NULL)
+		panic("registering an already registered vnet module: %s",
+		    vml_iter->vml_modinfo->vmi_name);
 	vml = malloc(sizeof(struct vnet_modlink), M_VIMAGE, M_NOWAIT);
+
+	/*
+	 * XXX we support only statically assigned module IDs at the time.
+	 * In principle modules should be able to get a dynamically
+	 * assigned ID at registration time.
+	 *
+	 * If a module is registered in multiple instances, then each
+	 * instance must have both iarg and iname set.
+	 */
+	if (vmi->vmi_id >= VNET_MOD_MAX)
+		panic("invalid vnet module ID: %d", vmi->vmi_id);
+	if (vmi->vmi_name == NULL)
+		panic("vnet module with no name: %d", vmi->vmi_id);
+	if ((iarg == NULL) ^ (iname == NULL))
+		panic("invalid vnet module instance: %s", vmi->vmi_name);
+
 	vml->vml_modinfo = vmi;
+	vml->vml_iarg = iarg;
+	vml->vml_iname = iname;
+
+	/* Check whether the module we depend on is already registered. */
+	if (vmi->vmi_dependson != vmi->vmi_id) {
+		TAILQ_FOREACH(vml_iter, &vnet_modlink_head, vml_mod_le)
+			if (vml_iter->vml_modinfo->vmi_id ==
+			    vmi->vmi_dependson)
+				break;	/* Depencency found, we are done. */
+		if (vml_iter == NULL) {
+#ifdef DEBUG_ORDERING
+			printf("dependency %d missing for vnet mod %s,"
+			    "postponing registration\n",
+			    vmi->vmi_dependson, vmi->vmi_name);
+#endif /* DEBUG_ORDERING */
+			TAILQ_INSERT_TAIL(&vnet_modpending_head, vml,
+			    vml_mod_le);
+			return;
+		}
+	}
+
+	vnet_mod_complete_registration(vml);
+}
+	
+void
+vnet_mod_complete_registration(struct vnet_modlink *vml)
+{
+	VNET_ITERATOR_DECL(vnet_iter);
+	struct vnet_modlink *vml_iter;
+
 	TAILQ_INSERT_TAIL(&vnet_modlink_head, vml, vml_mod_le);
+
+	VNET_FOREACH(vnet_iter) {
+		CURVNET_SET_QUIET(vnet_iter);
+		vnet_mod_constructor(vml);
+		CURVNET_RESTORE();
+	}
+
+	/* Check for pending modules depending on us. */
+	do {
+		TAILQ_FOREACH(vml_iter, &vnet_modpending_head, vml_mod_le)
+			if (vml_iter->vml_modinfo->vmi_dependson ==
+			    vml->vml_modinfo->vmi_id)
+				break;
+		if (vml_iter != NULL) {
+#ifdef DEBUG_ORDERING
+			printf("vnet mod %s now registering,"
+			    "dependency %d loaded\n",
+			    vml_iter->vml_modinfo->vmi_name,
+			    vml->vml_modinfo->vmi_id);
+#endif /* DEBUG_ORDERING */
+			TAILQ_REMOVE(&vnet_modpending_head, vml_iter,
+			    vml_mod_le);
+			vnet_mod_complete_registration(vml_iter);
+		}
+	} while (vml_iter != NULL);
+}
+
+static int vnet_mod_constructor(struct vnet_modlink *vml)
+{
+	const struct vnet_modinfo *vmi = vml->vml_modinfo;
+
+#ifdef DEBUG_ORDERING
+	printf("instantiating vnet_%s", vmi->vmi_name);
+	if (vml->vml_iarg)
+		printf("/%s", vml->vml_iname);
+	printf(": ");
+	if (vmi->vmi_struct_size)
+		printf("malloc(%zu); ", vmi->vmi_struct_size);
+	if (vmi->vmi_iattach != NULL)
+		printf("iattach()");
+	printf("\n");
+#endif
+
+#ifdef VIMAGE
+	if (vmi->vmi_struct_size) {
+		void *mem = malloc(vmi->vmi_struct_size, M_VNET,
+		    M_NOWAIT | M_ZERO);
+		if (mem == NULL) /* XXX should return error, not panic. */
+			panic("vi_alloc: malloc for %s\n", vmi->vmi_name);
+		curvnet->mod_data[vmi->vmi_id] = mem;
+	}
+#endif
+
+	if (vmi->vmi_iattach != NULL)
+		vmi->vmi_iattach(vml->vml_iarg);
+
+	return (0);
 }
 
 /*
@@ -75,7 +192,7 @@ vi_symlookup(struct kld_sym_lookup *lookup, char *symstr)
 		if (vml->vml_modinfo->vmi_symmap == NULL)
 			continue;
 		for (mapentry = vml->vml_modinfo->vmi_symmap;
-		     mapentry->name != NULL; mapentry++) {
+		    mapentry->name != NULL; mapentry++) {
 			if (strcmp(symstr, mapentry->name) == 0) {
 				lookup->symvalue = (u_long) mapentry->base;
 				lookup->symsize = mapentry->size;
@@ -91,8 +208,27 @@ vi_init(void *unused)
 {
 
 	TAILQ_INIT(&vnet_modlink_head);
+	TAILQ_INIT(&vnet_modpending_head);
+}
+
+static void
+vi_init_done(void *unused)
+{
+	struct vnet_modlink *vml_iter;
+
+	if (TAILQ_EMPTY(&vnet_modpending_head))
+		return;
+
+	printf("vnet modules with unresolved dependencies:\n");
+	TAILQ_FOREACH(vml_iter, &vnet_modpending_head, vml_mod_le)
+		printf("    %d:%s depending on %d\n",
+		    vml_iter->vml_modinfo->vmi_id,
+		    vml_iter->vml_modinfo->vmi_name,
+		    vml_iter->vml_modinfo->vmi_dependson);
+	panic("going nowhere without my vnet modules!");
 }
 
 SYSINIT(vimage, SI_SUB_VIMAGE, SI_ORDER_FIRST, vi_init, NULL);
+SYSINIT(vimage_done, SI_SUB_VIMAGE_DONE, SI_ORDER_FIRST, vi_init_done, NULL);
 
 #endif /* !VIMAGE_GLOBALS */
diff --git a/sys/kern/uipc_domain.c b/sys/kern/uipc_domain.c
index 9c9b05f..df35bcb 100644
--- a/sys/kern/uipc_domain.c
+++ b/sys/kern/uipc_domain.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/socketvar.h>
 #include <sys/systm.h>
+#include <sys/vimage.h>
 #include <vm/uma.h>
 
 /*
@@ -64,6 +65,8 @@ static void domainfinalize(void *);
 SYSINIT(domainfin, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST, domainfinalize,
     NULL);
 
+static vnet_attach_fn net_init_domain;
+
 static struct callout pffast_callout;
 static struct callout pfslow_callout;
 
@@ -100,6 +103,14 @@ struct pr_usrreqs nousrreqs = {
 	.pru_sopoll =		pru_sopoll_notsupp,
 };
 
+#ifndef VIMAGE_GLOBALS
+vnet_modinfo_t vnet_domain_modinfo = {
+	.vmi_id		= VNET_MOD_DOMAIN,
+	.vmi_name	= "domain",
+	.vmi_iattach	= net_init_domain
+};
+#endif
+
 static void
 protosw_init(struct protosw *pr)
 {
@@ -159,9 +170,10 @@ protosw_init(struct protosw *pr)
  * Note: you cant unload it again because a socket may be using it.
  * XXX can't fail at this time.
  */
-static void
-net_init_domain(struct domain *dp)
+static int
+net_init_domain(const void *arg)
 {
+	const struct domain *dp = arg;
 	struct protosw *pr;
 
 	if (dp->dom_init)
@@ -175,6 +187,7 @@ net_init_domain(struct domain *dp)
 	max_datalen = MHLEN - max_hdr;
 	if (max_datalen < 1)
 		panic("%s: max_datalen < 1", __func__);
+	return (0);
 }
 
 /*
@@ -210,7 +223,11 @@ net_add_domain(void *data)
 		    "domainfinalize()\n", dp->dom_name);
 #endif
 	mtx_unlock(&dom_mtx);
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register_multi(&vnet_domain_modinfo, dp, dp->dom_name);
+#else
 	net_init_domain(dp);
+#endif
 }
 
 static void
diff --git a/sys/net/if.c b/sys/net/if.c
index 0f76eb1..da3c041 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -183,9 +183,13 @@ static struct vnet_symmap vnet_net_symmap[] = {
 	VNET_SYMMAP_END
 };
 
-VNET_MOD_DECLARE(NET, net, vnet_net_iattach, vnet_net_idetach,
-    NONE, vnet_net_symmap)
-#endif
+static const vnet_modinfo_t vnet_net_modinfo = {
+	.vmi_id		= VNET_MOD_NET,
+	.vmi_name	= "net",
+	.vmi_symmap	= vnet_net_symmap,
+	.vmi_iattach	= vnet_net_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
 
 /*
  * System initialization
diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c
index 4fbbb2d..1c10f17 100644
--- a/sys/net/if_gif.c
+++ b/sys/net/if_gif.c
@@ -123,6 +123,15 @@ static int	gif_clone_create(struct if_clone *, int, caddr_t);
 static void	gif_clone_destroy(struct ifnet *);
 static int	vnet_gif_iattach(const void *);
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_gif_modinfo = {
+	.vmi_id		= VNET_MOD_GIF,
+	.vmi_name	= "gif",
+	.vmi_dependson	= VNET_MOD_NET,
+	.vmi_iattach	= vnet_gif_iattach
+};
+#endif
+
 IFC_SIMPLE_DECLARE(gif, 0);
 
 static int gifmodevent(module_t, int, void *);
@@ -282,7 +291,11 @@ gifmodevent(mod, type, data)
 	case MOD_LOAD:
 		mtx_init(&gif_mtx, "gif_mtx", NULL, MTX_DEF);
 
+#ifndef VIMAGE_GLOBALS
+		vnet_mod_register(&vnet_gif_modinfo);
+#else
 		vnet_gif_iattach(NULL);
+#endif
 		if_clone_attach(&gif_cloner);
 
 		break;
diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c
index 5748311..c31acef 100644
--- a/sys/net/if_loop.c
+++ b/sys/net/if_loop.c
@@ -111,6 +111,14 @@ static int	vnet_loif_iattach(const void *);
 struct ifnet *loif;			/* Used externally */
 #endif
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_loif_modinfo = {
+	.vmi_id		= VNET_MOD_LOIF,
+	.vmi_name	= "loif",
+	.vmi_iattach	= vnet_loif_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 IFC_SIMPLE_DECLARE(lo, 1);
 
 static void
@@ -170,7 +178,11 @@ loop_modevent(module_t mod, int type, void *data)
 
 	switch (type) {
 	case MOD_LOAD:
+#ifndef VIMAGE_GLOBALS
+		vnet_mod_register(&vnet_loif_modinfo);
+#else
 		vnet_loif_iattach(NULL);
+#endif
 		break;
 
 	case MOD_UNLOAD:
diff --git a/sys/net/route.c b/sys/net/route.c
index a1a3d8c..f1e13ad 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -108,6 +108,14 @@ static void rt_maskedcopy(struct sockaddr *,
 	    struct sockaddr *, struct sockaddr *);
 static int vnet_route_iattach(const void *);
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_rtable_modinfo = {
+	.vmi_id		= VNET_MOD_RTABLE,
+	.vmi_name	= "rtable",
+	.vmi_iattach	= vnet_route_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 /* compare two sockaddr structures */
 #define	sa_equal(a1, a2) (bcmp((a1), (a2), (a1)->sa_len) == 0)
 
@@ -161,7 +169,11 @@ route_init(void)
 		rt_numfibs = 1;
 	rn_init();	/* initialize all zeroes, all ones, mask table */
 
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_rtable_modinfo);
+#else
 	vnet_route_iattach(NULL);
+#endif
 }
 
 static int vnet_route_iattach(const void *unused __unused)
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index 3a44f5c..4aae7d3 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -120,6 +120,15 @@ static void	arptimer(void *);
 static void	in_arpinput(struct mbuf *);
 #endif
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_arp_modinfo = {
+	.vmi_id		= VNET_MOD_ARP,
+	.vmi_name	= "arp",
+	.vmi_dependson	= VNET_MOD_INET,
+	.vmi_iattach	= arp_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 #ifdef AF_INET
 void arp_ifscrub(struct ifnet *ifp, uint32_t addr);
 
@@ -808,7 +817,11 @@ static void
 arp_init(void)
 {
 
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_arp_modinfo);
+#else
 	arp_iattach(NULL);
+#endif
 
 	arpintrq.ifq_maxlen = 50;
 	mtx_init(&arpintrq.ifq_mtx, "arp_inq", NULL, MTX_DEF);
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index db53a23..70c85bf 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -224,6 +224,23 @@ int fw_one_pass;
 
 static void	ip_freef(struct ipqhead *, struct ipq *);
 
+#ifndef VIMAGE_GLOBALS
+static void vnet_inet_register(void);
+ 
+static const vnet_modinfo_t vnet_inet_modinfo = {
+	.vmi_id		= VNET_MOD_INET,
+	.vmi_name	= "inet",
+};
+ 
+static void vnet_inet_register()
+{
+  
+	vnet_mod_register(&vnet_inet_modinfo);
+}
+ 
+SYSINIT(inet, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, vnet_inet_register, 0);
+#endif
+
 /*
  * IP initialization: fill in IP protocol switch table.
  * All protocols not implemented in kernel go to raw IP protocol handler.
diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
index f0bb9d5..27c8e89 100644
--- a/sys/netinet6/ip6_input.c
+++ b/sys/netinet6/ip6_input.c
@@ -155,6 +155,25 @@ static int ip6_hopopts_input(u_int32_t *, u_int32_t *, struct mbuf **, int *);
 static struct mbuf *ip6_pullexthdr(struct mbuf *, size_t, int);
 #endif
 
+#ifndef VIMAGE_GLOBALS
+static void vnet_inet6_register(void);
+ 
+static const vnet_modinfo_t vnet_inet6_modinfo = {
+	.vmi_id		= VNET_MOD_INET6,
+	.vmi_name	= "inet6",
+	.vmi_dependson	= VNET_MOD_INET	/* XXX revisit - TCP/UDP needs this? */
+};
+ 
+static void
+vnet_inet6_register(void)
+{
+
+	vnet_mod_register(&vnet_inet6_modinfo);
+}
+ 
+SYSINIT(inet6, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, vnet_inet6_register, 0);
+#endif
+
 /*
  * IP6 initialization: fill in IP6 protocol switch table.
  * All protocols not implemented in kernel go to raw IP6 protocol handler.
diff --git a/sys/netipsec/ipsec.c b/sys/netipsec/ipsec.c
index 85d2897..a76afd2 100644
--- a/sys/netipsec/ipsec.c
+++ b/sys/netipsec/ipsec.c
@@ -244,6 +244,15 @@ static void vshiftl __P((unsigned char *, int, int));
 
 MALLOC_DEFINE(M_IPSEC_INPCB, "inpcbpolicy", "inpcb-resident ipsec policy");
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_ipsec_modinfo = {
+	.vmi_id		= VNET_MOD_IPSEC,
+	.vmi_name	= "ipsec",
+	.vmi_dependson	= VNET_MOD_INET,	/* XXX revisit - INET6 ? */
+	.vmi_iattach	= ipsec_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 void
 ipsec_init(void)
 {
@@ -1760,7 +1769,12 @@ static void
 ipsec_attach(void)
 {
 
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_ipsec_modinfo);
+#else
 	ipsec_iattach(NULL);
+#endif
+
 }
 
 static int
diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c
index 365ac13..07d7001 100644
--- a/sys/netipsec/xform_ah.c
+++ b/sys/netipsec/xform_ah.c
@@ -75,6 +75,15 @@
 
 static int ah_iattach(const void *);
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_ah_modinfo = {
+	.vmi_id		= VNET_MOD_AH,
+	.vmi_name	= "ipsec_ah",
+	.vmi_dependson	= VNET_MOD_IPSEC,
+	.vmi_iattach	= ah_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 /*
  * Return header size in bytes.  The old protocol did not support
  * the replay counter; the new protocol always includes the counter.
@@ -1223,7 +1232,11 @@ ah_attach(void)
 {
 
 	xform_register(&ah_xformsw);
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_ah_modinfo);
+#else
 	ah_iattach(NULL);
+#endif
 }
 
 static int
diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c
index 46ab8d8..6508c14 100644
--- a/sys/netipsec/xform_esp.c
+++ b/sys/netipsec/xform_esp.c
@@ -92,6 +92,15 @@ static int esp_input_cb(struct cryptop *op);
 static int esp_output_cb(struct cryptop *crp);
 static int esp_iattach(const void *);
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_esp_modinfo = {
+	.vmi_id		= VNET_MOD_ESP,
+	.vmi_name	= "ipsec_esp",
+	.vmi_dependson	= VNET_MOD_IPSEC,
+	.vmi_iattach	= esp_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 /*
  * NB: this is public for use by the PF_KEY support.
  * NB: if you add support here; be sure to add code to esp_attach below!
@@ -993,7 +1002,11 @@ esp_attach(void)
 {
 
 	xform_register(&esp_xformsw);
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_esp_modinfo);
+#else
 	esp_iattach(NULL);
+#endif
 }
 
 static int
diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c
index c4f0591..8e2f1c4 100644
--- a/sys/netipsec/xform_ipcomp.c
+++ b/sys/netipsec/xform_ipcomp.c
@@ -82,6 +82,15 @@ static int ipcomp_input_cb(struct cryptop *crp);
 static int ipcomp_output_cb(struct cryptop *crp);
 static int ipcomp_iattach(const void *);
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_ipcomp_modinfo = {
+	.vmi_id		= VNET_MOD_IPCOMP,
+	.vmi_name	= "ipsec_ipcomp",
+	.vmi_dependson	= VNET_MOD_IPSEC,
+	.vmi_iattach	= ipcomp_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 struct comp_algo *
 ipcomp_algorithm_lookup(int alg)
 {
@@ -602,7 +611,11 @@ ipcomp_attach(void)
 {
 
 	xform_register(&ipcomp_xformsw);
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_ipcomp_modinfo);
+#else
 	ipcomp_iattach(NULL);
+#endif
 }
 
 static int
diff --git a/sys/netipsec/xform_ipip.c b/sys/netipsec/xform_ipip.c
index f7949ec..4d97168 100644
--- a/sys/netipsec/xform_ipip.c
+++ b/sys/netipsec/xform_ipip.c
@@ -108,6 +108,16 @@ SYSCTL_V_STRUCT(V_NET, vnet_ipsec, _net_inet_ipip, IPSECCTL_STATS,
 #define	M_IPSEC	(M_AUTHIPHDR|M_AUTHIPDGM|M_DECRYPTED)
 
 static void _ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp);
+static int ipe4_iattach(const void *);
+
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_ipip_modinfo = {
+	.vmi_id		= VNET_MOD_IPIP,
+	.vmi_name	= "ipsec_ipip",
+	.vmi_dependson	= VNET_MOD_IPSEC,
+	.vmi_iattach	= ipe4_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
 
 #ifdef INET6
 /*
@@ -719,7 +729,11 @@ ipe4_attach(void)
 	(void) encap_attach_func(AF_INET6, -1,
 		ipe4_encapcheck, (struct protosw *)&ipe6_protosw, NULL);
 #endif
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_ipip_modinfo);
+#else
 	ipe4_iattach(NULL);
+#endif
 }
 SYSINIT(ipe4_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, ipe4_attach, NULL);
 #endif	/* IPSEC */
diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h
index 16fab6c..fa397c4 100644
--- a/sys/sys/kernel.h
+++ b/sys/sys/kernel.h
@@ -163,6 +163,7 @@ enum sysinit_sub_id {
 	SI_SUB_SWAP		= 0xc000000,	/* swap */
 	SI_SUB_INTRINSIC_POST	= 0xd000000,	/* proc 0 cleanup*/
 	SI_SUB_SYSCALLS		= 0xd800000,	/* register system calls */
+	SI_SUB_VIMAGE_DONE	= 0xdc00000,	/* vnet registration complete */
 	SI_SUB_KTHREAD_INIT	= 0xe000000,	/* init process*/
 	SI_SUB_KTHREAD_PAGE	= 0xe400000,	/* pageout daemon*/
 	SI_SUB_KTHREAD_VM	= 0xe800000,	/* vm daemon*/
diff --git a/sys/sys/vimage.h b/sys/sys/vimage.h
index b7aec29..7781b47 100644
--- a/sys/sys/vimage.h
+++ b/sys/sys/vimage.h
@@ -35,6 +35,15 @@
 
 #include <sys/queue.h>
 
+#if defined(VIMAGE) && defined(VIMAGE_GLOBALS)
+#error "You cannot have both option VIMAGE and option VIMAGE_GLOBALS!"
+#endif
+
+typedef int vnet_attach_fn(const void *);
+typedef int vnet_detach_fn(const void *);
+
+#ifndef VIMAGE_GLOBALS
+
 struct kld_sym_lookup;
 
 struct vnet_symmap {
@@ -42,27 +51,78 @@ struct vnet_symmap {
 	void	*base;
 	size_t	size;
 };
+typedef struct vnet_symmap vnet_symmap_t;
 
 struct vnet_modinfo {
+	u_int				 vmi_id;
+	u_int				 vmi_dependson;
 	char				*vmi_name;
+	vnet_attach_fn			*vmi_iattach;
+	vnet_detach_fn			*vmi_idetach;
+	size_t				 vmi_struct_size;
 	struct vnet_symmap		*vmi_symmap;
 };
+typedef struct vnet_modinfo vnet_modinfo_t;
 
 struct vnet_modlink {
-	TAILQ_ENTRY(vnet_modlink)	vml_mod_le;
+	TAILQ_ENTRY(vnet_modlink)	 vml_mod_le;
 	const struct vnet_modinfo	*vml_modinfo;
+	const void			*vml_iarg;
+	const char			*vml_iname;
 };
 
-#define VNET_MOD_DECLARE(m_name_uc, m_name_lc, m_iattach, m_idetach, 	\
-    m_dependson, m_symmap)						\
-	static const struct vnet_modinfo vnet_##m_name_lc##_modinfo = {	\
-		.vmi_name		= #m_name_lc,			\
-		.vmi_symmap		= m_symmap			\
-};
+#define	VNET_SYMMAP(mod, name)						\
+	{ #name, &(vnet_ ## mod ## _0._ ## name),			\
+	sizeof(vnet_ ## mod ## _0._ ## name) }
 
-#if defined(VIMAGE) && defined(VIMAGE_GLOBALS)
-#error "You cannot have both option VIMAGE and option VIMAGE_GLOBALS!"
-#endif
+#define	VNET_SYMMAP_END		{ NULL, 0 }
+
+/* stateful modules */
+#define	VNET_MOD_NET		 0	/* MUST be 0 - implicit dependency */
+#define	VNET_MOD_NETGRAPH	 1
+#define	VNET_MOD_INET		 2
+#define	VNET_MOD_INET6		 3
+#define	VNET_MOD_IPSEC		 4
+#define	VNET_MOD_IPFW		 5
+#define	VNET_MOD_DUMMYNET	 6
+#define	VNET_MOD_PF		 7
+#define	VNET_MOD_ALTQ		 8
+#define	VNET_MOD_IPX		 9
+#define	VNET_MOD_ATALK		10
+#define	VNET_MOD_ACCF_HTTP	11
+#define	VNET_MOD_IGMP		12
+
+/* stateless modules */
+#define	VNET_MOD_NG_ETHER	20
+#define	VNET_MOD_NG_IFACE	21
+#define	VNET_MOD_NG_EIFACE	22
+#define	VNET_MOD_ESP		23
+#define	VNET_MOD_IPIP		24
+#define	VNET_MOD_AH		25
+#define	VNET_MOD_IPCOMP	 	26	
+#define	VNET_MOD_GIF		27
+#define	VNET_MOD_ARP		28
+#define	VNET_MOD_RTABLE		29
+#define	VNET_MOD_LOIF		30
+#define	VNET_MOD_DOMAIN		31
+#define	VNET_MOD_DYNAMIC_START	32
+#define	VNET_MOD_MAX		64
+
+/* Sysctl virtualization macros need these name mappings bellow */
+#define	V_MOD_vnet_net		VNET_MOD_NET
+#define	V_MOD_vnet_netgraph	VNET_MOD_NETGRAPH
+#define	V_MOD_vnet_inet		VNET_MOD_INET
+#define	V_MOD_vnet_inet6	VNET_MOD_INET6
+#define	V_MOD_vnet_ipfw		VNET_MOD_IPFW
+#define	V_MOD_vnet_pf		VNET_MOD_PF
+#define	V_MOD_vnet_gif		VNET_MOD_GIF
+#define	V_MOD_vnet_ipsec	VNET_MOD_IPSEC
+
+int	vi_symlookup(struct kld_sym_lookup *, char *);
+void	vnet_mod_register(const struct vnet_modinfo *);
+void	vnet_mod_register_multi(const struct vnet_modinfo *, void *, char *);
+
+#endif /* !VIMAGE_GLOBALS */
 
 #ifdef VIMAGE_GLOBALS
 #define	VSYM(base, sym) (sym)
@@ -74,12 +134,6 @@ struct vnet_modlink {
 #endif
 #endif
 
-#define VNET_SYMMAP(mod, name)						\
-	{ #name, &(vnet_ ## mod ## _0._ ## name),			\
-	sizeof(vnet_ ## mod ## _0._ ## name) }
-
-#define VNET_SYMMAP_END		{ NULL, 0 }
-
 /* Non-VIMAGE null-macros */
 #define	IS_DEFAULT_VNET(arg) 1
 #define	CURVNET_SET(arg)
@@ -110,9 +164,6 @@ struct vnet_modlink {
 #define	G_hostname		VPROCG(hostname) /* global hostname */
 #define	V_domainname		VPROCG(domainname)
 
-int	vi_symlookup(struct kld_sym_lookup *, char *);
-void	vnet_mod_register(const struct vnet_modinfo *);
-
 /*
  * Size-guards for the vimage structures.
  * If you need to update the values you MUST increment __FreeBSD_version.