Introduce an infrastructure for dismantling vnet instances.

Vnet modules and protocol domains may now register destructor
functions to clean up and release per-module state.  The destructor
mechanisms can be triggered by invoking "vimage -d", or a future
equivalent command which will be provided via the new jail framework.

While this patch introduces numerous placeholder destructor functions,
many of those are currently incomplete, thus leaking memory or (even
worse) failing to stop all running timers.  Many of such issues are
already known and will be incrementaly fixed over the next weeks in
smaller incremental commits.

Apart from introducing new fields in structs ifnet, domain, protosw
and vnet_net, which requires the kernel and modules to be rebuilt, this
change should have no impact on nooptions VIMAGE builds, since vnet
destructors can only be called in VIMAGE kernels.  Moreover,
destructor functions should be in general compiled in only in
options VIMAGE builds, except for kernel modules which can be safely
kldunloaded at run time.

Bump __FreeBSD_version to 800097.
Reviewed by:	bz, julian
Approved by:	rwatson, kib (re), julian (mentor)

6 files changed, 103 insertions, 13 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index 1a5dcd2..35a5e0b 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -153,6 +153,9 @@ extern void	nd6_setmtu(struct ifnet *);
 #endif
 
 static int	vnet_net_iattach(const void *);
+#ifdef VIMAGE
+static int	vnet_net_idetach(const void *);
+#endif
 
 #ifdef VIMAGE_GLOBALS
 struct	ifnethead ifnet;	/* depend on static init XXX */
@@ -189,7 +192,10 @@ static const vnet_modinfo_t vnet_net_modinfo = {
 	.vmi_name	= "net",
 	.vmi_size	= sizeof(struct vnet_net),
 	.vmi_symmap	= vnet_net_symmap,
-	.vmi_iattach	= vnet_net_iattach
+	.vmi_iattach	= vnet_net_iattach,
+#ifdef VIMAGE
+	.vmi_idetach	= vnet_net_idetach
+#endif
 };
 #endif /* !VIMAGE_GLOBALS */
 
@@ -446,6 +452,22 @@ vnet_net_iattach(const void *unused __unused)
 	return (0);
 }
 
+#ifdef VIMAGE
+static int
+vnet_net_idetach(const void *unused __unused)
+{
+	INIT_VNET_NET(curvnet);
+
+	VNET_ASSERT(TAILQ_EMPTY(&V_ifnet));
+	VNET_ASSERT(TAILQ_EMPTY(&V_ifg_head));
+	VNET_ASSERT(SLIST_EMPTY(&V_ifklist.kl_list));
+
+	free((caddr_t)V_ifindex_table, M_IFNET);
+
+	return (0);
+}
+#endif
+
 void
 if_grow(void)
 {
@@ -688,6 +710,8 @@ if_attach_internal(struct ifnet *ifp, int vmove)
 
 #ifdef VIMAGE
 	ifp->if_vnet = curvnet;
+	if (ifp->if_home_vnet == NULL)
+		ifp->if_home_vnet = curvnet;
 #endif
 
 	if_addgroup(ifp, IFG_ALL);
diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c
index 69e14e9..c6f7285 100644
--- a/sys/net/if_gif.c
+++ b/sys/net/if_gif.c
@@ -303,12 +303,10 @@ gifmodevent(mod, type, data)
 		break;
 	case MOD_UNLOAD:
 		if_clone_detach(&gif_cloner);
-		mtx_destroy(&gif_mtx);
-#ifdef INET6
-#ifndef VIMAGE
-		V_ip6_gif_hlim = 0;	/* XXX -> vnet_gif_idetach() */
-#endif
+#ifdef VIMAGE
+		vnet_mod_deregister(&vnet_gif_modinfo);
 #endif
+		mtx_destroy(&gif_mtx);
 		break;
 	default:
 		return EOPNOTSUPP;
diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c
index 2bbccac..b0f22c8 100644
--- a/sys/net/if_loop.c
+++ b/sys/net/if_loop.c
@@ -105,6 +105,9 @@ int		looutput(struct ifnet *ifp, struct mbuf *m,
 static int	lo_clone_create(struct if_clone *, int, caddr_t);
 static void	lo_clone_destroy(struct ifnet *);
 static int	vnet_loif_iattach(const void *);
+#ifdef VIMAGE
+static int	vnet_loif_idetach(const void *);
+#endif
 
 #ifdef VIMAGE_GLOBALS
 struct ifnet *loif;			/* Used externally */
@@ -119,7 +122,10 @@ static const vnet_modinfo_t vnet_loif_modinfo = {
 	.vmi_id		= VNET_MOD_LOIF,
 	.vmi_dependson	= VNET_MOD_IF_CLONE,
 	.vmi_name	= "loif",
-	.vmi_iattach	= vnet_loif_iattach
+	.vmi_iattach	= vnet_loif_iattach,
+#ifdef VIMAGE
+	.vmi_idetach	= vnet_loif_idetach
+#endif
 };
 #endif /* !VIMAGE_GLOBALS */
 
@@ -128,12 +134,11 @@ IFC_SIMPLE_DECLARE(lo, 1);
 static void
 lo_clone_destroy(struct ifnet *ifp)
 {
-#ifdef INVARIANTS
-	INIT_VNET_NET(ifp->if_vnet);
-#endif
 
+#ifndef VIMAGE
 	/* XXX: destroying lo0 will lead to panics. */
 	KASSERT(V_loif != ifp, ("%s: destroying lo0", __func__));
+#endif
 
 	bpfdetach(ifp);
 	if_detach(ifp);
@@ -166,7 +171,8 @@ lo_clone_create(struct if_clone *ifc, int unit, caddr_t params)
 	return (0);
 }
 
-static int vnet_loif_iattach(const void *unused __unused)
+static int
+vnet_loif_iattach(const void *unused __unused)
 {
 	INIT_VNET_NET(curvnet);
 
@@ -175,7 +181,11 @@ static int vnet_loif_iattach(const void *unused __unused)
 #ifdef VIMAGE
 	V_lo_cloner = malloc(sizeof(*V_lo_cloner), M_LO_CLONER,
 	    M_WAITOK | M_ZERO);
+	V_lo_cloner_data = malloc(sizeof(*V_lo_cloner_data), M_LO_CLONER,
+	    M_WAITOK | M_ZERO);
 	bcopy(&lo_cloner, V_lo_cloner, sizeof(*V_lo_cloner));
+	bcopy(lo_cloner.ifc_data, V_lo_cloner_data, sizeof(*V_lo_cloner_data));
+	V_lo_cloner->ifc_data = V_lo_cloner_data;
 	if_clone_attach(V_lo_cloner);
 #else
 	if_clone_attach(&lo_cloner);
@@ -183,6 +193,21 @@ static int vnet_loif_iattach(const void *unused __unused)
 	return (0);
 }
 
+#ifdef VIMAGE
+static int
+vnet_loif_idetach(const void *unused __unused)
+{
+	INIT_VNET_NET(curvnet);
+
+	if_clone_detach(V_lo_cloner);
+	free(V_lo_cloner, M_LO_CLONER);
+	free(V_lo_cloner_data, M_LO_CLONER);
+	V_loif = NULL;
+
+	return (0);
+}
+#endif
+
 static int
 loop_modevent(module_t mod, int type, void *data)
 {
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index d811f22..2d0a0ab 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -71,6 +71,7 @@ struct	ether_header;
 struct	carp_if;
 struct  ifvlantrunk;
 struct	route;
+struct	vnet;
 #endif
 
 #include <sys/queue.h>		/* get TAILQ macros */
@@ -169,6 +170,9 @@ struct ifnet {
 		(struct ifnet *);
 	int	(*if_transmit)		/* initiate output routine */
 		(struct ifnet *, struct mbuf *);
+	void	(*if_reassign)		/* reassign to vnet routine */
+		(struct ifnet *, struct vnet *, char *);
+	struct	vnet *if_home_vnet;	/* where this ifnet originates from */
 	struct	ifaddr	*if_addr;	/* pointer to link-level address */
 	void	*if_llsoftc;		/* link layer softc */
 	int	if_drv_flags;		/* driver-managed status flags */
diff --git a/sys/net/route.c b/sys/net/route.c
index 8705a54..aaa38d7 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -99,12 +99,18 @@ static int	rttrash;		/* routes not in table but not freed */
 static void rt_maskedcopy(struct sockaddr *,
 	    struct sockaddr *, struct sockaddr *);
 static int vnet_route_iattach(const void *);
+#ifdef VIMAGE
+static int vnet_route_idetach(const void *);
+#endif
 
 #ifndef VIMAGE_GLOBALS
 static const vnet_modinfo_t vnet_rtable_modinfo = {
 	.vmi_id		= VNET_MOD_RTABLE,
 	.vmi_name	= "rtable",
-	.vmi_iattach	= vnet_route_iattach
+	.vmi_iattach	= vnet_route_iattach,
+#ifdef VIMAGE
+	.vmi_idetach	= vnet_route_idetach
+#endif
 };
 #endif /* !VIMAGE_GLOBALS */
 
@@ -194,7 +200,8 @@ route_init(void)
 #endif
 }
 
-static int vnet_route_iattach(const void *unused __unused)
+static int
+vnet_route_iattach(const void *unused __unused)
 {
 	INIT_VNET_NET(curvnet);
 	struct domain *dom;
@@ -235,6 +242,36 @@ static int vnet_route_iattach(const void *unused __unused)
 	return (0);
 }
 
+#ifdef VIMAGE
+static int
+vnet_route_idetach(const void *unused __unused)
+{
+	int table;
+	int fam;
+	struct domain *dom;
+	struct radix_node_head **rnh;
+
+	for (dom = domains; dom; dom = dom->dom_next) {
+		if (dom->dom_rtdetach) {
+			for (table = 0; table < rt_numfibs; table++) {
+				if ( (fam = dom->dom_family) == AF_INET ||
+				    table == 0) {
+					/* For now only AF_INET has > 1 tbl. */
+					rnh = rt_tables_get_rnh_ptr(table, fam);
+					if (rnh == NULL)
+						panic("%s: rnh NULL", __func__);
+					dom->dom_rtdetach((void **)rnh,
+					    dom->dom_rtoffset);
+				} else {
+					break;
+				}
+			}
+		}
+	}
+	return (0);
+}
+#endif
+
 #ifndef _SYS_SYSPROTO_H_
 struct setfib_args {
 	int     fibnum;
diff --git a/sys/net/vnet.h b/sys/net/vnet.h
index d36c303..696e472 100644
--- a/sys/net/vnet.h
+++ b/sys/net/vnet.h
@@ -51,6 +51,7 @@ struct vnet_net {
 
 	struct ifnet *		_loif;
 	struct if_clone *	_lo_cloner;
+	struct ifc_simple_data *_lo_cloner_data;
 
 	LIST_HEAD(, rawcb)	_rawcb_list;
 
@@ -87,6 +88,7 @@ extern struct vnet_net vnet_net_0;
 #define	V_ifklist		VNET_NET(ifklist)
 #define	V_ifnet			VNET_NET(ifnet)
 #define	V_lo_cloner		VNET_NET(lo_cloner)
+#define	V_lo_cloner_data	VNET_NET(lo_cloner_data)
 #define	V_loif			VNET_NET(loif)
 #define	V_rawcb_list		VNET_NET(rawcb_list)
 #define	V_rt_tables		VNET_NET(rt_tables)