Virtualize the pfil hooks so that different jails may chose different

packet filters. ALso allows ipfw to be enabled on on ejail and disabled
on another. In 8.0 it's a global setting.

Sitting aroung in tree waiting to commit for: 2 months
MFC after:	2 months

7 files changed, 107 insertions, 84 deletions

diff --git a/sys/netinet/ip_fastfwd.c b/sys/netinet/ip_fastfwd.c
index 78b6d30..0399393 100644
--- a/sys/netinet/ip_fastfwd.c
+++ b/sys/netinet/ip_fastfwd.c
@@ -351,10 +351,11 @@ ip_fastforward(struct mbuf *m)
 	/*
 	 * Run through list of ipfilter hooks for input packets
 	 */
-	if (!PFIL_HOOKED(&inet_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet_pfil_hook))
 		goto passin;
 
-	if (pfil_run_hooks(&inet_pfil_hook, &m, m->m_pkthdr.rcvif, PFIL_IN, NULL) ||
+	if (pfil_run_hooks(
+	    &V_inet_pfil_hook, &m, m->m_pkthdr.rcvif, PFIL_IN, NULL) ||
 	    m == NULL)
 		goto drop;
 
@@ -438,10 +439,10 @@ passin:
 	/*
 	 * Run through list of hooks for output packets.
 	 */
-	if (!PFIL_HOOKED(&inet_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet_pfil_hook))
 		goto passout;
 
-	if (pfil_run_hooks(&inet_pfil_hook, &m, ifp, PFIL_OUT, NULL) || m == NULL) {
+	if (pfil_run_hooks(&V_inet_pfil_hook, &m, ifp, PFIL_OUT, NULL) || m == NULL) {
 		goto drop;
 	}
 
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index 7886fa7..50b66b5 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -170,7 +170,7 @@ SYSCTL_VNET_INT(_net_inet_ip, OID_AUTO, check_interface, CTLFLAG_RW,
     &VNET_NAME(ip_checkinterface), 0,
     "Verify packet arrives on correct interface");
 
-struct pfil_head inet_pfil_hook;	/* Packet filter hooks */
+VNET_DEFINE(struct pfil_head, inet_pfil_hook);	/* Packet filter hooks */
 
 static struct netisr_handler ip_nh = {
 	.nh_name = "ip",
@@ -318,6 +318,13 @@ ip_init(void)
 	    NULL, UMA_ALIGN_PTR, 0);
 	maxnipq_update();
 
+	/* Initialize packet filter hooks. */
+	V_inet_pfil_hook.ph_type = PFIL_TYPE_AF;
+	V_inet_pfil_hook.ph_af = AF_INET;
+	if ((i = pfil_head_register(&V_inet_pfil_hook)) != 0)
+		printf("%s: WARNING: unable to register pfil hook, "
+			"error %d\n", __func__, i);
+
 #ifdef FLOWTABLE
 	TUNABLE_INT_FETCH("net.inet.ip.output_flowtable_size",
 	    &V_ip_output_flowtable_size);
@@ -348,13 +355,6 @@ ip_init(void)
 				ip_protox[pr->pr_protocol] = pr - inetsw;
 		}
 
-	/* Initialize packet filter hooks. */
-	inet_pfil_hook.ph_type = PFIL_TYPE_AF;
-	inet_pfil_hook.ph_af = AF_INET;
-	if ((i = pfil_head_register(&inet_pfil_hook)) != 0)
-		printf("%s: WARNING: unable to register pfil hook, "
-			"error %d\n", __func__, i);
-
 	/* Start ipport_tick. */
 	callout_init(&ipport_tick_callout, CALLOUT_MPSAFE);
 	callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
@@ -510,11 +510,11 @@ tooshort:
 	 */
 
 	/* Jump over all PFIL processing if hooks are not active. */
-	if (!PFIL_HOOKED(&inet_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet_pfil_hook))
 		goto passin;
 
 	odst = ip->ip_dst;
-	if (pfil_run_hooks(&inet_pfil_hook, &m, ifp, PFIL_IN, NULL) != 0)
+	if (pfil_run_hooks(&V_inet_pfil_hook, &m, ifp, PFIL_IN, NULL) != 0)
 		return;
 	if (m == NULL)			/* consumed by filter */
 		return;
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
index e222cda..b5be6fd 100644
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -489,12 +489,12 @@ sendit:
 #endif /* IPSEC */
 
 	/* Jump over all PFIL processing if hooks are not active. */
-	if (!PFIL_HOOKED(&inet_pfil_hook))
+	if (!PFIL_HOOKED(&V_inet_pfil_hook))
 		goto passout;
 
 	/* Run through list of hooks for output packets. */
 	odst.s_addr = ip->ip_dst.s_addr;
-	error = pfil_run_hooks(&inet_pfil_hook, &m, ifp, PFIL_OUT, inp);
+	error = pfil_run_hooks(&V_inet_pfil_hook, &m, ifp, PFIL_OUT, inp);
 	if (error != 0 || m == NULL)
 		goto done;
 
diff --git a/sys/netinet/ip_var.h b/sys/netinet/ip_var.h
index 448ba3d..a1d2166 100644
--- a/sys/netinet/ip_var.h
+++ b/sys/netinet/ip_var.h
@@ -244,14 +244,20 @@ extern int	(*ip_rsvp_vif)(struct socket *, struct sockopt *);
 extern void	(*ip_rsvp_force_done)(struct socket *);
 extern void	(*rsvp_input_p)(struct mbuf *m, int off);
 
-extern	struct pfil_head inet_pfil_hook;	/* packet filter hooks */
+VNET_DECLARE(struct pfil_head, inet_pfil_hook);	/* packet filter hooks */
+#define	V_inet_pfil_hook	VNET(inet_pfil_hook)
 
 void	in_delayed_cksum(struct mbuf *m);
 
 /* ipfw and dummynet hooks. Most are declared in raw_ip.c */
 struct ip_fw_args;
-extern int	(*ip_fw_chk_ptr)(struct ip_fw_args *args);
-extern int	(*ip_fw_ctl_ptr)(struct sockopt *);
+typedef int	(*ip_fw_chk_ptr_t)(struct ip_fw_args *args);
+typedef int	(*ip_fw_ctl_ptr_t)(struct sockopt *);
+VNET_DECLARE(ip_fw_chk_ptr_t, ip_fw_chk_ptr);
+VNET_DECLARE(ip_fw_ctl_ptr_t, ip_fw_ctl_ptr);
+#define	V_ip_fw_chk_ptr		VNET(ip_fw_chk_ptr)
+#define	V_ip_fw_ctl_ptr		VNET(ip_fw_ctl_ptr)
+
 extern int	(*ip_dn_ctl_ptr)(struct sockopt *);
 extern int	(*ip_dn_io_ptr)(struct mbuf **m, int dir, struct ip_fw_args *fwa);
 extern void	(*ip_dn_ruledel_ptr)(void *);		/* in ip_fw2.c */
diff --git a/sys/netinet/ipfw/ip_fw2.c b/sys/netinet/ipfw/ip_fw2.c
index e35669d..975d303 100644
--- a/sys/netinet/ipfw/ip_fw2.c
+++ b/sys/netinet/ipfw/ip_fw2.c
@@ -2495,6 +2495,10 @@ do {									\
 	}
 
 	IPFW_RLOCK(chain);
+	if (! V_ipfw_vnet_ready) { /* shutting down, leave NOW. */
+		IPFW_RUNLOCK(chain);
+		return (IP_FW_PASS);	/* accept */
+	}
 	mtag = m_tag_find(m, PACKET_TAG_DIVERT, NULL);
 	if (args->rule) {
 		/*
@@ -4637,29 +4641,21 @@ ipfw_init(void)
 		printf("limited to %d packets/entry by default\n",
 		    V_verbose_limit);
 
-	/*
-	 * Hook us up to pfil.
-	 * Eventually pfil will be per vnet.
-	 */
-	if ((error = ipfw_hook()) != 0) {
-		printf("ipfw_hook() error\n");
-		return (error);
-	}
-#ifdef INET6
-	if ((error = ipfw6_hook()) != 0) {
-		printf("ipfw6_hook() error\n");
-		return (error);
-	}
-#endif
-	/*
-	 * Other things that are only done the first time.
-	 * (now that we are guaranteed of success).
-	 */
-	ip_fw_ctl_ptr = ipfw_ctl;
-	ip_fw_chk_ptr = ipfw_chk;
 	return (error);
 }
 
+/**********************
+ * Called for the removal of the last instance only on module unload.
+ */
+static void
+ipfw_destroy(void)
+{
+
+	uma_zdestroy(ipfw_dyn_rule_zone);
+	IPFW_DYN_LOCK_DESTROY();
+	printf("IP firewall unloaded\n");
+}
+
 /****************
  * Stuff that must be initialized for every instance
  * (including the first of course).
@@ -4743,19 +4739,30 @@ vnet_ipfw_init(const void *unused)
 
 	/* First set up some values that are compile time options */
 	V_ipfw_vnet_ready = 1;		/* Open for business */
-	return (0);
-}
 
-/**********************
- * Called for the removal of the last instance only on module unload.
- */
-static void
-ipfw_destroy(void)
-{
+	/* Hook up the raw inputs */
+	V_ip_fw_ctl_ptr = ipfw_ctl;
+	V_ip_fw_chk_ptr = ipfw_chk;
 
-	uma_zdestroy(ipfw_dyn_rule_zone);
-	IPFW_DYN_LOCK_DESTROY();
-	printf("IP firewall unloaded\n");
+	/*
+	 * Hook us up to pfil.
+	 */
+	if (V_fw_enable) {
+		if ((error = ipfw_hook()) != 0) {
+			printf("ipfw_hook() error\n");
+			return (error);
+		}
+	}
+#ifdef INET6
+	if (V_fw6_enable) {
+		if ((error = ipfw6_hook()) != 0) {
+			printf("ipfw6_hook() error\n");
+			/* XXX should we unhook everything else? */
+			return (error);
+		}
+	}
+#endif
+	return (0);
 }
 
 /***********************
@@ -4767,9 +4774,18 @@ vnet_ipfw_uninit(const void *unused)
 	struct ip_fw *reap;
 
 	V_ipfw_vnet_ready = 0; /* tell new callers to go away */
-	callout_drain(&V_ipfw_timeout);
+	ipfw_unhook();
+#ifdef INET6
+	ipfw6_unhook();
+#endif
+	/* layer2 and other entrypoints still come in this way. */
+	V_ip_fw_chk_ptr = NULL;
+	V_ip_fw_ctl_ptr = NULL;
+	IPFW_WLOCK(&V_layer3_chain);
 	/* We wait on the wlock here until the last user leaves */
+	IPFW_WUNLOCK(&V_layer3_chain);
 	IPFW_WLOCK(&V_layer3_chain);
+	callout_drain(&V_ipfw_timeout);
 	flush_tables(&V_layer3_chain);
 	V_layer3_chain.reap = NULL;
 	free_chain(&V_layer3_chain, 1 /* kill default rule */);
@@ -4803,21 +4819,10 @@ ipfw_modevent(module_t mod, int type, void *unused)
 		/* Called once at module load or
 	 	 * system boot if compiled in. */
 		break;
-	case MOD_UNLOAD:
-		break;
 	case MOD_QUIESCE:
-		/* Yes, the unhooks can return errors, we can safely ignore
-		 * them. Eventually these will be done per jail as they
-		 * shut down. We will wait on each vnet's l3 lock as existing
-		 * callers go away.
-		 */
-		ipfw_unhook();
-#ifdef INET6
-		ipfw6_unhook();
-#endif
-		/* layer2 and other entrypoints still come in this way. */
-		ip_fw_chk_ptr = NULL;
-		ip_fw_ctl_ptr = NULL;
+		/* Called before unload. May veto unloading. */
+		break;
+	case MOD_UNLOAD:
 		/* Called during unload. */
 		break;
 	case MOD_SHUTDOWN:
@@ -4866,4 +4871,3 @@ SYSUNINIT(ipfw_destroy, IPFW_SI_SUB_FIREWALL, IPFW_MODULE_ORDER,
 VNET_SYSUNINIT(vnet_ipfw_uninit, IPFW_SI_SUB_FIREWALL, IPFW_VNET_ORDER,
 	    vnet_ipfw_uninit, NULL);
 
- 
diff --git a/sys/netinet/ipfw/ip_fw_pfil.c b/sys/netinet/ipfw/ip_fw_pfil.c
index ffffb59..db73084 100644
--- a/sys/netinet/ipfw/ip_fw_pfil.c
+++ b/sys/netinet/ipfw/ip_fw_pfil.c
@@ -515,42 +515,54 @@ ipfw6_unhook(void)
 int
 ipfw_chg_hook(SYSCTL_HANDLER_ARGS)
 {
-	int enable = *(int *)arg1;
+	int enable;
+	int oldenable;
 	int error;
 
-#ifdef VIMAGE /* Since enabling is global, only let base do it. */
-	if (! IS_DEFAULT_VNET(curvnet))
-		return (EPERM);
+	if (arg1 == &VNET_NAME(fw_enable)) {
+		enable = V_fw_enable;
+	}
+#ifdef INET6
+	else if (arg1 == &VNET_NAME(fw6_enable)) {
+		enable = V_fw6_enable;
+	}
 #endif
+	else 
+		return (EINVAL);
+
+	oldenable = enable;
+
 	error = sysctl_handle_int(oidp, &enable, 0, req);
+
 	if (error)
 		return (error);
 
 	enable = (enable) ? 1 : 0;
 
-	if (enable == *(int *)arg1)
+	if (enable == oldenable)
 		return (0);
 
-	if (arg1 == &V_fw_enable) {
+	if (arg1 == &VNET_NAME(fw_enable)) {
 		if (enable)
 			error = ipfw_hook();
 		else
 			error = ipfw_unhook();
+		if (error)
+			return (error);
+		V_fw_enable = enable;
 	}
 #ifdef INET6
-	if (arg1 == &V_fw6_enable) {
+	else if (arg1 == &VNET_NAME(fw6_enable)) {
 		if (enable)
 			error = ipfw6_hook();
 		else
 			error = ipfw6_unhook();
+		if (error)
+			return (error);
+		V_fw6_enable = enable;
 	}
 #endif
 
-	if (error)
-		return (error);
-
-	*(int *)arg1 = enable;
-
 	return (0);
 }
 
diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c
index b40ceb2..d612c2c 100644
--- a/sys/netinet/raw_ip.c
+++ b/sys/netinet/raw_ip.c
@@ -84,9 +84,9 @@ VNET_DEFINE(struct inpcbinfo, ripcbinfo);
  * The data hooks are not used here but it is convenient
  * to keep them all in one place.
  */
-int (*ip_fw_ctl_ptr)(struct sockopt *) = NULL;
+VNET_DEFINE(ip_fw_chk_ptr_t, ip_fw_chk_ptr) = NULL;
+VNET_DEFINE(ip_fw_ctl_ptr_t, ip_fw_ctl_ptr) = NULL;
 int (*ip_dn_ctl_ptr)(struct sockopt *) = NULL;
-int (*ip_fw_chk_ptr)(struct ip_fw_args *args) = NULL;
 int (*ip_dn_io_ptr)(struct mbuf **m, int dir, struct ip_fw_args *fwa) = NULL;
 
 /*
@@ -523,8 +523,8 @@ rip_ctloutput(struct socket *so, struct sockopt *sopt)
 		case IP_FW_TABLE_LIST:
 		case IP_FW_NAT_GET_CONFIG:
 		case IP_FW_NAT_GET_LOG:
-			if (ip_fw_ctl_ptr != NULL)
-				error = ip_fw_ctl_ptr(sopt);
+			if (V_ip_fw_ctl_ptr != NULL)
+				error = V_ip_fw_ctl_ptr(sopt);
 			else
 				error = ENOPROTOOPT;
 			break;
@@ -584,8 +584,8 @@ rip_ctloutput(struct socket *so, struct sockopt *sopt)
 		case IP_FW_TABLE_FLUSH:
 		case IP_FW_NAT_CFG:
 		case IP_FW_NAT_DEL:
-			if (ip_fw_ctl_ptr != NULL)
-				error = ip_fw_ctl_ptr(sopt);
+			if (V_ip_fw_ctl_ptr != NULL)
+				error = V_ip_fw_ctl_ptr(sopt);
 			else
 				error = ENOPROTOOPT;
 			break;