This commit enables partial operation of dummynet with kernels

compiled with "options VIMAGE".
As it is now, there is still a single instance of the pipes,
and it is only usable from vnet0 (the main instance).
Trying to use a pipe from a different vimage does not crash
the system as it did before, but the traffic coming out from
the pipe goes to the wrong place, and i still need to
figure out where.

Support for per-vimage pipes is almost there (just a matter of
uncommenting the VNET_* definitions for dn_cfg, plus putting into
the structure the remaining static variables), however i need
first to figure out how init/uninit work, and also to understand
where packets are ending up on exit from a pipe.

In summary: vimage support for dummynet is not complete yet,
but we are getting there.

3 files changed, 57 insertions, 31 deletions

diff --git a/sys/netinet/ipfw/ip_dn_io.c b/sys/netinet/ipfw/ip_dn_io.c
index eab9d9e..41bab2d 100644
--- a/sys/netinet/ipfw/ip_dn_io.c
+++ b/sys/netinet/ipfw/ip_dn_io.c
@@ -69,6 +69,7 @@ __FBSDID("$FreeBSD$");
  */
 
 struct dn_parms dn_cfg;
+//VNET_DEFINE(struct dn_parms, _base_dn_cfg);
 
 static long tick_last;		/* Last tick duration (usec). */
 static long tick_delta;		/* Last vs standard tick diff (usec). */
@@ -100,31 +101,34 @@ SYSCTL_DECL(_net_inet);
 SYSCTL_DECL(_net_inet_ip);
 SYSCTL_NODE(_net_inet_ip, OID_AUTO, dummynet, CTLFLAG_RW, 0, "Dummynet");
 
+/* wrapper to pass dn_cfg fields to SYSCTL_* */
+//#define DC(x)	(&(VNET_NAME(_base_dn_cfg).x))
+#define DC(x)	(&(dn_cfg.x))
 /* parameters */
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, hash_size,
-    CTLFLAG_RW, &dn_cfg.hash_size, 0, "Default hash table size");
+    CTLFLAG_RW, DC(hash_size), 0, "Default hash table size");
 SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, pipe_slot_limit,
-    CTLFLAG_RW, &dn_cfg.slot_limit, 0,
+    CTLFLAG_RW, DC(slot_limit), 0,
     "Upper limit in slots for pipe queue.");
 SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, pipe_byte_limit,
-    CTLFLAG_RW, &dn_cfg.byte_limit, 0,
+    CTLFLAG_RW, DC(byte_limit), 0,
     "Upper limit in bytes for pipe queue.");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, io_fast,
-    CTLFLAG_RW, &dn_cfg.io_fast, 0, "Enable fast dummynet io.");
+    CTLFLAG_RW, DC(io_fast), 0, "Enable fast dummynet io.");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, debug,
-    CTLFLAG_RW, &dn_cfg.debug, 0, "Dummynet debug level");
+    CTLFLAG_RW, DC(debug), 0, "Dummynet debug level");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, expire,
-    CTLFLAG_RW, &dn_cfg.expire, 0, "Expire empty queues/pipes");
+    CTLFLAG_RW, DC(expire), 0, "Expire empty queues/pipes");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, expire_cycle,
-    CTLFLAG_RD, &dn_cfg.expire_cycle, 0, "Expire cycle for queues/pipes");
+    CTLFLAG_RD, DC(expire_cycle), 0, "Expire cycle for queues/pipes");
 
 /* RED parameters */
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, red_lookup_depth,
-    CTLFLAG_RD, &dn_cfg.red_lookup_depth, 0, "Depth of RED lookup table");
+    CTLFLAG_RD, DC(red_lookup_depth), 0, "Depth of RED lookup table");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, red_avg_pkt_size,
-    CTLFLAG_RD, &dn_cfg.red_avg_pkt_size, 0, "RED Medium packet size");
+    CTLFLAG_RD, DC(red_avg_pkt_size), 0, "RED Medium packet size");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, red_max_pkt_size,
-    CTLFLAG_RD, &dn_cfg.red_max_pkt_size, 0, "RED Max packet size");
+    CTLFLAG_RD, DC(red_max_pkt_size), 0, "RED Max packet size");
 
 /* time adjustment */
 SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, tick_delta,
@@ -142,13 +146,13 @@ SYSCTL_LONG(_net_inet_ip_dummynet, OID_AUTO, tick_lost,
 
 /* statistics */
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, schk_count,
-    CTLFLAG_RD, &dn_cfg.schk_count, 0, "Number of schedulers");
+    CTLFLAG_RD, DC(schk_count), 0, "Number of schedulers");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, si_count,
-    CTLFLAG_RD, &dn_cfg.si_count, 0, "Number of scheduler instances");
+    CTLFLAG_RD, DC(si_count), 0, "Number of scheduler instances");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, fsk_count,
-    CTLFLAG_RD, &dn_cfg.fsk_count, 0, "Number of flowsets");
+    CTLFLAG_RD, DC(fsk_count), 0, "Number of flowsets");
 SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, queue_count,
-    CTLFLAG_RD, &dn_cfg.queue_count, 0, "Number of queues");
+    CTLFLAG_RD, DC(queue_count), 0, "Number of queues");
 SYSCTL_ULONG(_net_inet_ip_dummynet, OID_AUTO, io_pkt,
     CTLFLAG_RD, &io_pkt, 0,
     "Number of packets passed to dummynet.");
@@ -158,7 +162,7 @@ SYSCTL_ULONG(_net_inet_ip_dummynet, OID_AUTO, io_pkt_fast,
 SYSCTL_ULONG(_net_inet_ip_dummynet, OID_AUTO, io_pkt_drop,
     CTLFLAG_RD, &io_pkt_drop, 0,
     "Number of packets dropped by dummynet.");
-
+#undef DC
 SYSEND
 
 #endif
@@ -496,6 +500,8 @@ dummynet_task(void *context, int pending)
 	struct timeval t;
 	struct mq q = { NULL, NULL }; /* queue to accumulate results */
 
+	CURVNET_SET(context);
+
 	DN_BH_WLOCK();
 
 	/* Update number of lost(coalesced) ticks. */
@@ -560,6 +566,7 @@ dummynet_task(void *context, int pending)
 	dn_reschedule();
 	if (q.head != NULL)
 		dummynet_send(q.head);
+	CURVNET_RESTORE();
 }
 
 /*
diff --git a/sys/netinet/ipfw/ip_dn_private.h b/sys/netinet/ipfw/ip_dn_private.h
index 87ebeaa..f1a7f3f 100644
--- a/sys/netinet/ipfw/ip_dn_private.h
+++ b/sys/netinet/ipfw/ip_dn_private.h
@@ -150,6 +150,8 @@ struct dn_parms {
 	uint32_t expire;
 	uint32_t expire_cycle;	/* tick count */
 	
+	int init_done;
+
 	/* if the upper half is busy doing something long,
 	 * can set the busy flag and we will enqueue packets in
 	 * a queue for later processing.
@@ -354,6 +356,8 @@ enum {
 };
 
 extern struct dn_parms dn_cfg;
+//VNET_DECLARE(struct dn_parms, _base_dn_cfg);
+//#define dn_cfg              VNET(_base_dn_cfg)
 
 int dummynet_io(struct mbuf **, int , struct ip_fw_args *);
 void dummynet_task(void *context, int pending);
diff --git a/sys/netinet/ipfw/ip_dummynet.c b/sys/netinet/ipfw/ip_dummynet.c
index 08cddbe..0a6d16d 100644
--- a/sys/netinet/ipfw/ip_dummynet.c
+++ b/sys/netinet/ipfw/ip_dummynet.c
@@ -2112,14 +2112,10 @@ ip_dn_ctl(struct sockopt *sopt)
 static void
 ip_dn_init(void)
 {
-	static int init_done = 0;
-
-	if (init_done)
+	if (dn_cfg.init_done)
 		return;
-	init_done = 1;
-	if (bootverbose)
-		printf("DUMMYNET with IPv6 initialized (100131)\n");
-
+	printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet);
+	dn_cfg.init_done = 1;
 	/* Set defaults here. MSVC does not accept initializers,
 	 * and this is also useful for vimages
 	 */
@@ -2156,10 +2152,8 @@ ip_dn_init(void)
 	SLIST_INIT(&dn_cfg.schedlist);
 
 	DN_LOCK_INIT();
-	ip_dn_ctl_ptr = ip_dn_ctl;
-	ip_dn_io_ptr = dummynet_io;
 
-	TASK_INIT(&dn_task, 0, dummynet_task, NULL);
+	TASK_INIT(&dn_task, 0, dummynet_task, curvnet);
 	dn_tq = taskqueue_create_fast("dummynet", M_NOWAIT,
 	    taskqueue_thread_enqueue, &dn_tq);
 	taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet");
@@ -2173,13 +2167,16 @@ ip_dn_init(void)
 
 #ifdef KLD_MODULE
 static void
-ip_dn_destroy(void)
+ip_dn_destroy(int last)
 {
 	callout_drain(&dn_timeout);
 
 	DN_BH_WLOCK();
-	ip_dn_ctl_ptr = NULL;
-	ip_dn_io_ptr = NULL;
+	if (last) {
+		printf("%s removing last instance\n", __FUNCTION__);
+		ip_dn_ctl_ptr = NULL;
+		ip_dn_io_ptr = NULL;
+	}
 
 	dummynet_flush();
 	DN_BH_WUNLOCK();
@@ -2204,13 +2201,15 @@ dummynet_modevent(module_t mod, int type, void *data)
 			return EEXIST ;
 		}
 		ip_dn_init();
+		ip_dn_ctl_ptr = ip_dn_ctl;
+		ip_dn_io_ptr = dummynet_io;
 		return 0;
 	} else if (type == MOD_UNLOAD) {
 #if !defined(KLD_MODULE)
 		printf("dummynet statically compiled, cannot unload\n");
 		return EINVAL ;
 #else
-		ip_dn_destroy();
+		ip_dn_destroy(1 /* last */);
 		return 0;
 #endif
 	} else
@@ -2288,8 +2287,24 @@ static moduledata_t dummynet_mod = {
 	"dummynet", dummynet_modevent, NULL
 };
 
-DECLARE_MODULE(dummynet, dummynet_mod,
-	SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY-1);
+#define	DN_SI_SUB	SI_SUB_PROTO_IFATTACHDOMAIN
+#define	DN_MODEV_ORD	(SI_ORDER_ANY - 128) /* after ipfw */
+DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD);
 MODULE_DEPEND(dummynet, ipfw, 2, 2, 2);
 MODULE_VERSION(dummynet, 1);
+
+/*
+ * Starting up. Done in order after dummynet_modevent() has been called.
+ * VNET_SYSINIT is also called for each existing vnet and each new vnet.
+ */
+//VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL);
+ 
+/*
+ * Shutdown handlers up shop. These are done in REVERSE ORDER, but still
+ * after dummynet_modevent() has been called. Not called on reboot.
+ * VNET_SYSUNINIT is also called for each exiting vnet as it exits.
+ * or when the module is unloaded.
+ */
+//VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL);
+
 /* end of file */