Build on Jeff Roberson's linker-set based dynamic per-CPU allocator

(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator.  Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...).  This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.

Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack.  Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory.  Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.

Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy.  Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address.  When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.

This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.

Bump __FreeBSD_version and update UPDATING.

Portions submitted by:  bz
Reviewed by:            bz, zec
Discussed with:         gnn, jamie, jeff, jhb, julian, sam
Suggested by:           peter
Approved by:            re (kensmith)

8 files changed, 1 insertions, 39 deletions

diff --git a/sys/contrib/altq/altq/altq_subr.c b/sys/contrib/altq/altq/altq_subr.c
index ac36770..348128f 100644
--- a/sys/contrib/altq/altq/altq_subr.c
+++ b/sys/contrib/altq/altq/altq_subr.c
@@ -469,7 +469,6 @@ tbr_timeout(arg)
 	VNET_LIST_RLOCK();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
-		INIT_VNET_NET(vnet_iter);
 #endif
 		for (ifp = TAILQ_FIRST(&V_ifnet); ifp;
 		    ifp = TAILQ_NEXT(ifp, if_list)) {
diff --git a/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c b/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c
index 3e0e633..1480ed0 100644
--- a/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c
+++ b/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c
@@ -64,6 +64,7 @@ static const char rcsid[] = "@(#)$Id: ip_fil_freebsd.c,v 2.53.2.50 2007/09/20 12
 #endif
 #if __FreeBSD_version >= 800044
 # include <sys/vimage.h>
+# include <netinet/tcp_var.h>
 #else
 #define V_path_mtu_discovery path_mtu_discovery
 #define V_ipforwarding ipforwarding
@@ -92,9 +93,6 @@ static const char rcsid[] = "@(#)$Id: ip_fil_freebsd.c,v 2.53.2.50 2007/09/20 12
 #include <netinet/udp.h>
 #include <netinet/tcpip.h>
 #include <netinet/ip_icmp.h>
-#if defined(__FreeBSD_version) && (__FreeBSD_version >= 800056)
-# include <netinet/vinet.h>
-#endif
 #ifndef _KERNEL
 # include "netinet/ipf.h"
 #endif
@@ -213,7 +211,6 @@ char *s;
 
 int ipfattach()
 {
-	INIT_VNET_INET(curvnet);
 #ifdef USE_SPL
 	int s;
 #endif
@@ -265,7 +262,6 @@ int ipfattach()
  */
 int ipfdetach()
 {
-	INIT_VNET_INET(curvnet);
 #ifdef USE_SPL
 	int s;
 #endif
@@ -654,7 +650,6 @@ static int fr_send_ip(fin, m, mpp)
 fr_info_t *fin;
 mb_t *m, **mpp;
 {
-	INIT_VNET_INET(curvnet);
 	fr_info_t fnew;
 	ip_t *ip, *oip;
 	int hlen;
diff --git a/sys/contrib/pf/net/pf.c b/sys/contrib/pf/net/pf.c
index d0947b9..d59930f 100644
--- a/sys/contrib/pf/net/pf.c
+++ b/sys/contrib/pf/net/pf.c
@@ -119,9 +119,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet/udp_var.h>
 #include <netinet/icmp_var.h>
 #include <netinet/if_ether.h>
-#ifdef __FreeBSD__
-#include <netinet/vinet.h>
-#endif
 
 #ifndef __FreeBSD__
 #include <dev/rndvar.h>
@@ -141,7 +138,6 @@ __FBSDID("$FreeBSD$");
 #ifdef __FreeBSD__
 #include <netinet6/ip6_var.h>
 #include <netinet6/in6_pcb.h>
-#include <netinet6/vinet6.h>
 #endif
 #endif /* INET6 */
 
@@ -1762,7 +1758,6 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af,
     u_int8_t flags, u_int16_t win, u_int16_t mss, u_int8_t ttl, int tag,
     u_int16_t rtag, struct ether_header *eh, struct ifnet *ifp)
 {
-	INIT_VNET_INET(curvnet);
 	struct mbuf	*m;
 	int		 len, tlen;
 #ifdef INET
@@ -2926,7 +2921,6 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd, struct inpcb *inp_arg)
 pf_socket_lookup(int direction, struct pf_pdesc *pd)
 #endif
 {
-	INIT_VNET_INET(curvnet);
 	struct pf_addr		*saddr, *daddr;
 	u_int16_t		 sport, dport;
 #ifdef __FreeBSD__
@@ -3096,7 +3090,6 @@ pf_get_wscale(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
 u_int16_t
 pf_get_mss(struct mbuf *m, int off, u_int16_t th_off, sa_family_t af)
 {
-	INIT_VNET_INET(curvnet);
 	int		 hlen;
 	u_int8_t	 hdr[60];
 	u_int8_t	*opt, optlen;
@@ -3136,7 +3129,6 @@ u_int16_t
 pf_calc_mss(struct pf_addr *addr, sa_family_t af, u_int16_t offer)
 {
 #ifdef INET
-	INIT_VNET_INET(curvnet);
 	struct sockaddr_in	*dst;
 	struct route		 ro;
 #endif /* INET */
@@ -3239,7 +3231,6 @@ pf_test_tcp(struct pf_rule **rm, struct pf_state **sm, int direction,
     struct ifqueue *ifq)
 #endif
 {
-	INIT_VNET_INET(curvnet);
 	struct pf_rule		*nr = NULL;
 	struct pf_addr		*saddr = pd->src, *daddr = pd->dst;
 	struct tcphdr		*th = pd->hdr.tcp;
@@ -6094,7 +6085,6 @@ void
 pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
     struct pf_state *s, struct pf_pdesc *pd)
 {
-	INIT_VNET_INET(curvnet);
 	struct mbuf		*m0, *m1;
 	struct route		 iproute;
 	struct route		*ro = NULL;
@@ -6633,26 +6623,22 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t a
 		switch (p) {
 		case IPPROTO_TCP:
 		    {
-			INIT_VNET_INET(curvnet);
 			TCPSTAT_INC(tcps_rcvbadsum);
 			break;
 		    }
 		case IPPROTO_UDP:
 		    {
-			INIT_VNET_INET(curvnet);
 			UDPSTAT_INC(udps_badsum);
 			break;
 		    }
 		case IPPROTO_ICMP:
 		    {
-			INIT_VNET_INET(curvnet);
 			ICMPSTAT_INC(icps_checksum);
 			break;
 		    }
 #ifdef INET6
 		case IPPROTO_ICMPV6:
 		    {
-			INIT_VNET_INET6(curvnet);
 			ICMP6STAT_INC(icp6s_checksum);
 			break;
 		    }
diff --git a/sys/contrib/pf/net/pf_if.c b/sys/contrib/pf/net/pf_if.c
index 1972edc..7056640 100644
--- a/sys/contrib/pf/net/pf_if.c
+++ b/sys/contrib/pf/net/pf_if.c
@@ -115,9 +115,6 @@ void		 pfi_change_group_event(void * __unused, char *);
 void		 pfi_detach_group_event(void * __unused, struct ifg_group *);
 void		 pfi_ifaddr_event(void * __unused, struct ifnet *);
 
-#ifdef VIMAGE_GLOBALS
-extern struct ifgrouphead ifg_head;
-#endif
 #endif
 
 RB_PROTOTYPE(pfi_ifhead, pfi_kif, pfik_tree, pfi_if_compare);
@@ -129,7 +126,6 @@ RB_GENERATE(pfi_ifhead, pfi_kif, pfik_tree, pfi_if_compare);
 void
 pfi_initialize(void)
 {
-	INIT_VNET_NET(curvnet);
 
 	if (pfi_all != NULL)	/* already initialized */
 		return;
diff --git a/sys/contrib/pf/net/pf_ioctl.c b/sys/contrib/pf/net/pf_ioctl.c
index 436f765..36331f3 100644
--- a/sys/contrib/pf/net/pf_ioctl.c
+++ b/sys/contrib/pf/net/pf_ioctl.c
@@ -3703,7 +3703,6 @@ static int
 pf_check6_in(void *arg, struct mbuf **m, struct ifnet *ifp, int dir,
     struct inpcb *inp)
 {
-	INIT_VNET_NET(curvnet);
 
 	/*
 	 * IPv6 is not affected by ip_len/ip_off byte order changes.
diff --git a/sys/contrib/pf/net/pf_subr.c b/sys/contrib/pf/net/pf_subr.c
index a722aef..9314bf1 100644
--- a/sys/contrib/pf/net/pf_subr.c
+++ b/sys/contrib/pf/net/pf_subr.c
@@ -65,7 +65,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet/tcp_timer.h>
 #include <netinet/tcp_var.h>
 #include <netinet/if_ether.h>
-#include <netinet/vinet.h>
 #include <net/pfvar.h>
 
 /*
diff --git a/sys/contrib/pf/net/pfvar.h b/sys/contrib/pf/net/pfvar.h
index b12d389..2c176db 100644
--- a/sys/contrib/pf/net/pfvar.h
+++ b/sys/contrib/pf/net/pfvar.h
@@ -1855,12 +1855,4 @@ int	pf_osfp_match(struct pf_osfp_enlist *, pf_osfp_t);
 struct pf_os_fingerprint *
 	pf_osfp_validate(void);
 
-/*
- * Symbol translation macros
- */
-#define	INIT_VNET_PF(vnet) \
-	INIT_FROM_VNET(vnet, VNET_MOD_PF, struct vnet_pf, vnet_pf)
-
-#define	VNET_PF(sym)	VSYM(vnet_pf, sym)
-
 #endif /* _NET_PFVAR_H_ */
diff --git a/sys/contrib/rdma/rdma_cma.c b/sys/contrib/rdma/rdma_cma.c
index 23d56f1..1bf2d08 100644
--- a/sys/contrib/rdma/rdma_cma.c
+++ b/sys/contrib/rdma/rdma_cma.c
@@ -51,7 +51,6 @@ __FBSDID("$FreeBSD$");
 #include <net/if.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
-#include <netinet/vinet.h>
 
 #include <contrib/rdma/rdma_cm.h>
 #include <contrib/rdma/ib_cache.h>
@@ -1950,7 +1949,6 @@ err1:
 
 static int cma_alloc_any_port(struct kvl *ps, struct rdma_id_private *id_priv)
 {
-	INIT_VNET_INET(curvnet);
 	struct rdma_bind_list *bind_list;
 	int port, ret;
 
@@ -1995,7 +1993,6 @@ err1:
 
 static int cma_use_port(struct kvl *ps, struct rdma_id_private *id_priv)
 {
-	INIT_VNET_INET(curvnet);
 	struct rdma_id_private *cur_id;
 	struct sockaddr_in *sin, *cur_sin;
 	struct rdma_bind_list *bind_list;
@@ -2915,7 +2912,6 @@ static void cma_remove_one(struct ib_device *device)
 
 static int cma_init(void)
 {
-	INIT_VNET_INET(curvnet);
 	int ret;
 
 	LIST_INIT(&listen_any_list);