Merge multi-FIB IPv6 support from projects/multi-fibv6/head/:

Extend the so far IPv4-only support for multiple routing tables (FIBs)
introduced in r178888 to IPv6 providing feature parity.

This includes an extended rtalloc(9) KPI for IPv6, the necessary
adjustments to the network stack, and user land support as in netstat.

Sponsored by:	Cisco Systems, Inc.
Reviewed by:	melifaro (basically)
MFC after:	10 days

4 files changed, 122 insertions, 70 deletions

diff --git a/sys/net/flowtable.c b/sys/net/flowtable.c
index fac0f59..8501b18 100644
--- a/sys/net/flowtable.c
+++ b/sys/net/flowtable.c
@@ -374,7 +374,7 @@ SYSCTL_VNET_PROC(_net_inet_flowtable, OID_AUTO, stats, CTLTYPE_STRING|CTLFLAG_RD
 
 #ifndef RADIX_MPATH
 static void
-in_rtalloc_ign_wrapper(struct route *ro, uint32_t hash, u_int fibnum)
+rtalloc_ign_wrapper(struct route *ro, uint32_t hash, u_int fibnum)
 {
 
 	rtalloc_ign_fib(ro, 0, fibnum);
@@ -1315,7 +1315,7 @@ flowtable_alloc(char *name, int nentry, int flags)
 #ifdef RADIX_MPATH
 	ft->ft_rtalloc = rtalloc_mpath_fib;
 #else
-	ft->ft_rtalloc = in_rtalloc_ign_wrapper;
+	ft->ft_rtalloc = rtalloc_ign_wrapper;
 #endif
 	if (flags & FL_PCPU) {
 		ft->ft_lock = flowtable_pcpu_lock;
diff --git a/sys/net/if_faith.c b/sys/net/if_faith.c
index ca80883..db4c1d9 100644
--- a/sys/net/if_faith.c
+++ b/sys/net/if_faith.c
@@ -338,7 +338,7 @@ faithprefix(in6)
 	sin6.sin6_family = AF_INET6;
 	sin6.sin6_len = sizeof(struct sockaddr_in6);
 	sin6.sin6_addr = *in6;
-	rt = rtalloc1((struct sockaddr *)&sin6, 0, 0UL);
+	rt = in6_rtalloc1((struct sockaddr *)&sin6, 0, 0UL, RT_DEFAULT_FIB);
 	if (rt && rt->rt_ifp && rt->rt_ifp->if_type == IFT_FAITH &&
 	    (rt->rt_ifp->if_flags & IFF_UP) != 0)
 		ret = 1;
diff --git a/sys/net/route.c b/sys/net/route.c
index 296914b..c48a4ac 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -35,6 +35,7 @@
  ***********************************************************************/
 
 #include "opt_inet.h"
+#include "opt_inet6.h"
 #include "opt_route.h"
 #include "opt_mrouting.h"
 #include "opt_mpath.h"
@@ -72,7 +73,11 @@ SYSCTL_UINT(_net, OID_AUTO, fibs, CTLFLAG_RD, &rt_numfibs, 0, "");
 /*
  * Allow the boot code to allow LESS than RT_MAXFIBS to be used.
  * We can't do more because storage is statically allocated for now.
- * (for compatibility reasons.. this will change).
+ * (for compatibility reasons.. this will change. When this changes, code should
+ * be refactored to protocol independent parts and protocol dependent parts,
+ * probably hanging of domain(9) specific storage to not need the full
+ * fib * af RNH allocation etc. but allow tuning the number of tables per
+ * address family).
  */
 TUNABLE_INT("net.fibs", &rt_numfibs);
 
@@ -82,6 +87,9 @@ TUNABLE_INT("net.fibs", &rt_numfibs);
  * changes for the FIB of the caller when adding a new set of addresses
  * to an interface.  XXX this is a shotgun aproach to a problem that needs
  * a more fine grained solution.. that will come.
+ * XXX also has the problems getting the FIB from curthread which will not
+ * always work given the fib can be overridden and prefixes can be added
+ * from the network stack context.
  */
 u_int rt_add_addr_allfibs = 1;
 SYSCTL_UINT(_net, OID_AUTO, add_addr_allfibs, CTLFLAG_RW,
@@ -196,27 +204,23 @@ vnet_route_init(const void *unused __unused)
 	V_rtzone = uma_zcreate("rtentry", sizeof(struct rtentry), NULL, NULL,
 	    NULL, NULL, UMA_ALIGN_PTR, 0);
 	for (dom = domains; dom; dom = dom->dom_next) {
-		if (dom->dom_rtattach)  {
-			for  (table = 0; table < rt_numfibs; table++) {
-				if ( (fam = dom->dom_family) == AF_INET ||
-				    table == 0) {
- 			        	/* for now only AF_INET has > 1 table */
-					/* XXX MRT 
-					 * rtattach will be also called
-					 * from vfs_export.c but the
-					 * offset will be 0
-					 * (only for AF_INET and AF_INET6
-					 * which don't need it anyhow)
-					 */
-					rnh = rt_tables_get_rnh_ptr(table, fam);
-					if (rnh == NULL)
-						panic("%s: rnh NULL", __func__);
-					dom->dom_rtattach((void **)rnh,
-				    	    dom->dom_rtoffset);
-				} else {
-					break;
-				}
-			}
+		if (dom->dom_rtattach == NULL)
+			continue;
+
+		for  (table = 0; table < rt_numfibs; table++) {
+			fam = dom->dom_family;
+			if (table != 0 && fam != AF_INET6 && fam != AF_INET)
+				break;
+
+			/*
+			 * XXX MRT rtattach will be also called from
+			 * vfs_export.c but the offset will be 0 (only for
+			 * AF_INET and AF_INET6 which don't need it anyhow).
+			 */
+			rnh = rt_tables_get_rnh_ptr(table, fam);
+			if (rnh == NULL)
+				panic("%s: rnh NULL", __func__);
+			dom->dom_rtattach((void **)rnh, dom->dom_rtoffset);
 		}
 	}
 }
@@ -233,20 +237,19 @@ vnet_route_uninit(const void *unused __unused)
 	struct radix_node_head **rnh;
 
 	for (dom = domains; dom; dom = dom->dom_next) {
-		if (dom->dom_rtdetach) {
-			for (table = 0; table < rt_numfibs; table++) {
-				if ( (fam = dom->dom_family) == AF_INET ||
-				    table == 0) {
-					/* For now only AF_INET has > 1 tbl. */
-					rnh = rt_tables_get_rnh_ptr(table, fam);
-					if (rnh == NULL)
-						panic("%s: rnh NULL", __func__);
-					dom->dom_rtdetach((void **)rnh,
-					    dom->dom_rtoffset);
-				} else {
-					break;
-				}
-			}
+		if (dom->dom_rtdetach == NULL)
+			continue;
+
+		for (table = 0; table < rt_numfibs; table++) {
+			fam = dom->dom_family;
+
+			if (table != 0 && fam != AF_INET6 && fam != AF_INET)
+				break;
+
+			rnh = rt_tables_get_rnh_ptr(table, fam);
+			if (rnh == NULL)
+				panic("%s: rnh NULL", __func__);
+			dom->dom_rtdetach((void **)rnh, dom->dom_rtoffset);
 		}
 	}
 }
@@ -274,7 +277,8 @@ sys_setfib(struct thread *td, struct setfib_args *uap)
 void
 rtalloc(struct route *ro)
 {
-	rtalloc_ign_fib(ro, 0UL, 0);
+
+	rtalloc_ign_fib(ro, 0UL, RT_DEFAULT_FIB);
 }
 
 void
@@ -294,7 +298,7 @@ rtalloc_ign(struct route *ro, u_long ignore)
 		RTFREE(rt);
 		ro->ro_rt = NULL;
 	}
-	ro->ro_rt = rtalloc1_fib(&ro->ro_dst, 1, ignore, 0);
+	ro->ro_rt = rtalloc1_fib(&ro->ro_dst, 1, ignore, RT_DEFAULT_FIB);
 	if (ro->ro_rt)
 		RT_UNLOCK(ro->ro_rt);
 }
@@ -324,7 +328,8 @@ rtalloc_ign_fib(struct route *ro, u_long ignore, u_int fibnum)
 struct rtentry *
 rtalloc1(struct sockaddr *dst, int report, u_long ignflags)
 {
-	return (rtalloc1_fib(dst, report, ignflags, 0));
+
+	return (rtalloc1_fib(dst, report, ignflags, RT_DEFAULT_FIB));
 }
 
 struct rtentry *
@@ -339,8 +344,15 @@ rtalloc1_fib(struct sockaddr *dst, int report, u_long ignflags,
 	int needlock;
 
 	KASSERT((fibnum < rt_numfibs), ("rtalloc1_fib: bad fibnum"));
-	if (dst->sa_family != AF_INET)	/* Only INET supports > 1 fib now */
-		fibnum = 0;
+	switch (dst->sa_family) {
+	case AF_INET6:
+	case AF_INET:
+		/* We support multiple FIBs. */
+		break;
+	default:
+		fibnum = RT_DEFAULT_FIB;
+		break;
+	}
 	rnh = rt_tables_get_rnh(fibnum, dst->sa_family);
 	newrt = NULL;
 	if (rnh == NULL)
@@ -486,7 +498,8 @@ rtredirect(struct sockaddr *dst,
 	int flags,
 	struct sockaddr *src)
 {
-	rtredirect_fib(dst, gateway, netmask, flags, src, 0);
+
+	rtredirect_fib(dst, gateway, netmask, flags, src, RT_DEFAULT_FIB);
 }
 
 void
@@ -617,7 +630,8 @@ out:
 int
 rtioctl(u_long req, caddr_t data)
 {
-	return (rtioctl_fib(req, data, 0));
+
+	return (rtioctl_fib(req, data, RT_DEFAULT_FIB));
 }
 
 /*
@@ -647,7 +661,8 @@ rtioctl_fib(u_long req, caddr_t data, u_int fibnum)
 struct ifaddr *
 ifa_ifwithroute(int flags, struct sockaddr *dst, struct sockaddr *gateway)
 {
-	return (ifa_ifwithroute_fib(flags, dst, gateway, 0));
+
+	return (ifa_ifwithroute_fib(flags, dst, gateway, RT_DEFAULT_FIB));
 }
 
 struct ifaddr *
@@ -732,7 +747,9 @@ rtrequest(int req,
 	int flags,
 	struct rtentry **ret_nrt)
 {
-	return (rtrequest_fib(req, dst, gateway, netmask, flags, ret_nrt, 0));
+
+	return (rtrequest_fib(req, dst, gateway, netmask, flags, ret_nrt,
+	    RT_DEFAULT_FIB));
 }
 
 int
@@ -771,7 +788,8 @@ rtrequest_fib(int req,
 int
 rt_getifa(struct rt_addrinfo *info)
 {
-	return (rt_getifa_fib(info, 0));
+
+	return (rt_getifa_fib(info, RT_DEFAULT_FIB));
 }
 
 /*
@@ -1029,8 +1047,16 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 #define senderr(x) { error = x ; goto bad; }
 
 	KASSERT((fibnum < rt_numfibs), ("rtrequest1_fib: bad fibnum"));
-	if (dst->sa_family != AF_INET)	/* Only INET supports > 1 fib now */
-		fibnum = 0;
+	switch (dst->sa_family) {
+	case AF_INET6:
+	case AF_INET:
+		/* We support multiple FIBs. */
+		break;
+	default:
+		fibnum = RT_DEFAULT_FIB;
+		break;
+	}
+
 	/*
 	 * Find the correct routing tree to use for this Address Family
 	 */
@@ -1135,8 +1161,7 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 		rt->rt_flags = RTF_UP | flags;
 		rt->rt_fibnum = fibnum;
 		/*
-		 * Add the gateway. Possibly re-malloc-ing the storage for it
-		 * 
+		 * Add the gateway. Possibly re-malloc-ing the storage for it.
 		 */
 		RT_LOCK(rt);
 		if ((error = rt_setgate(rt, dst, gateway)) != 0) {
@@ -1182,12 +1207,15 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 
 #ifdef FLOWTABLE
 		rt0 = NULL;
-		/* XXX
-		 * "flow-table" only support IPv4 at the moment.
-		 * XXX-BZ as of r205066 it would support IPv6.
-		 */
+		/* "flow-table" only supports IPv6 and IPv4 at the moment. */
+		switch (dst->sa_family) {
+#ifdef INET6
+		case AF_INET6:
+#endif
 #ifdef INET
-		if (dst->sa_family == AF_INET) {
+		case AF_INET:
+#endif
+#if defined(INET6) || defined(INET)
 			rn = rnh->rnh_matchaddr(dst, rnh);
 			if (rn && ((rn->rn_flags & RNF_ROOT) == 0)) {
 				struct sockaddr *mask;
@@ -1226,9 +1254,9 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 					}
 				}
 			}
+#endif/* INET6 || INET */
 		}
-#endif
-#endif
+#endif /* FLOWTABLE */
 
 		/* XXX mtu manipulation will be done in rnh_addaddr -- itojun */
 		rn = rnh->rnh_addaddr(ndst, netmask, rnh, rt->rt_nodes);
@@ -1249,9 +1277,18 @@ rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt,
 		} 
 #ifdef FLOWTABLE
 		else if (rt0 != NULL) {
+			switch (dst->sa_family) {
+#ifdef INET6
+			case AF_INET6:
+				flowtable_route_flush(V_ip6_ft, rt0);
+				break;
+#endif
 #ifdef INET
-			flowtable_route_flush(V_ip_ft, rt0);
+			case AF_INET:
+				flowtable_route_flush(V_ip_ft, rt0);
+				break;
 #endif
+			}
 			RTFREE(rt0);
 		}
 #endif
@@ -1383,8 +1420,17 @@ rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum)
 		dst = ifa->ifa_addr;
 		netmask = ifa->ifa_netmask;
 	}
-	if ( dst->sa_family != AF_INET)
-		fibnum = 0;
+	if (dst->sa_len == 0)
+		return(EINVAL);
+	switch (dst->sa_family) {
+	case AF_INET6:
+	case AF_INET:
+		/* We support multiple FIBs. */
+		break;
+	default:
+		fibnum = RT_DEFAULT_FIB;
+		break;
+	}
 	if (fibnum == -1) {
 		if (rt_add_addr_allfibs == 0 && cmd == (int)RTM_ADD) {
 			startfib = endfib = curthread->td_proc->p_fibnum;
@@ -1397,8 +1443,6 @@ rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum)
 		startfib = fibnum;
 		endfib = fibnum;
 	}
-	if (dst->sa_len == 0)
-		return(EINVAL);
 
 	/*
 	 * If it's a delete, check that if it exists,
@@ -1422,9 +1466,7 @@ rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum)
 	 * Now go through all the requested tables (fibs) and do the
 	 * requested action. Realistically, this will either be fib 0
 	 * for protocols that don't do multiple tables or all the
-	 * tables for those that do. XXX For this version only AF_INET.
-	 * When that changes code should be refactored to protocol
-	 * independent parts and protocol dependent parts.
+	 * tables for those that do.
 	 */
 	for ( fibnum = startfib; fibnum <= endfib; fibnum++) {
 		if (cmd == RTM_DELETE) {
@@ -1564,12 +1606,14 @@ rtinit1(struct ifaddr *ifa, int cmd, int flags, int fibnum)
 	return (error);
 }
 
+#ifndef BURN_BRIDGES
 /* special one for inet internal use. may not use. */
 int
 rtinit_fib(struct ifaddr *ifa, int cmd, int flags)
 {
 	return (rtinit1(ifa, cmd, flags, -1));
 }
+#endif
 
 /*
  * Set up a routing table entry, normally
@@ -1579,7 +1623,7 @@ int
 rtinit(struct ifaddr *ifa, int cmd, int flags)
 {
 	struct sockaddr *dst;
-	int fib = 0;
+	int fib = RT_DEFAULT_FIB;
 
 	if (flags & RTF_HOST) {
 		dst = ifa->ifa_dstaddr;
@@ -1587,7 +1631,12 @@ rtinit(struct ifaddr *ifa, int cmd, int flags)
 		dst = ifa->ifa_addr;
 	}
 
-	if (dst->sa_family == AF_INET)
+	switch (dst->sa_family) {
+	case AF_INET6:
+	case AF_INET:
+		/* We do support multiple FIBs. */
 		fib = -1;
+		break;
+	}
 	return (rtinit1(ifa, cmd, flags, fib));
 }
diff --git a/sys/net/route.h b/sys/net/route.h
index 7e045b7..a400deb 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -111,6 +111,7 @@ struct rt_metrics {
  #endif
 #endif
 
+#define	RT_DEFAULT_FIB	0	/* Explicitly mark fib=0 restricted cases */
 extern u_int rt_numfibs;	/* number fo usable routing tables */
 /*
  * XXX kernel function pointer `rt_output' is visible to applications.
@@ -405,8 +406,10 @@ void	 rtredirect(struct sockaddr *, struct sockaddr *,
 int	 rtrequest(int, struct sockaddr *,
 	    struct sockaddr *, struct sockaddr *, int, struct rtentry **);
 
+#ifndef BURN_BRIDGES
 /* defaults to "all" FIBs */
 int	 rtinit_fib(struct ifaddr *, int, int);
+#endif
 
 /* XXX MRT NEW VERSIONS THAT USE FIBs
  * For now the protocol indepedent versions are the same as the AF_INET ones