MFC r313025, r313395, r314113, r314442, r315458, r315656

r313025:
Add tests for multi-fib IPv6 routing

PR:		196361
Submitted by:	jhujhiti@adjectivism.org
Reported by:	Jason Healy <jhealy@logn.net>
MFC after:	4 weeks
Sponsored by:	Spectra Logic Corp

r313395:
Add fibs_test:udp_dontroute6, another IPv6 multi-FIB test

PR:		196361
MFC after:	3 weeks
Sponsored by:	Spectra Logic Corp

r314113:
Remove tests/sys/netinet/fibs_tests's dependency on net/socat

Instead of bridging two tap interfaces with socat, just use an epair pair.

MFC after:	3 weeks
Sponsored by:	Spectra Logic Corp

r314442:
Add an ATF test for IPv6 SLAAC with multiple fibs

Tests that an interface can get a SLAAC address and that it inserts its
routes into the correct fib. Does not test anything to do with NDP.

PR:		196361
Reviewed by:	Erick Turnquist <jhujhiti@adjectivism.org>
MFC after:	3 weeks
Sponsored by:	Spectra Logic Corp
Differential Revision:	https://reviews.freebsd.org/D9776

r315458:
Constrain IPv6 routes to single FIBs when net.add_addr_allfibs=0

sys/netinet6/icmp6.c
	Use the interface's FIB for source address selection in ICMPv6 error
	responses.

sys/netinet6/in6.c
	In in6_newaddrmsg, announce arrival of local addresses on the
	interface's FIB only.  In in6_lltable_rtcheck, use a per-fib ND6
	cache instead of a single cache.

sys/netinet6/in6_src.c
	In in6_selectsrc, use the caller's fib instead of the default fib.
	In in6_selectsrc_socket, remove a superfluous check.

sys/netinet6/nd6.c
	In nd6_lle_event, use the interface's fib for routing socket
	messages.  In nd6_is_new_addr_neighbor, check all FIBs when trying
	to determine whether an address is a neighbor.  Also, simplify the
	code for point to point interfaces.

sys/netinet6/nd6.h
sys/netinet6/nd6.c
sys/netinet6/nd6_rtr.c
	Make defrouter_select fib-aware, and make all of its callers pass in
	the interface fib.

sys/netinet6/nd6_nbr.c
	When inputting a Neighbor Solicitation packet, consider the
	interface fib instead of the default fib for DAD.  Output NS and
	Neighbor Advertisement packets on the correct fib.

sys/netinet6/nd6_rtr.c
	Allow installing the same host route on different interfaces in
	different FIBs.  If rt_add_addr_allfibs=0, only install or delete
	the prefix route on the interface fib.

tests/sys/netinet/fibs_test.sh
	Clear some expected failures, but add a skip for the newly revealed
	BUG217871.

PR:		196361
Submitted by:	Erick Turnquist <jhujhiti@adjectivism.org>
Reported by:	Jason Healy <jhealy@logn.net>
Reviewed by:	asomers
MFC after:	3 weeks
Sponsored by:	Spectra Logic Corp
Differential Revision:	https://reviews.freebsd.org/D9451

r315656:
Fix back-to-back runs of sys/netinet/fibs_test;slaac_on_nondefault_fib6

This test was failing if run twice because rtadvd takes too long to die.
The rtadvd process from the first run was still running when the
second run created its interfaces.  The solution is to use SIGKILL during
the cleanup instead of SIGTERM so rtadvd will die faster.

While I'm here, randomize the addresses used for the test, which makes bugs
like this easier to spot, and fix the cleanup order to be the opposite of
the setup order

PR:		217871
MFC after:	18 days
X-MFC-With:	315458
Sponsored by:	Spectra Logic Corp

7 files changed, 140 insertions, 68 deletions

diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index d6d2784..a94ed8f 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -2147,7 +2147,7 @@ icmp6_reflect(struct mbuf *m, size_t off)
 		 * source address of the erroneous packet.
 		 */
 		in6_splitscope(&ip6->ip6_src, &dst6, &scopeid);
-		error = in6_selectsrc_addr(RT_DEFAULT_FIB, &dst6,
+		error = in6_selectsrc_addr(M_GETFIB(m), &dst6,
 		    scopeid, NULL, &src6, &hlim);
 
 		if (error) {
@@ -2289,7 +2289,7 @@ icmp6_redirect_input(struct mbuf *m, int off)
 	uint32_t scopeid;
 
 	in6_splitscope(&reddst6, &kdst, &scopeid);
-	if (fib6_lookup_nh_basic(RT_DEFAULT_FIB, &kdst, scopeid, 0, 0,&nh6)==0){
+	if (fib6_lookup_nh_basic(ifp->if_fib, &kdst, scopeid, 0, 0,&nh6)==0){
 		if ((nh6.nh_flags & NHF_GATEWAY) == 0) {
 			nd6log((LOG_ERR,
 			    "ICMP6 redirect rejected; no route "
diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index 8f1cae9..4df3564 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -159,6 +159,7 @@ in6_newaddrmsg(struct in6_ifaddr *ia, int cmd)
 	struct sockaddr_dl gateway;
 	struct sockaddr_in6 mask, addr;
 	struct rtentry rt;
+	int fibnum;
 
 	/*
 	 * initialize for rtmsg generation
@@ -176,8 +177,9 @@ in6_newaddrmsg(struct in6_ifaddr *ia, int cmd)
 	rt.rt_flags = RTF_HOST | RTF_STATIC;
 	if (cmd == RTM_ADD)
 		rt.rt_flags |= RTF_UP;
-	/* Announce arrival of local address to all FIBs. */
-	rt_newaddrmsg(cmd, &ia->ia_ifa, 0, &rt);
+	fibnum = V_rt_add_addr_allfibs ? RT_ALL_FIBS : ia62ifa(ia)->ifa_ifp->if_fib;
+	/* Announce arrival of local address to this FIB. */
+	rt_newaddrmsg_fib(cmd, &ia->ia_ifa, 0, &rt, fibnum);
 }
 
 int
@@ -2115,15 +2117,15 @@ in6_lltable_rtcheck(struct ifnet *ifp,
 	uint32_t scopeid;
 	int error;
 	char ip6buf[INET6_ADDRSTRLEN];
+	int fibnum;
 
 	KASSERT(l3addr->sa_family == AF_INET6,
 	    ("sin_family %d", l3addr->sa_family));
 
-	/* Our local addresses are always only installed on the default FIB. */
-
 	sin6 = (const struct sockaddr_in6 *)l3addr;
 	in6_splitscope(&sin6->sin6_addr, &dst, &scopeid);
-	error = fib6_lookup_nh_basic(RT_DEFAULT_FIB, &dst, scopeid, 0, 0, &nh6);
+	fibnum = V_rt_add_addr_allfibs ? RT_DEFAULT_FIB : ifp->if_fib;
+	error = fib6_lookup_nh_basic(fibnum, &dst, scopeid, 0, 0, &nh6);
 	if (error != 0 || (nh6.nh_flags & NHF_GATEWAY) || nh6.nh_ifp != ifp) {
 		struct ifaddr *ifa;
 		/*
diff --git a/sys/netinet6/in6_src.c b/sys/netinet6/in6_src.c
index 54c0a0a..59d0cb9 100644
--- a/sys/netinet6/in6_src.c
+++ b/sys/netinet6/in6_src.c
@@ -297,7 +297,7 @@ in6_selectsrc(uint32_t fibnum, struct sockaddr_in6 *dstsock,
 	 */
 	/* get the outgoing interface */
 	if ((error = in6_selectif(dstsock, opts, mopts, &ifp, oifp,
-	    (inp != NULL) ? inp->inp_inc.inc_fibnum : RT_DEFAULT_FIB)) != 0)
+	    (inp != NULL) ? inp->inp_inc.inc_fibnum : fibnum)) != 0)
 		return (error);
 
 #ifdef DIAGNOSTIC
@@ -563,7 +563,7 @@ in6_selectsrc_socket(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts,
 	uint32_t fibnum;
 	int error;
 
-	fibnum = (inp != NULL) ? inp->inp_inc.inc_fibnum : RT_DEFAULT_FIB;
+	fibnum = inp->inp_inc.inc_fibnum;
 	retifp = NULL;
 
 	error = in6_selectsrc(fibnum, dstsock, opts, inp, cred, &retifp, srcp);
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index ebaaedc..d7a8760 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -157,6 +157,7 @@ nd6_lle_event(void *arg __unused, struct llentry *lle, int evt)
 	struct sockaddr_dl gw;
 	struct ifnet *ifp;
 	int type;
+	int fibnum;
 
 	LLE_WLOCK_ASSERT(lle);
 
@@ -194,8 +195,9 @@ nd6_lle_event(void *arg __unused, struct llentry *lle, int evt)
 	rtinfo.rti_info[RTAX_DST] = (struct sockaddr *)&dst;
 	rtinfo.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&gw;
 	rtinfo.rti_addrs = RTA_DST | RTA_GATEWAY;
+	fibnum = V_rt_add_addr_allfibs ? RT_ALL_FIBS : ifp->if_fib;
 	rt_missmsg_fib(type, &rtinfo, RTF_HOST | RTF_LLDATA | (
-	    type == RTM_ADD ? RTF_UP: 0), 0, RT_DEFAULT_FIB);
+	    type == RTM_ADD ? RTF_UP: 0), 0, fibnum);
 }
 
 /*
@@ -1200,7 +1202,7 @@ nd6_purge(struct ifnet *ifp)
 
 	if (ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV) {
 		/* Refresh default router list. */
-		defrouter_select();
+		defrouter_select_fib(ifp->if_fib);
 	}
 }
 
@@ -1253,7 +1255,7 @@ static int
 nd6_is_new_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp)
 {
 	struct nd_prefix *pr;
-	struct ifaddr *dstaddr;
+	struct ifaddr *ifa;
 	struct rt_addrinfo info;
 	struct sockaddr_in6 rt_key;
 	const struct sockaddr *dst6;
@@ -1287,9 +1289,6 @@ nd6_is_new_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp)
 	bzero(&info, sizeof(info));
 	info.rti_info[RTAX_DST] = (struct sockaddr *)&rt_key;
 
-	/* Always use the default FIB here. XXME - why? */
-	fibnum = RT_DEFAULT_FIB;
-
 	/*
 	 * If the address matches one of our addresses,
 	 * it should be a neighbor.
@@ -1303,19 +1302,31 @@ restart:
 			continue;
 
 		if ((pr->ndpr_stateflags & NDPRF_ONLINK) == 0) {
-			/* Always use the default FIB here. */
 			dst6 = (const struct sockaddr *)&pr->ndpr_prefix;
 
-			genid = V_nd6_list_genid;
-			ND6_RUNLOCK();
-
-			/* Restore length field before retrying lookup */
-			rt_key.sin6_len = sizeof(rt_key);
-			error = rib_lookup_info(fibnum, dst6, 0, 0, &info);
+			/*
+			 * We only need to check all FIBs if add_addr_allfibs
+			 * is unset. If set, checking any FIB will suffice.
+			 */
+			fibnum = V_rt_add_addr_allfibs ? rt_numfibs - 1 : 0;
+			for (; fibnum < rt_numfibs; fibnum++) {
+				genid = V_nd6_list_genid;
+				ND6_RUNLOCK();
 
-			ND6_RLOCK();
-			if (genid != V_nd6_list_genid)
-				goto restart;
+				/*
+				 * Restore length field before
+				 * retrying lookup
+				 */
+				rt_key.sin6_len = sizeof(rt_key);
+				error = rib_lookup_info(fibnum, dst6, 0, 0,
+						        &info);
+
+				ND6_RLOCK();
+				if (genid != V_nd6_list_genid)
+					goto restart;
+				if (error == 0)
+					break;
+			}
 			if (error != 0)
 				continue;
 
@@ -1346,13 +1357,18 @@ restart:
 	 * If the address is assigned on the node of the other side of
 	 * a p2p interface, the address should be a neighbor.
 	 */
-	dstaddr = ifa_ifwithdstaddr((const struct sockaddr *)addr, RT_ALL_FIBS);
-	if (dstaddr != NULL) {
-		if (dstaddr->ifa_ifp == ifp) {
-			ifa_free(dstaddr);
-			return (1);
+	if (ifp->if_flags & IFF_POINTOPOINT) {
+		IF_ADDR_RLOCK(ifp);
+		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+			if (ifa->ifa_addr->sa_family != addr->sin6_family)
+				continue;
+			if (ifa->ifa_dstaddr != NULL &&
+			    sa_equal(addr, ifa->ifa_dstaddr)) {
+				IF_ADDR_RUNLOCK(ifp);
+				return 1;
+			}
 		}
-		ifa_free(dstaddr);
+		IF_ADDR_RUNLOCK(ifp);
 	}
 
 	/*
@@ -1485,7 +1501,7 @@ nd6_free(struct llentry **lnp, int gc)
 			/*
 			 * We need to unlock to avoid a LOR with rt6_flush() with the
 			 * rnh and for the calls to pfxlist_onlink_check() and
-			 * defrouter_select() in the block further down for calls
+			 * defrouter_select_fib() in the block further down for calls
 			 * into nd6_lookup().  We still hold a ref.
 			 */
 			LLE_WUNLOCK(ln);
@@ -1500,7 +1516,7 @@ nd6_free(struct llentry **lnp, int gc)
 
 		if (dr) {
 			/*
-			 * Since defrouter_select() does not affect the
+			 * Since defrouter_select_fib() does not affect the
 			 * on-link determination and MIP6 needs the check
 			 * before the default router selection, we perform
 			 * the check now.
@@ -1510,7 +1526,7 @@ nd6_free(struct llentry **lnp, int gc)
 			/*
 			 * Refresh default router list.
 			 */
-			defrouter_select();
+			defrouter_select_fib(dr->ifp->if_fib);
 		}
 
 		/*
@@ -2104,11 +2120,11 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 	 * Question: can we restrict the first condition to the "is_newentry"
 	 * case?
 	 * XXX: when we hear an RA from a new router with the link-layer
-	 * address option, defrouter_select() is called twice, since
+	 * address option, defrouter_select_fib() is called twice, since
 	 * defrtrlist_update called the function as well.  However, I believe
 	 * we can compromise the overhead, since it only happens the first
 	 * time.
-	 * XXX: although defrouter_select() should not have a bad effect
+	 * XXX: although defrouter_select_fib() should not have a bad effect
 	 * for those are not autoconfigured hosts, we explicitly avoid such
 	 * cases for safety.
 	 */
@@ -2117,7 +2133,7 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 		/*
 		 * guaranteed recursion
 		 */
-		defrouter_select();
+		defrouter_select_fib(ifp->if_fib);
 	}
 }
 
diff --git a/sys/netinet6/nd6.h b/sys/netinet6/nd6.h
index 9b9fa3d..243e954 100644
--- a/sys/netinet6/nd6.h
+++ b/sys/netinet6/nd6.h
@@ -469,6 +469,7 @@ void nd6_dad_stop(struct ifaddr *);
 void nd6_rs_input(struct mbuf *, int, int);
 void nd6_ra_input(struct mbuf *, int, int);
 void defrouter_reset(void);
+void defrouter_select_fib(int fibnum);
 void defrouter_select(void);
 void defrouter_ref(struct nd_defrouter *);
 void defrouter_rele(struct nd_defrouter *);
diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c
index 2ac1d7a..8bbdf85 100644
--- a/sys/netinet6/nd6_nbr.c
+++ b/sys/netinet6/nd6_nbr.c
@@ -262,8 +262,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 		bzero(&info, sizeof(info));
 		info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&rt_gateway;
 
-		/* Always use the default FIB. */
-		if (rib_lookup_info(RT_DEFAULT_FIB, (struct sockaddr *)&dst6,
+		if (rib_lookup_info(ifp->if_fib, (struct sockaddr *)&dst6,
 		    0, 0, &info) == 0) {
 			if ((info.rti_flags & RTF_ANNOUNCE) != 0 &&
 			    rt_gateway.sdl_family == AF_LINK) {
@@ -485,7 +484,7 @@ nd6_ns_output_fib(struct ifnet *ifp, const struct in6_addr *saddr6,
 			uint32_t scopeid;
 
 			in6_splitscope(&ip6->ip6_dst, &dst6, &scopeid);
-			error = in6_selectsrc_addr(RT_DEFAULT_FIB, &dst6,
+			error = in6_selectsrc_addr(fibnum, &dst6,
 			    scopeid, ifp, &src6, NULL);
 			if (error) {
 				char ip6buf[INET6_ADDRSTRLEN];
@@ -982,7 +981,7 @@ nd6_na_output_fib(struct ifnet *ifp, const struct in6_addr *daddr6_0,
 	 * Select a source whose scope is the same as that of the dest.
 	 */
 	in6_splitscope(&daddr6, &dst6, &scopeid);
-	error = in6_selectsrc_addr(RT_DEFAULT_FIB, &dst6,
+	error = in6_selectsrc_addr(fibnum, &dst6,
 	    scopeid, ifp, &src6, NULL);
 	if (error) {
 		char ip6buf[INET6_ADDRSTRLEN];
diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c
index a44a8b1..193ce99 100644
--- a/sys/netinet6/nd6_rtr.c
+++ b/sys/netinet6/nd6_rtr.c
@@ -500,7 +500,7 @@ defrouter_addreq(struct nd_defrouter *new)
 
 	error = in6_rtrequest(RTM_ADD, (struct sockaddr *)&def,
 	    (struct sockaddr *)&gate, (struct sockaddr *)&mask,
-	    RTF_GATEWAY, &newrt, RT_DEFAULT_FIB);
+	    RTF_GATEWAY, &newrt, new->ifp->if_fib);
 	if (newrt) {
 		nd6_rtmsg(RTM_ADD, newrt); /* tell user process */
 		RTFREE(newrt);
@@ -551,8 +551,8 @@ defrouter_rele(struct nd_defrouter *dr)
 
 /*
  * Remove the default route for a given router.
- * This is just a subroutine function for defrouter_select(), and should
- * not be called from anywhere else.
+ * This is just a subroutine function for defrouter_select_fib(), and
+ * should not be called from anywhere else.
  */
 static void
 defrouter_delreq(struct nd_defrouter *dr)
@@ -571,7 +571,7 @@ defrouter_delreq(struct nd_defrouter *dr)
 
 	in6_rtrequest(RTM_DELETE, (struct sockaddr *)&def,
 	    (struct sockaddr *)&gate,
-	    (struct sockaddr *)&mask, RTF_GATEWAY, &oldrt, RT_DEFAULT_FIB);
+	    (struct sockaddr *)&mask, RTF_GATEWAY, &oldrt, dr->ifp->if_fib);
 	if (oldrt) {
 		nd6_rtmsg(RTM_DELETE, oldrt);
 		RTFREE(oldrt);
@@ -698,11 +698,11 @@ defrouter_del(struct nd_defrouter *dr)
 
 	/*
 	 * If the router is the primary one, choose a new one.
-	 * Note that defrouter_select() will remove the current gateway
-	 * from the routing table.
+	 * Note that defrouter_select_fib() will remove the current
+         * gateway from the routing table.
 	 */
 	if (deldr)
-		defrouter_select();
+		defrouter_select_fib(deldr->ifp->if_fib);
 
 	/*
 	 * Release the list reference.
@@ -730,13 +730,23 @@ defrouter_del(struct nd_defrouter *dr)
  * even when the multipath routing is available, because we're not sure about
  * the benefits for stub hosts comparing to the risk of making the code
  * complicated and the possibility of introducing bugs.
+ *
+ * We maintain a single list of routers for multiple FIBs, only considering one
+ * at a time based on the receiving interface's FIB. If @fibnum is RT_ALL_FIBS,
+ * we do the whole thing multiple times.
  */
 void
-defrouter_select(void)
+defrouter_select_fib(int fibnum)
 {
 	struct nd_defrouter *dr, *selected_dr, *installed_dr;
 	struct llentry *ln = NULL;
 
+	if (fibnum == RT_ALL_FIBS) {
+		for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
+			defrouter_select_fib(fibnum);
+		}
+	}
+
 	ND6_RLOCK();
 	/*
 	 * Let's handle easy case (3) first:
@@ -755,7 +765,7 @@ defrouter_select(void)
 	selected_dr = installed_dr = NULL;
 	TAILQ_FOREACH(dr, &V_nd_defrouter, dr_entry) {
 		IF_AFDATA_RLOCK(dr->ifp);
-		if (selected_dr == NULL &&
+		if (selected_dr == NULL && dr->ifp->if_fib == fibnum &&
 		    (ln = nd6_lookup(&dr->rtaddr, 0, dr->ifp)) &&
 		    ND6_IS_LLINFO_PROBREACH(ln)) {
 			selected_dr = dr;
@@ -767,14 +777,17 @@ defrouter_select(void)
 			ln = NULL;
 		}
 
-		if (dr->installed) {
+		if (dr->installed && dr->ifp->if_fib == fibnum) {
 			if (installed_dr == NULL) {
 				installed_dr = dr;
 				defrouter_ref(installed_dr);
 			} else {
-				/* this should not happen.  warn for diagnosis. */
-				log(LOG_ERR,
-		    "defrouter_select: more than one router is installed\n");
+				/*
+				 * this should not happen.
+				 * warn for diagnosis.
+				 */
+				log(LOG_ERR, "defrouter_select_fib: more than "
+				             "one router is installed\n");
 			}
 		}
 	}
@@ -789,14 +802,24 @@ defrouter_select(void)
 	if (selected_dr == NULL) {
 		if (installed_dr == NULL ||
 		    TAILQ_NEXT(installed_dr, dr_entry) == NULL)
-			selected_dr = TAILQ_FIRST(&V_nd_defrouter);
+			dr = TAILQ_FIRST(&V_nd_defrouter);
 		else
-			selected_dr = TAILQ_NEXT(installed_dr, dr_entry);
-		defrouter_ref(selected_dr);
+			dr = TAILQ_NEXT(installed_dr, dr_entry);
+
+		/* Ensure we select a router for this FIB. */
+		TAILQ_FOREACH_FROM(dr, &V_nd_defrouter, dr_entry) {
+			if (dr->ifp->if_fib == fibnum) {
+				selected_dr = dr;
+				defrouter_ref(selected_dr);
+				break;
+			}
+		}
 	} else if (installed_dr != NULL) {
 		IF_AFDATA_RLOCK(installed_dr->ifp);
-		if ((ln = nd6_lookup(&installed_dr->rtaddr, 0, installed_dr->ifp)) &&
+		if ((ln = nd6_lookup(&installed_dr->rtaddr, 0,
+		                     installed_dr->ifp)) &&
 		    ND6_IS_LLINFO_PROBREACH(ln) &&
+		    installed_dr->ifp->if_fib == fibnum &&
 		    rtpref(selected_dr) <= rtpref(installed_dr)) {
 			defrouter_rele(selected_dr);
 			selected_dr = installed_dr;
@@ -808,18 +831,30 @@ defrouter_select(void)
 	ND6_RUNLOCK();
 
 	/*
-	 * If the selected router is different than the installed one,
-	 * remove the installed router and install the selected one.
-	 * Note that the selected router is never NULL here.
+	 * If we selected a router for this FIB and it's different
+	 * than the installed one, remove the installed router and
+	 * install the selected one in its place.
 	 */
 	if (installed_dr != selected_dr) {
 		if (installed_dr != NULL) {
 			defrouter_delreq(installed_dr);
 			defrouter_rele(installed_dr);
 		}
-		defrouter_addreq(selected_dr);
+		if (selected_dr != NULL)
+			defrouter_addreq(selected_dr);
 	}
-	defrouter_rele(selected_dr);
+	if (selected_dr != NULL)
+		defrouter_rele(selected_dr);
+}
+
+/*
+ * Maintain old KPI for default router selection.
+ * If unspecified, we can re-select routers for all FIBs.
+ */
+void
+defrouter_select(void)
+{
+	defrouter_select_fib(RT_ALL_FIBS);
 }
 
 /*
@@ -942,7 +977,7 @@ restart:
 	V_nd6_list_genid++;
 	ND6_WUNLOCK();
 
-	defrouter_select();
+	defrouter_select_fib(new->ifp->if_fib);
 
 	return (n);
 }
@@ -1731,7 +1766,7 @@ nd6_prefix_onlink_rtrequest(struct nd_prefix *pr, struct ifaddr *ifa)
 	struct rtentry *rt;
 	struct sockaddr_in6 mask6;
 	u_long rtflags;
-	int error, a_failure, fibnum;
+	int error, a_failure, fibnum, maxfib;
 
 	/*
 	 * in6_ifinit() sets nd6_rtrequest to ifa_rtrequest for all ifaddrs.
@@ -1742,8 +1777,15 @@ nd6_prefix_onlink_rtrequest(struct nd_prefix *pr, struct ifaddr *ifa)
 	mask6.sin6_addr = pr->ndpr_mask;
 	rtflags = (ifa->ifa_flags & ~IFA_RTSELF) | RTF_UP;
 
+	if(V_rt_add_addr_allfibs) {
+		fibnum = 0;
+		maxfib = rt_numfibs;
+	} else {
+		fibnum = ifa->ifa_ifp->if_fib;
+		maxfib = fibnum + 1;
+	}
 	a_failure = 0;
-	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
+	for (; fibnum < maxfib; fibnum++) {
 
 		rt = NULL;
 		error = in6_rtrequest(RTM_ADD,
@@ -1831,6 +1873,10 @@ nd6_prefix_onlink(struct nd_prefix *pr)
 		if ((opr->ndpr_stateflags & NDPRF_ONLINK) == 0)
 			continue;
 
+		if (!V_rt_add_addr_allfibs &&
+		    opr->ndpr_ifp->if_fib != pr->ndpr_ifp->if_fib)
+			continue;
+
 		if (opr->ndpr_plen == pr->ndpr_plen &&
 		    in6_are_prefix_equal(&pr->ndpr_prefix.sin6_addr,
 		    &opr->ndpr_prefix.sin6_addr, pr->ndpr_plen)) {
@@ -1891,7 +1937,7 @@ nd6_prefix_offlink(struct nd_prefix *pr)
 	struct rtentry *rt;
 	char ip6buf[INET6_ADDRSTRLEN];
 	uint64_t genid;
-	int fibnum, a_failure;
+	int fibnum, maxfib, a_failure;
 
 	ND6_ONLINK_LOCK_ASSERT();
 	ND6_UNLOCK_ASSERT();
@@ -1909,8 +1955,16 @@ nd6_prefix_offlink(struct nd_prefix *pr)
 	mask6.sin6_len = sizeof(sa6);
 	bcopy(&pr->ndpr_mask, &mask6.sin6_addr, sizeof(struct in6_addr));
 
+	if (V_rt_add_addr_allfibs) {
+		fibnum = 0;
+		maxfib = rt_numfibs;
+	} else {
+		fibnum = ifp->if_fib;
+		maxfib = fibnum + 1;
+	}
+
 	a_failure = 0;
-	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
+	for (; fibnum < maxfib; fibnum++) {
 		rt = NULL;
 		error = in6_rtrequest(RTM_DELETE, (struct sockaddr *)&sa6, NULL,
 		    (struct sockaddr *)&mask6, 0, &rt, fibnum);