Fix lagg failover due to missing notifications

When using lagg failover mode neither Gratuitous ARP (IPv4) or Unsolicited
Neighbour Advertisements (IPv6) are sent to notify other nodes that the
address may have moved.

This results is slow failover, dropped packets and network outages for the
lagg interface when the primary link goes down.

We now use the new if_link_state_change_cond with the force param set to
allow lagg to force through link state changes and hence fire a
ifnet_link_event which are now monitored by rip and nd6.

Upon receiving these events each protocol trigger the relevant
notifications:
* inet4 => Gratuitous ARP
* inet6 => Unsolicited Neighbour Announce

This also fixes the carp IPv6 NA's that stopped working after r251584 which
added the ipv6_route__llma route.

The new behavour can be controlled using the sysctls:
* net.link.ether.inet.arp_on_link
* net.inet6.icmp6.nd6_on_link

Also removed unused param from lagg_port_state and added descriptions for the
sysctls while here.

PR:		156226
MFC after:	1 month
Sponsored by:	Multiplay
Differential Revision:	https://reviews.freebsd.org/D4111

5 files changed, 164 insertions, 30 deletions

diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index 2b022ba..e815a96 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -114,7 +114,7 @@ VNET_DECLARE(int, icmp6_nodeinfo_oldmcprefix);
 #define V_icmp6_nodeinfo_oldmcprefix	VNET(icmp6_nodeinfo_oldmcprefix)
 
 /*
- * Definitions of some costant IP6 addresses.
+ * Definitions of some constant IP6 addresses.
  */
 const struct in6_addr in6addr_any = IN6ADDR_ANY_INIT;
 const struct in6_addr in6addr_loopback = IN6ADDR_LOOPBACK_INIT;
diff --git a/sys/netinet6/in6_var.h b/sys/netinet6/in6_var.h
index 007fd66..54fa1f5 100644
--- a/sys/netinet6/in6_var.h
+++ b/sys/netinet6/in6_var.h
@@ -399,6 +399,16 @@ struct	in6_rrenumreq {
 #define IA6_SIN6(ia)	(&((ia)->ia_addr))
 #define IA6_DSTSIN6(ia)	(&((ia)->ia_dstaddr))
 #define IFA_IN6(x)	(&((struct sockaddr_in6 *)((x)->ifa_addr))->sin6_addr)
+#define IFA_IN6_FLAGS(ifa)	((struct in6_ifaddr *)ifa)->ia6_flags
+#define IFA_ND6_NA_BASE_FLAGS(ifp, ifa)			\
+    (IFA_IN6_FLAGS(ifa) & IN6_IFF_ANYCAST ? 0 : ND_NA_FLAG_OVERRIDE) | \
+    ((V_ip6_forwarding && !(ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV && \
+    V_ip6_norbit_raif)) ? ND_NA_FLAG_ROUTER : 0)
+#define IFA_ND6_NA_UNSOLICITED_SKIP(ifa)		\
+    (IFA_IN6_FLAGS(ifa) & (IN6_IFF_DUPLICATED | IN6_IFF_DEPRECATED | \
+    IN6_IFF_TENTATIVE)) != 0
+#define IN6_MAX_ANYCAST_DELAY_TIME_MS 1000000
+#define IN6_BROADCAST_DELAY_TIME_MS 1000
 #define IFA_DSTIN6(x)	(&((struct sockaddr_in6 *)((x)->ifa_dstaddr))->sin6_addr)
 
 #define IFPR_IN6(x)	(&((struct sockaddr_in6 *)((x)->ifpr_prefix))->sin6_addr)
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 5e9a067..d0bcc9d 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/callout.h>
+#include <sys/random.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/socket.h>
@@ -102,8 +103,12 @@ VNET_DEFINE(int, nd6_maxnudhint) = 0;	/* max # of subsequent upper
 					 * layer hints */
 static VNET_DEFINE(int, nd6_maxqueuelen) = 1; /* max pkts cached in unresolved
 					 * ND entries */
+
+static VNET_DEFINE(int, nd6_on_link) = 1; /* Send unsolicited ND's on link up */
+
 #define	V_nd6_maxndopt			VNET(nd6_maxndopt)
 #define	V_nd6_maxqueuelen		VNET(nd6_maxqueuelen)
+#define	V_nd6_on_link			VNET(nd6_on_link)
 
 #ifdef ND6_DEBUG
 VNET_DEFINE(int, nd6_debug) = 1;
@@ -112,6 +117,7 @@ VNET_DEFINE(int, nd6_debug) = 0;
 #endif
 
 static eventhandler_tag lle_event_eh;
+static eventhandler_tag ifnet_link_event_eh;
 
 /* for debugging? */
 #if 0
@@ -196,6 +202,13 @@ nd6_lle_event(void *arg __unused, struct llentry *lle, int evt)
 	    type == RTM_ADD ? RTF_UP: 0), 0, RT_DEFAULT_FIB);
 }
 
+static void
+nd6_ifnet_link_event(void *arg __unused, struct ifnet *ifp, int linkstate)
+{
+
+	if (linkstate == LINK_STATE_UP && V_nd6_on_link)
+		nd6_na_output_unsolicited(ifp);
+}
 void
 nd6_init(void)
 {
@@ -211,9 +224,12 @@ nd6_init(void)
 	    nd6_slowtimo, curvnet);
 
 	nd6_dad_init();
-	if (IS_DEFAULT_VNET(curvnet))
+	if (IS_DEFAULT_VNET(curvnet)) {
 		lle_event_eh = EVENTHANDLER_REGISTER(lle_event, nd6_lle_event,
 		    NULL, EVENTHANDLER_PRI_ANY);
+		ifnet_link_event_eh = EVENTHANDLER_REGISTER(ifnet_link_event,
+		    nd6_ifnet_link_event, NULL, EVENTHANDLER_PRI_ANY);
+	}
 }
 
 #ifdef VIMAGE
@@ -223,8 +239,10 @@ nd6_destroy()
 
 	callout_drain(&V_nd6_slowtimo_ch);
 	callout_drain(&V_nd6_timer_ch);
-	if (IS_DEFAULT_VNET(curvnet))
+	if (IS_DEFAULT_VNET(curvnet)) {
 		EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh);
+		EVENTHANDLER_DEREGISTER(ifnet_link_event, ifnet_link_event_eh);
+	}
 }
 #endif
 
@@ -2457,13 +2475,18 @@ static int nd6_sysctl_prlist(SYSCTL_HANDLER_ARGS);
 SYSCTL_DECL(_net_inet6_icmp6);
 #endif
 SYSCTL_NODE(_net_inet6_icmp6, ICMPV6CTL_ND6_DRLIST, nd6_drlist,
-	CTLFLAG_RD, nd6_sysctl_drlist, "");
+    CTLFLAG_RD, nd6_sysctl_drlist, "List default routers");
 SYSCTL_NODE(_net_inet6_icmp6, ICMPV6CTL_ND6_PRLIST, nd6_prlist,
-	CTLFLAG_RD, nd6_sysctl_prlist, "");
+    CTLFLAG_RD, nd6_sysctl_prlist, "List prefixes");
 SYSCTL_INT(_net_inet6_icmp6, ICMPV6CTL_ND6_MAXQLEN, nd6_maxqueuelen,
-	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nd6_maxqueuelen), 1, "");
+    CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nd6_maxqueuelen), 1,
+    "Max packets cached in unresolved ND entries");
 SYSCTL_INT(_net_inet6_icmp6, OID_AUTO, nd6_gctimer,
-	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nd6_gctimer), (60 * 60 * 24), "");
+    CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nd6_gctimer), (60 * 60 * 24),
+    "Interface in seconds between garbage collection passes");
+SYSCTL_INT(_net_inet6_icmp6, OID_AUTO, nd6_on_link, CTLFLAG_VNET | CTLFLAG_RW,
+    &VNET_NAME(nd6_on_link), 0,
+    "Send unsolicited neighbor discovery on interface link up events");
 
 static int
 nd6_sysctl_drlist(SYSCTL_HANDLER_ARGS)
diff --git a/sys/netinet6/nd6.h b/sys/netinet6/nd6.h
index 8a0a56e..710a1ad 100644
--- a/sys/netinet6/nd6.h
+++ b/sys/netinet6/nd6.h
@@ -398,6 +398,10 @@ void nd6_init(void);
 #ifdef VIMAGE
 void nd6_destroy(void);
 #endif
+void nd6_na_output_unsolicited(struct ifnet *);
+void nd6_na_output_unsolicited_addr(struct ifnet *, const struct in6_addr *,
+    u_long);
+int nd6_na_unsolicited_addr_delay(struct ifaddr *);
 struct nd_ifinfo *nd6_ifattach(struct ifnet *);
 void nd6_ifdetach(struct nd_ifinfo *);
 int nd6_is_addr_neighbor(const struct sockaddr_in6 *, struct ifnet *);
diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c
index bf43fb6..0717e11 100644
--- a/sys/netinet6/nd6_nbr.c
+++ b/sys/netinet6/nd6_nbr.c
@@ -124,20 +124,16 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 	struct in6_addr saddr6 = ip6->ip6_src;
 	struct in6_addr daddr6 = ip6->ip6_dst;
 	struct in6_addr taddr6;
-	struct in6_addr myaddr6;
 	char *lladdr = NULL;
 	struct ifaddr *ifa = NULL;
+	u_long flags;
 	int lladdrlen = 0;
-	int anycast = 0, proxy = 0, tentative = 0;
+	int proxy = 0;
 	int tlladdr;
-	int rflag;
 	union nd_opts ndopts;
 	struct sockaddr_dl proxydl;
 	char ip6bufs[INET6_ADDRSTRLEN], ip6bufd[INET6_ADDRSTRLEN];
 
-	rflag = (V_ip6_forwarding) ? ND_NA_FLAG_ROUTER : 0;
-	if (ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV && V_ip6_norbit_raif)
-		rflag = 0;
 #ifndef PULLDOWN_TEST
 	IP6_EXTHDR_CHECK(m, off, icmp6len,);
 	nd_ns = (struct nd_neighbor_solicit *)((caddr_t)ip6 + off);
@@ -229,10 +225,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 	 * In implementation, we add target link-layer address by default.
 	 * We do not add one in MUST NOT cases.
 	 */
-	if (!IN6_IS_ADDR_MULTICAST(&daddr6))
-		tlladdr = 0;
-	else
-		tlladdr = 1;
+	tlladdr = !IN6_IS_ADDR_MULTICAST(&daddr6);
 
 	/*
 	 * Target address (taddr6) must be either:
@@ -289,9 +282,6 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 		 */
 		goto freeit;
 	}
-	myaddr6 = *IFA_IN6(ifa);
-	anycast = ((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_ANYCAST;
-	tentative = ((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_TENTATIVE;
 	if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_DUPLICATED)
 		goto freeit;
 
@@ -303,7 +293,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 		goto bad;
 	}
 
-	if (IN6_ARE_ADDR_EQUAL(&myaddr6, &saddr6)) {
+	if (IN6_ARE_ADDR_EQUAL(IFA_IN6(ifa), &saddr6)) {
 		nd6log((LOG_INFO, "nd6_ns_input: duplicate IP6 address %s\n",
 		    ip6_sprintf(ip6bufs, &saddr6)));
 		goto freeit;
@@ -321,7 +311,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 	 *
 	 * The processing is defined in RFC 2462.
 	 */
-	if (tentative) {
+	if (IFA_IN6_FLAGS(ifa) & IN6_IFF_TENTATIVE) {
 		/*
 		 * If source address is unspecified address, it is for
 		 * duplicate address detection.
@@ -335,6 +325,10 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 		goto freeit;
 	}
 
+	flags = IFA_ND6_NA_BASE_FLAGS(ifp, ifa);
+	if (proxy || !tlladdr)
+		flags &= ~ND_NA_FLAG_OVERRIDE;
+
 	/*
 	 * If the source address is unspecified address, entries must not
 	 * be created or updated.
@@ -349,20 +343,16 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 		in6_all = in6addr_linklocal_allnodes;
 		if (in6_setscope(&in6_all, ifp, NULL) != 0)
 			goto bad;
-		nd6_na_output_fib(ifp, &in6_all, &taddr6,
-		    ((anycast || proxy || !tlladdr) ? 0 : ND_NA_FLAG_OVERRIDE) |
-		    rflag, tlladdr, proxy ? (struct sockaddr *)&proxydl : NULL,
-		    M_GETFIB(m));
+		nd6_na_output_fib(ifp, &in6_all, &taddr6, flags, tlladdr,
+		    proxy ? (struct sockaddr *)&proxydl : NULL, M_GETFIB(m));
 		goto freeit;
 	}
 
 	nd6_cache_lladdr(ifp, &saddr6, lladdr, lladdrlen,
 	    ND_NEIGHBOR_SOLICIT, 0);
 
-	nd6_na_output_fib(ifp, &saddr6, &taddr6,
-	    ((anycast || proxy || !tlladdr) ? 0 : ND_NA_FLAG_OVERRIDE) |
-	    rflag | ND_NA_FLAG_SOLICITED, tlladdr,
-	    proxy ? (struct sockaddr *)&proxydl : NULL, M_GETFIB(m));
+	nd6_na_output_fib(ifp, &saddr6, &taddr6, flags | ND_NA_FLAG_SOLICITED,
+	    tlladdr, proxy ? (struct sockaddr *)&proxydl : NULL, M_GETFIB(m));
  freeit:
 	if (ifa != NULL)
 		ifa_free(ifa);
@@ -1597,3 +1587,110 @@ nd6_dad_na_input(struct ifaddr *ifa)
 		nd6_dad_rele(dp);
 	}
 }
+
+/*
+ * Send unsolicited neighbor advertisements for all interface addresses to
+ * notify other nodes of changes.
+ *
+ * This is a noop if the interface isn't up.
+ */
+void __noinline
+nd6_na_output_unsolicited(struct ifnet *ifp)
+{
+	int i, cnt, entries;
+	struct ifaddr *ifa;
+	struct ann {
+		struct in6_addr addr;
+		u_long flags;
+		int delay;
+	} *ann1, *head;
+
+	if (!(ifp->if_flags & IFF_UP))
+		return;
+
+	entries = 8;
+	cnt = 0;
+	head = malloc(sizeof(struct ann) * entries, M_TEMP, M_WAITOK);
+
+	/* Take a copy then process to avoid locking issues. */
+	IF_ADDR_RLOCK(ifp);
+	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+		if (ifa->ifa_addr->sa_family != AF_INET6 ||
+		    IFA_ND6_NA_UNSOLICITED_SKIP(ifa))
+			continue;
+
+		if (cnt == entries) {
+			ann1 = (struct ann*)realloc(head, sizeof(struct ann) *
+			    (entries + 8), M_TEMP, M_NOWAIT);
+			if (ann1 == NULL) {
+				log(LOG_INFO, "nd6_announce: realloc to %d "
+				    "entries failed\n", entries + 8);
+				/* Process what we have. */
+				break;
+			}
+			entries += 8;
+			head = ann1;
+		}
+
+		ann1 = head + cnt;
+		bcopy(IFA_IN6(ifa), &ann1->addr, sizeof(ann1->addr));
+		ann1->flags = IFA_ND6_NA_BASE_FLAGS(ifp, ifa);
+		ann1->delay = nd6_na_unsolicited_addr_delay(ifa);
+		cnt++;
+	}
+	IF_ADDR_RUNLOCK(ifp);
+
+	for (i = 0; i < cnt;) {
+		ann1 = head + i;
+		nd6_na_output_unsolicited_addr(ifp, &ann1->addr, ann1->flags);
+		i++;
+		if (i == cnt)
+			break;
+		DELAY(ann1->delay);
+	}
+	free(head, M_TEMP);
+}
+
+/*
+ * Return the delay required for announcements of the address as per RFC 4861.
+ */
+int
+nd6_na_unsolicited_addr_delay(struct ifaddr *ifa)
+{
+
+	if (IFA_IN6_FLAGS(ifa) & IN6_IFF_ANYCAST) {
+		/*
+		 * Random value between 0 and MAX_ANYCAST_DELAY_TIME
+		 * as per section 7.2.7.
+		 */
+		return (random() % IN6_MAX_ANYCAST_DELAY_TIME_MS);
+	}
+
+	/* Small delay as per section 7.2.6. */
+	return (IN6_BROADCAST_DELAY_TIME_MS);
+}
+
+/*
+ * Send an unsolicited neighbor advertisement for an address to notify other
+ * nodes of changes.
+ */
+void __noinline
+nd6_na_output_unsolicited_addr(struct ifnet *ifp, const struct in6_addr *addr,
+    u_long flags)
+{
+	int error;
+	struct in6_addr mcast;
+
+	mcast = in6addr_linklocal_allnodes;
+	if ((error = in6_setscope(&mcast, ifp, NULL)) != 0) {
+		/*
+		 * This shouldn't by possible as the only error is for loopback
+		 * address which we're not using.
+		 */
+		log(LOG_INFO, "in6_setscope: on mcast failed: %d\n", error);
+		return;
+	}
+	nd6_na_output(ifp, &mcast, addr, flags, 1, NULL);
+}
+
+