This main goals of this project are:

1. separating L2 tables (ARP, NDP) from the L3 routing tables
2. removing as much locking dependencies among these layers as
   possible to allow for some parallelism in the search operations
3. simplify the logic in the routing code,

The most notable end result is the obsolescent of the route
cloning (RTF_CLONING) concept, which translated into code reduction
in both IPv4 ARP and IPv6 NDP related modules, and size reduction in
struct rtentry{}. The change in design obsoletes the semantics of
RTF_CLONING, RTF_WASCLONE and RTF_LLINFO routing flags. The userland
applications such as "arp" and "ndp" have been modified to reflect
those changes. The output from "netstat -r" shows only the routing
entries.

Quite a few developers have contributed to this project in the
past: Glebius Smirnoff, Luigi Rizzo, Alessandro Cerri, and
Andre Oppermann. And most recently:

- Kip Macy revised the locking code completely, thus completing
  the last piece of the puzzle, Kip has also been conducting
  active functional testing
- Sam Leffler has helped me improving/refactoring the code, and
  provided valuable reviews
- Julian Elischer setup the perforce tree for me and has helped
  me maintaining that branch before the svn conversion

4 files changed, 67 insertions, 31 deletions

diff --git a/usr.sbin/arp/arp.c b/usr.sbin/arp/arp.c
index c8ef3e9..4154c3a1 100644
--- a/usr.sbin/arp/arp.c
+++ b/usr.sbin/arp/arp.c
@@ -370,7 +370,6 @@ set(int argc, char **argv)
 		if (addr->sin_addr.s_addr != dst->sin_addr.s_addr)	
 			break;
 		if (sdl->sdl_family == AF_LINK &&
-		    (rtm->rtm_flags & RTF_LLINFO) &&
 		    !(rtm->rtm_flags & RTF_GATEWAY) &&
 		    valid_type(sdl->sdl_type) )
 			break;
@@ -426,25 +425,46 @@ delete(char *host, int do_proxy)
 	struct sockaddr_inarp *addr, *dst;
 	struct rt_msghdr *rtm;
 	struct sockaddr_dl *sdl;
+	struct sockaddr_dl sdl_m;
 
 	dst = getaddr(host);
 	if (dst == NULL)
 		return (1);
 	dst->sin_other = do_proxy;
+
+	/*
+	 * setup the data structure to notify the kernel
+	 * it is the ARP entry the RTM_GET is interested
+	 * in
+	 */
+	bzero(&sdl_m, sizeof(sdl_m));
+	sdl_m.sdl_len = sizeof(sdl_m);
+	sdl_m.sdl_family = AF_LINK;
+
 	for (;;) {	/* try twice */
-		rtm = rtmsg(RTM_GET, dst, NULL);
+		rtm = rtmsg(RTM_GET, dst, &sdl_m);
 		if (rtm == NULL) {
 			warn("%s", host);
 			return (1);
 		}
 		addr = (struct sockaddr_inarp *)(rtm + 1);
 		sdl = (struct sockaddr_dl *)(SA_SIZE(addr) + (char *)addr);
-		if (addr->sin_addr.s_addr == dst->sin_addr.s_addr &&
-		    sdl->sdl_family == AF_LINK &&
-		    (rtm->rtm_flags & RTF_LLINFO) &&
+
+		/*
+		 * With the new L2/L3 restructure, the route 
+		 * returned is a prefix route. The important
+		 * piece of information from the previous
+		 * RTM_GET is the interface index. In the
+		 * case of ECMP, the kernel will traverse
+		 * the route group for the given entry.
+		 */
+		if (sdl->sdl_family == AF_LINK &&
 		    !(rtm->rtm_flags & RTF_GATEWAY) &&
-		    valid_type(sdl->sdl_type) )
-			break;	/* found it */
+		    valid_type(sdl->sdl_type) ) {
+			addr->sin_addr.s_addr = dst->sin_addr.s_addr;
+			break;
+		}
+
 		if (dst->sin_other & SIN_PROXY) {
 			fprintf(stderr, "delete: cannot locate %s\n",host);
 			return (1);
@@ -478,7 +498,11 @@ search(u_long addr, action_fn *action)
 	mib[2] = 0;
 	mib[3] = AF_INET;
 	mib[4] = NET_RT_FLAGS;
+#ifdef RTF_LLINFO
 	mib[5] = RTF_LLINFO;
+#else
+	mib[5] = 0;
+#endif	
 	if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0)
 		err(1, "route-sysctl-estimate");
 	if (needed == 0)	/* empty table */
@@ -563,15 +587,9 @@ print_entry(struct sockaddr_dl *sdl,
 		printf(" permanent");
 	if (addr->sin_other & SIN_PROXY)
 		printf(" published (proxy only)");
-	if (rtm->rtm_addrs & RTA_NETMASK) {
-		addr = (struct sockaddr_inarp *)
-			(SA_SIZE(sdl) + (char *)sdl);
-		if (addr->sin_addr.s_addr == 0xffffffff)
-			printf(" published");
-		if (addr->sin_len != 8)
-			printf("(weird)");
-	}
-        switch(sdl->sdl_type) {
+	if (rtm->rtm_flags & RTF_ANNOUNCE)
+		printf(" published");
+	switch(sdl->sdl_type) {
 	case IFT_ETHER:
                 printf(" [ethernet]");
                 break;
diff --git a/usr.sbin/ndp/ndp.c b/usr.sbin/ndp/ndp.c
index 86bd124..a158fb2 100644
--- a/usr.sbin/ndp/ndp.c
+++ b/usr.sbin/ndp/ndp.c
@@ -114,6 +114,11 @@
 	((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long))
 #define ADVANCE(x, n) (x += ROUNDUP((n)->sa_len))
 
+#define NEXTADDR(w, s) \
+	if (rtm->rtm_addrs & (w)) { \
+		bcopy((char *)&s, cp, sizeof(s)); cp += sizeof(s);}
+
+
 static pid_t pid;
 static int nflag;
 static int tflag;
@@ -428,7 +433,6 @@ set(argc, argv)
 	sdl = (struct sockaddr_dl *)(ROUNDUP(sin->sin6_len) + (char *)sin);
 	if (IN6_ARE_ADDR_EQUAL(&sin->sin6_addr, &sin_m.sin6_addr)) {
 		if (sdl->sdl_family == AF_LINK &&
-		    (rtm->rtm_flags & RTF_LLINFO) &&
 		    !(rtm->rtm_flags & RTF_GATEWAY)) {
 			switch (sdl->sdl_type) {
 			case IFT_ETHER: case IFT_FDDI: case IFT_ISO88023:
@@ -499,6 +503,7 @@ delete(host)
 {
 	struct sockaddr_in6 *sin = &sin_m;
 	register struct rt_msghdr *rtm = &m_rtmsg.m_rtm;
+	register char *cp = m_rtmsg.m_space;
 	struct sockaddr_dl *sdl;
 	struct addrinfo hints, *res;
 	int gai_error;
@@ -529,7 +534,6 @@ delete(host)
 	sdl = (struct sockaddr_dl *)(ROUNDUP(sin->sin6_len) + (char *)sin);
 	if (IN6_ARE_ADDR_EQUAL(&sin->sin6_addr, &sin_m.sin6_addr)) {
 		if (sdl->sdl_family == AF_LINK &&
-		    (rtm->rtm_flags & RTF_LLINFO) &&
 		    !(rtm->rtm_flags & RTF_GATEWAY)) {
 			goto delete;
 		}
@@ -545,6 +549,11 @@ delete:
 		printf("cannot locate %s\n", host);
 		return (1);
 	}
+        /* 
+         * need to reinit the field because it has rt_key
+         * but we want the actual address
+         */
+	NEXTADDR(RTA_DST, sin_m);
 	if (rtmsg(RTM_DELETE) == 0) {
 		struct sockaddr_in6 s6 = *sin; /* XXX: for safety */
 
@@ -603,7 +612,11 @@ again:;
 	mib[2] = 0;
 	mib[3] = AF_INET6;
 	mib[4] = NET_RT_FLAGS;
+#ifdef RTF_LLINFO
 	mib[5] = RTF_LLINFO;
+#else
+	mib[5] = 0;
+#endif
 	if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0)
 		err(1, "sysctl(PF_ROUTE estimate)");
 	if (needed > 0) {
@@ -893,9 +906,6 @@ rtmsg(cmd)
 	case RTM_GET:
 		rtm->rtm_addrs |= RTA_DST;
 	}
-#define NEXTADDR(w, s) \
-	if (rtm->rtm_addrs & (w)) { \
-		bcopy((char *)&s, cp, sizeof(s)); cp += SA_SIZE(&s);}
 
 	NEXTADDR(RTA_DST, sin_m);
 	NEXTADDR(RTA_GATEWAY, sdl_m);
@@ -1616,3 +1626,5 @@ ts_print(tvp)
 	(void)printf("%02d:%02d:%02d.%06u ",
 	    s / 3600, (s % 3600) / 60, s % 60, (u_int32_t)tvp->tv_usec);
 }
+
+#undef NEXTADDR
diff --git a/usr.sbin/ppp/route.c b/usr.sbin/ppp/route.c
index fdc6002..f5efe63 100644
--- a/usr.sbin/ppp/route.c
+++ b/usr.sbin/ppp/route.c
@@ -157,13 +157,21 @@ static struct bits {
   { RTF_DYNAMIC, 'D' },
   { RTF_MODIFIED, 'M' },
   { RTF_DONE, 'd' },
-  { RTF_CLONING, 'C' },
   { RTF_XRESOLVE, 'X' },
-  { RTF_LLINFO, 'L' },
+#ifdef RTF_CLONING
+  { RTF_CLONING, 'C' },
+#endif
   { RTF_STATIC, 'S' },
   { RTF_PROTO1, '1' },
   { RTF_PROTO2, '2' },
   { RTF_BLACKHOLE, 'B' },
+
+#ifdef RTF_LLINFO
+  { RTF_LLINFO, 'L' },
+#endif
+#ifdef RTF_CLONING  
+  { RTF_CLONING, 'C' },
+#endif
 #ifdef RTF_WASCLONED
   { RTF_WASCLONED, 'W' },
 #endif
diff --git a/usr.sbin/route6d/route6d.c b/usr.sbin/route6d/route6d.c
index 17ddb58..0205a03 100644
--- a/usr.sbin/route6d/route6d.c
+++ b/usr.sbin/route6d/route6d.c
@@ -1752,7 +1752,6 @@ rtrecv()
 			continue;
 		case RTM_LOSING:
 		case RTM_MISS:
-		case RTM_RESOLVE:
 		case RTM_GET:
 		case RTM_LOCK:
 			/* nothing to be done here */
@@ -1790,7 +1789,6 @@ rtrecv()
 		case RTM_ADD:
 		case RTM_LOSING:
 		case RTM_MISS:
-		case RTM_RESOLVE:
 		case RTM_GET:
 		case RTM_LOCK:
 			/* should already be handled */
@@ -2141,10 +2139,7 @@ ifrt(ifcp, again)
 			rrt->rrt_info.rip6_tag = htons(routetag & 0xffff);
 			rrt->rrt_info.rip6_metric = 1 + ifcp->ifc_metric;
 			rrt->rrt_info.rip6_plen = ifa->ifa_plen;
-			if (ifa->ifa_plen == 128)
-				rrt->rrt_flags = RTF_HOST;
-			else
-				rrt->rrt_flags = RTF_CLONING;
+			rrt->rrt_flags = RTF_HOST;
 			rrt->rrt_rflags |= RRTF_CHANGED;
 			applyplen(&rrt->rrt_info.rip6_dest, ifa->ifa_plen);
 			memset(&rrt->rrt_gw, 0, sizeof(struct in6_addr));
@@ -2433,7 +2428,6 @@ do { \
 	RTTYPE("LOCK", RTM_LOCK);
 	RTTYPE("OLDADD", RTM_OLDADD);
 	RTTYPE("OLDDEL", RTM_OLDDEL);
-	RTTYPE("RESOLVE", RTM_RESOLVE);
 	RTTYPE("NEWADDR", RTM_NEWADDR);
 	RTTYPE("DELADDR", RTM_DELADDR);
 	RTTYPE("IFINFO", RTM_IFINFO);
@@ -2484,7 +2478,9 @@ do { \
 #ifdef	RTF_MASK
 	RTFLAG("m", RTF_MASK);
 #endif
+#ifdef RTF_CLONING
 	RTFLAG("C", RTF_CLONING);
+#endif
 #ifdef RTF_CLONED
 	RTFLAG("c", RTF_CLONED);
 #endif
@@ -2495,7 +2491,9 @@ do { \
 	RTFLAG("W", RTF_WASCLONED);
 #endif
 	RTFLAG("X", RTF_XRESOLVE);
+#ifdef RTF_LLINFO
 	RTFLAG("L", RTF_LLINFO);
+#endif
 	RTFLAG("S", RTF_STATIC);
 	RTFLAG("B", RTF_BLACKHOLE);
 #ifdef RTF_PROTO3
@@ -2631,7 +2629,7 @@ rt_entry(rtm, again)
 
 	sin6_dst = sin6_gw = sin6_mask = sin6_genmask = sin6_ifp = 0;
 	if ((rtm->rtm_flags & RTF_UP) == 0 || rtm->rtm_flags &
-		(RTF_CLONING|RTF_XRESOLVE|RTF_LLINFO|RTF_BLACKHOLE)) {
+		(RTF_XRESOLVE|RTF_BLACKHOLE)) {
 		return;		/* not interested in the link route */
 	}
 	/* do not look at cloned routes */