This main goals of this project are:

1. separating L2 tables (ARP, NDP) from the L3 routing tables
2. removing as much locking dependencies among these layers as
   possible to allow for some parallelism in the search operations
3. simplify the logic in the routing code,

The most notable end result is the obsolescent of the route
cloning (RTF_CLONING) concept, which translated into code reduction
in both IPv4 ARP and IPv6 NDP related modules, and size reduction in
struct rtentry{}. The change in design obsoletes the semantics of
RTF_CLONING, RTF_WASCLONE and RTF_LLINFO routing flags. The userland
applications such as "arp" and "ndp" have been modified to reflect
those changes. The output from "netstat -r" shows only the routing
entries.

Quite a few developers have contributed to this project in the
past: Glebius Smirnoff, Luigi Rizzo, Alessandro Cerri, and
Andre Oppermann. And most recently:

- Kip Macy revised the locking code completely, thus completing
  the last piece of the puzzle, Kip has also been conducting
  active functional testing
- Sam Leffler has helped me improving/refactoring the code, and
  provided valuable reviews
- Julian Elischer setup the perforce tree for me and has helped
  me maintaining that branch before the svn conversion

15 files changed, 491 insertions, 571 deletions

diff --git a/sys/netinet/if_atm.c b/sys/netinet/if_atm.c
index 065f0c4..d534a18 100644
--- a/sys/netinet/if_atm.c
+++ b/sys/netinet/if_atm.c
@@ -127,10 +127,6 @@ atm_rtrequest(int req, struct rtentry *rt, struct rt_addrinfo *info)
 			break;
 		}
 
-		if ((rt->rt_flags & RTF_CLONING) != 0) {
-			printf("atm_rtrequest: cloning route detected?\n");
-			break;
-		}
 		if (gate->sa_family != AF_LINK ||
 		    gate->sa_len < sizeof(null_sdl)) {
 			log(LOG_DEBUG, "atm_rtrequest: bad gateway value");
@@ -332,8 +328,6 @@ atmresolve(struct rtentry *rt, struct mbuf *m, struct sockaddr *dst,
 			goto bad;	/* failed */
 		RT_REMREF(rt);		/* don't keep LL references */
 		if ((rt->rt_flags & RTF_GATEWAY) != 0 ||
-		    (rt->rt_flags & RTF_LLINFO) == 0 ||
-		    /* XXX: are we using LLINFO? */
 		    rt->rt_gateway->sa_family != AF_LINK) {
 			RT_UNLOCK(rt);
 			goto bad;
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index da7946d..9a67712 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -65,6 +65,7 @@ __FBSDID("$FreeBSD$");
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
+#include <net/if_llatbl.h>
 #include <netinet/if_ether.h>
 #include <netinet/vinet.h>
 
@@ -79,6 +80,7 @@ __FBSDID("$FreeBSD$");
 
 #define SIN(s) ((struct sockaddr_in *)s)
 #define SDL(s) ((struct sockaddr_dl *)s)
+#define LLTABLE(ifp)	((struct lltable *)(ifp)->if_afdata[AF_INET])
 
 SYSCTL_DECL(_net_link_ether);
 SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, "");
@@ -87,23 +89,13 @@ SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, "");
 #ifdef VIMAGE_GLOBALS
 static int	arpt_keep; /* once resolved, good for 20 more minutes */
 static int	arp_maxtries;
-static int	useloopback; /* use loopback interface for local traffic */
+int	useloopback; /* use loopback interface for local traffic */
 static int	arp_proxyall;
 #endif
 
 SYSCTL_V_INT(V_NET, vnet_inet, _net_link_ether_inet, OID_AUTO, max_age,
     CTLFLAG_RW, arpt_keep, 0, "ARP entry lifetime in seconds");
 
-#define	rt_expire rt_rmx.rmx_expire
-
-struct llinfo_arp {
-	struct	callout la_timer;
-	struct	rtentry *la_rt;
-	struct	mbuf *la_hold;	/* last packet until resolved/timeout */
-	u_short	la_preempt;	/* countdown for pre-expiry arps */
-	u_short	la_asked;	/* # requests sent */
-};
-
 static struct	ifqueue arpintrq;
 
 SYSCTL_V_INT(V_NET, vnet_inet, _net_link_ether_inet, OID_AUTO, maxtries,
@@ -117,196 +109,65 @@ SYSCTL_V_INT(V_NET, vnet_inet, _net_link_ether_inet, OID_AUTO, proxyall,
 	"Enable proxy ARP for all suitable requests");
 
 static void	arp_init(void);
-static void	arp_rtrequest(int, struct rtentry *, struct rt_addrinfo *);
-static void	arprequest(struct ifnet *,
+void		arprequest(struct ifnet *,
 			struct in_addr *, struct in_addr *, u_char *);
 static void	arpintr(struct mbuf *);
 static void	arptimer(void *);
-static struct rtentry
-		*arplookup(u_long, int, int, int);
 #ifdef INET
 static void	in_arpinput(struct mbuf *);
 #endif
 
+#ifdef AF_INET
+void arp_ifscrub(struct ifnet *ifp, uint32_t addr);
+
 /*
- * Timeout routine.
+ * called by in_ifscrub to remove entry from the table when
+ * the interface goes away
  */
-static void
-arptimer(void *arg)
+void
+arp_ifscrub(struct ifnet *ifp, uint32_t addr)
 {
-	struct rtentry *rt = (struct rtentry *)arg;
-
-	RT_LOCK_ASSERT(rt);
-	/*
-	 * The lock is needed to close a theoretical race
-	 * between spontaneous expiry and intentional removal.
-	 * We still got an extra reference on rtentry, so can
-	 * safely pass pointers to its contents.
-	 */
-	RT_UNLOCK(rt);
-
-	in_rtrequest(RTM_DELETE, rt_key(rt), NULL, rt_mask(rt), 0, NULL,
-	    rt->rt_fibnum);
+	struct sockaddr_in addr4;
+
+	bzero((void *)&addr4, sizeof(addr4));
+	addr4.sin_len    = sizeof(addr4);
+	addr4.sin_family = AF_INET;
+	addr4.sin_addr.s_addr = addr;
+	IF_AFDATA_LOCK(ifp);
+	lla_lookup(LLTABLE(ifp), (LLE_DELETE | LLE_IFADDR),
+	    (struct sockaddr *)&addr4);
+	IF_AFDATA_UNLOCK(ifp);
 }
+#endif
 
 /*
- * Parallel to llc_rtrequest.
+ * Timeout routine.  Age arp_tab entries periodically.
  */
 static void
-arp_rtrequest(int req, struct rtentry *rt, struct rt_addrinfo *info)
+arptimer(void *arg)
 {
-	INIT_VNET_NET(curvnet);
-	INIT_VNET_INET(curvnet);
-	struct sockaddr *gate;
-	struct llinfo_arp *la;
-	static struct sockaddr_dl null_sdl = {sizeof(null_sdl), AF_LINK};
-	struct in_ifaddr *ia;
-	struct ifaddr *ifa;
-
-	RT_LOCK_ASSERT(rt);
+	struct ifnet *ifp;
+	struct llentry   *lle = (struct llentry *)arg;
 
-	if (rt->rt_flags & RTF_GATEWAY)
+	if (lle == NULL) {
+		panic("%s: NULL entry!\n", __func__);
 		return;
-	gate = rt->rt_gateway;
-	la = (struct llinfo_arp *)rt->rt_llinfo;
-	switch (req) {
-
-	case RTM_ADD:
-		/*
-		 * XXX: If this is a manually added route to interface
-		 * such as older version of routed or gated might provide,
-		 * restore cloning bit.
-		 */
-		if ((rt->rt_flags & RTF_HOST) == 0 &&
-		    rt_mask(rt) != NULL &&
-		    SIN(rt_mask(rt))->sin_addr.s_addr != 0xffffffff)
-			rt->rt_flags |= RTF_CLONING;
-		if (rt->rt_flags & RTF_CLONING) {
-			/*
-			 * Case 1: This route should come from a route to iface.
-			 */
-			rt_setgate(rt, rt_key(rt),
-					(struct sockaddr *)&null_sdl);
-			gate = rt->rt_gateway;
-			SDL(gate)->sdl_type = rt->rt_ifp->if_type;
-			SDL(gate)->sdl_index = rt->rt_ifp->if_index;
-			rt->rt_expire = time_uptime;
-			break;
-		}
-		/* Announce a new entry if requested. */
-		if (rt->rt_flags & RTF_ANNOUNCE)
-			arprequest(rt->rt_ifp,
-			    &SIN(rt_key(rt))->sin_addr,
-			    &SIN(rt_key(rt))->sin_addr,
-			    (u_char *)LLADDR(SDL(gate)));
-		/*FALLTHROUGH*/
-	case RTM_RESOLVE:
-		if (gate->sa_family != AF_LINK ||
-		    gate->sa_len < sizeof(null_sdl)) {
-			log(LOG_DEBUG, "%s: bad gateway %s%s\n", __func__,
-			    inet_ntoa(SIN(rt_key(rt))->sin_addr),
-			    (gate->sa_family != AF_LINK) ?
-			    " (!AF_LINK)": "");
-			break;
-		}
-		SDL(gate)->sdl_type = rt->rt_ifp->if_type;
-		SDL(gate)->sdl_index = rt->rt_ifp->if_index;
-		if (la != 0)
-			break; /* This happens on a route change */
-		/*
-		 * Case 2:  This route may come from cloning, or a manual route
-		 * add with a LL address.
-		 */
-		R_Zalloc(la, struct llinfo_arp *, sizeof(*la));
-		rt->rt_llinfo = (caddr_t)la;
-		if (la == 0) {
-			log(LOG_DEBUG, "%s: malloc failed\n", __func__);
-			break;
-		}
-		/*
-		 * We are storing a route entry outside of radix tree. So,
-		 * it can be found and accessed by other means than radix
-		 * lookup. The routing code assumes that any rtentry detached
-		 * from radix can be destroyed safely. To prevent this, we
-		 * add an additional reference.
-		 */
-		RT_ADDREF(rt);
-		la->la_rt = rt;
-		rt->rt_flags |= RTF_LLINFO;
-		callout_init_mtx(&la->la_timer, &rt->rt_mtx,
-		    CALLOUT_RETURNUNLOCKED);
-
-#ifdef INET
+	}
+	ifp = lle->lle_tbl->llt_ifp;
+	IF_AFDATA_LOCK(ifp);
+	LLE_WLOCK(lle);
+	if ((lle->la_flags & LLE_DELETED) ||
+	    (time_second >= lle->la_expire)) {
+		if (!callout_pending(&lle->la_timer) &&
+		    callout_active(&lle->la_timer))
+			(void) llentry_free(lle);
+	} else {
 		/*
-		 * This keeps the multicast addresses from showing up
-		 * in `arp -a' listings as unresolved.  It's not actually
-		 * functional.  Then the same for broadcast.
+		 * Still valid, just drop our reference
 		 */
-		if (IN_MULTICAST(ntohl(SIN(rt_key(rt))->sin_addr.s_addr)) &&
-		    rt->rt_ifp->if_type != IFT_ARCNET) {
-			ETHER_MAP_IP_MULTICAST(&SIN(rt_key(rt))->sin_addr,
-					       LLADDR(SDL(gate)));
-			SDL(gate)->sdl_alen = 6;
-			rt->rt_expire = 0;
-		}
-		if (in_broadcast(SIN(rt_key(rt))->sin_addr, rt->rt_ifp)) {
-			memcpy(LLADDR(SDL(gate)), rt->rt_ifp->if_broadcastaddr,
-			       rt->rt_ifp->if_addrlen);
-			SDL(gate)->sdl_alen = rt->rt_ifp->if_addrlen;
-			rt->rt_expire = 0;
-		}
-#endif
-
-		TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
-			if (ia->ia_ifp == rt->rt_ifp &&
-			    SIN(rt_key(rt))->sin_addr.s_addr ==
-			    (IA_SIN(ia))->sin_addr.s_addr)
-				break;
-		}
-		if (ia) {
-		    /*
-		     * This test used to be
-		     *	if (loif.if_flags & IFF_UP)
-		     * It allowed local traffic to be forced
-		     * through the hardware by configuring the loopback down.
-		     * However, it causes problems during network configuration
-		     * for boards that can't receive packets they send.
-		     * It is now necessary to clear "useloopback" and remove
-		     * the route to force traffic out to the hardware.
-		     */
-			rt->rt_expire = 0;
-			bcopy(IF_LLADDR(rt->rt_ifp), LLADDR(SDL(gate)),
-			      SDL(gate)->sdl_alen = rt->rt_ifp->if_addrlen);
-			if (V_useloopback) {
-				rt->rt_ifp = V_loif;
-				rt->rt_rmx.rmx_mtu = V_loif->if_mtu;
-			}
-
-		    /*
-		     * make sure to set rt->rt_ifa to the interface
-		     * address we are using, otherwise we will have trouble
-		     * with source address selection.
-		     */
-			ifa = &ia->ia_ifa;
-			if (ifa != rt->rt_ifa) {
-				IFAFREE(rt->rt_ifa);
-				IFAREF(ifa);
-				rt->rt_ifa = ifa;
-			}
-		}
-		break;
-
-	case RTM_DELETE:
-		if (la == NULL)	/* XXX: at least CARP does this. */
-			break;
-		callout_stop(&la->la_timer);
-		rt->rt_llinfo = NULL;
-		rt->rt_flags &= ~RTF_LLINFO;
-		RT_REMREF(rt);
-		if (la->la_hold)
-			m_freem(la->la_hold);
-		Free((caddr_t)la);
+		LLE_FREE_LOCKED(lle);
 	}
+	IF_AFDATA_UNLOCK(ifp);
 }
 
 /*
@@ -315,14 +176,37 @@ arp_rtrequest(int req, struct rtentry *rt, struct rt_addrinfo *info)
  *	- arp header target ip address
  *	- arp header source ethernet address
  */
-static void
-arprequest(struct ifnet *ifp, struct in_addr *sip, struct in_addr *tip,
+void
+arprequest(struct ifnet *ifp, struct in_addr *sip, struct in_addr  *tip,
     u_char *enaddr)
 {
 	struct mbuf *m;
 	struct arphdr *ah;
 	struct sockaddr sa;
 
+	if (sip == NULL) {
+		/* XXX don't believe this can happen (or explain why) */
+		/*
+		 * The caller did not supply a source address, try to find
+		 * a compatible one among those assigned to this interface.
+		 */
+		struct ifaddr *ifa;
+
+		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+			if (!ifa->ifa_addr ||
+			    ifa->ifa_addr->sa_family != AF_INET)
+				continue;
+			sip = &SIN(ifa->ifa_addr)->sin_addr;
+			if (0 == ((sip->s_addr ^ tip->s_addr) &
+			    SIN(ifa->ifa_netmask)->sin_addr.s_addr) )
+				break;  /* found it. */
+		}
+		if (sip == NULL) {  
+			printf("%s: cannot find matching address\n", __func__);
+			return;
+		}
+	}
+
 	if ((m = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL)
 		return;
 	m->m_len = sizeof(*ah) + 2*sizeof(struct in_addr) +
@@ -345,8 +229,6 @@ arprequest(struct ifnet *ifp, struct in_addr *sip, struct in_addr *tip,
 	sa.sa_len = 2;
 	m->m_flags |= M_BCAST;
 	(*ifp->if_output)(ifp, m, &sa, (struct rtentry *)0);
-
-	return;
 }
 
 /*
@@ -365,16 +247,15 @@ arprequest(struct ifnet *ifp, struct in_addr *sip, struct in_addr *tip,
  */
 int
 arpresolve(struct ifnet *ifp, struct rtentry *rt0, struct mbuf *m,
-    struct sockaddr *dst, u_char *desten)
+	struct sockaddr *dst, u_char *desten, struct llentry **lle)
 {
 	INIT_VNET_INET(ifp->if_vnet);
-	struct llinfo_arp *la = NULL;
-	struct rtentry *rt = NULL;
-	struct sockaddr_dl *sdl;
-	int error;
-	int fibnum = -1;
+	struct llentry *la = 0;
+	u_int flags;
+	int error, renew;
 
-	if (m) {
+	*lle = NULL;
+	if (m != NULL) {
 		if (m->m_flags & M_BCAST) {
 			/* broadcast */
 			(void)memcpy(desten,
@@ -386,110 +267,78 @@ arpresolve(struct ifnet *ifp, struct rtentry *rt0, struct mbuf *m,
 			ETHER_MAP_IP_MULTICAST(&SIN(dst)->sin_addr, desten);
 			return (0);
 		}
-		fibnum = M_GETFIB(m);
 	}
 
-	if (rt0 != NULL) {
-		/* Look for a cached arp (ll) entry. */
-		if (m == NULL)
-			fibnum = rt0->rt_fibnum;
-		error = rt_check(&rt, &rt0, dst);
-		if (error) {
-			m_freem(m);
-			return error;
-		}
-		la = (struct llinfo_arp *)rt->rt_llinfo;
-		if (la == NULL)
-			RT_UNLOCK(rt);
-	}
+	flags = (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) ? 0 : LLE_CREATE;
 
-	/*
-	 * If we had no mbuf and no route, then hope the caller
-	 * has a fib in mind because we are running out of ideas.
-	 * I think this should not happen in current code.
-	 * (kmacy would know).
+	/* XXXXX
+	 * Since this function returns an llentry, the 
+	 * lock is held by the caller.
+	 * XXX if caller is required to hold lock, assert it
 	 */
-	if (fibnum == -1)
-		fibnum = curthread->td_proc->p_fibnum; /* last gasp */
-
+retry:
+	IF_AFDATA_LOCK(ifp);	
+	la = lla_lookup(LLTABLE(ifp), flags, dst);
+	IF_AFDATA_UNLOCK(ifp);	
 	if (la == NULL) {
-		/*
-		 * We enter this block if rt0 was NULL,
-		 * or if rt found by rt_check() didn't have llinfo.
-		 * we should get a cloned route, which since it should
-		 * come from the local interface should have a ll entry.
-		 * It may be incomplete but that's ok.
-		 */
-		rt = arplookup(SIN(dst)->sin_addr.s_addr, 1, 0, fibnum);
-		if (rt == NULL) {
-			log(LOG_DEBUG,
-			    "arpresolve: can't allocate route for %s\n",
-			    inet_ntoa(SIN(dst)->sin_addr));
-			m_freem(m);
-			return (EINVAL); /* XXX */
-		}
-		la = (struct llinfo_arp *)rt->rt_llinfo;
-		if (la == NULL) {
-			RT_UNLOCK(rt);
+		if (flags & LLE_CREATE)
 			log(LOG_DEBUG,
 			    "arpresolve: can't allocate llinfo for %s\n",
 			    inet_ntoa(SIN(dst)->sin_addr));
-			m_freem(m);
-			return (EINVAL); /* XXX */
-		}
-	}
-	sdl = SDL(rt->rt_gateway);
-	/*
-	 * Check the address family and length is valid, the address
-	 * is resolved; otherwise, try to resolve.
-	 */
-	if ((rt->rt_expire == 0 || rt->rt_expire > time_uptime) &&
-	    sdl->sdl_family == AF_LINK && sdl->sdl_alen != 0) {
-
-		bcopy(LLADDR(sdl), desten, sdl->sdl_alen);
+		m_freem(m);
+		return (EINVAL);
+	} 
 
+	if ((la->la_flags & LLE_VALID) &&
+	    ((la->la_flags & LLE_STATIC) || la->la_expire > time_uptime)) {
+		bcopy(&la->ll_addr, desten, ifp->if_addrlen);
 		/*
 		 * If entry has an expiry time and it is approaching,
-		 * send an ARP request.
+		 * see if we need to send an ARP request within this
+		 * arpt_down interval.
 		 */
-		if ((rt->rt_expire != 0) &&
-		    (time_uptime + la->la_preempt > rt->rt_expire)) {
-			struct in_addr sin = 
-			    SIN(rt->rt_ifa->ifa_addr)->sin_addr;
+		if (!(la->la_flags & LLE_STATIC) &&
+		    time_uptime + la->la_preempt > la->la_expire) {
+			arprequest(ifp, NULL,
+			    &SIN(dst)->sin_addr, IF_LLADDR(ifp));
 
 			la->la_preempt--;
-			RT_UNLOCK(rt);
-			arprequest(ifp, &sin, &SIN(dst)->sin_addr,
-			    IF_LLADDR(ifp));
-			return (0);
-		} 
-
-		RT_UNLOCK(rt);
-		return (0);
-	}
-	/*
-	 * If ARP is disabled or static on this interface, stop.
-	 * XXX
-	 * Probably should not allocate empty llinfo struct if we are
-	 * not going to be sending out an arp request.
-	 */
-	if (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) {
-		RT_UNLOCK(rt);
+		}
+		
+		*lle = la;
+		error = 0;
+		goto done;
+	} 
+			    
+	if (la->la_flags & LLE_STATIC) {   /* should not happen! */
+		log(LOG_DEBUG, "arpresolve: ouch, empty static llinfo for %s\n",
+		    inet_ntoa(SIN(dst)->sin_addr));
 		m_freem(m);
-		return (EINVAL);
+		error = EINVAL;
+		goto done;
+	}
+
+	renew = (la->la_asked == 0 || la->la_expire != time_uptime);
+	if ((renew || m != NULL) && (flags & LLE_EXCLUSIVE) == 0) {
+		flags |= LLE_EXCLUSIVE;
+		LLE_RUNLOCK(la);
+		goto retry;
 	}
 	/*
 	 * There is an arptab entry, but no ethernet address
 	 * response yet.  Replace the held mbuf with this
 	 * latest one.
 	 */
-	if (m) {
-		if (la->la_hold)
+	if (m != NULL) {
+		if (la->la_hold != NULL)
 			m_freem(la->la_hold);
 		la->la_hold = m;
+		if (renew == 0 && (flags & LLE_EXCLUSIVE)) {
+			flags &= ~LLE_EXCLUSIVE;
+			LLE_DOWNGRADE(la);
+		}
+		
 	}
-	KASSERT(rt->rt_expire > 0, ("sending ARP request for static entry"));
-
 	/*
 	 * Return EWOULDBLOCK if we have tried less than arp_maxtries. It
 	 * will be masked by ether_output(). Return EHOSTDOWN/EHOSTUNREACH
@@ -499,22 +348,24 @@ arpresolve(struct ifnet *ifp, struct rtentry *rt0, struct mbuf *m,
 	if (la->la_asked < V_arp_maxtries)
 		error = EWOULDBLOCK;	/* First request. */
 	else
-		error = (rt == rt0) ? EHOSTDOWN : EHOSTUNREACH;
+		error =
+		    (rt0->rt_flags & RTF_GATEWAY) ? EHOSTDOWN : EHOSTUNREACH;
 
-	if (la->la_asked == 0 || rt->rt_expire != time_uptime) {
-		struct in_addr sin =
-		    SIN(rt->rt_ifa->ifa_addr)->sin_addr;
-
-		rt->rt_expire = time_uptime;
-		callout_reset(&la->la_timer, hz, arptimer, rt);
+	if (renew) {
+		LLE_ADDREF(la);
+		la->la_expire = time_uptime;
+		callout_reset(&la->la_timer, hz, arptimer, la);
 		la->la_asked++;
-		RT_UNLOCK(rt);
-
-		arprequest(ifp, &sin, &SIN(dst)->sin_addr,
+		LLE_WUNLOCK(la);
+		arprequest(ifp, NULL, &SIN(dst)->sin_addr,
 		    IF_LLADDR(ifp));
-	} else
-		RT_UNLOCK(rt);
-
+		return (error);
+	}
+done:
+	if (flags & LLE_EXCLUSIVE)
+		LLE_WUNLOCK(la);
+	else
+		LLE_RUNLOCK(la);
 	return (error);
 }
 
@@ -598,21 +449,17 @@ in_arpinput(struct mbuf *m)
 {
 	struct arphdr *ah;
 	struct ifnet *ifp = m->m_pkthdr.rcvif;
-	struct llinfo_arp *la;
+	struct llentry *la = NULL;
 	struct rtentry *rt;
 	struct ifaddr *ifa;
 	struct in_ifaddr *ia;
-	struct sockaddr_dl *sdl;
 	struct sockaddr sa;
 	struct in_addr isaddr, itaddr, myaddr;
-	struct mbuf *hold;
 	u_int8_t *enaddr = NULL;
-	int op, rif_len;
+	int op, flags;
+	struct mbuf *m0;
 	int req_len;
 	int bridged = 0, is_bridge = 0;
-	u_int fibnum;
-	u_int goodfib = 0;
-	int firstpass = 1;
 #ifdef DEV_CARP
 	int carp_match = 0;
 #endif
@@ -649,7 +496,7 @@ in_arpinput(struct mbuf *m)
 	 */
 	LIST_FOREACH(ia, INADDR_HASH(itaddr.s_addr), ia_hash) {
 		if (((bridged && ia->ia_ifp->if_bridge != NULL) ||
-		    (ia->ia_ifp == ifp)) &&
+		    ia->ia_ifp == ifp) &&
 		    itaddr.s_addr == ia->ia_addr.sin_addr.s_addr)
 			goto match;
 #ifdef DEV_CARP
@@ -663,7 +510,7 @@ in_arpinput(struct mbuf *m)
 	}
 	LIST_FOREACH(ia, INADDR_HASH(isaddr.s_addr), ia_hash)
 		if (((bridged && ia->ia_ifp->if_bridge != NULL) ||
-		    (ia->ia_ifp == ifp)) &&
+		    ia->ia_ifp == ifp) &&
 		    isaddr.s_addr == ia->ia_addr.sin_addr.s_addr)
 			goto match;
 
@@ -729,191 +576,94 @@ match:
 	}
 	if (ifp->if_flags & IFF_STATICARP)
 		goto reply;
-	/*
-	 * We look for any FIB that has this address to find
-	 * the interface etc.
-	 * For sanity checks that are FIB independent we abort the loop.
-	 */
-	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
-		rt = arplookup(isaddr.s_addr,
-		    itaddr.s_addr == myaddr.s_addr, 0, fibnum);
-		if (rt == NULL)
-			continue;
-		
-		sdl = SDL(rt->rt_gateway);
-		/* Only call this once */
-		if (firstpass) {
-			sin.sin_addr.s_addr = isaddr.s_addr;
-			EVENTHANDLER_INVOKE(route_arp_update_event, rt,
-			    ar_sha(ah), (struct sockaddr *)&sin);
-		}
-		
-		la = (struct llinfo_arp *)rt->rt_llinfo;
-		if (la == NULL) {
-			RT_UNLOCK(rt);
-			continue;
-		}
 
-		if (firstpass) {
-			/* The following is not an error when doing bridging. */
-			if (!bridged && rt->rt_ifp != ifp
+	bzero(&sin, sizeof(sin));
+	sin.sin_len = sizeof(struct sockaddr_in);
+	sin.sin_family = AF_INET;
+	sin.sin_addr = isaddr;
+	flags = (itaddr.s_addr == myaddr.s_addr) ? LLE_CREATE : 0;
+	flags |= LLE_EXCLUSIVE;
+	IF_AFDATA_LOCK(ifp); 
+	la = lla_lookup(LLTABLE(ifp), flags, (struct sockaddr *)&sin);
+	IF_AFDATA_UNLOCK(ifp);
+	if (la != NULL) {
+		/* the following is not an error when doing bridging */
+		if (!bridged && la->lle_tbl->llt_ifp != ifp
 #ifdef DEV_CARP
-			    && (ifp->if_type != IFT_CARP || !carp_match)
+		    && (ifp->if_type != IFT_CARP || !carp_match)
 #endif
-			    ) {
-				if (log_arp_wrong_iface)
-					log(LOG_ERR, "arp: %s is on %s "
-						"but got reply from %*D "
-						"on %s\n",
-					    inet_ntoa(isaddr),
-					    rt->rt_ifp->if_xname,
-					    ifp->if_addrlen,
-					    (u_char *)ar_sha(ah), ":",
-					    ifp->if_xname);
-				RT_UNLOCK(rt);
-				break;
-			}
-			if (sdl->sdl_alen &&
-			    bcmp(ar_sha(ah), LLADDR(sdl), sdl->sdl_alen)) {
-				if (rt->rt_expire) {
-				    if (log_arp_movements)
-					log(LOG_INFO,
-					    "arp: %s moved from %*D to %*D "
-					    "on %s\n",
-					    inet_ntoa(isaddr),
-					    ifp->if_addrlen,
-					    (u_char *)LLADDR(sdl), ":",
-					    ifp->if_addrlen,
-					    (u_char *)ar_sha(ah), ":",
-					    ifp->if_xname);
-				} else {
-					RT_UNLOCK(rt);
-					if (log_arp_permanent_modify)
-						log(LOG_ERR,
-						    "arp: %*D attempts to "
-						    "modify permanent entry "
-						    "for %s on %s\n",
-						    ifp->if_addrlen,
-						    (u_char *)ar_sha(ah), ":",
-						    inet_ntoa(isaddr),
-						    ifp->if_xname);
-					break;
-				}
-			}
-			/*
-			 * sanity check for the address length.
-			 * XXX this does not work for protocols
-			 * with variable address length. -is
-			 */
-			if (sdl->sdl_alen &&
-			    sdl->sdl_alen != ah->ar_hln) {
-				log(LOG_WARNING,
-				    "arp from %*D: new addr len %d, was %d",
-				    ifp->if_addrlen, (u_char *) ar_sha(ah),
-				    ":", ah->ar_hln, sdl->sdl_alen);
+			) {
+			if (log_arp_wrong_iface)
+				log(LOG_ERR, "arp: %s is on %s "
+				    "but got reply from %*D on %s\n",
+				    inet_ntoa(isaddr),
+				    la->lle_tbl->llt_ifp->if_xname,
+				    ifp->if_addrlen, (u_char *)ar_sha(ah), ":",
+				    ifp->if_xname);
+			goto reply;
+		}
+		if ((la->la_flags & LLE_VALID) &&
+		    bcmp(ar_sha(ah), &la->ll_addr, ifp->if_addrlen)) {
+			if (la->la_flags & LLE_STATIC) {
+				log(LOG_ERR,
+				    "arp: %*D attempts to modify permanent "
+				    "entry for %s on %s\n",
+				    ifp->if_addrlen, (u_char *)ar_sha(ah), ":",
+				    inet_ntoa(isaddr), ifp->if_xname);
+				goto reply;
 			}
-			if (ifp->if_addrlen != ah->ar_hln) {
-				log(LOG_WARNING,
-				    "arp from %*D: addr len: "
-				    "new %d, i/f %d (ignored)",
-				    ifp->if_addrlen, (u_char *) ar_sha(ah),
-				    ":", ah->ar_hln, ifp->if_addrlen);
-				RT_UNLOCK(rt);
-				break;
+			if (log_arp_movements) {
+			        log(LOG_INFO, "arp: %s moved from %*D "
+				    "to %*D on %s\n",
+				    inet_ntoa(isaddr),
+				    ifp->if_addrlen,
+				    (u_char *)&la->ll_addr, ":",
+				    ifp->if_addrlen, (u_char *)ar_sha(ah), ":",
+				    ifp->if_xname);
 			}
-			firstpass = 0;
-			goodfib = fibnum;
 		}
-
-		/* Copy in the information received. */
-		(void)memcpy(LLADDR(sdl), ar_sha(ah),
-		    sdl->sdl_alen = ah->ar_hln);
-		/*
-		 * If we receive an arp from a token-ring station over
-		 * a token-ring nic then try to save the source routing info.
-		 * XXXMRT Only minimal Token Ring support for MRT.
-		 * Only do this on the first pass as if modifies the mbuf.
-		 */
-		if (ifp->if_type == IFT_ISO88025) {
-			struct iso88025_header *th = NULL;
-			struct iso88025_sockaddr_dl_data *trld;
-
-			/* force the fib loop to end after this pass */
-			fibnum = rt_numfibs - 1;
-
-			th = (struct iso88025_header *)m->m_pkthdr.header;
-			trld = SDL_ISO88025(sdl);
-			rif_len = TR_RCF_RIFLEN(th->rcf);
-			if ((th->iso88025_shost[0] & TR_RII) &&
-			    (rif_len > 2)) {
-				trld->trld_rcf = th->rcf;
-				trld->trld_rcf ^= htons(TR_RCF_DIR);
-				memcpy(trld->trld_route, th->rd, rif_len - 2);
-				trld->trld_rcf &= ~htons(TR_RCF_BCST_MASK);
-				/*
-				 * Set up source routing information for
-				 * reply packet (XXX)
-				 */
-				m->m_data -= rif_len;
-				m->m_len  += rif_len;
-				m->m_pkthdr.len += rif_len;
-			} else {
-				th->iso88025_shost[0] &= ~TR_RII;
-				trld->trld_rcf = 0;
-			}
-			m->m_data -= 8;
-			m->m_len  += 8;
-			m->m_pkthdr.len += 8;
-			th->rcf = trld->trld_rcf;
+		    
+		if (ifp->if_addrlen != ah->ar_hln) {
+			log(LOG_WARNING,
+			    "arp from %*D: addr len: new %d, i/f %d (ignored)",
+			    ifp->if_addrlen, (u_char *) ar_sha(ah), ":",
+			    ah->ar_hln, ifp->if_addrlen);
+			goto reply;
 		}
+		(void)memcpy(&la->ll_addr, ar_sha(ah), ifp->if_addrlen);
+		la->la_flags |= LLE_VALID;
 
-		if (rt->rt_expire) {
-			rt->rt_expire = time_uptime + V_arpt_keep;
+		if (!(la->la_flags & LLE_STATIC)) {
+			la->la_expire = time_uptime + V_arpt_keep;
 			callout_reset(&la->la_timer, hz * V_arpt_keep,
-			    arptimer, rt);
+			    arptimer, la);
 		}
 		la->la_asked = 0;
 		la->la_preempt = V_arp_maxtries;
-		hold = la->la_hold;
-		la->la_hold = NULL;
-		RT_UNLOCK(rt);
-		if (hold != NULL)
-			(*ifp->if_output)(ifp, hold, rt_key(rt), rt);
-	} /* end of FIB loop */
+		if (la->la_hold != NULL) {
+			m0 = la->la_hold;
+			la->la_hold = 0;
+			memcpy(&sa, L3_ADDR(la), sizeof(sa));
+			LLE_WUNLOCK(la);
+			
+			(*ifp->if_output)(ifp, m0, &sa, NULL);
+			return;
+		}
+	}
 reply:
-
-	/*
-	 * Decide if we have to respond to something.
-	 */
 	if (op != ARPOP_REQUEST)
 		goto drop;
+
 	if (itaddr.s_addr == myaddr.s_addr) {
 		/* Shortcut.. the receiving interface is the target. */
 		(void)memcpy(ar_tha(ah), ar_sha(ah), ah->ar_hln);
 		(void)memcpy(ar_sha(ah), enaddr, ah->ar_hln);
 	} else {
-		/* It's not asking for our address. But it still may
-		 * be something we should answer.
-		 *
-		 * XXX MRT
-		 * We assume that link level info is independent of
-		 * the table used and so we use whichever we can and don't
-		 * have a better option.
-		 */
-		/* Have we been asked to proxy for the target. */
-		rt = arplookup(itaddr.s_addr, 0, SIN_PROXY, goodfib);
-		if (rt == NULL) {
-			/* Nope, only intersted now if proxying everything. */
-			struct sockaddr_in sin;
-
+		if (la == NULL) {
 			if (!V_arp_proxyall)
 				goto drop;
 
-			bzero(&sin, sizeof sin);
-			sin.sin_family = AF_INET;
-			sin.sin_len = sizeof sin;
 			sin.sin_addr = itaddr;
-
 			/* XXX MRT use table 0 for arp reply  */
 			rt = in_rtalloc1((struct sockaddr *)&sin, 0, 0UL, 0);
 			if (!rt)
@@ -964,19 +714,18 @@ reply:
 			 * Otherwise we may conflict with the host we are
 			 * proxying for.
 			 */
-			if (rt->rt_ifp != ifp &&
-			    (rt->rt_ifp->if_bridge != ifp->if_bridge ||
+			if (la->lle_tbl->llt_ifp != ifp &&
+			    (la->lle_tbl->llt_ifp->if_bridge != ifp->if_bridge ||
 			    ifp->if_bridge == NULL)) {
-				RT_UNLOCK(rt);
 				goto drop;
 			}
-			sdl = SDL(rt->rt_gateway);
 			(void)memcpy(ar_tha(ah), ar_sha(ah), ah->ar_hln);
-			(void)memcpy(ar_sha(ah), LLADDR(sdl), ah->ar_hln);
-			RT_UNLOCK(rt);
+			(void)memcpy(ar_sha(ah), &la->ll_addr, ah->ar_hln);
 		}
 	}
 
+	if (la != NULL)
+		LLE_WUNLOCK(la);
 	if (itaddr.s_addr == myaddr.s_addr &&
 	    IN_LINKLOCAL(ntohl(itaddr.s_addr))) {
 		/* RFC 3927 link-local IPv4; always reply by broadcast. */
@@ -1002,68 +751,34 @@ reply:
 	return;
 
 drop:
+	if (la != NULL)
+		LLE_WUNLOCK(la);
 	m_freem(m);
 }
 #endif
 
-/*
- * Lookup or enter a new address in arptab.
- */
-static struct rtentry *
-arplookup(u_long addr, int create, int proxy, int fibnum)
-{
-	struct rtentry *rt;
-	struct sockaddr_inarp sin;
-	const char *why = 0;
-
-	bzero(&sin, sizeof(sin));
-	sin.sin_len = sizeof(sin);
-	sin.sin_family = AF_INET;
-	sin.sin_addr.s_addr = addr;
-	if (proxy)
-		sin.sin_other = SIN_PROXY;
-	rt = in_rtalloc1((struct sockaddr *)&sin, create, 0UL, fibnum);
-	if (rt == 0)
-		return (0);
-
-	if (rt->rt_flags & RTF_GATEWAY)
-		why = "host is not on local network";
-	else if ((rt->rt_flags & RTF_LLINFO) == 0)
-		why = "could not allocate llinfo";
-	else if (rt->rt_gateway->sa_family != AF_LINK)
-		why = "gateway route is not ours";
-
-	if (why) {
-#define	ISDYNCLONE(_rt) \
-	(((_rt)->rt_flags & (RTF_STATIC | RTF_WASCLONED)) == RTF_WASCLONED)
-		if (create)
-			log(LOG_DEBUG, "arplookup %s failed: %s\n",
-			    inet_ntoa(sin.sin_addr), why);
-		/*
-		 * If there are no references to this Layer 2 route,
-		 * and it is a cloned route, and not static, and
-		 * arplookup() is creating the route, then purge
-		 * it from the routing table as it is probably bogus.
-		 */
-		if (rt->rt_refcnt == 1 && ISDYNCLONE(rt))
-			rtexpunge(rt);
-		RTFREE_LOCKED(rt);
-		return (0);
-#undef ISDYNCLONE
-	} else {
-		RT_REMREF(rt);
-		return (rt);
-	}
-}
-
 void
 arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa)
 {
+	struct llentry *lle;
+
 	if (ntohl(IA_SIN(ifa)->sin_addr.s_addr) != INADDR_ANY)
 		arprequest(ifp, &IA_SIN(ifa)->sin_addr,
 				&IA_SIN(ifa)->sin_addr, IF_LLADDR(ifp));
-	ifa->ifa_rtrequest = arp_rtrequest;
-	ifa->ifa_flags |= RTF_CLONING;
+	/* 
+	 * interface address is considered static entry
+	 * because the output of the arp utility shows
+	 * that L2 entry as permanent
+	 */
+	IF_AFDATA_LOCK(ifp);
+	lle = lla_lookup(LLTABLE(ifp), (LLE_CREATE | LLE_IFADDR | LLE_STATIC),
+	    (struct sockaddr *)IA_SIN(ifa));
+	IF_AFDATA_UNLOCK(ifp);
+	if (lle == NULL)
+		log(LOG_INFO, "arp_ifinit: cannot create arp "
+		    "entry for interface address\n");
+	LLE_RUNLOCK(lle);
+	ifa->ifa_rtrequest = NULL;
 }
 
 void
@@ -1072,8 +787,7 @@ arp_ifinit2(struct ifnet *ifp, struct ifaddr *ifa, u_char *enaddr)
 	if (ntohl(IA_SIN(ifa)->sin_addr.s_addr) != INADDR_ANY)
 		arprequest(ifp, &IA_SIN(ifa)->sin_addr,
 				&IA_SIN(ifa)->sin_addr, enaddr);
-	ifa->ifa_rtrequest = arp_rtrequest;
-	ifa->ifa_flags |= RTF_CLONING;
+	ifa->ifa_rtrequest = NULL;
 }
 
 static void
diff --git a/sys/netinet/if_ether.h b/sys/netinet/if_ether.h
index 14df15f..ba4e2f6 100644
--- a/sys/netinet/if_ether.h
+++ b/sys/netinet/if_ether.h
@@ -109,8 +109,11 @@ struct sockaddr_inarp {
 extern u_char	ether_ipmulticast_min[ETHER_ADDR_LEN];
 extern u_char	ether_ipmulticast_max[ETHER_ADDR_LEN];
 
+struct llentry;
+
 int	arpresolve(struct ifnet *ifp, struct rtentry *rt,
-		struct mbuf *m, struct sockaddr *dst, u_char *desten);
+		    struct mbuf *m, struct sockaddr *dst, u_char *desten,
+		    struct llentry **lle);
 void	arp_ifinit(struct ifnet *, struct ifaddr *);
 void	arp_ifinit2(struct ifnet *, struct ifaddr *, u_char *);
 #endif
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index 9b6dd48..852ef6d 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/vimage.h>
 
 #include <net/if.h>
+#include <net/if_llatbl.h>
 #include <net/if_types.h>
 #include <net/route.h>
 
@@ -871,6 +872,8 @@ in_addprefix(struct in_ifaddr *target, int flags)
 	return (error);
 }
 
+extern void arp_ifscrub(struct ifnet *ifp, uint32_t addr);
+
 /*
  * If there is no other address in the system that can serve a route to the
  * same prefix, remove the route.  Hand over the route to the new address
@@ -893,6 +896,8 @@ in_scrubprefix(struct in_ifaddr *target)
 		prefix = target->ia_addr.sin_addr;
 		mask = target->ia_sockmask.sin_addr;
 		prefix.s_addr &= mask.s_addr;
+		/* remove arp cache */
+		arp_ifscrub(target->ia_ifp, IA_SIN(target)->sin_addr.s_addr);
 	}
 
 	TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
@@ -1015,3 +1020,238 @@ in_ifdetach(struct ifnet *ifp)
 	in_pcbpurgeif0(&V_udbinfo, ifp);
 	in_purgemaddrs(ifp);
 }
+
+#include <sys/syslog.h>
+#include <net/if_dl.h>
+#include <netinet/if_ether.h>
+
+struct in_llentry {
+	struct llentry		base;
+	struct sockaddr_in	l3_addr4;
+};
+
+static struct llentry *
+in_lltable_new(const struct sockaddr *l3addr, u_int flags)
+{
+	struct in_llentry *lle;
+
+	lle = malloc(sizeof(struct in_llentry), M_LLTABLE, M_DONTWAIT | M_ZERO);
+	if (lle == NULL)		/* NB: caller generates msg */
+		return NULL;
+
+	callout_init(&lle->base.la_timer, CALLOUT_MPSAFE);
+	/*
+	 * For IPv4 this will trigger "arpresolve" to generate
+	 * an ARP request.
+	 */
+	lle->base.la_expire = time_second; /* mark expired */
+	lle->l3_addr4 = *(const struct sockaddr_in *)l3addr;
+	lle->base.lle_refcnt = 1;
+	LLE_LOCK_INIT(&lle->base);
+	return &lle->base;
+}
+
+/*
+ * Deletes an address from the address table.
+ * This function is called by the timer functions
+ * such as arptimer() and nd6_llinfo_timer(), and
+ * the caller does the locking.
+ */
+static void
+in_lltable_free(struct lltable *llt, struct llentry *lle)
+{
+	free(lle, M_LLTABLE);
+}
+
+static int
+in_lltable_rtcheck(struct ifnet *ifp, const struct sockaddr *l3addr)
+{
+	struct rtentry *rt;
+
+	KASSERT(l3addr->sa_family == AF_INET,
+	    ("sin_family %d", l3addr->sa_family));
+
+	/* XXX rtalloc1 should take a const param */
+	rt = rtalloc1(__DECONST(struct sockaddr *, l3addr), 0, 0);
+	if (rt == NULL || (rt->rt_flags & RTF_GATEWAY) || rt->rt_ifp != ifp) {
+		log(LOG_INFO, "IPv4 address: \"%s\" is not on the network\n",
+		    inet_ntoa(((const struct sockaddr_in *)l3addr)->sin_addr));
+		if (rt != NULL)
+			RTFREE_LOCKED(rt);
+		return (EINVAL);
+	}
+	RTFREE_LOCKED(rt);
+	return 0;
+}
+
+/*
+ * Return NULL if not found or marked for deletion.
+ * If found return lle read locked.
+ */
+static struct llentry *
+in_lltable_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr)
+{
+	const struct sockaddr_in *sin = (const struct sockaddr_in *)l3addr;
+	struct ifnet *ifp = llt->llt_ifp;
+	struct llentry *lle;
+	struct llentries *lleh;
+	u_int hashkey;
+
+	IF_AFDATA_LOCK_ASSERT(ifp);
+	KASSERT(l3addr->sa_family == AF_INET,
+	    ("sin_family %d", l3addr->sa_family));
+
+	hashkey = sin->sin_addr.s_addr;
+	lleh = &llt->lle_head[LLATBL_HASH(hashkey, LLTBL_HASHMASK)];
+	LIST_FOREACH(lle, lleh, lle_next) {
+		if (lle->la_flags & LLE_DELETED)
+			continue;
+		if (bcmp(L3_ADDR(lle), l3addr, sizeof(struct sockaddr_in)) == 0)
+			break;
+	}
+	if (lle == NULL) {
+#ifdef DIAGNOSTICS
+		if (flags & LLE_DELETE)
+			log(LOG_INFO, "interface address is missing from cache = %p  in delete\n", lle);	
+#endif
+		if (!(flags & LLE_CREATE))
+			return (NULL);
+		/*
+		 * A route that covers the given address must have
+		 * been installed 1st because we are doing a resolution,
+		 * verify this.
+		 */
+		if (!(flags & LLE_IFADDR) &&
+		    in_lltable_rtcheck(ifp, l3addr) != 0)
+			goto done;
+
+		lle = in_lltable_new(l3addr, flags);
+		if (lle == NULL) {
+			log(LOG_INFO, "lla_lookup: new lle malloc failed\n");
+			goto done;
+		}
+		lle->la_flags = flags & ~LLE_CREATE;
+		if ((flags & (LLE_CREATE | LLE_IFADDR)) == (LLE_CREATE | LLE_IFADDR)) {
+			bcopy(IF_LLADDR(ifp), &lle->ll_addr, ifp->if_addrlen);
+			lle->la_flags |= (LLE_VALID | LLE_STATIC);
+		}
+
+		lle->lle_tbl  = llt;
+		lle->lle_head = lleh;
+		LIST_INSERT_HEAD(lleh, lle, lle_next);
+	} else if (flags & LLE_DELETE) {
+		if (!(lle->la_flags & LLE_IFADDR) || (flags & LLE_IFADDR)) {
+			LLE_WLOCK(lle);
+			lle->la_flags = LLE_DELETED;
+			LLE_WUNLOCK(lle);
+#ifdef DIAGNOSTICS
+			log(LOG_INFO, "ifaddr cache = %p  is deleted\n", lle);	
+#endif
+		}
+		lle = (void *)-1;
+		
+	}
+	if (lle != NULL && lle != (void *)-1) {
+		if (flags & LLE_EXCLUSIVE)
+			LLE_WLOCK(lle);
+		else
+			LLE_RLOCK(lle);
+	}
+done:
+	return (lle);
+}
+
+static int
+in_lltable_dump(struct lltable *llt, struct sysctl_req *wr)
+{
+#define	SIN(lle)	((struct sockaddr_in *) L3_ADDR(lle))
+	struct ifnet *ifp = llt->llt_ifp;
+	struct llentry *lle;
+	/* XXX stack use */
+	struct {
+		struct rt_msghdr	rtm;
+		struct sockaddr_inarp	sin;
+		struct sockaddr_dl	sdl;
+	} arpc;
+	int error, i;
+
+	/* XXXXX
+	 * current IFNET_RLOCK() is mapped to IFNET_WLOCK()
+	 * so it is okay to use this ASSERT, change it when
+	 * IFNET lock is finalized
+	 */
+	IFNET_WLOCK_ASSERT();
+
+	error = 0;
+	for (i = 0; i < LLTBL_HASHTBL_SIZE; i++) {
+		LIST_FOREACH(lle, &llt->lle_head[i], lle_next) {
+			struct sockaddr_dl *sdl;
+			
+			/* skip deleted entries */
+			if ((lle->la_flags & (LLE_DELETED|LLE_VALID)) != LLE_VALID)
+				continue;
+			/*
+			 * produce a msg made of:
+			 *  struct rt_msghdr;
+			 *  struct sockaddr_inarp; (IPv4)
+			 *  struct sockaddr_dl;
+			 */
+			bzero(&arpc, sizeof(arpc));
+			arpc.rtm.rtm_msglen = sizeof(arpc);
+			arpc.sin.sin_family = AF_INET;
+			arpc.sin.sin_len = sizeof(arpc.sin);
+			arpc.sin.sin_addr.s_addr = SIN(lle)->sin_addr.s_addr;
+
+			/* publish */
+			if (lle->la_flags & LLE_PUB) {
+				arpc.rtm.rtm_flags |= RTF_ANNOUNCE;
+				/* proxy only */
+				if (lle->la_flags & LLE_PROXY)
+					arpc.sin.sin_other = SIN_PROXY;
+			}
+
+			sdl = &arpc.sdl;
+			sdl->sdl_family = AF_LINK;
+			sdl->sdl_len = sizeof(*sdl);
+			sdl->sdl_alen = ifp->if_addrlen;
+			sdl->sdl_index = ifp->if_index;
+			sdl->sdl_type = ifp->if_type;
+			bcopy(&lle->ll_addr, LLADDR(sdl), ifp->if_addrlen);
+
+			arpc.rtm.rtm_rmx.rmx_expire =
+			    lle->la_flags & LLE_STATIC ? 0 : lle->la_expire;
+			arpc.rtm.rtm_flags |= RTF_HOST;
+			if (lle->la_flags & LLE_STATIC)
+				arpc.rtm.rtm_flags |= RTF_STATIC;
+			arpc.rtm.rtm_index = ifp->if_index;
+			error = SYSCTL_OUT(wr, &arpc, sizeof(arpc));
+			if (error)
+				break;
+		}
+	}
+	return error;
+#undef SIN
+}
+
+void *
+in_domifattach(struct ifnet *ifp)
+{   
+	struct lltable *llt = lltable_init(ifp, AF_INET);
+ 
+	if (llt != NULL) {
+		llt->llt_new = in_lltable_new;
+		llt->llt_free = in_lltable_free;
+		llt->llt_rtcheck = in_lltable_rtcheck;
+		llt->llt_lookup = in_lltable_lookup;
+		llt->llt_dump = in_lltable_dump;
+	}
+	return (llt);
+}
+
+void
+in_domifdetach(struct ifnet *ifp __unused, void *aux)
+{
+	struct lltable *llt = (struct lltable *)aux;
+
+	lltable_free(llt);
+}
diff --git a/sys/netinet/in_mcast.c b/sys/netinet/in_mcast.c
index e7916dd..6f5dfe1 100644
--- a/sys/netinet/in_mcast.c
+++ b/sys/netinet/in_mcast.c
@@ -1036,7 +1036,7 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt)
 
 			ro.ro_rt = NULL;
 			*(struct sockaddr_in *)&ro.ro_dst = gsa->sin;
-			in_rtalloc_ign(&ro, RTF_CLONING,
+			in_rtalloc_ign(&ro, 0,
 			   inp->inp_inc.inc_fibnum);
 			if (ro.ro_rt != NULL) {
 				ifp = ro.ro_rt->rt_ifp;
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index c5515da..84bff92 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -570,7 +570,7 @@ in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
 	 * Find out route to destination.
 	 */
 	if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
-		in_rtalloc_ign(&sro, RTF_CLONING, inp->inp_inc.inc_fibnum);
+		in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum);
 
 	/*
 	 * If we found a route, use the address corresponding to
diff --git a/sys/netinet/in_proto.c b/sys/netinet/in_proto.c
index a93f1f2..f3fbe0c 100644
--- a/sys/netinet/in_proto.c
+++ b/sys/netinet/in_proto.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
+#include <netinet/in_var.h>
 #include <netinet/ip.h>
 #include <netinet/ip_var.h>
 #include <netinet/ip_icmp.h>
@@ -364,7 +365,9 @@ struct domain inetdomain = {
 	.dom_rtattach =		in_inithead,
 #endif
 	.dom_rtoffset =		32,
-	.dom_maxrtkey =		sizeof(struct sockaddr_in)
+	.dom_maxrtkey =		sizeof(struct sockaddr_in),
+	.dom_ifattach =		in_domifattach,
+	.dom_ifdetach =		in_domifdetach
 };
 
 DOMAIN_SET(inet);
diff --git a/sys/netinet/in_rmx.c b/sys/netinet/in_rmx.c
index e61e36a..f9e9d98 100644
--- a/sys/netinet/in_rmx.c
+++ b/sys/netinet/in_rmx.c
@@ -75,8 +75,8 @@ in_addroute(void *v_arg, void *n_arg, struct radix_node_head *head,
 {
 	struct rtentry *rt = (struct rtentry *)treenodes;
 	struct sockaddr_in *sin = (struct sockaddr_in *)rt_key(rt);
-	struct radix_node *ret;
 
+	RADIX_NODE_HEAD_WLOCK_ASSERT(head);
 	/*
 	 * A little bit of help for both IP output and input:
 	 *   For host routes, we make sure that RTF_BROADCAST
@@ -106,31 +106,7 @@ in_addroute(void *v_arg, void *n_arg, struct radix_node_head *head,
 	if (!rt->rt_rmx.rmx_mtu && rt->rt_ifp)
 		rt->rt_rmx.rmx_mtu = rt->rt_ifp->if_mtu;
 
-	ret = rn_addroute(v_arg, n_arg, head, treenodes);
-	if (ret == NULL && rt->rt_flags & RTF_HOST) {
-		struct rtentry *rt2;
-		/*
-		 * We are trying to add a host route, but can't.
-		 * Find out if it is because of an
-		 * ARP entry and delete it if so.
-		 */
-		rt2 = in_rtalloc1((struct sockaddr *)sin, 0,
-		    RTF_CLONING|RTF_RNH_LOCKED, rt->rt_fibnum);
-		if (rt2) {
-			if (rt2->rt_flags & RTF_LLINFO &&
-			    rt2->rt_flags & RTF_HOST &&
-			    rt2->rt_gateway &&
-			    rt2->rt_gateway->sa_family == AF_LINK) {
-				rtexpunge(rt2);
-				RTFREE_LOCKED(rt2);
-				ret = rn_addroute(v_arg, n_arg, head,
-						  treenodes);
-			} else
-				RTFREE_LOCKED(rt2);
-		}
-	}
-
-	return ret;
+	return (rn_addroute(v_arg, n_arg, head, treenodes));
 }
 
 /*
@@ -187,13 +163,10 @@ in_clsroute(struct radix_node *rn, struct radix_node_head *head)
 	if (!(rt->rt_flags & RTF_UP))
 		return;			/* prophylactic measures */
 
-	if ((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST)
-		return;
-
 	if (rt->rt_flags & RTPRF_OURS)
 		return;
 
-	if (!(rt->rt_flags & (RTF_WASCLONED | RTF_DYNAMIC)))
+	if (!(rt->rt_flags & RTF_DYNAMIC))
 		return;
 
 	/*
@@ -434,7 +407,6 @@ in_ifadownkill(struct radix_node *rn, void *xap)
 		 * the routes that rtrequest() would have in any case,
 		 * so that behavior is not needed there.
 		 */
-		rt->rt_flags &= ~RTF_CLONING;
 		rtexpunge(rt);
 	}
 	RT_UNLOCK(rt);
diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h
index 7facfa8..73868ad 100644
--- a/sys/netinet/in_var.h
+++ b/sys/netinet/in_var.h
@@ -321,6 +321,9 @@ void	ip_input(struct mbuf *);
 int	in_ifadown(struct ifaddr *ifa, int);
 void	in_ifscrub(struct ifnet *, struct in_ifaddr *);
 struct	mbuf	*ip_fastforward(struct mbuf *);
+void	*in_domifattach(struct ifnet *);
+void	in_domifdetach(struct ifnet *, void *);
+
 
 /* XXX */
 void	 in_rtalloc_ign(struct route *ro, u_long ignflags, u_int fibnum);
diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 9f6bc31..6d96163 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -371,14 +371,6 @@ carp_setroute(struct carp_softc *sc, int cmd)
 			    (cmd == RTM_DELETE && count == 0))
 				rtinit(ifa, cmd, RTF_UP | RTF_HOST);
 		}
-#ifdef INET6
-		if (ifa->ifa_addr->sa_family == AF_INET6) {
-			if (cmd == RTM_ADD)
-				in6_ifaddloop(ifa);
-			else
-				in6_ifremloop(ifa);
-		}
-#endif /* INET6 */
 	}
 	splx(s);
 }
diff --git a/sys/netinet/ip_fastfwd.c b/sys/netinet/ip_fastfwd.c
index 407b08c..ea106df 100644
--- a/sys/netinet/ip_fastfwd.c
+++ b/sys/netinet/ip_fastfwd.c
@@ -128,7 +128,7 @@ ip_findroute(struct route *ro, struct in_addr dest, struct mbuf *m)
 	dst->sin_family = AF_INET;
 	dst->sin_len = sizeof(*dst);
 	dst->sin_addr.s_addr = dest.s_addr;
-	in_rtalloc_ign(ro, RTF_CLONING, M_GETFIB(m));
+	in_rtalloc_ign(ro, 0, M_GETFIB(m));
 
 	/*
 	 * Route there and interface still up?
diff --git a/sys/netinet/ip_fw2.c b/sys/netinet/ip_fw2.c
index e91fc9b..eae49c8 100644
--- a/sys/netinet/ip_fw2.c
+++ b/sys/netinet/ip_fw2.c
@@ -528,7 +528,7 @@ verify_path(struct in_addr src, struct ifnet *ifp, u_int fib)
 	dst->sin_family = AF_INET;
 	dst->sin_len = sizeof(*dst);
 	dst->sin_addr = src;
-	in_rtalloc_ign(&ro, RTF_CLONING, fib);
+	in_rtalloc_ign(&ro, 0, fib);
 
 	if (ro.ro_rt == NULL)
 		return 0;
@@ -620,7 +620,7 @@ verify_path6(struct in6_addr *src, struct ifnet *ifp)
 	dst->sin6_len = sizeof(*dst);
 	dst->sin6_addr = *src;
 	/* XXX MRT 0 for ipv6 at this time */
-	rtalloc_ign((struct route *)&ro, RTF_CLONING);
+	rtalloc_ign((struct route *)&ro, 0);
 
 	if (ro.ro_rt == NULL)
 		return 0;
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index c99c53e..4a169ff 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -1286,7 +1286,7 @@ ip_rtaddr(struct in_addr dst, u_int fibnum)
 	sin->sin_family = AF_INET;
 	sin->sin_len = sizeof(*sin);
 	sin->sin_addr = dst;
-	in_rtalloc_ign(&sro, RTF_CLONING, fibnum);
+	in_rtalloc_ign(&sro, 0, fibnum);
 
 	if (sro.ro_rt == NULL)
 		return (NULL);
@@ -1412,7 +1412,7 @@ ip_forward(struct mbuf *m, int srcrt)
 		sin->sin_family = AF_INET;
 		sin->sin_len = sizeof(*sin);
 		sin->sin_addr = ip->ip_dst;
-		in_rtalloc_ign(&ro, RTF_CLONING, M_GETFIB(m));
+		in_rtalloc_ign(&ro, 0, M_GETFIB(m));
 
 		rt = ro.ro_rt;
 
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
index 03cf56f..6cda8aa 100644
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -567,7 +567,6 @@ passout:
 		 * to avoid confusing lower layers.
 		 */
 		m->m_flags &= ~(M_PROTOFLAGS);
-
 		error = (*ifp->if_output)(ifp, m,
 				(struct sockaddr *)dst, ro->ro_rt);
 		goto done;
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index 2c2337e..e7a6dff 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -1659,7 +1659,7 @@ tcp_maxmtu(struct in_conninfo *inc, int *flags)
 		dst->sin_family = AF_INET;
 		dst->sin_len = sizeof(*dst);
 		dst->sin_addr = inc->inc_faddr;
-		in_rtalloc_ign(&sro, RTF_CLONING, inc->inc_fibnum);
+		in_rtalloc_ign(&sro, 0, inc->inc_fibnum);
 	}
 	if (sro.ro_rt != NULL) {
 		ifp = sro.ro_rt->rt_ifp;
@@ -1694,7 +1694,7 @@ tcp_maxmtu6(struct in_conninfo *inc, int *flags)
 		sro6.ro_dst.sin6_family = AF_INET6;
 		sro6.ro_dst.sin6_len = sizeof(struct sockaddr_in6);
 		sro6.ro_dst.sin6_addr = inc->inc6_faddr;
-		rtalloc_ign((struct route *)&sro6, RTF_CLONING);
+		rtalloc_ign((struct route *)&sro6, 0);
 	}
 	if (sro6.ro_rt != NULL) {
 		ifp = sro6.ro_rt->rt_ifp;