SMP locking for radix nodes.

11 files changed, 99 insertions, 23 deletions

diff --git a/sys/kern/vfs_export.c b/sys/kern/vfs_export.c
index 22927bd..b909b83 100644
--- a/sys/kern/vfs_export.c
+++ b/sys/kern/vfs_export.c
@@ -137,7 +137,7 @@ vfs_hang_addrlist(mp, nep, argp)
 			smask->sa_len = argp->ex_masklen;
 	}
 	i = saddr->sa_family;
-	if ((rnh = nep->ne_rtable[i]) == 0) {
+	if ((rnh = nep->ne_rtable[i]) == NULL) {
 		/*
 		 * Seems silly to initialize every AF when most are not used,
 		 * do so on demand here
@@ -148,13 +148,15 @@ vfs_hang_addrlist(mp, nep, argp)
 				    dom->dom_rtoffset);
 				break;
 			}
-		if ((rnh = nep->ne_rtable[i]) == 0) {
+		if ((rnh = nep->ne_rtable[i]) == NULL) {
 			error = ENOBUFS;
 			goto out;
 		}
 	}
+	RADIX_NODE_HEAD_LOCK(rnh);
 	rn = (*rnh->rnh_addaddr)(saddr, smask, rnh, np->netc_rnodes);
-	if (rn == 0 || np != (struct netcred *)rn) {	/* already exists */
+	RADIX_NODE_HEAD_UNLOCK(rnh);
+	if (rn == NULL || np != (struct netcred *)rn) {	/* already exists */
 		error = EPERM;
 		goto out;
 	}
@@ -197,9 +199,11 @@ vfs_free_addrlist(nep)
 
 	for (i = 0; i <= AF_MAX; i++)
 		if ((rnh = nep->ne_rtable[i])) {
+			RADIX_NODE_HEAD_LOCK(rnh);
 			(*rnh->rnh_walktree) (rnh, vfs_free_netcred, rnh);
+			RADIX_NODE_HEAD_DESTROY(rnh);
 			free(rnh, M_RTABLE);
-			nep->ne_rtable[i] = 0;
+			nep->ne_rtable[i] = NULL;	/* not SMP safe XXX */
 		}
 }
 
@@ -355,8 +359,10 @@ vfs_export_lookup(mp, nam)
 			saddr = nam;
 			rnh = nep->ne_rtable[saddr->sa_family];
 			if (rnh != NULL) {
+				RADIX_NODE_HEAD_LOCK(rnh);
 				np = (struct netcred *)
 				    (*rnh->rnh_matchaddr)(saddr, rnh);
+				RADIX_NODE_HEAD_UNLOCK(rnh);
 				if (np && np->netc_rnodes->rn_flags & RNF_ROOT)
 					np = NULL;
 			}
diff --git a/sys/net/if.c b/sys/net/if.c
index 42f2590..1f58605 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -533,7 +533,9 @@ if_detach(ifp)
 	for (i = 1; i <= AF_MAX; i++) {
 		if ((rnh = rt_tables[i]) == NULL)
 			continue;
+		RADIX_NODE_HEAD_LOCK(rnh);
 		(void) rnh->rnh_walktree(rnh, if_rtdel, ifp);
+		RADIX_NODE_HEAD_UNLOCK(rnh);
 	}
 
 	/* Announce that the interface is gone. */
diff --git a/sys/net/radix.c b/sys/net/radix.c
index e1742f3..f6cdadb 100644
--- a/sys/net/radix.c
+++ b/sys/net/radix.c
@@ -1024,6 +1024,7 @@ rn_inithead(head, off)
 	if (rnh == 0)
 		return (0);
 	Bzero(rnh, sizeof (*rnh));
+	RADIX_NODE_HEAD_LOCK_INIT(rnh);
 	*head = rnh;
 	t = rn_newpair(rn_zeros, off, rnh->rnh_nodes);
 	ttt = rnh->rnh_nodes + 2;
diff --git a/sys/net/radix.h b/sys/net/radix.h
index 3a65585..d9937ca 100644
--- a/sys/net/radix.h
+++ b/sys/net/radix.h
@@ -37,6 +37,9 @@
 #ifndef _RADIX_H_
 #define	_RADIX_H_
 
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
 #ifdef MALLOC_DECLARE
 MALLOC_DECLARE(M_RTABLE);
 #endif
@@ -138,6 +141,7 @@ struct radix_node_head {
 	void	(*rnh_close)	/* do something when the last ref drops */
 		(struct radix_node *rn, struct radix_node_head *head);
 	struct	radix_node rnh_nodes[3];	/* empty tree for common case */
+	struct	mtx rnh_mtx;
 };
 
 #ifndef _KERNEL
@@ -152,6 +156,13 @@ struct radix_node_head {
 #define Bzero(p, n) bzero((caddr_t)(p), (unsigned)(n));
 #define R_Malloc(p, t, n) (p = (t) malloc((unsigned long)(n), M_RTABLE, M_NOWAIT))
 #define Free(p) free((caddr_t)p, M_RTABLE);
+
+
+#define RADIX_NODE_HEAD_LOCK_INIT(rnh)	\
+    mtx_init(&(rnh)->rnh_mtx, "radix node head", NULL, MTX_DEF | MTX_RECURSE)
+#define RADIX_NODE_HEAD_LOCK(rnh)	mtx_lock(&(rnh)->rnh_mtx)
+#define RADIX_NODE_HEAD_UNLOCK(rnh)	mtx_unlock(&(rnh)->rnh_mtx)
+#define RADIX_NODE_HEAD_DESTROY(rnh)	mtx_destroy(&(rnh)->rnh_mtx)
 #endif /* _KERNEL */
 
 void	 rn_init(void);
diff --git a/sys/net/route.c b/sys/net/route.c
index 1727d5b..260f375 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -132,7 +132,12 @@ rtalloc1(dst, report, ignflags)
 	/*
 	 * Look up the address in the table for that Address Family
 	 */
-	if (rnh && (rn = rnh->rnh_matchaddr((caddr_t)dst, rnh)) &&
+	if (rnh == NULL) {
+		rtstat.rts_unreach++;
+		goto miss2;
+	}
+	RADIX_NODE_HEAD_LOCK(rnh);
+	if ((rn = rnh->rnh_matchaddr((caddr_t)dst, rnh)) &&
 	    ((rn->rn_flags & RNF_ROOT) == 0)) {
 		/*
 		 * If we find it and it's not the root node, then
@@ -178,6 +183,7 @@ rtalloc1(dst, report, ignflags)
 			rt_missmsg(RTM_ADD, &info, rt->rt_flags, 0);
 		} else
 			rt->rt_refcnt++;
+		RADIX_NODE_HEAD_UNLOCK(rnh);
 	} else {
 		/*
 		 * Either we hit the root or couldn't find any match,
@@ -185,7 +191,9 @@ rtalloc1(dst, report, ignflags)
 		 * "caint get there frm here"
 		 */
 		rtstat.rts_unreach++;
-	miss:	if (report) {
+	miss:
+		RADIX_NODE_HEAD_UNLOCK(rnh);
+	miss2:	if (report) {
 			/*
 			 * If required, report the failure to the supervising
 			 * Authorities.
@@ -211,8 +219,7 @@ rtfree(rt)
 	/*
 	 * find the tree for that address family
 	 */
-	register struct radix_node_head *rnh =
-		rt_tables[rt_key(rt)->sa_family];
+	struct radix_node_head *rnh = rt_tables[rt_key(rt)->sa_family];
 
 	if (rt == 0 || rnh == 0)
 		panic("rtfree");
@@ -222,7 +229,7 @@ rtfree(rt)
 	 * and there is a close function defined, call the close function
 	 */
 	rt->rt_refcnt--;
-	if(rnh->rnh_close && rt->rt_refcnt == 0) {
+	if (rnh->rnh_close && rt->rt_refcnt == 0) {
 		rnh->rnh_close((struct radix_node *)rt, rnh);
 	}
 
@@ -270,7 +277,8 @@ rtfree(rt)
 	}
 }
 
-#define	equal(a1, a2) (bcmp((caddr_t)(a1), (caddr_t)(a2), (a1)->sa_len) == 0)
+/* compare two sockaddr structures */
+#define	sa_equal(a1, a2) (bcmp((a1), (a2), (a1)->sa_len) == 0)
 
 /*
  * Force a routing table entry to the specified
@@ -306,7 +314,7 @@ rtredirect(dst, gateway, netmask, flags, src, rtp)
 	 * going down recently.
 	 */
 	if (!(flags & RTF_DONE) && rt &&
-	     (!equal(src, rt->rt_gateway) || rt->rt_ifa != ifa))
+	     (!sa_equal(src, rt->rt_gateway) || rt->rt_ifa != ifa))
 		error = EINVAL;
 	else if (ifa_ifwithaddr(gateway))
 		error = EHOSTUNREACH;
@@ -538,6 +546,7 @@ rtrequest1(req, info, ret_nrt)
 	 */
 	if ((rnh = rt_tables[dst->sa_family]) == 0)
 		senderr(EAFNOSUPPORT);
+	RADIX_NODE_HEAD_LOCK(rnh);
 	/*
 	 * If we are adding a host route then we don't want to put
 	 * a netmask in the tree, nor do we want to clone it.
@@ -763,6 +772,7 @@ rtrequest1(req, info, ret_nrt)
 		error = EOPNOTSUPP;
 	}
 bad:
+	RADIX_NODE_HEAD_UNLOCK(rnh);
 	splx(s);
 	return (error);
 #undef dst
@@ -1002,8 +1012,10 @@ rt_setgate(rt0, dst, gate)
 		struct rtfc_arg arg;
 		arg.rnh = rnh;
 		arg.rt0 = rt;
+		RADIX_NODE_HEAD_LOCK(rnh);
 		rnh->rnh_walktree_from(rnh, rt_key(rt), rt_mask(rt),
 				       rt_fixchange, &arg);
+		RADIX_NODE_HEAD_UNLOCK(rnh);
 	}
 
 	return 0;
@@ -1079,11 +1091,16 @@ rtinit(ifa, cmd, flags)
 		 * Look up an rtentry that is in the routing tree and
 		 * contains the correct info.
 		 */
-		if ((rnh = rt_tables[dst->sa_family]) == NULL ||
-		    (rn = rnh->rnh_lookup(dst, netmask, rnh)) == NULL ||
+		if ((rnh = rt_tables[dst->sa_family]) == NULL)
+			goto bad;
+		RADIX_NODE_HEAD_LOCK(rnh);
+		error = ((rn = rnh->rnh_lookup(dst, netmask, rnh)) == NULL ||
 		    (rn->rn_flags & RNF_ROOT) ||
 		    ((struct rtentry *)rn)->rt_ifa != ifa ||
-		    !equal(SA(rn->rn_key), dst)) {
+		    !sa_equal(SA(rn->rn_key), dst));
+		RADIX_NODE_HEAD_UNLOCK(rnh);
+		if (error) {
+bad:
 			if (m)
 				(void) m_free(m);
 			return (flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH);
diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c
index 40f2ee1..0c441ef 100644
--- a/sys/net/rtsock.c
+++ b/sys/net/rtsock.c
@@ -367,11 +367,15 @@ route_output(m, so)
 	case RTM_LOCK:
 		if ((rnh = rt_tables[dst->sa_family]) == 0) {
 			senderr(EAFNOSUPPORT);
-		} else if ((rt = (struct rtentry *)
-				rnh->rnh_lookup(dst, netmask, rnh)) != NULL)
+		}
+		RADIX_NODE_HEAD_LOCK(rnh);
+		rt = (struct rtentry *) rnh->rnh_lookup(dst, netmask, rnh);
+		RADIX_NODE_HEAD_UNLOCK(rnh);
+		if (rt != NULL)
 			rt->rt_refcnt++;
 		else
 			senderr(ESRCH);
+
 		switch(rtm->rtm_type) {
 
 		case RTM_GET:
@@ -1024,11 +1028,25 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS)
 
 	case NET_RT_DUMP:
 	case NET_RT_FLAGS:
-		for (i = 1; i <= AF_MAX; i++)
-			if ((rnh = rt_tables[i]) && (af == 0 || af == i) &&
-			    (error = rnh->rnh_walktree(rnh,
-							sysctl_dumpentry, &w)))
-				break;
+		if (af != 0) {
+			if ((rnh = rt_tables[af]) != NULL) {
+				RADIX_NODE_HEAD_LOCK(rnh);
+			    	error = rnh->rnh_walktree(rnh,
+				    sysctl_dumpentry, &w);
+				RADIX_NODE_HEAD_UNLOCK(rnh);
+			} else
+				error = EAFNOSUPPORT;
+		} else {
+			for (i = 1; i <= AF_MAX; i++)
+				if ((rnh = rt_tables[i]) != NULL) {
+					RADIX_NODE_HEAD_LOCK(rnh);
+					error = rnh->rnh_walktree(rnh,
+					    sysctl_dumpentry, &w);
+					RADIX_NODE_HEAD_UNLOCK(rnh);
+					if (error)
+						break;
+				}
+		}
 		break;
 
 	case NET_RT_IFLIST:
diff --git a/sys/netatm/atm_if.c b/sys/netatm/atm_if.c
index e5fdb6b..24618a7 100644
--- a/sys/netatm/atm_if.c
+++ b/sys/netatm/atm_if.c
@@ -869,7 +869,9 @@ atm_nif_detach(nip)
 	for (i = 1; i <= AF_MAX; i++) {
 		if ((rnh = rt_tables[i]) == NULL)
 			continue;
+		RADIX_NODE_HEAD_LOCK(rnh);
 		(void) rnh->rnh_walktree(rnh, atm_netif_rtdel, ifp);
+		RADIX_NODE_HEAD_UNLOCK(rnh);
 	}
 
 	/*
diff --git a/sys/netinet/in_rmx.c b/sys/netinet/in_rmx.c
index 08052fb..a495959 100644
--- a/sys/netinet/in_rmx.c
+++ b/sys/netinet/in_rmx.c
@@ -291,7 +291,9 @@ in_rtqtimo(void *rock)
 	arg.nextstop = time_second + rtq_timeout;
 	arg.draining = arg.updating = 0;
 	s = splnet();
+	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in_rtqkill, &arg);
+	RADIX_NODE_HEAD_UNLOCK(rnh);
 	splx(s);
 
 	/*
@@ -318,7 +320,9 @@ in_rtqtimo(void *rock)
 		arg.found = arg.killed = 0;
 		arg.updating = 1;
 		s = splnet();
+		RADIX_NODE_HEAD_LOCK(rnh);
 		rnh->rnh_walktree(rnh, in_rtqkill, &arg);
+		RADIX_NODE_HEAD_UNLOCK(rnh);
 		splx(s);
 	}
 
@@ -339,7 +343,9 @@ in_rtqdrain(void)
 	arg.draining = 1;
 	arg.updating = 0;
 	s = splnet();
+	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in_rtqkill, &arg);
+	RADIX_NODE_HEAD_UNLOCK(rnh);
 	splx(s);
 }
 
@@ -420,7 +426,9 @@ in_ifadown(struct ifaddr *ifa, int delete)
 	arg.rnh = rnh = rt_tables[AF_INET];
 	arg.ifa = ifa;
 	arg.del = delete;
+	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in_ifadownkill, &arg);
+	RADIX_NODE_HEAD_UNLOCK(rnh);
 	ifa->ifa_flags &= ~IFA_ROUTE;
 	return 0;
 }
diff --git a/sys/netinet6/in6_rmx.c b/sys/netinet6/in6_rmx.c
index 60732b2..14587bd 100644
--- a/sys/netinet6/in6_rmx.c
+++ b/sys/netinet6/in6_rmx.c
@@ -257,8 +257,7 @@ in6_clsroute(struct radix_node *rn, struct radix_node_head *head)
 	if ((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST)
 		return;
 
-	if ((rt->rt_flags & (RTF_WASCLONED | RTPRF_OURS))
-	   != RTF_WASCLONED)
+	if ((rt->rt_flags & (RTF_WASCLONED | RTPRF_OURS)) != RTF_WASCLONED)
 		return;
 
 	/*
@@ -347,7 +346,9 @@ in6_rtqtimo(void *rock)
 	arg.nextstop = time_second + rtq_timeout;
 	arg.draining = arg.updating = 0;
 	s = splnet();
+	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in6_rtqkill, &arg);
+	RADIX_NODE_HEAD_UNLOCK(rnh);
 	splx(s);
 
 	/*
@@ -374,7 +375,9 @@ in6_rtqtimo(void *rock)
 		arg.found = arg.killed = 0;
 		arg.updating = 1;
 		s = splnet();
+		RADIX_NODE_HEAD_LOCK(rnh);
 		rnh->rnh_walktree(rnh, in6_rtqkill, &arg);
+		RADIX_NODE_HEAD_UNLOCK(rnh);
 		splx(s);
 	}
 
@@ -426,7 +429,9 @@ in6_mtutimo(void *rock)
 	arg.rnh = rnh;
 	arg.nextstop = time_second + MTUTIMO_DEFAULT;
 	s = splnet();
+	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in6_mtuexpire, &arg);
+	RADIX_NODE_HEAD_UNLOCK(rnh);
 	splx(s);
 
 	atv.tv_usec = 0;
@@ -451,7 +456,9 @@ in6_rtqdrain()
 	arg.draining = 1;
 	arg.updating = 0;
 	s = splnet();
+	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, in6_rtqkill, &arg);
+	RADIX_NODE_HEAD_UNLOCK(rnh);
 	splx(s);
 }
 #endif
diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c
index efb4867..c920636 100644
--- a/sys/netinet6/nd6_rtr.c
+++ b/sys/netinet6/nd6_rtr.c
@@ -1908,7 +1908,9 @@ rt6_flush(gateway, ifp)
 	/* XXX: hack for KAME's link-local address kludge */
 	gateway->s6_addr16[1] = htons(ifp->if_index);
 
+	RADIX_NODE_HEAD_LOCK(rnh);
 	rnh->rnh_walktree(rnh, rt6_deleteroute, (void *)gateway);
+	RADIX_NODE_HEAD_UNLOCK(rnh);
 	splx(s);
 }
 
diff --git a/sys/nfsclient/bootp_subr.c b/sys/nfsclient/bootp_subr.c
index 50f3b1f..d77f8f7 100644
--- a/sys/nfsclient/bootp_subr.c
+++ b/sys/nfsclient/bootp_subr.c
@@ -375,7 +375,9 @@ bootpboot_p_rtlist(void)
 {
 
 	printf("Routing table:\n");
+	RADIX_NODE_LOCK(rt_tables[AF_INET]);	/* could sleep XXX */
 	bootpboot_p_tree(rt_tables[AF_INET]->rnh_treetop);
+	RADIX_NODE_UNLOCK(rt_tables[AF_INET]);
 }
 
 void