Introduce basic locking of global IPX address list 'ipx_ifaddr' using

a new rwlock, ipx_ifaddr_rw, wrapped with macros.  This locking is
necessary but not sufficient, in isolation, to satisfy the stability
requirements of a fully parallel IPX input path during interface
reconfiguration.

MFC after:	3 weeks

5 files changed, 61 insertions, 13 deletions

diff --git a/sys/netipx/ipx.c b/sys/netipx/ipx.c
index cd2240f..d19b920 100644
--- a/sys/netipx/ipx.c
+++ b/sys/netipx/ipx.c
@@ -65,8 +65,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/systm.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/priv.h>
+#include <sys/rwlock.h>
 #include <sys/sockio.h>
 #include <sys/socket.h>
 
@@ -78,9 +80,10 @@ __FBSDID("$FreeBSD$");
 #include <netipx/ipx_var.h>
 
 /*
- * XXXRW: Requires synchronization.
+ * The IPX-layer address list is protected by ipx_ifaddr_rw.
  */
-struct ipx_ifaddr *ipx_ifaddr;
+struct rwlock		 ipx_ifaddr_rw;
+struct ipx_ifaddr	*ipx_ifaddr;
 
 static void	ipx_ifscrub(struct ifnet *ifp, struct ipx_ifaddr *ia);
 static int	ipx_ifinit(struct ifnet *ifp, struct ipx_ifaddr *ia,
@@ -345,6 +348,8 @@ ipx_iaonnetof(struct ipx_addr *dst)
 	struct ipx_ifaddr *ia_maybe = NULL;
 	union ipx_net net = dst->x_net;
 
+	IPX_IFADDR_LOCK_ASSERT();
+
 	for (ia = ipx_ifaddr; ia != NULL; ia = ia->ia_next) {
 		if ((ifp = ia->ia_ifp) != NULL) {
 			if (ifp->if_flags & IFF_POINTOPOINT) {
diff --git a/sys/netipx/ipx_if.h b/sys/netipx/ipx_if.h
index 8b5d16c..fbf5547 100644
--- a/sys/netipx/ipx_if.h
+++ b/sys/netipx/ipx_if.h
@@ -105,7 +105,16 @@ struct	ipx_aliasreq {
 #define	ETHERTYPE_IPX		0x8137	/* Only  Ethernet_II Available */
 
 #ifdef	_KERNEL
-extern struct	ipx_ifaddr *ipx_ifaddr;
+extern struct rwlock		 ipx_ifaddr_rw;
+extern struct ipx_ifaddr	*ipx_ifaddr;
+
+#define	IPX_IFADDR_LOCK_INIT()		rw_init(&ipx_ifaddr_rw, "ipx_ifaddr_rw")
+#define	IPX_IFADDR_LOCK_ASSERT()	rw_assert(&ipx_ifaddr_rw, RA_LOCKED)
+#define	IPX_IFADDR_RLOCK()		rw_rlock(&ipx_ifaddr_rw)
+#define	IPX_IFADDR_RUNLOCK()		rw_runlock(&ipx_ifaddr_rw)
+#define	IPX_IFADDR_WLOCK()		rw_wlock(&ipx_ifaddr_rw)
+#define	IPX_IFADDR_WUNLOCK()		rw_wunlock(&ipx_ifaddr_rw)
+#define	IPX_IFADDR_RLOCK_ASSERT()	rw_assert(&ipx_ifaddr_rw, RA_WLOCKED)
 
 struct ipx_ifaddr	*ipx_iaonnetof(struct ipx_addr *dst);
 #endif
diff --git a/sys/netipx/ipx_input.c b/sys/netipx/ipx_input.c
index 6c29ec4..d0a9d4e 100644
--- a/sys/netipx/ipx_input.c
+++ b/sys/netipx/ipx_input.c
@@ -72,6 +72,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/socket.h>
 #include <sys/kernel.h>
 #include <sys/random.h>
+#include <sys/rwlock.h>
 #include <sys/sysctl.h>
 
 #include <net/if.h>
@@ -146,6 +147,7 @@ ipx_init(void)
 	LIST_INIT(&ipxrawpcb_list);
 
 	IPX_LIST_LOCK_INIT();
+	IPX_IFADDR_LOCK_INIT();
 
 	ipx_netmask.sipx_len = 6;
 	ipx_netmask.sipx_addr.x_net = ipx_broadnet;
@@ -253,11 +255,15 @@ ipxintr(struct mbuf *m)
 			 * If it is a broadcast to the net where it was
 			 * received from, treat it as ours.
 			 */
+			IPX_IFADDR_RLOCK();
 			for (ia = ipx_ifaddr; ia != NULL; ia = ia->ia_next)
 				if((ia->ia_ifa.ifa_ifp == m->m_pkthdr.rcvif) &&
 				   ipx_neteq(ia->ia_addr.sipx_addr,
-					     ipx->ipx_dna))
+					     ipx->ipx_dna)) {
+					IPX_IFADDR_RUNLOCK();
 					goto ours;
+				}
+			IPX_IFADDR_RUNLOCK();
 
 			/*
 			 * Look to see if I need to eat this packet.
@@ -278,12 +284,13 @@ ipxintr(struct mbuf *m)
 	 * Is this our packet? If not, forward.
 	 */
 	} else {
+		IPX_IFADDR_RLOCK();
 		for (ia = ipx_ifaddr; ia != NULL; ia = ia->ia_next)
 			if (ipx_hosteq(ipx->ipx_dna, ia->ia_addr.sipx_addr) &&
 			    (ipx_neteq(ipx->ipx_dna, ia->ia_addr.sipx_addr) ||
 			     ipx_neteqnn(ipx->ipx_dna.x_net, ipx_zeronet)))
 				break;
-
+		IPX_IFADDR_RUNLOCK();
 		if (ia == NULL) {
 			ipx_forward(m);
 			return;
@@ -370,13 +377,17 @@ ipx_forward(struct mbuf *m)
 	 * age the packet so we can eat it safely the second time around.
 	 */
 	if (ipx->ipx_dna.x_host.c_host[0] & 0x1) {
-		struct ipx_ifaddr *ia = ipx_iaonnetof(&ipx->ipx_dna);
+		struct ipx_ifaddr *ia;
 		struct ifnet *ifp;
+
+		IPX_IFADDR_RLOCK();
+		ia = ipx_iaonnetof(&ipx->ipx_dna);
 		if (ia != NULL) {
 			/* I'm gonna hafta eat this packet */
 			agedelta += IPX_MAXHOPS - ipx->ipx_tc;
 			ipx->ipx_tc = IPX_MAXHOPS;
 		}
+		IPX_IFADDR_RUNLOCK();
 		if ((ok_back = ipx_do_route(&ipx->ipx_sna,&ipx_sroute)) == 0) {
 			/* error = ENETUNREACH; He'll never get it! */
 			ipxstat.ipxs_noroute++;
diff --git a/sys/netipx/ipx_outputfl.c b/sys/netipx/ipx_outputfl.c
index f6be964..325a5f9 100644
--- a/sys/netipx/ipx_outputfl.c
+++ b/sys/netipx/ipx_outputfl.c
@@ -101,14 +101,18 @@ ipx_outputfl(struct mbuf *m0, struct route *ro, int flags)
 		 * short circuit routing lookup.
 		 */
 		if (flags & IPX_ROUTETOIF) {
-			struct ipx_ifaddr *ia = ipx_iaonnetof(&ipx->ipx_dna);
+			struct ipx_ifaddr *ia;
 
+			IPX_IFADDR_RLOCK();
+			ia = ipx_iaonnetof(&ipx->ipx_dna);
 			if (ia == NULL) {
+				IPX_IFADDR_RUNLOCK();
 				ipxstat.ipxs_noroute++;
 				error = ENETUNREACH;
 				goto bad;
 			}
 			ifp = ia->ia_ifp;
+			IPX_IFADDR_RUNLOCK();
 			goto gotif;
 		}
 		rtalloc_ign(ro, 0);
@@ -200,6 +204,7 @@ ipx_output_type20(struct mbuf *m)
 	/*
 	 * Now see if we have already seen this.
 	 */
+	IPX_IFADDR_RLOCK();
 	for (ia = ipx_ifaddr; ia != NULL; ia = ia->ia_next)
 		if(ia->ia_ifa.ifa_ifp == m->m_pkthdr.rcvif) {
 			if(tia == NULL)
@@ -207,15 +212,20 @@ ipx_output_type20(struct mbuf *m)
 
 			for (i=0;i<ipx->ipx_tc;i++,nbnet++)
 				if(ipx_neteqnn(ia->ia_addr.sipx_addr.x_net,
-							*nbnet))
+							*nbnet)) {
+					IPX_IFADDR_RUNLOCK();
 					goto bad;
+				}
 		}
+
 	/*
 	 * Don't route the packet if the interface where it come from
 	 * does not have an IPX address.
 	 */
-	if(tia == NULL)
+	if (tia == NULL) {
+		IPX_IFADDR_RUNLOCK();
 		goto bad;
+	}
 
 	/*
 	 * Add our receiving interface to the list.
@@ -266,6 +276,7 @@ ipx_output_type20(struct mbuf *m)
 			}
 skip_this: ;
 		}
+	IPX_IFADDR_RUNLOCK();
 
 bad:
 	m_freem(m);
diff --git a/sys/netipx/ipx_pcb.c b/sys/netipx/ipx_pcb.c
index a478c08..bfe68c3 100644
--- a/sys/netipx/ipx_pcb.c
+++ b/sys/netipx/ipx_pcb.c
@@ -222,10 +222,13 @@ ipx_pcbconnect(struct ipxpcb *ipxp, struct sockaddr *nam, struct thread *td)
 		 * If we found a route, use the address
 		 * corresponding to the outgoing interface
 		 */
-		if (ro->ro_rt != NULL && (ifp = ro->ro_rt->rt_ifp) != NULL)
+		if (ro->ro_rt != NULL && (ifp = ro->ro_rt->rt_ifp) != NULL) {
+			IPX_IFADDR_RLOCK();
 			for (ia = ipx_ifaddr; ia != NULL; ia = ia->ia_next)
 				if (ia->ia_ifp == ifp)
 					break;
+			IPX_IFADDR_RUNLOCK();
+		}
 		if (ia == NULL) {
 			u_short fport = sipx->sipx_addr.x_port;
 			sipx->sipx_addr.x_port = 0;
@@ -251,20 +254,29 @@ ipx_pcbconnect(struct ipxpcb *ipxp, struct sockaddr *nam, struct thread *td)
 		 * If we found a route, use the address
 		 * corresponding to the outgoing interface
 		 */
-		if (ro->ro_rt != NULL && (ifp = ro->ro_rt->rt_ifp) != NULL)
+		if (ro->ro_rt != NULL && (ifp = ro->ro_rt->rt_ifp) != NULL) {
+			IPX_IFADDR_RLOCK();
 			for (ia = ipx_ifaddr; ia != NULL; ia = ia->ia_next)
 				if (ia->ia_ifp == ifp)
 					break;
+			IPX_IFADDR_RUNLOCK();
+		}
 		if (ia == NULL) {
 			u_short fport = sipx->sipx_addr.x_port;
 			sipx->sipx_addr.x_port = 0;
 			ia = (struct ipx_ifaddr *)
 				ifa_ifwithdstaddr((struct sockaddr *)sipx);
 			sipx->sipx_addr.x_port = fport;
-			if (ia == NULL)
+			if (ia == NULL) {
+				IPX_IFADDR_RLOCK();
 				ia = ipx_iaonnetof(&sipx->sipx_addr);
-			if (ia == NULL)
+				IPX_IFADDR_RUNLOCK();
+			}
+			if (ia == NULL) {
+				IPX_IFADDR_RLOCK();
 				ia = ipx_ifaddr;
+				IPX_IFADDR_RUNLOCK();
+			}
 			if (ia == NULL)
 				return (EADDRNOTAVAIL);
 		}