Lock down a socket, milestone 1.

o Add a mutex (sb_mtx) to struct sockbuf. This protects the data in a
  socket buffer. The mutex in the receive buffer also protects the data
  in struct socket.

o Determine the lock strategy for each members in struct socket.

o Lock down the following members:

  - so_count
  - so_options
  - so_linger
  - so_state

o Remove *_locked() socket APIs.  Make the following socket APIs
  touching the members above now require a locked socket:

 - sodisconnect()
 - soisconnected()
 - soisconnecting()
 - soisdisconnected()
 - soisdisconnecting()
 - sofree()
 - soref()
 - sorele()
 - sorwakeup()
 - sotryfree()
 - sowakeup()
 - sowwakeup()

Reviewed by:	alfred

3 files changed, 109 insertions, 20 deletions

diff --git a/sys/netipx/ipx_pcb.c b/sys/netipx/ipx_pcb.c
index 8fe9796..ebe20bd 100644
--- a/sys/netipx/ipx_pcb.c
+++ b/sys/netipx/ipx_pcb.c
@@ -156,8 +156,12 @@ ipx_pcbconnect(ipxp, nam, td)
 	 */
 	ro = &ipxp->ipxp_route;
 	dst = &satoipx_addr(ro->ro_dst);
-	if (ipxp->ipxp_socket->so_options & SO_DONTROUTE)
+	SOCK_LOCK(ipxp->ipxp_socket);
+	if (ipxp->ipxp_socket->so_options & SO_DONTROUTE) {
+		SOCK_UNLOCK(ipxp->ipxp_socket);
 		goto flush;
+	}
+	SOCK_UNLOCK(ipxp->ipxp_socket);
 	if (!ipx_neteq(ipxp->ipxp_lastdst, sipx->sipx_addr))
 		goto flush;
 	if (!ipx_hosteq(ipxp->ipxp_lastdst, sipx->sipx_addr)) {
@@ -172,15 +176,18 @@ ipx_pcbconnect(ipxp, nam, td)
 		}
 	}/* else cached route is ok; do nothing */
 	ipxp->ipxp_lastdst = sipx->sipx_addr;
+	SOCK_LOCK(ipxp->ipxp_socket);
 	if ((ipxp->ipxp_socket->so_options & SO_DONTROUTE) == 0 && /*XXX*/
 	    (ro->ro_rt == NULL || ro->ro_rt->rt_ifp == NULL)) {
+		SOCK_UNLOCK(ipxp->ipxp_socket);
 		    /* No route yet, so try to acquire one */
 		    ro->ro_dst.sa_family = AF_IPX;
 		    ro->ro_dst.sa_len = sizeof(ro->ro_dst);
 		    *dst = sipx->sipx_addr;
 		    dst->x_port = 0;
 		    rtalloc(ro);
-	}
+	} else
+		SOCK_UNLOCK(ipxp->ipxp_socket);
 	if (ipx_neteqnn(ipxp->ipxp_laddr.x_net, ipx_zeronet)) {
 		/* 
 		 * If route is known or can be allocated now,
@@ -257,8 +264,12 @@ ipx_pcbdisconnect(ipxp)
 {
 
 	ipxp->ipxp_faddr = zeroipx_addr;
-	if (ipxp->ipxp_socket->so_state & SS_NOFDREF)
+	SOCK_LOCK(ipxp->ipxp_socket);
+	if (ipxp->ipxp_socket->so_state & SS_NOFDREF) {
+		SOCK_UNLOCK(ipxp->ipxp_socket);
 		ipx_pcbdetach(ipxp);
+	} else
+		SOCK_UNLOCK(ipxp->ipxp_socket);
 }
 
 void
@@ -268,6 +279,7 @@ ipx_pcbdetach(ipxp)
 	struct socket *so = ipxp->ipxp_socket;
 
 	so->so_pcb = 0;
+	SOCK_LOCK(so);
 	sotryfree(so);
 	if (ipxp->ipxp_route.ro_rt != NULL)
 		rtfree(ipxp->ipxp_route.ro_rt);
diff --git a/sys/netipx/ipx_usrreq.c b/sys/netipx/ipx_usrreq.c
index f0c803a..68d9f10 100644
--- a/sys/netipx/ipx_usrreq.c
+++ b/sys/netipx/ipx_usrreq.c
@@ -146,7 +146,9 @@ ipx_input(m, ipxp)
 	if (sbappendaddr(&ipxp->ipxp_socket->so_rcv, (struct sockaddr *)&ipx_ipx,
 	    m, (struct mbuf *)NULL) == 0)
 		goto bad;
+	SOCK_LOCK(ipxp->ipxp_socket);
 	sorwakeup(ipxp->ipxp_socket);
+	SOCK_UNLOCK(ipxp->ipxp_socket);
 	return;
 bad:
 	m_freem(m);
@@ -159,7 +161,9 @@ ipx_abort(ipxp)
 	struct socket *so = ipxp->ipxp_socket;
 
 	ipx_pcbdisconnect(ipxp);
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 }
 
 /*
@@ -186,7 +190,9 @@ ipx_drop(ipxp, errno)
 	}*/
 	so->so_error = errno;
 	ipx_pcbdisconnect(ipxp);
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 }
 
 static int
@@ -200,6 +206,7 @@ ipx_output(ipxp, m0)
 	register struct route *ro;
 	struct mbuf *m;
 	struct mbuf *mprev = NULL;
+	int soopts;
 
 	/*
 	 * Calculate data length.
@@ -261,9 +268,14 @@ ipx_output(ipxp, m0)
 	 * Output datagram.
 	 */
 	so = ipxp->ipxp_socket;
-	if (so->so_options & SO_DONTROUTE)
+	SOCK_LOCK(so);
+	if (so->so_options & SO_DONTROUTE) {
+		soopts = so->so_options & SO_BROADCAST;
+		SOCK_UNLOCK(so);
 		return (ipx_outputfl(m, (struct route *)NULL,
-		    (so->so_options & SO_BROADCAST) | IPX_ROUTETOIF));
+		    soopts | IPX_ROUTETOIF));
+	}
+	SOCK_UNLOCK(so);
 	/*
 	 * Use cached route for previous datagram if
 	 * possible.  If the previous net was the same
@@ -306,7 +318,10 @@ ipx_output(ipxp, m0)
 	}
 	ipxp->ipxp_lastdst = ipx->ipx_dna;
 #endif /* ancient_history */
-	return (ipx_outputfl(m, ro, so->so_options & SO_BROADCAST));
+	SOCK_LOCK(so);
+	soopts = so->so_options & SO_BROADCAST;
+	SOCK_UNLOCK(so);
+	return (ipx_outputfl(m, ro, soopts));
 }
 
 int
@@ -429,8 +444,11 @@ ipx_usr_abort(so)
 	s = splnet();
 	ipx_pcbdetach(ipxp);
 	splx(s);
+	SOCK_LOCK(so);
 	sotryfree(so);
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 	return (0);
 }
 
@@ -480,8 +498,11 @@ ipx_connect(so, nam, td)
 	s = splnet();
 	error = ipx_pcbconnect(ipxp, nam, td);
 	splx(s);
-	if (error == 0)
+	if (error == 0) {
+		SOCK_LOCK(so);
 		soisconnected(so);
+		SOCK_UNLOCK(so);
+	}
 	return (error);
 }
 
@@ -512,7 +533,9 @@ ipx_disconnect(so)
 	s = splnet();
 	ipx_pcbdisconnect(ipxp);
 	splx(s);
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 	return (0);
 }
 
diff --git a/sys/netipx/spx_usrreq.c b/sys/netipx/spx_usrreq.c
index 189d741..4bdfa55 100644
--- a/sys/netipx/spx_usrreq.c
+++ b/sys/netipx/spx_usrreq.c
@@ -163,6 +163,7 @@ spx_input(m, ipxp)
 
 	so = ipxp->ipxp_socket;
 
+	SOCK_LOCK(so);
 	if (so->so_options & SO_DEBUG || traceallspxs) {
 		ostate = cb->s_state;
 		spx_savesi = *si;
@@ -170,6 +171,7 @@ spx_input(m, ipxp)
 	if (so->so_options & SO_ACCEPTCONN) {
 		struct spxpcb *ocb = cb;
 
+		SOCK_UNLOCK(so);
 		so = sonewconn(so, 0);
 		if (so == NULL) {
 			goto drop;
@@ -193,7 +195,8 @@ spx_input(m, ipxp)
 		cb->s_flags = ocb->s_flags;	/* preserve sockopts */
 		cb->s_flags2 = ocb->s_flags2;	/* preserve sockopts */
 		cb->s_state = TCPS_LISTEN;
-	}
+	} else
+		SOCK_UNLOCK(so);
 
 	/*
 	 * Packet received on connection.
@@ -258,7 +261,9 @@ spx_input(m, ipxp)
 		ipxp->ipxp_fport =  si->si_sport;
 		cb->s_timer[SPXT_REXMT] = 0;
 		cb->s_timer[SPXT_KEEP] = SPXTV_KEEP;
+		SOCK_LOCK(so);
 		soisconnected(so);
+		SOCK_UNLOCK(so);
 		cb->s_state = TCPS_ESTABLISHED;
 		spxstat.spxs_accepts++;
 		}
@@ -285,7 +290,9 @@ spx_input(m, ipxp)
 		cb->s_dport = ipxp->ipxp_fport =  si->si_sport;
 		cb->s_timer[SPXT_REXMT] = 0;
 		cb->s_flags |= SF_ACKNOW;
+		SOCK_LOCK(so);
 		soisconnected(so);
+		SOCK_UNLOCK(so);
 		cb->s_state = TCPS_ESTABLISHED;
 		/* Use roundtrip time of connection request for initial rtt */
 		if (cb->s_rtt) {
@@ -297,8 +304,12 @@ spx_input(m, ipxp)
 			    cb->s_rtt = 0;
 		}
 	}
-	if (so->so_options & SO_DEBUG || traceallspxs)
+	SOCK_LOCK(so);
+	if (so->so_options & SO_DEBUG || traceallspxs) {
+		SOCK_UNLOCK(so);
 		spx_trace(SA_INPUT, (u_char)ostate, cb, &spx_savesi, 0);
+	} else
+		SOCK_UNLOCK(so);
 
 	m->m_len -= sizeof(struct ipx);
 	m->m_pkthdr.len -= sizeof(struct ipx);
@@ -319,15 +330,23 @@ dropwithreset:
 	si->si_ack = ntohs(si->si_ack);
 	si->si_alo = ntohs(si->si_alo);
 	m_freem(dtom(si));
-	if (cb->s_ipxpcb->ipxp_socket->so_options & SO_DEBUG || traceallspxs)
+	SOCK_LOCK(cb->s_ipxpcb->ipxp_socket);
+	if (cb->s_ipxpcb->ipxp_socket->so_options & SO_DEBUG || traceallspxs) {
+		SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 		spx_trace(SA_DROP, (u_char)ostate, cb, &spx_savesi, 0);
+	} else
+		SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 	return;
 
 drop:
 bad:
+	SOCK_LOCK(cb->s_ipxpcb->ipxp_socket);
 	if (cb == 0 || cb->s_ipxpcb->ipxp_socket->so_options & SO_DEBUG ||
-            traceallspxs)
+            traceallspxs) {
+		SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 		spx_trace(SA_DROP, (u_char)ostate, cb, &spx_savesi, 0);
+	} else
+		SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 	m_freem(m);
 }
 
@@ -462,7 +481,9 @@ register struct spx *si;
 		else
 			break;
 	}
+	SOCK_LOCK(so);
 	sowwakeup(so);
+	SOCK_UNLOCK(so);
 	cb->s_rack = si->si_ack;
 update_window:
 	if (SSEQ_LT(cb->s_snxt, cb->s_rack))
@@ -501,10 +522,13 @@ update_window:
 			} /* else queue this packet; */
 		} else {
 			/*register struct socket *so = cb->s_ipxpcb->ipxp_socket;
+			SOCK_LOCK(so);
 			if (so->so_state && SS_NOFDREF) {
+				SOCK_UNLOCK(so);
 				spx_close(cb);
-			} else
-				       would crash system*/
+			} else {
+				SOCK_UNLOCK(so);
+				       would crash system } */
 			spx_istat.notyet++;
 			m_freem(dtom(si));
 			return (0);
@@ -565,8 +589,11 @@ present:
 				cb->s_oobflags &= ~SF_IOOB;
 				if (so->so_rcv.sb_cc)
 					so->so_oobmark = so->so_rcv.sb_cc;
-				else
+				else {
+					SOCK_LOCK(so);
 					so->so_state |= SS_RCVATMARK;
+					SOCK_UNLOCK(so);
+				}
 			}
 			q = q->si_prev;
 			remque(q->si_next);
@@ -596,7 +623,9 @@ present:
 					MCHTYPE(m, MT_OOBDATA);
 					spx_newchecks[1]++;
 					so->so_oobmark = 0;
+					SOCK_LOCK(so);
 					so->so_state &= ~SS_RCVATMARK;
+					SOCK_UNLOCK(so);
 				}
 				if (packetp == 0) {
 					m->m_data += SPINC;
@@ -622,8 +651,11 @@ present:
 		  } else
 			break;
 	}
-	if (wakeup)
+	if (wakeup) {
+		SOCK_LOCK(so);
 		sorwakeup(so);
+		SOCK_UNLOCK(so);
+	}
 	return (0);
 }
 
@@ -1027,8 +1059,12 @@ send:
 		si->si_cc |= SPX_SP;
 	} else {
 		cb->s_outx = 3;
-		if (so->so_options & SO_DEBUG || traceallspxs)
+		SOCK_LOCK(so);
+		if (so->so_options & SO_DEBUG || traceallspxs) {
+			SOCK_UNLOCK(so);
 			spx_trace(SA_OUTPUT, cb->s_state, cb, si, 0);
+		} else
+			SOCK_UNLOCK(so);
 		return (0);
 	}
 	/*
@@ -1090,13 +1126,20 @@ send:
 			si->si_sum = 0xffff;
 
 		cb->s_outx = 4;
-		if (so->so_options & SO_DEBUG || traceallspxs)
+		SOCK_LOCK(so);
+		if (so->so_options & SO_DEBUG || traceallspxs) {
+			SOCK_UNLOCK(so);
 			spx_trace(SA_OUTPUT, cb->s_state, cb, si, 0);
+			SOCK_LOCK(so);
+		}
 
-		if (so->so_options & SO_DONTROUTE)
+		if (so->so_options & SO_DONTROUTE) {
+			SOCK_UNLOCK(so);
 			error = ipx_outputfl(m, (struct route *)NULL, IPX_ROUTETOIF);
-		else
+		} else {
+			SOCK_UNLOCK(so);
 			error = ipx_outputfl(m, &cb->s_ipxpcb->ipxp_route, 0);
+		}
 	}
 	if (error) {
 		return (error);
@@ -1413,7 +1456,9 @@ spx_connect(so, nam, td)
 	error = ipx_pcbconnect(ipxp, nam, td);
 	if (error)
 		goto spx_connect_end;
+	SOCK_LOCK(so);
 	soisconnecting(so);
+	SOCK_UNLOCK(so);
 	spxstat.spxs_connattempt++;
 	cb->s_state = TCPS_SYN_SENT;
 	cb->s_did = 0;
@@ -1535,12 +1580,15 @@ spx_rcvoob(so, m, flags)
 	ipxp = sotoipxpcb(so);
 	cb = ipxtospxpcb(ipxp);
 
+	SOCK_LOCK(so);
 	if ((cb->s_oobflags & SF_IOOB) || so->so_oobmark ||
 	    (so->so_state & SS_RCVATMARK)) {
+		SOCK_UNLOCK(so);
 		m->m_len = 1;
 		*mtod(m, caddr_t) = cb->s_iobc;
 		return (0);
 	}
+	SOCK_UNLOCK(so);
 	return (EINVAL);
 }
 
@@ -1684,7 +1732,9 @@ spx_close(cb)
 	m_free(dtom(cb->s_ipx));
 	FREE(cb, M_PCB);
 	ipxp->ipxp_pcb = 0;
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 	ipx_pcbdetach(ipxp);
 	spxstat.spxs_closed++;
 	return ((struct spxpcb *)NULL);
@@ -1887,13 +1937,17 @@ spx_timers(cb, timer)
 		spxstat.spxs_keeptimeo++;
 		if (cb->s_state < TCPS_ESTABLISHED)
 			goto dropit;
+		SOCK_LOCK(cb->s_ipxpcb->ipxp_socket);
 		if (cb->s_ipxpcb->ipxp_socket->so_options & SO_KEEPALIVE) {
+			SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 		    	if (cb->s_idle >= SPXTV_MAXIDLE)
 				goto dropit;
 			spxstat.spxs_keepprobe++;
 			spx_output(cb, (struct mbuf *)NULL);
-		} else
+		} else {
+			SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 			cb->s_idle = 0;
+		}
 		cb->s_timer[SPXT_KEEP] = SPXTV_KEEP;
 		break;
 	dropit: