Lock down a socket, milestone 1.

o Add a mutex (sb_mtx) to struct sockbuf. This protects the data in a
  socket buffer. The mutex in the receive buffer also protects the data
  in struct socket.

o Determine the lock strategy for each members in struct socket.

o Lock down the following members:

  - so_count
  - so_options
  - so_linger
  - so_state

o Remove *_locked() socket APIs.  Make the following socket APIs
  touching the members above now require a locked socket:

 - sodisconnect()
 - soisconnected()
 - soisconnecting()
 - soisdisconnected()
 - soisdisconnecting()
 - sofree()
 - soref()
 - sorele()
 - sorwakeup()
 - sotryfree()
 - sowakeup()
 - sowwakeup()

Reviewed by:	alfred

1 files changed, 67 insertions, 13 deletions

diff --git a/sys/netipx/spx_usrreq.c b/sys/netipx/spx_usrreq.c
index 189d741..4bdfa55 100644
--- a/sys/netipx/spx_usrreq.c
+++ b/sys/netipx/spx_usrreq.c
@@ -163,6 +163,7 @@ spx_input(m, ipxp)
 
 	so = ipxp->ipxp_socket;
 
+	SOCK_LOCK(so);
 	if (so->so_options & SO_DEBUG || traceallspxs) {
 		ostate = cb->s_state;
 		spx_savesi = *si;
@@ -170,6 +171,7 @@ spx_input(m, ipxp)
 	if (so->so_options & SO_ACCEPTCONN) {
 		struct spxpcb *ocb = cb;
 
+		SOCK_UNLOCK(so);
 		so = sonewconn(so, 0);
 		if (so == NULL) {
 			goto drop;
@@ -193,7 +195,8 @@ spx_input(m, ipxp)
 		cb->s_flags = ocb->s_flags;	/* preserve sockopts */
 		cb->s_flags2 = ocb->s_flags2;	/* preserve sockopts */
 		cb->s_state = TCPS_LISTEN;
-	}
+	} else
+		SOCK_UNLOCK(so);
 
 	/*
 	 * Packet received on connection.
@@ -258,7 +261,9 @@ spx_input(m, ipxp)
 		ipxp->ipxp_fport =  si->si_sport;
 		cb->s_timer[SPXT_REXMT] = 0;
 		cb->s_timer[SPXT_KEEP] = SPXTV_KEEP;
+		SOCK_LOCK(so);
 		soisconnected(so);
+		SOCK_UNLOCK(so);
 		cb->s_state = TCPS_ESTABLISHED;
 		spxstat.spxs_accepts++;
 		}
@@ -285,7 +290,9 @@ spx_input(m, ipxp)
 		cb->s_dport = ipxp->ipxp_fport =  si->si_sport;
 		cb->s_timer[SPXT_REXMT] = 0;
 		cb->s_flags |= SF_ACKNOW;
+		SOCK_LOCK(so);
 		soisconnected(so);
+		SOCK_UNLOCK(so);
 		cb->s_state = TCPS_ESTABLISHED;
 		/* Use roundtrip time of connection request for initial rtt */
 		if (cb->s_rtt) {
@@ -297,8 +304,12 @@ spx_input(m, ipxp)
 			    cb->s_rtt = 0;
 		}
 	}
-	if (so->so_options & SO_DEBUG || traceallspxs)
+	SOCK_LOCK(so);
+	if (so->so_options & SO_DEBUG || traceallspxs) {
+		SOCK_UNLOCK(so);
 		spx_trace(SA_INPUT, (u_char)ostate, cb, &spx_savesi, 0);
+	} else
+		SOCK_UNLOCK(so);
 
 	m->m_len -= sizeof(struct ipx);
 	m->m_pkthdr.len -= sizeof(struct ipx);
@@ -319,15 +330,23 @@ dropwithreset:
 	si->si_ack = ntohs(si->si_ack);
 	si->si_alo = ntohs(si->si_alo);
 	m_freem(dtom(si));
-	if (cb->s_ipxpcb->ipxp_socket->so_options & SO_DEBUG || traceallspxs)
+	SOCK_LOCK(cb->s_ipxpcb->ipxp_socket);
+	if (cb->s_ipxpcb->ipxp_socket->so_options & SO_DEBUG || traceallspxs) {
+		SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 		spx_trace(SA_DROP, (u_char)ostate, cb, &spx_savesi, 0);
+	} else
+		SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 	return;
 
 drop:
 bad:
+	SOCK_LOCK(cb->s_ipxpcb->ipxp_socket);
 	if (cb == 0 || cb->s_ipxpcb->ipxp_socket->so_options & SO_DEBUG ||
-            traceallspxs)
+            traceallspxs) {
+		SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 		spx_trace(SA_DROP, (u_char)ostate, cb, &spx_savesi, 0);
+	} else
+		SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 	m_freem(m);
 }
 
@@ -462,7 +481,9 @@ register struct spx *si;
 		else
 			break;
 	}
+	SOCK_LOCK(so);
 	sowwakeup(so);
+	SOCK_UNLOCK(so);
 	cb->s_rack = si->si_ack;
 update_window:
 	if (SSEQ_LT(cb->s_snxt, cb->s_rack))
@@ -501,10 +522,13 @@ update_window:
 			} /* else queue this packet; */
 		} else {
 			/*register struct socket *so = cb->s_ipxpcb->ipxp_socket;
+			SOCK_LOCK(so);
 			if (so->so_state && SS_NOFDREF) {
+				SOCK_UNLOCK(so);
 				spx_close(cb);
-			} else
-				       would crash system*/
+			} else {
+				SOCK_UNLOCK(so);
+				       would crash system } */
 			spx_istat.notyet++;
 			m_freem(dtom(si));
 			return (0);
@@ -565,8 +589,11 @@ present:
 				cb->s_oobflags &= ~SF_IOOB;
 				if (so->so_rcv.sb_cc)
 					so->so_oobmark = so->so_rcv.sb_cc;
-				else
+				else {
+					SOCK_LOCK(so);
 					so->so_state |= SS_RCVATMARK;
+					SOCK_UNLOCK(so);
+				}
 			}
 			q = q->si_prev;
 			remque(q->si_next);
@@ -596,7 +623,9 @@ present:
 					MCHTYPE(m, MT_OOBDATA);
 					spx_newchecks[1]++;
 					so->so_oobmark = 0;
+					SOCK_LOCK(so);
 					so->so_state &= ~SS_RCVATMARK;
+					SOCK_UNLOCK(so);
 				}
 				if (packetp == 0) {
 					m->m_data += SPINC;
@@ -622,8 +651,11 @@ present:
 		  } else
 			break;
 	}
-	if (wakeup)
+	if (wakeup) {
+		SOCK_LOCK(so);
 		sorwakeup(so);
+		SOCK_UNLOCK(so);
+	}
 	return (0);
 }
 
@@ -1027,8 +1059,12 @@ send:
 		si->si_cc |= SPX_SP;
 	} else {
 		cb->s_outx = 3;
-		if (so->so_options & SO_DEBUG || traceallspxs)
+		SOCK_LOCK(so);
+		if (so->so_options & SO_DEBUG || traceallspxs) {
+			SOCK_UNLOCK(so);
 			spx_trace(SA_OUTPUT, cb->s_state, cb, si, 0);
+		} else
+			SOCK_UNLOCK(so);
 		return (0);
 	}
 	/*
@@ -1090,13 +1126,20 @@ send:
 			si->si_sum = 0xffff;
 
 		cb->s_outx = 4;
-		if (so->so_options & SO_DEBUG || traceallspxs)
+		SOCK_LOCK(so);
+		if (so->so_options & SO_DEBUG || traceallspxs) {
+			SOCK_UNLOCK(so);
 			spx_trace(SA_OUTPUT, cb->s_state, cb, si, 0);
+			SOCK_LOCK(so);
+		}
 
-		if (so->so_options & SO_DONTROUTE)
+		if (so->so_options & SO_DONTROUTE) {
+			SOCK_UNLOCK(so);
 			error = ipx_outputfl(m, (struct route *)NULL, IPX_ROUTETOIF);
-		else
+		} else {
+			SOCK_UNLOCK(so);
 			error = ipx_outputfl(m, &cb->s_ipxpcb->ipxp_route, 0);
+		}
 	}
 	if (error) {
 		return (error);
@@ -1413,7 +1456,9 @@ spx_connect(so, nam, td)
 	error = ipx_pcbconnect(ipxp, nam, td);
 	if (error)
 		goto spx_connect_end;
+	SOCK_LOCK(so);
 	soisconnecting(so);
+	SOCK_UNLOCK(so);
 	spxstat.spxs_connattempt++;
 	cb->s_state = TCPS_SYN_SENT;
 	cb->s_did = 0;
@@ -1535,12 +1580,15 @@ spx_rcvoob(so, m, flags)
 	ipxp = sotoipxpcb(so);
 	cb = ipxtospxpcb(ipxp);
 
+	SOCK_LOCK(so);
 	if ((cb->s_oobflags & SF_IOOB) || so->so_oobmark ||
 	    (so->so_state & SS_RCVATMARK)) {
+		SOCK_UNLOCK(so);
 		m->m_len = 1;
 		*mtod(m, caddr_t) = cb->s_iobc;
 		return (0);
 	}
+	SOCK_UNLOCK(so);
 	return (EINVAL);
 }
 
@@ -1684,7 +1732,9 @@ spx_close(cb)
 	m_free(dtom(cb->s_ipx));
 	FREE(cb, M_PCB);
 	ipxp->ipxp_pcb = 0;
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 	ipx_pcbdetach(ipxp);
 	spxstat.spxs_closed++;
 	return ((struct spxpcb *)NULL);
@@ -1887,13 +1937,17 @@ spx_timers(cb, timer)
 		spxstat.spxs_keeptimeo++;
 		if (cb->s_state < TCPS_ESTABLISHED)
 			goto dropit;
+		SOCK_LOCK(cb->s_ipxpcb->ipxp_socket);
 		if (cb->s_ipxpcb->ipxp_socket->so_options & SO_KEEPALIVE) {
+			SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 		    	if (cb->s_idle >= SPXTV_MAXIDLE)
 				goto dropit;
 			spxstat.spxs_keepprobe++;
 			spx_output(cb, (struct mbuf *)NULL);
-		} else
+		} else {
+			SOCK_UNLOCK(cb->s_ipxpcb->ipxp_socket);
 			cb->s_idle = 0;
+		}
 		cb->s_timer[SPXT_KEEP] = SPXTV_KEEP;
 		break;
 	dropit: