Lock down a socket, milestone 1.

o Add a mutex (sb_mtx) to struct sockbuf. This protects the data in a
  socket buffer. The mutex in the receive buffer also protects the data
  in struct socket.

o Determine the lock strategy for each members in struct socket.

o Lock down the following members:

  - so_count
  - so_options
  - so_linger
  - so_state

o Remove *_locked() socket APIs.  Make the following socket APIs
  touching the members above now require a locked socket:

 - sodisconnect()
 - soisconnected()
 - soisconnecting()
 - soisdisconnected()
 - soisdisconnecting()
 - sofree()
 - soref()
 - sorele()
 - sorwakeup()
 - sotryfree()
 - sowakeup()
 - sowwakeup()

Reviewed by:	alfred

4 files changed, 129 insertions, 24 deletions

diff --git a/sys/netns/idp_usrreq.c b/sys/netns/idp_usrreq.c
index 5203961..6ca054d 100644
--- a/sys/netns/idp_usrreq.c
+++ b/sys/netns/idp_usrreq.c
@@ -97,7 +97,9 @@ idp_input(m, nsp)
 	if (sbappendaddr(&nsp->nsp_socket->so_rcv, (struct sockaddr *)&idp_ns,
 	    m, (struct mbuf *)0) == 0)
 		goto bad;
+	SOCK_LOCK(nsp->nsp_socket);
 	sorwakeup(nsp->nsp_socket);
+	SOCK_UNLOCK(nsp->nsp_socket);
 	return;
 bad:
 	m_freem(m);
@@ -109,7 +111,9 @@ idp_abort(nsp)
 	struct socket *so = nsp->nsp_socket;
 
 	ns_pcbdisconnect(nsp);
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 }
 /*
  * Drop connection, reporting
@@ -133,7 +137,9 @@ idp_drop(nsp, errno)
 	}*/
 	so->so_error = errno;
 	ns_pcbdisconnect(nsp);
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 }
 
 int noIdpRoute;
@@ -148,6 +154,7 @@ idp_output(nsp, m0)
 	register struct route *ro;
 	struct mbuf *mprev;
 	extern int idpcksum;
+	int soopts;
 
 	/*
 	 * Calculate data length.
@@ -211,9 +218,14 @@ idp_output(nsp, m0)
 	 * Output datagram.
 	 */
 	so = nsp->nsp_socket;
-	if (so->so_options & SO_DONTROUTE)
+	SOCK_LOCK(so);
+	if (so->so_options & SO_DONTROUTE) {
+		soopts = so->so_options & SO_BROADCAST;
+		SO_UNLOCK(so);
 		return (ns_output(m, (struct route *)0,
-		    (so->so_options & SO_BROADCAST) | NS_ROUTETOIF));
+		    soopts | NS_ROUTETOIF));
+	}
+	SO_UNLOCK(so);
 	/*
 	 * Use cached route for previous datagram if
 	 * possible.  If the previous net was the same
@@ -257,7 +269,10 @@ idp_output(nsp, m0)
 	nsp->nsp_lastdst = idp->idp_dna;
 #endif /* ancient_history */
 	if (noIdpRoute) ro = 0;
-	return (ns_output(m, ro, so->so_options & SO_BROADCAST));
+	SOCK_LOCK(so);
+	soopts = so->so_options & SO_BROADCAST;
+	SOCK_UNLOCK(so);
+	return (ns_output(m, ro, soopts));
 }
 /* ARGSUSED */
 idp_ctloutput(req, so, level, name, value)
@@ -427,8 +442,11 @@ idp_usrreq(so, req, m, nam, control)
 			break;
 		}
 		error = ns_pcbconnect(nsp, nam);
-		if (error == 0)
+		if (error == 0) {
+			SOCK_LOCK(so);
 			soisconnected(so);
+			SOCK_UNLOCK(so);
+		}
 		break;
 
 	case PRU_CONNECT2:
@@ -445,7 +463,9 @@ idp_usrreq(so, req, m, nam, control)
 			break;
 		}
 		ns_pcbdisconnect(nsp);
+		SOCK_LOCK(so);
 		soisdisconnected(so);
+		SOCK_UNLOCK(so);
 		break;
 
 	case PRU_SHUTDOWN:
@@ -491,8 +511,11 @@ idp_usrreq(so, req, m, nam, control)
 
 	case PRU_ABORT:
 		ns_pcbdetach(nsp);
+		SOCK_LOCK(so);
 		sotryfree(so);
+		SOCK_LOCK(so);
 		soisdisconnected(so);	/* XXX huh, called after sofree()? */
+		SOCK_UNLOCK(so);
 		break;
 
 	case PRU_SOCKADDR:
@@ -546,10 +569,13 @@ idp_raw_usrreq(so, req, m, nam, control)
 
 	case PRU_ATTACH:
 
+		SOCK_LOCK(so);
 		if (!(so->so_state & SS_PRIV) || (nsp != NULL)) {
+			SOCK_UNLOCK(so);
 			error = EINVAL;
 			break;
 		}
+		SOCK_UNLOCK(so);
 		error = ns_pcballoc(so, &nsrawpcb);
 		if (error)
 			break;
diff --git a/sys/netns/ns.c b/sys/netns/ns.c
index 7af5acf..0c1d272 100644
--- a/sys/netns/ns.c
+++ b/sys/netns/ns.c
@@ -107,8 +107,12 @@ ns_control(so, cmd, data, ifp)
 		return (0);
 	}
 
-	if ((so->so_state & SS_PRIV) == 0)
+	SOCK_LOCK(so);
+	if ((so->so_state & SS_PRIV) == 0) {
+		SOCK_UNLOCK(so);
 		return (EPERM);
+	}
+	SOCK_UNLOCK(so);
 
 	switch (cmd) {
 	case SIOCAIFADDR:
diff --git a/sys/netns/ns_pcb.c b/sys/netns/ns_pcb.c
index 01301c0..9e06710 100644
--- a/sys/netns/ns_pcb.c
+++ b/sys/netns/ns_pcb.c
@@ -94,9 +94,13 @@ ns_pcbbind(nsp, nam)
 	if (lport) {
 		u_short aport = ntohs(lport);
 
+		SOCK_LOCK(so);
 		if (aport < NSPORT_RESERVED &&
-		    (nsp->nsp_socket->so_state & SS_PRIV) == 0)
+		    (nsp->nsp_socket->so_state & SS_PRIV) == 0) {
+			SOCK_UNLOCK(so);
 			return (EACCES);
+		}
+		SOCK_UNLOCK(so);
 		if (ns_pcblookup(&zerons_addr, lport, 0))
 			return (EADDRINUSE);
 	}
@@ -148,8 +152,12 @@ ns_pcbconnect(nsp, nam)
 	 */
 	ro = &nsp->nsp_route;
 	dst = &satons_addr(ro->ro_dst);
-	if (nsp->nsp_socket->so_options & SO_DONTROUTE)
+	SOCK_LOCK(nsp->nsp_socket);
+	if (nsp->nsp_socket->so_options & SO_DONTROUTE) {
+		SOCK_UNLOCK(nsp->nsp_socket);
 		goto flush;
+	}
+	SOCK_UNLOCK(nsp->nsp_socket);
 	if (!ns_neteq(nsp->nsp_lastdst, sns->sns_addr))
 		goto flush;
 	if (!ns_hosteq(nsp->nsp_lastdst, sns->sns_addr)) {
@@ -165,16 +173,19 @@ ns_pcbconnect(nsp, nam)
 		}
 	}/* else cached route is ok; do nothing */
 	nsp->nsp_lastdst = sns->sns_addr;
+	SOCK_LOCK(nsp->nsp_socket);
 	if ((nsp->nsp_socket->so_options & SO_DONTROUTE) == 0 && /*XXX*/
 	    (ro->ro_rt == (struct rtentry *)0 ||
 	     ro->ro_rt->rt_ifp == (struct ifnet *)0)) {
+		    SOCK_UNLOCK(nsp->nsp_socket);
 		    /* No route yet, so try to acquire one */
 		    ro->ro_dst.sa_family = AF_NS;
 		    ro->ro_dst.sa_len = sizeof(ro->ro_dst);
 		    *dst = sns->sns_addr;
 		    dst->x_port = 0;
 		    rtalloc(ro);
-	}
+	} else
+		SOCK_UNLOCK(nsp->nsp_socket);
 	if (ns_neteqnn(nsp->nsp_laddr.x_net, ns_zeronet)) {
 		/*
 		 * If route is known or can be allocated now,
@@ -222,8 +233,12 @@ ns_pcbdisconnect(nsp)
 {
 
 	nsp->nsp_faddr = zerons_addr;
-	if (nsp->nsp_socket->so_state & SS_NOFDREF)
+	SOCK_LOCK(so);
+	if (nsp->nsp_socket->so_state & SS_NOFDREF) {
+		SOCK_UNLOCK(so);
 		ns_pcbdetach(nsp);
+	} else
+		SOCK_UNLOCK(so);
 }
 
 ns_pcbdetach(nsp)
@@ -232,6 +247,7 @@ ns_pcbdetach(nsp)
 	struct socket *so = nsp->nsp_socket;
 
 	so->so_pcb = 0;
+	SOCK_LOCK(so);
 	sotryfree(so);
 	if (nsp->nsp_route.ro_rt)
 		rtfree(nsp->nsp_route.ro_rt);
diff --git a/sys/netns/spp_usrreq.c b/sys/netns/spp_usrreq.c
index 8a14ff0..78bfec7 100644
--- a/sys/netns/spp_usrreq.c
+++ b/sys/netns/spp_usrreq.c
@@ -106,6 +106,7 @@ spp_input(m, nsp)
 	si->si_alo = ntohs(si->si_alo);
 
 	so = nsp->nsp_socket;
+	SOCK_LOCK(so);
 	if (so->so_options & SO_DEBUG || traceallspps) {
 		ostate = cb->s_state;
 		spp_savesi = *si;
@@ -113,6 +114,7 @@ spp_input(m, nsp)
 	if (so->so_options & SO_ACCEPTCONN) {
 		struct sppcb *ocb = cb;
 
+		SOCK_UNLOCK(so);
 		so = sonewconn(so, 0);
 		if (so == 0) {
 			goto drop;
@@ -136,7 +138,8 @@ spp_input(m, nsp)
 		cb->s_flags = ocb->s_flags;	/* preserve sockopts */
 		cb->s_flags2 = ocb->s_flags2;	/* preserve sockopts */
 		cb->s_state = TCPS_LISTEN;
-	}
+	} else
+		SOCK_UNLOCK(so);
 
 	/*
 	 * Packet received on connection.
@@ -207,7 +210,9 @@ spp_input(m, nsp)
 		nsp->nsp_fport =  si->si_sport;
 		cb->s_timer[SPPT_REXMT] = 0;
 		cb->s_timer[SPPT_KEEP] = SPPTV_KEEP;
+		SOCK_LOCK(so);
 		soisconnected(so);
+		SOCK_UNLOCK(so);
 		cb->s_state = TCPS_ESTABLISHED;
 		sppstat.spps_accepts++;
 		}
@@ -234,7 +239,9 @@ spp_input(m, nsp)
 		cb->s_dport = nsp->nsp_fport =  si->si_sport;
 		cb->s_timer[SPPT_REXMT] = 0;
 		cb->s_flags |= SF_ACKNOW;
+		SOCK_LOCK(so);
 		soisconnected(so);
+		SOCK_UNLOCK(so);
 		cb->s_state = TCPS_ESTABLISHED;
 		/* Use roundtrip time of connection request for initial rtt */
 		if (cb->s_rtt) {
@@ -246,8 +253,12 @@ spp_input(m, nsp)
 			    cb->s_rtt = 0;
 		}
 	}
-	if (so->so_options & SO_DEBUG || traceallspps)
+	SOCK_LOCK(so);
+	if (so->so_options & SO_DEBUG || traceallspps) {
+		SOCK_UNLOCK(so);
 		spp_trace(SA_INPUT, (u_char)ostate, cb, &spp_savesi, 0);
+	} else
+		SOCK_UNLOCK(so);
 
 	m->m_len -= sizeof (struct idp);
 	m->m_pkthdr.len -= sizeof (struct idp);
@@ -268,15 +279,23 @@ dropwithreset:
 	si->si_ack = ntohs(si->si_ack);
 	si->si_alo = ntohs(si->si_alo);
 	ns_error(dtom(si), NS_ERR_NOSOCK, 0);
-	if (cb->s_nspcb->nsp_socket->so_options & SO_DEBUG || traceallspps)
+	SOCK_LOCK(cb->s_nspcb->nsp_socket);
+	if (cb->s_nspcb->nsp_socket->so_options & SO_DEBUG || traceallspps) {
+		SOCK_UNLOCK(cb->s_nspcb->nsp_socket);
 		spp_trace(SA_DROP, (u_char)ostate, cb, &spp_savesi, 0);
+	} else
+		SOCK_UNLOCK(cb->s_nspcb->nsp_socket);
 	return;
 
 drop:
 bad:
+	SOCK_LOCK(cb->s_nspcb->nsp_socket);
 	if (cb == 0 || cb->s_nspcb->nsp_socket->so_options & SO_DEBUG ||
-            traceallspps)
+            traceallspps) {
+		SOCK_UNLOCK(cb->s_nspcb->nsp_socket);
 		spp_trace(SA_DROP, (u_char)ostate, cb, &spp_savesi, 0);
+	} else
+		SOCK_UNLOCK(cb->s_nspcb->nsp_socket);
 	m_freem(m);
 }
 
@@ -410,7 +429,9 @@ register struct spidp *si;
 		else
 			break;
 	}
+	SOCK_LOCK(so);
 	sowwakeup(so);
+	SOCK_UNLOCK(so);
 	cb->s_rack = si->si_ack;
 update_window:
 	if (SSEQ_LT(cb->s_snxt, cb->s_rack))
@@ -449,11 +470,14 @@ update_window:
 			} /* else queue this packet; */
 		} else {
 			/*register struct socket *so = cb->s_nspcb->nsp_socket;
+			SOCK_LOCK(so);
 			if (so->so_state && SS_NOFDREF) {
+				SOCK_UNLOCK(so);
 				ns_error(dtom(si), NS_ERR_NOSOCK, 0);
 				(void)spp_close(cb);
-			} else
-				       would crash system*/
+			} else {
+				SOCK_UNLOCK(so);
+				       would crash system }*/
 			spp_istat.notyet++;
 			ns_error(dtom(si), NS_ERR_FULLUP, 0);
 			return (0);
@@ -514,8 +538,11 @@ present:
 				cb->s_oobflags &= ~SF_IOOB;
 				if (so->so_rcv.sb_cc)
 					so->so_oobmark = so->so_rcv.sb_cc;
-				else
+				else {
+					SOCK_LOCK(so);
 					so->so_state |= SS_RCVATMARK;
+					SOCK_UNLOCK(so);
+				}
 			}
 			q = q->si_prev;
 			remque(q->si_next);
@@ -545,7 +572,9 @@ present:
 					MCHTYPE(m, MT_OOBDATA);
 					spp_newchecks[1]++;
 					so->so_oobmark = 0;
+					SOCK_LOCK(so);
 					so->so_state &= ~SS_RCVATMARK;
+					SOCK_UNLOCK(so);
 				}
 				if (packetp == 0) {
 					m->m_data += SPINC;
@@ -571,7 +600,11 @@ present:
 		  } else
 			break;
 	}
-	if (wakeup) sorwakeup(so);
+	if (wakeup) {
+		SOCK_LOCK(so);
+		sorwakeup(so);
+		SOCK_UNLOCK(so);
+	}
 	return (0);
 }
 
@@ -1015,8 +1048,12 @@ send:
 		si->si_cc |= SP_SP;
 	} else {
 		cb->s_outx = 3;
-		if (so->so_options & SO_DEBUG || traceallspps)
+		SOCK_LOCK(so);
+		if (so->so_options & SO_DEBUG || traceallspps) {
+			SOCK_UNLOCK(so);
 			spp_trace(SA_OUTPUT, cb->s_state, cb, si, 0);
+		} else
+			SOCK_UNLOCK(so);
 		return (0);
 	}
 	/*
@@ -1082,13 +1119,20 @@ send:
 			si->si_sum = 0xffff;
 
 		cb->s_outx = 4;
-		if (so->so_options & SO_DEBUG || traceallspps)
+		SOCK_LOCK(so);
+		if (so->so_options & SO_DEBUG || traceallspps) {
+			SOCK_UNLOCK(so);
 			spp_trace(SA_OUTPUT, cb->s_state, cb, si, 0);
+			SOCK_LOCK(so);
+		}
 
-		if (so->so_options & SO_DONTROUTE)
+		if (so->so_options & SO_DONTROUTE) {
+			SOCK_UNLOCK(so);
 			error = ns_output(m, (struct route *)0, NS_ROUTETOIF);
-		else
+		} else {
+			SOCK_UNLOCK(so);
 			error = ns_output(m, &cb->s_nspcb->nsp_route, 0);
+		}
 	}
 	if (error) {
 		return (error);
@@ -1369,7 +1413,9 @@ spp_usrreq(so, req, m, nam, controlp)
 		error = ns_pcbconnect(nsp, nam);
 		if (error)
 			break;
+		SOCK_LOCK(so);
 		soisconnecting(so);
+		SOCK_UNLOCK(so);
 		sppstat.spps_connattempt++;
 		cb->s_state = TCPS_SYN_SENT;
 		cb->s_did = 0;
@@ -1443,12 +1489,15 @@ spp_usrreq(so, req, m, nam, controlp)
 		break;
 
 	case PRU_RCVOOB:
+		SOCK_LOCK(so);
 		if ((cb->s_oobflags & SF_IOOB) || so->so_oobmark ||
 		    (so->so_state & SS_RCVATMARK)) {
+			SOCK_UNLOCK(so);
 			m->m_len = 1;
 			*mtod(m, caddr_t) = cb->s_iobc;
 			break;
-		}
+		} else
+			SOCK_UNLOCK(so);
 		error = EINVAL;
 		break;
 
@@ -1496,8 +1545,12 @@ spp_usrreq(so, req, m, nam, controlp)
 	default:
 		panic("sp_usrreq");
 	}
-	if (cb && (so->so_options & SO_DEBUG || traceallspps))
+	SOCK_LOCK(so);
+	if (cb && (so->so_options & SO_DEBUG || traceallspps)) {
+		SOCK_UNLOCK(so);
 		spp_trace(SA_USER, (u_char)ostate, cb, (struct spidp *)0, req);
+	} else
+		SOCK_UNLOCK(so);
 release:
 	if (controlp != NULL)
 		m_freem(controlp);
@@ -1574,7 +1627,9 @@ spp_close(cb)
 	(void) m_free(dtom(cb->s_idp));
 	(void) m_free(dtom(cb));
 	nsp->nsp_pcb = 0;
+	SOCK_LOCK(so);
 	soisdisconnected(so);
+	SOCK_UNLOCK(so);
 	ns_pcbdetach(nsp);
 	sppstat.spps_closed++;
 	return ((struct sppcb *)0);
@@ -1783,13 +1838,17 @@ spp_timers(cb, timer)
 		sppstat.spps_keeptimeo++;
 		if (cb->s_state < TCPS_ESTABLISHED)
 			goto dropit;
+		SOCK_LOCK(cb->s_nspcb->nsp_socket);
 		if (cb->s_nspcb->nsp_socket->so_options & SO_KEEPALIVE) {
+			SOCK_UNLOCK(cb->s_nspcb->nsp_socket);
 		    	if (cb->s_idle >= SPPTV_MAXIDLE)
 				goto dropit;
 			sppstat.spps_keepprobe++;
 			(void) spp_output(cb, (struct mbuf *) 0);
-		} else
+		} else {
+			SOCK_UNLOCK(cb->s_nspcb->nsp_socket);
 			cb->s_idle = 0;
+		}
 		cb->s_timer[SPPT_KEEP] = SPPTV_KEEP;
 		break;
 	dropit: