The socket field so_state is used to hold a variety of socket related

flags relating to several aspects of socket functionality.  This change
breaks out several bits relating to send and receive operation into a
new per-socket buffer field, sb_state, in order to facilitate locking.
This is required because, in order to provide more granular locking of
sockets, different state fields have different locking properties.  The
following fields are moved to sb_state:

  SS_CANTRCVMORE            (so_state)
  SS_CANTSENDMORE           (so_state)
  SS_RCVATMARK              (so_state)

Rename respectively to:

  SBS_CANTRCVMORE           (so_rcv.sb_state)
  SBS_CANTSENDMORE          (so_snd.sb_state)
  SBS_RCVATMARK             (so_rcv.sb_state)

This facilitates locking by isolating fields to be located with other
identically locked fields, and permits greater granularity in socket
locking by avoiding storing fields with different locking semantics in
the same short (avoiding locking conflicts).  In the future, we may
wish to coallesce sb_state and sb_flags; for the time being I leave
them separate and there is no additional memory overhead due to the
packing/alignment of shorts in the socket buffer structure.

17 files changed, 85 insertions, 65 deletions

diff --git a/sys/fs/fifofs/fifo_vnops.c b/sys/fs/fifofs/fifo_vnops.c
index 8ecd33e..80fe7b6 100644
--- a/sys/fs/fifofs/fifo_vnops.c
+++ b/sys/fs/fifofs/fifo_vnops.c
@@ -211,7 +211,7 @@ fail1:
 		}
 		fip->fi_readers = fip->fi_writers = 0;
 		wso->so_snd.sb_lowat = PIPE_BUF;
-		rso->so_state |= SS_CANTRCVMORE;
+		rso->so_rcv.sb_state |= SBS_CANTRCVMORE;
 		vp->v_fifoinfo = fip;
 	}
 
@@ -229,7 +229,7 @@ fail1:
 	if (ap->a_mode & FREAD) {
 		fip->fi_readers++;
 		if (fip->fi_readers == 1) {
-			fip->fi_writesock->so_state &= ~SS_CANTSENDMORE;
+			fip->fi_writesock->so_snd.sb_state &= ~SBS_CANTSENDMORE;
 			if (fip->fi_writers > 0) {
 				wakeup(&fip->fi_writers);
 				sowwakeup(fip->fi_writesock);
@@ -243,7 +243,7 @@ fail1:
 		}
 		fip->fi_writers++;
 		if (fip->fi_writers == 1) {
-			fip->fi_readsock->so_state &= ~SS_CANTRCVMORE;
+			fip->fi_readsock->so_rcv.sb_state &= ~SBS_CANTRCVMORE;
 			if (fip->fi_readers > 0) {
 				wakeup(&fip->fi_readers);
 				sorwakeup(fip->fi_writesock);
@@ -447,7 +447,7 @@ filt_fiforead(struct knote *kn, long hint)
 	struct socket *so = (struct socket *)kn->kn_hook;
 
 	kn->kn_data = so->so_rcv.sb_cc;
-	if (so->so_state & SS_CANTRCVMORE) {
+	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 		kn->kn_flags |= EV_EOF;
 		return (1);
 	}
@@ -471,7 +471,7 @@ filt_fifowrite(struct knote *kn, long hint)
 	struct socket *so = (struct socket *)kn->kn_hook;
 
 	kn->kn_data = sbspace(&so->so_snd);
-	if (so->so_state & SS_CANTSENDMORE) {
+	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
 		return (1);
 	}
diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index 5f14608..a580b43 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -168,7 +168,7 @@ soo_ioctl(fp, cmd, data, active_cred, td)
 		return (0);
 
 	case SIOCATMARK:
-		*(int *)data = (so->so_state&SS_RCVATMARK) != 0;
+		*(int *)data = (so->so_rcv.sb_state & SBS_RCVATMARK) != 0;
 		return (0);
 	}
 	/*
@@ -207,13 +207,13 @@ soo_stat(fp, ub, active_cred, td)
 	bzero((caddr_t)ub, sizeof (*ub));
 	ub->st_mode = S_IFSOCK;
 	/*
-	 * If SS_CANTRCVMORE is set, but there's still data left in the
+	 * If SBS_CANTRCVMORE is set, but there's still data left in the
 	 * receive buffer, the socket is still readable.
 	 */
-	if ((so->so_state & SS_CANTRCVMORE) == 0 ||
+	if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0 ||
 	    so->so_rcv.sb_cc != 0)
 		ub->st_mode |= S_IRUSR | S_IRGRP | S_IROTH;
-	if ((so->so_state & SS_CANTSENDMORE) == 0)
+	if ((so->so_snd.sb_state & SBS_CANTSENDMORE) == 0)
 		ub->st_mode |= S_IWUSR | S_IWGRP | S_IWOTH;
 	ub->st_size = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
 	ub->st_uid = so->so_cred->cr_uid;
diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
index 7dbc19d..2d50f32 100644
--- a/sys/kern/uipc_sockbuf.c
+++ b/sys/kern/uipc_sockbuf.c
@@ -153,7 +153,9 @@ soisdisconnecting(so)
 {
 
 	so->so_state &= ~SS_ISCONNECTING;
-	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
+	so->so_state |= SS_ISDISCONNECTING;
+	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	wakeup(&so->so_timeo);
 	sowwakeup(so);
 	sorwakeup(so);
@@ -165,7 +167,9 @@ soisdisconnected(so)
 {
 
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
-	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
+	so->so_state |= SS_ISDISCONNECTED;
+	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	wakeup(&so->so_timeo);
 	sbdrop(&so->so_snd, so->so_snd.sb_cc);
 	sowwakeup(so);
@@ -270,7 +274,7 @@ socantsendmore(so)
 	struct socket *so;
 {
 
-	so->so_state |= SS_CANTSENDMORE;
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sowwakeup(so);
 }
 
@@ -279,7 +283,7 @@ socantrcvmore(so)
 	struct socket *so;
 {
 
-	so->so_state |= SS_CANTRCVMORE;
+	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
 	sorwakeup(so);
 }
 
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 145a5a6..fb9043c 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -608,7 +608,7 @@ restart:
 		goto out;
 	do {
 		s = splnet();
-		if (so->so_state & SS_CANTSENDMORE)
+		if (so->so_snd.sb_state & SBS_CANTSENDMORE)
 			snderr(EPIPE);
 		if (so->so_error) {
 			error = so->so_error;
@@ -758,7 +758,7 @@ restart:
 			    so->so_options |= SO_DONTROUTE;
 		    s = splnet();				/* XXX */
 		    /*
-		     * XXX all the SS_CANTSENDMORE checks previously
+		     * XXX all the SBS_CANTSENDMORE checks previously
 		     * done could be out of date.  We could have recieved
 		     * a reset packet in an interrupt or maybe we slept
 		     * while doing page faults in uiomove() etc. We could
@@ -920,7 +920,7 @@ restart:
 				so->so_error = 0;
 			goto release;
 		}
-		if (so->so_state & SS_CANTRCVMORE) {
+		if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 			if (m)
 				goto dontblock;
 			else
@@ -1035,7 +1035,7 @@ dontblock:
 		else
 		    KASSERT(m->m_type == MT_DATA || m->m_type == MT_HEADER,
 			("m->m_type == %d", m->m_type));
-		so->so_state &= ~SS_RCVATMARK;
+		so->so_rcv.sb_state &= ~SBS_RCVATMARK;
 		len = uio->uio_resid;
 		if (so->so_oobmark && len > so->so_oobmark - offset)
 			len = so->so_oobmark - offset;
@@ -1125,7 +1125,7 @@ dontblock:
 			if ((flags & MSG_PEEK) == 0) {
 				so->so_oobmark -= len;
 				if (so->so_oobmark == 0) {
-					so->so_state |= SS_RCVATMARK;
+					so->so_rcv.sb_state |= SBS_RCVATMARK;
 					break;
 				}
 			} else {
@@ -1145,7 +1145,7 @@ dontblock:
 		 */
 		while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
 		    !sosendallatonce(so) && nextrecord == NULL) {
-			if (so->so_error || so->so_state & SS_CANTRCVMORE)
+			if (so->so_error || so->so_rcv.sb_state & SBS_CANTRCVMORE)
 				break;
 			/*
 			 * Notify the protocol that some data has been
@@ -1192,7 +1192,7 @@ dontblock:
 			(*pr->pr_usrreqs->pru_rcvd)(so, flags);
 	}
 	if (orig_resid == uio->uio_resid && orig_resid &&
-	    (flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) {
+	    (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
 		sbunlock(&so->so_rcv);
 		splx(s);
 		goto restart;
@@ -1834,7 +1834,7 @@ sopoll(struct socket *so, int events, struct ucred *active_cred,
 			revents |= events & (POLLOUT | POLLWRNORM);
 
 	if (events & (POLLPRI | POLLRDBAND))
-		if (so->so_oobmark || (so->so_state & SS_RCVATMARK))
+		if (so->so_oobmark || (so->so_rcv.sb_state & SBS_RCVATMARK))
 			revents |= events & (POLLPRI | POLLRDBAND);
 
 	if (revents == 0) {
@@ -1905,7 +1905,7 @@ filt_soread(struct knote *kn, long hint)
 	int result;
 
 	kn->kn_data = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
-	if (so->so_state & SS_CANTRCVMORE) {
+	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 		kn->kn_flags |= EV_EOF;
 		kn->kn_fflags = so->so_error;
 		result = 1;
@@ -1938,7 +1938,7 @@ filt_sowrite(struct knote *kn, long hint)
 	int result;
 
 	kn->kn_data = sbspace(&so->so_snd);
-	if (so->so_state & SS_CANTSENDMORE) {
+	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
 		kn->kn_fflags = so->so_error;
 		result = 1;
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index 7dbc19d..2d50f32 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -153,7 +153,9 @@ soisdisconnecting(so)
 {
 
 	so->so_state &= ~SS_ISCONNECTING;
-	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
+	so->so_state |= SS_ISDISCONNECTING;
+	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	wakeup(&so->so_timeo);
 	sowwakeup(so);
 	sorwakeup(so);
@@ -165,7 +167,9 @@ soisdisconnected(so)
 {
 
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
-	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
+	so->so_state |= SS_ISDISCONNECTED;
+	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	wakeup(&so->so_timeo);
 	sbdrop(&so->so_snd, so->so_snd.sb_cc);
 	sowwakeup(so);
@@ -270,7 +274,7 @@ socantsendmore(so)
 	struct socket *so;
 {
 
-	so->so_state |= SS_CANTSENDMORE;
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sowwakeup(so);
 }
 
@@ -279,7 +283,7 @@ socantrcvmore(so)
 	struct socket *so;
 {
 
-	so->so_state |= SS_CANTRCVMORE;
+	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
 	sorwakeup(so);
 }
 
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 18a5e24..5ebb7dd 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -292,7 +292,7 @@ accept1(td, uap, compat)
 		goto noconnection;
 	}
 	while (TAILQ_EMPTY(&head->so_comp) && head->so_error == 0) {
-		if (head->so_state & SS_CANTRCVMORE) {
+		if (head->so_rcv.sb_state & SBS_CANTRCVMORE) {
 			head->so_error = ECONNABORTED;
 			break;
 		}
@@ -1842,7 +1842,7 @@ retry_lookup:
 		 * before going to the extra work of constituting the sf_buf.
 		 */
 		if ((so->so_state & SS_NBIO) && sbspace(&so->so_snd) <= 0) {
-			if (so->so_state & SS_CANTSENDMORE)
+			if (so->so_snd.sb_state & SBS_CANTSENDMORE)
 				error = EPIPE;
 			else
 				error = EAGAIN;
@@ -2001,8 +2001,8 @@ retry_space:
 		 * blocks before the pru_send (or more accurately, any blocking
 		 * results in a loop back to here to re-check).
 		 */
-		if ((so->so_state & SS_CANTSENDMORE) || so->so_error) {
-			if (so->so_state & SS_CANTSENDMORE) {
+		if ((so->so_snd.sb_state & SBS_CANTSENDMORE) || so->so_error) {
+			if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 				error = EPIPE;
 			} else {
 				error = so->so_error;
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index aa435f2..18a4274 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -378,7 +378,7 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 			}
 		}
 
-		if (so->so_state & SS_CANTSENDMORE) {
+		if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 			error = EPIPE;
 			break;
 		}
diff --git a/sys/netatm/atm_aal5.c b/sys/netatm/atm_aal5.c
index 81fcce8..04e5d7e 100644
--- a/sys/netatm/atm_aal5.c
+++ b/sys/netatm/atm_aal5.c
@@ -767,7 +767,7 @@ atm_aal5_cpcs_data(tok, m)
 	 * that there's room in the socket buffer
 	 */
 	if (((so->so_state & SS_ISCONNECTED) == 0) ||
-	    (so->so_state & SS_CANTRCVMORE) ||
+	    (so->so_rcv.sb_state & SBS_CANTRCVMORE) ||
 	    (len > sbspace(&so->so_rcv))) {
 		atm_sock_stat.as_indrop[atp->atp_type]++;
 		KB_FREEALL(m);
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c b/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
index 1503076..09cb139 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
@@ -1024,7 +1024,7 @@ ng_btsocket_rfcomm_session_task(ng_btsocket_rfcomm_session_p s)
 {
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
-	if (s->l2so->so_state & SS_CANTRCVMORE) {
+	if (s->l2so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 		NG_BTSOCKET_RFCOMM_INFO(
 "%s: L2CAP connection has been terminated, so=%p, so_state=%#x, so_count=%d, " \
 "state=%d, flags=%#x\n", __func__, s->l2so, s->l2so->so_state, 
@@ -1357,7 +1357,7 @@ ng_btsocket_rfcomm_session_accept(ng_btsocket_rfcomm_session_p s0)
 	ACCEPT_LOCK();
 	if (TAILQ_EMPTY(&s0->l2so->so_comp)) {
 		ACCEPT_UNLOCK();
-		if (s0->l2so->so_state & SS_CANTRCVMORE)
+		if (s0->l2so->so_rcv.sb_state & SBS_CANTRCVMORE)
 			return (ECONNABORTED);
 		return (EWOULDBLOCK);
 	}
diff --git a/sys/netgraph/ng_ksocket.c b/sys/netgraph/ng_ksocket.c
index badc69c..f243318 100644
--- a/sys/netgraph/ng_ksocket.c
+++ b/sys/netgraph/ng_ksocket.c
@@ -1144,7 +1144,7 @@ sendit:		/* Forward data with optional peer sockaddr as meta info */
 	 * If the peer has closed the connection, forward a 0-length mbuf
 	 * to indicate end-of-file.
 	 */
-	if (so->so_state & SS_CANTRCVMORE && !(priv->flags & KSF_EOFSEEN)) {
+	if (so->so_rcv.sb_state & SBS_CANTRCVMORE && !(priv->flags & KSF_EOFSEEN)) {
 		MGETHDR(m, waitflag, MT_DATA);
 		if (m != NULL) {
 			m->m_len = m->m_pkthdr.len = 0;
@@ -1171,7 +1171,7 @@ ng_ksocket_check_accept(priv_p priv)
 	}
 	/* Unlocked read. */
 	if (TAILQ_EMPTY(&head->so_comp)) {
-		if (head->so_state & SS_CANTRCVMORE)
+		if (head->so_rcv.sb_state & SBS_CANTRCVMORE)
 			return ECONNABORTED;
 		return EWOULDBLOCK;
 	}
diff --git a/sys/netinet/accf_http.c b/sys/netinet/accf_http.c
index ffae8b5..d7843bc 100644
--- a/sys/netinet/accf_http.c
+++ b/sys/netinet/accf_http.c
@@ -161,7 +161,7 @@ static void
 sohashttpget(struct socket *so, void *arg, int waitflag)
 {
 
-	if ((so->so_state & SS_CANTRCVMORE) == 0 && !sbfull(&so->so_rcv)) {
+	if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0 && !sbfull(&so->so_rcv)) {
 		struct mbuf *m;
 		char *cmp;
 		int	cmplen, cc;
@@ -214,7 +214,7 @@ soparsehttpvers(struct socket *so, void *arg, int waitflag)
 	struct mbuf *m, *n;
 	int	i, cc, spaces, inspaces;
 
-	if ((so->so_state & SS_CANTRCVMORE) != 0 || sbfull(&so->so_rcv))
+	if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) != 0 || sbfull(&so->so_rcv))
 		goto fallout;
 
 	m = so->so_rcv.sb_mb;
@@ -301,7 +301,7 @@ soishttpconnected(struct socket *so, void *arg, int waitflag)
 	int ccleft, copied;
 
 	DPRINT("start");
-	if ((so->so_state & SS_CANTRCVMORE) != 0 || sbfull(&so->so_rcv))
+	if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) != 0 || sbfull(&so->so_rcv))
 		goto gotit;
 
 	/*
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index 04b4d2c..3fa801d 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -355,7 +355,7 @@ present:
 		flags = q->tqe_th->th_flags & TH_FIN;
 		nq = LIST_NEXT(q, tqe_q);
 		LIST_REMOVE(q, tqe_q);
-		if (so->so_state & SS_CANTRCVMORE)
+		if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 			m_freem(q->tqe_m);
 		else
 			sbappendstream(&so->so_rcv, q->tqe_m);
@@ -1237,7 +1237,7 @@ after_listen:
 #endif
 			 * Add data to socket buffer.
 			 */
-			if (so->so_state & SS_CANTRCVMORE) {
+			if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 				m_freem(m);
 			} else {
 				m_adj(m, drop_hdrlen);	/* delayed header drop */
@@ -2143,7 +2143,7 @@ process_ACK:
 		 * we should release the tp also, and use a
 		 * compressed state.
 		 */
-				if (so->so_state & SS_CANTRCVMORE) {
+				if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 					soisdisconnected(so);
 					callout_reset(tp->tt_2msl, tcp_maxidle,
 						      tcp_timer_2msl, tp);
@@ -2250,7 +2250,7 @@ step6:
 			so->so_oobmark = so->so_rcv.sb_cc +
 			    (tp->rcv_up - tp->rcv_nxt) - 1;
 			if (so->so_oobmark == 0)
-				so->so_state |= SS_RCVATMARK;
+				so->so_rcv.sb_state |= SBS_RCVATMARK;
 			sohasoutofband(so);
 			tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA);
 		}
@@ -2311,7 +2311,7 @@ dodata:							/* XXX */
 			tcpstat.tcps_rcvpack++;
 			tcpstat.tcps_rcvbyte += tlen;
 			ND6_HINT(tp);
-			if (so->so_state & SS_CANTRCVMORE)
+			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 				m_freem(m);
 			else
 				sbappendstream(&so->so_rcv, m);
diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c
index 04b4d2c..3fa801d 100644
--- a/sys/netinet/tcp_reass.c
+++ b/sys/netinet/tcp_reass.c
@@ -355,7 +355,7 @@ present:
 		flags = q->tqe_th->th_flags & TH_FIN;
 		nq = LIST_NEXT(q, tqe_q);
 		LIST_REMOVE(q, tqe_q);
-		if (so->so_state & SS_CANTRCVMORE)
+		if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 			m_freem(q->tqe_m);
 		else
 			sbappendstream(&so->so_rcv, q->tqe_m);
@@ -1237,7 +1237,7 @@ after_listen:
 #endif
 			 * Add data to socket buffer.
 			 */
-			if (so->so_state & SS_CANTRCVMORE) {
+			if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 				m_freem(m);
 			} else {
 				m_adj(m, drop_hdrlen);	/* delayed header drop */
@@ -2143,7 +2143,7 @@ process_ACK:
 		 * we should release the tp also, and use a
 		 * compressed state.
 		 */
-				if (so->so_state & SS_CANTRCVMORE) {
+				if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 					soisdisconnected(so);
 					callout_reset(tp->tt_2msl, tcp_maxidle,
 						      tcp_timer_2msl, tp);
@@ -2250,7 +2250,7 @@ step6:
 			so->so_oobmark = so->so_rcv.sb_cc +
 			    (tp->rcv_up - tp->rcv_nxt) - 1;
 			if (so->so_oobmark == 0)
-				so->so_state |= SS_RCVATMARK;
+				so->so_rcv.sb_state |= SBS_RCVATMARK;
 			sohasoutofband(so);
 			tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA);
 		}
@@ -2311,7 +2311,7 @@ dodata:							/* XXX */
 			tcpstat.tcps_rcvpack++;
 			tcpstat.tcps_rcvbyte += tlen;
 			ND6_HINT(tp);
-			if (so->so_state & SS_CANTRCVMORE)
+			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 				m_freem(m);
 			else
 				sbappendstream(&so->so_rcv, m);
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 89ddc79..fa95920 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -650,7 +650,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
 	if (inp == NULL) {
 		/*
 		 * OOPS! we lost a race, the TCP session got reset after
-		 * we checked SS_CANTSENDMORE, eg: while doing uiomove or a
+		 * we checked SBS_CANTSENDMORE, eg: while doing uiomove or a
 		 * network interrupt in the non-splnet() section of sosend().
 		 */
 		if (m)
@@ -788,7 +788,7 @@ tcp_usr_rcvoob(struct socket *so, struct mbuf *m, int flags)
 
 	COMMON_START();
 	if ((so->so_oobmark == 0 &&
-	     (so->so_state & SS_RCVATMARK) == 0) ||
+	     (so->so_rcv.sb_state & SBS_RCVATMARK) == 0) ||
 	    so->so_options & SO_OOBINLINE ||
 	    tp->t_oobflags & TCPOOB_HADDATA) {
 		error = EINVAL;
diff --git a/sys/netipx/spx_usrreq.c b/sys/netipx/spx_usrreq.c
index ea47aba..eb34fb7 100644
--- a/sys/netipx/spx_usrreq.c
+++ b/sys/netipx/spx_usrreq.c
@@ -567,7 +567,7 @@ present:
 				if (so->so_rcv.sb_cc)
 					so->so_oobmark = so->so_rcv.sb_cc;
 				else
-					so->so_state |= SS_RCVATMARK;
+					so->so_rcv.sb_state |= SBS_RCVATMARK;
 			}
 			q = q->si_prev;
 			remque(q->si_next);
@@ -597,7 +597,7 @@ present:
 					MCHTYPE(m, MT_OOBDATA);
 					spx_newchecks[1]++;
 					so->so_oobmark = 0;
-					so->so_state &= ~SS_RCVATMARK;
+					so->so_rcv.sb_state &= ~SBS_RCVATMARK;
 				}
 				if (packetp == 0) {
 					m->m_data += SPINC;
@@ -1537,7 +1537,7 @@ spx_rcvoob(so, m, flags)
 	cb = ipxtospxpcb(ipxp);
 
 	if ((cb->s_oobflags & SF_IOOB) || so->so_oobmark ||
-	    (so->so_state & SS_RCVATMARK)) {
+	    (so->so_rcv.sb_state & SBS_RCVATMARK)) {
 		m->m_len = 1;
 		*mtod(m, caddr_t) = cb->s_iobc;
 		return (0);
diff --git a/sys/netsmb/smb_trantcp.c b/sys/netsmb/smb_trantcp.c
index 1832388..5972dee 100644
--- a/sys/netsmb/smb_trantcp.c
+++ b/sys/netsmb/smb_trantcp.c
@@ -414,8 +414,8 @@ nbssn_recv(struct nbpcb *nbp, struct mbuf **mpp, int *lenp,
 		 * If we don't have one waiting, return.
 		 */
 		error = nbssn_recvhdr(nbp, &len, &rpcode, MSG_DONTWAIT, td);
-		if (so->so_state &
-		    (SS_ISDISCONNECTING | SS_ISDISCONNECTED | SS_CANTRCVMORE)) {
+		if ((so->so_state & (SS_ISDISCONNECTING | SS_ISDISCONNECTED)) ||
+		    (so->so_rcv.sb_state & SBS_CANTRCVMORE)) {
 			nbp->nbp_state = NBST_CLOSED;
 			NBDEBUG("session closed by peer\n");
 			return ECONNRESET;
diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index 84a1747..bae1b4d 100644
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -109,6 +109,7 @@ struct socket {
 		int	sb_lowat;	/* low water mark */
 		int	sb_timeo;	/* timeout for read/write */
 		short	sb_flags;	/* flags, see below */
+		short	sb_state;	/* (c/d) socket state on sockbuf */
 	} so_rcv, so_snd;
 #define	SB_MAX		(256*1024)	/* default for max chars in sockbuf */
 #define	SB_LOCK		0x01		/* lock on data queue */
@@ -177,24 +178,35 @@ extern struct mtx accept_mtx;
 #define	SOCK_UNLOCK(_so)		SOCKBUF_UNLOCK(&(_so)->so_rcv)
 #define	SOCK_LOCK_ASSERT(_so)		SOCKBUF_LOCK_ASSERT(&(_so)->so_rcv)
 
-/*
+/*-
  * Socket state bits.
+ *
+ * Historically, this bits were all kept in the so_state field.  For
+ * locking reasons, they are now in multiple fields, as they are
+ * locked differently.  so_state maintains basic socket state protected
+ * by the socket lock.  so_qstate holds information about the socket
+ * accept queues.  Each socket buffer also has a state field holding
+ * information relevant to that socket buffer (can't send, rcv).  Many
+ * fields will be read without locks to improve performance and avoid
+ * lock order issues.  However, this approach must be used with caution.
  */
 #define	SS_NOFDREF		0x0001	/* no file table ref any more */
 #define	SS_ISCONNECTED		0x0002	/* socket connected to a peer */
 #define	SS_ISCONNECTING		0x0004	/* in process of connecting to peer */
 #define	SS_ISDISCONNECTING	0x0008	/* in process of disconnecting */
-#define	SS_CANTSENDMORE		0x0010	/* can't send more data to peer */
-#define	SS_CANTRCVMORE		0x0020	/* can't receive more data from peer */
-#define	SS_RCVATMARK		0x0040	/* at mark on input */
-
 #define	SS_NBIO			0x0100	/* non-blocking ops */
 #define	SS_ASYNC		0x0200	/* async i/o notify */
 #define	SS_ISCONFIRMING		0x0400	/* deciding to accept connection req */
-
 #define	SS_ISDISCONNECTED	0x2000	/* socket disconnected from peer */
 
 /*
+ * Socket state bits now stored in the socket buffer state field.
+ */
+#define	SBS_CANTSENDMORE	0x0010	/* can't send more data to peer */
+#define	SBS_CANTRCVMORE		0x0020	/* can't receive more data from peer */
+#define	SBS_RCVATMARK		0x0040	/* at mark on input */
+
+/*
  * Socket state bits stored in so_qstate.
  */
 #define	SQ_INCOMP		0x0800	/* unaccepted, incomplete connection */
@@ -261,7 +273,7 @@ struct xsocket {
 /* can we read something from so? */
 #define	soreadable(so) \
     ((so)->so_rcv.sb_cc >= (so)->so_rcv.sb_lowat || \
-	((so)->so_state & SS_CANTRCVMORE) || \
+	((so)->so_rcv.sb_state & SBS_CANTRCVMORE) || \
 	!TAILQ_EMPTY(&(so)->so_comp) || (so)->so_error)
 
 /* can we write something to so? */
@@ -269,7 +281,7 @@ struct xsocket {
     ((sbspace(&(so)->so_snd) >= (so)->so_snd.sb_lowat && \
 	(((so)->so_state&SS_ISCONNECTED) || \
 	  ((so)->so_proto->pr_flags&PR_CONNREQUIRED)==0)) || \
-     ((so)->so_state & SS_CANTSENDMORE) || \
+     ((so)->so_snd.sb_state & SBS_CANTSENDMORE) || \
      (so)->so_error)
 
 /* adjust counters in sb reflecting allocation of m */