Reduce the number of unnecessary unlock-relocks on socket buffer mutexes

associated with performing a wakeup on the socket buffer:

- When performing an sbappend*() followed by a so[rw]wakeup(), explicitly
  acquire the socket buffer lock and use the _locked() variants of both
  calls.  Note that the _locked() sowakeup() versions unlock the mutex on
  return.  This is done in uipc_send(), divert_packet(), mroute
  socket_send(), raw_append(), tcp_reass(), tcp_input(), and udp_append().

- When the socket buffer lock is dropped before a sowakeup(), remove the
  explicit unlock and use the _locked() sowakeup() variant.  This is done
  in soisdisconnecting(), soisdisconnected() when setting the can't send/
  receive flags and dropping data, and in uipc_rcvd() which adjusting
  back-pressure on the sockets.

For UNIX domain sockets running mpsafe with a contention-intensive SMP
mysql benchmark, this results in a 1.6% query rate improvement due to
reduce mutex costs.

9 files changed, 64 insertions, 49 deletions

diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
index e366c3c..efcea0c 100644
--- a/sys/kern/uipc_sockbuf.c
+++ b/sys/kern/uipc_sockbuf.c
@@ -169,13 +169,11 @@ soisdisconnecting(so)
 	SOCK_UNLOCK(so);
 	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	sorwakeup_locked(so);
 	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	sowwakeup_locked(so);
 	wakeup(&so->so_timeo);
-	sowwakeup(so);
-	sorwakeup(so);
 }
 
 void
@@ -188,20 +186,19 @@ soisdisconnected(so)
 	 * SOCKBUF_LOCK(&so->so_rcv) even though they are the same mutex to
 	 * avoid introducing the assumption  that they are the same.
 	 */
+	/* XXXRW: so_state locking? */
 	SOCK_LOCK(so);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= SS_ISDISCONNECTED;
 	SOCK_UNLOCK(so);
 	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	sorwakeup_locked(so);
 	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sbdrop_locked(&so->so_snd, so->so_snd.sb_cc);
-	SOCKBUF_UNLOCK(&so->so_snd);
+	sowwakeup_locked(so);
 	wakeup(&so->so_timeo);
-	sowwakeup(so);
-	sorwakeup(so);
 }
 
 /*
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index e366c3c..efcea0c 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -169,13 +169,11 @@ soisdisconnecting(so)
 	SOCK_UNLOCK(so);
 	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	sorwakeup_locked(so);
 	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	sowwakeup_locked(so);
 	wakeup(&so->so_timeo);
-	sowwakeup(so);
-	sorwakeup(so);
 }
 
 void
@@ -188,20 +186,19 @@ soisdisconnected(so)
 	 * SOCKBUF_LOCK(&so->so_rcv) even though they are the same mutex to
 	 * avoid introducing the assumption  that they are the same.
 	 */
+	/* XXXRW: so_state locking? */
 	SOCK_LOCK(so);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= SS_ISDISCONNECTED;
 	SOCK_UNLOCK(so);
 	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	sorwakeup_locked(so);
 	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sbdrop_locked(&so->so_snd, so->so_snd.sb_cc);
-	SOCKBUF_UNLOCK(&so->so_snd);
+	sowwakeup_locked(so);
 	wakeup(&so->so_timeo);
-	sowwakeup(so);
-	sorwakeup(so);
 }
 
 /*
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 862b7be..95e4874 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -294,8 +294,7 @@ uipc_rcvd(struct socket *so, int flags)
 		    newhiwat, RLIM_INFINITY);
 		unp->unp_cc = so->so_rcv.sb_cc;
 		SOCKBUF_UNLOCK(&so->so_rcv);
-		SOCKBUF_UNLOCK(&so2->so_snd);
-		sowwakeup(so2);
+		sowwakeup_locked(so2);
 		break;
 
 	default:
@@ -355,8 +354,7 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 			from = &sun_noname;
 		SOCKBUF_LOCK(&so2->so_rcv);
 		if (sbappendaddr_locked(&so2->so_rcv, from, m, control)) {
-			SOCKBUF_UNLOCK(&so2->so_rcv);
-			sorwakeup(so2);
+			sorwakeup_locked(so2);
 			m = NULL;
 			control = NULL;
 		} else {
@@ -412,8 +410,7 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 		(void)chgsbsize(so->so_cred->cr_uidinfo, &so->so_snd.sb_hiwat,
 		    newhiwat, RLIM_INFINITY);
 		unp->unp_conn->unp_cc = so2->so_rcv.sb_cc;
-		SOCKBUF_UNLOCK(&so2->so_rcv);
-		sorwakeup(so2);
+		sorwakeup_locked(so2);
 		m = NULL;
 		break;
 
diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c
index ef4bb85..f31a7f8 100644
--- a/sys/netinet/ip_divert.c
+++ b/sys/netinet/ip_divert.c
@@ -228,12 +228,14 @@ divert_packet(struct mbuf *m, int incoming)
 		/* XXX why does only one socket match? */
 		if (inp->inp_lport == nport) {
 			sa = inp->inp_socket;
-			if (sbappendaddr(&sa->so_rcv,
+			SOCKBUF_LOCK(&sa->so_rcv);
+			if (sbappendaddr_locked(&sa->so_rcv,
 			    (struct sockaddr *)&divsrc, m,
-			    (struct mbuf *)0) == 0)
+			    (struct mbuf *)0) == 0) {
 				sa = NULL;	/* force mbuf reclaim below */
-			else
-				sorwakeup(sa);
+				SOCKBUF_UNLOCK(&sa->so_rcv);
+			} else
+				sorwakeup_locked(sa);
 			INP_UNLOCK(inp);
 			break;
 		}
diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c
index c4ef487..9230d02 100644
--- a/sys/netinet/ip_mroute.c
+++ b/sys/netinet/ip_mroute.c
@@ -1307,10 +1307,13 @@ static int
 socket_send(struct socket *s, struct mbuf *mm, struct sockaddr_in *src)
 {
     if (s) {
-	if (sbappendaddr(&s->so_rcv, (struct sockaddr *)src, mm, NULL) != 0) {
-	    sorwakeup(s);
+	SOCKBUF_LOCK(&s->so_rcv);
+	if (sbappendaddr_locked(&s->so_rcv, (struct sockaddr *)src, mm,
+	    NULL) != 0) {
+	    sorwakeup_locked(s);
 	    return 0;
 	}
+	SOCKBUF_UNLOCK(&s->so_rcv);
     }
     m_freem(mm);
     return -1;
diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c
index 85eaaf2..dd79877 100644
--- a/sys/netinet/raw_ip.c
+++ b/sys/netinet/raw_ip.c
@@ -160,18 +160,22 @@ raw_append(struct inpcb *last, struct ip *ip, struct mbuf *n)
 #endif
 	if (!policyfail) {
 		struct mbuf *opts = NULL;
+		struct socket *so;
 
+		so = last->inp_socket;
 		if ((last->inp_flags & INP_CONTROLOPTS) ||
-		    (last->inp_socket->so_options & SO_TIMESTAMP))
+		    (so->so_options & SO_TIMESTAMP))
 			ip_savecontrol(last, &opts, ip, n);
-		if (sbappendaddr(&last->inp_socket->so_rcv,
+		SOCKBUF_LOCK(&so->so_rcv);
+		if (sbappendaddr_locked(&so->so_rcv,
 		    (struct sockaddr *)&ripsrc, n, opts) == 0) {
 			/* should notify about lost packet */
 			m_freem(n);
 			if (opts)
 				m_freem(opts);
+			SOCKBUF_UNLOCK(&so->so_rcv);
 		} else
-			sorwakeup(last->inp_socket);
+			sorwakeup_locked(so);
 	} else
 		m_freem(n);
 	return policyfail;
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index 5b37327..ee6d3ce 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -353,22 +353,24 @@ present:
 	q = LIST_FIRST(&tp->t_segq);
 	if (!q || q->tqe_th->th_seq != tp->rcv_nxt)
 		return (0);
+	SOCKBUF_LOCK(&so->so_rcv);
 	do {
 		tp->rcv_nxt += q->tqe_len;
 		flags = q->tqe_th->th_flags & TH_FIN;
 		nq = LIST_NEXT(q, tqe_q);
 		LIST_REMOVE(q, tqe_q);
+		/* Unlocked read. */
 		if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 			m_freem(q->tqe_m);
 		else
-			sbappendstream(&so->so_rcv, q->tqe_m);
+			sbappendstream_locked(&so->so_rcv, q->tqe_m);
 		uma_zfree(tcp_reass_zone, q);
 		tp->t_segqlen--;
 		tcp_reass_qsize--;
 		q = nq;
 	} while (q && q->tqe_th->th_seq == tp->rcv_nxt);
 	ND6_HINT(tp);
-	sorwakeup(so);
+	sorwakeup_locked(so);
 	return (flags);
 }
 
@@ -1262,13 +1264,15 @@ after_listen:
 #endif
 			 * Add data to socket buffer.
 			 */
+			/* Unlocked read. */
+			SOCKBUF_LOCK(&so->so_rcv);
 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 				m_freem(m);
 			} else {
 				m_adj(m, drop_hdrlen);	/* delayed header drop */
-				sbappendstream(&so->so_rcv, m);
+				sbappendstream_locked(&so->so_rcv, m);
 			}
-			sorwakeup(so);
+			sorwakeup_locked(so);
 			if (DELAY_ACK(tp)) {
 				tp->t_flags |= TF_DELACK;
 			} else {
@@ -2153,8 +2157,7 @@ process_ACK:
 			tp->snd_wnd -= acked;
 			ourfinisacked = 0;
 		}
-		SOCKBUF_UNLOCK(&so->so_snd);
-		sowwakeup(so);
+		sowwakeup_locked(so);
 		/* detect una wraparound */
 		if ((tcp_do_newreno || tp->sack_enable) && 
 		    !IN_FASTRECOVERY(tp) &&
@@ -2363,11 +2366,13 @@ dodata:							/* XXX */
 			tcpstat.tcps_rcvpack++;
 			tcpstat.tcps_rcvbyte += tlen;
 			ND6_HINT(tp);
+			/* Unlocked read. */
+			SOCKBUF_LOCK(&so->so_rcv);
 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 				m_freem(m);
 			else
-				sbappendstream(&so->so_rcv, m);
-			sorwakeup(so);
+				sbappendstream_locked(&so->so_rcv, m);
+			sorwakeup_locked(so);
 		} else {
 			thflags = tcp_reass(tp, th, &tlen, m);
 			tp->t_flags |= TF_ACKNOW;
diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c
index 5b37327..ee6d3ce 100644
--- a/sys/netinet/tcp_reass.c
+++ b/sys/netinet/tcp_reass.c
@@ -353,22 +353,24 @@ present:
 	q = LIST_FIRST(&tp->t_segq);
 	if (!q || q->tqe_th->th_seq != tp->rcv_nxt)
 		return (0);
+	SOCKBUF_LOCK(&so->so_rcv);
 	do {
 		tp->rcv_nxt += q->tqe_len;
 		flags = q->tqe_th->th_flags & TH_FIN;
 		nq = LIST_NEXT(q, tqe_q);
 		LIST_REMOVE(q, tqe_q);
+		/* Unlocked read. */
 		if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 			m_freem(q->tqe_m);
 		else
-			sbappendstream(&so->so_rcv, q->tqe_m);
+			sbappendstream_locked(&so->so_rcv, q->tqe_m);
 		uma_zfree(tcp_reass_zone, q);
 		tp->t_segqlen--;
 		tcp_reass_qsize--;
 		q = nq;
 	} while (q && q->tqe_th->th_seq == tp->rcv_nxt);
 	ND6_HINT(tp);
-	sorwakeup(so);
+	sorwakeup_locked(so);
 	return (flags);
 }
 
@@ -1262,13 +1264,15 @@ after_listen:
 #endif
 			 * Add data to socket buffer.
 			 */
+			/* Unlocked read. */
+			SOCKBUF_LOCK(&so->so_rcv);
 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 				m_freem(m);
 			} else {
 				m_adj(m, drop_hdrlen);	/* delayed header drop */
-				sbappendstream(&so->so_rcv, m);
+				sbappendstream_locked(&so->so_rcv, m);
 			}
-			sorwakeup(so);
+			sorwakeup_locked(so);
 			if (DELAY_ACK(tp)) {
 				tp->t_flags |= TF_DELACK;
 			} else {
@@ -2153,8 +2157,7 @@ process_ACK:
 			tp->snd_wnd -= acked;
 			ourfinisacked = 0;
 		}
-		SOCKBUF_UNLOCK(&so->so_snd);
-		sowwakeup(so);
+		sowwakeup_locked(so);
 		/* detect una wraparound */
 		if ((tcp_do_newreno || tp->sack_enable) && 
 		    !IN_FASTRECOVERY(tp) &&
@@ -2363,11 +2366,13 @@ dodata:							/* XXX */
 			tcpstat.tcps_rcvpack++;
 			tcpstat.tcps_rcvbyte += tlen;
 			ND6_HINT(tp);
+			/* Unlocked read. */
+			SOCKBUF_LOCK(&so->so_rcv);
 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 				m_freem(m);
 			else
-				sbappendstream(&so->so_rcv, m);
-			sorwakeup(so);
+				sbappendstream_locked(&so->so_rcv, m);
+			sorwakeup_locked(so);
 		} else {
 			thflags = tcp_reass(tp, th, &tlen, m);
 			tp->t_flags |= TF_ACKNOW;
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 9c541db..0997f9a 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -447,6 +447,7 @@ udp_append(last, ip, n, off)
 	int off;
 {
 	struct sockaddr *append_sa;
+	struct socket *so;
 	struct mbuf *opts = 0;
 
 	INP_LOCK_ASSERT(last);
@@ -496,13 +497,17 @@ udp_append(last, ip, n, off)
 #endif
 	append_sa = (struct sockaddr *)&udp_in;
 	m_adj(n, off);
-	if (sbappendaddr(&last->inp_socket->so_rcv, append_sa, n, opts) == 0) {
+
+	so = last->inp_socket;
+	SOCKBUF_LOCK(&so->so_rcv);
+	if (sbappendaddr_locked(&so->so_rcv, append_sa, n, opts) == 0) {
 		m_freem(n);
 		if (opts)
 			m_freem(opts);
 		udpstat.udps_fullsock++;
+		SOCKBUF_UNLOCK(&so->so_rcv);
 	} else
-		sorwakeup(last->inp_socket);
+		sorwakeup_locked(so);
 }
 
 /*