Reduce the number of unnecessary unlock-relocks on socket buffer mutexes

associated with performing a wakeup on the socket buffer:

- When performing an sbappend*() followed by a so[rw]wakeup(), explicitly
  acquire the socket buffer lock and use the _locked() variants of both
  calls.  Note that the _locked() sowakeup() versions unlock the mutex on
  return.  This is done in uipc_send(), divert_packet(), mroute
  socket_send(), raw_append(), tcp_reass(), tcp_input(), and udp_append().

- When the socket buffer lock is dropped before a sowakeup(), remove the
  explicit unlock and use the _locked() sowakeup() variant.  This is done
  in soisdisconnecting(), soisdisconnected() when setting the can't send/
  receive flags and dropping data, and in uipc_rcvd() which adjusting
  back-pressure on the sockets.

For UNIX domain sockets running mpsafe with a contention-intensive SMP
mysql benchmark, this results in a 1.6% query rate improvement due to
reduce mutex costs.

1 files changed, 5 insertions, 8 deletions

diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index e366c3c..efcea0c 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -169,13 +169,11 @@ soisdisconnecting(so)
 	SOCK_UNLOCK(so);
 	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	sorwakeup_locked(so);
 	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
-	SOCKBUF_UNLOCK(&so->so_snd);
+	sowwakeup_locked(so);
 	wakeup(&so->so_timeo);
-	sowwakeup(so);
-	sorwakeup(so);
 }
 
 void
@@ -188,20 +186,19 @@ soisdisconnected(so)
 	 * SOCKBUF_LOCK(&so->so_rcv) even though they are the same mutex to
 	 * avoid introducing the assumption  that they are the same.
 	 */
+	/* XXXRW: so_state locking? */
 	SOCK_LOCK(so);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= SS_ISDISCONNECTED;
 	SOCK_UNLOCK(so);
 	SOCKBUF_LOCK(&so->so_rcv);
 	so->so_rcv.sb_state |= SBS_CANTRCVMORE;
-	SOCKBUF_UNLOCK(&so->so_rcv);
+	sorwakeup_locked(so);
 	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sbdrop_locked(&so->so_snd, so->so_snd.sb_cc);
-	SOCKBUF_UNLOCK(&so->so_snd);
+	sowwakeup_locked(so);
 	wakeup(&so->so_timeo);
-	sowwakeup(so);
-	sorwakeup(so);
 }
 
 /*