Reduce the number of unnecessary unlock-relocks on socket buffer mutexes

associated with performing a wakeup on the socket buffer:

- When performing an sbappend*() followed by a so[rw]wakeup(), explicitly
  acquire the socket buffer lock and use the _locked() variants of both
  calls.  Note that the _locked() sowakeup() versions unlock the mutex on
  return.  This is done in uipc_send(), divert_packet(), mroute
  socket_send(), raw_append(), tcp_reass(), tcp_input(), and udp_append().

- When the socket buffer lock is dropped before a sowakeup(), remove the
  explicit unlock and use the _locked() sowakeup() variant.  This is done
  in soisdisconnecting(), soisdisconnected() when setting the can't send/
  receive flags and dropping data, and in uipc_rcvd() which adjusting
  back-pressure on the sockets.

For UNIX domain sockets running mpsafe with a contention-intensive SMP
mysql benchmark, this results in a 1.6% query rate improvement due to
reduce mutex costs.

1 files changed, 7 insertions, 2 deletions

diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 9c541db..0997f9a 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -447,6 +447,7 @@ udp_append(last, ip, n, off)
 	int off;
 {
 	struct sockaddr *append_sa;
+	struct socket *so;
 	struct mbuf *opts = 0;
 
 	INP_LOCK_ASSERT(last);
@@ -496,13 +497,17 @@ udp_append(last, ip, n, off)
 #endif
 	append_sa = (struct sockaddr *)&udp_in;
 	m_adj(n, off);
-	if (sbappendaddr(&last->inp_socket->so_rcv, append_sa, n, opts) == 0) {
+
+	so = last->inp_socket;
+	SOCKBUF_LOCK(&so->so_rcv);
+	if (sbappendaddr_locked(&so->so_rcv, append_sa, n, opts) == 0) {
 		m_freem(n);
 		if (opts)
 			m_freem(opts);
 		udpstat.udps_fullsock++;
+		SOCKBUF_UNLOCK(&so->so_rcv);
 	} else
-		sorwakeup(last->inp_socket);
+		sorwakeup_locked(so);
 }
 
 /*