Lock down a socket, milestone 1.

o Add a mutex (sb_mtx) to struct sockbuf. This protects the data in a
  socket buffer. The mutex in the receive buffer also protects the data
  in struct socket.

o Determine the lock strategy for each members in struct socket.

o Lock down the following members:

  - so_count
  - so_options
  - so_linger
  - so_state

o Remove *_locked() socket APIs.  Make the following socket APIs
  touching the members above now require a locked socket:

 - sodisconnect()
 - soisconnected()
 - soisconnecting()
 - soisdisconnected()
 - soisdisconnecting()
 - sofree()
 - soref()
 - sorele()
 - sorwakeup()
 - sotryfree()
 - sowakeup()
 - sowwakeup()

Reviewed by:	alfred

2 files changed, 49 insertions, 16 deletions

diff --git a/sys/fs/fifofs/fifo_vnops.c b/sys/fs/fifofs/fifo_vnops.c
index 36e8f7c..71ba03c 100644
--- a/sys/fs/fifofs/fifo_vnops.c
+++ b/sys/fs/fifofs/fifo_vnops.c
@@ -203,26 +203,32 @@ fifo_open(ap)
 		}
 		fip->fi_readers = fip->fi_writers = 0;
 		wso->so_snd.sb_lowat = PIPE_BUF;
+		SOCK_LOCK(rso);
 		rso->so_state |= SS_CANTRCVMORE;
+		SOCK_UNLOCK(rso);
 	}
 	if (ap->a_mode & FREAD) {
 		fip->fi_readers++;
 		if (fip->fi_readers == 1) {
+			SOCK_LOCK(fip->fi_writesock);
 			fip->fi_writesock->so_state &= ~SS_CANTSENDMORE;
 			if (fip->fi_writers > 0) {
 				wakeup((caddr_t)&fip->fi_writers);
 				sowwakeup(fip->fi_writesock);
 			}
+			SOCK_UNLOCK(fip->fi_writesock);
 		}
 	}
 	if (ap->a_mode & FWRITE) {
 		fip->fi_writers++;
 		if (fip->fi_writers == 1) {
+			SOCK_LOCK(fip->fi_readsock);
 			fip->fi_readsock->so_state &= ~SS_CANTRCVMORE;
 			if (fip->fi_readers > 0) {
 				wakeup((caddr_t)&fip->fi_readers);
 				sorwakeup(fip->fi_writesock);
 			}
+			SOCK_UNLOCK(fip->fi_readsock);
 		}
 	}
 	if ((ap->a_mode & FREAD) && (ap->a_mode & O_NONBLOCK) == 0) {
@@ -282,15 +288,21 @@ fifo_read(ap)
 #endif
 	if (uio->uio_resid == 0)
 		return (0);
-	if (ap->a_ioflag & IO_NDELAY)
+	if (ap->a_ioflag & IO_NDELAY) {
+		SOCK_LOCK(rso);
 		rso->so_state |= SS_NBIO;
+		SOCK_UNLOCK(rso);
+	}
 	startresid = uio->uio_resid;
 	VOP_UNLOCK(ap->a_vp, 0, td);
 	error = soreceive(rso, (struct sockaddr **)0, uio, (struct mbuf **)0,
 	    (struct mbuf **)0, (int *)0);
 	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY, td);
-	if (ap->a_ioflag & IO_NDELAY)
+	if (ap->a_ioflag & IO_NDELAY) {
+		SOCK_LOCK(rso);
 		rso->so_state &= ~SS_NBIO;
+		SOCK_UNLOCK(rso);
+	}
 	return (error);
 }
 
@@ -315,14 +327,20 @@ fifo_write(ap)
 	if (ap->a_uio->uio_rw != UIO_WRITE)
 		panic("fifo_write mode");
 #endif
-	if (ap->a_ioflag & IO_NDELAY)
+	if (ap->a_ioflag & IO_NDELAY) {
+		SOCK_LOCK(wso);
 		wso->so_state |= SS_NBIO;
+		SOCK_UNLOCK(wso);
+	}
 	VOP_UNLOCK(ap->a_vp, 0, td);
 	error = sosend(wso, (struct sockaddr *)0, ap->a_uio, 0,
 		       (struct mbuf *)0, 0, td);
 	vn_lock(ap->a_vp, LK_EXCLUSIVE | LK_RETRY, td);
-	if (ap->a_ioflag & IO_NDELAY)
+	if (ap->a_ioflag & IO_NDELAY) {
+		SOCK_LOCK(wso);
 		wso->so_state &= ~SS_NBIO;
+		SOCK_UNLOCK(wso);
+	}
 	return (error);
 }
 
@@ -412,10 +430,13 @@ filt_fiforead(struct knote *kn, long hint)
 	struct socket *so = (struct socket *)kn->kn_hook;
 
 	kn->kn_data = so->so_rcv.sb_cc;
+	SOCK_LOCK(so);
 	if (so->so_state & SS_CANTRCVMORE) {
+		SOCK_UNLOCK(so);
 		kn->kn_flags |= EV_EOF;
 		return (1);
 	}
+	SOCK_UNLOCK(so);
 	kn->kn_flags &= ~EV_EOF;
 	return (kn->kn_data > 0);
 }
@@ -436,10 +457,13 @@ filt_fifowrite(struct knote *kn, long hint)
 	struct socket *so = (struct socket *)kn->kn_hook;
 
 	kn->kn_data = sbspace(&so->so_snd);
+	SOCK_LOCK(so);
 	if (so->so_state & SS_CANTSENDMORE) {
+		SOCK_UNLOCK(so);
 		kn->kn_flags |= EV_EOF;
 		return (1);
 	}
+	SOCK_UNLOCK(so);
 	kn->kn_flags &= ~EV_EOF;
 	return (kn->kn_data >= so->so_snd.sb_lowat);
 }
diff --git a/sys/fs/portalfs/portal_vnops.c b/sys/fs/portalfs/portal_vnops.c
index b345c57..23b471f 100644
--- a/sys/fs/portalfs/portal_vnops.c
+++ b/sys/fs/portalfs/portal_vnops.c
@@ -43,24 +43,26 @@
  */
 
 #include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/sysproto.h>
-#include <sys/kernel.h>
-#include <sys/time.h>
-#include <sys/proc.h>
-#include <sys/filedesc.h>
-#include <sys/vnode.h>
 #include <sys/fcntl.h>
 #include <sys/file.h>
-#include <sys/stat.h>
-#include <sys/mount.h>
+#include <sys/filedesc.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
 #include <sys/malloc.h>
-#include <sys/namei.h>
 #include <sys/mbuf.h>
+#include <sys/mount.h>
+#include <sys/mutex.h>
+#include <sys/namei.h>
+#include <sys/proc.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
+#include <sys/stat.h>
+#include <sys/systm.h>
+#include <sys/sysproto.h>
+#include <sys/time.h>
 #include <sys/un.h>
 #include <sys/unpcb.h>
+#include <sys/vnode.h>
 #include <fs/portalfs/portal.h>
 
 static int portal_fileid = PORTAL_ROOTFILEID+1;
@@ -182,8 +184,12 @@ portal_connect(so, so2)
 	if (so->so_type != so2->so_type)
 		return (EPROTOTYPE);
 
-	if ((so2->so_options & SO_ACCEPTCONN) == 0)
+	SOCK_LOCK(so2);
+	if ((so2->so_options & SO_ACCEPTCONN) == 0) {
+		SOCK_UNLOCK(so2);
 		return (ECONNREFUSED);
+	}
+	SOCK_UNLOCK(so2);
 
 	if ((so3 = sonewconn(so2, 0)) == 0)
 		return (ECONNREFUSED);
@@ -280,14 +286,17 @@ portal_open(ap)
 	 * and keep polling the reference count.   XXX.
 	 */
 	s = splnet();
+	SOCK_LOCK(so);
 	while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
 		if (fmp->pm_server->f_count == 1) {
+			SOCK_UNLOCK(so);
 			error = ECONNREFUSED;
 			splx(s);
 			goto bad;
 		}
-		(void) tsleep((caddr_t) &so->so_timeo, PSOCK, "portalcon", 5 * hz);
+		(void) msleep((caddr_t) &so->so_timeo, SOCK_MTX(so), PSOCK, "portalcon", 5 * hz);
 	}
+	SOCK_UNLOCK(so);
 	splx(s);
 
 	if (so->so_error) {