Give struct socket structures a ref counting interface similar to

vnodes.  This will hopefully serve as a base from which we can
expand the MP code.  We currently do not attempt to obtain any
mutex or SX locks, but the door is open to add them when we nail
down exactly how that part of it is going to work.

23 files changed, 223 insertions, 182 deletions

diff --git a/sys/compat/svr4/svr4_stream.c b/sys/compat/svr4/svr4_stream.c
index cb19305..bf1c15a 100644
--- a/sys/compat/svr4/svr4_stream.c
+++ b/sys/compat/svr4/svr4_stream.c
@@ -150,7 +150,6 @@ svr4_sendit(td, s, mp, flags)
 	register struct msghdr *mp;
 	int flags;
 {
-	struct file *fp;
 	struct uio auio;
 	register struct iovec *iov;
 	register int i;
@@ -163,8 +162,7 @@ svr4_sendit(td, s, mp, flags)
 	struct uio ktruio;
 #endif
 
-	error = holdsock(td->td_proc->p_fd, s, &fp);
-	if (error)
+	if ((error = fgetsock(td, s, &so, NULL)) != 0)
 		return (error);
 	auio.uio_iov = mp->msg_iov;
 	auio.uio_iovcnt = mp->msg_iovlen;
@@ -176,16 +174,14 @@ svr4_sendit(td, s, mp, flags)
 	iov = mp->msg_iov;
 	for (i = 0; i < mp->msg_iovlen; i++, iov++) {
 		if ((auio.uio_resid += iov->iov_len) < 0) {
-			fdrop(fp, td);
-			return (EINVAL);
+			error = EINVAL;
+			goto done1;
 		}
 	}
 	if (mp->msg_name) {
 		error = getsockaddr(&to, mp->msg_name, mp->msg_namelen);
-		if (error) {
-			fdrop(fp, td);
-			return (error);
-		}
+		if (error)
+			goto done1;
 	} else {
 		to = 0;
 	}
@@ -211,7 +207,6 @@ svr4_sendit(td, s, mp, flags)
 	}
 #endif
 	len = auio.uio_resid;
-	so = (struct socket *)fp->f_data;
 	error = so->so_proto->pr_usrreqs->pru_sosend(so, to, &auio, 0, control,
 						     flags, td);
 	if (error) {
@@ -239,7 +234,8 @@ svr4_sendit(td, s, mp, flags)
 bad:
 	if (to)
 		FREE(to, M_SONAME);
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 	return (error);
 }
 
@@ -250,7 +246,6 @@ svr4_recvit(td, s, mp, namelenp)
 	register struct msghdr *mp;
 	caddr_t namelenp;
 {
-	struct file *fp;
 	struct uio auio;
 	register struct iovec *iov;
 	register int i;
@@ -264,8 +259,7 @@ svr4_recvit(td, s, mp, namelenp)
 	struct uio ktruio;
 #endif
 
-	error = holdsock(td->td_proc->p_fd, s, &fp);
-	if (error)
+	if ((error = fgetsock(td, s, &so, NULL)) != 0)
 		return (error);
 	auio.uio_iov = mp->msg_iov;
 	auio.uio_iovcnt = mp->msg_iovlen;
@@ -277,8 +271,8 @@ svr4_recvit(td, s, mp, namelenp)
 	iov = mp->msg_iov;
 	for (i = 0; i < mp->msg_iovlen; i++, iov++) {
 		if ((auio.uio_resid += iov->iov_len) < 0) {
-			fdrop(fp, td);
-			return (EINVAL);
+			error = EINVAL;
+			goto done1;
 		}
 	}
 #ifdef KTRACE
@@ -291,7 +285,6 @@ svr4_recvit(td, s, mp, namelenp)
 	}
 #endif
 	len = auio.uio_resid;
-	so = (struct socket *)fp->f_data;
 	error = so->so_proto->pr_usrreqs->pru_soreceive(so, &fromsa, &auio,
 	    (struct mbuf **)0, mp->msg_control ? &control : (struct mbuf **)0,
 	    &mp->msg_flags);
@@ -365,7 +358,8 @@ out:
 		FREE(fromsa, M_SONAME);
 	if (control)
 		m_freem(control);
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 	return (error);
 }
 
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 1d8d038..d5febf7 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -60,6 +60,8 @@
 #include <sys/unistd.h>
 #include <sys/resourcevar.h>
 #include <sys/event.h>
+#include <sys/sx.h>
+#include <sys/socketvar.h>
 
 #include <machine/limits.h>
 
@@ -1425,6 +1427,51 @@ fgetvp_write(struct thread *td, int fd, struct vnode **vpp)
 	return(_fgetvp(td, fd, vpp, FWRITE));
 }
 
+/*
+ * Like fget() but loads the underlying socket, or returns an error if
+ * the descriptor does not represent a socket.
+ *
+ * We bump the ref count on the returned socket.  XXX Also obtain the SX lock in
+ * the future.
+ */
+int
+fgetsock(struct thread *td, int fd, struct socket **spp, u_int *fflagp)
+{
+	struct filedesc *fdp;
+	struct file *fp;
+	struct socket *so;
+
+	GIANT_REQUIRED;
+	fdp = td->td_proc->p_fd;
+	*spp = NULL;
+	if (fflagp)
+		*fflagp = 0;
+	if ((u_int)fd >= fdp->fd_nfiles)
+		return(EBADF);
+	if ((fp = fdp->fd_ofiles[fd]) == NULL)
+		return(EBADF);
+	if (fp->f_type != DTYPE_SOCKET)
+		return(ENOTSOCK);
+	if (fp->f_data == NULL)
+		return(EINVAL);
+	so = (struct socket *)fp->f_data;
+	if (fflagp)
+		*fflagp = fp->f_flag;
+	soref(so);
+	*spp = so;
+	return(0);
+}
+
+/*
+ * Drop the reference count on the the socket and XXX release the SX lock in
+ * the future.  The last reference closes the socket.
+ */
+void
+fputsock(struct socket *so)
+{
+	sorele(so);
+}
+
 int
 fdrop(fp, td)
 	struct file *fp;
diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index 8822fcb..04a419a 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -182,6 +182,12 @@ soo_stat(fp, ub, td)
 	return ((*so->so_proto->pr_usrreqs->pru_sense)(so, ub));
 }
 
+/*
+ * API socket close on file pointer.  We call soclose() to close the 
+ * socket (including initiating closing protocols).  soclose() will
+ * sorele() the file reference but the actual socket will not go away
+ * until the socket's ref count hits 0.
+ */
 /* ARGSUSED */
 int
 soo_close(fp, td)
@@ -189,10 +195,12 @@ soo_close(fp, td)
 	struct thread *td;
 {
 	int error = 0;
+	struct socket *so;
 
 	fp->f_ops = &badfileops;
-	if (fp->f_data)
-		error = soclose((struct socket *)fp->f_data);
-	fp->f_data = 0;
+	if ((so = (struct socket *)fp->f_data) != NULL) {
+		fp->f_data = NULL;
+		error = soclose(so);
+	}
 	return (error);
 }
diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
index c969e7d..122fcda 100644
--- a/sys/kern/uipc_sockbuf.c
+++ b/sys/kern/uipc_sockbuf.c
@@ -210,6 +210,8 @@ sodropablereq(head)
  * then we allocate a new structure, propoerly linked into the
  * data structure of the original socket, and return this.
  * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED.
+ *
+ * note: the ref count on the socket is 0 on return
  */
 struct socket *
 sonewconn(head, connstatus)
@@ -246,7 +248,7 @@ sonewconn3(head, connstatus, td)
 		so->so_cred = crhold(head->so_cred);
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) ||
 	    (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
-		sodealloc(so);
+		sotryfree(so);
 		return ((struct socket *)0);
 	}
 
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 10c8585..caae630c 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -91,6 +91,10 @@ SYSCTL_DECL(_kern_ipc);
 static int somaxconn = SOMAXCONN;
 SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW,
     &somaxconn, 0, "Maximum pending socket connection queue size");
+static int numopensockets;
+SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD,
+    &numopensockets, 0, "Number of open sockets");
+
 
 /*
  * Socket operation routines.
@@ -106,6 +110,8 @@ SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW,
  * Note that it would probably be better to allocate socket
  * and PCB at the same time, but I'm not convinced that all
  * the protocols can be easily modified to do this.
+ *
+ * soalloc() returns a socket with a ref count of 0.
  */
 struct socket *
 soalloc(waitok)
@@ -119,11 +125,17 @@ soalloc(waitok)
 		bzero(so, sizeof *so);
 		so->so_gencnt = ++so_gencnt;
 		so->so_zone = socket_zone;
+		/* sx_init(&so->so_sxlock, "socket sxlock"); */
 		TAILQ_INIT(&so->so_aiojobq);
+		++numopensockets;
 	}
 	return so;
 }
 
+/*
+ * socreate returns a socket with a ref count of 1.  The socket should be
+ * closed with soclose().
+ */
 int
 socreate(dom, aso, type, proto, td)
 	int dom;
@@ -162,10 +174,11 @@ socreate(dom, aso, type, proto, td)
 	so->so_type = type;
 	so->so_cred = crhold(td->td_proc->p_ucred);
 	so->so_proto = prp;
+	soref(so);
 	error = (*prp->pr_usrreqs->pru_attach)(so, proto, td);
 	if (error) {
 		so->so_state |= SS_NOFDREF;
-		sofree(so);
+		sorele(so);
 		return (error);
 	}
 	*aso = so;
@@ -186,11 +199,11 @@ sobind(so, nam, td)
 	return (error);
 }
 
-void
-sodealloc(so)
-	struct socket *so;
+static void
+sodealloc(struct socket *so)
 {
 
+	KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count));
 	so->so_gencnt = ++so_gencnt;
 	if (so->so_rcv.sb_hiwat)
 		(void)chgsbsize(so->so_cred->cr_uidinfo,
@@ -210,7 +223,9 @@ sodealloc(so)
 	}
 #endif
 	crfree(so->so_cred);
+	/* sx_destroy(&so->so_sxlock); */
 	zfree(so->so_zone, so);
+	--numopensockets;
 }
 
 int
@@ -242,6 +257,8 @@ sofree(so)
 {
 	struct socket *head = so->so_head;
 
+	KASSERT(so->so_count == 0, ("socket %p so_count not 0", so));
+
 	if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0)
 		return;
 	if (head != NULL) {
@@ -272,6 +289,10 @@ sofree(so)
  * Close a socket on last file table reference removal.
  * Initiate disconnect if connected.
  * Free socket when disconnect complete.
+ *
+ * This function will sorele() the socket.  Note that soclose() may be
+ * called prior to the ref count reaching zero.  The actual socket
+ * structure will not be freed until the ref count reaches zero.
  */
 int
 soclose(so)
@@ -329,7 +350,7 @@ discard:
 	if (so->so_state & SS_NOFDREF)
 		panic("soclose: NOFDREF");
 	so->so_state |= SS_NOFDREF;
-	sofree(so);
+	sorele(so);
 	splx(s);
 	return (error);
 }
@@ -345,7 +366,7 @@ soabort(so)
 
 	error = (*so->so_proto->pr_usrreqs->pru_abort)(so);
 	if (error) {
-		sofree(so);
+		sotryfree(so);	/* note: does not decrement the ref count */
 		return error;
 	}
 	return (0);
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index c969e7d..122fcda 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -210,6 +210,8 @@ sodropablereq(head)
  * then we allocate a new structure, propoerly linked into the
  * data structure of the original socket, and return this.
  * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED.
+ *
+ * note: the ref count on the socket is 0 on return
  */
 struct socket *
 sonewconn(head, connstatus)
@@ -246,7 +248,7 @@ sonewconn3(head, connstatus, td)
 		so->so_cred = crhold(head->so_cred);
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) ||
 	    (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
-		sodealloc(so);
+		sotryfree(so);
 		return ((struct socket *)0);
 	}
 
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index a0ca7cc..ee68cfe 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -139,7 +139,7 @@ socket(td, uap)
 			fdrop(fp, td);
 		}
 	} else {
-		fp->f_data = (caddr_t)so;
+		fp->f_data = (caddr_t)so;	/* already has ref count */
 		fp->f_flag = FREAD|FWRITE;
 		fp->f_ops = &socketops;
 		fp->f_type = DTYPE_SOCKET;
@@ -164,22 +164,19 @@ bind(td, uap)
 		int	namelen;
 	} */ *uap;
 {
-	struct file *fp;
 	struct sockaddr *sa;
+	struct socket *sp;
 	int error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
-	if (error)
-		goto done2;
-	error = getsockaddr(&sa, uap->name, uap->namelen);
-	if (error) {
-		fdrop(fp, td);
+	if ((error = fgetsock(td, uap->s, &sp, NULL)) != 0)
 		goto done2;
-	}
-	error = sobind((struct socket *)fp->f_data, sa, td);
+	if ((error = getsockaddr(&sa, uap->name, uap->namelen)) != 0)
+		goto done1;
+	error = sobind(sp, sa, td);
 	FREE(sa, M_SONAME);
-	fdrop(fp, td);
+done1:
+	fputsock(sp);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -197,14 +194,13 @@ listen(td, uap)
 		int	backlog;
 	} */ *uap;
 {
-	struct file *fp;
+	struct socket *sp;
 	int error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
-	if (error == 0) {
-		error = solisten((struct socket *)fp->f_data, uap->backlog, td);
-		fdrop(fp, td);
+	if ((error = fgetsock(td, uap->s, &sp, NULL)) == 0) {
+		error = solisten(sp, uap->backlog, td);
+		fputsock(sp);
 	}
 	mtx_unlock(&Giant);
 	return(error);
@@ -225,13 +221,12 @@ accept1(td, uap, compat)
 	int compat;
 {
 	struct filedesc *fdp;
-	struct file *lfp = NULL;
 	struct file *nfp = NULL;
 	struct sockaddr *sa;
 	int namelen, error, s;
 	struct socket *head, *so;
 	int fd;
-	short fflag;		/* type must match fp->f_flag */
+	u_int fflag;
 
 	mtx_lock(&Giant);
 	fdp = td->td_proc->p_fd;
@@ -241,11 +236,10 @@ accept1(td, uap, compat)
 		if(error)
 			goto done2;
 	}
-	error = holdsock(fdp, uap->s, &lfp);
+	error = fgetsock(td, uap->s, &head, &fflag);
 	if (error)
 		goto done2;
 	s = splnet();
-	head = (struct socket *)lfp->f_data;
 	if ((head->so_options & SO_ACCEPTCONN) == 0) {
 		splx(s);
 		error = EINVAL;
@@ -286,7 +280,6 @@ accept1(td, uap, compat)
 	TAILQ_REMOVE(&head->so_comp, so, so_list);
 	head->so_qlen--;
 
-	fflag = lfp->f_flag;
 	error = falloc(td, &nfp, &fd);
 	if (error) {
 		/*
@@ -312,7 +305,8 @@ accept1(td, uap, compat)
 	if (head->so_sigio != NULL)
 		fsetown(fgetown(head->so_sigio), &so->so_sigio);
 
-	nfp->f_data = (caddr_t)so;
+	soref(so);			/* file descriptor reference */
+	nfp->f_data = (caddr_t)so;	/* nfp has ref count from falloc */
 	nfp->f_flag = fflag;
 	nfp->f_ops = &socketops;
 	nfp->f_type = DTYPE_SOCKET;
@@ -375,7 +369,7 @@ noconnection:
 done:
 	if (nfp != NULL)
 		fdrop(nfp, td);
-	fdrop(lfp, td);
+	fputsock(head);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -420,35 +414,31 @@ connect(td, uap)
 		int	namelen;
 	} */ *uap;
 {
-	struct file *fp;
-	register struct socket *so;
+	struct socket *so;
 	struct sockaddr *sa;
 	int error, s;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
-	if (error)
+	if ((error = fgetsock(td, uap->s, &so, NULL)) != 0)
 		goto done2;
-	so = (struct socket *)fp->f_data;
 	if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
 		error = EALREADY;
-		goto done;
+		goto done1;
 	}
 	error = getsockaddr(&sa, uap->name, uap->namelen);
 	if (error)
-		goto done;
+		goto done1;
 	error = soconnect(so, sa, td);
 	if (error)
 		goto bad;
 	if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
 		FREE(sa, M_SONAME);
 		error = EINPROGRESS;
-		goto done;
+		goto done1;
 	}
 	s = splnet();
 	while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
-		error = tsleep((caddr_t)&so->so_timeo, PSOCK | PCATCH,
-		    "connec", 0);
+		error = tsleep((caddr_t)&so->so_timeo, PSOCK | PCATCH, "connec", 0);
 		if (error)
 			break;
 	}
@@ -462,8 +452,8 @@ bad:
 	FREE(sa, M_SONAME);
 	if (error == ERESTART)
 		error = EINTR;
-done:
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -499,12 +489,12 @@ socketpair(td, uap)
 		goto free2;
 	fhold(fp1);
 	sv[0] = fd;
-	fp1->f_data = (caddr_t)so1;
+	fp1->f_data = (caddr_t)so1;	/* so1 already has ref count */
 	error = falloc(td, &fp2, &fd);
 	if (error)
 		goto free3;
 	fhold(fp2);
-	fp2->f_data = (caddr_t)so2;
+	fp2->f_data = (caddr_t)so2;	/* so2 already has ref count */
 	sv[1] = fd;
 	error = soconnect2(so1, so2);
 	if (error)
@@ -552,12 +542,11 @@ sendit(td, s, mp, flags)
 	register struct msghdr *mp;
 	int flags;
 {
-	struct file *fp;
 	struct uio auio;
 	register struct iovec *iov;
 	register int i;
 	struct mbuf *control;
-	struct sockaddr *to;
+	struct sockaddr *to = NULL;
 	int len, error;
 	struct socket *so;
 #ifdef KTRACE
@@ -565,8 +554,7 @@ sendit(td, s, mp, flags)
 	struct uio ktruio;
 #endif
 
-	error = holdsock(td->td_proc->p_fd, s, &fp);
-	if (error)
+	if ((error = fgetsock(td, s, &so, NULL)) != 0)
 		return (error);
 	auio.uio_iov = mp->msg_iov;
 	auio.uio_iovcnt = mp->msg_iovlen;
@@ -578,18 +566,14 @@ sendit(td, s, mp, flags)
 	iov = mp->msg_iov;
 	for (i = 0; i < mp->msg_iovlen; i++, iov++) {
 		if ((auio.uio_resid += iov->iov_len) < 0) {
-			fdrop(fp, td);
-			return (EINVAL);
+			error = EINVAL;
+			goto bad;
 		}
 	}
 	if (mp->msg_name) {
 		error = getsockaddr(&to, mp->msg_name, mp->msg_namelen);
-		if (error) {
-			fdrop(fp, td);
-			return (error);
-		}
-	} else {
-		to = 0;
+		if (error)
+			goto bad;
 	}
 	if (mp->msg_control) {
 		if (mp->msg_controllen < sizeof(struct cmsghdr)
@@ -633,7 +617,6 @@ sendit(td, s, mp, flags)
 	}
 #endif
 	len = auio.uio_resid;
-	so = (struct socket *)fp->f_data;
 	error = so->so_proto->pr_usrreqs->pru_sosend(so, to, &auio, 0, control,
 						     flags, td);
 	if (error) {
@@ -659,7 +642,7 @@ sendit(td, s, mp, flags)
 	}
 #endif
 bad:
-	fdrop(fp, td);
+	fputsock(so);
 	if (to)
 		FREE(to, M_SONAME);
 	return (error);
@@ -834,7 +817,6 @@ recvit(td, s, mp, namelenp)
 	register struct msghdr *mp;
 	caddr_t namelenp;
 {
-	struct file *fp;
 	struct uio auio;
 	register struct iovec *iov;
 	register int i;
@@ -848,8 +830,7 @@ recvit(td, s, mp, namelenp)
 	struct uio ktruio;
 #endif
 
-	error = holdsock(td->td_proc->p_fd, s, &fp);
-	if (error)
+	if ((error = fgetsock(td, s, &so, NULL)) != 0)
 		return (error);
 	auio.uio_iov = mp->msg_iov;
 	auio.uio_iovcnt = mp->msg_iovlen;
@@ -861,7 +842,7 @@ recvit(td, s, mp, namelenp)
 	iov = mp->msg_iov;
 	for (i = 0; i < mp->msg_iovlen; i++, iov++) {
 		if ((auio.uio_resid += iov->iov_len) < 0) {
-			fdrop(fp, td);
+			fputsock(so);
 			return (EINVAL);
 		}
 	}
@@ -875,7 +856,6 @@ recvit(td, s, mp, namelenp)
 	}
 #endif
 	len = auio.uio_resid;
-	so = (struct socket *)fp->f_data;
 	error = so->so_proto->pr_usrreqs->pru_soreceive(so, &fromsa, &auio,
 	    (struct mbuf **)0, mp->msg_control ? &control : (struct mbuf **)0,
 	    &mp->msg_flags);
@@ -975,7 +955,7 @@ recvit(td, s, mp, namelenp)
 		mp->msg_controllen = ctlbuf - (caddr_t)mp->msg_control;
 	}
 out:
-	fdrop(fp, td);
+	fputsock(so);
 	if (fromsa)
 		FREE(fromsa, M_SONAME);
 	if (control)
@@ -1196,14 +1176,13 @@ shutdown(td, uap)
 		int	how;
 	} */ *uap;
 {
-	struct file *fp;
+	struct socket *so;
 	int error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
-	if (error == 0) {
-		error = soshutdown((struct socket *)fp->f_data, uap->how);
-		fdrop(fp, td);
+	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
+		error = soshutdown(so, uap->how);
+		fputsock(so);
 	}
 	mtx_unlock(&Giant);
 	return(error);
@@ -1224,7 +1203,7 @@ setsockopt(td, uap)
 		int	valsize;
 	} */ *uap;
 {
-	struct file *fp;
+	struct socket *so;
 	struct sockopt sopt;
 	int error;
 
@@ -1234,16 +1213,15 @@ setsockopt(td, uap)
 		return (EINVAL);
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
-	if (error == 0) {
+	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
 		sopt.sopt_dir = SOPT_SET;
 		sopt.sopt_level = uap->level;
 		sopt.sopt_name = uap->name;
 		sopt.sopt_val = uap->val;
 		sopt.sopt_valsize = uap->valsize;
 		sopt.sopt_td = td;
-		error = sosetopt((struct socket *)fp->f_data, &sopt);
-		fdrop(fp, td);
+		error = sosetopt(so, &sopt);
+		fputsock(so);
 	}
 	mtx_unlock(&Giant);
 	return(error);
@@ -1265,24 +1243,20 @@ getsockopt(td, uap)
 	} */ *uap;
 {
 	int	valsize, error;
-	struct	file *fp;
+	struct  socket *so;
 	struct	sockopt sopt;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
-	if (error)
+	if ((error = fgetsock(td, uap->s, &so, NULL)) != 0)
 		goto done2;
 	if (uap->val) {
 		error = copyin((caddr_t)uap->avalsize, (caddr_t)&valsize,
 		    sizeof (valsize));
-		if (error) {
-			fdrop(fp, td);
-			goto done2;
-		}
+		if (error)
+			goto done1;
 		if (valsize < 0) {
-			fdrop(fp, td);
 			error = EINVAL;
-			goto done2;
+			goto done1;
 		}
 	} else {
 		valsize = 0;
@@ -1295,13 +1269,14 @@ getsockopt(td, uap)
 	sopt.sopt_valsize = (size_t)valsize; /* checked non-negative above */
 	sopt.sopt_td = td;
 
-	error = sogetopt((struct socket *)fp->f_data, &sopt);
+	error = sogetopt(so, &sopt);
 	if (error == 0) {
 		valsize = sopt.sopt_valsize;
 		error = copyout((caddr_t)&valsize,
 				(caddr_t)uap->avalsize, sizeof (valsize));
 	}
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -1323,21 +1298,16 @@ getsockname1(td, uap, compat)
 	} */ *uap;
 	int compat;
 {
-	struct file *fp;
-	register struct socket *so;
+	struct socket *so;
 	struct sockaddr *sa;
 	int len, error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->fdes, &fp);
-	if (error)
+	if ((error = fgetsock(td, uap->fdes, &so, NULL)) != 0)
 		goto done2;
 	error = copyin((caddr_t)uap->alen, (caddr_t)&len, sizeof (len));
-	if (error) {
-		fdrop(fp, td);
-		goto done2;
-	}
-	so = (struct socket *)fp->f_data;
+	if (error)
+		goto done1;
 	sa = 0;
 	error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &sa);
 	if (error)
@@ -1360,7 +1330,8 @@ gotnothing:
 bad:
 	if (sa)
 		FREE(sa, M_SONAME);
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -1408,26 +1379,20 @@ getpeername1(td, uap, compat)
 	} */ *uap;
 	int compat;
 {
-	struct file *fp;
-	register struct socket *so;
+	struct socket *so;
 	struct sockaddr *sa;
 	int len, error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->fdes, &fp);
-	if (error)
+	if ((error = fgetsock(td, uap->fdes, &so, NULL)) != 0)
 		goto done2;
-	so = (struct socket *)fp->f_data;
 	if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
-		fdrop(fp, td);
 		error = ENOTCONN;
-		goto done2;
+		goto done1;
 	}
 	error = copyin((caddr_t)uap->alen, (caddr_t)&len, sizeof (len));
-	if (error) {
-		fdrop(fp, td);
-		goto done2;
-	}
+	if (error)
+		goto done1;
 	sa = 0;
 	error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, &sa);
 	if (error)
@@ -1450,7 +1415,8 @@ gotnothing:
 bad:
 	if (sa)
 		FREE(sa, M_SONAME);
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -1550,33 +1516,6 @@ getsockaddr(namp, uaddr, len)
 }
 
 /*
- * holdsock() - load the struct file pointer associated
- * with a socket into *fpp.  If an error occurs, non-zero
- * will be returned and *fpp will be set to NULL.
- */
-int
-holdsock(fdp, fdes, fpp)
-	struct filedesc *fdp;
-	int fdes;
-	struct file **fpp;
-{
-	register struct file *fp = NULL;
-	int error = 0;
-
-	if ((unsigned)fdes >= fdp->fd_nfiles ||
-	    (fp = fdp->fd_ofiles[fdes]) == NULL) {
-		error = EBADF;
-	} else if (fp->f_type != DTYPE_SOCKET) {
-		error = ENOTSOCK;
-		fp = NULL;
-	} else {
-		fhold(fp);
-	}
-	*fpp = fp;
-	return(error);
-}
-
-/*
  * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
  * XXX - The sf_buf functions are currently private to sendfile(2), so have
  * been made static, but may be useful in the future for doing zero-copy in
@@ -1678,10 +1617,9 @@ sf_buf_free(caddr_t addr, void *args)
 int
 sendfile(struct thread *td, struct sendfile_args *uap)
 {
-	struct file *fp = NULL;
 	struct vnode *vp;
 	struct vm_object *obj;
-	struct socket *so;
+	struct socket *so = NULL;
 	struct mbuf *m;
 	struct sf_buf *sf;
 	struct vm_page *pg;
@@ -1701,10 +1639,8 @@ sendfile(struct thread *td, struct sendfile_args *uap)
 		error = EINVAL;
 		goto done;
 	}
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
-	if (error)
+	if ((error = fgetsock(td, uap->s, &so, NULL)) != 0)
 		goto done;
-	so = (struct socket *)fp->f_data;
 	if (so->so_type != SOCK_STREAM) {
 		error = EINVAL;
 		goto done;
@@ -1988,8 +1924,9 @@ done:
 	}
 	if (vp)
 		vrele(vp);
-	if (fp)
-		fdrop(fp, td);
+	if (so)
+		fputsock(so);
 	mtx_unlock(&Giant);
 	return (error);
 }
+
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index b8b7044..0cd408f 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -935,7 +935,7 @@ unp_drop(unp, errno)
 		if (unp->unp_addr)
 			FREE(unp->unp_addr, M_SONAME);
 		zfree(unp_zone, unp);
-		sofree(so);
+		sotryfree(so);
 	}
 }
 
diff --git a/sys/net/raw_cb.c b/sys/net/raw_cb.c
index 97dbc10..0dac2e5 100644
--- a/sys/net/raw_cb.c
+++ b/sys/net/raw_cb.c
@@ -97,7 +97,7 @@ raw_detach(rp)
 	struct socket *so = rp->rcb_socket;
 
 	so->so_pcb = 0;
-	sofree(so);
+	sotryfree(so);
 	LIST_REMOVE(rp, list);
 #ifdef notdef
 	if (rp->rcb_laddr)
diff --git a/sys/net/raw_usrreq.c b/sys/net/raw_usrreq.c
index 555bd3f..3138b10 100644
--- a/sys/net/raw_usrreq.c
+++ b/sys/net/raw_usrreq.c
@@ -142,8 +142,8 @@ raw_uabort(struct socket *so)
 	if (rp == 0)
 		return EINVAL;
 	raw_disconnect(rp);
-	sofree(so);
-	soisdisconnected(so);
+	sotryfree(so);
+	soisdisconnected(so);	/* XXX huh? called after the sofree()? */
 	return 0;
 }
 
diff --git a/sys/netatalk/ddp_pcb.c b/sys/netatalk/ddp_pcb.c
index 2dbf0f9..7c5f857 100644
--- a/sys/netatalk/ddp_pcb.c
+++ b/sys/netatalk/ddp_pcb.c
@@ -441,7 +441,7 @@ at_pcbdetach( struct socket *so, struct ddpcb *ddp)
 {
     soisdisconnected( so );
     so->so_pcb = 0;
-    sofree( so );
+    sotryfree(so);
 
     /* remove ddp from ddp_ports list */
     if ( ddp->ddp_lsat.sat_port != ATADDR_ANYPORT &&
diff --git a/sys/netatalk/ddp_usrreq.c b/sys/netatalk/ddp_usrreq.c
index 2dbf0f9..7c5f857 100644
--- a/sys/netatalk/ddp_usrreq.c
+++ b/sys/netatalk/ddp_usrreq.c
@@ -441,7 +441,7 @@ at_pcbdetach( struct socket *so, struct ddpcb *ddp)
 {
     soisdisconnected( so );
     so->so_pcb = 0;
-    sofree( so );
+    sotryfree(so);
 
     /* remove ddp from ddp_ports list */
     if ( ddp->ddp_lsat.sat_port != ATADDR_ANYPORT &&
diff --git a/sys/netatm/atm_socket.c b/sys/netatm/atm_socket.c
index 6d87791..534fec5 100644
--- a/sys/netatm/atm_socket.c
+++ b/sys/netatm/atm_socket.c
@@ -176,7 +176,7 @@ atm_sock_detach(so)
 	 * Break links and free control blocks
 	 */
 	so->so_pcb = NULL;
-	sofree(so);
+	sotryfree(so);
 
 	atm_free((caddr_t)atp);
 
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 4ee4b0d..4fecd6a 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -563,7 +563,7 @@ in_pcbdetach(inp)
 	inp->inp_gencnt = ++ipi->ipi_gencnt;
 	in_pcbremlists(inp);
 	so->so_pcb = 0;
-	sofree(so);
+	sotryfree(so);
 	if (inp->inp_options)
 		(void)m_free(inp->inp_options);
 	if (rt) {
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index a82ab84..6612848 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -606,7 +606,7 @@ in6_pcbdetach(inp)
 	inp->inp_gencnt = ++ipi->ipi_gencnt;
 	in_pcbremlists(inp);
 	sotoinpcb(so) = 0;
-	sofree(so);
+	sotryfree(so);
 
 	if (inp->in6p_options)
 		m_freem(inp->in6p_options);
diff --git a/sys/netipx/ipx_pcb.c b/sys/netipx/ipx_pcb.c
index fc0c6dd..ea038fe 100644
--- a/sys/netipx/ipx_pcb.c
+++ b/sys/netipx/ipx_pcb.c
@@ -268,7 +268,7 @@ ipx_pcbdetach(ipxp)
 	struct socket *so = ipxp->ipxp_socket;
 
 	so->so_pcb = 0;
-	sofree(so);
+	sotryfree(so);
 	if (ipxp->ipxp_route.ro_rt != NULL)
 		rtfree(ipxp->ipxp_route.ro_rt);
 	remque(ipxp);
diff --git a/sys/netipx/ipx_usrreq.c b/sys/netipx/ipx_usrreq.c
index fdd6917..7cddcfc 100644
--- a/sys/netipx/ipx_usrreq.c
+++ b/sys/netipx/ipx_usrreq.c
@@ -426,7 +426,7 @@ ipx_usr_abort(so)
 	s = splnet();
 	ipx_pcbdetach(ipxp);
 	splx(s);
-	sofree(so);
+	sotryfree(so);
 	soisdisconnected(so);
 	return (0);
 }
diff --git a/sys/netnatm/natm.c b/sys/netnatm/natm.c
index 257aa76..af80e11 100644
--- a/sys/netnatm/natm.c
+++ b/sys/netnatm/natm.c
@@ -133,7 +133,7 @@ natm_usr_detach(struct socket *so)
      */
     npcb_free(npcb, NPCB_DESTROY);	/* drain */
     so->so_pcb = NULL;
-    sofree(so);
+    sotryfree(so);
  out:
     splx(s);
     return (error);
@@ -481,7 +481,7 @@ struct proc *p;
 
       npcb_free(npcb, NPCB_DESTROY);	/* drain */
       so->so_pcb = NULL;
-      sofree(so);
+      sotryfree(so);
 
       break;
 
diff --git a/sys/netns/idp_usrreq.c b/sys/netns/idp_usrreq.c
index a03770f0..5203961 100644
--- a/sys/netns/idp_usrreq.c
+++ b/sys/netns/idp_usrreq.c
@@ -491,8 +491,8 @@ idp_usrreq(so, req, m, nam, control)
 
 	case PRU_ABORT:
 		ns_pcbdetach(nsp);
-		sofree(so);
-		soisdisconnected(so);
+		sotryfree(so);
+		soisdisconnected(so);	/* XXX huh, called after sofree()? */
 		break;
 
 	case PRU_SOCKADDR:
diff --git a/sys/netns/ns_pcb.c b/sys/netns/ns_pcb.c
index 85f7548..01301c0 100644
--- a/sys/netns/ns_pcb.c
+++ b/sys/netns/ns_pcb.c
@@ -232,7 +232,7 @@ ns_pcbdetach(nsp)
 	struct socket *so = nsp->nsp_socket;
 
 	so->so_pcb = 0;
-	sofree(so);
+	sotryfree(so);
 	if (nsp->nsp_route.ro_rt)
 		rtfree(nsp->nsp_route.ro_rt);
 	remque(nsp);
diff --git a/sys/nfsserver/nfs_syscalls.c b/sys/nfsserver/nfs_syscalls.c
index 17f90f7..73935f8 100644
--- a/sys/nfsserver/nfs_syscalls.c
+++ b/sys/nfsserver/nfs_syscalls.c
@@ -143,9 +143,12 @@ nfssvc(struct thread *td, struct nfssvc_args *uap)
 		error = copyin(uap->argp, (caddr_t)&nfsdarg, sizeof(nfsdarg));
 		if (error)
 			goto done2;
-		error = holdsock(td->td_proc->p_fd, nfsdarg.sock, &fp);
-		if (error)
+		if ((error = fget(td, nfsdarg.sock, &fp)) != 0)
+			goto done2;
+		if (fp->f_type != DTYPE_SOCKET) {
+			fdrop(fp, td);
 			goto done2;
+		}
 		/*
 		 * Get the client address for connected sockets.
 		 */
diff --git a/sys/sys/file.h b/sys/sys/file.h
index 589bffd..f552167 100644
--- a/sys/sys/file.h
+++ b/sys/sys/file.h
@@ -50,6 +50,7 @@ struct thread;
 struct uio;
 struct knote;
 struct vnode;
+struct socket;
 
 /*
  * Kernel descriptor table.
@@ -119,6 +120,9 @@ int fgetvp __P((struct thread *td, int fd, struct vnode **vpp));
 int fgetvp_read __P((struct thread *td, int fd, struct vnode **vpp));
 int fgetvp_write __P((struct thread *td, int fd, struct vnode **vpp));
 
+int fgetsock __P((struct thread *td, int fd, struct socket **spp, u_int *fflagp));
+void fputsock __P((struct socket *sp));
+
 static __inline void
 fhold(fp)
 	struct file *fp;
diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index ec22fa7..f868dcd 100644
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -38,6 +38,7 @@
 #define _SYS_SOCKETVAR_H_
 
 #include <sys/queue.h>			/* for TAILQ macros */
+#include <sys/sx.h>			/* SX locks */
 #include <sys/selinfo.h>		/* for struct selinfo */
 
 /*
@@ -52,6 +53,7 @@ struct accept_filter;
 
 struct socket {
 	struct	vm_zone *so_zone;	/* zone we were allocated from */
+	int	so_count;		/* reference count */
 	short	so_type;		/* generic type, see socket.h */
 	short	so_options;		/* from socket call, see socket.h */
 	short	so_linger;		/* time to linger while closing */
@@ -244,6 +246,28 @@ struct xsocket {
 	} \
 }
 
+/*
+ * soref()/sorele() ref-count the socket structure.  Note that you must
+ * still explicitly close the socket, but the last ref count will free
+ * the structure.
+ */
+
+#define soref(so)	do {			\
+				++so->so_count; \
+			} while (0)
+
+#define sorele(so)	do {				\
+				if (so->so_count <= 0)	\
+					panic("sorele");\
+				if (--so->so_count == 0)\
+					sofree(so);	\
+			} while (0)
+
+#define sotryfree(so)	do {				\
+				if (so->so_count == 0)	\
+					sofree(so);	\
+			} while(0)
+
 #define	sorwakeup(so)	do { \
 			  if (sb_notify(&(so)->so_rcv)) \
 			    sowakeup((so), &(so)->so_rcv); \
@@ -360,7 +384,6 @@ int	soconnect __P((struct socket *so, struct sockaddr *nam, struct thread *td));
 int	soconnect2 __P((struct socket *so1, struct socket *so2));
 int	socreate __P((int dom, struct socket **aso, int type, int proto,
 	    struct thread *td));
-void	sodealloc __P((struct socket *so));
 int	sodisconnect __P((struct socket *so));
 void	sofree __P((struct socket *so));
 int	sogetopt __P((struct socket *so, struct sockopt *sopt));