Untangle the nfs send and receive queue locking a little. One lock

routine was [ab]used for two different things, and you couldn't tell from
the wait channel which one had wedged.
Catch a few things missing from NFS_NOSERVER.

4 files changed, 85 insertions, 61 deletions

diff --git a/sys/nfsserver/nfs.h b/sys/nfsserver/nfs.h
index 0ed90c0..bc15a7c 100644
--- a/sys/nfsserver/nfs.h
+++ b/sys/nfsserver/nfs.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs.h	8.4 (Berkeley) 5/1/95
- * $Id: nfs.h,v 1.43 1998/08/23 03:07:16 wollman Exp $
+ * $Id: nfs.h,v 1.44 1998/09/07 05:42:15 bde Exp $
  */
 
 #ifndef _NFS_NFS_H_
@@ -594,8 +594,10 @@ int	nfs_send __P((struct socket *, struct sockaddr *, struct mbuf *,
 int	nfs_rephead __P((int, struct nfsrv_descript *, struct nfssvc_sock *,
 			 int, int, u_quad_t *, struct mbuf **, struct mbuf **,
 			 caddr_t *));
-int	nfs_sndlock __P((int *, int *, struct nfsreq *));
-void	nfs_sndunlock __P((int *, int *));
+int	nfs_sndlock __P((struct nfsreq *));
+void	nfs_sndunlock __P((struct nfsreq *));
+int	nfs_slplock __P((struct nfssvc_sock *, int));
+void	nfs_slpunlock __P((struct nfssvc_sock *));
 int	nfs_disct __P((struct mbuf **, caddr_t *, int, int, caddr_t *));
 int	nfs_vinvalbuf __P((struct vnode *, int, struct ucred *, struct proc *,
 			   int));
diff --git a/sys/nfsserver/nfs_srvsock.c b/sys/nfsserver/nfs_srvsock.c
index 8c1b812..8c58d00 100644
--- a/sys/nfsserver/nfs_srvsock.c
+++ b/sys/nfsserver/nfs_srvsock.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs_socket.c	8.5 (Berkeley) 3/30/95
- * $Id: nfs_socket.c,v 1.48 1998/12/07 21:58:44 archie Exp $
+ * $Id: nfs_socket.c,v 1.49 1998/12/30 00:37:43 hoek Exp $
  */
 
 /*
@@ -137,7 +137,7 @@ struct callout_handle	nfs_timer_handle;
 
 static int	nfs_msg __P((struct proc *,char *,char *));
 static int	nfs_rcvlock __P((struct nfsreq *));
-static void	nfs_rcvunlock __P((int *flagp, int *statep));
+static void	nfs_rcvunlock __P((struct nfsreq *));
 static void	nfs_realign __P((struct mbuf *m, int hsiz));
 static int	nfs_receive __P((struct nfsreq *rep, struct sockaddr **aname,
 				 struct mbuf **mp));
@@ -233,7 +233,6 @@ nfs_connect(nmp, rep)
 			goto bad;
 		}
 	} else {
-		/* XXX should not use mbuf */
 		error = soconnect(so, nmp->nm_nam, p);
 		if (error)
 			goto bad;
@@ -393,7 +392,7 @@ nfs_safedisconnect(nmp)
 	dummyreq.r_nmp = nmp;
 	nfs_rcvlock(&dummyreq);
 	nfs_disconnect(nmp);
-	nfs_rcvunlock(&nmp->nm_flag, &nmp->nm_state);
+	nfs_rcvunlock(&dummyreq);
 }
 
 /*
@@ -519,8 +518,7 @@ nfs_receive(rep, aname, mp)
 	 * until we have an entire rpc request/reply.
 	 */
 	if (sotype != SOCK_DGRAM) {
-		error = nfs_sndlock(&rep->r_nmp->nm_flag, &rep->r_nmp->nm_state,
-				    rep);
+		error = nfs_sndlock(rep);
 		if (error)
 			return (error);
 tryagain:
@@ -534,16 +532,14 @@ tryagain:
 		 * mount point.
 		 */
 		if (rep->r_mrep || (rep->r_flags & R_SOFTTERM)) {
-			nfs_sndunlock(&rep->r_nmp->nm_flag,
-				      &rep->r_nmp->nm_state);
+			nfs_sndunlock(rep);
 			return (EINTR);
 		}
 		so = rep->r_nmp->nm_so;
 		if (!so) {
 			error = nfs_reconnect(rep);
 			if (error) {
-				nfs_sndunlock(&rep->r_nmp->nm_flag,
-					      &rep->r_nmp->nm_state);
+				nfs_sndunlock(rep);
 				return (error);
 			}
 			goto tryagain;
@@ -555,14 +551,13 @@ tryagain:
 			if (error) {
 				if (error == EINTR || error == ERESTART ||
 				    (error = nfs_reconnect(rep)) != 0) {
-					nfs_sndunlock(&rep->r_nmp->nm_flag,
-						      &rep->r_nmp->nm_state);
+					nfs_sndunlock(rep);
 					return (error);
 				}
 				goto tryagain;
 			}
 		}
-		nfs_sndunlock(&rep->r_nmp->nm_flag, &rep->r_nmp->nm_state);
+		nfs_sndunlock(rep);
 		if (sotype == SOCK_STREAM) {
 			aio.iov_base = (caddr_t) &len;
 			aio.iov_len = sizeof(u_int32_t);
@@ -669,15 +664,13 @@ errout:
 				    "receive error %d from nfs server %s\n",
 				    error,
 				 rep->r_nmp->nm_mountp->mnt_stat.f_mntfromname);
-			error = nfs_sndlock(&rep->r_nmp->nm_flag,
-					    &rep->r_nmp->nm_state, rep);
+			error = nfs_sndlock(rep);
 			if (!error)
 				error = nfs_reconnect(rep);
 			if (!error)
 				goto tryagain;
 			else
-				nfs_sndunlock(&rep->r_nmp->nm_flag,
-					&rep->r_nmp->nm_state);
+				nfs_sndunlock(rep);
 		}
 	} else {
 		if ((so = rep->r_nmp->nm_so) == NULL)
@@ -756,7 +749,7 @@ nfs_reply(myrep)
 		 * Get the next Rpc reply off the socket
 		 */
 		error = nfs_receive(myrep, &nam, &mrep);
-		nfs_rcvunlock(&nmp->nm_flag, &nmp->nm_state);
+		nfs_rcvunlock(myrep);
 		if (error) {
 
 			/*
@@ -1016,12 +1009,12 @@ tryagain:
 		nmp->nm_sent < nmp->nm_cwnd)) {
 		splx(s);
 		if (nmp->nm_soflags & PR_CONNREQUIRED)
-			error = nfs_sndlock(&nmp->nm_flag, &nmp->nm_state, rep);
+			error = nfs_sndlock(rep);
 		if (!error) {
 			m = m_copym(m, 0, M_COPYALL, M_WAIT);
 			error = nfs_send(nmp->nm_so, nmp->nm_nam, m, rep);
 			if (nmp->nm_soflags & PR_CONNREQUIRED)
-				nfs_sndunlock(&nmp->nm_flag, &nmp->nm_state);
+				nfs_sndunlock(rep);
 		}
 		if (!error && (rep->r_flags & R_MUSTRESEND) == 0) {
 			nmp->nm_sent += NFS_CWNDSCALE;
@@ -1497,11 +1490,10 @@ nfs_sigintr(nmp, rep, p)
  * in progress when a reconnect is necessary.
  */
 int
-nfs_sndlock(flagp, statep, rep)
-	register int *flagp;
-	register int *statep;
+nfs_sndlock(rep)
 	struct nfsreq *rep;
 {
+	register int *statep = &rep->r_nmp->nm_state;
 	struct proc *p;
 	int slpflag = 0, slptimeo = 0;
 
@@ -1515,7 +1507,7 @@ nfs_sndlock(flagp, statep, rep)
 		if (nfs_sigintr(rep->r_nmp, rep, p))
 			return (EINTR);
 		*statep |= NFSSTA_WANTSND;
-		(void) tsleep((caddr_t)flagp, slpflag | (PZERO - 1),
+		(void) tsleep((caddr_t)statep, slpflag | (PZERO - 1),
 			"nfsndlck", slptimeo);
 		if (slpflag == PCATCH) {
 			slpflag = 0;
@@ -1530,17 +1522,17 @@ nfs_sndlock(flagp, statep, rep)
  * Unlock the stream socket for others.
  */
 void
-nfs_sndunlock(flagp, statep)
-	register int *flagp;
-	register int *statep;
+nfs_sndunlock(rep)
+	struct nfsreq *rep;
 {
+	register int *statep = &rep->r_nmp->nm_state;
 
 	if ((*statep & NFSSTA_SNDLOCK) == 0)
 		panic("nfs sndunlock");
 	*statep &= ~NFSSTA_SNDLOCK;
 	if (*statep & NFSSTA_WANTSND) {
 		*statep &= ~NFSSTA_WANTSND;
-		wakeup((caddr_t)flagp);
+		wakeup((caddr_t)statep);
 	}
 }
 
@@ -1548,11 +1540,10 @@ static int
 nfs_rcvlock(rep)
 	register struct nfsreq *rep;
 {
-	register int *flagp = &rep->r_nmp->nm_flag;
 	register int *statep = &rep->r_nmp->nm_state;
 	int slpflag, slptimeo = 0;
 
-	if (*flagp & NFSMNT_INT)
+	if (rep->r_nmp->nm_flag & NFSMNT_INT)
 		slpflag = PCATCH;
 	else
 		slpflag = 0;
@@ -1560,7 +1551,7 @@ nfs_rcvlock(rep)
 		if (nfs_sigintr(rep->r_nmp, rep, rep->r_procp))
 			return (EINTR);
 		*statep |= NFSSTA_WANTRCV;
-		(void) tsleep((caddr_t)flagp, slpflag | (PZERO - 1), "nfsrcvlk",
+		(void) tsleep((caddr_t)statep, slpflag | (PZERO - 1), "nfsrcvlk",
 			slptimeo);
 		/*
 		 * If our reply was recieved while we were sleeping,
@@ -1583,17 +1574,17 @@ nfs_rcvlock(rep)
  * Unlock the stream socket for others.
  */
 static void
-nfs_rcvunlock(flagp, statep)
-	register int *flagp;
-	register int *statep;
+nfs_rcvunlock(rep)
+	register struct nfsreq *rep;
 {
+	register int *statep = &rep->r_nmp->nm_state;
 
 	if ((*statep & NFSSTA_RCVLOCK) == 0)
 		panic("nfs rcvunlock");
 	*statep &= ~NFSSTA_RCVLOCK;
 	if (*statep & NFSSTA_WANTRCV) {
 		*statep &= ~NFSSTA_WANTRCV;
-		wakeup((caddr_t)flagp);
+		wakeup((caddr_t)statep);
 	}
 }
 
diff --git a/sys/nfsserver/nfs_syscalls.c b/sys/nfsserver/nfs_syscalls.c
index c7c2a48..38b5af2 100644
--- a/sys/nfsserver/nfs_syscalls.c
+++ b/sys/nfsserver/nfs_syscalls.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs_syscalls.c	8.5 (Berkeley) 3/30/95
- * $Id: nfs_syscalls.c,v 1.46 1999/02/16 10:49:53 dfr Exp $
+ * $Id: nfs_syscalls.c,v 1.47 1999/02/18 09:19:41 dfr Exp $
  */
 
 #include <sys/param.h>
@@ -89,7 +89,9 @@ extern int nfsrvw_procrastinate_v3;
 struct nfssvc_sock *nfs_udpsock, *nfs_cltpsock;
 static int nuidhash_max = NFS_MAXUIDHASH;
 
+#ifndef NFS_NOSERVER
 static void	nfsrv_zapsock __P((struct nfssvc_sock *slp));
+#endif
 static int	nfssvc_iod __P((struct proc *));
 
 #define	TRUE	1
@@ -466,8 +468,6 @@ nfssvc_nfsd(nsd, argp, p)
 	register struct mbuf *m;
 	register int siz;
 	register struct nfssvc_sock *slp;
-	register struct socket *so;
-	register int *solockp;
 	struct nfsd *nfsd = nsd->nsd_nfsd;
 	struct nfsrv_descript *nd = NULL;
 	struct mbuf *mreq;
@@ -527,13 +527,10 @@ nfssvc_nfsd(nsd, argp, p)
 					nfsrv_zapsock(slp);
 				else if (slp->ns_flag & SLP_NEEDQ) {
 					slp->ns_flag &= ~SLP_NEEDQ;
-					(void) nfs_sndlock(&slp->ns_solock,
-						&slp->ns_solock,
-						(struct nfsreq *)0);
+					(void) nfs_slplock(slp, 1);
 					nfsrv_rcv(slp->ns_so, (caddr_t)slp,
 						M_WAIT);
-					nfs_sndunlock(&slp->ns_solock,
-						&slp->ns_solock);
+					nfs_slpunlock(slp);
 				}
 				error = nfsrv_dorec(slp, nfsd, &nd);
 				cur_usec = nfs_curusec();
@@ -561,12 +558,7 @@ nfssvc_nfsd(nsd, argp, p)
 			continue;
 		}
 		splx(s);
-		so = slp->ns_so;
-		sotype = so->so_type;
-		if (so->so_proto->pr_flags & PR_CONNREQUIRED)
-			solockp = &slp->ns_solock;
-		else
-			solockp = (int *)0;
+		sotype = slp->ns_so->so_type;
 		if (nd) {
 		    getmicrotime(&nd->nd_starttime);
 		    if (nd->nd_nam2)
@@ -692,11 +684,10 @@ nfssvc_nfsd(nsd, argp, p)
 				M_PREPEND(m, NFSX_UNSIGNED, M_WAIT);
 				*mtod(m, u_int32_t *) = htonl(0x80000000 | siz);
 			}
-			if (solockp)
-				(void) nfs_sndlock(solockp, solockp, 
-					(struct nfsreq *)0);
+			if (slp->ns_so->so_proto->pr_flags & PR_CONNREQUIRED)
+				(void) nfs_slplock(slp, 1);
 			if (slp->ns_flag & SLP_VALID)
-			    error = nfs_send(so, nd->nd_nam2, m, NULL);
+			    error = nfs_send(slp->ns_so, nd->nd_nam2, m, NULL);
 			else {
 			    error = EPIPE;
 			    m_freem(m);
@@ -709,8 +700,8 @@ nfssvc_nfsd(nsd, argp, p)
 				m_freem(nd->nd_mrep);
 			if (error == EPIPE)
 				nfsrv_zapsock(slp);
-			if (solockp)
-				nfs_sndunlock(solockp, solockp);
+			if (slp->ns_so->so_proto->pr_flags & PR_CONNREQUIRED)
+				nfs_slpunlock(slp);
 			if (error == EINTR || error == ERESTART) {
 				free((caddr_t)nd, M_NFSRVDESC);
 				nfsrv_slpderef(slp);
@@ -835,6 +826,44 @@ nfsrv_slpderef(slp)
 }
 
 /*
+ * Lock a socket against others.
+ */
+int
+nfs_slplock(slp, wait)
+	register struct nfssvc_sock *slp;
+	int wait;
+{
+	int *statep = &slp->ns_solock;
+
+	if (!wait && (*statep & NFSSTA_SNDLOCK))
+		return(0);	/* already locked, fail */
+	while (*statep & NFSSTA_SNDLOCK) {
+		*statep |= NFSSTA_WANTSND;
+		(void) tsleep((caddr_t)statep, PZERO - 1, "nfsslplck", 0);
+	}
+	*statep |= NFSSTA_SNDLOCK;
+	return (1);
+}
+
+/*
+ * Unlock the stream socket for others.
+ */
+void
+nfs_slpunlock(slp)
+	register struct nfssvc_sock *slp;
+{
+	int *statep = &slp->ns_solock;
+
+	if ((*statep & NFSSTA_SNDLOCK) == 0)
+		panic("nfs slpunlock");
+	*statep &= ~NFSSTA_SNDLOCK;
+	if (*statep & NFSSTA_WANTSND) {
+		*statep &= ~NFSSTA_WANTSND;
+		wakeup((caddr_t)statep);
+	}
+}
+
+/*
  * Initialize the data structures for the server.
  * Handshake with any new nfsds starting up to avoid any chance of
  * corruption.
diff --git a/sys/nfsserver/nfsrvstats.h b/sys/nfsserver/nfsrvstats.h
index 0ed90c0..bc15a7c 100644
--- a/sys/nfsserver/nfsrvstats.h
+++ b/sys/nfsserver/nfsrvstats.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs.h	8.4 (Berkeley) 5/1/95
- * $Id: nfs.h,v 1.43 1998/08/23 03:07:16 wollman Exp $
+ * $Id: nfs.h,v 1.44 1998/09/07 05:42:15 bde Exp $
  */
 
 #ifndef _NFS_NFS_H_
@@ -594,8 +594,10 @@ int	nfs_send __P((struct socket *, struct sockaddr *, struct mbuf *,
 int	nfs_rephead __P((int, struct nfsrv_descript *, struct nfssvc_sock *,
 			 int, int, u_quad_t *, struct mbuf **, struct mbuf **,
 			 caddr_t *));
-int	nfs_sndlock __P((int *, int *, struct nfsreq *));
-void	nfs_sndunlock __P((int *, int *));
+int	nfs_sndlock __P((struct nfsreq *));
+void	nfs_sndunlock __P((struct nfsreq *));
+int	nfs_slplock __P((struct nfssvc_sock *, int));
+void	nfs_slpunlock __P((struct nfssvc_sock *));
 int	nfs_disct __P((struct mbuf **, caddr_t *, int, int, caddr_t *));
 int	nfs_vinvalbuf __P((struct vnode *, int, struct ucred *, struct proc *,
 			   int));