Fix a nasty hang connected with write gathering. Also add debug print

statements to bits of the server which helped me find the hang.

12 files changed, 156 insertions, 44 deletions

diff --git a/sys/nfs/nfs.h b/sys/nfs/nfs.h
index 59033bf..a3c45bd 100644
--- a/sys/nfs/nfs.h
+++ b/sys/nfs/nfs.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs.h	8.4 (Berkeley) 5/1/95
- * $Id: nfs.h,v 1.23 1997/03/27 20:01:03 guido Exp $
+ * $Id: nfs.h,v 1.24 1997/03/29 12:34:33 bde Exp $
  */
 
 #ifndef _NFS_NFS_H_
@@ -517,6 +517,23 @@ extern int nfsd_head_flag;
  	 !bcmp((caddr_t)&(o)->nd_cr, (caddr_t)&(n)->nd_cr, \
 		sizeof (struct ucred)))
 
+#ifdef NFS_DEBUG
+
+extern int nfs_debug;
+#define NFS_DEBUG_ASYNCIO	1 /* asynchronous i/o */
+#define NFS_DEBUG_WG		2 /* server write gathering */
+#define NFS_DEBUG_RC		4 /* server request caching */
+
+#define NFS_DPF(cat, args)					\
+	do {							\
+		if (nfs_debug & NFS_DEBUG_##cat) printf args;	\
+	} while (0)
+
+#else
+
+#define NFS_DPF(cat, args)
+
+#endif
 
 int	nfs_init __P((struct vfsconf *vfsp));
 int	nfs_reply __P((struct nfsreq *));
diff --git a/sys/nfs/nfs_serv.c b/sys/nfs/nfs_serv.c
index a0d5136..7946328 100644
--- a/sys/nfs/nfs_serv.c
+++ b/sys/nfs/nfs_serv.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs_serv.c	8.3 (Berkeley) 1/12/94
- * $Id: nfs_serv.c,v 1.39 1997/03/25 05:13:40 peter Exp $
+ * $Id: nfs_serv.c,v 1.40 1997/03/29 12:40:18 bde Exp $
  */
 
 /*
@@ -999,6 +999,7 @@ nfsmout:
 		owp = wp;
 		wp = wp->nd_tq.le_next;
 	    }
+	    NFS_DPF(WG, ("Q%03x", nfsd->nd_retxid & 0xfff));
 	    if (owp) {
 		LIST_INSERT_AFTER(owp, nfsd, nd_tq);
 	    } else {
@@ -1050,6 +1051,7 @@ loop1:
 		    break;
 		if (nfsd->nd_mreq)
 		    continue;
+		NFS_DPF(WG, ("P%03x", nfsd->nd_retxid & 0xfff));
 		LIST_REMOVE(nfsd, nd_tq);
 		LIST_REMOVE(nfsd, nd_hash);
 		splx(s);
@@ -1126,6 +1128,7 @@ loop1:
 		 */
 		swp = nfsd;
 		do {
+		    NFS_DPF(WG, ("R%03x", nfsd->nd_retxid & 0xfff));
 		    if (error) {
 			nfsm_writereply(NFSX_WCCDATA(v3), v3);
 			if (v3) {
@@ -1185,6 +1188,7 @@ loop1:
 	s = splsoftclock();
 	for (nfsd = slp->ns_tq.lh_first; nfsd; nfsd = nfsd->nd_tq.le_next)
 		if (nfsd->nd_mreq) {
+		    NFS_DPF(WG, ("X%03x", nfsd->nd_retxid & 0xfff));
 		    LIST_REMOVE(nfsd, nd_tq);
 		    *mrq = nfsd->nd_mreq;
 		    *ndp = nfsd;
@@ -1209,7 +1213,10 @@ nfsrvw_coalesce(owp, nfsd)
 {
         register int overlap;
         register struct mbuf *mp;
+	struct nfsrv_descript *p;
 
+	NFS_DPF(WG, ("C%03x-%03x",
+		     nfsd->nd_retxid & 0xfff, owp->nd_retxid & 0xfff));
         LIST_REMOVE(nfsd, nd_hash);
         LIST_REMOVE(nfsd, nd_tq);
         if (owp->nd_eoff < nfsd->nd_eoff) {
@@ -1232,6 +1239,16 @@ nfsrvw_coalesce(owp, nfsd)
             owp->nd_stable == NFSV3WRITE_UNSTABLE)
             owp->nd_stable = NFSV3WRITE_DATASYNC;
         LIST_INSERT_HEAD(&owp->nd_coalesce, nfsd, nd_tq);
+
+	/*
+	 * If nfsd had anything else coalesced into it, transfer them
+	 * to owp, otherwise their replies will never get sent.
+	 */
+	for (p = nfsd->nd_coalesce.lh_first; p;
+	     p = nfsd->nd_coalesce.lh_first) {
+	    LIST_REMOVE(p, nd_tq);
+	    LIST_INSERT_HEAD(&owp->nd_coalesce, p, nd_tq);
+	}
 }
 
 /*
diff --git a/sys/nfs/nfs_srvcache.c b/sys/nfs/nfs_srvcache.c
index 1555cc3..6121db0 100644
--- a/sys/nfs/nfs_srvcache.c
+++ b/sys/nfs/nfs_srvcache.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs_srvcache.c	8.3 (Berkeley) 3/30/95
- * $Id$
+ * $Id: nfs_srvcache.c,v 1.11 1997/02/22 09:42:40 peter Exp $
  */
 
 #ifndef NFS_NOSERVER 
@@ -184,6 +184,7 @@ loop:
 	    rp = rp->rc_hash.le_next) {
 	    if (nd->nd_retxid == rp->rc_xid && nd->nd_procnum == rp->rc_proc &&
 		netaddr_match(NETFAMILY(rp), &rp->rc_haddr, nd->nd_nam)) {
+		        NFS_DPF(RC, ("H%03x", rp->rc_xid & 0xfff));
 			if ((rp->rc_flag & RC_LOCKED) != 0) {
 				rp->rc_flag |= RC_WANTED;
 				(void) tsleep((caddr_t)rp, PZERO-1, "nfsrc", 0);
@@ -224,6 +225,7 @@ loop:
 		}
 	}
 	nfsstats.srvcache_misses++;
+	NFS_DPF(RC, ("M%03x", nd->nd_retxid & 0xfff));
 	if (numnfsrvcache < desirednfsrvcache) {
 		rp = (struct nfsrvcache *)malloc((u_long)sizeof *rp,
 		    M_NFSD, M_WAITOK);
@@ -289,6 +291,7 @@ loop:
 	    rp = rp->rc_hash.le_next) {
 	    if (nd->nd_retxid == rp->rc_xid && nd->nd_procnum == rp->rc_proc &&
 		netaddr_match(NETFAMILY(rp), &rp->rc_haddr, nd->nd_nam)) {
+			NFS_DPF(RC, ("U%03x", rp->rc_xid & 0xfff));
 			if ((rp->rc_flag & RC_LOCKED) != 0) {
 				rp->rc_flag |= RC_WANTED;
 				(void) tsleep((caddr_t)rp, PZERO-1, "nfsrc", 0);
@@ -319,6 +322,7 @@ loop:
 			return;
 		}
 	}
+	NFS_DPF(RC, ("L%03x", nd->nd_retxid & 0xfff));
 }
 
 /*
diff --git a/sys/nfs/nfsmount.h b/sys/nfs/nfsmount.h
index 67121c3..d119a20 100644
--- a/sys/nfs/nfsmount.h
+++ b/sys/nfs/nfsmount.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfsmount.h	8.3 (Berkeley) 3/30/95
- * $Id$
+ * $Id: nfsmount.h,v 1.11 1997/02/22 09:42:48 peter Exp $
  */
 
 
@@ -95,22 +95,6 @@ struct	nfsmount {
  */
 #define VFSTONFS(mp)	((struct nfsmount *)((mp)->mnt_data))
 
-#ifdef NFS_DEBUG
-
-extern int nfs_debug;
-#define NFS_DEBUG_ASYNCIO	1
-
-#define NFS_DPF(cat, args)					\
-	do {							\
-		if (nfs_debug & NFS_DEBUG_##cat) printf args;	\
-	} while (0)
-
-#else
-
-#define NFS_DPF(cat, args)
-
-#endif
-
 #endif /* KERNEL */
 
 #endif
diff --git a/sys/nfsclient/nfs.h b/sys/nfsclient/nfs.h
index 59033bf..a3c45bd 100644
--- a/sys/nfsclient/nfs.h
+++ b/sys/nfsclient/nfs.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs.h	8.4 (Berkeley) 5/1/95
- * $Id: nfs.h,v 1.23 1997/03/27 20:01:03 guido Exp $
+ * $Id: nfs.h,v 1.24 1997/03/29 12:34:33 bde Exp $
  */
 
 #ifndef _NFS_NFS_H_
@@ -517,6 +517,23 @@ extern int nfsd_head_flag;
  	 !bcmp((caddr_t)&(o)->nd_cr, (caddr_t)&(n)->nd_cr, \
 		sizeof (struct ucred)))
 
+#ifdef NFS_DEBUG
+
+extern int nfs_debug;
+#define NFS_DEBUG_ASYNCIO	1 /* asynchronous i/o */
+#define NFS_DEBUG_WG		2 /* server write gathering */
+#define NFS_DEBUG_RC		4 /* server request caching */
+
+#define NFS_DPF(cat, args)					\
+	do {							\
+		if (nfs_debug & NFS_DEBUG_##cat) printf args;	\
+	} while (0)
+
+#else
+
+#define NFS_DPF(cat, args)
+
+#endif
 
 int	nfs_init __P((struct vfsconf *vfsp));
 int	nfs_reply __P((struct nfsreq *));
diff --git a/sys/nfsclient/nfsargs.h b/sys/nfsclient/nfsargs.h
index 59033bf..a3c45bd 100644
--- a/sys/nfsclient/nfsargs.h
+++ b/sys/nfsclient/nfsargs.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs.h	8.4 (Berkeley) 5/1/95
- * $Id: nfs.h,v 1.23 1997/03/27 20:01:03 guido Exp $
+ * $Id: nfs.h,v 1.24 1997/03/29 12:34:33 bde Exp $
  */
 
 #ifndef _NFS_NFS_H_
@@ -517,6 +517,23 @@ extern int nfsd_head_flag;
  	 !bcmp((caddr_t)&(o)->nd_cr, (caddr_t)&(n)->nd_cr, \
 		sizeof (struct ucred)))
 
+#ifdef NFS_DEBUG
+
+extern int nfs_debug;
+#define NFS_DEBUG_ASYNCIO	1 /* asynchronous i/o */
+#define NFS_DEBUG_WG		2 /* server write gathering */
+#define NFS_DEBUG_RC		4 /* server request caching */
+
+#define NFS_DPF(cat, args)					\
+	do {							\
+		if (nfs_debug & NFS_DEBUG_##cat) printf args;	\
+	} while (0)
+
+#else
+
+#define NFS_DPF(cat, args)
+
+#endif
 
 int	nfs_init __P((struct vfsconf *vfsp));
 int	nfs_reply __P((struct nfsreq *));
diff --git a/sys/nfsclient/nfsmount.h b/sys/nfsclient/nfsmount.h
index 67121c3..d119a20 100644
--- a/sys/nfsclient/nfsmount.h
+++ b/sys/nfsclient/nfsmount.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfsmount.h	8.3 (Berkeley) 3/30/95
- * $Id$
+ * $Id: nfsmount.h,v 1.11 1997/02/22 09:42:48 peter Exp $
  */
 
 
@@ -95,22 +95,6 @@ struct	nfsmount {
  */
 #define VFSTONFS(mp)	((struct nfsmount *)((mp)->mnt_data))
 
-#ifdef NFS_DEBUG
-
-extern int nfs_debug;
-#define NFS_DEBUG_ASYNCIO	1
-
-#define NFS_DPF(cat, args)					\
-	do {							\
-		if (nfs_debug & NFS_DEBUG_##cat) printf args;	\
-	} while (0)
-
-#else
-
-#define NFS_DPF(cat, args)
-
-#endif
-
 #endif /* KERNEL */
 
 #endif
diff --git a/sys/nfsclient/nfsstats.h b/sys/nfsclient/nfsstats.h
index 59033bf..a3c45bd 100644
--- a/sys/nfsclient/nfsstats.h
+++ b/sys/nfsclient/nfsstats.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs.h	8.4 (Berkeley) 5/1/95
- * $Id: nfs.h,v 1.23 1997/03/27 20:01:03 guido Exp $
+ * $Id: nfs.h,v 1.24 1997/03/29 12:34:33 bde Exp $
  */
 
 #ifndef _NFS_NFS_H_
@@ -517,6 +517,23 @@ extern int nfsd_head_flag;
  	 !bcmp((caddr_t)&(o)->nd_cr, (caddr_t)&(n)->nd_cr, \
 		sizeof (struct ucred)))
 
+#ifdef NFS_DEBUG
+
+extern int nfs_debug;
+#define NFS_DEBUG_ASYNCIO	1 /* asynchronous i/o */
+#define NFS_DEBUG_WG		2 /* server write gathering */
+#define NFS_DEBUG_RC		4 /* server request caching */
+
+#define NFS_DPF(cat, args)					\
+	do {							\
+		if (nfs_debug & NFS_DEBUG_##cat) printf args;	\
+	} while (0)
+
+#else
+
+#define NFS_DPF(cat, args)
+
+#endif
 
 int	nfs_init __P((struct vfsconf *vfsp));
 int	nfs_reply __P((struct nfsreq *));
diff --git a/sys/nfsserver/nfs.h b/sys/nfsserver/nfs.h
index 59033bf..a3c45bd 100644
--- a/sys/nfsserver/nfs.h
+++ b/sys/nfsserver/nfs.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs.h	8.4 (Berkeley) 5/1/95
- * $Id: nfs.h,v 1.23 1997/03/27 20:01:03 guido Exp $
+ * $Id: nfs.h,v 1.24 1997/03/29 12:34:33 bde Exp $
  */
 
 #ifndef _NFS_NFS_H_
@@ -517,6 +517,23 @@ extern int nfsd_head_flag;
  	 !bcmp((caddr_t)&(o)->nd_cr, (caddr_t)&(n)->nd_cr, \
 		sizeof (struct ucred)))
 
+#ifdef NFS_DEBUG
+
+extern int nfs_debug;
+#define NFS_DEBUG_ASYNCIO	1 /* asynchronous i/o */
+#define NFS_DEBUG_WG		2 /* server write gathering */
+#define NFS_DEBUG_RC		4 /* server request caching */
+
+#define NFS_DPF(cat, args)					\
+	do {							\
+		if (nfs_debug & NFS_DEBUG_##cat) printf args;	\
+	} while (0)
+
+#else
+
+#define NFS_DPF(cat, args)
+
+#endif
 
 int	nfs_init __P((struct vfsconf *vfsp));
 int	nfs_reply __P((struct nfsreq *));
diff --git a/sys/nfsserver/nfs_serv.c b/sys/nfsserver/nfs_serv.c
index a0d5136..7946328 100644
--- a/sys/nfsserver/nfs_serv.c
+++ b/sys/nfsserver/nfs_serv.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs_serv.c	8.3 (Berkeley) 1/12/94
- * $Id: nfs_serv.c,v 1.39 1997/03/25 05:13:40 peter Exp $
+ * $Id: nfs_serv.c,v 1.40 1997/03/29 12:40:18 bde Exp $
  */
 
 /*
@@ -999,6 +999,7 @@ nfsmout:
 		owp = wp;
 		wp = wp->nd_tq.le_next;
 	    }
+	    NFS_DPF(WG, ("Q%03x", nfsd->nd_retxid & 0xfff));
 	    if (owp) {
 		LIST_INSERT_AFTER(owp, nfsd, nd_tq);
 	    } else {
@@ -1050,6 +1051,7 @@ loop1:
 		    break;
 		if (nfsd->nd_mreq)
 		    continue;
+		NFS_DPF(WG, ("P%03x", nfsd->nd_retxid & 0xfff));
 		LIST_REMOVE(nfsd, nd_tq);
 		LIST_REMOVE(nfsd, nd_hash);
 		splx(s);
@@ -1126,6 +1128,7 @@ loop1:
 		 */
 		swp = nfsd;
 		do {
+		    NFS_DPF(WG, ("R%03x", nfsd->nd_retxid & 0xfff));
 		    if (error) {
 			nfsm_writereply(NFSX_WCCDATA(v3), v3);
 			if (v3) {
@@ -1185,6 +1188,7 @@ loop1:
 	s = splsoftclock();
 	for (nfsd = slp->ns_tq.lh_first; nfsd; nfsd = nfsd->nd_tq.le_next)
 		if (nfsd->nd_mreq) {
+		    NFS_DPF(WG, ("X%03x", nfsd->nd_retxid & 0xfff));
 		    LIST_REMOVE(nfsd, nd_tq);
 		    *mrq = nfsd->nd_mreq;
 		    *ndp = nfsd;
@@ -1209,7 +1213,10 @@ nfsrvw_coalesce(owp, nfsd)
 {
         register int overlap;
         register struct mbuf *mp;
+	struct nfsrv_descript *p;
 
+	NFS_DPF(WG, ("C%03x-%03x",
+		     nfsd->nd_retxid & 0xfff, owp->nd_retxid & 0xfff));
         LIST_REMOVE(nfsd, nd_hash);
         LIST_REMOVE(nfsd, nd_tq);
         if (owp->nd_eoff < nfsd->nd_eoff) {
@@ -1232,6 +1239,16 @@ nfsrvw_coalesce(owp, nfsd)
             owp->nd_stable == NFSV3WRITE_UNSTABLE)
             owp->nd_stable = NFSV3WRITE_DATASYNC;
         LIST_INSERT_HEAD(&owp->nd_coalesce, nfsd, nd_tq);
+
+	/*
+	 * If nfsd had anything else coalesced into it, transfer them
+	 * to owp, otherwise their replies will never get sent.
+	 */
+	for (p = nfsd->nd_coalesce.lh_first; p;
+	     p = nfsd->nd_coalesce.lh_first) {
+	    LIST_REMOVE(p, nd_tq);
+	    LIST_INSERT_HEAD(&owp->nd_coalesce, p, nd_tq);
+	}
 }
 
 /*
diff --git a/sys/nfsserver/nfs_srvcache.c b/sys/nfsserver/nfs_srvcache.c
index 1555cc3..6121db0 100644
--- a/sys/nfsserver/nfs_srvcache.c
+++ b/sys/nfsserver/nfs_srvcache.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs_srvcache.c	8.3 (Berkeley) 3/30/95
- * $Id$
+ * $Id: nfs_srvcache.c,v 1.11 1997/02/22 09:42:40 peter Exp $
  */
 
 #ifndef NFS_NOSERVER 
@@ -184,6 +184,7 @@ loop:
 	    rp = rp->rc_hash.le_next) {
 	    if (nd->nd_retxid == rp->rc_xid && nd->nd_procnum == rp->rc_proc &&
 		netaddr_match(NETFAMILY(rp), &rp->rc_haddr, nd->nd_nam)) {
+		        NFS_DPF(RC, ("H%03x", rp->rc_xid & 0xfff));
 			if ((rp->rc_flag & RC_LOCKED) != 0) {
 				rp->rc_flag |= RC_WANTED;
 				(void) tsleep((caddr_t)rp, PZERO-1, "nfsrc", 0);
@@ -224,6 +225,7 @@ loop:
 		}
 	}
 	nfsstats.srvcache_misses++;
+	NFS_DPF(RC, ("M%03x", nd->nd_retxid & 0xfff));
 	if (numnfsrvcache < desirednfsrvcache) {
 		rp = (struct nfsrvcache *)malloc((u_long)sizeof *rp,
 		    M_NFSD, M_WAITOK);
@@ -289,6 +291,7 @@ loop:
 	    rp = rp->rc_hash.le_next) {
 	    if (nd->nd_retxid == rp->rc_xid && nd->nd_procnum == rp->rc_proc &&
 		netaddr_match(NETFAMILY(rp), &rp->rc_haddr, nd->nd_nam)) {
+			NFS_DPF(RC, ("U%03x", rp->rc_xid & 0xfff));
 			if ((rp->rc_flag & RC_LOCKED) != 0) {
 				rp->rc_flag |= RC_WANTED;
 				(void) tsleep((caddr_t)rp, PZERO-1, "nfsrc", 0);
@@ -319,6 +322,7 @@ loop:
 			return;
 		}
 	}
+	NFS_DPF(RC, ("L%03x", nd->nd_retxid & 0xfff));
 }
 
 /*
diff --git a/sys/nfsserver/nfsrvstats.h b/sys/nfsserver/nfsrvstats.h
index 59033bf..a3c45bd 100644
--- a/sys/nfsserver/nfsrvstats.h
+++ b/sys/nfsserver/nfsrvstats.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)nfs.h	8.4 (Berkeley) 5/1/95
- * $Id: nfs.h,v 1.23 1997/03/27 20:01:03 guido Exp $
+ * $Id: nfs.h,v 1.24 1997/03/29 12:34:33 bde Exp $
  */
 
 #ifndef _NFS_NFS_H_
@@ -517,6 +517,23 @@ extern int nfsd_head_flag;
  	 !bcmp((caddr_t)&(o)->nd_cr, (caddr_t)&(n)->nd_cr, \
 		sizeof (struct ucred)))
 
+#ifdef NFS_DEBUG
+
+extern int nfs_debug;
+#define NFS_DEBUG_ASYNCIO	1 /* asynchronous i/o */
+#define NFS_DEBUG_WG		2 /* server write gathering */
+#define NFS_DEBUG_RC		4 /* server request caching */
+
+#define NFS_DPF(cat, args)					\
+	do {							\
+		if (nfs_debug & NFS_DEBUG_##cat) printf args;	\
+	} while (0)
+
+#else
+
+#define NFS_DPF(cat, args)
+
+#endif
 
 int	nfs_init __P((struct vfsconf *vfsp));
 int	nfs_reply __P((struct nfsreq *));