Change the experimental NFSv4 client so that it does not do

the NFSv4 Close operations until ncl_inactive(). This is
necessary so that the Open StateIDs are available for doing
I/O on mmap'd files after VOP_CLOSE(). I also changed some
indentation for the nfscl_getclose() function.

Approved by:	kib (mentor)

5 files changed, 79 insertions, 63 deletions

diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h
index 27465e8..08bcafc 100644
--- a/sys/fs/nfs/nfs_var.h
+++ b/sys/fs/nfs/nfs_var.h
@@ -351,7 +351,7 @@ int nfsrpc_openrpc(struct nfsmount *, vnode_t, u_int8_t *, int, u_int8_t *, int,
     u_int32_t, struct ucred *, NFSPROC_T *, int, int);
 int nfsrpc_opendowngrade(vnode_t, u_int32_t, struct nfsclopen *,
     struct ucred *, NFSPROC_T *);
-int nfsrpc_close(vnode_t, struct ucred *, NFSPROC_T *);
+int nfsrpc_close(vnode_t, int, NFSPROC_T *);
 int nfsrpc_closerpc(struct nfsrv_descript *, struct nfsmount *,
     struct nfsclopen *, struct ucred *, NFSPROC_T *, int);
 int nfsrpc_openconfirm(vnode_t, u_int8_t *, int, struct nfsclopen *,
@@ -457,8 +457,7 @@ void nfscl_initiate_recovery(struct nfsclclient *);
 int nfscl_hasexpired(struct nfsclclient *, u_int32_t, NFSPROC_T *);
 void nfscl_dumpstate(struct nfsmount *, int, int, int, int);
 void nfscl_dupopen(vnode_t, int);
-int nfscl_getclose(vnode_t, struct ucred *, NFSPROC_T *,
-    struct nfsclclient **, struct nfsclopenhead *);
+int nfscl_getclose(vnode_t, struct nfsclclient **, struct nfsclopenhead *);
 int nfscl_deleg(mount_t, struct nfsclclient *, u_int8_t *, int,
     struct ucred *, NFSPROC_T *, struct nfscldeleg **);
 void nfscl_lockinit(struct nfsv4lock *);
diff --git a/sys/fs/nfsclient/nfs_clnode.c b/sys/fs/nfsclient/nfs_clnode.c
index 188b72b..5148357 100644
--- a/sys/fs/nfsclient/nfs_clnode.c
+++ b/sys/fs/nfsclient/nfs_clnode.c
@@ -190,6 +190,13 @@ ncl_inactive(struct vop_inactive_args *ap)
 	np = VTONFS(ap->a_vp);
 	if (prtactive && vrefcnt(ap->a_vp) != 0)
 		vprint("ncl_inactive: pushing active", ap->a_vp);
+
+	/*
+	 * Since mmap()'d files to I/O after VOP_CLOSE(), the NFSv4 Close
+	 * operations are delayed until now.
+	 */
+	(void) nfsrpc_close(ap->a_vp, 1, td);
+
 	if (ap->a_vp->v_type != VDIR) {
 		sp = np->n_sillyrename;
 		np->n_sillyrename = NULL;
diff --git a/sys/fs/nfsclient/nfs_clrpcops.c b/sys/fs/nfsclient/nfs_clrpcops.c
index 9287d70..b42415a 100644
--- a/sys/fs/nfsclient/nfs_clrpcops.c
+++ b/sys/fs/nfsclient/nfs_clrpcops.c
@@ -550,7 +550,7 @@ nfsmout:
  * V4 Close operation.
  */
 APPLESTATIC int
-nfsrpc_close(vnode_t vp, struct ucred *cred, NFSPROC_T *p)
+nfsrpc_close(vnode_t vp, int doclose, NFSPROC_T *p)
 {
 	struct nfsclclient *clp;
 	struct nfsclopenhead oh;
@@ -558,11 +558,14 @@ nfsrpc_close(vnode_t vp, struct ucred *cred, NFSPROC_T *p)
 
 	if (vnode_vtype(vp) != VREG)
 		return (0);
-	error = nfscl_getclose(vp, cred, p, &clp, &oh);
+	if (doclose)
+		error = nfscl_getclose(vp, &clp, &oh);
+	else
+		error = nfscl_getclose(vp, &clp, NULL);
 	if (error)
 		return (error);
 
-	if (!LIST_EMPTY(&oh))
+	if (doclose && !LIST_EMPTY(&oh))
 		nfsrpc_doclose(VFSTONFS(vnode_mount(vp)), &oh, p);
 	nfscl_clientrelease(clp);
 	return (0);
@@ -997,7 +1000,7 @@ nfsrpc_setattr(vnode_t vp, struct vattr *vap, NFSACL_T *aclp,
 		if (lckp != NULL)
 			nfscl_lockderef(lckp);
 		if (!openerr)
-			(void) nfsrpc_close(vp, cred, p);
+			(void) nfsrpc_close(vp, 0, p);
 		if (error == NFSERR_GRACE || error == NFSERR_STALESTATEID ||
 		    error == NFSERR_STALEDONTRECOVER || error == NFSERR_DELAY ||
 		    error == NFSERR_OLDSTATEID) {
diff --git a/sys/fs/nfsclient/nfs_clstate.c b/sys/fs/nfsclient/nfs_clstate.c
index 501fab0..e3cf76a 100644
--- a/sys/fs/nfsclient/nfs_clstate.c
+++ b/sys/fs/nfsclient/nfs_clstate.c
@@ -663,6 +663,9 @@ nfscl_openrelease(struct nfsclopen *op, int error, int candelete)
  * client data structures to do the SetClientId/SetClientId_confirm,
  * but will release that lock and return the clientid with a refernce
  * count on it.
+ * If the p argument is NULL, it will not do the SetClientId/Confirm
+ * and the cred argument is not used, so it can be NULL too.
+ * It always clpp with a reference count on it, unless returning an error.
  */
 APPLESTATIC int
 nfscl_getcl(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
@@ -2748,8 +2751,8 @@ nfscl_dupopen(vnode_t vp, int dupopens)
  * on ohp.
  */
 APPLESTATIC int
-nfscl_getclose(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
-    struct nfsclclient **clpp, struct nfsclopenhead *ohp)
+nfscl_getclose(vnode_t vp, struct nfsclclient **clpp,
+    struct nfsclopenhead *ohp)
 {
 	struct nfsclclient *clp;
 	struct nfsclowner *owp, *nowp;
@@ -2758,12 +2761,13 @@ nfscl_getclose(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 	struct nfsfh *nfhp;
 	int error, notdecr, candelete;
 
-	error = nfscl_getcl(vp, cred, p, &clp);
+	error = nfscl_getcl(vp, NULL, NULL, &clp);
 	if (error)
 		return (error);
 	*clpp = clp;
 
-	LIST_INIT(ohp);
+	if (ohp != NULL)
+		LIST_INIT(ohp);
 	nfhp = VTONFS(vp)->n_fhp;
 	notdecr = 1;
 	NFSLOCKCLSTATE();
@@ -2798,49 +2802,56 @@ nfscl_getclose(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 
 	/* Now process the opens against the server. */
 	LIST_FOREACH(owp, &clp->nfsc_owner, nfsow_list) {
-	    op = LIST_FIRST(&owp->nfsow_open);
-	    while (op != NULL) {
-		nop = LIST_NEXT(op, nfso_list);
-		if (op->nfso_fhlen == nfhp->nfh_len &&
-		    !NFSBCMP(op->nfso_fh, nfhp->nfh_fh, nfhp->nfh_len)) {
-		    /* Found an open, decrement cnt if possible */
-		    if (notdecr && op->nfso_opencnt > 0) {
-			notdecr = 0;
-			op->nfso_opencnt--;
-		    }
-		    /*
-		     * There are more opens, so just return after
-		     * putting any opens already found back in the
-		     * state list.
-		     */
-		    if (op->nfso_opencnt > 0) {
-			/* reuse op, since we're returning */
-			op = LIST_FIRST(ohp);
-			while (op != NULL) {
-			    nop = LIST_NEXT(op, nfso_list);
-			    LIST_REMOVE(op, nfso_list);
-			    LIST_INSERT_HEAD(&op->nfso_own->nfsow_open,
-				op, nfso_list);
-			    op = nop;
-			}
-			NFSUNLOCKCLSTATE();
-			LIST_INIT(ohp);
-			return (0);
-		    }
+		op = LIST_FIRST(&owp->nfsow_open);
+		while (op != NULL) {
+			nop = LIST_NEXT(op, nfso_list);
+			if (op->nfso_fhlen == nfhp->nfh_len &&
+			    !NFSBCMP(op->nfso_fh, nfhp->nfh_fh,
+			    nfhp->nfh_len)) {
+				/* Found an open, decrement cnt if possible */
+				if (notdecr && op->nfso_opencnt > 0) {
+					notdecr = 0;
+					op->nfso_opencnt--;
+				}
+				/*
+				 * There are more opens, so just return after
+				 * putting any opens already found back in the
+				 * state list.
+				 */
+				if (op->nfso_opencnt > 0) {
+					if (ohp != NULL) {
+					    /* Reattach open until later */
+					    op = LIST_FIRST(ohp);
+					    while (op != NULL) {
+						nop = LIST_NEXT(op, nfso_list);
+						LIST_REMOVE(op, nfso_list);
+						LIST_INSERT_HEAD(
+						    &op->nfso_own->nfsow_open,
+						    op, nfso_list);
+						op = nop;
+					    }
+					    LIST_INIT(ohp);
+					}
+					NFSUNLOCKCLSTATE();
+					return (0);
+				}
 
-		    /*
-		     * Move this entry to the list of opens to be returned.
-		     * (If we find other open(s) still in use, it will be
-		     *  put back in the state list in the code just above.)
-		     */
-		    LIST_REMOVE(op, nfso_list);
-		    LIST_INSERT_HEAD(ohp, op, nfso_list);
+				/*
+				 * Move this entry to the list of opens to be
+				 * returned. (If we find other open(s) still in
+				 * use, it will be put back in the state list
+				 * in the code just above.)
+				 */
+				if (ohp != NULL) {
+					LIST_REMOVE(op, nfso_list);
+					LIST_INSERT_HEAD(ohp, op, nfso_list);
+				}
+			}
+			op = nop;
 		}
-		op = nop;
-	    }
 	}
 
-	if (dp != NULL) {
+	if (dp != NULL && ohp != NULL) {
 		/*
 		 * If we are flushing all writes against the server for this
 		 * file upon close, we do not need to keep the local opens
@@ -2869,8 +2880,8 @@ nfscl_getclose(vnode_t vp, struct ucred *cred, NFSPROC_T *p,
 		}
 	}
 	NFSUNLOCKCLSTATE();
-	if (notdecr)
-	    printf("nfscl: never fnd open\n");
+	if (notdecr && ohp == NULL)
+		printf("nfscl: never fnd open\n");
 	return (0);
 }
 
diff --git a/sys/fs/nfsclient/nfs_clvnops.c b/sys/fs/nfsclient/nfs_clvnops.c
index 62808a4..4b320ac 100644
--- a/sys/fs/nfsclient/nfs_clvnops.c
+++ b/sys/fs/nfsclient/nfs_clvnops.c
@@ -517,8 +517,7 @@ nfs_open(struct vop_open_args *ap)
 			error = ncl_vinvalbuf(vp, V_SAVE, ap->a_td, 1);
 			if (error == EINTR || error == EIO) {
 				if (NFS_ISV4(vp))
-					(void) nfsrpc_close(vp, ap->a_cred,
-					    ap->a_td);
+					(void) nfsrpc_close(vp, 0, ap->a_td);
 				return (error);
 			}
 			np->n_attrstamp = 0;
@@ -527,8 +526,7 @@ nfs_open(struct vop_open_args *ap)
 			error = VOP_GETATTR(vp, &vattr, ap->a_cred);
 			if (error) {
 				if (NFS_ISV4(vp))
-					(void) nfsrpc_close(vp, ap->a_cred,
-					    ap->a_td);
+					(void) nfsrpc_close(vp, 0, ap->a_td);
 				return (error);
 			}
 			mtx_lock(&np->n_mtx);
@@ -549,8 +547,7 @@ nfs_open(struct vop_open_args *ap)
 			error = VOP_GETATTR(vp, &vattr, ap->a_cred);
 			if (error) {
 				if (NFS_ISV4(vp))
-					(void) nfsrpc_close(vp, ap->a_cred,
-					    ap->a_td);
+					(void) nfsrpc_close(vp, 0, ap->a_td);
 				return (error);
 			}
 			mtx_lock(&np->n_mtx);
@@ -562,8 +559,8 @@ nfs_open(struct vop_open_args *ap)
 				error = ncl_vinvalbuf(vp, V_SAVE, ap->a_td, 1);
 				if (error == EINTR || error == EIO) {
 					if (NFS_ISV4(vp))
-						(void) nfsrpc_close(vp,
-						    ap->a_cred, ap->a_td);
+						(void) nfsrpc_close(vp, 0,
+						    ap->a_td);
 					return (error);
 				}
 				mtx_lock(&np->n_mtx);
@@ -583,8 +580,7 @@ nfs_open(struct vop_open_args *ap)
 			error = ncl_vinvalbuf(vp, V_SAVE, ap->a_td, 1);
 			if (error) {
 				if (NFS_ISV4(vp))
-					(void) nfsrpc_close(vp, ap->a_cred,
-					    ap->a_td);
+					(void) nfsrpc_close(vp, 0, ap->a_td);
 				return (error);
 			}
 			mtx_lock(&np->n_mtx);
@@ -745,7 +741,7 @@ nfs_close(struct vop_close_args *ap)
 		/*
 		 * and do the close.
 		 */
-		ret = nfsrpc_close(vp, cred, ap->a_td);
+		ret = nfsrpc_close(vp, 0, ap->a_td);
 		if (!error && ret)
 			error = ret;
 		if (error)