Add checks for VI_DOOMED and vn_lock() failures to the

experimental NFS server, to handle the case where an
exported file system is forced dismounted while an RPC
is in progress. Further commits will fix the cases where
a mount point is used when the associated vnode isn't locked.

Reviewed by:	kib
MFC after:	2 weeks

4 files changed, 99 insertions, 51 deletions

diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c
index ae4a676..e39d8cc 100644
--- a/sys/fs/nfsserver/nfs_nfsdport.c
+++ b/sys/fs/nfsserver/nfs_nfsdport.c
@@ -153,6 +153,10 @@ nfsvno_accchk(struct vnode *vp, accmode_t accmode, struct ucred *cred,
 	struct vattr vattr;
 	int error = 0, getret = 0;
 
+	if (vpislocked == 0) {
+		if (vn_lock(vp, LK_SHARED) != 0)
+			return (EPERM);
+	}
 	if (accmode & VWRITE) {
 		/* Just vn_writechk() changed to check rdonly */
 		/*
@@ -166,7 +170,7 @@ nfsvno_accchk(struct vnode *vp, accmode_t accmode, struct ucred *cred,
 			case VREG:
 			case VDIR:
 			case VLNK:
-				return (EROFS);
+				error = EROFS;
 			default:
 				break;
 			}
@@ -176,11 +180,14 @@ nfsvno_accchk(struct vnode *vp, accmode_t accmode, struct ucred *cred,
 		 * the inode, try to free it up once.  If
 		 * we fail, we can't allow writing.
 		 */
-		if (vp->v_vflag & VV_TEXT)
-			return (ETXTBSY);
+		if ((vp->v_vflag & VV_TEXT) != 0 && error == 0)
+			error = ETXTBSY;
+	}
+	if (error != 0) {
+		if (vpislocked == 0)
+			VOP_UNLOCK(vp, 0);
+		return (error);
 	}
-	if (vpislocked == 0)
-		vn_lock(vp, LK_SHARED | LK_RETRY);
 
 	/*
 	 * Should the override still be applied when ACLs are enabled?
@@ -1097,9 +1104,11 @@ nfsvno_rename(struct nameidata *fromndp, struct nameidata *tondp,
 		goto out;
 	}
 	if (ndflag & ND_NFSV4) {
-		NFSVOPLOCK(fvp, LK_EXCLUSIVE | LK_RETRY, p);
-		error = nfsrv_checkremove(fvp, 0, p);
-		NFSVOPUNLOCK(fvp, 0, p);
+		if (vn_lock(fvp, LK_EXCLUSIVE) == 0) {
+			error = nfsrv_checkremove(fvp, 0, p);
+			VOP_UNLOCK(fvp, 0);
+		} else
+			error = EPERM;
 		if (tvp && !error)
 			error = nfsrv_checkremove(tvp, 1, p);
 	} else {
@@ -1156,13 +1165,16 @@ nfsvno_link(struct nameidata *ndp, struct vnode *vp, struct ucred *cred,
 			error = EXDEV;
 	}
 	if (!error) {
-		NFSVOPLOCK(vp, LK_EXCLUSIVE | LK_RETRY, p);
-		error = VOP_LINK(ndp->ni_dvp, vp, &ndp->ni_cnd);
+		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
+		if ((vp->v_iflag & VI_DOOMED) == 0)
+			error = VOP_LINK(ndp->ni_dvp, vp, &ndp->ni_cnd);
+		else
+			error = EPERM;
 		if (ndp->ni_dvp == vp)
 			vrele(ndp->ni_dvp);
 		else
 			vput(ndp->ni_dvp);
-		NFSVOPUNLOCK(vp, 0, p);
+		VOP_UNLOCK(vp, 0);
 	} else {
 		if (ndp->ni_dvp == ndp->ni_vp)
 			vrele(ndp->ni_dvp);
@@ -2793,6 +2805,11 @@ nfsvno_advlock(struct vnode *vp, int ftype, u_int64_t first,
 
 	if (nfsrv_dolocallocks == 0)
 		return (0);
+
+	/* Check for VI_DOOMED here, so that VOP_ADVLOCK() isn't performed. */
+	if ((vp->v_iflag & VI_DOOMED) != 0)
+		return (EPERM);
+
 	fl.l_whence = SEEK_SET;
 	fl.l_type = ftype;
 	fl.l_start = (off_t)first;
diff --git a/sys/fs/nfsserver/nfs_nfsdserv.c b/sys/fs/nfsserver/nfs_nfsdserv.c
index b4a4126..bd5bf7c 100644
--- a/sys/fs/nfsserver/nfs_nfsdserv.c
+++ b/sys/fs/nfsserver/nfs_nfsdserv.c
@@ -2676,9 +2676,12 @@ nfsrvd_open(struct nfsrv_descript *nd, __unused int isdgram,
 		};
 		stp->ls_flags |= NFSLCK_RECLAIM;
 		vp = dp;
-		NFSVOPLOCK(vp, LK_EXCLUSIVE | LK_RETRY, p);
-		nd->nd_repstat = nfsrv_opencheck(clientid, &stateid, stp, vp,
-		    nd, p, nd->nd_repstat);
+		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
+		if ((vp->v_iflag & VI_DOOMED) == 0)
+			nd->nd_repstat = nfsrv_opencheck(clientid, &stateid,
+			    stp, vp, nd, p, nd->nd_repstat);
+		else
+			nd->nd_repstat = NFSERR_PERM;
 	} else {
 		nd->nd_repstat = NFSERR_BADXDR;
 		vrele(dp);
diff --git a/sys/fs/nfsserver/nfs_nfsdsocket.c b/sys/fs/nfsserver/nfs_nfsdsocket.c
index 8917e92..6ff79d2 100644
--- a/sys/fs/nfsserver/nfs_nfsdsocket.c
+++ b/sys/fs/nfsserver/nfs_nfsdsocket.c
@@ -902,13 +902,15 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram,
 				nd->nd_repstat = NFSERR_XDEV;
 				break;
 			}
-			VREF(vp);
-			VREF(savevp);
 			if (nfsv4_opflag[op].modifyfs)
 				NFS_STARTWRITE(NULL, &mp);
-			NFSVOPLOCK(savevp, LK_EXCLUSIVE | LK_RETRY, p);
-			error = (*(nfsrv4_ops2[op]))(nd, isdgram, savevp,
-			    vp, p, &savevpnes, &vpnes);
+			if (vn_lock(savevp, LK_EXCLUSIVE) == 0) {
+				VREF(vp);
+				VREF(savevp);
+				error = (*(nfsrv4_ops2[op]))(nd, isdgram,
+				    savevp, vp, p, &savevpnes, &vpnes);
+			} else
+				nd->nd_repstat = NFSERR_PERM;
 			if (nfsv4_opflag[op].modifyfs)
 				NFS_ENDWRITE(mp);
 		    } else {
@@ -916,12 +918,15 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram,
 				panic("nfsrvd_compound");
 			if (nfsv4_opflag[op].needscfh) {
 				if (vp != NULL) {
+					if (nfsv4_opflag[op].modifyfs)
+						NFS_STARTWRITE(NULL, &mp);
 					if (vn_lock(vp, nfsv4_opflag[op].lktype)
-					    != 0)
+					    == 0)
+						VREF(vp);
+					else
 						nd->nd_repstat = NFSERR_PERM;
-				} else
+				} else {
 					nd->nd_repstat = NFSERR_NOFILEHANDLE;
-				if (nd->nd_repstat != 0) {
 					if (op == NFSV4OP_SETATTR) {
 						/*
 						 * Setattr reply requires a
@@ -934,11 +939,9 @@ nfsrvd_compound(struct nfsrv_descript *nd, int isdgram,
 					}
 					break;
 				}
-				VREF(vp);
-				if (nfsv4_opflag[op].modifyfs)
-					NFS_STARTWRITE(NULL, &mp);
-				error = (*(nfsrv4_ops0[op]))(nd, isdgram, vp,
-				    p, &vpnes);
+				if (nd->nd_repstat == 0)
+					error = (*(nfsrv4_ops0[op]))(nd,
+					    isdgram, vp, p, &vpnes);
 				if (nfsv4_opflag[op].modifyfs)
 					NFS_ENDWRITE(mp);
 			} else {
diff --git a/sys/fs/nfsserver/nfs_nfsdstate.c b/sys/fs/nfsserver/nfs_nfsdstate.c
index 829e7f6..6e66d54 100644
--- a/sys/fs/nfsserver/nfs_nfsdstate.c
+++ b/sys/fs/nfsserver/nfs_nfsdstate.c
@@ -1659,7 +1659,7 @@ tryagain:
 			if (new_stp->ls_flags & bits & NFSLCK_ACCESSBITS) {
 			    ret = nfsrv_clientconflict(tstp->ls_clp, &haslock,
 				vp, p);
-			    if (ret) {
+			    if (ret == 1) {
 				/*
 				* nfsrv_clientconflict unlocks state
 				 * when it returns non-zero.
@@ -1667,13 +1667,17 @@ tryagain:
 				lckstp = NULL;
 				goto tryagain;
 			    }
-			    NFSUNLOCKSTATE();
+			    if (ret == 0)
+				NFSUNLOCKSTATE();
 			    if (haslock) {
 				NFSLOCKV4ROOTMUTEX();
 				nfsv4_unlock(&nfsv4rootfs_lock, 1);
 				NFSUNLOCKV4ROOTMUTEX();
 			    }
-			    return (NFSERR_OPENMODE);
+			    if (ret == 2)
+				return (NFSERR_PERM);
+			    else
+				return (NFSERR_OPENMODE);
 			}
 		    }
 		}
@@ -1826,7 +1830,7 @@ tryagain:
 		    other_lop = NULL;
 		}
 		ret = nfsrv_clientconflict(lop->lo_stp->ls_clp,&haslock,vp,p);
-		if (ret) {
+		if (ret == 1) {
 		    if (filestruct_locked != 0) {
 			/* Roll back local locks. */
 			nfsrv_locallock_rollback(vp, lfp, p);
@@ -1845,7 +1849,7 @@ tryagain:
 		 * Found a conflicting lock, so record the conflict and
 		 * return the error.
 		 */
-		if (cfp) {
+		if (cfp != NULL && ret == 0) {
 		    cfp->cl_clientid.lval[0]=lop->lo_stp->ls_stateid.other[0];
 		    cfp->cl_clientid.lval[1]=lop->lo_stp->ls_stateid.other[1];
 		    cfp->cl_first = lop->lo_first;
@@ -1855,20 +1859,23 @@ tryagain:
 		    NFSBCOPY(lop->lo_stp->ls_owner, cfp->cl_owner,
 			cfp->cl_ownerlen);
 		}
-		if (new_stp->ls_flags & NFSLCK_RECLAIM)
+		if (ret == 2)
+		    error = NFSERR_PERM;
+		else if (new_stp->ls_flags & NFSLCK_RECLAIM)
 		    error = NFSERR_RECLAIMCONFLICT;
 		else if (new_stp->ls_flags & NFSLCK_CHECK)
 		    error = NFSERR_LOCKED;
 		else
 		    error = NFSERR_DENIED;
-		if (filestruct_locked != 0) {
+		if (filestruct_locked != 0 && ret == 0) {
 			/* Roll back local locks. */
 			NFSUNLOCKSTATE();
 			nfsrv_locallock_rollback(vp, lfp, p);
 			NFSLOCKSTATE();
 			nfsrv_unlocklf(lfp);
 		}
-		NFSUNLOCKSTATE();
+		if (ret == 0)
+			NFSUNLOCKSTATE();
 		if (haslock) {
 			NFSLOCKV4ROOTMUTEX();
 			nfsv4_unlock(&nfsv4rootfs_lock, 1);
@@ -2120,18 +2127,21 @@ tryagain:
 		     ((stp->ls_flags & NFSLCK_ACCESSBITS) &
 		      ((new_stp->ls_flags>>NFSLCK_SHIFT)&NFSLCK_ACCESSBITS)))){
 			ret = nfsrv_clientconflict(stp->ls_clp,&haslock,vp,p);
-			if (ret) {
+			if (ret == 1) {
 				/*
 				 * nfsrv_clientconflict() unlocks
 				 * state when it returns non-zero.
 				 */
 				goto tryagain;
 			}
-			if (new_stp->ls_flags & NFSLCK_RECLAIM)
+			if (ret == 2)
+				error = NFSERR_PERM;
+			else if (new_stp->ls_flags & NFSLCK_RECLAIM)
 				error = NFSERR_RECLAIMCONFLICT;
 			else
 				error = NFSERR_SHAREDENIED;
-			NFSUNLOCKSTATE();
+			if (ret == 0)
+				NFSUNLOCKSTATE();
 			if (haslock) {
 				NFSLOCKV4ROOTMUTEX();
 				nfsv4_unlock(&nfsv4rootfs_lock, 1);
@@ -2394,7 +2404,7 @@ tryagain:
 		       ((stp->ls_flags & NFSLCK_ACCESSBITS) &
 		        ((new_stp->ls_flags>>NFSLCK_SHIFT)&NFSLCK_ACCESSBITS))){
 			ret = nfsrv_clientconflict(stp->ls_clp,&haslock,vp,p);
-			if (ret) {
+			if (ret == 1) {
 				/*
 				 * nfsrv_clientconflict() unlocks state
 				 * when it returns non-zero.
@@ -2404,11 +2414,14 @@ tryagain:
 				openstp = NULL;
 				goto tryagain;
 			}
-			if (new_stp->ls_flags & NFSLCK_RECLAIM)
+			if (ret == 2)
+				error = NFSERR_PERM;
+			else if (new_stp->ls_flags & NFSLCK_RECLAIM)
 				error = NFSERR_RECLAIMCONFLICT;
 			else
 				error = NFSERR_SHAREDENIED;
-			NFSUNLOCKSTATE();
+			if (ret == 0)
+				NFSUNLOCKSTATE();
 			if (haslock) {
 				NFSLOCKV4ROOTMUTEX();
 				nfsv4_unlock(&nfsv4rootfs_lock, 1);
@@ -4080,10 +4093,13 @@ nfsrv_updatestable(NFSPROC_T *p)
 	NFSVNO_SETATTRVAL(&nva, size, 0);
 	vp = NFSFPVNODE(sf->nsf_fp);
 	NFS_STARTWRITE(vp, &mp);
-	NFSVOPLOCK(vp, LK_EXCLUSIVE | LK_RETRY, p);
-	error = nfsvno_setattr(vp, &nva, NFSFPCRED(sf->nsf_fp), p, NULL);
+	if (vn_lock(vp, LK_EXCLUSIVE) == 0) {
+		error = nfsvno_setattr(vp, &nva, NFSFPCRED(sf->nsf_fp), p,
+		    NULL);
+		VOP_UNLOCK(vp, 0);
+	} else
+		error = EPERM;
 	NFS_ENDWRITE(mp);
-	NFSVOPUNLOCK(vp, 0, p);
 	if (!error)
 	    error = NFSD_RDWR(UIO_WRITE, vp,
 		(caddr_t)&sf->nsf_rec, sizeof (struct nfsf_rec), (off_t)0,
@@ -4211,10 +4227,11 @@ nfsrv_checkstable(struct nfsclient *clp)
  * Return 0 to indicate the conflict can't be revoked and 1 to indicate
  * the revocation worked and the conflicting client is "bye, bye", so it
  * can be tried again.
+ * Return 2 to indicate that the vnode is VI_DOOMED after vn_lock().
  * Unlocks State before a non-zero value is returned.
  */
 static int
-nfsrv_clientconflict(struct nfsclient *clp, int *haslockp, __unused vnode_t vp,
+nfsrv_clientconflict(struct nfsclient *clp, int *haslockp, vnode_t vp,
     NFSPROC_T *p)
 {
 	int gotlock, lktype;
@@ -4238,7 +4255,10 @@ nfsrv_clientconflict(struct nfsclient *clp, int *haslockp, __unused vnode_t vp,
 		NFSUNLOCKV4ROOTMUTEX();
 		*haslockp = 1;
 		vn_lock(vp, lktype | LK_RETRY);
-		return (1);
+		if ((vp->v_iflag & VI_DOOMED) != 0)
+			return (2);
+		else
+			return (1);
 	}
 	NFSUNLOCKSTATE();
 
@@ -4254,7 +4274,6 @@ nfsrv_clientconflict(struct nfsclient *clp, int *haslockp, __unused vnode_t vp,
 	return (1);
 }
 
-
 /*
  * Resolve a delegation conflict.
  * Returns 0 to indicate the conflict was resolved without sleeping.
@@ -4403,6 +4422,13 @@ nfsrv_delegconflict(struct nfsstate *stp, int *haslockp, NFSPROC_T *p,
 		NFSUNLOCKV4ROOTMUTEX();
 		*haslockp = 1;
 		vn_lock(vp, lktype | LK_RETRY);
+		if ((vp->v_iflag & VI_DOOMED) != 0) {
+			*haslockp = 0;
+			NFSLOCKV4ROOTMUTEX();
+			nfsv4_unlock(&nfsv4rootfs_lock, 1);
+			NFSUNLOCKV4ROOTMUTEX();
+			return (NFSERR_PERM);
+		}
 		return (-1);
 	}
 
@@ -4594,12 +4620,11 @@ nfsd_recalldelegation(vnode_t vp, NFSPROC_T *p)
 	NFSGETNANOTIME(&mytime);
 	starttime = (u_int32_t)mytime.tv_sec;
 	do {
-		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
-		if ((vp->v_iflag & VI_DOOMED) == 0)
+		if (vn_lock(vp, LK_EXCLUSIVE) == 0) {
 			error = nfsrv_checkremove(vp, 0, p);
-		else
+			VOP_UNLOCK(vp, 0);
+		} else
 			error = EPERM;
-		VOP_UNLOCK(vp, 0);
 		if (error == NFSERR_DELAY) {
 			NFSGETNANOTIME(&mytime);
 			if (((u_int32_t)mytime.tv_sec - starttime) >