Change the second argument of vflush() to an integer that specifies

the number of references on the filesystem root vnode to be both
expected and released. Many filesystems hold an extra reference on
the filesystem root vnode, which must be accounted for when
determining if the filesystem is busy and then released if it isn't
busy. The old `skipvp' approach required individual filesystem
xxx_unmount functions to re-implement much of vflush()'s logic to
deal with the root vnode.

All 9 filesystems that hold an extra reference on the root vnode
got the logic wrong in the case of forced unmounts, so `umount -f'
would always fail if there were any extra root vnode references.
Fix this issue centrally in vflush(), now that we can.

This commit also fixes a vnode reference leak in devfs, which could
result in idle devfs filesystems that refuse to unmount.

Reviewed by:	phk, bp

30 files changed, 118 insertions, 333 deletions

diff --git a/sys/coda/coda_vfsops.c b/sys/coda/coda_vfsops.c
index a640cf4..8bcd36f 100644
--- a/sys/coda/coda_vfsops.c
+++ b/sys/coda/coda_vfsops.c
@@ -259,7 +259,7 @@ coda_unmount(vfsp, mntflags, p)
 
 	active = coda_kill(vfsp, NOT_DOWNCALL);
 	mi->mi_rootvp->v_flag &= ~VROOT;
-	error = vflush(mi->mi_vfsp, NULLVP, FORCECLOSE);
+	error = vflush(mi->mi_vfsp, 0, FORCECLOSE);
 	printf("coda_unmount: active = %d, vflush active %d\n", active, error);
 	error = 0;
 	/* I'm going to take this out to allow lookups to go through. I'm
diff --git a/sys/fs/cd9660/cd9660_vfsops.c b/sys/fs/cd9660/cd9660_vfsops.c
index a569c53..0b42933 100644
--- a/sys/fs/cd9660/cd9660_vfsops.c
+++ b/sys/fs/cd9660/cd9660_vfsops.c
@@ -537,7 +537,7 @@ cd9660_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp))
 		return EBUSY;
 #endif
-	if ((error = vflush(mp, NULLVP, flags)))
+	if ((error = vflush(mp, 0, flags)))
 		return (error);
 
 	isomp = VFSTOISOFS(mp);
diff --git a/sys/fs/coda/coda_vfsops.c b/sys/fs/coda/coda_vfsops.c
index a640cf4..8bcd36f 100644
--- a/sys/fs/coda/coda_vfsops.c
+++ b/sys/fs/coda/coda_vfsops.c
@@ -259,7 +259,7 @@ coda_unmount(vfsp, mntflags, p)
 
 	active = coda_kill(vfsp, NOT_DOWNCALL);
 	mi->mi_rootvp->v_flag &= ~VROOT;
-	error = vflush(mi->mi_vfsp, NULLVP, FORCECLOSE);
+	error = vflush(mi->mi_vfsp, 0, FORCECLOSE);
 	printf("coda_unmount: active = %d, vflush active %d\n", active, error);
 	error = 0;
 	/* I'm going to take this out to allow lookups to go through. I'm
diff --git a/sys/fs/devfs/devfs_vfsops.c b/sys/fs/devfs/devfs_vfsops.c
index 6ab9cf1..2d1aa90 100644
--- a/sys/fs/devfs/devfs_vfsops.c
+++ b/sys/fs/devfs/devfs_vfsops.c
@@ -118,26 +118,16 @@ devfs_unmount(mp, mntflags, p)
 {
 	int error;
 	int flags = 0;
-	struct vnode *rootvp;
 	struct devfs_mount *fmp;
 
-	error = devfs_root(mp, &rootvp);
-	if (error)
-		return (error);
 	fmp = VFSTODEVFS(mp);
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	if (rootvp->v_usecount > 2) {
-		vrele(rootvp);
-		return (EBUSY);
-	}
-	error = vflush(mp, rootvp, flags);
+	/* There is 1 extra root vnode reference from devfs_mount(). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 	devfs_purge(fmp->dm_rootdir);
-	vput(rootvp);
-	vrele(rootvp);
-	vgone(rootvp);
 	mp->mnt_data = 0;
 	lockdestroy(&fmp->dm_lock);
 	free(fmp, M_DEVFS);
diff --git a/sys/fs/fdescfs/fdesc_vfsops.c b/sys/fs/fdescfs/fdesc_vfsops.c
index 900e1de..a758bb8 100644
--- a/sys/fs/fdescfs/fdesc_vfsops.c
+++ b/sys/fs/fdescfs/fdesc_vfsops.c
@@ -114,7 +114,6 @@ fdesc_unmount(mp, mntflags, p)
 {
 	int error;
 	int flags = 0;
-	struct vnode *rootvp = VFSTOFDESC(mp)->f_root;
 
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
@@ -123,21 +122,14 @@ fdesc_unmount(mp, mntflags, p)
 	 * Clear out buffer cache.  I don't think we
 	 * ever get anything cached at this level at the
 	 * moment, but who knows...
+	 *
+	 * There is 1 extra root vnode reference corresponding
+	 * to f_root.
 	 */
-	if (rootvp->v_usecount > 1)
-		return (EBUSY);
-	if ((error = vflush(mp, rootvp, flags)) != 0)
+	if ((error = vflush(mp, 1, flags)) != 0)
 		return (error);
 
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(rootvp);
-	/*
 	 * Finally, throw away the fdescmount structure
 	 */
 	free(mp->mnt_data, M_FDESCMNT);	/* XXX */
diff --git a/sys/fs/hpfs/hpfs_vfsops.c b/sys/fs/hpfs/hpfs_vfsops.c
index ec68f86..46e197d 100644
--- a/sys/fs/hpfs/hpfs_vfsops.c
+++ b/sys/fs/hpfs/hpfs_vfsops.c
@@ -477,7 +477,7 @@ hpfs_unmount(
 
 	dprintf(("hpfs_unmount: vflushing...\n"));
 	
-	error = vflush(mp,NULLVP,flags);
+	error = vflush(mp, 0, flags);
 	if (error) {
 		printf("hpfs_unmount: vflush failed: %d\n",error);
 		return (error);
diff --git a/sys/fs/msdosfs/msdosfs_vfsops.c b/sys/fs/msdosfs/msdosfs_vfsops.c
index 1d9b131..abaeb7f 100644
--- a/sys/fs/msdosfs/msdosfs_vfsops.c
+++ b/sys/fs/msdosfs/msdosfs_vfsops.c
@@ -251,7 +251,7 @@ msdosfs_mount(mp, path, data, ndp, p)
 			flags = WRITECLOSE;
 			if (mp->mnt_flag & MNT_FORCE)
 				flags |= FORCECLOSE;
-			error = vflush(mp, NULLVP, flags);
+			error = vflush(mp, 0, flags);
 		}
 		if (!error && (mp->mnt_flag & MNT_RELOAD))
 			/* not yet implemented */
@@ -749,7 +749,7 @@ msdosfs_unmount(mp, mntflags, p)
 	flags = 0;
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	error = vflush(mp, NULLVP, flags);
+	error = vflush(mp, 0, flags);
 	if (error)
 		return error;
 	pmp = VFSTOMSDOSFS(mp);
diff --git a/sys/fs/ntfs/ntfs_vfsops.c b/sys/fs/ntfs/ntfs_vfsops.c
index b6fde8e..526decd 100644
--- a/sys/fs/ntfs/ntfs_vfsops.c
+++ b/sys/fs/ntfs/ntfs_vfsops.c
@@ -603,7 +603,7 @@ out1:
 	for(i=0;i<NTFS_SYSNODESNUM;i++)
 		if(ntmp->ntm_sysvn[i]) vrele(ntmp->ntm_sysvn[i]);
 
-	if (vflush(mp,NULLVP,0))
+	if (vflush(mp, 0, 0))
 		dprintf(("ntfs_mountfs: vflush failed\n"));
 
 out:
@@ -651,7 +651,7 @@ ntfs_unmount(
 		flags |= FORCECLOSE;
 
 	dprintf(("ntfs_unmount: vflushing...\n"));
-	error = vflush(mp,NULLVP,flags | SKIPSYSTEM);
+	error = vflush(mp, 0, flags | SKIPSYSTEM);
 	if (error) {
 		printf("ntfs_unmount: vflush failed: %d\n",error);
 		return (error);
@@ -667,7 +667,7 @@ ntfs_unmount(
 		 if(ntmp->ntm_sysvn[i]) vrele(ntmp->ntm_sysvn[i]);
 
 	/* vflush system vnodes */
-	error = vflush(mp,NULLVP,flags);
+	error = vflush(mp, 0, flags);
 	if (error)
 		printf("ntfs_unmount: vflush failed(sysnodes): %d\n",error);
 
diff --git a/sys/fs/nullfs/null_vfsops.c b/sys/fs/nullfs/null_vfsops.c
index dfaefe3..8573398 100644
--- a/sys/fs/nullfs/null_vfsops.c
+++ b/sys/fs/nullfs/null_vfsops.c
@@ -229,7 +229,6 @@ nullfs_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *vp = MOUNTTONULLMOUNT(mp)->nullm_rootvp;
 	void *mntdata;
 	int error;
 	int flags = 0;
@@ -239,31 +238,11 @@ nullfs_unmount(mp, mntflags, p)
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
 
-	error = VFS_ROOT(mp, &vp);
-	if (error)
-		return (error);
-	if (vp->v_usecount > 2) {
-		NULLFSDEBUG("nullfs_unmount: rootvp is busy(%d)\n",
-		    vp->v_usecount);
-		vput(vp);
-		return (EBUSY);
-	}
-	error = vflush(mp, vp, flags);
+	/* There is 1 extra root vnode reference (nullm_rootvp). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
-#ifdef NULLFS_DEBUG
-	vprint("alias root of lower", vp);
-#endif
-	vput(vp);
-	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(vp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(vp);
 	/*
 	 * Finally, throw away the null_mount structure
 	 */
diff --git a/sys/fs/nwfs/nwfs_vfsops.c b/sys/fs/nwfs/nwfs_vfsops.c
index c9084ef..c978722 100644
--- a/sys/fs/nwfs/nwfs_vfsops.c
+++ b/sys/fs/nwfs/nwfs_vfsops.c
@@ -237,35 +237,16 @@ nwfs_unmount(struct mount *mp, int mntflags, struct proc *p)
 {
 	struct nwmount *nmp = VFSTONWFS(mp);
 	struct ncp_conn *conn;
-	struct vnode *vp;
 	int error, flags;
 
 	NCPVODEBUG("nwfs_unmount: flags=%04x\n",mntflags);
 	flags = 0;
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	error = VFS_ROOT(mp,&vp);
-	if (error) return (error);
-	if (vp->v_usecount > 2) {
-		printf("nwfs_unmount: usecnt=%d\n",vp->v_usecount);
-		vput(vp);
-		return (EBUSY);
-	}
-	error = vflush(mp, vp, flags);
-	if (error) {
-		vput(vp);
+	/* There is 1 extra root vnode reference from nwfs_mount(). */
+	error = vflush(mp, 1, flags);
+	if (error)
 		return (error);
-	}
-	/*
-	 * There are two reference counts and one lock to get rid of here.
-	 */
-	NCPVODEBUG("v_use: %d\n",vp->v_usecount);
-	vput(vp);
-	NCPVODEBUG("v_use after vput: %d\n",vp->v_usecount);
-	vrele(vp);
-	NCPVODEBUG("v_use after vrele: %d\n",vp->v_usecount);
-	vgone(vp);
-	NCPVODEBUG("v_gone finished !!!!\n");
 	conn = NWFSTOCONN(nmp);
 	ncp_conn_puthandle(nmp->connh,NULL,0);
 	if (ncp_conn_lock(conn,p,p->p_ucred,NCPM_WRITE | NCPM_EXECUTE) == 0) {
diff --git a/sys/fs/portalfs/portal_vfsops.c b/sys/fs/portalfs/portal_vfsops.c
index 5146456..423cd38 100644
--- a/sys/fs/portalfs/portal_vfsops.c
+++ b/sys/fs/portalfs/portal_vfsops.c
@@ -155,7 +155,6 @@ portal_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *rootvp = VFSTOPORTAL(mp)->pm_root;
 	int error, flags = 0;
 
 
@@ -172,21 +171,12 @@ portal_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp, 1))
 		return (EBUSY);
 #endif
-	if (rootvp->v_usecount > 1)
-		return (EBUSY);
-	error = vflush(mp, rootvp, flags);
+	/* There is 1 extra root vnode reference (pm_root). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(rootvp);
-	/*
 	 * Shutdown the socket.  This will cause the select in the
 	 * daemon to wake up, and then the accept will get ECONNABORTED
 	 * which it interprets as a request to go and bury itself.
diff --git a/sys/fs/smbfs/smbfs_vfsops.c b/sys/fs/smbfs/smbfs_vfsops.c
index c0f9ee1..b5196c0 100644
--- a/sys/fs/smbfs/smbfs_vfsops.c
+++ b/sys/fs/smbfs/smbfs_vfsops.c
@@ -254,7 +254,6 @@ static int
 smbfs_unmount(struct mount *mp, int mntflags, struct proc *p)
 {
 	struct smbmount *smp = VFSTOSMBFS(mp);
-	struct vnode *vp;
 	struct smb_cred scred;
 	int error, flags;
 
@@ -262,22 +261,10 @@ smbfs_unmount(struct mount *mp, int mntflags, struct proc *p)
 	flags = 0;
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	error = VFS_ROOT(mp, &vp);
+	/* There is 1 extra root vnode reference from smbfs_mount(). */
+	error = vflush(mp, 1, flags);
 	if (error)
-		return (error);
-	if (vp->v_usecount > 2) {
-		printf("smbfs_unmount: usecnt=%d\n", vp->v_usecount);
-		vput(vp);
-		return EBUSY;
-	}
-	error = vflush(mp, vp, flags);
-	if (error) {
-		vput(vp);
 		return error;
-	}
-	vput(vp);
-	vrele(vp);
-	vgone(vp);
 	smb_makescred(&scred, p, p->p_ucred);
 	smb_share_put(smp->sm_share, &scred);
 	mp->mnt_data = (qaddr_t)0;
diff --git a/sys/fs/umapfs/umap_vfsops.c b/sys/fs/umapfs/umap_vfsops.c
index e86467d..280ded9 100644
--- a/sys/fs/umapfs/umap_vfsops.c
+++ b/sys/fs/umapfs/umap_vfsops.c
@@ -260,7 +260,6 @@ umapfs_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *umapm_rootvp = MOUNTTOUMAPMOUNT(mp)->umapm_rootvp;
 	int error;
 	int flags = 0;
 
@@ -281,23 +280,11 @@ umapfs_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp, 1))
 		return (EBUSY);
 #endif
-	if (umapm_rootvp->v_usecount > 1)
-		return (EBUSY);
-	error = vflush(mp, umapm_rootvp, flags);
+	/* There is 1 extra root vnode reference (umapm_rootvp). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
-#ifdef DEBUG
-	vprint("alias root of lower", umapm_rootvp);
-#endif
-	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(umapm_rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(umapm_rootvp);
 	/*
 	 * Finally, throw away the umap_mount structure
 	 */
diff --git a/sys/fs/unionfs/union_vfsops.c b/sys/fs/unionfs/union_vfsops.c
index 5018f09..d38f31e 100644
--- a/sys/fs/unionfs/union_vfsops.c
+++ b/sys/fs/unionfs/union_vfsops.c
@@ -307,7 +307,6 @@ union_unmount(mp, mntflags, p)
 	struct proc *p;
 {
 	struct union_mount *um = MOUNTTOUNIONMOUNT(mp);
-	struct vnode *um_rootvp;
 	int error;
 	int freeing;
 	int flags = 0;
@@ -317,9 +316,6 @@ union_unmount(mp, mntflags, p)
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
 
-	if ((error = union_root(mp, &um_rootvp)) != 0)
-		return (error);
-
 	/*
 	 * Keep flushing vnodes from the mount list.
 	 * This is needed because of the un_pvp held
@@ -329,14 +325,13 @@ union_unmount(mp, mntflags, p)
 	 * (d) times, where (d) is the maximum tree depth
 	 * in the filesystem.
 	 */
-	for (freeing = 0; vflush(mp, um_rootvp, flags) != 0;) {
+	for (freeing = 0; (error = vflush(mp, 0, flags)) != 0;) {
 		struct vnode *vp;
 		int n;
 
 		/* count #vnodes held on mount list */
-		for (n = 0, vp = LIST_FIRST(&mp->mnt_vnodelist);
-				vp != NULLVP;
-				vp = LIST_NEXT(vp, v_mntvnodes))
+		n = 0;
+		LIST_FOREACH(vp, &mp->mnt_vnodelist, v_mntvnodes)
 			n++;
 
 		/* if this is unchanged then stop */
@@ -347,15 +342,10 @@ union_unmount(mp, mntflags, p)
 		freeing = n;
 	}
 
-	/* At this point the root vnode should have a single reference */
-	if (um_rootvp->v_usecount > 1) {
-		vput(um_rootvp);
-		return (EBUSY);
-	}
+	/* If the most recent vflush failed, the filesystem is still busy. */
+	if (error)
+		return (error);
 
-#ifdef DEBUG
-	vprint("union root", um_rootvp);
-#endif	 
 	/*
 	 * Discard references to upper and lower target vnodes.
 	 */
@@ -364,14 +354,6 @@ union_unmount(mp, mntflags, p)
 	vrele(um->um_uppervp);
 	crfree(um->um_cred);
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vput(um_rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(um_rootvp);
-	/*
 	 * Finally, throw away the union_mount structure
 	 */
 	free(mp->mnt_data, M_UNIONFSMNT);	/* XXX */
diff --git a/sys/gnu/ext2fs/ext2_vfsops.c b/sys/gnu/ext2fs/ext2_vfsops.c
index 5f7118e..05bb202 100644
--- a/sys/gnu/ext2fs/ext2_vfsops.c
+++ b/sys/gnu/ext2fs/ext2_vfsops.c
@@ -823,7 +823,7 @@ ext2_flushfiles(mp, flags, p)
 	ump = VFSTOUFS(mp);
 #if QUOTA
 	if (mp->mnt_flag & MNT_QUOTA) {
-		if ((error = vflush(mp, NULLVP, SKIPSYSTEM|flags)) != 0)
+		if ((error = vflush(mp, 0, SKIPSYSTEM|flags)) != 0)
 			return (error);
 		for (i = 0; i < MAXQUOTAS; i++) {
 			if (ump->um_quotas[i] == NULLVP)
@@ -836,7 +836,7 @@ ext2_flushfiles(mp, flags, p)
 		 */
 	}
 #endif
-	error = vflush(mp, NULLVP, flags);
+	error = vflush(mp, 0, flags);
 	return (error);
 }
 
diff --git a/sys/gnu/fs/ext2fs/ext2_vfsops.c b/sys/gnu/fs/ext2fs/ext2_vfsops.c
index 5f7118e..05bb202 100644
--- a/sys/gnu/fs/ext2fs/ext2_vfsops.c
+++ b/sys/gnu/fs/ext2fs/ext2_vfsops.c
@@ -823,7 +823,7 @@ ext2_flushfiles(mp, flags, p)
 	ump = VFSTOUFS(mp);
 #if QUOTA
 	if (mp->mnt_flag & MNT_QUOTA) {
-		if ((error = vflush(mp, NULLVP, SKIPSYSTEM|flags)) != 0)
+		if ((error = vflush(mp, 0, SKIPSYSTEM|flags)) != 0)
 			return (error);
 		for (i = 0; i < MAXQUOTAS; i++) {
 			if (ump->um_quotas[i] == NULLVP)
@@ -836,7 +836,7 @@ ext2_flushfiles(mp, flags, p)
 		 */
 	}
 #endif
-	error = vflush(mp, NULLVP, flags);
+	error = vflush(mp, 0, flags);
 	return (error);
 }
 
diff --git a/sys/isofs/cd9660/cd9660_vfsops.c b/sys/isofs/cd9660/cd9660_vfsops.c
index a569c53..0b42933 100644
--- a/sys/isofs/cd9660/cd9660_vfsops.c
+++ b/sys/isofs/cd9660/cd9660_vfsops.c
@@ -537,7 +537,7 @@ cd9660_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp))
 		return EBUSY;
 #endif
-	if ((error = vflush(mp, NULLVP, flags)))
+	if ((error = vflush(mp, 0, flags)))
 		return (error);
 
 	isomp = VFSTOISOFS(mp);
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index cf2ea29..2f4dc8d 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -1627,10 +1627,22 @@ vdrop(vp)
 /*
  * Remove any vnodes in the vnode table belonging to mount point mp.
  *
- * If MNT_NOFORCE is specified, there should not be any active ones,
+ * If FORCECLOSE is not specified, there should not be any active ones,
  * return error if any are found (nb: this is a user error, not a
- * system error). If MNT_FORCE is specified, detach any active vnodes
+ * system error). If FORCECLOSE is specified, detach any active vnodes
  * that are found.
+ *
+ * If WRITECLOSE is set, only flush out regular file vnodes open for
+ * writing.
+ *
+ * SKIPSYSTEM causes any vnodes marked VSYSTEM to be skipped.
+ *
+ * `rootrefs' specifies the base reference count for the root vnode
+ * of this filesystem. The root vnode is considered busy if its
+ * v_usecount exceeds this value. On a successful return, vflush()
+ * will call vrele() on the root vnode exactly rootrefs times.
+ * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must
+ * be zero.
  */
 #ifdef DIAGNOSTIC
 static int busyprt = 0;		/* print out busy vnodes */
@@ -1638,15 +1650,26 @@ SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "");
 #endif
 
 int
-vflush(mp, skipvp, flags)
+vflush(mp, rootrefs, flags)
 	struct mount *mp;
-	struct vnode *skipvp;
+	int rootrefs;
 	int flags;
 {
 	struct proc *p = curproc;	/* XXX */
-	struct vnode *vp, *nvp;
-	int busy = 0;
+	struct vnode *vp, *nvp, *rootvp = NULL;
+	int busy = 0, error;
 
+	if (rootrefs > 0) {
+		KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0,
+		    ("vflush: bad args"));
+		/*
+		 * Get the filesystem root vnode. We can vput() it
+		 * immediately, since with rootrefs > 0, it won't go away.
+		 */
+		if ((error = VFS_ROOT(mp, &rootvp)) != 0)
+			return (error);
+		vput(rootvp);
+	}
 	mtx_lock(&mntvnode_mtx);
 loop:
 	for (vp = LIST_FIRST(&mp->mnt_vnodelist); vp; vp = nvp) {
@@ -1657,11 +1680,6 @@ loop:
 		if (vp->v_mount != mp)
 			goto loop;
 		nvp = LIST_NEXT(vp, v_mntvnodes);
-		/*
-		 * Skip over a selected vnode.
-		 */
-		if (vp == skipvp)
-			continue;
 
 		mtx_lock(&vp->v_interlock);
 		/*
@@ -1717,8 +1735,24 @@ loop:
 		busy++;
 	}
 	mtx_unlock(&mntvnode_mtx);
+	if (rootrefs > 0 && (flags & FORCECLOSE) == 0) {
+		/*
+		 * If just the root vnode is busy, and if its refcount
+		 * is equal to `rootrefs', then go ahead and kill it.
+		 */
+		mtx_lock(&rootvp->v_interlock);
+		KASSERT(busy > 0, ("vflush: not busy"));
+		KASSERT(rootvp->v_usecount >= rootrefs, ("vflush: rootrefs"));
+		if (busy == 1 && rootvp->v_usecount == rootrefs) {
+			vgonel(rootvp, p);
+			busy = 0;
+		} else
+			mtx_unlock(&rootvp->v_interlock);
+	}
 	if (busy)
 		return (EBUSY);
+	for (; rootrefs > 0; rootrefs--)
+		vrele(rootvp);
 	return (0);
 }
 
diff --git a/sys/miscfs/fdesc/fdesc_vfsops.c b/sys/miscfs/fdesc/fdesc_vfsops.c
index 900e1de..a758bb8 100644
--- a/sys/miscfs/fdesc/fdesc_vfsops.c
+++ b/sys/miscfs/fdesc/fdesc_vfsops.c
@@ -114,7 +114,6 @@ fdesc_unmount(mp, mntflags, p)
 {
 	int error;
 	int flags = 0;
-	struct vnode *rootvp = VFSTOFDESC(mp)->f_root;
 
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
@@ -123,21 +122,14 @@ fdesc_unmount(mp, mntflags, p)
 	 * Clear out buffer cache.  I don't think we
 	 * ever get anything cached at this level at the
 	 * moment, but who knows...
+	 *
+	 * There is 1 extra root vnode reference corresponding
+	 * to f_root.
 	 */
-	if (rootvp->v_usecount > 1)
-		return (EBUSY);
-	if ((error = vflush(mp, rootvp, flags)) != 0)
+	if ((error = vflush(mp, 1, flags)) != 0)
 		return (error);
 
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(rootvp);
-	/*
 	 * Finally, throw away the fdescmount structure
 	 */
 	free(mp->mnt_data, M_FDESCMNT);	/* XXX */
diff --git a/sys/miscfs/nullfs/null_vfsops.c b/sys/miscfs/nullfs/null_vfsops.c
index dfaefe3..8573398 100644
--- a/sys/miscfs/nullfs/null_vfsops.c
+++ b/sys/miscfs/nullfs/null_vfsops.c
@@ -229,7 +229,6 @@ nullfs_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *vp = MOUNTTONULLMOUNT(mp)->nullm_rootvp;
 	void *mntdata;
 	int error;
 	int flags = 0;
@@ -239,31 +238,11 @@ nullfs_unmount(mp, mntflags, p)
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
 
-	error = VFS_ROOT(mp, &vp);
-	if (error)
-		return (error);
-	if (vp->v_usecount > 2) {
-		NULLFSDEBUG("nullfs_unmount: rootvp is busy(%d)\n",
-		    vp->v_usecount);
-		vput(vp);
-		return (EBUSY);
-	}
-	error = vflush(mp, vp, flags);
+	/* There is 1 extra root vnode reference (nullm_rootvp). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
-#ifdef NULLFS_DEBUG
-	vprint("alias root of lower", vp);
-#endif
-	vput(vp);
-	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(vp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(vp);
 	/*
 	 * Finally, throw away the null_mount structure
 	 */
diff --git a/sys/miscfs/portal/portal_vfsops.c b/sys/miscfs/portal/portal_vfsops.c
index 5146456..423cd38 100644
--- a/sys/miscfs/portal/portal_vfsops.c
+++ b/sys/miscfs/portal/portal_vfsops.c
@@ -155,7 +155,6 @@ portal_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *rootvp = VFSTOPORTAL(mp)->pm_root;
 	int error, flags = 0;
 
 
@@ -172,21 +171,12 @@ portal_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp, 1))
 		return (EBUSY);
 #endif
-	if (rootvp->v_usecount > 1)
-		return (EBUSY);
-	error = vflush(mp, rootvp, flags);
+	/* There is 1 extra root vnode reference (pm_root). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(rootvp);
-	/*
 	 * Shutdown the socket.  This will cause the select in the
 	 * daemon to wake up, and then the accept will get ECONNABORTED
 	 * which it interprets as a request to go and bury itself.
diff --git a/sys/miscfs/umapfs/umap_vfsops.c b/sys/miscfs/umapfs/umap_vfsops.c
index e86467d..280ded9 100644
--- a/sys/miscfs/umapfs/umap_vfsops.c
+++ b/sys/miscfs/umapfs/umap_vfsops.c
@@ -260,7 +260,6 @@ umapfs_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *umapm_rootvp = MOUNTTOUMAPMOUNT(mp)->umapm_rootvp;
 	int error;
 	int flags = 0;
 
@@ -281,23 +280,11 @@ umapfs_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp, 1))
 		return (EBUSY);
 #endif
-	if (umapm_rootvp->v_usecount > 1)
-		return (EBUSY);
-	error = vflush(mp, umapm_rootvp, flags);
+	/* There is 1 extra root vnode reference (umapm_rootvp). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
-#ifdef DEBUG
-	vprint("alias root of lower", umapm_rootvp);
-#endif
-	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(umapm_rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(umapm_rootvp);
 	/*
 	 * Finally, throw away the umap_mount structure
 	 */
diff --git a/sys/miscfs/union/union_vfsops.c b/sys/miscfs/union/union_vfsops.c
index 5018f09..d38f31e 100644
--- a/sys/miscfs/union/union_vfsops.c
+++ b/sys/miscfs/union/union_vfsops.c
@@ -307,7 +307,6 @@ union_unmount(mp, mntflags, p)
 	struct proc *p;
 {
 	struct union_mount *um = MOUNTTOUNIONMOUNT(mp);
-	struct vnode *um_rootvp;
 	int error;
 	int freeing;
 	int flags = 0;
@@ -317,9 +316,6 @@ union_unmount(mp, mntflags, p)
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
 
-	if ((error = union_root(mp, &um_rootvp)) != 0)
-		return (error);
-
 	/*
 	 * Keep flushing vnodes from the mount list.
 	 * This is needed because of the un_pvp held
@@ -329,14 +325,13 @@ union_unmount(mp, mntflags, p)
 	 * (d) times, where (d) is the maximum tree depth
 	 * in the filesystem.
 	 */
-	for (freeing = 0; vflush(mp, um_rootvp, flags) != 0;) {
+	for (freeing = 0; (error = vflush(mp, 0, flags)) != 0;) {
 		struct vnode *vp;
 		int n;
 
 		/* count #vnodes held on mount list */
-		for (n = 0, vp = LIST_FIRST(&mp->mnt_vnodelist);
-				vp != NULLVP;
-				vp = LIST_NEXT(vp, v_mntvnodes))
+		n = 0;
+		LIST_FOREACH(vp, &mp->mnt_vnodelist, v_mntvnodes)
 			n++;
 
 		/* if this is unchanged then stop */
@@ -347,15 +342,10 @@ union_unmount(mp, mntflags, p)
 		freeing = n;
 	}
 
-	/* At this point the root vnode should have a single reference */
-	if (um_rootvp->v_usecount > 1) {
-		vput(um_rootvp);
-		return (EBUSY);
-	}
+	/* If the most recent vflush failed, the filesystem is still busy. */
+	if (error)
+		return (error);
 
-#ifdef DEBUG
-	vprint("union root", um_rootvp);
-#endif	 
 	/*
 	 * Discard references to upper and lower target vnodes.
 	 */
@@ -364,14 +354,6 @@ union_unmount(mp, mntflags, p)
 	vrele(um->um_uppervp);
 	crfree(um->um_cred);
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vput(um_rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(um_rootvp);
-	/*
 	 * Finally, throw away the union_mount structure
 	 */
 	free(mp->mnt_data, M_UNIONFSMNT);	/* XXX */
diff --git a/sys/msdosfs/msdosfs_vfsops.c b/sys/msdosfs/msdosfs_vfsops.c
index 1d9b131..abaeb7f 100644
--- a/sys/msdosfs/msdosfs_vfsops.c
+++ b/sys/msdosfs/msdosfs_vfsops.c
@@ -251,7 +251,7 @@ msdosfs_mount(mp, path, data, ndp, p)
 			flags = WRITECLOSE;
 			if (mp->mnt_flag & MNT_FORCE)
 				flags |= FORCECLOSE;
-			error = vflush(mp, NULLVP, flags);
+			error = vflush(mp, 0, flags);
 		}
 		if (!error && (mp->mnt_flag & MNT_RELOAD))
 			/* not yet implemented */
@@ -749,7 +749,7 @@ msdosfs_unmount(mp, mntflags, p)
 	flags = 0;
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	error = vflush(mp, NULLVP, flags);
+	error = vflush(mp, 0, flags);
 	if (error)
 		return error;
 	pmp = VFSTOMSDOSFS(mp);
diff --git a/sys/nfs/nfs_vfsops.c b/sys/nfs/nfs_vfsops.c
index d5e8972..3c76ed5 100644
--- a/sys/nfs/nfs_vfsops.c
+++ b/sys/nfs/nfs_vfsops.c
@@ -953,8 +953,6 @@ nfs_unmount(mp, mntflags, p)
 	struct proc *p;
 {
 	register struct nfsmount *nmp;
-	struct nfsnode *np;
-	struct vnode *vp;
 	int error, flags = 0;
 
 	if (mntflags & MNT_FORCE)
@@ -962,36 +960,20 @@ nfs_unmount(mp, mntflags, p)
 	nmp = VFSTONFS(mp);
 	/*
 	 * Goes something like this..
-	 * - Check for activity on the root vnode (other than ourselves).
-	 * - Call vflush() to clear out vnodes for this file system,
-	 *   except for the root vnode.
-	 * - Decrement reference on the vnode representing remote root.
+	 * - Call vflush() to clear out vnodes for this file system
 	 * - Close the socket
 	 * - Free up the data structures
 	 */
 	/*
-	 * We need to decrement the ref. count on the nfsnode representing
-	 * the remote root.  See comment in mountnfs().  The VFS unmount()
-	 * has done vput on this vnode, otherwise we would get deadlock!
-	 */
-	error = nfs_nget(mp, (nfsfh_t *)nmp->nm_fh, nmp->nm_fhsize, &np);
-	if (error)
-		return(error);
-	vp = NFSTOV(np);
-	if (vp->v_usecount > 2) {
-		vput(vp);
-		return (EBUSY);
-	}
-
-	/*
 	 * Must handshake with nqnfs_clientd() if it is active.
 	 */
 	nmp->nm_state |= NFSSTA_DISMINPROG;
 	while (nmp->nm_inprog != NULLVP)
 		(void) tsleep((caddr_t)&lbolt, PSOCK, "nfsdism", 0);
-	error = vflush(mp, vp, flags);
+
+	/* We hold 1 extra ref on the root vnode; see comment in mountnfs(). */
+	error = vflush(mp, 1, flags);
 	if (error) {
-		vput(vp);
 		nmp->nm_state &= ~NFSSTA_DISMINPROG;
 		return (error);
 	}
@@ -1003,12 +985,6 @@ nfs_unmount(mp, mntflags, p)
 	if (nmp->nm_flag & (NFSMNT_NQNFS | NFSMNT_KERB))
 		nmp->nm_state |= NFSSTA_DISMNT;
 
-	/*
-	 * There are two reference counts and one lock to get rid of here.
-	 */
-	vput(vp);
-	vrele(vp);
-	vgone(vp);
 	nfs_disconnect(nmp);
 	FREE(nmp->nm_nam, M_SONAME);
 
diff --git a/sys/nfsclient/nfs_vfsops.c b/sys/nfsclient/nfs_vfsops.c
index d5e8972..3c76ed5 100644
--- a/sys/nfsclient/nfs_vfsops.c
+++ b/sys/nfsclient/nfs_vfsops.c
@@ -953,8 +953,6 @@ nfs_unmount(mp, mntflags, p)
 	struct proc *p;
 {
 	register struct nfsmount *nmp;
-	struct nfsnode *np;
-	struct vnode *vp;
 	int error, flags = 0;
 
 	if (mntflags & MNT_FORCE)
@@ -962,36 +960,20 @@ nfs_unmount(mp, mntflags, p)
 	nmp = VFSTONFS(mp);
 	/*
 	 * Goes something like this..
-	 * - Check for activity on the root vnode (other than ourselves).
-	 * - Call vflush() to clear out vnodes for this file system,
-	 *   except for the root vnode.
-	 * - Decrement reference on the vnode representing remote root.
+	 * - Call vflush() to clear out vnodes for this file system
 	 * - Close the socket
 	 * - Free up the data structures
 	 */
 	/*
-	 * We need to decrement the ref. count on the nfsnode representing
-	 * the remote root.  See comment in mountnfs().  The VFS unmount()
-	 * has done vput on this vnode, otherwise we would get deadlock!
-	 */
-	error = nfs_nget(mp, (nfsfh_t *)nmp->nm_fh, nmp->nm_fhsize, &np);
-	if (error)
-		return(error);
-	vp = NFSTOV(np);
-	if (vp->v_usecount > 2) {
-		vput(vp);
-		return (EBUSY);
-	}
-
-	/*
 	 * Must handshake with nqnfs_clientd() if it is active.
 	 */
 	nmp->nm_state |= NFSSTA_DISMINPROG;
 	while (nmp->nm_inprog != NULLVP)
 		(void) tsleep((caddr_t)&lbolt, PSOCK, "nfsdism", 0);
-	error = vflush(mp, vp, flags);
+
+	/* We hold 1 extra ref on the root vnode; see comment in mountnfs(). */
+	error = vflush(mp, 1, flags);
 	if (error) {
-		vput(vp);
 		nmp->nm_state &= ~NFSSTA_DISMINPROG;
 		return (error);
 	}
@@ -1003,12 +985,6 @@ nfs_unmount(mp, mntflags, p)
 	if (nmp->nm_flag & (NFSMNT_NQNFS | NFSMNT_KERB))
 		nmp->nm_state |= NFSSTA_DISMNT;
 
-	/*
-	 * There are two reference counts and one lock to get rid of here.
-	 */
-	vput(vp);
-	vrele(vp);
-	vgone(vp);
 	nfs_disconnect(nmp);
 	FREE(nmp->nm_nam, M_SONAME);
 
diff --git a/sys/ntfs/ntfs_vfsops.c b/sys/ntfs/ntfs_vfsops.c
index b6fde8e..526decd 100644
--- a/sys/ntfs/ntfs_vfsops.c
+++ b/sys/ntfs/ntfs_vfsops.c
@@ -603,7 +603,7 @@ out1:
 	for(i=0;i<NTFS_SYSNODESNUM;i++)
 		if(ntmp->ntm_sysvn[i]) vrele(ntmp->ntm_sysvn[i]);
 
-	if (vflush(mp,NULLVP,0))
+	if (vflush(mp, 0, 0))
 		dprintf(("ntfs_mountfs: vflush failed\n"));
 
 out:
@@ -651,7 +651,7 @@ ntfs_unmount(
 		flags |= FORCECLOSE;
 
 	dprintf(("ntfs_unmount: vflushing...\n"));
-	error = vflush(mp,NULLVP,flags | SKIPSYSTEM);
+	error = vflush(mp, 0, flags | SKIPSYSTEM);
 	if (error) {
 		printf("ntfs_unmount: vflush failed: %d\n",error);
 		return (error);
@@ -667,7 +667,7 @@ ntfs_unmount(
 		 if(ntmp->ntm_sysvn[i]) vrele(ntmp->ntm_sysvn[i]);
 
 	/* vflush system vnodes */
-	error = vflush(mp,NULLVP,flags);
+	error = vflush(mp, 0, flags);
 	if (error)
 		printf("ntfs_unmount: vflush failed(sysnodes): %d\n",error);
 
diff --git a/sys/nwfs/nwfs_vfsops.c b/sys/nwfs/nwfs_vfsops.c
index c9084ef..c978722 100644
--- a/sys/nwfs/nwfs_vfsops.c
+++ b/sys/nwfs/nwfs_vfsops.c
@@ -237,35 +237,16 @@ nwfs_unmount(struct mount *mp, int mntflags, struct proc *p)
 {
 	struct nwmount *nmp = VFSTONWFS(mp);
 	struct ncp_conn *conn;
-	struct vnode *vp;
 	int error, flags;
 
 	NCPVODEBUG("nwfs_unmount: flags=%04x\n",mntflags);
 	flags = 0;
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	error = VFS_ROOT(mp,&vp);
-	if (error) return (error);
-	if (vp->v_usecount > 2) {
-		printf("nwfs_unmount: usecnt=%d\n",vp->v_usecount);
-		vput(vp);
-		return (EBUSY);
-	}
-	error = vflush(mp, vp, flags);
-	if (error) {
-		vput(vp);
+	/* There is 1 extra root vnode reference from nwfs_mount(). */
+	error = vflush(mp, 1, flags);
+	if (error)
 		return (error);
-	}
-	/*
-	 * There are two reference counts and one lock to get rid of here.
-	 */
-	NCPVODEBUG("v_use: %d\n",vp->v_usecount);
-	vput(vp);
-	NCPVODEBUG("v_use after vput: %d\n",vp->v_usecount);
-	vrele(vp);
-	NCPVODEBUG("v_use after vrele: %d\n",vp->v_usecount);
-	vgone(vp);
-	NCPVODEBUG("v_gone finished !!!!\n");
 	conn = NWFSTOCONN(nmp);
 	ncp_conn_puthandle(nmp->connh,NULL,0);
 	if (ncp_conn_lock(conn,p,p->p_ucred,NCPM_WRITE | NCPM_EXECUTE) == 0) {
diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index cba084c..e530d4a 100644
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -579,7 +579,7 @@ void	vdrop __P((struct vnode *));
 int	vfinddev __P((dev_t dev, enum vtype type, struct vnode **vpp));
 void	vfs_add_vnodeops __P((const void *));
 void	vfs_rm_vnodeops __P((const void *));
-int	vflush __P((struct mount *mp, struct vnode *skipvp, int flags));
+int	vflush __P((struct mount *mp, int rootrefs, int flags));
 int	vget __P((struct vnode *vp, int lockflag, struct proc *p));
 void	vgone __P((struct vnode *vp));
 void	vgonel __P((struct vnode *vp, struct proc *p));
diff --git a/sys/ufs/ffs/ffs_vfsops.c b/sys/ufs/ffs/ffs_vfsops.c
index 04ba155..fd77a17 100644
--- a/sys/ufs/ffs/ffs_vfsops.c
+++ b/sys/ufs/ffs/ffs_vfsops.c
@@ -898,7 +898,7 @@ ffs_flushfiles(mp, flags, p)
 #ifdef QUOTA
 	if (mp->mnt_flag & MNT_QUOTA) {
 		int i;
-		error = vflush(mp, NULLVP, SKIPSYSTEM|flags);
+		error = vflush(mp, 0, SKIPSYSTEM|flags);
 		if (error)
 			return (error);
 		for (i = 0; i < MAXQUOTAS; i++) {
@@ -913,7 +913,7 @@ ffs_flushfiles(mp, flags, p)
 	}
 #endif
 	if (ump->um_devvp->v_flag & VCOPYONWRITE) {
-		if ((error = vflush(mp, NULL, SKIPSYSTEM | flags)) != 0)
+		if ((error = vflush(mp, 0, SKIPSYSTEM | flags)) != 0)
 			return (error);
 		ffs_snapshot_unmount(mp);
 		/*
@@ -924,7 +924,7 @@ ffs_flushfiles(mp, flags, p)
         /*
 	 * Flush all the files.
 	 */
-	if ((error = vflush(mp, NULL, flags)) != 0)
+	if ((error = vflush(mp, 0, flags)) != 0)
 		return (error);
 	/*
 	 * Flush filesystem metadata.