Change the second argument of vflush() to an integer that specifies

the number of references on the filesystem root vnode to be both
expected and released. Many filesystems hold an extra reference on
the filesystem root vnode, which must be accounted for when
determining if the filesystem is busy and then released if it isn't
busy. The old `skipvp' approach required individual filesystem
xxx_unmount functions to re-implement much of vflush()'s logic to
deal with the root vnode.

All 9 filesystems that hold an extra reference on the root vnode
got the logic wrong in the case of forced unmounts, so `umount -f'
would always fail if there were any extra root vnode references.
Fix this issue centrally in vflush(), now that we can.

This commit also fixes a vnode reference leak in devfs, which could
result in idle devfs filesystems that refuse to unmount.

Reviewed by:	phk, bp

13 files changed, 31 insertions, 143 deletions

diff --git a/sys/fs/cd9660/cd9660_vfsops.c b/sys/fs/cd9660/cd9660_vfsops.c
index a569c53..0b42933 100644
--- a/sys/fs/cd9660/cd9660_vfsops.c
+++ b/sys/fs/cd9660/cd9660_vfsops.c
@@ -537,7 +537,7 @@ cd9660_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp))
 		return EBUSY;
 #endif
-	if ((error = vflush(mp, NULLVP, flags)))
+	if ((error = vflush(mp, 0, flags)))
 		return (error);
 
 	isomp = VFSTOISOFS(mp);
diff --git a/sys/fs/coda/coda_vfsops.c b/sys/fs/coda/coda_vfsops.c
index a640cf4..8bcd36f 100644
--- a/sys/fs/coda/coda_vfsops.c
+++ b/sys/fs/coda/coda_vfsops.c
@@ -259,7 +259,7 @@ coda_unmount(vfsp, mntflags, p)
 
 	active = coda_kill(vfsp, NOT_DOWNCALL);
 	mi->mi_rootvp->v_flag &= ~VROOT;
-	error = vflush(mi->mi_vfsp, NULLVP, FORCECLOSE);
+	error = vflush(mi->mi_vfsp, 0, FORCECLOSE);
 	printf("coda_unmount: active = %d, vflush active %d\n", active, error);
 	error = 0;
 	/* I'm going to take this out to allow lookups to go through. I'm
diff --git a/sys/fs/devfs/devfs_vfsops.c b/sys/fs/devfs/devfs_vfsops.c
index 6ab9cf1..2d1aa90 100644
--- a/sys/fs/devfs/devfs_vfsops.c
+++ b/sys/fs/devfs/devfs_vfsops.c
@@ -118,26 +118,16 @@ devfs_unmount(mp, mntflags, p)
 {
 	int error;
 	int flags = 0;
-	struct vnode *rootvp;
 	struct devfs_mount *fmp;
 
-	error = devfs_root(mp, &rootvp);
-	if (error)
-		return (error);
 	fmp = VFSTODEVFS(mp);
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	if (rootvp->v_usecount > 2) {
-		vrele(rootvp);
-		return (EBUSY);
-	}
-	error = vflush(mp, rootvp, flags);
+	/* There is 1 extra root vnode reference from devfs_mount(). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 	devfs_purge(fmp->dm_rootdir);
-	vput(rootvp);
-	vrele(rootvp);
-	vgone(rootvp);
 	mp->mnt_data = 0;
 	lockdestroy(&fmp->dm_lock);
 	free(fmp, M_DEVFS);
diff --git a/sys/fs/fdescfs/fdesc_vfsops.c b/sys/fs/fdescfs/fdesc_vfsops.c
index 900e1de..a758bb8 100644
--- a/sys/fs/fdescfs/fdesc_vfsops.c
+++ b/sys/fs/fdescfs/fdesc_vfsops.c
@@ -114,7 +114,6 @@ fdesc_unmount(mp, mntflags, p)
 {
 	int error;
 	int flags = 0;
-	struct vnode *rootvp = VFSTOFDESC(mp)->f_root;
 
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
@@ -123,21 +122,14 @@ fdesc_unmount(mp, mntflags, p)
 	 * Clear out buffer cache.  I don't think we
 	 * ever get anything cached at this level at the
 	 * moment, but who knows...
+	 *
+	 * There is 1 extra root vnode reference corresponding
+	 * to f_root.
 	 */
-	if (rootvp->v_usecount > 1)
-		return (EBUSY);
-	if ((error = vflush(mp, rootvp, flags)) != 0)
+	if ((error = vflush(mp, 1, flags)) != 0)
 		return (error);
 
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(rootvp);
-	/*
 	 * Finally, throw away the fdescmount structure
 	 */
 	free(mp->mnt_data, M_FDESCMNT);	/* XXX */
diff --git a/sys/fs/hpfs/hpfs_vfsops.c b/sys/fs/hpfs/hpfs_vfsops.c
index ec68f86..46e197d 100644
--- a/sys/fs/hpfs/hpfs_vfsops.c
+++ b/sys/fs/hpfs/hpfs_vfsops.c
@@ -477,7 +477,7 @@ hpfs_unmount(
 
 	dprintf(("hpfs_unmount: vflushing...\n"));
 	
-	error = vflush(mp,NULLVP,flags);
+	error = vflush(mp, 0, flags);
 	if (error) {
 		printf("hpfs_unmount: vflush failed: %d\n",error);
 		return (error);
diff --git a/sys/fs/msdosfs/msdosfs_vfsops.c b/sys/fs/msdosfs/msdosfs_vfsops.c
index 1d9b131..abaeb7f 100644
--- a/sys/fs/msdosfs/msdosfs_vfsops.c
+++ b/sys/fs/msdosfs/msdosfs_vfsops.c
@@ -251,7 +251,7 @@ msdosfs_mount(mp, path, data, ndp, p)
 			flags = WRITECLOSE;
 			if (mp->mnt_flag & MNT_FORCE)
 				flags |= FORCECLOSE;
-			error = vflush(mp, NULLVP, flags);
+			error = vflush(mp, 0, flags);
 		}
 		if (!error && (mp->mnt_flag & MNT_RELOAD))
 			/* not yet implemented */
@@ -749,7 +749,7 @@ msdosfs_unmount(mp, mntflags, p)
 	flags = 0;
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	error = vflush(mp, NULLVP, flags);
+	error = vflush(mp, 0, flags);
 	if (error)
 		return error;
 	pmp = VFSTOMSDOSFS(mp);
diff --git a/sys/fs/ntfs/ntfs_vfsops.c b/sys/fs/ntfs/ntfs_vfsops.c
index b6fde8e..526decd 100644
--- a/sys/fs/ntfs/ntfs_vfsops.c
+++ b/sys/fs/ntfs/ntfs_vfsops.c
@@ -603,7 +603,7 @@ out1:
 	for(i=0;i<NTFS_SYSNODESNUM;i++)
 		if(ntmp->ntm_sysvn[i]) vrele(ntmp->ntm_sysvn[i]);
 
-	if (vflush(mp,NULLVP,0))
+	if (vflush(mp, 0, 0))
 		dprintf(("ntfs_mountfs: vflush failed\n"));
 
 out:
@@ -651,7 +651,7 @@ ntfs_unmount(
 		flags |= FORCECLOSE;
 
 	dprintf(("ntfs_unmount: vflushing...\n"));
-	error = vflush(mp,NULLVP,flags | SKIPSYSTEM);
+	error = vflush(mp, 0, flags | SKIPSYSTEM);
 	if (error) {
 		printf("ntfs_unmount: vflush failed: %d\n",error);
 		return (error);
@@ -667,7 +667,7 @@ ntfs_unmount(
 		 if(ntmp->ntm_sysvn[i]) vrele(ntmp->ntm_sysvn[i]);
 
 	/* vflush system vnodes */
-	error = vflush(mp,NULLVP,flags);
+	error = vflush(mp, 0, flags);
 	if (error)
 		printf("ntfs_unmount: vflush failed(sysnodes): %d\n",error);
 
diff --git a/sys/fs/nullfs/null_vfsops.c b/sys/fs/nullfs/null_vfsops.c
index dfaefe3..8573398 100644
--- a/sys/fs/nullfs/null_vfsops.c
+++ b/sys/fs/nullfs/null_vfsops.c
@@ -229,7 +229,6 @@ nullfs_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *vp = MOUNTTONULLMOUNT(mp)->nullm_rootvp;
 	void *mntdata;
 	int error;
 	int flags = 0;
@@ -239,31 +238,11 @@ nullfs_unmount(mp, mntflags, p)
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
 
-	error = VFS_ROOT(mp, &vp);
-	if (error)
-		return (error);
-	if (vp->v_usecount > 2) {
-		NULLFSDEBUG("nullfs_unmount: rootvp is busy(%d)\n",
-		    vp->v_usecount);
-		vput(vp);
-		return (EBUSY);
-	}
-	error = vflush(mp, vp, flags);
+	/* There is 1 extra root vnode reference (nullm_rootvp). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
-#ifdef NULLFS_DEBUG
-	vprint("alias root of lower", vp);
-#endif
-	vput(vp);
-	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(vp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(vp);
 	/*
 	 * Finally, throw away the null_mount structure
 	 */
diff --git a/sys/fs/nwfs/nwfs_vfsops.c b/sys/fs/nwfs/nwfs_vfsops.c
index c9084ef..c978722 100644
--- a/sys/fs/nwfs/nwfs_vfsops.c
+++ b/sys/fs/nwfs/nwfs_vfsops.c
@@ -237,35 +237,16 @@ nwfs_unmount(struct mount *mp, int mntflags, struct proc *p)
 {
 	struct nwmount *nmp = VFSTONWFS(mp);
 	struct ncp_conn *conn;
-	struct vnode *vp;
 	int error, flags;
 
 	NCPVODEBUG("nwfs_unmount: flags=%04x\n",mntflags);
 	flags = 0;
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	error = VFS_ROOT(mp,&vp);
-	if (error) return (error);
-	if (vp->v_usecount > 2) {
-		printf("nwfs_unmount: usecnt=%d\n",vp->v_usecount);
-		vput(vp);
-		return (EBUSY);
-	}
-	error = vflush(mp, vp, flags);
-	if (error) {
-		vput(vp);
+	/* There is 1 extra root vnode reference from nwfs_mount(). */
+	error = vflush(mp, 1, flags);
+	if (error)
 		return (error);
-	}
-	/*
-	 * There are two reference counts and one lock to get rid of here.
-	 */
-	NCPVODEBUG("v_use: %d\n",vp->v_usecount);
-	vput(vp);
-	NCPVODEBUG("v_use after vput: %d\n",vp->v_usecount);
-	vrele(vp);
-	NCPVODEBUG("v_use after vrele: %d\n",vp->v_usecount);
-	vgone(vp);
-	NCPVODEBUG("v_gone finished !!!!\n");
 	conn = NWFSTOCONN(nmp);
 	ncp_conn_puthandle(nmp->connh,NULL,0);
 	if (ncp_conn_lock(conn,p,p->p_ucred,NCPM_WRITE | NCPM_EXECUTE) == 0) {
diff --git a/sys/fs/portalfs/portal_vfsops.c b/sys/fs/portalfs/portal_vfsops.c
index 5146456..423cd38 100644
--- a/sys/fs/portalfs/portal_vfsops.c
+++ b/sys/fs/portalfs/portal_vfsops.c
@@ -155,7 +155,6 @@ portal_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *rootvp = VFSTOPORTAL(mp)->pm_root;
 	int error, flags = 0;
 
 
@@ -172,21 +171,12 @@ portal_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp, 1))
 		return (EBUSY);
 #endif
-	if (rootvp->v_usecount > 1)
-		return (EBUSY);
-	error = vflush(mp, rootvp, flags);
+	/* There is 1 extra root vnode reference (pm_root). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(rootvp);
-	/*
 	 * Shutdown the socket.  This will cause the select in the
 	 * daemon to wake up, and then the accept will get ECONNABORTED
 	 * which it interprets as a request to go and bury itself.
diff --git a/sys/fs/smbfs/smbfs_vfsops.c b/sys/fs/smbfs/smbfs_vfsops.c
index c0f9ee1..b5196c0 100644
--- a/sys/fs/smbfs/smbfs_vfsops.c
+++ b/sys/fs/smbfs/smbfs_vfsops.c
@@ -254,7 +254,6 @@ static int
 smbfs_unmount(struct mount *mp, int mntflags, struct proc *p)
 {
 	struct smbmount *smp = VFSTOSMBFS(mp);
-	struct vnode *vp;
 	struct smb_cred scred;
 	int error, flags;
 
@@ -262,22 +261,10 @@ smbfs_unmount(struct mount *mp, int mntflags, struct proc *p)
 	flags = 0;
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
-	error = VFS_ROOT(mp, &vp);
+	/* There is 1 extra root vnode reference from smbfs_mount(). */
+	error = vflush(mp, 1, flags);
 	if (error)
-		return (error);
-	if (vp->v_usecount > 2) {
-		printf("smbfs_unmount: usecnt=%d\n", vp->v_usecount);
-		vput(vp);
-		return EBUSY;
-	}
-	error = vflush(mp, vp, flags);
-	if (error) {
-		vput(vp);
 		return error;
-	}
-	vput(vp);
-	vrele(vp);
-	vgone(vp);
 	smb_makescred(&scred, p, p->p_ucred);
 	smb_share_put(smp->sm_share, &scred);
 	mp->mnt_data = (qaddr_t)0;
diff --git a/sys/fs/umapfs/umap_vfsops.c b/sys/fs/umapfs/umap_vfsops.c
index e86467d..280ded9 100644
--- a/sys/fs/umapfs/umap_vfsops.c
+++ b/sys/fs/umapfs/umap_vfsops.c
@@ -260,7 +260,6 @@ umapfs_unmount(mp, mntflags, p)
 	int mntflags;
 	struct proc *p;
 {
-	struct vnode *umapm_rootvp = MOUNTTOUMAPMOUNT(mp)->umapm_rootvp;
 	int error;
 	int flags = 0;
 
@@ -281,23 +280,11 @@ umapfs_unmount(mp, mntflags, p)
 	if (mntinvalbuf(mp, 1))
 		return (EBUSY);
 #endif
-	if (umapm_rootvp->v_usecount > 1)
-		return (EBUSY);
-	error = vflush(mp, umapm_rootvp, flags);
+	/* There is 1 extra root vnode reference (umapm_rootvp). */
+	error = vflush(mp, 1, flags);
 	if (error)
 		return (error);
 
-#ifdef DEBUG
-	vprint("alias root of lower", umapm_rootvp);
-#endif
-	/*
-	 * Release reference on underlying root vnode
-	 */
-	vrele(umapm_rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(umapm_rootvp);
 	/*
 	 * Finally, throw away the umap_mount structure
 	 */
diff --git a/sys/fs/unionfs/union_vfsops.c b/sys/fs/unionfs/union_vfsops.c
index 5018f09..d38f31e 100644
--- a/sys/fs/unionfs/union_vfsops.c
+++ b/sys/fs/unionfs/union_vfsops.c
@@ -307,7 +307,6 @@ union_unmount(mp, mntflags, p)
 	struct proc *p;
 {
 	struct union_mount *um = MOUNTTOUNIONMOUNT(mp);
-	struct vnode *um_rootvp;
 	int error;
 	int freeing;
 	int flags = 0;
@@ -317,9 +316,6 @@ union_unmount(mp, mntflags, p)
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
 
-	if ((error = union_root(mp, &um_rootvp)) != 0)
-		return (error);
-
 	/*
 	 * Keep flushing vnodes from the mount list.
 	 * This is needed because of the un_pvp held
@@ -329,14 +325,13 @@ union_unmount(mp, mntflags, p)
 	 * (d) times, where (d) is the maximum tree depth
 	 * in the filesystem.
 	 */
-	for (freeing = 0; vflush(mp, um_rootvp, flags) != 0;) {
+	for (freeing = 0; (error = vflush(mp, 0, flags)) != 0;) {
 		struct vnode *vp;
 		int n;
 
 		/* count #vnodes held on mount list */
-		for (n = 0, vp = LIST_FIRST(&mp->mnt_vnodelist);
-				vp != NULLVP;
-				vp = LIST_NEXT(vp, v_mntvnodes))
+		n = 0;
+		LIST_FOREACH(vp, &mp->mnt_vnodelist, v_mntvnodes)
 			n++;
 
 		/* if this is unchanged then stop */
@@ -347,15 +342,10 @@ union_unmount(mp, mntflags, p)
 		freeing = n;
 	}
 
-	/* At this point the root vnode should have a single reference */
-	if (um_rootvp->v_usecount > 1) {
-		vput(um_rootvp);
-		return (EBUSY);
-	}
+	/* If the most recent vflush failed, the filesystem is still busy. */
+	if (error)
+		return (error);
 
-#ifdef DEBUG
-	vprint("union root", um_rootvp);
-#endif	 
 	/*
 	 * Discard references to upper and lower target vnodes.
 	 */
@@ -364,14 +354,6 @@ union_unmount(mp, mntflags, p)
 	vrele(um->um_uppervp);
 	crfree(um->um_cred);
 	/*
-	 * Release reference on underlying root vnode
-	 */
-	vput(um_rootvp);
-	/*
-	 * And blow it away for future re-use
-	 */
-	vgone(um_rootvp);
-	/*
 	 * Finally, throw away the union_mount structure
 	 */
 	free(mp->mnt_data, M_UNIONFSMNT);	/* XXX */