- Add a ref count to the mount structure. Sleep for up to 3 seconds in

   vfs_mount_destroy waiting for this ref to hit 0.  We don't print an
   error if we are rebooting as the root mount always retains some refernces
   by init proc.
 - Acquire a mnt ref for every vnode allocated to a mount point.  Drop this
   ref only once vdestroy() has been called and the mount has been freed.
 - No longer NULL the v_mount pointer in delmntque() so that we may release
   the ref after vgone() has been called.  This allows us to guarantee
   that the mount point structure will be valid until the last vnode has
   lost its last ref.
 - Fix a few places that rely on checking v_mount to detect recycling.

Sponsored by:	Isilon Systems, Inc.
MFC After:	1 week

3 files changed, 61 insertions, 15 deletions

diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index 2015a43..0e7aa86 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -405,6 +405,24 @@ nmount(td, uap)
  * Various utility functions
  */
 
+void
+vfs_ref(struct mount *mp)
+{
+
+	MNT_ILOCK(mp);
+	MNT_REF(mp);
+	MNT_IUNLOCK(mp);
+}
+
+void
+vfs_rel(struct mount *mp)
+{
+
+	MNT_ILOCK(mp);
+	MNT_REL(mp);
+	MNT_IUNLOCK(mp);
+}
+
 /*
  * Allocate and initialize the mount point struct.
  */
@@ -420,9 +438,10 @@ vfs_mount_alloc(struct vnode *vp, struct vfsconf *vfsp,
 	mtx_init(&mp->mnt_mtx, "struct mount mtx", NULL, MTX_DEF);
 	lockinit(&mp->mnt_lock, PVFS, "vfslock", 0, 0);
 	(void) vfs_busy(mp, LK_NOWAIT, 0, td);
+	mp->mnt_ref = 0;
 	mp->mnt_op = vfsp->vfc_vfsops;
 	mp->mnt_vfc = vfsp;
-	vfsp->vfc_refcount++;
+	vfsp->vfc_refcount++;	/* XXX Unlocked */
 	mp->mnt_stat.f_type = vfsp->vfc_typenum;
 	mp->mnt_flag |= vfsp->vfc_flags & MNT_VISFLAGMASK;
 	strlcpy(mp->mnt_stat.f_fstypename, vfsp->vfc_name, MFSNAMELEN);
@@ -445,8 +464,19 @@ vfs_mount_alloc(struct vnode *vp, struct vfsconf *vfsp,
 static void
 vfs_mount_destroy(struct mount *mp, struct thread *td)
 {
+	int i;
 
 	MNT_ILOCK(mp);
+	for (i = 0; mp->mnt_ref && i < 3; i++)
+		msleep(mp, MNT_MTX(mp), PVFS, "mntref", hz);
+	/*
+	 * This will always cause a 3 second delay in rebooting due to
+	 * refs on the root mountpoint that never go away.  Most of these
+	 * are held by init which never exits.
+	 */
+	if (i == 3 && (!rebooting || bootverbose))
+		printf("Mount point %s had %d dangling refs\n",
+		    mp->mnt_stat.f_mntonname, mp->mnt_ref);
 	if (mp->mnt_holdcnt != 0) {
 		printf("Waiting for mount point to be unheld\n");
 		while (mp->mnt_holdcnt != 0) {
@@ -1305,8 +1335,8 @@ devfs_fixup(struct thread *td)
 	TAILQ_INSERT_TAIL(&mountlist, mp, mnt_list);
 	mtx_unlock(&mountlist_mtx);
 	VOP_UNLOCK(vp, 0, td);
-	vfs_unbusy(mp, td);
 	vput(dvp);
+	vfs_unbusy(mp, td);
 
 	/* Unlink the no longer needed /dev/dev -> / symlink */
 	kern_unlink(td, "/dev/dev", UIO_SYSSPACE);
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index ca85214..5017c72 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -785,6 +785,8 @@ vdestroy(struct vnode *vp)
 		mtx_destroy(&vp->v_pollinfo->vpi_lock);
 		uma_zfree(vnodepoll_zone, vp->v_pollinfo);
 	}
+	if (vp->v_mount)
+		vfs_rel(vp->v_mount);
 #ifdef INVARIANTS
 	/* XXX Elsewhere we can detect an already freed vnode via NULL v_op. */
 	vp->v_op = NULL;
@@ -918,7 +920,6 @@ getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops,
 	else if (mp == NULL)
 		printf("NULL mp in getnewvnode()\n");
 #endif
-	delmntque(vp);
 	if (mp != NULL) {
 		insmntque(vp, mp);
 		bo->bo_bsize = mp->mnt_stat.f_iosize;
@@ -939,15 +940,15 @@ delmntque(struct vnode *vp)
 {
 	struct mount *mp;
 
-	if (vp->v_mount == NULL)
-		return;
 	mp = vp->v_mount;
+	if (mp == NULL)
+		return;
 	MNT_ILOCK(mp);
-	vp->v_mount = NULL;
 	VNASSERT(mp->mnt_nvnodelistsize > 0, vp,
 		("bad mount point vnode list size"));
 	TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
 	mp->mnt_nvnodelistsize--;
+	/* mnt ref is released in vdestroy. */
 	MNT_IUNLOCK(mp);
 }
 
@@ -960,12 +961,13 @@ insmntque(struct vnode *vp, struct mount *mp)
 
 	vp->v_mount = mp;
 	VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)"));
-	MNT_ILOCK(vp->v_mount);
+	MNT_ILOCK(mp);
+	MNT_REF(mp);
 	TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
 	VNASSERT(mp->mnt_nvnodelistsize >= 0, vp,
 		("neg mount point vnode list size"));
 	mp->mnt_nvnodelistsize++;
-	MNT_IUNLOCK(vp->v_mount);
+	MNT_IUNLOCK(mp);
 }
 
 /*
diff --git a/sys/sys/mount.h b/sys/sys/mount.h
index ccec2fc..6c741cb 100644
--- a/sys/sys/mount.h
+++ b/sys/sys/mount.h
@@ -134,21 +134,30 @@ struct vfsopt;
  * array of operations and an instance record.  The filesystems are
  * put on a doubly linked list.
  *
+ * Lock reference:
+ * 	m - mountlist_mtx
+ *	i - interlock
+ *	l - mnt_lock
+ *
+ * Unmarked fields are considered stable as long as a ref is held.
+ *
  */
 struct mount {
-	TAILQ_ENTRY(mount) mnt_list;		/* mount list */
+	TAILQ_ENTRY(mount) mnt_list;		/* (m) mount list */
 	struct vfsops	*mnt_op;		/* operations on fs */
 	struct vfsconf	*mnt_vfc;		/* configuration info */
 	struct vnode	*mnt_vnodecovered;	/* vnode we mounted on */
 	struct vnode	*mnt_syncer;		/* syncer vnode */
-	struct vnodelst	mnt_nvnodelist;		/* list of vnodes this mount */
 	struct lock	mnt_lock;		/* mount structure lock */
 	struct mtx	mnt_mtx;		/* mount structure interlock */
-	int		mnt_writeopcount;	/* write syscalls in progress */
+	int		mnt_ref;		/* (i) Reference count */
+	struct vnodelst	mnt_nvnodelist;		/* (i) list of vnodes */
+	int		mnt_nvnodelistsize;	/* (i) # of vnodes */
+	int		mnt_writeopcount;	/* (i) write syscalls pending */
+	int		mnt_kern_flag;		/* (i) kernel only flags */
 	u_int		mnt_flag;		/* flags shared with user */
 	struct vfsoptlist *mnt_opt;		/* current mount options */
 	struct vfsoptlist *mnt_optnew;		/* new options passed to fs */
-	int		mnt_kern_flag;		/* kernel only flags */
 	int		mnt_maxsymlinklen;	/* max size of short symlink */
 	struct statfs	mnt_stat;		/* cache of filesystem stats */
 	struct ucred	*mnt_cred;		/* credentials of mounter */
@@ -158,7 +167,6 @@ struct mount {
 	struct netexport *mnt_export;		/* export list */
 	struct label	*mnt_mntlabel;		/* MAC label for the mount */
 	struct label	*mnt_fslabel;		/* MAC label for the fs */
-	int		mnt_nvnodelistsize;	/* # of vnodes on this mount */
 	u_int		mnt_hashseed;		/* Random seed for vfs_hash */
 	int		mnt_markercnt;		/* marker vnodes in use */
 	int		mnt_holdcnt;		/* hold count */
@@ -186,6 +194,12 @@ void          __mnt_vnode_markerfree(struct vnode **mvp, struct mount *mp);
 #define	MNT_ILOCK(mp)	mtx_lock(&(mp)->mnt_mtx)
 #define	MNT_IUNLOCK(mp)	mtx_unlock(&(mp)->mnt_mtx)
 #define	MNT_MTX(mp)	(&(mp)->mnt_mtx)
+#define	MNT_REF(mp)	(mp)->mnt_ref++
+#define	MNT_REL(mp)	do {						\
+	(mp)->mnt_ref--;						\
+	if ((mp)->mnt_ref == 0)						\
+		wakeup((mp));						\
+} while (0)
 
 #endif /* _KERNEL */
 
@@ -629,9 +643,7 @@ int	vfs_filteropt(struct vfsoptlist *, const char **legal);
 int	vfs_scanopt(struct vfsoptlist *opts, const char *name, const char *fmt, ...);
 int	vfs_setpublicfs			    /* set publicly exported fs */
 	    (struct mount *, struct netexport *, struct export_args *);
-int	vfs_lock(struct mount *);         /* lock a vfs */
 void	vfs_msync(struct mount *, int);
-void	vfs_unlock(struct mount *);       /* unlock a vfs */
 int	vfs_busy(struct mount *, int, struct mtx *, struct thread *);
 int	vfs_export			 /* process mount export info */
 	    (struct mount *, struct export_args *);
@@ -643,6 +655,8 @@ int	vfs_modevent(module_t, int, void *);
 void	vfs_mount_error(struct mount *, const char *, ...);
 void	vfs_mountroot(void);			/* mount our root filesystem */
 void	vfs_mountedfrom(struct mount *, const char *from);
+void	vfs_ref(struct mount *);
+void	vfs_rel(struct mount *);
 int	vfs_suser(struct mount *, struct thread *);
 void	vfs_unbusy(struct mount *, struct thread *);
 void	vfs_unmountall(void);