3 files changed, 110 insertions, 6 deletions

diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index d0c9f41..34d5602 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -498,6 +498,16 @@ vfs_mount_destroy(struct mount *mp, struct thread *td)
 		}
 		printf("mount point write ops completed\n");
 	}
+	if (mp->mnt_secondary_writes > 0) {
+		printf("Waiting for mount point secondary write ops\n");
+		while (mp->mnt_secondary_writes > 0) {
+			mp->mnt_kern_flag |= MNTK_SUSPEND;
+			msleep(&mp->mnt_secondary_writes,
+			       MNT_MTX(mp),
+			       PZERO, "mntdestroy3", 0);
+		}
+		printf("mount point secondary write ops completed\n");
+	}
 	MNT_IUNLOCK(mp);
 	mp->mnt_vfc->vfc_refcount--;
 	if (!TAILQ_EMPTY(&mp->mnt_nvnodelist))
@@ -508,10 +518,13 @@ vfs_mount_destroy(struct mount *mp, struct thread *td)
 		wakeup(mp);
 	if (mp->mnt_writeopcount != 0)
 		panic("vfs_mount_destroy: nonzero writeopcount");
+	if (mp->mnt_secondary_writes != 0)
+		panic("vfs_mount_destroy: nonzero secondary_writes");
 	if (mp->mnt_nvnodelistsize != 0)
 		panic("vfs_mount_destroy: nonzero nvnodelistsize");
 	mp->mnt_writeopcount = -1000;
 	mp->mnt_nvnodelistsize = -1000;
+	mp->mnt_secondary_writes = -1000;
 	MNT_IUNLOCK(mp);
 	mtx_destroy(&mp->mnt_mtx);
 #ifdef MAC
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index add6991..c286f0b 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -2052,12 +2052,19 @@ vrele(struct vnode *vp)
 	 * We must call VOP_INACTIVE with the node locked. Mark
 	 * as VI_DOINGINACT to avoid recursion.
 	 */
+	vp->v_iflag |= VI_OWEINACT;
 	if (vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK, td) == 0) {
 		VI_LOCK(vp);
-		vinactive(vp, td);
+		if (vp->v_usecount > 0)
+			vp->v_iflag &= ~VI_OWEINACT;
+		if (vp->v_iflag & VI_OWEINACT)
+			vinactive(vp, td);
 		VOP_UNLOCK(vp, 0, td);
-	} else
+	} else {
 		VI_LOCK(vp);
+		if (vp->v_usecount > 0)
+			vp->v_iflag &= ~VI_OWEINACT;
+	}
 	vdropl(vp);
 }
 
@@ -2104,9 +2111,14 @@ vput(struct vnode *vp)
 	if (VOP_ISLOCKED(vp, NULL) != LK_EXCLUSIVE) {
 		error = VOP_LOCK(vp, LK_EXCLUPGRADE|LK_INTERLOCK|LK_NOWAIT, td);
 		VI_LOCK(vp);
-		if (error)
+		if (error) {
+			if (vp->v_usecount > 0)
+				vp->v_iflag &= ~VI_OWEINACT;
 			goto done;
+		}
 	}
+	if (vp->v_usecount > 0)
+		vp->v_iflag &= ~VI_OWEINACT;
 	if (vp->v_iflag & VI_OWEINACT)
 		vinactive(vp, td);
 	VOP_UNLOCK(vp, 0, td);
@@ -2368,6 +2380,7 @@ vgonel(struct vnode *vp)
 	struct thread *td;
 	int oweinact;
 	int active;
+	struct mount *mp;
 
 	CTR1(KTR_VFS, "vgonel: vp %p", vp);
 	ASSERT_VOP_LOCKED(vp, "vgonel");
@@ -2396,8 +2409,9 @@ vgonel(struct vnode *vp)
 	 * Clean out any buffers associated with the vnode.
 	 * If the flush fails, just toss the buffers.
 	 */
+	mp = NULL;
 	if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd))
-		(void) vn_write_suspend_wait(vp, NULL, V_WAIT);
+		(void) vn_start_secondary_write(vp, &mp, V_WAIT);
 	if (vinvalbuf(vp, V_SAVE, td, 0, 0) != 0)
 		vinvalbuf(vp, 0, td, 0, 0);
 
@@ -2418,6 +2432,8 @@ vgonel(struct vnode *vp)
 	 */
 	if (VOP_RECLAIM(vp, td))
 		panic("vgone: cannot reclaim");
+	if (mp != NULL)
+		vn_finished_secondary_write(mp);
 	VNASSERT(vp->v_object == NULL, vp,
 	    ("vop_reclaim left v_object vp=%p, tag=%s", vp, vp->v_tag));
 	/*
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index fda74f1..79a233d 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -951,6 +951,58 @@ vn_write_suspend_wait(vp, mp, flags)
 }
 
 /*
+ * Secondary suspension. Used by operations such as vop_inactive
+ * routines that are needed by the higher level functions. These
+ * are allowed to proceed until all the higher level functions have
+ * completed (indicated by mnt_writeopcount dropping to zero). At that
+ * time, these operations are halted until the suspension is over.
+ */
+int
+vn_start_secondary_write(vp, mpp, flags)
+	struct vnode *vp;
+	struct mount **mpp;
+	int flags;
+{
+	struct mount *mp;
+	int error;
+
+ retry:
+	if (vp != NULL) {
+		if ((error = VOP_GETWRITEMOUNT(vp, mpp)) != 0) {
+			*mpp = NULL;
+			if (error != EOPNOTSUPP)
+				return (error);
+			return (0);
+		}
+	}
+	/*
+	 * If we are not suspended or have not yet reached suspended
+	 * mode, then let the operation proceed.
+	 */
+	if ((mp = *mpp) == NULL)
+		return (0);
+	MNT_ILOCK(mp);
+	if ((mp->mnt_kern_flag & MNTK_SUSPENDED) == 0) {
+		mp->mnt_secondary_writes++;
+		mp->mnt_secondary_accwrites++;
+		MNT_IUNLOCK(mp);
+		return (0);
+	}
+	if (flags & V_NOWAIT) {
+		MNT_IUNLOCK(mp);
+		return (EWOULDBLOCK);
+	}
+	/*
+	 * Wait for the suspension to finish.
+	 */
+	error = msleep(&mp->mnt_flag, MNT_MTX(mp),
+		       (PUSER - 1) | (flags & PCATCH) | PDROP, "suspfs", 0);
+	if (error == 0)
+		goto retry;
+	return (error);
+}
+
+/*
  * Filesystem write operation has completed. If we are suspending and this
  * operation is the last one, notify the suspender that the suspension is
  * now in effect.
@@ -971,6 +1023,30 @@ vn_finished_write(mp)
 	MNT_IUNLOCK(mp);
 }
 
+
+/*
+ * Filesystem secondary write operation has completed. If we are
+ * suspending and this operation is the last one, notify the suspender
+ * that the suspension is now in effect.
+ */
+void
+vn_finished_secondary_write(mp)
+	struct mount *mp;
+{
+	if (mp == NULL)
+		return;
+	MNT_ILOCK(mp);
+	mp->mnt_secondary_writes--;
+	if (mp->mnt_secondary_writes < 0)
+		panic("vn_finished_secondary_write: neg cnt");
+	if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 &&
+	    mp->mnt_secondary_writes <= 0)
+		wakeup(&mp->mnt_secondary_writes);
+	MNT_IUNLOCK(mp);
+}
+
+
+
 /*
  * Request a filesystem to suspend write operations.
  */
@@ -991,12 +1067,11 @@ vfs_write_suspend(mp)
 		    MNT_MTX(mp), (PUSER - 1)|PDROP, "suspwt", 0);
 	else
 		MNT_IUNLOCK(mp);
-	if ((error = VFS_SYNC(mp, MNT_WAIT, td)) != 0) {
+	if ((error = VFS_SYNC(mp, MNT_SUSPEND, td)) != 0) {
 		vfs_write_resume(mp);
 		return (error);
 	}
 	MNT_ILOCK(mp);
-	mp->mnt_kern_flag |= MNTK_SUSPENDED;
 unlock:
 	MNT_IUNLOCK(mp);
 	return (error);