Block secondary writes while expunging active unlinked files.

Fix detection of active unlinked files by checking VI_OWEINACT and
VI_DOINGINACT in addition to v_usecount.

Defer inactive handling for unlinked files if the file system is mostly
suspended (secondary writes being blocked).

Perform deferred inactive handling after the file system is resumed.

5 files changed, 119 insertions, 5 deletions

diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 79a233d..f598360 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -982,7 +982,7 @@ vn_start_secondary_write(vp, mpp, flags)
 	if ((mp = *mpp) == NULL)
 		return (0);
 	MNT_ILOCK(mp);
-	if ((mp->mnt_kern_flag & MNTK_SUSPENDED) == 0) {
+	if ((mp->mnt_kern_flag & (MNTK_SUSPENDED | MNTK_SUSPEND2)) == 0) {
 		mp->mnt_secondary_writes++;
 		mp->mnt_secondary_accwrites++;
 		MNT_IUNLOCK(mp);
@@ -1087,7 +1087,8 @@ vfs_write_resume(mp)
 
 	MNT_ILOCK(mp);
 	if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) {
-		mp->mnt_kern_flag &= ~(MNTK_SUSPEND | MNTK_SUSPENDED);
+		mp->mnt_kern_flag &= ~(MNTK_SUSPEND | MNTK_SUSPEND2 |
+				       MNTK_SUSPENDED);
 		wakeup(&mp->mnt_writeopcount);
 		wakeup(&mp->mnt_flag);
 	}
diff --git a/sys/sys/mount.h b/sys/sys/mount.h
index ea22326..5c4bb87 100644
--- a/sys/sys/mount.h
+++ b/sys/sys/mount.h
@@ -304,6 +304,7 @@ void          __mnt_vnode_markerfree(struct vnode **mvp, struct mount *mp);
 #define MNTK_UNMOUNT	0x01000000	/* unmount in progress */
 #define	MNTK_MWAIT	0x02000000	/* waiting for unmount to finish */
 #define	MNTK_SUSPEND	0x08000000	/* request write suspension */
+#define	MNTK_SUSPEND2	0x04000000	/* block secondary writes */
 #define	MNTK_SUSPENDED	0x10000000	/* write operations are suspended */
 #define	MNTK_MPSAFE	0x20000000	/* Filesystem is MPSAFE. */
 #define	MNTK_NOKNOTE	0x80000000	/* Don't send KNOTEs from VOP hooks */
diff --git a/sys/ufs/ffs/ffs_snapshot.c b/sys/ufs/ffs/ffs_snapshot.c
index fb69fc2..fcd713a 100644
--- a/sys/ufs/ffs/ffs_snapshot.c
+++ b/sys/ufs/ffs/ffs_snapshot.c
@@ -159,6 +159,7 @@ static int snapacct_ufs2(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *,
 static int mapacct_ufs2(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *,
     struct fs *, ufs_lbn_t, int);
 static int readblock(struct vnode *vp, struct buf *, ufs2_daddr_t);
+static void process_deferred_inactive(struct mount *);
 
 /*
  * To ensure the consistency of snapshots across crashes, we must
@@ -268,10 +269,12 @@ restart:
 	}
 	VOP_LEASE(nd.ni_dvp, td, KERNCRED, LEASE_WRITE);
 	error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vat);
+	vhold(nd.ni_dvp);
 	vput(nd.ni_dvp);
 	if (error) {
 		NDFREE(&nd, NDF_ONLY_PNBUF);
 		vn_finished_write(wrtmp);
+		vdrop(nd.ni_dvp);
 		return (error);
 	}
 	vp = nd.ni_vp;
@@ -496,7 +499,9 @@ loop:
 		VI_LOCK(xvp);
 		MNT_IUNLOCK(mp);
 		if ((xvp->v_iflag & VI_DOOMED) ||
-		    xvp->v_usecount == 0 || xvp->v_type == VNON ||
+		    (xvp->v_usecount == 0 &&
+		     (xvp->v_iflag & (VI_OWEINACT | VI_DOINGINACT)) == 0) ||
+		    xvp->v_type == VNON ||
 		    (VTOI(xvp)->i_flags & SF_SNAPSHOT)) {
 			VI_UNLOCK(xvp);
 			MNT_ILOCK(mp);
@@ -511,22 +516,36 @@ loop:
 			MNT_ILOCK(mp);
 			continue;
 		}
+		vholdl(xvp);
 		if (vn_lock(xvp, LK_EXCLUSIVE | LK_INTERLOCK, td) != 0) {
 			MNT_ILOCK(mp);
 			MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp);
+			vdrop(xvp);
 			goto loop;
 		}
+		VI_LOCK(xvp);
+		if (xvp->v_usecount == 0 &&
+		    (xvp->v_iflag & (VI_OWEINACT | VI_DOINGINACT)) == 0) {
+			VI_UNLOCK(xvp);
+			VOP_UNLOCK(xvp, 0, td);
+			vdrop(xvp);
+			MNT_ILOCK(mp);
+			continue;
+		}
+		VI_UNLOCK(xvp);
 		if (snapdebug)
 			vprint("ffs_snapshot: busy vnode", xvp);
 		if (VOP_GETATTR(xvp, &vat, td->td_ucred, td) == 0 &&
 		    vat.va_nlink > 0) {
 			VOP_UNLOCK(xvp, 0, td);
+			vdrop(xvp);
 			MNT_ILOCK(mp);
 			continue;
 		}
 		xp = VTOI(xvp);
 		if (ffs_checkfreefile(copy_fs, vp, xp->i_number)) {
 			VOP_UNLOCK(xvp, 0, td);
+			vdrop(xvp);
 			MNT_ILOCK(mp);
 			continue;
 		}
@@ -557,6 +576,7 @@ loop:
 			error = ffs_freefile(ump, copy_fs, vp, xp->i_number,
 			    xp->i_mode);
 		VOP_UNLOCK(xvp, 0, td);
+		vdrop(xvp);
 		if (error) {
 			free(copy_fs->fs_csp, M_UFSMNT);
 			bawrite(sbp);
@@ -567,6 +587,7 @@ loop:
 		MNT_ILOCK(mp);
 	}
 	MNT_IUNLOCK(mp);
+	vdrop(nd.ni_dvp);
 	/*
 	 * If there already exist snapshots on this filesystem, grab a
 	 * reference to their shared lock. If this is the first snapshot
@@ -783,6 +804,7 @@ out:
 	else
 		VOP_UNLOCK(vp, 0, td);
 	vn_finished_write(wrtmp);
+	process_deferred_inactive(mp);
 	return (error);
 }
 
@@ -2237,4 +2259,69 @@ readblock(vp, bp, lbn)
 	return (bp->b_error);
 }
 
+
+/*
+ * Process file deletes that were deferred by ufs_inactive() due to
+ * the file system being suspended.
+ */
+static void
+process_deferred_inactive(struct mount *mp)
+{
+	struct vnode *vp, *mvp;
+	struct thread *td;
+	int error;
+
+	td = curthread;
+	(void) vn_start_secondary_write(NULL, &mp, V_WAIT);
+	MNT_ILOCK(mp);
+ loop:
+	MNT_VNODE_FOREACH(vp, mp, mvp) {
+		VI_LOCK(vp);
+		if ((vp->v_iflag & (VI_DOOMED | VI_OWEINACT)) != VI_OWEINACT ||
+		    vp->v_usecount > 0 ||
+		    vp->v_type == VNON) {
+			VI_UNLOCK(vp);
+			continue;
+		}
+		MNT_IUNLOCK(mp);
+		vholdl(vp);
+		error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK, td);
+		if (error != 0) {
+			vdrop(vp);
+			MNT_ILOCK(mp);
+			if (error == ENOENT)
+				continue;	/* vnode recycled */
+			MNT_VNODE_FOREACH_ABORT_ILOCKED(mp, mvp);
+			goto loop;
+		}
+		VI_LOCK(vp);
+		if ((vp->v_iflag & VI_OWEINACT) == 0) {
+			VI_UNLOCK(vp);
+			VOP_UNLOCK(vp, 0, td);
+			vdrop(vp);
+			MNT_ILOCK(mp);
+			continue;
+		}
+		
+		VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp,
+			 ("process_deferred_inactive: "
+			  "recursed on VI_DOINGINACT"));
+		vp->v_iflag |= VI_DOINGINACT;
+		vp->v_iflag &= ~VI_OWEINACT;
+		VI_UNLOCK(vp);
+		(void) VOP_INACTIVE(vp, td);
+		VI_LOCK(vp);
+		VNASSERT(vp->v_iflag & VI_DOINGINACT, vp,
+			 ("process_deferred_inactive: lost VI_DOINGINACT"));
+		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
+			 ("process_deferred_inactive: got VI_OWEINACT"));
+		vp->v_iflag &= ~VI_DOINGINACT;
+		VI_UNLOCK(vp);
+		VOP_UNLOCK(vp, 0, td);
+		vdrop(vp);
+		MNT_ILOCK(mp);
+	}
+	MNT_IUNLOCK(mp);
+	vn_finished_secondary_write(mp);
+}
 #endif
diff --git a/sys/ufs/ffs/ffs_vfsops.c b/sys/ufs/ffs/ffs_vfsops.c
index edfc8eb..6ad9bc7 100644
--- a/sys/ufs/ffs/ffs_vfsops.c
+++ b/sys/ufs/ffs/ffs_vfsops.c
@@ -1224,7 +1224,7 @@ loop:
 					  secondary_accwrites) != 0)
 			goto loop;	/* More work needed */
 		mtx_assert(MNT_MTX(mp), MA_OWNED);
-		mp->mnt_kern_flag |= MNTK_SUSPENDED;
+		mp->mnt_kern_flag |= MNTK_SUSPEND2 | MNTK_SUSPENDED;
 		MNT_IUNLOCK(mp);
 		suspended = 1;
 	} else
diff --git a/sys/ufs/ufs/ufs_inode.c b/sys/ufs/ufs/ufs_inode.c
index 091756c..b85c9a3 100644
--- a/sys/ufs/ufs/ufs_inode.c
+++ b/sys/ufs/ufs/ufs_inode.c
@@ -86,7 +86,32 @@ ufs_inactive(ap)
 	if (ip->i_effnlink == 0 && DOINGSOFTDEP(vp))
 		softdep_releasefile(ip);
 	if (ip->i_nlink <= 0 && (vp->v_mount->mnt_flag & MNT_RDONLY) == 0) {
-		(void) vn_start_secondary_write(vp, &mp, V_WAIT);
+	loop:
+		if (vn_start_secondary_write(vp, &mp, V_NOWAIT) != 0) {
+			/* Cannot delete file while file system is suspended */
+			if ((vp->v_iflag & VI_DOOMED) != 0) {
+				/* Cannot return before file is deleted */
+				(void) vn_start_secondary_write(vp, &mp,
+								V_WAIT);
+			} else {
+				MNT_ILOCK(mp);
+				if ((mp->mnt_kern_flag &
+				     (MNTK_SUSPEND2 | MNTK_SUSPENDED)) == 0) {
+					MNT_IUNLOCK(mp);
+					goto loop;
+				}
+				/*
+				 * Fail to inactivate vnode now and
+				 * let ffs_snapshot() clean up after
+				 * it has resumed the file system.
+				 */
+				VI_LOCK(vp);
+				vp->v_iflag |= VI_OWEINACT;
+				VI_UNLOCK(vp);
+				MNT_IUNLOCK(mp);
+				return (0);
+			}
+		}
 #ifdef QUOTA
 		if (!getinoquota(ip))
 			(void)chkiq(ip, -1, NOCRED, FORCE);