Try to unbreak the vnode locking around vop_reclaim() (based mostly on

patch from kan@).

Pull bufobj_invalbuf() out of vinvalbuf() and make g_vfs call it on
close.  This is not yet a generally safe function, but for this very
specific use it is safe.  This solves the problem with buffers not
being flushed by unmount or after failed mount attempts.

7 files changed, 40 insertions, 39 deletions

diff --git a/sys/geom/geom_vfs.c b/sys/geom/geom_vfs.c
index c875897..d8f6c1d 100644
--- a/sys/geom/geom_vfs.c
+++ b/sys/geom/geom_vfs.c
@@ -153,6 +153,7 @@ g_vfs_open(struct vnode *vp, struct g_consumer **cpp, const char *fsname, int wr
 	bo->bo_ops = g_vfs_bufops;
 	bo->bo_private = cp;
 	bo->bo_bsize = pp->sectorsize;
+	gp->softc = bo;
 
 	return (error);
 }
@@ -161,9 +162,12 @@ void
 g_vfs_close(struct g_consumer *cp, struct thread *td)
 {
 	struct g_geom *gp;
+	struct bufobj *bo;
 
 	g_topology_assert();
 
 	gp = cp->geom;
+	bo = gp->softc;
+	bufobj_invalbuf(bo, V_SAVE, td, 0, 0);
 	g_wither_geom_close(gp, ENXIO);
 }
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 7c83674..d72227a 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -84,7 +84,7 @@ static MALLOC_DEFINE(M_NETADDR, "Export Host", "Export host address structure");
 static void	delmntque(struct vnode *vp);
 static void	insmntque(struct vnode *vp, struct mount *mp);
 static void	vlruvp(struct vnode *vp);
-static int	flushbuflist(struct bufv *bufv, int flags, struct vnode *vp,
+static int	flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo,
 		    int slpflag, int slptimeo);
 static void	syncer_shutdown(void *arg, int howto);
 static int	vtryrecycle(struct vnode *vp);
@@ -927,22 +927,14 @@ insmntque(struct vnode *vp, struct mount *mp)
 }
 
 /*
- * Flush out and invalidate all buffers associated with a vnode.
+ * Flush out and invalidate all buffers associated with a bufobj
  * Called with the underlying object locked.
  */
 int
-vinvalbuf(vp, flags, td, slpflag, slptimeo)
-	struct vnode *vp;
-	int flags;
-	struct thread *td;
-	int slpflag, slptimeo;
+bufobj_invalbuf(struct bufobj *bo, int flags, struct thread *td, int slpflag, int slptimeo)
 {
 	int error;
-	struct bufobj *bo;
-
-	ASSERT_VOP_LOCKED(vp, "vinvalbuf");
 
-	bo = &vp->v_bufobj;
 	BO_LOCK(bo);
 	if (flags & V_SAVE) {
 		error = bufobj_wwait(bo, slpflag, slptimeo);
@@ -970,10 +962,10 @@ vinvalbuf(vp, flags, td, slpflag, slptimeo)
 	 */
 	do {
 		error = flushbuflist(&bo->bo_clean,
-		    flags, vp, slpflag, slptimeo);
+		    flags, bo, slpflag, slptimeo);
 		if (error == 0)
 			error = flushbuflist(&bo->bo_dirty,
-			    flags, vp, slpflag, slptimeo);
+			    flags, bo, slpflag, slptimeo);
 		if (error != 0 && error != EAGAIN) {
 			BO_UNLOCK(bo);
 			return (error);
@@ -990,7 +982,7 @@ vinvalbuf(vp, flags, td, slpflag, slptimeo)
 		BO_UNLOCK(bo);
 		if (bo->bo_object != NULL) {
 			VM_OBJECT_LOCK(bo->bo_object);
-			vm_object_pip_wait(bo->bo_object, "vnvlbx");
+			vm_object_pip_wait(bo->bo_object, "bovlbx");
 			VM_OBJECT_UNLOCK(bo->bo_object);
 		}
 		BO_LOCK(bo);
@@ -1018,21 +1010,31 @@ vinvalbuf(vp, flags, td, slpflag, slptimeo)
 }
 
 /*
+ * Flush out and invalidate all buffers associated with a vnode.
+ * Called with the underlying object locked.
+ */
+int
+vinvalbuf(struct vnode *vp, int flags, struct thread *td, int slpflag, int slptimeo)
+{
+
+	ASSERT_VOP_LOCKED(vp, "vinvalbuf");
+	return (bufobj_invalbuf(&vp->v_bufobj, flags, td, slpflag, slptimeo));
+}
+
+/*
  * Flush out buffers on the specified list.
  *
  */
 static int
-flushbuflist(bufv, flags, vp, slpflag, slptimeo)
+flushbuflist(bufv, flags, bo, slpflag, slptimeo)
 	struct bufv *bufv;
 	int flags;
-	struct vnode *vp;
+	struct bufobj *bo;
 	int slpflag, slptimeo;
 {
 	struct buf *bp, *nbp;
 	int retval, error;
-	struct bufobj *bo;
 
-	bo = &vp->v_bufobj;
 	ASSERT_BO_LOCKED(bo);
 
 	retval = 0;
@@ -1049,31 +1051,23 @@ flushbuflist(bufv, flags, vp, slpflag, slptimeo)
 			BO_LOCK(bo);
 			return (error != ENOLCK ? error : EAGAIN);
 		}
+		if (bp->b_bufobj != bo) {	/* XXX: necessary ? */
+			BO_LOCK(bo);
+			return (EAGAIN);
+		}
 		/*
 		 * XXX Since there are no node locks for NFS, I
 		 * believe there is a slight chance that a delayed
 		 * write will occur while sleeping just above, so
-		 * check for it.  Note that vfs_bio_awrite expects
-		 * buffers to reside on a queue, while bwrite and
-		 * brelse do not.
+		 * check for it.
 		 */
 		if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) &&
-			(flags & V_SAVE)) {
-
-			if (bp->b_vp == vp) {
-				if (bp->b_flags & B_CLUSTEROK) {
-					vfs_bio_awrite(bp);
-				} else {
-					bremfree(bp);
-					bp->b_flags |= B_ASYNC;
-					bwrite(bp);
-				}
-			} else {
-				bremfree(bp);
-				(void) bwrite(bp);
-			}
+		    (flags & V_SAVE)) {
+			bremfree(bp);
+			bp->b_flags |= B_ASYNC;
+			bwrite(bp);
 			BO_LOCK(bo);
-			return (EAGAIN);
+			return (EAGAIN);	/* XXX: why not loop ? */
 		}
 		bremfree(bp);
 		bp->b_flags |= (B_INVAL | B_NOCACHE | B_RELBUF);
diff --git a/sys/nfsclient/nfs_vnops.c b/sys/nfsclient/nfs_vnops.c
index 5a90982..1c41c93 100644
--- a/sys/nfsclient/nfs_vnops.c
+++ b/sys/nfsclient/nfs_vnops.c
@@ -538,7 +538,9 @@ nfs_close(struct vop_close_args *ap)
 		    error = nfs_flush(vp, MNT_WAIT, ap->a_td, cm);
 		    /* np->n_flag &= ~NMODIFIED; */
 		} else {
+		    VOP_LOCK(vp, LK_EXCLUSIVE, curthread);
 		    error = nfs_vinvalbuf(vp, V_SAVE, ap->a_td, 1);
+		    VOP_UNLOCK(vp, 0, curthread);
 		}
 	    }
  	    /* 
diff --git a/sys/sys/bufobj.h b/sys/sys/bufobj.h
index 7b0243e..caad00f 100644
--- a/sys/sys/bufobj.h
+++ b/sys/sys/bufobj.h
@@ -125,6 +125,7 @@ struct bufobj {
 
 void bufobj_wdrop(struct bufobj *bo);
 void bufobj_wref(struct bufobj *bo);
+int bufobj_invalbuf(struct bufobj *bo, int flags, struct thread *td, int slpflag, int slptimeo);
 int bufobj_wwait(struct bufobj *bo, int slpflag, int timeo);
 int bufsync(struct bufobj *bo, int waitfor, struct thread *td);
 
diff --git a/sys/ufs/ffs/ffs_vfsops.c b/sys/ufs/ffs/ffs_vfsops.c
index 0499224..6635e2b 100644
--- a/sys/ufs/ffs/ffs_vfsops.c
+++ b/sys/ufs/ffs/ffs_vfsops.c
@@ -794,7 +794,6 @@ ffs_mountfs(devvp, mp, td)
 out:
 	if (bp)
 		brelse(bp);
-	vinvalbuf(devvp, V_SAVE, td, 0, 0);
 	if (cp != NULL) {
 		DROP_GIANT();
 		g_topology_lock();
@@ -956,7 +955,6 @@ ffs_unmount(mp, mntflags, td)
 			return (error);
 		}
 	}
-	vinvalbuf(ump->um_devvp, V_SAVE, td, 0, 0);
 	DROP_GIANT();
 	g_topology_lock();
 	g_vfs_close(ump->um_cp, td);
diff --git a/sys/ufs/ufs/ufs_inode.c b/sys/ufs/ufs/ufs_inode.c
index 7db044c..4cebcd2 100644
--- a/sys/ufs/ufs/ufs_inode.c
+++ b/sys/ufs/ufs/ufs_inode.c
@@ -173,12 +173,12 @@ ufs_reclaim(ap)
 		}
 	}
 #endif
+	vnode_destroy_vobject(vp);
 #ifdef UFS_DIRHASH
 	if (ip->i_dirhash != NULL)
 		ufsdirhash_free(ip);
 #endif
 	UFS_IFREE(ump, ip);
 	vp->v_data = 0;
-	vnode_destroy_vobject(vp);
 	return (0);
 }
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index 0f9c3eb..af483ed 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -158,6 +158,7 @@ vnode_destroy_vobject(struct vnode *vp)
 	obj = vp->v_object;
 	if (obj == NULL)
 		return;
+	VOP_LOCK(vp, LK_EXCLUSIVE, curthread);
 	vp->v_object = NULL;
 	VM_OBJECT_LOCK(obj);
 	if (obj->ref_count == 0) {
@@ -180,6 +181,7 @@ vnode_destroy_vobject(struct vnode *vp)
 		vm_pager_deallocate(obj);
 		VM_OBJECT_UNLOCK(obj);
 	}
+	VOP_UNLOCK(vp, 0, curthread);
 }