This patch removes the VOP_BWRITE() vector.

VOP_BWRITE() was a hack which made it possible for NFS client
side to use struct buf with non-bio backing.

This patch takes a more general approach and adds a bp->b_op
vector where more methods can be added.

The success of this patch depends on bp->b_op being initialized
all relevant places for some value of "relevant" which is not
easy to determine.  For now the buffers have grown a b_magic
element which will make such issues a tiny bit easier to debug.

19 files changed, 74 insertions, 66 deletions

diff --git a/sys/coda/coda_vnops.c b/sys/coda/coda_vnops.c
index 0692737..e8829d3 100644
--- a/sys/coda/coda_vnops.c
+++ b/sys/coda/coda_vnops.c
@@ -137,7 +137,6 @@ struct vnodeopv_entry_desc coda_vnodeop_entries[] = {
     { &vop_islocked_desc, coda_islocked },	/* islocked */
     { &vop_pathconf_desc, coda_vop_error },	/* pathconf */
     { &vop_advlock_desc, coda_vop_nop },	/* advlock */
-    { &vop_bwrite_desc, coda_vop_error },	/* bwrite */
     { &vop_lease_desc, coda_vop_nop },		/* lease */
     { &vop_poll_desc, (vop_t *) vop_stdpoll },
     { &vop_getpages_desc, coda_fbsd_getpages },	/* pager intf.*/
diff --git a/sys/fs/coda/coda_vnops.c b/sys/fs/coda/coda_vnops.c
index 0692737..e8829d3 100644
--- a/sys/fs/coda/coda_vnops.c
+++ b/sys/fs/coda/coda_vnops.c
@@ -137,7 +137,6 @@ struct vnodeopv_entry_desc coda_vnodeop_entries[] = {
     { &vop_islocked_desc, coda_islocked },	/* islocked */
     { &vop_pathconf_desc, coda_vop_error },	/* pathconf */
     { &vop_advlock_desc, coda_vop_nop },	/* advlock */
-    { &vop_bwrite_desc, coda_vop_error },	/* bwrite */
     { &vop_lease_desc, coda_vop_nop },		/* lease */
     { &vop_poll_desc, (vop_t *) vop_stdpoll },
     { &vop_getpages_desc, coda_fbsd_getpages },	/* pager intf.*/
diff --git a/sys/fs/hpfs/hpfs_vnops.c b/sys/fs/hpfs/hpfs_vnops.c
index a52020f..500fd6c 100644
--- a/sys/fs/hpfs/hpfs_vnops.c
+++ b/sys/fs/hpfs/hpfs_vnops.c
@@ -1359,7 +1359,6 @@ struct vnodeopv_entry_desc hpfs_vnodeop_entries[] = {
 	{ &vop_getpages_desc, (vop_t *) hpfs_getpages },
 	{ &vop_putpages_desc, (vop_t *) hpfs_putpages },
 	{ &vop_strategy_desc, (vop_t *)hpfs_strategy },
-	{ &vop_bwrite_desc, (vop_t *)vop_stdbwrite },
 	{ &vop_read_desc, (vop_t *)hpfs_read },
 	{ &vop_write_desc, (vop_t *)hpfs_write },
 	{ &vop_ioctl_desc, (vop_t *)hpfs_ioctl },
diff --git a/sys/fs/ntfs/ntfs_vnops.c b/sys/fs/ntfs/ntfs_vnops.c
index 710c0d1..66ac769 100644
--- a/sys/fs/ntfs/ntfs_vnops.c
+++ b/sys/fs/ntfs/ntfs_vnops.c
@@ -875,7 +875,6 @@ struct vnodeopv_entry_desc ntfs_vnodeop_entries[] = {
 	{ &vop_getpages_desc, (vop_t *) ntfs_getpages },
 	{ &vop_putpages_desc, (vop_t *) ntfs_putpages },
 	{ &vop_strategy_desc, (vop_t *)ntfs_strategy },
-	{ &vop_bwrite_desc, (vop_t *)vop_stdbwrite },
 	{ &vop_read_desc, (vop_t *)ntfs_read },
 	{ &vop_write_desc, (vop_t *)ntfs_write },
 
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 4705903..87a17a8 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -58,6 +58,11 @@ static MALLOC_DEFINE(M_BIOBUF, "BIO buffer", "BIO buffer");
 
 struct	bio_ops bioops;		/* I/O operation notification */
 
+struct	buf_ops buf_ops_bio = {
+	"buf_ops_bio",
+	bwrite
+};
+
 struct buf *buf;		/* buffer header pool */
 struct swqueue bswlist;
 struct mtx buftimelock;		/* Interlock on setting prio and timo */
@@ -1685,6 +1690,8 @@ restart:
 		bp->b_bcount = 0;
 		bp->b_npages = 0;
 		bp->b_dirtyoff = bp->b_dirtyend = 0;
+		bp->b_magic = B_MAGIC_BIO;
+		bp->b_op = &buf_ops_bio;
 
 		LIST_INIT(&bp->b_dep);
 
@@ -2088,7 +2095,7 @@ vfs_setdirty(struct buf *bp)
  *	case it is returned with B_INVAL clear and B_CACHE set based on the
  *	backing VM.
  *
- *	getblk() also forces a VOP_BWRITE() for any B_DELWRI buffer whos
+ *	getblk() also forces a BUF_WRITE() for any B_DELWRI buffer whos
  *	B_CACHE bit is clear.
  *	
  *	What this means, basically, is that the caller should use B_CACHE to
diff --git a/sys/kern/vfs_cluster.c b/sys/kern/vfs_cluster.c
index de6b67e..8a6e045 100644
--- a/sys/kern/vfs_cluster.c
+++ b/sys/kern/vfs_cluster.c
@@ -755,6 +755,8 @@ cluster_wbuild(vp, size, start_lbn, len)
 		 */
 		TAILQ_INIT(&bp->b_cluster.cluster_head);
 		bp->b_bcount = 0;
+		bp->b_magic = tbp->b_magic;
+		bp->b_op = tbp->b_op;
 		bp->b_bufsize = 0;
 		bp->b_npages = 0;
 		if (tbp->b_wcred != NOCRED) {
diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c
index d0ed3bb..cdbf668 100644
--- a/sys/kern/vfs_default.c
+++ b/sys/kern/vfs_default.c
@@ -80,7 +80,6 @@ vop_t **default_vnodeop_p;
 static struct vnodeopv_entry_desc default_vnodeop_entries[] = {
 	{ &vop_default_desc,		(vop_t *) vop_eopnotsupp },
 	{ &vop_advlock_desc,		(vop_t *) vop_einval },
-	{ &vop_bwrite_desc,		(vop_t *) vop_stdbwrite },
 	{ &vop_close_desc,		(vop_t *) vop_null },
 	{ &vop_createvobject_desc,	(vop_t *) vop_stdcreatevobject },
 	{ &vop_destroyvobject_desc,	(vop_t *) vop_stddestroyvobject },
@@ -333,13 +332,6 @@ vop_stdpoll(ap)
 	return (vn_pollrecord(ap->a_vp, ap->a_p, ap->a_events));
 }
 
-int
-vop_stdbwrite(ap)
-	struct vop_bwrite_args *ap;
-{
-	return (bwrite(ap->a_bp));
-}
-
 /*
  * Stubs to use when there is no locking to be done on the underlying object.
  * A minimal shared lock is necessary to ensure that the underlying object
diff --git a/sys/kern/vfs_export.c b/sys/kern/vfs_export.c
index f10edb2..29dec8a 100644
--- a/sys/kern/vfs_export.c
+++ b/sys/kern/vfs_export.c
@@ -748,7 +748,7 @@ vinvalbuf(vp, flags, cred, p, slpflag, slptimeo)
 			 * believe there is a slight chance that a delayed
 			 * write will occur while sleeping just above, so
 			 * check for it.  Note that vfs_bio_awrite expects
-			 * buffers to reside on a queue, while VOP_BWRITE and
+			 * buffers to reside on a queue, while BUF_WRITE and
 			 * brelse do not.
 			 */
 			if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) &&
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index f10edb2..29dec8a 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -748,7 +748,7 @@ vinvalbuf(vp, flags, cred, p, slpflag, slptimeo)
 			 * believe there is a slight chance that a delayed
 			 * write will occur while sleeping just above, so
 			 * check for it.  Note that vfs_bio_awrite expects
-			 * buffers to reside on a queue, while VOP_BWRITE and
+			 * buffers to reside on a queue, while BUF_WRITE and
 			 * brelse do not.
 			 */
 			if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) &&
diff --git a/sys/kern/vnode_if.src b/sys/kern/vnode_if.src
index dd22b80..2fb59d3 100644
--- a/sys/kern/vnode_if.src
+++ b/sys/kern/vnode_if.src
@@ -484,14 +484,6 @@ vop_freeblks {
 };
 
 #
-#% bwrite	vp	L L L
-#
-vop_bwrite {
-	IN struct vnode *vp;
-	IN struct buf *bp;
-};
-
-#
 #% getacl	vp	L L L
 #
 vop_getacl {
diff --git a/sys/nfs/nfs_bio.c b/sys/nfs/nfs_bio.c
index 0c30ca3..442af96 100644
--- a/sys/nfs/nfs_bio.c
+++ b/sys/nfs/nfs_bio.c
@@ -63,6 +63,23 @@
 #include <nfs/nqnfs.h>
 #include <nfs/nfsnode.h>
 
+/*
+ * Just call nfs_writebp() with the force argument set to 1.
+ *
+ * NOTE: B_DONE may or may not be set in a_bp on call.
+ */
+static int
+nfs_bwrite(struct buf *bp)
+{
+	return (nfs_writebp(bp, 1, curproc));
+}
+
+struct buf_ops buf_ops_nfs = {
+	"buf_ops_nfs",
+	nfs_bwrite
+};
+
+
 static struct buf *nfs_getcacheblk __P((struct vnode *vp, daddr_t bn, int size,
 					struct proc *p));
 
@@ -890,6 +907,8 @@ again:
 				bcount += n;
 				allocbuf(bp, bcount);
 				bp->b_flags |= save;
+				bp->b_magic = B_MAGIC_NFS;
+				bp->b_op = &buf_ops_nfs;
 			}
 		} else {
 			/*
diff --git a/sys/nfs/nfs_vnops.c b/sys/nfs/nfs_vnops.c
index d889cf4..2a5564e 100644
--- a/sys/nfs/nfs_vnops.c
+++ b/sys/nfs/nfs_vnops.c
@@ -129,7 +129,6 @@ static int	nfsspec_access __P((struct vop_access_args *));
 static int	nfs_readlink __P((struct vop_readlink_args *));
 static int	nfs_print __P((struct vop_print_args *));
 static int	nfs_advlock __P((struct vop_advlock_args *));
-static int	nfs_bwrite __P((struct vop_bwrite_args *));
 /*
  * Global vfs data structures for nfs
  */
@@ -139,7 +138,6 @@ static struct vnodeopv_entry_desc nfsv2_vnodeop_entries[] = {
 	{ &vop_access_desc,		(vop_t *) nfs_access },
 	{ &vop_advlock_desc,		(vop_t *) nfs_advlock },
 	{ &vop_bmap_desc,		(vop_t *) nfs_bmap },
-	{ &vop_bwrite_desc,		(vop_t *) nfs_bwrite },
 	{ &vop_close_desc,		(vop_t *) nfs_close },
 	{ &vop_create_desc,		(vop_t *) nfs_create },
 	{ &vop_fsync_desc,		(vop_t *) nfs_fsync },
@@ -3090,23 +3088,10 @@ nfs_print(ap)
 }
 
 /*
- * Just call nfs_writebp() with the force argument set to 1.
- *
- * NOTE: B_DONE may or may not be set in a_bp on call.
- */
-static int
-nfs_bwrite(ap)
-	struct vop_bwrite_args /* {
-		struct vnode *a_bp;
-	} */ *ap;
-{
-	return (nfs_writebp(ap->a_bp, 1, curproc));
-}
-
-/*
- * This is a clone of vn_bwrite(), except that B_WRITEINPROG isn't set unless
- * the force flag is one and it also handles the B_NEEDCOMMIT flag.  We set
- * B_CACHE if this is a VMIO buffer.
+ * This is the "real" nfs::bwrite(struct buf*).
+ * B_WRITEINPROG isn't set unless the force flag is one and it 
+ * handles the B_NEEDCOMMIT flag.
+ * We set B_CACHE if this is a VMIO buffer.
  */
 int
 nfs_writebp(bp, force, procp)
diff --git a/sys/nfsclient/nfs_bio.c b/sys/nfsclient/nfs_bio.c
index 0c30ca3..442af96 100644
--- a/sys/nfsclient/nfs_bio.c
+++ b/sys/nfsclient/nfs_bio.c
@@ -63,6 +63,23 @@
 #include <nfs/nqnfs.h>
 #include <nfs/nfsnode.h>
 
+/*
+ * Just call nfs_writebp() with the force argument set to 1.
+ *
+ * NOTE: B_DONE may or may not be set in a_bp on call.
+ */
+static int
+nfs_bwrite(struct buf *bp)
+{
+	return (nfs_writebp(bp, 1, curproc));
+}
+
+struct buf_ops buf_ops_nfs = {
+	"buf_ops_nfs",
+	nfs_bwrite
+};
+
+
 static struct buf *nfs_getcacheblk __P((struct vnode *vp, daddr_t bn, int size,
 					struct proc *p));
 
@@ -890,6 +907,8 @@ again:
 				bcount += n;
 				allocbuf(bp, bcount);
 				bp->b_flags |= save;
+				bp->b_magic = B_MAGIC_NFS;
+				bp->b_op = &buf_ops_nfs;
 			}
 		} else {
 			/*
diff --git a/sys/nfsclient/nfs_vnops.c b/sys/nfsclient/nfs_vnops.c
index d889cf4..2a5564e 100644
--- a/sys/nfsclient/nfs_vnops.c
+++ b/sys/nfsclient/nfs_vnops.c
@@ -129,7 +129,6 @@ static int	nfsspec_access __P((struct vop_access_args *));
 static int	nfs_readlink __P((struct vop_readlink_args *));
 static int	nfs_print __P((struct vop_print_args *));
 static int	nfs_advlock __P((struct vop_advlock_args *));
-static int	nfs_bwrite __P((struct vop_bwrite_args *));
 /*
  * Global vfs data structures for nfs
  */
@@ -139,7 +138,6 @@ static struct vnodeopv_entry_desc nfsv2_vnodeop_entries[] = {
 	{ &vop_access_desc,		(vop_t *) nfs_access },
 	{ &vop_advlock_desc,		(vop_t *) nfs_advlock },
 	{ &vop_bmap_desc,		(vop_t *) nfs_bmap },
-	{ &vop_bwrite_desc,		(vop_t *) nfs_bwrite },
 	{ &vop_close_desc,		(vop_t *) nfs_close },
 	{ &vop_create_desc,		(vop_t *) nfs_create },
 	{ &vop_fsync_desc,		(vop_t *) nfs_fsync },
@@ -3090,23 +3088,10 @@ nfs_print(ap)
 }
 
 /*
- * Just call nfs_writebp() with the force argument set to 1.
- *
- * NOTE: B_DONE may or may not be set in a_bp on call.
- */
-static int
-nfs_bwrite(ap)
-	struct vop_bwrite_args /* {
-		struct vnode *a_bp;
-	} */ *ap;
-{
-	return (nfs_writebp(ap->a_bp, 1, curproc));
-}
-
-/*
- * This is a clone of vn_bwrite(), except that B_WRITEINPROG isn't set unless
- * the force flag is one and it also handles the B_NEEDCOMMIT flag.  We set
- * B_CACHE if this is a VMIO buffer.
+ * This is the "real" nfs::bwrite(struct buf*).
+ * B_WRITEINPROG isn't set unless the force flag is one and it 
+ * handles the B_NEEDCOMMIT flag.
+ * We set B_CACHE if this is a VMIO buffer.
  */
 int
 nfs_writebp(bp, force, procp)
diff --git a/sys/ntfs/ntfs_vnops.c b/sys/ntfs/ntfs_vnops.c
index 710c0d1..66ac769 100644
--- a/sys/ntfs/ntfs_vnops.c
+++ b/sys/ntfs/ntfs_vnops.c
@@ -875,7 +875,6 @@ struct vnodeopv_entry_desc ntfs_vnodeop_entries[] = {
 	{ &vop_getpages_desc, (vop_t *) ntfs_getpages },
 	{ &vop_putpages_desc, (vop_t *) ntfs_putpages },
 	{ &vop_strategy_desc, (vop_t *)ntfs_strategy },
-	{ &vop_bwrite_desc, (vop_t *)vop_stdbwrite },
 	{ &vop_read_desc, (vop_t *)ntfs_read },
 	{ &vop_write_desc, (vop_t *)ntfs_write },
 
diff --git a/sys/sys/buf.h b/sys/sys/buf.h
index eaa3384..ebcf9d2 100644
--- a/sys/sys/buf.h
+++ b/sys/sys/buf.h
@@ -68,6 +68,13 @@ extern struct bio_ops {
 	int	(*io_countdeps) __P((struct buf *, int));
 } bioops;
 
+struct buf_ops {
+	char	*bop_name;
+	int	(*bop_write) __P((struct buf *));
+};
+
+extern struct buf_ops buf_ops_bio;
+
 /*
  * The buffer header describes an I/O operation in the kernel.
  *
@@ -99,6 +106,10 @@ struct buf {
 #define	b_ioflags	b_io.bio_flags
 #define	b_pblkno	b_io.bio_pblkno
 #define	b_resid		b_io.bio_resid
+	struct buf_ops	*b_op;
+	unsigned		b_magic;
+#define B_MAGIC_BIO	0x10b10b10
+#define B_MAGIC_NFS	0x67238234
 	void	(*b_iodone) __P((struct buf *));
 	off_t	b_offset;		/* Offset into file. */
 	LIST_ENTRY(buf) b_hash;		/* Hash chain. */
@@ -410,7 +421,9 @@ bufq_first(struct buf_queue_head *head)
 	return (TAILQ_FIRST(&head->queue));
 }
 
-#define BUF_WRITE(bp)		VOP_BWRITE((bp)->b_vp, (bp))
+#define BUF_WRITE(bp)					\
+	(bp)->b_op->bop_write(bp)
+
 #define BUF_STRATEGY(bp)	VOP_STRATEGY((bp)->b_vp, (bp))
 
 static __inline void
diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index 4f1ca3f..760fad7 100644
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -540,7 +540,6 @@ struct ucred;
 struct uio;
 struct vattr;
 struct vnode;
-struct vop_bwrite_args;
 
 extern int	(*lease_check_hook) __P((struct vop_lease_args *));
 
@@ -620,7 +619,6 @@ int	vfs_object_create __P((struct vnode *vp, struct proc *p,
 void	vfs_timestamp __P((struct timespec *));
 void	vfs_write_resume __P((struct mount *mp));
 void	vfs_write_suspend __P((struct mount *mp));
-int	vop_stdbwrite __P((struct vop_bwrite_args *ap));
 int	vop_stdgetwritemount __P((struct vop_getwritemount_args *));
 int	vop_stdinactive __P((struct vop_inactive_args *));
 int	vop_stdislocked __P((struct vop_islocked_args *));
diff --git a/sys/ufs/mfs/mfs_vnops.c b/sys/ufs/mfs/mfs_vnops.c
index f69bb8d..3c6a236 100644
--- a/sys/ufs/mfs/mfs_vnops.c
+++ b/sys/ufs/mfs/mfs_vnops.c
@@ -70,7 +70,6 @@ vop_t **mfs_vnodeop_p;
 static struct vnodeopv_entry_desc mfs_vnodeop_entries[] = {
 	{ &vop_default_desc,		(vop_t *) mfs_badop },
 	{ &vop_bmap_desc,		(vop_t *) mfs_bmap },
-	{ &vop_bwrite_desc,		(vop_t *) vop_defaultop },
 	{ &vop_close_desc,		(vop_t *) mfs_close },
 	{ &vop_createvobject_desc,	(vop_t *) vop_stdcreatevobject },
 	{ &vop_destroyvobject_desc,	(vop_t *) vop_stddestroyvobject },
diff --git a/sys/vm/vm_pager.c b/sys/vm/vm_pager.c
index f8cea1b..b13c9c0 100644
--- a/sys/vm/vm_pager.c
+++ b/sys/vm/vm_pager.c
@@ -355,6 +355,8 @@ initpbuf(struct buf *bp)
 	bp->b_ioflags = 0;
 	bp->b_iodone = NULL;
 	bp->b_error = 0;
+	bp->b_magic = B_MAGIC_BIO;
+	bp->b_op = &buf_ops_bio;
 	BUF_LOCK(bp, LK_EXCLUSIVE);
 }