Add DTrace support to the new NFS client. This is essentially

cloned from the old NFS client, plus additions for NFSv4. A
review of this code is in progress, however it was felt by the
reviewer that it could go in now, before code slush. Any changes
required by the review can be committed as bug fixes later.

10 files changed, 940 insertions, 5 deletions

diff --git a/sys/fs/nfs/nfs_commonkrpc.c b/sys/fs/nfs/nfs_commonkrpc.c
index ba1ac4e..72df6c5 100644
--- a/sys/fs/nfs/nfs_commonkrpc.c
+++ b/sys/fs/nfs/nfs_commonkrpc.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_inet6.h"
+#include "opt_kdtrace.h"
 #include "opt_kgssapi.h"
 #include "opt_nfs.h"
 
@@ -64,6 +65,28 @@ __FBSDID("$FreeBSD$");
 
 #include <fs/nfs/nfsport.h>
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+dtrace_nfsclient_nfs23_start_probe_func_t
+		dtrace_nfscl_nfs234_start_probe;
+
+dtrace_nfsclient_nfs23_done_probe_func_t
+		dtrace_nfscl_nfs234_done_probe;
+
+/*
+ * Registered probes by RPC type.
+ */
+uint32_t	nfscl_nfs2_start_probes[NFS_NPROCS + 1];
+uint32_t	nfscl_nfs2_done_probes[NFS_NPROCS + 1];
+
+uint32_t	nfscl_nfs3_start_probes[NFS_NPROCS + 1];
+uint32_t	nfscl_nfs3_done_probes[NFS_NPROCS + 1];
+
+uint32_t	nfscl_nfs4_start_probes[NFS_NPROCS + 1];
+uint32_t	nfscl_nfs4_done_probes[NFS_NPROCS + 1];
+#endif
+
 NFSSTATESPINLOCK;
 NFSREQSPINLOCK;
 extern struct nfsstats newnfsstats;
@@ -568,6 +591,29 @@ newnfs_request(struct nfsrv_descript *nd, struct nfsmount *nmp,
 		if ((nd->nd_flag & ND_NFSV4) && procnum == NFSV4PROC_COMPOUND)
 			MALLOC(rep, struct nfsreq *, sizeof(struct nfsreq),
 			    M_NFSDREQ, M_WAITOK);
+#ifdef KDTRACE_HOOKS
+		if (dtrace_nfscl_nfs234_start_probe != NULL) {
+			uint32_t probe_id;
+			int probe_procnum;
+	
+			if (nd->nd_flag & ND_NFSV4) {
+				probe_id =
+				    nfscl_nfs4_start_probes[nd->nd_procnum];
+				probe_procnum = nd->nd_procnum;
+			} else if (nd->nd_flag & ND_NFSV3) {
+				probe_id = nfscl_nfs3_start_probes[procnum];
+				probe_procnum = procnum;
+			} else {
+				probe_id =
+				    nfscl_nfs2_start_probes[nd->nd_procnum];
+				probe_procnum = procnum;
+			}
+			if (probe_id != 0)
+				(dtrace_nfscl_nfs234_start_probe)
+				    (probe_id, vp, nd->nd_mreq, cred,
+				     probe_procnum);
+		}
+#endif
 	}
 	trycnt = 0;
 tryagain:
@@ -762,6 +808,27 @@ tryagain:
 		}
 	}
 
+#ifdef KDTRACE_HOOKS
+	if (nmp != NULL && dtrace_nfscl_nfs234_done_probe != NULL) {
+		uint32_t probe_id;
+		int probe_procnum;
+
+		if (nd->nd_flag & ND_NFSV4) {
+			probe_id = nfscl_nfs4_done_probes[nd->nd_procnum];
+			probe_procnum = nd->nd_procnum;
+		} else if (nd->nd_flag & ND_NFSV3) {
+			probe_id = nfscl_nfs3_done_probes[procnum];
+			probe_procnum = procnum;
+		} else {
+			probe_id = nfscl_nfs2_done_probes[nd->nd_procnum];
+			probe_procnum = procnum;
+		}
+		if (probe_id != 0)
+			(dtrace_nfscl_nfs234_done_probe)(probe_id, vp,
+			    nd->nd_mreq, cred, probe_procnum, 0);
+	}
+#endif
+
 	m_freem(nd->nd_mreq);
 	AUTH_DESTROY(auth);
 	if (rep != NULL)
diff --git a/sys/fs/nfs/nfsport.h b/sys/fs/nfs/nfsport.h
index c21482d..9866ef7 100644
--- a/sys/fs/nfs/nfsport.h
+++ b/sys/fs/nfs/nfsport.h
@@ -267,6 +267,7 @@
  * NFS_V3NPROCS is one greater than the last V3 op and NFS_NPROCS is
  * one greater than the last number.
  */
+#ifndef	NFS_V3NPROCS
 #define	NFS_V3NPROCS		22
 
 #define	NFSPROC_LOOKUPP		22
@@ -293,6 +294,7 @@
  * Must be defined as one higher than the last Proc# above.
  */
 #define	NFSV4_NPROCS		41
+#endif	/* NFS_V3NPROCS */
 
 /*
  * Stats structure
@@ -358,7 +360,9 @@ struct ext_nfsstats {
 /*
  * Define NFS_NPROCS as NFSV4_NPROCS for the experimental kernel code.
  */
+#ifndef	NFS_NPROCS
 #define	NFS_NPROCS		NFSV4_NPROCS
+#endif
 
 #include <fs/nfs/nfskpiport.h>
 #include <fs/nfs/nfsdport.h>
diff --git a/sys/fs/nfs/nfsproto.h b/sys/fs/nfs/nfsproto.h
index 6b66c1f..a15f93b 100644
--- a/sys/fs/nfs/nfsproto.h
+++ b/sys/fs/nfs/nfsproto.h
@@ -226,6 +226,48 @@
 #define	NFSPROC_COMMIT		21
 
 /*
+ * The lower numbers -> 21 are used by NFSv2 and v3. These define higher
+ * numbers used by NFSv4.
+ * NFS_V3NPROCS is one greater than the last V3 op and NFS_NPROCS is
+ * one greater than the last number.
+ */
+#ifndef	NFS_V3NPROCS
+#define	NFS_V3NPROCS		22
+
+#define	NFSPROC_LOOKUPP		22
+#define	NFSPROC_SETCLIENTID	23
+#define	NFSPROC_SETCLIENTIDCFRM	24
+#define	NFSPROC_LOCK		25
+#define	NFSPROC_LOCKU		26
+#define	NFSPROC_OPEN		27
+#define	NFSPROC_CLOSE		28
+#define	NFSPROC_OPENCONFIRM	29
+#define	NFSPROC_LOCKT		30
+#define	NFSPROC_OPENDOWNGRADE	31
+#define	NFSPROC_RENEW		32
+#define	NFSPROC_PUTROOTFH	33
+#define	NFSPROC_RELEASELCKOWN	34
+#define	NFSPROC_DELEGRETURN	35
+#define	NFSPROC_RETDELEGREMOVE	36
+#define	NFSPROC_RETDELEGRENAME1	37
+#define	NFSPROC_RETDELEGRENAME2	38
+#define	NFSPROC_GETACL		39
+#define	NFSPROC_SETACL		40
+
+/*
+ * Must be defined as one higher than the last Proc# above.
+ */
+#define	NFSV4_NPROCS		41
+#endif	/* NFS_V3NPROCS */
+
+/*
+ * Define NFS_NPROCS as NFSV4_NPROCS for the experimental kernel code.
+ */
+#ifndef	NFS_NPROCS
+#define	NFS_NPROCS		NFSV4_NPROCS
+#endif
+
+/*
  * NFSPROC_NOOP is a fake op# that can't be the same as any V2/3/4 Procedure
  * or Operation#. Since the NFS V4 Op #s go higher, use NFSV4OP_NOPS, which
  * is one greater than the highest Op#.
diff --git a/sys/fs/nfsclient/nfs_clbio.c b/sys/fs/nfsclient/nfs_clbio.c
index 95138ec..b33c8a5 100644
--- a/sys/fs/nfsclient/nfs_clbio.c
+++ b/sys/fs/nfsclient/nfs_clbio.c
@@ -35,6 +35,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_kdtrace.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bio.h>
@@ -55,6 +57,7 @@ __FBSDID("$FreeBSD$");
 #include <fs/nfsclient/nfsmount.h>
 #include <fs/nfsclient/nfs.h>
 #include <fs/nfsclient/nfsnode.h>
+#include <fs/nfsclient/nfs_kdtrace.h>
 
 extern int newnfs_directio_allow_mmap;
 extern struct nfsstats newnfsstats;
@@ -399,6 +402,7 @@ nfs_bioread_check_cons(struct vnode *vp, struct thread *td, struct ucred *cred)
 				goto out;
 		}
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		error = VOP_GETATTR(vp, &vattr, cred);
 		if (error)
 			goto out;
@@ -915,6 +919,7 @@ ncl_write(struct vop_write_args *ap)
 #endif
 flush_and_restart:
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			error = ncl_vinvalbuf(vp, V_SAVE, td, 1);
 			if (error)
 				return (error);
@@ -928,6 +933,7 @@ flush_and_restart:
 	 */
 	if (ioflag & IO_APPEND) {
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		error = VOP_GETATTR(vp, &vattr, cred);
 		if (error)
 			return (error);
@@ -1748,6 +1754,7 @@ ncl_doio(struct vnode *vp, struct buf *bp, struct ucred *cr, struct thread *td,
 			mtx_lock(&np->n_mtx);
 			np->n_flag |= NWRITEERR;
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			mtx_unlock(&np->n_mtx);
 		    }
 		    bp->b_dirtyoff = bp->b_dirtyend = 0;
diff --git a/sys/fs/nfsclient/nfs_clkdtrace.c b/sys/fs/nfsclient/nfs_clkdtrace.c
new file mode 100644
index 0000000..c7db3a4
--- /dev/null
+++ b/sys/fs/nfsclient/nfs_clkdtrace.c
@@ -0,0 +1,600 @@
+/*-
+ * Copyright (c) 2009 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed at the University of Cambridge Computer
+ * Laboratory with support from a grant from Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_bsd.h>
+
+#include <fs/nfs/nfsproto.h>
+
+#include <fs/nfsclient/nfs_kdtrace.h>
+
+/*
+ * dtnfscl is a DTrace provider that tracks the intent to perform RPCs
+ * in the NFS client, as well as acess to and maintenance of the access and
+ * attribute caches.  This is not quite the same as RPCs, because NFS may
+ * issue multiple RPC transactions in the event that authentication fails,
+ * there's a jukebox error, or none at all if the access or attribute cache
+ * hits.  However, it cleanly represents the logical layer between RPC
+ * transmission and vnode/vfs operations, providing access to state linking
+ * the two.
+ */
+
+static int	dtnfsclient_unload(void);
+static void	dtnfsclient_getargdesc(void *, dtrace_id_t, void *,
+		    dtrace_argdesc_t *);
+static void	dtnfsclient_provide(void *, dtrace_probedesc_t *);
+static void	dtnfsclient_destroy(void *, dtrace_id_t, void *);
+static void	dtnfsclient_enable(void *, dtrace_id_t, void *);
+static void	dtnfsclient_disable(void *, dtrace_id_t, void *);
+static void	dtnfsclient_load(void *);
+
+static dtrace_pattr_t dtnfsclient_attr = {
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+};
+
+/*
+ * Description of NFSv4, NFSv3 and (optional) NFSv2 probes for a procedure.
+ */
+struct dtnfsclient_rpc {
+	char		*nr_v4_name;
+	char		*nr_v3_name;	/* Or NULL if none. */
+	char		*nr_v2_name;	/* Or NULL if none. */
+
+	/*
+	 * IDs for the start and done cases, for NFSv2, NFSv3 and NFSv4.
+	 */
+	uint32_t	 nr_v2_id_start, nr_v2_id_done;
+	uint32_t	 nr_v3_id_start, nr_v3_id_done;
+	uint32_t	 nr_v4_id_start, nr_v4_id_done;
+};
+
+/*
+ * This table is indexed by NFSv3 procedure number, but also used for NFSv2
+ * procedure names and NFSv4 operations.
+ */
+static struct dtnfsclient_rpc	dtnfsclient_rpcs[NFS_NPROCS + 1] = {
+	{ "null", "null", "null" },
+	{ "getattr", "getattr", "getattr" },
+	{ "setattr", "setattr", "setattr" },
+	{ "lookup", "lookup", "lookup" },
+	{ "access", "access", "noop" },
+	{ "readlink", "readlink", "readlink" },
+	{ "read", "read", "read" },
+	{ "write", "write", "write" },
+	{ "create", "create", "create" },
+	{ "mkdir", "mkdir", "mkdir" },
+	{ "symlink", "symlink", "symlink" },
+	{ "mknod", "mknod" },
+	{ "remove", "remove", "remove" },
+	{ "rmdir", "rmdir", "rmdir" },
+	{ "rename", "rename", "rename" },
+	{ "link", "link", "link" },
+	{ "readdir", "readdir", "readdir" },
+	{ "readdirplus", "readdirplus" },
+	{ "fsstat", "fsstat", "statfs" },
+	{ "fsinfo", "fsinfo" },
+	{ "pathconf", "pathconf" },
+	{ "commit", "commit" },
+	{ "lookupp" },
+	{ "setclientid" },
+	{ "setclientidcfrm" },
+	{ "lock" },
+	{ "locku" },
+	{ "open" },
+	{ "close" },
+	{ "openconfirm" },
+	{ "lockt" },
+	{ "opendowngrade" },
+	{ "renew" },
+	{ "putrootfh" },
+	{ "releaselckown" },
+	{ "delegreturn" },
+	{ "retdelegremove" },
+	{ "retdelegrename1" },
+	{ "retdelegrename2" },
+	{ "getacl" },
+	{ "setacl" },
+	{ "noop", "noop", "noop" }
+};
+
+/*
+ * Module name strings.
+ */
+static char	*dtnfsclient_accesscache_str = "accesscache";
+static char	*dtnfsclient_attrcache_str = "attrcache";
+static char	*dtnfsclient_nfs2_str = "nfs2";
+static char	*dtnfsclient_nfs3_str = "nfs3";
+static char	*dtnfsclient_nfs4_str = "nfs4";
+
+/*
+ * Function name strings.
+ */
+static char	*dtnfsclient_flush_str = "flush";
+static char	*dtnfsclient_load_str = "load";
+static char	*dtnfsclient_get_str = "get";
+
+/*
+ * Name strings.
+ */
+static char	*dtnfsclient_done_str = "done";
+static char	*dtnfsclient_hit_str = "hit";
+static char	*dtnfsclient_miss_str = "miss";
+static char	*dtnfsclient_start_str = "start";
+
+static dtrace_pops_t dtnfsclient_pops = {
+	dtnfsclient_provide,
+	NULL,
+	dtnfsclient_enable,
+	dtnfsclient_disable,
+	NULL,
+	NULL,
+	dtnfsclient_getargdesc,
+	NULL,
+	NULL,
+	dtnfsclient_destroy
+};
+
+static dtrace_provider_id_t	dtnfsclient_id;
+
+/*
+ * Most probes are generated from the above RPC table, but for access and
+ * attribute caches, we have specific IDs we recognize and handle specially
+ * in various spots.
+ */
+extern uint32_t	nfscl_accesscache_flush_done_id;
+extern uint32_t	nfscl_accesscache_get_hit_id;
+extern uint32_t	nfscl_accesscache_get_miss_id;
+extern uint32_t	nfscl_accesscache_load_done_id;
+
+extern uint32_t	nfscl_attrcache_flush_done_id;
+extern uint32_t	nfscl_attrcache_get_hit_id;
+extern uint32_t	nfscl_attrcache_get_miss_id;
+extern uint32_t	nfscl_attrcache_load_done_id;
+
+/*
+ * When tracing on a procedure is enabled, the DTrace ID for an RPC event is
+ * stored in one of these two NFS client-allocated arrays; 0 indicates that
+ * the event is not being traced so probes should not be called.
+ *
+ * For simplicity, we allocate both v2, v3 and v4 arrays as NFS_NPROCS + 1, and
+ * the v2, v3 arrays are simply sparse.
+ */
+extern uint32_t			nfscl_nfs2_start_probes[NFS_NPROCS + 1];
+extern uint32_t			nfscl_nfs2_done_probes[NFS_NPROCS + 1];
+
+extern uint32_t			nfscl_nfs3_start_probes[NFS_NPROCS + 1];
+extern uint32_t			nfscl_nfs3_done_probes[NFS_NPROCS + 1];
+
+extern uint32_t			nfscl_nfs4_start_probes[NFS_NPROCS + 1];
+extern uint32_t			nfscl_nfs4_done_probes[NFS_NPROCS + 1];
+
+/*
+ * Look up a DTrace probe ID to see if it's associated with a "done" event --
+ * if so, we will return a fourth argument type of "int".
+ */
+static int
+dtnfs234_isdoneprobe(dtrace_id_t id)
+{
+	int i;
+
+	for (i = 0; i < NFS_NPROCS + 1; i++) {
+		if (dtnfsclient_rpcs[i].nr_v4_id_done == id ||
+		    dtnfsclient_rpcs[i].nr_v3_id_done == id ||
+		    dtnfsclient_rpcs[i].nr_v2_id_done == id)
+			return (1);
+	}
+	return (0);
+}
+
+static void
+dtnfsclient_getargdesc(void *arg, dtrace_id_t id, void *parg,
+    dtrace_argdesc_t *desc)
+{
+	const char *p = NULL;
+
+	if (id == nfscl_accesscache_flush_done_id ||
+	    id == nfscl_attrcache_flush_done_id ||
+	    id == nfscl_attrcache_get_miss_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfscl_accesscache_get_hit_id ||
+	    id == nfscl_accesscache_get_miss_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "uid_t";
+			break;
+		case 2:
+			p = "uint32_t";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfscl_accesscache_load_done_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "uid_t";
+			break;
+		case 2:
+			p = "uint32_t";
+			break;
+		case 3:
+			p = "int";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfscl_attrcache_get_hit_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct vattr *";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfscl_attrcache_load_done_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct vattr *";
+			break;
+		case 2:
+			p = "int";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct mbuf *";
+			break;
+		case 2:
+			p = "struct ucred *";
+			break;
+		case 3:
+			p = "int";
+			break;
+		case 4:
+			if (dtnfs234_isdoneprobe(id)) {
+				p = "int";
+				break;
+			}
+			/* FALLSTHROUGH */
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	}
+	if (p != NULL)
+		strlcpy(desc->dtargd_native, p, sizeof(desc->dtargd_native));
+}
+
+static void
+dtnfsclient_provide(void *arg, dtrace_probedesc_t *desc)
+{
+	int i;
+
+	if (desc != NULL)
+		return;
+
+	/*
+	 * Register access cache probes.
+	 */
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_flush_str, dtnfsclient_done_str) == 0) {
+		nfscl_accesscache_flush_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_flush_str, dtnfsclient_done_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_get_str, dtnfsclient_hit_str) == 0) {
+		nfscl_accesscache_get_hit_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_get_str, dtnfsclient_hit_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_get_str, dtnfsclient_miss_str) == 0) {
+		nfscl_accesscache_get_miss_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_get_str, dtnfsclient_miss_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_load_str, dtnfsclient_done_str) == 0) {
+		nfscl_accesscache_load_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_load_str, dtnfsclient_done_str, 0, NULL);
+	}
+
+	/*
+	 * Register attribute cache probes.
+	 */
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_flush_str, dtnfsclient_done_str) == 0) {
+		nfscl_attrcache_flush_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_flush_str, dtnfsclient_done_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_get_str, dtnfsclient_hit_str) == 0) {
+		nfscl_attrcache_get_hit_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_get_str, dtnfsclient_hit_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_get_str, dtnfsclient_miss_str) == 0) {
+		nfscl_attrcache_get_miss_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_get_str, dtnfsclient_miss_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_load_str, dtnfsclient_done_str) == 0) {
+		nfscl_attrcache_load_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_load_str, dtnfsclient_done_str, 0, NULL);
+	}
+
+	/*
+	 * Register NFSv2 RPC procedures; note sparseness check for each slot
+	 * in the NFSv3, NFSv4 procnum-indexed array.
+	 */
+	for (i = 0; i < NFS_NPROCS + 1; i++) {
+		if (dtnfsclient_rpcs[i].nr_v2_name != NULL &&
+		    dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs2_str,
+		    dtnfsclient_rpcs[i].nr_v2_name, dtnfsclient_start_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v2_id_start =
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs2_str,
+			    dtnfsclient_rpcs[i].nr_v2_name,
+			    dtnfsclient_start_str, 0,
+			    &nfscl_nfs2_start_probes[i]);
+		}
+		if (dtnfsclient_rpcs[i].nr_v2_name != NULL &&
+		    dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs2_str,
+		    dtnfsclient_rpcs[i].nr_v2_name, dtnfsclient_done_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v2_id_done = 
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs2_str,
+			    dtnfsclient_rpcs[i].nr_v2_name,
+			    dtnfsclient_done_str, 0,
+			    &nfscl_nfs2_done_probes[i]);
+		}
+	}
+
+	/*
+	 * Register NFSv3 RPC procedures; note sparseness check for each slot
+	 * in the NFSv4 procnum-indexed array.
+	 */
+	for (i = 0; i < NFS_NPROCS + 1; i++) {
+		if (dtnfsclient_rpcs[i].nr_v3_name != NULL &&
+		    dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs3_str,
+		    dtnfsclient_rpcs[i].nr_v3_name, dtnfsclient_start_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v3_id_start =
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs3_str,
+			    dtnfsclient_rpcs[i].nr_v3_name,
+			    dtnfsclient_start_str, 0,
+			    &nfscl_nfs3_start_probes[i]);
+		}
+		if (dtnfsclient_rpcs[i].nr_v3_name != NULL &&
+		    dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs3_str,
+		    dtnfsclient_rpcs[i].nr_v3_name, dtnfsclient_done_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v3_id_done = 
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs3_str,
+			    dtnfsclient_rpcs[i].nr_v3_name,
+			    dtnfsclient_done_str, 0,
+			    &nfscl_nfs3_done_probes[i]);
+		}
+	}
+
+	/*
+	 * Register NFSv4 RPC procedures.
+	 */
+	for (i = 0; i < NFS_NPROCS + 1; i++) {
+		if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs4_str,
+		    dtnfsclient_rpcs[i].nr_v4_name, dtnfsclient_start_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v4_id_start =
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs4_str,
+			    dtnfsclient_rpcs[i].nr_v4_name,
+			    dtnfsclient_start_str, 0,
+			    &nfscl_nfs4_start_probes[i]);
+		}
+		if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs4_str,
+		    dtnfsclient_rpcs[i].nr_v4_name, dtnfsclient_done_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v4_id_done = 
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs4_str,
+			    dtnfsclient_rpcs[i].nr_v4_name,
+			    dtnfsclient_done_str, 0,
+			    &nfscl_nfs4_done_probes[i]);
+		}
+	}
+}
+
+static void
+dtnfsclient_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+}
+
+static void
+dtnfsclient_enable(void *arg, dtrace_id_t id, void *parg)
+{
+	uint32_t *p = parg;
+	void *f = dtrace_probe;
+
+	if (id == nfscl_accesscache_flush_done_id)
+		dtrace_nfscl_accesscache_flush_done_probe = f;
+	else if (id == nfscl_accesscache_get_hit_id)
+		dtrace_nfscl_accesscache_get_hit_probe = f;
+	else if (id == nfscl_accesscache_get_miss_id)
+		dtrace_nfscl_accesscache_get_miss_probe = f;
+	else if (id == nfscl_accesscache_load_done_id)
+		dtrace_nfscl_accesscache_load_done_probe = f;
+	else if (id == nfscl_attrcache_flush_done_id)
+		dtrace_nfscl_attrcache_flush_done_probe = f;
+	else if (id == nfscl_attrcache_get_hit_id)
+		dtrace_nfscl_attrcache_get_hit_probe = f;
+	else if (id == nfscl_attrcache_get_miss_id)
+		dtrace_nfscl_attrcache_get_miss_probe = f;
+	else if (id == nfscl_attrcache_load_done_id)
+		dtrace_nfscl_attrcache_load_done_probe = f;
+	else
+		*p = id;
+}
+
+static void
+dtnfsclient_disable(void *arg, dtrace_id_t id, void *parg)
+{
+	uint32_t *p = parg;
+
+	if (id == nfscl_accesscache_flush_done_id)
+		dtrace_nfscl_accesscache_flush_done_probe = NULL;
+	else if (id == nfscl_accesscache_get_hit_id)
+		dtrace_nfscl_accesscache_get_hit_probe = NULL;
+	else if (id == nfscl_accesscache_get_miss_id)
+		dtrace_nfscl_accesscache_get_miss_probe = NULL;
+	else if (id == nfscl_accesscache_load_done_id)
+		dtrace_nfscl_accesscache_load_done_probe = NULL;
+	else if (id == nfscl_attrcache_flush_done_id)
+		dtrace_nfscl_attrcache_flush_done_probe = NULL;
+	else if (id == nfscl_attrcache_get_hit_id)
+		dtrace_nfscl_attrcache_get_hit_probe = NULL;
+	else if (id == nfscl_attrcache_get_miss_id)
+		dtrace_nfscl_attrcache_get_miss_probe = NULL;
+	else if (id == nfscl_attrcache_load_done_id)
+		dtrace_nfscl_attrcache_load_done_probe = NULL;
+	else
+		*p = 0;
+}
+
+static void
+dtnfsclient_load(void *dummy)
+{
+
+	if (dtrace_register("nfscl", &dtnfsclient_attr,
+	    DTRACE_PRIV_USER, NULL, &dtnfsclient_pops, NULL,
+	    &dtnfsclient_id) != 0)
+		return;
+
+	dtrace_nfscl_nfs234_start_probe =
+	    (dtrace_nfsclient_nfs23_start_probe_func_t)dtrace_probe;
+	dtrace_nfscl_nfs234_done_probe =
+	    (dtrace_nfsclient_nfs23_done_probe_func_t)dtrace_probe;
+}
+
+
+static int
+dtnfsclient_unload()
+{
+
+	dtrace_nfscl_nfs234_start_probe = NULL;
+	dtrace_nfscl_nfs234_done_probe = NULL;
+
+	return (dtrace_unregister(dtnfsclient_id));
+}
+
+static int
+dtnfsclient_modevent(module_t mod __unused, int type, void *data __unused)
+{
+	int error = 0;
+
+	switch (type) {
+	case MOD_LOAD:
+		break;
+
+	case MOD_UNLOAD:
+		break;
+
+	case MOD_SHUTDOWN:
+		break;
+
+	default:
+		error = EOPNOTSUPP;
+		break;
+	}
+
+	return (error);
+}
+
+SYSINIT(dtnfsclient_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
+    dtnfsclient_load, NULL);
+SYSUNINIT(dtnfsclient_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
+    dtnfsclient_unload, NULL);
+
+DEV_MODULE(dtnfscl, dtnfsclient_modevent, NULL);
+MODULE_VERSION(dtnfscl, 1);
+MODULE_DEPEND(dtnfscl, dtrace, 1, 1, 1);
+MODULE_DEPEND(dtnfscl, opensolaris, 1, 1, 1);
+MODULE_DEPEND(dtnfscl, nfscl, 1, 1, 1);
+MODULE_DEPEND(dtnfscl, nfscommon, 1, 1, 1);
diff --git a/sys/fs/nfsclient/nfs_clnode.c b/sys/fs/nfsclient/nfs_clnode.c
index d4d652e..d15de6e 100644
--- a/sys/fs/nfsclient/nfs_clnode.c
+++ b/sys/fs/nfsclient/nfs_clnode.c
@@ -35,6 +35,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_kdtrace.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/fcntl.h>
@@ -53,6 +55,7 @@ __FBSDID("$FreeBSD$");
 #include <fs/nfsclient/nfsnode.h>
 #include <fs/nfsclient/nfsmount.h>
 #include <fs/nfsclient/nfs.h>
+#include <fs/nfsclient/nfs_kdtrace.h>
 
 #include <nfs/nfs_lock.h>
 
@@ -300,7 +303,9 @@ ncl_invalcaches(struct vnode *vp)
 	mtx_lock(&np->n_mtx);
 	for (i = 0; i < NFS_ACCESSCACHESIZE; i++)
 		np->n_accesscache[i].stamp = 0;
+	KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp);
 	np->n_attrstamp = 0;
+	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 	mtx_unlock(&np->n_mtx);
 }
 
diff --git a/sys/fs/nfsclient/nfs_clport.c b/sys/fs/nfsclient/nfs_clport.c
index 4d88bd2..0538382 100644
--- a/sys/fs/nfsclient/nfs_clport.c
+++ b/sys/fs/nfsclient/nfs_clport.c
@@ -34,6 +34,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_kdtrace.h"
+
 /*
  * generally, I don't like #includes inside .h files, but it seems to
  * be the easiest way to handle the port.
@@ -43,6 +45,26 @@ __FBSDID("$FreeBSD$");
 #include <netinet/if_ether.h>
 #include <net/if_types.h>
 
+#include <fs/nfsclient/nfs_kdtrace.h>
+
+#ifdef KDTRACE_HOOKS
+dtrace_nfsclient_attrcache_flush_probe_func_t
+		dtrace_nfscl_attrcache_flush_done_probe;
+uint32_t	nfscl_attrcache_flush_done_id;
+
+dtrace_nfsclient_attrcache_get_hit_probe_func_t
+		dtrace_nfscl_attrcache_get_hit_probe;
+uint32_t	nfscl_attrcache_get_hit_id;
+
+dtrace_nfsclient_attrcache_get_miss_probe_func_t
+		dtrace_nfscl_attrcache_get_miss_probe;
+uint32_t	nfscl_attrcache_get_miss_id;
+
+dtrace_nfsclient_attrcache_load_probe_func_t
+		dtrace_nfscl_attrcache_load_done_probe;
+uint32_t	nfscl_attrcache_load_done_id;
+#endif /* !KDTRACE_HOOKS */
+
 extern u_int32_t newnfs_true, newnfs_false, newnfs_xdrneg1;
 extern struct vop_vector newnfs_vnodeops;
 extern struct vop_vector newnfs_fifoops;
@@ -407,6 +429,7 @@ nfscl_loadattrcache(struct vnode **vpp, struct nfsvattr *nap, void *nvaper,
 				 */
 				vap->va_size = np->n_size;
 				np->n_attrstamp = 0;
+				KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			} else if (np->n_flag & NMODIFIED) {
 				/*
 				 * We've modified the file: Use the larger
@@ -439,9 +462,11 @@ nfscl_loadattrcache(struct vnode **vpp, struct nfsvattr *nap, void *nvaper,
 	 * We detect this by for the mtime moving back. We invalidate the 
 	 * attrcache when this happens.
 	 */
-	if (timespeccmp(&mtime_save, &vap->va_mtime, >))
+	if (timespeccmp(&mtime_save, &vap->va_mtime, >)) {
 		/* Size changed or mtime went backwards */
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
+	}
 	if (vaper != NULL) {
 		NFSBCOPY((caddr_t)vap, (caddr_t)vaper, sizeof(*vap));
 		if (np->n_flag & NCHG) {
@@ -451,6 +476,10 @@ nfscl_loadattrcache(struct vnode **vpp, struct nfsvattr *nap, void *nvaper,
 				vaper->va_mtime = np->n_mtim;
 		}
 	}
+#ifdef KDTRACE_HOOKS
+	if (np->n_attrstamp != 0)
+		KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, vap, 0);
+#endif
 	NFSUNLOCKNODE(np);
 	return (0);
 }
diff --git a/sys/fs/nfsclient/nfs_clsubs.c b/sys/fs/nfsclient/nfs_clsubs.c
index 0c05631..214dfb7 100644
--- a/sys/fs/nfsclient/nfs_clsubs.c
+++ b/sys/fs/nfsclient/nfs_clsubs.c
@@ -35,6 +35,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_kdtrace.h"
+
 /*
  * These functions support the macros and help fiddle mbuf chains for
  * the nfs op functions. They do things like create the rpc header and
@@ -68,6 +70,7 @@ __FBSDID("$FreeBSD$");
 #include <fs/nfsclient/nfsnode.h>
 #include <fs/nfsclient/nfsmount.h>
 #include <fs/nfsclient/nfs.h>
+#include <fs/nfsclient/nfs_kdtrace.h>
 
 #include <netinet/in.h>
 
@@ -238,6 +241,7 @@ ncl_getattrcache(struct vnode *vp, struct vattr *vaper)
 #ifdef NFS_ACDEBUG
 		mtx_unlock(&Giant);	/* ncl_printf() */
 #endif
+		KDTRACE_NFS_ATTRCACHE_GET_MISS(vp);
 		return( ENOENT);
 	}
 	newnfsstats.attrcache_hits++;
@@ -267,6 +271,7 @@ ncl_getattrcache(struct vnode *vp, struct vattr *vaper)
 #ifdef NFS_ACDEBUG
 	mtx_unlock(&Giant);	/* ncl_printf() */
 #endif
+	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap);
 	return (0);
 }
 
diff --git a/sys/fs/nfsclient/nfs_clvnops.c b/sys/fs/nfsclient/nfs_clvnops.c
index 3e1c66d..0a1d6a2 100644
--- a/sys/fs/nfsclient/nfs_clvnops.c
+++ b/sys/fs/nfsclient/nfs_clvnops.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
  * vnode op calls for Sun NFS version 2, 3 and 4
  */
 
+#include "opt_kdtrace.h"
 #include "opt_inet.h"
 
 #include <sys/param.h>
@@ -70,6 +71,7 @@ __FBSDID("$FreeBSD$");
 #include <fs/nfsclient/nfsnode.h>
 #include <fs/nfsclient/nfsmount.h>
 #include <fs/nfsclient/nfs.h>
+#include <fs/nfsclient/nfs_kdtrace.h>
 
 #include <net/if.h>
 #include <netinet/in.h>
@@ -77,6 +79,24 @@ __FBSDID("$FreeBSD$");
 
 #include <nfs/nfs_lock.h>
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+dtrace_nfsclient_accesscache_flush_probe_func_t
+		dtrace_nfscl_accesscache_flush_done_probe;
+uint32_t	nfscl_accesscache_flush_done_id;
+
+dtrace_nfsclient_accesscache_get_probe_func_t
+		dtrace_nfscl_accesscache_get_hit_probe,
+		dtrace_nfscl_accesscache_get_miss_probe;
+uint32_t	nfscl_accesscache_get_hit_id;
+uint32_t	nfscl_accesscache_get_miss_id;
+
+dtrace_nfsclient_accesscache_load_probe_func_t
+		dtrace_nfscl_accesscache_load_done_probe;
+uint32_t	nfscl_accesscache_load_done_id;
+#endif /* !KDTRACE_HOOKS */
+
 /* Defs */
 #define	TRUE	1
 #define	FALSE	0
@@ -298,9 +318,15 @@ nfs34_access_otw(struct vnode *vp, int wmode, struct thread *td,
 		mtx_unlock(&np->n_mtx);
 		if (retmode != NULL)
 			*retmode = rmode;
+		KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, cred->cr_uid, rmode, 0);
 	} else if (NFS_ISV4(vp)) {
 		error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0);
 	}
+#ifdef KDTRACE_HOOKS
+	if (error != 0)
+		KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, cred->cr_uid, 0,
+		    error);
+#endif
 	return (error);
 }
 
@@ -397,6 +423,14 @@ nfs_access(struct vop_access_args *ap)
 			}
 		}
 		mtx_unlock(&np->n_mtx);
+#ifdef KDTRACE_HOOKS
+		if (gotahit != 0)
+			KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp,
+			    ap->a_cred->cr_uid, mode);
+		else
+			KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp,
+			    ap->a_cred->cr_uid, mode);
+#endif
 		if (gotahit == 0) {
 			/*
 			 * Either a no, or a don't know.  Go to the wire.
@@ -507,6 +541,7 @@ nfs_open(struct vop_open_args *ap)
 		}
 		mtx_lock(&np->n_mtx);
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		if (vp->v_type == VDIR)
 			np->n_direofoffset = 0;
 		mtx_unlock(&np->n_mtx);
@@ -692,8 +727,10 @@ nfs_close(struct vop_close_args *ap)
 	     *     is the cause of some caching/coherency issue that might
 	     *     crop up.)
  	     */
-	    if (VFSTONFS(vp->v_mount)->nm_negnametimeo == 0)
+	    if (VFSTONFS(vp->v_mount)->nm_negnametimeo == 0) {
 		    np->n_attrstamp = 0;
+		    KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
+	    }
 	    if (np->n_flag & NWRITEERR) {
 		np->n_flag &= ~NWRITEERR;
 		error = np->n_error;
@@ -949,6 +986,7 @@ nfs_setattrrpc(struct vnode *vp, struct vattr *vap, struct ucred *cred,
 			np->n_accesscache[i].stamp = 0;
 		np->n_flag |= NDELEGMOD;
 		mtx_unlock(&np->n_mtx);
+		KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp);
 	}
 	error = nfsrpc_setattr(vp, vap, NULL, cred, td, &nfsva, &attrflag,
 	    NULL);
@@ -1030,6 +1068,7 @@ nfs_lookup(struct vop_lookup_args *ap)
 		    !(newnp->n_flag & NMODIFIED)) {
 			mtx_lock(&newnp->n_mtx);
 			newnp->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(newvp);
 			mtx_unlock(&newnp->n_mtx);
 		}
 		if (nfscl_nodeleg(newvp, 0) == 0 ||
@@ -1233,6 +1272,7 @@ nfs_lookup(struct vop_lookup_args *ap)
 			 */
 			mtx_lock(&np->n_mtx);
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(newvp);
 			mtx_unlock(&np->n_mtx);
 		}
 	}
@@ -1412,8 +1452,10 @@ nfs_mknodrpc(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 	dnp = VTONFS(dvp);
 	mtx_lock(&dnp->n_mtx);
 	dnp->n_flag |= NMODIFIED;
-	if (!dattrflag)
+	if (!dattrflag) {
 		dnp->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&dnp->n_mtx);
 	return (error);
 }
@@ -1566,8 +1608,10 @@ again:
 	}
 	mtx_lock(&dnp->n_mtx);
 	dnp->n_flag |= NMODIFIED;
-	if (!dattrflag)
+	if (!dattrflag) {
 		dnp->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&dnp->n_mtx);
 	return (error);
 }
@@ -1630,6 +1674,7 @@ nfs_remove(struct vop_remove_args *ap)
 	mtx_lock(&np->n_mtx);
 	np->n_attrstamp = 0;
 	mtx_unlock(&np->n_mtx);
+	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 	return (error);
 }
 
@@ -1678,8 +1723,10 @@ nfs_removerpc(struct vnode *dvp, struct vnode *vp, char *name,
 		(void) nfscl_loadattrcache(&dvp, &dnfsva, NULL, NULL, 0, 1);
 	mtx_lock(&dnp->n_mtx);
 	dnp->n_flag |= NMODIFIED;
-	if (!dattrflag)
+	if (!dattrflag) {
 		dnp->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&dnp->n_mtx);
 	if (error && NFS_ISV4(dvp))
 		error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0);
@@ -1857,6 +1904,7 @@ nfs_renamerpc(struct vnode *fdvp, struct vnode *fvp, char *fnameptr,
 	} else {
 		fdnp->n_attrstamp = 0;
 		mtx_unlock(&fdnp->n_mtx);
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(fdvp);
 	}
 	mtx_lock(&tdnp->n_mtx);
 	tdnp->n_flag |= NMODIFIED;
@@ -1866,6 +1914,7 @@ nfs_renamerpc(struct vnode *fdvp, struct vnode *fvp, char *fnameptr,
 	} else {
 		tdnp->n_attrstamp = 0;
 		mtx_unlock(&tdnp->n_mtx);
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(tdvp);
 	}
 	if (error && NFS_ISV4(fdvp))
 		error = nfscl_maperr(td, error, (uid_t)0, (gid_t)0);
@@ -1908,6 +1957,7 @@ nfs_link(struct vop_link_args *ap)
 	} else {
 		tdnp->n_attrstamp = 0;
 		mtx_unlock(&tdnp->n_mtx);
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(tdvp);
 	}
 	if (attrflag)
 		(void) nfscl_loadattrcache(&vp, &nfsva, NULL, NULL, 0, 1);
@@ -1916,6 +1966,7 @@ nfs_link(struct vop_link_args *ap)
 		mtx_lock(&np->n_mtx);
 		np->n_attrstamp = 0;
 		mtx_unlock(&np->n_mtx);
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 	}
 	/*
 	 * If negative lookup caching is enabled, I might as well
@@ -2002,6 +2053,7 @@ nfs_symlink(struct vop_symlink_args *ap)
 	} else {
 		dnp->n_attrstamp = 0;
 		mtx_unlock(&dnp->n_mtx);
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
 	}
 	return (error);
 }
@@ -2037,6 +2089,7 @@ nfs_mkdir(struct vop_mkdir_args *ap)
 	} else {
 		dnp->n_attrstamp = 0;
 		mtx_unlock(&dnp->n_mtx);
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
 	}
 	if (nfhp) {
 		ret = nfscl_nget(dvp->v_mount, dvp, nfhp, cnp, cnp->cn_thread,
@@ -2105,6 +2158,7 @@ nfs_rmdir(struct vop_rmdir_args *ap)
 	} else {
 		dnp->n_attrstamp = 0;
 		mtx_unlock(&dnp->n_mtx);
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
 	}
 
 	cache_purge(dvp);
@@ -2946,12 +3000,14 @@ nfs_advlock(struct vop_advlock_args *ap)
 		if (ap->a_op == F_SETLK) {
 			if ((np->n_flag & NMODIFIED) == 0) {
 				np->n_attrstamp = 0;
+				KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 				ret = VOP_GETATTR(vp, &va, cred);
 			}
 			if ((np->n_flag & NMODIFIED) || ret ||
 			    np->n_change != va.va_filerev) {
 				(void) ncl_vinvalbuf(vp, V_SAVE, td, 1);
 				np->n_attrstamp = 0;
+				KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 				ret = VOP_GETATTR(vp, &va, cred);
 				if (!ret) {
 					np->n_mtime = va.va_mtime;
diff --git a/sys/fs/nfsclient/nfs_kdtrace.h b/sys/fs/nfsclient/nfs_kdtrace.h
new file mode 100644
index 0000000..f8f143f
--- /dev/null
+++ b/sys/fs/nfsclient/nfs_kdtrace.h
@@ -0,0 +1,120 @@
+/*-
+ * Copyright (c) 2009 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed at the University of Cambridge Computer
+ * Laboratory with support from a grant from Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _NFSCL_NFS_KDTRACE_H_
+#define	_NFSCL_NFS_KDTRACE_H_
+
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+/*
+ * Definitions for NFS access cache probes.
+ */
+extern uint32_t	nfscl_accesscache_flush_done_id;
+extern uint32_t	nfscl_accesscache_get_hit_id;
+extern uint32_t	nfscl_accesscache_get_miss_id;
+extern uint32_t	nfscl_accesscache_load_done_id;
+
+#define	KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp)	do {			\
+	if (dtrace_nfscl_accesscache_flush_done_probe != NULL)		\
+		(dtrace_nfscl_accesscache_flush_done_probe)(		\
+		    nfscl_accesscache_flush_done_id, (vp));		\
+} while (0)
+
+#define	KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp, uid, mode)	do {		\
+	if (dtrace_nfscl_accesscache_get_hit_probe != NULL)		\
+		(dtrace_nfscl_accesscache_get_hit_probe)(		\
+		    nfscl_accesscache_get_hit_id, (vp), (uid),		\
+		    (mode));						\
+} while (0)
+	
+#define	KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp, uid, mode)	do {		\
+	if (dtrace_nfscl_accesscache_get_miss_probe != NULL)		\
+		(dtrace_nfscl_accesscache_get_miss_probe)(		\
+		    nfscl_accesscache_get_miss_id, (vp), (uid),		\
+		    (mode));						\
+} while (0)
+
+#define	KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, uid, rmode, error) do {	\
+	if (dtrace_nfscl_accesscache_load_done_probe != NULL)		\
+		(dtrace_nfscl_accesscache_load_done_probe)(		\
+		    nfscl_accesscache_load_done_id, (vp), (uid),	\
+		    (rmode), (error));					\
+} while (0)
+
+/*
+ * Definitions for NFS attribute cache probes.
+ */
+extern uint32_t	nfscl_attrcache_flush_done_id;
+extern uint32_t	nfscl_attrcache_get_hit_id;
+extern uint32_t	nfscl_attrcache_get_miss_id;
+extern uint32_t	nfscl_attrcache_load_done_id;
+
+#define	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp)	do {			\
+	if (dtrace_nfscl_attrcache_flush_done_probe != NULL)		\
+		(dtrace_nfscl_attrcache_flush_done_probe)(		\
+		    nfscl_attrcache_flush_done_id, (vp));		\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap)	do {			\
+	if (dtrace_nfscl_attrcache_get_hit_probe != NULL)		\
+		(dtrace_nfscl_attrcache_get_hit_probe)(			\
+		    nfscl_attrcache_get_hit_id, (vp), (vap));		\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_GET_MISS(vp)	do {			\
+	if (dtrace_nfscl_attrcache_get_miss_probe != NULL)		\
+		(dtrace_nfscl_attrcache_get_miss_probe)(		\
+			    nfscl_attrcache_get_miss_id, (vp));		\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, vap, error)	do {		\
+	if (dtrace_nfscl_attrcache_load_done_probe != NULL)		\
+		(dtrace_nfscl_attrcache_load_done_probe)(		\
+		    nfscl_attrcache_load_done_id, (vp), (vap),		\
+		    (error));						\
+} while (0)
+
+#else /* !KDTRACE_HOOKS */
+
+#define	KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp)
+#define	KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp, uid, mode)
+#define	KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp, uid, mode)
+#define	KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, uid, rmode, error)
+
+#define	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp)
+#define	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap)
+#define	KDTRACE_NFS_ATTRCACHE_GET_MISS(vp)
+#define	KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, vap, error)
+
+#endif /* KDTRACE_HOOKS */
+
+#endif /* !_NFSCL_NFS_KDTRACE_H_ */