Add DTrace probes to the NFS access and attribute caches. Access cache

events are:

  nfsclient:accesscache:flush:done
  nfsclient:accesscache:get:hit
  nfsclient:accesscache:get:miss
  nfsclient:accesscache:load:done

They pass the vnode, uid, and requested or loaded access mode (if any);
the load event may also report a load error if the RPC fails.

The attribute cache events are:

  nfsclient:attrcache:flush:done
  nfsclient:attrcache:get:hit
  nfsclient:attrcache:get:miss
  nfsclient:attrcache:load:done

They pass the vnode, optionally the vattr if one is present (hit or load),
and in the case of a load event, also a possible RPC error.

MFC after:	1 month
Sponsored by:	Google, Inc.

6 files changed, 536 insertions, 75 deletions

diff --git a/sys/cddl/dev/dtnfsclient/dtnfsclient.c b/sys/cddl/dev/dtnfsclient/dtnfsclient.c
index a44b478..8444ae2 100644
--- a/sys/cddl/dev/dtnfsclient/dtnfsclient.c
+++ b/sys/cddl/dev/dtnfsclient/dtnfsclient.c
@@ -44,11 +44,13 @@ __FBSDID("$FreeBSD$");
 
 /*
  * dtnfsclient is a DTrace provider that tracks the intent to perform RPCs
- * in the NFS client.  This is not quite the same as RPCs, because NFS may
+ * in the NFS client, as well as acess to and maintenance of the access and
+ * attribute caches.  This is not quite the same as RPCs, because NFS may
  * issue multiple RPC transactions in the event that authentication fails,
- * there's a jukebox error, etc.  However, it cleanly represents the logical
- * layer between RPC transmission and vnode/vfs operations, providing access
- * to state linking the two.
+ * there's a jukebox error, or none at all if the access or attribute cache
+ * hits.  However, it cleanly represents the logical layer between RPC
+ * transmission and vnode/vfs operations, providing access to state linking
+ * the two.
  */
 
 static int	dtnfsclient_unload(void);
@@ -68,6 +70,9 @@ static dtrace_pattr_t dtnfsclient_attr = {
 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
 };
 
+/*
+ * Descrition of NFSv3 and (optional) NFSv2 probes for a procedure.
+ */
 struct dtnfsclient_rpc {
 	char		*nr_v3_name;
 	char		*nr_v2_name;	/* Or NULL if none. */
@@ -109,11 +114,28 @@ static struct dtnfsclient_rpc	dtnfsclient_rpcs[NFS_NPROCS] = {
 	{ "noop" },
 };
 
-static char	*dtnfsclient_v2modulename = "nfs2";
-static char	*dtnfsclient_v3modulename = "nfs3";
+/*
+ * Module name strings.
+ */
+static char	*dtnfsclient_accesscache_str = "accesscache";
+static char	*dtnfsclient_attrcache_str = "attrcache";
+static char	*dtnfsclient_nfs2_str = "nfs2";
+static char	*dtnfsclient_nfs3_str = "nfs3";
+
+/*
+ * Function name strings.
+ */
+static char	*dtnfsclient_flush_str = "flush";
+static char	*dtnfsclient_load_str = "load";
+static char	*dtnfsclient_get_str = "get";
 
-static char	*dtnfsclient_start = "start";
-static char	*dtnfsclient_done = "done";
+/*
+ * Name strings.
+ */
+static char	*dtnfsclient_done_str = "done";
+static char	*dtnfsclient_hit_str = "hit";
+static char	*dtnfsclient_miss_str = "miss";
+static char	*dtnfsclient_start_str = "start";
 
 static dtrace_pops_t dtnfsclient_pops = {
 	dtnfsclient_provide,
@@ -131,7 +153,22 @@ static dtrace_pops_t dtnfsclient_pops = {
 static dtrace_provider_id_t	dtnfsclient_id;
 
 /*
- * When tracing on a procedure is enabled, the DTrace ID for the event is
+ * Most probes are generated from the above RPC table, but for access and
+ * attribute caches, we have specific IDs we recognize and handle specially
+ * in various spots.
+ */
+extern uint32_t	nfsclient_accesscache_flush_done_id;
+extern uint32_t	nfsclient_accesscache_get_hit_id;
+extern uint32_t	nfsclient_accesscache_get_miss_id;
+extern uint32_t	nfsclient_accesscache_load_done_id;
+
+extern uint32_t	nfsclient_attrcache_flush_done_id;
+extern uint32_t	nfsclient_attrcache_get_hit_id;
+extern uint32_t	nfsclient_attrcache_get_miss_id;
+extern uint32_t	nfsclient_attrcache_load_done_id;
+
+/*
+ * When tracing on a procedure is enabled, the DTrace ID for an RPC event is
  * stored in one of these two NFS client-allocated arrays; 0 indicates that
  * the event is not being traced so probes should not be called.
  *
@@ -167,28 +204,102 @@ dtnfsclient_getargdesc(void *arg, dtrace_id_t id, void *parg,
 {
 	const char *p = NULL;
 
-	switch (desc->dtargd_ndx) {
-	case 0:
-		p = "struct vnode *";
-		break;
-	case 1:
-		p = "struct mbuf *";
-		break;
-	case 2:
-		p = "struct ucred *";
-		break;
-	case 3:
-		p = "int";
-		break;
-	case 4:
-		if (dtnfs23_isdoneprobe(id)) {
+	if (id == nfsclient_accesscache_flush_done_id ||
+	    id == nfsclient_attrcache_flush_done_id ||
+	    id == nfsclient_attrcache_get_miss_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfsclient_accesscache_get_hit_id ||
+	    id == nfsclient_accesscache_get_miss_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "uid_t";
+			break;
+		case 2:
+			p = "uint32_t";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfsclient_accesscache_load_done_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "uid_t";
+			break;
+		case 2:
+			p = "uint32_t";
+			break;
+		case 3:
 			p = "int";
 			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfsclient_attrcache_get_hit_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct vattr *";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfsclient_attrcache_load_done_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct vattr *";
+			break;
+		case 2:
+			p = "int";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct mbuf *";
+			break;
+		case 2:
+			p = "struct ucred *";
+			break;
+		case 3:
+			p = "int";
+			break;
+		case 4:
+			if (dtnfs23_isdoneprobe(id)) {
+				p = "int";
+				break;
+			}
+			/* FALLSTHROUGH */
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
 		}
-		/* FALLSTHROUGH */
-	default:
-		desc->dtargd_ndx = DTRACE_ARGNONE;
-		break;
 	}
 	if (p != NULL)
 		strlcpy(desc->dtargd_native, p, sizeof(desc->dtargd_native));
@@ -203,56 +314,112 @@ dtnfsclient_provide(void *arg, dtrace_probedesc_t *desc)
 		return;
 
 	/*
-	 * First, register NFSv2 RPC procedures; note sparseness check for
-	 * each slot in the NFSv3 procnum-indexed array.
+	 * Register access cache probes.
+	 */
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_flush_str, dtnfsclient_done_str) == 0) {
+		nfsclient_accesscache_flush_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_flush_str, dtnfsclient_done_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_get_str, dtnfsclient_hit_str) == 0) {
+		nfsclient_accesscache_get_hit_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_get_str, dtnfsclient_hit_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_get_str, dtnfsclient_miss_str) == 0) {
+		nfsclient_accesscache_get_miss_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_get_str, dtnfsclient_miss_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_load_str, dtnfsclient_done_str) == 0) {
+		nfsclient_accesscache_load_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_load_str, dtnfsclient_done_str, 0, NULL);
+	}
+
+	/*
+	 * Register attribute cache probes.
+	 */
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_flush_str, dtnfsclient_done_str) == 0) {
+		nfsclient_attrcache_flush_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_flush_str, dtnfsclient_done_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_get_str, dtnfsclient_hit_str) == 0) {
+		nfsclient_attrcache_get_hit_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_get_str, dtnfsclient_hit_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_get_str, dtnfsclient_miss_str) == 0) {
+		nfsclient_attrcache_get_miss_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_get_str, dtnfsclient_miss_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_load_str, dtnfsclient_done_str) == 0) {
+		nfsclient_attrcache_load_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_load_str, dtnfsclient_done_str, 0, NULL);
+	}
+
+	/*
+	 * Register NFSv2 RPC procedures; note sparseness check for each slot
+	 * in the NFSv3 procnum-indexed array.
 	 */
 	for (i = 0; i < NFS_NPROCS; i++) {
 		if (dtnfsclient_rpcs[i].nr_v2_name != NULL &&
-		    dtrace_probe_lookup(dtnfsclient_id,
-		    dtnfsclient_v2modulename, dtnfsclient_rpcs[i].nr_v2_name,
-		    dtnfsclient_start) == 0) {
+		    dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs2_str,
+		    dtnfsclient_rpcs[i].nr_v2_name, dtnfsclient_start_str) ==
+		    0) {
 			dtnfsclient_rpcs[i].nr_v2_id_start =
 			    dtrace_probe_create(dtnfsclient_id,
-			    dtnfsclient_v2modulename,
+			    dtnfsclient_nfs2_str,
 			    dtnfsclient_rpcs[i].nr_v2_name,
-			    dtnfsclient_start, 0,
+			    dtnfsclient_start_str, 0,
 			    &nfsclient_nfs2_start_probes[i]);
 		}
 		if (dtnfsclient_rpcs[i].nr_v2_name != NULL &&
-		    dtrace_probe_lookup(dtnfsclient_id,
-		    dtnfsclient_v2modulename, dtnfsclient_rpcs[i].nr_v2_name,
-		    dtnfsclient_done) == 0) {
+		    dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs2_str,
+		    dtnfsclient_rpcs[i].nr_v2_name, dtnfsclient_done_str) ==
+		    0) {
 			dtnfsclient_rpcs[i].nr_v2_id_done = 
 			    dtrace_probe_create(dtnfsclient_id,
-			    dtnfsclient_v2modulename,
+			    dtnfsclient_nfs2_str,
 			    dtnfsclient_rpcs[i].nr_v2_name,
-			    dtnfsclient_done, 0,
+			    dtnfsclient_done_str, 0,
 			    &nfsclient_nfs2_done_probes[i]);
 		}
 	}
 
 	/*
-	 * Now, register NFSv3 RPC procedures.
+	 * Register NFSv3 RPC procedures.
 	 */
 	for (i = 0; i < NFS_NPROCS; i++) {
-		if (dtrace_probe_lookup(dtnfsclient_id,
-		    dtnfsclient_v3modulename, dtnfsclient_rpcs[i].nr_v3_name,
-		    dtnfsclient_start) == 0) {
+		if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs3_str,
+		    dtnfsclient_rpcs[i].nr_v3_name, dtnfsclient_start_str) ==
+		    0) {
 			dtnfsclient_rpcs[i].nr_v3_id_start =
 			    dtrace_probe_create(dtnfsclient_id,
-			    dtnfsclient_v3modulename,
+			    dtnfsclient_nfs3_str,
 			    dtnfsclient_rpcs[i].nr_v3_name,
-			    dtnfsclient_start, 0,
+			    dtnfsclient_start_str, 0,
 			    &nfsclient_nfs3_start_probes[i]);
 		}
-		if (dtrace_probe_lookup(dtnfsclient_id,
-		    dtnfsclient_v3modulename, dtnfsclient_rpcs[i].nr_v3_name,
-		    dtnfsclient_done) == 0) {
+		if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs3_str,
+		    dtnfsclient_rpcs[i].nr_v3_name, dtnfsclient_done_str) ==
+		    0) {
 			dtnfsclient_rpcs[i].nr_v3_id_done = 
 			    dtrace_probe_create(dtnfsclient_id,
-			    dtnfsclient_v3modulename,
+			    dtnfsclient_nfs3_str,
 			    dtnfsclient_rpcs[i].nr_v3_name,
-			    dtnfsclient_done, 0,
+			    dtnfsclient_done_str, 0,
 			    &nfsclient_nfs3_done_probes[i]);
 		}
 	}
@@ -267,8 +434,26 @@ static void
 dtnfsclient_enable(void *arg, dtrace_id_t id, void *parg)
 {
 	uint32_t *p = parg;
-
-	*p = id;
+	void *f = dtrace_probe;
+
+	if (id == nfsclient_accesscache_flush_done_id)
+		dtrace_nfsclient_accesscache_flush_done_probe = f;
+	else if (id == nfsclient_accesscache_get_hit_id)
+		dtrace_nfsclient_accesscache_get_hit_probe = f;
+	else if (id == nfsclient_accesscache_get_miss_id)
+		dtrace_nfsclient_accesscache_get_miss_probe = f;
+	else if (id == nfsclient_accesscache_load_done_id)
+		dtrace_nfsclient_accesscache_load_done_probe = f;
+	else if (id == nfsclient_attrcache_flush_done_id)
+		dtrace_nfsclient_attrcache_flush_done_probe = f;
+	else if (id == nfsclient_attrcache_get_hit_id)
+		dtrace_nfsclient_attrcache_get_hit_probe = f;
+	else if (id == nfsclient_attrcache_get_miss_id)
+		dtrace_nfsclient_attrcache_get_miss_probe = f;
+	else if (id == nfsclient_attrcache_load_done_id)
+		dtrace_nfsclient_attrcache_load_done_probe = f;
+	else
+		*p = id;
 }
 
 static void
@@ -276,7 +461,24 @@ dtnfsclient_disable(void *arg, dtrace_id_t id, void *parg)
 {
 	uint32_t *p = parg;
 
-	*p = 0;
+	if (id == nfsclient_accesscache_flush_done_id)
+		dtrace_nfsclient_accesscache_flush_done_probe = NULL;
+	else if (id == nfsclient_accesscache_get_hit_id)
+		dtrace_nfsclient_accesscache_get_hit_probe = NULL;
+	else if (id == nfsclient_accesscache_get_miss_id)
+		dtrace_nfsclient_accesscache_get_miss_probe = NULL;
+	else if (id == nfsclient_accesscache_load_done_id)
+		dtrace_nfsclient_accesscache_load_done_probe = NULL;
+	else if (id == nfsclient_attrcache_flush_done_id)
+		dtrace_nfsclient_attrcache_flush_done_probe = NULL;
+	else if (id == nfsclient_attrcache_get_hit_id)
+		dtrace_nfsclient_attrcache_get_hit_probe = NULL;
+	else if (id == nfsclient_attrcache_get_miss_id)
+		dtrace_nfsclient_attrcache_get_miss_probe = NULL;
+	else if (id == nfsclient_attrcache_load_done_id)
+		dtrace_nfsclient_attrcache_load_done_probe = NULL;
+	else
+		*p = 0;
 }
 
 static void
diff --git a/sys/nfsclient/nfs_bio.c b/sys/nfsclient/nfs_bio.c
index ef863e5..97820f0 100644
--- a/sys/nfsclient/nfs_bio.c
+++ b/sys/nfsclient/nfs_bio.c
@@ -35,6 +35,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_kdtrace.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bio.h>
@@ -61,6 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <nfsclient/nfs.h>
 #include <nfsclient/nfsmount.h>
 #include <nfsclient/nfsnode.h>
+#include <nfsclient/nfs_kdtrace.h>
 
 #include <nfs4client/nfs4.h>
 
@@ -403,6 +406,7 @@ nfs_bioread_check_cons(struct vnode *vp, struct thread *td, struct ucred *cred)
 				goto out;
 		}
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		error = VOP_GETATTR(vp, &vattr, cred);
 		if (error)
 			goto out;
@@ -915,6 +919,7 @@ nfs_write(struct vop_write_args *ap)
 #endif
 flush_and_restart:
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			error = nfs_vinvalbuf(vp, V_SAVE, td, 1);
 			if (error)
 				return (error);
@@ -928,6 +933,7 @@ flush_and_restart:
 	 */
 	if (ioflag & IO_APPEND) {
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		error = VOP_GETATTR(vp, &vattr, cred);
 		if (error)
 			return (error);
@@ -1756,6 +1762,7 @@ nfs_doio(struct vnode *vp, struct buf *bp, struct ucred *cr, struct thread *td)
 			mtx_lock(&np->n_mtx);
 			np->n_flag |= NWRITEERR;
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			mtx_unlock(&np->n_mtx);
 		    }
 		    bp->b_dirtyoff = bp->b_dirtyend = 0;
diff --git a/sys/nfsclient/nfs_kdtrace.h b/sys/nfsclient/nfs_kdtrace.h
new file mode 100644
index 0000000..4bd4557
--- /dev/null
+++ b/sys/nfsclient/nfs_kdtrace.h
@@ -0,0 +1,121 @@
+/*-
+ * Copyright (c) 2009 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed at the University of Cambridge Computer
+ * Laboratory with support from a grant from Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _NFSCLIENT_NFS_KDTRACE_H_
+#define	_NFSCLIENT_NFS_KDTRACE_H_
+
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+/*
+ * Definitions for NFS access cache probes.
+ */
+extern uint32_t nfsclient_accesscache_flush_done_id;
+extern uint32_t nfsclient_accesscache_get_hit_id;
+extern uint32_t nfsclient_accesscache_get_miss_id;
+extern uint32_t nfsclient_accesscache_load_done_id;
+
+#define	KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp)	do {			\
+	if (dtrace_nfsclient_accesscache_flush_done_probe != NULL)	\
+		(dtrace_nfsclient_accesscache_flush_done_probe)(	\
+		    nfsclient_accesscache_flush_done_id, (vp));		\
+} while (0)
+
+#define	KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp, uid, mode)	do {		\
+	if (dtrace_nfsclient_accesscache_get_hit_probe != NULL)		\
+		(dtrace_nfsclient_accesscache_get_hit_probe)(		\
+		    nfsclient_accesscache_get_hit_id, (vp), (uid),	\
+		    (mode));						\
+} while (0)
+	
+#define	KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp, uid, mode)	do {		\
+	if (dtrace_nfsclient_accesscache_get_miss_probe != NULL)	\
+		(dtrace_nfsclient_accesscache_get_miss_probe)(		\
+		    nfsclient_accesscache_get_miss_id, (vp), (uid),	\
+		    (mode));						\
+} while (0)
+
+#define	KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, uid, rmode, error) do {	\
+	if (error && dtrace_nfsclient_accesscache_load_done_probe !=	\
+	    NULL)							\
+		(dtrace_nfsclient_accesscache_load_done_probe)(		\
+		    nfsclient_accesscache_load_done_id, (vp), (uid),	\
+		    (rmode), (error));					\
+} while (0)
+
+/*
+ * Definitions for NFS attribute cache probes.
+ */
+extern uint32_t nfsclient_attrcache_flush_done_id;
+extern uint32_t nfsclient_attrcache_get_hit_id;
+extern uint32_t nfsclient_attrcache_get_miss_id;
+extern uint32_t nfsclient_attrcache_load_done_id;
+
+#define	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp)	do {			\
+	if (dtrace_nfsclient_attrcache_flush_done_probe != NULL)	\
+		(dtrace_nfsclient_attrcache_flush_done_probe)(		\
+		    nfsclient_attrcache_flush_done_id, (vp));		\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap)	do {			\
+	if (dtrace_nfsclient_attrcache_get_hit_probe != NULL)		\
+		(dtrace_nfsclient_attrcache_get_hit_probe)(		\
+		    nfsclient_attrcache_get_hit_id, (vp), (vap));	\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_GET_MISS(vp)	do {			\
+	if (dtrace_nfsclient_attrcache_get_miss_probe != NULL)		\
+		(dtrace_nfsclient_attrcache_get_miss_probe)(		\
+			    nfsclient_attrcache_get_miss_id, (vp));	\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, vap, error)	do {		\
+	if (dtrace_nfsclient_attrcache_load_done_probe != NULL)		\
+		(dtrace_nfsclient_attrcache_load_done_probe)(		\
+		    nfsclient_attrcache_load_done_id, (vp), (vap),	\
+		    (error));						\
+} while (0)
+
+#else /* !KDTRACE_HOOKS */
+
+#define	KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp)
+#define	KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp, uid, mode)
+#define	KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp, uid, mode)
+#define	KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, uid, rmode, error)
+
+#define	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp)
+#define	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap)
+#define	KDTRACE_NFS_ATTRCACHE_GET_MISS(vp)
+#define	KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, vap, error)
+
+#endif /* KDTRACE_HOOKS */
+
+#endif /* !_NFSCLIENT_NFS_KDTRACE_H_ */
diff --git a/sys/nfsclient/nfs_subs.c b/sys/nfsclient/nfs_subs.c
index be6ab92..8d80616 100644
--- a/sys/nfsclient/nfs_subs.c
+++ b/sys/nfsclient/nfs_subs.c
@@ -41,6 +41,8 @@ __FBSDID("$FreeBSD$");
  * copy data between mbuf chains and uio lists.
  */
 
+#include "opt_kdtrace.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -69,6 +71,7 @@ __FBSDID("$FreeBSD$");
 #include <nfs/nfsproto.h>
 #include <nfsclient/nfs.h>
 #include <nfsclient/nfsnode.h>
+#include <nfsclient/nfs_kdtrace.h>
 #include <nfs/xdr_subs.h>
 #include <nfsclient/nfsm_subs.h>
 #include <nfsclient/nfsmount.h>
@@ -81,6 +84,24 @@ __FBSDID("$FreeBSD$");
  */
 #include <machine/stdarg.h>
 
+#ifdef KDTRACE_HOOKS
+dtrace_nfsclient_attrcache_flush_probe_func_t
+    dtrace_nfsclient_attrcache_flush_done_probe;
+uint32_t nfsclient_attrcache_flush_done_id;
+
+dtrace_nfsclient_attrcache_get_hit_probe_func_t
+    dtrace_nfsclient_attrcache_get_hit_probe;
+uint32_t nfsclient_attrcache_get_hit_id;
+
+dtrace_nfsclient_attrcache_get_miss_probe_func_t
+    dtrace_nfsclient_attrcache_get_miss_probe;
+uint32_t nfsclient_attrcache_get_miss_id;
+
+dtrace_nfsclient_attrcache_load_probe_func_t
+    dtrace_nfsclient_attrcache_load_done_probe;
+uint32_t nfsclient_attrcache_load_done_id;
+#endif /* !KDTRACE_HOOKS */
+
 /*
  * Data items converted to xdr at startup, since they are constant
  * This is kinda hokey, but may save a little time doing byte swaps
@@ -556,7 +577,7 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 	struct vnode *vp = *vpp;
 	struct vattr *vap;
 	struct nfs_fattr *fp;
-	struct nfsnode *np;
+	struct nfsnode *np = NULL;
 	int32_t t1;
 	caddr_t cp2;
 	int rdev;
@@ -566,12 +587,15 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 	struct timespec mtime, mtime_save;
 	int v3 = NFS_ISV3(vp);
 	struct thread *td = curthread;
+	int error = 0;
 
 	md = *mdp;
 	t1 = (mtod(md, caddr_t) + md->m_len) - *dposp;
 	cp2 = nfsm_disct(mdp, dposp, NFSX_FATTR(v3), t1, M_WAIT);
-	if (cp2 == NULL)
-		return EBADRPC;
+	if (cp2 == NULL) {
+		error = EBADRPC;
+		goto out;
+	}
 	fp = (struct nfs_fattr *)cp2;
 	if (v3) {
 		vtyp = nfsv3tov_type(fp->fa_type);
@@ -684,6 +708,7 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 				 */
 				vap->va_size = np->n_size;
 				np->n_attrstamp = 0;
+				KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			} else if (np->n_flag & NMODIFIED) {
 				/*
 				 * We've modified the file: Use the larger
@@ -716,9 +741,11 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 	 * We detect this by for the mtime moving back. We invalidate the 
 	 * attrcache when this happens.
 	 */
-	if (timespeccmp(&mtime_save, &vap->va_mtime, >))
+	if (timespeccmp(&mtime_save, &vap->va_mtime, >)) {
 		/* Size changed or mtime went backwards */
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
+	}
 	if (vaper != NULL) {
 		bcopy((caddr_t)vap, (caddr_t)vaper, sizeof(*vap));
 		if (np->n_flag & NCHG) {
@@ -729,7 +756,13 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 		}
 	}
 	mtx_unlock(&np->n_mtx);
-	return (0);
+out:
+#ifdef KDRACE_HOOKS
+	if (np != NULL)
+		KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, error == 0 ? &np->n_vattr
+		    : NULL, error);
+#endif
+	return (error);
 }
 
 #ifdef NFS_ACDEBUG
@@ -794,7 +827,8 @@ nfs_getattrcache(struct vnode *vp, struct vattr *vaper)
 	if ((time_second - np->n_attrstamp) >= timeo) {
 		nfsstats.attrcache_misses++;
 		mtx_unlock(&np->n_mtx);
-		return( ENOENT);
+		KDTRACE_NFS_ATTRCACHE_GET_MISS(vp);
+		return (ENOENT);
 	}
 	nfsstats.attrcache_hits++;
 	if (vap->va_size != np->n_size) {
@@ -823,6 +857,7 @@ nfs_getattrcache(struct vnode *vp, struct vattr *vaper)
 #ifdef NFS_ACDEBUG
 	mtx_unlock(&Giant);	/* nfs_printf() */
 #endif
+	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap);
 	return (0);
 }
 
diff --git a/sys/nfsclient/nfs_vnops.c b/sys/nfsclient/nfs_vnops.c
index a209327..3592280 100644
--- a/sys/nfsclient/nfs_vnops.c
+++ b/sys/nfsclient/nfs_vnops.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_inet.h"
+#include "opt_kdtrace.h"
 
 #include <sys/param.h>
 #include <sys/kernel.h>
@@ -76,6 +77,7 @@ __FBSDID("$FreeBSD$");
 #include <nfsclient/nfs.h>
 #include <nfsclient/nfsnode.h>
 #include <nfsclient/nfsmount.h>
+#include <nfsclient/nfs_kdtrace.h>
 #include <nfsclient/nfs_lock.h>
 #include <nfs/xdr_subs.h>
 #include <nfsclient/nfsm_subs.h>
@@ -85,6 +87,26 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in_var.h>
 #include <netinet/vinet.h>
 
+#include <machine/stdarg.h>
+
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+dtrace_nfsclient_accesscache_flush_probe_func_t
+    dtrace_nfsclient_accesscache_flush_done_probe;
+uint32_t nfsclient_accesscache_flush_done_id;
+
+dtrace_nfsclient_accesscache_get_probe_func_t
+    dtrace_nfsclient_accesscache_get_hit_probe,
+    dtrace_nfsclient_accesscache_get_miss_probe;
+uint32_t nfsclient_accesscache_get_hit_id;
+uint32_t nfsclient_accesscache_get_miss_id;
+
+dtrace_nfsclient_accesscache_load_probe_func_t
+    dtrace_nfsclient_accesscache_load_done_probe;
+uint32_t nfsclient_accesscache_load_done_id;
+#endif /* !KDTRACE_HOOKS */
+
 /* Defs */
 #define	TRUE	1
 #define	FALSE	0
@@ -313,9 +335,11 @@ nfs3_access_otw(struct vnode *vp, int wmode, struct thread *td,
 		mtx_unlock(&np->n_mtx);
 		if (retmode != NULL)
 			*retmode = rmode;
+		KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, cred->cr_uid, rmode, 0);
 	}
 	m_freem(mrep);
 nfsmout:
+	KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, cred->cr_uid, 0, error);
 	return (error);
 }
 
@@ -401,6 +425,14 @@ nfs_access(struct vop_access_args *ap)
 			}
 		}
 		mtx_unlock(&np->n_mtx);
+#ifdef KDTRACE_HOOKS
+		if (gotahit)
+			KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp,
+			    ap->a_cred->cr_uid, mode);
+		else
+			KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp,
+			    ap->a_cred->cr_uid, mode);
+#endif
 		if (gotahit == 0) {
 			/*
 			 * Either a no, or a don't know.  Go to the wire.
@@ -494,6 +526,7 @@ nfs_open(struct vop_open_args *ap)
 		if (error == EINTR || error == EIO)
 			return (error);
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		if (vp->v_type == VDIR)
 			np->n_direofoffset = 0;
 		error = VOP_GETATTR(vp, &vattr, ap->a_cred);
@@ -510,6 +543,7 @@ nfs_open(struct vop_open_args *ap)
 		    td->td_proc == NULL ||
 		    np->n_ac_ts_pid != td->td_proc->p_pid) {
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		}
 		mtx_unlock(&np->n_mtx);						
 		error = VOP_GETATTR(vp, &vattr, ap->a_cred);
@@ -868,6 +902,7 @@ nfs_setattrrpc(struct vnode *vp, struct vattr *vap, struct ucred *cred)
 		for (i = 0; i < NFS_ACCESSCACHESIZE; i++)
 			np->n_accesscache[i].stamp = 0;
 		mtx_unlock(&np->n_mtx);
+		KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp);
 		nfsm_wcc_data(vp, wccflag);
 	} else
 		nfsm_loadattr(vp, NULL);
@@ -1421,8 +1456,10 @@ nfsmout:
 	}
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
 	return (error);
 }
@@ -1553,8 +1590,10 @@ nfsmout:
 	}
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
 	return (error);
 }
@@ -1618,6 +1657,7 @@ nfs_remove(struct vop_remove_args *ap)
 	} else if (!np->n_sillyrename)
 		error = nfs_sillyrename(dvp, vp, cnp);
 	np->n_attrstamp = 0;
+	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 	return (error);
 }
 
@@ -1663,8 +1703,10 @@ nfs_removerpc(struct vnode *dvp, const char *name, int namelen,
 nfsmout:
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
 	return (error);
 }
@@ -1809,10 +1851,14 @@ nfsmout:
 	mtx_lock(&(VTONFS(tdvp))->n_mtx);
 	VTONFS(tdvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(tdvp))->n_mtx);
-	if (!fwccflag)
+	if (!fwccflag) {
 		VTONFS(fdvp)->n_attrstamp = 0;
-	if (!twccflag)
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(fdvp);
+	}
+	if (!twccflag) {
 		VTONFS(tdvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(tdvp);
+	}
 	return (error);
 }
 
@@ -1860,10 +1906,14 @@ nfsmout:
 	mtx_lock(&(VTONFS(tdvp))->n_mtx);
 	VTONFS(tdvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(tdvp))->n_mtx);
-	if (!attrflag)
+	if (!attrflag) {
 		VTONFS(vp)->n_attrstamp = 0;
-	if (!wccflag)
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
+	}
+	if (!wccflag) {
 		VTONFS(tdvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(tdvp);
+	}
 	return (error);
 }
 
@@ -1948,8 +1998,10 @@ nfsmout:
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	return (error);
 }
 
@@ -2004,8 +2056,10 @@ nfsmout:
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	if (error == 0 && newvp == NULL) {
 		error = nfs_lookitup(dvp, cnp->cn_nameptr, len, cnp->cn_cred,
 			cnp->cn_thread, &np);
@@ -2054,8 +2108,10 @@ nfsmout:
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	cache_purge(dvp);
 	cache_purge(vp);
 	/*
diff --git a/sys/sys/dtrace_bsd.h b/sys/sys/dtrace_bsd.h
index 737a47a..f323284 100644
--- a/sys/sys/dtrace_bsd.h
+++ b/sys/sys/dtrace_bsd.h
@@ -35,6 +35,8 @@
 struct mbuf;
 struct trapframe;
 struct thread;
+struct vattr;
+struct vnode;
 
 /*
  * Cyclic clock function type definition used to hook the cyclic
@@ -94,14 +96,52 @@ typedef	void (*dtrace_malloc_probe_func_t)(u_int32_t, uintptr_t arg0,
 
 extern dtrace_malloc_probe_func_t   dtrace_malloc_probe;
 
-/* The dtnfsclient provider hooks into the NFS[23] client. */
-typedef void (*dtrace_nfsclient_nfs23_start_probe_func_t)(u_int32_t,
+/* dtnfsclient NFSv3 access cache provider hooks. */
+typedef void (*dtrace_nfsclient_accesscache_flush_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_accesscache_flush_probe_func_t
+    dtrace_nfsclient_accesscache_flush_done_probe;
+
+typedef void (*dtrace_nfsclient_accesscache_get_probe_func_t)(uint32_t,
+    struct vnode *, uid_t, uint32_t);
+extern dtrace_nfsclient_accesscache_get_probe_func_t
+    dtrace_nfsclient_accesscache_get_hit_probe,
+    dtrace_nfsclient_accesscache_get_miss_probe;
+
+typedef void (*dtrace_nfsclient_accesscache_load_probe_func_t)(uint32_t,
+    struct vnode *, uid_t, uint32_t, int);
+extern dtrace_nfsclient_accesscache_load_probe_func_t
+    dtrace_nfsclient_accesscache_load_done_probe;
+
+/* dtnfsclient NFSv[23] attribute cache provider hooks. */
+typedef void (*dtrace_nfsclient_attrcache_flush_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_attrcache_flush_probe_func_t
+    dtrace_nfsclient_attrcache_flush_done_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_get_hit_probe_func_t)(uint32_t,
+    struct vnode *, struct vattr *);
+extern dtrace_nfsclient_attrcache_get_hit_probe_func_t
+    dtrace_nfsclient_attrcache_get_hit_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_get_miss_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_attrcache_get_miss_probe_func_t
+    dtrace_nfsclient_attrcache_get_miss_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_load_probe_func_t)(uint32_t,
+    struct vnode *, struct vattr *, int);
+extern dtrace_nfsclient_attrcache_load_probe_func_t
+    dtrace_nfsclient_attrcache_load_done_probe;
+
+/* dtnfsclient NFSv[23] RPC provider hooks. */
+typedef void (*dtrace_nfsclient_nfs23_start_probe_func_t)(uint32_t,
     struct vnode *, struct mbuf *, struct ucred *, int);
-typedef void (*dtrace_nfsclient_nfs23_done_probe_func_t)(u_int32_t,
-    struct vnode *, struct mbuf *, struct ucred *, int, int);
-
 extern dtrace_nfsclient_nfs23_start_probe_func_t
     dtrace_nfsclient_nfs23_start_probe;
+
+typedef void (*dtrace_nfsclient_nfs23_done_probe_func_t)(uint32_t,
+    struct vnode *, struct mbuf *, struct ucred *, int, int);
 extern dtrace_nfsclient_nfs23_done_probe_func_t
     dtrace_nfsclient_nfs23_done_probe;