Implement a rudimentary access cache for the Coda kernel module,

modeled on the access cache found in NFS, smbfs, and the Linux coda
module.  This is a positive access cache of a single entry per file,
tracking recently granted rights, but unlike NFS and smbfs,
supporting explicit invalidation by the distributed file system.

For each cnode, maintain a C_ACCCACHE flag indicating the validity
of the cache, and a cached uid and mode tracking recently granted
positive access control decisions.

Prefer the cache to venus_access() in VOP_ACCESS() if it is valid,
and when we must fall back to venus_access(), update the cache.

Allow Venus to clear the access cache, either the whole cache on
CODA_FLUSH, or just entries for a specific uid on CODA_PURGEUSER.
Unlike the Coda module on Linux, we don't flush all entries on a
user purge using a generation number, we instead walk present
cnodes and clear only entries for the specific user, meaning it is
somewhat more expensive but won't hit all users.

Since the Coda module is agressive about not keeping around
unopened cnodes, the utility of the cache is somewhat limited for
files, but works will for directories.  We should make Coda less
agressive about GCing cnodes in VOP_INACTIVE() in order to improve
the effectiveness of in-kernel caching of attributes and access
rights.

MFC after:	1 month

3 files changed, 117 insertions, 28 deletions

diff --git a/sys/fs/coda/cnode.h b/sys/fs/coda/cnode.h
index 3352496..f2797fc 100644
--- a/sys/fs/coda/cnode.h
+++ b/sys/fs/coda/cnode.h
@@ -106,6 +106,8 @@ struct cnode {
 	struct vattr	 c_vattr; 	/* attributes */
 	char		*c_symlink;	/* pointer to symbolic link */
 	u_short		 c_symlen;	/* length of symbolic link */
+	uid_t		 c_cached_uid;	/* cached uid */
+	mode_t		 c_cached_mode;	/* cached access mode */
 	struct cnode	*c_next;	/* links if on FreeBSD machine */
 };
 #define	VTOC(vp)	((struct cnode *)(vp)->v_data)
@@ -114,6 +116,7 @@ struct cnode {
 /* flags */
 #define	C_VATTR		0x01	/* Validity of vattr in the cnode */
 #define	C_SYMLINK	0x02	/* Validity of symlink pointer in the Code */
+#define	C_ACCCACHE	0x04	/* Validity of access cache */
 #define	C_WANTED	0x08	/* Set if lock wanted */
 #define	C_LOCKED	0x10	/* Set if lock held */
 #define	C_UNMOUNTING	0X20	/* Set if unmounting */
@@ -121,6 +124,7 @@ struct cnode {
 
 #define	VALID_VATTR(cp)		((cp->c_flags) & C_VATTR)
 #define	VALID_SYMLINK(cp)	((cp->c_flags) & C_SYMLINK)
+#define	VALID_ACCCACHE(cp)	((cp->c_flags) & C_ACCCACHE)
 #define	IS_UNMOUNTING(cp)	((cp)->c_flags & C_UNMOUNTING)
 
 struct vcomm {
diff --git a/sys/fs/coda/coda_subr.c b/sys/fs/coda/coda_subr.c
index ed081c2..1586133 100644
--- a/sys/fs/coda/coda_subr.c
+++ b/sys/fs/coda/coda_subr.c
@@ -180,6 +180,57 @@ coda_find(CodaFid *fid)
 }
 
 /*
+ * Clear all cached access control decisions from Coda.
+ */
+static void
+coda_acccache_purge(struct mount *mnt)
+{
+	struct cnode *cp;
+	int hash;
+
+	for (hash = 0; hash < CODA_CACHESIZE; hash++) {
+		for (cp = coda_cache[hash]; cp != NULL;
+		    cp = CNODE_NEXT(cp)) {
+			if (CTOV(cp)->v_mount == mnt && VALID_ACCCACHE(cp)) {
+				CODADEBUG(CODA_FLUSH, myprintf(("acccache "
+				    "purge fid %s uid %d mode 0x%x\n",
+				    coda_f2s(&cp->c_fid), cp->c_cached_uid,
+				    cp->c_cached_mode)););
+				cp->c_flags &= ~C_ACCCACHE;
+			}
+		}
+	}
+}
+
+/*
+ * When a user loses their tokens (or other related events), we invalidate
+ * any cached access rights in the access cache.  In the Linux version of
+ * Coda, we maintain a global epoch and simply bump it to invalidate all
+ * cached results generated in the epoch.  For now, we walk all cnodes and
+ * manually invalidate just that uid in FreeBSD.
+ */
+static void
+coda_acccache_purgeuser(struct mount *mnt, uid_t uid)
+{
+	struct cnode *cp;
+	int hash;
+
+	for (hash = 0; hash < CODA_CACHESIZE; hash++) {
+		for (cp = coda_cache[hash]; cp != NULL;
+		    cp = CNODE_NEXT(cp)) {
+			if (CTOV(cp)->v_mount == mnt &&
+			    VALID_ACCCACHE(cp) && (cp->c_cached_uid == uid)) {
+				CODADEBUG(CODA_PURGEUSER, myprintf((
+				    "acccache purgeuser fid %s uid %d mode "
+				    "0x%x\n", coda_f2s(&cp->c_fid),
+				    cp->c_cached_uid, cp->c_cached_mode)););
+				cp->c_flags &= ~C_ACCCACHE;
+			}
+		}
+	}
+}
+
+/*
  * coda_kill is called as a side effect to vcopen.  To prevent any cnodes
  * left around from an earlier run of a venus or warden from causing problems
  * with the new instance, mark any outstanding cnodes as dying.  Future
@@ -244,6 +295,7 @@ coda_flush(struct coda_mntinfo *mnt, enum dc_status dcstat)
 	coda_clstat.ncalls++;
 	coda_clstat.reqs[CODA_FLUSH]++;
 
+	coda_acccache_purge(mnt->mi_vfsp);
 	cache_purgevfs(mnt->mi_vfsp);
 	for (hash = 0; hash < CODA_CACHESIZE; hash++) {
 		for (cp = coda_cache[hash]; cp != NULL;
@@ -415,13 +467,16 @@ handleDownCall(struct coda_mntinfo *mnt, int opcode, union outputArgs *out)
 		coda_clstat.reqs[CODA_PURGEUSER]++;
 
 		/* XXX - need to prevent fsync's. */
-#if 0
+
+		/*
+		 * Purge any access cache entries for the uid.
+		 */
 #ifdef CODA_COMPAT_5
-	  	coda_nc_purge_user(out->coda_purgeuser.cred.cr_uid,
-		    IS_DOWNCALL);
+	  	coda_acccache_purgeuser(mnt->mi_vfsp,
+		    out->coda_purgeuser.cred.cr_uid);
 #else
-		coda_nc_purge_user(out->coda_purgeuser.uid, IS_DOWNCALL);
-#endif
+		coda_acccache_purgeuser(mnt->mi_vfsp,
+		    out->coda_purgeuser.uid);
 #endif
 		/*
 		 * For now, we flush the entire namecache, but this is
@@ -442,7 +497,7 @@ handleDownCall(struct coda_mntinfo *mnt, int opcode, union outputArgs *out)
 		if (cp != NULL) {
 			vref(CTOV(cp));
 			cache_purge(CTOV(cp));
-			cp->c_flags &= ~C_VATTR;
+			cp->c_flags &= ~(C_VATTR | C_ACCCACHE);
 			ASSERT_VOP_LOCKED(CTOV(cp), "coda HandleDownCall");
 			if (CTOV(cp)->v_vflag & VV_TEXT)
 				error = coda_vmflush(cp);
@@ -466,7 +521,7 @@ handleDownCall(struct coda_mntinfo *mnt, int opcode, union outputArgs *out)
 		if (cp != NULL) {
 			vref(CTOV(cp));
 			cache_purge(CTOV(cp));
-			cp->c_flags &= ~C_VATTR;
+			cp->c_flags &= ~(C_VATTR | C_ACCCACHE);
 			CODADEBUG(CODA_ZAPDIR, myprintf(("zapdir: fid = %s, "
 			    "refcnt = %d\n", coda_f2s(&cp->c_fid),
 			    CTOV(cp)->v_usecount - 1)););
@@ -487,7 +542,7 @@ handleDownCall(struct coda_mntinfo *mnt, int opcode, union outputArgs *out)
 		if (cp != NULL) {
 			vref(CTOV(cp));
 			cache_purge(CTOV(cp));
-			cp->c_flags &= ~C_VATTR;
+			cp->c_flags &= ~(C_VATTR | C_ACCCACHE);
 			ASSERT_VOP_LOCKED(CTOV(cp), "coda HandleDownCall");
 			if (!(IS_DIR(out->coda_purgefid.Fid))
 			    && (CTOV(cp)->v_vflag & VV_TEXT))
diff --git a/sys/fs/coda/coda_vnops.c b/sys/fs/coda/coda_vnops.c
index b572412..cc8f812 100644
--- a/sys/fs/coda/coda_vnops.c
+++ b/sys/fs/coda/coda_vnops.c
@@ -77,9 +77,7 @@ __FBSDID("$FreeBSD$");
  */
 static int coda_attr_cache = 1;		/* Set to cache attributes. */
 static int coda_symlink_cache = 1;	/* Set to cache symbolic links. */
-#if 0
 static int coda_access_cache = 1;	/* Set to cache some access checks. */
-#endif
 
 /*
  * Structure to keep track of vfs calls.
@@ -588,7 +586,7 @@ coda_setattr(struct vop_setattr_args *ap)
 		coda_print_vattr(vap);
 	error = venus_setattr(vtomi(vp), &cp->c_fid, vap, cred, td->td_proc);
 	if (!error)
-		cp->c_flags &= ~C_VATTR;
+		cp->c_flags &= ~(C_VATTR | C_ACCCACHE);
 
 	/*
 	 * XXX: Since we now share vm objects between layers, this is
@@ -615,6 +613,7 @@ coda_access(struct vop_access_args *ap)
 	struct ucred *cred = ap->a_cred;
 	struct thread *td = ap->a_td;
 	/* locals */
+	int error;
 
 	MARK_ENTRY(CODA_ACCESS_STATS);
 
@@ -632,16 +631,41 @@ coda_access(struct vop_access_args *ap)
 	}
 
 	/*
-	 * XXXRW: We should add an actual access cache here, similar to the
-	 * one found in NFS, the Linux Coda module, etc.
-	 *
-	 * In principle it could be as simple as caching the uid and granted
-	 * access mode (as in NFS), but we also need invalidation.  The Coda
-	 * module on Linux does this using a global generation number which
-	 * is bumped on an access control cache flush, whereas NFS does it
-	 * with a timeout.
+	 * We maintain a one-entry LRU positive access cache with each cnode.
+	 * In principle we could also track negative results, and for more
+	 * than one uid, but we don't yet.  Venus is responsible for
+	 * invalidating this cache as required.
 	 */
-	return (venus_access(vtomi(vp), &cp->c_fid, mode, cred, td->td_proc));
+	if (coda_access_cache && VALID_ACCCACHE(cp) &&
+	    (cred->cr_uid == cp->c_cached_uid) &&
+	    (mode & cp->c_cached_mode) == mode) {
+		MARK_INT_SAT(CODA_ACCESS_STATS);
+		return (0);
+	}
+	error = venus_access(vtomi(vp), &cp->c_fid, mode, cred, td->td_proc);
+	if (error == 0 && coda_access_cache) {
+		/*-
+		 * When we have a new successful request, we consider three
+		 * cases:
+		 *
+		 * - No initialized access cache, in which case cache the
+		 *   result.
+		 * - Cached result for a different user, in which case we
+		 *   replace the entry.
+		 * - Cached result for the same user, in which case we add
+		 *   any newly granted rights to the cached mode.
+		 *
+		 * XXXRW: If we ever move to something more interesting than
+		 * uid-based token lookup, we'll need to change this.
+		 */
+		cp->c_flags |= C_ACCCACHE;
+		if (cp->c_cached_uid != cred->cr_uid) {
+			cp->c_cached_mode = mode;
+			cp->c_cached_uid = cred->cr_uid;
+		} else
+			cp->c_cached_mode |= mode;
+	}
+	return (error);
 }
 
 int
@@ -1110,7 +1134,7 @@ coda_remove(struct vop_remove_args *ap)
 	 * changed, so invalidate its attr cache also.
 	 */
 	VTOC(dvp)->c_flags &= ~C_VATTR;
-	VTOC(vp)->c_flags &= ~C_VATTR;
+	VTOC(vp)->c_flags &= ~(C_VATTR | C_ACCCACHE);
 	error = venus_remove(vtomi(dvp), &cp->c_fid, nm, len, cred,
 	    td->td_proc);
 	cache_purge(vp);
@@ -1170,6 +1194,8 @@ int
 coda_rename(struct vop_rename_args *ap)
 {
 	/* true args */
+	struct vnode *fvp = ap->a_fvp;
+	struct vnode *tvp = ap->a_tvp;
 	struct vnode *odvp = ap->a_fdvp;
 	struct cnode *odcp = VTOC(odvp);
 	struct componentname  *fcnp = ap->a_fcnp;
@@ -1214,11 +1240,13 @@ coda_rename(struct vop_rename_args *ap)
 	cache_purge(ndvp);
 
 	/*
-	 * Invalidate the parent's attr cache, the modification time has
+	 * Invalidate parent directories as modification times have changed.
+	 * Invalidate access cache on renamed file as rights may have
 	 * changed.
 	 */
 	VTOC(odvp)->c_flags &= ~C_VATTR;
 	VTOC(ndvp)->c_flags &= ~C_VATTR;
+	VTOC(fvp)->c_flags &= ~C_ACCCACHE;
 	if (flen+1 > CODA_MAXNAMLEN) {
 		MARK_INT_FAIL(CODA_RENAME_STATS);
 		error = EINVAL;
@@ -1235,23 +1263,25 @@ exit:
 	CODADEBUG(CODA_RENAME, myprintf(("in rename result %d\n",error)););
 
 	/*
-	 * XXX - do we need to call cache pureg on the moved vnode?
+	 * Update namecache to reflect that the names of various objects may
+	 * have changed (or gone away entirely).
 	 */
-	cache_purge(ap->a_fvp);
+	cache_purge(fvp);
+	cache_purge(tvp);
 
 	/*
 	 * Release parents first, then children.
 	 */
 	vrele(odvp);
-	if (ap->a_tvp) {
-		if (ap->a_tvp == ndvp)
+	if (tvp) {
+		if (tvp == ndvp)
 			vrele(ndvp);
 		else
 			vput(ndvp);
-		vput(ap->a_tvp);
+		vput(tvp);
 	} else
 		vput(ndvp);
-	vrele(ap->a_fvp);
+	vrele(fvp);
 	return (error);
 }