Add DEXITCODE plumbing to NFS.

Isilon has the concept of an in-memory exit-code ring that saves the last exit
code of a function and allows for stack tracing. This is very helpful when
debugging tough issues.

This patch is essentially a no-op for BSD at this point, until we upstream
the dexitcode logic itself. The patch adds DEXITCODE calls to every NFS
function that returns an errno error code. A number of code paths were also
reorganized to have single exit paths, to reduce code duplication.

Submitted by:   David Kwan <dkwan@isilon.com>
Reviewed by:    rmacklem
Approved by:    zml (mentor)
MFC after:      2 weeks

5 files changed, 209 insertions, 107 deletions

diff --git a/sys/fs/nfs/nfs_commonacl.c b/sys/fs/nfs/nfs_commonacl.c
index c4875d8..1d731c5 100644
--- a/sys/fs/nfs/nfs_commonacl.c
+++ b/sys/fs/nfs/nfs_commonacl.c
@@ -59,7 +59,8 @@ nfsrv_dissectace(struct nfsrv_descript *nd, struct acl_entry *acep,
 	mask = fxdr_unsigned(u_int32_t, *tl++);
 	len = fxdr_unsigned(int, *tl);
 	if (len < 0) {
-		return (NFSERR_BADXDR);
+		error = NFSERR_BADXDR;
+		goto nfsmout;
 	} else if (len == 0) {
 		/* Netapp filers return a 0 length who for nil users */
 		acep->ae_tag = ACL_UNDEFINED_TAG;
@@ -68,7 +69,8 @@ nfsrv_dissectace(struct nfsrv_descript *nd, struct acl_entry *acep,
 		acep->ae_entry_type = ACL_ENTRY_TYPE_DENY;
 		if (acesizep)
 			*acesizep = 4 * NFSX_UNSIGNED;
-		return (0);
+		error = 0;
+		goto nfsmout;
 	}
 	if (len > NFSV4_SMALLSTR)
 		name = malloc(len + 1, M_NFSSTRING, M_WAITOK);
@@ -78,7 +80,7 @@ nfsrv_dissectace(struct nfsrv_descript *nd, struct acl_entry *acep,
 	if (error) {
 		if (len > NFSV4_SMALLSTR)
 			free(name, M_NFSSTRING);
-		return (error);
+		goto nfsmout;
 	}
 	if (len == 6) {
 		if (!NFSBCMP(name, "OWNER@", 6)) {
@@ -171,8 +173,9 @@ nfsrv_dissectace(struct nfsrv_descript *nd, struct acl_entry *acep,
 	*aceerrp = aceerr;
 	if (acesizep)
 		*acesizep = NFSM_RNDUP(len) + (4 * NFSX_UNSIGNED);
-	return (0);
+	error = 0;
 nfsmout:
+	NFSEXITCODE(error);
 	return (error);
 }
 
@@ -184,6 +187,7 @@ nfsrv_acemasktoperm(u_int32_t acetype, u_int32_t mask, int owner,
     enum vtype type, acl_perm_t *permp)
 {
 	acl_perm_t perm = 0x0;
+	int error = 0;
 
 	if (mask & NFSV4ACE_READDATA) {
 		mask &= ~NFSV4ACE_READDATA;
@@ -257,10 +261,15 @@ nfsrv_acemasktoperm(u_int32_t acetype, u_int32_t mask, int owner,
 		mask &= ~NFSV4ACE_SYNCHRONIZE;
 		perm |= ACL_SYNCHRONIZE;
 	}
-	if (mask != 0)
-		return (NFSERR_ATTRNOTSUPP);
+	if (mask != 0) {
+		error = NFSERR_ATTRNOTSUPP;
+		goto out;
+	}
 	*permp = perm;
-	return (0);
+
+out:
+	NFSEXITCODE(error);
+	return (error);
 }
 
 /* local functions */
@@ -445,19 +454,26 @@ nfsrv_setacl(vnode_t vp, NFSACL_T *aclp, struct ucred *cred,
 {
 	int error;
 
-	if (nfsrv_useacl == 0 || nfs_supportsnfsv4acls(vp) == 0)
-		return (NFSERR_ATTRNOTSUPP);
+	if (nfsrv_useacl == 0 || nfs_supportsnfsv4acls(vp) == 0) {
+		error = NFSERR_ATTRNOTSUPP;
+		goto out;
+	}
 	/*
 	 * With NFSv4 ACLs, chmod(2) may need to add additional entries.
 	 * Make sure it has enough room for that - splitting every entry
 	 * into two and appending "canonical six" entries at the end.
 	 * Cribbed out of kern/vfs_acl.c - Rick M.
 	 */
-	if (aclp->acl_cnt > (ACL_MAX_ENTRIES - 6) / 2)
-		return (NFSERR_ATTRNOTSUPP);
+	if (aclp->acl_cnt > (ACL_MAX_ENTRIES - 6) / 2) {
+		error = NFSERR_ATTRNOTSUPP;
+		goto out;
+	}
 	error = VOP_ACLCHECK(vp, ACL_TYPE_NFS4, aclp, cred, p);
 	if (!error)
 		error = VOP_SETACL(vp, ACL_TYPE_NFS4, aclp, cred, p);
+
+out:
+	NFSEXITCODE(error);
 	return (error);
 }
 
diff --git a/sys/fs/nfs/nfs_commonkrpc.c b/sys/fs/nfs/nfs_commonkrpc.c
index f658ac8..ca629d2 100644
--- a/sys/fs/nfs/nfs_commonkrpc.c
+++ b/sys/fs/nfs/nfs_commonkrpc.c
@@ -166,7 +166,7 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
 	CLIENT *client;
 	struct netconfig *nconf;
 	struct socket *so;
-	int one = 1, retries, error;
+	int one = 1, retries, error = 0;
 	struct thread *td = curthread;
 
 	/*
@@ -222,7 +222,7 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
 	    nrp->nr_soproto, td->td_ucred, td);
 	if (error) {
 		td->td_ucred = origcred;
-		return (error);
+		goto out;
 	}
 	do {
 	    if (error != 0 && pktscale > 2)
@@ -253,7 +253,7 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
 	soclose(so);
 	if (error) {
 		td->td_ucred = origcred;
-		return (error);
+		goto out;
 	}
 
 	client = clnt_reconnect_create(nconf, saddr, nrp->nr_prog,
@@ -307,7 +307,10 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
 
 	/* Restore current thread's credentials. */
 	td->td_ucred = origcred;
-	return (0);
+
+out:
+	NFSEXITCODE(error);
+	return (error);
 }
 
 /*
diff --git a/sys/fs/nfs/nfs_commonport.c b/sys/fs/nfs/nfs_commonport.c
index 2853854..4935b70 100644
--- a/sys/fs/nfs/nfs_commonport.c
+++ b/sys/fs/nfs/nfs_commonport.c
@@ -329,6 +329,7 @@ nfsvno_pathconf(struct vnode *vp, int flag, register_t *retf,
 		};
 		error = 0;
 	}
+	NFSEXITCODE(error);
 	return (error);
 }
 
@@ -427,6 +428,7 @@ nfssvc_nfscommon(struct thread *td, struct nfssvc_args *uap)
 	int error;
 
 	error = nfssvc_call(td, uap, td->td_ucred);
+	NFSEXITCODE(error);
 	return (error);
 }
 
@@ -439,9 +441,9 @@ nfssvc_call(struct thread *p, struct nfssvc_args *uap, struct ucred *cred)
 	if (uap->flag & NFSSVC_IDNAME) {
 		error = copyin(uap->argp, (caddr_t)&nid, sizeof (nid));
 		if (error)
-			return (error);
+			goto out;
 		error = nfssvc_idname(&nid);
-		return (error);
+		goto out;
 	} else if (uap->flag & NFSSVC_GETSTATS) {
 		error = copyout(&newnfsstats,
 		    CAST_USER_ADDR_T(uap->argp), sizeof (newnfsstats));
@@ -503,7 +505,7 @@ nfssvc_call(struct thread *p, struct nfssvc_args *uap, struct ucred *cred)
 				    sizeof(newnfsstats.cbrpccnt));
 			}
 		}
-		return (error);
+		goto out;
 	} else if (uap->flag & NFSSVC_NFSUSERDPORT) {
 		u_short sockport;
 
@@ -515,6 +517,9 @@ nfssvc_call(struct thread *p, struct nfssvc_args *uap, struct ucred *cred)
 		nfsrv_nfsuserddelport();
 		error = 0;
 	}
+
+out:
+	NFSEXITCODE(error);
 	return (error);
 }
 
@@ -569,7 +574,7 @@ nfscommon_modevent(module_t mod, int type, void *data)
 	switch (type) {
 	case MOD_LOAD:
 		if (loaded)
-			return (0);
+			goto out;
 		newnfs_portinit();
 		mtx_init(&nfs_nameid_mutex, "nfs_nameid_mutex", NULL, MTX_DEF);
 		mtx_init(&nfs_sockl_mutex, "nfs_sockl_mutex", NULL, MTX_DEF);
@@ -606,6 +611,9 @@ nfscommon_modevent(module_t mod, int type, void *data)
 		error = EOPNOTSUPP;
 		break;
 	}
+
+out:
+	NFSEXITCODE(error);
 	return error;
 }
 static moduledata_t nfscommon_mod = {
diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c
index 4a75557..880381a 100644
--- a/sys/fs/nfs/nfs_commonsubs.c
+++ b/sys/fs/nfs/nfs_commonsubs.c
@@ -179,8 +179,10 @@ nfsm_mbufuio(struct nfsrv_descript *nd, struct uio *uiop, int siz)
 	len = NFSMTOD(mp, caddr_t) + mbuf_len(mp) - mbufcp;
 	rem = NFSM_RNDUP(siz) - siz;
 	while (siz > 0) {
-		if (uiop->uio_iovcnt <= 0 || uiop->uio_iov == NULL)
-			return (EBADRPC);
+		if (uiop->uio_iovcnt <= 0 || uiop->uio_iov == NULL) {
+			error = EBADRPC;
+			goto out;
+		}
 		left = uiop->uio_iov->iov_len;
 		uiocp = uiop->uio_iov->iov_base;
 		if (left > siz)
@@ -189,8 +191,10 @@ nfsm_mbufuio(struct nfsrv_descript *nd, struct uio *uiop, int siz)
 		while (left > 0) {
 			while (len == 0) {
 				mp = mbuf_next(mp);
-				if (mp == NULL)
-					return (EBADRPC);
+				if (mp == NULL) {
+					error = EBADRPC;
+					goto out;
+				}
 				mbufcp = NFSMTOD(mp, caddr_t);
 				len = mbuf_len(mp);
 			}
@@ -231,6 +235,9 @@ nfsm_mbufuio(struct nfsrv_descript *nd, struct uio *uiop, int siz)
 		else
 			nd->nd_dpos += rem;
 	}
+
+out:
+	NFSEXITCODE2(error, nd);
 	return (error);
 }
 #endif	/* !APPLE */
@@ -308,9 +315,10 @@ nfsm_dissct(struct nfsrv_descript *nd, int siz)
 APPLESTATIC int
 nfsm_advance(struct nfsrv_descript *nd, int offs, int left)
 {
+	int error = 0;
 
 	if (offs == 0)
-		return (0);
+		goto out;
 	/*
 	 * A negative offs should be considered a serious problem.
 	 */
@@ -330,13 +338,18 @@ nfsm_advance(struct nfsrv_descript *nd, int offs, int left)
 	while (offs > left) {
 		offs -= left;
 		nd->nd_md = mbuf_next(nd->nd_md);
-		if (nd->nd_md == NULL)
-			return (EBADRPC);
+		if (nd->nd_md == NULL) {
+			error = EBADRPC;
+			goto out;
+		}
 		left = mbuf_len(nd->nd_md);
 		nd->nd_dpos = NFSMTOD(nd->nd_md, caddr_t);
 	}
 	nd->nd_dpos += offs;
-	return (0);
+
+out:
+	NFSEXITCODE(error);
+	return (error);
 }
 
 /*
@@ -620,8 +633,10 @@ nfsm_getfh(struct nfsrv_descript *nd, struct nfsfh **nfhpp)
 	if (nd->nd_flag & (ND_NFSV3 | ND_NFSV4)) {
 		NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 		if ((len = fxdr_unsigned(int, *tl)) <= 0 ||
-			len > NFSX_FHMAX)
-			return (EBADRPC);
+			len > NFSX_FHMAX) {
+			error = EBADRPC;
+			goto nfsmout;
+		}
 	} else
 		len = NFSX_V2FH;
 	MALLOC(nfhp, struct nfsfh *, sizeof (struct nfsfh) + len,
@@ -629,11 +644,12 @@ nfsm_getfh(struct nfsrv_descript *nd, struct nfsfh **nfhpp)
 	error = nfsrv_mtostr(nd, nfhp->nfh_fh, len);
 	if (error) {
 		FREE((caddr_t)nfhp, M_NFSFH);
-		return (error);
+		goto nfsmout;
 	}
 	nfhp->nfh_len = len;
 	*nfhpp = nfhp;
 nfsmout:
+	NFSEXITCODE2(error, nd);
 	return (error);
 }
 
@@ -670,7 +686,7 @@ nfsrv_dissectacl(struct nfsrv_descript *nd, NFSACL_T *aclp, int *aclerrp,
 		else
 			error = nfsrv_skipace(nd, &acesize);
 		if (error)
-			return (error);
+			goto nfsmout;
 		aclsize += acesize;
 	}
 	if (aclp && !aceerr)
@@ -680,6 +696,7 @@ nfsrv_dissectacl(struct nfsrv_descript *nd, NFSACL_T *aclp, int *aclerrp,
 	if (aclsizep)
 		*aclsizep = aclsize;
 nfsmout:
+	NFSEXITCODE2(error, nd);
 	return (error);
 }
 
@@ -697,6 +714,7 @@ nfsrv_skipace(struct nfsrv_descript *nd, int *acesizep)
 	error = nfsm_advance(nd, NFSM_RNDUP(len), -1);
 nfsmout:
 	*acesizep = NFSM_RNDUP(len) + (4 * NFSX_UNSIGNED);
+	NFSEXITCODE2(error, nd);
 	return (error);
 }
 
@@ -715,8 +733,10 @@ nfsrv_getattrbits(struct nfsrv_descript *nd, nfsattrbit_t *attrbitp, int *cntp,
 
 	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 	cnt = fxdr_unsigned(int, *tl);
-	if (cnt < 0)
-		return (NFSERR_BADXDR);
+	if (cnt < 0) {
+		error = NFSERR_BADXDR;
+		goto nfsmout;
+	}
 	if (cnt > NFSATTRBIT_MAXWORDS) {
 		outcnt = NFSATTRBIT_MAXWORDS;
 		if (retnotsupp)
@@ -735,6 +755,7 @@ nfsrv_getattrbits(struct nfsrv_descript *nd, nfsattrbit_t *attrbitp, int *cntp,
 	if (cntp)
 		*cntp = NFSX_UNSIGNED + (cnt * NFSX_UNSIGNED);
 nfsmout:
+	NFSEXITCODE2(error, nd);
 	return (error);
 }
 
@@ -756,7 +777,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
     u_int32_t *leasep, u_int32_t *rderrp, NFSPROC_T *p, struct ucred *cred)
 {
 	u_int32_t *tl;
-	int i = 0, j, k, l, m, bitpos, attrsum = 0;
+	int i = 0, j, k, l = 0, m, bitpos, attrsum = 0;
 	int error, tfhsize, aceerr, attrsize, cnt, retnotsup;
 	u_char *cp, *cp2, namestr[NFSV4_SMALLSTR + 1];
 	nfsattrbit_t attrbits, retattrbits, checkattrbits;
@@ -783,7 +804,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 		error = nfsrv_getattrbits(nd, &attrbits, NULL, NULL);
 	}
 	if (error)
-		return (error);
+		goto nfsmout;
 
 	if (compare) {
 		*retcmpp = retnotsup;
@@ -854,7 +875,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 			    error = nfsrv_getattrbits(nd, &nap->na_suppattr,
 				&cnt, &retnotsup);
 			if (error)
-			    return (error);
+			    goto nfsmout;
 			if (compare && !(*retcmpp)) {
 			   NFSSETSUPP_ATTRBIT(&checkattrbits);
 			   if (!NFSEQUAL_ATTRBIT(&retattrbits, &checkattrbits)
@@ -1015,7 +1036,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 				    &cnt, p);
 				if (error) {
 				    acl_free(naclp);
-				    return (error);
+				    goto nfsmout;
 				}
 				if (aceerr || nfsrv_compareacl(aclp, naclp))
 				    *retcmpp = NFSERR_NOTSAME;
@@ -1034,7 +1055,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 				error = nfsrv_dissectacl(nd, NULL, &aceerr,
 				    &cnt, p);
 			    if (error)
-				return (error);
+				goto nfsmout;
 			}
 			attrsum += cnt;
 			break;
@@ -1123,7 +1144,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 		case NFSATTRBIT_FILEHANDLE:
 			error = nfsm_getfh(nd, &tnfhp);
 			if (error)
-				return (error);
+				goto nfsmout;
 			tfhsize = tnfhp->nfh_len;
 			if (compare) {
 				if (!(*retcmpp) &&
@@ -1189,7 +1210,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 		case NFSATTRBIT_FSLOCATIONS:
 			error = nfsrv_getrefstr(nd, &cp, &cp2, &l, &m);
 			if (error)
-				return (error);
+				goto nfsmout;
 			attrsum += l;
 			if (compare && !(*retcmpp)) {
 				refp = nfsv4root_getreferral(vp, NULL, 0);
@@ -1365,8 +1386,10 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 		case NFSATTRBIT_OWNER:
 			NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 			j = fxdr_unsigned(int, *tl);
-			if (j < 0)
-				return (NFSERR_BADXDR);
+			if (j < 0) {
+				error = NFSERR_BADXDR;
+				goto nfsmout;
+			}
 			attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(j));
 			if (j > NFSV4_SMALLSTR)
 				cp = malloc(j + 1, M_NFSSTRING, M_WAITOK);
@@ -1376,7 +1399,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 			if (error) {
 				if (j > NFSV4_SMALLSTR)
 					free(cp, M_NFSSTRING);
-				return (error);
+				goto nfsmout;
 			}
 			if (compare) {
 			    if (!(*retcmpp)) {
@@ -1396,8 +1419,10 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 		case NFSATTRBIT_OWNERGROUP:
 			NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 			j = fxdr_unsigned(int, *tl);
-			if (j < 0)
-				return (NFSERR_BADXDR);
+			if (j < 0) {
+				error =  NFSERR_BADXDR;
+				goto nfsmout;
+			}
 			attrsum += (NFSX_UNSIGNED + NFSM_RNDUP(j));
 			if (j > NFSV4_SMALLSTR)
 				cp = malloc(j + 1, M_NFSSTRING, M_WAITOK);
@@ -1407,7 +1432,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 			if (error) {
 				if (j > NFSV4_SMALLSTR)
 					free(cp, M_NFSSTRING);
-				return (error);
+				goto nfsmout;
 			}
 			if (compare) {
 			    if (!(*retcmpp)) {
@@ -1713,6 +1738,7 @@ nfsv4_loadattr(struct nfsrv_descript *nd, vnode_t vp,
 			error = nfsm_advance(nd, attrsize - attrsum, -1);
 	}
 nfsmout:
+	NFSEXITCODE2(error, nd);
 	return (error);
 }
 
@@ -1909,8 +1935,10 @@ nfsrv_mtostr(struct nfsrv_descript *nd, char *str, int siz)
 		siz -= xfer;
 		if (siz > 0) {
 			mp = mbuf_next(mp);
-			if (mp == NULL)
-				return (EBADRPC);
+			if (mp == NULL) {
+				error = EBADRPC;
+				goto out;
+			}
 			cp = NFSMTOD(mp, caddr_t);
 			len = mbuf_len(mp);
 		} else {
@@ -1927,6 +1955,9 @@ nfsrv_mtostr(struct nfsrv_descript *nd, char *str, int siz)
 		else
 			nd->nd_dpos += rem;
 	}
+
+out:
+	NFSEXITCODE2(error, nd);
 	return (error);
 }
 
@@ -2576,9 +2607,12 @@ nfsv4_strtouid(u_char *str, int len, uid_t *uidp, NFSPROC_T *p)
 	u_char *cp;
 	struct nfsusrgrp *usrp;
 	int cnt, ret;
+	int error = 0;
 
-	if (len == 0)
-		return (NFSERR_BADOWNER);
+	if (len == 0) {
+		error = NFSERR_BADOWNER;
+		goto out;
+	}
 	/*
 	 * Look for an '@'.
 	 */
@@ -2609,7 +2643,8 @@ tryagain:
 	if (len == 6 && !NFSBCMP(str, "nobody", 6)) {
 		*uidp = nfsrv_defaultuid;
 		NFSUNLOCKNAMEID();
-		return (0);
+		error = 0;
+		goto out;
 	}
 
 	LIST_FOREACH(usrp, NFSUSERNAMEHASH(str, len), lug_namehash) {
@@ -2621,7 +2656,8 @@ tryagain:
 			TAILQ_REMOVE(&nfsuserlruhead, usrp, lug_lru);
 			TAILQ_INSERT_TAIL(&nfsuserlruhead, usrp, lug_lru);
 			NFSUNLOCKNAMEID();
-			return (0);
+			error = 0;
+			goto out;
 		}
 	}
 	NFSUNLOCKNAMEID();
@@ -2630,7 +2666,11 @@ tryagain:
 	    str, p);
 	if (ret == 0 && cnt < 2)
 		goto tryagain;
-	return (NFSERR_BADOWNER);
+	error = NFSERR_BADOWNER;
+
+out:
+	NFSEXITCODE(error);
+	return (error);
 }
 
 /*
@@ -2756,9 +2796,12 @@ nfsv4_strtogid(u_char *str, int len, gid_t *gidp, NFSPROC_T *p)
 	u_char *cp;
 	struct nfsusrgrp *usrp;
 	int cnt, ret;
+	int error = 0;
 
-	if (len == 0)
-		return (NFSERR_BADOWNER);
+	if (len == 0) {
+		error =  NFSERR_BADOWNER;
+		goto out;
+	}
 	/*
 	 * Look for an '@'.
 	 */
@@ -2787,7 +2830,8 @@ tryagain:
 	if (len == 7 && !NFSBCMP(str, "nogroup", 7)) {
 		*gidp = nfsrv_defaultgid;
 		NFSUNLOCKNAMEID();
-		return (0);
+		error = 0;
+		goto out;
 	}
 
 	LIST_FOREACH(usrp, NFSGROUPNAMEHASH(str, len), lug_namehash) {
@@ -2799,7 +2843,8 @@ tryagain:
 			TAILQ_REMOVE(&nfsuserlruhead, usrp, lug_lru);
 			TAILQ_INSERT_TAIL(&nfsuserlruhead, usrp, lug_lru);
 			NFSUNLOCKNAMEID();
-			return (0);
+			error = 0;
+			goto out;
 		}
 	}
 	NFSUNLOCKNAMEID();
@@ -2808,7 +2853,11 @@ tryagain:
 	    str, p);
 	if (ret == 0 && cnt < 2)
 		goto tryagain;
-	return (NFSERR_BADOWNER);
+	error = NFSERR_BADOWNER;
+
+out:
+	NFSEXITCODE(error);
+	return (error);
 }
 
 /*
@@ -2853,7 +2902,8 @@ nfsrv_nfsuserdport(u_short port, NFSPROC_T *p)
 	NFSLOCKNAMEID();
 	if (nfsrv_nfsuserd) {
 		NFSUNLOCKNAMEID();
-		return (EPERM);
+		error = EPERM;
+		goto out;
 	}
 	nfsrv_nfsuserd = 1;
 	NFSUNLOCKNAMEID();
@@ -2879,6 +2929,8 @@ nfsrv_nfsuserdport(u_short port, NFSPROC_T *p)
 		NFSSOCKADDRFREE(rp->nr_nam);
 		nfsrv_nfsuserd = 0;
 	}
+out:
+	NFSEXITCODE(error);
 	return (error);
 }
 
@@ -2918,7 +2970,8 @@ nfsrv_getuser(int procnum, uid_t uid, gid_t gid, char *name, NFSPROC_T *p)
 	NFSLOCKNAMEID();
 	if (nfsrv_nfsuserd == 0) {
 		NFSUNLOCKNAMEID();
-		return (EPERM);
+		error = EPERM;
+		goto out;
 	}
 	NFSUNLOCKNAMEID();
 	nd = &nfsd;
@@ -2944,6 +2997,8 @@ nfsrv_getuser(int procnum, uid_t uid, gid_t gid, char *name, NFSPROC_T *p)
 		mbuf_freem(nd->nd_mrep);
 		error = nd->nd_repstat;
 	}
+out:
+	NFSEXITCODE(error);
 	return (error);
 }
 
@@ -3000,7 +3055,7 @@ nfssvc_idname(struct nfsd_idargs *nidp)
 	    NFSUNLOCKNAMEID();
 	    if (error)
 		free(cp, M_NFSSTRING);
-	    return (error);
+	    goto out;
 	}
 
 	/*
@@ -3013,7 +3068,7 @@ nfssvc_idname(struct nfsd_idargs *nidp)
 	    nidp->nid_namelen);
 	if (error) {
 		free((caddr_t)newusrp, M_NFSUSERGROUP);
-		return (error);
+		goto out;
 	}
 	newusrp->lug_namelen = nidp->nid_namelen;
 
@@ -3088,6 +3143,8 @@ nfssvc_idname(struct nfsd_idargs *nidp)
 	} else
 		FREE((caddr_t)newusrp, M_NFSUSERGROUP);
 	NFSUNLOCKNAMEID();
+out:
+	NFSEXITCODE(error);
 	return (error);
 }
 
@@ -3117,6 +3174,7 @@ nfsrv_checkutf8(u_int8_t *cp, int len)
 	int cnt = 0, gotd = 0, shift = 0;
 	u_int8_t byte;
 	static int utf8_shift[5] = { 7, 11, 16, 21, 26 };
+	int error = 0;
 
 	/*
 	 * Here are what the variables are used for:
@@ -3133,14 +3191,18 @@ nfsrv_checkutf8(u_int8_t *cp, int len)
 		if (cnt > 0) {
 			/* This handles the 10xxxxxx bytes */
 			if ((*cp & 0xc0) != 0x80 ||
-			    (gotd && (*cp & 0x20)))
-				return (NFSERR_INVAL);
+			    (gotd && (*cp & 0x20))) {
+				error = NFSERR_INVAL;
+				goto out;
+			}
 			gotd = 0;
 			val <<= 6;
 			val |= (*cp & 0x3f);
 			cnt--;
-			if (cnt == 0 && (val >> shift) == 0x0)
-				return (NFSERR_INVAL);
+			if (cnt == 0 && (val >> shift) == 0x0) {
+				error = NFSERR_INVAL;
+				goto out;
+			}
 		} else if (*cp & 0x80) {
 			/* first byte of multi byte char */
 			byte = *cp;
@@ -3148,8 +3210,10 @@ nfsrv_checkutf8(u_int8_t *cp, int len)
 				cnt++;
 				byte <<= 1;
 			}
-			if (cnt == 0 || cnt == 6)
-				return (NFSERR_INVAL);
+			if (cnt == 0 || cnt == 6) {
+				error = NFSERR_INVAL;
+				goto out;
+			}
 			val = (*cp & (0x3f >> cnt));
 			shift = utf8_shift[cnt - 1];
 			if (cnt == 2 && val == 0xd)
@@ -3160,8 +3224,11 @@ nfsrv_checkutf8(u_int8_t *cp, int len)
 		len--;
 	}
 	if (cnt > 0)
-		return (NFSERR_INVAL);
-	return (0);
+		error = NFSERR_INVAL;
+
+out:
+	NFSEXITCODE(error);
+	return (error);
 }
 
 /*
@@ -3182,7 +3249,7 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 {
 	u_int32_t *tl;
 	u_char *cp = NULL, *cp2 = NULL, *cp3, *str;
-	int i, j, len, stringlen, cnt, slen, siz, xdrsum, error, nsrv;
+	int i, j, len, stringlen, cnt, slen, siz, xdrsum, error = 0, nsrv;
 	struct list {
 		SLIST_ENTRY(list) next;
 		int len;
@@ -3200,15 +3267,20 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 	 */
 	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 	len = fxdr_unsigned(int, *tl);
-	if (len < 0 || len > 10240)
-		return (NFSERR_BADXDR);
+	if (len < 0 || len > 10240) {
+		error = NFSERR_BADXDR;
+		goto nfsmout;
+	}
 	if (len == 0) {
 		NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
-		if (*tl != 0)
-			return (NFSERR_BADXDR);
+		if (*tl != 0) {
+			error = NFSERR_BADXDR;
+			goto nfsmout;
+		}
 		*nilp = 1;
 		*sump = 2 * NFSX_UNSIGNED;
-		return (0);
+		error = 0;
+		goto nfsmout;
 	}
 	cp = malloc(len + 1, M_NFSSTRING, M_WAITOK);
 	error = nfsrv_mtostr(nd, cp, len);
@@ -3218,10 +3290,8 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 		if (cnt <= 0)
 			error = NFSERR_BADXDR;
 	}
-	if (error) {
-		free(cp, M_NFSSTRING);
-		return (error);
-	}
+	if (error)
+		goto nfsmout;
 
 	/*
 	 * Now, loop through the location list and make up the srvlist.
@@ -3235,9 +3305,8 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 		NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 		nsrv = fxdr_unsigned(int, *tl);
 		if (nsrv <= 0) {
-			free(cp, M_NFSSTRING);
-			free(cp2, M_NFSSTRING);
-			return (NFSERR_BADXDR);
+			error = NFSERR_BADXDR;
+			goto nfsmout;
 		}
 
 		/*
@@ -3246,9 +3315,8 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 		NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 		len = fxdr_unsigned(int, *tl);
 		if (len <= 0 || len > 1024) {
-			free(cp, M_NFSSTRING);
-			free(cp2, M_NFSSTRING);
-			return (NFSERR_BADXDR);
+			error = NFSERR_BADXDR;
+			goto nfsmout;
 		}
 		nfsrv_refstrbigenough(siz + len + 3, &cp2, &cp3, &slen);
 		if (cp3 != cp2) {
@@ -3256,11 +3324,8 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 			siz++;
 		}
 		error = nfsrv_mtostr(nd, cp3, len);
-		if (error) {
-			free(cp, M_NFSSTRING);
-			free(cp2, M_NFSSTRING);
-			return (error);
-		}
+		if (error)
+			goto nfsmout;
 		cp3 += len;
 		*cp3++ = ':';
 		siz += (len + 1);
@@ -3272,18 +3337,14 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 			NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 			len = fxdr_unsigned(int, *tl);
 			if (len <= 0 || len > 1024) {
-				free(cp, M_NFSSTRING);
-				free(cp2, M_NFSSTRING);
-				return (NFSERR_BADXDR);
+				error = NFSERR_BADXDR;
+				goto nfsmout;
 			}
 			lsp = (struct list *)malloc(sizeof (struct list)
 			    + len, M_TEMP, M_WAITOK);
 			error = nfsrv_mtostr(nd, lsp->host, len);
-			if (error) {
-				free(cp, M_NFSSTRING);
-				free(cp2, M_NFSSTRING);
-				return (error);
-			}
+			if (error)
+				goto nfsmout;
 			xdrsum += NFSX_UNSIGNED + NFSM_RNDUP(len);
 			lsp->len = len;
 			SLIST_INSERT_HEAD(&head, lsp, next);
@@ -3295,17 +3356,13 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 		NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 		len = fxdr_unsigned(int, *tl);
 		if (len <= 0 || len > 1024) {
-			free(cp, M_NFSSTRING);
-			free(cp2, M_NFSSTRING);
-			return (NFSERR_BADXDR);
+			error = NFSERR_BADXDR;
+			goto nfsmout;
 		}
 		nfsrv_refstrbigenough(siz + len + 1, &cp2, &cp3, &slen);
 		error = nfsrv_mtostr(nd, cp3, len);
-		if (error) {
-			free(cp, M_NFSSTRING);
-			free(cp2, M_NFSSTRING);
-			return (error);
-		}
+		if (error)
+			goto nfsmout;
 		xdrsum += NFSX_UNSIGNED + NFSM_RNDUP(len);
 		str = cp3;
 		stringlen = len;
@@ -3328,12 +3385,14 @@ nfsrv_getrefstr(struct nfsrv_descript *nd, u_char **fsrootp, u_char **srvp,
 	*fsrootp = cp;
 	*srvp = cp2;
 	*sump = xdrsum;
+	NFSEXITCODE2(0, nd);
 	return (0);
 nfsmout:
 	if (cp != NULL)
 		free(cp, M_NFSSTRING);
 	if (cp2 != NULL)
 		free(cp2, M_NFSSTRING);
+	NFSEXITCODE2(error, nd);
 	return (error);
 }
 
diff --git a/sys/fs/nfs/nfsdport.h b/sys/fs/nfs/nfsdport.h
index a3f05db..529ada2 100644
--- a/sys/fs/nfs/nfsdport.h
+++ b/sys/fs/nfs/nfsdport.h
@@ -57,6 +57,22 @@ struct nfsexstuff {
 	int	nes_secflavors[MAXSECFLAVORS];	/* and the flavors */
 };
 
+/*
+ * These are NO-OPS for BSD until Isilon upstreams EXITCODE support.
+ * EXITCODE is an in-memory ring buffer that holds the routines failing status.
+ * This is a valuable tool to use when debugging and analyzing issues.
+ * In addition to recording a routine's failing status, it offers
+ * logging of routines for call stack tracing.
+ * EXITCODE should be used only in routines that return a true errno value, as
+ * that value will be formatted to a displayable errno string.  Routines that 
+ * return regular int status that are not true errno should not set EXITCODE.
+ * If you want to log routine tracing, you can add EXITCODE(0) to any routine.
+ * NFS extended the EXITCODE with EXITCODE2 to record either the routine's
+ * exit errno status or the nd_repstat.
+ */
+#define	NFSEXITCODE(error)
+#define	NFSEXITCODE2(error, nd)
+
 #define	NFSVNO_EXINIT(e)		((e)->nes_exflag = 0)
 #define	NFSVNO_EXPORTED(e)		((e)->nes_exflag & MNT_EXPORTED)
 #define	NFSVNO_EXRDONLY(e)		((e)->nes_exflag & MNT_EXRDONLY)