Move the NFSv4.1 client patches over from projects/nfsv4.1-client

to head. I don't think the NFS client behaviour will change unless
the new "minorversion=1" mount option is used. It includes basic
NFSv4.1 support plus support for pNFS using the Files Layout only.
All problems detecting during an NFSv4.1 Bakeathon testing event
in June 2012 have been resolved in this code and it has been tested
against the NFSv4.1 server available to me.
Although not reviewed, I believe that kib@ has looked at it.

9 files changed, 910 insertions, 107 deletions

diff --git a/sys/fs/nfs/nfs.h b/sys/fs/nfs/nfs.h
index 3d588a5..3cc8c8f 100644
--- a/sys/fs/nfs/nfs.h
+++ b/sys/fs/nfs/nfs.h
@@ -50,6 +50,7 @@
 #define	NFS_MAXREXMIT	100		/* Stop counting after this many */
 #define	NFSV4_CALLBACKTIMEO (2 * NFS_HZ) /* Timeout in ticks */
 #define	NFSV4_CALLBACKRETRY 5		/* Number of retries before failure */
+#define	NFSV4_CBSLOTS	8		/* Number of slots for session */
 #define	NFSV4_CBRETRYCNT 4		/* # of CBRecall retries upon err */
 #define	NFSV4_UPCALLTIMEO (15 * NFS_HZ)	/* Timeout in ticks for upcalls */
 					/* to gssd or nfsuserd */
@@ -100,6 +101,9 @@
 #ifndef	NFSCLDELEGHIGHWATER
 #define	NFSCLDELEGHIGHWATER	10000	/* limit for client delegations */
 #endif
+#ifndef	NFSCLLAYOUTHIGHWATER
+#define	NFSCLLAYOUTHIGHWATER	10000	/* limit for client pNFS layouts */
+#endif
 #ifndef NFSNOOPEN			/* Inactive open owner (sec) */
 #define	NFSNOOPEN		120
 #endif
@@ -529,6 +533,7 @@ struct nfsrv_descript {
 	nfsquad_t		nd_clientid;	/* Implied clientid */
 	int			nd_gssnamelen;	/* principal name length */
 	char			*nd_gssname;	/* principal name */
+	uint32_t		*nd_slotseq;	/* ptr to slot seq# in req */
 };
 
 #define	nd_princlen	nd_gssnamelen
@@ -560,6 +565,8 @@ struct nfsrv_descript {
 #define	ND_EXGSSPRIVACY		0x00400000
 #define	ND_INCRSEQID		0x00800000
 #define	ND_NFSCL		0x01000000
+#define	ND_NFSV41		0x02000000
+#define	ND_HASSEQUENCE		0x04000000
 
 /*
  * ND_GSS should be the "or" of all GSS type authentications.
@@ -572,6 +579,7 @@ struct nfsv4_opflag {
 	int	savereply;
 	int	modifyfs;
 	int	lktype;
+	int	needsseq;
 };
 
 /*
@@ -645,6 +653,15 @@ struct nfsv4lock {
 #define	NFSACCCHK_VPNOTLOCKED		0
 #define	NFSACCCHK_VPISLOCKED		1
 
+/*
+ * Slot for the NFSv4.1 Sequence Op.
+ */
+struct nfsslot {
+	int		nfssl_inprog;
+	uint32_t	nfssl_seq;
+	struct mbuf	*nfssl_reply;
+};
+
 #endif	/* _KERNEL */
 
 #endif	/* _NFS_NFS_H */
diff --git a/sys/fs/nfs/nfs_commonkrpc.c b/sys/fs/nfs/nfs_commonkrpc.c
index 7b72ced..6640c1f 100644
--- a/sys/fs/nfs/nfs_commonkrpc.c
+++ b/sys/fs/nfs/nfs_commonkrpc.c
@@ -76,23 +76,27 @@ dtrace_nfsclient_nfs23_done_probe_func_t
 /*
  * Registered probes by RPC type.
  */
-uint32_t	nfscl_nfs2_start_probes[NFS_NPROCS + 1];
-uint32_t	nfscl_nfs2_done_probes[NFS_NPROCS + 1];
+uint32_t	nfscl_nfs2_start_probes[NFSV41_NPROCS + 1];
+uint32_t	nfscl_nfs2_done_probes[NFSV41_NPROCS + 1];
 
-uint32_t	nfscl_nfs3_start_probes[NFS_NPROCS + 1];
-uint32_t	nfscl_nfs3_done_probes[NFS_NPROCS + 1];
+uint32_t	nfscl_nfs3_start_probes[NFSV41_NPROCS + 1];
+uint32_t	nfscl_nfs3_done_probes[NFSV41_NPROCS + 1];
 
-uint32_t	nfscl_nfs4_start_probes[NFS_NPROCS + 1];
-uint32_t	nfscl_nfs4_done_probes[NFS_NPROCS + 1];
+uint32_t	nfscl_nfs4_start_probes[NFSV41_NPROCS + 1];
+uint32_t	nfscl_nfs4_done_probes[NFSV41_NPROCS + 1];
 #endif
 
 NFSSTATESPINLOCK;
 NFSREQSPINLOCK;
+NFSDLOCKMUTEX;
 extern struct nfsstats newnfsstats;
 extern struct nfsreqhead nfsd_reqq;
 extern int nfscl_ticks;
 extern void (*ncl_call_invalcaches)(struct vnode *);
+extern int nfs_numnfscbd;
+extern int nfscl_debuglevel;
 
+SVCPOOL		*nfscbd_pool;
 static int	nfsrv_gsscallbackson = 0;
 static int	nfs_bufpackets = 4;
 static int	nfs_reconnects;
@@ -167,6 +171,7 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
 	struct socket *so;
 	int one = 1, retries, error = 0;
 	struct thread *td = curthread;
+	SVCXPRT *xprt;
 	struct timeval timo;
 
 	/*
@@ -277,6 +282,24 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp,
 				retries = nmp->nm_retry;
 		} else
 			retries = INT_MAX;
+		if (NFSHASNFSV4N(nmp)) {
+			/*
+			 * Make sure the nfscbd_pool doesn't get destroyed
+			 * while doing this.
+			 */
+			NFSD_LOCK();
+			if (nfs_numnfscbd > 0) {
+				nfs_numnfscbd++;
+				NFSD_UNLOCK();
+				xprt = svc_vc_create_backchannel(nfscbd_pool);
+				CLNT_CONTROL(client, CLSET_BACKCHANNEL, xprt);
+				NFSD_LOCK();
+				nfs_numnfscbd--;
+				if (nfs_numnfscbd == 0)
+					wakeup(&nfs_numnfscbd);
+			}
+			NFSD_UNLOCK();
+		}
 	} else {
 		/*
 		 * Three cases:
@@ -468,12 +491,13 @@ int
 newnfs_request(struct nfsrv_descript *nd, struct nfsmount *nmp,
     struct nfsclient *clp, struct nfssockreq *nrp, vnode_t vp,
     struct thread *td, struct ucred *cred, u_int32_t prog, u_int32_t vers,
-    u_char *retsum, int toplevel, u_int64_t *xidp)
+    u_char *retsum, int toplevel, u_int64_t *xidp, struct nfsclsession *sep)
 {
-	u_int32_t *tl;
+	u_int32_t retseq, retval, *tl;
 	time_t waituntil;
-	int i, j, set_sigset = 0, timeo;
+	int i = 0, j = 0, opcnt, set_sigset = 0, slot;
 	int trycnt, error = 0, usegssname = 0, secflavour = AUTH_SYS;
+	int freeslot, timeo;
 	u_int16_t procnum;
 	u_int trylater_delay = 1;
 	struct nfs_feedback_arg nf;
@@ -670,7 +694,9 @@ newnfs_request(struct nfsrv_descript *nd, struct nfsmount *nmp,
 #endif
 	}
 	trycnt = 0;
+	freeslot = -1;		/* Set to slot that needs to be free'd */
 tryagain:
+	slot = -1;		/* Slot that needs a sequence# increment. */
 	/*
 	 * This timeout specifies when a new socket should be created,
 	 * along with new xid values. For UDP, this should be done
@@ -772,11 +798,66 @@ tryagain:
 	nd->nd_dpos = NFSMTOD(nd->nd_md, caddr_t);
 	nd->nd_repstat = 0;
 	if (nd->nd_procnum != NFSPROC_NULL) {
+		/* If sep == NULL, set it to the default in nmp. */
+		if (sep == NULL && nmp != NULL)
+			sep = NFSMNT_MDSSESSION(nmp);
 		/*
 		 * and now the actual NFS xdr.
 		 */
 		NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 		nd->nd_repstat = fxdr_unsigned(u_int32_t, *tl);
+		if (nd->nd_repstat >= 10000)
+			NFSCL_DEBUG(1, "proc=%d reps=%d\n", (int)nd->nd_procnum,
+			    (int)nd->nd_repstat);
+
+		/*
+		 * Get rid of the tag, return count and SEQUENCE result for
+		 * NFSv4.
+		 */
+		if ((nd->nd_flag & ND_NFSV4) != 0) {
+			NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
+			i = fxdr_unsigned(int, *tl);
+			error = nfsm_advance(nd, NFSM_RNDUP(i), -1);
+			if (error)
+				goto nfsmout;
+			NFSM_DISSECT(tl, u_int32_t *, 3 * NFSX_UNSIGNED);
+			opcnt = fxdr_unsigned(int, *tl++);
+			i = fxdr_unsigned(int, *tl++);
+			j = fxdr_unsigned(int, *tl);
+			if (j >= 10000)
+				NFSCL_DEBUG(1, "fop=%d fst=%d\n", i, j);
+			/*
+			 * If the first op is Sequence, free up the slot.
+			 */
+			if (nmp != NULL && i == NFSV4OP_SEQUENCE && j != 0)
+				NFSCL_DEBUG(1, "failed seq=%d\n", j);
+			if (nmp != NULL && i == NFSV4OP_SEQUENCE && j == 0) {
+				NFSM_DISSECT(tl, uint32_t *, NFSX_V4SESSIONID +
+				    5 * NFSX_UNSIGNED);
+				mtx_lock(&sep->nfsess_mtx);
+				tl += NFSX_V4SESSIONID / NFSX_UNSIGNED;
+				retseq = fxdr_unsigned(uint32_t, *tl++);
+				slot = fxdr_unsigned(int, *tl++);
+				freeslot = slot;
+				if (retseq != sep->nfsess_slotseq[slot])
+					printf("retseq diff 0x%x\n", retseq);
+				retval = fxdr_unsigned(uint32_t, *++tl);
+				if ((retval + 1) < sep->nfsess_foreslots)
+					sep->nfsess_foreslots = (retval + 1);
+				else if ((retval + 1) > sep->nfsess_foreslots)
+					sep->nfsess_foreslots = (retval < 64) ?
+					    (retval + 1) : 64;
+				mtx_unlock(&sep->nfsess_mtx);
+
+				/* Grab the op and status for the next one. */
+				if (opcnt > 1) {
+					NFSM_DISSECT(tl, uint32_t *,
+					    2 * NFSX_UNSIGNED);
+					i = fxdr_unsigned(int, *tl++);
+					j = fxdr_unsigned(int, *tl);
+				}
+			}
+		}
 		if (nd->nd_repstat != 0) {
 			if (((nd->nd_repstat == NFSERR_DELAY ||
 			      nd->nd_repstat == NFSERR_GRACE) &&
@@ -784,7 +865,9 @@ tryagain:
 			     nd->nd_procnum != NFSPROC_DELEGRETURN &&
 			     nd->nd_procnum != NFSPROC_SETATTR &&
 			     nd->nd_procnum != NFSPROC_READ &&
+			     nd->nd_procnum != NFSPROC_READDS &&
 			     nd->nd_procnum != NFSPROC_WRITE &&
+			     nd->nd_procnum != NFSPROC_WRITEDS &&
 			     nd->nd_procnum != NFSPROC_OPEN &&
 			     nd->nd_procnum != NFSPROC_CREATE &&
 			     nd->nd_procnum != NFSPROC_OPENCONFIRM &&
@@ -801,6 +884,13 @@ tryagain:
 				while (NFSD_MONOSEC < waituntil)
 					(void) nfs_catnap(PZERO, 0, "nfstry");
 				trylater_delay *= 2;
+				if (slot != -1) {
+					mtx_lock(&sep->nfsess_mtx);
+					sep->nfsess_slotseq[slot]++;
+					*nd->nd_slotseq = txdr_unsigned(
+					    sep->nfsess_slotseq[slot]);
+					mtx_unlock(&sep->nfsess_mtx);
+				}
 				m_freem(nd->nd_mrep);
 				nd->nd_mrep = NULL;
 				goto tryagain;
@@ -817,34 +907,22 @@ tryagain:
 					(*ncl_call_invalcaches)(vp);
 			}
 		}
-
-		/*
-		 * Get rid of the tag, return count, and PUTFH result for V4.
-		 */
-		if (nd->nd_flag & ND_NFSV4) {
-			NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
-			i = fxdr_unsigned(int, *tl);
-			error = nfsm_advance(nd, NFSM_RNDUP(i), -1);
-			if (error)
-				goto nfsmout;
-			NFSM_DISSECT(tl, u_int32_t *, 3 * NFSX_UNSIGNED);
-			i = fxdr_unsigned(int, *++tl);
-
+		if ((nd->nd_flag & ND_NFSV4) != 0) {
+			/* Free the slot, as required. */
+			if (freeslot != -1)
+				nfsv4_freeslot(sep, freeslot);
 			/*
-			 * If the first op's status is non-zero, mark that
-			 * there is no more data to process.
+			 * If this op is Putfh, throw its results away.
 			 */
-			if (*++tl)
-				nd->nd_flag |= ND_NOMOREDATA;
-
-			/*
-			 * If the first op is Putfh, throw its results away
-			 * and toss the op# and status for the first op.
-			 */
-			if (nmp != NULL && i == NFSV4OP_PUTFH && *tl == 0) {
+			if (j >= 10000)
+				NFSCL_DEBUG(1, "nop=%d nst=%d\n", i, j);
+			if (nmp != NULL && i == NFSV4OP_PUTFH && j == 0) {
 				NFSM_DISSECT(tl,u_int32_t *,2 * NFSX_UNSIGNED);
 				i = fxdr_unsigned(int, *tl++);
 				j = fxdr_unsigned(int, *tl);
+				if (j >= 10000)
+					NFSCL_DEBUG(1, "n2op=%d n2st=%d\n", i,
+					    j);
 				/*
 				 * All Compounds that do an Op that must
 				 * be in sequence consist of NFSV4OP_PUTFH
@@ -867,19 +945,20 @@ tryagain:
 				      j != NFSERR_RESOURCE &&
 				      j != NFSERR_NOFILEHANDLE)))		 
 					nd->nd_flag |= ND_INCRSEQID;
-				/*
-				 * If the first op's status is non-zero, mark
-				 * that there is no more data to process.
-				 */
-				if (j)
-					nd->nd_flag |= ND_NOMOREDATA;
 			}
+			/*
+			 * If this op's status is non-zero, mark
+			 * that there is no more data to process.
+			 */
+			if (j)
+				nd->nd_flag |= ND_NOMOREDATA;
 
 			/*
 			 * If R_DONTRECOVER is set, replace the stale error
 			 * reply, so that recovery isn't initiated.
 			 */
 			if ((nd->nd_repstat == NFSERR_STALECLIENTID ||
+			     nd->nd_repstat == NFSERR_BADSESSION ||
 			     nd->nd_repstat == NFSERR_STALESTATEID) &&
 			    rep != NULL && (rep->r_flags & R_DONTRECOVER))
 				nd->nd_repstat = NFSERR_STALEDONTRECOVER;
diff --git a/sys/fs/nfs/nfs_commonport.c b/sys/fs/nfs/nfs_commonport.c
index fe9b94a..545b995 100644
--- a/sys/fs/nfs/nfs_commonport.c
+++ b/sys/fs/nfs/nfs_commonport.c
@@ -106,6 +106,12 @@ MALLOC_DEFINE(M_NEWNFSDIROFF, "NFSCL diroffdiroff",
     "New NFS directory offset data");
 MALLOC_DEFINE(M_NEWNFSDROLLBACK, "NFSD rollback",
     "New NFS local lock rollback");
+MALLOC_DEFINE(M_NEWNFSLAYOUT, "NFSCL layout", "NFSv4.1 Layout");
+MALLOC_DEFINE(M_NEWNFSFLAYOUT, "NFSCL flayout", "NFSv4.1 File Layout");
+MALLOC_DEFINE(M_NEWNFSDEVINFO, "NFSCL devinfo", "NFSv4.1 Device Info");
+MALLOC_DEFINE(M_NEWNFSSOCKREQ, "NFSCL sockreq", "NFS Sock Req");
+MALLOC_DEFINE(M_NEWNFSCLDS, "NFSCL session", "NFSv4.1 Session");
+MALLOC_DEFINE(M_NEWNFSLAYRECALL, "NFSCL layrecall", "NFSv4.1 Layout Recall");
 
 /*
  * Definition of mutex locks.
diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c
index 5fe8315..95aa7bd 100644
--- a/sys/fs/nfs/nfs_commonsubs.c
+++ b/sys/fs/nfs/nfs_commonsubs.c
@@ -85,47 +85,66 @@ NFSSOCKMUTEX;
  * non-idempotent Ops.
  * Define it here, since it is used by both the client and server.
  */
-struct nfsv4_opflag nfsv4_opflag[NFSV4OP_NOPS] = {
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* undef */
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* undef */
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* undef */
-	{ 0, 1, 0, 0, LK_SHARED },		/* Access */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* Close */
-	{ 0, 2, 0, 1, LK_EXCLUSIVE },		/* Commit */
-	{ 1, 2, 1, 1, LK_EXCLUSIVE },		/* Create */
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* Delegpurge */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* Delegreturn */
-	{ 0, 1, 0, 0, LK_SHARED },		/* Getattr */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* GetFH */
-	{ 2, 1, 1, 1, LK_EXCLUSIVE },		/* Link */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* Lock */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* LockT */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* LockU */
-	{ 1, 1, 0, 0, LK_EXCLUSIVE },		/* Lookup */
-	{ 1, 1, 0, 0, LK_EXCLUSIVE },		/* Lookupp */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* NVerify */
-	{ 1, 1, 0, 1, LK_EXCLUSIVE },		/* Open */
-	{ 1, 1, 0, 0, LK_EXCLUSIVE },		/* OpenAttr */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* OpenConfirm */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* OpenDowngrade */
-	{ 1, 0, 0, 0, LK_EXCLUSIVE },		/* PutFH */
-	{ 1, 0, 0, 0, LK_EXCLUSIVE },		/* PutPubFH */
-	{ 1, 0, 0, 0, LK_EXCLUSIVE },		/* PutRootFH */
-	{ 0, 1, 0, 0, LK_SHARED },		/* Read */
-	{ 0, 1, 0, 0, LK_SHARED },		/* Readdir */
-	{ 0, 1, 0, 0, LK_SHARED },		/* ReadLink */
-	{ 0, 2, 1, 1, LK_EXCLUSIVE },		/* Remove */
-	{ 2, 1, 1, 1, LK_EXCLUSIVE },		/* Rename */
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* Renew */
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* RestoreFH */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* SaveFH */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* SecInfo */
-	{ 0, 2, 1, 1, LK_EXCLUSIVE },		/* Setattr */
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* SetClientID */
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* SetClientIDConfirm */
-	{ 0, 1, 0, 0, LK_EXCLUSIVE },		/* Verify */
-	{ 0, 2, 1, 1, LK_EXCLUSIVE },		/* Write */
-	{ 0, 0, 0, 0, LK_EXCLUSIVE },		/* ReleaseLockOwner */
+struct nfsv4_opflag nfsv4_opflag[NFSV41_NOPS] = {
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* undef */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* undef */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* undef */
+	{ 0, 1, 0, 0, LK_SHARED, 1 },			/* Access */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Close */
+	{ 0, 2, 0, 1, LK_EXCLUSIVE, 1 },		/* Commit */
+	{ 1, 2, 1, 1, LK_EXCLUSIVE, 1 },		/* Create */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Delegpurge */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Delegreturn */
+	{ 0, 1, 0, 0, LK_SHARED, 1 },			/* Getattr */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* GetFH */
+	{ 2, 1, 1, 1, LK_EXCLUSIVE, 1 },		/* Link */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Lock */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* LockT */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* LockU */
+	{ 1, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Lookup */
+	{ 1, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Lookupp */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* NVerify */
+	{ 1, 1, 0, 1, LK_EXCLUSIVE, 1 },		/* Open */
+	{ 1, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* OpenAttr */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* OpenConfirm */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* OpenDowngrade */
+	{ 1, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* PutFH */
+	{ 1, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* PutPubFH */
+	{ 1, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* PutRootFH */
+	{ 0, 1, 0, 0, LK_SHARED, 1 },			/* Read */
+	{ 0, 1, 0, 0, LK_SHARED, 1 },			/* Readdir */
+	{ 0, 1, 0, 0, LK_SHARED, 1 },			/* ReadLink */
+	{ 0, 2, 1, 1, LK_EXCLUSIVE, 1 },		/* Remove */
+	{ 2, 1, 1, 1, LK_EXCLUSIVE, 1 },		/* Rename */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Renew */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* RestoreFH */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* SaveFH */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* SecInfo */
+	{ 0, 2, 1, 1, LK_EXCLUSIVE, 1 },		/* Setattr */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* SetClientID */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* SetClientIDConfirm */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Verify */
+	{ 0, 2, 1, 1, LK_EXCLUSIVE, 1 },		/* Write */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* ReleaseLockOwner */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Backchannel Ctrl */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Bind Conn to Sess */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0 },		/* Exchange ID */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0 },		/* Create Session */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0 },		/* Destroy Session */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Free StateID */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Get Dir Deleg */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Get Device Info */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Get Device List */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Layout Commit */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Layout Get */
+	{ 0, 1, 0, 0, LK_EXCLUSIVE, 1 },		/* Layout Return */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Secinfo No name */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Sequence */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Set SSV */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Test StateID */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Want Delegation */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 0 },		/* Destroy ClientID */
+	{ 0, 0, 0, 0, LK_EXCLUSIVE, 1 },		/* Reclaim Complete */
 };
 #endif	/* !APPLEKEXT */
 
@@ -147,9 +166,9 @@ static struct nfsuserlruhead nfsuserlruhead;
  * marked 0 in this array, the code will still work, just not quite as
  * efficiently.)
  */
-static int nfs_bigreply[NFS_NPROCS] = { 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0,
+int nfs_bigreply[NFSV41_NPROCS] = { 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-    0, 0, 0, 0, 0 };
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 };
 
 /* local functions */
 static int nfsrv_skipace(struct nfsrv_descript *nd, int *acesizep);
@@ -1857,7 +1876,7 @@ nfsv4_getref(struct nfsv4lock *lp, int *isleptp, void *mutex,
 		if (isleptp)
 			*isleptp = 1;
 		(void) nfsmsleep(&lp->nfslock_lock, mutex,
-		    PZERO - 1, "nfsv4lck", NULL);
+		    PZERO - 1, "nfsv4gr", NULL);
 	}
 	if (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0)
 		return;
@@ -3016,7 +3035,7 @@ nfsrv_getuser(int procnum, uid_t uid, gid_t gid, char *name, NFSPROC_T *p)
 		(void) nfsm_strtom(nd, name, len);
 	}
 	error = newnfs_request(nd, NULL, NULL, &nfsrv_nfsuserdsock, NULL, NULL,
-		cred, RPCPROG_NFSUSERD, RPCNFSUSERD_VERS, NULL, 0, NULL);
+		cred, RPCPROG_NFSUSERD, RPCNFSUSERD_VERS, NULL, 0, NULL, NULL);
 	NFSFREECRED(cred);
 	if (!error) {
 		mbuf_freem(nd->nd_mrep);
@@ -3510,3 +3529,240 @@ newnfs_sndunlock(int *flagp)
 	NFSUNLOCKSOCK();
 }
 
+APPLESTATIC int
+nfsv4_getipaddr(struct nfsrv_descript *nd, struct sockaddr_storage *sa,
+    int *isudp)
+{
+	struct sockaddr_in *sad;
+	struct sockaddr_in6 *sad6;
+	struct in_addr saddr;
+	uint32_t portnum, *tl;
+	int af = 0, i, j, k;
+	char addr[64], protocol[5], *cp;
+	int cantparse = 0, error = 0;
+	uint16_t portv;
+
+	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
+	i = fxdr_unsigned(int, *tl);
+	if (i >= 3 && i <= 4) {
+		error = nfsrv_mtostr(nd, protocol, i);
+		if (error)
+			goto nfsmout;
+		if (strcmp(protocol, "tcp") == 0) {
+			af = AF_INET;
+			*isudp = 0;
+		} else if (strcmp(protocol, "udp") == 0) {
+			af = AF_INET;
+			*isudp = 1;
+		} else if (strcmp(protocol, "tcp6") == 0) {
+			af = AF_INET6;
+			*isudp = 0;
+		} else if (strcmp(protocol, "udp6") == 0) {
+			af = AF_INET6;
+			*isudp = 1;
+		} else
+			cantparse = 1;
+	} else {
+		cantparse = 1;
+		if (i > 0) {
+			error = nfsm_advance(nd, NFSM_RNDUP(i), -1);
+			if (error)
+				goto nfsmout;
+		}
+	}
+	NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
+	i = fxdr_unsigned(int, *tl);
+	if (i < 0) {
+		error = NFSERR_BADXDR;
+		goto nfsmout;
+	} else if (cantparse == 0 && i >= 11 && i < 64) {
+		/*
+		 * The shortest address is 11chars and the longest is < 64.
+		 */
+		error = nfsrv_mtostr(nd, addr, i);
+		if (error)
+			goto nfsmout;
+
+		/* Find the port# at the end and extract that. */
+		i = strlen(addr);
+		k = 0;
+		cp = &addr[i - 1];
+		/* Count back two '.'s from end to get port# field. */
+		for (j = 0; j < i; j++) {
+			if (*cp == '.') {
+				k++;
+				if (k == 2)
+					break;
+			}
+			cp--;
+		}
+		if (k == 2) {
+			/*
+			 * The NFSv4 port# is appended as .N.N, where N is
+			 * a decimal # in the range 0-255, just like an inet4
+			 * address. Cheat and use inet_aton(), which will
+			 * return a Class A address and then shift the high
+			 * order 8bits over to convert it to the port#.
+			 */
+			*cp++ = '\0';
+			if (inet_aton(cp, &saddr) == 1) {
+				portnum = ntohl(saddr.s_addr);
+				portv = (uint16_t)((portnum >> 16) |
+				    (portnum & 0xff));
+			} else
+				cantparse = 1;
+		} else
+			cantparse = 1;
+		if (cantparse == 0) {
+			if (af == AF_INET) {
+				sad = (struct sockaddr_in *)sa;
+				if (inet_pton(af, addr, &sad->sin_addr) == 1) {
+					sad->sin_len = sizeof(*sad);
+					sad->sin_family = AF_INET;
+					sad->sin_port = htons(portv);
+					return (0);
+				}
+			} else {
+				sad6 = (struct sockaddr_in6 *)sa;
+				if (inet_pton(af, addr, &sad6->sin6_addr)
+				    == 1) {
+					sad6->sin6_len = sizeof(*sad6);
+					sad6->sin6_family = AF_INET6;
+					sad6->sin6_port = htons(portv);
+					return (0);
+				}
+			}
+		}
+	} else {
+		if (i > 0) {
+			error = nfsm_advance(nd, NFSM_RNDUP(i), -1);
+			if (error)
+				goto nfsmout;
+		}
+	}
+	error = EPERM;
+nfsmout:
+	return (error);
+}
+
+/*
+ * Handle an NFSv4.1 Sequence request for the session.
+ */
+int
+nfsv4_seqsession(uint32_t seqid, uint32_t slotid, uint32_t highslot,
+    struct nfsslot *slots, struct mbuf **reply, uint16_t maxslot)
+{
+	int error;
+
+	error = 0;
+	*reply = NULL;
+	if (slotid > maxslot)
+		return (NFSERR_BADSLOT);
+	if (seqid == slots[slotid].nfssl_seq) {
+		/* A retry. */
+		if (slots[slotid].nfssl_inprog != 0)
+			error = NFSERR_DELAY;
+		else if (slots[slotid].nfssl_reply != NULL) {
+			*reply = slots[slotid].nfssl_reply;
+			slots[slotid].nfssl_reply = NULL;
+			slots[slotid].nfssl_inprog = 1;
+		} else
+			error = NFSERR_SEQMISORDERED;
+	} else if ((slots[slotid].nfssl_seq + 1) == seqid) {
+		m_freem(slots[slotid].nfssl_reply);
+		slots[slotid].nfssl_reply = NULL;
+		slots[slotid].nfssl_inprog = 1;
+		slots[slotid].nfssl_seq++;
+	} else
+		error = NFSERR_SEQMISORDERED;
+	return (error);
+}
+
+/*
+ * Cache this reply for the slot.
+ */
+void
+nfsv4_seqsess_cacherep(uint32_t slotid, struct nfsslot *slots, struct mbuf *rep)
+{
+
+	slots[slotid].nfssl_reply = rep;
+	slots[slotid].nfssl_inprog = 0;
+}
+
+/*
+ * Generate the xdr for an NFSv4.1 Sequence Operation.
+ */
+APPLESTATIC void
+nfsv4_setsequence(struct nfsrv_descript *nd, struct nfsclsession *sep,
+    int dont_replycache)
+{
+	uint32_t *tl, slotseq = 0;
+	int i, maxslot, slotpos;
+	uint64_t bitval;
+	uint8_t sessionid[NFSX_V4SESSIONID];
+
+	/* Find an unused slot. */
+	slotpos = -1;
+	maxslot = -1;
+	mtx_lock(&sep->nfsess_mtx);
+	do {
+		bitval = 1;
+		for (i = 0; i < sep->nfsess_foreslots; i++) {
+			if ((bitval & sep->nfsess_slots) == 0) {
+				slotpos = i;
+				sep->nfsess_slots |= bitval;
+				sep->nfsess_slotseq[i]++;
+				slotseq = sep->nfsess_slotseq[i];
+				break;
+			}
+			bitval <<= 1;
+		}
+		if (slotpos == -1)
+			(void)mtx_sleep(&sep->nfsess_slots, &sep->nfsess_mtx,
+			    PZERO, "nfsclseq", 0);
+	} while (slotpos == -1);
+	/* Now, find the highest slot in use. (nfsc_slots is 64bits) */
+	bitval = 1;
+	for (i = 0; i < 64; i++) {
+		if ((bitval & sep->nfsess_slots) != 0)
+			maxslot = i;
+		bitval <<= 1;
+	}
+	bcopy(sep->nfsess_sessionid, sessionid, NFSX_V4SESSIONID);
+	mtx_unlock(&sep->nfsess_mtx);
+	KASSERT(maxslot >= 0, ("nfscl_setsequence neg maxslot"));
+
+	/* Build the Sequence arguments. */
+	NFSM_BUILD(tl, uint32_t *, NFSX_V4SESSIONID + 4 * NFSX_UNSIGNED);
+	bcopy(sessionid, tl, NFSX_V4SESSIONID);
+	tl += NFSX_V4SESSIONID / NFSX_UNSIGNED;
+	nd->nd_slotseq = tl;
+	*tl++ = txdr_unsigned(slotseq);
+	*tl++ = txdr_unsigned(slotpos);
+	*tl++ = txdr_unsigned(maxslot);
+	if (dont_replycache == 0)
+		*tl = newnfs_true;
+	else
+		*tl = newnfs_false;
+	nd->nd_flag |= ND_HASSEQUENCE;
+}
+
+/*
+ * Free a session slot.
+ */
+APPLESTATIC void
+nfsv4_freeslot(struct nfsclsession *sep, int slot)
+{
+	uint64_t bitval;
+
+	bitval = 1;
+	if (slot > 0)
+		bitval <<= slot;
+	mtx_lock(&sep->nfsess_mtx);
+	if ((bitval & sep->nfsess_slots) == 0)
+		printf("freeing free slot!!\n");
+	sep->nfsess_slots &= ~bitval;
+	wakeup(&sep->nfsess_slots);
+	mtx_unlock(&sep->nfsess_mtx);
+}
+
diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h
index 8e68d42..a13f880 100644
--- a/sys/fs/nfs/nfs_var.h
+++ b/sys/fs/nfs/nfs_var.h
@@ -69,9 +69,12 @@ struct nfsclowner;
 struct nfsclopen;
 struct nfsclopenhead;
 struct nfsclclient;
+struct nfsclsession;
 struct nfscllockowner;
 struct nfscllock;
 struct nfscldeleg;
+struct nfscllayout;
+struct nfscldevinfo;
 struct nfsv4lock;
 struct nfsvattr;
 struct nfs_vattr;
@@ -257,11 +260,18 @@ int nfsrv_mtostr(struct nfsrv_descript *, char *, int);
 int nfsrv_checkutf8(u_int8_t *, int);
 int newnfs_sndlock(int *);
 void newnfs_sndunlock(int *);
+int nfsv4_getipaddr(struct nfsrv_descript *, struct sockaddr_storage *,
+    int *);
+int nfsv4_seqsession(uint32_t, uint32_t, uint32_t, struct nfsslot *,
+    struct mbuf **, uint16_t);
+void nfsv4_seqsess_cacherep(uint32_t, struct nfsslot *, struct mbuf *);
+void nfsv4_setsequence(struct nfsrv_descript *, struct nfsclsession *, int);
+void nfsv4_freeslot(struct nfsclsession *, int);
 
 /* nfs_clcomsubs.c */
 void nfsm_uiombuf(struct nfsrv_descript *, struct uio *, int);
 void nfscl_reqstart(struct nfsrv_descript *, int, struct nfsmount *,
-    u_int8_t *, int, u_int32_t **);
+    u_int8_t *, int, u_int32_t **, struct nfsclsession *);
 nfsuint64 *nfscl_getcookie(struct nfsnode *, off_t off, int);
 void nfscl_fillsattr(struct nfsrv_descript *, struct vattr *,
       vnode_t, int, u_int32_t);
@@ -360,12 +370,12 @@ int nfsrpc_closerpc(struct nfsrv_descript *, struct nfsmount *,
     struct nfsclopen *, struct ucred *, NFSPROC_T *, int);
 int nfsrpc_openconfirm(vnode_t, u_int8_t *, int, struct nfsclopen *,
     struct ucred *, NFSPROC_T *);
-int nfsrpc_setclient(struct nfsmount *, struct nfsclclient *,
+int nfsrpc_setclient(struct nfsmount *, struct nfsclclient *, int,
     struct ucred *, NFSPROC_T *);
 int nfsrpc_getattr(vnode_t, struct ucred *, NFSPROC_T *,
     struct nfsvattr *, void *);
 int nfsrpc_getattrnovp(struct nfsmount *, u_int8_t *, int, int,
-    struct ucred *, NFSPROC_T *, struct nfsvattr *, u_int64_t *);
+    struct ucred *, NFSPROC_T *, struct nfsvattr *, u_int64_t *, uint32_t *);
 int nfsrpc_setattr(vnode_t, struct vattr *, NFSACL_T *, struct ucred *,
     NFSPROC_T *, struct nfsvattr *, int *, void *);
 int nfsrpc_lookup(vnode_t, char *, int, struct ucred *, NFSPROC_T *,
@@ -404,7 +414,7 @@ int nfsrpc_readdir(vnode_t, struct uio *, nfsuint64 *, struct ucred *,
 int nfsrpc_readdirplus(vnode_t, struct uio *, nfsuint64 *, 
     struct ucred *, NFSPROC_T *, struct nfsvattr *, int *, int *, void *);
 int nfsrpc_commit(vnode_t, u_quad_t, int, struct ucred *,
-    NFSPROC_T *, u_char *, struct nfsvattr *, int *, void *);
+    NFSPROC_T *, struct nfsvattr *, int *, void *);
 int nfsrpc_advlock(vnode_t, off_t, int, struct flock *, int,
     struct ucred *, NFSPROC_T *, void *, int);
 int nfsrpc_lockt(struct nfsrv_descript *, vnode_t,
@@ -419,7 +429,7 @@ int nfsrpc_fsinfo(vnode_t, struct nfsfsinfo *, struct ucred *,
     NFSPROC_T *, struct nfsvattr *, int *, void *);
 int nfsrpc_pathconf(vnode_t, struct nfsv3_pathconf *,
     struct ucred *, NFSPROC_T *, struct nfsvattr *, int *, void *);
-int nfsrpc_renew(struct nfsclclient *, struct ucred *,
+int nfsrpc_renew(struct nfsclclient *, struct nfsclds *, struct ucred *,
     NFSPROC_T *);
 int nfsrpc_rellockown(struct nfsmount *, struct nfscllockowner *, uint8_t *,
     int, struct ucred *, NFSPROC_T *);
@@ -429,16 +439,42 @@ int nfsrpc_delegreturn(struct nfscldeleg *, struct ucred *,
     struct nfsmount *, NFSPROC_T *, int);
 int nfsrpc_getacl(vnode_t, struct ucred *, NFSPROC_T *, NFSACL_T *, void *);
 int nfsrpc_setacl(vnode_t, struct ucred *, NFSPROC_T *, NFSACL_T *, void *);
+int nfsrpc_exchangeid(struct nfsmount *, struct nfsclclient *,
+    struct nfssockreq *, uint32_t, struct nfsclds **, struct ucred *,
+    NFSPROC_T *);
+int nfsrpc_createsession(struct nfsmount *, struct nfsclsession *,
+    struct nfssockreq *, uint32_t, int, struct ucred *, NFSPROC_T *);
+int nfsrpc_destroysession(struct nfsmount *, struct nfsclclient *,
+    struct ucred *, NFSPROC_T *);
+int nfsrpc_destroyclient(struct nfsmount *, struct nfsclclient *,
+    struct ucred *, NFSPROC_T *);
+int nfsrpc_layoutget(struct nfsmount *, uint8_t *, int, int, uint64_t, uint64_t,
+    uint64_t, int, nfsv4stateid_t *, int *, struct nfsclflayouthead *,
+    struct ucred *, NFSPROC_T *, void *);
+int nfsrpc_getdeviceinfo(struct nfsmount *, uint8_t *, int, uint32_t *,
+    struct nfscldevinfo **, struct ucred *, NFSPROC_T *);
+int nfsrpc_layoutcommit(struct nfsmount *, uint8_t *, int, int,
+    uint64_t, uint64_t, uint64_t, nfsv4stateid_t *, int, int, uint8_t *,
+    struct ucred *, NFSPROC_T *, void *);
+int nfsrpc_layoutreturn(struct nfsmount *, uint8_t *, int, int, int, uint32_t,
+    int, uint64_t, uint64_t, nfsv4stateid_t *, int, uint32_t *, struct ucred *,
+    NFSPROC_T *, void *);
+int nfsrpc_reclaimcomplete(struct nfsmount *, struct ucred *, NFSPROC_T *);
+int nfscl_doiods(vnode_t, struct uio *, int *, int *, uint32_t,
+    struct ucred *, NFSPROC_T *);
+int nfscl_findlayoutforio(struct nfscllayout *, uint64_t, uint32_t,
+    struct nfsclflayout **);
+void nfscl_freenfsclds(struct nfsclds *);
 
 /* nfs_clstate.c */
 int nfscl_open(vnode_t, u_int8_t *, int, u_int32_t, int,
     struct ucred *, NFSPROC_T *, struct nfsclowner **, struct nfsclopen **,
     int *, int *, int);
-int nfscl_getstateid(vnode_t, u_int8_t *, int, u_int32_t, struct ucred *,
+int nfscl_getstateid(vnode_t, u_int8_t *, int, u_int32_t, int, struct ucred *,
     NFSPROC_T *, nfsv4stateid_t *, void **);
 void nfscl_ownerrelease(struct nfsclowner *, int, int, int);
 void nfscl_openrelease(struct nfsclopen *, int, int);
-int nfscl_getcl(vnode_t, struct ucred *, NFSPROC_T *,
+int nfscl_getcl(struct mount *, struct ucred *, NFSPROC_T *, int,
     struct nfsclclient **);
 struct nfsclclient *nfscl_findcl(struct nfsmount *);
 void nfscl_clientrelease(struct nfsclclient *);
@@ -490,6 +526,21 @@ void nfscl_deleggetmodtime(vnode_t, struct timespec *);
 int nfscl_tryclose(struct nfsclopen *, struct ucred *,
     struct nfsmount *, NFSPROC_T *);
 void nfscl_cleanup(NFSPROC_T *);
+int nfscl_layout(struct nfsmount *, vnode_t, u_int8_t *, int, nfsv4stateid_t *,
+    int, struct nfsclflayouthead *, struct nfscllayout **, struct ucred *,
+    NFSPROC_T *);
+struct nfscllayout *nfscl_getlayout(struct nfsclclient *, uint8_t *, int,
+    uint64_t, struct nfsclflayout **, int *);
+void nfscl_rellayout(struct nfscllayout *, int);
+struct nfscldevinfo *nfscl_getdevinfo(struct nfsclclient *, uint8_t *,
+    struct nfscldevinfo *);
+void nfscl_reldevinfo(struct nfscldevinfo *);
+int nfscl_adddevinfo(struct nfsmount *, struct nfscldevinfo *,
+    struct nfsclflayout *);
+void nfscl_freelayout(struct nfscllayout *);
+void nfscl_freeflayout(struct nfsclflayout *);
+void nfscl_freedevinfo(struct nfscldevinfo *);
+int nfscl_layoutcommit(vnode_t, NFSPROC_T *);
 
 /* nfs_clport.c */
 int nfscl_nget(mount_t, vnode_t, struct nfsfh *,
@@ -588,7 +639,8 @@ void newnfs_restore_sigmask(struct thread *, sigset_t *);
 int newnfs_msleep(struct thread *, void *, struct mtx *, int, char *, int);
 int newnfs_request(struct nfsrv_descript *, struct nfsmount *,
     struct nfsclient *, struct nfssockreq *, vnode_t, NFSPROC_T *,
-    struct ucred *, u_int32_t, u_int32_t, u_char *, int, u_int64_t *);
+    struct ucred *, u_int32_t, u_int32_t, u_char *, int, u_int64_t *,
+    struct nfsclsession *);
 int newnfs_connect(struct nfsmount *, struct nfssockreq *,
     struct ucred *, NFSPROC_T *, int);
 void newnfs_disconnect(struct nfssockreq *);
diff --git a/sys/fs/nfs/nfscl.h b/sys/fs/nfs/nfscl.h
index 1ff4d8b..bda8d48 100644
--- a/sys/fs/nfs/nfscl.h
+++ b/sys/fs/nfs/nfscl.h
@@ -49,7 +49,7 @@ struct nfsv4node {
  */
 #define	NFSCL_REQSTART(n, p, v) 					\
 	nfscl_reqstart((n), (p), VFSTONFS((v)->v_mount), 		\
-	    VTONFS(v)->n_fhp->nfh_fh, VTONFS(v)->n_fhp->nfh_len, NULL)
+	    VTONFS(v)->n_fhp->nfh_fh, VTONFS(v)->n_fhp->nfh_len, NULL, NULL)
 
 /*
  * These two macros convert between a lease duration and renew interval.
diff --git a/sys/fs/nfs/nfsclstate.h b/sys/fs/nfs/nfsclstate.h
index 868e7cf..aa2bfee 100644
--- a/sys/fs/nfs/nfsclstate.h
+++ b/sys/fs/nfs/nfsclstate.h
@@ -40,26 +40,75 @@ LIST_HEAD(nfsclhead, nfsclclient);
 LIST_HEAD(nfsclownerhead, nfsclowner);
 TAILQ_HEAD(nfscldeleghead, nfscldeleg);
 LIST_HEAD(nfscldeleghash, nfscldeleg);
+TAILQ_HEAD(nfscllayouthead, nfscllayout);
+LIST_HEAD(nfscllayouthash, nfscllayout);
+LIST_HEAD(nfsclflayouthead, nfsclflayout);
+LIST_HEAD(nfscldevinfohead, nfscldevinfo);
+LIST_HEAD(nfsclrecalllayouthead, nfsclrecalllayout);
 #define	NFSCLDELEGHASHSIZE	256
-#define	NFSCLDELEGHASH(c, f, l)						\
+#define	NFSCLDELEGHASH(c, f, l)							\
 	(&((c)->nfsc_deleghash[ncl_hash((f), (l)) % NFSCLDELEGHASHSIZE]))
+#define	NFSCLLAYOUTHASHSIZE	256
+#define	NFSCLLAYOUTHASH(c, f, l)						\
+	(&((c)->nfsc_layouthash[ncl_hash((f), (l)) % NFSCLLAYOUTHASHSIZE]))
+
+/* Structure for NFSv4.1 session stuff. */
+struct nfsclsession {
+	struct mtx	nfsess_mtx;
+	struct nfsslot	nfsess_cbslots[NFSV4_CBSLOTS];
+	nfsquad_t	nfsess_clientid;
+	uint32_t	nfsess_slotseq[64];	/* Max for 64bit nm_slots */
+	uint64_t	nfsess_slots;
+	uint32_t	nfsess_sequenceid;
+	uint32_t	nfsess_maxcache;	/* Max size for cached reply. */
+	uint16_t	nfsess_foreslots;
+	uint16_t	nfsess_backslots;
+	uint8_t		nfsess_sessionid[NFSX_V4SESSIONID];
+};
+
+/*
+ * This structure holds the session, clientid and related information
+ * needed for an NFSv4.1 Meta Data Server (MDS) or Data Server (DS).
+ * It is malloc'd to the correct length.
+ */
+struct nfsclds {
+	TAILQ_ENTRY(nfsclds)	nfsclds_list;
+	struct nfsclsession	nfsclds_sess;
+	struct mtx		nfsclds_mtx;
+	struct nfssockreq	*nfsclds_sockp;
+	time_t			nfsclds_expire;
+	uint16_t		nfsclds_flags;
+	uint16_t		nfsclds_servownlen;
+	uint8_t			nfsclds_verf[NFSX_VERF];
+	uint8_t			nfsclds_serverown[0];
+};
+
+/*
+ * Flags for nfsclds_flags.
+ */
+#define	NFSCLDS_HASWRITEVERF	0x0001
+#define	NFSCLDS_MDS		0x0002
+#define	NFSCLDS_DS		0x0004
 
 struct nfsclclient {
 	LIST_ENTRY(nfsclclient) nfsc_list;
 	struct nfsclownerhead	nfsc_owner;
 	struct nfscldeleghead	nfsc_deleg;
 	struct nfscldeleghash	nfsc_deleghash[NFSCLDELEGHASHSIZE];
-	struct nfsv4lock nfsc_lock;
-	struct proc	*nfsc_renewthread;
-	struct nfsmount	*nfsc_nmp;
-	nfsquad_t	nfsc_clientid;
-	time_t		nfsc_expire;
-	u_int32_t	nfsc_clientidrev;
-	u_int32_t	nfsc_renew;
-	u_int32_t	nfsc_cbident;
-	u_int16_t	nfsc_flags;
-	u_int16_t	nfsc_idlen;
-	u_int8_t	nfsc_id[1];	/* Malloc'd to correct length */
+	struct nfscllayouthead	nfsc_layout;
+	struct nfscllayouthash	nfsc_layouthash[NFSCLLAYOUTHASHSIZE];
+	struct nfscldevinfohead	nfsc_devinfo;
+	struct nfsv4lock	nfsc_lock;
+	struct proc		*nfsc_renewthread;
+	struct nfsmount		*nfsc_nmp;
+	time_t			nfsc_expire;
+	u_int32_t		nfsc_clientidrev;
+	u_int32_t		nfsc_rev;
+	u_int32_t		nfsc_renew;
+	u_int32_t		nfsc_cbident;
+	u_int16_t		nfsc_flags;
+	u_int16_t		nfsc_idlen;
+	u_int8_t		nfsc_id[1];	/* Malloc'd to correct length */
 };
 
 /*
@@ -176,6 +225,141 @@ struct nfscllockownerfh {
 };
 
 /*
+ * MALLOC'd to the correct length to accommodate the file handle.
+ */
+struct nfscllayout {
+	TAILQ_ENTRY(nfscllayout)	nfsly_list;
+	LIST_ENTRY(nfscllayout)		nfsly_hash;
+	nfsv4stateid_t			nfsly_stateid;
+	struct nfsv4lock		nfsly_lock;
+	uint64_t			nfsly_filesid[2];
+	uint64_t			nfsly_lastbyte;
+	struct nfsclflayouthead		nfsly_flayread;
+	struct nfsclflayouthead		nfsly_flayrw;
+	struct nfsclrecalllayouthead	nfsly_recall;
+	time_t				nfsly_timestamp;
+	struct nfsclclient		*nfsly_clp;
+	uint16_t			nfsly_flags;
+	uint16_t			nfsly_fhlen;
+	uint8_t				nfsly_fh[1];
+};
+
+/*
+ * Flags for nfsly_flags.
+ */
+#define	NFSLY_FILES		0x0001
+#define	NFSLY_BLOCK		0x0002
+#define	NFSLY_OBJECT		0x0004
+#define	NFSLY_RECALL		0x0008
+#define	NFSLY_RECALLFILE	0x0010
+#define	NFSLY_RECALLFSID	0x0020
+#define	NFSLY_RECALLALL		0x0040
+#define	NFSLY_RETONCLOSE	0x0080
+#define	NFSLY_WRITTEN		0x0100	/* Has been used to write to a DS. */
+
+/*
+ * MALLOC'd to the correct length to accommodate the file handle list.
+ * These hang off of nfsly_flayread and nfsly_flayrw, sorted in increasing
+ * offset order.
+ * The nfsly_flayread list holds the ones with iomode == NFSLAYOUTIOMODE_READ,
+ * whereas the nfsly_flayrw holds the ones with iomode == NFSLAYOUTIOMODE_RW.
+ */
+struct nfsclflayout {
+	LIST_ENTRY(nfsclflayout)	nfsfl_list;
+	uint8_t				nfsfl_dev[NFSX_V4DEVICEID];
+	uint64_t			nfsfl_off;
+	uint64_t			nfsfl_end;
+	uint64_t			nfsfl_patoff;
+	struct nfscldevinfo		*nfsfl_devp;
+	uint32_t			nfsfl_iomode;
+	uint32_t			nfsfl_util;
+	uint32_t			nfsfl_stripe1;
+	uint16_t			nfsfl_flags;
+	uint16_t			nfsfl_fhcnt;
+	struct nfsfh			*nfsfl_fh[1];	/* FH list for DS */
+};
+
+/*
+ * Flags for nfsfl_flags.
+ */
+#define	NFSFL_RECALL	0x0001		/* File layout has been recalled */
+
+/*
+ * Structure that is used to store a LAYOUTRECALL.
+ */
+struct nfsclrecalllayout {
+	LIST_ENTRY(nfsclrecalllayout)	nfsrecly_list;
+	uint64_t			nfsrecly_off;
+	uint64_t			nfsrecly_len;
+	int				nfsrecly_recalltype;
+	uint32_t			nfsrecly_iomode;
+	uint32_t			nfsrecly_stateseqid;
+};
+
+/*
+ * Stores the NFSv4.1 Device Info. Malloc'd to the correct length to
+ * store the list of network connections and list of indices.
+ * nfsdi_data[] is allocated the following way:
+ * - nfsdi_addrcnt * struct nfsclds
+ * - stripe indices, each stored as one byte, since there can be many
+ *   of them. (This implies a limit of 256 on nfsdi_addrcnt, since the
+ *   indices select which address.)
+ */
+struct nfscldevinfo {
+	LIST_ENTRY(nfscldevinfo)	nfsdi_list;
+	uint8_t				nfsdi_deviceid[NFSX_V4DEVICEID];
+	struct nfsclclient		*nfsdi_clp;
+	uint32_t			nfsdi_refcnt;
+	uint32_t			nfsdi_layoutrefs;
+	uint16_t			nfsdi_stripecnt;
+	uint16_t			nfsdi_addrcnt;
+	struct nfsclds			*nfsdi_data[0];
+};
+
+/* These inline functions return values from nfsdi_data[]. */
+/*
+ * Return a pointer to the address at "pos".
+ */
+static __inline struct nfsclds **
+nfsfldi_addr(struct nfscldevinfo *ndi, int pos)
+{
+
+	if (pos >= ndi->nfsdi_addrcnt)
+		return (NULL);
+	return (&ndi->nfsdi_data[pos]);
+}
+
+/*
+ * Return the Nth ("pos") stripe index.
+ */
+static __inline int
+nfsfldi_stripeindex(struct nfscldevinfo *ndi, int pos)
+{
+	uint8_t *valp;
+
+	if (pos >= ndi->nfsdi_stripecnt)
+		return (-1);
+	valp = (uint8_t *)&ndi->nfsdi_data[ndi->nfsdi_addrcnt];
+	valp += pos;
+	return ((int)*valp);
+}
+
+/*
+ * Set the Nth ("pos") stripe index to "val".
+ */
+static __inline void
+nfsfldi_setstripeindex(struct nfscldevinfo *ndi, int pos, uint8_t val)
+{
+	uint8_t *valp;
+
+	if (pos >= ndi->nfsdi_stripecnt)
+		return;
+	valp = (uint8_t *)&ndi->nfsdi_data[ndi->nfsdi_addrcnt];
+	valp += pos;
+	*valp = val;
+}
+
+/*
  * Macro for incrementing the seqid#.
  */
 #define	NFSCL_INCRSEQID(s, n)	do { 					\
diff --git a/sys/fs/nfs/nfsport.h b/sys/fs/nfs/nfsport.h
index eb026bc..3f22b7a 100644
--- a/sys/fs/nfs/nfsport.h
+++ b/sys/fs/nfs/nfsport.h
@@ -228,6 +228,34 @@
  */
 #define	NFSV4OP_NOPS		40
 
+/*
+ * Additional Ops for NFSv4.1.
+ */
+#define	NFSV4OP_BACKCHANNELCTL	40
+#define	NFSV4OP_BINDCONNTOSESS	41
+#define	NFSV4OP_EXCHANGEID	42
+#define	NFSV4OP_CREATESESSION	43
+#define	NFSV4OP_DESTROYSESSION	44
+#define	NFSV4OP_FREESTATEID	45
+#define	NFSV4OP_GETDIRDELEG	46
+#define	NFSV4OP_GETDEVINFO	47
+#define	NFSV4OP_GETDEVLIST	48
+#define	NFSV4OP_LAYOUTCOMMIT	49
+#define	NFSV4OP_LAYOUTGET	50
+#define	NFSV4OP_LAYOUTRETURN	51
+#define	NFSV4OP_SECINFONONAME	52
+#define	NFSV4OP_SEQUENCE	53
+#define	NFSV4OP_SETSSV		54
+#define	NFSV4OP_TESTSTATEID	55
+#define	NFSV4OP_WANTDELEG	56
+#define	NFSV4OP_DESTROYCLIENTID	57
+#define	NFSV4OP_RECLAIMCOMPL	58
+
+/*
+ * Must be one more than last op#.
+ */
+#define	NFSV41_NOPS		59
+
 /* Quirky case if the illegal op code */
 #define	NFSV4OP_OPILLEGAL	10044
 
@@ -261,6 +289,20 @@
 #define	NFSV4OP_CBNOPS		5
 
 /*
+ * Additional Callback Ops for NFSv4.1 only. Not yet in nfsstats.
+ */
+#define	NFSV4OP_CBLAYOUTRECALL	5
+#define	NFSV4OP_CBNOTIFY	6
+#define	NFSV4OP_CBPUSHDELEG	7
+#define	NFSV4OP_CBRECALLANY	8
+#define	NFSV4OP_CBRECALLOBJAVAIL 9
+#define	NFSV4OP_CBRECALLSLOT	10
+#define	NFSV4OP_CBSEQUENCE	11
+#define	NFSV4OP_CBWANTCANCELLED	12
+#define	NFSV4OP_CBNOTIFYLOCK	13
+#define	NFSV4OP_CBNOTIFYDEVID	14
+
+/*
  * The lower numbers -> 21 are used by NFSv2 and v3. These define higher
  * numbers used by NFSv4.
  * NFS_V3NPROCS is one greater than the last V3 op and NFS_NPROCS is
@@ -293,6 +335,27 @@
  * Must be defined as one higher than the last Proc# above.
  */
 #define	NFSV4_NPROCS		41
+
+/* Additional procedures for NFSv4.1. */
+#define	NFSPROC_EXCHANGEID	41
+#define	NFSPROC_CREATESESSION	42
+#define	NFSPROC_DESTROYSESSION	43
+#define	NFSPROC_DESTROYCLIENT	44
+#define	NFSPROC_FREESTATEID	45
+#define	NFSPROC_LAYOUTGET	46
+#define	NFSPROC_GETDEVICEINFO	47
+#define	NFSPROC_LAYOUTCOMMIT	48
+#define	NFSPROC_LAYOUTRETURN	49
+#define	NFSPROC_RECLAIMCOMPL	50
+#define	NFSPROC_WRITEDS		51
+#define	NFSPROC_READDS		52
+#define	NFSPROC_COMMITDS	53
+
+/*
+ * Must be defined as one higher than the last NFSv4.1 Proc# above.
+ */
+#define	NFSV41_NPROCS		54
+
 #endif	/* NFS_V3NPROCS */
 
 /*
@@ -368,13 +431,13 @@ struct ext_nfsstats {
 #include <fs/nfs/rpcv2.h>
 #include <fs/nfs/nfsproto.h>
 #include <fs/nfs/nfs.h>
+#include <fs/nfs/nfsclstate.h>
 #include <fs/nfs/nfs_var.h>
 #include <fs/nfs/nfsm_subs.h>
 #include <fs/nfs/nfsrvcache.h>
 #include <fs/nfs/nfsrvstate.h>
 #include <fs/nfs/xdr_subs.h>
 #include <fs/nfs/nfscl.h>
-#include <fs/nfs/nfsclstate.h>
 #include <nfsclient/nfsargs.h>
 #include <fs/nfsclient/nfsmount.h>
 
@@ -583,6 +646,8 @@ void nfsrvd_rcv(struct socket *, void *, int);
 #define	NFSPROCLISTUNLOCK()	sx_sunlock(&allproc_lock)
 #define	NFSLOCKSOCKREQ(r)	mtx_lock(&((r)->nr_mtx))
 #define	NFSUNLOCKSOCKREQ(r)	mtx_unlock(&((r)->nr_mtx))
+#define	NFSLOCKDS(d)		mtx_lock(&((d)->nfsclds_mtx))
+#define	NFSUNLOCKDS(d)		mtx_unlock(&((d)->nfsclds_mtx))
 
 /*
  * Use these macros to initialize/free a mutex.
@@ -672,6 +737,12 @@ MALLOC_DECLARE(M_NEWNFSV4NODE);
 MALLOC_DECLARE(M_NEWNFSDIRECTIO);
 MALLOC_DECLARE(M_NEWNFSMNT);
 MALLOC_DECLARE(M_NEWNFSDROLLBACK);
+MALLOC_DECLARE(M_NEWNFSLAYOUT);
+MALLOC_DECLARE(M_NEWNFSFLAYOUT);
+MALLOC_DECLARE(M_NEWNFSDEVINFO);
+MALLOC_DECLARE(M_NEWNFSSOCKREQ);
+MALLOC_DECLARE(M_NEWNFSCLDS);
+MALLOC_DECLARE(M_NEWNFSLAYRECALL);
 #define	M_NFSRVCACHE	M_NEWNFSRVCACHE
 #define	M_NFSDCLIENT	M_NEWNFSDCLIENT
 #define	M_NFSDSTATE	M_NEWNFSDSTATE
@@ -691,6 +762,12 @@ MALLOC_DECLARE(M_NEWNFSDROLLBACK);
 #define	M_NFSV4NODE	M_NEWNFSV4NODE
 #define	M_NFSDIRECTIO	M_NEWNFSDIRECTIO
 #define	M_NFSDROLLBACK	M_NEWNFSDROLLBACK
+#define	M_NFSLAYOUT	M_NEWNFSLAYOUT
+#define	M_NFSFLAYOUT	M_NEWNFSFLAYOUT
+#define	M_NFSDEVINFO	M_NEWNFSDEVINFO
+#define	M_NFSSOCKREQ	M_NEWNFSSOCKREQ
+#define	M_NFSCLDS	M_NEWNFSCLDS
+#define	M_NFSLAYRECALL	M_NEWNFSLAYRECALL
 
 #define	NFSINT_SIGMASK(set) 						\
 	(SIGISMEMBER(set, SIGINT) || SIGISMEMBER(set, SIGTERM) ||	\
@@ -759,12 +836,16 @@ void newnfs_realign(struct mbuf **);
  */
 #define	NFSSTA_HASWRITEVERF	0x00040000  /* Has write verifier */
 #define	NFSSTA_GOTFSINFO	0x00100000  /* Got the fsinfo */
+#define	NFSSTA_NOLAYOUTCOMMIT	0x04000000  /* Don't do LayoutCommit */
+#define	NFSSTA_SESSPERSIST	0x08000000  /* Has a persistent session */
 #define	NFSSTA_TIMEO		0x10000000  /* Experiencing a timeout */
 #define	NFSSTA_LOCKTIMEO	0x20000000  /* Experiencing a lockd timeout */
 #define	NFSSTA_HASSETFSID	0x40000000  /* Has set the fsid */
+#define	NFSSTA_PNFS		0x80000000  /* pNFS is enabled */
 
 #define	NFSHASNFSV3(n)		((n)->nm_flag & NFSMNT_NFSV3)
 #define	NFSHASNFSV4(n)		((n)->nm_flag & NFSMNT_NFSV4)
+#define	NFSHASNFSV4N(n)		((n)->nm_minorvers > 0)
 #define	NFSHASNFSV3OR4(n)	((n)->nm_flag & (NFSMNT_NFSV3 | NFSMNT_NFSV4))
 #define	NFSHASGOTFSINFO(n)	((n)->nm_state & NFSSTA_GOTFSINFO)
 #define	NFSHASHASSETFSID(n)	((n)->nm_state & NFSSTA_HASSETFSID)
@@ -781,6 +862,10 @@ void newnfs_realign(struct mbuf **);
 #define	NFSHASPRIVACY(n)	((n)->nm_flag & NFSMNT_PRIVACY)
 #define	NFSSETWRITEVERF(n)	((n)->nm_state |= NFSSTA_HASWRITEVERF)
 #define	NFSSETHASSETFSID(n)	((n)->nm_state |= NFSSTA_HASSETFSID)
+#define	NFSHASPNFSOPT(n)	((n)->nm_flag & NFSMNT_PNFS)
+#define	NFSHASNOLAYOUTCOMMIT(n)	((n)->nm_state & NFSSTA_NOLAYOUTCOMMIT)
+#define	NFSHASSESSPERSIST(n)	((n)->nm_state & NFSSTA_SESSPERSIST)
+#define	NFSHASPNFS(n)		((n)->nm_state & NFSSTA_PNFS)
 
 /*
  * Gets the stats field out of the mount structure.
diff --git a/sys/fs/nfs/nfsproto.h b/sys/fs/nfs/nfsproto.h
index 5ae2e3d..6836661 100644
--- a/sys/fs/nfs/nfsproto.h
+++ b/sys/fs/nfs/nfsproto.h
@@ -62,7 +62,9 @@
 #define	NFS_MINPACKET	20
 #define	NFS_FABLKSIZE	512	/* Size in bytes of a block wrt fa_blocks */
 #define	NFSV4_MINORVERSION	0	/* V4 Minor version */
+#define	NFSV41_MINORVERSION	1	/* V4 Minor version */
 #define	NFSV4_CBVERS		1	/* V4 CB Version */
+#define	NFSV41_CBVERS		4	/* V4.1 CB Version */
 #define	NFSV4_SMALLSTR	50		/* Strings small enough for stack */
 
 /* Stat numbers for rpc returns (version 2, 3 and 4) */
@@ -145,6 +147,46 @@
 #define	NFSERR_ADMINREVOKED	10047
 #define	NFSERR_CBPATHDOWN	10048
 
+/* NFSv4.1 specific errors. */
+#define	NFSERR_BADIOMODE	10049
+#define	NFSERR_BADLAYOUT	10050
+#define	NFSERR_BADSESSIONDIGEST	10051
+#define	NFSERR_BADSESSION	10052
+#define	NFSERR_BADSLOT		10053
+#define	NFSERR_COMPLETEALREADY	10054
+#define	NFSERR_NOTBNDTOSESS	10055
+#define	NFSERR_DELEGALREADYWANT	10056
+#define	NFSERR_BACKCHANBUSY	10057
+#define	NFSERR_LAYOUTTRYLATER	10058
+#define	NFSERR_LAYOUTUNAVAIL	10059
+#define	NFSERR_NOMATCHLAYOUT	10060
+#define	NFSERR_RECALLCONFLICT	10061
+#define	NFSERR_UNKNLAYOUTTYPE	10062
+#define	NFSERR_SEQMISORDERED	10063
+#define	NFSERR_SEQUENCEPOS	10064
+#define	NFSERR_REQTOOBIG	10065
+#define	NFSERR_REPTOOBIG	10066
+#define	NFSERR_REPTOOBIGTOCACHE	10067
+#define	NFSERR_RETRYUNCACHEDREP	10068
+#define	NFSERR_UNSAFECOMPOUND	10069
+#define	NFSERR_TOOMANYOPS	10070
+#define	NFSERR_OPNOTINSESS	10071
+#define	NFSERR_HASHALGUNSUPP	10072
+#define	NFSERR_CLIENTIDBUSY	10074
+#define	NFSERR_PNFSIOHOLE	10075
+#define	NFSERR_SEQFALSERETRY	10076
+#define	NFSERR_BADHIGHSLOT	10077
+#define	NFSERR_DEADSESSION	10078
+#define	NFSERR_ENCRALGUNSUPP	10079
+#define	NFSERR_PNFSNOLAYOUT	10080
+#define	NFSERR_NOTONLYOP	10081
+#define	NFSERR_WRONGCRED	10082
+#define	NFSERR_WRONGTYPE	10083
+#define	NFSERR_DIRDELEGUNAVAIL	10084
+#define	NFSERR_REJECTDELEG	10085
+#define	NFSERR_RETURNCONFLICT	10086
+#define	NFSERR_DELEGREVOKED	10087
+
 #define	NFSERR_STALEWRITEVERF	30001	/* Fake return for nfs_commit() */
 #define	NFSERR_DONTREPLY	30003	/* Don't process request */
 #define	NFSERR_RETVOID		30004	/* Return void, not error */
@@ -189,6 +231,8 @@
 #define	NFSX_V4SPECDATA		(2 * NFSX_UNSIGNED)
 #define	NFSX_V4TIME		(NFSX_HYPER + NFSX_UNSIGNED)
 #define	NFSX_V4SETTIME		(NFSX_UNSIGNED + NFSX_V4TIME)
+#define	NFSX_V4SESSIONID	16
+#define	NFSX_V4DEVICEID		16
 
 /* sizes common to multiple NFS versions */
 #define	NFSX_FHMAX		(NFSX_V4FHMAX)
@@ -258,6 +302,27 @@
  * Must be defined as one higher than the last Proc# above.
  */
 #define	NFSV4_NPROCS		41
+
+/* Additional procedures for NFSv4.1. */
+#define	NFSPROC_EXCHANGEID	41
+#define	NFSPROC_CREATESESSION	42
+#define	NFSPROC_DESTROYSESSION	43
+#define	NFSPROC_DESTROYCLIENT	44
+#define	NFSPROC_FREESTATEID	45
+#define	NFSPROC_LAYOUTGET	46
+#define	NFSPROC_GETDEVICEINFO	47
+#define	NFSPROC_LAYOUTCOMMIT	48
+#define	NFSPROC_LAYOUTRETURN	49
+#define	NFSPROC_RECLAIMCOMPL	50
+#define	NFSPROC_WRITEDS		51
+#define	NFSPROC_READDS		52
+#define	NFSPROC_COMMITDS	53
+
+/*
+ * Must be defined as one higher than the last NFSv4.1 Proc# above.
+ */
+#define	NFSV41_NPROCS		54
+
 #endif	/* NFS_V3NPROCS */
 
 /*
@@ -269,10 +334,10 @@
 
 /*
  * NFSPROC_NOOP is a fake op# that can't be the same as any V2/3/4 Procedure
- * or Operation#. Since the NFS V4 Op #s go higher, use NFSV4OP_NOPS, which
+ * or Operation#. Since the NFS V4 Op #s go higher, use NFSV41_NOPS, which
  * is one greater than the highest Op#.
  */
-#define	NFSPROC_NOOP		NFSV4OP_NOPS
+#define	NFSPROC_NOOP		NFSV41_NOPS
 
 /* Actual Version 2 procedure numbers */
 #define	NFSV2PROC_NULL		0
@@ -406,6 +471,7 @@
 #define	NFSSTATEID_PUTALLZERO		0
 #define	NFSSTATEID_PUTALLONE		1
 #define	NFSSTATEID_PUTSTATEID		2
+#define	NFSSTATEID_PUTSEQIDZERO		3
 
 /*
  * Bits for share access and deny.
@@ -462,12 +528,70 @@
 #define	NFSCREATE_UNCHECKED		0
 #define	NFSCREATE_GUARDED		1
 #define	NFSCREATE_EXCLUSIVE		2
+#define	NFSCREATE_EXCLUSIVE41		3
 
 #define	NFSV3FSINFO_LINK		0x01
 #define	NFSV3FSINFO_SYMLINK		0x02
 #define	NFSV3FSINFO_HOMOGENEOUS		0x08
 #define	NFSV3FSINFO_CANSETTIME		0x10
 
+/* Flags for Exchange ID */
+#define	NFSV4EXCH_SUPPMOVEDREFER	0x00000001
+#define	NFSV4EXCH_SUPPMOVEDMIGR	0x00000002
+#define	NFSV4EXCH_BINDPRINCSTATEID	0x00000100
+#define	NFSV4EXCH_USENONPNFS		0x00010000
+#define	NFSV4EXCH_USEPNFSMDS		0x00020000
+#define	NFSV4EXCH_USEPNFSDS		0x00040000
+#define	NFSV4EXCH_MASKPNFS		0x00070000
+#define	NFSV4EXCH_UPDCONFIRMEDRECA	0x40000000
+#define	NFSV4EXCH_CONFIRMEDR		0x80000000
+
+/* State Protects */
+#define	NFSV4EXCH_SP4NONE		0
+#define	NFSV4EXCH_SP4MACHCRED		1
+#define	NFSV4EXCH_SP4SSV		2
+
+/* Flags for Create Session */
+#define	NFSV4CRSESS_PERSIST		0x00000001
+#define	NFSV4CRSESS_CONNBACKCHAN	0x00000002
+#define	NFSV4CRSESS_CONNRDMA		0x00000004
+
+/* Flags for Sequence */
+#define	NFSV4SEQ_CBPATHDOWN		0x00000001
+#define	NFSV4SEQ_CBGSSCONTEXPIRING	0x00000002
+#define	NFSV4SEQ_CBGSSCONTEXPIRED	0x00000004
+#define	NFSV4SEQ_EXPIREDALLSTATEREVOKED	0x00000008
+#define	NFSV4SEQ_EXPIREDSOMESTATEREVOKED 0x00000010
+#define	NFSV4SEQ_ADMINSTATEREVOKED	0x00000020
+#define	NFSV4SEQ_RECALLABLESTATEREVOKED	0x00000040
+#define	NFSV4SEQ_LEASEMOVED		0x00000080
+#define	NFSV4SEQ_RESTARTRECLAIMNEEDED	0x00000100
+#define	NFSV4SEQ_CBPATHDOWNSESSION	0x00000200
+#define	NFSV4SEQ_BACKCHANNELFAULT	0x00000400
+#define	NFSV4SEQ_DEVIDCHANGED		0x00000800
+#define	NFSV4SEQ_DEVIDDELETED		0x00001000
+
+/* Flags for Layout. */
+#define	NFSLAYOUTRETURN_FILE		1
+#define	NFSLAYOUTRETURN_FSID		2
+#define	NFSLAYOUTRETURN_ALL		3
+
+#define	NFSLAYOUT_NFSV4_1_FILES		0x1
+#define	NFSLAYOUT_OSD2_OBJECTS		0x2
+#define	NFSLAYOUT_BLOCK_VOLUME		0x3
+
+#define	NFSLAYOUTIOMODE_READ		1
+#define	NFSLAYOUTIOMODE_RW		2
+#define	NFSLAYOUTIOMODE_ANY		3
+
+/* Flags for Get Device Info. */
+#define	NFSDEVICEIDNOTIFY_CHANGEBIT	0x1
+#define	NFSDEVICEIDNOTIFY_DELETEBIT	0x2
+
+/* Flags for File Layout. */
+#define	NFSFLAYUTIL_DENSE		0x1
+#define	NFSFLAYUTIL_COMMIT_THRU_MDS	0x2
+
 /* Conversion macros */
 #define	vtonfsv2_mode(t,m) 						\
 		txdr_unsigned(((t) == VFIFO) ? MAKEIMODE(VCHR, (m)) : 	\