Implement support for RPCSEC_GSS authentication to both the NFS client

and server. This replaces the RPC implementation of the NFS client and
server with the newer RPC implementation originally developed
(actually ported from the userland sunrpc code) to support the NFS
Lock Manager.  I have tested this code extensively and I believe it is
stable and that performance is at least equal to the legacy RPC
implementation.

The NFS code currently contains support for both the new RPC
implementation and the older legacy implementation inherited from the
original NFS codebase. The default is to use the new implementation -
add the NFS_LEGACYRPC option to fall back to the old code. When I
merge this support back to RELENG_7, I will probably change this so
that users have to 'opt in' to get the new code.

To use RPCSEC_GSS on either client or server, you must build a kernel
which includes the KGSSAPI option and the crypto device. On the
userland side, you must build at least a new libc, mountd, mount_nfs
and gssd. You must install new versions of /etc/rc.d/gssd and
/etc/rc.d/nfsd and add 'gssd_enable=YES' to /etc/rc.conf.

As long as gssd is running, you should be able to mount an NFS
filesystem from a server that requires RPCSEC_GSS authentication. The
mount itself can happen without any kerberos credentials but all
access to the filesystem will be denied unless the accessing user has
a valid ticket file in the standard place (/tmp/krb5cc_<uid>). There
is currently no support for situations where the ticket file is in a
different place, such as when the user logged in via SSH and has
delegated credentials from that login. This restriction is also
present in Solaris and Linux. In theory, we could improve this in
future, possibly using Brooks Davis' implementation of variant
symlinks.

Supporting RPCSEC_GSS on a server is nearly as simple. You must create
service creds for the server in the form 'nfs/<fqdn>@<REALM>' and
install them in /etc/krb5.keytab. The standard heimdal utility ktutil
makes this fairly easy. After the service creds have been created, you
can add a '-sec=krb5' option to /etc/exports and restart both mountd
and nfsd.

The only other difference an administrator should notice is that nfsd
doesn't fork to create service threads any more. In normal operation,
there will be two nfsd processes, one in userland waiting for TCP
connections and one in the kernel handling requests. The latter
process will create as many kthreads as required - these should be
visible via 'top -H'. The code has some support for varying the number
of service threads according to load but initially at least, nfsd uses
a fixed number of threads according to the value supplied to its '-n'
option.

Sponsored by:	Isilon Systems
MFC after:	1 month

11 files changed, 1373 insertions, 65 deletions

diff --git a/sys/nfsserver/nfs.h b/sys/nfsserver/nfs.h
index beb9359..2709377 100644
--- a/sys/nfsserver/nfs.h
+++ b/sys/nfsserver/nfs.h
@@ -89,13 +89,26 @@
  * Structures for the nfssvc(2) syscall. Not that anyone but nfsd and mount_nfs
  * should ever try and use it.
  */
-struct nfsd_args {
+
+/*
+ * Add a socket to monitor for NFS requests.
+ */
+struct nfsd_addsock_args {
 	int	sock;		/* Socket to serve */
 	caddr_t	name;		/* Client addr for connection based sockets */
 	int	namelen;	/* Length of name */
 };
 
 /*
+ * Start processing requests.
+ */
+struct nfsd_nfsd_args {
+	const char *principal;	/* GSS-API service principal name */
+	int	minthreads;	/* minimum service thread count */
+	int	maxthreads;	/* maximum service thread count */
+};
+
+/*
  * XXX to allow amd to include nfs.h without nfsproto.h
  */
 #ifdef NFS_NPROCS
@@ -105,8 +118,9 @@ struct nfsd_args {
 /*
  * Flags for nfssvc() system call.
  */
-#define	NFSSVC_NFSD	0x004
+#define	NFSSVC_OLDNFSD	0x004
 #define	NFSSVC_ADDSOCK	0x008
+#define	NFSSVC_NFSD	0x010
 
 /*
  * vfs.nfsrv sysctl(3) identifiers
@@ -167,6 +181,7 @@ extern int32_t (*nfsrv3_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
 #define	NWDELAYHASH(sock, f) \
 	(&(sock)->ns_wdelayhashtbl[(*((u_int32_t *)(f))) % NFS_WDELAYHASHSIZ])
 
+#ifdef NFS_LEGACYRPC
 /*
  * Network address hash list element
  */
@@ -257,11 +272,37 @@ struct nfsrv_descript {
 	struct timeval		nd_starttime;	/* Time RPC initiated */
 	fhandle_t		nd_fh;		/* File handle */
 	struct ucred		*nd_cr;		/* Credentials */
+	int			nd_credflavor;	/* Security flavor */
 };
 
+#else
+
+/*
+ * This structure is used by the server for describing each request.
+ */
+struct nfsrv_descript {
+	struct mbuf		*nd_mrep;	/* Request mbuf list */
+	struct mbuf		*nd_md;		/* Current dissect mbuf */
+	struct mbuf		*nd_mreq;	/* Reply mbuf list */
+	struct sockaddr		*nd_nam;	/* and socket addr */
+	struct sockaddr		*nd_nam2;	/* return socket addr */
+	caddr_t			nd_dpos;	/* Current dissect pos */
+	u_int32_t		nd_procnum;	/* RPC # */
+	int			nd_stable;	/* storage type */
+	int			nd_flag;	/* nd_flag */
+	int			nd_repstat;	/* Reply status */
+	fhandle_t		nd_fh;		/* File handle */
+	struct ucred		*nd_cr;		/* Credentials */
+	int			nd_credflavor;	/* Security flavor */
+};
+
+#endif
+
 /* Bits for "nd_flag" */
 #define ND_NFSV3	0x08
 
+#ifdef NFS_LEGACYRPC
+
 extern TAILQ_HEAD(nfsd_head, nfsd) nfsd_head;
 extern int nfsd_head_flag;
 #define	NFSD_CHECKSLP	0x01
@@ -273,6 +314,8 @@ extern int nfsd_head_flag;
 		((o)->nd_eoff >= (n)->nd_off && \
 		 !bcmp((caddr_t)&(o)->nd_fh, (caddr_t)&(n)->nd_fh, NFSX_V3FH))
 
+#endif
+
 /*
  * Defines for WebNFS
  */
@@ -315,38 +358,42 @@ extern int nfs_debug;
 
 #endif
 
+#ifdef NFS_LEGACYRPC
+int	netaddr_match(int, union nethostaddr *, struct sockaddr *);
 int	nfs_getreq(struct nfsrv_descript *, struct nfsd *, int);
 int	nfsrv_send(struct socket *, struct sockaddr *, struct mbuf *);
-struct mbuf *nfs_rephead(int, struct nfsrv_descript *, int, struct mbuf **,
-	    caddr_t *);
+int	nfsrv_dorec(struct nfssvc_sock *, struct nfsd *,
+	    struct nfsrv_descript **);
 int	nfs_slplock(struct nfssvc_sock *, int);
 void	nfs_slpunlock(struct nfssvc_sock *);
+void	nfsrv_initcache(void);
+void	nfsrv_destroycache(void);
+void	nfsrv_timer(void *);
+int	nfsrv_getcache(struct nfsrv_descript *, struct mbuf **);
+void	nfsrv_updatecache(struct nfsrv_descript *, int, struct mbuf *);
+void	nfsrv_cleancache(void);
+void	nfsrv_rcv(struct socket *so, void *arg, int waitflag);
+void	nfsrv_slpderef(struct nfssvc_sock *slp);
+void	nfsrv_wakenfsd(struct nfssvc_sock *slp);
+int	nfsrv_writegather(struct nfsrv_descript **, struct nfssvc_sock *,
+	    struct mbuf **);
+#endif
+struct mbuf *nfs_rephead(int, struct nfsrv_descript *, int, struct mbuf **,
+	    caddr_t *);
 void	nfsm_srvfattr(struct nfsrv_descript *, struct vattr *,
 	    struct nfs_fattr *);
 void	nfsm_srvwcc(struct nfsrv_descript *, int, struct vattr *, int,
 	    struct vattr *, struct mbuf **, char **);
 void	nfsm_srvpostopattr(struct nfsrv_descript *, int, struct vattr *,
 	    struct mbuf **, char **);
-int	netaddr_match(int, union nethostaddr *, struct sockaddr *);
-int	nfs_namei(struct nameidata *, fhandle_t *, int,
-	    struct nfssvc_sock *, struct sockaddr *, struct mbuf **,
+int	nfs_namei(struct nameidata *, struct nfsrv_descript *, fhandle_t *,
+	    int, struct nfssvc_sock *, struct sockaddr *, struct mbuf **,
 	    caddr_t *, struct vnode **, int, struct vattr *, int *, int);
 void	nfsm_adj(struct mbuf *, int, int);
 int	nfsm_mbuftouio(struct mbuf **, struct uio *, int, caddr_t *);
-void	nfsrv_initcache(void);
-void	nfsrv_destroycache(void);
-void	nfsrv_timer(void *);
-int	nfsrv_dorec(struct nfssvc_sock *, struct nfsd *,
-	    struct nfsrv_descript **);
-int	nfsrv_getcache(struct nfsrv_descript *, struct mbuf **);
-void	nfsrv_updatecache(struct nfsrv_descript *, int, struct mbuf *);
-void	nfsrv_cleancache(void);
 void	nfsrv_init(int);
 int	nfsrv_errmap(struct nfsrv_descript *, int);
 void	nfsrvw_sort(gid_t *, int);
-void	nfsrv_wakenfsd(struct nfssvc_sock *slp);
-int	nfsrv_writegather(struct nfsrv_descript **, struct nfssvc_sock *,
-	    struct mbuf **);
 
 int	nfsrv3_access(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	    struct mbuf **mrq);
@@ -354,8 +401,9 @@ int	nfsrv_commit(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	    struct mbuf **mrq);
 int	nfsrv_create(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	    struct mbuf **mrq);
-int	nfsrv_fhtovp(fhandle_t *, int, struct vnode **, int *, struct ucred *,
-	    struct nfssvc_sock *, struct sockaddr *, int *, int);
+int	nfsrv_fhtovp(fhandle_t *, int, struct vnode **, int *,
+	    struct nfsrv_descript *, struct nfssvc_sock *, struct sockaddr *,
+	    int *, int);
 int	nfsrv_setpublicfs(struct mount *, struct netexport *,
 	    struct export_args *);
 int	nfs_ispublicfh(fhandle_t *);
@@ -399,8 +447,6 @@ int	nfsrv_symlink(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	    struct mbuf **mrq);
 int	nfsrv_write(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	    struct mbuf **mrq);
-void	nfsrv_rcv(struct socket *so, void *arg, int waitflag);
-void	nfsrv_slpderef(struct nfssvc_sock *slp);
 #endif	/* _KERNEL */
 
 #endif
diff --git a/sys/nfsserver/nfs_fha.c b/sys/nfsserver/nfs_fha.c
new file mode 100644
index 0000000..a2e4ca3
--- /dev/null
+++ b/sys/nfsserver/nfs_fha.c
@@ -0,0 +1,597 @@
+/*-
+ * Copyright (c) 2008 Isilon Inc http://www.isilon.com/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/sysproto.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/vnode.h>
+#include <sys/malloc.h>
+#include <sys/mount.h>
+#include <sys/mbuf.h>
+#include <sys/sbuf.h>
+
+#include <rpc/rpc.h>
+#include <nfs/xdr_subs.h>
+#include <nfs/rpcv2.h>
+#include <nfs/nfsproto.h>
+#include <nfsserver/nfs.h>
+#include <nfsserver/nfsm_subs.h>
+#include <nfsserver/nfs_fha.h>
+
+#ifndef NFS_LEGACYRPC
+
+static MALLOC_DEFINE(M_NFS_FHA, "NFS FHA", "NFS FHA");
+
+/* Sysctl defaults. */
+#define DEF_BIN_SHIFT		18 /* 256k */
+#define DEF_MAX_NFSDS_PER_FH	8
+#define DEF_MAX_REQS_PER_NFSD	4
+
+struct fha_ctls {
+	u_int32_t bin_shift;
+	u_int32_t max_nfsds_per_fh;
+	u_int32_t max_reqs_per_nfsd;
+} fha_ctls;
+
+struct sysctl_ctx_list fha_clist;
+
+SYSCTL_DECL(_vfs_nfsrv);
+SYSCTL_DECL(_vfs_nfsrv_fha);
+
+/* Static sysctl node for the fha from the top-level vfs_nfsrv node. */
+SYSCTL_NODE(_vfs_nfsrv, OID_AUTO, fha, CTLFLAG_RD, 0, "fha node");
+
+/* This is the global structure that represents the state of the fha system. */
+static struct fha_global {
+	struct fha_hash_entry_list *hashtable;
+	u_long hashmask;
+} g_fha;
+
+/* 
+ * These are the entries in the filehandle hash. They talk about a specific 
+ * file, requests against which are being handled by one or more nfsds. We keep
+ * a chain of nfsds against the file. We only have more than one if reads are 
+ * ongoing, and then only if the reads affect disparate regions of the file.
+ *
+ * In general, we want to assign a new request to an existing nfsd if it is 
+ * going to contend with work happening already on that nfsd, or if the 
+ * operation is a read and the nfsd is already handling a proximate read. We 
+ * do this to avoid jumping around in the read stream unnecessarily, and to 
+ * avoid contention between threads over single files.
+ */
+struct fha_hash_entry {
+	LIST_ENTRY(fha_hash_entry) link;
+	u_int64_t fh;
+	u_int16_t num_reads;
+	u_int16_t num_writes;
+	u_int8_t num_threads;
+	struct svcthread_list threads;
+};
+LIST_HEAD(fha_hash_entry_list, fha_hash_entry);
+
+/* A structure used for passing around data internally. */
+struct fha_info {
+	u_int64_t fh;
+	off_t offset;
+	int locktype;
+};
+
+static int fhe_stats_sysctl(SYSCTL_HANDLER_ARGS);
+ 
+static void
+nfs_fha_init(void *foo)
+{
+
+	/*
+	 * A small hash table to map filehandles to fha_hash_entry
+	 * structures.
+	 */
+	g_fha.hashtable = hashinit(256, M_NFS_FHA, &g_fha.hashmask);
+
+	/*
+	 * Initialize the sysctl context list for the fha module.
+	 */
+	sysctl_ctx_init(&fha_clist);
+
+	fha_ctls.bin_shift = DEF_BIN_SHIFT;
+	fha_ctls.max_nfsds_per_fh = DEF_MAX_NFSDS_PER_FH;
+	fha_ctls.max_reqs_per_nfsd = DEF_MAX_REQS_PER_NFSD;
+
+	SYSCTL_ADD_UINT(&fha_clist, SYSCTL_STATIC_CHILDREN(_vfs_nfsrv_fha),
+	    OID_AUTO, "bin_shift", CTLFLAG_RW,
+	    &fha_ctls.bin_shift, 0, "For FHA reads, no two requests will "
+	    "contend if they're 2^(bin_shift) bytes apart");
+
+	SYSCTL_ADD_UINT(&fha_clist, SYSCTL_STATIC_CHILDREN(_vfs_nfsrv_fha),
+	    OID_AUTO, "max_nfsds_per_fh", CTLFLAG_RW,
+	    &fha_ctls.max_nfsds_per_fh, 0, "Maximum nfsd threads that "
+	    "should be working on requests for the same file handle");
+
+	SYSCTL_ADD_UINT(&fha_clist, SYSCTL_STATIC_CHILDREN(_vfs_nfsrv_fha),
+	    OID_AUTO, "max_reqs_per_nfsd", CTLFLAG_RW,
+	    &fha_ctls.max_reqs_per_nfsd, 0, "Maximum requests that "
+	    "single nfsd thread should be working on at any time");
+
+	SYSCTL_ADD_OID(&fha_clist, SYSCTL_STATIC_CHILDREN(_vfs_nfsrv_fha), 
+	    OID_AUTO, "fhe_stats", CTLTYPE_STRING | CTLFLAG_RD, 0, 0,
+	    fhe_stats_sysctl, "A", "");
+}
+
+static void
+nfs_fha_uninit(void *foo)
+{
+
+	hashdestroy(g_fha.hashtable, M_NFS_FHA, g_fha.hashmask);
+}
+
+SYSINIT(nfs_fha, SI_SUB_ROOT_CONF, SI_ORDER_ANY, nfs_fha_init, NULL);
+SYSUNINIT(nfs_fha, SI_SUB_ROOT_CONF, SI_ORDER_ANY, nfs_fha_uninit, NULL);
+
+/* 
+ * This just specifies that offsets should obey affinity when within
+ * the same 1Mbyte (1<<20) chunk for the file (reads only for now).
+ */
+static void
+fha_extract_info(struct svc_req *req, struct fha_info *i)
+{
+	struct mbuf *md = req->rq_args;
+	fhandle_t fh;
+	caddr_t dpos = mtod(md, caddr_t);
+	static u_int64_t random_fh = 0;
+	int error;
+	int v3 = (req->rq_vers == 3);
+	u_int32_t *tl;
+	rpcproc_t procnum;
+
+	/* 
+	 * We start off with a random fh. If we get a reasonable
+	 * procnum, we set the fh. If there's a concept of offset 
+	 * that we're interested in, we set that.
+	 */
+	i->fh = ++random_fh;
+	i->offset = 0;
+	i->locktype = LK_EXCLUSIVE;
+	
+	/*
+	 * Extract the procnum and convert to v3 form if necessary.
+	 */
+	procnum = req->rq_proc;
+	if (!v3)
+		procnum = nfsrv_nfsv3_procid[procnum];
+
+	/* 
+	 * We do affinity for most. However, we divide a realm of affinity 
+	 * by file offset so as to allow for concurrent random access. We 
+	 * only do this for reads today, but this may change when IFS supports 
+	 * efficient concurrent writes.
+	 */
+	if (procnum == NFSPROC_FSSTAT ||
+	    procnum == NFSPROC_FSINFO ||
+	    procnum == NFSPROC_PATHCONF ||
+	    procnum == NFSPROC_NOOP || 
+	    procnum == NFSPROC_NULL)
+		goto out;
+	
+	/* Grab the filehandle. */
+	error = nfsm_srvmtofh_xx(&fh, v3, &md, &dpos);
+	if (error)
+		goto out;
+
+	i->fh = *(const u_int64_t *)(fh.fh_fid.fid_data);
+
+	/* Content ourselves with zero offset for all but reads. */
+	if (procnum != NFSPROC_READ)
+		goto out;
+
+	if (v3) {
+		tl = nfsm_dissect_xx_nonblock(2 * NFSX_UNSIGNED, &md, &dpos);
+		if (tl == NULL)
+			goto out;
+		i->offset = fxdr_hyper(tl);
+	} else {
+		tl = nfsm_dissect_xx_nonblock(NFSX_UNSIGNED, &md, &dpos);
+		if (tl == NULL)
+			goto out;
+		i->offset = fxdr_unsigned(u_int32_t, *tl);
+	}
+ out:
+	switch (procnum) {
+	case NFSPROC_NULL:
+	case NFSPROC_GETATTR:
+	case NFSPROC_LOOKUP:
+	case NFSPROC_ACCESS:
+	case NFSPROC_READLINK:
+	case NFSPROC_READ:
+	case NFSPROC_READDIR:
+	case NFSPROC_READDIRPLUS:
+		i->locktype = LK_SHARED;
+		break;
+	case NFSPROC_SETATTR:
+	case NFSPROC_WRITE:
+	case NFSPROC_CREATE:
+	case NFSPROC_MKDIR:
+	case NFSPROC_SYMLINK:
+	case NFSPROC_MKNOD:
+	case NFSPROC_REMOVE:
+	case NFSPROC_RMDIR:
+	case NFSPROC_RENAME:
+	case NFSPROC_LINK:
+	case NFSPROC_FSSTAT:
+	case NFSPROC_FSINFO:
+	case NFSPROC_PATHCONF:
+	case NFSPROC_COMMIT:
+	case NFSPROC_NOOP:
+		i->locktype = LK_EXCLUSIVE;
+		break;
+	}
+}
+
+static struct fha_hash_entry *
+fha_hash_entry_new(u_int64_t fh)
+{
+	struct fha_hash_entry *e;
+
+	e = malloc(sizeof(*e), M_NFS_FHA, M_WAITOK);
+	e->fh = fh;
+	e->num_reads = 0;
+	e->num_writes = 0;
+	e->num_threads = 0;
+	LIST_INIT(&e->threads);
+	
+	return e;
+}
+
+static void
+fha_hash_entry_destroy(struct fha_hash_entry *e)
+{
+
+	if (e->num_reads + e->num_writes)
+		panic("nonempty fhe");
+	free(e, M_NFS_FHA);
+}
+
+static void
+fha_hash_entry_remove(struct fha_hash_entry *e)
+{
+
+	LIST_REMOVE(e, link);
+	fha_hash_entry_destroy(e);
+}
+
+static struct fha_hash_entry *
+fha_hash_entry_lookup(SVCPOOL *pool, u_int64_t fh)
+{
+	struct fha_hash_entry *fhe, *new_fhe;
+
+	LIST_FOREACH(fhe, &g_fha.hashtable[fh % g_fha.hashmask], link) {
+		if (fhe->fh == fh)
+			break;
+	}
+
+	if (!fhe) {
+		/* Allocate a new entry. */
+		mtx_unlock(&pool->sp_lock);
+		new_fhe = fha_hash_entry_new(fh);
+		mtx_lock(&pool->sp_lock);
+
+		/* Double-check to make sure we still need the new entry. */
+		LIST_FOREACH(fhe, &g_fha.hashtable[fh % g_fha.hashmask], link) {
+			if (fhe->fh == fh)
+				break;
+		}
+		if (!fhe) {
+			fhe = new_fhe;
+			LIST_INSERT_HEAD(&g_fha.hashtable[fh % g_fha.hashmask],
+			    fhe, link);
+		} else {
+			fha_hash_entry_destroy(new_fhe);
+		}
+	}
+
+	return fhe;
+}
+
+static void
+fha_hash_entry_add_thread(struct fha_hash_entry *fhe, SVCTHREAD *thread)
+{
+	LIST_INSERT_HEAD(&fhe->threads, thread, st_alink);
+	fhe->num_threads++;
+}
+
+static void
+fha_hash_entry_remove_thread(struct fha_hash_entry *fhe, SVCTHREAD *thread)
+{
+
+	LIST_REMOVE(thread, st_alink);
+	fhe->num_threads--;
+}
+
+/* 
+ * Account for an ongoing operation associated with this file.
+ */
+static void
+fha_hash_entry_add_op(struct fha_hash_entry *fhe, int locktype, int count)
+{
+
+	if (LK_EXCLUSIVE == locktype)
+		fhe->num_writes += count;
+	else
+		fhe->num_reads += count;
+}
+
+static SVCTHREAD *
+get_idle_thread(SVCPOOL *pool)
+{
+	SVCTHREAD *st;
+
+	LIST_FOREACH(st, &pool->sp_idlethreads, st_ilink) {
+		if (st->st_xprt == NULL && STAILQ_EMPTY(&st->st_reqs))
+			return (st);
+	}
+	return (NULL);
+}
+
+
+/* 
+ * Get the service thread currently associated with the fhe that is
+ * appropriate to handle this operation.
+ */
+SVCTHREAD *
+fha_hash_entry_choose_thread(SVCPOOL *pool, struct fha_hash_entry *fhe,
+    struct fha_info *i, SVCTHREAD *this_thread);
+
+SVCTHREAD *
+fha_hash_entry_choose_thread(SVCPOOL *pool, struct fha_hash_entry *fhe,
+    struct fha_info *i, SVCTHREAD *this_thread)
+{
+	SVCTHREAD *thread, *min_thread = NULL;
+	int req_count, min_count = 0;
+	off_t offset1, offset2;
+
+	LIST_FOREACH(thread, &fhe->threads, st_alink) {
+		req_count = thread->st_reqcount;
+
+		/* If there are any writes in progress, use the first thread. */
+		if (fhe->num_writes) {
+#if 0
+			ITRACE_CURPROC(ITRACE_NFS, ITRACE_INFO, 
+			    "fha: %p(%d)w", thread, req_count);
+#endif
+			return (thread);
+		}
+
+		/* 
+		 * Check for read locality, making sure that we won't 
+		 * exceed our per-thread load limit in the process. 
+		 */
+		offset1 = i->offset >> fha_ctls.bin_shift;
+		offset2 = STAILQ_FIRST(&thread->st_reqs)->rq_p3
+			>> fha_ctls.bin_shift;
+		if (offset1 == offset2) {
+			if ((fha_ctls.max_reqs_per_nfsd == 0) ||
+			    (req_count < fha_ctls.max_reqs_per_nfsd)) {
+#if 0
+				ITRACE_CURPROC(ITRACE_NFS, ITRACE_INFO, 
+				    "fha: %p(%d)r", thread, req_count);
+#endif
+				return (thread);
+			}
+		}
+
+		/* 
+		 * We don't have a locality match, so skip this thread,
+		 * but keep track of the most attractive thread in case 
+		 * we need to come back to it later.
+		 */
+#if 0
+		ITRACE_CURPROC(ITRACE_NFS, ITRACE_INFO, 
+		    "fha: %p(%d)s off1 %llu off2 %llu", thread, 
+		    req_count, offset1, offset2);
+#endif
+		if ((min_thread == NULL) || (req_count < min_count)) {
+			min_count = req_count;
+			min_thread = thread;
+		}
+	}
+
+	/* 
+	 * We didn't find a good match yet. See if we can add 
+	 * a new thread to this file handle entry's thread list.
+	 */
+	if ((fha_ctls.max_nfsds_per_fh == 0) || 
+	    (fhe->num_threads < fha_ctls.max_nfsds_per_fh)) {
+		/* 
+		 * We can add a new thread, so try for an idle thread 
+		 * first, and fall back to this_thread if none are idle. 
+		 */
+		if (STAILQ_EMPTY(&this_thread->st_reqs)) {
+			thread = this_thread;
+#if 0
+			ITRACE_CURPROC(ITRACE_NFS, ITRACE_INFO, 
+			    "fha: %p(%d)t", thread, thread->st_reqcount);
+#endif
+		} else if ((thread = get_idle_thread(pool))) {
+#if 0
+			ITRACE_CURPROC(ITRACE_NFS, ITRACE_INFO, 
+			    "fha: %p(%d)i", thread, thread->st_reqcount);
+#endif
+		} else { 
+			thread = this_thread;
+#if 0
+			ITRACE_CURPROC(ITRACE_NFS, ITRACE_INFO, 
+			    "fha: %p(%d)b", thread, thread->st_reqcount);
+#endif
+		}
+		fha_hash_entry_add_thread(fhe, thread);
+	} else {
+		/* 
+		 * We don't want to use any more threads for this file, so 
+		 * go back to the most attractive nfsd we're already using.
+		 */
+		thread = min_thread;
+	}
+
+	return (thread);
+}
+
+/* 
+ * After getting a request, try to assign it to some thread. Usually we
+ * handle it ourselves.
+ */
+SVCTHREAD *
+fha_assign(SVCTHREAD *this_thread, struct svc_req *req)
+{
+	SVCPOOL *pool;
+	SVCTHREAD *thread;
+	struct fha_info i;
+	struct fha_hash_entry *fhe;
+
+	/*
+	 * Only do placement if this is an NFS request.
+	 */
+	if (req->rq_prog != NFS_PROG)
+		return (this_thread);
+
+	if (req->rq_vers != 2 && req->rq_vers != 3)
+		return (this_thread);
+
+	pool = req->rq_xprt->xp_pool;
+	fha_extract_info(req, &i);
+
+	/* 
+	 * We save the offset associated with this request for later 
+	 * nfsd matching.
+	 */
+	fhe = fha_hash_entry_lookup(pool, i.fh);
+	req->rq_p1 = fhe;
+	req->rq_p2 = i.locktype;
+	req->rq_p3 = i.offset;
+	
+	/* 
+	 * Choose a thread, taking into consideration locality, thread load,
+	 * and the number of threads already working on this file.
+	 */
+	thread = fha_hash_entry_choose_thread(pool, fhe, &i, this_thread);
+	KASSERT(thread, ("fha_assign: NULL thread!"));
+	fha_hash_entry_add_op(fhe, i.locktype, 1);
+
+	return (thread);
+}
+
+/* 
+ * Called when we're done with an operation. The request has already
+ * been de-queued.
+ */
+void
+fha_nd_complete(SVCTHREAD *thread, struct svc_req *req)
+{
+	struct fha_hash_entry *fhe = req->rq_p1;
+
+	/*
+	 * This may be called for reqs that didn't go through
+	 * fha_assign (e.g. extra NULL ops used for RPCSEC_GSS.
+	 */
+	if (!fhe)
+		return;
+
+	fha_hash_entry_add_op(fhe, req->rq_p2, -1);
+
+	if (thread->st_reqcount == 0) {
+		fha_hash_entry_remove_thread(fhe, thread);
+		if (0 == fhe->num_reads + fhe->num_writes)
+			fha_hash_entry_remove(fhe);
+	}
+}
+
+extern SVCPOOL *nfsrv_pool;
+
+static int
+fhe_stats_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	int error, count, i;
+	struct sbuf sb;
+	struct fha_hash_entry *fhe;
+	bool_t first = TRUE;
+	SVCTHREAD *thread;
+
+	sbuf_new(&sb, NULL, 4096, SBUF_FIXEDLEN);
+
+	if (!nfsrv_pool) {
+		sbuf_printf(&sb, "NFSD not running\n");
+		goto out;
+	}
+
+	mtx_lock(&nfsrv_pool->sp_lock);
+	count = 0;
+	for (i = 0; i <= g_fha.hashmask; i++)
+		if (!LIST_EMPTY(&g_fha.hashtable[i]))
+			count++;
+
+	if (count == 0) {
+		sbuf_printf(&sb, "No file handle entries.\n");
+		goto out;
+	}
+
+	for (i = 0; i <= g_fha.hashmask; i++) {
+		LIST_FOREACH(fhe, &g_fha.hashtable[i], link) {
+			sbuf_printf(&sb, "%sfhe %p: {\n", first ? "" : ", ", fhe);
+
+			sbuf_printf(&sb, "    fh: %ju\n", (uintmax_t) fhe->fh);
+			sbuf_printf(&sb, "    num_reads: %d\n", fhe->num_reads);
+			sbuf_printf(&sb, "    num_writes: %d\n", fhe->num_writes);
+			sbuf_printf(&sb, "    num_threads: %d\n", fhe->num_threads);
+
+			LIST_FOREACH(thread, &fhe->threads, st_alink) {
+				sbuf_printf(&sb, "    thread %p (count %d)\n",
+				    thread, thread->st_reqcount);
+			}
+
+			sbuf_printf(&sb, "}");
+			first = FALSE;
+
+			/* Limit the output. */
+			if (++count > 128) {
+				sbuf_printf(&sb, "...");
+				break;
+			}
+		}
+	}
+
+ out:
+	if (nfsrv_pool)
+		mtx_unlock(&nfsrv_pool->sp_lock);
+	sbuf_trim(&sb);
+	sbuf_finish(&sb);
+	error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
+	sbuf_delete(&sb);
+	return (error);
+}
+
+#endif /* !NFS_LEGACYRPC */
diff --git a/sys/nfsserver/nfs_fha.h b/sys/nfsserver/nfs_fha.h
new file mode 100644
index 0000000..e7344ed
--- /dev/null
+++ b/sys/nfsserver/nfs_fha.h
@@ -0,0 +1,28 @@
+/*-
+ * Copyright (c) 2008 Isilon Inc http://www.isilon.com/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/* $FreeBSD$ */
+
+void fha_nd_complete(SVCTHREAD *, struct svc_req *);
+SVCTHREAD *fha_assign(SVCTHREAD *, struct svc_req *);
diff --git a/sys/nfsserver/nfs_serv.c b/sys/nfsserver/nfs_serv.c
index a0a0308..d528769 100644
--- a/sys/nfsserver/nfs_serv.c
+++ b/sys/nfsserver/nfs_serv.c
@@ -142,8 +142,10 @@ SYSCTL_STRUCT(_vfs_nfsrv, NFS_NFSRVSTATS, nfsrvstats, CTLFLAG_RW,
 
 static int	nfsrv_access(struct vnode *, accmode_t, struct ucred *,
 		    int, int);
+#ifdef NFS_LEGACYRPC
 static void	nfsrvw_coalesce(struct nfsrv_descript *,
 		    struct nfsrv_descript *);
+#endif
 
 /*
  * Clear nameidata fields that are tested in nsfmout cleanup code prior
@@ -216,7 +218,7 @@ nfsrv3_access(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	fhp = &nfh.fh_generic;
 	nfsm_srvmtofh(fhp);
 	tl = nfsm_dissect_nonblock(u_int32_t *, NFSX_UNSIGNED);
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	if (error) {
 		nfsm_reply(NFSX_UNSIGNED);
@@ -283,7 +285,7 @@ nfsrv_getattr(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	vfslocked = 0;
 	fhp = &nfh.fh_generic;
 	nfsm_srvmtofh(fhp);
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp, nam,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp, nam,
 	    &rdonly, TRUE);
 	if (error) {
 		nfsm_reply(0);
@@ -392,7 +394,7 @@ nfsrv_setattr(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	/*
 	 * Now that we have all the fields, lets do it.
 	 */
-	error = nfsrv_fhtovp(fhp, 1, &vp, &tvfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &tvfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	vfslocked = nfsrv_lockedpair(vfslocked, tvfslocked);
 	if (error) {
@@ -505,7 +507,7 @@ nfsrv_lookup(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	nd.ni_cnd.cn_cred = cred;
 	nd.ni_cnd.cn_nameiop = LOOKUP;
 	nd.ni_cnd.cn_flags = LOCKLEAF | SAVESTART | MPSAFE;
-	error = nfs_namei(&nd, fhp, len, slp, nam, &md, &dpos,
+	error = nfs_namei(&nd, nfsd, fhp, len, slp, nam, &md, &dpos,
 		&dirp, v3, &dirattr, &dirattr_ret, pubflag);
 	vfslocked = NDHASGIANT(&nd);
 
@@ -715,7 +717,7 @@ nfsrv_readlink(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	uiop->uio_rw = UIO_READ;
 	uiop->uio_segflg = UIO_SYSSPACE;
 	uiop->uio_td = NULL;
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	if (error) {
 		nfsm_reply(2 * NFSX_UNSIGNED);
@@ -811,7 +813,7 @@ nfsrv_read(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	 * as well.
 	 */
 
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	if (error) {
 		vp = NULL;
@@ -1112,7 +1114,7 @@ nfsrv_write(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 		error = 0;
 		goto nfsmout;
 	}
-	error = nfsrv_fhtovp(fhp, 1, &vp, &tvfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &tvfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	vfslocked = nfsrv_lockedpair(vfslocked, tvfslocked);
 	if (error) {
@@ -1227,6 +1229,16 @@ nfsmout:
 	return(error);
 }
 
+#ifdef NFS_LEGACYRPC
+
+/*
+ * XXX dfr - write gathering isn't supported by the new RPC code since
+ * its really only useful for NFSv2. If there is a real need, we could
+ * attempt to fit it into the filehandle affinity system, e.g. by
+ * looking to see if there are queued write requests that overlap this
+ * one.
+ */
+
 /*
  * For the purposes of write gathering, we must decide if the credential
  * associated with two pending requests have equivilent privileges.  Since
@@ -1432,7 +1444,7 @@ loop1:
 		cred = nfsd->nd_cr;
 		v3 = (nfsd->nd_flag & ND_NFSV3);
 		forat_ret = aftat_ret = 1;
-		error = nfsrv_fhtovp(&nfsd->nd_fh, 1, &vp, &vfslocked, cred,
+		error = nfsrv_fhtovp(&nfsd->nd_fh, 1, &vp, &vfslocked, nfsd,
 		    slp, nfsd->nd_nam, &rdonly, TRUE);
 		if (!error) {
 		    if (v3)
@@ -1634,6 +1646,8 @@ nfsrvw_coalesce(struct nfsrv_descript *owp, struct nfsrv_descript *nfsd)
 	}
 }
 
+#endif
+
 /*
  * nfs create service
  * now does a truncate to 0 length via. setattr if it already exists
@@ -1697,7 +1711,7 @@ nfsrv_create(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	 * be valid at all if an error occurs so we have to invalidate it
 	 * prior to calling nfsm_reply ( which might goto nfsmout ).
 	 */
-	error = nfs_namei(&nd, fhp, len, slp, nam, &md, &dpos,
+	error = nfs_namei(&nd, nfsd, fhp, len, slp, nam, &md, &dpos,
 		&dirp, v3, &dirfor, &dirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &nd);
 	if (dirp && !v3) {
@@ -1987,7 +2001,7 @@ nfsrv_mknod(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	 * nfsmout.
 	 */
 
-	error = nfs_namei(&nd, fhp, len, slp, nam, &md, &dpos,
+	error = nfs_namei(&nd, nfsd, fhp, len, slp, nam, &md, &dpos,
 		&dirp, v3, &dirfor, &dirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &nd);
 	if (error) {
@@ -2169,7 +2183,7 @@ nfsrv_remove(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	nd.ni_cnd.cn_cred = cred;
 	nd.ni_cnd.cn_nameiop = DELETE;
 	nd.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF | MPSAFE;
-	error = nfs_namei(&nd, fhp, len, slp, nam, &md, &dpos,
+	error = nfs_namei(&nd, nfsd, fhp, len, slp, nam, &md, &dpos,
 		&dirp, v3,  &dirfor, &dirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &nd);
 	if (dirp && !v3) {
@@ -2296,7 +2310,7 @@ nfsrv_rename(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	fromnd.ni_cnd.cn_cred = cred;
 	fromnd.ni_cnd.cn_nameiop = DELETE;
 	fromnd.ni_cnd.cn_flags = WANTPARENT | SAVESTART | MPSAFE;
-	error = nfs_namei(&fromnd, ffhp, len, slp, nam, &md,
+	error = nfs_namei(&fromnd, nfsd, ffhp, len, slp, nam, &md,
 		&dpos, &fdirp, v3, &fdirfor, &fdirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &fromnd);
 	if (fdirp && !v3) {
@@ -2319,7 +2333,7 @@ nfsrv_rename(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	tond.ni_cnd.cn_cred = cred;
 	tond.ni_cnd.cn_nameiop = RENAME;
 	tond.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF | NOCACHE | SAVESTART | MPSAFE;
-	error = nfs_namei(&tond, tfhp, len2, slp, nam, &md,
+	error = nfs_namei(&tond, nfsd, tfhp, len2, slp, nam, &md,
 		&dpos, &tdirp, v3, &tdirfor, &tdirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &tond);
 	if (tdirp && !v3) {
@@ -2512,7 +2526,7 @@ nfsrv_link(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	nfsm_srvmtofh(dfhp);
 	nfsm_srvnamesiz(len);
 
-	error = nfsrv_fhtovp(fhp, TRUE, &vp, &tvfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, TRUE, &vp, &tvfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	vfslocked = nfsrv_lockedpair(vfslocked, tvfslocked);
 	if (error) {
@@ -2535,7 +2549,7 @@ nfsrv_link(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	nd.ni_cnd.cn_cred = cred;
 	nd.ni_cnd.cn_nameiop = CREATE;
 	nd.ni_cnd.cn_flags = LOCKPARENT | MPSAFE | MPSAFE;
-	error = nfs_namei(&nd, dfhp, len, slp, nam, &md, &dpos,
+	error = nfs_namei(&nd, nfsd, dfhp, len, slp, nam, &md, &dpos,
 		&dirp, v3, &dirfor, &dirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &nd);
 	if (dirp && !v3) {
@@ -2664,7 +2678,7 @@ nfsrv_symlink(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	nd.ni_cnd.cn_cred = cred;
 	nd.ni_cnd.cn_nameiop = CREATE;
 	nd.ni_cnd.cn_flags = LOCKPARENT | SAVESTART | MPSAFE;
-	error = nfs_namei(&nd, fhp, len, slp, nam, &md, &dpos,
+	error = nfs_namei(&nd, nfsd, fhp, len, slp, nam, &md, &dpos,
 		&dirp, v3, &dirfor, &dirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &nd);
 	if (error == 0) {
@@ -2847,7 +2861,7 @@ nfsrv_mkdir(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	nd.ni_cnd.cn_nameiop = CREATE;
 	nd.ni_cnd.cn_flags = LOCKPARENT | MPSAFE;
 
-	error = nfs_namei(&nd, fhp, len, slp, nam, &md, &dpos,
+	error = nfs_namei(&nd, nfsd, fhp, len, slp, nam, &md, &dpos,
 		&dirp, v3, &dirfor, &dirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &nd);
 	if (dirp && !v3) {
@@ -3005,7 +3019,7 @@ nfsrv_rmdir(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	nd.ni_cnd.cn_cred = cred;
 	nd.ni_cnd.cn_nameiop = DELETE;
 	nd.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF | MPSAFE;
-	error = nfs_namei(&nd, fhp, len, slp, nam, &md, &dpos,
+	error = nfs_namei(&nd, nfsd, fhp, len, slp, nam, &md, &dpos,
 		&dirp, v3, &dirfor, &dirfor_ret, FALSE);
 	vfslocked = nfsrv_lockedpair_nd(vfslocked, &nd);
 	if (dirp && !v3) {
@@ -3180,7 +3194,7 @@ nfsrv_readdir(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	if (siz > xfer)
 		siz = xfer;
 	fullsiz = siz;
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	if (!error && vp->v_type != VDIR) {
 		error = ENOTDIR;
@@ -3474,7 +3488,7 @@ nfsrv_readdirplus(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	if (siz > xfer)
 		siz = xfer;
 	fullsiz = siz;
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	if (!error && vp->v_type != VDIR) {
 		error = ENOTDIR;
@@ -3815,7 +3829,7 @@ nfsrv_commit(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	off = fxdr_hyper(tl);
 	tl += 2;
 	cnt = fxdr_unsigned(int, *tl);
-	error = nfsrv_fhtovp(fhp, 1, &vp, &tvfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &tvfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	vfslocked = nfsrv_lockedpair(vfslocked, tvfslocked);
 	if (error) {
@@ -3960,7 +3974,7 @@ nfsrv_statfs(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	vfslocked = 0;
 	fhp = &nfh.fh_generic;
 	nfsm_srvmtofh(fhp);
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	if (error) {
 		nfsm_reply(NFSX_UNSIGNED);
@@ -4055,7 +4069,7 @@ nfsrv_fsinfo(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	fhp = &nfh.fh_generic;
 	vfslocked = 0;
 	nfsm_srvmtofh(fhp);
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	if (error) {
 		nfsm_reply(NFSX_UNSIGNED);
@@ -4080,10 +4094,7 @@ nfsrv_fsinfo(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	 * There should be filesystem VFS OP(s) to get this information.
 	 * For now, assume ufs.
 	 */
-	if (slp->ns_so->so_type == SOCK_DGRAM)
-		pref = NFS_MAXDGRAMDATA;
-	else
-		pref = NFS_MAXDATA;
+	pref = NFS_SRVMAXDATA(nfsd);
 	sip->fs_rtmax = txdr_unsigned(pref);
 	sip->fs_rtpref = txdr_unsigned(pref);
 	sip->fs_rtmult = txdr_unsigned(NFS_FABLKSIZE);
@@ -4133,7 +4144,7 @@ nfsrv_pathconf(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	vfslocked = 0;
 	fhp = &nfh.fh_generic;
 	nfsm_srvmtofh(fhp);
-	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, cred, slp,
+	error = nfsrv_fhtovp(fhp, 1, &vp, &vfslocked, nfsd, slp,
 	    nam, &rdonly, TRUE);
 	if (error) {
 		nfsm_reply(NFSX_UNSIGNED);
diff --git a/sys/nfsserver/nfs_srvcache.c b/sys/nfsserver/nfs_srvcache.c
index 1e36a45..5121690 100644
--- a/sys/nfsserver/nfs_srvcache.c
+++ b/sys/nfsserver/nfs_srvcache.c
@@ -57,6 +57,8 @@ __FBSDID("$FreeBSD$");
 #include <nfsserver/nfs.h>
 #include <nfsserver/nfsrvcache.h>
 
+#ifdef NFS_LEGACYRPC
+
 static long numnfsrvcache;
 static long desirednfsrvcache;
 
@@ -385,3 +387,5 @@ nfsrv_cleancache(void)
 	}
 	numnfsrvcache = 0;
 }
+
+#endif /* NFS_LEGACYRPC */
diff --git a/sys/nfsserver/nfs_srvkrpc.c b/sys/nfsserver/nfs_srvkrpc.c
new file mode 100644
index 0000000..509dc58
--- /dev/null
+++ b/sys/nfsserver/nfs_srvkrpc.c
@@ -0,0 +1,565 @@
+/*-
+ * Copyright (c) 1989, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * Rick Macklem at The University of Guelph.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)nfs_syscalls.c	8.5 (Berkeley) 3/30/95
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_inet6.h"
+#include "opt_kgssapi.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/sysproto.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/file.h>
+#include <sys/filedesc.h>
+#include <sys/vnode.h>
+#include <sys/malloc.h>
+#include <sys/mount.h>
+#include <sys/priv.h>
+#include <sys/proc.h>
+#include <sys/bio.h>
+#include <sys/buf.h>
+#include <sys/mbuf.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/domain.h>
+#include <sys/protosw.h>
+#include <sys/namei.h>
+#include <sys/fcntl.h>
+#include <sys/lockf.h>
+
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#ifdef INET6
+#include <net/if.h>
+#include <netinet6/in6_var.h>
+#endif
+
+#include <rpc/rpc.h>
+#include <rpc/rpcsec_gss.h>
+#include <rpc/replay.h>
+
+#include <nfs/xdr_subs.h>
+#include <nfs/rpcv2.h>
+#include <nfs/nfsproto.h>
+#include <nfsserver/nfs.h>
+#include <nfsserver/nfsm_subs.h>
+#include <nfsserver/nfsrvcache.h>
+#include <nfsserver/nfs_fha.h>
+
+#ifndef NFS_LEGACYRPC
+
+static MALLOC_DEFINE(M_NFSSVC, "nfss_srvsock", "Nfs server structure");
+
+MALLOC_DEFINE(M_NFSRVDESC, "nfss_srvdesc", "NFS server socket descriptor");
+MALLOC_DEFINE(M_NFSD, "nfss_daemon", "Nfs server daemon structure");
+
+#define	TRUE	1
+#define	FALSE	0
+
+SYSCTL_DECL(_vfs_nfsrv);
+
+SVCPOOL		*nfsrv_pool;
+int		nfsd_waiting = 0;
+int		nfsrv_numnfsd = 0;
+static int	nfs_realign_test;
+static int	nfs_realign_count;
+struct callout	nfsrv_callout;
+static eventhandler_tag nfsrv_nmbclusters_tag;
+
+static int	nfs_privport = 0;
+SYSCTL_INT(_vfs_nfsrv, NFS_NFSPRIVPORT, nfs_privport, CTLFLAG_RW,
+    &nfs_privport, 0,
+    "Only allow clients using a privileged port");
+SYSCTL_INT(_vfs_nfsrv, OID_AUTO, gatherdelay, CTLFLAG_RW,
+    &nfsrvw_procrastinate, 0,
+    "Delay value for write gathering");
+SYSCTL_INT(_vfs_nfsrv, OID_AUTO, gatherdelay_v3, CTLFLAG_RW,
+    &nfsrvw_procrastinate_v3, 0,
+    "Delay in seconds for NFSv3 write gathering");
+SYSCTL_INT(_vfs_nfsrv, OID_AUTO, realign_test, CTLFLAG_RW,
+	    &nfs_realign_test, 0, "");
+SYSCTL_INT(_vfs_nfsrv, OID_AUTO, realign_count, CTLFLAG_RW,
+	    &nfs_realign_count, 0, "");
+
+static int	nfssvc_addsock(struct file *, struct thread *);
+static int	nfssvc_nfsd(struct thread *, struct nfsd_nfsd_args *);
+
+extern u_long sb_max_adj;
+
+int32_t (*nfsrv3_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
+    struct nfssvc_sock *slp, struct mbuf **mreqp) = {
+	nfsrv_null,
+	nfsrv_getattr,
+	nfsrv_setattr,
+	nfsrv_lookup,
+	nfsrv3_access,
+	nfsrv_readlink,
+	nfsrv_read,
+	nfsrv_write,
+	nfsrv_create,
+	nfsrv_mkdir,
+	nfsrv_symlink,
+	nfsrv_mknod,
+	nfsrv_remove,
+	nfsrv_rmdir,
+	nfsrv_rename,
+	nfsrv_link,
+	nfsrv_readdir,
+	nfsrv_readdirplus,
+	nfsrv_statfs,
+	nfsrv_fsinfo,
+	nfsrv_pathconf,
+	nfsrv_commit,
+	nfsrv_noop
+};
+
+/*
+ * NFS server system calls
+ */
+
+/*
+ * Nfs server psuedo system call for the nfsd's
+ * Based on the flag value it either:
+ * - adds a socket to the selection list
+ * - remains in the kernel as an nfsd
+ * - remains in the kernel as an nfsiod
+ * For INET6 we suppose that nfsd provides only IN6P_IPV6_V6ONLY sockets
+ * and that mountd provides
+ *  - sockaddr with no IPv4-mapped addresses
+ *  - mask for both INET and INET6 families if there is IPv4-mapped overlap
+ */
+#ifndef _SYS_SYSPROTO_H_
+struct nfssvc_args {
+	int flag;
+	caddr_t argp;
+};
+#endif
+int
+nfssvc(struct thread *td, struct nfssvc_args *uap)
+{
+	struct file *fp;
+	struct nfsd_addsock_args addsockarg;
+	struct nfsd_nfsd_args nfsdarg;
+	int error;
+
+	KASSERT(!mtx_owned(&Giant), ("nfssvc(): called with Giant"));
+
+	error = priv_check(td, PRIV_NFS_DAEMON);
+	if (error)
+		return (error);
+	if (uap->flag & NFSSVC_ADDSOCK) {
+		error = copyin(uap->argp, (caddr_t)&addsockarg,
+		    sizeof(addsockarg));
+		if (error)
+			return (error);
+		if ((error = fget(td, addsockarg.sock, &fp)) != 0)
+			return (error);
+		if (fp->f_type != DTYPE_SOCKET) {
+			fdrop(fp, td);
+			return (error);	/* XXXRW: Should be EINVAL? */
+		}
+		error = nfssvc_addsock(fp, td);
+		fdrop(fp, td);
+	} else if (uap->flag & NFSSVC_OLDNFSD) {
+		error = nfssvc_nfsd(td, NULL);
+	} else if (uap->flag & NFSSVC_NFSD) {
+		if (!uap->argp) 
+			return (EINVAL);
+		error = copyin(uap->argp, (caddr_t)&nfsdarg,
+		    sizeof(nfsdarg));
+		if (error)
+			return (error);
+		error = nfssvc_nfsd(td, &nfsdarg);
+	} else {
+		error = ENXIO;
+	}
+	if (error == EINTR || error == ERESTART)
+		error = 0;
+	return (error);
+}
+
+/*
+ * Generate the rpc reply header
+ * siz arg. is used to decide if adding a cluster is worthwhile
+ */
+struct mbuf *
+nfs_rephead(int siz, struct nfsrv_descript *nd, int err,
+    struct mbuf **mbp, caddr_t *bposp)
+{
+	u_int32_t *tl;
+	struct mbuf *mreq;
+	caddr_t bpos;
+	struct mbuf *mb;
+
+	if (err == EBADRPC)
+		return (NULL);
+
+	nd->nd_repstat = err;
+	if (err && (nd->nd_flag & ND_NFSV3) == 0)	/* XXX recheck */
+		siz = 0;
+
+	MGET(mreq, M_WAIT, MT_DATA);
+
+	/*
+	 * If this is a big reply, use a cluster
+	 */
+	mreq->m_len = 0;
+	if (siz >= MINCLSIZE) {
+		MCLGET(mreq, M_WAIT);
+	}
+	mb = mreq;
+	bpos = mtod(mb, caddr_t);
+
+	if (err != NFSERR_RETVOID) {
+		tl = nfsm_build(u_int32_t *, NFSX_UNSIGNED);
+		if (err)
+			*tl = txdr_unsigned(nfsrv_errmap(nd, err));
+		else
+			*tl = 0;
+	}
+
+	*mbp = mb;
+	*bposp = bpos;
+	if (err != 0 && err != NFSERR_RETVOID)
+		nfsrvstats.srvrpc_errs++;
+
+	return (mreq);
+}
+
+/*
+ *	nfs_realign:
+ *
+ *	Check for badly aligned mbuf data and realign by copying the unaligned
+ *	portion of the data into a new mbuf chain and freeing the portions
+ *	of the old chain that were replaced.
+ *
+ *	We cannot simply realign the data within the existing mbuf chain
+ *	because the underlying buffers may contain other rpc commands and
+ *	we cannot afford to overwrite them.
+ *
+ *	We would prefer to avoid this situation entirely.  The situation does
+ *	not occur with NFS/UDP and is supposed to only occassionally occur
+ *	with TCP.  Use vfs.nfs.realign_count and realign_test to check this.
+ */
+static void
+nfs_realign(struct mbuf **pm)	/* XXX COMMON */
+{
+	struct mbuf *m;
+	struct mbuf *n = NULL;
+	int off = 0;
+
+	++nfs_realign_test;
+	while ((m = *pm) != NULL) {
+		if ((m->m_len & 0x3) || (mtod(m, intptr_t) & 0x3)) {
+			MGET(n, M_WAIT, MT_DATA);
+			if (m->m_len >= MINCLSIZE) {
+				MCLGET(n, M_WAIT);
+			}
+			n->m_len = 0;
+			break;
+		}
+		pm = &m->m_next;
+	}
+
+	/*
+	 * If n is non-NULL, loop on m copying data, then replace the
+	 * portion of the chain that had to be realigned.
+	 */
+	if (n != NULL) {
+		++nfs_realign_count;
+		while (m) {
+			m_copyback(n, off, m->m_len, mtod(m, caddr_t));
+			off += m->m_len;
+			m = m->m_next;
+		}
+		m_freem(*pm);
+		*pm = n;
+	}
+}
+
+static void
+nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
+{
+	rpcproc_t procnum;
+	int32_t (*proc)(struct nfsrv_descript *nd, struct nfssvc_sock *slp,
+	    struct mbuf **mreqp);
+	int flag;
+	struct nfsrv_descript nd;
+	struct mbuf *mreq, *mrep;
+	int error;
+
+	if (rqst->rq_vers == NFS_VER2) {
+		if (rqst->rq_proc > NFSV2PROC_STATFS) {
+			svcerr_noproc(rqst);
+			svc_freereq(rqst);
+			return;
+		}
+		procnum = nfsrv_nfsv3_procid[rqst->rq_proc];
+		flag = 0;
+	} else {
+		if (rqst->rq_proc >= NFS_NPROCS) {
+			svcerr_noproc(rqst);
+			svc_freereq(rqst);
+			return;
+		}
+		procnum = rqst->rq_proc;
+		flag = ND_NFSV3;
+	}
+	proc = nfsrv3_procs[procnum];
+
+	mreq = mrep = NULL;
+	mreq = rqst->rq_args;
+	rqst->rq_args = NULL;
+	nfs_realign(&mreq);
+
+	/*
+	 * Note: we want rq_addr, not svc_getrpccaller -
+	 * NFS_SRVMAXDATA uses a NULL value for nd_nam2 to detect TCP
+	 * mounts.
+	 */
+	memset(&nd, 0, sizeof(nd));
+	nd.nd_md = nd.nd_mrep = mreq;
+	nd.nd_dpos = mtod(mreq, caddr_t);
+	nd.nd_nam = (struct sockaddr *) &xprt->xp_ltaddr;
+	nd.nd_nam2 = rqst->rq_addr;
+	nd.nd_procnum = procnum;
+	nd.nd_cr = NULL;
+	nd.nd_flag = flag;
+
+	if (proc != nfsrv_null) {
+		if (!svc_getcred(rqst, &nd.nd_cr, &nd.nd_credflavor)) {
+			svcerr_weakauth(rqst);
+			svc_freereq(rqst);
+			return;
+		}
+#ifdef MAC
+		mac_cred_associate_nfsd(nd.nd_cr);
+#endif
+	}
+	nfsrvstats.srvrpccnt[nd.nd_procnum]++;
+
+	error = proc(&nd, NULL, &mrep);
+
+	if (nd.nd_cr)
+		crfree(nd.nd_cr);
+
+	if (mrep == NULL) {
+		svcerr_decode(rqst);
+		svc_freereq(rqst);
+		return;
+	}
+	if (error && error != NFSERR_RETVOID) {
+		svcerr_systemerr(rqst);
+		svc_freereq(rqst);
+		return;
+	}
+	if (!svc_sendreply_mbuf(rqst, mrep))
+		svcerr_systemerr(rqst);
+	svc_freereq(rqst);
+}
+
+/*
+ * Adds a socket to the list for servicing by nfsds.
+ */
+static int
+nfssvc_addsock(struct file *fp, struct thread *td)
+{
+	int siz;
+	struct socket *so;
+	int error;
+	SVCXPRT *xprt;
+
+	so = fp->f_data;
+
+	siz = sb_max_adj;
+	error = soreserve(so, siz, siz);
+	if (error) {
+		return (error);
+	}
+
+	/*
+	 * Steal the socket from userland so that it doesn't close
+	 * unexpectedly.
+	 */
+	if (so->so_type == SOCK_DGRAM)
+		xprt = svc_dg_create(nfsrv_pool, so, 0, 0);
+	else
+		xprt = svc_vc_create(nfsrv_pool, so, 0, 0);
+	if (xprt) {
+		fp->f_ops = &badfileops;
+		fp->f_data = NULL;
+		svc_reg(xprt, NFS_PROG, NFS_VER2, nfssvc_program, NULL);
+		svc_reg(xprt, NFS_PROG, NFS_VER3, nfssvc_program, NULL);
+	}
+
+	return (0);
+}
+
+/*
+ * Called by nfssvc() for nfsds. Just loops around servicing rpc requests
+ * until it is killed by a signal.
+ */
+static int
+nfssvc_nfsd(struct thread *td, struct nfsd_nfsd_args *args)
+{
+#ifdef KGSSAPI
+	char principal[128];
+	int error;
+#endif
+
+#ifdef KGSSAPI
+	if (args) {
+		error = copyinstr(args->principal, principal,
+		    sizeof(principal), NULL);
+		if (error)
+			return (error);
+	} else {
+		snprintf(principal, sizeof(principal), "nfs@%s", hostname);
+	}
+#endif
+
+	/*
+	 * Only the first nfsd actually does any work. The RPC code
+	 * adds threads to it as needed. Any extra processes offered
+	 * by nfsd just exit. If nfsd is new enough, it will call us
+	 * once with a structure that specifies how many threads to
+	 * use.
+	 */
+	NFSD_LOCK();
+	if (nfsrv_numnfsd == 0) {
+		nfsrv_numnfsd++;
+
+		NFSD_UNLOCK();
+
+#ifdef KGSSAPI
+		rpc_gss_set_svc_name(principal, "kerberosv5",
+		    GSS_C_INDEFINITE, NFS_PROG, NFS_VER2);
+		rpc_gss_set_svc_name(principal, "kerberosv5",
+		    GSS_C_INDEFINITE, NFS_PROG, NFS_VER3);
+#endif
+
+		if (args) {
+			nfsrv_pool->sp_minthreads = args->minthreads;
+			nfsrv_pool->sp_maxthreads = args->maxthreads;
+		} else {
+			nfsrv_pool->sp_minthreads = 4;
+			nfsrv_pool->sp_maxthreads = 4;
+		}
+			
+		svc_run(nfsrv_pool);
+
+#ifdef KGSSAPI
+		rpc_gss_clear_svc_name(NFS_PROG, NFS_VER2);
+		rpc_gss_clear_svc_name(NFS_PROG, NFS_VER3);
+#endif
+
+		NFSD_LOCK();
+		nfsrv_numnfsd--;
+		nfsrv_init(TRUE);
+	}
+	NFSD_UNLOCK();
+
+	return (0);
+}
+
+/*
+ * Size the NFS server's duplicate request cache at 1/2 the
+ * nmbclusters, floating within a (64, 2048) range. This is to
+ * prevent all mbuf clusters being tied up in the NFS dupreq
+ * cache for small values of nmbclusters.
+ */
+static size_t
+nfsrv_replay_size(void)
+{
+	size_t replaysiz;
+
+	replaysiz = nmbclusters / 2;
+	if (replaysiz > NFSRVCACHE_MAX_SIZE)
+		replaysiz = NFSRVCACHE_MAX_SIZE;
+	if (replaysiz < NFSRVCACHE_MIN_SIZE)
+		replaysiz = NFSRVCACHE_MIN_SIZE;
+	replaysiz *= MCLBYTES;
+
+	return (replaysiz);
+}
+
+/*
+ * Called when nmbclusters changes - we resize the replay cache
+ * accordingly.
+ */
+static void
+nfsrv_nmbclusters_change(void *tag)
+{
+
+	if (nfsrv_pool)
+		replay_setsize(nfsrv_pool->sp_rcache, nfsrv_replay_size());
+}
+
+/*
+ * Initialize the data structures for the server.
+ * Handshake with any new nfsds starting up to avoid any chance of
+ * corruption.
+ */
+void
+nfsrv_init(int terminating)
+{
+
+	NFSD_LOCK_ASSERT();
+
+	if (terminating) {
+		NFSD_UNLOCK();
+		EVENTHANDLER_DEREGISTER(nmbclusters_change,
+		    nfsrv_nmbclusters_tag);
+		svcpool_destroy(nfsrv_pool);
+		nfsrv_pool = NULL;
+		NFSD_LOCK();
+	} else
+		nfs_pub.np_valid = 0;
+
+	NFSD_UNLOCK();
+
+	nfsrv_pool = svcpool_create("nfsd", SYSCTL_STATIC_CHILDREN(_vfs_nfsrv));
+	nfsrv_pool->sp_rcache = replay_newcache(nfsrv_replay_size());
+	nfsrv_pool->sp_assign = fha_assign;
+	nfsrv_pool->sp_done = fha_nd_complete;
+	nfsrv_nmbclusters_tag = EVENTHANDLER_REGISTER(nmbclusters_change,
+	    nfsrv_nmbclusters_change, NULL, EVENTHANDLER_PRI_FIRST);
+
+	NFSD_LOCK();
+}
+
+#endif /* !NFS_LEGACYRPC */
diff --git a/sys/nfsserver/nfs_srvsock.c b/sys/nfsserver/nfs_srvsock.c
index f4362f4..6f42e31 100644
--- a/sys/nfsserver/nfs_srvsock.c
+++ b/sys/nfsserver/nfs_srvsock.c
@@ -70,6 +70,8 @@ __FBSDID("$FreeBSD$");
 
 #include <security/mac/mac_framework.h>
 
+#ifdef NFS_LEGACYRPC
+
 #define	TRUE	1
 #define	FALSE	0
 
@@ -383,6 +385,7 @@ nfs_getreq(struct nfsrv_descript *nd, struct nfsd *nfsd, int has_header)
 		}
 		if (len > 0)
 			nfsm_adv(nfsm_rndup(len));
+		nd->nd_credflavor = RPCAUTH_UNIX;
 	} else {
 		nd->nd_repstat = (NFSERR_AUTHERR | AUTH_REJECTCRED);
 		nd->nd_procnum = NFSPROC_NOOP;
@@ -809,3 +812,5 @@ nfsrv_timer(void *arg)
 	NFSD_UNLOCK();
 	callout_reset(&nfsrv_callout, nfsrv_ticks, nfsrv_timer, NULL);
 }
+
+#endif /* NFS_LEGACYRPC */
diff --git a/sys/nfsserver/nfs_srvsubs.c b/sys/nfsserver/nfs_srvsubs.c
index d738fdf..2cce8be 100644
--- a/sys/nfsserver/nfs_srvsubs.c
+++ b/sys/nfsserver/nfs_srvsubs.c
@@ -93,10 +93,12 @@ static const nfstype nfsv2_type[9] = { NFNON, NFREG, NFDIR, NFBLK, NFCHR,
 
 int nfsrv_ticks;
 
+#ifdef NFS_LEGACYRPC
 struct nfssvc_sockhead nfssvc_sockhead;
 int nfssvc_sockhead_flag;
 struct nfsd_head nfsd_head;
 int nfsd_head_flag;
+#endif
 
 static int nfssvc_offset = SYS_nfssvc;
 static struct sysent nfssvc_prev_sysent;
@@ -545,12 +547,18 @@ nfsrv_modevent(module_t mod, int type, void *data)
 		if (nfsrv_ticks < 1)
 			nfsrv_ticks = 1;
 
+#ifdef NFS_LEGACYRPC
 		nfsrv_initcache();	/* Init the server request cache */
 		NFSD_LOCK();
 		nfsrv_init(0);		/* Init server data structures */
 		callout_init(&nfsrv_callout, CALLOUT_MPSAFE);
 		NFSD_UNLOCK();
 		nfsrv_timer(0);
+#else
+		NFSD_LOCK();
+		nfsrv_init(0);		/* Init server data structures */
+		NFSD_UNLOCK();
+#endif
 
 		error = syscall_register(&nfssvc_offset, &nfssvc_sysent,
 		    &nfssvc_prev_sysent);
@@ -568,7 +576,9 @@ nfsrv_modevent(module_t mod, int type, void *data)
 		if (registered)
 			syscall_deregister(&nfssvc_offset, &nfssvc_prev_sysent);
 		callout_drain(&nfsrv_callout);
+#ifdef NFS_LEGACYRPC
 		nfsrv_destroycache();	/* Free the server request cache */
+#endif
 		mtx_destroy(&nfsd_mtx);
 		break;
 	default:
@@ -604,8 +614,9 @@ MODULE_VERSION(nfsserver, 1);
  * released by the caller.
  */
 int
-nfs_namei(struct nameidata *ndp, fhandle_t *fhp, int len,
-    struct nfssvc_sock *slp, struct sockaddr *nam, struct mbuf **mdp,
+nfs_namei(struct nameidata *ndp, struct nfsrv_descript *nfsd,
+    fhandle_t *fhp, int len, struct nfssvc_sock *slp,
+    struct sockaddr *nam, struct mbuf **mdp,
     caddr_t *dposp, struct vnode **retdirp, int v3, struct vattr *retdirattrp,
     int *retdirattr_retp, int pubflag)
 {
@@ -667,7 +678,7 @@ nfs_namei(struct nameidata *ndp, fhandle_t *fhp, int len,
 	 * Extract and set starting directory.
 	 */
 	error = nfsrv_fhtovp(fhp, FALSE, &dp, &dvfslocked,
-	    ndp->ni_cnd.cn_cred, slp, nam, &rdonly, pubflag);
+	    nfsd, slp, nam, &rdonly, pubflag);
 	if (error)
 		goto out;
 	vfslocked = VFS_LOCK_GIANT(dp->v_mount);
@@ -1079,17 +1090,21 @@ nfsm_srvfattr(struct nfsrv_descript *nfsd, struct vattr *vap,
  */
 int
 nfsrv_fhtovp(fhandle_t *fhp, int lockflag, struct vnode **vpp, int *vfslockedp,
-    struct ucred *cred, struct nfssvc_sock *slp, struct sockaddr *nam,
-    int *rdonlyp, int pubflag)
+    struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
+    struct sockaddr *nam, int *rdonlyp, int pubflag)
 {
 	struct mount *mp;
 	int i;
-	struct ucred *credanon;
+	struct ucred *cred, *credanon;
 	int error, exflags;
 #ifdef MNT_EXNORESPORT		/* XXX needs mountd and /etc/exports help yet */
 	struct sockaddr_int *saddr;
 #endif
+	int credflavor;
 	int vfslocked;
+	int numsecflavors, *secflavors;
+	int v3 = nfsd->nd_flag & ND_NFSV3;
+	int mountreq;
 
 	*vfslockedp = 0;
 	*vpp = NULL;
@@ -1104,9 +1119,35 @@ nfsrv_fhtovp(fhandle_t *fhp, int lockflag, struct vnode **vpp, int *vfslockedp,
 	if (!mp)
 		return (ESTALE);
 	vfslocked = VFS_LOCK_GIANT(mp);
-	error = VFS_CHECKEXP(mp, nam, &exflags, &credanon);
+	error = VFS_CHECKEXP(mp, nam, &exflags, &credanon,
+	    &numsecflavors, &secflavors);
 	if (error)
 		goto out;
+	credflavor = nfsd->nd_credflavor;
+	for (i = 0; i < numsecflavors; i++) {
+		if (secflavors[i] == credflavor)
+			break;
+	}
+	if (i == numsecflavors) {
+		/*
+		 * RFC 2623 section 2.3.2 - allow certain procedures
+		 * used at NFS client mount time even if they have
+		 * weak authentication.
+		 */
+		mountreq = FALSE;
+		if (v3) {
+			if (nfsd->nd_procnum == NFSPROC_FSINFO)
+				mountreq = TRUE;
+		} else {
+			if (nfsd->nd_procnum == NFSPROC_FSSTAT
+			    || nfsd->nd_procnum == NFSPROC_GETATTR)
+				mountreq = TRUE;
+		}
+		if (!mountreq) {
+			error = NFSERR_AUTHERR | AUTH_REJECTCRED;
+			goto out;
+		}
+	}
 	error = VFS_FHTOVP(mp, &fhp->fh_fid, vpp);
 	if (error)
 		goto out;
@@ -1126,6 +1167,7 @@ nfsrv_fhtovp(fhandle_t *fhp, int lockflag, struct vnode **vpp, int *vfslockedp,
 	/*
 	 * Check/setup credentials.
 	 */
+	cred = nfsd->nd_cr;
 	if (cred->cr_uid == 0 || (exflags & MNT_EXPORTANON)) {
 		cred->cr_uid = credanon->cr_uid;
 		for (i = 0; i < credanon->cr_ngroups && i < NGROUPS; i++)
@@ -1168,6 +1210,8 @@ nfs_ispublicfh(fhandle_t *fhp)
 	return (TRUE);
 }
 
+#ifdef NFS_LEGACYRPC
+
 /*
  * This function compares two net addresses by family and returns TRUE
  * if they are the same host.
@@ -1210,6 +1254,8 @@ netaddr_match(int family, union nethostaddr *haddr, struct sockaddr *nam)
 	return (0);
 }
 
+#endif
+
 /*
  * Map errnos to NFS error numbers. For Version 3 also filter out error
  * numbers not specified for the associated procedure.
@@ -1364,13 +1410,12 @@ nfsm_clget_xx(u_int32_t **tl, struct mbuf *mb, struct mbuf **mp,
 }
 
 int
-nfsm_srvmtofh_xx(fhandle_t *f, struct nfsrv_descript *nfsd, struct mbuf **md,
-    caddr_t *dpos)
+nfsm_srvmtofh_xx(fhandle_t *f, int v3, struct mbuf **md, caddr_t *dpos)
 {
 	u_int32_t *tl;
 	int fhlen;
 
-	if (nfsd->nd_flag & ND_NFSV3) {
+	if (v3) {
 		tl = nfsm_dissect_xx_nonblock(NFSX_UNSIGNED, md, dpos);
 		if (tl == NULL)
 			return EBADRPC;
diff --git a/sys/nfsserver/nfs_syscalls.c b/sys/nfsserver/nfs_syscalls.c
index d5e5e67..0c936c9 100644
--- a/sys/nfsserver/nfs_syscalls.c
+++ b/sys/nfsserver/nfs_syscalls.c
@@ -73,6 +73,8 @@ __FBSDID("$FreeBSD$");
 #include <nfsserver/nfsm_subs.h>
 #include <nfsserver/nfsrvcache.h>
 
+#ifdef NFS_LEGACYRPC
+
 static MALLOC_DEFINE(M_NFSSVC, "nfss_srvsock", "Nfs server structure");
 
 MALLOC_DEFINE(M_NFSRVDESC, "nfss_srvdesc", "NFS server socket descriptor");
@@ -130,7 +132,7 @@ nfssvc(struct thread *td, struct nfssvc_args *uap)
 {
 	struct file *fp;
 	struct sockaddr *nam;
-	struct nfsd_args nfsdarg;
+	struct nfsd_addsock_args nfsdarg;
 	int error;
 
 	KASSERT(!mtx_owned(&Giant), ("nfssvc(): called with Giant"));
@@ -170,7 +172,7 @@ nfssvc(struct thread *td, struct nfssvc_args *uap)
 		}
 		error = nfssvc_addsock(fp, nam);
 		fdrop(fp, td);
-	} else if (uap->flag & NFSSVC_NFSD) {
+	} else if (uap->flag & NFSSVC_OLDNFSD) {
 		error = nfssvc_nfsd();
 	} else {
 		error = ENXIO;
@@ -727,3 +729,5 @@ nfsrv_init(int terminating)
 	TAILQ_INSERT_TAIL(&nfssvc_sockhead, nfs_cltpsock, ns_chain);
 #endif
 }
+
+#endif /* NFS_LEGACYRPC */
diff --git a/sys/nfsserver/nfsm_subs.h b/sys/nfsserver/nfsm_subs.h
index f2dafc4..fc9b76d 100644
--- a/sys/nfsserver/nfsm_subs.h
+++ b/sys/nfsserver/nfsm_subs.h
@@ -75,8 +75,7 @@
 int	nfsm_srvstrsiz_xx(int *s, int m, struct mbuf **md, caddr_t *dpos);
 int	nfsm_srvnamesiz_xx(int *s, int m, struct mbuf **md, caddr_t *dpos);
 int	nfsm_srvnamesiz0_xx(int *s, int m, struct mbuf **md, caddr_t *dpos);
-int	nfsm_srvmtofh_xx(fhandle_t *f, struct nfsrv_descript *nfsd,
-	    struct mbuf **md, caddr_t *dpos);
+int	nfsm_srvmtofh_xx(fhandle_t *f, int v3, struct mbuf **md, caddr_t *dpos);
 int	nfsm_srvsattr_xx(struct vattr *a, struct mbuf **md, caddr_t *dpos);
 
 #define	nfsm_srvstrsiz(s, m) \
@@ -112,7 +111,7 @@ do { \
 #define nfsm_srvmtofh(f) \
 do { \
 	int t1; \
-	t1 = nfsm_srvmtofh_xx((f), nfsd, &md, &dpos); \
+	t1 = nfsm_srvmtofh_xx((f), nfsd->nd_flag & ND_NFSV3, &md, &dpos); \
 	if (t1) { \
 		error = t1; \
 		nfsm_reply(0); \
diff --git a/sys/nfsserver/nfsrvcache.h b/sys/nfsserver/nfsrvcache.h
index 66176f4..9c527e9 100644
--- a/sys/nfsserver/nfsrvcache.h
+++ b/sys/nfsserver/nfsrvcache.h
@@ -44,6 +44,8 @@
 #define NFSRVCACHE_MAX_SIZE	2048
 #define NFSRVCACHE_MIN_SIZE	  64
 
+#ifdef NFS_LEGACYRPC
+
 struct nfsrvcache {
 	TAILQ_ENTRY(nfsrvcache) rc_lru;		/* LRU chain */
 	LIST_ENTRY(nfsrvcache) rc_hash;		/* Hash chain */
@@ -83,3 +85,5 @@ struct nfsrvcache {
 #define	RC_NAM		0x40
 
 #endif
+
+#endif