Implement support for RPCSEC_GSS authentication to both the NFS client

and server. This replaces the RPC implementation of the NFS client and
server with the newer RPC implementation originally developed
(actually ported from the userland sunrpc code) to support the NFS
Lock Manager.  I have tested this code extensively and I believe it is
stable and that performance is at least equal to the legacy RPC
implementation.

The NFS code currently contains support for both the new RPC
implementation and the older legacy implementation inherited from the
original NFS codebase. The default is to use the new implementation -
add the NFS_LEGACYRPC option to fall back to the old code. When I
merge this support back to RELENG_7, I will probably change this so
that users have to 'opt in' to get the new code.

To use RPCSEC_GSS on either client or server, you must build a kernel
which includes the KGSSAPI option and the crypto device. On the
userland side, you must build at least a new libc, mountd, mount_nfs
and gssd. You must install new versions of /etc/rc.d/gssd and
/etc/rc.d/nfsd and add 'gssd_enable=YES' to /etc/rc.conf.

As long as gssd is running, you should be able to mount an NFS
filesystem from a server that requires RPCSEC_GSS authentication. The
mount itself can happen without any kerberos credentials but all
access to the filesystem will be denied unless the accessing user has
a valid ticket file in the standard place (/tmp/krb5cc_<uid>). There
is currently no support for situations where the ticket file is in a
different place, such as when the user logged in via SSH and has
delegated credentials from that login. This restriction is also
present in Solaris and Linux. In theory, we could improve this in
future, possibly using Brooks Davis' implementation of variant
symlinks.

Supporting RPCSEC_GSS on a server is nearly as simple. You must create
service creds for the server in the form 'nfs/<fqdn>@<REALM>' and
install them in /etc/krb5.keytab. The standard heimdal utility ktutil
makes this fairly easy. After the service creds have been created, you
can add a '-sec=krb5' option to /etc/exports and restart both mountd
and nfsd.

The only other difference an administrator should notice is that nfsd
doesn't fork to create service threads any more. In normal operation,
there will be two nfsd processes, one in userland waiting for TCP
connections and one in the kernel handling requests. The latter
process will create as many kthreads as required - these should be
visible via 'top -H'. The code has some support for varying the number
of service threads according to load but initially at least, nfsd uses
a fixed number of threads according to the value supplied to its '-n'
option.

Sponsored by:	Isilon Systems
MFC after:	1 month

1 files changed, 66 insertions, 91 deletions

diff --git a/sys/rpc/svc_dg.c b/sys/rpc/svc_dg.c
index 666b952..72721b0 100644
--- a/sys/rpc/svc_dg.c
+++ b/sys/rpc/svc_dg.c
@@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
+#include <sys/sx.h>
 #include <sys/systm.h>
 #include <sys/uio.h>
 
@@ -61,10 +62,10 @@ __FBSDID("$FreeBSD$");
 #include <rpc/rpc_com.h>
 
 static enum xprt_stat svc_dg_stat(SVCXPRT *);
-static bool_t svc_dg_recv(SVCXPRT *, struct rpc_msg *);
-static bool_t svc_dg_reply(SVCXPRT *, struct rpc_msg *);
-static bool_t svc_dg_getargs(SVCXPRT *, xdrproc_t, void *);
-static bool_t svc_dg_freeargs(SVCXPRT *, xdrproc_t, void *);
+static bool_t svc_dg_recv(SVCXPRT *, struct rpc_msg *,
+    struct sockaddr **, struct mbuf **);
+static bool_t svc_dg_reply(SVCXPRT *, struct rpc_msg *,
+    struct sockaddr *, struct mbuf *);
 static void svc_dg_destroy(SVCXPRT *);
 static bool_t svc_dg_control(SVCXPRT *, const u_int, void *);
 static void svc_dg_soupcall(struct socket *so, void *arg, int waitflag);
@@ -72,9 +73,7 @@ static void svc_dg_soupcall(struct socket *so, void *arg, int waitflag);
 static struct xp_ops svc_dg_ops = {
 	.xp_recv =	svc_dg_recv,
 	.xp_stat =	svc_dg_stat,
-	.xp_getargs =	svc_dg_getargs,
 	.xp_reply =	svc_dg_reply,
-	.xp_freeargs =	svc_dg_freeargs,
 	.xp_destroy =	svc_dg_destroy,
 	.xp_control =	svc_dg_control,
 };
@@ -116,9 +115,8 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 		return (NULL);
 	}
 
-	xprt = mem_alloc(sizeof (SVCXPRT));
-	memset(xprt, 0, sizeof (SVCXPRT));
-	mtx_init(&xprt->xp_lock, "xprt->xp_lock", NULL, MTX_DEF);
+	xprt = svc_xprt_alloc();
+	sx_init(&xprt->xp_lock, "xprt->xp_lock");
 	xprt->xp_pool = pool;
 	xprt->xp_socket = so;
 	xprt->xp_p1 = NULL;
@@ -129,16 +127,9 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	if (error)
 		goto freedata;
 
-	xprt->xp_ltaddr.buf = mem_alloc(sizeof (struct sockaddr_storage));
-	xprt->xp_ltaddr.maxlen = sizeof (struct sockaddr_storage);
-	xprt->xp_ltaddr.len = sa->sa_len;
-	memcpy(xprt->xp_ltaddr.buf, sa, sa->sa_len);
+	memcpy(&xprt->xp_ltaddr, sa, sa->sa_len);
 	free(sa, M_SONAME);
 
-	xprt->xp_rtaddr.buf = mem_alloc(sizeof (struct sockaddr_storage));
-	xprt->xp_rtaddr.maxlen = sizeof (struct sockaddr_storage);
-	xprt->xp_rtaddr.len = 0;
-
 	xprt_register(xprt);
 
 	SOCKBUF_LOCK(&so->so_rcv);
@@ -151,7 +142,7 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 freedata:
 	(void) printf(svc_dg_str, __no_mem_str);
 	if (xprt) {
-		(void) mem_free(xprt, sizeof (SVCXPRT));
+		svc_xprt_free(xprt);
 	}
 	return (NULL);
 }
@@ -161,34 +152,34 @@ static enum xprt_stat
 svc_dg_stat(SVCXPRT *xprt)
 {
 
+	if (soreadable(xprt->xp_socket))
+		return (XPRT_MOREREQS);
+
 	return (XPRT_IDLE);
 }
 
 static bool_t
-svc_dg_recv(SVCXPRT *xprt, struct rpc_msg *msg)
+svc_dg_recv(SVCXPRT *xprt, struct rpc_msg *msg,
+    struct sockaddr **addrp, struct mbuf **mp)
 {
 	struct uio uio;
 	struct sockaddr *raddr;
 	struct mbuf *mreq;
+	XDR xdrs;
 	int error, rcvflag;
 
 	/*
+	 * Serialise access to the socket.
+	 */
+	sx_xlock(&xprt->xp_lock);
+
+	/*
 	 * The socket upcall calls xprt_active() which will eventually
 	 * cause the server to call us here. We attempt to read a
 	 * packet from the socket and process it. If the read fails,
 	 * we have drained all pending requests so we call
 	 * xprt_inactive().
-	 *
-	 * The lock protects us in the case where a new packet arrives
-	 * on the socket after our call to soreceive fails with
-	 * EWOULDBLOCK - the call to xprt_active() in the upcall will
-	 * happen only after our call to xprt_inactive() which ensures
-	 * that we will remain active. It might be possible to use
-	 * SOCKBUF_LOCK for this - its not clear to me what locks are
-	 * held during the upcall.
 	 */
-	mtx_lock(&xprt->xp_lock);
-
 	uio.uio_resid = 1000000000;
 	uio.uio_td = curthread;
 	mreq = NULL;
@@ -196,8 +187,19 @@ svc_dg_recv(SVCXPRT *xprt, struct rpc_msg *msg)
 	error = soreceive(xprt->xp_socket, &raddr, &uio, &mreq, NULL, &rcvflag);
 
 	if (error == EWOULDBLOCK) {
-		xprt_inactive(xprt);
-		mtx_unlock(&xprt->xp_lock);
+		/*
+		 * We must re-test for readability after taking the
+		 * lock to protect us in the case where a new packet
+		 * arrives on the socket after our call to soreceive
+		 * fails with EWOULDBLOCK. The pool lock protects us
+		 * from racing the upcall after our soreadable() call
+		 * returns false.
+		 */
+		mtx_lock(&xprt->xp_pool->sp_lock);
+		if (!soreadable(xprt->xp_socket))
+			xprt_inactive_locked(xprt);
+		mtx_unlock(&xprt->xp_pool->sp_lock);
+		sx_xunlock(&xprt->xp_lock);
 		return (FALSE);
 	}
 
@@ -208,45 +210,52 @@ svc_dg_recv(SVCXPRT *xprt, struct rpc_msg *msg)
 		xprt->xp_socket->so_rcv.sb_flags &= ~SB_UPCALL;
 		SOCKBUF_UNLOCK(&xprt->xp_socket->so_rcv);
 		xprt_inactive(xprt);
-		mtx_unlock(&xprt->xp_lock);
+		sx_xunlock(&xprt->xp_lock);
 		return (FALSE);
 	}
 
-	mtx_unlock(&xprt->xp_lock);
-
-	KASSERT(raddr->sa_len < xprt->xp_rtaddr.maxlen,
-	    ("Unexpected remote address length"));
-	memcpy(xprt->xp_rtaddr.buf, raddr, raddr->sa_len);
-	xprt->xp_rtaddr.len = raddr->sa_len;
-	free(raddr, M_SONAME);
+	sx_xunlock(&xprt->xp_lock);
 
-	xdrmbuf_create(&xprt->xp_xdrreq, mreq, XDR_DECODE);
-	if (! xdr_callmsg(&xprt->xp_xdrreq, msg)) {
-		XDR_DESTROY(&xprt->xp_xdrreq);
+	xdrmbuf_create(&xdrs, mreq, XDR_DECODE);
+	if (! xdr_callmsg(&xdrs, msg)) {
+		XDR_DESTROY(&xdrs);
 		return (FALSE);
 	}
-	xprt->xp_xid = msg->rm_xid;
+
+	*addrp = raddr;
+	*mp = xdrmbuf_getall(&xdrs);
+	XDR_DESTROY(&xdrs);
 
 	return (TRUE);
 }
 
 static bool_t
-svc_dg_reply(SVCXPRT *xprt, struct rpc_msg *msg)
+svc_dg_reply(SVCXPRT *xprt, struct rpc_msg *msg,
+    struct sockaddr *addr, struct mbuf *m)
 {
+	XDR xdrs;
 	struct mbuf *mrep;
-	bool_t stat = FALSE;
+	bool_t stat = TRUE;
 	int error;
 
 	MGETHDR(mrep, M_WAIT, MT_DATA);
-	MCLGET(mrep, M_WAIT);
 	mrep->m_len = 0;
 
-	xdrmbuf_create(&xprt->xp_xdrrep, mrep, XDR_ENCODE);
-	msg->rm_xid = xprt->xp_xid;
-	if (xdr_replymsg(&xprt->xp_xdrrep, msg)) {
+	xdrmbuf_create(&xdrs, mrep, XDR_ENCODE);
+
+	if (msg->rm_reply.rp_stat == MSG_ACCEPTED &&
+	    msg->rm_reply.rp_acpt.ar_stat == SUCCESS) {
+		if (!xdr_replymsg(&xdrs, msg))
+			stat = FALSE;
+		else
+			xdrmbuf_append(&xdrs, m);
+	} else {
+		stat = xdr_replymsg(&xdrs, msg);
+	}
+
+	if (stat) {
 		m_fixhdr(mrep);
-		error = sosend(xprt->xp_socket,
-		    (struct sockaddr *) xprt->xp_rtaddr.buf, NULL, mrep, NULL,
+		error = sosend(xprt->xp_socket, addr, NULL, mrep, NULL,
 		    0, curthread);
 		if (!error) {
 			stat = TRUE;
@@ -255,61 +264,29 @@ svc_dg_reply(SVCXPRT *xprt, struct rpc_msg *msg)
 		m_freem(mrep);
 	}
 
-	/*
-	 * This frees the request mbuf chain as well. The reply mbuf
-	 * chain was consumed by sosend.
-	 */
-	XDR_DESTROY(&xprt->xp_xdrreq);
-	XDR_DESTROY(&xprt->xp_xdrrep);
+	XDR_DESTROY(&xdrs);
 	xprt->xp_p2 = NULL;
 
 	return (stat);
 }
 
-static bool_t
-svc_dg_getargs(SVCXPRT *xprt, xdrproc_t xdr_args, void *args_ptr)
-{
-
-	return (xdr_args(&xprt->xp_xdrreq, args_ptr));
-}
-
-static bool_t
-svc_dg_freeargs(SVCXPRT *xprt, xdrproc_t xdr_args, void *args_ptr)
-{
-	XDR xdrs;
-
-	/*
-	 * Free the request mbuf here - this allows us to handle
-	 * protocols where not all requests have replies
-	 * (i.e. NLM). Note that xdrmbuf_destroy handles being called
-	 * twice correctly - the mbuf will only be freed once.
-	 */
-	XDR_DESTROY(&xprt->xp_xdrreq);
-
-	xdrs.x_op = XDR_FREE;
-	return (xdr_args(&xdrs, args_ptr));
-}
-
 static void
 svc_dg_destroy(SVCXPRT *xprt)
 {
+
 	SOCKBUF_LOCK(&xprt->xp_socket->so_rcv);
 	xprt->xp_socket->so_upcallarg = NULL;
 	xprt->xp_socket->so_upcall = NULL;
 	xprt->xp_socket->so_rcv.sb_flags &= ~SB_UPCALL;
 	SOCKBUF_UNLOCK(&xprt->xp_socket->so_rcv);
 
-	xprt_unregister(xprt);
-
-	mtx_destroy(&xprt->xp_lock);
+	sx_destroy(&xprt->xp_lock);
 	if (xprt->xp_socket)
 		(void)soclose(xprt->xp_socket);
 
-	if (xprt->xp_rtaddr.buf)
-		(void) mem_free(xprt->xp_rtaddr.buf, xprt->xp_rtaddr.maxlen);
-	if (xprt->xp_ltaddr.buf)
-		(void) mem_free(xprt->xp_ltaddr.buf, xprt->xp_ltaddr.maxlen);
-	(void) mem_free(xprt, sizeof (SVCXPRT));
+	if (xprt->xp_netid)
+		(void) mem_free(xprt->xp_netid, strlen(xprt->xp_netid) + 1);
+	svc_xprt_free(xprt);
 }
 
 static bool_t
@@ -328,7 +305,5 @@ svc_dg_soupcall(struct socket *so, void *arg, int waitflag)
 {
 	SVCXPRT *xprt = (SVCXPRT *) arg;
 
-	mtx_lock(&xprt->xp_lock);
 	xprt_active(xprt);
-	mtx_unlock(&xprt->xp_lock);
 }