Fix two races in the server side krpc w.r.t upcalls:

  Add a flag so that soupcall_clear() is only called once to cancel
  an upcall.
  Move the test for xprt_registered in the upcall down to after the
  mtx_lock() of the pool mutex, to catch the case where it is
  unregistered while the upcall is waiting for the mutex.
Also, move the mtx_destroy() of the pool mutex to after SVC_RELEASE(),
so that it isn't destroyed before the upcalls are disabled.

Reviewed by:	dfr, jhb
Tested by:	pho
Approved by:	kib (mentor)

3 files changed, 20 insertions, 7 deletions

diff --git a/sys/rpc/svc.c b/sys/rpc/svc.c
index 8af9e80..fbddafb 100644
--- a/sys/rpc/svc.c
+++ b/sys/rpc/svc.c
@@ -175,12 +175,12 @@ svcpool_destroy(SVCPOOL *pool)
 		mtx_lock(&pool->sp_lock);
 	}
 
-	mtx_destroy(&pool->sp_lock);
-
 	TAILQ_FOREACH_SAFE(xprt, &cleanup, xp_link, nxprt) {
 		SVC_RELEASE(xprt);
 	}
 
+	mtx_destroy(&pool->sp_lock);
+
 	if (pool->sp_rcache)
 		replay_freecache(pool->sp_rcache);
 
@@ -353,15 +353,16 @@ xprt_active(SVCXPRT *xprt)
 {
 	SVCPOOL *pool = xprt->xp_pool;
 
+	mtx_lock(&pool->sp_lock);
+
 	if (!xprt->xp_registered) {
 		/*
 		 * Race with xprt_unregister - we lose.
 		 */
+		mtx_unlock(&pool->sp_lock);
 		return;
 	}
 
-	mtx_lock(&pool->sp_lock);
-
 	if (!xprt->xp_active) {
 		TAILQ_INSERT_TAIL(&pool->sp_active, xprt, xp_alink);
 		xprt->xp_active = TRUE;
diff --git a/sys/rpc/svc.h b/sys/rpc/svc.h
index 73c4414..1bd3054 100644
--- a/sys/rpc/svc.h
+++ b/sys/rpc/svc.h
@@ -166,6 +166,7 @@ typedef struct __rpc_svcxprt {
 	int		xp_idletimeout; /* idle time before closing */
 	time_t		xp_lastactive;	/* time of last RPC */
 	u_int64_t	xp_sockref;	/* set by nfsv4 to identify socket */
+	int		xp_upcallset;	/* socket upcall is set up */
 #else
 	int		xp_fd;
 	u_short		xp_port;	 /* associated port number */
diff --git a/sys/rpc/svc_vc.c b/sys/rpc/svc_vc.c
index be366de..7249270 100644
--- a/sys/rpc/svc_vc.c
+++ b/sys/rpc/svc_vc.c
@@ -160,6 +160,7 @@ svc_vc_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	solisten(so, SOMAXCONN, curthread);
 
 	SOCKBUF_LOCK(&so->so_rcv);
+	xprt->xp_upcallset = 1;
 	soupcall_set(so, SO_RCV, svc_vc_soupcall, xprt);
 	SOCKBUF_UNLOCK(&so->so_rcv);
 
@@ -234,6 +235,7 @@ svc_vc_create_conn(SVCPOOL *pool, struct socket *so, struct sockaddr *raddr)
 	xprt_register(xprt);
 
 	SOCKBUF_LOCK(&so->so_rcv);
+	xprt->xp_upcallset = 1;
 	soupcall_set(so, SO_RCV, svc_vc_soupcall, xprt);
 	SOCKBUF_UNLOCK(&so->so_rcv);
 
@@ -352,7 +354,10 @@ svc_vc_rendezvous_recv(SVCXPRT *xprt, struct rpc_msg *msg,
 
 	if (error) {
 		SOCKBUF_LOCK(&xprt->xp_socket->so_rcv);
-		soupcall_clear(xprt->xp_socket, SO_RCV);
+		if (xprt->xp_upcallset) {
+			xprt->xp_upcallset = 0;
+			soupcall_clear(xprt->xp_socket, SO_RCV);
+		}
 		SOCKBUF_UNLOCK(&xprt->xp_socket->so_rcv);
 		xprt_inactive(xprt);
 		sx_xunlock(&xprt->xp_lock);
@@ -397,7 +402,10 @@ static void
 svc_vc_destroy_common(SVCXPRT *xprt)
 {
 	SOCKBUF_LOCK(&xprt->xp_socket->so_rcv);
-	soupcall_clear(xprt->xp_socket, SO_RCV);
+	if (xprt->xp_upcallset) {
+		xprt->xp_upcallset = 0;
+		soupcall_clear(xprt->xp_socket, SO_RCV);
+	}
 	SOCKBUF_UNLOCK(&xprt->xp_socket->so_rcv);
 
 	sx_destroy(&xprt->xp_lock);
@@ -632,7 +640,10 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
 
 		if (error) {
 			SOCKBUF_LOCK(&xprt->xp_socket->so_rcv);
-			soupcall_clear(xprt->xp_socket, SO_RCV);
+			if (xprt->xp_upcallset) {
+				xprt->xp_upcallset = 0;
+				soupcall_clear(xprt->xp_socket, SO_RCV);
+			}
 			SOCKBUF_UNLOCK(&xprt->xp_socket->so_rcv);
 			xprt_inactive(xprt);
 			cd->strm_stat = XPRT_DIED;