Mfp4 CH=177274,177280,177284-177285,177297,177324-177325

  VNET socket push back:
  try to minimize the number of places where we have to switch vnets
  and narrow down the time we stay switched.  Add assertions to the
  socket code to catch possibly unset vnets as seen in r204147.

  While this reduces the number of vnet recursion in some places like
  NFS, POSIX local sockets and some netgraph, .. recursions are
  impossible to fix.

  The current expectations are documented at the beginning of
  uipc_socket.c along with the other information there.

  Sponsored by: The FreeBSD Foundation
  Sponsored by: CK Software GmbH
  Reviewed by:  jhb
  Tested by:    zec

Tested by:	Mikolaj Golub (to.my.trociny gmail.com)
MFC after:	2 weeks

6 files changed, 3 insertions, 28 deletions

diff --git a/sys/rpc/clnt_dg.c b/sys/rpc/clnt_dg.c
index d35a6a9..58d55ff 100644
--- a/sys/rpc/clnt_dg.c
+++ b/sys/rpc/clnt_dg.c
@@ -200,14 +200,11 @@ clnt_dg_create(
 		return (NULL);
 	}
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
 		rpc_createerr.cf_stat = RPC_TLIERROR;
 		rpc_createerr.cf_error.re_errno = 0;
-		CURVNET_RESTORE();
 		return (NULL);
 	}
-	CURVNET_RESTORE();
 
 	/*
 	 * Find the receive and the send size
diff --git a/sys/rpc/clnt_vc.c b/sys/rpc/clnt_vc.c
index 9882407..9f36bba 100644
--- a/sys/rpc/clnt_vc.c
+++ b/sys/rpc/clnt_vc.c
@@ -220,9 +220,7 @@ clnt_vc_create(
 		}
 	}
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
-		CURVNET_RESTORE();
 		goto err;
 	}
 
@@ -245,7 +243,6 @@ clnt_vc_create(
 		sopt.sopt_valsize = sizeof(one);
 		sosetopt(so, &sopt);
 	}
-	CURVNET_RESTORE();
 
 	ct->ct_closeit = FALSE;
 
diff --git a/sys/rpc/rpc_generic.c b/sys/rpc/rpc_generic.c
index 1c6aa82..fd39350 100644
--- a/sys/rpc/rpc_generic.c
+++ b/sys/rpc/rpc_generic.c
@@ -187,7 +187,9 @@ __rpc_socket2sockinfo(struct socket *so, struct __rpc_sockinfo *sip)
 	struct sockopt opt;
 	int error;
 
+	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
+	CURVNET_RESTORE();
 	if (error)
 		return 0;
 
@@ -825,7 +827,6 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 	sa->sa_len = salen;
 
 	if (*portp == 0) {
-		CURVNET_SET(so->so_vnet);
 		bzero(&opt, sizeof(opt));
 		opt.sopt_dir = SOPT_GET;
 		opt.sopt_level = proto;
@@ -834,14 +835,12 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 		opt.sopt_valsize = sizeof(old);
 		error = sogetopt(so, &opt);
 		if (error) {
-			CURVNET_RESTORE();
 			goto out;
 		}
 
 		opt.sopt_dir = SOPT_SET;
 		opt.sopt_val = &portlow;
 		error = sosetopt(so, &opt);
-		CURVNET_RESTORE();
 		if (error)
 			goto out;
 	}
@@ -852,9 +851,7 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 		if (error) {
 			opt.sopt_dir = SOPT_SET;
 			opt.sopt_val = &old;
-			CURVNET_SET(so->so_vnet);
 			sosetopt(so, &opt);
-			CURVNET_RESTORE();
 		}
 	}
 out:
diff --git a/sys/rpc/svc_dg.c b/sys/rpc/svc_dg.c
index 9d7696b..adfe2b5 100644
--- a/sys/rpc/svc_dg.c
+++ b/sys/rpc/svc_dg.c
@@ -103,10 +103,8 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	struct sockaddr* sa;
 	int error;
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
 		printf(svc_dg_str, svc_dg_err1);
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 	/*
@@ -116,7 +114,6 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	recvsize = __rpc_get_t_size(si.si_af, si.si_proto, (int)recvsize);
 	if ((sendsize == 0) || (recvsize == 0)) {
 		printf(svc_dg_str, svc_dg_err2);
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 
@@ -128,6 +125,7 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt->xp_p2 = NULL;
 	xprt->xp_ops = &svc_dg_ops;
 
+	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
 	CURVNET_RESTORE();
 	if (error)
diff --git a/sys/rpc/svc_generic.c b/sys/rpc/svc_generic.c
index e6e8acd..f8b70de 100644
--- a/sys/rpc/svc_generic.c
+++ b/sys/rpc/svc_generic.c
@@ -230,14 +230,11 @@ svc_tli_create(
 		/*
 		 * It is an open socket. Get the transport info.
 		 */
-		CURVNET_SET(so->so_vnet);
 		if (!__rpc_socket2sockinfo(so, &si)) {
 			printf(
 		"svc_tli_create: could not get transport information\n");
-			CURVNET_RESTORE();
 			return (NULL);
 		}
-		CURVNET_RESTORE();
 	}
 
 	/*
@@ -264,9 +261,7 @@ svc_tli_create(
 		"svc_tli_create: could not bind to requested address\n");
 				goto freedata;
 			}
-			CURVNET_SET(so->so_vnet);
 			solisten(so, (int)bindaddr->qlen, curthread);
-			CURVNET_RESTORE();
 		}
 			
 	}
diff --git a/sys/rpc/svc_vc.c b/sys/rpc/svc_vc.c
index 999cad1..fddcf58 100644
--- a/sys/rpc/svc_vc.c
+++ b/sys/rpc/svc_vc.c
@@ -154,10 +154,8 @@ svc_vc_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt->xp_p2 = NULL;
 	xprt->xp_ops = &svc_vc_rendezvous_ops;
 
-	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
 	if (error) {
-		CURVNET_RESTORE();
 		goto cleanup_svc_vc_create;
 	}
 
@@ -167,7 +165,6 @@ svc_vc_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt_register(xprt);
 
 	solisten(so, SOMAXCONN, curthread);
-	CURVNET_RESTORE();
 
 	SOCKBUF_LOCK(&so->so_rcv);
 	xprt->xp_upcallset = 1;
@@ -200,10 +197,8 @@ svc_vc_create_conn(SVCPOOL *pool, struct socket *so, struct sockaddr *raddr)
 	opt.sopt_name = SO_KEEPALIVE;
 	opt.sopt_val = &one;
 	opt.sopt_valsize = sizeof(one);
-	CURVNET_SET(so->so_vnet);
 	error = sosetopt(so, &opt);
 	if (error) {
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 
@@ -216,11 +211,9 @@ svc_vc_create_conn(SVCPOOL *pool, struct socket *so, struct sockaddr *raddr)
 		opt.sopt_valsize = sizeof(one);
 		error = sosetopt(so, &opt);
 		if (error) {
-			CURVNET_RESTORE();
 			return (NULL);
 		}
 	}
-	CURVNET_RESTORE();
 
 	cd = mem_alloc(sizeof(*cd));
 	cd->strm_stat = XPRT_IDLE;
@@ -635,10 +628,8 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
 		uio.uio_td = curthread;
 		m = NULL;
 		rcvflag = MSG_DONTWAIT;
-		CURVNET_SET(xprt->xp_socket->so_vnet);
 		error = soreceive(xprt->xp_socket, NULL, &uio, &m, NULL,
 		    &rcvflag);
-		CURVNET_RESTORE();
 
 		if (error == EWOULDBLOCK) {
 			/*