Mfp4 CH=177274,177280,177284-177285,177297,177324-177325

  VNET socket push back:
  try to minimize the number of places where we have to switch vnets
  and narrow down the time we stay switched.  Add assertions to the
  socket code to catch possibly unset vnets as seen in r204147.

  While this reduces the number of vnet recursion in some places like
  NFS, POSIX local sockets and some netgraph, .. recursions are
  impossible to fix.

  The current expectations are documented at the beginning of
  uipc_socket.c along with the other information there.

  Sponsored by: The FreeBSD Foundation
  Sponsored by: CK Software GmbH
  Reviewed by:  jhb
  Tested by:    zec

Tested by:	Mikolaj Golub (to.my.trociny gmail.com)
MFC after:	2 weeks

23 files changed, 179 insertions, 103 deletions

diff --git a/sys/fs/nfsclient/nfs_clvfsops.c b/sys/fs/nfsclient/nfs_clvfsops.c
index d6db84f..b2d4864 100644
--- a/sys/fs/nfsclient/nfs_clvfsops.c
+++ b/sys/fs/nfsclient/nfs_clvfsops.c
@@ -452,10 +452,12 @@ ncl_mountroot(struct mount *mp)
 		sin.sin_family = AF_INET;
 		sin.sin_len = sizeof(sin);
                 /* XXX MRT use table 0 for this sort of thing */
+		CURVNET_SET(TD_TO_VNET(td));
 		error = rtrequest(RTM_ADD, (struct sockaddr *)&sin,
 		    (struct sockaddr *)&nd->mygateway,
 		    (struct sockaddr *)&mask,
 		    RTF_UP | RTF_GATEWAY, NULL);
+		CURVNET_RESTORE();
 		if (error)
 			panic("nfs_mountroot: RTM_ADD: %d", error);
 	}
diff --git a/sys/fs/portalfs/portal_vnops.c b/sys/fs/portalfs/portal_vnops.c
index 64e5afd..6fcc1ce 100644
--- a/sys/fs/portalfs/portal_vnops.c
+++ b/sys/fs/portalfs/portal_vnops.c
@@ -60,6 +60,8 @@
 
 #include <fs/portalfs/portal.h>
 
+#include <net/vnet.h>
+
 static int portal_fileid = PORTAL_ROOTFILEID+1;
 
 static void	portal_closefd(struct thread *td, int fd);
@@ -185,8 +187,12 @@ portal_connect(so, so2)
 	if ((so2->so_options & SO_ACCEPTCONN) == 0)
 		return (ECONNREFUSED);
 
-	if ((so3 = sonewconn(so2, 0)) == 0)
+	CURVNET_SET(so2->so_vnet);
+	if ((so3 = sonewconn(so2, 0)) == 0) {
+		CURVNET_RESTORE();
 		return (ECONNREFUSED);
+	}
+	CURVNET_RESTORE();
 
 	unp2 = sotounpcb(so2);
 	unp3 = sotounpcb(so3);
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index d59bcb4..bca83b5 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -78,6 +78,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/ktrace.h>
 #endif
 
+#include <net/vnet.h>
+
 #include <security/audit/audit.h>
 
 #include <vm/uma.h>
@@ -2330,7 +2332,9 @@ fputsock(struct socket *so)
 
 	ACCEPT_LOCK();
 	SOCK_LOCK(so);
+	CURVNET_SET(so->so_vnet);
 	sorele(so);
+	CURVNET_RESTORE();
 }
 
 /*
diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index 2766ca4..c9b0534 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -80,9 +80,7 @@ soo_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
 	if (error)
 		return (error);
 #endif
-	CURVNET_SET(so->so_vnet);
 	error = soreceive(so, 0, uio, 0, 0, 0);
-	CURVNET_RESTORE();
 	return (error);
 }
 
@@ -123,7 +121,6 @@ soo_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
 	struct socket *so = fp->f_data;
 	int error = 0;
 
-	CURVNET_SET(so->so_vnet);
 	switch (cmd) {
 	case FIONBIO:
 		SOCK_LOCK(so);
@@ -210,14 +207,18 @@ soo_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
 		 */
 		if (IOCGROUP(cmd) == 'i')
 			error = ifioctl(so, cmd, data, td);
-		else if (IOCGROUP(cmd) == 'r')
+		else if (IOCGROUP(cmd) == 'r') {
+			CURVNET_SET(so->so_vnet);
 			error = rtioctl_fib(cmd, data, so->so_fibnum);
-		else
+			CURVNET_RESTORE();
+		} else {
+			CURVNET_SET(so->so_vnet);
 			error = ((*so->so_proto->pr_usrreqs->pru_control)
 			    (so, cmd, data, 0, td));
+			CURVNET_RESTORE();
+		}
 		break;
 	}
-	CURVNET_RESTORE();
 	return (error);
 }
 
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 738bef9..54a050f 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -92,6 +92,12 @@
  * from a listen queue to a file descriptor, in order to prevent garbage
  * collection of the socket at an untimely moment.  For a number of reasons,
  * these interfaces are not preferred, and should be avoided.
+ * 
+ * NOTE: With regard to VNETs the general rule is that callers do not set
+ * curvnet. Exceptions to this rule include soabort(), sodisconnect(),
+ * sofree() (and with that sorele(), sotryfree()), as well as sonewconn()
+ * and sorflush(), which are usually called from a pre-set VNET context.
+ * sopoll() currently does not need a VNET context to be set.
  */
 
 #include <sys/cdefs.h>
@@ -175,6 +181,10 @@ int	maxsockets;
 MALLOC_DEFINE(M_SONAME, "soname", "socket name");
 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block");
 
+#define	VNET_SO_ASSERT(so)						\
+	VNET_ASSERT(curvnet != NULL,					\
+	    ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so)));
+
 static int somaxconn = SOMAXCONN;
 static int sysctl_somaxconn(SYSCTL_HANDLER_ARGS);
 /* XXX: we dont have SYSCTL_USHORT */
@@ -295,6 +305,8 @@ soalloc(struct vnet *vnet)
 	so->so_gencnt = ++so_gencnt;
 	++numopensockets;
 #ifdef VIMAGE
+	VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p",
+	    __func__, __LINE__, so));
 	vnet->vnet_sockcnt++;
 	so->so_vnet = vnet;
 #endif
@@ -318,6 +330,8 @@ sodealloc(struct socket *so)
 	so->so_gencnt = ++so_gencnt;
 	--numopensockets;	/* Could be below, but faster here. */
 #ifdef VIMAGE
+	VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p",
+	    __func__, __LINE__, so));
 	so->so_vnet->vnet_sockcnt--;
 #endif
 	mtx_unlock(&so_global_mtx);
@@ -457,6 +471,7 @@ sonewconn(struct socket *head, int connstatus)
 #endif
 	knlist_init_mtx(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv));
 	knlist_init_mtx(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd));
+	VNET_SO_ASSERT(head);
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) ||
 	    (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
 		sodealloc(so);
@@ -531,8 +546,12 @@ sobind(struct socket *so, struct sockaddr *nam, struct thread *td)
 int
 solisten(struct socket *so, int backlog, struct thread *td)
 {
+	int error;
 
-	return ((*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td));
+	CURVNET_SET(so->so_vnet);
+	error = (*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td);
+	CURVNET_RESTORE();
+	return error;
 }
 
 int
@@ -620,6 +639,7 @@ sofree(struct socket *so)
 	SOCK_UNLOCK(so);
 	ACCEPT_UNLOCK();
 
+	VNET_SO_ASSERT(so);
 	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
 		(*pr->pr_domain->dom_dispose)(so->so_rcv.sb_mb);
 	if (pr->pr_usrreqs->pru_detach != NULL)
@@ -749,6 +769,7 @@ soabort(struct socket *so)
 	KASSERT(so->so_state & SS_NOFDREF, ("soabort: !SS_NOFDREF"));
 	KASSERT((so->so_state & SQ_COMP) == 0, ("soabort: SQ_COMP"));
 	KASSERT((so->so_state & SQ_INCOMP) == 0, ("soabort: SQ_INCOMP"));
+	VNET_SO_ASSERT(so);
 
 	if (so->so_proto->pr_usrreqs->pru_abort != NULL)
 		(*so->so_proto->pr_usrreqs->pru_abort)(so);
@@ -766,7 +787,10 @@ soaccept(struct socket *so, struct sockaddr **nam)
 	KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF"));
 	so->so_state &= ~SS_NOFDREF;
 	SOCK_UNLOCK(so);
+
+	CURVNET_SET(so->so_vnet);
 	error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam);
+	CURVNET_RESTORE();
 	return (error);
 }
 
@@ -804,8 +828,12 @@ soconnect(struct socket *so, struct sockaddr *nam, struct thread *td)
 int
 soconnect2(struct socket *so1, struct socket *so2)
 {
+	int error;
 
-	return ((*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2));
+	CURVNET_SET(so1->so_vnet);
+	error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2);
+	CURVNET_RESTORE();
+	return (error);
 }
 
 int
@@ -817,6 +845,7 @@ sodisconnect(struct socket *so)
 		return (ENOTCONN);
 	if (so->so_state & SS_ISDISCONNECTING)
 		return (EALREADY);
+	VNET_SO_ASSERT(so);
 	error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so);
 	return (error);
 }
@@ -1080,6 +1109,7 @@ sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio,
 	 * there are probably other places that this also happens.  We must
 	 * rethink this.
 	 */
+	VNET_SO_ASSERT(so);
 	error = (*so->so_proto->pr_usrreqs->pru_send)(so,
 	    (flags & MSG_OOB) ? PRUS_OOB :
 	/*
@@ -1267,6 +1297,7 @@ restart:
 			 * places that this also happens.  We must rethink
 			 * this.
 			 */
+			VNET_SO_ASSERT(so);
 			error = (*so->so_proto->pr_usrreqs->pru_send)(so,
 			    (flags & MSG_OOB) ? PRUS_OOB :
 			/*
@@ -1333,6 +1364,7 @@ soreceive_rcvoob(struct socket *so, struct uio *uio, int flags)
 	int error;
 
 	KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0"));
+	VNET_SO_ASSERT(so);
 
 	m = m_get(M_WAIT, MT_DATA);
 	error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK);
@@ -1441,8 +1473,10 @@ soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio,
 	if (mp != NULL)
 		*mp = NULL;
 	if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING)
-	    && uio->uio_resid)
+	    && uio->uio_resid) {
+		VNET_SO_ASSERT(so);
 		(*pr->pr_usrreqs->pru_rcvd)(so, 0);
+	}
 
 	error = sblock(&so->so_rcv, SBLOCKWAIT(flags));
 	if (error)
@@ -1589,6 +1623,7 @@ dontblock:
 			cm->m_next = NULL;
 			if (pr->pr_domain->dom_externalize != NULL) {
 				SOCKBUF_UNLOCK(&so->so_rcv);
+				VNET_SO_ASSERT(so);
 				error = (*pr->pr_domain->dom_externalize)
 				    (cm, controlp);
 				SOCKBUF_LOCK(&so->so_rcv);
@@ -1804,6 +1839,7 @@ dontblock:
 			 */
 			if (pr->pr_flags & PR_WANTRCVD) {
 				SOCKBUF_UNLOCK(&so->so_rcv);
+				VNET_SO_ASSERT(so);
 				(*pr->pr_usrreqs->pru_rcvd)(so, flags);
 				SOCKBUF_LOCK(&so->so_rcv);
 			}
@@ -1850,6 +1886,7 @@ dontblock:
 		if (!(flags & MSG_SOCALLBCK) &&
 		    (pr->pr_flags & PR_WANTRCVD)) {
 			SOCKBUF_UNLOCK(&so->so_rcv);
+			VNET_SO_ASSERT(so);
 			(*pr->pr_usrreqs->pru_rcvd)(so, flags);
 			SOCKBUF_LOCK(&so->so_rcv);
 		}
@@ -2046,6 +2083,7 @@ deliver:
 		    (((flags & MSG_WAITALL) && uio->uio_resid > 0) ||
 		     !(flags & MSG_SOCALLBCK))) {
 			SOCKBUF_UNLOCK(sb);
+			VNET_SO_ASSERT(so);
 			(*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags);
 			SOCKBUF_LOCK(sb);
 		}
@@ -2256,9 +2294,13 @@ int
 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
     struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
 {
+	int error;
 
-	return (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, mp0,
+	CURVNET_SET(so->so_vnet);
+	error = (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, mp0,
 	    controlp, flagsp));
+	CURVNET_RESTORE();
+	return (error);
 }
 
 int
@@ -2269,17 +2311,19 @@ soshutdown(struct socket *so, int how)
 
 	if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR))
 		return (EINVAL);
+
+	CURVNET_SET(so->so_vnet);
 	if (pr->pr_usrreqs->pru_flush != NULL) {
 	        (*pr->pr_usrreqs->pru_flush)(so, how);
 	}
 	if (how != SHUT_WR)
 		sorflush(so);
 	if (how != SHUT_RD) {
-		CURVNET_SET(so->so_vnet);
 		error = (*pr->pr_usrreqs->pru_shutdown)(so);
 		CURVNET_RESTORE();
 		return (error);
 	}
+	CURVNET_RESTORE();
 	return (0);
 }
 
@@ -2290,6 +2334,8 @@ sorflush(struct socket *so)
 	struct protosw *pr = so->so_proto;
 	struct sockbuf asb;
 
+	VNET_SO_ASSERT(so);
+
 	/*
 	 * In order to avoid calling dom_dispose with the socket buffer mutex
 	 * held, and in order to generally avoid holding the lock for a long
@@ -2303,7 +2349,6 @@ sorflush(struct socket *so)
 	 * socket buffer.  Don't let our acquire be interrupted by a signal
 	 * despite any existing socket disposition on interruptable waiting.
 	 */
-	CURVNET_SET(so->so_vnet);
 	socantrcvmore(so);
 	(void) sblock(sb, SBL_WAIT | SBL_NOINTR);
 
@@ -2327,7 +2372,6 @@ sorflush(struct socket *so)
 	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
 		(*pr->pr_domain->dom_dispose)(asb.sb_mb);
 	sbrelease_internal(&asb, so);
-	CURVNET_RESTORE();
 }
 
 /*
@@ -2392,11 +2436,14 @@ sosetopt(struct socket *so, struct sockopt *sopt)
 	struct mac extmac;
 #endif
 
+	CURVNET_SET(so->so_vnet);
 	error = 0;
 	if (sopt->sopt_level != SOL_SOCKET) {
-		if (so->so_proto && so->so_proto->pr_ctloutput)
-			return ((*so->so_proto->pr_ctloutput)
-				  (so, sopt));
+		if (so->so_proto && so->so_proto->pr_ctloutput) {
+			error = (*so->so_proto->pr_ctloutput)(so, sopt);
+			CURVNET_RESTORE();
+			return (error);
+		}
 		error = ENOPROTOOPT;
 	} else {
 		switch (sopt->sopt_name) {
@@ -2592,6 +2639,7 @@ sosetopt(struct socket *so, struct sockopt *sopt)
 		}
 	}
 bad:
+	CURVNET_RESTORE();
 	return (error);
 }
 
@@ -2635,13 +2683,15 @@ sogetopt(struct socket *so, struct sockopt *sopt)
 	struct mac extmac;
 #endif
 
+	CURVNET_SET(so->so_vnet);
 	error = 0;
 	if (sopt->sopt_level != SOL_SOCKET) {
-		if (so->so_proto && so->so_proto->pr_ctloutput) {
-			return ((*so->so_proto->pr_ctloutput)
-				  (so, sopt));
-		} else
-			return (ENOPROTOOPT);
+		if (so->so_proto && so->so_proto->pr_ctloutput)
+			error = (*so->so_proto->pr_ctloutput)(so, sopt);
+		else
+			error = ENOPROTOOPT;
+		CURVNET_RESTORE();
+		return (error);
 	} else {
 		switch (sopt->sopt_name) {
 #ifdef INET
@@ -2725,11 +2775,11 @@ integer:
 			error = sooptcopyin(sopt, &extmac, sizeof(extmac),
 			    sizeof(extmac));
 			if (error)
-				return (error);
+				goto bad;
 			error = mac_getsockopt_label(sopt->sopt_td->td_ucred,
 			    so, &extmac);
 			if (error)
-				return (error);
+				goto bad;
 			error = sooptcopyout(sopt, &extmac, sizeof extmac);
 #else
 			error = EOPNOTSUPP;
@@ -2741,11 +2791,11 @@ integer:
 			error = sooptcopyin(sopt, &extmac, sizeof(extmac),
 			    sizeof(extmac));
 			if (error)
-				return (error);
+				goto bad;
 			error = mac_getsockopt_peerlabel(
 			    sopt->sopt_td->td_ucred, so, &extmac);
 			if (error)
-				return (error);
+				goto bad;
 			error = sooptcopyout(sopt, &extmac, sizeof extmac);
 #else
 			error = EOPNOTSUPP;
@@ -2768,8 +2818,12 @@ integer:
 			error = ENOPROTOOPT;
 			break;
 		}
-		return (error);
 	}
+#ifdef MAC
+bad:
+#endif
+	CURVNET_RESTORE();
+	return (error);
 }
 
 /* XXX; prepare mbuf for (__FreeBSD__ < 3) routines. */
@@ -2903,6 +2957,10 @@ sopoll(struct socket *so, int events, struct ucred *active_cred,
     struct thread *td)
 {
 
+	/*
+	 * We do not need to set or assert curvnet as long as everyone uses
+	 * sopoll_generic().
+	 */
 	return (so->so_proto->pr_usrreqs->pru_sopoll(so, events, active_cred,
 	    td));
 }
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 958c807..d3326d4 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -260,14 +260,9 @@ listen(td, uap)
 		so = fp->f_data;
 #ifdef MAC
 		error = mac_socket_check_listen(td->td_ucred, so);
-		if (error == 0) {
+		if (error == 0)
 #endif
-			CURVNET_SET(so->so_vnet);
 			error = solisten(so, uap->backlog, td);
-			CURVNET_RESTORE();
-#ifdef MAC
-		}
-#endif
 		fdrop(fp, td);
 	}
 	return(error);
@@ -428,9 +423,7 @@ kern_accept(struct thread *td, int s, struct sockaddr **name,
 	tmp = fflag & FASYNC;
 	(void) fo_ioctl(nfp, FIOASYNC, &tmp, td->td_ucred, td);
 	sa = 0;
-	CURVNET_SET(so->so_vnet);
 	error = soaccept(so, &sa);
-	CURVNET_RESTORE();
 	if (error) {
 		/*
 		 * return a namelen of zero for older code which might
@@ -981,11 +974,9 @@ kern_recvit(td, s, mp, fromseg, controlp)
 		ktruio = cloneuio(&auio);
 #endif
 	len = auio.uio_resid;
-	CURVNET_SET(so->so_vnet);
 	error = soreceive(so, &fromsa, &auio, (struct mbuf **)0,
 	    (mp->msg_control || controlp) ? &control : (struct mbuf **)0,
 	    &mp->msg_flags);
-	CURVNET_RESTORE();
 	if (error) {
 		if (auio.uio_resid != (int)len && (error == ERESTART ||
 		    error == EINTR || error == EWOULDBLOCK))
@@ -1331,9 +1322,7 @@ kern_setsockopt(td, s, level, name, val, valseg, valsize)
 	error = getsock(td->td_proc->p_fd, s, &fp, NULL);
 	if (error == 0) {
 		so = fp->f_data;
-		CURVNET_SET(so->so_vnet);
 		error = sosetopt(so, &sopt);
-		CURVNET_RESTORE();
 		fdrop(fp, td);
 	}
 	return(error);
@@ -1412,9 +1401,7 @@ kern_getsockopt(td, s, level, name, val, valseg, valsize)
 	error = getsock(td->td_proc->p_fd, s, &fp, NULL);
 	if (error == 0) {
 		so = fp->f_data;
-		CURVNET_SET(so->so_vnet);
 		error = sogetopt(so, &sopt);
-		CURVNET_RESTORE();
 		*valsize = sopt.sopt_valsize;
 		fdrop(fp, td);
 	}
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index d049688..4bad386 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -1311,7 +1311,9 @@ unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	}
 	if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
 		if (so2->so_options & SO_ACCEPTCONN) {
+			CURVNET_SET(so2->so_vnet);
 			so3 = sonewconn(so2, 0);
+			CURVNET_RESTORE();
 		} else
 			so3 = NULL;
 		if (so3 == NULL) {
@@ -2215,8 +2217,14 @@ unp_gc(__unused void *arg, int pending)
 	 * struct files associated with these sockets but leave each socket
 	 * with one remaining ref.
 	 */
-	for (i = 0; i < total; i++)
-		sorflush(unref[i]->f_data);
+	for (i = 0; i < total; i++) {
+		struct socket *so;
+
+		so = unref[i]->f_data;
+		CURVNET_SET(so->so_vnet);
+		sorflush(so);
+		CURVNET_RESTORE();
+	}
 
 	/*
 	 * And finally release the sockets so they can be reclaimed.
diff --git a/sys/net/if.c b/sys/net/if.c
index ccd4689..2e21ae9 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -2440,10 +2440,13 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 	int error;
 	int oif_flags;
 
+	CURVNET_SET(so->so_vnet);
 	switch (cmd) {
 	case SIOCGIFCONF:
 	case OSIOCGIFCONF:
-		return (ifconf(cmd, data));
+		error = ifconf(cmd, data);
+		CURVNET_RESTORE();
+		return (error);
 
 #ifdef COMPAT_FREEBSD32
 	case SIOCGIFCONF32:
@@ -2455,7 +2458,9 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 			ifc.ifc_len = ifc32->ifc_len;
 			ifc.ifc_buf = PTRIN(ifc32->ifc_buf);
 
-			return (ifconf(SIOCGIFCONF, (void *)&ifc));
+			error = ifconf(SIOCGIFCONF, (void *)&ifc);
+			CURVNET_RESTORE();
+			return (error);
 		}
 #endif
 	}
@@ -2465,42 +2470,55 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 #ifdef VIMAGE
 	case SIOCSIFRVNET:
 		error = priv_check(td, PRIV_NET_SETIFVNET);
-		if (error)
-			return (error);
-		return (if_vmove_reclaim(td, ifr->ifr_name, ifr->ifr_jid));
+		if (error == 0)
+			error = if_vmove_reclaim(td, ifr->ifr_name,
+			    ifr->ifr_jid);
+		CURVNET_RESTORE();
+		return (error);
 #endif
 	case SIOCIFCREATE:
 	case SIOCIFCREATE2:
 		error = priv_check(td, PRIV_NET_IFCREATE);
-		if (error)
-			return (error);
-		return (if_clone_create(ifr->ifr_name, sizeof(ifr->ifr_name),
-			cmd == SIOCIFCREATE2 ? ifr->ifr_data : NULL));
+		if (error == 0)
+			error = if_clone_create(ifr->ifr_name,
+			    sizeof(ifr->ifr_name),
+			    cmd == SIOCIFCREATE2 ? ifr->ifr_data : NULL);
+		CURVNET_RESTORE();
+		return (error);
 	case SIOCIFDESTROY:
 		error = priv_check(td, PRIV_NET_IFDESTROY);
-		if (error)
-			return (error);
-		return if_clone_destroy(ifr->ifr_name);
+		if (error == 0)
+			error = if_clone_destroy(ifr->ifr_name);
+		CURVNET_RESTORE();
+		return (error);
 
 	case SIOCIFGCLONERS:
-		return (if_clone_list((struct if_clonereq *)data));
+		error = if_clone_list((struct if_clonereq *)data);
+		CURVNET_RESTORE();
+		return (error);
 	case SIOCGIFGMEMB:
-		return (if_getgroupmembers((struct ifgroupreq *)data));
+		error = if_getgroupmembers((struct ifgroupreq *)data);
+		CURVNET_RESTORE();
+		return (error);
 	}
 
 	ifp = ifunit_ref(ifr->ifr_name);
-	if (ifp == NULL)
+	if (ifp == NULL) {
+		CURVNET_RESTORE();
 		return (ENXIO);
+	}
 
 	error = ifhwioctl(cmd, ifp, data, td);
 	if (error != ENOIOCTL) {
 		if_rele(ifp);
+		CURVNET_RESTORE();
 		return (error);
 	}
 
 	oif_flags = ifp->if_flags;
 	if (so->so_proto == NULL) {
 		if_rele(ifp);
+		CURVNET_RESTORE();
 		return (EOPNOTSUPP);
 	}
 #ifndef COMPAT_43
@@ -2575,6 +2593,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 #endif
 	}
 	if_rele(ifp);
+	CURVNET_RESTORE();
 	return (error);
 }
 
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c b/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
index 0ac24e5..efca5389 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
@@ -50,6 +50,9 @@
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+
+#include <net/vnet.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
@@ -592,8 +595,11 @@ ng_btsocket_l2cap_process_l2ca_con_ind(struct ng_mesg *msg,
 		 * space then create new socket and set proper source address.
 		 */
 
-		if (pcb->so->so_qlen <= pcb->so->so_qlimit)
+		if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
+			CURVNET_SET(pcb->so->so_vnet);
 			so1 = sonewconn(pcb->so, 0);
+			CURVNET_RESTORE();
+		}
 
 		if (so1 == NULL) {
 			result = NG_L2CAP_NO_RESOURCES;
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c b/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
index 1fd9eb3..f07da3a 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
@@ -52,6 +52,9 @@
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 #include <sys/uio.h>
+
+#include <net/vnet.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
@@ -1163,8 +1166,11 @@ ng_btsocket_rfcomm_connect_ind(ng_btsocket_rfcomm_session_p s, int channel)
 
 	mtx_lock(&pcb->pcb_mtx);
 
-	if (pcb->so->so_qlen <= pcb->so->so_qlimit)
+	if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
+		CURVNET_SET(pcb->so->so_vnet);
 		so1 = sonewconn(pcb->so, 0);
+		CURVNET_RESTORE();
+	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c b/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
index e1bff57..85b68dc 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
@@ -50,6 +50,9 @@
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+
+#include <net/vnet.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
@@ -477,8 +480,11 @@ ng_btsocket_sco_process_lp_con_ind(struct ng_mesg *msg,
 		 * space then create new socket and set proper source address.
 		 */
 
-		if (pcb->so->so_qlen <= pcb->so->so_qlimit)
+		if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
+			CURVNET_SET(pcb->so->so_vnet);
 			so1 = sonewconn(pcb->so, 0);
+			CURVNET_RESTORE();
+		}
 
 		if (so1 == NULL) {
 			status = 0x0d; /* Rejected due to limited resources */
diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c
index 39c9aa7..1329c40 100644
--- a/sys/netinet/sctp_input.c
+++ b/sys/netinet/sctp_input.c
@@ -2675,8 +2675,10 @@ sctp_handle_cookie_echo(struct mbuf *m, int iphlen, int offset,
 			oso = (*inp_p)->sctp_socket;
 			atomic_add_int(&(*stcb)->asoc.refcnt, 1);
 			SCTP_TCB_UNLOCK((*stcb));
+			CURVNET_SET(oso->so_vnet);
 			so = sonewconn(oso, 0
 			    );
+			CURVNET_RESTORE();
 			SCTP_TCB_LOCK((*stcb));
 			atomic_subtract_int(&(*stcb)->asoc.refcnt, 1);
 
diff --git a/sys/netinet/sctp_peeloff.c b/sys/netinet/sctp_peeloff.c
index 748e23f..548966d 100644
--- a/sys/netinet/sctp_peeloff.c
+++ b/sys/netinet/sctp_peeloff.c
@@ -167,8 +167,10 @@ sctp_get_peeloff(struct socket *head, sctp_assoc_t assoc_id, int *error)
 	}
 	atomic_add_int(&stcb->asoc.refcnt, 1);
 	SCTP_TCB_UNLOCK(stcb);
+	CURVNET_SET(head->so_vnet);
 	newso = sonewconn(head, SS_ISCONNECTED
 	    );
+	CURVNET_RESTORE();
 	if (newso == NULL) {
 		SCTPDBG(SCTP_DEBUG_PEEL1, "sctp_peeloff:sonewconn failed\n");
 		SCTP_LTRACE_ERR_RET(NULL, stcb, NULL, SCTP_FROM_SCTP_PEELOFF, ENOMEM);
diff --git a/sys/nfsclient/bootp_subr.c b/sys/nfsclient/bootp_subr.c
index cff72c7..9fcd03f 100644
--- a/sys/nfsclient/bootp_subr.c
+++ b/sys/nfsclient/bootp_subr.c
@@ -584,8 +584,6 @@ bootpc_call(struct bootpc_globalcontext *gctx, struct thread *td)
 	int retry;
 	const char *s;
 
-	CURVNET_SET(TD_TO_VNET(td));
-
 	/*
 	 * Create socket and set its recieve timeout.
 	 */
@@ -962,7 +960,6 @@ gotreply:
 out:
 	soclose(so);
 out0:
-	CURVNET_RESTORE();
 	return error;
 }
 
@@ -977,8 +974,6 @@ bootpc_fakeup_interface(struct bootpc_ifcontext *ifctx,
 	struct ifaddr *ifa;
 	struct sockaddr_dl *sdl;
 
-	CURVNET_SET(TD_TO_VNET(td));
-
 	error = socreate(AF_INET, &ifctx->so, SOCK_DGRAM, 0, td->td_ucred, td);
 	if (error != 0)
 		panic("nfs_boot: socreate, error=%d", error);
@@ -1053,8 +1048,6 @@ bootpc_fakeup_interface(struct bootpc_ifcontext *ifctx,
 		      ifctx->ireq.ifr_name);
 	ifctx->sdl = sdl;
 
-	CURVNET_RESTORE();
-
 	return error;
 }
 
@@ -1630,6 +1623,7 @@ bootpc_init(void)
 	/*
 	 * Find a network interface.
 	 */
+	CURVNET_SET(TD_TO_VNET(td));
 #ifdef BOOTP_WIRED_TO
 	printf("bootpc_init: wired to interface '%s'\n",
 	       __XSTRING(BOOTP_WIRED_TO));
@@ -1676,6 +1670,7 @@ bootpc_init(void)
 		ifctx = ifctx->next;
 	}
 	IFNET_RUNLOCK();
+	CURVNET_RESTORE();
 
 	if (gctx->interfaces == NULL || gctx->interfaces->ifp == NULL) {
 #ifdef BOOTP_WIRED_TO
diff --git a/sys/nfsclient/krpc_subr.c b/sys/nfsclient/krpc_subr.c
index c2a8450..0639b6c 100644
--- a/sys/nfsclient/krpc_subr.c
+++ b/sys/nfsclient/krpc_subr.c
@@ -216,8 +216,6 @@ krpc_call(struct sockaddr_in *sa, u_int prog, u_int vers, u_int func,
 	nam = mhead = NULL;
 	from = NULL;
 
-	CURVNET_SET(TD_TO_VNET(td));
-
 	/*
 	 * Create socket and set its recieve timeout.
 	 */
@@ -430,7 +428,6 @@ krpc_call(struct sockaddr_in *sa, u_int prog, u_int vers, u_int func,
 	if (mhead) m_freem(mhead);
 	if (from) free(from, M_SONAME);
 	soclose(so);
-	CURVNET_RESTORE();
 	return error;
 }
 
diff --git a/sys/nfsclient/nfs_diskless.c b/sys/nfsclient/nfs_diskless.c
index aacecff..b574945 100644
--- a/sys/nfsclient/nfs_diskless.c
+++ b/sys/nfsclient/nfs_diskless.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/jail.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mount.h>
@@ -202,6 +203,7 @@ nfs_setup_diskless(void)
 		return;
 	}
 	ifa = NULL;
+	CURVNET_SET(TD_TO_VNET(curthread));
 	IFNET_RLOCK();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@@ -213,12 +215,14 @@ nfs_setup_diskless(void)
 					  LLADDR(&ourdl),
 					  sdl->sdl_alen)) {
 				    IFNET_RUNLOCK();
+				    CURVNET_RESTORE();
 				    goto match_done;
 				}
 			}
 		}
 	}
 	IFNET_RUNLOCK();
+	CURVNET_RESTORE();
 	printf("nfs_diskless: no interface\n");
 	return;	/* no matching interface */
 match_done:
diff --git a/sys/nfsclient/nfs_vfsops.c b/sys/nfsclient/nfs_vfsops.c
index fc54ff6..3700708 100644
--- a/sys/nfsclient/nfs_vfsops.c
+++ b/sys/nfsclient/nfs_vfsops.c
@@ -428,7 +428,6 @@ nfs_mountroot(struct mount *mp)
 	char buf[128];
 	char *cp;
 
-	CURVNET_SET(TD_TO_VNET(td));
 
 #if defined(BOOTP_NFSROOT) && defined(BOOTP)
 	bootpc_init();		/* use bootp to get nfs_diskless filled in */
@@ -437,7 +436,6 @@ nfs_mountroot(struct mount *mp)
 #endif
 
 	if (nfs_diskless_valid == 0) {
-		CURVNET_RESTORE();
 		return (-1);
 	}
 	if (nfs_diskless_valid == 1)
@@ -504,10 +502,12 @@ nfs_mountroot(struct mount *mp)
 		sin.sin_family = AF_INET;
 		sin.sin_len = sizeof(sin);
                 /* XXX MRT use table 0 for this sort of thing */
+		CURVNET_SET(TD_TO_VNET(td));
 		error = rtrequest(RTM_ADD, (struct sockaddr *)&sin,
 		    (struct sockaddr *)&nd->mygateway,
 		    (struct sockaddr *)&mask,
 		    RTF_UP | RTF_GATEWAY, NULL);
+		CURVNET_RESTORE();
 		if (error)
 			panic("nfs_mountroot: RTM_ADD: %d", error);
 	}
@@ -525,7 +525,6 @@ nfs_mountroot(struct mount *mp)
 	nd->root_args.hostname = buf;
 	if ((error = nfs_mountdiskless(buf,
 	    &nd->root_saddr, &nd->root_args, td, &vp, mp)) != 0) {
-		CURVNET_RESTORE();
 		return (error);
 	}
 
@@ -539,7 +538,6 @@ nfs_mountroot(struct mount *mp)
 	    sizeof (prison0.pr_hostname));
 	mtx_unlock(&prison0.pr_mtx);
 	inittodr(ntohl(nd->root_time));
-	CURVNET_RESTORE();
 	return (0);
 }
 
diff --git a/sys/rpc/clnt_dg.c b/sys/rpc/clnt_dg.c
index d35a6a9..58d55ff 100644
--- a/sys/rpc/clnt_dg.c
+++ b/sys/rpc/clnt_dg.c
@@ -200,14 +200,11 @@ clnt_dg_create(
 		return (NULL);
 	}
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
 		rpc_createerr.cf_stat = RPC_TLIERROR;
 		rpc_createerr.cf_error.re_errno = 0;
-		CURVNET_RESTORE();
 		return (NULL);
 	}
-	CURVNET_RESTORE();
 
 	/*
 	 * Find the receive and the send size
diff --git a/sys/rpc/clnt_vc.c b/sys/rpc/clnt_vc.c
index 9882407..9f36bba 100644
--- a/sys/rpc/clnt_vc.c
+++ b/sys/rpc/clnt_vc.c
@@ -220,9 +220,7 @@ clnt_vc_create(
 		}
 	}
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
-		CURVNET_RESTORE();
 		goto err;
 	}
 
@@ -245,7 +243,6 @@ clnt_vc_create(
 		sopt.sopt_valsize = sizeof(one);
 		sosetopt(so, &sopt);
 	}
-	CURVNET_RESTORE();
 
 	ct->ct_closeit = FALSE;
 
diff --git a/sys/rpc/rpc_generic.c b/sys/rpc/rpc_generic.c
index 1c6aa82..fd39350 100644
--- a/sys/rpc/rpc_generic.c
+++ b/sys/rpc/rpc_generic.c
@@ -187,7 +187,9 @@ __rpc_socket2sockinfo(struct socket *so, struct __rpc_sockinfo *sip)
 	struct sockopt opt;
 	int error;
 
+	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
+	CURVNET_RESTORE();
 	if (error)
 		return 0;
 
@@ -825,7 +827,6 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 	sa->sa_len = salen;
 
 	if (*portp == 0) {
-		CURVNET_SET(so->so_vnet);
 		bzero(&opt, sizeof(opt));
 		opt.sopt_dir = SOPT_GET;
 		opt.sopt_level = proto;
@@ -834,14 +835,12 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 		opt.sopt_valsize = sizeof(old);
 		error = sogetopt(so, &opt);
 		if (error) {
-			CURVNET_RESTORE();
 			goto out;
 		}
 
 		opt.sopt_dir = SOPT_SET;
 		opt.sopt_val = &portlow;
 		error = sosetopt(so, &opt);
-		CURVNET_RESTORE();
 		if (error)
 			goto out;
 	}
@@ -852,9 +851,7 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 		if (error) {
 			opt.sopt_dir = SOPT_SET;
 			opt.sopt_val = &old;
-			CURVNET_SET(so->so_vnet);
 			sosetopt(so, &opt);
-			CURVNET_RESTORE();
 		}
 	}
 out:
diff --git a/sys/rpc/svc_dg.c b/sys/rpc/svc_dg.c
index 9d7696b..adfe2b5 100644
--- a/sys/rpc/svc_dg.c
+++ b/sys/rpc/svc_dg.c
@@ -103,10 +103,8 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	struct sockaddr* sa;
 	int error;
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
 		printf(svc_dg_str, svc_dg_err1);
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 	/*
@@ -116,7 +114,6 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	recvsize = __rpc_get_t_size(si.si_af, si.si_proto, (int)recvsize);
 	if ((sendsize == 0) || (recvsize == 0)) {
 		printf(svc_dg_str, svc_dg_err2);
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 
@@ -128,6 +125,7 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt->xp_p2 = NULL;
 	xprt->xp_ops = &svc_dg_ops;
 
+	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
 	CURVNET_RESTORE();
 	if (error)
diff --git a/sys/rpc/svc_generic.c b/sys/rpc/svc_generic.c
index e6e8acd..f8b70de 100644
--- a/sys/rpc/svc_generic.c
+++ b/sys/rpc/svc_generic.c
@@ -230,14 +230,11 @@ svc_tli_create(
 		/*
 		 * It is an open socket. Get the transport info.
 		 */
-		CURVNET_SET(so->so_vnet);
 		if (!__rpc_socket2sockinfo(so, &si)) {
 			printf(
 		"svc_tli_create: could not get transport information\n");
-			CURVNET_RESTORE();
 			return (NULL);
 		}
-		CURVNET_RESTORE();
 	}
 
 	/*
@@ -264,9 +261,7 @@ svc_tli_create(
 		"svc_tli_create: could not bind to requested address\n");
 				goto freedata;
 			}
-			CURVNET_SET(so->so_vnet);
 			solisten(so, (int)bindaddr->qlen, curthread);
-			CURVNET_RESTORE();
 		}
 			
 	}
diff --git a/sys/rpc/svc_vc.c b/sys/rpc/svc_vc.c
index 999cad1..fddcf58 100644
--- a/sys/rpc/svc_vc.c
+++ b/sys/rpc/svc_vc.c
@@ -154,10 +154,8 @@ svc_vc_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt->xp_p2 = NULL;
 	xprt->xp_ops = &svc_vc_rendezvous_ops;
 
-	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
 	if (error) {
-		CURVNET_RESTORE();
 		goto cleanup_svc_vc_create;
 	}
 
@@ -167,7 +165,6 @@ svc_vc_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt_register(xprt);
 
 	solisten(so, SOMAXCONN, curthread);
-	CURVNET_RESTORE();
 
 	SOCKBUF_LOCK(&so->so_rcv);
 	xprt->xp_upcallset = 1;
@@ -200,10 +197,8 @@ svc_vc_create_conn(SVCPOOL *pool, struct socket *so, struct sockaddr *raddr)
 	opt.sopt_name = SO_KEEPALIVE;
 	opt.sopt_val = &one;
 	opt.sopt_valsize = sizeof(one);
-	CURVNET_SET(so->so_vnet);
 	error = sosetopt(so, &opt);
 	if (error) {
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 
@@ -216,11 +211,9 @@ svc_vc_create_conn(SVCPOOL *pool, struct socket *so, struct sockaddr *raddr)
 		opt.sopt_valsize = sizeof(one);
 		error = sosetopt(so, &opt);
 		if (error) {
-			CURVNET_RESTORE();
 			return (NULL);
 		}
 	}
-	CURVNET_RESTORE();
 
 	cd = mem_alloc(sizeof(*cd));
 	cd->strm_stat = XPRT_IDLE;
@@ -635,10 +628,8 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
 		uio.uio_td = curthread;
 		m = NULL;
 		rcvflag = MSG_DONTWAIT;
-		CURVNET_SET(xprt->xp_socket->so_vnet);
 		error = soreceive(xprt->xp_socket, NULL, &uio, &m, NULL,
 		    &rcvflag);
-		CURVNET_RESTORE();
 
 		if (error == EWOULDBLOCK) {
 			/*