summaryrefslogtreecommitdiffstats
path: root/sys/kern
diff options
context:
space:
mode:
Diffstat (limited to 'sys/kern')
-rw-r--r--sys/kern/init_main.c4
-rw-r--r--sys/kern/kern_fork.c6
-rw-r--r--sys/kern/kern_linker.c9
-rw-r--r--sys/kern/kern_vimage.c58
-rw-r--r--sys/kern/subr_pcpu.c4
-rw-r--r--sys/kern/sys_socket.c10
-rw-r--r--sys/kern/uipc_socket.c44
-rw-r--r--sys/kern/uipc_syscalls.c17
-rw-r--r--sys/kern/uipc_usrreq.c5
9 files changed, 142 insertions, 15 deletions
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index a8abad8..96ebf36 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -74,6 +74,7 @@ __FBSDID("$FreeBSD$");
#include <sys/malloc.h>
#include <sys/conf.h>
#include <sys/cpuset.h>
+#include <sys/vimage.h>
#include <machine/cpu.h>
@@ -452,6 +453,9 @@ proc0_init(void *dummy __unused)
p->p_ucred->cr_uidinfo = uifind(0);
p->p_ucred->cr_ruidinfo = uifind(0);
p->p_ucred->cr_prison = NULL; /* Don't jail it. */
+#ifdef VIMAGE
+ p->p_ucred->cr_vnet = LIST_FIRST(&vnet_head);
+#endif
#ifdef AUDIT
audit_cred_kproc0(p->p_ucred);
#endif
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index e7e98b1..76695e7 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -68,6 +68,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sdt.h>
#include <sys/sx.h>
#include <sys/signalvar.h>
+#include <sys/vimage.h>
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
@@ -523,6 +524,11 @@ again:
td2->td_sigmask = td->td_sigmask;
td2->td_flags = TDF_INMEM;
+#ifdef VIMAGE
+ td2->td_vnet = NULL;
+ td2->td_vnet_lpush = NULL;
+#endif
+
/*
* Duplicate sub-structures as needed.
* Increase reference counts on shared objects.
diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
index 9350713..2237107 100644
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -993,6 +993,12 @@ kern_kldload(struct thread *td, const char *file, int *fileid)
return (error);
/*
+ * It's possible that kldloaded module will attach a new ifnet,
+ * so vnet context must be set when this ocurs.
+ */
+ CURVNET_SET(TD_TO_VNET(td));
+
+ /*
* If file does not contain a qualified name or any dot in it
* (kldname.ko, or kldname.ver.ko) treat it as an interface
* name.
@@ -1019,6 +1025,7 @@ kern_kldload(struct thread *td, const char *file, int *fileid)
*fileid = lf->id;
unlock:
KLD_UNLOCK();
+ CURVNET_RESTORE();
return (error);
}
@@ -1056,6 +1063,7 @@ kern_kldunload(struct thread *td, int fileid, int flags)
if ((error = priv_check(td, PRIV_KLD_UNLOAD)) != 0)
return (error);
+ CURVNET_SET(TD_TO_VNET(td));
KLD_LOCK();
lf = linker_find_file_by_id(fileid);
if (lf) {
@@ -1092,6 +1100,7 @@ kern_kldunload(struct thread *td, int fileid, int flags)
PMC_CALL_HOOK(td, PMC_FN_KLD_UNLOAD, (void *) &pkm);
#endif
KLD_UNLOCK();
+ CURVNET_RESTORE();
return (error);
}
diff --git a/sys/kern/kern_vimage.c b/sys/kern/kern_vimage.c
index 310e328..21d502e 100644
--- a/sys/kern/kern_vimage.c
+++ b/sys/kern/kern_vimage.c
@@ -31,6 +31,8 @@
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
+#include "opt_ddb.h"
+
#include <sys/param.h>
#include <sys/types.h>
#include <sys/kernel.h>
@@ -38,6 +40,9 @@ __FBSDID("$FreeBSD$");
#include <sys/malloc.h>
#include <sys/systm.h>
#include <sys/vimage.h>
+#ifdef DDB
+#include <ddb/ddb.h>
+#endif
#ifndef VIMAGE_GLOBALS
@@ -51,8 +56,6 @@ static int vnet_mod_constructor(struct vnet_modlink *);
static int vnet_mod_destructor(struct vnet_modlink *);
#ifdef VIMAGE
-/* curvnet should be thread-local - this is only a temporary step. */
-struct vnet *curvnet;
struct vnet_list_head vnet_head;
#endif
@@ -183,7 +186,8 @@ vnet_mod_deregister_multi(const struct vnet_modinfo *vmi, void *iarg,
free(vml, M_VIMAGE);
}
-static int vnet_mod_constructor(struct vnet_modlink *vml)
+static int
+vnet_mod_constructor(struct vnet_modlink *vml)
{
const struct vnet_modinfo *vmi = vml->vml_modinfo;
@@ -303,7 +307,9 @@ vi_init(void *unused)
if (vnet == NULL)
panic("vi_alloc: malloc failed");
LIST_INSERT_HEAD(&vnet_head, vnet, vnet_le);
+ vnet->vnet_magic_n = VNET_MAGIC_N;
+ /* We MUST clear curvnet in vi_init_done before going SMP. */
curvnet = LIST_FIRST(&vnet_head);
#endif
}
@@ -313,6 +319,10 @@ vi_init_done(void *unused)
{
struct vnet_modlink *vml_iter;
+#ifdef VIMAGE
+ curvnet = NULL;
+#endif
+
if (TAILQ_EMPTY(&vnet_modpending_head))
return;
@@ -327,5 +337,45 @@ vi_init_done(void *unused)
SYSINIT(vimage, SI_SUB_VIMAGE, SI_ORDER_FIRST, vi_init, NULL);
SYSINIT(vimage_done, SI_SUB_VIMAGE_DONE, SI_ORDER_FIRST, vi_init_done, NULL);
-
#endif /* !VIMAGE_GLOBALS */
+
+#ifdef VIMAGE
+#ifdef DDB
+static void
+db_vnet_ptr(void *arg)
+{
+
+ if (arg)
+ db_printf(" %p", arg);
+ else
+#if SIZE_MAX == UINT32_MAX /* 32-bit arch */
+ db_printf(" 0");
+#else /* 64-bit arch, most probaly... */
+ db_printf(" 0");
+#endif
+}
+
+DB_SHOW_COMMAND(vnets, db_show_vnets)
+{
+ VNET_ITERATOR_DECL(vnet_iter);
+
+#if SIZE_MAX == UINT32_MAX /* 32-bit arch */
+ db_printf(" vnet ifs socks");
+ db_printf(" net inet inet6 ipsec netgraph\n");
+#else /* 64-bit arch, most probaly... */
+ db_printf(" vnet ifs socks");
+ db_printf(" net inet inet6 ipsec netgraph\n");
+#endif
+ VNET_FOREACH(vnet_iter) {
+ db_printf("%p %3d %5d",
+ vnet_iter, vnet_iter->ifccnt, vnet_iter->sockcnt);
+ db_vnet_ptr(vnet_iter->mod_data[VNET_MOD_NET]);
+ db_vnet_ptr(vnet_iter->mod_data[VNET_MOD_INET]);
+ db_vnet_ptr(vnet_iter->mod_data[VNET_MOD_INET6]);
+ db_vnet_ptr(vnet_iter->mod_data[VNET_MOD_IPSEC]);
+ db_vnet_ptr(vnet_iter->mod_data[VNET_MOD_NETGRAPH]);
+ db_printf("\n");
+ }
+}
+#endif
+#endif /* VIMAGE */
diff --git a/sys/kern/subr_pcpu.c b/sys/kern/subr_pcpu.c
index ea25aa7..74082c2 100644
--- a/sys/kern/subr_pcpu.c
+++ b/sys/kern/subr_pcpu.c
@@ -135,6 +135,10 @@ show_pcpu(struct pcpu *pc)
db_printf("none\n");
db_show_mdpcpu(pc);
+#ifdef VIMAGE
+ db_printf("curvnet = %p\n", pc->pc_curthread->td_vnet);
+#endif
+
#ifdef WITNESS
db_printf("spin locks held:\n");
witness_list_locks(&pc->pc_spinlocks);
diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index 813b1dd..61b0361 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
#include <sys/stat.h>
#include <sys/uio.h>
#include <sys/ucred.h>
+#include <sys/vimage.h>
#include <net/if.h>
#include <net/route.h>
@@ -74,16 +75,19 @@ soo_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
int flags, struct thread *td)
{
struct socket *so = fp->f_data;
-#ifdef MAC
int error;
+#ifdef MAC
SOCK_LOCK(so);
error = mac_socket_check_receive(active_cred, so);
SOCK_UNLOCK(so);
if (error)
return (error);
#endif
- return (soreceive(so, 0, uio, 0, 0, 0));
+ CURVNET_SET(so->so_vnet);
+ error = soreceive(so, 0, uio, 0, 0, 0);
+ CURVNET_RESTORE();
+ return (error);
}
/* ARGSUSED */
@@ -125,6 +129,7 @@ soo_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
struct socket *so = fp->f_data;
int error = 0;
+ CURVNET_SET(so->so_vnet);
switch (cmd) {
case FIONBIO:
SOCK_LOCK(so);
@@ -205,6 +210,7 @@ soo_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
(so, cmd, data, 0, td));
break;
}
+ CURVNET_RESTORE();
return (error);
}
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index a2f7c05..0fd3bee 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -264,7 +264,7 @@ SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL);
* soalloc() returns a socket with a ref count of 0.
*/
static struct socket *
-soalloc(void)
+soalloc(struct vnet *vnet)
{
struct socket *so;
@@ -286,7 +286,8 @@ soalloc(void)
so->so_gencnt = ++so_gencnt;
++numopensockets;
#ifdef VIMAGE
- so->so_vnet = curvnet;
+ ++vnet->sockcnt; /* locked with so_global_mtx */
+ so->so_vnet = vnet;
#endif
mtx_unlock(&so_global_mtx);
return (so);
@@ -307,6 +308,9 @@ sodealloc(struct socket *so)
mtx_lock(&so_global_mtx);
so->so_gencnt = ++so_gencnt;
--numopensockets; /* Could be below, but faster here. */
+#ifdef VIMAGE
+ --so->so_vnet->sockcnt;
+#endif
mtx_unlock(&so_global_mtx);
if (so->so_rcv.sb_hiwat)
(void)chgsbsize(so->so_cred->cr_uidinfo,
@@ -356,7 +360,7 @@ socreate(int dom, struct socket **aso, int type, int proto,
if (prp->pr_type != type)
return (EPROTOTYPE);
- so = soalloc();
+ so = soalloc(TD_TO_VNET(td));
if (so == NULL)
return (ENOBUFS);
@@ -382,7 +386,9 @@ socreate(int dom, struct socket **aso, int type, int proto,
* Auto-sizing of socket buffers is managed by the protocols and
* the appropriate flags must be set in the pru_attach function.
*/
+ CURVNET_SET(so->so_vnet);
error = (*prp->pr_usrreqs->pru_attach)(so, proto, td);
+ CURVNET_RESTORE();
if (error) {
KASSERT(so->so_count == 1, ("socreate: so_count %d",
so->so_count));
@@ -424,7 +430,8 @@ sonewconn(struct socket *head, int connstatus)
if (over)
#endif
return (NULL);
- so = soalloc();
+ VNET_ASSERT(head->so_vnet);
+ so = soalloc(head->so_vnet);
if (so == NULL)
return (NULL);
if ((head->so_options & SO_ACCEPTFILTER) != 0)
@@ -496,8 +503,12 @@ sonewconn(struct socket *head, int connstatus)
int
sobind(struct socket *so, struct sockaddr *nam, struct thread *td)
{
+ int error;
- return ((*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td));
+ CURVNET_SET(so->so_vnet);
+ error = (*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td);
+ CURVNET_RESTORE();
+ return error;
}
/*
@@ -645,6 +656,7 @@ soclose(struct socket *so)
KASSERT(!(so->so_state & SS_NOFDREF), ("soclose: SS_NOFDREF on enter"));
+ CURVNET_SET(so->so_vnet);
funsetown(&so->so_sigio);
if (so->so_state & SS_ISCONNECTED) {
if ((so->so_state & SS_ISDISCONNECTING) == 0) {
@@ -696,6 +708,7 @@ drop:
KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF"));
so->so_state |= SS_NOFDREF;
sorele(so);
+ CURVNET_RESTORE();
return (error);
}
@@ -771,7 +784,9 @@ soconnect(struct socket *so, struct sockaddr *nam, struct thread *td)
* biting us.
*/
so->so_error = 0;
+ CURVNET_SET(so->so_vnet);
error = (*so->so_proto->pr_usrreqs->pru_connect)(so, nam, td);
+ CURVNET_RESTORE();
}
return (error);
@@ -1287,9 +1302,13 @@ int
sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
{
+ int error;
- return (so->so_proto->pr_usrreqs->pru_sosend(so, addr, uio, top,
- control, flags, td));
+ CURVNET_SET(so->so_vnet);
+ error = so->so_proto->pr_usrreqs->pru_sosend(so, addr, uio, top,
+ control, flags, td);
+ CURVNET_RESTORE();
+ return (error);
}
/*
@@ -2037,6 +2056,7 @@ int
soshutdown(struct socket *so, int how)
{
struct protosw *pr = so->so_proto;
+ int error;
if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR))
return (EINVAL);
@@ -2045,8 +2065,12 @@ soshutdown(struct socket *so, int how)
}
if (how != SHUT_WR)
sorflush(so);
- if (how != SHUT_RD)
- return ((*pr->pr_usrreqs->pru_shutdown)(so));
+ if (how != SHUT_RD) {
+ CURVNET_SET(so->so_vnet);
+ error = (*pr->pr_usrreqs->pru_shutdown)(so);
+ CURVNET_RESTORE();
+ return (error);
+ }
return (0);
}
@@ -2070,6 +2094,7 @@ sorflush(struct socket *so)
* socket buffer. Don't let our acquire be interrupted by a signal
* despite any existing socket disposition on interruptable waiting.
*/
+ CURVNET_SET(so->so_vnet);
socantrcvmore(so);
(void) sblock(sb, SBL_WAIT | SBL_NOINTR);
@@ -2093,6 +2118,7 @@ sorflush(struct socket *so)
if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
(*pr->pr_domain->dom_dispose)(asb.sb_mb);
sbrelease_internal(&asb, so);
+ CURVNET_RESTORE();
}
/*
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 662b3abd..5689aeb 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sysctl.h>
#include <sys/uio.h>
#include <sys/vnode.h>
+#include <sys/vimage.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
@@ -264,7 +265,9 @@ listen(td, uap)
if (error)
goto done;
#endif
+ CURVNET_SET(so->so_vnet);
error = solisten(so, uap->backlog, td);
+ CURVNET_RESTORE();
#ifdef MAC
done:
#endif
@@ -429,7 +432,9 @@ kern_accept(struct thread *td, int s, struct sockaddr **name,
tmp = fflag & FASYNC;
(void) fo_ioctl(nfp, FIOASYNC, &tmp, td->td_ucred, td);
sa = 0;
+ CURVNET_SET(so->so_vnet);
error = soaccept(so, &sa);
+ CURVNET_RESTORE();
if (error) {
/*
* return a namelen of zero for older code which might
@@ -976,9 +981,11 @@ kern_recvit(td, s, mp, fromseg, controlp)
ktruio = cloneuio(&auio);
#endif
len = auio.uio_resid;
+ CURVNET_SET(so->so_vnet);
error = soreceive(so, &fromsa, &auio, (struct mbuf **)0,
(mp->msg_control || controlp) ? &control : (struct mbuf **)0,
&mp->msg_flags);
+ CURVNET_RESTORE();
if (error) {
if (auio.uio_resid != (int)len && (error == ERESTART ||
error == EINTR || error == EWOULDBLOCK))
@@ -1322,7 +1329,9 @@ kern_setsockopt(td, s, level, name, val, valseg, valsize)
error = getsock(td->td_proc->p_fd, s, &fp, NULL);
if (error == 0) {
so = fp->f_data;
+ CURVNET_SET(so->so_vnet);
error = sosetopt(so, &sopt);
+ CURVNET_RESTORE();
fdrop(fp, td);
}
return(error);
@@ -1400,7 +1409,9 @@ kern_getsockopt(td, s, level, name, val, valseg, valsize)
error = getsock(td->td_proc->p_fd, s, &fp, NULL);
if (error == 0) {
so = fp->f_data;
+ CURVNET_SET(so->so_vnet);
error = sogetopt(so, &sopt);
+ CURVNET_RESTORE();
*valsize = sopt.sopt_valsize;
fdrop(fp, td);
}
@@ -1463,7 +1474,9 @@ kern_getsockname(struct thread *td, int fd, struct sockaddr **sa,
return (error);
so = fp->f_data;
*sa = NULL;
+ CURVNET_SET(so->so_vnet);
error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, sa);
+ CURVNET_RESTORE();
if (error)
goto bad;
if (*sa == NULL)
@@ -1564,7 +1577,9 @@ kern_getpeername(struct thread *td, int fd, struct sockaddr **sa,
goto done;
}
*sa = NULL;
+ CURVNET_SET(so->so_vnet);
error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, sa);
+ CURVNET_RESTORE();
if (error)
goto bad;
if (*sa == NULL)
@@ -2176,9 +2191,11 @@ retry_space:
goto done;
}
SOCKBUF_UNLOCK(&so->so_snd);
+ CURVNET_SET(so->so_vnet);
/* Avoid error aliasing. */
err = (*so->so_proto->pr_usrreqs->pru_send)
(so, 0, m, NULL, NULL, td);
+ CURVNET_RESTORE();
if (err == 0) {
/*
* We need two counters to get the
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 6ddc837..2f33008 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -90,6 +90,7 @@ __FBSDID("$FreeBSD$");
#include <sys/un.h>
#include <sys/unpcb.h>
#include <sys/vnode.h>
+#include <sys/vimage.h>
#ifdef DDB
#include <ddb/ddb.h>
@@ -1647,6 +1648,10 @@ static void
unp_init(void)
{
+#ifdef VIMAGE
+ if (!IS_DEFAULT_VNET(curvnet))
+ return;
+#endif
unp_zone = uma_zcreate("unpcb", sizeof(struct unpcb), NULL, NULL,
NULL, NULL, UMA_ALIGN_PTR, 0);
if (unp_zone == NULL)
OpenPOWER on IntegriCloud