summaryrefslogtreecommitdiffstats
path: root/sys/kern
diff options
context:
space:
mode:
Diffstat (limited to 'sys/kern')
-rw-r--r--sys/kern/init_main.c1
-rw-r--r--sys/kern/kern_fork.c119
-rw-r--r--sys/kern/kern_thread.c46
-rw-r--r--sys/kern/kern_umtx.c16
-rw-r--r--sys/kern/sched_4bsd.c23
-rw-r--r--sys/kern/sched_ule.c21
-rw-r--r--sys/kern/uipc_usrreq.c81
7 files changed, 209 insertions, 98 deletions
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index b86ea91..80a0907 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -460,6 +460,7 @@ proc0_init(void *dummy __unused)
td->td_pri_class = PRI_TIMESHARE;
td->td_user_pri = PUSER;
td->td_base_user_pri = PUSER;
+ td->td_lend_user_pri = PRI_MAX;
td->td_priority = PVM;
td->td_base_pri = PUSER;
td->td_oncpu = 0;
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 126c668..61d9531 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -97,9 +97,7 @@ struct fork_args {
/* ARGSUSED */
int
-fork(td, uap)
- struct thread *td;
- struct fork_args *uap;
+fork(struct thread *td, struct fork_args *uap)
{
int error;
struct proc *p2;
@@ -135,9 +133,7 @@ vfork(td, uap)
}
int
-rfork(td, uap)
- struct thread *td;
- struct rfork_args *uap;
+rfork(struct thread *td, struct rfork_args *uap)
{
struct proc *p2;
int error;
@@ -197,12 +193,59 @@ sysctl_kern_randompid(SYSCTL_HANDLER_ARGS)
SYSCTL_PROC(_kern, OID_AUTO, randompid, CTLTYPE_INT|CTLFLAG_RW,
0, 0, sysctl_kern_randompid, "I", "Random PID modulus");
+static int
+fork_norfproc(struct thread *td, int flags, struct proc **procp)
+{
+ int error;
+ struct proc *p1;
+
+ KASSERT((flags & RFPROC) == 0,
+ ("fork_norfproc called with RFPROC set"));
+ p1 = td->td_proc;
+ *procp = NULL;
+
+ if (((p1->p_flag & (P_HADTHREADS|P_SYSTEM)) == P_HADTHREADS) &&
+ (flags & (RFCFDG | RFFDG))) {
+ PROC_LOCK(p1);
+ if (thread_single(SINGLE_BOUNDARY)) {
+ PROC_UNLOCK(p1);
+ return (ERESTART);
+ }
+ PROC_UNLOCK(p1);
+ }
+
+ error = vm_forkproc(td, NULL, NULL, NULL, flags);
+ if (error)
+ goto fail;
+
+ /*
+ * Close all file descriptors.
+ */
+ if (flags & RFCFDG) {
+ struct filedesc *fdtmp;
+ fdtmp = fdinit(td->td_proc->p_fd);
+ fdfree(td);
+ p1->p_fd = fdtmp;
+ }
+
+ /*
+ * Unshare file descriptors (from parent).
+ */
+ if (flags & RFFDG)
+ fdunshare(p1, td);
+
+fail:
+ if (((p1->p_flag & (P_HADTHREADS|P_SYSTEM)) == P_HADTHREADS) &&
+ (flags & (RFCFDG | RFFDG))) {
+ PROC_LOCK(p1);
+ thread_single_end();
+ PROC_UNLOCK(p1);
+ }
+ return (error);
+}
+
int
-fork1(td, flags, pages, procp)
- struct thread *td;
- int flags;
- int pages;
- struct proc **procp;
+fork1(struct thread *td, int flags, int pages, struct proc **procp)
{
struct proc *p1, *p2, *pptr;
struct proc *newproc;
@@ -227,47 +270,8 @@ fork1(td, flags, pages, procp)
* Here we don't create a new process, but we divorce
* certain parts of a process from itself.
*/
- if ((flags & RFPROC) == 0) {
- if (((p1->p_flag & (P_HADTHREADS|P_SYSTEM)) == P_HADTHREADS) &&
- (flags & (RFCFDG | RFFDG))) {
- PROC_LOCK(p1);
- if (thread_single(SINGLE_BOUNDARY)) {
- PROC_UNLOCK(p1);
- return (ERESTART);
- }
- PROC_UNLOCK(p1);
- }
-
- error = vm_forkproc(td, NULL, NULL, NULL, flags);
- if (error)
- goto norfproc_fail;
-
- /*
- * Close all file descriptors.
- */
- if (flags & RFCFDG) {
- struct filedesc *fdtmp;
- fdtmp = fdinit(td->td_proc->p_fd);
- fdfree(td);
- p1->p_fd = fdtmp;
- }
-
- /*
- * Unshare file descriptors (from parent).
- */
- if (flags & RFFDG)
- fdunshare(p1, td);
-
-norfproc_fail:
- if (((p1->p_flag & (P_HADTHREADS|P_SYSTEM)) == P_HADTHREADS) &&
- (flags & (RFCFDG | RFFDG))) {
- PROC_LOCK(p1);
- thread_single_end();
- PROC_UNLOCK(p1);
- }
- *procp = NULL;
- return (error);
- }
+ if ((flags & RFPROC) == 0)
+ return (fork_norfproc(td, flags, procp));
/*
* XXX
@@ -539,6 +543,7 @@ again:
td2->td_sigstk = td->td_sigstk;
td2->td_sigmask = td->td_sigmask;
td2->td_flags = TDF_INMEM;
+ td2->td_lend_user_pri = PRI_MAX;
#ifdef VIMAGE
td2->td_vnet = NULL;
@@ -798,10 +803,8 @@ fail1:
* is called from the MD fork_trampoline() entry point.
*/
void
-fork_exit(callout, arg, frame)
- void (*callout)(void *, struct trapframe *);
- void *arg;
- struct trapframe *frame;
+fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
+ struct trapframe *frame)
{
struct proc *p;
struct thread *td;
@@ -855,9 +858,7 @@ fork_exit(callout, arg, frame)
* first parameter and is called when returning to a new userland process.
*/
void
-fork_return(td, frame)
- struct thread *td;
- struct trapframe *frame;
+fork_return(struct thread *td, struct trapframe *frame)
{
userret(td, frame);
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index 5f07590..7161a99 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -81,15 +81,54 @@ MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN);
static void thread_zombie(struct thread *);
+#define TID_BUFFER_SIZE 1024
+
struct mtx tid_lock;
static struct unrhdr *tid_unrhdr;
-
+static lwpid_t tid_buffer[TID_BUFFER_SIZE];
+static int tid_head, tid_tail;
static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash");
struct tidhashhead *tidhashtbl;
u_long tidhash;
struct rwlock tidhash_lock;
+static lwpid_t
+tid_alloc(void)
+{
+ lwpid_t tid;
+
+ tid = alloc_unr(tid_unrhdr);
+ if (tid != -1)
+ return (tid);
+ mtx_lock(&tid_lock);
+ if (tid_head == tid_tail) {
+ mtx_unlock(&tid_lock);
+ return (-1);
+ }
+ tid = tid_buffer[tid_head++];
+ tid_head %= TID_BUFFER_SIZE;
+ mtx_unlock(&tid_lock);
+ return (tid);
+}
+
+static void
+tid_free(lwpid_t tid)
+{
+ lwpid_t tmp_tid = -1;
+
+ mtx_lock(&tid_lock);
+ if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) {
+ tmp_tid = tid_buffer[tid_head++];
+ tid_head = (tid_head + 1) % TID_BUFFER_SIZE;
+ }
+ tid_buffer[tid_tail++] = tid;
+ tid_tail %= TID_BUFFER_SIZE;
+ mtx_unlock(&tid_lock);
+ if (tmp_tid != -1)
+ free_unr(tid_unrhdr, tmp_tid);
+}
+
/*
* Prepare a thread for use.
*/
@@ -102,7 +141,7 @@ thread_ctor(void *mem, int size, void *arg, int flags)
td->td_state = TDS_INACTIVE;
td->td_oncpu = NOCPU;
- td->td_tid = alloc_unr(tid_unrhdr);
+ td->td_tid = tid_alloc();
/*
* Note that td_critnest begins life as 1 because the thread is not
@@ -110,6 +149,7 @@ thread_ctor(void *mem, int size, void *arg, int flags)
* end of a context switch.
*/
td->td_critnest = 1;
+ td->td_lend_user_pri = PRI_MAX;
EVENTHANDLER_INVOKE(thread_ctor, td);
#ifdef AUDIT
audit_thread_alloc(td);
@@ -155,7 +195,7 @@ thread_dtor(void *mem, int size, void *arg)
osd_thread_exit(td);
EVENTHANDLER_INVOKE(thread_dtor, td);
- free_unr(tid_unrhdr, td->td_tid);
+ tid_free(td->td_tid);
}
/*
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index 43570ce..e7b9b32 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -1407,17 +1407,19 @@ umtx_propagate_priority(struct thread *td)
for (;;) {
td = pi->pi_owner;
- if (td == NULL)
+ if (td == NULL || td == curthread)
return;
MPASS(td->td_proc != NULL);
MPASS(td->td_proc->p_magic == P_MAGIC);
- if (UPRI(td) <= pri)
- return;
-
thread_lock(td);
- sched_lend_user_prio(td, pri);
+ if (td->td_lend_user_pri > pri)
+ sched_lend_user_prio(td, pri);
+ else {
+ thread_unlock(td);
+ break;
+ }
thread_unlock(td);
/*
@@ -3587,8 +3589,8 @@ umtx_thread_cleanup(struct thread *td)
pi->pi_owner = NULL;
TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
}
+ mtx_unlock_spin(&umtx_lock);
thread_lock(td);
- td->td_flags &= ~TDF_UBORROWING;
+ sched_unlend_user_prio(td, PRI_MAX);
thread_unlock(td);
- mtx_unlock_spin(&umtx_lock);
}
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index 9face64..6278126 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -879,25 +879,23 @@ sched_prio(struct thread *td, u_char prio)
void
sched_user_prio(struct thread *td, u_char prio)
{
- u_char oldprio;
THREAD_LOCK_ASSERT(td, MA_OWNED);
td->td_base_user_pri = prio;
- if (td->td_flags & TDF_UBORROWING && td->td_user_pri <= prio)
+ if (td->td_lend_user_pri <= prio)
return;
- oldprio = td->td_user_pri;
td->td_user_pri = prio;
}
void
sched_lend_user_prio(struct thread *td, u_char prio)
{
- u_char oldprio;
THREAD_LOCK_ASSERT(td, MA_OWNED);
- td->td_flags |= TDF_UBORROWING;
- oldprio = td->td_user_pri;
- td->td_user_pri = prio;
+ if (prio < td->td_lend_user_pri)
+ td->td_lend_user_pri = prio;
+ if (prio < td->td_user_pri)
+ td->td_user_pri = prio;
}
void
@@ -907,12 +905,11 @@ sched_unlend_user_prio(struct thread *td, u_char prio)
THREAD_LOCK_ASSERT(td, MA_OWNED);
base_pri = td->td_base_user_pri;
- if (prio >= base_pri) {
- td->td_flags &= ~TDF_UBORROWING;
- sched_user_prio(td, base_pri);
- } else {
- sched_lend_user_prio(td, prio);
- }
+ td->td_lend_user_pri = prio;
+ if (prio > base_pri)
+ td->td_user_pri = base_pri;
+ else
+ td->td_user_pri = prio;
}
void
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index 030c98d..fb30fdb 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -1677,8 +1677,8 @@ sched_user_prio(struct thread *td, u_char prio)
{
td->td_base_user_pri = prio;
- if (td->td_flags & TDF_UBORROWING && td->td_user_pri <= prio)
- return;
+ if (td->td_lend_user_pri <= prio)
+ return;
td->td_user_pri = prio;
}
@@ -1687,8 +1687,10 @@ sched_lend_user_prio(struct thread *td, u_char prio)
{
THREAD_LOCK_ASSERT(td, MA_OWNED);
- td->td_flags |= TDF_UBORROWING;
- td->td_user_pri = prio;
+ if (prio < td->td_lend_user_pri)
+ td->td_lend_user_pri = prio;
+ if (prio < td->td_user_pri)
+ td->td_user_pri = prio;
}
void
@@ -1698,12 +1700,11 @@ sched_unlend_user_prio(struct thread *td, u_char prio)
THREAD_LOCK_ASSERT(td, MA_OWNED);
base_pri = td->td_base_user_pri;
- if (prio >= base_pri) {
- td->td_flags &= ~TDF_UBORROWING;
- sched_user_prio(td, base_pri);
- } else {
- sched_lend_user_prio(td, prio);
- }
+ td->td_lend_user_pri = prio;
+ if (prio > base_pri)
+ td->td_user_pri = base_pri;
+ else
+ td->td_user_pri = prio;
}
/*
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 63437bc..8920201 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -76,6 +76,7 @@ __FBSDID("$FreeBSD$");
#include <sys/namei.h>
#include <sys/proc.h>
#include <sys/protosw.h>
+#include <sys/queue.h>
#include <sys/resourcevar.h>
#include <sys/rwlock.h>
#include <sys/socket.h>
@@ -115,6 +116,13 @@ static struct unp_head unp_shead; /* (l) List of stream sockets. */
static struct unp_head unp_dhead; /* (l) List of datagram sockets. */
static struct unp_head unp_sphead; /* (l) List of seqpacket sockets. */
+struct unp_defer {
+ SLIST_ENTRY(unp_defer) ud_link;
+ struct file *ud_fp;
+};
+static SLIST_HEAD(, unp_defer) unp_defers;
+static int unp_defers_count;
+
static const struct sockaddr sun_noname = { sizeof(sun_noname), AF_LOCAL };
/*
@@ -126,6 +134,13 @@ static const struct sockaddr sun_noname = { sizeof(sun_noname), AF_LOCAL };
static struct task unp_gc_task;
/*
+ * The close of unix domain sockets attached as SCM_RIGHTS is
+ * postponed to the taskqueue, to avoid arbitrary recursion depth.
+ * The attached sockets might have another sockets attached.
+ */
+static struct task unp_defer_task;
+
+/*
* Both send and receive buffers are allocated PIPSIZ bytes of buffering for
* stream sockets, although the total for sender and receiver is actually
* only PIPSIZ.
@@ -162,8 +177,11 @@ SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW,
&unpsp_sendspace, 0, "Default seqpacket send space.");
SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW,
&unpsp_recvspace, 0, "Default seqpacket receive space.");
-SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0,
+SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0,
"File descriptors in flight.");
+SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD,
+ &unp_defers_count, 0,
+ "File descriptors deferred to taskqueue for close.");
/*
* Locking and synchronization:
@@ -213,6 +231,7 @@ SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0,
*/
static struct rwlock unp_link_rwlock;
static struct mtx unp_list_lock;
+static struct mtx unp_defers_lock;
#define UNP_LINK_LOCK_INIT() rw_init(&unp_link_rwlock, \
"unp_link_rwlock")
@@ -234,6 +253,11 @@ static struct mtx unp_list_lock;
#define UNP_LIST_LOCK() mtx_lock(&unp_list_lock)
#define UNP_LIST_UNLOCK() mtx_unlock(&unp_list_lock)
+#define UNP_DEFERRED_LOCK_INIT() mtx_init(&unp_defers_lock, \
+ "unp_defer", NULL, MTX_DEF)
+#define UNP_DEFERRED_LOCK() mtx_lock(&unp_defers_lock)
+#define UNP_DEFERRED_UNLOCK() mtx_unlock(&unp_defers_lock)
+
#define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \
"unp_mtx", "unp_mtx", \
MTX_DUPOK|MTX_DEF|MTX_RECURSE)
@@ -259,8 +283,9 @@ static void unp_init(void);
static int unp_internalize(struct mbuf **, struct thread *);
static void unp_internalize_fp(struct file *);
static int unp_externalize(struct mbuf *, struct mbuf **);
-static void unp_externalize_fp(struct file *);
+static int unp_externalize_fp(struct file *);
static struct mbuf *unp_addsockcred(struct thread *, struct mbuf *);
+static void unp_process_defers(void * __unused, int);
/*
* Definitions of protocols supported in the LOCAL domain.
@@ -1764,9 +1789,12 @@ unp_init(void)
LIST_INIT(&unp_dhead);
LIST_INIT(&unp_shead);
LIST_INIT(&unp_sphead);
+ SLIST_INIT(&unp_defers);
TASK_INIT(&unp_gc_task, 0, unp_gc, NULL);
+ TASK_INIT(&unp_defer_task, 0, unp_process_defers, NULL);
UNP_LINK_LOCK_INIT();
UNP_LIST_LOCK_INIT();
+ UNP_DEFERRED_LOCK_INIT();
}
static int
@@ -1970,9 +1998,45 @@ fptounp(struct file *fp)
static void
unp_discard(struct file *fp)
{
+ struct unp_defer *dr;
+
+ if (unp_externalize_fp(fp)) {
+ dr = malloc(sizeof(*dr), M_TEMP, M_WAITOK);
+ dr->ud_fp = fp;
+ UNP_DEFERRED_LOCK();
+ SLIST_INSERT_HEAD(&unp_defers, dr, ud_link);
+ UNP_DEFERRED_UNLOCK();
+ atomic_add_int(&unp_defers_count, 1);
+ taskqueue_enqueue(taskqueue_thread, &unp_defer_task);
+ } else
+ (void) closef(fp, (struct thread *)NULL);
+}
- unp_externalize_fp(fp);
- (void) closef(fp, (struct thread *)NULL);
+static void
+unp_process_defers(void *arg __unused, int pending)
+{
+ struct unp_defer *dr;
+ SLIST_HEAD(, unp_defer) drl;
+ int count;
+
+ SLIST_INIT(&drl);
+ for (;;) {
+ UNP_DEFERRED_LOCK();
+ if (SLIST_FIRST(&unp_defers) == NULL) {
+ UNP_DEFERRED_UNLOCK();
+ break;
+ }
+ SLIST_SWAP(&unp_defers, &drl, unp_defer);
+ UNP_DEFERRED_UNLOCK();
+ count = 0;
+ while ((dr = SLIST_FIRST(&drl)) != NULL) {
+ SLIST_REMOVE_HEAD(&drl, ud_link);
+ closef(dr->ud_fp, NULL);
+ free(dr, M_TEMP);
+ count++;
+ }
+ atomic_add_int(&unp_defers_count, -count);
+ }
}
static void
@@ -1990,16 +2054,21 @@ unp_internalize_fp(struct file *fp)
UNP_LINK_WUNLOCK();
}
-static void
+static int
unp_externalize_fp(struct file *fp)
{
struct unpcb *unp;
+ int ret;
UNP_LINK_WLOCK();
- if ((unp = fptounp(fp)) != NULL)
+ if ((unp = fptounp(fp)) != NULL) {
unp->unp_msgcount--;
+ ret = 1;
+ } else
+ ret = 0;
unp_rights--;
UNP_LINK_WUNLOCK();
+ return (ret);
}
/*
OpenPOWER on IntegriCloud