diff options
Diffstat (limited to 'sys/kern')
-rw-r--r-- | sys/kern/init_main.c | 1 | ||||
-rw-r--r-- | sys/kern/kern_fork.c | 119 | ||||
-rw-r--r-- | sys/kern/kern_thread.c | 46 | ||||
-rw-r--r-- | sys/kern/kern_umtx.c | 16 | ||||
-rw-r--r-- | sys/kern/sched_4bsd.c | 23 | ||||
-rw-r--r-- | sys/kern/sched_ule.c | 21 | ||||
-rw-r--r-- | sys/kern/uipc_usrreq.c | 81 |
7 files changed, 209 insertions, 98 deletions
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index b86ea91..80a0907 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -460,6 +460,7 @@ proc0_init(void *dummy __unused) td->td_pri_class = PRI_TIMESHARE; td->td_user_pri = PUSER; td->td_base_user_pri = PUSER; + td->td_lend_user_pri = PRI_MAX; td->td_priority = PVM; td->td_base_pri = PUSER; td->td_oncpu = 0; diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 126c668..61d9531 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -97,9 +97,7 @@ struct fork_args { /* ARGSUSED */ int -fork(td, uap) - struct thread *td; - struct fork_args *uap; +fork(struct thread *td, struct fork_args *uap) { int error; struct proc *p2; @@ -135,9 +133,7 @@ vfork(td, uap) } int -rfork(td, uap) - struct thread *td; - struct rfork_args *uap; +rfork(struct thread *td, struct rfork_args *uap) { struct proc *p2; int error; @@ -197,12 +193,59 @@ sysctl_kern_randompid(SYSCTL_HANDLER_ARGS) SYSCTL_PROC(_kern, OID_AUTO, randompid, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_kern_randompid, "I", "Random PID modulus"); +static int +fork_norfproc(struct thread *td, int flags, struct proc **procp) +{ + int error; + struct proc *p1; + + KASSERT((flags & RFPROC) == 0, + ("fork_norfproc called with RFPROC set")); + p1 = td->td_proc; + *procp = NULL; + + if (((p1->p_flag & (P_HADTHREADS|P_SYSTEM)) == P_HADTHREADS) && + (flags & (RFCFDG | RFFDG))) { + PROC_LOCK(p1); + if (thread_single(SINGLE_BOUNDARY)) { + PROC_UNLOCK(p1); + return (ERESTART); + } + PROC_UNLOCK(p1); + } + + error = vm_forkproc(td, NULL, NULL, NULL, flags); + if (error) + goto fail; + + /* + * Close all file descriptors. + */ + if (flags & RFCFDG) { + struct filedesc *fdtmp; + fdtmp = fdinit(td->td_proc->p_fd); + fdfree(td); + p1->p_fd = fdtmp; + } + + /* + * Unshare file descriptors (from parent). + */ + if (flags & RFFDG) + fdunshare(p1, td); + +fail: + if (((p1->p_flag & (P_HADTHREADS|P_SYSTEM)) == P_HADTHREADS) && + (flags & (RFCFDG | RFFDG))) { + PROC_LOCK(p1); + thread_single_end(); + PROC_UNLOCK(p1); + } + return (error); +} + int -fork1(td, flags, pages, procp) - struct thread *td; - int flags; - int pages; - struct proc **procp; +fork1(struct thread *td, int flags, int pages, struct proc **procp) { struct proc *p1, *p2, *pptr; struct proc *newproc; @@ -227,47 +270,8 @@ fork1(td, flags, pages, procp) * Here we don't create a new process, but we divorce * certain parts of a process from itself. */ - if ((flags & RFPROC) == 0) { - if (((p1->p_flag & (P_HADTHREADS|P_SYSTEM)) == P_HADTHREADS) && - (flags & (RFCFDG | RFFDG))) { - PROC_LOCK(p1); - if (thread_single(SINGLE_BOUNDARY)) { - PROC_UNLOCK(p1); - return (ERESTART); - } - PROC_UNLOCK(p1); - } - - error = vm_forkproc(td, NULL, NULL, NULL, flags); - if (error) - goto norfproc_fail; - - /* - * Close all file descriptors. - */ - if (flags & RFCFDG) { - struct filedesc *fdtmp; - fdtmp = fdinit(td->td_proc->p_fd); - fdfree(td); - p1->p_fd = fdtmp; - } - - /* - * Unshare file descriptors (from parent). - */ - if (flags & RFFDG) - fdunshare(p1, td); - -norfproc_fail: - if (((p1->p_flag & (P_HADTHREADS|P_SYSTEM)) == P_HADTHREADS) && - (flags & (RFCFDG | RFFDG))) { - PROC_LOCK(p1); - thread_single_end(); - PROC_UNLOCK(p1); - } - *procp = NULL; - return (error); - } + if ((flags & RFPROC) == 0) + return (fork_norfproc(td, flags, procp)); /* * XXX @@ -539,6 +543,7 @@ again: td2->td_sigstk = td->td_sigstk; td2->td_sigmask = td->td_sigmask; td2->td_flags = TDF_INMEM; + td2->td_lend_user_pri = PRI_MAX; #ifdef VIMAGE td2->td_vnet = NULL; @@ -798,10 +803,8 @@ fail1: * is called from the MD fork_trampoline() entry point. */ void -fork_exit(callout, arg, frame) - void (*callout)(void *, struct trapframe *); - void *arg; - struct trapframe *frame; +fork_exit(void (*callout)(void *, struct trapframe *), void *arg, + struct trapframe *frame) { struct proc *p; struct thread *td; @@ -855,9 +858,7 @@ fork_exit(callout, arg, frame) * first parameter and is called when returning to a new userland process. */ void -fork_return(td, frame) - struct thread *td; - struct trapframe *frame; +fork_return(struct thread *td, struct trapframe *frame) { userret(td, frame); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index 5f07590..7161a99 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -81,15 +81,54 @@ MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN); static void thread_zombie(struct thread *); +#define TID_BUFFER_SIZE 1024 + struct mtx tid_lock; static struct unrhdr *tid_unrhdr; - +static lwpid_t tid_buffer[TID_BUFFER_SIZE]; +static int tid_head, tid_tail; static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash"); struct tidhashhead *tidhashtbl; u_long tidhash; struct rwlock tidhash_lock; +static lwpid_t +tid_alloc(void) +{ + lwpid_t tid; + + tid = alloc_unr(tid_unrhdr); + if (tid != -1) + return (tid); + mtx_lock(&tid_lock); + if (tid_head == tid_tail) { + mtx_unlock(&tid_lock); + return (-1); + } + tid = tid_buffer[tid_head++]; + tid_head %= TID_BUFFER_SIZE; + mtx_unlock(&tid_lock); + return (tid); +} + +static void +tid_free(lwpid_t tid) +{ + lwpid_t tmp_tid = -1; + + mtx_lock(&tid_lock); + if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) { + tmp_tid = tid_buffer[tid_head++]; + tid_head = (tid_head + 1) % TID_BUFFER_SIZE; + } + tid_buffer[tid_tail++] = tid; + tid_tail %= TID_BUFFER_SIZE; + mtx_unlock(&tid_lock); + if (tmp_tid != -1) + free_unr(tid_unrhdr, tmp_tid); +} + /* * Prepare a thread for use. */ @@ -102,7 +141,7 @@ thread_ctor(void *mem, int size, void *arg, int flags) td->td_state = TDS_INACTIVE; td->td_oncpu = NOCPU; - td->td_tid = alloc_unr(tid_unrhdr); + td->td_tid = tid_alloc(); /* * Note that td_critnest begins life as 1 because the thread is not @@ -110,6 +149,7 @@ thread_ctor(void *mem, int size, void *arg, int flags) * end of a context switch. */ td->td_critnest = 1; + td->td_lend_user_pri = PRI_MAX; EVENTHANDLER_INVOKE(thread_ctor, td); #ifdef AUDIT audit_thread_alloc(td); @@ -155,7 +195,7 @@ thread_dtor(void *mem, int size, void *arg) osd_thread_exit(td); EVENTHANDLER_INVOKE(thread_dtor, td); - free_unr(tid_unrhdr, td->td_tid); + tid_free(td->td_tid); } /* diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c index 43570ce..e7b9b32 100644 --- a/sys/kern/kern_umtx.c +++ b/sys/kern/kern_umtx.c @@ -1407,17 +1407,19 @@ umtx_propagate_priority(struct thread *td) for (;;) { td = pi->pi_owner; - if (td == NULL) + if (td == NULL || td == curthread) return; MPASS(td->td_proc != NULL); MPASS(td->td_proc->p_magic == P_MAGIC); - if (UPRI(td) <= pri) - return; - thread_lock(td); - sched_lend_user_prio(td, pri); + if (td->td_lend_user_pri > pri) + sched_lend_user_prio(td, pri); + else { + thread_unlock(td); + break; + } thread_unlock(td); /* @@ -3587,8 +3589,8 @@ umtx_thread_cleanup(struct thread *td) pi->pi_owner = NULL; TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); } + mtx_unlock_spin(&umtx_lock); thread_lock(td); - td->td_flags &= ~TDF_UBORROWING; + sched_unlend_user_prio(td, PRI_MAX); thread_unlock(td); - mtx_unlock_spin(&umtx_lock); } diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index 9face64..6278126 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -879,25 +879,23 @@ sched_prio(struct thread *td, u_char prio) void sched_user_prio(struct thread *td, u_char prio) { - u_char oldprio; THREAD_LOCK_ASSERT(td, MA_OWNED); td->td_base_user_pri = prio; - if (td->td_flags & TDF_UBORROWING && td->td_user_pri <= prio) + if (td->td_lend_user_pri <= prio) return; - oldprio = td->td_user_pri; td->td_user_pri = prio; } void sched_lend_user_prio(struct thread *td, u_char prio) { - u_char oldprio; THREAD_LOCK_ASSERT(td, MA_OWNED); - td->td_flags |= TDF_UBORROWING; - oldprio = td->td_user_pri; - td->td_user_pri = prio; + if (prio < td->td_lend_user_pri) + td->td_lend_user_pri = prio; + if (prio < td->td_user_pri) + td->td_user_pri = prio; } void @@ -907,12 +905,11 @@ sched_unlend_user_prio(struct thread *td, u_char prio) THREAD_LOCK_ASSERT(td, MA_OWNED); base_pri = td->td_base_user_pri; - if (prio >= base_pri) { - td->td_flags &= ~TDF_UBORROWING; - sched_user_prio(td, base_pri); - } else { - sched_lend_user_prio(td, prio); - } + td->td_lend_user_pri = prio; + if (prio > base_pri) + td->td_user_pri = base_pri; + else + td->td_user_pri = prio; } void diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index 030c98d..fb30fdb 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -1677,8 +1677,8 @@ sched_user_prio(struct thread *td, u_char prio) { td->td_base_user_pri = prio; - if (td->td_flags & TDF_UBORROWING && td->td_user_pri <= prio) - return; + if (td->td_lend_user_pri <= prio) + return; td->td_user_pri = prio; } @@ -1687,8 +1687,10 @@ sched_lend_user_prio(struct thread *td, u_char prio) { THREAD_LOCK_ASSERT(td, MA_OWNED); - td->td_flags |= TDF_UBORROWING; - td->td_user_pri = prio; + if (prio < td->td_lend_user_pri) + td->td_lend_user_pri = prio; + if (prio < td->td_user_pri) + td->td_user_pri = prio; } void @@ -1698,12 +1700,11 @@ sched_unlend_user_prio(struct thread *td, u_char prio) THREAD_LOCK_ASSERT(td, MA_OWNED); base_pri = td->td_base_user_pri; - if (prio >= base_pri) { - td->td_flags &= ~TDF_UBORROWING; - sched_user_prio(td, base_pri); - } else { - sched_lend_user_prio(td, prio); - } + td->td_lend_user_pri = prio; + if (prio > base_pri) + td->td_user_pri = base_pri; + else + td->td_user_pri = prio; } /* diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c index 63437bc..8920201 100644 --- a/sys/kern/uipc_usrreq.c +++ b/sys/kern/uipc_usrreq.c @@ -76,6 +76,7 @@ __FBSDID("$FreeBSD$"); #include <sys/namei.h> #include <sys/proc.h> #include <sys/protosw.h> +#include <sys/queue.h> #include <sys/resourcevar.h> #include <sys/rwlock.h> #include <sys/socket.h> @@ -115,6 +116,13 @@ static struct unp_head unp_shead; /* (l) List of stream sockets. */ static struct unp_head unp_dhead; /* (l) List of datagram sockets. */ static struct unp_head unp_sphead; /* (l) List of seqpacket sockets. */ +struct unp_defer { + SLIST_ENTRY(unp_defer) ud_link; + struct file *ud_fp; +}; +static SLIST_HEAD(, unp_defer) unp_defers; +static int unp_defers_count; + static const struct sockaddr sun_noname = { sizeof(sun_noname), AF_LOCAL }; /* @@ -126,6 +134,13 @@ static const struct sockaddr sun_noname = { sizeof(sun_noname), AF_LOCAL }; static struct task unp_gc_task; /* + * The close of unix domain sockets attached as SCM_RIGHTS is + * postponed to the taskqueue, to avoid arbitrary recursion depth. + * The attached sockets might have another sockets attached. + */ +static struct task unp_defer_task; + +/* * Both send and receive buffers are allocated PIPSIZ bytes of buffering for * stream sockets, although the total for sender and receiver is actually * only PIPSIZ. @@ -162,8 +177,11 @@ SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW, &unpsp_sendspace, 0, "Default seqpacket send space."); SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW, &unpsp_recvspace, 0, "Default seqpacket receive space."); -SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0, +SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0, "File descriptors in flight."); +SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD, + &unp_defers_count, 0, + "File descriptors deferred to taskqueue for close."); /* * Locking and synchronization: @@ -213,6 +231,7 @@ SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0, */ static struct rwlock unp_link_rwlock; static struct mtx unp_list_lock; +static struct mtx unp_defers_lock; #define UNP_LINK_LOCK_INIT() rw_init(&unp_link_rwlock, \ "unp_link_rwlock") @@ -234,6 +253,11 @@ static struct mtx unp_list_lock; #define UNP_LIST_LOCK() mtx_lock(&unp_list_lock) #define UNP_LIST_UNLOCK() mtx_unlock(&unp_list_lock) +#define UNP_DEFERRED_LOCK_INIT() mtx_init(&unp_defers_lock, \ + "unp_defer", NULL, MTX_DEF) +#define UNP_DEFERRED_LOCK() mtx_lock(&unp_defers_lock) +#define UNP_DEFERRED_UNLOCK() mtx_unlock(&unp_defers_lock) + #define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \ "unp_mtx", "unp_mtx", \ MTX_DUPOK|MTX_DEF|MTX_RECURSE) @@ -259,8 +283,9 @@ static void unp_init(void); static int unp_internalize(struct mbuf **, struct thread *); static void unp_internalize_fp(struct file *); static int unp_externalize(struct mbuf *, struct mbuf **); -static void unp_externalize_fp(struct file *); +static int unp_externalize_fp(struct file *); static struct mbuf *unp_addsockcred(struct thread *, struct mbuf *); +static void unp_process_defers(void * __unused, int); /* * Definitions of protocols supported in the LOCAL domain. @@ -1764,9 +1789,12 @@ unp_init(void) LIST_INIT(&unp_dhead); LIST_INIT(&unp_shead); LIST_INIT(&unp_sphead); + SLIST_INIT(&unp_defers); TASK_INIT(&unp_gc_task, 0, unp_gc, NULL); + TASK_INIT(&unp_defer_task, 0, unp_process_defers, NULL); UNP_LINK_LOCK_INIT(); UNP_LIST_LOCK_INIT(); + UNP_DEFERRED_LOCK_INIT(); } static int @@ -1970,9 +1998,45 @@ fptounp(struct file *fp) static void unp_discard(struct file *fp) { + struct unp_defer *dr; + + if (unp_externalize_fp(fp)) { + dr = malloc(sizeof(*dr), M_TEMP, M_WAITOK); + dr->ud_fp = fp; + UNP_DEFERRED_LOCK(); + SLIST_INSERT_HEAD(&unp_defers, dr, ud_link); + UNP_DEFERRED_UNLOCK(); + atomic_add_int(&unp_defers_count, 1); + taskqueue_enqueue(taskqueue_thread, &unp_defer_task); + } else + (void) closef(fp, (struct thread *)NULL); +} - unp_externalize_fp(fp); - (void) closef(fp, (struct thread *)NULL); +static void +unp_process_defers(void *arg __unused, int pending) +{ + struct unp_defer *dr; + SLIST_HEAD(, unp_defer) drl; + int count; + + SLIST_INIT(&drl); + for (;;) { + UNP_DEFERRED_LOCK(); + if (SLIST_FIRST(&unp_defers) == NULL) { + UNP_DEFERRED_UNLOCK(); + break; + } + SLIST_SWAP(&unp_defers, &drl, unp_defer); + UNP_DEFERRED_UNLOCK(); + count = 0; + while ((dr = SLIST_FIRST(&drl)) != NULL) { + SLIST_REMOVE_HEAD(&drl, ud_link); + closef(dr->ud_fp, NULL); + free(dr, M_TEMP); + count++; + } + atomic_add_int(&unp_defers_count, -count); + } } static void @@ -1990,16 +2054,21 @@ unp_internalize_fp(struct file *fp) UNP_LINK_WUNLOCK(); } -static void +static int unp_externalize_fp(struct file *fp) { struct unpcb *unp; + int ret; UNP_LINK_WLOCK(); - if ((unp = fptounp(fp)) != NULL) + if ((unp = fptounp(fp)) != NULL) { unp->unp_msgcount--; + ret = 1; + } else + ret = 0; unp_rights--; UNP_LINK_WUNLOCK(); + return (ret); } /* |