diff options
Diffstat (limited to 'lib/libkse/thread/thr_kern.c')
| -rw-r--r-- | lib/libkse/thread/thr_kern.c | 2573 |
1 files changed, 0 insertions, 2573 deletions
diff --git a/lib/libkse/thread/thr_kern.c b/lib/libkse/thread/thr_kern.c deleted file mode 100644 index 6563ca7..0000000 --- a/lib/libkse/thread/thr_kern.c +++ /dev/null @@ -1,2573 +0,0 @@ -/* - * Copyright (C) 2003 Daniel M. Eischen <deischen@freebsd.org> - * Copyright (C) 2002 Jonathon Mini <mini@freebsd.org> - * Copyright (c) 1995-1998 John Birrell <jb@cimlogic.com.au> - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by John Birrell. - * 4. Neither the name of the author nor the names of any co-contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - -#include <sys/types.h> -#include <sys/kse.h> -#include <sys/ptrace.h> -#include <sys/signalvar.h> -#include <sys/queue.h> -#include <machine/atomic.h> -#include <machine/sigframe.h> - -#include <assert.h> -#include <errno.h> -#include <signal.h> -#include <stdlib.h> -#include <string.h> -#include <time.h> -#include <ucontext.h> -#include <unistd.h> - -#include "atomic_ops.h" -#include "thr_private.h" -#include "libc_private.h" -#ifdef NOTYET -#include "spinlock.h" -#endif - -/* #define DEBUG_THREAD_KERN */ -#ifdef DEBUG_THREAD_KERN -#define DBG_MSG stdout_debug -#else -#define DBG_MSG(x...) -#endif - -/* - * Define a high water mark for the maximum number of threads that - * will be cached. Once this level is reached, any extra threads - * will be free()'d. - */ -#define MAX_CACHED_THREADS 100 -/* - * Define high water marks for the maximum number of KSEs and KSE groups - * that will be cached. Because we support 1:1 threading, there could have - * same number of KSEs and KSE groups as threads. Once these levels are - * reached, any extra KSE and KSE groups will be free()'d. - */ -#define MAX_CACHED_KSES ((_thread_scope_system <= 0) ? 50 : 100) -#define MAX_CACHED_KSEGS ((_thread_scope_system <= 0) ? 50 : 100) - -#define KSE_SET_MBOX(kse, thrd) \ - (kse)->k_kcb->kcb_kmbx.km_curthread = &(thrd)->tcb->tcb_tmbx - -#define KSE_SET_EXITED(kse) (kse)->k_flags |= KF_EXITED - -/* - * Macros for manipulating the run queues. The priority queue - * routines use the thread's pqe link and also handle the setting - * and clearing of the thread's THR_FLAGS_IN_RUNQ flag. - */ -#define KSE_RUNQ_INSERT_HEAD(kse, thrd) \ - _pq_insert_head(&(kse)->k_schedq->sq_runq, thrd) -#define KSE_RUNQ_INSERT_TAIL(kse, thrd) \ - _pq_insert_tail(&(kse)->k_schedq->sq_runq, thrd) -#define KSE_RUNQ_REMOVE(kse, thrd) \ - _pq_remove(&(kse)->k_schedq->sq_runq, thrd) -#define KSE_RUNQ_FIRST(kse) \ - ((_libkse_debug == 0) ? \ - _pq_first(&(kse)->k_schedq->sq_runq) : \ - _pq_first_debug(&(kse)->k_schedq->sq_runq)) - -#define KSE_RUNQ_THREADS(kse) ((kse)->k_schedq->sq_runq.pq_threads) - -#define THR_NEED_CANCEL(thrd) \ - (((thrd)->cancelflags & THR_CANCELLING) != 0 && \ - ((thrd)->cancelflags & PTHREAD_CANCEL_DISABLE) == 0 && \ - (((thrd)->cancelflags & THR_AT_CANCEL_POINT) != 0 || \ - ((thrd)->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0)) - -#define THR_NEED_ASYNC_CANCEL(thrd) \ - (((thrd)->cancelflags & THR_CANCELLING) != 0 && \ - ((thrd)->cancelflags & PTHREAD_CANCEL_DISABLE) == 0 && \ - (((thrd)->cancelflags & THR_AT_CANCEL_POINT) == 0 && \ - ((thrd)->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0)) - -/* - * We've got to keep track of everything that is allocated, not only - * to have a speedy free list, but also so they can be deallocated - * after a fork(). - */ -static TAILQ_HEAD(, kse) active_kseq; -static TAILQ_HEAD(, kse) free_kseq; -static TAILQ_HEAD(, kse_group) free_kse_groupq; -static TAILQ_HEAD(, kse_group) active_kse_groupq; -static TAILQ_HEAD(, kse_group) gc_ksegq; -static struct lock kse_lock; /* also used for kseg queue */ -static int free_kse_count = 0; -static int free_kseg_count = 0; -static TAILQ_HEAD(, pthread) free_threadq; -static struct lock thread_lock; -static int free_thread_count = 0; -static int inited = 0; -static int active_kse_count = 0; -static int active_kseg_count = 0; -static u_int64_t next_uniqueid = 1; - -LIST_HEAD(thread_hash_head, pthread); -#define THREAD_HASH_QUEUES 127 -static struct thread_hash_head thr_hashtable[THREAD_HASH_QUEUES]; -#define THREAD_HASH(thrd) ((unsigned long)thrd % THREAD_HASH_QUEUES) - -/* Lock for thread tcb constructor/destructor */ -static pthread_mutex_t _tcb_mutex; - -#ifdef DEBUG_THREAD_KERN -static void dump_queues(struct kse *curkse); -#endif -static void kse_check_completed(struct kse *kse); -static void kse_check_waitq(struct kse *kse); -static void kse_fini(struct kse *curkse); -static void kse_reinit(struct kse *kse, int sys_scope); -static void kse_sched_multi(struct kse_mailbox *kmbx); -static void kse_sched_single(struct kse_mailbox *kmbx); -static void kse_switchout_thread(struct kse *kse, struct pthread *thread); -static void kse_wait(struct kse *kse, struct pthread *td_wait, int sigseq); -static void kse_free_unlocked(struct kse *kse); -static void kse_destroy(struct kse *kse); -static void kseg_free_unlocked(struct kse_group *kseg); -static void kseg_init(struct kse_group *kseg); -static void kseg_reinit(struct kse_group *kseg); -static void kseg_destroy(struct kse_group *kseg); -static void kse_waitq_insert(struct pthread *thread); -static void kse_wakeup_multi(struct kse *curkse); -static struct kse_mailbox *kse_wakeup_one(struct pthread *thread); -static void thr_cleanup(struct kse *kse, struct pthread *curthread); -static void thr_link(struct pthread *thread); -static void thr_resume_wrapper(int sig, siginfo_t *, ucontext_t *); -static void thr_resume_check(struct pthread *curthread, ucontext_t *ucp); -static int thr_timedout(struct pthread *thread, struct timespec *curtime); -static void thr_unlink(struct pthread *thread); -static void thr_destroy(struct pthread *curthread, struct pthread *thread); -static void thread_gc(struct pthread *thread); -static void kse_gc(struct pthread *thread); -static void kseg_gc(struct pthread *thread); - -static __inline void -thr_accounting(struct pthread *thread) -{ - if ((thread->slice_usec != -1) && - (thread->slice_usec <= TIMESLICE_USEC) && - (thread->attr.sched_policy != SCHED_FIFO)) { - thread->slice_usec += (thread->tcb->tcb_tmbx.tm_uticks - + thread->tcb->tcb_tmbx.tm_sticks) * _clock_res_usec; - /* Check for time quantum exceeded: */ - if (thread->slice_usec > TIMESLICE_USEC) - thread->slice_usec = -1; - } - thread->tcb->tcb_tmbx.tm_uticks = 0; - thread->tcb->tcb_tmbx.tm_sticks = 0; -} - -/* - * This is called after a fork(). - * No locks need to be taken here since we are guaranteed to be - * single threaded. - * - * XXX - * POSIX says for threaded process, fork() function is used - * only to run new programs, and the effects of calling functions - * that require certain resources between the call to fork() and - * the call to an exec function are undefined. - * - * It is not safe to free memory after fork(), because these data - * structures may be in inconsistent state. - */ -void -_kse_single_thread(struct pthread *curthread) -{ -#ifdef NOTYET - struct kse *kse; - struct kse_group *kseg; - struct pthread *thread; - - _thr_spinlock_init(); - *__malloc_lock = (spinlock_t)_SPINLOCK_INITIALIZER; - if (__isthreaded) { - _thr_rtld_fini(); - _thr_signal_deinit(); - } - __isthreaded = 0; - /* - * Restore signal mask early, so any memory problems could - * dump core. - */ - __sys_sigprocmask(SIG_SETMASK, &curthread->sigmask, NULL); - _thread_active_threads = 1; - - curthread->kse->k_kcb->kcb_kmbx.km_curthread = NULL; - curthread->attr.flags &= ~PTHREAD_SCOPE_PROCESS; - curthread->attr.flags |= PTHREAD_SCOPE_SYSTEM; - - /* - * Enter a loop to remove and free all threads other than - * the running thread from the active thread list: - */ - while ((thread = TAILQ_FIRST(&_thread_list)) != NULL) { - THR_GCLIST_REMOVE(thread); - /* - * Remove this thread from the list (the current - * thread will be removed but re-added by libpthread - * initialization. - */ - TAILQ_REMOVE(&_thread_list, thread, tle); - /* Make sure this isn't the running thread: */ - if (thread != curthread) { - _thr_stack_free(&thread->attr); - if (thread->specific != NULL) - free(thread->specific); - thr_destroy(curthread, thread); - } - } - - TAILQ_INIT(&curthread->mutexq); /* initialize mutex queue */ - curthread->joiner = NULL; /* no joining threads yet */ - curthread->refcount = 0; - SIGEMPTYSET(curthread->sigpend); /* clear pending signals */ - - /* Don't free thread-specific data as the caller may require it */ - - /* Free the free KSEs: */ - while ((kse = TAILQ_FIRST(&free_kseq)) != NULL) { - TAILQ_REMOVE(&free_kseq, kse, k_qe); - kse_destroy(kse); - } - free_kse_count = 0; - - /* Free the active KSEs: */ - while ((kse = TAILQ_FIRST(&active_kseq)) != NULL) { - TAILQ_REMOVE(&active_kseq, kse, k_qe); - kse_destroy(kse); - } - active_kse_count = 0; - - /* Free the free KSEGs: */ - while ((kseg = TAILQ_FIRST(&free_kse_groupq)) != NULL) { - TAILQ_REMOVE(&free_kse_groupq, kseg, kg_qe); - kseg_destroy(kseg); - } - free_kseg_count = 0; - - /* Free the active KSEGs: */ - while ((kseg = TAILQ_FIRST(&active_kse_groupq)) != NULL) { - TAILQ_REMOVE(&active_kse_groupq, kseg, kg_qe); - kseg_destroy(kseg); - } - active_kseg_count = 0; - - /* Free the free threads. */ - while ((thread = TAILQ_FIRST(&free_threadq)) != NULL) { - TAILQ_REMOVE(&free_threadq, thread, tle); - thr_destroy(curthread, thread); - } - free_thread_count = 0; - - /* Free the to-be-gc'd threads. */ - while ((thread = TAILQ_FIRST(&_thread_gc_list)) != NULL) { - TAILQ_REMOVE(&_thread_gc_list, thread, gcle); - thr_destroy(curthread, thread); - } - TAILQ_INIT(&gc_ksegq); - _gc_count = 0; - - if (inited != 0) { - /* - * Destroy these locks; they'll be recreated to assure they - * are in the unlocked state. - */ - _lock_destroy(&kse_lock); - _lock_destroy(&thread_lock); - _lock_destroy(&_thread_list_lock); - inited = 0; - } - - /* We're no longer part of any lists */ - curthread->tlflags = 0; - - /* - * After a fork, we are still operating on the thread's original - * stack. Don't clear the THR_FLAGS_USER from the thread's - * attribute flags. - */ - - /* Initialize the threads library. */ - curthread->kse = NULL; - curthread->kseg = NULL; - _kse_initial = NULL; - _libpthread_init(curthread); -#else - int i; - - /* Reset the current thread and KSE lock data. */ - for (i = 0; i < curthread->locklevel; i++) { - _lockuser_reinit(&curthread->lockusers[i], (void *)curthread); - } - curthread->locklevel = 0; - for (i = 0; i < curthread->kse->k_locklevel; i++) { - _lockuser_reinit(&curthread->kse->k_lockusers[i], - (void *)curthread->kse); - _LCK_SET_PRIVATE2(&curthread->kse->k_lockusers[i], NULL); - } - curthread->kse->k_locklevel = 0; - - /* - * Reinitialize the thread and signal locks so that - * sigaction() will work after a fork(). - */ - _lock_reinit(&curthread->lock, LCK_ADAPTIVE, _thr_lock_wait, - _thr_lock_wakeup); - _lock_reinit(&_thread_signal_lock, LCK_ADAPTIVE, _kse_lock_wait, - _kse_lock_wakeup); - - _thr_spinlock_init(); - if (__isthreaded) { - _thr_rtld_fini(); - _thr_signal_deinit(); - } - __isthreaded = 0; - curthread->kse->k_kcb->kcb_kmbx.km_curthread = NULL; - curthread->attr.flags |= PTHREAD_SCOPE_SYSTEM; - - /* - * After a fork, it is possible that an upcall occurs in - * the parent KSE that fork()'d before the child process - * is fully created and before its vm space is copied. - * During the upcall, the tcb is set to null or to another - * thread, and this is what gets copied in the child process - * when the vm space is cloned sometime after the upcall - * occurs. Note that we shouldn't have to set the kcb, but - * we do it for completeness. - */ - _kcb_set(curthread->kse->k_kcb); - _tcb_set(curthread->kse->k_kcb, curthread->tcb); - - /* After a fork(), there child should have no pending signals. */ - sigemptyset(&curthread->sigpend); - - /* - * Restore signal mask early, so any memory problems could - * dump core. - */ - sigprocmask(SIG_SETMASK, &curthread->sigmask, NULL); - _thread_active_threads = 1; -#endif -} - -/* - * This is used to initialize housekeeping and to initialize the - * KSD for the KSE. - */ -void -_kse_init(void) -{ - if (inited == 0) { - TAILQ_INIT(&active_kseq); - TAILQ_INIT(&active_kse_groupq); - TAILQ_INIT(&free_kseq); - TAILQ_INIT(&free_kse_groupq); - TAILQ_INIT(&free_threadq); - TAILQ_INIT(&gc_ksegq); - if (_lock_init(&kse_lock, LCK_ADAPTIVE, - _kse_lock_wait, _kse_lock_wakeup, calloc) != 0) - PANIC("Unable to initialize free KSE queue lock"); - if (_lock_init(&thread_lock, LCK_ADAPTIVE, - _kse_lock_wait, _kse_lock_wakeup, calloc) != 0) - PANIC("Unable to initialize free thread queue lock"); - if (_lock_init(&_thread_list_lock, LCK_ADAPTIVE, - _kse_lock_wait, _kse_lock_wakeup, calloc) != 0) - PANIC("Unable to initialize thread list lock"); - _pthread_mutex_init(&_tcb_mutex, NULL); - active_kse_count = 0; - active_kseg_count = 0; - _gc_count = 0; - inited = 1; - } -} - -/* - * This is called when the first thread (other than the initial - * thread) is created. - */ -int -_kse_setthreaded(int threaded) -{ - sigset_t sigset; - - if ((threaded != 0) && (__isthreaded == 0)) { - SIGFILLSET(sigset); - __sys_sigprocmask(SIG_SETMASK, &sigset, &_thr_initial->sigmask); - - /* - * Tell the kernel to create a KSE for the initial thread - * and enable upcalls in it. - */ - _kse_initial->k_flags |= KF_STARTED; - - if (_thread_scope_system <= 0) { - _thr_initial->attr.flags &= ~PTHREAD_SCOPE_SYSTEM; - _kse_initial->k_kseg->kg_flags &= ~KGF_SINGLE_THREAD; - _kse_initial->k_kcb->kcb_kmbx.km_curthread = NULL; - } - else { - /* - * For bound thread, kernel reads mailbox pointer - * once, we'd set it here before calling kse_create. - */ - _tcb_set(_kse_initial->k_kcb, _thr_initial->tcb); - KSE_SET_MBOX(_kse_initial, _thr_initial); - _kse_initial->k_kcb->kcb_kmbx.km_flags |= KMF_BOUND; - } - - /* - * Locking functions in libc are required when there are - * threads other than the initial thread. - */ - _thr_rtld_init(); - - __isthreaded = 1; - if (kse_create(&_kse_initial->k_kcb->kcb_kmbx, 0) != 0) { - _kse_initial->k_flags &= ~KF_STARTED; - __isthreaded = 0; - PANIC("kse_create() failed\n"); - return (-1); - } - _thr_initial->tcb->tcb_tmbx.tm_lwp = - _kse_initial->k_kcb->kcb_kmbx.km_lwp; - _thread_activated = 1; - -#ifndef SYSTEM_SCOPE_ONLY - if (_thread_scope_system <= 0) { - /* Set current thread to initial thread */ - _tcb_set(_kse_initial->k_kcb, _thr_initial->tcb); - KSE_SET_MBOX(_kse_initial, _thr_initial); - _thr_start_sig_daemon(); - _thr_setmaxconcurrency(); - } - else -#endif - __sys_sigprocmask(SIG_SETMASK, &_thr_initial->sigmask, - NULL); - } - return (0); -} - -/* - * Lock wait and wakeup handlers for KSE locks. These are only used by - * KSEs, and should never be used by threads. KSE locks include the - * KSE group lock (used for locking the scheduling queue) and the - * kse_lock defined above. - * - * When a KSE lock attempt blocks, the entire KSE blocks allowing another - * KSE to run. For the most part, it doesn't make much sense to try and - * schedule another thread because you need to lock the scheduling queue - * in order to do that. And since the KSE lock is used to lock the scheduling - * queue, you would just end up blocking again. - */ -void -_kse_lock_wait(struct lock *lock __unused, struct lockuser *lu) -{ - struct kse *curkse = (struct kse *)_LCK_GET_PRIVATE(lu); - struct timespec ts; - int saved_flags; - - if (curkse->k_kcb->kcb_kmbx.km_curthread != NULL) - PANIC("kse_lock_wait does not disable upcall.\n"); - /* - * Enter a loop to wait until we get the lock. - */ - ts.tv_sec = 0; - ts.tv_nsec = 1000000; /* 1 sec */ - while (!_LCK_GRANTED(lu)) { - /* - * Yield the kse and wait to be notified when the lock - * is granted. - */ - saved_flags = curkse->k_kcb->kcb_kmbx.km_flags; - curkse->k_kcb->kcb_kmbx.km_flags |= KMF_NOUPCALL | - KMF_NOCOMPLETED; - kse_release(&ts); - curkse->k_kcb->kcb_kmbx.km_flags = saved_flags; - } -} - -void -_kse_lock_wakeup(struct lock *lock, struct lockuser *lu) -{ - struct kse *curkse; - struct kse *kse; - struct kse_mailbox *mbx; - - curkse = _get_curkse(); - kse = (struct kse *)_LCK_GET_PRIVATE(lu); - - if (kse == curkse) - PANIC("KSE trying to wake itself up in lock"); - else { - mbx = &kse->k_kcb->kcb_kmbx; - _lock_grant(lock, lu); - /* - * Notify the owning kse that it has the lock. - * It is safe to pass invalid address to kse_wakeup - * even if the mailbox is not in kernel at all, - * and waking up a wrong kse is also harmless. - */ - kse_wakeup(mbx); - } -} - -/* - * Thread wait and wakeup handlers for thread locks. These are only used - * by threads, never by KSEs. Thread locks include the per-thread lock - * (defined in its structure), and condition variable and mutex locks. - */ -void -_thr_lock_wait(struct lock *lock __unused, struct lockuser *lu) -{ - struct pthread *curthread = (struct pthread *)lu->lu_private; - - do { - THR_LOCK_SWITCH(curthread); - THR_SET_STATE(curthread, PS_LOCKWAIT); - _thr_sched_switch_unlocked(curthread); - } while (!_LCK_GRANTED(lu)); -} - -void -_thr_lock_wakeup(struct lock *lock __unused, struct lockuser *lu) -{ - struct pthread *thread; - struct pthread *curthread; - struct kse_mailbox *kmbx; - - curthread = _get_curthread(); - thread = (struct pthread *)_LCK_GET_PRIVATE(lu); - - THR_SCHED_LOCK(curthread, thread); - _lock_grant(lock, lu); - kmbx = _thr_setrunnable_unlocked(thread); - THR_SCHED_UNLOCK(curthread, thread); - if (kmbx != NULL) - kse_wakeup(kmbx); -} - -kse_critical_t -_kse_critical_enter(void) -{ - kse_critical_t crit; - - crit = (kse_critical_t)_kcb_critical_enter(); - return (crit); -} - -void -_kse_critical_leave(kse_critical_t crit) -{ - struct pthread *curthread; - - _kcb_critical_leave((struct kse_thr_mailbox *)crit); - if ((crit != NULL) && ((curthread = _get_curthread()) != NULL)) - THR_YIELD_CHECK(curthread); -} - -int -_kse_in_critical(void) -{ - return (_kcb_in_critical()); -} - -void -_thr_critical_enter(struct pthread *thread) -{ - thread->critical_count++; -} - -void -_thr_critical_leave(struct pthread *thread) -{ - thread->critical_count--; - THR_YIELD_CHECK(thread); -} - -void -_thr_sched_switch(struct pthread *curthread) -{ - struct kse *curkse; - - (void)_kse_critical_enter(); - curkse = _get_curkse(); - KSE_SCHED_LOCK(curkse, curkse->k_kseg); - _thr_sched_switch_unlocked(curthread); -} - -/* - * XXX - We may need to take the scheduling lock before calling - * this, or perhaps take the lock within here before - * doing anything else. - */ -void -_thr_sched_switch_unlocked(struct pthread *curthread) -{ - struct kse *curkse; - volatile int resume_once = 0; - ucontext_t *uc; - - /* We're in the scheduler, 5 by 5: */ - curkse = curthread->kse; - - curthread->need_switchout = 1; /* The thread yielded on its own. */ - curthread->critical_yield = 0; /* No need to yield anymore. */ - - /* Thread can unlock the scheduler lock. */ - curthread->lock_switch = 1; - - if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM) - kse_sched_single(&curkse->k_kcb->kcb_kmbx); - else { - if (__predict_false(_libkse_debug != 0)) { - /* - * Because debugger saves single step status in thread - * mailbox's tm_dflags, we can safely clear single - * step status here. the single step status will be - * restored by kse_switchin when the thread is - * switched in again. This also lets uts run in full - * speed. - */ - ptrace(PT_CLEARSTEP, curkse->k_kcb->kcb_kmbx.km_lwp, - (caddr_t) 1, 0); - } - - KSE_SET_SWITCH(curkse); - _thread_enter_uts(curthread->tcb, curkse->k_kcb); - } - - /* - * Unlock the scheduling queue and leave the - * critical region. - */ - /* Don't trust this after a switch! */ - curkse = curthread->kse; - - curthread->lock_switch = 0; - KSE_SCHED_UNLOCK(curkse, curkse->k_kseg); - _kse_critical_leave(&curthread->tcb->tcb_tmbx); - - /* - * This thread is being resumed; check for cancellations. - */ - if (THR_NEED_ASYNC_CANCEL(curthread) && !THR_IN_CRITICAL(curthread)) { - uc = alloca(sizeof(ucontext_t)); - resume_once = 0; - THR_GETCONTEXT(uc); - if (resume_once == 0) { - resume_once = 1; - curthread->check_pending = 0; - thr_resume_check(curthread, uc); - } - } - THR_ACTIVATE_LAST_LOCK(curthread); -} - -/* - * This is the scheduler for a KSE which runs a scope system thread. - * The multi-thread KSE scheduler should also work for a single threaded - * KSE, but we use a separate scheduler so that it can be fine-tuned - * to be more efficient (and perhaps not need a separate stack for - * the KSE, allowing it to use the thread's stack). - */ - -static void -kse_sched_single(struct kse_mailbox *kmbx) -{ - struct kse *curkse; - struct pthread *curthread; - struct timespec ts; - sigset_t sigmask; - int i, sigseqno, level, first = 0; - - curkse = (struct kse *)kmbx->km_udata; - curthread = curkse->k_curthread; - - if (__predict_false((curkse->k_flags & KF_INITIALIZED) == 0)) { - /* Setup this KSEs specific data. */ - _kcb_set(curkse->k_kcb); - _tcb_set(curkse->k_kcb, curthread->tcb); - curkse->k_flags |= KF_INITIALIZED; - first = 1; - curthread->active = 1; - - /* Setup kernel signal masks for new thread. */ - __sys_sigprocmask(SIG_SETMASK, &curthread->sigmask, NULL); - /* - * Enter critical region, this is meanless for bound thread, - * It is used to let other code work, those code want mailbox - * to be cleared. - */ - (void)_kse_critical_enter(); - } else { - /* - * Bound thread always has tcb set, this prevent some - * code from blindly setting bound thread tcb to NULL, - * buggy code ? - */ - _tcb_set(curkse->k_kcb, curthread->tcb); - } - - curthread->critical_yield = 0; - curthread->need_switchout = 0; - - /* - * Lock the scheduling queue. - * - * There is no scheduling queue for single threaded KSEs, - * but we need a lock for protection regardless. - */ - if (curthread->lock_switch == 0) - KSE_SCHED_LOCK(curkse, curkse->k_kseg); - - /* - * This has to do the job of kse_switchout_thread(), only - * for a single threaded KSE/KSEG. - */ - - switch (curthread->state) { - case PS_MUTEX_WAIT: - case PS_COND_WAIT: - if (THR_NEED_CANCEL(curthread)) { - curthread->interrupted = 1; - curthread->continuation = _thr_finish_cancellation; - THR_SET_STATE(curthread, PS_RUNNING); - } - break; - - case PS_LOCKWAIT: - /* - * This state doesn't timeout. - */ - curthread->wakeup_time.tv_sec = -1; - curthread->wakeup_time.tv_nsec = -1; - level = curthread->locklevel - 1; - if (_LCK_GRANTED(&curthread->lockusers[level])) - THR_SET_STATE(curthread, PS_RUNNING); - break; - - case PS_DEAD: - /* Unlock the scheduling queue and exit the KSE and thread. */ - thr_cleanup(curkse, curthread); - KSE_SCHED_UNLOCK(curkse, curkse->k_kseg); - PANIC("bound thread shouldn't get here\n"); - break; - - case PS_JOIN: - if (THR_NEED_CANCEL(curthread)) { - curthread->join_status.thread = NULL; - THR_SET_STATE(curthread, PS_RUNNING); - } else { - /* - * This state doesn't timeout. - */ - curthread->wakeup_time.tv_sec = -1; - curthread->wakeup_time.tv_nsec = -1; - } - break; - - case PS_SUSPENDED: - if (THR_NEED_CANCEL(curthread)) { - curthread->interrupted = 1; - THR_SET_STATE(curthread, PS_RUNNING); - } else { - /* - * These states don't timeout. - */ - curthread->wakeup_time.tv_sec = -1; - curthread->wakeup_time.tv_nsec = -1; - } - break; - - case PS_RUNNING: - if ((curthread->flags & THR_FLAGS_SUSPENDED) != 0 && - !THR_NEED_CANCEL(curthread)) { - THR_SET_STATE(curthread, PS_SUSPENDED); - /* - * These states don't timeout. - */ - curthread->wakeup_time.tv_sec = -1; - curthread->wakeup_time.tv_nsec = -1; - } - break; - - case PS_SIGWAIT: - PANIC("bound thread does not have SIGWAIT state\n"); - - case PS_SLEEP_WAIT: - PANIC("bound thread does not have SLEEP_WAIT state\n"); - - case PS_SIGSUSPEND: - PANIC("bound thread does not have SIGSUSPEND state\n"); - - case PS_DEADLOCK: - /* - * These states don't timeout and don't need - * to be in the waiting queue. - */ - curthread->wakeup_time.tv_sec = -1; - curthread->wakeup_time.tv_nsec = -1; - break; - - default: - PANIC("Unknown state\n"); - break; - } - - while (curthread->state != PS_RUNNING) { - sigseqno = curkse->k_sigseqno; - if (curthread->check_pending != 0) { - /* - * Install pending signals into the frame, possible - * cause mutex or condvar backout. - */ - curthread->check_pending = 0; - SIGFILLSET(sigmask); - - /* - * Lock out kernel signal code when we are processing - * signals, and get a fresh copy of signal mask. - */ - __sys_sigprocmask(SIG_SETMASK, &sigmask, - &curthread->sigmask); - for (i = 1; i <= _SIG_MAXSIG; i++) { - if (SIGISMEMBER(curthread->sigmask, i)) - continue; - if (SIGISMEMBER(curthread->sigpend, i)) - (void)_thr_sig_add(curthread, i, - &curthread->siginfo[i-1]); - } - __sys_sigprocmask(SIG_SETMASK, &curthread->sigmask, - NULL); - /* The above code might make thread runnable */ - if (curthread->state == PS_RUNNING) - break; - } - THR_DEACTIVATE_LAST_LOCK(curthread); - kse_wait(curkse, curthread, sigseqno); - THR_ACTIVATE_LAST_LOCK(curthread); - if (curthread->wakeup_time.tv_sec >= 0) { - KSE_GET_TOD(curkse, &ts); - if (thr_timedout(curthread, &ts)) { - /* Indicate the thread timedout: */ - curthread->timeout = 1; - /* Make the thread runnable. */ - THR_SET_STATE(curthread, PS_RUNNING); - } - } - } - - if (curthread->lock_switch == 0) { - /* Unlock the scheduling queue. */ - KSE_SCHED_UNLOCK(curkse, curkse->k_kseg); - } - - DBG_MSG("Continuing bound thread %p\n", curthread); - if (first) { - _kse_critical_leave(&curthread->tcb->tcb_tmbx); - pthread_exit(curthread->start_routine(curthread->arg)); - } -} - -#ifdef DEBUG_THREAD_KERN -static void -dump_queues(struct kse *curkse) -{ - struct pthread *thread; - - DBG_MSG("Threads in waiting queue:\n"); - TAILQ_FOREACH(thread, &curkse->k_kseg->kg_schedq.sq_waitq, pqe) { - DBG_MSG(" thread %p, state %d, blocked %d\n", - thread, thread->state, thread->blocked); - } -} -#endif - -/* - * This is the scheduler for a KSE which runs multiple threads. - */ -static void -kse_sched_multi(struct kse_mailbox *kmbx) -{ - struct kse *curkse; - struct pthread *curthread, *td_wait; - int ret; - - curkse = (struct kse *)kmbx->km_udata; - THR_ASSERT(curkse->k_kcb->kcb_kmbx.km_curthread == NULL, - "Mailbox not null in kse_sched_multi"); - - /* Check for first time initialization: */ - if (__predict_false((curkse->k_flags & KF_INITIALIZED) == 0)) { - /* Setup this KSEs specific data. */ - _kcb_set(curkse->k_kcb); - - /* Set this before grabbing the context. */ - curkse->k_flags |= KF_INITIALIZED; - } - - /* - * No current thread anymore, calling _get_curthread in UTS - * should dump core - */ - _tcb_set(curkse->k_kcb, NULL); - - /* If this is an upcall; take the scheduler lock. */ - if (!KSE_IS_SWITCH(curkse)) - KSE_SCHED_LOCK(curkse, curkse->k_kseg); - else - KSE_CLEAR_SWITCH(curkse); - - if (KSE_IS_IDLE(curkse)) { - KSE_CLEAR_IDLE(curkse); - curkse->k_kseg->kg_idle_kses--; - } - - /* - * Now that the scheduler lock is held, get the current - * thread. The KSE's current thread cannot be safely - * examined without the lock because it could have returned - * as completed on another KSE. See kse_check_completed(). - */ - curthread = curkse->k_curthread; - - /* - * If the current thread was completed in another KSE, then - * it will be in the run queue. Don't mark it as being blocked. - */ - if ((curthread != NULL) && - ((curthread->flags & THR_FLAGS_IN_RUNQ) == 0) && - (curthread->need_switchout == 0)) { - /* - * Assume the current thread is blocked; when the - * completed threads are checked and if the current - * thread is among the completed, the blocked flag - * will be cleared. - */ - curthread->blocked = 1; - DBG_MSG("Running thread %p is now blocked in kernel.\n", - curthread); - } - - /* Check for any unblocked threads in the kernel. */ - kse_check_completed(curkse); - - /* - * Check for threads that have timed-out. - */ - kse_check_waitq(curkse); - - /* - * Switchout the current thread, if necessary, as the last step - * so that it is inserted into the run queue (if it's runnable) - * _after_ any other threads that were added to it above. - */ - if (curthread == NULL) - ; /* Nothing to do here. */ - else if ((curthread->need_switchout == 0) && DBG_CAN_RUN(curthread) && - (curthread->blocked == 0) && (THR_IN_CRITICAL(curthread))) { - /* - * Resume the thread and tell it to yield when - * it leaves the critical region. - */ - curthread->critical_yield = 1; - curthread->active = 1; - if ((curthread->flags & THR_FLAGS_IN_RUNQ) != 0) - KSE_RUNQ_REMOVE(curkse, curthread); - curkse->k_curthread = curthread; - curthread->kse = curkse; - DBG_MSG("Continuing thread %p in critical region\n", - curthread); - kse_wakeup_multi(curkse); - KSE_SCHED_UNLOCK(curkse, curkse->k_kseg); - ret = _thread_switch(curkse->k_kcb, curthread->tcb, 1); - if (ret != 0) - PANIC("Can't resume thread in critical region\n"); - } - else if ((curthread->flags & THR_FLAGS_IN_RUNQ) == 0) { - curthread->tcb->tcb_tmbx.tm_lwp = 0; - kse_switchout_thread(curkse, curthread); - } - curkse->k_curthread = NULL; - -#ifdef DEBUG_THREAD_KERN - dump_queues(curkse); -#endif - - /* Check if there are no threads ready to run: */ - while (((curthread = KSE_RUNQ_FIRST(curkse)) == NULL) && - (curkse->k_kseg->kg_threadcount != 0) && - ((curkse->k_flags & KF_TERMINATED) == 0)) { - /* - * Wait for a thread to become active or until there are - * no more threads. - */ - td_wait = KSE_WAITQ_FIRST(curkse); - kse_wait(curkse, td_wait, 0); - kse_check_completed(curkse); - kse_check_waitq(curkse); - } - - /* Check for no more threads: */ - if ((curkse->k_kseg->kg_threadcount == 0) || - ((curkse->k_flags & KF_TERMINATED) != 0)) { - /* - * Normally this shouldn't return, but it will if there - * are other KSEs running that create new threads that - * are assigned to this KSE[G]. For instance, if a scope - * system thread were to create a scope process thread - * and this kse[g] is the initial kse[g], then that newly - * created thread would be assigned to us (the initial - * kse[g]). - */ - kse_wakeup_multi(curkse); - KSE_SCHED_UNLOCK(curkse, curkse->k_kseg); - kse_fini(curkse); - /* never returns */ - } - - THR_ASSERT(curthread != NULL, - "Return from kse_wait/fini without thread."); - THR_ASSERT(curthread->state != PS_DEAD, - "Trying to resume dead thread!"); - KSE_RUNQ_REMOVE(curkse, curthread); - - /* - * Make the selected thread the current thread. - */ - curkse->k_curthread = curthread; - - /* - * Make sure the current thread's kse points to this kse. - */ - curthread->kse = curkse; - - /* - * Reset the time slice if this thread is running for the first - * time or running again after using its full time slice allocation. - */ - if (curthread->slice_usec == -1) - curthread->slice_usec = 0; - - /* Mark the thread active. */ - curthread->active = 1; - - /* - * The thread's current signal frame will only be NULL if it - * is being resumed after being blocked in the kernel. In - * this case, and if the thread needs to run down pending - * signals or needs a cancellation check, we need to add a - * signal frame to the thread's context. - */ - if (curthread->lock_switch == 0 && curthread->state == PS_RUNNING && - (curthread->check_pending != 0 || - THR_NEED_ASYNC_CANCEL(curthread)) && - !THR_IN_CRITICAL(curthread)) { - curthread->check_pending = 0; - signalcontext(&curthread->tcb->tcb_tmbx.tm_context, 0, - (__sighandler_t *)thr_resume_wrapper); - } - kse_wakeup_multi(curkse); - /* - * Continue the thread at its current frame: - */ - if (curthread->lock_switch != 0) { - /* - * This thread came from a scheduler switch; it will - * unlock the scheduler lock and set the mailbox. - */ - ret = _thread_switch(curkse->k_kcb, curthread->tcb, 0); - } else { - /* This thread won't unlock the scheduler lock. */ - KSE_SCHED_UNLOCK(curkse, curkse->k_kseg); - ret = _thread_switch(curkse->k_kcb, curthread->tcb, 1); - } - if (ret != 0) - PANIC("Thread has returned from _thread_switch"); - - /* This point should not be reached. */ - PANIC("Thread has returned from _thread_switch"); -} - -static void -thr_resume_wrapper(int sig __unused, siginfo_t *siginfo __unused, - ucontext_t *ucp) -{ - struct pthread *curthread = _get_curthread(); - struct kse *curkse; - int ret, err_save = errno; - - DBG_MSG(">>> sig wrapper\n"); - if (curthread->lock_switch) - PANIC("thr_resume_wrapper, lock_switch != 0\n"); - thr_resume_check(curthread, ucp); - errno = err_save; - _kse_critical_enter(); - curkse = curthread->kse; - curthread->tcb->tcb_tmbx.tm_context = *ucp; - ret = _thread_switch(curkse->k_kcb, curthread->tcb, 1); - if (ret != 0) - PANIC("thr_resume_wrapper: thread has returned " - "from _thread_switch"); - /* THR_SETCONTEXT(ucp); */ /* not work, why ? */ -} - -static void -thr_resume_check(struct pthread *curthread, ucontext_t *ucp) -{ - _thr_sig_rundown(curthread, ucp); - - if (THR_NEED_ASYNC_CANCEL(curthread)) - pthread_testcancel(); -} - -/* - * Clean up a thread. This must be called with the thread's KSE - * scheduling lock held. The thread must be a thread from the - * KSE's group. - */ -static void -thr_cleanup(struct kse *curkse, struct pthread *thread) -{ - struct pthread *joiner; - struct kse_mailbox *kmbx = NULL; - int sys_scope; - - thread->active = 0; - thread->need_switchout = 0; - thread->lock_switch = 0; - thread->check_pending = 0; - - if ((joiner = thread->joiner) != NULL) { - /* Joinee scheduler lock held; joiner won't leave. */ - if (joiner->kseg == curkse->k_kseg) { - if (joiner->join_status.thread == thread) { - joiner->join_status.thread = NULL; - joiner->join_status.ret = thread->ret; - (void)_thr_setrunnable_unlocked(joiner); - } - } else { - KSE_SCHED_UNLOCK(curkse, curkse->k_kseg); - /* The joiner may have removed itself and exited. */ - if (_thr_ref_add(thread, joiner, 0) == 0) { - KSE_SCHED_LOCK(curkse, joiner->kseg); - if (joiner->join_status.thread == thread) { - joiner->join_status.thread = NULL; - joiner->join_status.ret = thread->ret; - kmbx = _thr_setrunnable_unlocked(joiner); - } - KSE_SCHED_UNLOCK(curkse, joiner->kseg); - _thr_ref_delete(thread, joiner); - if (kmbx != NULL) - kse_wakeup(kmbx); - } - KSE_SCHED_LOCK(curkse, curkse->k_kseg); - } - thread->attr.flags |= PTHREAD_DETACHED; - } - - if (!(sys_scope = (thread->attr.flags & PTHREAD_SCOPE_SYSTEM))) { - /* - * Remove the thread from the KSEG's list of threads. - */ - KSEG_THRQ_REMOVE(thread->kseg, thread); - /* - * Migrate the thread to the main KSE so that this - * KSE and KSEG can be cleaned when their last thread - * exits. - */ - thread->kseg = _kse_initial->k_kseg; - thread->kse = _kse_initial; - } - - /* - * We can't hold the thread list lock while holding the - * scheduler lock. - */ - KSE_SCHED_UNLOCK(curkse, curkse->k_kseg); - DBG_MSG("Adding thread %p to GC list\n", thread); - KSE_LOCK_ACQUIRE(curkse, &_thread_list_lock); - thread->tlflags |= TLFLAGS_GC_SAFE; - THR_GCLIST_ADD(thread); - KSE_LOCK_RELEASE(curkse, &_thread_list_lock); - if (sys_scope) { - /* - * System scope thread is single thread group, - * when thread is exited, its kse and ksegrp should - * be recycled as well. - * kse upcall stack belongs to thread, clear it here. - */ - curkse->k_stack.ss_sp = 0; - curkse->k_stack.ss_size = 0; - kse_exit(); - PANIC("kse_exit() failed for system scope thread"); - } - KSE_SCHED_LOCK(curkse, curkse->k_kseg); -} - -void -_thr_gc(struct pthread *curthread) -{ - thread_gc(curthread); - kse_gc(curthread); - kseg_gc(curthread); -} - -static void -thread_gc(struct pthread *curthread) -{ - struct pthread *td, *td_next; - kse_critical_t crit; - TAILQ_HEAD(, pthread) worklist; - - TAILQ_INIT(&worklist); - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock); - - /* Check the threads waiting for GC. */ - for (td = TAILQ_FIRST(&_thread_gc_list); td != NULL; td = td_next) { - td_next = TAILQ_NEXT(td, gcle); - if ((td->tlflags & TLFLAGS_GC_SAFE) == 0) - continue; - else if (((td->attr.flags & PTHREAD_SCOPE_SYSTEM) != 0) && - ((td->kse->k_kcb->kcb_kmbx.km_flags & KMF_DONE) == 0)) { - /* - * The thread and KSE are operating on the same - * stack. Wait for the KSE to exit before freeing - * the thread's stack as well as everything else. - */ - continue; - } - /* - * Remove the thread from the GC list. If the thread - * isn't yet detached, it will get added back to the - * GC list at a later time. - */ - THR_GCLIST_REMOVE(td); - DBG_MSG("Freeing thread %p stack\n", td); - /* - * We can free the thread stack since it's no longer - * in use. - */ - _thr_stack_free(&td->attr); - if (((td->attr.flags & PTHREAD_DETACHED) != 0) && - (td->refcount == 0)) { - /* - * The thread has detached and is no longer - * referenced. It is safe to remove all - * remnants of the thread. - */ - THR_LIST_REMOVE(td); - TAILQ_INSERT_HEAD(&worklist, td, gcle); - } - } - KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock); - _kse_critical_leave(crit); - - while ((td = TAILQ_FIRST(&worklist)) != NULL) { - TAILQ_REMOVE(&worklist, td, gcle); - /* - * XXX we don't free initial thread and its kse - * (if thread is a bound thread), because there might - * have some code referencing initial thread and kse. - */ - if (td == _thr_initial) { - DBG_MSG("Initial thread won't be freed\n"); - continue; - } - - if ((td->attr.flags & PTHREAD_SCOPE_SYSTEM) != 0) { - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock); - kse_free_unlocked(td->kse); - kseg_free_unlocked(td->kseg); - KSE_LOCK_RELEASE(curthread->kse, &kse_lock); - _kse_critical_leave(crit); - } - DBG_MSG("Freeing thread %p\n", td); - _thr_free(curthread, td); - } -} - -static void -kse_gc(struct pthread *curthread) -{ - kse_critical_t crit; - TAILQ_HEAD(, kse) worklist; - struct kse *kse; - - if (free_kse_count <= MAX_CACHED_KSES) - return; - TAILQ_INIT(&worklist); - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock); - while (free_kse_count > MAX_CACHED_KSES) { - kse = TAILQ_FIRST(&free_kseq); - TAILQ_REMOVE(&free_kseq, kse, k_qe); - TAILQ_INSERT_HEAD(&worklist, kse, k_qe); - free_kse_count--; - } - KSE_LOCK_RELEASE(curthread->kse, &kse_lock); - _kse_critical_leave(crit); - - while ((kse = TAILQ_FIRST(&worklist))) { - TAILQ_REMOVE(&worklist, kse, k_qe); - kse_destroy(kse); - } -} - -static void -kseg_gc(struct pthread *curthread) -{ - kse_critical_t crit; - TAILQ_HEAD(, kse_group) worklist; - struct kse_group *kseg; - - if (free_kseg_count <= MAX_CACHED_KSEGS) - return; - TAILQ_INIT(&worklist); - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock); - while (free_kseg_count > MAX_CACHED_KSEGS) { - kseg = TAILQ_FIRST(&free_kse_groupq); - TAILQ_REMOVE(&free_kse_groupq, kseg, kg_qe); - free_kseg_count--; - TAILQ_INSERT_HEAD(&worklist, kseg, kg_qe); - } - KSE_LOCK_RELEASE(curthread->kse, &kse_lock); - _kse_critical_leave(crit); - - while ((kseg = TAILQ_FIRST(&worklist))) { - TAILQ_REMOVE(&worklist, kseg, kg_qe); - kseg_destroy(kseg); - } -} - -/* - * Only new threads that are running or suspended may be scheduled. - */ -int -_thr_schedule_add(struct pthread *curthread, struct pthread *newthread) -{ - kse_critical_t crit; - int ret; - - /* Add the new thread. */ - thr_link(newthread); - - /* - * If this is the first time creating a thread, make sure - * the mailbox is set for the current thread. - */ - if ((newthread->attr.flags & PTHREAD_SCOPE_SYSTEM) != 0) { - /* We use the thread's stack as the KSE's stack. */ - newthread->kse->k_kcb->kcb_kmbx.km_stack.ss_sp = - newthread->attr.stackaddr_attr; - newthread->kse->k_kcb->kcb_kmbx.km_stack.ss_size = - newthread->attr.stacksize_attr; - - /* - * No need to lock the scheduling queue since the - * KSE/KSEG pair have not yet been started. - */ - KSEG_THRQ_ADD(newthread->kseg, newthread); - /* this thread never gives up kse */ - newthread->active = 1; - newthread->kse->k_curthread = newthread; - newthread->kse->k_kcb->kcb_kmbx.km_flags = KMF_BOUND; - newthread->kse->k_kcb->kcb_kmbx.km_func = - (kse_func_t *)kse_sched_single; - newthread->kse->k_kcb->kcb_kmbx.km_quantum = 0; - KSE_SET_MBOX(newthread->kse, newthread); - /* - * This thread needs a new KSE and KSEG. - */ - newthread->kse->k_flags &= ~KF_INITIALIZED; - newthread->kse->k_flags |= KF_STARTED; - /* Fire up! */ - ret = kse_create(&newthread->kse->k_kcb->kcb_kmbx, 1); - if (ret != 0) - ret = errno; - } - else { - /* - * Lock the KSE and add the new thread to its list of - * assigned threads. If the new thread is runnable, also - * add it to the KSE's run queue. - */ - crit = _kse_critical_enter(); - KSE_SCHED_LOCK(curthread->kse, newthread->kseg); - KSEG_THRQ_ADD(newthread->kseg, newthread); - if (newthread->state == PS_RUNNING) - THR_RUNQ_INSERT_TAIL(newthread); - if ((newthread->kse->k_flags & KF_STARTED) == 0) { - /* - * This KSE hasn't been started yet. Start it - * outside of holding the lock. - */ - newthread->kse->k_flags |= KF_STARTED; - newthread->kse->k_kcb->kcb_kmbx.km_func = - (kse_func_t *)kse_sched_multi; - newthread->kse->k_kcb->kcb_kmbx.km_flags = 0; - kse_create(&newthread->kse->k_kcb->kcb_kmbx, 0); - } else if ((newthread->state == PS_RUNNING) && - KSE_IS_IDLE(newthread->kse)) { - /* - * The thread is being scheduled on another KSEG. - */ - kse_wakeup_one(newthread); - } - KSE_SCHED_UNLOCK(curthread->kse, newthread->kseg); - _kse_critical_leave(crit); - ret = 0; - } - if (ret != 0) - thr_unlink(newthread); - - return (ret); -} - -void -kse_waitq_insert(struct pthread *thread) -{ - struct pthread *td; - - if (thread->wakeup_time.tv_sec == -1) - TAILQ_INSERT_TAIL(&thread->kse->k_schedq->sq_waitq, thread, - pqe); - else { - td = TAILQ_FIRST(&thread->kse->k_schedq->sq_waitq); - while ((td != NULL) && (td->wakeup_time.tv_sec != -1) && - ((td->wakeup_time.tv_sec < thread->wakeup_time.tv_sec) || - ((td->wakeup_time.tv_sec == thread->wakeup_time.tv_sec) && - (td->wakeup_time.tv_nsec <= thread->wakeup_time.tv_nsec)))) - td = TAILQ_NEXT(td, pqe); - if (td == NULL) - TAILQ_INSERT_TAIL(&thread->kse->k_schedq->sq_waitq, - thread, pqe); - else - TAILQ_INSERT_BEFORE(td, thread, pqe); - } - thread->flags |= THR_FLAGS_IN_WAITQ; -} - -/* - * This must be called with the scheduling lock held. - */ -static void -kse_check_completed(struct kse *kse) -{ - struct pthread *thread; - struct kse_thr_mailbox *completed; - int sig; - - if ((completed = kse->k_kcb->kcb_kmbx.km_completed) != NULL) { - kse->k_kcb->kcb_kmbx.km_completed = NULL; - while (completed != NULL) { - thread = completed->tm_udata; - DBG_MSG("Found completed thread %p, name %s\n", - thread, - (thread->name == NULL) ? "none" : thread->name); - thread->blocked = 0; - if (thread != kse->k_curthread) { - thr_accounting(thread); - if ((thread->flags & THR_FLAGS_SUSPENDED) != 0) - THR_SET_STATE(thread, PS_SUSPENDED); - else - KSE_RUNQ_INSERT_TAIL(kse, thread); - if ((thread->kse != kse) && - (thread->kse->k_curthread == thread)) { - /* - * Remove this thread from its - * previous KSE so that it (the KSE) - * doesn't think it is still active. - */ - thread->kse->k_curthread = NULL; - thread->active = 0; - } - } - if ((sig = thread->tcb->tcb_tmbx.tm_syncsig.si_signo) - != 0) { - if (SIGISMEMBER(thread->sigmask, sig)) - SIGADDSET(thread->sigpend, sig); - else if (THR_IN_CRITICAL(thread)) - kse_thr_interrupt(NULL, KSE_INTR_SIGEXIT, sig); - else - (void)_thr_sig_add(thread, sig, - &thread->tcb->tcb_tmbx.tm_syncsig); - thread->tcb->tcb_tmbx.tm_syncsig.si_signo = 0; - } - completed = completed->tm_next; - } - } -} - -/* - * This must be called with the scheduling lock held. - */ -static void -kse_check_waitq(struct kse *kse) -{ - struct pthread *pthread; - struct timespec ts; - - KSE_GET_TOD(kse, &ts); - - /* - * Wake up threads that have timedout. This has to be - * done before adding the current thread to the run queue - * so that a CPU intensive thread doesn't get preference - * over waiting threads. - */ - while (((pthread = KSE_WAITQ_FIRST(kse)) != NULL) && - thr_timedout(pthread, &ts)) { - /* Remove the thread from the wait queue: */ - KSE_WAITQ_REMOVE(kse, pthread); - DBG_MSG("Found timedout thread %p in waitq\n", pthread); - - /* Indicate the thread timedout: */ - pthread->timeout = 1; - - /* Add the thread to the priority queue: */ - if ((pthread->flags & THR_FLAGS_SUSPENDED) != 0) - THR_SET_STATE(pthread, PS_SUSPENDED); - else { - THR_SET_STATE(pthread, PS_RUNNING); - KSE_RUNQ_INSERT_TAIL(kse, pthread); - } - } -} - -static int -thr_timedout(struct pthread *thread, struct timespec *curtime) -{ - if (thread->wakeup_time.tv_sec < 0) - return (0); - else if (thread->wakeup_time.tv_sec > curtime->tv_sec) - return (0); - else if ((thread->wakeup_time.tv_sec == curtime->tv_sec) && - (thread->wakeup_time.tv_nsec > curtime->tv_nsec)) - return (0); - else - return (1); -} - -/* - * This must be called with the scheduling lock held. - * - * Each thread has a time slice, a wakeup time (used when it wants - * to wait for a specified amount of time), a run state, and an - * active flag. - * - * When a thread gets run by the scheduler, the active flag is - * set to non-zero (1). When a thread performs an explicit yield - * or schedules a state change, it enters the scheduler and the - * active flag is cleared. When the active flag is still seen - * set in the scheduler, that means that the thread is blocked in - * the kernel (because it is cleared before entering the scheduler - * in all other instances). - * - * The wakeup time is only set for those states that can timeout. - * It is set to (-1, -1) for all other instances. - * - * The thread's run state, aside from being useful when debugging, - * is used to place the thread in an appropriate queue. There - * are 2 basic queues: - * - * o run queue - queue ordered by priority for all threads - * that are runnable - * o waiting queue - queue sorted by wakeup time for all threads - * that are not otherwise runnable (not blocked - * in kernel, not waiting for locks) - * - * The thread's time slice is used for round-robin scheduling - * (the default scheduling policy). While a SCHED_RR thread - * is runnable it's time slice accumulates. When it reaches - * the time slice interval, it gets reset and added to the end - * of the queue of threads at its priority. When a thread no - * longer becomes runnable (blocks in kernel, waits, etc), its - * time slice is reset. - * - * The job of kse_switchout_thread() is to handle all of the above. - */ -static void -kse_switchout_thread(struct kse *kse, struct pthread *thread) -{ - int level; - int i; - int restart; - siginfo_t siginfo; - - /* - * Place the currently running thread into the - * appropriate queue(s). - */ - DBG_MSG("Switching out thread %p, state %d\n", thread, thread->state); - - THR_DEACTIVATE_LAST_LOCK(thread); - if (thread->blocked != 0) { - thread->active = 0; - thread->need_switchout = 0; - /* This thread must have blocked in the kernel. */ - /* - * Check for pending signals and cancellation for - * this thread to see if we need to interrupt it - * in the kernel. - */ - if (THR_NEED_CANCEL(thread)) { - kse_thr_interrupt(&thread->tcb->tcb_tmbx, - KSE_INTR_INTERRUPT, 0); - } else if (thread->check_pending != 0) { - for (i = 1; i <= _SIG_MAXSIG; ++i) { - if (SIGISMEMBER(thread->sigpend, i) && - !SIGISMEMBER(thread->sigmask, i)) { - restart = _thread_sigact[i - 1].sa_flags & SA_RESTART; - kse_thr_interrupt(&thread->tcb->tcb_tmbx, - restart ? KSE_INTR_RESTART : KSE_INTR_INTERRUPT, 0); - break; - } - } - } - } - else { - switch (thread->state) { - case PS_MUTEX_WAIT: - case PS_COND_WAIT: - if (THR_NEED_CANCEL(thread)) { - thread->interrupted = 1; - thread->continuation = _thr_finish_cancellation; - THR_SET_STATE(thread, PS_RUNNING); - } else { - /* Insert into the waiting queue: */ - KSE_WAITQ_INSERT(kse, thread); - } - break; - - case PS_LOCKWAIT: - /* - * This state doesn't timeout. - */ - thread->wakeup_time.tv_sec = -1; - thread->wakeup_time.tv_nsec = -1; - level = thread->locklevel - 1; - if (!_LCK_GRANTED(&thread->lockusers[level])) - KSE_WAITQ_INSERT(kse, thread); - else - THR_SET_STATE(thread, PS_RUNNING); - break; - - case PS_SLEEP_WAIT: - case PS_SIGWAIT: - if (THR_NEED_CANCEL(thread)) { - thread->interrupted = 1; - THR_SET_STATE(thread, PS_RUNNING); - } else { - KSE_WAITQ_INSERT(kse, thread); - } - break; - - case PS_JOIN: - if (THR_NEED_CANCEL(thread)) { - thread->join_status.thread = NULL; - THR_SET_STATE(thread, PS_RUNNING); - } else { - /* - * This state doesn't timeout. - */ - thread->wakeup_time.tv_sec = -1; - thread->wakeup_time.tv_nsec = -1; - - /* Insert into the waiting queue: */ - KSE_WAITQ_INSERT(kse, thread); - } - break; - - case PS_SIGSUSPEND: - case PS_SUSPENDED: - if (THR_NEED_CANCEL(thread)) { - thread->interrupted = 1; - THR_SET_STATE(thread, PS_RUNNING); - } else { - /* - * These states don't timeout. - */ - thread->wakeup_time.tv_sec = -1; - thread->wakeup_time.tv_nsec = -1; - - /* Insert into the waiting queue: */ - KSE_WAITQ_INSERT(kse, thread); - } - break; - - case PS_DEAD: - /* - * The scheduler is operating on a different - * stack. It is safe to do garbage collecting - * here. - */ - thr_cleanup(kse, thread); - return; - break; - - case PS_RUNNING: - if ((thread->flags & THR_FLAGS_SUSPENDED) != 0 && - !THR_NEED_CANCEL(thread)) - THR_SET_STATE(thread, PS_SUSPENDED); - break; - - case PS_DEADLOCK: - /* - * These states don't timeout. - */ - thread->wakeup_time.tv_sec = -1; - thread->wakeup_time.tv_nsec = -1; - - /* Insert into the waiting queue: */ - KSE_WAITQ_INSERT(kse, thread); - break; - - default: - PANIC("Unknown state\n"); - break; - } - - thr_accounting(thread); - if (thread->state == PS_RUNNING) { - if (thread->slice_usec == -1) { - /* - * The thread exceeded its time quantum or - * it yielded the CPU; place it at the tail - * of the queue for its priority. - */ - KSE_RUNQ_INSERT_TAIL(kse, thread); - } else { - /* - * The thread hasn't exceeded its interval - * Place it at the head of the queue for its - * priority. - */ - KSE_RUNQ_INSERT_HEAD(kse, thread); - } - } - } - thread->active = 0; - thread->need_switchout = 0; - if (thread->check_pending != 0) { - /* Install pending signals into the frame. */ - thread->check_pending = 0; - KSE_LOCK_ACQUIRE(kse, &_thread_signal_lock); - for (i = 1; i <= _SIG_MAXSIG; i++) { - if (SIGISMEMBER(thread->sigmask, i)) - continue; - if (SIGISMEMBER(thread->sigpend, i)) - (void)_thr_sig_add(thread, i, - &thread->siginfo[i-1]); - else if (SIGISMEMBER(_thr_proc_sigpending, i) && - _thr_getprocsig_unlocked(i, &siginfo)) { - (void)_thr_sig_add(thread, i, &siginfo); - } - } - KSE_LOCK_RELEASE(kse, &_thread_signal_lock); - } -} - -/* - * This function waits for the smallest timeout value of any waiting - * thread, or until it receives a message from another KSE. - * - * This must be called with the scheduling lock held. - */ -static void -kse_wait(struct kse *kse, struct pthread *td_wait, int sigseqno) -{ - struct timespec ts, ts_sleep; - int saved_flags; - - if ((td_wait == NULL) || (td_wait->wakeup_time.tv_sec < 0)) { - /* Limit sleep to no more than 1 minute. */ - ts_sleep.tv_sec = 60; - ts_sleep.tv_nsec = 0; - } else { - KSE_GET_TOD(kse, &ts); - TIMESPEC_SUB(&ts_sleep, &td_wait->wakeup_time, &ts); - if (ts_sleep.tv_sec > 60) { - ts_sleep.tv_sec = 60; - ts_sleep.tv_nsec = 0; - } - } - /* Don't sleep for negative times. */ - if ((ts_sleep.tv_sec >= 0) && (ts_sleep.tv_nsec >= 0)) { - KSE_SET_IDLE(kse); - kse->k_kseg->kg_idle_kses++; - KSE_SCHED_UNLOCK(kse, kse->k_kseg); - if ((kse->k_kseg->kg_flags & KGF_SINGLE_THREAD) && - (kse->k_sigseqno != sigseqno)) - ; /* don't sleep */ - else { - saved_flags = kse->k_kcb->kcb_kmbx.km_flags; - kse->k_kcb->kcb_kmbx.km_flags |= KMF_NOUPCALL; - kse_release(&ts_sleep); - kse->k_kcb->kcb_kmbx.km_flags = saved_flags; - } - KSE_SCHED_LOCK(kse, kse->k_kseg); - if (KSE_IS_IDLE(kse)) { - KSE_CLEAR_IDLE(kse); - kse->k_kseg->kg_idle_kses--; - } - } -} - -/* - * Avoid calling this kse_exit() so as not to confuse it with the - * system call of the same name. - */ -static void -kse_fini(struct kse *kse) -{ - /* struct kse_group *free_kseg = NULL; */ - struct timespec ts; - struct pthread *td; - - /* - * Check to see if this is one of the main kses. - */ - if (kse->k_kseg != _kse_initial->k_kseg) { - PANIC("shouldn't get here"); - /* This is for supporting thread groups. */ -#ifdef NOT_YET - /* Remove this KSE from the KSEG's list of KSEs. */ - KSE_SCHED_LOCK(kse, kse->k_kseg); - TAILQ_REMOVE(&kse->k_kseg->kg_kseq, kse, k_kgqe); - kse->k_kseg->kg_ksecount--; - if (TAILQ_EMPTY(&kse->k_kseg->kg_kseq)) - free_kseg = kse->k_kseg; - KSE_SCHED_UNLOCK(kse, kse->k_kseg); - - /* - * Add this KSE to the list of free KSEs along with - * the KSEG if is now orphaned. - */ - KSE_LOCK_ACQUIRE(kse, &kse_lock); - if (free_kseg != NULL) - kseg_free_unlocked(free_kseg); - kse_free_unlocked(kse); - KSE_LOCK_RELEASE(kse, &kse_lock); - kse_exit(); - /* Never returns. */ - PANIC("kse_exit()"); -#endif - } else { - /* - * We allow program to kill kse in initial group (by - * lowering the concurrency). - */ - if ((kse != _kse_initial) && - ((kse->k_flags & KF_TERMINATED) != 0)) { - KSE_SCHED_LOCK(kse, kse->k_kseg); - TAILQ_REMOVE(&kse->k_kseg->kg_kseq, kse, k_kgqe); - kse->k_kseg->kg_ksecount--; - /* - * Migrate thread to _kse_initial if its lastest - * kse it ran on is the kse. - */ - td = TAILQ_FIRST(&kse->k_kseg->kg_threadq); - while (td != NULL) { - if (td->kse == kse) - td->kse = _kse_initial; - td = TAILQ_NEXT(td, kle); - } - KSE_SCHED_UNLOCK(kse, kse->k_kseg); - KSE_LOCK_ACQUIRE(kse, &kse_lock); - kse_free_unlocked(kse); - KSE_LOCK_RELEASE(kse, &kse_lock); - /* Make sure there is always at least one is awake */ - KSE_WAKEUP(_kse_initial); - kse_exit(); - /* Never returns. */ - PANIC("kse_exit() failed for initial kseg"); - } - KSE_SCHED_LOCK(kse, kse->k_kseg); - KSE_SET_IDLE(kse); - kse->k_kseg->kg_idle_kses++; - KSE_SCHED_UNLOCK(kse, kse->k_kseg); - ts.tv_sec = 120; - ts.tv_nsec = 0; - kse->k_kcb->kcb_kmbx.km_flags = 0; - kse_release(&ts); - /* Never reach */ - } -} - -void -_thr_set_timeout(const struct timespec *timeout) -{ - struct pthread *curthread = _get_curthread(); - struct timespec ts; - - /* Reset the timeout flag for the running thread: */ - curthread->timeout = 0; - - /* Check if the thread is to wait forever: */ - if (timeout == NULL) { - /* - * Set the wakeup time to something that can be recognised as - * different to an actual time of day: - */ - curthread->wakeup_time.tv_sec = -1; - curthread->wakeup_time.tv_nsec = -1; - } - /* Check if no waiting is required: */ - else if ((timeout->tv_sec == 0) && (timeout->tv_nsec == 0)) { - /* Set the wake up time to 'immediately': */ - curthread->wakeup_time.tv_sec = 0; - curthread->wakeup_time.tv_nsec = 0; - } else { - /* Calculate the time for the current thread to wakeup: */ - KSE_GET_TOD(curthread->kse, &ts); - TIMESPEC_ADD(&curthread->wakeup_time, &ts, timeout); - } -} - -void -_thr_panic_exit(char *file, int line, char *msg) -{ - char buf[256]; - - snprintf(buf, sizeof(buf), "(%s:%d) %s\n", file, line, msg); - __sys_write(2, buf, strlen(buf)); - abort(); -} - -void -_thr_setrunnable(struct pthread *curthread, struct pthread *thread) -{ - kse_critical_t crit; - struct kse_mailbox *kmbx; - - crit = _kse_critical_enter(); - KSE_SCHED_LOCK(curthread->kse, thread->kseg); - kmbx = _thr_setrunnable_unlocked(thread); - KSE_SCHED_UNLOCK(curthread->kse, thread->kseg); - _kse_critical_leave(crit); - if ((kmbx != NULL) && (__isthreaded != 0)) - kse_wakeup(kmbx); -} - -struct kse_mailbox * -_thr_setrunnable_unlocked(struct pthread *thread) -{ - struct kse_mailbox *kmbx = NULL; - - if ((thread->kseg->kg_flags & KGF_SINGLE_THREAD) != 0) { - /* No silly queues for these threads. */ - if ((thread->flags & THR_FLAGS_SUSPENDED) != 0) - THR_SET_STATE(thread, PS_SUSPENDED); - else { - THR_SET_STATE(thread, PS_RUNNING); - kmbx = kse_wakeup_one(thread); - } - - } else if (thread->state != PS_RUNNING) { - if ((thread->flags & THR_FLAGS_IN_WAITQ) != 0) - KSE_WAITQ_REMOVE(thread->kse, thread); - if ((thread->flags & THR_FLAGS_SUSPENDED) != 0) - THR_SET_STATE(thread, PS_SUSPENDED); - else { - THR_SET_STATE(thread, PS_RUNNING); - if ((thread->blocked == 0) && (thread->active == 0) && - (thread->flags & THR_FLAGS_IN_RUNQ) == 0) - THR_RUNQ_INSERT_TAIL(thread); - /* - * XXX - Threads are not yet assigned to specific - * KSEs; they are assigned to the KSEG. So - * the fact that a thread's KSE is waiting - * doesn't necessarily mean that it will be - * the KSE that runs the thread after the - * lock is granted. But we don't know if the - * other KSEs within the same KSEG are also - * in a waiting state or not so we err on the - * side of caution and wakeup the thread's - * last known KSE. We ensure that the - * threads KSE doesn't change while it's - * scheduling lock is held so it is safe to - * reference it (the KSE). If the KSE wakes - * up and doesn't find any more work it will - * again go back to waiting so no harm is - * done. - */ - kmbx = kse_wakeup_one(thread); - } - } - return (kmbx); -} - -static struct kse_mailbox * -kse_wakeup_one(struct pthread *thread) -{ - struct kse *ke; - - if (KSE_IS_IDLE(thread->kse)) { - KSE_CLEAR_IDLE(thread->kse); - thread->kseg->kg_idle_kses--; - return (&thread->kse->k_kcb->kcb_kmbx); - } else { - TAILQ_FOREACH(ke, &thread->kseg->kg_kseq, k_kgqe) { - if (KSE_IS_IDLE(ke)) { - KSE_CLEAR_IDLE(ke); - ke->k_kseg->kg_idle_kses--; - return (&ke->k_kcb->kcb_kmbx); - } - } - } - return (NULL); -} - -static void -kse_wakeup_multi(struct kse *curkse) -{ - struct kse *ke; - int tmp; - - if ((tmp = KSE_RUNQ_THREADS(curkse)) && curkse->k_kseg->kg_idle_kses) { - TAILQ_FOREACH(ke, &curkse->k_kseg->kg_kseq, k_kgqe) { - if (KSE_IS_IDLE(ke)) { - KSE_CLEAR_IDLE(ke); - ke->k_kseg->kg_idle_kses--; - KSE_WAKEUP(ke); - if (--tmp == 0) - break; - } - } - } -} - -/* - * Allocate a new KSEG. - * - * We allow the current thread to be NULL in the case that this - * is the first time a KSEG is being created (library initialization). - * In this case, we don't need to (and can't) take any locks. - */ -struct kse_group * -_kseg_alloc(struct pthread *curthread) -{ - struct kse_group *kseg = NULL; - kse_critical_t crit; - - if ((curthread != NULL) && (free_kseg_count > 0)) { - /* Use the kse lock for the kseg queue. */ - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock); - if ((kseg = TAILQ_FIRST(&free_kse_groupq)) != NULL) { - TAILQ_REMOVE(&free_kse_groupq, kseg, kg_qe); - free_kseg_count--; - active_kseg_count++; - TAILQ_INSERT_TAIL(&active_kse_groupq, kseg, kg_qe); - } - KSE_LOCK_RELEASE(curthread->kse, &kse_lock); - _kse_critical_leave(crit); - if (kseg) - kseg_reinit(kseg); - } - - /* - * If requested, attempt to allocate a new KSE group only if the - * KSE allocation was successful and a KSE group wasn't found in - * the free list. - */ - if ((kseg == NULL) && - ((kseg = (struct kse_group *)malloc(sizeof(*kseg))) != NULL)) { - if (_pq_alloc(&kseg->kg_schedq.sq_runq, - THR_MIN_PRIORITY, THR_LAST_PRIORITY) != 0) { - free(kseg); - kseg = NULL; - } else { - kseg_init(kseg); - /* Add the KSEG to the list of active KSEGs. */ - if (curthread != NULL) { - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock); - active_kseg_count++; - TAILQ_INSERT_TAIL(&active_kse_groupq, - kseg, kg_qe); - KSE_LOCK_RELEASE(curthread->kse, &kse_lock); - _kse_critical_leave(crit); - } else { - active_kseg_count++; - TAILQ_INSERT_TAIL(&active_kse_groupq, - kseg, kg_qe); - } - } - } - return (kseg); -} - -static void -kseg_init(struct kse_group *kseg) -{ - kseg_reinit(kseg); - _lock_init(&kseg->kg_lock, LCK_ADAPTIVE, _kse_lock_wait, - _kse_lock_wakeup, calloc); -} - -static void -kseg_reinit(struct kse_group *kseg) -{ - TAILQ_INIT(&kseg->kg_kseq); - TAILQ_INIT(&kseg->kg_threadq); - TAILQ_INIT(&kseg->kg_schedq.sq_waitq); - kseg->kg_threadcount = 0; - kseg->kg_ksecount = 0; - kseg->kg_idle_kses = 0; - kseg->kg_flags = 0; -} - -/* - * This must be called with the kse lock held and when there are - * no more threads that reference it. - */ -static void -kseg_free_unlocked(struct kse_group *kseg) -{ - TAILQ_REMOVE(&active_kse_groupq, kseg, kg_qe); - TAILQ_INSERT_HEAD(&free_kse_groupq, kseg, kg_qe); - free_kseg_count++; - active_kseg_count--; -} - -void -_kseg_free(struct kse_group *kseg) -{ - struct kse *curkse; - kse_critical_t crit; - - crit = _kse_critical_enter(); - curkse = _get_curkse(); - KSE_LOCK_ACQUIRE(curkse, &kse_lock); - kseg_free_unlocked(kseg); - KSE_LOCK_RELEASE(curkse, &kse_lock); - _kse_critical_leave(crit); -} - -static void -kseg_destroy(struct kse_group *kseg) -{ - _lock_destroy(&kseg->kg_lock); - _pq_free(&kseg->kg_schedq.sq_runq); - free(kseg); -} - -/* - * Allocate a new KSE. - * - * We allow the current thread to be NULL in the case that this - * is the first time a KSE is being created (library initialization). - * In this case, we don't need to (and can't) take any locks. - */ -struct kse * -_kse_alloc(struct pthread *curthread, int sys_scope) -{ - struct kse *kse = NULL; - char *stack; - kse_critical_t crit; - int i; - - if ((curthread != NULL) && (free_kse_count > 0)) { - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock); - /* Search for a finished KSE. */ - kse = TAILQ_FIRST(&free_kseq); - while ((kse != NULL) && - ((kse->k_kcb->kcb_kmbx.km_flags & KMF_DONE) == 0)) { - kse = TAILQ_NEXT(kse, k_qe); - } - if (kse != NULL) { - DBG_MSG("found an unused kse.\n"); - TAILQ_REMOVE(&free_kseq, kse, k_qe); - free_kse_count--; - TAILQ_INSERT_TAIL(&active_kseq, kse, k_qe); - active_kse_count++; - } - KSE_LOCK_RELEASE(curthread->kse, &kse_lock); - _kse_critical_leave(crit); - if (kse != NULL) - kse_reinit(kse, sys_scope); - } - if ((kse == NULL) && - ((kse = (struct kse *)malloc(sizeof(*kse))) != NULL)) { - if (sys_scope != 0) - stack = NULL; - else if ((stack = malloc(KSE_STACKSIZE)) == NULL) { - free(kse); - return (NULL); - } - bzero(kse, sizeof(*kse)); - - /* Initialize KCB without the lock. */ - if ((kse->k_kcb = _kcb_ctor(kse)) == NULL) { - if (stack != NULL) - free(stack); - free(kse); - return (NULL); - } - - /* Initialize the lockusers. */ - for (i = 0; i < MAX_KSE_LOCKLEVEL; i++) { - _lockuser_init(&kse->k_lockusers[i], (void *)kse); - _LCK_SET_PRIVATE2(&kse->k_lockusers[i], NULL); - } - /* _lock_init(kse->k_lock, ...) */ - - if (curthread != NULL) { - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock); - } - kse->k_flags = 0; - TAILQ_INSERT_TAIL(&active_kseq, kse, k_qe); - active_kse_count++; - if (curthread != NULL) { - KSE_LOCK_RELEASE(curthread->kse, &kse_lock); - _kse_critical_leave(crit); - } - /* - * Create the KSE context. - * Scope system threads (one thread per KSE) are not required - * to have a stack for an unneeded kse upcall. - */ - if (!sys_scope) { - kse->k_kcb->kcb_kmbx.km_func = (kse_func_t *)kse_sched_multi; - kse->k_stack.ss_sp = stack; - kse->k_stack.ss_size = KSE_STACKSIZE; - } else { - kse->k_kcb->kcb_kmbx.km_func = (kse_func_t *)kse_sched_single; - kse->k_stack.ss_sp = NULL; - kse->k_stack.ss_size = 0; - } - kse->k_kcb->kcb_kmbx.km_udata = (void *)kse; - kse->k_kcb->kcb_kmbx.km_quantum = 20000; - /* - * We need to keep a copy of the stack in case it - * doesn't get used; a KSE running a scope system - * thread will use that thread's stack. - */ - kse->k_kcb->kcb_kmbx.km_stack = kse->k_stack; - } - return (kse); -} - -static void -kse_reinit(struct kse *kse, int sys_scope) -{ - if (!sys_scope) { - kse->k_kcb->kcb_kmbx.km_func = (kse_func_t *)kse_sched_multi; - if (kse->k_stack.ss_sp == NULL) { - /* XXX check allocation failure */ - kse->k_stack.ss_sp = (char *) malloc(KSE_STACKSIZE); - kse->k_stack.ss_size = KSE_STACKSIZE; - } - kse->k_kcb->kcb_kmbx.km_quantum = 20000; - } else { - kse->k_kcb->kcb_kmbx.km_func = (kse_func_t *)kse_sched_single; - if (kse->k_stack.ss_sp) - free(kse->k_stack.ss_sp); - kse->k_stack.ss_sp = NULL; - kse->k_stack.ss_size = 0; - kse->k_kcb->kcb_kmbx.km_quantum = 0; - } - kse->k_kcb->kcb_kmbx.km_stack = kse->k_stack; - kse->k_kcb->kcb_kmbx.km_udata = (void *)kse; - kse->k_kcb->kcb_kmbx.km_curthread = NULL; - kse->k_kcb->kcb_kmbx.km_flags = 0; - kse->k_curthread = NULL; - kse->k_kseg = 0; - kse->k_schedq = 0; - kse->k_locklevel = 0; - kse->k_flags = 0; - kse->k_error = 0; - kse->k_cpu = 0; - kse->k_sigseqno = 0; -} - -void -kse_free_unlocked(struct kse *kse) -{ - TAILQ_REMOVE(&active_kseq, kse, k_qe); - active_kse_count--; - kse->k_kseg = NULL; - kse->k_kcb->kcb_kmbx.km_quantum = 20000; - kse->k_flags = 0; - TAILQ_INSERT_HEAD(&free_kseq, kse, k_qe); - free_kse_count++; -} - -void -_kse_free(struct pthread *curthread, struct kse *kse) -{ - kse_critical_t crit; - - if (curthread == NULL) - kse_free_unlocked(kse); - else { - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &kse_lock); - kse_free_unlocked(kse); - KSE_LOCK_RELEASE(curthread->kse, &kse_lock); - _kse_critical_leave(crit); - } -} - -static void -kse_destroy(struct kse *kse) -{ - int i; - - if (kse->k_stack.ss_sp != NULL) - free(kse->k_stack.ss_sp); - _kcb_dtor(kse->k_kcb); - for (i = 0; i < MAX_KSE_LOCKLEVEL; ++i) - _lockuser_destroy(&kse->k_lockusers[i]); - _lock_destroy(&kse->k_lock); - free(kse); -} - -struct pthread * -_thr_alloc(struct pthread *curthread) -{ - kse_critical_t crit; - struct pthread *thread = NULL; - int i; - - if (curthread != NULL) { - if (GC_NEEDED()) - _thr_gc(curthread); - if (free_thread_count > 0) { - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &thread_lock); - if ((thread = TAILQ_FIRST(&free_threadq)) != NULL) { - TAILQ_REMOVE(&free_threadq, thread, tle); - free_thread_count--; - } - KSE_LOCK_RELEASE(curthread->kse, &thread_lock); - _kse_critical_leave(crit); - } - } - if ((thread == NULL) && - ((thread = malloc(sizeof(struct pthread))) != NULL)) { - bzero(thread, sizeof(struct pthread)); - thread->siginfo = calloc(_SIG_MAXSIG, sizeof(siginfo_t)); - if (thread->siginfo == NULL) { - free(thread); - return (NULL); - } - if (curthread) { - _pthread_mutex_lock(&_tcb_mutex); - thread->tcb = _tcb_ctor(thread, 0 /* not initial tls */); - _pthread_mutex_unlock(&_tcb_mutex); - } else { - thread->tcb = _tcb_ctor(thread, 1 /* initial tls */); - } - if (thread->tcb == NULL) { - free(thread->siginfo); - free(thread); - return (NULL); - } - /* - * Initialize thread locking. - * Lock initializing needs malloc, so don't - * enter critical region before doing this! - */ - if (_lock_init(&thread->lock, LCK_ADAPTIVE, - _thr_lock_wait, _thr_lock_wakeup, calloc) != 0) - PANIC("Cannot initialize thread lock"); - for (i = 0; i < MAX_THR_LOCKLEVEL; i++) { - _lockuser_init(&thread->lockusers[i], (void *)thread); - _LCK_SET_PRIVATE2(&thread->lockusers[i], - (void *)thread); - } - } - return (thread); -} - -void -_thr_free(struct pthread *curthread, struct pthread *thread) -{ - kse_critical_t crit; - - DBG_MSG("Freeing thread %p\n", thread); - if (thread->name) { - free(thread->name); - thread->name = NULL; - } - if ((curthread == NULL) || (free_thread_count >= MAX_CACHED_THREADS)) { - thr_destroy(curthread, thread); - } else { - /* Add the thread to the free thread list. */ - crit = _kse_critical_enter(); - KSE_LOCK_ACQUIRE(curthread->kse, &thread_lock); - TAILQ_INSERT_TAIL(&free_threadq, thread, tle); - free_thread_count++; - KSE_LOCK_RELEASE(curthread->kse, &thread_lock); - _kse_critical_leave(crit); - } -} - -static void -thr_destroy(struct pthread *curthread, struct pthread *thread) -{ - int i; - - for (i = 0; i < MAX_THR_LOCKLEVEL; i++) - _lockuser_destroy(&thread->lockusers[i]); - _lock_destroy(&thread->lock); - if (curthread) { - _pthread_mutex_lock(&_tcb_mutex); - _tcb_dtor(thread->tcb); - _pthread_mutex_unlock(&_tcb_mutex); - } else { - _tcb_dtor(thread->tcb); - } - free(thread->siginfo); - free(thread); -} - -/* - * Add an active thread: - * - * o Assign the thread a unique id (which GDB uses to track - * threads. - * o Add the thread to the list of all threads and increment - * number of active threads. - */ -static void -thr_link(struct pthread *thread) -{ - kse_critical_t crit; - struct kse *curkse; - - crit = _kse_critical_enter(); - curkse = _get_curkse(); - KSE_LOCK_ACQUIRE(curkse, &_thread_list_lock); - /* - * Initialize the unique id (which GDB uses to track - * threads), add the thread to the list of all threads, - * and - */ - thread->uniqueid = next_uniqueid++; - THR_LIST_ADD(thread); - _thread_active_threads++; - KSE_LOCK_RELEASE(curkse, &_thread_list_lock); - _kse_critical_leave(crit); -} - -/* - * Remove an active thread. - */ -static void -thr_unlink(struct pthread *thread) -{ - kse_critical_t crit; - struct kse *curkse; - - crit = _kse_critical_enter(); - curkse = _get_curkse(); - KSE_LOCK_ACQUIRE(curkse, &_thread_list_lock); - THR_LIST_REMOVE(thread); - _thread_active_threads--; - KSE_LOCK_RELEASE(curkse, &_thread_list_lock); - _kse_critical_leave(crit); -} - -void -_thr_hash_add(struct pthread *thread) -{ - struct thread_hash_head *head; - - head = &thr_hashtable[THREAD_HASH(thread)]; - LIST_INSERT_HEAD(head, thread, hle); -} - -void -_thr_hash_remove(struct pthread *thread) -{ - LIST_REMOVE(thread, hle); -} - -struct pthread * -_thr_hash_find(struct pthread *thread) -{ - struct pthread *td; - struct thread_hash_head *head; - - head = &thr_hashtable[THREAD_HASH(thread)]; - LIST_FOREACH(td, head, hle) { - if (td == thread) - return (thread); - } - return (NULL); -} - -void -_thr_debug_check_yield(struct pthread *curthread) -{ - /* - * Note that TMDF_SUSPEND is set after process is suspended. - * When we are being debugged, every suspension in process - * will cause all KSEs to schedule an upcall in kernel, unless the - * KSE is in critical region. - * If the function is being called, it means the KSE is no longer - * in critical region, if the TMDF_SUSPEND is set by debugger - * before KSE leaves critical region, we will catch it here, else - * if the flag is changed during testing, it also not a problem, - * because the change only occurs after a process suspension event - * occurs. A suspension event will always cause KSE to schedule an - * upcall, in the case, because we are not in critical region, - * upcall will be scheduled sucessfully, the flag will be checked - * again in kse_sched_multi, we won't back until the flag - * is cleared by debugger, the flag will be cleared in next - * suspension event. - */ - if (!DBG_CAN_RUN(curthread)) { - if ((curthread->attr.flags & PTHREAD_SCOPE_SYSTEM) == 0) - _thr_sched_switch(curthread); - else - kse_thr_interrupt(&curthread->tcb->tcb_tmbx, - KSE_INTR_DBSUSPEND, 0); - } -} |
