summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--lib/libkse/thread/thr_cancel.c12
-rw-r--r--lib/libkse/thread/thr_concurrency.c108
-rw-r--r--lib/libkse/thread/thr_cond.c8
-rw-r--r--lib/libkse/thread/thr_create.c7
-rw-r--r--lib/libkse/thread/thr_detach.c15
-rw-r--r--lib/libkse/thread/thr_exit.c7
-rw-r--r--lib/libkse/thread/thr_find_thread.c3
-rw-r--r--lib/libkse/thread/thr_join.c14
-rw-r--r--lib/libkse/thread/thr_kern.c161
-rw-r--r--lib/libkse/thread/thr_mutex.c35
-rw-r--r--lib/libkse/thread/thr_nanosleep.c4
-rw-r--r--lib/libkse/thread/thr_priority_queue.c9
-rw-r--r--lib/libkse/thread/thr_private.h46
-rw-r--r--lib/libkse/thread/thr_sigsuspend.c5
-rw-r--r--lib/libkse/thread/thr_sigwait.c5
-rw-r--r--lib/libkse/thread/thr_spinlock.c31
-rw-r--r--lib/libkse/thread/thr_yield.c5
-rw-r--r--lib/libpthread/arch/i386/i386/thr_switch.S6
-rw-r--r--lib/libpthread/thread/thr_cancel.c12
-rw-r--r--lib/libpthread/thread/thr_concurrency.c108
-rw-r--r--lib/libpthread/thread/thr_cond.c8
-rw-r--r--lib/libpthread/thread/thr_create.c7
-rw-r--r--lib/libpthread/thread/thr_detach.c15
-rw-r--r--lib/libpthread/thread/thr_exit.c7
-rw-r--r--lib/libpthread/thread/thr_find_thread.c3
-rw-r--r--lib/libpthread/thread/thr_join.c14
-rw-r--r--lib/libpthread/thread/thr_kern.c161
-rw-r--r--lib/libpthread/thread/thr_mutex.c35
-rw-r--r--lib/libpthread/thread/thr_nanosleep.c4
-rw-r--r--lib/libpthread/thread/thr_priority_queue.c9
-rw-r--r--lib/libpthread/thread/thr_private.h46
-rw-r--r--lib/libpthread/thread/thr_sigsuspend.c5
-rw-r--r--lib/libpthread/thread/thr_sigwait.c5
-rw-r--r--lib/libpthread/thread/thr_spinlock.c31
-rw-r--r--lib/libpthread/thread/thr_yield.c5
35 files changed, 640 insertions, 316 deletions
diff --git a/lib/libkse/thread/thr_cancel.c b/lib/libkse/thread/thr_cancel.c
index 23e0dfa..fc05b55 100644
--- a/lib/libkse/thread/thr_cancel.c
+++ b/lib/libkse/thread/thr_cancel.c
@@ -217,7 +217,6 @@ static void
testcancel(struct pthread *curthread)
{
/* Take the scheduling lock while fiddling with the state: */
- THR_SCHED_LOCK(curthread, curthread);
if (checkcancel(curthread) != 0) {
/* Unlock before exiting: */
@@ -227,8 +226,6 @@ testcancel(struct pthread *curthread)
pthread_exit(PTHREAD_CANCELED);
PANIC("cancel");
}
-
- THR_SCHED_UNLOCK(curthread, curthread);
}
void
@@ -236,23 +233,29 @@ _pthread_testcancel(void)
{
struct pthread *curthread = _get_curthread();
+ THR_SCHED_LOCK(curthread, curthread);
testcancel(curthread);
+ THR_SCHED_UNLOCK(curthread, curthread);
}
void
_thr_enter_cancellation_point(struct pthread *thread)
{
/* Look for a cancellation before we block: */
+ THR_SCHED_LOCK(thread, thread);
testcancel(thread);
thread->cancelflags |= THR_AT_CANCEL_POINT;
+ THR_SCHED_UNLOCK(thread, thread);
}
void
_thr_leave_cancellation_point(struct pthread *thread)
{
+ THR_SCHED_LOCK(thread, thread);
thread->cancelflags &= ~THR_AT_CANCEL_POINT;
/* Look for a cancellation after we unblock: */
testcancel(thread);
+ THR_SCHED_UNLOCK(thread, thread);
}
static void
@@ -263,9 +266,12 @@ finish_cancellation(void *arg)
curthread->continuation = NULL;
curthread->interrupted = 0;
+ THR_SCHED_LOCK(curthread, curthread);
if ((curthread->cancelflags & THR_CANCEL_NEEDED) != 0) {
curthread->cancelflags &= ~THR_CANCEL_NEEDED;
+ THR_SCHED_UNLOCK(curthread, curthread);
_thr_exit_cleanup();
pthread_exit(PTHREAD_CANCELED);
}
+ THR_SCHED_UNLOCK(curthread, curthread);
}
diff --git a/lib/libkse/thread/thr_concurrency.c b/lib/libkse/thread/thr_concurrency.c
index a4b055f..db84121 100644
--- a/lib/libkse/thread/thr_concurrency.c
+++ b/lib/libkse/thread/thr_concurrency.c
@@ -28,6 +28,8 @@
*/
#include <errno.h>
#include <pthread.h>
+#include <sys/types.h>
+#include <sys/sysctl.h>
#include "thr_private.h"
@@ -52,14 +54,8 @@ _pthread_getconcurrency(void)
int
_pthread_setconcurrency(int new_level)
{
- struct pthread *curthread;
- struct kse *newkse;
- kse_critical_t crit;
- int kse_count;
- int i;
int ret;
-
if (new_level < 0)
ret = EINVAL;
else if (new_level == level)
@@ -71,50 +67,72 @@ _pthread_setconcurrency(int new_level)
DBG_MSG("Can't enable threading.\n");
ret = EAGAIN;
} else {
- ret = 0;
- curthread = _get_curthread();
- /* Race condition, but so what. */
- kse_count = _kse_initial->k_kseg->kg_ksecount;
- for (i = kse_count; i < new_level; i++) {
- newkse = _kse_alloc(curthread);
- if (newkse == NULL) {
- DBG_MSG("Can't alloc new KSE.\n");
- ret = EAGAIN;
- break;
- }
- newkse->k_kseg = _kse_initial->k_kseg;
- newkse->k_schedq = _kse_initial->k_schedq;
- newkse->k_curthread = NULL;
- crit = _kse_critical_enter();
+ ret = _thr_setconcurrency(new_level);
+ if (ret == 0)
+ level = new_level;
+ }
+ return (ret);
+}
+
+int
+_thr_setconcurrency(int new_level)
+{
+ struct pthread *curthread;
+ struct kse *newkse;
+ kse_critical_t crit;
+ int kse_count;
+ int i;
+ int ret;
+
+ ret = 0;
+ curthread = _get_curthread();
+ /* Race condition, but so what. */
+ kse_count = _kse_initial->k_kseg->kg_ksecount;
+ for (i = kse_count; i < new_level; i++) {
+ newkse = _kse_alloc(curthread);
+ if (newkse == NULL) {
+ DBG_MSG("Can't alloc new KSE.\n");
+ ret = EAGAIN;
+ break;
+ }
+ newkse->k_kseg = _kse_initial->k_kseg;
+ newkse->k_schedq = _kse_initial->k_schedq;
+ newkse->k_curthread = NULL;
+ crit = _kse_critical_enter();
+ KSE_SCHED_LOCK(curthread->kse, newkse->k_kseg);
+ TAILQ_INSERT_TAIL(&newkse->k_kseg->kg_kseq,
+ newkse, k_kgqe);
+ newkse->k_kseg->kg_ksecount++;
+ KSE_SCHED_UNLOCK(curthread->kse, newkse->k_kseg);
+ if (kse_create(&newkse->k_mbx, 0) != 0) {
KSE_SCHED_LOCK(curthread->kse, newkse->k_kseg);
- TAILQ_INSERT_TAIL(&newkse->k_kseg->kg_kseq,
+ TAILQ_REMOVE(&newkse->k_kseg->kg_kseq,
newkse, k_kgqe);
- newkse->k_kseg->kg_ksecount++;
+ newkse->k_kseg->kg_ksecount--;
KSE_SCHED_UNLOCK(curthread->kse, newkse->k_kseg);
- if (_ksd_setprivate(&_kse_initial->k_ksd) != 0) {
- /* This should never happen. */
- PANIC("pthread_setconcurrency: Unable to "
- "set KSE specific data");
- }
- newkse->k_flags |= KF_INITIALIZED;
- if (kse_create(&newkse->k_mbx, 0) != 0) {
- _ksd_setprivate(&curthread->kse->k_ksd);
- KSE_SCHED_LOCK(curthread->kse, newkse->k_kseg);
- TAILQ_REMOVE(&newkse->k_kseg->kg_kseq,
- newkse, k_kgqe);
- newkse->k_kseg->kg_ksecount--;
- KSE_SCHED_UNLOCK(curthread->kse, newkse->k_kseg);
- _kse_critical_leave(crit);
- _kse_free(curthread, newkse);
- DBG_MSG("kse_create syscall failed.\n");
- ret = EAGAIN;
- break;
- }
- _ksd_setprivate(&curthread->kse->k_ksd);
+ _kse_critical_leave(crit);
+ _kse_free(curthread, newkse);
+ DBG_MSG("kse_create syscall failed.\n");
+ ret = EAGAIN;
+ break;
+ } else {
_kse_critical_leave(crit);
}
- if (ret == 0)
- level = new_level;
}
return (ret);
}
+
+int
+_thr_setmaxconcurrency(void)
+{
+ int vcpu;
+ int len;
+ int ret;
+
+ len = sizeof(vcpu);
+ ret = sysctlbyname("kern.threads.virtual_cpu", &vcpu, &len, NULL, NULL);
+ if (ret == 0 && vcpu > 0)
+ ret = _thr_setconcurrency(vcpu);
+ return (ret);
+}
+
diff --git a/lib/libkse/thread/thr_cond.c b/lib/libkse/thread/thr_cond.c
index 537c264..32a152d 100644
--- a/lib/libkse/thread/thr_cond.c
+++ b/lib/libkse/thread/thr_cond.c
@@ -267,6 +267,7 @@ _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
* lock); we should be able to safely
* set the state.
*/
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_COND_WAIT);
/* Remember the CV: */
@@ -280,6 +281,7 @@ _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
_thr_sched_switch(curthread);
curthread->data.cond = NULL;
+ THR_UNLOCK_SWITCH(curthread);
/*
* XXX - This really isn't a good check
@@ -324,8 +326,10 @@ _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
* that the mutex be reaquired prior to
* cancellation.
*/
- if (done != 0)
+ if (done != 0) {
rval = _mutex_cv_lock(mutex);
+ unlock_mutex = 1;
+ }
}
}
break;
@@ -475,6 +479,7 @@ _pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
* lock); we should be able to safely
* set the state.
*/
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_COND_WAIT);
/* Remember the CV: */
@@ -488,6 +493,7 @@ _pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
_thr_sched_switch(curthread);
curthread->data.cond = NULL;
+ THR_UNLOCK_SWITCH(curthread);
/*
* XXX - This really isn't a good check
diff --git a/lib/libkse/thread/thr_create.c b/lib/libkse/thread/thr_create.c
index deb26de..4c65d3c 100644
--- a/lib/libkse/thread/thr_create.c
+++ b/lib/libkse/thread/thr_create.c
@@ -297,6 +297,13 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
* pair if necessary.
*/
ret = _thr_schedule_add(curthread, new_thread);
+ if (ret != 0) {
+ KSE_LOCK_ACQUIRE(curthread->kse,
+ &_thread_list_lock);
+ THR_LIST_REMOVE(new_thread);
+ KSE_LOCK_RELEASE(curthread->kse,
+ &_thread_list_lock);
+ }
_kse_critical_leave(crit);
if (ret != 0)
free_thread(curthread, new_thread);
diff --git a/lib/libkse/thread/thr_detach.c b/lib/libkse/thread/thr_detach.c
index 30b7ce5..66db912 100644
--- a/lib/libkse/thread/thr_detach.c
+++ b/lib/libkse/thread/thr_detach.c
@@ -44,8 +44,6 @@ _pthread_detach(pthread_t pthread)
{
struct pthread *curthread = _get_curthread();
struct pthread *joiner;
- kse_critical_t crit;
- int dead;
int rval = 0;
/* Check for invalid calling parameters: */
@@ -56,7 +54,6 @@ _pthread_detach(pthread_t pthread)
else if ((rval = _thr_ref_add(curthread, pthread,
/*include dead*/1)) != 0) {
/* Return an error: */
- _thr_leave_cancellation_point(curthread);
}
/* Check if the thread is already detached: */
@@ -91,20 +88,11 @@ _pthread_detach(pthread_t pthread)
}
joiner = NULL;
}
- dead = (pthread->flags & THR_FLAGS_GC_SAFE) != 0;
THR_SCHED_UNLOCK(curthread, pthread);
-
- if (dead != 0) {
- crit = _kse_critical_enter();
- KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
- THR_GCLIST_ADD(pthread);
- KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock);
- _kse_critical_leave(crit);
- }
_thr_ref_delete(curthread, pthread);
/* See if there is a thread waiting in pthread_join(): */
- if (joiner != NULL) {
+ if (joiner != NULL && _thr_ref_add(curthread, joiner, 0) == 0) {
/* Lock the joiner before fiddling with it. */
THR_SCHED_LOCK(curthread, joiner);
if (joiner->join_status.thread == pthread) {
@@ -118,6 +106,7 @@ _pthread_detach(pthread_t pthread)
_thr_setrunnable_unlocked(joiner);
}
THR_SCHED_UNLOCK(curthread, joiner);
+ _thr_ref_delete(curthread, joiner);
}
}
diff --git a/lib/libkse/thread/thr_exit.c b/lib/libkse/thread/thr_exit.c
index 4a82b12..8435f43 100644
--- a/lib/libkse/thread/thr_exit.c
+++ b/lib/libkse/thread/thr_exit.c
@@ -104,7 +104,9 @@ _pthread_exit(void *status)
* Flag this thread as exiting. Threads should now be prevented
* from joining to this thread.
*/
+ THR_SCHED_LOCK(curthread, curthread);
curthread->flags |= THR_FLAGS_EXITING;
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Save the return value: */
curthread->ret = status;
@@ -121,10 +123,11 @@ _pthread_exit(void *status)
}
/* This thread will never be re-scheduled. */
- THR_SCHED_LOCK(curthread, curthread);
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_DEAD);
- THR_SCHED_UNLOCK(curthread, curthread);
_thr_sched_switch(curthread);
+ /* Never reach! */
+ THR_UNLOCK_SWITCH(curthread);
/* This point should not be reached. */
PANIC("Dead thread has resumed");
diff --git a/lib/libkse/thread/thr_find_thread.c b/lib/libkse/thread/thr_find_thread.c
index b5cae66..0c813db 100644
--- a/lib/libkse/thread/thr_find_thread.c
+++ b/lib/libkse/thread/thr_find_thread.c
@@ -85,6 +85,9 @@ _thr_ref_delete(struct pthread *curthread, struct pthread *thread)
KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
thread->refcount--;
curthread->critical_count--;
+ if ((thread->refcount == 0) &&
+ (thread->flags & THR_FLAGS_GC_SAFE) != 0)
+ THR_GCLIST_ADD(thread);
KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock);
_kse_critical_leave(crit);
}
diff --git a/lib/libkse/thread/thr_join.c b/lib/libkse/thread/thr_join.c
index 77e3a8c..7cbc192 100644
--- a/lib/libkse/thread/thr_join.c
+++ b/lib/libkse/thread/thr_join.c
@@ -70,16 +70,16 @@ _pthread_join(pthread_t pthread, void **thread_return)
return (ESRCH);
}
+ THR_SCHED_LOCK(curthread, pthread);
/* Check if this thread has been detached: */
if ((pthread->attr.flags & PTHREAD_DETACHED) != 0) {
+ THR_SCHED_UNLOCK(curthread, pthread);
/* Remove the reference and return an error: */
_thr_ref_delete(curthread, pthread);
ret = ESRCH;
} else {
/* Lock the target thread while checking its state. */
- THR_SCHED_LOCK(curthread, pthread);
- if ((pthread->state == PS_DEAD) ||
- ((pthread->flags & THR_FLAGS_EXITING) != 0)) {
+ if (pthread->state == PS_DEAD) {
if (thread_return != NULL)
/* Return the thread's return value: */
*thread_return = pthread->ret;
@@ -123,15 +123,13 @@ _pthread_join(pthread_t pthread, void **thread_return)
THR_SCHED_UNLOCK(curthread, pthread);
_thr_ref_delete(curthread, pthread);
- THR_SCHED_LOCK(curthread, curthread);
- if (curthread->join_status.thread == pthread)
- THR_SET_STATE(curthread, PS_JOIN);
- THR_SCHED_UNLOCK(curthread, curthread);
-
+ THR_LOCK_SWITCH(curthread);
while (curthread->join_status.thread == pthread) {
+ THR_SET_STATE(curthread, PS_JOIN);
/* Schedule the next thread: */
_thr_sched_switch(curthread);
}
+ THR_UNLOCK_SWITCH(curthread);
/*
* The thread return value and error are set by the
diff --git a/lib/libkse/thread/thr_kern.c b/lib/libkse/thread/thr_kern.c
index 63283f7..5762219 100644
--- a/lib/libkse/thread/thr_kern.c
+++ b/lib/libkse/thread/thr_kern.c
@@ -95,14 +95,7 @@ __FBSDID("$FreeBSD$");
_pq_remove(&(kse)->k_schedq->sq_runq, thrd)
#define KSE_RUNQ_FIRST(kse) _pq_first(&(kse)->k_schedq->sq_runq)
-/*
- * XXX - Remove when David commits kernel changes to support these.
- */
-#ifndef KMF_NOUPCALL
-#define KMF_NOUPCALL 0x01
-#define KMF_NOCOMPLETED 0x02
-#endif
-
+#define KSE_RUNQ_THREADS(kse) ((kse)->k_schedq->sq_runq.pq_threads)
/*
* We've got to keep track of everything that is allocated, not only
@@ -140,11 +133,11 @@ static void kseg_free_unlocked(struct kse_group *kseg);
static void kseg_init(struct kse_group *kseg);
static void kseg_reinit(struct kse_group *kseg);
static void kse_waitq_insert(struct pthread *thread);
+static void kse_wakeup_multi(struct kse *curkse);
+static void kse_wakeup_one(struct pthread *thread);
static void thr_cleanup(struct kse *kse, struct pthread *curthread);
-#ifdef NOT_YET
static void thr_resume_wrapper(int unused_1, siginfo_t *unused_2,
ucontext_t *ucp);
-#endif
static void thr_resume_check(struct pthread *curthread, ucontext_t *ucp,
struct pthread_sigframe *psf);
static int thr_timedout(struct pthread *thread, struct timespec *curtime);
@@ -368,11 +361,13 @@ _kse_setthreaded(int threaded)
_kse_initial->k_flags |= KF_STARTED;
if (kse_create(&_kse_initial->k_mbx, 0) != 0) {
_kse_initial->k_flags &= ~KF_STARTED;
+ __isthreaded = 0;
/* may abort() */
DBG_MSG("kse_create failed\n");
return (-1);
}
KSE_SET_MBOX(_kse_initial, _thr_initial);
+ _thr_setmaxconcurrency();
}
return (0);
}
@@ -466,16 +461,14 @@ _thr_lock_wait(struct lock *lock, struct lockuser *lu)
while (_LCK_BUSY(lu) && count < 300)
count++;
while (_LCK_BUSY(lu)) {
- THR_SCHED_LOCK(curthread, curthread);
+ THR_LOCK_SWITCH(curthread);
if (_LCK_BUSY(lu)) {
/* Wait for the lock: */
atomic_store_rel_int(&curthread->need_wakeup, 1);
THR_SET_STATE(curthread, PS_LOCKWAIT);
- THR_SCHED_UNLOCK(curthread, curthread);
_thr_sched_switch(curthread);
}
- else
- THR_SCHED_UNLOCK(curthread, curthread);
+ THR_UNLOCK_SWITCH(curthread);
}
}
@@ -484,14 +477,26 @@ _thr_lock_wakeup(struct lock *lock, struct lockuser *lu)
{
struct pthread *thread;
struct pthread *curthread;
+ int unlock;
curthread = _get_curthread();
thread = (struct pthread *)_LCK_GET_PRIVATE(lu);
- THR_SCHED_LOCK(curthread, thread);
+ unlock = 0;
+ if (curthread->kseg == thread->kseg) {
+ /* Not already locked */
+ if (curthread->lock_switch == 0) {
+ THR_SCHED_LOCK(curthread, thread);
+ unlock = 1;
+ }
+ } else {
+ THR_SCHED_LOCK(curthread, thread);
+ unlock = 1;
+ }
_thr_setrunnable_unlocked(thread);
atomic_store_rel_int(&thread->need_wakeup, 0);
- THR_SCHED_UNLOCK(curthread, thread);
+ if (unlock)
+ THR_SCHED_UNLOCK(curthread, thread);
}
kse_critical_t
@@ -541,12 +546,12 @@ void
_thr_sched_switch(struct pthread *curthread)
{
struct pthread_sigframe psf;
- kse_critical_t crit;
struct kse *curkse;
volatile int once = 0;
/* We're in the scheduler, 5 by 5: */
- crit = _kse_critical_enter();
+ THR_ASSERT(curthread->lock_switch, "lock_switch");
+ THR_ASSERT(_kse_in_critical(), "not in critical region");
curkse = _get_curkse();
curthread->need_switchout = 1; /* The thread yielded on its own. */
@@ -568,7 +573,11 @@ _thr_sched_switch(struct pthread *curthread)
*/
if ((once == 0) && (!THR_IN_CRITICAL(curthread))) {
once = 1;
+ curthread->critical_count++;
+ THR_UNLOCK_SWITCH(curthread);
+ curthread->critical_count--;
thr_resume_check(curthread, &curthread->tmbx.tm_context, &psf);
+ THR_LOCK_SWITCH(curthread);
}
}
@@ -734,14 +743,20 @@ kse_sched_multi(struct kse *curkse)
KSE_CLEAR_WAIT(curkse);
}
- /* Lock the scheduling lock. */
- KSE_SCHED_LOCK(curkse, curkse->k_kseg);
+ curthread = curkse->k_curthread;
+ if (curthread == NULL || curthread->lock_switch == 0) {
+ /*
+ * curthread was preempted by upcall, it is not a volunteer
+ * context switch. Lock the scheduling lock.
+ */
+ KSE_SCHED_LOCK(curkse, curkse->k_kseg);
+ }
/*
* If the current thread was completed in another KSE, then
* it will be in the run queue. Don't mark it as being blocked.
*/
- if (((curthread = curkse->k_curthread) != NULL) &&
+ if ((curthread != NULL) &&
((curthread->flags & THR_FLAGS_IN_RUNQ) == 0) &&
(curthread->need_switchout == 0)) {
/*
@@ -774,17 +789,22 @@ kse_sched_multi(struct kse *curkse)
* Resume the thread and tell it to yield when
* it leaves the critical region.
*/
- curthread->critical_yield = 0;
+ curthread->critical_yield = 1;
curthread->active = 1;
if ((curthread->flags & THR_FLAGS_IN_RUNQ) != 0)
KSE_RUNQ_REMOVE(curkse, curthread);
curkse->k_curthread = curthread;
curthread->kse = curkse;
- KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
DBG_MSG("Continuing thread %p in critical region\n",
curthread);
- ret = _thread_switch(&curthread->tmbx,
- &curkse->k_mbx.km_curthread);
+ if (curthread->lock_switch) {
+ KSE_SCHED_LOCK(curkse, curkse->k_kseg);
+ ret = _thread_switch(&curthread->tmbx, 0);
+ } else {
+ KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
+ ret = _thread_switch(&curthread->tmbx,
+ &curkse->k_mbx.km_curthread);
+ }
if (ret != 0)
PANIC("Can't resume thread in critical region\n");
}
@@ -792,6 +812,8 @@ kse_sched_multi(struct kse *curkse)
kse_switchout_thread(curkse, curthread);
curkse->k_curthread = NULL;
+ kse_wakeup_multi(curkse);
+
/* This has to be done without the scheduling lock held. */
KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
kse_check_signals(curkse);
@@ -868,6 +890,8 @@ kse_sched_multi(struct kse *curkse)
curframe = curthread->curframe;
curthread->curframe = NULL;
+ kse_wakeup_multi(curkse);
+
/* Unlock the scheduling queue: */
KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
@@ -881,10 +905,13 @@ kse_sched_multi(struct kse *curkse)
#ifdef NOT_YET
if ((curframe == NULL) && ((curthread->check_pending != 0) ||
(((curthread->cancelflags & THR_AT_CANCEL_POINT) == 0) &&
- ((curthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0)))) {
+ ((curthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0))))
+ signalcontext(&curthread->tmbx.tm_context, 0,
+ (__sighandler_t *)thr_resume_wrapper);
+#else
+ if ((curframe == NULL) && (curthread->check_pending != 0))
signalcontext(&curthread->tmbx.tm_context, 0,
(__sighandler_t *)thr_resume_wrapper);
- }
#endif
#ifdef GS_HACK
/* XXX - The kernel sometimes forgets to restore %gs properly. */
@@ -893,7 +920,13 @@ kse_sched_multi(struct kse *curkse)
/*
* Continue the thread at its current frame:
*/
- ret = _thread_switch(&curthread->tmbx, &curkse->k_mbx.km_curthread);
+ if (curthread->lock_switch) {
+ KSE_SCHED_LOCK(curkse, curkse->k_kseg);
+ ret = _thread_switch(&curthread->tmbx, 0);
+ } else {
+ ret = _thread_switch(&curthread->tmbx,
+ &curkse->k_mbx.km_curthread);
+ }
if (ret != 0)
PANIC("Thread has returned from _thread_switch");
@@ -932,7 +965,6 @@ kse_check_signals(struct kse *curkse)
}
}
-#ifdef NOT_YET
static void
thr_resume_wrapper(int unused_1, siginfo_t *unused_2, ucontext_t *ucp)
{
@@ -940,7 +972,6 @@ thr_resume_wrapper(int unused_1, siginfo_t *unused_2, ucontext_t *ucp)
thr_resume_check(curthread, ucp, NULL);
}
-#endif
static void
thr_resume_check(struct pthread *curthread, ucontext_t *ucp,
@@ -960,9 +991,11 @@ thr_resume_check(struct pthread *curthread, ucontext_t *ucp,
_thr_sig_rundown(curthread, ucp, psf);
}
+#ifdef NOT_YET
if (((curthread->cancelflags & THR_AT_CANCEL_POINT) == 0) &&
((curthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0))
pthread_testcancel();
+#endif
}
/*
@@ -1071,6 +1104,7 @@ _thr_gc(struct pthread *curthread)
* referenced. It is safe to remove all
* remnants of the thread.
*/
+ THR_LIST_REMOVE(td);
TAILQ_INSERT_HEAD(&worklist, td, gcle);
}
}
@@ -1172,11 +1206,11 @@ _thr_schedule_add(struct pthread *curthread, struct pthread *newthread)
if (need_start != 0)
kse_create(&newthread->kse->k_mbx, 0);
else if ((newthread->state == PS_RUNNING) &&
- KSE_WAITING(newthread->kse)) {
+ KSE_IS_IDLE(newthread->kse)) {
/*
* The thread is being scheduled on another KSEG.
*/
- KSE_WAKEUP(newthread->kse);
+ kse_wakeup_one(newthread);
}
ret = 0;
}
@@ -1326,6 +1360,8 @@ kse_switchout_thread(struct kse *kse, struct pthread *thread)
*/
DBG_MSG("Switching out thread %p, state %d\n", thread, thread->state);
if (thread->blocked != 0) {
+ thread->active = 0;
+ thread->need_switchout = 0;
/* This thread must have blocked in the kernel. */
/* thread->slice_usec = -1;*/ /* restart timeslice */
/*
@@ -1346,6 +1382,8 @@ kse_switchout_thread(struct kse *kse, struct pthread *thread)
* stack. It is safe to do garbage collecting
* here.
*/
+ thread->active = 0;
+ thread->need_switchout = 0;
thr_cleanup(kse, thread);
return;
break;
@@ -1456,14 +1494,18 @@ kse_wait(struct kse *kse, struct pthread *td_wait)
}
/* Don't sleep for negative times. */
if ((ts_sleep.tv_sec >= 0) && (ts_sleep.tv_nsec >= 0)) {
- KSE_SET_WAIT(kse);
+ KSE_SET_IDLE(kse);
+ kse->k_kseg->kg_idle_kses++;
KSE_SCHED_UNLOCK(kse, kse->k_kseg);
saved_flags = kse->k_mbx.km_flags;
kse->k_mbx.km_flags |= KMF_NOUPCALL;
kse_release(&ts_sleep);
kse->k_mbx.km_flags = saved_flags;
- KSE_CLEAR_WAIT(kse);
KSE_SCHED_LOCK(kse, kse->k_kseg);
+ if (KSE_IS_IDLE(kse)) {
+ KSE_CLEAR_IDLE(kse);
+ kse->k_kseg->kg_idle_kses--;
+ }
}
}
@@ -1495,11 +1537,13 @@ kse_fini(struct kse *kse)
* Add this KSE to the list of free KSEs along with
* the KSEG if is now orphaned.
*/
+#ifdef NOT_YET
KSE_LOCK_ACQUIRE(kse, &kse_lock);
if (free_kseg != NULL)
kseg_free_unlocked(free_kseg);
kse_free_unlocked(kse);
KSE_LOCK_RELEASE(kse, &kse_lock);
+#endif
kse_exit();
/* Never returns. */
} else {
@@ -1620,7 +1664,7 @@ _thr_setrunnable_unlocked(struct pthread *thread)
if ((thread->kseg->kg_flags & KGF_SINGLE_THREAD) != 0)
/* No silly queues for these threads. */
THR_SET_STATE(thread, PS_RUNNING);
- else {
+ else if (thread->state != PS_RUNNING) {
if ((thread->flags & THR_FLAGS_IN_WAITQ) != 0)
KSE_WAITQ_REMOVE(thread->kse, thread);
THR_SET_STATE(thread, PS_RUNNING);
@@ -1641,8 +1685,47 @@ _thr_setrunnable_unlocked(struct pthread *thread)
* (the KSE). If the KSE wakes up and doesn't find any more
* work it will again go back to waiting so no harm is done.
*/
- if (KSE_WAITING(thread->kse))
+ kse_wakeup_one(thread);
+}
+
+static void
+kse_wakeup_one(struct pthread *thread)
+{
+ struct kse *ke;
+
+ if (KSE_IS_IDLE(thread->kse)) {
+ KSE_CLEAR_IDLE(thread->kse);
+ thread->kseg->kg_idle_kses--;
KSE_WAKEUP(thread->kse);
+ } else {
+ TAILQ_FOREACH(ke, &thread->kseg->kg_kseq, k_kgqe) {
+ if (KSE_IS_IDLE(ke)) {
+ KSE_CLEAR_IDLE(ke);
+ ke->k_kseg->kg_idle_kses--;
+ KSE_WAKEUP(ke);
+ return;
+ }
+ }
+ }
+}
+
+static void
+kse_wakeup_multi(struct kse *curkse)
+{
+ struct kse *ke;
+ int tmp;
+
+ if ((tmp = KSE_RUNQ_THREADS(curkse)) && curkse->k_kseg->kg_idle_kses) {
+ TAILQ_FOREACH(ke, &curkse->k_kseg->kg_kseq, k_kgqe) {
+ if (KSE_IS_IDLE(ke)) {
+ KSE_CLEAR_IDLE(ke);
+ ke->k_kseg->kg_idle_kses--;
+ KSE_WAKEUP(ke);
+ if (--tmp == 0)
+ break;
+ }
+ }
+ }
}
struct pthread *
@@ -1876,7 +1959,7 @@ kse_reinit(struct kse *kse)
sigemptyset(&kse->k_sigmask);
bzero(&kse->k_sigq, sizeof(kse->k_sigq));
kse->k_check_sigq = 0;
- kse->k_flags = KF_INITIALIZED;
+ kse->k_flags = 0;
kse->k_waiting = 0;
kse->k_error = 0;
kse->k_cpu = 0;
@@ -1948,6 +2031,7 @@ _thr_alloc(struct pthread *curthread)
free_thread_count--;
}
KSE_LOCK_RELEASE(curthread->kse, &thread_lock);
+ _kse_critical_leave(crit);
}
}
if (thread == NULL)
@@ -1971,9 +2055,6 @@ _thr_free(struct pthread *curthread, struct pthread *thread)
}
else {
crit = _kse_critical_enter();
- KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
- THR_LIST_REMOVE(thread);
- KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock);
KSE_LOCK_ACQUIRE(curthread->kse, &thread_lock);
TAILQ_INSERT_HEAD(&free_threadq, thread, tle);
free_thread_count++;
diff --git a/lib/libkse/thread/thr_mutex.c b/lib/libkse/thread/thr_mutex.c
index 1ae12ea9..bb97db1 100644
--- a/lib/libkse/thread/thr_mutex.c
+++ b/lib/libkse/thread/thr_mutex.c
@@ -325,6 +325,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
/* Lock the mutex for the running thread: */
(*mutex)->m_owner = curthread;
+ THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
@@ -335,6 +336,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
(*mutex)->m_prio = curthread->active_priority;
(*mutex)->m_saved_prio =
curthread->inherited_priority;
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*mutex);
@@ -358,6 +360,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
/* Lock the mutex for the running thread: */
(*mutex)->m_owner = curthread;
+ THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
@@ -371,7 +374,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
curthread->inherited_priority;
curthread->inherited_priority =
(*mutex)->m_prio;
-
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*mutex);
TAILQ_INSERT_TAIL(&curthread->mutexq,
@@ -503,6 +506,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_MUTEX_WAIT);
/* Unlock the mutex structure: */
@@ -510,6 +514,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
/* Schedule the next thread: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
}
break;
@@ -520,6 +525,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
/* Lock the mutex for this thread: */
(*m)->m_owner = curthread;
+ THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
@@ -529,7 +535,6 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* Make sure the thread's scheduling lock is
* held while priorities are adjusted.
*/
- THR_SCHED_LOCK(curthread, curthread);
(*m)->m_prio = curthread->active_priority;
(*m)->m_saved_prio =
curthread->inherited_priority;
@@ -561,17 +566,18 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
- THR_SET_STATE(curthread, PS_MUTEX_WAIT);
-
if (curthread->active_priority > (*m)->m_prio)
/* Adjust priorities: */
mutex_priority_adjust(curthread, *m);
+ THR_LOCK_SWITCH(curthread);
+ THR_SET_STATE(curthread, PS_MUTEX_WAIT);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
/* Schedule the next thread: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
}
break;
@@ -591,6 +597,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
*/
(*m)->m_owner = curthread;
+ THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
@@ -601,7 +608,6 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* scheduling lock is held while priorities
* are adjusted.
*/
- THR_SCHED_LOCK(curthread, curthread);
curthread->active_priority = (*m)->m_prio;
(*m)->m_saved_prio =
curthread->inherited_priority;
@@ -636,6 +642,8 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
+
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_MUTEX_WAIT);
/* Unlock the mutex structure: */
@@ -643,7 +651,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
/* Schedule the next thread: */
_thr_sched_switch(curthread);
-
+ THR_UNLOCK_SWITCH(curthread);
/*
* The threads priority may have changed while
* waiting for the mutex causing a ceiling
@@ -749,9 +757,15 @@ _mutex_cv_unlock(pthread_mutex_t *m)
int
_mutex_cv_lock(pthread_mutex_t *m)
{
+ struct pthread *curthread;
int ret;
- if ((ret = _pthread_mutex_lock(m)) == 0)
+
+ curthread = _get_curthread();
+ if ((ret = _pthread_mutex_lock(m)) == 0) {
+ THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
(*m)->m_refcount--;
+ THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
+ }
return (ret);
}
@@ -807,6 +821,8 @@ mutex_self_lock(struct pthread *curthread, pthread_mutex_t m)
* What SS2 define as a 'normal' mutex. Intentionally
* deadlock on attempts to get a lock you already own.
*/
+
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_DEADLOCK);
/* Unlock the mutex structure: */
@@ -814,6 +830,7 @@ mutex_self_lock(struct pthread *curthread, pthread_mutex_t m)
/* Schedule the next thread: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
break;
case PTHREAD_MUTEX_RECURSIVE:
@@ -917,12 +934,12 @@ mutex_unlock_common(pthread_mutex_t *m, int add_reference)
curthread->active_priority =
MAX(curthread->inherited_priority,
curthread->base_priority);
- THR_SCHED_UNLOCK(curthread, curthread);
/*
* This thread now owns one less priority mutex.
*/
curthread->priority_mutex_count--;
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Remove the mutex from the threads queue. */
MUTEX_ASSERT_IS_OWNED(*m);
@@ -974,12 +991,12 @@ mutex_unlock_common(pthread_mutex_t *m, int add_reference)
curthread->active_priority =
MAX(curthread->inherited_priority,
curthread->base_priority);
- THR_SCHED_UNLOCK(curthread, curthread);
/*
* This thread now owns one less priority mutex.
*/
curthread->priority_mutex_count--;
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Remove the mutex from the threads queue. */
MUTEX_ASSERT_IS_OWNED(*m);
diff --git a/lib/libkse/thread/thr_nanosleep.c b/lib/libkse/thread/thr_nanosleep.c
index bec3b66..e83f4f4 100644
--- a/lib/libkse/thread/thr_nanosleep.c
+++ b/lib/libkse/thread/thr_nanosleep.c
@@ -60,14 +60,14 @@ _nanosleep(const struct timespec *time_to_sleep,
/* Calculate the time for the current thread to wake up: */
TIMESPEC_ADD(&curthread->wakeup_time, &ts, time_to_sleep);
- THR_SCHED_LOCK(curthread, curthread);
+ THR_LOCK_SWITCH(curthread);
curthread->interrupted = 0;
THR_SET_STATE(curthread, PS_SLEEP_WAIT);
- THR_SCHED_UNLOCK(curthread, curthread);
/* Reschedule the current thread to sleep: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
/* Calculate the remaining time to sleep: */
KSE_GET_TOD(curthread->kse, &ts1);
diff --git a/lib/libkse/thread/thr_priority_queue.c b/lib/libkse/thread/thr_priority_queue.c
index 3f261f8..2822aa8 100644
--- a/lib/libkse/thread/thr_priority_queue.c
+++ b/lib/libkse/thread/thr_priority_queue.c
@@ -126,6 +126,7 @@ _pq_init(pq_queue_t *pq)
/* Initialize the priority queue: */
TAILQ_INIT(&pq->pq_queue);
pq->pq_flags = 0;
+ pq->pq_threads = 0;
}
return (ret);
}
@@ -151,10 +152,10 @@ _pq_remove(pq_queue_t *pq, pthread_t pthread)
* from the priority queue when _pq_first is called.
*/
TAILQ_REMOVE(&pq->pq_lists[prio].pl_head, pthread, pqe);
-
+ pq->pq_threads--;
/* This thread is now longer in the priority queue. */
pthread->flags &= ~THR_FLAGS_IN_RUNQ;
-
+
PQ_CLEAR_ACTIVE(pq);
}
@@ -177,7 +178,7 @@ _pq_insert_head(pq_queue_t *pq, pthread_t pthread)
if (pq->pq_lists[prio].pl_queued == 0)
/* Insert the list into the priority queue: */
pq_insert_prio_list(pq, prio);
-
+ pq->pq_threads++;
/* Mark this thread as being in the priority queue. */
pthread->flags |= THR_FLAGS_IN_RUNQ;
@@ -203,7 +204,7 @@ _pq_insert_tail(pq_queue_t *pq, pthread_t pthread)
if (pq->pq_lists[prio].pl_queued == 0)
/* Insert the list into the priority queue: */
pq_insert_prio_list(pq, prio);
-
+ pq->pq_threads++;
/* Mark this thread as being in the priority queue. */
pthread->flags |= THR_FLAGS_IN_RUNQ;
diff --git a/lib/libkse/thread/thr_private.h b/lib/libkse/thread/thr_private.h
index fc2e4ee..4d43242 100644
--- a/lib/libkse/thread/thr_private.h
+++ b/lib/libkse/thread/thr_private.h
@@ -143,6 +143,7 @@ typedef struct pq_queue {
int pq_size; /* number of priority lists */
#define PQF_ACTIVE 0x0001
int pq_flags;
+ int pq_threads;
} pq_queue_t;
/*
@@ -197,6 +198,7 @@ struct kse {
#define KF_STARTED 0x0001 /* kernel kse created */
#define KF_INITIALIZED 0x0002 /* initialized on 1st upcall */
int k_waiting;
+ int k_idle; /* kse is idle */
int k_error; /* syscall errno in critical */
int k_cpu; /* CPU ID when bound */
int k_done; /* this KSE is done */
@@ -294,11 +296,15 @@ do { \
#define KSE_SET_WAIT(kse) atomic_store_rel_int(&(kse)->k_waiting, 1)
-#define KSE_CLEAR_WAIT(kse) atomic_set_acq_int(&(kse)->k_waiting, 0)
+#define KSE_CLEAR_WAIT(kse) atomic_store_rel_int(&(kse)->k_waiting, 0)
#define KSE_WAITING(kse) (kse)->k_waiting != 0
#define KSE_WAKEUP(kse) kse_wakeup(&(kse)->k_mbx)
+#define KSE_SET_IDLE(kse) ((kse)->k_idle = 1)
+#define KSE_CLEAR_IDLE(kse) ((kse)->k_idle = 0)
+#define KSE_IS_IDLE(kse) ((kse)->k_idle != 0)
+
/*
* TailQ initialization values.
*/
@@ -658,6 +664,7 @@ struct pthread {
/* Thread state: */
enum pthread_state state;
+ int lock_switch;
/*
* Number of microseconds accumulated by this thread when
@@ -803,8 +810,11 @@ struct pthread {
#define THR_YIELD_CHECK(thrd) \
do { \
if (((thrd)->critical_yield != 0) && \
- !(THR_IN_CRITICAL(thrd))) \
+ !(THR_IN_CRITICAL(thrd))) { \
+ THR_LOCK_SWITCH(thrd); \
_thr_sched_switch(thrd); \
+ THR_UNLOCK_SWITCH(thrd); \
+ } \
else if (((thrd)->check_pending != 0) && \
!(THR_IN_CRITICAL(thrd))) \
_thr_sig_check_pending(thrd); \
@@ -828,15 +838,31 @@ do { \
_lock_release((lck), \
&(thrd)->lockusers[(thrd)->locklevel - 1]); \
(thrd)->locklevel--; \
- if ((thrd)->locklevel != 0) \
+ if ((thrd)->lock_switch) \
; \
- else if ((thrd)->critical_yield != 0) \
- _thr_sched_switch(thrd); \
- else if ((thrd)->check_pending != 0) \
- _thr_sig_check_pending(thrd); \
+ else { \
+ THR_YIELD_CHECK(thrd); \
+ } \
} \
} while (0)
+#define THR_LOCK_SWITCH(thrd) \
+do { \
+ THR_ASSERT(!(thrd)->lock_switch, "context switch locked"); \
+ _kse_critical_enter(); \
+ KSE_SCHED_LOCK((thrd)->kse, (thrd)->kseg); \
+ (thrd)->lock_switch = 1; \
+} while (0)
+
+#define THR_UNLOCK_SWITCH(thrd) \
+do { \
+ THR_ASSERT((thrd)->lock_switch, "context switch not locked"); \
+ THR_ASSERT(_kse_in_critical(), "Er,not in critical region"); \
+ (thrd)->lock_switch = 0; \
+ KSE_SCHED_UNLOCK((thrd)->kse, (thrd)->kseg); \
+ _kse_critical_leave(&thrd->tmbx); \
+} while (0)
+
/*
* For now, threads will have their own lock separate from their
* KSE scheduling lock.
@@ -907,8 +933,6 @@ do { \
KSE_SCHED_UNLOCK((curthr)->kse, (thr)->kseg); \
(curthr)->locklevel--; \
_kse_critical_leave((curthr)->critical[(curthr)->locklevel]); \
- if ((curthr)->locklevel == 0) \
- THR_YIELD_CHECK(curthr); \
} while (0)
#define THR_CRITICAL_ENTER(thr) (thr)->critical_count++
@@ -917,7 +941,9 @@ do { \
if (((thr)->critical_yield != 0) && \
((thr)->critical_count == 0)) { \
(thr)->critical_yield = 0; \
+ THR_LOCK_SWITCH(thr); \
_thr_sched_switch(thr); \
+ THR_UNLOCK_SWITCH(thr); \
} \
} while (0)
@@ -1092,6 +1118,8 @@ void _thr_sigframe_restore(struct pthread *thread, struct pthread_sigframe *psf)
void _thr_seterrno(struct pthread *, int);
void _thr_enter_cancellation_point(struct pthread *);
void _thr_leave_cancellation_point(struct pthread *);
+int _thr_setconcurrency(int new_level);
+int _thr_setmaxconcurrency(void);
/* XXX - Stuff that goes away when my sources get more up to date. */
/* #include <sys/kse.h> */
diff --git a/lib/libkse/thread/thr_sigsuspend.c b/lib/libkse/thread/thr_sigsuspend.c
index 7ce027a..9ada1b2 100644
--- a/lib/libkse/thread/thr_sigsuspend.c
+++ b/lib/libkse/thread/thr_sigsuspend.c
@@ -53,13 +53,14 @@ _sigsuspend(const sigset_t *set)
memcpy(&curthread->tmbx.tm_context.uc_sigmask,
set, sizeof(sigset_t));
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_SIGSUSPEND);
- THR_SCHED_UNLOCK(curthread, curthread);
-
/* Wait for a signal: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
+
/* Always return an interrupted error: */
errno = EINTR;
diff --git a/lib/libkse/thread/thr_sigwait.c b/lib/libkse/thread/thr_sigwait.c
index b955251..7b5a31f 100644
--- a/lib/libkse/thread/thr_sigwait.c
+++ b/lib/libkse/thread/thr_sigwait.c
@@ -134,11 +134,10 @@ _sigwait(const sigset_t *set, int *sig)
curthread->data.sigwait = &waitset;
/* Wait for a signal: */
- THR_SCHED_LOCK(curthread, curthread);
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_SIGWAIT);
- THR_SCHED_UNLOCK(curthread, curthread);
_thr_sched_switch(curthread);
-
+ THR_UNLOCK_SWITCH(curthread);
/* Return the signal number to the caller: */
*sig = curthread->signo;
diff --git a/lib/libkse/thread/thr_spinlock.c b/lib/libkse/thread/thr_spinlock.c
index cb71a46..56691dd 100644
--- a/lib/libkse/thread/thr_spinlock.c
+++ b/lib/libkse/thread/thr_spinlock.c
@@ -33,12 +33,8 @@
*
*/
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <sched.h>
-#include <pthread.h>
-#include <unistd.h>
+#include <sys/types.h>
+#include <machine/atomic.h>
#include <libc_private.h>
#include "spinlock.h"
@@ -52,7 +48,12 @@
void
_spinunlock(spinlock_t *lck)
{
- lck->access_lock = 0;
+ kse_critical_t crit;
+
+ crit = (kse_critical_t)lck->fname;
+ atomic_store_rel_long(&lck->access_lock, 0);
+ if (crit != NULL)
+ _kse_critical_leave(crit);
}
@@ -65,14 +66,21 @@ _spinunlock(spinlock_t *lck)
void
_spinlock(spinlock_t *lck)
{
+ kse_critical_t crit;
+
/*
* Try to grab the lock and loop if another thread grabs
* it before we do.
*/
+ if (_kse_isthreaded())
+ crit = _kse_critical_enter();
+ else
+ crit = NULL;
while(_atomic_lock(&lck->access_lock)) {
while (lck->access_lock)
;
}
+ lck->fname = (char *)crit;
}
/*
@@ -88,12 +96,5 @@ _spinlock(spinlock_t *lck)
void
_spinlock_debug(spinlock_t *lck, char *fname, int lineno)
{
- /*
- * Try to grab the lock and loop if another thread grabs
- * it before we do.
- */
- while(_atomic_lock(&lck->access_lock)) {
- while (lck->access_lock)
- ;
- }
+ _spinlock(lck);
}
diff --git a/lib/libkse/thread/thr_yield.c b/lib/libkse/thread/thr_yield.c
index acaa3c5..dfe7278 100644
--- a/lib/libkse/thread/thr_yield.c
+++ b/lib/libkse/thread/thr_yield.c
@@ -46,8 +46,9 @@ _sched_yield(void)
curthread->slice_usec = -1;
/* Schedule the next thread: */
+ THR_LOCK_SWITCH(curthread);
_thr_sched_switch(curthread);
-
+ THR_UNLOCK_SWITCH(curthread);
/* Always return no error. */
return(0);
}
@@ -62,5 +63,7 @@ _pthread_yield(void)
curthread->slice_usec = -1;
/* Schedule the next thread: */
+ THR_LOCK_SWITCH(curthread);
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
}
diff --git a/lib/libpthread/arch/i386/i386/thr_switch.S b/lib/libpthread/arch/i386/i386/thr_switch.S
index 4438b90..f8da6b6 100644
--- a/lib/libpthread/arch/i386/i386/thr_switch.S
+++ b/lib/libpthread/arch/i386/i386/thr_switch.S
@@ -103,9 +103,11 @@ ENTRY(_thread_switch)
* all registers are now moved out of mailbox,
* it's safe to set current thread pointer
*/
+ cmpl $0, %ecx
+ je 8f
movl %ebx,(%ecx)
- popl %ecx /* %ecx off stack */
+8: popl %ecx /* %ecx off stack */
popl %ebx /* %ebx off stack */
popfl /* flags off stack */
-5: ret /* %eip off stack */
+9: ret /* %eip off stack */
diff --git a/lib/libpthread/thread/thr_cancel.c b/lib/libpthread/thread/thr_cancel.c
index 23e0dfa..fc05b55 100644
--- a/lib/libpthread/thread/thr_cancel.c
+++ b/lib/libpthread/thread/thr_cancel.c
@@ -217,7 +217,6 @@ static void
testcancel(struct pthread *curthread)
{
/* Take the scheduling lock while fiddling with the state: */
- THR_SCHED_LOCK(curthread, curthread);
if (checkcancel(curthread) != 0) {
/* Unlock before exiting: */
@@ -227,8 +226,6 @@ testcancel(struct pthread *curthread)
pthread_exit(PTHREAD_CANCELED);
PANIC("cancel");
}
-
- THR_SCHED_UNLOCK(curthread, curthread);
}
void
@@ -236,23 +233,29 @@ _pthread_testcancel(void)
{
struct pthread *curthread = _get_curthread();
+ THR_SCHED_LOCK(curthread, curthread);
testcancel(curthread);
+ THR_SCHED_UNLOCK(curthread, curthread);
}
void
_thr_enter_cancellation_point(struct pthread *thread)
{
/* Look for a cancellation before we block: */
+ THR_SCHED_LOCK(thread, thread);
testcancel(thread);
thread->cancelflags |= THR_AT_CANCEL_POINT;
+ THR_SCHED_UNLOCK(thread, thread);
}
void
_thr_leave_cancellation_point(struct pthread *thread)
{
+ THR_SCHED_LOCK(thread, thread);
thread->cancelflags &= ~THR_AT_CANCEL_POINT;
/* Look for a cancellation after we unblock: */
testcancel(thread);
+ THR_SCHED_UNLOCK(thread, thread);
}
static void
@@ -263,9 +266,12 @@ finish_cancellation(void *arg)
curthread->continuation = NULL;
curthread->interrupted = 0;
+ THR_SCHED_LOCK(curthread, curthread);
if ((curthread->cancelflags & THR_CANCEL_NEEDED) != 0) {
curthread->cancelflags &= ~THR_CANCEL_NEEDED;
+ THR_SCHED_UNLOCK(curthread, curthread);
_thr_exit_cleanup();
pthread_exit(PTHREAD_CANCELED);
}
+ THR_SCHED_UNLOCK(curthread, curthread);
}
diff --git a/lib/libpthread/thread/thr_concurrency.c b/lib/libpthread/thread/thr_concurrency.c
index a4b055f..db84121 100644
--- a/lib/libpthread/thread/thr_concurrency.c
+++ b/lib/libpthread/thread/thr_concurrency.c
@@ -28,6 +28,8 @@
*/
#include <errno.h>
#include <pthread.h>
+#include <sys/types.h>
+#include <sys/sysctl.h>
#include "thr_private.h"
@@ -52,14 +54,8 @@ _pthread_getconcurrency(void)
int
_pthread_setconcurrency(int new_level)
{
- struct pthread *curthread;
- struct kse *newkse;
- kse_critical_t crit;
- int kse_count;
- int i;
int ret;
-
if (new_level < 0)
ret = EINVAL;
else if (new_level == level)
@@ -71,50 +67,72 @@ _pthread_setconcurrency(int new_level)
DBG_MSG("Can't enable threading.\n");
ret = EAGAIN;
} else {
- ret = 0;
- curthread = _get_curthread();
- /* Race condition, but so what. */
- kse_count = _kse_initial->k_kseg->kg_ksecount;
- for (i = kse_count; i < new_level; i++) {
- newkse = _kse_alloc(curthread);
- if (newkse == NULL) {
- DBG_MSG("Can't alloc new KSE.\n");
- ret = EAGAIN;
- break;
- }
- newkse->k_kseg = _kse_initial->k_kseg;
- newkse->k_schedq = _kse_initial->k_schedq;
- newkse->k_curthread = NULL;
- crit = _kse_critical_enter();
+ ret = _thr_setconcurrency(new_level);
+ if (ret == 0)
+ level = new_level;
+ }
+ return (ret);
+}
+
+int
+_thr_setconcurrency(int new_level)
+{
+ struct pthread *curthread;
+ struct kse *newkse;
+ kse_critical_t crit;
+ int kse_count;
+ int i;
+ int ret;
+
+ ret = 0;
+ curthread = _get_curthread();
+ /* Race condition, but so what. */
+ kse_count = _kse_initial->k_kseg->kg_ksecount;
+ for (i = kse_count; i < new_level; i++) {
+ newkse = _kse_alloc(curthread);
+ if (newkse == NULL) {
+ DBG_MSG("Can't alloc new KSE.\n");
+ ret = EAGAIN;
+ break;
+ }
+ newkse->k_kseg = _kse_initial->k_kseg;
+ newkse->k_schedq = _kse_initial->k_schedq;
+ newkse->k_curthread = NULL;
+ crit = _kse_critical_enter();
+ KSE_SCHED_LOCK(curthread->kse, newkse->k_kseg);
+ TAILQ_INSERT_TAIL(&newkse->k_kseg->kg_kseq,
+ newkse, k_kgqe);
+ newkse->k_kseg->kg_ksecount++;
+ KSE_SCHED_UNLOCK(curthread->kse, newkse->k_kseg);
+ if (kse_create(&newkse->k_mbx, 0) != 0) {
KSE_SCHED_LOCK(curthread->kse, newkse->k_kseg);
- TAILQ_INSERT_TAIL(&newkse->k_kseg->kg_kseq,
+ TAILQ_REMOVE(&newkse->k_kseg->kg_kseq,
newkse, k_kgqe);
- newkse->k_kseg->kg_ksecount++;
+ newkse->k_kseg->kg_ksecount--;
KSE_SCHED_UNLOCK(curthread->kse, newkse->k_kseg);
- if (_ksd_setprivate(&_kse_initial->k_ksd) != 0) {
- /* This should never happen. */
- PANIC("pthread_setconcurrency: Unable to "
- "set KSE specific data");
- }
- newkse->k_flags |= KF_INITIALIZED;
- if (kse_create(&newkse->k_mbx, 0) != 0) {
- _ksd_setprivate(&curthread->kse->k_ksd);
- KSE_SCHED_LOCK(curthread->kse, newkse->k_kseg);
- TAILQ_REMOVE(&newkse->k_kseg->kg_kseq,
- newkse, k_kgqe);
- newkse->k_kseg->kg_ksecount--;
- KSE_SCHED_UNLOCK(curthread->kse, newkse->k_kseg);
- _kse_critical_leave(crit);
- _kse_free(curthread, newkse);
- DBG_MSG("kse_create syscall failed.\n");
- ret = EAGAIN;
- break;
- }
- _ksd_setprivate(&curthread->kse->k_ksd);
+ _kse_critical_leave(crit);
+ _kse_free(curthread, newkse);
+ DBG_MSG("kse_create syscall failed.\n");
+ ret = EAGAIN;
+ break;
+ } else {
_kse_critical_leave(crit);
}
- if (ret == 0)
- level = new_level;
}
return (ret);
}
+
+int
+_thr_setmaxconcurrency(void)
+{
+ int vcpu;
+ int len;
+ int ret;
+
+ len = sizeof(vcpu);
+ ret = sysctlbyname("kern.threads.virtual_cpu", &vcpu, &len, NULL, NULL);
+ if (ret == 0 && vcpu > 0)
+ ret = _thr_setconcurrency(vcpu);
+ return (ret);
+}
+
diff --git a/lib/libpthread/thread/thr_cond.c b/lib/libpthread/thread/thr_cond.c
index 537c264..32a152d 100644
--- a/lib/libpthread/thread/thr_cond.c
+++ b/lib/libpthread/thread/thr_cond.c
@@ -267,6 +267,7 @@ _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
* lock); we should be able to safely
* set the state.
*/
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_COND_WAIT);
/* Remember the CV: */
@@ -280,6 +281,7 @@ _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
_thr_sched_switch(curthread);
curthread->data.cond = NULL;
+ THR_UNLOCK_SWITCH(curthread);
/*
* XXX - This really isn't a good check
@@ -324,8 +326,10 @@ _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
* that the mutex be reaquired prior to
* cancellation.
*/
- if (done != 0)
+ if (done != 0) {
rval = _mutex_cv_lock(mutex);
+ unlock_mutex = 1;
+ }
}
}
break;
@@ -475,6 +479,7 @@ _pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
* lock); we should be able to safely
* set the state.
*/
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_COND_WAIT);
/* Remember the CV: */
@@ -488,6 +493,7 @@ _pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
_thr_sched_switch(curthread);
curthread->data.cond = NULL;
+ THR_UNLOCK_SWITCH(curthread);
/*
* XXX - This really isn't a good check
diff --git a/lib/libpthread/thread/thr_create.c b/lib/libpthread/thread/thr_create.c
index deb26de..4c65d3c 100644
--- a/lib/libpthread/thread/thr_create.c
+++ b/lib/libpthread/thread/thr_create.c
@@ -297,6 +297,13 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
* pair if necessary.
*/
ret = _thr_schedule_add(curthread, new_thread);
+ if (ret != 0) {
+ KSE_LOCK_ACQUIRE(curthread->kse,
+ &_thread_list_lock);
+ THR_LIST_REMOVE(new_thread);
+ KSE_LOCK_RELEASE(curthread->kse,
+ &_thread_list_lock);
+ }
_kse_critical_leave(crit);
if (ret != 0)
free_thread(curthread, new_thread);
diff --git a/lib/libpthread/thread/thr_detach.c b/lib/libpthread/thread/thr_detach.c
index 30b7ce5..66db912 100644
--- a/lib/libpthread/thread/thr_detach.c
+++ b/lib/libpthread/thread/thr_detach.c
@@ -44,8 +44,6 @@ _pthread_detach(pthread_t pthread)
{
struct pthread *curthread = _get_curthread();
struct pthread *joiner;
- kse_critical_t crit;
- int dead;
int rval = 0;
/* Check for invalid calling parameters: */
@@ -56,7 +54,6 @@ _pthread_detach(pthread_t pthread)
else if ((rval = _thr_ref_add(curthread, pthread,
/*include dead*/1)) != 0) {
/* Return an error: */
- _thr_leave_cancellation_point(curthread);
}
/* Check if the thread is already detached: */
@@ -91,20 +88,11 @@ _pthread_detach(pthread_t pthread)
}
joiner = NULL;
}
- dead = (pthread->flags & THR_FLAGS_GC_SAFE) != 0;
THR_SCHED_UNLOCK(curthread, pthread);
-
- if (dead != 0) {
- crit = _kse_critical_enter();
- KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
- THR_GCLIST_ADD(pthread);
- KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock);
- _kse_critical_leave(crit);
- }
_thr_ref_delete(curthread, pthread);
/* See if there is a thread waiting in pthread_join(): */
- if (joiner != NULL) {
+ if (joiner != NULL && _thr_ref_add(curthread, joiner, 0) == 0) {
/* Lock the joiner before fiddling with it. */
THR_SCHED_LOCK(curthread, joiner);
if (joiner->join_status.thread == pthread) {
@@ -118,6 +106,7 @@ _pthread_detach(pthread_t pthread)
_thr_setrunnable_unlocked(joiner);
}
THR_SCHED_UNLOCK(curthread, joiner);
+ _thr_ref_delete(curthread, joiner);
}
}
diff --git a/lib/libpthread/thread/thr_exit.c b/lib/libpthread/thread/thr_exit.c
index 4a82b12..8435f43 100644
--- a/lib/libpthread/thread/thr_exit.c
+++ b/lib/libpthread/thread/thr_exit.c
@@ -104,7 +104,9 @@ _pthread_exit(void *status)
* Flag this thread as exiting. Threads should now be prevented
* from joining to this thread.
*/
+ THR_SCHED_LOCK(curthread, curthread);
curthread->flags |= THR_FLAGS_EXITING;
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Save the return value: */
curthread->ret = status;
@@ -121,10 +123,11 @@ _pthread_exit(void *status)
}
/* This thread will never be re-scheduled. */
- THR_SCHED_LOCK(curthread, curthread);
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_DEAD);
- THR_SCHED_UNLOCK(curthread, curthread);
_thr_sched_switch(curthread);
+ /* Never reach! */
+ THR_UNLOCK_SWITCH(curthread);
/* This point should not be reached. */
PANIC("Dead thread has resumed");
diff --git a/lib/libpthread/thread/thr_find_thread.c b/lib/libpthread/thread/thr_find_thread.c
index b5cae66..0c813db 100644
--- a/lib/libpthread/thread/thr_find_thread.c
+++ b/lib/libpthread/thread/thr_find_thread.c
@@ -85,6 +85,9 @@ _thr_ref_delete(struct pthread *curthread, struct pthread *thread)
KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
thread->refcount--;
curthread->critical_count--;
+ if ((thread->refcount == 0) &&
+ (thread->flags & THR_FLAGS_GC_SAFE) != 0)
+ THR_GCLIST_ADD(thread);
KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock);
_kse_critical_leave(crit);
}
diff --git a/lib/libpthread/thread/thr_join.c b/lib/libpthread/thread/thr_join.c
index 77e3a8c..7cbc192 100644
--- a/lib/libpthread/thread/thr_join.c
+++ b/lib/libpthread/thread/thr_join.c
@@ -70,16 +70,16 @@ _pthread_join(pthread_t pthread, void **thread_return)
return (ESRCH);
}
+ THR_SCHED_LOCK(curthread, pthread);
/* Check if this thread has been detached: */
if ((pthread->attr.flags & PTHREAD_DETACHED) != 0) {
+ THR_SCHED_UNLOCK(curthread, pthread);
/* Remove the reference and return an error: */
_thr_ref_delete(curthread, pthread);
ret = ESRCH;
} else {
/* Lock the target thread while checking its state. */
- THR_SCHED_LOCK(curthread, pthread);
- if ((pthread->state == PS_DEAD) ||
- ((pthread->flags & THR_FLAGS_EXITING) != 0)) {
+ if (pthread->state == PS_DEAD) {
if (thread_return != NULL)
/* Return the thread's return value: */
*thread_return = pthread->ret;
@@ -123,15 +123,13 @@ _pthread_join(pthread_t pthread, void **thread_return)
THR_SCHED_UNLOCK(curthread, pthread);
_thr_ref_delete(curthread, pthread);
- THR_SCHED_LOCK(curthread, curthread);
- if (curthread->join_status.thread == pthread)
- THR_SET_STATE(curthread, PS_JOIN);
- THR_SCHED_UNLOCK(curthread, curthread);
-
+ THR_LOCK_SWITCH(curthread);
while (curthread->join_status.thread == pthread) {
+ THR_SET_STATE(curthread, PS_JOIN);
/* Schedule the next thread: */
_thr_sched_switch(curthread);
}
+ THR_UNLOCK_SWITCH(curthread);
/*
* The thread return value and error are set by the
diff --git a/lib/libpthread/thread/thr_kern.c b/lib/libpthread/thread/thr_kern.c
index 63283f7..5762219 100644
--- a/lib/libpthread/thread/thr_kern.c
+++ b/lib/libpthread/thread/thr_kern.c
@@ -95,14 +95,7 @@ __FBSDID("$FreeBSD$");
_pq_remove(&(kse)->k_schedq->sq_runq, thrd)
#define KSE_RUNQ_FIRST(kse) _pq_first(&(kse)->k_schedq->sq_runq)
-/*
- * XXX - Remove when David commits kernel changes to support these.
- */
-#ifndef KMF_NOUPCALL
-#define KMF_NOUPCALL 0x01
-#define KMF_NOCOMPLETED 0x02
-#endif
-
+#define KSE_RUNQ_THREADS(kse) ((kse)->k_schedq->sq_runq.pq_threads)
/*
* We've got to keep track of everything that is allocated, not only
@@ -140,11 +133,11 @@ static void kseg_free_unlocked(struct kse_group *kseg);
static void kseg_init(struct kse_group *kseg);
static void kseg_reinit(struct kse_group *kseg);
static void kse_waitq_insert(struct pthread *thread);
+static void kse_wakeup_multi(struct kse *curkse);
+static void kse_wakeup_one(struct pthread *thread);
static void thr_cleanup(struct kse *kse, struct pthread *curthread);
-#ifdef NOT_YET
static void thr_resume_wrapper(int unused_1, siginfo_t *unused_2,
ucontext_t *ucp);
-#endif
static void thr_resume_check(struct pthread *curthread, ucontext_t *ucp,
struct pthread_sigframe *psf);
static int thr_timedout(struct pthread *thread, struct timespec *curtime);
@@ -368,11 +361,13 @@ _kse_setthreaded(int threaded)
_kse_initial->k_flags |= KF_STARTED;
if (kse_create(&_kse_initial->k_mbx, 0) != 0) {
_kse_initial->k_flags &= ~KF_STARTED;
+ __isthreaded = 0;
/* may abort() */
DBG_MSG("kse_create failed\n");
return (-1);
}
KSE_SET_MBOX(_kse_initial, _thr_initial);
+ _thr_setmaxconcurrency();
}
return (0);
}
@@ -466,16 +461,14 @@ _thr_lock_wait(struct lock *lock, struct lockuser *lu)
while (_LCK_BUSY(lu) && count < 300)
count++;
while (_LCK_BUSY(lu)) {
- THR_SCHED_LOCK(curthread, curthread);
+ THR_LOCK_SWITCH(curthread);
if (_LCK_BUSY(lu)) {
/* Wait for the lock: */
atomic_store_rel_int(&curthread->need_wakeup, 1);
THR_SET_STATE(curthread, PS_LOCKWAIT);
- THR_SCHED_UNLOCK(curthread, curthread);
_thr_sched_switch(curthread);
}
- else
- THR_SCHED_UNLOCK(curthread, curthread);
+ THR_UNLOCK_SWITCH(curthread);
}
}
@@ -484,14 +477,26 @@ _thr_lock_wakeup(struct lock *lock, struct lockuser *lu)
{
struct pthread *thread;
struct pthread *curthread;
+ int unlock;
curthread = _get_curthread();
thread = (struct pthread *)_LCK_GET_PRIVATE(lu);
- THR_SCHED_LOCK(curthread, thread);
+ unlock = 0;
+ if (curthread->kseg == thread->kseg) {
+ /* Not already locked */
+ if (curthread->lock_switch == 0) {
+ THR_SCHED_LOCK(curthread, thread);
+ unlock = 1;
+ }
+ } else {
+ THR_SCHED_LOCK(curthread, thread);
+ unlock = 1;
+ }
_thr_setrunnable_unlocked(thread);
atomic_store_rel_int(&thread->need_wakeup, 0);
- THR_SCHED_UNLOCK(curthread, thread);
+ if (unlock)
+ THR_SCHED_UNLOCK(curthread, thread);
}
kse_critical_t
@@ -541,12 +546,12 @@ void
_thr_sched_switch(struct pthread *curthread)
{
struct pthread_sigframe psf;
- kse_critical_t crit;
struct kse *curkse;
volatile int once = 0;
/* We're in the scheduler, 5 by 5: */
- crit = _kse_critical_enter();
+ THR_ASSERT(curthread->lock_switch, "lock_switch");
+ THR_ASSERT(_kse_in_critical(), "not in critical region");
curkse = _get_curkse();
curthread->need_switchout = 1; /* The thread yielded on its own. */
@@ -568,7 +573,11 @@ _thr_sched_switch(struct pthread *curthread)
*/
if ((once == 0) && (!THR_IN_CRITICAL(curthread))) {
once = 1;
+ curthread->critical_count++;
+ THR_UNLOCK_SWITCH(curthread);
+ curthread->critical_count--;
thr_resume_check(curthread, &curthread->tmbx.tm_context, &psf);
+ THR_LOCK_SWITCH(curthread);
}
}
@@ -734,14 +743,20 @@ kse_sched_multi(struct kse *curkse)
KSE_CLEAR_WAIT(curkse);
}
- /* Lock the scheduling lock. */
- KSE_SCHED_LOCK(curkse, curkse->k_kseg);
+ curthread = curkse->k_curthread;
+ if (curthread == NULL || curthread->lock_switch == 0) {
+ /*
+ * curthread was preempted by upcall, it is not a volunteer
+ * context switch. Lock the scheduling lock.
+ */
+ KSE_SCHED_LOCK(curkse, curkse->k_kseg);
+ }
/*
* If the current thread was completed in another KSE, then
* it will be in the run queue. Don't mark it as being blocked.
*/
- if (((curthread = curkse->k_curthread) != NULL) &&
+ if ((curthread != NULL) &&
((curthread->flags & THR_FLAGS_IN_RUNQ) == 0) &&
(curthread->need_switchout == 0)) {
/*
@@ -774,17 +789,22 @@ kse_sched_multi(struct kse *curkse)
* Resume the thread and tell it to yield when
* it leaves the critical region.
*/
- curthread->critical_yield = 0;
+ curthread->critical_yield = 1;
curthread->active = 1;
if ((curthread->flags & THR_FLAGS_IN_RUNQ) != 0)
KSE_RUNQ_REMOVE(curkse, curthread);
curkse->k_curthread = curthread;
curthread->kse = curkse;
- KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
DBG_MSG("Continuing thread %p in critical region\n",
curthread);
- ret = _thread_switch(&curthread->tmbx,
- &curkse->k_mbx.km_curthread);
+ if (curthread->lock_switch) {
+ KSE_SCHED_LOCK(curkse, curkse->k_kseg);
+ ret = _thread_switch(&curthread->tmbx, 0);
+ } else {
+ KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
+ ret = _thread_switch(&curthread->tmbx,
+ &curkse->k_mbx.km_curthread);
+ }
if (ret != 0)
PANIC("Can't resume thread in critical region\n");
}
@@ -792,6 +812,8 @@ kse_sched_multi(struct kse *curkse)
kse_switchout_thread(curkse, curthread);
curkse->k_curthread = NULL;
+ kse_wakeup_multi(curkse);
+
/* This has to be done without the scheduling lock held. */
KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
kse_check_signals(curkse);
@@ -868,6 +890,8 @@ kse_sched_multi(struct kse *curkse)
curframe = curthread->curframe;
curthread->curframe = NULL;
+ kse_wakeup_multi(curkse);
+
/* Unlock the scheduling queue: */
KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
@@ -881,10 +905,13 @@ kse_sched_multi(struct kse *curkse)
#ifdef NOT_YET
if ((curframe == NULL) && ((curthread->check_pending != 0) ||
(((curthread->cancelflags & THR_AT_CANCEL_POINT) == 0) &&
- ((curthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0)))) {
+ ((curthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0))))
+ signalcontext(&curthread->tmbx.tm_context, 0,
+ (__sighandler_t *)thr_resume_wrapper);
+#else
+ if ((curframe == NULL) && (curthread->check_pending != 0))
signalcontext(&curthread->tmbx.tm_context, 0,
(__sighandler_t *)thr_resume_wrapper);
- }
#endif
#ifdef GS_HACK
/* XXX - The kernel sometimes forgets to restore %gs properly. */
@@ -893,7 +920,13 @@ kse_sched_multi(struct kse *curkse)
/*
* Continue the thread at its current frame:
*/
- ret = _thread_switch(&curthread->tmbx, &curkse->k_mbx.km_curthread);
+ if (curthread->lock_switch) {
+ KSE_SCHED_LOCK(curkse, curkse->k_kseg);
+ ret = _thread_switch(&curthread->tmbx, 0);
+ } else {
+ ret = _thread_switch(&curthread->tmbx,
+ &curkse->k_mbx.km_curthread);
+ }
if (ret != 0)
PANIC("Thread has returned from _thread_switch");
@@ -932,7 +965,6 @@ kse_check_signals(struct kse *curkse)
}
}
-#ifdef NOT_YET
static void
thr_resume_wrapper(int unused_1, siginfo_t *unused_2, ucontext_t *ucp)
{
@@ -940,7 +972,6 @@ thr_resume_wrapper(int unused_1, siginfo_t *unused_2, ucontext_t *ucp)
thr_resume_check(curthread, ucp, NULL);
}
-#endif
static void
thr_resume_check(struct pthread *curthread, ucontext_t *ucp,
@@ -960,9 +991,11 @@ thr_resume_check(struct pthread *curthread, ucontext_t *ucp,
_thr_sig_rundown(curthread, ucp, psf);
}
+#ifdef NOT_YET
if (((curthread->cancelflags & THR_AT_CANCEL_POINT) == 0) &&
((curthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0))
pthread_testcancel();
+#endif
}
/*
@@ -1071,6 +1104,7 @@ _thr_gc(struct pthread *curthread)
* referenced. It is safe to remove all
* remnants of the thread.
*/
+ THR_LIST_REMOVE(td);
TAILQ_INSERT_HEAD(&worklist, td, gcle);
}
}
@@ -1172,11 +1206,11 @@ _thr_schedule_add(struct pthread *curthread, struct pthread *newthread)
if (need_start != 0)
kse_create(&newthread->kse->k_mbx, 0);
else if ((newthread->state == PS_RUNNING) &&
- KSE_WAITING(newthread->kse)) {
+ KSE_IS_IDLE(newthread->kse)) {
/*
* The thread is being scheduled on another KSEG.
*/
- KSE_WAKEUP(newthread->kse);
+ kse_wakeup_one(newthread);
}
ret = 0;
}
@@ -1326,6 +1360,8 @@ kse_switchout_thread(struct kse *kse, struct pthread *thread)
*/
DBG_MSG("Switching out thread %p, state %d\n", thread, thread->state);
if (thread->blocked != 0) {
+ thread->active = 0;
+ thread->need_switchout = 0;
/* This thread must have blocked in the kernel. */
/* thread->slice_usec = -1;*/ /* restart timeslice */
/*
@@ -1346,6 +1382,8 @@ kse_switchout_thread(struct kse *kse, struct pthread *thread)
* stack. It is safe to do garbage collecting
* here.
*/
+ thread->active = 0;
+ thread->need_switchout = 0;
thr_cleanup(kse, thread);
return;
break;
@@ -1456,14 +1494,18 @@ kse_wait(struct kse *kse, struct pthread *td_wait)
}
/* Don't sleep for negative times. */
if ((ts_sleep.tv_sec >= 0) && (ts_sleep.tv_nsec >= 0)) {
- KSE_SET_WAIT(kse);
+ KSE_SET_IDLE(kse);
+ kse->k_kseg->kg_idle_kses++;
KSE_SCHED_UNLOCK(kse, kse->k_kseg);
saved_flags = kse->k_mbx.km_flags;
kse->k_mbx.km_flags |= KMF_NOUPCALL;
kse_release(&ts_sleep);
kse->k_mbx.km_flags = saved_flags;
- KSE_CLEAR_WAIT(kse);
KSE_SCHED_LOCK(kse, kse->k_kseg);
+ if (KSE_IS_IDLE(kse)) {
+ KSE_CLEAR_IDLE(kse);
+ kse->k_kseg->kg_idle_kses--;
+ }
}
}
@@ -1495,11 +1537,13 @@ kse_fini(struct kse *kse)
* Add this KSE to the list of free KSEs along with
* the KSEG if is now orphaned.
*/
+#ifdef NOT_YET
KSE_LOCK_ACQUIRE(kse, &kse_lock);
if (free_kseg != NULL)
kseg_free_unlocked(free_kseg);
kse_free_unlocked(kse);
KSE_LOCK_RELEASE(kse, &kse_lock);
+#endif
kse_exit();
/* Never returns. */
} else {
@@ -1620,7 +1664,7 @@ _thr_setrunnable_unlocked(struct pthread *thread)
if ((thread->kseg->kg_flags & KGF_SINGLE_THREAD) != 0)
/* No silly queues for these threads. */
THR_SET_STATE(thread, PS_RUNNING);
- else {
+ else if (thread->state != PS_RUNNING) {
if ((thread->flags & THR_FLAGS_IN_WAITQ) != 0)
KSE_WAITQ_REMOVE(thread->kse, thread);
THR_SET_STATE(thread, PS_RUNNING);
@@ -1641,8 +1685,47 @@ _thr_setrunnable_unlocked(struct pthread *thread)
* (the KSE). If the KSE wakes up and doesn't find any more
* work it will again go back to waiting so no harm is done.
*/
- if (KSE_WAITING(thread->kse))
+ kse_wakeup_one(thread);
+}
+
+static void
+kse_wakeup_one(struct pthread *thread)
+{
+ struct kse *ke;
+
+ if (KSE_IS_IDLE(thread->kse)) {
+ KSE_CLEAR_IDLE(thread->kse);
+ thread->kseg->kg_idle_kses--;
KSE_WAKEUP(thread->kse);
+ } else {
+ TAILQ_FOREACH(ke, &thread->kseg->kg_kseq, k_kgqe) {
+ if (KSE_IS_IDLE(ke)) {
+ KSE_CLEAR_IDLE(ke);
+ ke->k_kseg->kg_idle_kses--;
+ KSE_WAKEUP(ke);
+ return;
+ }
+ }
+ }
+}
+
+static void
+kse_wakeup_multi(struct kse *curkse)
+{
+ struct kse *ke;
+ int tmp;
+
+ if ((tmp = KSE_RUNQ_THREADS(curkse)) && curkse->k_kseg->kg_idle_kses) {
+ TAILQ_FOREACH(ke, &curkse->k_kseg->kg_kseq, k_kgqe) {
+ if (KSE_IS_IDLE(ke)) {
+ KSE_CLEAR_IDLE(ke);
+ ke->k_kseg->kg_idle_kses--;
+ KSE_WAKEUP(ke);
+ if (--tmp == 0)
+ break;
+ }
+ }
+ }
}
struct pthread *
@@ -1876,7 +1959,7 @@ kse_reinit(struct kse *kse)
sigemptyset(&kse->k_sigmask);
bzero(&kse->k_sigq, sizeof(kse->k_sigq));
kse->k_check_sigq = 0;
- kse->k_flags = KF_INITIALIZED;
+ kse->k_flags = 0;
kse->k_waiting = 0;
kse->k_error = 0;
kse->k_cpu = 0;
@@ -1948,6 +2031,7 @@ _thr_alloc(struct pthread *curthread)
free_thread_count--;
}
KSE_LOCK_RELEASE(curthread->kse, &thread_lock);
+ _kse_critical_leave(crit);
}
}
if (thread == NULL)
@@ -1971,9 +2055,6 @@ _thr_free(struct pthread *curthread, struct pthread *thread)
}
else {
crit = _kse_critical_enter();
- KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
- THR_LIST_REMOVE(thread);
- KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock);
KSE_LOCK_ACQUIRE(curthread->kse, &thread_lock);
TAILQ_INSERT_HEAD(&free_threadq, thread, tle);
free_thread_count++;
diff --git a/lib/libpthread/thread/thr_mutex.c b/lib/libpthread/thread/thr_mutex.c
index 1ae12ea9..bb97db1 100644
--- a/lib/libpthread/thread/thr_mutex.c
+++ b/lib/libpthread/thread/thr_mutex.c
@@ -325,6 +325,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
/* Lock the mutex for the running thread: */
(*mutex)->m_owner = curthread;
+ THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
@@ -335,6 +336,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
(*mutex)->m_prio = curthread->active_priority;
(*mutex)->m_saved_prio =
curthread->inherited_priority;
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*mutex);
@@ -358,6 +360,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
/* Lock the mutex for the running thread: */
(*mutex)->m_owner = curthread;
+ THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
@@ -371,7 +374,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex)
curthread->inherited_priority;
curthread->inherited_priority =
(*mutex)->m_prio;
-
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Add to the list of owned mutexes: */
MUTEX_ASSERT_NOT_OWNED(*mutex);
TAILQ_INSERT_TAIL(&curthread->mutexq,
@@ -503,6 +506,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_MUTEX_WAIT);
/* Unlock the mutex structure: */
@@ -510,6 +514,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
/* Schedule the next thread: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
}
break;
@@ -520,6 +525,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
/* Lock the mutex for this thread: */
(*m)->m_owner = curthread;
+ THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
@@ -529,7 +535,6 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* Make sure the thread's scheduling lock is
* held while priorities are adjusted.
*/
- THR_SCHED_LOCK(curthread, curthread);
(*m)->m_prio = curthread->active_priority;
(*m)->m_saved_prio =
curthread->inherited_priority;
@@ -561,17 +566,18 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
- THR_SET_STATE(curthread, PS_MUTEX_WAIT);
-
if (curthread->active_priority > (*m)->m_prio)
/* Adjust priorities: */
mutex_priority_adjust(curthread, *m);
+ THR_LOCK_SWITCH(curthread);
+ THR_SET_STATE(curthread, PS_MUTEX_WAIT);
/* Unlock the mutex structure: */
THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
/* Schedule the next thread: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
}
break;
@@ -591,6 +597,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
*/
(*m)->m_owner = curthread;
+ THR_SCHED_LOCK(curthread, curthread);
/* Track number of priority mutexes owned: */
curthread->priority_mutex_count++;
@@ -601,7 +608,6 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* scheduling lock is held while priorities
* are adjusted.
*/
- THR_SCHED_LOCK(curthread, curthread);
curthread->active_priority = (*m)->m_prio;
(*m)->m_saved_prio =
curthread->inherited_priority;
@@ -636,6 +642,8 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
* region (holding the mutex lock); we should
* be able to safely set the state.
*/
+
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_MUTEX_WAIT);
/* Unlock the mutex structure: */
@@ -643,7 +651,7 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *m)
/* Schedule the next thread: */
_thr_sched_switch(curthread);
-
+ THR_UNLOCK_SWITCH(curthread);
/*
* The threads priority may have changed while
* waiting for the mutex causing a ceiling
@@ -749,9 +757,15 @@ _mutex_cv_unlock(pthread_mutex_t *m)
int
_mutex_cv_lock(pthread_mutex_t *m)
{
+ struct pthread *curthread;
int ret;
- if ((ret = _pthread_mutex_lock(m)) == 0)
+
+ curthread = _get_curthread();
+ if ((ret = _pthread_mutex_lock(m)) == 0) {
+ THR_LOCK_ACQUIRE(curthread, &(*m)->m_lock);
(*m)->m_refcount--;
+ THR_LOCK_RELEASE(curthread, &(*m)->m_lock);
+ }
return (ret);
}
@@ -807,6 +821,8 @@ mutex_self_lock(struct pthread *curthread, pthread_mutex_t m)
* What SS2 define as a 'normal' mutex. Intentionally
* deadlock on attempts to get a lock you already own.
*/
+
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_DEADLOCK);
/* Unlock the mutex structure: */
@@ -814,6 +830,7 @@ mutex_self_lock(struct pthread *curthread, pthread_mutex_t m)
/* Schedule the next thread: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
break;
case PTHREAD_MUTEX_RECURSIVE:
@@ -917,12 +934,12 @@ mutex_unlock_common(pthread_mutex_t *m, int add_reference)
curthread->active_priority =
MAX(curthread->inherited_priority,
curthread->base_priority);
- THR_SCHED_UNLOCK(curthread, curthread);
/*
* This thread now owns one less priority mutex.
*/
curthread->priority_mutex_count--;
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Remove the mutex from the threads queue. */
MUTEX_ASSERT_IS_OWNED(*m);
@@ -974,12 +991,12 @@ mutex_unlock_common(pthread_mutex_t *m, int add_reference)
curthread->active_priority =
MAX(curthread->inherited_priority,
curthread->base_priority);
- THR_SCHED_UNLOCK(curthread, curthread);
/*
* This thread now owns one less priority mutex.
*/
curthread->priority_mutex_count--;
+ THR_SCHED_UNLOCK(curthread, curthread);
/* Remove the mutex from the threads queue. */
MUTEX_ASSERT_IS_OWNED(*m);
diff --git a/lib/libpthread/thread/thr_nanosleep.c b/lib/libpthread/thread/thr_nanosleep.c
index bec3b66..e83f4f4 100644
--- a/lib/libpthread/thread/thr_nanosleep.c
+++ b/lib/libpthread/thread/thr_nanosleep.c
@@ -60,14 +60,14 @@ _nanosleep(const struct timespec *time_to_sleep,
/* Calculate the time for the current thread to wake up: */
TIMESPEC_ADD(&curthread->wakeup_time, &ts, time_to_sleep);
- THR_SCHED_LOCK(curthread, curthread);
+ THR_LOCK_SWITCH(curthread);
curthread->interrupted = 0;
THR_SET_STATE(curthread, PS_SLEEP_WAIT);
- THR_SCHED_UNLOCK(curthread, curthread);
/* Reschedule the current thread to sleep: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
/* Calculate the remaining time to sleep: */
KSE_GET_TOD(curthread->kse, &ts1);
diff --git a/lib/libpthread/thread/thr_priority_queue.c b/lib/libpthread/thread/thr_priority_queue.c
index 3f261f8..2822aa8 100644
--- a/lib/libpthread/thread/thr_priority_queue.c
+++ b/lib/libpthread/thread/thr_priority_queue.c
@@ -126,6 +126,7 @@ _pq_init(pq_queue_t *pq)
/* Initialize the priority queue: */
TAILQ_INIT(&pq->pq_queue);
pq->pq_flags = 0;
+ pq->pq_threads = 0;
}
return (ret);
}
@@ -151,10 +152,10 @@ _pq_remove(pq_queue_t *pq, pthread_t pthread)
* from the priority queue when _pq_first is called.
*/
TAILQ_REMOVE(&pq->pq_lists[prio].pl_head, pthread, pqe);
-
+ pq->pq_threads--;
/* This thread is now longer in the priority queue. */
pthread->flags &= ~THR_FLAGS_IN_RUNQ;
-
+
PQ_CLEAR_ACTIVE(pq);
}
@@ -177,7 +178,7 @@ _pq_insert_head(pq_queue_t *pq, pthread_t pthread)
if (pq->pq_lists[prio].pl_queued == 0)
/* Insert the list into the priority queue: */
pq_insert_prio_list(pq, prio);
-
+ pq->pq_threads++;
/* Mark this thread as being in the priority queue. */
pthread->flags |= THR_FLAGS_IN_RUNQ;
@@ -203,7 +204,7 @@ _pq_insert_tail(pq_queue_t *pq, pthread_t pthread)
if (pq->pq_lists[prio].pl_queued == 0)
/* Insert the list into the priority queue: */
pq_insert_prio_list(pq, prio);
-
+ pq->pq_threads++;
/* Mark this thread as being in the priority queue. */
pthread->flags |= THR_FLAGS_IN_RUNQ;
diff --git a/lib/libpthread/thread/thr_private.h b/lib/libpthread/thread/thr_private.h
index fc2e4ee..4d43242 100644
--- a/lib/libpthread/thread/thr_private.h
+++ b/lib/libpthread/thread/thr_private.h
@@ -143,6 +143,7 @@ typedef struct pq_queue {
int pq_size; /* number of priority lists */
#define PQF_ACTIVE 0x0001
int pq_flags;
+ int pq_threads;
} pq_queue_t;
/*
@@ -197,6 +198,7 @@ struct kse {
#define KF_STARTED 0x0001 /* kernel kse created */
#define KF_INITIALIZED 0x0002 /* initialized on 1st upcall */
int k_waiting;
+ int k_idle; /* kse is idle */
int k_error; /* syscall errno in critical */
int k_cpu; /* CPU ID when bound */
int k_done; /* this KSE is done */
@@ -294,11 +296,15 @@ do { \
#define KSE_SET_WAIT(kse) atomic_store_rel_int(&(kse)->k_waiting, 1)
-#define KSE_CLEAR_WAIT(kse) atomic_set_acq_int(&(kse)->k_waiting, 0)
+#define KSE_CLEAR_WAIT(kse) atomic_store_rel_int(&(kse)->k_waiting, 0)
#define KSE_WAITING(kse) (kse)->k_waiting != 0
#define KSE_WAKEUP(kse) kse_wakeup(&(kse)->k_mbx)
+#define KSE_SET_IDLE(kse) ((kse)->k_idle = 1)
+#define KSE_CLEAR_IDLE(kse) ((kse)->k_idle = 0)
+#define KSE_IS_IDLE(kse) ((kse)->k_idle != 0)
+
/*
* TailQ initialization values.
*/
@@ -658,6 +664,7 @@ struct pthread {
/* Thread state: */
enum pthread_state state;
+ int lock_switch;
/*
* Number of microseconds accumulated by this thread when
@@ -803,8 +810,11 @@ struct pthread {
#define THR_YIELD_CHECK(thrd) \
do { \
if (((thrd)->critical_yield != 0) && \
- !(THR_IN_CRITICAL(thrd))) \
+ !(THR_IN_CRITICAL(thrd))) { \
+ THR_LOCK_SWITCH(thrd); \
_thr_sched_switch(thrd); \
+ THR_UNLOCK_SWITCH(thrd); \
+ } \
else if (((thrd)->check_pending != 0) && \
!(THR_IN_CRITICAL(thrd))) \
_thr_sig_check_pending(thrd); \
@@ -828,15 +838,31 @@ do { \
_lock_release((lck), \
&(thrd)->lockusers[(thrd)->locklevel - 1]); \
(thrd)->locklevel--; \
- if ((thrd)->locklevel != 0) \
+ if ((thrd)->lock_switch) \
; \
- else if ((thrd)->critical_yield != 0) \
- _thr_sched_switch(thrd); \
- else if ((thrd)->check_pending != 0) \
- _thr_sig_check_pending(thrd); \
+ else { \
+ THR_YIELD_CHECK(thrd); \
+ } \
} \
} while (0)
+#define THR_LOCK_SWITCH(thrd) \
+do { \
+ THR_ASSERT(!(thrd)->lock_switch, "context switch locked"); \
+ _kse_critical_enter(); \
+ KSE_SCHED_LOCK((thrd)->kse, (thrd)->kseg); \
+ (thrd)->lock_switch = 1; \
+} while (0)
+
+#define THR_UNLOCK_SWITCH(thrd) \
+do { \
+ THR_ASSERT((thrd)->lock_switch, "context switch not locked"); \
+ THR_ASSERT(_kse_in_critical(), "Er,not in critical region"); \
+ (thrd)->lock_switch = 0; \
+ KSE_SCHED_UNLOCK((thrd)->kse, (thrd)->kseg); \
+ _kse_critical_leave(&thrd->tmbx); \
+} while (0)
+
/*
* For now, threads will have their own lock separate from their
* KSE scheduling lock.
@@ -907,8 +933,6 @@ do { \
KSE_SCHED_UNLOCK((curthr)->kse, (thr)->kseg); \
(curthr)->locklevel--; \
_kse_critical_leave((curthr)->critical[(curthr)->locklevel]); \
- if ((curthr)->locklevel == 0) \
- THR_YIELD_CHECK(curthr); \
} while (0)
#define THR_CRITICAL_ENTER(thr) (thr)->critical_count++
@@ -917,7 +941,9 @@ do { \
if (((thr)->critical_yield != 0) && \
((thr)->critical_count == 0)) { \
(thr)->critical_yield = 0; \
+ THR_LOCK_SWITCH(thr); \
_thr_sched_switch(thr); \
+ THR_UNLOCK_SWITCH(thr); \
} \
} while (0)
@@ -1092,6 +1118,8 @@ void _thr_sigframe_restore(struct pthread *thread, struct pthread_sigframe *psf)
void _thr_seterrno(struct pthread *, int);
void _thr_enter_cancellation_point(struct pthread *);
void _thr_leave_cancellation_point(struct pthread *);
+int _thr_setconcurrency(int new_level);
+int _thr_setmaxconcurrency(void);
/* XXX - Stuff that goes away when my sources get more up to date. */
/* #include <sys/kse.h> */
diff --git a/lib/libpthread/thread/thr_sigsuspend.c b/lib/libpthread/thread/thr_sigsuspend.c
index 7ce027a..9ada1b2 100644
--- a/lib/libpthread/thread/thr_sigsuspend.c
+++ b/lib/libpthread/thread/thr_sigsuspend.c
@@ -53,13 +53,14 @@ _sigsuspend(const sigset_t *set)
memcpy(&curthread->tmbx.tm_context.uc_sigmask,
set, sizeof(sigset_t));
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_SIGSUSPEND);
- THR_SCHED_UNLOCK(curthread, curthread);
-
/* Wait for a signal: */
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
+
/* Always return an interrupted error: */
errno = EINTR;
diff --git a/lib/libpthread/thread/thr_sigwait.c b/lib/libpthread/thread/thr_sigwait.c
index b955251..7b5a31f 100644
--- a/lib/libpthread/thread/thr_sigwait.c
+++ b/lib/libpthread/thread/thr_sigwait.c
@@ -134,11 +134,10 @@ _sigwait(const sigset_t *set, int *sig)
curthread->data.sigwait = &waitset;
/* Wait for a signal: */
- THR_SCHED_LOCK(curthread, curthread);
+ THR_LOCK_SWITCH(curthread);
THR_SET_STATE(curthread, PS_SIGWAIT);
- THR_SCHED_UNLOCK(curthread, curthread);
_thr_sched_switch(curthread);
-
+ THR_UNLOCK_SWITCH(curthread);
/* Return the signal number to the caller: */
*sig = curthread->signo;
diff --git a/lib/libpthread/thread/thr_spinlock.c b/lib/libpthread/thread/thr_spinlock.c
index cb71a46..56691dd 100644
--- a/lib/libpthread/thread/thr_spinlock.c
+++ b/lib/libpthread/thread/thr_spinlock.c
@@ -33,12 +33,8 @@
*
*/
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <sched.h>
-#include <pthread.h>
-#include <unistd.h>
+#include <sys/types.h>
+#include <machine/atomic.h>
#include <libc_private.h>
#include "spinlock.h"
@@ -52,7 +48,12 @@
void
_spinunlock(spinlock_t *lck)
{
- lck->access_lock = 0;
+ kse_critical_t crit;
+
+ crit = (kse_critical_t)lck->fname;
+ atomic_store_rel_long(&lck->access_lock, 0);
+ if (crit != NULL)
+ _kse_critical_leave(crit);
}
@@ -65,14 +66,21 @@ _spinunlock(spinlock_t *lck)
void
_spinlock(spinlock_t *lck)
{
+ kse_critical_t crit;
+
/*
* Try to grab the lock and loop if another thread grabs
* it before we do.
*/
+ if (_kse_isthreaded())
+ crit = _kse_critical_enter();
+ else
+ crit = NULL;
while(_atomic_lock(&lck->access_lock)) {
while (lck->access_lock)
;
}
+ lck->fname = (char *)crit;
}
/*
@@ -88,12 +96,5 @@ _spinlock(spinlock_t *lck)
void
_spinlock_debug(spinlock_t *lck, char *fname, int lineno)
{
- /*
- * Try to grab the lock and loop if another thread grabs
- * it before we do.
- */
- while(_atomic_lock(&lck->access_lock)) {
- while (lck->access_lock)
- ;
- }
+ _spinlock(lck);
}
diff --git a/lib/libpthread/thread/thr_yield.c b/lib/libpthread/thread/thr_yield.c
index acaa3c5..dfe7278 100644
--- a/lib/libpthread/thread/thr_yield.c
+++ b/lib/libpthread/thread/thr_yield.c
@@ -46,8 +46,9 @@ _sched_yield(void)
curthread->slice_usec = -1;
/* Schedule the next thread: */
+ THR_LOCK_SWITCH(curthread);
_thr_sched_switch(curthread);
-
+ THR_UNLOCK_SWITCH(curthread);
/* Always return no error. */
return(0);
}
@@ -62,5 +63,7 @@ _pthread_yield(void)
curthread->slice_usec = -1;
/* Schedule the next thread: */
+ THR_LOCK_SWITCH(curthread);
_thr_sched_switch(curthread);
+ THR_UNLOCK_SWITCH(curthread);
}
OpenPOWER on IntegriCloud