Treat initial thread as scope system thread when KSE mode is not activated

yet, so we can protect some locking code from being interrupted by signal
handling. When KSE mode is turned on, reset the thread flag to scope process
except we are running in 1:1 mode which we needn't turn it off.
Also remove some unused member variables in structure kse.

Tested by: deischen

12 files changed, 47 insertions, 81 deletions

diff --git a/lib/libpthread/thread/thr_init.c b/lib/libpthread/thread/thr_init.c
index be3ad80..7f60994 100644
--- a/lib/libpthread/thread/thr_init.c
+++ b/lib/libpthread/thread/thr_init.c
@@ -268,9 +268,7 @@ _libpthread_init(struct pthread *curthread)
 	_kse_initial->k_kseg = _kseg_alloc(NULL);
 	if (_kse_initial->k_kseg == NULL)
 		PANIC("Can't allocate initial kseg.");
-#ifdef SYSTEM_SCOPE_ONLY
 	_kse_initial->k_kseg->kg_flags |= KGF_SINGLE_THREAD;
-#endif
 	_kse_initial->k_schedq = &_kse_initial->k_kseg->kg_schedq;
 
 	TAILQ_INSERT_TAIL(&_kse_initial->k_kseg->kg_kseq, _kse_initial, k_kgqe);
@@ -309,6 +307,9 @@ _libpthread_init(struct pthread *curthread)
 	_kcb_set(_thr_initial->kse->k_kcb);
 	_tcb_set(_thr_initial->kse->k_kcb, _thr_initial->tcb);
 	_thr_initial->kse->k_flags |= KF_INITIALIZED;
+
+	_thr_signal_init();
+	_kse_critical_leave(&_thr_initial->tcb->tcb_tmbx);
 }
 
 /*
@@ -322,9 +323,7 @@ init_main_thread(struct pthread *thread)
 
 	/* Setup the thread attributes. */
 	thread->attr = _pthread_attr_default;
-#ifdef SYSTEM_SCOPE_ONLY
 	thread->attr.flags |= PTHREAD_SCOPE_SYSTEM;
-#endif
 	/*
 	 * Set up the thread stack.
 	 *
diff --git a/lib/libpthread/thread/thr_kern.c b/lib/libpthread/thread/thr_kern.c
index ae151d7..33b4c54 100644
--- a/lib/libpthread/thread/thr_kern.c
+++ b/lib/libpthread/thread/thr_kern.c
@@ -411,6 +411,9 @@ _kse_setthreaded(int threaded)
 	sigset_t sigset;
 
 	if ((threaded != 0) && (__isthreaded == 0)) {
+		SIGFILLSET(sigset);
+		__sys_sigprocmask(SIG_SETMASK, &sigset, &_thr_initial->sigmask);
+
 		/*
 		 * Tell the kernel to create a KSE for the initial thread
 		 * and enable upcalls in it.
@@ -425,10 +428,11 @@ _kse_setthreaded(int threaded)
 		_tcb_set(_kse_initial->k_kcb, _thr_initial->tcb);
 		KSE_SET_MBOX(_kse_initial, _thr_initial);
 		_kse_initial->k_kcb->kcb_kmbx.km_flags |= KMF_BOUND;
+#else
+		_thr_initial->attr.flags &= ~PTHREAD_SCOPE_SYSTEM;
+		_kse_initial->k_kseg->kg_flags &= ~KGF_SINGLE_THREAD;
+		_kse_initial->k_kcb->kcb_kmbx.km_curthread = NULL;
 #endif
-		SIGFILLSET(sigset);
-		__sys_sigprocmask(SIG_SETMASK, &sigset, &_thr_initial->sigmask);
-		_thr_signal_init();
 
 		/*
 		 * Locking functions in libc are required when there are
@@ -963,12 +967,6 @@ kse_sched_multi(struct kse_mailbox *kmbx)
 	 */
 	_tcb_set(curkse->k_kcb, NULL);
 
-	/* This may have returned from a kse_release(). */
-	if (KSE_WAITING(curkse)) {
-		DBG_MSG("Entered upcall when KSE is waiting.");
-		KSE_CLEAR_WAIT(curkse);
-	}
-
 	/* If this is an upcall; take the scheduler lock. */
 	if (curkse->k_switch == 0)
 		KSE_SCHED_LOCK(curkse, curkse->k_kseg);
@@ -1156,16 +1154,16 @@ thr_resume_wrapper(int sig, siginfo_t *siginfo, ucontext_t *ucp)
 {
 	struct pthread *curthread = _get_curthread();
 	struct kse *curkse;
-	int ret, err_save = curthread->error;
+	int ret, err_save = errno;
 
 	DBG_MSG(">>> sig wrapper\n");
 	if (curthread->lock_switch)
 		PANIC("thr_resume_wrapper, lock_switch != 0\n");
 	thr_resume_check(curthread, ucp, NULL);
+	errno = err_save;
 	_kse_critical_enter();
 	curkse = _get_curkse();
 	curthread->tcb->tcb_tmbx.tm_context = *ucp;
-	curthread->error = err_save;
 	ret = _thread_switch(curkse->k_kcb, curthread->tcb, 1);
 	if (ret != 0)
 		PANIC("thr_resume_wrapper: thread has returned "
@@ -2290,11 +2288,7 @@ kse_reinit(struct kse *kse, int sys_scope)
 	kse->k_kseg = 0;
 	kse->k_schedq = 0;
 	kse->k_locklevel = 0;
-	SIGEMPTYSET(kse->k_sigmask);
-	bzero(&kse->k_sigq, sizeof(kse->k_sigq));
-	kse->k_check_sigq = 0;
 	kse->k_flags = 0;
-	kse->k_waiting = 0;
 	kse->k_idle = 0;
 	kse->k_error = 0;
 	kse->k_cpu = 0;
diff --git a/lib/libpthread/thread/thr_nanosleep.c b/lib/libpthread/thread/thr_nanosleep.c
index 8e709ff..57bbef9 100644
--- a/lib/libpthread/thread/thr_nanosleep.c
+++ b/lib/libpthread/thread/thr_nanosleep.c
@@ -55,8 +55,7 @@ _nanosleep(const struct timespec *time_to_sleep,
 		errno = EINVAL;
 		ret = -1;
 	} else {
-		if (!_kse_isthreaded() ||
-		    (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM))
+		if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM)
 			return (__sys_nanosleep(time_to_sleep, time_remaining));
 			
 		KSE_GET_TOD(curthread->kse, &ts);
diff --git a/lib/libpthread/thread/thr_private.h b/lib/libpthread/thread/thr_private.h
index 0664aca..c1ea592 100644
--- a/lib/libpthread/thread/thr_private.h
+++ b/lib/libpthread/thread/thr_private.h
@@ -188,14 +188,10 @@ struct kse {
 	struct lock		k_lock;
 	struct lockuser		k_lockusers[MAX_KSE_LOCKLEVEL];
 	int			k_locklevel;
-	sigset_t		k_sigmask;
-	struct sigstatus	k_sigq[_SIG_MAXSIG];
 	stack_t			k_stack;
-	int			k_check_sigq;
 	int			k_flags;
 #define	KF_STARTED			0x0001	/* kernel kse created */
 #define	KF_INITIALIZED			0x0002	/* initialized on 1st upcall */
-	int			k_waiting;
 	int			k_idle;		/* kse is idle */
 	int			k_error;	/* syscall errno in critical */
 	int			k_cpu;		/* CPU ID when bound */
@@ -294,11 +290,6 @@ do { \
 #define KSE_WAITQ_INSERT(kse, thrd)	kse_waitq_insert(thrd)
 #define	KSE_WAITQ_FIRST(kse)		TAILQ_FIRST(&(kse)->k_schedq->sq_waitq)
 
-#define	KSE_SET_WAIT(kse) 	atomic_store_rel_int(&(kse)->k_waiting, 1)
-
-#define	KSE_CLEAR_WAIT(kse) 	atomic_store_rel_int(&(kse)->k_waiting, 0)
-
-#define	KSE_WAITING(kse)	(kse)->k_waiting != 0
 #define	KSE_WAKEUP(kse)		kse_wakeup(&(kse)->k_kcb->kcb_kmbx)
 
 #define	KSE_SET_IDLE(kse)	((kse)->k_idle = 1)
@@ -733,9 +724,6 @@ struct pthread {
 	 */
 	int			interrupted;
 
-	/* Signal number when in state PS_SIGWAIT: */
-	int			signo;
-
 	/*
 	 * Set to non-zero when this thread has entered a critical
 	 * region.  We allow for recursive entries into critical regions.
diff --git a/lib/libpthread/thread/thr_sig.c b/lib/libpthread/thread/thr_sig.c
index 3c68212..01b0914 100644
--- a/lib/libpthread/thread/thr_sig.c
+++ b/lib/libpthread/thread/thr_sig.c
@@ -310,24 +310,6 @@ _thr_sig_handler(int sig, siginfo_t *info, ucontext_t *ucp)
 
 	DBG_MSG(">>> _thr_sig_handler(%d)\n", sig);
 
-	curkse = _get_curkse();
-	if ((curkse == NULL) || ((curkse->k_flags & KF_STARTED) == 0)) {
-		/* Upcalls are not yet started; just call the handler. */
-		sigfunc = _thread_sigact[sig - 1].sa_sigaction;
-		if (((__sighandler_t *)sigfunc != SIG_DFL) &&
-		    ((__sighandler_t *)sigfunc != SIG_IGN) &&
-		    (sigfunc != (__siginfohandler_t *)_thr_sig_handler)) {
-			if (((_thread_sigact[sig - 1].sa_flags & SA_SIGINFO)
-			    != 0) || (info == NULL))
-				(*(sigfunc))(sig, info, ucp);
-			else
-				(*(sigfunc))(sig,
-				    (siginfo_t*)(intptr_t)info->si_code, ucp);
-		}
-
-		return;
-	}
-
 	curthread = _get_curthread();
 	if (curthread == NULL)
 		PANIC("No current thread.\n");
@@ -359,11 +341,11 @@ _thr_sig_handler(int sig, siginfo_t *info, ucontext_t *ucp)
 	}
 
 	/* It is now safe to invoke signal handler */
-	err_save = curthread->error;
+	err_save = errno;
 	timeout_save = curthread->timeout;
 	intr_save = curthread->interrupted;
-	/* Get a fresh copy of signal mask from kernel, for thread dump only */
-	__sys_sigprocmask(SIG_SETMASK, NULL, &curthread->sigmask);
+	/* Get a fresh copy of signal mask, for thread dump only */
+	curthread->sigmask = ucp->uc_sigmask;
 	_kse_critical_enter();
 	KSE_LOCK_ACQUIRE(curkse, &_thread_signal_lock);
 	sigfunc = _thread_sigact[sig - 1].sa_sigaction;
@@ -389,15 +371,20 @@ _thr_sig_handler(int sig, siginfo_t *info, ucontext_t *ucp)
 				 ucp);
 	} else {
 		if ((__sighandler_t *)sigfunc == SIG_DFL) {
-			if (sigprop(sig) & SA_KILL)
-				kse_thr_interrupt(NULL, KSE_INTR_SIGEXIT, sig);
+			if (sigprop(sig) & SA_KILL) {
+				if (_kse_isthreaded())
+					kse_thr_interrupt(NULL,
+						 KSE_INTR_SIGEXIT, sig);
+				else
+					kill(getpid(), sig);
+			}
 #ifdef NOTYET
 			else if (sigprop(sig) & SA_STOP)
 				kse_thr_interrupt(NULL, KSE_INTR_JOBSTOP, sig);
 #endif
 		}
 	}
-	curthread->error = err_save;
+	errno = err_save;
 	curthread->timeout = timeout_save;
 	curthread->interrupted = intr_save;
 	_kse_critical_enter();
@@ -445,13 +432,13 @@ thr_sig_invoke_handler(struct pthread *curthread, int sig, siginfo_t *info,
 	}
 	KSE_LOCK_RELEASE(curkse, &_thread_signal_lock);
 	KSE_SCHED_UNLOCK(curkse, curkse->k_kseg);
-	_kse_critical_leave(&curthread->tcb->tcb_tmbx);
 	/*
 	 * We are processing buffered signals, synchronize working
 	 * signal mask into kernel.
 	 */
 	if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM)
 		__sys_sigprocmask(SIG_SETMASK, &curthread->sigmask, NULL);
+	_kse_critical_leave(&curthread->tcb->tcb_tmbx);
 	ucp->uc_sigmask = sigmask;
 	if (((__sighandler_t *)sigfunc != SIG_DFL) &&
 	    ((__sighandler_t *)sigfunc != SIG_IGN)) {
@@ -462,8 +449,13 @@ thr_sig_invoke_handler(struct pthread *curthread, int sig, siginfo_t *info,
 			    ucp);
 	} else {
 		if ((__sighandler_t *)sigfunc == SIG_DFL) {
-			if (sigprop(sig) & SA_KILL)
-				kse_thr_interrupt(NULL, KSE_INTR_SIGEXIT, sig);
+			if (sigprop(sig) & SA_KILL) {
+				if (_kse_isthreaded())
+					kse_thr_interrupt(NULL,
+						 KSE_INTR_SIGEXIT, sig);
+				else
+					kill(getpid(), sig);
+			}
 #ifdef NOTYET
 			else if (sigprop(sig) & SA_STOP)
 				kse_thr_interrupt(NULL, KSE_INTR_JOBSTOP, sig);
@@ -1049,7 +1041,6 @@ thr_sigframe_restore(struct pthread *thread, struct pthread_sigframe *psf)
 		PANIC("invalid pthread_sigframe\n");
 	thread->flags = psf->psf_flags;
 	thread->interrupted = psf->psf_interrupted;
-	thread->signo = psf->psf_signo;
 	thread->state = psf->psf_state;
 	thread->data = psf->psf_wait_data;
 	thread->wakeup_time = psf->psf_wakeup_time;
@@ -1062,7 +1053,6 @@ thr_sigframe_save(struct pthread *thread, struct pthread_sigframe *psf)
 	psf->psf_flags =
 	    thread->flags & (THR_FLAGS_PRIVATE | THR_FLAGS_IN_TDLIST);
 	psf->psf_interrupted = thread->interrupted;
-	psf->psf_signo = thread->signo;
 	psf->psf_state = thread->state;
 	psf->psf_wait_data = thread->data;
 	psf->psf_wakeup_time = thread->wakeup_time;
@@ -1074,7 +1064,10 @@ _thr_signal_init(void)
 	struct sigaction act;
 	__siginfohandler_t *sigfunc;
 	int i;
+	sigset_t sigset;
 
+	SIGFILLSET(sigset);
+	__sys_sigprocmask(SIG_SETMASK, &sigset, &_thr_initial->sigmask);
 	/* Enter a loop to get the existing signal status: */
 	for (i = 1; i <= _SIG_MAXSIG; i++) {
 		/* Check for signals which cannot be trapped: */
@@ -1111,11 +1104,13 @@ _thr_signal_init(void)
 	act.sa_flags = SA_SIGINFO | SA_RESTART;
 	act.sa_sigaction = (__siginfohandler_t *)&_thr_sig_handler;
 	if (__sys_sigaction(SIGINFO, &act, NULL) != 0) {
+		__sys_sigprocmask(SIG_SETMASK, &_thr_initial->sigmask, NULL);
 		/*
 		 * Abort this process if signal initialisation fails:
 		 */
 		PANIC("Cannot initialize signal handler");
 	}
+	__sys_sigprocmask(SIG_SETMASK, &_thr_initial->sigmask, NULL);
 }
 
 void
diff --git a/lib/libpthread/thread/thr_sigaction.c b/lib/libpthread/thread/thr_sigaction.c
index c4f5be8..ed84c20 100644
--- a/lib/libpthread/thread/thr_sigaction.c
+++ b/lib/libpthread/thread/thr_sigaction.c
@@ -52,9 +52,6 @@ _sigaction(int sig, const struct sigaction * act, struct sigaction * oact)
 		errno = EINVAL;
 		ret = -1;
 	} else {
-		if (!_kse_isthreaded())
-			return __sys_sigaction(sig, act, oact);
-
 		if (act)
 			newact = *act;
 
diff --git a/lib/libpthread/thread/thr_sigmask.c b/lib/libpthread/thread/thr_sigmask.c
index 635bac2..15740f4 100644
--- a/lib/libpthread/thread/thr_sigmask.c
+++ b/lib/libpthread/thread/thr_sigmask.c
@@ -49,9 +49,6 @@ _pthread_sigmask(int how, const sigset_t *set, sigset_t *oset)
 	sigset_t oldset, newset;
 	int ret;
 
-	if (! _kse_isthreaded())
-		_kse_setthreaded(1);
-
 	if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM) {
 		ret = __sys_sigprocmask(how, set, oset);
 		if (ret != 0)
diff --git a/lib/libpthread/thread/thr_sigpending.c b/lib/libpthread/thread/thr_sigpending.c
index ad5354b..7a0a76d 100644
--- a/lib/libpthread/thread/thr_sigpending.c
+++ b/lib/libpthread/thread/thr_sigpending.c
@@ -55,8 +55,7 @@ _sigpending(sigset_t *set)
 		ret = EINVAL;
 	}
 	else {
-		if (!_kse_isthreaded() ||
-		    (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM))
+		if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM)
 			return (__sys_sigpending(set));
 
 		crit = _kse_critical_enter();
diff --git a/lib/libpthread/thread/thr_sigprocmask.c b/lib/libpthread/thread/thr_sigprocmask.c
index 262848a..d87df58 100644
--- a/lib/libpthread/thread/thr_sigprocmask.c
+++ b/lib/libpthread/thread/thr_sigprocmask.c
@@ -46,9 +46,10 @@ _sigprocmask(int how, const sigset_t *set, sigset_t *oset)
 {
 	int ret;
 
-	if (_kse_isthreaded() == 0)
-		ret = __sys_sigprocmask(how, set, oset);
-	else
-		ret = pthread_sigmask(how, set, oset);
+	ret = pthread_sigmask(how, set, oset);
+	if (ret) {
+		errno = ret;
+		ret = -1;
+	}
 	return (ret);
 }
diff --git a/lib/libpthread/thread/thr_sigsuspend.c b/lib/libpthread/thread/thr_sigsuspend.c
index e3402a4..4d3aa4e 100644
--- a/lib/libpthread/thread/thr_sigsuspend.c
+++ b/lib/libpthread/thread/thr_sigsuspend.c
@@ -47,8 +47,7 @@ _sigsuspend(const sigset_t *set)
 	sigset_t	oldmask, newmask;
 	int             ret = -1;
 
-	if (!_kse_isthreaded() ||
-	    (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM))
+	if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM)
 		return (__sys_sigsuspend(set));
 
 	/* Check if a new signal set was provided by the caller: */
diff --git a/lib/libpthread/thread/thr_sigwait.c b/lib/libpthread/thread/thr_sigwait.c
index d3c0038..43f9321 100644
--- a/lib/libpthread/thread/thr_sigwait.c
+++ b/lib/libpthread/thread/thr_sigwait.c
@@ -55,8 +55,7 @@ lib_sigtimedwait(const sigset_t *set, siginfo_t *info,
 	kse_critical_t  crit;
 	siginfo_t	siginfo;
 
-	if (!_kse_isthreaded() ||
-	    (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM)) {
+	if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM) {
 		if (info == NULL)
 			info = &siginfo;
 		return (__sys_sigtimedwait((sigset_t *)set, info,
diff --git a/lib/libpthread/thread/thr_yield.c b/lib/libpthread/thread/thr_yield.c
index 1133784..5c24113 100644
--- a/lib/libpthread/thread/thr_yield.c
+++ b/lib/libpthread/thread/thr_yield.c
@@ -42,7 +42,7 @@ _sched_yield(void)
 {
 	struct pthread	*curthread = _get_curthread();
 
-	if (!_kse_isthreaded() || curthread->attr.flags & PTHREAD_SCOPE_SYSTEM)
+	if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM)
 		return (__sys_sched_yield());
 
 	/* Reset the accumulated time slice value for the current thread: */
@@ -60,8 +60,7 @@ _pthread_yield(void)
 {
 	struct pthread	*curthread = _get_curthread();
 
-	if (!_kse_isthreaded() ||
-	    curthread->attr.flags & PTHREAD_SCOPE_SYSTEM) {
+	if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM) {
 		__sys_sched_yield();
 		return;
 	}