Add an i386-specifc hack to always set %gs. There still seems

to be instances where the kernel doesn't  properly save and/or
restore it.

Use noupcall and nocompleted flags in the KSE mailbox.  These
require kernel changes to work which will be committed sometime
later.  Things still work without the changes.

Remove the general kse entry function and use two different
functions -- one for scope system threads and one for scope
process threads.  The scope system function is not yet enabled
and we use the same function for all threads at the moment.

Keep a copy of the KSE stack for the case that a KSE runs
a scope system thread and uses the same stack as the thread
(no upcalls are generated, so a separate stack isn't needed).
This isn't enabled yet.

Use a separate field for the KSE waiting flag.  It isn't
correct to use the mailbox flags field.

The following fixes were provided by David Xu:

  o Initialize condition variable locks with thread versions
    of the low-level locking functions instead of the kse versions.

  o Enable threading before creating the first thread instead
    of after.

  o Don't enter critical regions when trying to malloc/free
    or call functions that malloc/free.

  o Take the scheduling lock when inheriting thread attributes.

  o Check the attribute's stack pointer instead of the
    attributes stack size for null when allocating a
    thread's stack.

  o Add a kseg reinit function so we don't have to destroy and
    then recreate the same lock.

  o Check the return value of kse_create() and return an
    appropriate error if it fails.

  o Don't forget to destroy a thread's locks when freeing it.

  o Examine the correct flags word for checking to see if
    a thread is in a synchronization queue.

Things should now work on an SMP kernel.

1 files changed, 39 insertions, 15 deletions

diff --git a/lib/libpthread/thread/thr_create.c b/lib/libpthread/thread/thr_create.c
index 74cb134..c38b267 100644
--- a/lib/libpthread/thread/thr_create.c
+++ b/lib/libpthread/thread/thr_create.c
@@ -103,11 +103,20 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	if (_thr_initial == NULL)
 		_libpthread_init(NULL);
 
-	crit = _kse_critical_enter();
+	/*
+	 * Turn on threaded mode, if failed, it is unnecessary to
+	 * do further work.
+	 */
+	if (_kse_isthreaded() == 0 && _kse_setthreaded(1)) {
+		return (EAGAIN);
+	}
 	curthread = _get_curthread();
-	curkse = curthread->kse;
 
-	/* Allocate memory for the thread structure: */
+	/*
+	 * Allocate memory for the thread structure.
+	 * Some functions use malloc, so don't put it
+	 * in a critical region.
+	 */
 	if ((new_thread = _thr_alloc(curthread)) == NULL) {
 		/* Insufficient memory to create a thread: */
 		ret = EAGAIN;
@@ -135,9 +144,13 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			if (kse != NULL)
 				_kse_free(curthread, kse);
 			if ((new_thread->attr.flags & THR_STACK_USER) == 0) {
+				crit = _kse_critical_enter();
+				curkse = _get_curkse();
 				KSE_LOCK_ACQUIRE(curkse, &_thread_list_lock);
+				/* Stack routines don't use malloc/free. */
 				_thr_stack_free(&new_thread->attr);
 				KSE_LOCK_RELEASE(curkse, &_thread_list_lock);
+				_kse_critical_leave(crit);
 			}
 			_thr_free(curthread, new_thread);
 		}
@@ -169,8 +182,13 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			/* Initialize the signal frame: */
 			new_thread->curframe = NULL;
 
-			/* Initialize the machine context: */
+			/*
+			 * Initialize the machine context.
+			 * Enter a critical region to get consistent context.
+			 */
+			crit = _kse_critical_enter();
 			THR_GETCONTEXT(&new_thread->tmbx.tm_context);
+			_kse_critical_leave(crit);
 			new_thread->tmbx.tm_udata = new_thread;
 			new_thread->tmbx.tm_context.uc_sigmask =
 			    new_thread->sigmask;
@@ -178,17 +196,20 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			    new_thread->attr.stacksize_attr;
 			new_thread->tmbx.tm_context.uc_stack.ss_sp =
 			    new_thread->attr.stackaddr_attr;
-
 			makecontext(&new_thread->tmbx.tm_context,
 			    (void (*)(void))thread_start, 4, new_thread,
 			    start_routine, arg);
-
 			/*
 			 * Check if this thread is to inherit the scheduling
 			 * attributes from its parent:
 			 */
 			if ((new_thread->attr.flags & PTHREAD_INHERIT_SCHED) != 0) {
-				/* Copy the scheduling attributes: */
+				/*
+				 * Copy the scheduling attributes.
+				 * Lock the scheduling lock to get consistent
+				 * scheduling parameters.
+				 */
+				THR_SCHED_LOCK(curthread, curthread);
 				new_thread->base_priority =
 				    curthread->base_priority &
 				    ~THR_SIGNAL_PRIORITY;
@@ -197,6 +218,7 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 				    ~THR_SIGNAL_PRIORITY;
 				new_thread->attr.sched_policy =
 				    curthread->attr.sched_policy;
+				THR_SCHED_UNLOCK(curthread, curthread);
 			} else {
 				/*
 				 * Use just the thread priority, leaving the
@@ -212,7 +234,11 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			/* Initialize the mutex queue: */
 			TAILQ_INIT(&new_thread->mutexq);
 
-			/* Initialize thread locking. */
+			/*
+			 * Initialize thread locking.
+			 * Lock initializing needs malloc, so don't
+			 * enter critical region before doing this!
+			 */
 			if (_lock_init(&new_thread->lock, LCK_ADAPTIVE,
 			    _thr_lock_wait, _thr_lock_wakeup) != 0)
 				PANIC("Cannot initialize thread lock");
@@ -245,23 +271,24 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			}
 			else {
 				kse->k_curthread = NULL;
+#ifdef NOT_YET
 				kse->k_kseg->kg_flags |= KGF_SINGLE_THREAD;
+#endif
 				new_thread->kse = kse;
 				new_thread->kseg = kse->k_kseg;
 				kse->k_mbx.km_udata = kse;
 				kse->k_mbx.km_curthread = NULL;
 			}
-			KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
 
+			crit = _kse_critical_enter();
+			KSE_LOCK_ACQUIRE(curthread->kse, &_thread_list_lock);
 			/*
 			 * Initialise the unique id which GDB uses to
 			 * track threads.
 			 */
 			new_thread->uniqueid = next_uniqueid++;
-
 			/* Add the thread to the linked list of all threads: */
 			THR_LIST_ADD(new_thread);
-
 			KSE_LOCK_RELEASE(curthread->kse, &_thread_list_lock);
 
 			/*
@@ -269,15 +296,12 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			 * pair if necessary.
 			 */
 			_thr_schedule_add(curthread, new_thread);
+			_kse_critical_leave(crit);
 
 			/* Return a pointer to the thread structure: */
 			(*thread) = new_thread;
 		}
 	}
-	_kse_critical_leave(crit);
-
-	if ((ret == 0) && (_kse_isthreaded() == 0))
-		_kse_setthreaded(1);
 
 	/* Return the status: */
 	return (ret);