Make libthr async-signal-safe without costly signal masking. The guidlines I

followed are: Only 3 functions (pthread_cancel, pthread_setcancelstate,
pthread_setcanceltype) are required to be async-signal-safe by POSIX. None of
the rest of the pthread api is required to be async-signal-safe. This means
that only the three mentioned functions are safe to use from inside
signal handlers.
However, there are certain system/libc calls that are
cancellation points that a caller may call from within a signal handler,
and since they are cancellation points calls have to be made into libthr
to test for cancellation and exit the thread if necessary. So, the
cancellation test and thread exit code paths must be async-signal-safe
as well. A summary of the changes follows:

o Almost all of the code paths that masked signals, as well as locking the
  pthread structure now lock only the pthread structure.
o Signals are masked (and left that way) as soon as a thread enters
  pthread_exit().
o The active and dead threads locks now explicitly require that signals
  are masked.
o Access to the isdead field of the pthread structure is protected by both
  the active and dead list locks for writing. Either one is sufficient for
  reading.
o The thread state and type fields have been combined into one three-state
  switch to make it easier to read without requiring a lock. It doesn't need
  a lock for writing (and therefore for reading either) because only the
  current thread can write to it and it is an integer value.
o The thread state field of the pthread structure has been eliminated. It
  was an unnecessary field that mostly duplicated the flags field, but
  required additional locking that would make a lot more code paths require
  signal masking. Any truly unique values (such as PS_DEAD) have been
  reborn as separate members of the pthread structure.
o Since the mutex and condvar pthread functions are not async-signal-safe
  there is no need to muck about with the wait queues when handling
  a signal ...
o ... which also removes the need for wrapping signal handlers and sigaction(2).
o The condvar and mutex async-cancellation code had to be revised as a result
  of some of these changes, which resulted in semi-unrelated changes which
  would have been difficult to work on as a separate commit, so they are
  included as well.

The only part of the changes I am worried about is related to locking for
the pthread joining fields. But, I will take a closer look at them once this
mega-patch is committed.

1 files changed, 44 insertions, 30 deletions

diff --git a/lib/libthr/thread/thr_join.c b/lib/libthr/thread/thr_join.c
index fd7cff5..8ff4c1a 100644
--- a/lib/libthr/thread/thr_join.c
+++ b/lib/libthr/thread/thr_join.c
@@ -44,21 +44,15 @@ _pthread_join(pthread_t pthread, void **thread_return)
 	int ret = 0;
 	pthread_t thread;
  
-	_thread_enter_cancellation_point();
-
 	/* Check if the caller has specified an invalid thread: */
-	if (pthread->magic != PTHREAD_MAGIC) {
+	if (pthread->magic != PTHREAD_MAGIC)
 		/* Invalid thread: */
-		_thread_leave_cancellation_point();
 		return(EINVAL);
-	}
 
 	/* Check if the caller has specified itself: */
-	if (pthread == curthread) {
+	if (pthread == curthread)
 		/* Avoid a deadlock condition: */
-		_thread_leave_cancellation_point();
 		return(EDEADLK);
-	}
 
 	/*
 	 * Search for the specified thread in the list of active threads.  This
@@ -66,35 +60,38 @@ _pthread_join(pthread_t pthread, void **thread_return)
 	 * the searches in _thread_list and _dead_list (as well as setting up
 	 * join/detach state) have to be done atomically.
 	 */
+	_thread_sigblock();
 	DEAD_LIST_LOCK;
 	THREAD_LIST_LOCK;
-	TAILQ_FOREACH(thread, &_thread_list, tle)
-		if (thread == pthread) {
-			UMTX_LOCK(&pthread->lock);
-			break;
+	if (!pthread->isdead) {
+		TAILQ_FOREACH(thread, &_thread_list, tle) {
+			if (thread == pthread) {
+				PTHREAD_LOCK(pthread);
+				break;
+			}
 		}
-
-	if (thread == NULL)
-		/*
-		 * Search for the specified thread in the list of dead threads:
-		 */
-		TAILQ_FOREACH(thread, &_dead_list, dle)
+	} else {
+		TAILQ_FOREACH(thread, &_dead_list, dle) {
 			if (thread == pthread) {
-				UMTX_LOCK(&pthread->lock);
+				PTHREAD_LOCK(pthread);
 				break;
 			}
+		}
+	}
 
 	/* Check if the thread was not found or has been detached: */
 	if (thread == NULL) {
 		THREAD_LIST_UNLOCK;
 		DEAD_LIST_UNLOCK;
+		_thread_sigunblock();
 		ret = ESRCH;
 		goto out;
 	}
 	if ((pthread->attr.flags & PTHREAD_DETACHED) != 0) {
-		UMTX_UNLOCK(&pthread->lock);
+		PTHREAD_UNLOCK(pthread);
 		THREAD_LIST_UNLOCK;
 		DEAD_LIST_UNLOCK;
+		_thread_sigunblock();
 		ret = EINVAL;
 		goto out;
 	}
@@ -102,44 +99,60 @@ _pthread_join(pthread_t pthread, void **thread_return)
 	if (pthread->joiner != NULL) {
 		/* Multiple joiners are not supported. */
 		/* XXXTHR - support multiple joiners. */
-		UMTX_UNLOCK(&pthread->lock);
+		PTHREAD_UNLOCK(pthread);
 		THREAD_LIST_UNLOCK;
 		DEAD_LIST_UNLOCK;
+		_thread_sigunblock();
 		ret = ENOTSUP;
 		goto out;
 
 	}
 
 	/* Check if the thread is not dead: */
-	if (pthread->state != PS_DEAD) {
+	if (!pthread->isdead) {
 		/* Set the running thread to be the joiner: */
 		pthread->joiner = curthread;
-		UMTX_UNLOCK(&pthread->lock);
-		_thread_critical_enter(curthread);
+		PTHREAD_UNLOCK(pthread);
 
 		/* Keep track of which thread we're joining to: */
 		curthread->join_status.thread = pthread;
 
 		while (curthread->join_status.thread == pthread) {
-			PTHREAD_SET_STATE(curthread, PS_JOIN);
 			/* Wait for our signal to wake up. */
-			_thread_critical_exit(curthread);
 			THREAD_LIST_UNLOCK;
 			DEAD_LIST_UNLOCK;
+			_thread_sigunblock();
+			if (curthread->cancellation != CS_NULL)
+				pthread->joiner = NULL;
+			_thread_enter_cancellation_point();
+
+			/*
+			 * XXX - Workaround to make a join a cancellation
+			 *	 point. Must find a better solution.
+			 */
+			PTHREAD_LOCK(curthread);
+			curthread->flags |= PTHREAD_FLAGS_SUSPENDED;
+			PTHREAD_UNLOCK(curthread);
 			ret = _thread_suspend(curthread, NULL);
 			if (ret != 0 && ret != EAGAIN && ret != EINTR)
 				PANIC("Unable to suspend in join.");
+			PTHREAD_LOCK(curthread);
+			curthread->flags &= ~PTHREAD_FLAGS_SUSPENDED;
+			PTHREAD_UNLOCK(curthread);
+			if (curthread->cancellation != CS_NULL)
+				pthread->joiner = NULL;
+			_thread_leave_cancellation_point();
 
 			/*
 			 * XXX - For correctness reasons.
 			 * We must aquire these in the same order and also
-			 * importantly, release in the same order, order because
+			 * importantly, release in the same order because
 			 * otherwise we might deadlock with the joined thread
 			 * when we attempt to release one of these locks.
 			 */
+			_thread_sigblock();
 			DEAD_LIST_LOCK;
 			THREAD_LIST_LOCK;
-			_thread_critical_enter(curthread);
 		}
 
 		/*
@@ -149,9 +162,9 @@ _pthread_join(pthread_t pthread, void **thread_return)
 		ret = curthread->join_status.error;
 		if ((ret == 0) && (thread_return != NULL))
 			*thread_return = curthread->join_status.ret;
-		_thread_critical_exit(curthread);
 		THREAD_LIST_UNLOCK;
 		DEAD_LIST_UNLOCK;
+		_thread_sigunblock();
 	} else {
 		/*
 		 * The thread exited (is dead) without being detached, and no
@@ -166,11 +179,12 @@ _pthread_join(pthread_t pthread, void **thread_return)
 
 		/* Free all remaining memory allocated to the thread. */
 		pthread->attr.flags |= PTHREAD_DETACHED;
-		UMTX_UNLOCK(&pthread->lock);
+		PTHREAD_UNLOCK(pthread);
 		TAILQ_REMOVE(&_dead_list, pthread, dle);
 		deadlist_free_onethread(pthread);
 		THREAD_LIST_UNLOCK;
 		DEAD_LIST_UNLOCK;
+		_thread_sigunblock();
 	}
 
 out: