pthread_mutex_lock(), pthread_cond_trywait(), and pthread_cond_wait() are

not allowed to return EINTR, but use of pthread_suspend_np() could cause
EINTR to be returned.  To fix this, restructure pthread_suspend_np() so that
it does not interrupt a thread that is waiting on a mutex or condition, and
keep enough state around that pthread_resume_np() can fix things up
afterwards.

Reviewed by:	deischen

6 files changed, 125 insertions, 36 deletions

diff --git a/lib/libkse/thread/thr_cancel.c b/lib/libkse/thread/thr_cancel.c
index 82ddbb8..c6df52e 100644
--- a/lib/libkse/thread/thr_cancel.c
+++ b/lib/libkse/thread/thr_cancel.c
@@ -37,15 +37,6 @@ pthread_cancel(pthread_t pthread)
 				pthread->cancelflags |= PTHREAD_CANCELLING;
 				break;
 
-			case PS_SUSPENDED:
-				/*
-				 * This thread isn't in any scheduling
-				 * queues; just change it's state:
-				 */
-				pthread->cancelflags |= PTHREAD_CANCELLING;
-				PTHREAD_SET_STATE(pthread, PS_RUNNING);
-				break;
-
 			case PS_SPINBLOCK:
 			case PS_FDR_WAIT:
 			case PS_FDW_WAIT:
@@ -67,6 +58,20 @@ pthread_cancel(pthread_t pthread)
 				PTHREAD_NEW_STATE(pthread,PS_RUNNING);
 				break;
 
+			case PS_SUSPENDED:
+				if (pthread->suspended == SUSP_NO ||
+				    pthread->suspended == SUSP_YES ||
+				    pthread->suspended == SUSP_NOWAIT) {
+					/*
+					 * This thread isn't in any scheduling
+					 * queues; just change it's state:
+					 */
+					pthread->cancelflags |=
+					    PTHREAD_CANCELLING;
+					PTHREAD_SET_STATE(pthread, PS_RUNNING);
+					break;
+				}
+				/* FALLTHROUGH */
 			case PS_MUTEX_WAIT:
 			case PS_COND_WAIT:
 			case PS_FDLR_WAIT:
diff --git a/lib/libkse/thread/thr_cond.c b/lib/libkse/thread/thr_cond.c
index 78ee042..49062be 100644
--- a/lib/libkse/thread/thr_cond.c
+++ b/lib/libkse/thread/thr_cond.c
@@ -285,7 +285,6 @@ pthread_cond_wait(pthread_cond_t * cond, pthread_mutex_t * mutex)
 		if (interrupted != 0) {
 			if (_thread_run->continuation != NULL)
 				_thread_run->continuation((void *) _thread_run);
-			rval = EINTR;
 		}
 
 		_thread_leave_cancellation_point();
@@ -455,7 +454,6 @@ pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
 		if (interrupted != 0) {
 			if (_thread_run->continuation != NULL)
 				_thread_run->continuation((void *) _thread_run);
-			rval = EINTR;
 		}
 
 		_thread_leave_cancellation_point();
@@ -489,9 +487,18 @@ pthread_cond_signal(pthread_cond_t * cond)
 		switch ((*cond)->c_type) {
 		/* Fast condition variable: */
 		case COND_TYPE_FAST:
-			if ((pthread = cond_queue_deq(*cond)) != NULL)
-				/* Allow the thread to run: */
-				PTHREAD_NEW_STATE(pthread,PS_RUNNING);
+			if ((pthread = cond_queue_deq(*cond)) != NULL) {
+				/*
+				 * Unless the thread is currently suspended,
+				 * allow it to run.  If the thread is suspended,
+				 * make a note that the thread isn't in a wait
+				 * queue any more.
+				 */
+				if (pthread->state != PS_SUSPENDED)
+					PTHREAD_NEW_STATE(pthread,PS_RUNNING);
+				else
+					pthread->suspended = SUSP_NOWAIT;
+			}
 
 			/* Check for no more waiters: */
 			if (TAILQ_FIRST(&(*cond)->c_queue) == NULL)
@@ -546,7 +553,16 @@ pthread_cond_broadcast(pthread_cond_t * cond)
 			 * condition queue:
 			 */
 			while ((pthread = cond_queue_deq(*cond)) != NULL) {
-				PTHREAD_NEW_STATE(pthread,PS_RUNNING);
+				/*
+				 * Unless the thread is currently suspended,
+				 * allow it to run.  If the thread is suspended,
+				 * make a note that the thread isn't in a wait
+				 * queue any more.
+				 */
+				if (pthread->state != PS_SUSPENDED)
+					PTHREAD_NEW_STATE(pthread,PS_RUNNING);
+				else
+					pthread->suspended = SUSP_NOWAIT;
 			}
 
 			/* There are no more waiting threads: */
diff --git a/lib/libkse/thread/thr_mutex.c b/lib/libkse/thread/thr_mutex.c
index c97c86b..12fdc8e 100644
--- a/lib/libkse/thread/thr_mutex.c
+++ b/lib/libkse/thread/thr_mutex.c
@@ -612,7 +612,6 @@ pthread_mutex_lock(pthread_mutex_t * mutex)
 		 */
 		if (_thread_run->interrupted != 0) {
 			mutex_queue_remove(*mutex, _thread_run);
-			ret = EINTR;
 		}
 
 		/* Unlock the mutex structure: */
@@ -777,11 +776,20 @@ mutex_unlock_common(pthread_mutex_t * mutex, int add_reference)
 				if (((*mutex)->m_owner =
 			  	    mutex_queue_deq(*mutex)) != NULL) {
 					/*
-					 * Allow the new owner of the mutex to
-					 * run:
+					 * Unless the new owner of the mutex is
+					 * currently suspended, allow the owner
+					 * to run.  If the thread is suspended,
+					 * make a note that the thread isn't in
+					 * a wait queue any more.
 					 */
-					PTHREAD_NEW_STATE((*mutex)->m_owner,
-					    PS_RUNNING);
+					if (((*mutex)->m_owner->state !=
+					    PS_SUSPENDED)) {
+						PTHREAD_NEW_STATE((*mutex)->m_owner,
+						    PS_RUNNING);
+					} else {
+						(*mutex)->m_owner->suspended =
+						    SUSP_NOWAIT;
+					}
 
 					/*
 					 * Add the mutex to the threads list of
@@ -899,11 +907,20 @@ mutex_unlock_common(pthread_mutex_t * mutex, int add_reference)
 						(*mutex)->m_prio;
 
 					/*
-					 * Allow the new owner of the mutex to
-					 * run:
+					 * Unless the new owner of the mutex is
+					 * currently suspended, allow the owner
+					 * to run.  If the thread is suspended,
+					 * make a note that the thread isn't in
+					 * a wait queue any more.
 					 */
-					PTHREAD_NEW_STATE((*mutex)->m_owner,
-					    PS_RUNNING);
+					if (((*mutex)->m_owner->state !=
+					    PS_SUSPENDED)) {
+						PTHREAD_NEW_STATE((*mutex)->m_owner,
+						    PS_RUNNING);
+					} else {
+						(*mutex)->m_owner->suspended =
+						    SUSP_NOWAIT;
+					}
 				}
 			}
 			break;
@@ -1019,11 +1036,20 @@ mutex_unlock_common(pthread_mutex_t * mutex, int add_reference)
 					    (*mutex)->m_prio;
 
 					/*
-					 * Allow the new owner of the mutex to
-					 * run:
+					 * Unless the new owner of the mutex is
+					 * currently suspended, allow the owner
+					 * to run.  If the thread is suspended,
+					 * make a note that the thread isn't in
+					 * a wait queue any more.
 					 */
-					PTHREAD_NEW_STATE((*mutex)->m_owner,
-					    PS_RUNNING);
+					if (((*mutex)->m_owner->state !=
+					    PS_SUSPENDED)) {
+						PTHREAD_NEW_STATE((*mutex)->m_owner,
+						    PS_RUNNING);
+					} else {
+						(*mutex)->m_owner->suspended =
+						    SUSP_NOWAIT;
+					}
 				}
 			}
 			break;
diff --git a/lib/libkse/thread/thr_private.h b/lib/libkse/thread/thr_private.h
index a86cfc6..6b48f23 100644
--- a/lib/libkse/thread/thr_private.h
+++ b/lib/libkse/thread/thr_private.h
@@ -350,6 +350,17 @@ struct pthread_attr {
 #define PTHREAD_CREATE_SUSPENDED		1
 
 /*
+ * Additional state for a thread suspended with pthread_suspend_np().
+ */
+enum pthread_susp {
+	SUSP_NO,	/* Not suspended. */
+	SUSP_YES,	/* Suspended. */
+	SUSP_NOWAIT,	/* Suspended, was in a mutex or condition queue. */
+	SUSP_MUTEX_WAIT,/* Suspended, still in a mutex queue. */
+	SUSP_COND_WAIT	/* Suspended, still in a condition queue. */
+};
+
+/*
  * Miscellaneous definitions.
  */
 #define PTHREAD_STACK_DEFAULT			65536
@@ -577,7 +588,7 @@ struct pthread {
 #define PTHREAD_CANCEL_NEEDED		0x0010
 	int	cancelflags;
 
-	int	suspended;
+	enum	pthread_susp suspended;
 
 	thread_continuation_t	continuation;
 
diff --git a/lib/libkse/thread/thr_resume_np.c b/lib/libkse/thread/thr_resume_np.c
index fae355a..32b3d05 100644
--- a/lib/libkse/thread/thr_resume_np.c
+++ b/lib/libkse/thread/thr_resume_np.c
@@ -40,12 +40,14 @@
 int
 pthread_resume_np(pthread_t thread)
 {
-	int ret;
+	int	ret;
+	enum	pthread_susp old_suspended;
 
 	/* Find the thread in the list of active threads: */
 	if ((ret = _find_thread(thread)) == 0) {
 		/* Cancel any pending suspensions: */
-		thread->suspended = 0;
+		old_suspended = thread->suspended;
+		thread->suspended = SUSP_NO;
 
 		/* Is it currently suspended? */
 		if (thread->state == PS_SUSPENDED) {
@@ -55,9 +57,28 @@ pthread_resume_np(pthread_t thread)
 			 */
 			_thread_kern_sig_defer();
 
-			/* Allow the thread to run. */
-			PTHREAD_SET_STATE(thread,PS_RUNNING);
-			PTHREAD_PRIOQ_INSERT_TAIL(thread);
+			switch (old_suspended) {
+			case SUSP_MUTEX_WAIT:
+				/* Set the thread's state back. */
+				PTHREAD_SET_STATE(thread,PS_MUTEX_WAIT);
+				break;
+			case SUSP_COND_WAIT:
+				/* Set the thread's state back. */
+				PTHREAD_SET_STATE(thread,PS_COND_WAIT);
+				break;
+			case SUSP_NOWAIT:
+				/* Allow the thread to run. */
+				PTHREAD_SET_STATE(thread,PS_RUNNING);
+				PTHREAD_WAITQ_REMOVE(thread);
+				PTHREAD_PRIOQ_INSERT_TAIL(thread);
+				break;
+			case SUSP_NO:
+			case SUSP_YES:
+				/* Allow the thread to run. */
+				PTHREAD_SET_STATE(thread,PS_RUNNING);
+				PTHREAD_PRIOQ_INSERT_TAIL(thread);
+				break;
+			}
 
 			/*
 			 * Undefer and handle pending signals, yielding if
diff --git a/lib/libkse/thread/thr_suspend_np.c b/lib/libkse/thread/thr_suspend_np.c
index 9b08115..083f552 100644
--- a/lib/libkse/thread/thr_suspend_np.c
+++ b/lib/libkse/thread/thr_suspend_np.c
@@ -91,13 +91,23 @@ pthread_suspend_np(pthread_t thread)
 			break;
 
 		case PS_MUTEX_WAIT:
+			/* Mark the thread as suspended and still in a queue. */
+			thread->suspended = SUSP_MUTEX_WAIT;
+
+			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+			break;
 		case PS_COND_WAIT:
+			/* Mark the thread as suspended and still in a queue. */
+			thread->suspended = SUSP_COND_WAIT;
+
+			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+			break;
 		case PS_FDLR_WAIT:
 		case PS_FDLW_WAIT:
 		case PS_FILE_WAIT:
 		case PS_JOIN:
 			/* Mark the thread as suspended: */
-			thread->suspended = 1;
+			thread->suspended = SUSP_YES;
 
 			/*
 			 * Threads in these states may be in queues.
@@ -134,7 +144,7 @@ pthread_suspend_np(pthread_t thread)
 static void
 finish_suspension(void *arg)
 {
-	if (_thread_run->suspended != 0)
+	if (_thread_run->suspended != SUSP_NO)
 		_thread_kern_sched_state(PS_SUSPENDED, __FILE__, __LINE__);
 }