Fix pthread_suspend_np/pthread_resume_np. For the record, suspending a

thread waiting on an event (I/O, condvar, etc) will, when resumed using
pthread_resume_np, return with EINTR.  For example, suspending and resuming
a thread blocked on read() will not requeue the thread for the read, but
will return -1 with errno = EINTR.  If the suspended thread is in a critical
region, the thread is suspended as soon as it leaves the critical region.

Fix a bogon in pthread_kill() where a signal was being delivered twice
to threads waiting in sigwait().

Reported by (suspend/resume bug):	jdp
Reviewed by:	jasone

27 files changed, 375 insertions, 93 deletions

diff --git a/lib/libc_r/uthread/pthread_private.h b/lib/libc_r/uthread/pthread_private.h
index 962eb86..07c8bec 100644
--- a/lib/libc_r/uthread/pthread_private.h
+++ b/lib/libc_r/uthread/pthread_private.h
@@ -105,7 +105,7 @@
 		else							\
 			TAILQ_INSERT_BEFORE(tid,thrd,pqe);		\
 	}								\
-	(thrd)->flags | PTHREAD_FLAGS_IN_WAITQ;				\
+	(thrd)->flags |= PTHREAD_FLAGS_IN_WAITQ;			\
 } while (0)
 #define PTHREAD_WAITQ_CLEARACTIVE()
 #define PTHREAD_WAITQ_SETACTIVE()
@@ -576,6 +576,8 @@ struct pthread {
 #define PTHREAD_CANCEL_NEEDED		0x0010
 	int	cancelflags;
 
+	int	suspended;
+
 	thread_continuation_t	continuation;
 
 	/*
diff --git a/lib/libc_r/uthread/uthread_cancel.c b/lib/libc_r/uthread/uthread_cancel.c
index f22bfb5..82ddbb8 100644
--- a/lib/libc_r/uthread/uthread_cancel.c
+++ b/lib/libc_r/uthread/uthread_cancel.c
@@ -37,6 +37,15 @@ pthread_cancel(pthread_t pthread)
 				pthread->cancelflags |= PTHREAD_CANCELLING;
 				break;
 
+			case PS_SUSPENDED:
+				/*
+				 * This thread isn't in any scheduling
+				 * queues; just change it's state:
+				 */
+				pthread->cancelflags |= PTHREAD_CANCELLING;
+				PTHREAD_SET_STATE(pthread, PS_RUNNING);
+				break;
+
 			case PS_SPINBLOCK:
 			case PS_FDR_WAIT:
 			case PS_FDW_WAIT:
@@ -52,7 +61,6 @@ pthread_cancel(pthread_t pthread)
 			case PS_WAIT_WAIT:
 			case PS_SIGSUSPEND:
 			case PS_SIGWAIT:
-			case PS_SUSPENDED:
 				/* Interrupt and resume: */
 				pthread->interrupted = 1;
 				pthread->cancelflags |= PTHREAD_CANCELLING;
diff --git a/lib/libc_r/uthread/uthread_cond.c b/lib/libc_r/uthread/uthread_cond.c
index d236607..78ee042 100644
--- a/lib/libc_r/uthread/uthread_cond.c
+++ b/lib/libc_r/uthread/uthread_cond.c
@@ -282,8 +282,11 @@ pthread_cond_wait(pthread_cond_t * cond, pthread_mutex_t * mutex)
 			break;
 		}
 
-		if (interrupted != 0 && _thread_run->continuation != NULL)
-			_thread_run->continuation((void *) _thread_run);
+		if (interrupted != 0) {
+			if (_thread_run->continuation != NULL)
+				_thread_run->continuation((void *) _thread_run);
+			rval = EINTR;
+		}
 
 		_thread_leave_cancellation_point();
 	}
@@ -449,8 +452,11 @@ pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
 			break;
 		}
 
-		if (interrupted != 0 && _thread_run->continuation != NULL)
-			_thread_run->continuation((void *) _thread_run);
+		if (interrupted != 0) {
+			if (_thread_run->continuation != NULL)
+				_thread_run->continuation((void *) _thread_run);
+			rval = EINTR;
+		}
 
 		_thread_leave_cancellation_point();
 	}
diff --git a/lib/libc_r/uthread/uthread_create.c b/lib/libc_r/uthread/uthread_create.c
index b8a1c46..3a80611 100644
--- a/lib/libc_r/uthread/uthread_create.c
+++ b/lib/libc_r/uthread/uthread_create.c
@@ -299,10 +299,9 @@ pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			/* Add the thread to the linked list of all threads: */
 			TAILQ_INSERT_HEAD(&_thread_list, new_thread, tle);
 
-			if (pattr->suspend == PTHREAD_CREATE_SUSPENDED) {
+			if (pattr->suspend == PTHREAD_CREATE_SUSPENDED)
 				new_thread->state = PS_SUSPENDED;
-				PTHREAD_WAITQ_INSERT(new_thread);
-			} else {
+			else {
 				new_thread->state = PS_RUNNING;
 				PTHREAD_PRIOQ_INSERT_TAIL(new_thread);
 			}
diff --git a/lib/libc_r/uthread/uthread_kern.c b/lib/libc_r/uthread/uthread_kern.c
index 3e0ff17..8b333e4 100644
--- a/lib/libc_r/uthread/uthread_kern.c
+++ b/lib/libc_r/uthread/uthread_kern.c
@@ -184,8 +184,10 @@ __asm__("fnsave %0": :"m"(*fdata));
 			switch (_thread_run->state) {
 			case PS_DEAD:
 			case PS_STATE_MAX: /* to silence -Wall */
+			case PS_SUSPENDED:
 				/*
-				 * Dead threads are not placed in any queue:
+				 * Dead and suspended threads are not placed
+				 * in any queue:
 				 */
 				break;
 
@@ -227,7 +229,6 @@ __asm__("fnsave %0": :"m"(*fdata));
 			case PS_SIGSUSPEND:
 			case PS_SIGTHREAD:
 			case PS_SIGWAIT:
-			case PS_SUSPENDED:
 			case PS_WAIT_WAIT:
 				/* No timeouts for these states: */
 				_thread_run->wakeup_time.tv_sec = -1;
diff --git a/lib/libc_r/uthread/uthread_mutex.c b/lib/libc_r/uthread/uthread_mutex.c
index 6d75ea5..c97c86b 100644
--- a/lib/libc_r/uthread/uthread_mutex.c
+++ b/lib/libc_r/uthread/uthread_mutex.c
@@ -610,8 +610,10 @@ pthread_mutex_lock(pthread_mutex_t * mutex)
 		 * Check to see if this thread was interrupted and
 		 * is still in the mutex queue of waiting threads:
 		 */
-		if (_thread_run->interrupted != 0)
+		if (_thread_run->interrupted != 0) {
 			mutex_queue_remove(*mutex, _thread_run);
+			ret = EINTR;
+		}
 
 		/* Unlock the mutex structure: */
 		_SPINUNLOCK(&(*mutex)->lock);
diff --git a/lib/libc_r/uthread/uthread_resume_np.c b/lib/libc_r/uthread/uthread_resume_np.c
index 98ec718..fae355a 100644
--- a/lib/libc_r/uthread/uthread_resume_np.c
+++ b/lib/libc_r/uthread/uthread_resume_np.c
@@ -44,8 +44,11 @@ pthread_resume_np(pthread_t thread)
 
 	/* Find the thread in the list of active threads: */
 	if ((ret = _find_thread(thread)) == 0) {
-		/* The thread exists. Is it suspended? */
-		if (thread->state != PS_SUSPENDED) {
+		/* Cancel any pending suspensions: */
+		thread->suspended = 0;
+
+		/* Is it currently suspended? */
+		if (thread->state == PS_SUSPENDED) {
 			/*
 			 * Defer signals to protect the scheduling queues
 			 * from access by the signal handler:
@@ -53,7 +56,8 @@ pthread_resume_np(pthread_t thread)
 			_thread_kern_sig_defer();
 
 			/* Allow the thread to run. */
-			PTHREAD_NEW_STATE(thread,PS_RUNNING);
+			PTHREAD_SET_STATE(thread,PS_RUNNING);
+			PTHREAD_PRIOQ_INSERT_TAIL(thread);
 
 			/*
 			 * Undefer and handle pending signals, yielding if
diff --git a/lib/libc_r/uthread/uthread_sig.c b/lib/libc_r/uthread/uthread_sig.c
index dc6d209..86ded7f 100644
--- a/lib/libc_r/uthread/uthread_sig.c
+++ b/lib/libc_r/uthread/uthread_sig.c
@@ -149,7 +149,7 @@ _thread_sig_handler(int sig, int code, ucontext_t * scp)
 				signal_lock.access_lock = 0;
 			else {
 				sigaddset(&pthread->sigmask, sig);
-				
+
 				/*
 				 * Make sure not to deliver the same signal to
 				 * the thread twice.  sigpend is potentially
@@ -160,7 +160,7 @@ _thread_sig_handler(int sig, int code, ucontext_t * scp)
 				 */
 				if (sigismember(&pthread->sigpend, sig))
 					sigdelset(&pthread->sigpend, sig);
-			
+
 				signal_lock.access_lock = 0;
 				_thread_sig_deliver(pthread, sig);
 				sigdelset(&pthread->sigmask, sig);
@@ -461,6 +461,7 @@ handle_state_change(pthread_t pthread)
 		case PS_RUNNING:
 		case PS_SIGTHREAD:
 		case PS_STATE_MAX:
+		case PS_SUSPENDED:
 			break;
 
 		/*
@@ -492,7 +493,6 @@ handle_state_change(pthread_t pthread)
 		case PS_SIGWAIT:
 		case PS_SLEEP_WAIT:
 		case PS_SPINBLOCK:
-		case PS_SUSPENDED:
 		case PS_WAIT_WAIT:
 			if ((pthread->flags & PTHREAD_FLAGS_IN_WAITQ) != 0) {
 				PTHREAD_WAITQ_REMOVE(pthread);
@@ -628,10 +628,12 @@ _thread_sig_send(pthread_t pthread, int sig)
 		    !sigismember(&pthread->sigmask, sig)) {
 			/* Perform any state changes due to signal arrival: */
 			thread_sig_check_state(pthread, sig);
+			/* Increment the pending signal count. */
+			sigaddset(&pthread->sigpend,sig);
+		} else {
+			/* Increment the pending signal count. */
+			sigaddset(&pthread->sigpend,sig);
 		}
-
-		/* Increment the pending signal count. */
-		sigaddset(&pthread->sigpend,sig);
 	}
 }
 
diff --git a/lib/libc_r/uthread/uthread_suspend_np.c b/lib/libc_r/uthread/uthread_suspend_np.c
index ea9b1f8..9b08115 100644
--- a/lib/libc_r/uthread/uthread_suspend_np.c
+++ b/lib/libc_r/uthread/uthread_suspend_np.c
@@ -36,6 +36,8 @@
 #include <pthread.h>
 #include "pthread_private.h"
 
+static void	finish_suspension(void *arg);
+
 /* Suspend a thread: */
 int
 pthread_suspend_np(pthread_t thread)
@@ -44,22 +46,81 @@ pthread_suspend_np(pthread_t thread)
 
 	/* Find the thread in the list of active threads: */
 	if ((ret = _find_thread(thread)) == 0) {
-		/* The thread exists. Is it running? */
-		if (thread->state != PS_RUNNING &&
-		    thread->state != PS_SUSPENDED) {
-			/* The thread operation has been interrupted */
-			_thread_seterrno(thread,EINTR);
-			thread->interrupted = 1;
-		}
-
 		/*
 		 * Defer signals to protect the scheduling queues from
 		 * access by the signal handler:
 		 */
 		_thread_kern_sig_defer();
 
-		/* Suspend the thread. */
-		PTHREAD_NEW_STATE(thread,PS_SUSPENDED);
+		switch (thread->state) {
+		case PS_RUNNING:
+			/*
+			 * Remove the thread from the priority queue and
+			 * set the state to suspended:
+			 */
+			PTHREAD_PRIOQ_REMOVE(thread);
+			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+			break;
+
+		case PS_SPINBLOCK:
+		case PS_FDR_WAIT:
+		case PS_FDW_WAIT:
+		case PS_POLL_WAIT:
+		case PS_SELECT_WAIT:
+			/*
+			 * Remove these threads from the work queue
+			 * and mark the operation as interrupted:
+			 */
+			if ((thread->flags & PTHREAD_FLAGS_IN_WORKQ) != 0)
+				PTHREAD_WORKQ_REMOVE(thread);
+			_thread_seterrno(thread,EINTR);
+			thread->interrupted = 1;
+
+			/* FALLTHROUGH */
+		case PS_SIGTHREAD:
+		case PS_SLEEP_WAIT:
+		case PS_WAIT_WAIT:
+		case PS_SIGSUSPEND:
+		case PS_SIGWAIT:
+			/*
+			 * Remove these threads from the waiting queue and
+			 * set their state to suspended:
+			 */
+			PTHREAD_WAITQ_REMOVE(thread);
+			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+			break;
+
+		case PS_MUTEX_WAIT:
+		case PS_COND_WAIT:
+		case PS_FDLR_WAIT:
+		case PS_FDLW_WAIT:
+		case PS_FILE_WAIT:
+		case PS_JOIN:
+			/* Mark the thread as suspended: */
+			thread->suspended = 1;
+
+			/*
+			 * Threads in these states may be in queues.
+			 * In order to preserve queue integrity, the
+			 * cancelled thread must remove itself from the
+			 * queue.  Mark the thread as interrupted and
+			 * set the state to running.  When the thread
+			 * resumes, it will remove itself from the queue
+			 * and call the suspension completion routine.
+			 */
+			thread->interrupted = 1;
+			_thread_seterrno(thread, EINTR);
+			PTHREAD_NEW_STATE(thread, PS_RUNNING);
+			thread->continuation = finish_suspension;
+			break;
+
+		case PS_DEAD:
+		case PS_DEADLOCK:
+		case PS_STATE_MAX:
+		case PS_SUSPENDED:
+			/* Nothing needs to be done: */
+			break;
+		}
 
 		/*
 		 * Undefer and handle pending signals, yielding if
@@ -69,4 +130,13 @@ pthread_suspend_np(pthread_t thread)
 	}
 	return(ret);
 }
+
+static void
+finish_suspension(void *arg)
+{
+	if (_thread_run->suspended != 0)
+		_thread_kern_sched_state(PS_SUSPENDED, __FILE__, __LINE__);
+}
+
+
 #endif
diff --git a/lib/libkse/thread/thr_cancel.c b/lib/libkse/thread/thr_cancel.c
index f22bfb5..82ddbb8 100644
--- a/lib/libkse/thread/thr_cancel.c
+++ b/lib/libkse/thread/thr_cancel.c
@@ -37,6 +37,15 @@ pthread_cancel(pthread_t pthread)
 				pthread->cancelflags |= PTHREAD_CANCELLING;
 				break;
 
+			case PS_SUSPENDED:
+				/*
+				 * This thread isn't in any scheduling
+				 * queues; just change it's state:
+				 */
+				pthread->cancelflags |= PTHREAD_CANCELLING;
+				PTHREAD_SET_STATE(pthread, PS_RUNNING);
+				break;
+
 			case PS_SPINBLOCK:
 			case PS_FDR_WAIT:
 			case PS_FDW_WAIT:
@@ -52,7 +61,6 @@ pthread_cancel(pthread_t pthread)
 			case PS_WAIT_WAIT:
 			case PS_SIGSUSPEND:
 			case PS_SIGWAIT:
-			case PS_SUSPENDED:
 				/* Interrupt and resume: */
 				pthread->interrupted = 1;
 				pthread->cancelflags |= PTHREAD_CANCELLING;
diff --git a/lib/libkse/thread/thr_cond.c b/lib/libkse/thread/thr_cond.c
index d236607..78ee042 100644
--- a/lib/libkse/thread/thr_cond.c
+++ b/lib/libkse/thread/thr_cond.c
@@ -282,8 +282,11 @@ pthread_cond_wait(pthread_cond_t * cond, pthread_mutex_t * mutex)
 			break;
 		}
 
-		if (interrupted != 0 && _thread_run->continuation != NULL)
-			_thread_run->continuation((void *) _thread_run);
+		if (interrupted != 0) {
+			if (_thread_run->continuation != NULL)
+				_thread_run->continuation((void *) _thread_run);
+			rval = EINTR;
+		}
 
 		_thread_leave_cancellation_point();
 	}
@@ -449,8 +452,11 @@ pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
 			break;
 		}
 
-		if (interrupted != 0 && _thread_run->continuation != NULL)
-			_thread_run->continuation((void *) _thread_run);
+		if (interrupted != 0) {
+			if (_thread_run->continuation != NULL)
+				_thread_run->continuation((void *) _thread_run);
+			rval = EINTR;
+		}
 
 		_thread_leave_cancellation_point();
 	}
diff --git a/lib/libkse/thread/thr_create.c b/lib/libkse/thread/thr_create.c
index b8a1c46..3a80611 100644
--- a/lib/libkse/thread/thr_create.c
+++ b/lib/libkse/thread/thr_create.c
@@ -299,10 +299,9 @@ pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			/* Add the thread to the linked list of all threads: */
 			TAILQ_INSERT_HEAD(&_thread_list, new_thread, tle);
 
-			if (pattr->suspend == PTHREAD_CREATE_SUSPENDED) {
+			if (pattr->suspend == PTHREAD_CREATE_SUSPENDED)
 				new_thread->state = PS_SUSPENDED;
-				PTHREAD_WAITQ_INSERT(new_thread);
-			} else {
+			else {
 				new_thread->state = PS_RUNNING;
 				PTHREAD_PRIOQ_INSERT_TAIL(new_thread);
 			}
diff --git a/lib/libkse/thread/thr_kern.c b/lib/libkse/thread/thr_kern.c
index 3e0ff17..8b333e4 100644
--- a/lib/libkse/thread/thr_kern.c
+++ b/lib/libkse/thread/thr_kern.c
@@ -184,8 +184,10 @@ __asm__("fnsave %0": :"m"(*fdata));
 			switch (_thread_run->state) {
 			case PS_DEAD:
 			case PS_STATE_MAX: /* to silence -Wall */
+			case PS_SUSPENDED:
 				/*
-				 * Dead threads are not placed in any queue:
+				 * Dead and suspended threads are not placed
+				 * in any queue:
 				 */
 				break;
 
@@ -227,7 +229,6 @@ __asm__("fnsave %0": :"m"(*fdata));
 			case PS_SIGSUSPEND:
 			case PS_SIGTHREAD:
 			case PS_SIGWAIT:
-			case PS_SUSPENDED:
 			case PS_WAIT_WAIT:
 				/* No timeouts for these states: */
 				_thread_run->wakeup_time.tv_sec = -1;
diff --git a/lib/libkse/thread/thr_mutex.c b/lib/libkse/thread/thr_mutex.c
index 6d75ea5..c97c86b 100644
--- a/lib/libkse/thread/thr_mutex.c
+++ b/lib/libkse/thread/thr_mutex.c
@@ -610,8 +610,10 @@ pthread_mutex_lock(pthread_mutex_t * mutex)
 		 * Check to see if this thread was interrupted and
 		 * is still in the mutex queue of waiting threads:
 		 */
-		if (_thread_run->interrupted != 0)
+		if (_thread_run->interrupted != 0) {
 			mutex_queue_remove(*mutex, _thread_run);
+			ret = EINTR;
+		}
 
 		/* Unlock the mutex structure: */
 		_SPINUNLOCK(&(*mutex)->lock);
diff --git a/lib/libkse/thread/thr_private.h b/lib/libkse/thread/thr_private.h
index 962eb86..07c8bec 100644
--- a/lib/libkse/thread/thr_private.h
+++ b/lib/libkse/thread/thr_private.h
@@ -105,7 +105,7 @@
 		else							\
 			TAILQ_INSERT_BEFORE(tid,thrd,pqe);		\
 	}								\
-	(thrd)->flags | PTHREAD_FLAGS_IN_WAITQ;				\
+	(thrd)->flags |= PTHREAD_FLAGS_IN_WAITQ;			\
 } while (0)
 #define PTHREAD_WAITQ_CLEARACTIVE()
 #define PTHREAD_WAITQ_SETACTIVE()
@@ -576,6 +576,8 @@ struct pthread {
 #define PTHREAD_CANCEL_NEEDED		0x0010
 	int	cancelflags;
 
+	int	suspended;
+
 	thread_continuation_t	continuation;
 
 	/*
diff --git a/lib/libkse/thread/thr_resume_np.c b/lib/libkse/thread/thr_resume_np.c
index 98ec718..fae355a 100644
--- a/lib/libkse/thread/thr_resume_np.c
+++ b/lib/libkse/thread/thr_resume_np.c
@@ -44,8 +44,11 @@ pthread_resume_np(pthread_t thread)
 
 	/* Find the thread in the list of active threads: */
 	if ((ret = _find_thread(thread)) == 0) {
-		/* The thread exists. Is it suspended? */
-		if (thread->state != PS_SUSPENDED) {
+		/* Cancel any pending suspensions: */
+		thread->suspended = 0;
+
+		/* Is it currently suspended? */
+		if (thread->state == PS_SUSPENDED) {
 			/*
 			 * Defer signals to protect the scheduling queues
 			 * from access by the signal handler:
@@ -53,7 +56,8 @@ pthread_resume_np(pthread_t thread)
 			_thread_kern_sig_defer();
 
 			/* Allow the thread to run. */
-			PTHREAD_NEW_STATE(thread,PS_RUNNING);
+			PTHREAD_SET_STATE(thread,PS_RUNNING);
+			PTHREAD_PRIOQ_INSERT_TAIL(thread);
 
 			/*
 			 * Undefer and handle pending signals, yielding if
diff --git a/lib/libkse/thread/thr_sig.c b/lib/libkse/thread/thr_sig.c
index dc6d209..86ded7f 100644
--- a/lib/libkse/thread/thr_sig.c
+++ b/lib/libkse/thread/thr_sig.c
@@ -149,7 +149,7 @@ _thread_sig_handler(int sig, int code, ucontext_t * scp)
 				signal_lock.access_lock = 0;
 			else {
 				sigaddset(&pthread->sigmask, sig);
-				
+
 				/*
 				 * Make sure not to deliver the same signal to
 				 * the thread twice.  sigpend is potentially
@@ -160,7 +160,7 @@ _thread_sig_handler(int sig, int code, ucontext_t * scp)
 				 */
 				if (sigismember(&pthread->sigpend, sig))
 					sigdelset(&pthread->sigpend, sig);
-			
+
 				signal_lock.access_lock = 0;
 				_thread_sig_deliver(pthread, sig);
 				sigdelset(&pthread->sigmask, sig);
@@ -461,6 +461,7 @@ handle_state_change(pthread_t pthread)
 		case PS_RUNNING:
 		case PS_SIGTHREAD:
 		case PS_STATE_MAX:
+		case PS_SUSPENDED:
 			break;
 
 		/*
@@ -492,7 +493,6 @@ handle_state_change(pthread_t pthread)
 		case PS_SIGWAIT:
 		case PS_SLEEP_WAIT:
 		case PS_SPINBLOCK:
-		case PS_SUSPENDED:
 		case PS_WAIT_WAIT:
 			if ((pthread->flags & PTHREAD_FLAGS_IN_WAITQ) != 0) {
 				PTHREAD_WAITQ_REMOVE(pthread);
@@ -628,10 +628,12 @@ _thread_sig_send(pthread_t pthread, int sig)
 		    !sigismember(&pthread->sigmask, sig)) {
 			/* Perform any state changes due to signal arrival: */
 			thread_sig_check_state(pthread, sig);
+			/* Increment the pending signal count. */
+			sigaddset(&pthread->sigpend,sig);
+		} else {
+			/* Increment the pending signal count. */
+			sigaddset(&pthread->sigpend,sig);
 		}
-
-		/* Increment the pending signal count. */
-		sigaddset(&pthread->sigpend,sig);
 	}
 }
 
diff --git a/lib/libkse/thread/thr_suspend_np.c b/lib/libkse/thread/thr_suspend_np.c
index ea9b1f8..9b08115 100644
--- a/lib/libkse/thread/thr_suspend_np.c
+++ b/lib/libkse/thread/thr_suspend_np.c
@@ -36,6 +36,8 @@
 #include <pthread.h>
 #include "pthread_private.h"
 
+static void	finish_suspension(void *arg);
+
 /* Suspend a thread: */
 int
 pthread_suspend_np(pthread_t thread)
@@ -44,22 +46,81 @@ pthread_suspend_np(pthread_t thread)
 
 	/* Find the thread in the list of active threads: */
 	if ((ret = _find_thread(thread)) == 0) {
-		/* The thread exists. Is it running? */
-		if (thread->state != PS_RUNNING &&
-		    thread->state != PS_SUSPENDED) {
-			/* The thread operation has been interrupted */
-			_thread_seterrno(thread,EINTR);
-			thread->interrupted = 1;
-		}
-
 		/*
 		 * Defer signals to protect the scheduling queues from
 		 * access by the signal handler:
 		 */
 		_thread_kern_sig_defer();
 
-		/* Suspend the thread. */
-		PTHREAD_NEW_STATE(thread,PS_SUSPENDED);
+		switch (thread->state) {
+		case PS_RUNNING:
+			/*
+			 * Remove the thread from the priority queue and
+			 * set the state to suspended:
+			 */
+			PTHREAD_PRIOQ_REMOVE(thread);
+			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+			break;
+
+		case PS_SPINBLOCK:
+		case PS_FDR_WAIT:
+		case PS_FDW_WAIT:
+		case PS_POLL_WAIT:
+		case PS_SELECT_WAIT:
+			/*
+			 * Remove these threads from the work queue
+			 * and mark the operation as interrupted:
+			 */
+			if ((thread->flags & PTHREAD_FLAGS_IN_WORKQ) != 0)
+				PTHREAD_WORKQ_REMOVE(thread);
+			_thread_seterrno(thread,EINTR);
+			thread->interrupted = 1;
+
+			/* FALLTHROUGH */
+		case PS_SIGTHREAD:
+		case PS_SLEEP_WAIT:
+		case PS_WAIT_WAIT:
+		case PS_SIGSUSPEND:
+		case PS_SIGWAIT:
+			/*
+			 * Remove these threads from the waiting queue and
+			 * set their state to suspended:
+			 */
+			PTHREAD_WAITQ_REMOVE(thread);
+			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+			break;
+
+		case PS_MUTEX_WAIT:
+		case PS_COND_WAIT:
+		case PS_FDLR_WAIT:
+		case PS_FDLW_WAIT:
+		case PS_FILE_WAIT:
+		case PS_JOIN:
+			/* Mark the thread as suspended: */
+			thread->suspended = 1;
+
+			/*
+			 * Threads in these states may be in queues.
+			 * In order to preserve queue integrity, the
+			 * cancelled thread must remove itself from the
+			 * queue.  Mark the thread as interrupted and
+			 * set the state to running.  When the thread
+			 * resumes, it will remove itself from the queue
+			 * and call the suspension completion routine.
+			 */
+			thread->interrupted = 1;
+			_thread_seterrno(thread, EINTR);
+			PTHREAD_NEW_STATE(thread, PS_RUNNING);
+			thread->continuation = finish_suspension;
+			break;
+
+		case PS_DEAD:
+		case PS_DEADLOCK:
+		case PS_STATE_MAX:
+		case PS_SUSPENDED:
+			/* Nothing needs to be done: */
+			break;
+		}
 
 		/*
 		 * Undefer and handle pending signals, yielding if
@@ -69,4 +130,13 @@ pthread_suspend_np(pthread_t thread)
 	}
 	return(ret);
 }
+
+static void
+finish_suspension(void *arg)
+{
+	if (_thread_run->suspended != 0)
+		_thread_kern_sched_state(PS_SUSPENDED, __FILE__, __LINE__);
+}
+
+
 #endif
diff --git a/lib/libpthread/thread/thr_cancel.c b/lib/libpthread/thread/thr_cancel.c
index f22bfb5..82ddbb8 100644
--- a/lib/libpthread/thread/thr_cancel.c
+++ b/lib/libpthread/thread/thr_cancel.c
@@ -37,6 +37,15 @@ pthread_cancel(pthread_t pthread)
 				pthread->cancelflags |= PTHREAD_CANCELLING;
 				break;
 
+			case PS_SUSPENDED:
+				/*
+				 * This thread isn't in any scheduling
+				 * queues; just change it's state:
+				 */
+				pthread->cancelflags |= PTHREAD_CANCELLING;
+				PTHREAD_SET_STATE(pthread, PS_RUNNING);
+				break;
+
 			case PS_SPINBLOCK:
 			case PS_FDR_WAIT:
 			case PS_FDW_WAIT:
@@ -52,7 +61,6 @@ pthread_cancel(pthread_t pthread)
 			case PS_WAIT_WAIT:
 			case PS_SIGSUSPEND:
 			case PS_SIGWAIT:
-			case PS_SUSPENDED:
 				/* Interrupt and resume: */
 				pthread->interrupted = 1;
 				pthread->cancelflags |= PTHREAD_CANCELLING;
diff --git a/lib/libpthread/thread/thr_cond.c b/lib/libpthread/thread/thr_cond.c
index d236607..78ee042 100644
--- a/lib/libpthread/thread/thr_cond.c
+++ b/lib/libpthread/thread/thr_cond.c
@@ -282,8 +282,11 @@ pthread_cond_wait(pthread_cond_t * cond, pthread_mutex_t * mutex)
 			break;
 		}
 
-		if (interrupted != 0 && _thread_run->continuation != NULL)
-			_thread_run->continuation((void *) _thread_run);
+		if (interrupted != 0) {
+			if (_thread_run->continuation != NULL)
+				_thread_run->continuation((void *) _thread_run);
+			rval = EINTR;
+		}
 
 		_thread_leave_cancellation_point();
 	}
@@ -449,8 +452,11 @@ pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
 			break;
 		}
 
-		if (interrupted != 0 && _thread_run->continuation != NULL)
-			_thread_run->continuation((void *) _thread_run);
+		if (interrupted != 0) {
+			if (_thread_run->continuation != NULL)
+				_thread_run->continuation((void *) _thread_run);
+			rval = EINTR;
+		}
 
 		_thread_leave_cancellation_point();
 	}
diff --git a/lib/libpthread/thread/thr_create.c b/lib/libpthread/thread/thr_create.c
index b8a1c46..3a80611 100644
--- a/lib/libpthread/thread/thr_create.c
+++ b/lib/libpthread/thread/thr_create.c
@@ -299,10 +299,9 @@ pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			/* Add the thread to the linked list of all threads: */
 			TAILQ_INSERT_HEAD(&_thread_list, new_thread, tle);
 
-			if (pattr->suspend == PTHREAD_CREATE_SUSPENDED) {
+			if (pattr->suspend == PTHREAD_CREATE_SUSPENDED)
 				new_thread->state = PS_SUSPENDED;
-				PTHREAD_WAITQ_INSERT(new_thread);
-			} else {
+			else {
 				new_thread->state = PS_RUNNING;
 				PTHREAD_PRIOQ_INSERT_TAIL(new_thread);
 			}
diff --git a/lib/libpthread/thread/thr_kern.c b/lib/libpthread/thread/thr_kern.c
index 3e0ff17..8b333e4 100644
--- a/lib/libpthread/thread/thr_kern.c
+++ b/lib/libpthread/thread/thr_kern.c
@@ -184,8 +184,10 @@ __asm__("fnsave %0": :"m"(*fdata));
 			switch (_thread_run->state) {
 			case PS_DEAD:
 			case PS_STATE_MAX: /* to silence -Wall */
+			case PS_SUSPENDED:
 				/*
-				 * Dead threads are not placed in any queue:
+				 * Dead and suspended threads are not placed
+				 * in any queue:
 				 */
 				break;
 
@@ -227,7 +229,6 @@ __asm__("fnsave %0": :"m"(*fdata));
 			case PS_SIGSUSPEND:
 			case PS_SIGTHREAD:
 			case PS_SIGWAIT:
-			case PS_SUSPENDED:
 			case PS_WAIT_WAIT:
 				/* No timeouts for these states: */
 				_thread_run->wakeup_time.tv_sec = -1;
diff --git a/lib/libpthread/thread/thr_mutex.c b/lib/libpthread/thread/thr_mutex.c
index 6d75ea5..c97c86b 100644
--- a/lib/libpthread/thread/thr_mutex.c
+++ b/lib/libpthread/thread/thr_mutex.c
@@ -610,8 +610,10 @@ pthread_mutex_lock(pthread_mutex_t * mutex)
 		 * Check to see if this thread was interrupted and
 		 * is still in the mutex queue of waiting threads:
 		 */
-		if (_thread_run->interrupted != 0)
+		if (_thread_run->interrupted != 0) {
 			mutex_queue_remove(*mutex, _thread_run);
+			ret = EINTR;
+		}
 
 		/* Unlock the mutex structure: */
 		_SPINUNLOCK(&(*mutex)->lock);
diff --git a/lib/libpthread/thread/thr_private.h b/lib/libpthread/thread/thr_private.h
index 962eb86..07c8bec 100644
--- a/lib/libpthread/thread/thr_private.h
+++ b/lib/libpthread/thread/thr_private.h
@@ -105,7 +105,7 @@
 		else							\
 			TAILQ_INSERT_BEFORE(tid,thrd,pqe);		\
 	}								\
-	(thrd)->flags | PTHREAD_FLAGS_IN_WAITQ;				\
+	(thrd)->flags |= PTHREAD_FLAGS_IN_WAITQ;			\
 } while (0)
 #define PTHREAD_WAITQ_CLEARACTIVE()
 #define PTHREAD_WAITQ_SETACTIVE()
@@ -576,6 +576,8 @@ struct pthread {
 #define PTHREAD_CANCEL_NEEDED		0x0010
 	int	cancelflags;
 
+	int	suspended;
+
 	thread_continuation_t	continuation;
 
 	/*
diff --git a/lib/libpthread/thread/thr_resume_np.c b/lib/libpthread/thread/thr_resume_np.c
index 98ec718..fae355a 100644
--- a/lib/libpthread/thread/thr_resume_np.c
+++ b/lib/libpthread/thread/thr_resume_np.c
@@ -44,8 +44,11 @@ pthread_resume_np(pthread_t thread)
 
 	/* Find the thread in the list of active threads: */
 	if ((ret = _find_thread(thread)) == 0) {
-		/* The thread exists. Is it suspended? */
-		if (thread->state != PS_SUSPENDED) {
+		/* Cancel any pending suspensions: */
+		thread->suspended = 0;
+
+		/* Is it currently suspended? */
+		if (thread->state == PS_SUSPENDED) {
 			/*
 			 * Defer signals to protect the scheduling queues
 			 * from access by the signal handler:
@@ -53,7 +56,8 @@ pthread_resume_np(pthread_t thread)
 			_thread_kern_sig_defer();
 
 			/* Allow the thread to run. */
-			PTHREAD_NEW_STATE(thread,PS_RUNNING);
+			PTHREAD_SET_STATE(thread,PS_RUNNING);
+			PTHREAD_PRIOQ_INSERT_TAIL(thread);
 
 			/*
 			 * Undefer and handle pending signals, yielding if
diff --git a/lib/libpthread/thread/thr_sig.c b/lib/libpthread/thread/thr_sig.c
index dc6d209..86ded7f 100644
--- a/lib/libpthread/thread/thr_sig.c
+++ b/lib/libpthread/thread/thr_sig.c
@@ -149,7 +149,7 @@ _thread_sig_handler(int sig, int code, ucontext_t * scp)
 				signal_lock.access_lock = 0;
 			else {
 				sigaddset(&pthread->sigmask, sig);
-				
+
 				/*
 				 * Make sure not to deliver the same signal to
 				 * the thread twice.  sigpend is potentially
@@ -160,7 +160,7 @@ _thread_sig_handler(int sig, int code, ucontext_t * scp)
 				 */
 				if (sigismember(&pthread->sigpend, sig))
 					sigdelset(&pthread->sigpend, sig);
-			
+
 				signal_lock.access_lock = 0;
 				_thread_sig_deliver(pthread, sig);
 				sigdelset(&pthread->sigmask, sig);
@@ -461,6 +461,7 @@ handle_state_change(pthread_t pthread)
 		case PS_RUNNING:
 		case PS_SIGTHREAD:
 		case PS_STATE_MAX:
+		case PS_SUSPENDED:
 			break;
 
 		/*
@@ -492,7 +493,6 @@ handle_state_change(pthread_t pthread)
 		case PS_SIGWAIT:
 		case PS_SLEEP_WAIT:
 		case PS_SPINBLOCK:
-		case PS_SUSPENDED:
 		case PS_WAIT_WAIT:
 			if ((pthread->flags & PTHREAD_FLAGS_IN_WAITQ) != 0) {
 				PTHREAD_WAITQ_REMOVE(pthread);
@@ -628,10 +628,12 @@ _thread_sig_send(pthread_t pthread, int sig)
 		    !sigismember(&pthread->sigmask, sig)) {
 			/* Perform any state changes due to signal arrival: */
 			thread_sig_check_state(pthread, sig);
+			/* Increment the pending signal count. */
+			sigaddset(&pthread->sigpend,sig);
+		} else {
+			/* Increment the pending signal count. */
+			sigaddset(&pthread->sigpend,sig);
 		}
-
-		/* Increment the pending signal count. */
-		sigaddset(&pthread->sigpend,sig);
 	}
 }
 
diff --git a/lib/libpthread/thread/thr_suspend_np.c b/lib/libpthread/thread/thr_suspend_np.c
index ea9b1f8..9b08115 100644
--- a/lib/libpthread/thread/thr_suspend_np.c
+++ b/lib/libpthread/thread/thr_suspend_np.c
@@ -36,6 +36,8 @@
 #include <pthread.h>
 #include "pthread_private.h"
 
+static void	finish_suspension(void *arg);
+
 /* Suspend a thread: */
 int
 pthread_suspend_np(pthread_t thread)
@@ -44,22 +46,81 @@ pthread_suspend_np(pthread_t thread)
 
 	/* Find the thread in the list of active threads: */
 	if ((ret = _find_thread(thread)) == 0) {
-		/* The thread exists. Is it running? */
-		if (thread->state != PS_RUNNING &&
-		    thread->state != PS_SUSPENDED) {
-			/* The thread operation has been interrupted */
-			_thread_seterrno(thread,EINTR);
-			thread->interrupted = 1;
-		}
-
 		/*
 		 * Defer signals to protect the scheduling queues from
 		 * access by the signal handler:
 		 */
 		_thread_kern_sig_defer();
 
-		/* Suspend the thread. */
-		PTHREAD_NEW_STATE(thread,PS_SUSPENDED);
+		switch (thread->state) {
+		case PS_RUNNING:
+			/*
+			 * Remove the thread from the priority queue and
+			 * set the state to suspended:
+			 */
+			PTHREAD_PRIOQ_REMOVE(thread);
+			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+			break;
+
+		case PS_SPINBLOCK:
+		case PS_FDR_WAIT:
+		case PS_FDW_WAIT:
+		case PS_POLL_WAIT:
+		case PS_SELECT_WAIT:
+			/*
+			 * Remove these threads from the work queue
+			 * and mark the operation as interrupted:
+			 */
+			if ((thread->flags & PTHREAD_FLAGS_IN_WORKQ) != 0)
+				PTHREAD_WORKQ_REMOVE(thread);
+			_thread_seterrno(thread,EINTR);
+			thread->interrupted = 1;
+
+			/* FALLTHROUGH */
+		case PS_SIGTHREAD:
+		case PS_SLEEP_WAIT:
+		case PS_WAIT_WAIT:
+		case PS_SIGSUSPEND:
+		case PS_SIGWAIT:
+			/*
+			 * Remove these threads from the waiting queue and
+			 * set their state to suspended:
+			 */
+			PTHREAD_WAITQ_REMOVE(thread);
+			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+			break;
+
+		case PS_MUTEX_WAIT:
+		case PS_COND_WAIT:
+		case PS_FDLR_WAIT:
+		case PS_FDLW_WAIT:
+		case PS_FILE_WAIT:
+		case PS_JOIN:
+			/* Mark the thread as suspended: */
+			thread->suspended = 1;
+
+			/*
+			 * Threads in these states may be in queues.
+			 * In order to preserve queue integrity, the
+			 * cancelled thread must remove itself from the
+			 * queue.  Mark the thread as interrupted and
+			 * set the state to running.  When the thread
+			 * resumes, it will remove itself from the queue
+			 * and call the suspension completion routine.
+			 */
+			thread->interrupted = 1;
+			_thread_seterrno(thread, EINTR);
+			PTHREAD_NEW_STATE(thread, PS_RUNNING);
+			thread->continuation = finish_suspension;
+			break;
+
+		case PS_DEAD:
+		case PS_DEADLOCK:
+		case PS_STATE_MAX:
+		case PS_SUSPENDED:
+			/* Nothing needs to be done: */
+			break;
+		}
 
 		/*
 		 * Undefer and handle pending signals, yielding if
@@ -69,4 +130,13 @@ pthread_suspend_np(pthread_t thread)
 	}
 	return(ret);
 }
+
+static void
+finish_suspension(void *arg)
+{
+	if (_thread_run->suspended != 0)
+		_thread_kern_sched_state(PS_SUSPENDED, __FILE__, __LINE__);
+}
+
+
 #endif