Revamp suspend and resume. While I'm here add pthread_suspend_all_np()

and pthread_resume_all_np().  These suspend and resume all threads except
the current thread, respectively.  The existing functions pthread_single_np()
and pthread_multi_np(), which formerly had no effect, now exhibit the same
behaviour and pthread_suspend_all_np() and pthread_resume_all_np().  These
functions have been added mostly for the native java port.

Don't allow the uthread kernel pipe to use the same descriptors as
stdio.  Mostily submitted by Oswald Buddenhagen <ossi@kde.org>.

Correct some minor style nits.

13 files changed, 231 insertions, 307 deletions

diff --git a/lib/libkse/thread/thr_cancel.c b/lib/libkse/thread/thr_cancel.c
index b6b070f..d9324ab 100644
--- a/lib/libkse/thread/thr_cancel.c
+++ b/lib/libkse/thread/thr_cancel.c
@@ -78,20 +78,6 @@ _pthread_cancel(pthread_t pthread)
 				break;
 
 			case PS_SUSPENDED:
-				if (pthread->suspended == SUSP_NO ||
-				    pthread->suspended == SUSP_YES ||
-				    pthread->suspended == SUSP_JOIN ||
-				    pthread->suspended == SUSP_NOWAIT) {
-					/*
-					 * This thread isn't in any scheduling
-					 * queues; just change it's state:
-					 */
-					pthread->cancelflags |=
-					    PTHREAD_CANCELLING;
-					PTHREAD_SET_STATE(pthread, PS_RUNNING);
-					break;
-				}
-				/* FALLTHROUGH */
 			case PS_MUTEX_WAIT:
 			case PS_COND_WAIT:
 			case PS_FDLR_WAIT:
@@ -109,7 +95,7 @@ _pthread_cancel(pthread_t pthread)
 				 */
 				pthread->interrupted = 1;
 				pthread->cancelflags |= PTHREAD_CANCEL_NEEDED;
-				PTHREAD_NEW_STATE(pthread,PS_RUNNING);
+				PTHREAD_NEW_STATE(pthread, PS_RUNNING);
 				pthread->continuation = finish_cancellation;
 				break;
 
diff --git a/lib/libkse/thread/thr_cond.c b/lib/libkse/thread/thr_cond.c
index 7f3fe7a..cb45725 100644
--- a/lib/libkse/thread/thr_cond.c
+++ b/lib/libkse/thread/thr_cond.c
@@ -524,15 +524,9 @@ _pthread_cond_signal(pthread_cond_t * cond)
 
 			if ((pthread = cond_queue_deq(*cond)) != NULL) {
 				/*
-				 * Unless the thread is currently suspended,
-				 * allow it to run.  If the thread is suspended,
-				 * make a note that the thread isn't in a wait
-				 * queue any more.
+				 * Wake up the signaled thread:
 				 */
-				if (pthread->state != PS_SUSPENDED)
-					PTHREAD_NEW_STATE(pthread,PS_RUNNING);
-				else
-					pthread->suspended = SUSP_NOWAIT;
+				PTHREAD_NEW_STATE(pthread, PS_RUNNING);
 			}
 
 			/* Check for no more waiters: */
@@ -596,15 +590,9 @@ _pthread_cond_broadcast(pthread_cond_t * cond)
 			 */
 			while ((pthread = cond_queue_deq(*cond)) != NULL) {
 				/*
-				 * Unless the thread is currently suspended,
-				 * allow it to run.  If the thread is suspended,
-				 * make a note that the thread isn't in a wait
-				 * queue any more.
+				 * Wake up the signaled thread:
 				 */
-				if (pthread->state != PS_SUSPENDED)
-					PTHREAD_NEW_STATE(pthread,PS_RUNNING);
-				else
-					pthread->suspended = SUSP_NOWAIT;
+				PTHREAD_NEW_STATE(pthread, PS_RUNNING);
 			}
 
 			/* There are no more waiting threads: */
diff --git a/lib/libkse/thread/thr_exit.c b/lib/libkse/thread/thr_exit.c
index c9513cf..fd90e29 100644
--- a/lib/libkse/thread/thr_exit.c
+++ b/lib/libkse/thread/thr_exit.c
@@ -202,22 +202,8 @@ _pthread_exit(void *status)
 		pthread = curthread->joiner;
 		curthread->joiner = NULL;
 
-		switch (pthread->suspended) {
-		case SUSP_JOIN:
-			/*
-			 * The joining thread is suspended.  Change the
-			 * suspension state to make the thread runnable when it
-			 * is resumed:
-			 */
-			pthread->suspended = SUSP_NO;
-			break;
-		case SUSP_NO:
-			/* Make the joining thread runnable: */
-			PTHREAD_NEW_STATE(pthread, PS_RUNNING);
-			break;
-		default:
-			PANIC("Unreachable code reached");
-		}
+		/* Make the joining thread runnable: */
+		PTHREAD_NEW_STATE(pthread, PS_RUNNING);
 
 		/* Set the return value for the joining thread: */
 		pthread->join_status.ret = curthread->ret;
diff --git a/lib/libkse/thread/thr_init.c b/lib/libkse/thread/thr_init.c
index 2790748..74db740 100644
--- a/lib/libkse/thread/thr_init.c
+++ b/lib/libkse/thread/thr_init.c
@@ -201,20 +201,21 @@ _thread_init(void)
 			PANIC("Can't open console");
 		if (setlogin("root") == -1)
 			PANIC("Can't set login to root");
-		if (__sys_ioctl(fd,TIOCSCTTY, (char *) NULL) == -1)
+		if (__sys_ioctl(fd, TIOCSCTTY, (char *) NULL) == -1)
 			PANIC("Can't set controlling terminal");
-		if (__sys_dup2(fd,0) == -1 ||
-		    __sys_dup2(fd,1) == -1 ||
-		    __sys_dup2(fd,2) == -1)
+		if (__sys_dup2(fd, 0) == -1 ||
+		    __sys_dup2(fd, 1) == -1 ||
+		    __sys_dup2(fd, 2) == -1)
 			PANIC("Can't dup2");
 	}
 
 	/* Get the standard I/O flags before messing with them : */
-	for (i = 0; i < 3; i++)
+	for (i = 0; i < 3; i++) {
 		if (((_pthread_stdio_flags[i] =
-		    __sys_fcntl(i,F_GETFL, NULL)) == -1) &&
+		    __sys_fcntl(i, F_GETFL, NULL)) == -1) &&
 		    (errno != EBADF))
 			PANIC("Cannot get stdio flags");
+	}
 
 	/*
 	 * Create a pipe that is written to by the signal handler to prevent
@@ -224,8 +225,21 @@ _thread_init(void)
 		/* Cannot create pipe, so abort: */
 		PANIC("Cannot create kernel pipe");
 	}
+
+	/*
+	 * Make sure the pipe does not get in the way of stdio:
+	 */
+	for (i = 0; i < 2; i++) {
+		if (_thread_kern_pipe[i] < 3) {
+			fd = __sys_fcntl(_thread_kern_pipe[i], F_DUPFD, 3);
+			if (fd == -1)
+			    PANIC("Cannot create kernel pipe");
+			__sys_close(_thread_kern_pipe[i]);
+			_thread_kern_pipe[i] = fd;
+		}
+	}
 	/* Get the flags for the read pipe: */
-	else if ((flags = __sys_fcntl(_thread_kern_pipe[0], F_GETFL, NULL)) == -1) {
+	if ((flags = __sys_fcntl(_thread_kern_pipe[0], F_GETFL, NULL)) == -1) {
 		/* Abort this application: */
 		PANIC("Cannot get kernel read pipe flags");
 	}
diff --git a/lib/libkse/thread/thr_multi_np.c b/lib/libkse/thread/thr_multi_np.c
index c1a069f..bd42365 100644
--- a/lib/libkse/thread/thr_multi_np.c
+++ b/lib/libkse/thread/thr_multi_np.c
@@ -31,16 +31,20 @@
  *
  * $FreeBSD$
  */
-#include <string.h>
 #include <pthread.h>
-#include "pthread_private.h"
+#include <pthread_np.h>
 
 __weak_reference(_pthread_multi_np, pthread_multi_np);
 
 int
 _pthread_multi_np()
 {
+
 	/* Return to multi-threaded scheduling mode: */
-	_thread_single = NULL;
-	return(0);
+	/*
+	 * XXX - Do we want to do this?
+	 * __is_threaded = 1;
+	 */
+	pthread_resume_all_np();
+	return (0);
 }
diff --git a/lib/libkse/thread/thr_mutex.c b/lib/libkse/thread/thr_mutex.c
index 0f67b4b..86e0b8b 100644
--- a/lib/libkse/thread/thr_mutex.c
+++ b/lib/libkse/thread/thr_mutex.c
@@ -887,21 +887,9 @@ mutex_unlock_common(pthread_mutex_t * mutex, int add_reference)
 				 */
 				if (((*mutex)->m_owner =
 			  	    mutex_queue_deq(*mutex)) != NULL) {
-					/*
-					 * Unless the new owner of the mutex is
-					 * currently suspended, allow the owner
-					 * to run.  If the thread is suspended,
-					 * make a note that the thread isn't in
-					 * a wait queue any more.
-					 */
-					if (((*mutex)->m_owner->state !=
-					    PS_SUSPENDED)) {
-						PTHREAD_NEW_STATE((*mutex)->m_owner,
-						    PS_RUNNING);
-					} else {
-						(*mutex)->m_owner->suspended =
-						    SUSP_NOWAIT;
-					}
+					/* Make the new owner runnable: */
+					PTHREAD_NEW_STATE((*mutex)->m_owner,
+					    PS_RUNNING);
 
 					/*
 					 * Add the mutex to the threads list of
@@ -1019,20 +1007,10 @@ mutex_unlock_common(pthread_mutex_t * mutex, int add_reference)
 						(*mutex)->m_prio;
 
 					/*
-					 * Unless the new owner of the mutex is
-					 * currently suspended, allow the owner
-					 * to run.  If the thread is suspended,
-					 * make a note that the thread isn't in
-					 * a wait queue any more.
+					 * Make the new owner runnable:
 					 */
-					if (((*mutex)->m_owner->state !=
-					    PS_SUSPENDED)) {
-						PTHREAD_NEW_STATE((*mutex)->m_owner,
-						    PS_RUNNING);
-					} else {
-						(*mutex)->m_owner->suspended =
-						    SUSP_NOWAIT;
-					}
+					PTHREAD_NEW_STATE((*mutex)->m_owner,
+					    PS_RUNNING);
 				}
 			}
 			break;
@@ -1148,20 +1126,10 @@ mutex_unlock_common(pthread_mutex_t * mutex, int add_reference)
 					    (*mutex)->m_prio;
 
 					/*
-					 * Unless the new owner of the mutex is
-					 * currently suspended, allow the owner
-					 * to run.  If the thread is suspended,
-					 * make a note that the thread isn't in
-					 * a wait queue any more.
+					 * Make the new owner runnable:
 					 */
-					if (((*mutex)->m_owner->state !=
-					    PS_SUSPENDED)) {
-						PTHREAD_NEW_STATE((*mutex)->m_owner,
-						    PS_RUNNING);
-					} else {
-						(*mutex)->m_owner->suspended =
-						    SUSP_NOWAIT;
-					}
+					PTHREAD_NEW_STATE((*mutex)->m_owner,
+					    PS_RUNNING);
 				}
 			}
 			break;
diff --git a/lib/libkse/thread/thr_priority_queue.c b/lib/libkse/thread/thr_priority_queue.c
index 55d742b..b700d97 100644
--- a/lib/libkse/thread/thr_priority_queue.c
+++ b/lib/libkse/thread/thr_priority_queue.c
@@ -164,52 +164,74 @@ _pq_remove(pq_queue_t *pq, pthread_t pthread)
 void
 _pq_insert_head(pq_queue_t *pq, pthread_t pthread)
 {
-	int prio = pthread->active_priority;
+	int prio;
 
 	/*
-	 * Make some assertions when debugging is enabled:
+	 * Don't insert suspended threads into the priority queue.
+	 * The caller is responsible for setting the threads state.
 	 */
-	_PQ_ASSERT_INACTIVE("_pq_insert_head: pq_active");
-	_PQ_SET_ACTIVE();
-	_PQ_ASSERT_NOT_QUEUED(pthread,
-	    "_pq_insert_head: Already in priority queue");
-	_PQ_ASSERT_PROTECTED("_pq_insert_head: prioq not protected!");
-
-	TAILQ_INSERT_HEAD(&pq->pq_lists[prio].pl_head, pthread, pqe);
-	if (pq->pq_lists[prio].pl_queued == 0)
-		/* Insert the list into the priority queue: */
-		pq_insert_prio_list(pq, prio);
-
-	/* Mark this thread as being in the priority queue. */
-	pthread->flags |= PTHREAD_FLAGS_IN_PRIOQ;
+	if ((pthread->flags & PTHREAD_FLAGS_SUSPENDED) != 0) {
+		/* Make sure the threads state is suspended. */
+		if (pthread->state != PS_SUSPENDED)
+			PTHREAD_SET_STATE(pthread, PS_SUSPENDED);
+	} else {
+		/*
+		 * Make some assertions when debugging is enabled:
+		 */
+		_PQ_ASSERT_INACTIVE("_pq_insert_head: pq_active");
+		_PQ_SET_ACTIVE();
+		_PQ_ASSERT_NOT_QUEUED(pthread,
+		    "_pq_insert_head: Already in priority queue");
+		_PQ_ASSERT_PROTECTED("_pq_insert_head: prioq not protected!");
+
+		prio = pthread->active_priority;
+		TAILQ_INSERT_HEAD(&pq->pq_lists[prio].pl_head, pthread, pqe);
+		if (pq->pq_lists[prio].pl_queued == 0)
+			/* Insert the list into the priority queue: */
+			pq_insert_prio_list(pq, prio);
+
+		/* Mark this thread as being in the priority queue. */
+		pthread->flags |= PTHREAD_FLAGS_IN_PRIOQ;
 
-	_PQ_CLEAR_ACTIVE();
+		_PQ_CLEAR_ACTIVE();
+	}
 }
 
 
 void
 _pq_insert_tail(pq_queue_t *pq, pthread_t pthread)
 {
-	int prio = pthread->active_priority;
+	int prio;
 
 	/*
-	 * Make some assertions when debugging is enabled:
+	 * Don't insert suspended threads into the priority queue.
+	 * The caller is responsible for setting the threads state.
 	 */
-	_PQ_ASSERT_INACTIVE("_pq_insert_tail: pq_active");
-	_PQ_SET_ACTIVE();
-	_PQ_ASSERT_NOT_QUEUED(pthread,
-	    "_pq_insert_tail: Already in priority queue");
-	_PQ_ASSERT_PROTECTED("_pq_insert_tail: prioq not protected!");
-
-	TAILQ_INSERT_TAIL(&pq->pq_lists[prio].pl_head, pthread, pqe);
-	if (pq->pq_lists[prio].pl_queued == 0)
-		/* Insert the list into the priority queue: */
-		pq_insert_prio_list(pq, prio);
-
-	/* Mark this thread as being in the priority queue. */
-	pthread->flags |= PTHREAD_FLAGS_IN_PRIOQ;
+	if ((pthread->flags & PTHREAD_FLAGS_SUSPENDED) != 0) {
+		/* Make sure the threads state is suspended. */
+		if (pthread->state != PS_SUSPENDED)
+			PTHREAD_SET_STATE(pthread, PS_SUSPENDED);
+	} else {
+		/*
+		 * Make some assertions when debugging is enabled:
+		 */
+		_PQ_ASSERT_INACTIVE("_pq_insert_tail: pq_active");
+		_PQ_SET_ACTIVE();
+		_PQ_ASSERT_NOT_QUEUED(pthread,
+		    "_pq_insert_tail: Already in priority queue");
+		_PQ_ASSERT_PROTECTED("_pq_insert_tail: prioq not protected!");
+
+		prio = pthread->active_priority;
+		TAILQ_INSERT_TAIL(&pq->pq_lists[prio].pl_head, pthread, pqe);
+		if (pq->pq_lists[prio].pl_queued == 0)
+			/* Insert the list into the priority queue: */
+			pq_insert_prio_list(pq, prio);
+
+		/* Mark this thread as being in the priority queue. */
+		pthread->flags |= PTHREAD_FLAGS_IN_PRIOQ;
 
-	_PQ_CLEAR_ACTIVE();
+		_PQ_CLEAR_ACTIVE();
+	}
 }
 
 
@@ -237,6 +259,17 @@ _pq_first(pq_queue_t *pq)
 
 			/* Mark the list as not being in the queue: */
 			pql->pl_queued = 0;
+		} else if ((pthread->flags & PTHREAD_FLAGS_SUSPENDED) != 0) {
+			/*
+			 * This thread is suspended; remove it from the
+			 * list and ensure its state is suspended.
+			 */
+			TAILQ_REMOVE(&pql->pl_head, pthread, pqe);
+			PTHREAD_SET_STATE(pthread, PS_SUSPENDED);
+
+			/* This thread is now longer in the priority queue. */
+			pthread->flags &= ~PTHREAD_FLAGS_IN_PRIOQ;
+			pthread = NULL;
 		}
 	}
 
diff --git a/lib/libkse/thread/thr_private.h b/lib/libkse/thread/thr_private.h
index 04023fb..3fef49c 100644
--- a/lib/libkse/thread/thr_private.h
+++ b/lib/libkse/thread/thr_private.h
@@ -189,14 +189,15 @@
 	if ((thrd)->state != newstate) {				\
 		if ((thrd)->state == PS_RUNNING) {			\
 			PTHREAD_PRIOQ_REMOVE(thrd);			\
+			PTHREAD_SET_STATE(thrd, newstate);		\
 			PTHREAD_WAITQ_INSERT(thrd);			\
 		} else if (newstate == PS_RUNNING) { 			\
 			PTHREAD_WAITQ_REMOVE(thrd);			\
+			PTHREAD_SET_STATE(thrd, newstate);		\
 			PTHREAD_PRIOQ_INSERT_TAIL(thrd);		\
 		}							\
 	}								\
 	_thread_kern_new_state = 0;					\
-	PTHREAD_SET_STATE(thrd, newstate);				\
 } while (0)
 #else
 #define PTHREAD_ASSERT(cond, msg)
@@ -400,18 +401,6 @@ struct pthread_attr {
 #define PTHREAD_CREATE_SUSPENDED		1
 
 /*
- * Additional state for a thread suspended with pthread_suspend_np().
- */
-enum pthread_susp {
-	SUSP_NO,	/* Not suspended. */
-	SUSP_YES,	/* Suspended. */
-	SUSP_JOIN,	/* Suspended, joining. */
-	SUSP_NOWAIT,	/* Suspended, was in a mutex or condition queue. */
-	SUSP_MUTEX_WAIT,/* Suspended, still in a mutex queue. */
-	SUSP_COND_WAIT	/* Suspended, still in a condition queue. */
-};
-
-/*
  * Miscellaneous definitions.
  */
 #define PTHREAD_STACK_DEFAULT			65536
@@ -684,8 +673,6 @@ struct pthread {
 #define PTHREAD_CANCEL_NEEDED		0x0010
 	int	cancelflags;
 
-	enum pthread_susp	suspended;
-
 	thread_continuation_t	continuation;
 
 	/*
@@ -802,7 +789,8 @@ struct pthread {
 #define PTHREAD_FLAGS_IN_FDQ	0x0040	/* in fd lock queue using qe link */
 #define PTHREAD_FLAGS_IN_CONDQ	0x0080	/* in condition queue using sqe link*/
 #define PTHREAD_FLAGS_IN_MUTEXQ	0x0100	/* in mutex queue using sqe link */
-#define PTHREAD_FLAGS_TRACE	0x0200	/* for debugging purposes */
+#define	PTHREAD_FLAGS_SUSPENDED	0x0200	/* thread is suspended */
+#define PTHREAD_FLAGS_TRACE	0x0400	/* for debugging purposes */
 #define PTHREAD_FLAGS_IN_SYNCQ	\
     (PTHREAD_FLAGS_IN_CONDQ | PTHREAD_FLAGS_IN_MUTEXQ)
 
@@ -880,17 +868,6 @@ SCLASS struct pthread   * volatile _last_user_thread
 ;
 #endif
 
-/*
- * Ptr to the thread running in single-threaded mode or NULL if
- * running multi-threaded (default POSIX behaviour).
- */
-SCLASS struct pthread   * volatile _thread_single
-#ifdef GLOBAL_PTHREAD_PRIVATE
-= NULL;
-#else
-;
-#endif
-
 /* List of all threads: */
 SCLASS TAILQ_HEAD(, pthread)	_thread_list
 #ifdef GLOBAL_PTHREAD_PRIVATE
diff --git a/lib/libkse/thread/thr_resume_np.c b/lib/libkse/thread/thr_resume_np.c
index 9cbcf85..ed20b6a 100644
--- a/lib/libkse/thread/thr_resume_np.c
+++ b/lib/libkse/thread/thr_resume_np.c
@@ -35,62 +35,77 @@
 #include <pthread.h>
 #include "pthread_private.h"
 
+static void	resume_common(struct pthread *);
+
 __weak_reference(_pthread_resume_np, pthread_resume_np);
+__weak_reference(_pthread_resume_all_np, pthread_resume_all_np);
 
 /* Resume a thread: */
 int
 _pthread_resume_np(pthread_t thread)
 {
-	int	ret;
-	enum	pthread_susp old_suspended;
+	int ret;
 
 	/* Find the thread in the list of active threads: */
 	if ((ret = _find_thread(thread)) == 0) {
-		/* Cancel any pending suspensions: */
-		old_suspended = thread->suspended;
-		thread->suspended = SUSP_NO;
+		/*
+		 * Defer signals to protect the scheduling queues
+		 * from access by the signal handler:
+		 */
+		_thread_kern_sig_defer();
+
+		if ((thread->flags & PTHREAD_FLAGS_SUSPENDED) != 0)
+			resume_common(thread);
+
+		/*
+		 * Undefer and handle pending signals, yielding if
+		 * necessary:
+		 */
+		_thread_kern_sig_undefer();
+	}
+	return (ret);
+}
+
+void
+_pthread_resume_all_np(void)
+{
+	struct pthread	*curthread = _get_curthread();
+	struct pthread	*thread;
+
+	/*
+	 * Defer signals to protect the scheduling queues from access
+	 * by the signal handler:
+	 */
+	_thread_kern_sig_defer();
+
+	TAILQ_FOREACH(thread, &_thread_list, tle) {
+		if ((thread != curthread) &&
+		    ((thread->flags & PTHREAD_FLAGS_SUSPENDED) != 0))
+			resume_common(thread);
+	}
 
-		/* Is it currently suspended? */
-		if (thread->state == PS_SUSPENDED) {
-			/*
-			 * Defer signals to protect the scheduling queues
-			 * from access by the signal handler:
-			 */
-			_thread_kern_sig_defer();
+	/*
+	 * Undefer and handle pending signals, yielding if necessary:
+	 */
+	_thread_kern_sig_undefer();
+}
 
-			switch (old_suspended) {
-			case SUSP_MUTEX_WAIT:
-				/* Set the thread's state back. */
-				PTHREAD_SET_STATE(thread,PS_MUTEX_WAIT);
-				break;
-			case SUSP_COND_WAIT:
-				/* Set the thread's state back. */
-				PTHREAD_SET_STATE(thread,PS_COND_WAIT);
-				break;
-			case SUSP_JOIN:
-				/* Set the thread's state back. */
-				PTHREAD_SET_STATE(thread,PS_JOIN);
-				break;
-			case SUSP_NOWAIT:
-				/* Allow the thread to run. */
-				PTHREAD_SET_STATE(thread,PS_RUNNING);
-				PTHREAD_WAITQ_REMOVE(thread);
-				PTHREAD_PRIOQ_INSERT_TAIL(thread);
-				break;
-			case SUSP_NO:
-			case SUSP_YES:
-				/* Allow the thread to run. */
-				PTHREAD_SET_STATE(thread,PS_RUNNING);
-				PTHREAD_PRIOQ_INSERT_TAIL(thread);
-				break;
-			}
+static void
+resume_common(struct pthread *thread)
+{
+	/* Clear the suspend flag: */
+	thread->flags &= ~PTHREAD_FLAGS_SUSPENDED;
 
-			/*
-			 * Undefer and handle pending signals, yielding if
-			 * necessary:
-			 */
-			_thread_kern_sig_undefer();
-		}
+	/*
+	 * If the thread's state is suspended, that means it is
+	 * now runnable but not in any scheduling queue.  Set the
+	 * state to running and insert it into the run queue.
+	 */
+	if (thread->state == PS_SUSPENDED) {
+		PTHREAD_SET_STATE(thread, PS_RUNNING);
+		if (thread->priority_mutex_count > 0)
+			PTHREAD_PRIOQ_INSERT_HEAD(thread);
+		else
+			PTHREAD_PRIOQ_INSERT_TAIL(thread);
 	}
-	return(ret);
 }
diff --git a/lib/libkse/thread/thr_sig.c b/lib/libkse/thread/thr_sig.c
index 1bd93b7..7aa9b53 100644
--- a/lib/libkse/thread/thr_sig.c
+++ b/lib/libkse/thread/thr_sig.c
@@ -54,7 +54,7 @@ static void	thread_sigframe_save(struct pthread *thread,
 static void	thread_sig_invoke_handler(int sig, siginfo_t *info,
 		    ucontext_t *ucp);
 
-/* #define DEBUG_SIGNAL */
+/*#define DEBUG_SIGNAL*/
 #ifdef DEBUG_SIGNAL
 #define DBG_MSG		stdout_debug
 #else
@@ -375,7 +375,8 @@ thread_sig_find(int sig)
 				return (NULL);
 			}
 			else if ((handler_installed != 0) &&
-			    !sigismember(&pthread->sigmask, sig)) {
+			    !sigismember(&pthread->sigmask, sig) &&
+			    ((pthread->flags & PTHREAD_FLAGS_SUSPENDED) == 0)) {
 				if (pthread->state == PS_SIGSUSPEND) {
 					if (suspended_thread == NULL)
 						suspended_thread = pthread;
@@ -791,10 +792,17 @@ thread_sig_add(struct pthread *pthread, int sig, int has_args)
 		/*
 		 * The thread should be removed from all scheduling
 		 * queues at this point.  Raise the priority and place
-		 * the thread in the run queue.
+		 * the thread in the run queue.  It is also possible
+		 * for a signal to be sent to a suspended thread,
+		 * mostly via pthread_kill().  If a thread is suspended,
+		 * don't insert it into the priority queue; just set
+		 * its state to suspended and it will run the signal
+		 * handler when it is resumed.
 		 */
 		pthread->active_priority |= PTHREAD_SIGNAL_PRIORITY;
-		if (thread_is_active == 0)
+		if ((pthread->flags & PTHREAD_FLAGS_SUSPENDED) != 0)
+			PTHREAD_SET_STATE(pthread, PS_SUSPENDED);
+		else if (thread_is_active == 0)
 			PTHREAD_PRIOQ_INSERT_TAIL(pthread);
 	}
 }
diff --git a/lib/libkse/thread/thr_single_np.c b/lib/libkse/thread/thr_single_np.c
index 85471b8..1ee5e79 100644
--- a/lib/libkse/thread/thr_single_np.c
+++ b/lib/libkse/thread/thr_single_np.c
@@ -31,17 +31,19 @@
  *
  * $FreeBSD$
  */
-#include <string.h>
 #include <pthread.h>
-#include "pthread_private.h"
+#include <pthread_np.h>
 
 __weak_reference(_pthread_single_np, pthread_single_np);
 
 int _pthread_single_np()
 {
-	struct pthread	*curthread = _get_curthread();
 
 	/* Enter single-threaded (non-POSIX) scheduling mode: */
-	_thread_single = curthread;
-	return(0);
+	pthread_suspend_all_np();
+	/*
+	 * XXX - Do we want to do this?
+	 * __is_threaded = 0;
+	 */
+	return (0);
 }
diff --git a/lib/libkse/thread/thr_spinlock.c b/lib/libkse/thread/thr_spinlock.c
index 7333709..e05aa4a 100644
--- a/lib/libkse/thread/thr_spinlock.c
+++ b/lib/libkse/thread/thr_spinlock.c
@@ -93,7 +93,7 @@ _spinlock_debug(spinlock_t *lck, char *fname, int lineno)
 		cnt++;
 		if (cnt > 100) {
 			char str[256];
-			snprintf(str, sizeof(str), "%s - Warning: Thread %p attempted to lock %p from %s (%d) was left locked from %s (%d)\n", _getprogname(), curthread, lck, fname, lineno, lck->fname, lck->lineno);
+			snprintf(str, sizeof(str), "%s - Warning: Thread %p attempted to lock %p from %s (%d) was left locked from %s (%d)\n", getprogname(), curthread, lck, fname, lineno, lck->fname, lck->lineno);
 			__sys_write(2,str,strlen(str));
 			__sleep(1);
 			cnt = 0;
diff --git a/lib/libkse/thread/thr_suspend_np.c b/lib/libkse/thread/thr_suspend_np.c
index 0e272ff..952baa3 100644
--- a/lib/libkse/thread/thr_suspend_np.c
+++ b/lib/libkse/thread/thr_suspend_np.c
@@ -35,9 +35,10 @@
 #include <pthread.h>
 #include "pthread_private.h"
 
-static void	finish_suspension(void *arg);
+static void	suspend_common(struct pthread *thread);
 
 __weak_reference(_pthread_suspend_np, pthread_suspend_np);
+__weak_reference(_pthread_suspend_all_np, pthread_suspend_all_np);
 
 /* Suspend a thread: */
 int
@@ -45,100 +46,19 @@ _pthread_suspend_np(pthread_t thread)
 {
 	int ret;
 
+	/* Suspending the current thread doesn't make sense. */
+	if (thread == _get_curthread())
+		ret = EDEADLK;
+
 	/* Find the thread in the list of active threads: */
-	if ((ret = _find_thread(thread)) == 0) {
+	else if ((ret = _find_thread(thread)) == 0) {
 		/*
 		 * Defer signals to protect the scheduling queues from
 		 * access by the signal handler:
 		 */
 		_thread_kern_sig_defer();
 
-		switch (thread->state) {
-		case PS_RUNNING:
-			/*
-			 * Remove the thread from the priority queue and
-			 * set the state to suspended:
-			 */
-			PTHREAD_PRIOQ_REMOVE(thread);
-			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
-			break;
-
-		case PS_SPINBLOCK:
-		case PS_FDR_WAIT:
-		case PS_FDW_WAIT:
-		case PS_POLL_WAIT:
-		case PS_SELECT_WAIT:
-			/*
-			 * Remove these threads from the work queue
-			 * and mark the operation as interrupted:
-			 */
-			if ((thread->flags & PTHREAD_FLAGS_IN_WORKQ) != 0)
-				PTHREAD_WORKQ_REMOVE(thread);
-			_thread_seterrno(thread,EINTR);
-
-			/* FALLTHROUGH */
-		case PS_SLEEP_WAIT:
-			thread->interrupted = 1;
-
-			/* FALLTHROUGH */
-		case PS_SIGTHREAD:
-		case PS_WAIT_WAIT:
-		case PS_SIGSUSPEND:
-		case PS_SIGWAIT:
-			/*
-			 * Remove these threads from the waiting queue and
-			 * set their state to suspended:
-			 */
-			PTHREAD_WAITQ_REMOVE(thread);
-			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
-			break;
-
-		case PS_MUTEX_WAIT:
-			/* Mark the thread as suspended and still in a queue. */
-			thread->suspended = SUSP_MUTEX_WAIT;
-
-			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
-			break;
-		case PS_COND_WAIT:
-			/* Mark the thread as suspended and still in a queue. */
-			thread->suspended = SUSP_COND_WAIT;
-
-			PTHREAD_SET_STATE(thread, PS_SUSPENDED);
-			break;
-		case PS_JOIN:
-			/* Mark the thread as suspended and joining: */
-			thread->suspended = SUSP_JOIN;
-
-			PTHREAD_NEW_STATE(thread, PS_SUSPENDED);
-			break;
-		case PS_FDLR_WAIT:
-		case PS_FDLW_WAIT:
-		case PS_FILE_WAIT:
-			/* Mark the thread as suspended: */
-			thread->suspended = SUSP_YES;
-
-			/*
-			 * Threads in these states may be in queues.
-			 * In order to preserve queue integrity, the
-			 * cancelled thread must remove itself from the
-			 * queue.  Mark the thread as interrupted and
-			 * set the state to running.  When the thread
-			 * resumes, it will remove itself from the queue
-			 * and call the suspension completion routine.
-			 */
-			thread->interrupted = 1;
-			_thread_seterrno(thread, EINTR);
-			PTHREAD_NEW_STATE(thread, PS_RUNNING);
-			thread->continuation = finish_suspension;
-			break;
-
-		case PS_DEAD:
-		case PS_DEADLOCK:
-		case PS_STATE_MAX:
-		case PS_SUSPENDED:
-			/* Nothing needs to be done: */
-			break;
-		}
+		suspend_common(thread);
 
 		/*
 		 * Undefer and handle pending signals, yielding if
@@ -146,16 +66,39 @@ _pthread_suspend_np(pthread_t thread)
 		 */
 		_thread_kern_sig_undefer();
 	}
-	return(ret);
+	return (ret);
 }
 
-static void
-finish_suspension(void *arg)
+void
+_pthread_suspend_all_np(void)
 {
 	struct pthread	*curthread = _get_curthread();
+	struct pthread	*thread;
 
-	if (curthread->suspended != SUSP_NO)
-		_thread_kern_sched_state(PS_SUSPENDED, __FILE__, __LINE__);
-}
+	/*
+	 * Defer signals to protect the scheduling queues from
+	 * access by the signal handler:
+	 */
+	_thread_kern_sig_defer();
+
+	TAILQ_FOREACH(thread, &_thread_list, tle) {
+		if (thread != curthread)
+			suspend_common(thread);
+	}
 
+	/*
+	 * Undefer and handle pending signals, yielding if
+	 * necessary:
+	 */
+	_thread_kern_sig_undefer();
+}
 
+void
+suspend_common(struct pthread *thread)
+{
+	thread->flags |= PTHREAD_FLAGS_SUSPENDED;
+	if (thread->flags & PTHREAD_FLAGS_IN_PRIOQ) {
+		PTHREAD_PRIOQ_REMOVE(thread);
+		PTHREAD_SET_STATE(thread, PS_SUSPENDED);
+	}
+}