In suspend_common(), don't wait for a thread which is in creation, because

pthread_suspend_all_np() may have already suspended its parent thread.
Add locking code in pthread_suspend_all_np() to only allow one thread
to suspend other threads, this eliminates a deadlock where two or more
threads try to suspend each others.

5 files changed, 64 insertions, 4 deletions

diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c
index e1e304a..c29e8c3 100644
--- a/lib/libthr/thread/thr_init.c
+++ b/lib/libthr/thread/thr_init.c
@@ -120,6 +120,10 @@ struct umutex	_rwlock_static_lock = DEFAULT_UMUTEX;
 struct umutex	_keytable_lock = DEFAULT_UMUTEX;
 struct urwlock	_thr_list_lock = DEFAULT_URWLOCK;
 struct umutex	_thr_event_lock = DEFAULT_UMUTEX;
+struct umutex	_suspend_all_lock = DEFAULT_UMUTEX;
+struct pthread	*_single_thread;
+int		_suspend_all_cycle;
+int		_suspend_all_waiters;
 
 int	__pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *);
 int	__pthread_mutex_lock(pthread_mutex_t *);
@@ -441,11 +445,14 @@ init_private(void)
 	_thr_umutex_init(&_keytable_lock);
 	_thr_urwlock_init(&_thr_atfork_lock);
 	_thr_umutex_init(&_thr_event_lock);
+	_thr_umutex_init(&_suspend_all_lock);
 	_thr_once_init();
 	_thr_spinlock_init();
 	_thr_list_init();
 	_thr_wake_addr_init();
 	_sleepq_init();
+	_single_thread = NULL;
+	_suspend_all_waiters = 0;
 
 	/*
 	 * Avoid reinitializing some things if they don't need to be,
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index ba272fe..fb76290 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -721,6 +721,10 @@ extern struct umutex	_rwlock_static_lock __hidden;
 extern struct umutex	_keytable_lock __hidden;
 extern struct urwlock	_thr_list_lock __hidden;
 extern struct umutex	_thr_event_lock __hidden;
+extern struct umutex	_suspend_all_lock __hidden;
+extern int		_suspend_all_waiters __hidden;
+extern int		_suspend_all_cycle __hidden;
+extern struct pthread	*_single_thread __hidden;
 
 /*
  * Function prototype definitions.
@@ -777,6 +781,8 @@ int	_thr_setscheduler(lwpid_t, int, const struct sched_param *) __hidden;
 void	_thr_signal_prefork(void) __hidden;
 void	_thr_signal_postfork(void) __hidden;
 void	_thr_signal_postfork_child(void) __hidden;
+void	_thr_suspend_all_lock(struct pthread *) __hidden;
+void	_thr_suspend_all_unlock(struct pthread *) __hidden;
 void	_thr_try_gc(struct pthread *, struct pthread *) __hidden;
 int	_rtp_to_schedparam(const struct rtprio *rtp, int *policy,
 		struct sched_param *param) __hidden;
diff --git a/lib/libthr/thread/thr_resume_np.c b/lib/libthr/thread/thr_resume_np.c
index a3066bc..53377da 100644
--- a/lib/libthr/thread/thr_resume_np.c
+++ b/lib/libthr/thread/thr_resume_np.c
@@ -63,7 +63,11 @@ _pthread_resume_all_np(void)
 {
 	struct pthread *curthread = _get_curthread();
 	struct pthread *thread;
+	int old_nocancel;
 
+	old_nocancel = curthread->no_cancel;
+	curthread->no_cancel = 1;
+	_thr_suspend_all_lock(curthread);
 	/* Take the thread list lock: */
 	THREAD_LIST_RDLOCK(curthread);
 
@@ -77,6 +81,9 @@ _pthread_resume_all_np(void)
 
 	/* Release the thread list lock: */
 	THREAD_LIST_UNLOCK(curthread);
+	_thr_suspend_all_unlock(curthread);
+	curthread->no_cancel = old_nocancel;
+	_thr_testcancel(curthread);
 }
 
 static void
diff --git a/lib/libthr/thread/thr_sig.c b/lib/libthr/thread/thr_sig.c
index 3dee8b7..d2be994 100644
--- a/lib/libthr/thread/thr_sig.c
+++ b/lib/libthr/thread/thr_sig.c
@@ -356,7 +356,8 @@ check_suspend(struct pthread *curthread)
 		(THR_FLAGS_NEED_SUSPEND | THR_FLAGS_SUSPENDED))
 		!= THR_FLAGS_NEED_SUSPEND))
 		return;
-
+	if (curthread == _single_thread)
+		return;
 	if (curthread->force_exit)
 		return;
 
diff --git a/lib/libthr/thread/thr_suspend_np.c b/lib/libthr/thread/thr_suspend_np.c
index d5868e8..284619d 100644
--- a/lib/libthr/thread/thr_suspend_np.c
+++ b/lib/libthr/thread/thr_suspend_np.c
@@ -70,14 +70,48 @@ _pthread_suspend_np(pthread_t thread)
 }
 
 void
+_thr_suspend_all_lock(struct pthread *curthread)
+{
+	int old;
+
+	THR_LOCK_ACQUIRE(curthread, &_suspend_all_lock);
+	while (_single_thread != NULL) {
+		old = _suspend_all_cycle;
+		_suspend_all_waiters++;
+		THR_LOCK_RELEASE(curthread, &_suspend_all_lock);
+		_thr_umtx_wait_uint(&_suspend_all_cycle, old, NULL, 0);
+		THR_LOCK_ACQUIRE(curthread, &_suspend_all_lock);
+		_suspend_all_waiters--;
+	}
+	_single_thread = curthread;
+	THR_LOCK_RELEASE(curthread, &_suspend_all_lock);
+}
+
+void
+_thr_suspend_all_unlock(struct pthread *curthread)
+{
+
+	THR_LOCK_ACQUIRE(curthread, &_suspend_all_lock);
+	_single_thread = NULL;
+	if (_suspend_all_waiters != 0) {
+		_suspend_all_cycle++;
+		_thr_umtx_wake(&_suspend_all_cycle, INT_MAX, 0);
+	}
+	THR_LOCK_RELEASE(curthread, &_suspend_all_lock);
+}
+
+void
 _pthread_suspend_all_np(void)
 {
 	struct pthread *curthread = _get_curthread();
 	struct pthread *thread;
+	int old_nocancel;
 	int ret;
 
+	old_nocancel = curthread->no_cancel;
+	curthread->no_cancel = 1;
+	_thr_suspend_all_lock(curthread);
 	THREAD_LIST_RDLOCK(curthread);
-
 	TAILQ_FOREACH(thread, &_thread_list, tle) {
 		if (thread != curthread) {
 			THR_THREAD_LOCK(curthread, thread);
@@ -115,19 +149,24 @@ restart:
 			THR_THREAD_UNLOCK(curthread, thread);
 		}
 	}
-
 	THREAD_LIST_UNLOCK(curthread);
+	_thr_suspend_all_unlock(curthread);
+	curthread->no_cancel = old_nocancel;
+	_thr_testcancel(curthread);
 }
 
 static int
 suspend_common(struct pthread *curthread, struct pthread *thread,
 	int waitok)
 {
-	long tmp;
+	uint32_t tmp;
 
 	while (thread->state != PS_DEAD &&
 	      !(thread->flags & THR_FLAGS_SUSPENDED)) {
 		thread->flags |= THR_FLAGS_NEED_SUSPEND;
+		/* Thread is in creation. */
+		if (thread->tid == TID_TERMINATED)
+			return (1);
 		tmp = thread->cycle;
 		_thr_send_sig(thread, SIGCANCEL);
 		THR_THREAD_UNLOCK(curthread, thread);