Import my recent 1:1 threading working. some features improved includes:

 1. fast simple type mutex.
 2. __thread tls works.
 3. asynchronous cancellation works ( using signal ).
 4. thread synchronization is fully based on umtx, mainly, condition
    variable and other synchronization objects were rewritten by using
    umtx directly. those objects can be shared between processes via
    shared memory, it has to change ABI which does not happen yet.
 5. default stack size is increased to 1M on 32 bits platform, 2M for
    64 bits platform.
As the result, some mysql super-smack benchmarks show performance is
improved massivly.

Okayed by: jeff, mtm, rwatson, scottl

1 files changed, 238 insertions, 370 deletions

diff --git a/lib/libthr/thread/thr_cond.c b/lib/libthr/thread/thr_cond.c
index b2cbb83..cd50c4d 100644
--- a/lib/libthr/thread/thr_cond.c
+++ b/lib/libthr/thread/thr_cond.c
@@ -1,476 +1,344 @@
 /*
- * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
+ * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by John Birrell.
- * 4. Neither the name of the author nor the names of any co-contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
  *
- * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD$
  */
+
 #include <stdlib.h>
 #include <errno.h>
 #include <string.h>
 #include <pthread.h>
+#include <limits.h>
+
 #include "thr_private.h"
 
 /*
- * Proctect two different threads calling a pthread_cond_* function
- * from accidentally initializing the condition variable twice.
+ * Prototypes
  */
-static spinlock_t static_cond_lock = _SPINLOCK_INITIALIZER;
+static int cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr);
+static int cond_wait_common(pthread_cond_t *cond, pthread_mutex_t *mutex,
+		    const struct timespec *abstime, int cancel);
+static int cond_signal_common(pthread_cond_t *cond, int broadcast);
 
 /*
- * Prototypes
+ * Double underscore versions are cancellation points.  Single underscore
+ * versions are not and are provided for libc internal usage (which
+ * shouldn't introduce cancellation points).
  */
-static inline int	cond_init(pthread_cond_t *);
-static pthread_t	cond_queue_deq(pthread_cond_t);
-static void		cond_queue_remove(pthread_cond_t, pthread_t);
-static void		cond_queue_enq(pthread_cond_t, pthread_t);
-static int		cond_signal(pthread_cond_t *, int);
-static int		cond_wait_common(pthread_cond_t *,
-			    pthread_mutex_t *, const struct timespec *);
+__weak_reference(__pthread_cond_wait, pthread_cond_wait);
+__weak_reference(__pthread_cond_timedwait, pthread_cond_timedwait);
 
 __weak_reference(_pthread_cond_init, pthread_cond_init);
 __weak_reference(_pthread_cond_destroy, pthread_cond_destroy);
-__weak_reference(_pthread_cond_wait, pthread_cond_wait);
-__weak_reference(_pthread_cond_timedwait, pthread_cond_timedwait);
 __weak_reference(_pthread_cond_signal, pthread_cond_signal);
 __weak_reference(_pthread_cond_broadcast, pthread_cond_broadcast);
 
-#define	COND_LOCK(c)						\
-do {								\
-	if (umtx_lock(&(c)->c_lock, curthread->thr_id))		\
-		abort();					\
-} while (0)
-
-#define	COND_UNLOCK(c)						\
-do {								\
-	if (umtx_unlock(&(c)->c_lock, curthread->thr_id))	\
-		abort();					\
-} while (0)
-
-
-/* Reinitialize a condition variable to defaults. */
-int
-_cond_reinit(pthread_cond_t *cond)
+static int
+cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
 {
-	if (cond == NULL)
-		return (EINVAL);
-
-	if (*cond == NULL)
-		return (pthread_cond_init(cond, NULL));
+	pthread_cond_t	pcond;
+	int             rval = 0;
 
-	/*
-	 * Initialize the condition variable structure:
-	 */
-	TAILQ_INIT(&(*cond)->c_queue);
-	(*cond)->c_flags = COND_FLAGS_INITED;
-	(*cond)->c_type = COND_TYPE_FAST;
-	(*cond)->c_mutex = NULL;
-	(*cond)->c_seqno = 0;
-	bzero(&(*cond)->c_lock, sizeof((*cond)->c_lock));
-
-	return (0);
+	if ((pcond = (pthread_cond_t)
+	    malloc(sizeof(struct pthread_cond))) == NULL) {
+		rval = ENOMEM;
+	} else {
+		/*
+		 * Initialise the condition variable structure:
+		 */
+		_thr_umtx_init(&pcond->c_lock);
+		pcond->c_seqno = 0;
+		pcond->c_waiters = 0;
+		pcond->c_wakeups = 0;
+		if (cond_attr == NULL || *cond_attr == NULL) {
+			pcond->c_pshared = 0;
+			pcond->c_clockid = CLOCK_REALTIME;
+		} else {
+			pcond->c_pshared = (*cond_attr)->c_pshared;
+			pcond->c_clockid = (*cond_attr)->c_clockid;
+		}
+		*cond = pcond;
+	}
+	/* Return the completion status: */
+	return (rval);
 }
 
-int
-_pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
+static int
+init_static(struct pthread *thread, pthread_cond_t *cond)
 {
-	enum pthread_cond_type type;
-	pthread_cond_t	pcond;
+	int ret;
 
-	if (cond == NULL)
-		return (EINVAL);
+	THR_LOCK_ACQUIRE(thread, &_cond_static_lock);
 
-	/*
-	 * Check if a pointer to a condition variable attribute
-	 * structure was passed by the caller: 
-	 */
-	if (cond_attr != NULL && *cond_attr != NULL)
-		type = (*cond_attr)->c_type;
+	if (*cond == NULL)
+		ret = cond_init(cond, NULL);
 	else
-		/* Default to a fast condition variable: */
-		type = COND_TYPE_FAST;
-
-	/* Process according to condition variable type: */
-	switch (type) {
-	case COND_TYPE_FAST:
-		break;
-	default:
-		return (EINVAL);
-		break;
-	}
+		ret = 0;
 
-	if ((pcond = (pthread_cond_t)
-	    malloc(sizeof(struct pthread_cond))) == NULL)
-		return (ENOMEM);
-	/*
-	 * Initialise the condition variable
-	 * structure:
-	 */
-	TAILQ_INIT(&pcond->c_queue);
-	pcond->c_flags |= COND_FLAGS_INITED;
-	pcond->c_type = type;
-	pcond->c_mutex = NULL;
-	pcond->c_seqno = 0;
-	bzero(&pcond->c_lock, sizeof(pcond->c_lock));
+	THR_LOCK_RELEASE(thread, &_cond_static_lock);
 
-	*cond = pcond;
-
-	return (0);
+	return (ret);
 }
 
 int
-_pthread_cond_destroy(pthread_cond_t *cond)
+_pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
 {
-	/*
-	 * Short circuit for a statically initialized condvar
-	 * that is being destroyed without having been used.
-	 */
-	if (*cond == PTHREAD_COND_INITIALIZER)
-		return (0);
-
-	COND_LOCK(*cond);
-
-	/*
-	 * Free the memory allocated for the condition
-	 * variable structure:
-	 */
-	free(*cond);
 
-	/*
-	 * NULL the caller's pointer now that the condition
-	 * variable has been destroyed:
-	 */
 	*cond = NULL;
-
-	return (0);
+	return (cond_init(cond, cond_attr));
 }
 
 int
-_pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
+_pthread_cond_destroy(pthread_cond_t *cond)
 {
-	int rval;
+	struct pthread_cond	*cv;
+	struct pthread		*curthread = _get_curthread();
+	int			rval = 0;
+
+	if (*cond == NULL)
+		rval = EINVAL;
+	else {
+		/* Lock the condition variable structure: */
+		THR_LOCK_ACQUIRE(curthread, &(*cond)->c_lock);
+		if ((*cond)->c_waiters + (*cond)->c_wakeups != 0) {
+			THR_LOCK_RELEASE(curthread, &(*cond)->c_lock);
+			return (EBUSY);
+		}
+
+		/*
+		 * NULL the caller's pointer now that the condition
+		 * variable has been destroyed:
+		 */
+		cv = *cond;
+		*cond = NULL;
 
-	rval = cond_wait_common(cond, mutex, NULL);
+		/* Unlock the condition variable structure: */
+		THR_LOCK_RELEASE(curthread, &cv->c_lock);
 
-	/* This should never happen. */
-	if (rval == ETIMEDOUT)
-		abort();
+		/* Free the cond lock structure: */
 
+		/*
+		 * Free the memory allocated for the condition
+		 * variable structure:
+		 */
+		free(cv);
+
+	}
+	/* Return the completion status: */
 	return (rval);
 }
 
-int
-_pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
-		       const struct timespec * abstime)
+struct cond_cancel_info
 {
-	if (abstime == NULL || abstime->tv_nsec >= 1000000000)
-		return (EINVAL);
+	pthread_mutex_t	*mutex;
+	pthread_cond_t	*cond;
+	long		seqno;
+};
+
+static void
+cond_cancel_handler(void *arg)
+{
+	struct pthread *curthread = _get_curthread();
+	struct cond_cancel_info *cci = (struct cond_cancel_info *)arg;
+	pthread_cond_t cv;
+
+	cv = *(cci->cond);
+	THR_LOCK_ACQUIRE(curthread, &cv->c_lock);
+	if (cv->c_seqno != cci->seqno && cv->c_wakeups != 0) {
+		if (cv->c_waiters > 0) {
+			cv->c_seqno++;
+			_thr_umtx_wake(&cv->c_seqno, 1);
+		} else
+			cv->c_wakeups--;
+	} else {
+		cv->c_waiters--;
+	}
+	THR_LOCK_RELEASE(curthread, &cv->c_lock);
 
-	return (cond_wait_common(cond, mutex, abstime));
+	_mutex_cv_lock(cci->mutex);
 }
 
 static int
-cond_wait_common(pthread_cond_t * cond, pthread_mutex_t * mutex,
-	         const struct timespec * abstime)
+cond_wait_common(pthread_cond_t *cond, pthread_mutex_t *mutex,
+	const struct timespec *abstime, int cancel)
 {
-	int	rval = 0;
-	int	mtxrval;
+	struct pthread	*curthread = _get_curthread();
+	struct timespec ts, ts2, *tsp;
+	struct cond_cancel_info cci;
+	pthread_cond_t  cv;
+	long		seq, oldseq;
+	int		oldcancel;
+	int		ret = 0;
 
-
-	if (cond == NULL)
-		return (EINVAL);
 	/*
-	 * If the condition variable is statically initialized, perform dynamic
-	 * initialization.
+	 * If the condition variable is statically initialized,
+	 * perform the dynamic initialization:
 	 */
-	if (*cond == PTHREAD_COND_INITIALIZER && (rval = cond_init(cond)) != 0)
-		return (rval);
-
-	if ((*cond)->c_type != COND_TYPE_FAST)
-		return (EINVAL);
-
-	COND_LOCK(*cond);
-
-	/*
-	 * If the condvar was statically allocated, properly
-	 * initialize the tail queue.
-	 */
-	if (((*cond)->c_flags & COND_FLAGS_INITED) == 0) {
-		TAILQ_INIT(&(*cond)->c_queue);
-		(*cond)->c_flags |= COND_FLAGS_INITED;
-	}
-
-	if ((mutex == NULL) || (((*cond)->c_mutex != NULL) &&
-	    ((*cond)->c_mutex != *mutex))) {
-		COND_UNLOCK(*cond);
-		return (EINVAL);
-	} 
-	/* Remember the mutex */
-	(*cond)->c_mutex = *mutex;
-
-	_thread_enter_cancellation_point();
-	if ((rval = _mutex_cv_unlock(mutex)) != 0) {
-		if (rval == -1){
-			printf("mutex unlock by condvar failed!");
-			fflush(stdout);
-			abort();
-		}
-		_thread_leave_cancellation_point();
-		COND_UNLOCK(*cond);
-		return (rval);
+	if (__predict_false(*cond == NULL &&
+	    (ret = init_static(curthread, cond)) != 0))
+		return (ret);
+
+	cv = *cond;
+	THR_LOCK_ACQUIRE(curthread, &cv->c_lock);
+	ret = _mutex_cv_unlock(mutex);
+	if (ret) {
+		THR_LOCK_RELEASE(curthread, &cv->c_lock);
+		return (ret);
 	}
+	oldseq = seq = cv->c_seqno;
+	cci.mutex = mutex;
+	cci.cond  = cond;
+	cci.seqno = oldseq;
 
-	/*
-	 * We need to protect the queue operations.  It also 
-	 * protects the pthread flag field.  This is
-	 * dropped before calling _thread_suspend() and reaquired
-	 * when we return.
-	 */
-	PTHREAD_LOCK(curthread);
-
-	/*
-	 * Queue the running thread on the condition
-	 * variable and wait to be signaled.
-	 */
-	cond_queue_enq(*cond, curthread);
+	cv->c_waiters++;
 	do {
-		PTHREAD_UNLOCK(curthread);
-		COND_UNLOCK(*cond);
-		if (curthread->cancellation == CS_PENDING) {
-			/*
-			 * Posix says we must lock the mutex
-			 * even if we're being canceled.
-			 */
-			_mutex_cv_lock(mutex);
-			_thread_leave_cancellation_point();
-			PANIC("Shouldn't have come back.");
-		}
-		rval = _thread_suspend(curthread, (struct timespec *)abstime);
-		if (rval != 0 && rval != ETIMEDOUT && rval != EINTR) {
-			printf("thread suspend returned an invalid value");
-			fflush(stdout);
-			abort();
-		}
-		COND_LOCK(*cond);
-		PTHREAD_LOCK(curthread);
-		if (rval == ETIMEDOUT) {
-			/*
-			 * Condition may have been signaled between the
-			 * time the thread timed out and locked the condvar.
-			 * If it wasn't, manually remove it from the queue.
-			 */
-			if ((curthread->flags & PTHREAD_FLAGS_IN_CONDQ) == 0)
-				rval = 0;
-			else
-				cond_queue_remove(*cond, curthread);
+		THR_LOCK_RELEASE(curthread, &cv->c_lock);
+
+		if (abstime != NULL) {
+			clock_gettime(cv->c_clockid, &ts);
+			TIMESPEC_SUB(&ts2, abstime, &ts);
+			tsp = &ts2;
+		} else
+			tsp = NULL;
+
+		if (cancel) {
+			THR_CLEANUP_PUSH(curthread, cond_cancel_handler, &cci);
+			oldcancel = _thr_cancel_enter(curthread);
+			ret = _thr_umtx_wait(&cv->c_seqno, seq, tsp);
+			_thr_cancel_leave(curthread, oldcancel);
+			THR_CLEANUP_POP(curthread, 0);
+		} else {
+			ret = _thr_umtx_wait(&cv->c_seqno, seq, tsp);
 		}
-	} while ((curthread->flags & PTHREAD_FLAGS_IN_CONDQ) != 0);
 
-	PTHREAD_UNLOCK(curthread);
-	COND_UNLOCK(*cond);
-	mtxrval = _mutex_cv_lock(mutex);
+		THR_LOCK_ACQUIRE(curthread, &cv->c_lock);
+		seq = cv->c_seqno;
+		if (abstime != NULL && ret == ETIMEDOUT)
+			break;
 
-	/* If the mutex failed return that error. */
-	if (mtxrval == -1) {
-		printf("mutex lock from condvar failed!");
-		fflush(stdout);
-		abort();
+		/*
+		 * loop if we have never been told to wake up
+		 * or we lost a race.
+		 */
+	} while (seq == oldseq || cv->c_wakeups == 0);
+	
+	if (seq != oldseq && cv->c_wakeups != 0) {
+		cv->c_wakeups--;
+		ret = 0;
+	} else {
+		cv->c_waiters--;
 	}
-	if (mtxrval != 0)
-		rval = mtxrval;
-
-	_thread_leave_cancellation_point();
-	return (rval);
+	THR_LOCK_RELEASE(curthread, &cv->c_lock);
+	_mutex_cv_lock(mutex);
+	return (ret);
 }
 
 int
-_pthread_cond_signal(pthread_cond_t * cond)
+_pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
 {
-	return (cond_signal(cond, 0));
+
+	return (cond_wait_common(cond, mutex, NULL, 0));
 }
 
 int
-_pthread_cond_broadcast(pthread_cond_t * cond)
+__pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
 {
-	return (cond_signal(cond, 1));
+
+	return (cond_wait_common(cond, mutex, NULL, 1));
 }
 
-static int
-cond_signal(pthread_cond_t * cond, int broadcast)
+int
+_pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
+		       const struct timespec * abstime)
 {
-	int             rval = 0;
-	pthread_t       pthread;
 
-	if (cond == NULL)
-		return (EINVAL);
-       /*
-        * If the condition variable is statically initialized, perform dynamic
-        * initialization.
-        */
-	if (*cond == PTHREAD_COND_INITIALIZER && (rval = cond_init(cond)) != 0)
-		return (rval);
-
-	if ((*cond)->c_type != COND_TYPE_FAST)
+	if (abstime == NULL || abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
+	    abstime->tv_nsec >= 1000000000)
 		return (EINVAL);
-	COND_LOCK(*cond);
 
-	/*
-	 * Enter a loop to bring all (or only one) threads off the
-	 * condition queue:
-	 */
-	do {
-		/*
-		 * Wake up the signaled thread. It will be returned
-		 * to us locked.
-		 */
-		if ((pthread = cond_queue_deq(*cond)) != NULL) {
-			PTHREAD_WAKE(pthread);
-			PTHREAD_UNLOCK(pthread);
-		}
-	} while (broadcast && pthread != NULL);
-
-	COND_UNLOCK(*cond);
-	return (rval);
+	return (cond_wait_common(cond, mutex, abstime, 0));
 }
 
-void
-_cond_wait_backout(pthread_t pthread)
-{
-	pthread_cond_t	cond;
-
-	cond = pthread->data.cond;
-	if (cond == NULL)
-		return;
-
-	/* Process according to condition variable type: */
-	switch (cond->c_type) {
-	/* Fast condition variable: */
-	case COND_TYPE_FAST:
-		cond_queue_remove(cond, pthread);
-		break;
-	default:
-		break;
-	}
-}
-
-/*
- * Dequeue a waiting thread from the head of a condition queue in
- * descending priority order.
- */
-static pthread_t
-cond_queue_deq(pthread_cond_t cond)
+int
+__pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex,
+		       const struct timespec *abstime)
 {
-	pthread_t pthread;
 
-	while ((pthread = TAILQ_FIRST(&cond->c_queue)) != NULL) {
-		PTHREAD_LOCK(pthread);
-		cond_queue_remove(cond, pthread);
-
-		/*
-		 * Only exit the loop when we find a thread
-		 * that hasn't been canceled.
-		 */
-		if (pthread->cancellation == CS_NULL)
-			break;
-		else
-			PTHREAD_UNLOCK(pthread);
-	}
+	if (abstime == NULL || abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
+	    abstime->tv_nsec >= 1000000000)
+		return (EINVAL);
 
-	return(pthread);
+	return (cond_wait_common(cond, mutex, abstime, 1));
 }
 
-/*
- * Remove a waiting thread from a condition queue in descending priority
- * order.
- */
-static void
-cond_queue_remove(pthread_cond_t cond, pthread_t pthread)
+static int
+cond_signal_common(pthread_cond_t *cond, int broadcast)
 {
+	struct pthread	*curthread = _get_curthread();
+	pthread_cond_t	cv;
+	int		ret = 0, oldwaiters;
+
 	/*
-	 * Because pthread_cond_timedwait() can timeout as well
-	 * as be signaled by another thread, it is necessary to
-	 * guard against removing the thread from the queue if
-	 * it isn't in the queue.
+	 * If the condition variable is statically initialized, perform dynamic
+	 * initialization.
 	 */
-	if (pthread->flags & PTHREAD_FLAGS_IN_CONDQ) {
-		TAILQ_REMOVE(&cond->c_queue, pthread, sqe);
-		pthread->flags &= ~PTHREAD_FLAGS_IN_CONDQ;
+	if (__predict_false(*cond == NULL &&
+	    (ret = init_static(curthread, cond)) != 0))
+		return (ret);
+
+	cv = *cond;
+	/* Lock the condition variable structure. */
+	THR_LOCK_ACQUIRE(curthread, &cv->c_lock);
+	if (cv->c_waiters) {
+		if (!broadcast) {
+			cv->c_wakeups++;
+			cv->c_waiters--;
+			cv->c_seqno++;
+			_thr_umtx_wake(&cv->c_seqno, 1);
+		} else {
+			oldwaiters = cv->c_waiters;
+			cv->c_wakeups += cv->c_waiters;
+			cv->c_waiters = 0;
+			cv->c_seqno++;
+			_thr_umtx_wake(&cv->c_seqno, oldwaiters);
+		}
 	}
-	/* Check for no more waiters. */
-	if (TAILQ_FIRST(&cond->c_queue) == NULL)
-		cond->c_mutex = NULL;
+	THR_LOCK_RELEASE(curthread, &cv->c_lock);
+	return (ret);
 }
 
-/*
- * Enqueue a waiting thread to a condition queue in descending priority
- * order.
- */
-static void
-cond_queue_enq(pthread_cond_t cond, pthread_t pthread)
+int
+_pthread_cond_signal(pthread_cond_t * cond)
 {
-	pthread_t tid = TAILQ_LAST(&cond->c_queue, cond_head);
-	char *name;
-
-	name = pthread->name ? pthread->name : "unknown";
-	if ((pthread->flags & PTHREAD_FLAGS_IN_CONDQ) != 0)
-		_thread_printf(2, "Thread (%s:%ld) already on condq\n",
-		    pthread->name, pthread->thr_id);
-	if ((pthread->flags & PTHREAD_FLAGS_IN_MUTEXQ) != 0)
-		_thread_printf(2, "Thread (%s:%ld) already on mutexq\n",
-		    pthread->name, pthread->thr_id);
-	PTHREAD_ASSERT_NOT_IN_SYNCQ(pthread);
 
-	/*
-	 * For the common case of all threads having equal priority,
-	 * we perform a quick check against the priority of the thread
-	 * at the tail of the queue.
-	 */
-	if ((tid == NULL) || (pthread->active_priority <= tid->active_priority))
-		TAILQ_INSERT_TAIL(&cond->c_queue, pthread, sqe);
-	else {
-		tid = TAILQ_FIRST(&cond->c_queue);
-		while (pthread->active_priority <= tid->active_priority)
-			tid = TAILQ_NEXT(tid, sqe);
-		TAILQ_INSERT_BEFORE(tid, pthread, sqe);
-	}
-	pthread->flags |= PTHREAD_FLAGS_IN_CONDQ;
-	pthread->data.cond = cond;
+	return (cond_signal_common(cond, 0));
 }
 
-static inline int
-cond_init(pthread_cond_t *cond)
+int
+_pthread_cond_broadcast(pthread_cond_t * cond)
 {
-	int error = 0;
-	_SPINLOCK(&static_cond_lock);
-	if (*cond == PTHREAD_COND_INITIALIZER)
-		error = _pthread_cond_init(cond, NULL);
-	_SPINUNLOCK(&static_cond_lock);
-	return (error);
-}
 
+	return (cond_signal_common(cond, 1));
+}