Import my recent 1:1 threading working. some features improved includes:

 1. fast simple type mutex.
 2. __thread tls works.
 3. asynchronous cancellation works ( using signal ).
 4. thread synchronization is fully based on umtx, mainly, condition
    variable and other synchronization objects were rewritten by using
    umtx directly. those objects can be shared between processes via
    shared memory, it has to change ABI which does not happen yet.
 5. default stack size is increased to 1M on 32 bits platform, 2M for
    64 bits platform.
As the result, some mysql super-smack benchmarks show performance is
improved massivly.

Okayed by: jeff, mtm, rwatson, scottl

1 files changed, 173 insertions, 159 deletions

diff --git a/lib/libthr/thread/thr_sem.c b/lib/libthr/thread/thr_sem.c
index c634034..bd952ef 100644
--- a/lib/libthr/thread/thr_sem.c
+++ b/lib/libthr/thread/thr_sem.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (C) 2005 David Xu <davidxu@freebsd.org>.
  * Copyright (C) 2000 Jason Evans <jasone@freebsd.org>.
  * All rights reserved.
  * 
@@ -29,227 +30,240 @@
  * $FreeBSD$
  */
 
-#include <stdlib.h>
+#include "namespace.h"
+#include <sys/queue.h>
 #include <errno.h>
-#include <semaphore.h>
+#include <fcntl.h>
 #include <pthread.h>
+#include <semaphore.h>
+#include <stdlib.h>
+#include <time.h>
+#include <_semaphore.h>
+#include "un-namespace.h"
+
 #include "thr_private.h"
 
-#define _SEM_CHECK_VALIDITY(sem)		\
-	if ((*(sem))->magic != SEM_MAGIC) {	\
-		errno = EINVAL;			\
-		retval = -1;			\
-		goto RETURN;			\
-	}
 
 __weak_reference(_sem_init, sem_init);
 __weak_reference(_sem_destroy, sem_destroy);
-__weak_reference(_sem_open, sem_open);
-__weak_reference(_sem_close, sem_close);
-__weak_reference(_sem_unlink, sem_unlink);
-__weak_reference(_sem_wait, sem_wait);
+__weak_reference(_sem_getvalue, sem_getvalue);
 __weak_reference(_sem_trywait, sem_trywait);
+__weak_reference(_sem_wait, sem_wait);
+__weak_reference(_sem_timedwait, sem_timedwait);
 __weak_reference(_sem_post, sem_post);
-__weak_reference(_sem_getvalue, sem_getvalue);
 
 
-int
-_sem_init(sem_t *sem, int pshared, unsigned int value)
+static inline int
+sem_check_validity(sem_t *sem)
 {
-	int	retval;
 
-	/*
-	 * Range check the arguments.
-	 */
-	if (pshared != 0) {
-		/*
-		 * The user wants a semaphore that can be shared among
-		 * processes, which this implementation can't do.  Sounds like a
-		 * permissions problem to me (yeah right).
-		 */
-		errno = EPERM;
-		retval = -1;
-		goto RETURN;
+	if ((sem != NULL) && ((*sem)->magic == SEM_MAGIC))
+		return (0);
+	else {
+		errno = EINVAL;
+		return (-1);
 	}
+}
+
+static sem_t
+sem_alloc(unsigned int value, semid_t semid, int system_sem)
+{
+	sem_t sem;
 
 	if (value > SEM_VALUE_MAX) {
 		errno = EINVAL;
-		retval = -1;
-		goto RETURN;
+		return (NULL);
 	}
 
-	*sem = (sem_t)malloc(sizeof(struct sem));
-	if (*sem == NULL) {
+	sem = (sem_t)malloc(sizeof(struct sem));
+	if (sem == NULL) {
 		errno = ENOSPC;
-		retval = -1;
-		goto RETURN;
+		return (NULL);
 	}
-
+	_thr_umtx_init((umtx_t *)&sem->lock);
 	/*
-	 * Initialize the semaphore.
+	 * Fortunatly count and nwaiters are adjacency, so we can
+	 * use umtx_wait to wait on it, umtx_wait needs an address
+	 * can be accessed as a long interger.
 	 */
-	if (pthread_mutex_init(&(*sem)->lock, NULL) != 0) {
-		free(*sem);
-		errno = ENOSPC;
-		retval = -1;
-		goto RETURN;
-	}
-
-	if (pthread_cond_init(&(*sem)->gtzero, NULL) != 0) {
-		pthread_mutex_destroy(&(*sem)->lock);
-		free(*sem);
-		errno = ENOSPC;
-		retval = -1;
-		goto RETURN;
-	}
-	
-	(*sem)->count = (u_int32_t)value;
-	(*sem)->nwaiters = 0;
-	(*sem)->magic = SEM_MAGIC;
-
-	retval = 0;
-  RETURN:
-	return retval;
+	sem->count = (u_int32_t)value;
+	sem->nwaiters = 0;
+	sem->magic = SEM_MAGIC;
+	sem->semid = semid;
+	sem->syssem = system_sem;
+	return (sem);
 }
 
 int
-_sem_destroy(sem_t *sem)
+_sem_init(sem_t *sem, int pshared, unsigned int value)
 {
-	int	retval;
-	
-	_SEM_CHECK_VALIDITY(sem);
-
-	/* Make sure there are no waiters. */
-	pthread_mutex_lock(&(*sem)->lock);
-	if ((*sem)->nwaiters > 0) {
-		pthread_mutex_unlock(&(*sem)->lock);
-		errno = EBUSY;
-		retval = -1;
-		goto RETURN;
-	}
-	pthread_mutex_unlock(&(*sem)->lock);
-	
-	pthread_mutex_destroy(&(*sem)->lock);
-	pthread_cond_destroy(&(*sem)->gtzero);
-	(*sem)->magic = 0;
+	semid_t semid;
 
-	free(*sem);
+	semid = (semid_t)SEM_USER;
+	if ((pshared != 0) && (ksem_init(&semid, value) != 0))
+		return (-1);
 
-	retval = 0;
-  RETURN:
-	return retval;
-}
-
-sem_t *
-_sem_open(const char *name, int oflag, ...)
-{
-	errno = ENOSYS;
-	return SEM_FAILED;
+	(*sem) = sem_alloc(value, semid, pshared);
+	if ((*sem) == NULL) {
+		if (pshared != 0)
+			ksem_destroy(semid);
+		return (-1);
+	}
+	return (0);
 }
 
 int
-_sem_close(sem_t *sem)
+_sem_destroy(sem_t *sem)
 {
-	errno = ENOSYS;
-	return -1;
-}
+	int retval;
 
-int
-_sem_unlink(const char *name)
-{
-	errno = ENOSYS;
-	return -1;
+	if (sem_check_validity(sem) != 0)
+		return (-1);
+
+	/*
+	 * If this is a system semaphore let the kernel track it otherwise
+	 * make sure there are no waiters.
+	 */
+	if ((*sem)->syssem != 0)
+		retval = ksem_destroy((*sem)->semid);
+	else {
+		retval = 0;
+		(*sem)->magic = 0;
+	}
+	if (retval == 0)
+		free(*sem);
+	return (retval);
 }
 
 int
-_sem_wait(sem_t *sem)
+_sem_getvalue(sem_t * __restrict sem, int * __restrict sval)
 {
-	int	retval;
+	int retval;
 
-	_thread_enter_cancellation_point();
-	
-	_SEM_CHECK_VALIDITY(sem);
-
-	pthread_mutex_lock(&(*sem)->lock);
+	if (sem_check_validity(sem) != 0)
+		return (-1);
 
-	while ((*sem)->count == 0) {
-		(*sem)->nwaiters++;
-		pthread_cond_wait(&(*sem)->gtzero, &(*sem)->lock);
-		(*sem)->nwaiters--;
+	if ((*sem)->syssem != 0)
+		retval = ksem_getvalue((*sem)->semid, sval);
+        else {
+		*sval = (int)(*sem)->count;
+		retval = 0;
 	}
-	(*sem)->count--;
-
-	pthread_mutex_unlock(&(*sem)->lock);
-
-	retval = 0;
-  RETURN:
-	_thread_leave_cancellation_point();
-	return retval;
+	return (retval);
 }
 
 int
 _sem_trywait(sem_t *sem)
 {
-	int	retval;
+	int val;
 
-	_SEM_CHECK_VALIDITY(sem);
+	if (sem_check_validity(sem) != 0)
+		return (-1);
 
-	pthread_mutex_lock(&(*sem)->lock);
+	if ((*sem)->syssem != 0)
+ 		return (ksem_trywait((*sem)->semid));
 
-	if ((*sem)->count > 0) {
-		(*sem)->count--;
-		retval = 0;
-	} else {
-		errno = EAGAIN;
-		retval = -1;
+	while ((val = (*sem)->count) > 0) {
+		if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1))
+			return (0);
 	}
-	
-	pthread_mutex_unlock(&(*sem)->lock);
-
-  RETURN:
-	return retval;
+	errno = EAGAIN;
+	return (-1);
 }
 
 int
-_sem_post(sem_t *sem)
+_sem_wait(sem_t *sem)
 {
-	int	retval;
+	struct pthread *curthread;
+	int val, oldcancel, retval;
+
+	if (sem_check_validity(sem) != 0)
+		return (-1);
+
+	curthread = _get_curthread();
+	if ((*sem)->syssem != 0) {
+		oldcancel = _thr_cancel_enter(curthread);
+		retval = ksem_wait((*sem)->semid);
+		_thr_cancel_leave(curthread, oldcancel);
+		return (retval);
+	}
 
-	_SEM_CHECK_VALIDITY(sem);
+	_pthread_testcancel();
+	do {
+		while ((val = (*sem)->count) > 0) {
+			if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1))
+				return (0);
+		}
+		oldcancel = _thr_cancel_enter(curthread);
+		retval = _thr_umtx_wait((umtx_t *)&(*sem)->count, 0, NULL);
+		_thr_cancel_leave(curthread, oldcancel);
+	} while (retval == 0);
+	errno = retval;
+	return (-1);
+}
+
+int
+_sem_timedwait(sem_t * __restrict sem, struct timespec * __restrict abstime)
+{
+	struct timespec ts, ts2;
+	struct pthread *curthread;
+	int val, oldcancel, retval;
+
+	if (sem_check_validity(sem) != 0)
+		return (-1);
+
+	curthread = _get_curthread();
+	if ((*sem)->syssem != 0) {
+		oldcancel = _thr_cancel_enter(curthread);
+		retval = ksem_timedwait((*sem)->semid, abstime);
+		_thr_cancel_leave(curthread, oldcancel);
+		return (retval);
+	}
 
 	/*
-	 * sem_post() is required to be safe to call from within signal
-	 * handlers.  Thus, we must defer signals.
+	 * The timeout argument is only supposed to
+	 * be checked if the thread would have blocked.
 	 */
-	pthread_mutex_lock(&(*sem)->lock);
-
-	/* GIANT_LOCK(curthread); */
-
-	(*sem)->count++;
-	if ((*sem)->nwaiters > 0)
-		pthread_cond_signal(&(*sem)->gtzero);
-
-	/* GIANT_UNLOCK(curthread); */
-
-	pthread_mutex_unlock(&(*sem)->lock);
-
-	retval = 0;
-  RETURN:
-	return retval;
+	_pthread_testcancel();
+	do {
+		while ((val = (*sem)->count) > 0) {
+			if (atomic_cmpset_acq_int(&(*sem)->count, val, val - 1))
+				return (0);
+		}
+		if (abstime == NULL) {
+			errno = EINVAL;
+			return (-1);
+		}
+		clock_gettime(CLOCK_REALTIME, &ts);
+		TIMESPEC_SUB(&ts2, abstime, &ts);
+		oldcancel = _thr_cancel_enter(curthread);
+		retval = _thr_umtx_wait((umtx_t *)&(*sem)->count, 0, &ts2);
+		_thr_cancel_leave(curthread, oldcancel);
+	} while (retval == 0);
+	errno = retval;
+	return (-1);
 }
 
 int
-_sem_getvalue(sem_t *sem, int *sval)
+_sem_post(sem_t *sem)
 {
-	int	retval;
-
-	_SEM_CHECK_VALIDITY(sem);
+	int val, retval;
+	
+	if (sem_check_validity(sem) != 0)
+		return (-1);
 
-	pthread_mutex_lock(&(*sem)->lock);
-	*sval = (int)(*sem)->count;
-	pthread_mutex_unlock(&(*sem)->lock);
+	if ((*sem)->syssem != 0)
+		return (ksem_post((*sem)->semid));
 
-	retval = 0;
-  RETURN:
-	return retval;
+	/*
+	 * sem_post() is required to be safe to call from within
+	 * signal handlers, these code should work as that.
+	 */
+	do {
+		val = (*sem)->count;
+	} while (!atomic_cmpset_acq_int(&(*sem)->count, val, val + 1));
+	retval = _thr_umtx_wake((umtx_t *)&(*sem)->count, val + 1);
+	if (retval > 0)
+		retval = 0;
+	return (retval);
 }