Use UMTX_OP_WAIT_UINT_PRIVATE and UMTX_OP_WAKE_PRIVATE to save

time in kernel(avoid VM lookup).

9 files changed, 20 insertions, 19 deletions

diff --git a/lib/libthr/thread/thr_create.c b/lib/libthr/thread/thr_create.c
index 018000a..f73a6c9 100644
--- a/lib/libthr/thread/thr_create.c
+++ b/lib/libthr/thread/thr_create.c
@@ -192,7 +192,7 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 		new_thread->tid = TID_TERMINATED;
 		if (new_thread->flags & THR_FLAGS_NEED_SUSPEND) {
 			new_thread->cycle++;
-			_thr_umtx_wake(&new_thread->cycle, INT_MAX);
+			_thr_umtx_wake(&new_thread->cycle, INT_MAX, 0);
 		}
 		THR_THREAD_UNLOCK(curthread, new_thread);
 		THREAD_LIST_LOCK(curthread);
diff --git a/lib/libthr/thread/thr_exit.c b/lib/libthr/thread/thr_exit.c
index b78be1f..df0170a 100644
--- a/lib/libthr/thread/thr_exit.c
+++ b/lib/libthr/thread/thr_exit.c
@@ -125,7 +125,7 @@ _pthread_exit(void *status)
 	curthread->state = PS_DEAD;
 	if (curthread->flags & THR_FLAGS_NEED_SUSPEND) {
 		curthread->cycle++;
-		_thr_umtx_wake(&curthread->cycle, INT_MAX);
+		_thr_umtx_wake(&curthread->cycle, INT_MAX, 0);
 	}
 	THR_UNLOCK(curthread);
 	/*
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index 68b5d53..dbbdda4 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -315,7 +315,7 @@ struct pthread {
 	struct umutex		lock;
 
 	/* Internal condition variable cycle number. */
-	long			cycle;
+	uint32_t		cycle;
 
 	/* How many low level locks the thread held. */
 	int			locklevel;
diff --git a/lib/libthr/thread/thr_resume_np.c b/lib/libthr/thread/thr_resume_np.c
index b477a66..50b561a 100644
--- a/lib/libthr/thread/thr_resume_np.c
+++ b/lib/libthr/thread/thr_resume_np.c
@@ -87,5 +87,5 @@ resume_common(struct pthread *thread)
 	/* Clear the suspend flag: */
 	thread->flags &= ~THR_FLAGS_NEED_SUSPEND;
 	thread->cycle++;
-	_thr_umtx_wake(&thread->cycle, 1);
+	_thr_umtx_wake(&thread->cycle, 1, 0);
 }
diff --git a/lib/libthr/thread/thr_sem.c b/lib/libthr/thread/thr_sem.c
index ada158c..0dd5599 100644
--- a/lib/libthr/thread/thr_sem.c
+++ b/lib/libthr/thread/thr_sem.c
@@ -179,7 +179,7 @@ sem_cancel_handler(void *arg)
 
 	atomic_add_int(&(*sem)->nwaiters, -1);
 	if ((*sem)->nwaiters && (*sem)->count)
-		_thr_umtx_wake(&(*sem)->count, 1);
+		_thr_umtx_wake(&(*sem)->count, 1, 0);
 }
 
 int
@@ -208,7 +208,7 @@ _sem_wait(sem_t *sem)
 		atomic_add_int(&(*sem)->nwaiters, 1);
 		THR_CLEANUP_PUSH(curthread, sem_cancel_handler, sem);
 		_thr_cancel_enter(curthread);
-		retval = _thr_umtx_wait_uint(&(*sem)->count, 0, NULL);
+		retval = _thr_umtx_wait_uint(&(*sem)->count, 0, NULL, 0);
 		_thr_cancel_leave(curthread);
 		THR_CLEANUP_POP(curthread, 0);
 		atomic_add_int(&(*sem)->nwaiters, -1);
@@ -255,7 +255,7 @@ _sem_timedwait(sem_t * __restrict sem,
 		atomic_add_int(&(*sem)->nwaiters, 1);
 		THR_CLEANUP_PUSH(curthread, sem_cancel_handler, sem);
 		_thr_cancel_enter(curthread);
-		retval = _thr_umtx_wait_uint(&(*sem)->count, 0, &ts2);
+		retval = _thr_umtx_wait_uint((uint32_t*)&(*sem)->count, 0, &ts2, 0);
 		_thr_cancel_leave(curthread);
 		THR_CLEANUP_POP(curthread, 0);
 		atomic_add_int(&(*sem)->nwaiters, -1);
@@ -283,7 +283,7 @@ _sem_post(sem_t *sem)
 	atomic_add_rel_int(&(*sem)->count, 1);
 
 	if ((*sem)->nwaiters) {
-		retval = _thr_umtx_wake(&(*sem)->count, 1);
+		retval = _thr_umtx_wake(&(*sem)->count, 1, 0);
 		if (retval != 0)
 			retval = -1;
 	}
diff --git a/lib/libthr/thread/thr_sig.c b/lib/libthr/thread/thr_sig.c
index fe1555a..9615b06 100644
--- a/lib/libthr/thread/thr_sig.c
+++ b/lib/libthr/thread/thr_sig.c
@@ -89,7 +89,7 @@ _thr_ast(struct pthread *curthread)
 void
 _thr_suspend_check(struct pthread *curthread)
 {
-	long cycle;
+	uint32_t cycle;
 	int err;
 
 	if (curthread->force_exit)
@@ -114,7 +114,7 @@ _thr_suspend_check(struct pthread *curthread)
 		cycle = curthread->cycle;
 
 		/* Wake the thread suspending us. */
-		_thr_umtx_wake(&curthread->cycle, INT_MAX);
+		_thr_umtx_wake(&curthread->cycle, INT_MAX, 0);
 
 		/*
 		 * if we are from pthread_exit, we don't want to
@@ -124,7 +124,7 @@ _thr_suspend_check(struct pthread *curthread)
 			break;
 		curthread->flags |= THR_FLAGS_SUSPENDED;
 		THR_UMUTEX_UNLOCK(curthread, &(curthread)->lock);
-		_thr_umtx_wait(&curthread->cycle, cycle, NULL);
+		_thr_umtx_wait_uint(&curthread->cycle, cycle, NULL, 0);
 		THR_UMUTEX_LOCK(curthread, &(curthread)->lock);
 		curthread->flags &= ~THR_FLAGS_SUSPENDED;
 	}
diff --git a/lib/libthr/thread/thr_suspend_np.c b/lib/libthr/thread/thr_suspend_np.c
index 3e48656..7d4a2e2 100644
--- a/lib/libthr/thread/thr_suspend_np.c
+++ b/lib/libthr/thread/thr_suspend_np.c
@@ -130,7 +130,7 @@ suspend_common(struct pthread *curthread, struct pthread *thread,
 		THR_THREAD_UNLOCK(curthread, thread);
 		_thr_send_sig(thread, SIGCANCEL);
 		if (waitok) {
-			_thr_umtx_wait(&thread->cycle, tmp, NULL);
+			_thr_umtx_wait_uint(&thread->cycle, tmp, NULL, 0);
 			THR_THREAD_LOCK(curthread, thread);
 		} else {
 			THR_THREAD_LOCK(curthread, thread);
diff --git a/lib/libthr/thread/thr_umtx.c b/lib/libthr/thread/thr_umtx.c
index ff8c60e..3efe884 100644
--- a/lib/libthr/thread/thr_umtx.c
+++ b/lib/libthr/thread/thr_umtx.c
@@ -94,19 +94,20 @@ _thr_umtx_wait(volatile long *mtx, long id, const struct timespec *timeout)
 }
 
 int
-_thr_umtx_wait_uint(volatile u_int *mtx, u_int id, const struct timespec *timeout)
+_thr_umtx_wait_uint(volatile u_int *mtx, u_int id, const struct timespec *timeout, int shared)
 {
 	if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 &&
 		timeout->tv_nsec <= 0)))
 		return (ETIMEDOUT);
-	return _umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAIT_UINT, id, 0,
-		__DECONST(void*, timeout));
+	return _umtx_op_err(__DEVOLATILE(void *, mtx), 
+			shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 0,
+			__DECONST(void*, timeout));
 }
 
 int
-_thr_umtx_wake(volatile void *mtx, int nr_wakeup)
+_thr_umtx_wake(volatile void *mtx, int nr_wakeup, int shared)
 {
-	return _umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAKE,
+	return _umtx_op_err(__DEVOLATILE(void *, mtx), shared ? UMTX_OP_WAKE : UMTX_OP_WAKE_PRIVATE,
 		nr_wakeup, 0, 0);
 }
 
diff --git a/lib/libthr/thread/thr_umtx.h b/lib/libthr/thread/thr_umtx.h
index 752e7b2..0ef75a7 100644
--- a/lib/libthr/thread/thr_umtx.h
+++ b/lib/libthr/thread/thr_umtx.h
@@ -46,8 +46,8 @@ void _thr_umutex_init(struct umutex *mtx) __hidden;
 int _thr_umtx_wait(volatile long *mtx, long exp,
 	const struct timespec *timeout) __hidden;
 int _thr_umtx_wait_uint(volatile u_int *mtx, u_int exp,
-	const struct timespec *timeout) __hidden;
-int _thr_umtx_wake(volatile void *mtx, int count) __hidden;
+	const struct timespec *timeout, int shared) __hidden;
+int _thr_umtx_wake(volatile void *mtx, int count, int shared) __hidden;
 int _thr_ucond_wait(struct ucond *cv, struct umutex *m,
         const struct timespec *timeout, int check_unpaking) __hidden;
 void _thr_ucond_init(struct ucond *cv) __hidden;