Add locking support for rtld.

4 files changed, 229 insertions, 4 deletions

diff --git a/lib/libthr/thread/thr_kern.c b/lib/libthr/thread/thr_kern.c
index 4c451f1..aa7dce6 100644
--- a/lib/libthr/thread/thr_kern.c
+++ b/lib/libthr/thread/thr_kern.c
@@ -51,13 +51,11 @@ _thr_setthreaded(int threaded)
 		return (0);
 
 	__isthreaded = threaded;
-#if 0
 	if (threaded != 0) {
 		_thr_rtld_init();
 	} else {
 		_thr_rtld_fini();
 	}
-#endif
 	return (0);
 }
 
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index 60be095..df41c9e 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -509,6 +509,13 @@ struct pthread {
 	(((thrd)->locklevel > 0) ||			\
 	((thrd)->critical_count > 0))
 
+#define	THR_CRITICAL_ENTER(thrd)			\
+	(thrd)->critical_count++
+
+#define	THR_CRITICAL_LEAVE(thrd)			\
+	(thrd)->critical_count--;			\
+	_thr_ast(thrd);
+
 #define THR_UMTX_TRYLOCK(thrd, lck)			\
 	_thr_umtx_trylock((lck), (thrd)->tid)
 
diff --git a/lib/libthr/thread/thr_rtld.c b/lib/libthr/thread/thr_rtld.c
new file mode 100644
index 0000000..b1ba384
--- /dev/null
+++ b/lib/libthr/thread/thr_rtld.c
@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) 2006, David Xu <davidxu@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ *
+ */
+
+ /*
+  * A lockless rwlock for rtld.
+  */
+#include <sys/cdefs.h>
+#include <stdlib.h>
+
+#include "rtld_lock.h"
+#include "thr_private.h"
+
+#define CACHE_LINE_SIZE		64
+#define WAFLAG			0x1
+#define RC_INCR			0x2
+
+static int	_thr_rtld_clr_flag(int);
+static void	*_thr_rtld_lock_create(void);
+static void	_thr_rtld_lock_destroy(void *);
+static void	_thr_rtld_lock_release(void *);
+static void	_thr_rtld_rlock_acquire(void *);
+static int	_thr_rtld_set_flag(int);
+static void	_thr_rtld_wlock_acquire(void *);
+
+struct rtld_lock {
+	volatile int		lock;
+	volatile int		rd_waiters;
+	volatile int		wr_waiters;
+	volatile umtx_t		rd_cv;
+	volatile umtx_t		wr_cv;
+	void			*base;
+};
+
+static void *
+_thr_rtld_lock_create(void)
+{
+	void			*base;
+	char			*p;
+	uintptr_t		r;
+	struct rtld_lock	*l;
+
+	THR_ASSERT(sizeof(struct rtld_lock) <= CACHE_LINE_SIZE,
+		"rtld_lock too large");
+	base = calloc(1, CACHE_LINE_SIZE);
+	p = (char *)base;
+	if ((uintptr_t)p % CACHE_LINE_SIZE != 0) {
+		free(base);
+		base = calloc(1, 2 * CACHE_LINE_SIZE);
+		p = (char *)base;
+		if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0)
+			p += CACHE_LINE_SIZE - r;
+	}
+	l = (struct rtld_lock *)p;
+	l->base = base;
+	return (l);
+}
+
+static void
+_thr_rtld_lock_destroy(void *lock)
+{
+	struct rtld_lock *l = (struct rtld_lock *)lock;
+	free(l->base);
+}
+
+static void
+_thr_rtld_rlock_acquire(void *lock)
+{
+	struct pthread		*curthread;
+	struct rtld_lock	*l;
+	umtx_t			v;
+
+	curthread = _get_curthread();
+	l = (struct rtld_lock *)lock;
+
+	THR_CRITICAL_ENTER(curthread);
+	atomic_add_acq_int(&l->lock, RC_INCR);
+	if (!(l->lock & WAFLAG))
+		return;
+	v = l->rd_cv;
+	atomic_add_int(&l->rd_waiters, 1);
+	while (l->lock & WAFLAG) {
+		_thr_umtx_wait(&l->rd_cv, v, NULL);
+		v = l->rd_cv;
+	}
+	atomic_add_int(&l->rd_waiters, -1);
+}
+
+static void
+_thr_rtld_wlock_acquire(void *lock)
+{
+	struct pthread		*curthread;
+	struct rtld_lock	*l;
+	umtx_t			v;
+
+	curthread = _get_curthread();
+	l = (struct rtld_lock *)lock;
+
+	_thr_signal_block(curthread);
+	for (;;) {
+		if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG))
+			return;
+		v = l->wr_cv;
+		atomic_add_int(&l->wr_waiters, 1);
+		while (l->lock != 0) {
+			_thr_umtx_wait(&l->wr_cv, v, NULL);
+			v = l->wr_cv;
+		}
+		atomic_add_int(&l->wr_waiters, -1);
+	}
+}
+
+static void
+_thr_rtld_lock_release(void *lock)
+{
+	struct pthread		*curthread;
+	struct rtld_lock	*l;
+
+	curthread = _get_curthread();
+	l = (struct rtld_lock *)lock;
+	
+	if ((l->lock & WAFLAG) == 0) {
+		atomic_add_rel_int(&l->lock, -RC_INCR);
+		if (l->wr_waiters) {
+			atomic_add_long(&l->wr_cv, 1);
+			_thr_umtx_wake(&l->wr_cv, l->wr_waiters);
+		}
+		THR_CRITICAL_LEAVE(curthread);
+	} else {
+		atomic_add_rel_int(&l->lock, -WAFLAG);
+		if (l->wr_waiters) {
+			atomic_add_long(&l->wr_cv, 1);
+			_thr_umtx_wake(&l->wr_cv, l->wr_waiters);
+		} else if (l->rd_waiters) {
+			atomic_add_long(&l->rd_cv, 1);
+			_thr_umtx_wake(&l->rd_cv, l->rd_waiters);
+		}
+		_thr_signal_unblock(curthread);
+	}
+}
+
+static int
+_thr_rtld_set_flag(int mask)
+{
+	/*
+	 * The caller's code in rtld-elf is broken, it is not signal safe,
+	 * just return zero to fool it.
+	 */
+	return (0);
+}
+
+static int
+_thr_rtld_clr_flag(int mask)
+{
+	return (0);
+}
+
+void
+_thr_rtld_init(void)
+{
+	struct RtldLockInfo	li;
+	struct pthread		*curthread;
+	umtx_t dummy;
+
+	curthread = _get_curthread();
+
+	/* force to resolve _umtx_op PLT */
+	_umtx_op((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0);
+
+	li.lock_create  = _thr_rtld_lock_create;
+	li.lock_destroy = _thr_rtld_lock_destroy;
+	li.rlock_acquire = _thr_rtld_rlock_acquire;
+	li.wlock_acquire = _thr_rtld_wlock_acquire;
+	li.lock_release  = _thr_rtld_lock_release;
+	li.thread_set_flag = _thr_rtld_set_flag;
+	li.thread_clr_flag = _thr_rtld_clr_flag;
+	li.at_fork = NULL;
+	
+	/* mask signals, also force to resolve __sys_sigprocmask PLT */
+	_thr_signal_block(curthread);
+	_rtld_thread_init(&li);
+	_thr_signal_unblock(curthread);
+}
+
+void
+_thr_rtld_fini(void)
+{
+	struct pthread	*curthread;
+
+	curthread = _get_curthread();
+	_thr_signal_block(curthread);
+	_rtld_thread_init(NULL);
+	_thr_signal_unblock(curthread);
+}
diff --git a/lib/libthr/thread/thr_sig.c b/lib/libthr/thread/thr_sig.c
index 0924646..335fa09 100644
--- a/lib/libthr/thread/thr_sig.c
+++ b/lib/libthr/thread/thr_sig.c
@@ -50,8 +50,6 @@ sigcancel_handler(int sig, siginfo_t *info, ucontext_t *ucp)
 {
 	struct pthread *curthread = _get_curthread();
 
-	if (curthread->cancelflags & THR_CANCEL_AT_POINT)
-		pthread_testcancel();
 	_thr_ast(curthread);
 }
 
@@ -59,6 +57,9 @@ void
 _thr_ast(struct pthread *curthread)
 {
 	if (!THR_IN_CRITICAL(curthread)) {
+		if (__predict_false(curthread->cancelflags &
+			THR_CANCEL_AT_POINT))
+			_pthread_testcancel();
 		if (__predict_false((curthread->flags &
 		    (THR_FLAGS_NEED_SUSPEND | THR_FLAGS_SUSPENDED))
 			== THR_FLAGS_NEED_SUSPEND))