Attempt to eliminate PLT relocations from rwlock aquire/release

path, making them suitable for direct use by the dynamic loader.

Register libpthread-specific locking API with rtld on startup.

This still has some rough edges with signals which should be
addresses later.

Approved by:	re (scottl)

12 files changed, 263 insertions, 21 deletions

diff --git a/lib/libkse/Makefile b/lib/libkse/Makefile
index d5e51e5..c25ccf4 100644
--- a/lib/libkse/Makefile
+++ b/lib/libkse/Makefile
@@ -14,12 +14,16 @@ CFLAGS+=-I${.CURDIR}/../libc/include -I${.CURDIR}/thread \
 	-I${.CURDIR}/../../include
 CFLAGS+=-I${.CURDIR}/arch/${MACHINE_ARCH}/include
 CFLAGS+=-I${.CURDIR}/sys
+CFLAGS+=-I${.CURDIR}/../../libexec/rtld-elf
+CFLAGS+=-fno-builtin
 
 # Uncomment this if you want libpthread to contain debug information for
 # thread locking.
 CFLAGS+=-D_LOCK_DEBUG
 #CFLAGS+= -g
 
+LDADD+= -Wl,--version-script=${.CURDIR}/pthread.map
+
 # enable extra internal consistancy checks
 CFLAGS+=-D_PTHREADS_INVARIANTS -Wall
 
@@ -28,5 +32,6 @@ PRECIOUSLIB=	yes
 
 .include "${.CURDIR}/thread/Makefile.inc"
 .include "${.CURDIR}/sys/Makefile.inc"
+.include "${.CURDIR}/support/Makefile.inc"
 
 .include <bsd.lib.mk>
diff --git a/lib/libkse/support/Makefile.inc b/lib/libkse/support/Makefile.inc
new file mode 100644
index 0000000..27f6524
--- /dev/null
+++ b/lib/libkse/support/Makefile.inc
@@ -0,0 +1,10 @@
+# $FreeBSD$
+
+.PATH:	 ${.CURDIR}/support
+
+SRCS += thr_support.c
+SOBJS+= thr_libc.So
+
+thr_libc.So: thr_support.So
+	${CC} -nostdlib -L/usr/lib -o ${.TARGET} -Wl,-x,-r ${.ALLSRC} -lc_pic
+
diff --git a/lib/libkse/support/thr_support.c b/lib/libkse/support/thr_support.c
new file mode 100644
index 0000000..7ffa5d7
--- /dev/null
+++ b/lib/libkse/support/thr_support.c
@@ -0,0 +1,55 @@
+/*-
+ * Copyright 2003 Alexander Kabaev.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/kse.h>
+#include <signal.h>
+#include <string.h>
+
+__strong_reference(clock_gettime, _thr_clock_gettime);
+__strong_reference(kse_exit, _thr_kse_exit);
+__strong_reference(kse_wakeup, _thr_kse_wakeup);
+__strong_reference(kse_create, _thr_kse_create);
+__strong_reference(kse_thr_interrupt, _thr_kse_thr_interrupt);
+__strong_reference(kse_release, _thr_kse_release);
+
+__strong_reference(sigaction, _thr_sigaction);
+__strong_reference(sigprocmask, _thr_sigprocmask);
+__strong_reference(sigemptyset, _thr_sigemptyset);
+__strong_reference(sigaddset, _thr_sigaddset);
+__strong_reference(sigfillset, _thr_sigfillset);
+__strong_reference(sigismember, _thr_sigismember);
+__strong_reference(sigdelset, _thr_sigdelset);
+
+__strong_reference(memset, _thr_memset);
+__strong_reference(memcpy, _thr_memcpy);
+__strong_reference(strcpy, _thr_strcpy);
+__strong_reference(strlen, _thr_strlen);
+__strong_reference(bzero, _thr_bzero);
+
diff --git a/lib/libkse/sys/Makefile.inc b/lib/libkse/sys/Makefile.inc
index 0b5663d..ac659e8 100644
--- a/lib/libkse/sys/Makefile.inc
+++ b/lib/libkse/sys/Makefile.inc
@@ -4,3 +4,4 @@
 
 SRCS+=	thr_error.c _atomic_lock.S ksd.c thr_enter_uts.S thr_getcontext.S \
 	thr_switch.S lock.c
+
diff --git a/lib/libkse/thread/Makefile.inc b/lib/libkse/thread/Makefile.inc
index ac7c844..cc4f340 100644
--- a/lib/libkse/thread/Makefile.inc
+++ b/lib/libkse/thread/Makefile.inc
@@ -71,6 +71,7 @@ SRCS+= \
 	thr_read.c \
 	thr_readv.c \
 	thr_resume_np.c \
+	thr_rtld.c \
 	thr_rwlock.c \
 	thr_rwlockattr.c \
 	thr_select.c \
diff --git a/lib/libkse/thread/thr_cond.c b/lib/libkse/thread/thr_cond.c
index c71a1ea..8bf49bc 100644
--- a/lib/libkse/thread/thr_cond.c
+++ b/lib/libkse/thread/thr_cond.c
@@ -355,6 +355,8 @@ _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
 	return (rval);
 }
 
+__strong_reference(_pthread_cond_wait, _thr_cond_wait);
+
 int
 __pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
 {
@@ -632,6 +634,8 @@ _pthread_cond_signal(pthread_cond_t * cond)
 	return (rval);
 }
 
+__strong_reference(_pthread_cond_signal, _thr_cond_signal);
+
 int
 _pthread_cond_broadcast(pthread_cond_t * cond)
 {
@@ -673,7 +677,7 @@ _pthread_cond_broadcast(pthread_cond_t * cond)
 			/* There are no more waiting threads: */
 			(*cond)->c_mutex = NULL;
 			break;
-	
+
 		/* Trap invalid condition variable types: */
 		default:
 			/* Return an invalid argument error: */
@@ -689,6 +693,8 @@ _pthread_cond_broadcast(pthread_cond_t * cond)
 	return (rval);
 }
 
+__strong_reference(_pthread_cond_broadcast, _thr_cond_broadcast);
+
 void
 _cond_wait_backout(struct pthread *curthread)
 {
diff --git a/lib/libkse/thread/thr_exit.c b/lib/libkse/thread/thr_exit.c
index 22f187b..6dbef07 100644
--- a/lib/libkse/thread/thr_exit.c
+++ b/lib/libkse/thread/thr_exit.c
@@ -47,16 +47,12 @@ __weak_reference(_pthread_exit, pthread_exit);
 void
 _thr_exit(char *fname, int lineno, char *msg)
 {
-	char s[256];
 
-	/* Prepare an error message string: */
-	snprintf(s, sizeof(s),
+	/* Write an error message to the standard error file descriptor: */
+	_thread_printf(2,
 	    "Fatal error '%s' at line %d in file %s (errno = %d)\n",
 	    msg, lineno, fname, errno);
 
-	/* Write the string to the standard error file descriptor: */
-	__sys_write(2, s, strlen(s));
-
 	abort();
 }
 
diff --git a/lib/libkse/thread/thr_init.c b/lib/libkse/thread/thr_init.c
index 433f246..ceee041 100644
--- a/lib/libkse/thread/thr_init.c
+++ b/lib/libkse/thread/thr_init.c
@@ -304,6 +304,8 @@ _libpthread_init(struct pthread *curthread)
 	_thr_initial->kse->k_curthread = _thr_initial;
 	_thr_initial->kse->k_flags |= KF_INITIALIZED;
 	_kse_initial->k_curthread = _thr_initial;
+	
+	_thr_rtld_init();
 }
 
 /*
diff --git a/lib/libkse/thread/thr_mutex.c b/lib/libkse/thread/thr_mutex.c
index 4e3e79b..2df59ef 100644
--- a/lib/libkse/thread/thr_mutex.c
+++ b/lib/libkse/thread/thr_mutex.c
@@ -718,6 +718,8 @@ __pthread_mutex_lock(pthread_mutex_t *m)
 	return (ret);
 }
 
+__strong_reference(__pthread_mutex_lock, _thr_mutex_lock);
+
 int
 _pthread_mutex_lock(pthread_mutex_t *m)
 {
@@ -748,6 +750,8 @@ _pthread_mutex_unlock(pthread_mutex_t *m)
 	return (mutex_unlock_common(m, /* add reference */ 0));
 }
 
+__strong_reference(_pthread_mutex_unlock, _thr_mutex_unlock);
+
 int
 _mutex_cv_unlock(pthread_mutex_t *m)
 {
diff --git a/lib/libkse/thread/thr_private.h b/lib/libkse/thread/thr_private.h
index a6d3494..092f86d 100644
--- a/lib/libkse/thread/thr_private.h
+++ b/lib/libkse/thread/thr_private.h
@@ -794,6 +794,11 @@ struct pthread {
 	struct pthread_specific_elem	*specific;
 	int				specific_data_count;
 
+	/*
+	 * Current locks bitmap for rtld.
+	 */
+	int	rtld_bits;
+
 	/* Cleanup handlers Link List */
 	struct pthread_cleanup *cleanup;
 	char			*fname;	/* Ptr to source file name  */
@@ -1084,6 +1089,8 @@ int	_pthread_mutexattr_init(pthread_mutexattr_t *);
 int	_pthread_mutexattr_destroy(pthread_mutexattr_t *);
 int	_pthread_mutexattr_settype(pthread_mutexattr_t *, int);
 int	_pthread_once(pthread_once_t *, void (*) (void));
+int	_pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *);
+int	_pthread_rwlock_destroy (pthread_rwlock_t *);
 struct pthread *_pthread_self(void);
 int	_pthread_setspecific(pthread_key_t, const void *);
 struct pthread *_thr_alloc(struct pthread *);
@@ -1128,6 +1135,19 @@ void	_thr_leave_cancellation_point(struct pthread *);
 int	_thr_setconcurrency(int new_level);
 int	_thr_setmaxconcurrency(void);
 
+/*
+ * Aliases for _pthread functions. Should be called instead of
+ * originals if PLT replocation is unwanted at runtme.
+ */
+int	_thr_cond_broadcast(pthread_cond_t *);
+int	_thr_cond_signal(pthread_cond_t *);
+int	_thr_cond_wait(pthread_cond_t *, pthread_mutex_t *);
+int	_thr_mutex_lock(pthread_mutex_t *);
+int	_thr_mutex_unlock(pthread_mutex_t *);
+int	_thr_rwlock_rdlock (pthread_rwlock_t *);
+int	_thr_rwlock_wrlock (pthread_rwlock_t *);
+int	_thr_rwlock_unlock (pthread_rwlock_t *);
+
 /* XXX - Stuff that goes away when my sources get more up to date. */
 /* #include <sys/kse.h> */
 #ifdef SYS_KSE_H
diff --git a/lib/libkse/thread/thr_rtld.c b/lib/libkse/thread/thr_rtld.c
new file mode 100644
index 0000000..c1ee77e
--- /dev/null
+++ b/lib/libkse/thread/thr_rtld.c
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2001 Alexander Kabaev
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+#include <sys/cdefs.h>
+#include <rtld_lock.h>
+#include "thr_private.h"
+
+static void *_thr_rtld_lock_create();
+static void  _thr_rtld_lock_destroy(void *);
+static void  _thr_rtld_rlock_acquire(void *);
+static void  _thr_rtld_wlock_acquire(void *);
+static void  _thr_rtld_lock_release(void *);
+static int   _thr_rtld_set_flag(int);
+static int   _thr_rtld_clr_flag(int);
+
+static void *
+_thr_rtld_lock_create()
+{
+	pthread_rwlock_t prwlock;
+	if (_pthread_rwlock_init(&prwlock, NULL))
+		return (NULL);
+	return (prwlock);
+}
+
+static void
+_thr_rtld_lock_destroy(void *lock)
+{
+	pthread_rwlock_t prwlock = (pthread_rwlock_t)lock;
+	if (prwlock != NULL)
+		_pthread_rwlock_destroy(&prwlock);
+}
+
+static void
+_thr_rtld_rlock_acquire(void *lock)
+{
+	pthread_rwlock_t prwlock = (pthread_rwlock_t)lock;
+	_thr_rwlock_rdlock(&prwlock);
+}
+
+static void
+_thr_rtld_wlock_acquire(void *lock)
+{
+	pthread_rwlock_t prwlock = (pthread_rwlock_t)lock;
+	_thr_rwlock_wrlock(&prwlock);
+}
+
+static void
+_thr_rtld_lock_release(void *lock)
+{
+	pthread_rwlock_t prwlock = (pthread_rwlock_t)lock;
+	_thr_rwlock_unlock(&prwlock);
+}
+
+
+static int
+_thr_rtld_set_flag(int mask)
+{
+	struct pthread *curthread;
+	int bits;
+
+	curthread = _get_curthread();
+
+	if (curthread != NULL) {
+		bits = curthread->rtld_bits;
+		curthread->rtld_bits |= mask;
+	} else {
+		bits = 0;
+		PANIC("No current thread in rtld call");
+	}
+
+	return (bits);
+}
+
+static int
+_thr_rtld_clr_flag(int mask)
+{
+	struct pthread *curthread;
+	int bits;
+
+	curthread = _get_curthread();
+
+	if (curthread != NULL) {
+		bits = curthread->rtld_bits;
+		curthread->rtld_bits &= ~mask;
+	} else {
+		bits = 0;
+		PANIC("No current thread in rtld call");
+	}
+	return (bits);
+}
+
+void
+_thr_rtld_init()
+{
+	struct RtldLockInfo li;
+        li.lock_create  = _thr_rtld_lock_create;
+        li.lock_destroy = _thr_rtld_lock_destroy;
+        li.rlock_acquire = _thr_rtld_rlock_acquire;
+        li.wlock_acquire = _thr_rtld_wlock_acquire;
+        li.lock_release  = _thr_rtld_lock_release;
+        li.thread_set_flag = _thr_rtld_set_flag;
+        li.thread_clr_flag = _thr_rtld_clr_flag;
+        li.at_fork = NULL;
+	_rtld_thread_init(&li);
+}
+
+void
+_thr_rtld_fini()
+{
+	_rtld_thread_init(NULL);
+}
diff --git a/lib/libkse/thread/thr_rwlock.c b/lib/libkse/thread/thr_rwlock.c
index 5f1d8e7..f9559ab 100644
--- a/lib/libkse/thread/thr_rwlock.c
+++ b/lib/libkse/thread/thr_rwlock.c
@@ -46,7 +46,6 @@ __weak_reference(_pthread_rwlock_trywrlock, pthread_rwlock_trywrlock);
 __weak_reference(_pthread_rwlock_unlock, pthread_rwlock_unlock);
 __weak_reference(_pthread_rwlock_wrlock, pthread_rwlock_wrlock);
 
-
 /*
  * Prototypes
  */
@@ -158,16 +157,16 @@ _pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
 	}
 
 	/* grab the monitor lock */
-	if ((ret = _pthread_mutex_lock(&prwlock->lock)) != 0)
+	if ((ret = _thr_mutex_lock(&prwlock->lock)) != 0)
 		return (ret);
 
 	/* give writers priority over readers */
 	while (prwlock->blocked_writers || prwlock->state < 0) {
-		ret = _pthread_cond_wait(&prwlock->read_signal, &prwlock->lock);
+		ret = _thr_cond_wait(&prwlock->read_signal, &prwlock->lock);
 
 		if (ret != 0) {
 			/* can't do a whole lot if this fails */
-			_pthread_mutex_unlock(&prwlock->lock);
+			_thr_mutex_unlock(&prwlock->lock);
 			return (ret);
 		}
 	}
@@ -184,11 +183,13 @@ _pthread_rwlock_rdlock (pthread_rwlock_t *rwlock)
 	 * lock.  Decrementing 'state' is no good because we probably
 	 * don't have the monitor lock.
 	 */
-	_pthread_mutex_unlock(&prwlock->lock);
+	_thr_mutex_unlock(&prwlock->lock);
 
 	return (ret);
 }
 
+__strong_reference(_pthread_rwlock_rdlock, _thr_rwlock_rdlock);
+
 int
 _pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
 {
@@ -276,28 +277,30 @@ _pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
 		return (EINVAL);
 
 	/* grab the monitor lock */
-	if ((ret = _pthread_mutex_lock(&prwlock->lock)) != 0)
+	if ((ret = _thr_mutex_lock(&prwlock->lock)) != 0)
 		return (ret);
 
 	if (prwlock->state > 0) {
 		if (--prwlock->state == 0 && prwlock->blocked_writers)
-			ret = _pthread_cond_signal(&prwlock->write_signal);
+			ret = _thr_cond_signal(&prwlock->write_signal);
 	} else if (prwlock->state < 0) {
 		prwlock->state = 0;
 
 		if (prwlock->blocked_writers)
-			ret = _pthread_cond_signal(&prwlock->write_signal);
+			ret = _thr_cond_signal(&prwlock->write_signal);
 		else
-			ret = _pthread_cond_broadcast(&prwlock->read_signal);
+			ret = _thr_cond_broadcast(&prwlock->read_signal);
 	} else
 		ret = EINVAL;
 
 	/* see the comment on this in pthread_rwlock_rdlock */
-	_pthread_mutex_unlock(&prwlock->lock);
+	_thr_mutex_unlock(&prwlock->lock);
 
 	return (ret);
 }
 
+__strong_reference(_pthread_rwlock_unlock, _thr_rwlock_unlock);
+
 int
 _pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
 {
@@ -318,17 +321,17 @@ _pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
 	}
 
 	/* grab the monitor lock */
-	if ((ret = _pthread_mutex_lock(&prwlock->lock)) != 0)
+	if ((ret = _thr_mutex_lock(&prwlock->lock)) != 0)
 		return (ret);
 
 	while (prwlock->state != 0) {
 		++prwlock->blocked_writers;
 
-		ret = _pthread_cond_wait(&prwlock->write_signal, &prwlock->lock);
+		ret = _thr_cond_wait(&prwlock->write_signal, &prwlock->lock);
 
 		if (ret != 0) {
 			--prwlock->blocked_writers;
-			_pthread_mutex_unlock(&prwlock->lock);
+			_thr_mutex_unlock(&prwlock->lock);
 			return (ret);
 		}
 
@@ -339,7 +342,9 @@ _pthread_rwlock_wrlock (pthread_rwlock_t *rwlock)
 	prwlock->state = -1;
 
 	/* see the comment on this in pthread_rwlock_rdlock */
-	_pthread_mutex_unlock(&prwlock->lock);
+	_thr_mutex_unlock(&prwlock->lock);
 
 	return (ret);
 }
+
+__strong_reference(_pthread_rwlock_wrlock, _thr_rwlock_wrlock);