Import my recent 1:1 threading working. some features improved includes:

 1. fast simple type mutex.
 2. __thread tls works.
 3. asynchronous cancellation works ( using signal ).
 4. thread synchronization is fully based on umtx, mainly, condition
    variable and other synchronization objects were rewritten by using
    umtx directly. those objects can be shared between processes via
    shared memory, it has to change ABI which does not happen yet.
 5. default stack size is increased to 1M on 32 bits platform, 2M for
    64 bits platform.
As the result, some mysql super-smack benchmarks show performance is
improved massivly.

Okayed by: jeff, mtm, rwatson, scottl

4 files changed, 162 insertions, 103 deletions

diff --git a/lib/libthr/arch/amd64/Makefile.inc b/lib/libthr/arch/amd64/Makefile.inc
index c822c8c..6e6d577 100644
--- a/lib/libthr/arch/amd64/Makefile.inc
+++ b/lib/libthr/arch/amd64/Makefile.inc
@@ -1,5 +1,5 @@
 #$FreeBSD$
 
-.PATH:	 ${.CURDIR}/sys ${.CURDIR}/arch/${MACHINE_ARCH}/${MACHINE_ARCH}
+.PATH:	${.CURDIR}/arch/${MACHINE_ARCH}/${MACHINE_ARCH}
 
-SRCS+=	_setcurthread.c
+SRCS+=	pthread_md.c
diff --git a/lib/libthr/arch/amd64/amd64/_setcurthread.c b/lib/libthr/arch/amd64/amd64/_setcurthread.c
deleted file mode 100644
index 51395c1..0000000
--- a/lib/libthr/arch/amd64/amd64/_setcurthread.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright (c) 2004, David Xu <davidxu@freebsd.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice unmodified, this list of conditions, and the following
- *    disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/types.h>
-#include <sys/ucontext.h>
-
-#include <pthread.h>
-#include <machine/sysarch.h>
-
-#include "thr_private.h"
-#include "rtld_tls.h"
-
-struct tcb {
-	struct tcb	*tcb_self;      /* required by rtld */
-	void		*tcb_dtv;       /* required by rtld */
-	struct pthread	*tcb_thread;
-};
-
-void
-_retire_thread(void *entry)
-{
-	struct tcb *tcb = (struct tcb *)entry;
-
-	_rtld_free_tls(tcb, sizeof(struct tcb), 16);
-}
-
-void *
-_set_curthread(ucontext_t *uc, struct pthread *thr, int *err)
-{
-	struct tcb *tcb;
-	void *oldtls;
-
-	*err = 0;
-
-	if (thr->arch_id != NULL && uc == NULL) {
-		amd64_set_fsbase(thr->arch_id);
-		return (thr->arch_id);
-	}	
-
-	if (uc == NULL) {
-		__asm __volatile("movq %%fs:0, %0" : "=r" (oldtls));
-        } else {
-		oldtls = NULL;
-        }
-
-	/*
-	 * Allocate and initialise a new TLS block with enough extra
-	 * space for our self pointer.
-	 */
-	tcb = _rtld_allocate_tls(oldtls, sizeof(struct tcb), 16);
-
-	/*
-	 * Cache the address of the thread structure here, after
-	 * rtld's two words of private space.
- 	 */
-	tcb->tcb_thread = thr;
-
-	if (uc == NULL)
-		amd64_set_fsbase(tcb);
-	return (tcb);
-}
-
-pthread_t
-_get_curthread(void)
-{
-	extern pthread_t _thread_initial;
-	pthread_t td;
-
-	if (_thread_initial == NULL)
-		return (NULL);
-	__asm __volatile("movq %%fs:%1, %0"	\
-		: "=r" (td)         		\
-		: "m" (*(long *)(__offsetof(struct tcb, tcb_thread))));
-
-	return (td);
-}
diff --git a/lib/libthr/arch/amd64/amd64/pthread_md.c b/lib/libthr/arch/amd64/amd64/pthread_md.c
new file mode 100644
index 0000000..d2477df
--- /dev/null
+++ b/lib/libthr/arch/amd64/amd64/pthread_md.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2003 Daniel Eischen <deischen@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Neither the name of the author nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/types.h>
+#include <rtld_tls.h>
+
+#include "pthread_md.h"
+
+/*
+ * The constructors.
+ */
+struct tcb *
+_tcb_ctor(struct pthread *thread, int initial)
+{
+	struct tcb *tcb;
+	void *oldtls;
+
+	if (initial)
+		__asm __volatile("movq %%fs:0, %0" : "=r" (oldtls));
+	else
+		oldtls = NULL;
+	tcb = _rtld_allocate_tls(oldtls, sizeof(struct tcb), 16);
+	if (tcb)
+		tcb->tcb_thread = thread;
+	return (tcb);
+}
+
+void
+_tcb_dtor(struct tcb *tcb)
+{
+	_rtld_free_tls(tcb, sizeof(struct tcb), 16);
+}
diff --git a/lib/libthr/arch/amd64/include/pthread_md.h b/lib/libthr/arch/amd64/include/pthread_md.h
new file mode 100644
index 0000000..4500f6b
--- /dev/null
+++ b/lib/libthr/arch/amd64/include/pthread_md.h
@@ -0,0 +1,103 @@
+/*-
+ * Copyright (C) 2003 David Xu <davidxu@freebsd.org>
+ * Copyright (c) 2001 Daniel Eischen <deischen@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Neither the name of the author nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * Machine-dependent thread prototypes/definitions.
+ */
+#ifndef _PTHREAD_MD_H_
+#define	_PTHREAD_MD_H_
+
+#include <stddef.h>
+#include <sys/types.h>
+#include <machine/sysarch.h>
+#include <ucontext.h>
+
+#define	DTV_OFFSET		offsetof(struct tcb, tcb_dtv)
+
+/*
+ * Variant II tcb, first two members are required by rtld,
+ * %fs points to the structure.
+ */
+struct tcb {
+	struct tcb		*tcb_self;	/* required by rtld */
+	void			*tcb_dtv;	/* required by rtld */
+	struct pthread		*tcb_thread;
+	void			*tcb_spare[1];
+};
+
+/*
+ * Evaluates to the byte offset of the per-tcb variable name.
+ */
+#define	__tcb_offset(name)	__offsetof(struct tcb, name)
+
+/*
+ * Evaluates to the type of the per-tcb variable name.
+ */
+#define	__tcb_type(name)	__typeof(((struct tcb *)0)->name)
+
+/*
+ * Evaluates to the value of the per-tcb variable name.
+ */
+#define	TCB_GET64(name) ({					\
+	__tcb_type(name) __result;				\
+								\
+	u_long __i;						\
+	__asm __volatile("movq %%fs:%1, %0"			\
+	    : "=r" (__i)					\
+	    : "m" (*(u_long *)(__tcb_offset(name))));		\
+	__result = (__tcb_type(name))__i;			\
+								\
+	__result;						\
+})
+
+struct tcb	*_tcb_ctor(struct pthread *, int);
+void		_tcb_dtor(struct tcb *tcb);
+
+static __inline void
+_tcb_set(struct tcb *tcb)
+{
+	amd64_set_fsbase(tcb);
+}
+
+static __inline struct tcb *
+_tcb_get(void)
+{
+	return (TCB_GET64(tcb_self));
+}
+
+extern struct pthread *_thr_initial;
+
+static __inline struct pthread *
+_get_curthread(void)
+{
+	if (_thr_initial)
+		return (TCB_GET64(tcb_thread));
+	return (NULL);
+}
+#endif