Merge remote-tracking branch 'origin/stable/10' into devel

15 files changed, 514 insertions, 28 deletions

diff --git a/lib/libc/Versions.def b/lib/libc/Versions.def
index 8452c7d..e348308 100644
--- a/lib/libc/Versions.def
+++ b/lib/libc/Versions.def
@@ -27,6 +27,10 @@ FBSD_1.3 {
 FBSD_1.4 {
 } FBSD_1.3;
 
+# This version was first added to 12.0-current.
+FBSD_1.5 {
+} FBSD_1.4;
+
 
 # This is our private namespace.  Any global interfaces that are
 # strictly for use only by other FreeBSD applications and libraries
@@ -35,4 +39,4 @@ FBSD_1.4 {
 #
 # Please do NOT increment the version of this namespace.
 FBSDprivate_1.0 {
-} FBSD_1.4;
+} FBSD_1.5;
diff --git a/lib/libc/include/libc_private.h b/lib/libc/include/libc_private.h
index 6e2ff66..c219f55 100644
--- a/lib/libc/include/libc_private.h
+++ b/lib/libc/include/libc_private.h
@@ -264,6 +264,12 @@ extern const char *__progname;
 void _malloc_thread_cleanup(void);
 
 /*
+ * This function is used by the threading libraries to notify libc that a
+ * thread is exiting, so its thread-local dtors should be called.
+ */
+void __cxa_thread_call_dtors(void);
+
+/*
  * These functions are used by the threading libraries in order to protect
  * malloc across fork().
  */
diff --git a/lib/libc/stdio/fgetln.c b/lib/libc/stdio/fgetln.c
index 1779de2..6546768 100644
--- a/lib/libc/stdio/fgetln.c
+++ b/lib/libc/stdio/fgetln.c
@@ -159,6 +159,7 @@ fgetln(FILE *fp, size_t *lenp)
 
 error:
 	*lenp = 0;		/* ??? */
+	fp->_flags |= __SERR;
 	FUNLOCKFILE(fp);
 	return (NULL);		/* ??? */
 }
diff --git a/lib/libc/stdlib/Makefile.inc b/lib/libc/stdlib/Makefile.inc
index 75204f5..9bffd7e 100644
--- a/lib/libc/stdlib/Makefile.inc
+++ b/lib/libc/stdlib/Makefile.inc
@@ -5,7 +5,7 @@
 .PATH: ${.CURDIR}/${LIBC_ARCH}/stdlib ${.CURDIR}/stdlib
 
 MISRCS+=_Exit.c a64l.c abort.c abs.c atexit.c atof.c atoi.c atol.c atoll.c \
-	bsearch.c div.c exit.c getenv.c getopt.c getopt_long.c \
+	bsearch.c cxa_thread_atexit.c div.c exit.c getenv.c getopt.c getopt_long.c \
 	getsubopt.c hcreate.c heapsort.c imaxabs.c imaxdiv.c \
 	insque.c l64a.c labs.c ldiv.c llabs.c lldiv.c lsearch.c \
 	merge.c ptsname.c qsort.c qsort_r.c quick_exit.c radixsort.c rand.c \
diff --git a/lib/libc/stdlib/Symbol.map b/lib/libc/stdlib/Symbol.map
index 4b8ef5b..f6b62a9 100644
--- a/lib/libc/stdlib/Symbol.map
+++ b/lib/libc/stdlib/Symbol.map
@@ -104,8 +104,13 @@ FBSD_1.3 {
 	strtouq_l;
 };
 
+FBSD_1.5 {
+	__cxa_thread_atexit;
+};
+
 FBSDprivate_1.0 {
 	__system;
 	_system;
 	__libc_system;
+	__cxa_thread_call_dtors;
 };
diff --git a/lib/libc/stdlib/cxa_thread_atexit.c b/lib/libc/stdlib/cxa_thread_atexit.c
new file mode 100644
index 0000000..c966731
--- /dev/null
+++ b/lib/libc/stdlib/cxa_thread_atexit.c
@@ -0,0 +1,140 @@
+/*-
+ * Copyright (c) 2016 Mahdi Mokhtari <mokhi64@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/queue.h>
+#include "namespace.h"
+#include <errno.h>
+#include <link.h>
+#include <pthread.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "un-namespace.h"
+#include "libc_private.h"
+
+/*
+ * C++11 introduces the thread_local scope (like __thread with some
+ * additions).  As a key-feature it should support non-trivial
+ * destructors, registered with __cxa_thread_atexit() to be executed
+ * at the thread termination.
+ *
+ * The implemention keeps a _Thread_local list of destructors per each
+ * thread, and calls __cxa_thread_call_dtors() on each thread's exit
+ * to do cleanup.  For a thread calling exit(3), in particular, for
+ * the initial thread returning from main(), we call
+ * __cxa_thread_call_dtors() inside exit().
+ *
+ * It could be possible that a dynamically loaded library, use
+ * thread_local variable but is dlclose()'d before thread exit.  The
+ * destructor of this variable will then try to access the address,
+ * for calling it but it's unloaded, so it'll crash.  We're using
+ * __elf_phdr_match_addr() to detect and prevent such cases and so
+ * prevent the crash.
+ */
+
+#define CXA_DTORS_ITERATIONS 4
+
+struct cxa_thread_dtor {
+	void *obj;
+	void (*func)(void *);
+	void *dso;
+	LIST_ENTRY(cxa_thread_dtor) entry;
+};
+static _Thread_local LIST_HEAD(dtor_list, cxa_thread_dtor) dtors =
+    LIST_HEAD_INITIALIZER(dtors);
+
+int
+__cxa_thread_atexit(void (*dtor_func)(void *), void *obj, void *dso_symbol)
+{
+	struct cxa_thread_dtor *new_dtor;
+
+	new_dtor = malloc(sizeof(*new_dtor));
+	if (new_dtor == NULL) {
+		errno = ENOMEM; /* forcibly override malloc(3) error */
+		return (-1);
+	}
+
+	new_dtor->obj = obj;
+	new_dtor->func = dtor_func;
+	new_dtor->dso = dso_symbol;
+	LIST_INSERT_HEAD(&dtors, new_dtor, entry);
+	return (0);
+}
+
+static void
+walk_cb_call(struct cxa_thread_dtor *dtor)
+{
+	struct dl_phdr_info phdr_info;
+
+	if (_rtld_addr_phdr(dtor->dso, &phdr_info) &&
+	    __elf_phdr_match_addr(&phdr_info, dtor->func))
+		dtor->func(dtor->obj);
+	else
+		fprintf(stderr, "__cxa_thread_call_dtors: dtr %p from "
+		    "unloaded dso, skipping\n", (void *)(dtor->func));
+}
+
+static void
+walk_cb_nocall(struct cxa_thread_dtor *dtor __unused)
+{
+}
+
+static void
+cxa_thread_walk(void (*cb)(struct cxa_thread_dtor *))
+{
+	struct cxa_thread_dtor *dtor, *tdtor;
+
+	LIST_FOREACH_SAFE(dtor, &dtors, entry, tdtor) {
+		LIST_REMOVE(dtor, entry);
+		cb(dtor);
+		free(dtor);
+	}
+}
+
+/*
+ * This is the callback function we use to call destructors, once for
+ * each thread.  It is called in exit(3) in libc/stdlib/exit.c and
+ * before exit_thread() in libthr/thread/thr_exit.c.
+ */
+void
+__cxa_thread_call_dtors(void)
+{
+	int i;
+
+	for (i = 0; i < CXA_DTORS_ITERATIONS && !LIST_EMPTY(&dtors); i++)
+		cxa_thread_walk(walk_cb_call);
+
+	if (!LIST_EMPTY(&dtors)) {
+		fprintf(stderr, "Thread %p is exiting with more "
+		    "thread-specific dtors created after %d iterations "
+		    "of destructor calls\n",
+		    _pthread_self(), i);
+		cxa_thread_walk(walk_cb_nocall);
+	}
+}
diff --git a/lib/libc/stdlib/exit.c b/lib/libc/stdlib/exit.c
index 145eb9d..b8afede 100644
--- a/lib/libc/stdlib/exit.c
+++ b/lib/libc/stdlib/exit.c
@@ -64,6 +64,12 @@ exit(status)
 
 	_thread_autoinit_dummy_decl = 1;
 
+	/*
+	 * We're dealing with cleaning up thread_local destructors in the case of
+	 * the process termination through main() exit.
+	 * Other cases are handled elsewhere.
+	 */
+	__cxa_thread_call_dtors();
 	__cxa_finalize(NULL);
 	if (__cleanup)
 		(*__cleanup)();
diff --git a/lib/libc/sys/aio_fsync.2 b/lib/libc/sys/aio_fsync.2
index 7eb1a3b..b271dbf 100644
--- a/lib/libc/sys/aio_fsync.2
+++ b/lib/libc/sys/aio_fsync.2
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 15, 2016
+.Dd August 19, 2016
 .Dt AIO_FSYNC 2
 .Os
 .Sh NAME
@@ -74,16 +74,14 @@ the call returns without having enqueued the request.
 .Pp
 The
 .Fa iocb->aio_sigevent
-structure can be used to request notification of the request's
+structure can be used to request notification of the operation's
 completion as described in
 .Xr aio 4 .
 .Sh RESTRICTIONS
-The asynchronous I/O Control Block structure pointed to by
+The Asynchronous I/O Control Block structure pointed to by
 .Fa iocb
 must remain valid until the
 operation has completed.
-For this reason, use of auto (stack) variables
-for these objects is discouraged.
 .Pp
 The asynchronous I/O control buffer
 .Fa iocb
@@ -91,9 +89,8 @@ should be zeroed before the
 .Fn aio_fsync
 call to avoid passing bogus context information to the kernel.
 .Pp
-Modifications of the Asynchronous I/O Control Block structure or the
-buffer contents after the request has been enqueued, but before the
-request has completed, are not allowed.
+Modification of the Asynchronous I/O Control Block structure is not allowed
+while the request is queued.
 .Sh RETURN VALUES
 .Rv -std aio_fsync
 .Sh ERRORS
diff --git a/lib/libc/sys/aio_mlock.2 b/lib/libc/sys/aio_mlock.2
index 03e2df7..76f67ef 100644
--- a/lib/libc/sys/aio_mlock.2
+++ b/lib/libc/sys/aio_mlock.2
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 15, 2016
+.Dd August 19, 2016
 .Dt AIO_MLOCK 2
 .Os
 .Sh NAME
@@ -67,7 +67,7 @@ then the call returns without having enqueued the request.
 .Pp
 The
 .Fa iocb->aio_sigevent
-structure can be used to request notification of the request's
+structure can be used to request notification of the operation's
 completion as described in
 .Xr aio 4 .
 .Sh RESTRICTIONS
@@ -77,8 +77,6 @@ and the buffer that the
 .Fa iocb->aio_buf
 member of that structure references must remain valid until the
 operation has completed.
-For this reason, use of auto (stack) variables
-for these objects is discouraged.
 .Pp
 The asynchronous I/O control buffer
 .Fa iocb
@@ -87,8 +85,8 @@ should be zeroed before the
 call to avoid passing bogus context information to the kernel.
 .Pp
 Modifications of the Asynchronous I/O Control Block structure or the
-buffer contents after the request has been enqueued, but before the
-request has completed, are not allowed.
+memory mapping described by the virtual address range are not allowed
+while the request is queued.
 .Sh RETURN VALUES
 .Rv -std aio_mlock
 .Sh ERRORS
diff --git a/lib/libc/sys/aio_read.2 b/lib/libc/sys/aio_read.2
index 69960d5..96d84cc 100644
--- a/lib/libc/sys/aio_read.2
+++ b/lib/libc/sys/aio_read.2
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 15, 2016
+.Dd August 19, 2016
 .Dt AIO_READ 2
 .Os
 .Sh NAME
@@ -82,7 +82,7 @@ not be referenced after the request is enqueued.
 .Pp
 The
 .Fa iocb->aio_sigevent
-structure can be used to request notification of the request's
+structure can be used to request notification of the operation's
 completion as described in
 .Xr aio 4 .
 .Sh RESTRICTIONS
@@ -92,8 +92,6 @@ and the buffer that the
 .Fa iocb->aio_buf
 member of that structure references must remain valid until the
 operation has completed.
-For this reason, use of auto (stack) variables
-for these objects is discouraged.
 .Pp
 The asynchronous I/O control buffer
 .Fa iocb
@@ -102,8 +100,7 @@ should be zeroed before the
 call to avoid passing bogus context information to the kernel.
 .Pp
 Modifications of the Asynchronous I/O Control Block structure or the
-buffer contents after the request has been enqueued, but before the
-request has completed, are not allowed.
+buffer contents are not allowed while the request is queued.
 .Pp
 If the file offset in
 .Fa iocb->aio_offset
diff --git a/lib/libc/sys/aio_write.2 b/lib/libc/sys/aio_write.2
index 076ce50..2148913 100644
--- a/lib/libc/sys/aio_write.2
+++ b/lib/libc/sys/aio_write.2
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 15, 2016
+.Dd August 19, 2016
 .Dt AIO_WRITE 2
 .Os
 .Sh NAME
@@ -88,7 +88,7 @@ be referenced after the request is enqueued.
 .Pp
 The
 .Fa iocb->aio_sigevent
-structure can be used to request notification of the request's
+structure can be used to request notification of the operation's
 completion as described in
 .Xr aio 4 .
 .Sh RESTRICTIONS
@@ -98,8 +98,6 @@ and the buffer that the
 .Fa iocb->aio_buf
 member of that structure references must remain valid until the
 operation has completed.
-For this reason, use of auto (stack) variables
-for these objects is discouraged.
 .Pp
 The asynchronous I/O control buffer
 .Fa iocb
@@ -108,8 +106,7 @@ should be zeroed before the
 system call to avoid passing bogus context information to the kernel.
 .Pp
 Modifications of the Asynchronous I/O Control Block structure or the
-buffer contents after the request has been enqueued, but before the
-request has completed, are not allowed.
+buffer contents are not allowed while the request is queued.
 .Pp
 If the file offset in
 .Fa iocb->aio_offset
diff --git a/lib/libc/sys/ptrace.2 b/lib/libc/sys/ptrace.2
index fc5c0ea..0e8c8bd 100644
--- a/lib/libc/sys/ptrace.2
+++ b/lib/libc/sys/ptrace.2
@@ -151,6 +151,11 @@ The process ID of the new child process will also be present in the
 .Va pl_child_pid
 member of
 .Vt "struct ptrace_lwpinfo" .
+If the new child process was created via
+.Xr vfork 2 ,
+the traced process's stop will also include the
+.Dv PL_FLAG_VFORKED
+flag.
 Note that new child processes will be attached with the default
 tracing event mask;
 they do not inherit the event mask of the traced process.
@@ -173,6 +178,33 @@ Note that new processes do not report an event for the creation of their
 initial thread,
 and exiting processes do not report an event for the termination of the
 last thread.
+.It Dv PTRACE_VFORK
+Report a stop event when a parent process resumes after a
+.Xr vfork 2 .
+.Pp
+When a thread in the traced process creates a new child process via
+.Xr vfork 2 ,
+the stop that reports
+.Dv PL_FLAG_FORKED
+and
+.Dv PL_FLAG_SCX
+occurs just after the child process is created,
+but before the thread waits for the child process to stop sharing process
+memory.
+If a debugger is not tracing the new child process,
+it must ensure that no breakpoints are enabled in the shared process
+memory before detaching from the new child process.
+This means that no breakpoints are enabled in the parent process either.
+.Pp
+The
+.Dv PTRACE_VFORK
+flag enables a new stop that indicates when the new child process stops
+sharing the process memory of the parent process.
+A debugger can reinsert breakpoints in the parent process and resume it
+in response to this event.
+This event is indicated by setting the
+.Dv PL_FLAG_VFORK_DONE
+flag.
 .El
 .Pp
 The default tracing event mask when attaching to a process via
@@ -501,6 +533,16 @@ is enabled.
 Note that this event is not reported when the last LWP in a process exits.
 The termination of the last thread is reported via a normal process exit
 event.
+.It PL_FLAG_VFORKED
+Indicates that the thread is returning from a call to
+.Xr vfork 2
+that created a new child process.
+This flag is set in addition to
+.Dv PL_FLAG_FORKED .
+.It PL_FLAG_VFORK_DONE
+Indicates that the thread has resumed after a child process created via
+.Xr vfork 2
+has stopped sharing its address space with the traced process.
 .El
 .It pl_sigmask
 The current signal mask of the LWP
diff --git a/lib/libc/tests/stdlib/Makefile b/lib/libc/tests/stdlib/Makefile
index 2f79858..15d8cbc 100644
--- a/lib/libc/tests/stdlib/Makefile
+++ b/lib/libc/tests/stdlib/Makefile
@@ -1,8 +1,14 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 ATF_TESTS_C+=		heapsort_test
 ATF_TESTS_C+=		mergesort_test
 ATF_TESTS_C+=		qsort_test
+.if ${COMPILER_FEATURES:Mc++11}
+ATF_TESTS_CXX+=		cxa_thread_atexit_test
+ATF_TESTS_CXX+=		cxa_thread_atexit_nothr_test
+.endif
 
 TESTSDIR=	${TESTSBASE}/lib/libc/stdlib
 
@@ -34,6 +40,11 @@ PROGS+=		h_getopt h_getopt_long
 
 CFLAGS+=	-I${.CURDIR}
 
+CXXFLAGS.cxa_thread_atexit_test+=	-std=c++11
+CXXFLAGS.cxa_thread_atexit_nothr_test+=	-std=c++11
+DPADD.cxa_thread_atexit_test+=		${LIBPTHREAD}
+LDADD.cxa_thread_atexit_test+=		-lpthread
+
 .for t in h_getopt h_getopt_long
 CFLAGS.$t+=	-I${LIBNETBSD_SRCDIR} -I${SRCTOP}/contrib/netbsd-tests
 LDFLAGS.$t+=	-L${LIBNETBSD_OBJDIR}
diff --git a/lib/libc/tests/stdlib/cxa_thread_atexit_nothr_test.cc b/lib/libc/tests/stdlib/cxa_thread_atexit_nothr_test.cc
new file mode 100644
index 0000000..3ac3602
--- /dev/null
+++ b/lib/libc/tests/stdlib/cxa_thread_atexit_nothr_test.cc
@@ -0,0 +1,102 @@
+/*-
+ * Copyright (c) 2016 Mahdi Mokhtari <mokhi64@gmail.com>
+ * Copyright (c) 2016 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <dlfcn.h>
+#include <atf-c++.hpp>
+#include <cstdio>
+#include <cstdlib>
+
+static FILE *output = NULL;
+
+struct Foo {
+	Foo() { ATF_REQUIRE(fprintf(output, "Created\n") > 0); }
+	~Foo() { ATF_REQUIRE(fprintf(output, "Destroyed\n") > 0); }
+	void use() { ATF_REQUIRE(fprintf(output, "Used\n") > 0); }
+};
+
+static thread_local Foo f;
+
+/*
+ * This test must not be linked to libpthread.
+ */
+ATF_TEST_CASE_WITHOUT_HEAD(cxx__nothr);
+ATF_TEST_CASE_BODY(cxx__nothr)
+{
+	void *libthr_handle;
+
+	/* Avoid coredump during f construction. */
+	output = stderr;
+
+	libthr_handle = dlopen("libthr.so.3", RTLD_LAZY | RTLD_GLOBAL |
+	    RTLD_NOLOAD);
+	ATF_REQUIRE(libthr_handle == NULL);
+}
+
+static void
+check_local_main(void)
+{
+	static const char out_log[] = "Created\nUsed\nDestroyed\n";
+
+	fflush(output);
+	ATF_REQUIRE(atf::utils::compare_file("test_main.txt", out_log));
+}
+
+ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_local_main);
+ATF_TEST_CASE_BODY(cxx__thread_local_main)
+{
+
+	ATF_REQUIRE((output = fopen("test_main.txt", "w")) != NULL);
+	f.use();
+	atexit(check_local_main);
+}
+
+extern "C" int __cxa_thread_atexit(void (*)(void *), void *, void *);
+
+static void
+again(void *arg)
+{
+
+	__cxa_thread_atexit(again, arg, &output);
+}
+
+ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_inf_dtors);
+ATF_TEST_CASE_BODY(cxx__thread_inf_dtors)
+{
+
+	again(NULL);
+}
+
+ATF_INIT_TEST_CASES(tcs)
+{
+
+	ATF_ADD_TEST_CASE(tcs, cxx__nothr);
+	ATF_ADD_TEST_CASE(tcs, cxx__thread_local_main);
+	ATF_ADD_TEST_CASE(tcs, cxx__thread_inf_dtors);
+}
diff --git a/lib/libc/tests/stdlib/cxa_thread_atexit_test.cc b/lib/libc/tests/stdlib/cxa_thread_atexit_test.cc
new file mode 100644
index 0000000..ded91c9
--- /dev/null
+++ b/lib/libc/tests/stdlib/cxa_thread_atexit_test.cc
@@ -0,0 +1,180 @@
+/*-
+ * Copyright (c) 2016 Mahdi Mokhtari <mokhi64@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <dlfcn.h>
+#include <atf-c++.hpp>
+#include <cstdio>
+#include <cstdlib>
+#include <thread>
+
+static FILE *output = NULL;
+
+struct Foo {
+	Foo() { ATF_REQUIRE(fprintf(output, "Created\n") > 0); }
+	~Foo() { ATF_REQUIRE(fprintf(output, "Destroyed\n") > 0); }
+	void use() { ATF_REQUIRE(fprintf(output, "Used\n") > 0); }
+};
+
+struct Bar {
+	Bar() {}
+	~Bar() {
+		thread_local static Foo foo;
+		ATF_REQUIRE(fprintf(output, "DIED\n") > 0);
+	}
+	void use() {}
+};
+
+extern "C" int __cxa_thread_atexit(void (*)(void *), void *, void *);
+
+static void
+again(void *arg)
+{
+
+	__cxa_thread_atexit(again, arg, &output);
+}
+
+struct Baz {
+	Baz() {}
+	~Baz() {
+		again(NULL);
+	}
+	void use() {}
+};
+
+static thread_local Foo f;
+static thread_local Foo g;
+static thread_local Bar h;
+static thread_local Baz e;
+
+/*
+ * This test must be linked to libpthread.
+ */
+ATF_TEST_CASE_WITHOUT_HEAD(cxx__thr);
+ATF_TEST_CASE_BODY(cxx__thr)
+{
+	void *libthr_handle;
+
+	/* Avoid coredump during f construction. */
+	output = stderr;
+
+	libthr_handle = dlopen("libthr.so.3", RTLD_LAZY | RTLD_GLOBAL |
+	    RTLD_NOLOAD);
+	ATF_REQUIRE(libthr_handle != NULL);
+	dlclose(libthr_handle);
+}
+
+/*
+ * In this test f.use() will test cxa_thread_atexit() in non-threaded mode.
+ * After f.use() main will be threaded and we'll have one additional thread
+ * with its own TLS data.
+ */
+ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_local_before);
+ATF_TEST_CASE_BODY(cxx__thread_local_before)
+{
+	static const char out_log[] = "Created\nCreated\nUsed\nCreated\n"
+	    "Created\nUsed\nCreated\nDIED\nDestroyed\nDestroyed\nDestroyed\n";
+
+	ATF_REQUIRE((output = fopen("test_before.txt", "w")) != NULL);
+
+	f.use();
+	std::thread t([]() { f.use(); });
+	t.join();
+
+	fflush(output);
+
+	ATF_REQUIRE(atf::utils::compare_file("test_before.txt", out_log));
+}
+
+/*
+ * In this test, f.use() will test __cxa_thread_atexit()
+ * in threaded mode (but still in main-threaed).
+ */
+ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_local_after);
+ATF_TEST_CASE_BODY(cxx__thread_local_after)
+{
+	static const char out_log[] = "Created\nCreated\nUsed\nCreated\n"
+	    "DIED\nDestroyed\nDestroyed\nDestroyed\nCreated\nCreated\nUsed\n";
+
+	ATF_REQUIRE((output = fopen("test_after.txt", "w")) != NULL);
+
+	std::thread t([]() { g.use(); });
+	t.join();
+	sleep(1);
+	g.use();
+
+	fflush(output);
+
+	ATF_REQUIRE(atf::utils::compare_file("test_after.txt", out_log));
+}
+
+/*
+ * In this test, we register a new dtor while dtors are being run
+ * in __cxa_thread_atexit().
+ */
+ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_local_add_while_calling_dtors);
+ATF_TEST_CASE_BODY(cxx__thread_local_add_while_calling_dtors)
+{
+	static const char out_log[] = "Created\nCreated\nCreated\nDIED\n"
+	    "Destroyed\nDestroyed\nDestroyed\n";
+
+	ATF_REQUIRE((output = fopen("test_add_meanwhile.txt", "w")) != NULL);
+
+	std::thread t([]() { h.use(); });
+	t.join();
+	sleep(1);
+
+	fflush(output);
+
+	ATF_REQUIRE(atf::utils::compare_file("test_add_meanwhile.txt", out_log));
+}
+
+ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_inf_dtors);
+ATF_TEST_CASE_BODY(cxx__thread_inf_dtors)
+{
+
+	/*
+	 * Only added to make isolated run of this test not
+	 * coredumping.  Construction of Foo objects require filled
+	 * output.
+	 */
+	output = stderr;
+
+	std::thread t([]() { e.use(); });
+	t.join();
+}
+
+ATF_INIT_TEST_CASES(tcs)
+{
+
+	ATF_ADD_TEST_CASE(tcs, cxx__thr);
+	ATF_ADD_TEST_CASE(tcs, cxx__thread_local_before);
+	ATF_ADD_TEST_CASE(tcs, cxx__thread_local_after);
+	ATF_ADD_TEST_CASE(tcs, cxx__thread_local_add_while_calling_dtors);
+	ATF_ADD_TEST_CASE(tcs, cxx__thread_inf_dtors);
+}