Merge sync of head

19 files changed, 851 insertions, 127 deletions

diff --git a/libexec/rtld-elf/Makefile b/libexec/rtld-elf/Makefile
index cc30132..2bed4c7 100644
--- a/libexec/rtld-elf/Makefile
+++ b/libexec/rtld-elf/Makefile
@@ -42,17 +42,18 @@ CFLAGS+=	-fPIC
 CFLAGS+=	-fpic
 .endif
 CFLAGS+=	-DPIC $(DEBUG)
+.if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386"
+CFLAGS+=	-fvisibility=hidden
+.endif
 LDFLAGS+=	-shared -Wl,-Bsymbolic
-DPADD=		${LIBC_PIC}
-LDADD=		-lc_pic
+LIBADD=		c_pic
 
 .if ${MACHINE_CPUARCH} == "arm"
 # Some of the required math functions (div & mod) are implemented in
 # libcompiler_rt on ARM. The library also needs to be placed first to be
 # correctly linked. As some of the functions are used before we have
 # shared libraries.
-DPADD+=		${LIBCOMPILER_RT}
-LDADD+=		-lcompiler_rt
+LIBADD+=	compiler_rt
 .endif
 
 
diff --git a/libexec/rtld-elf/Symbol.map b/libexec/rtld-elf/Symbol.map
index 5ea7d7e..4adc2ad 100644
--- a/libexec/rtld-elf/Symbol.map
+++ b/libexec/rtld-elf/Symbol.map
@@ -30,5 +30,6 @@ FBSDprivate_1.0 {
     _rtld_atfork_post;
     _rtld_addr_phdr;
     _rtld_get_stack_prot;
+    _rtld_is_dlopened;
     _r_debug_postinit;
 };
diff --git a/libexec/rtld-elf/aarch64/reloc.c b/libexec/rtld-elf/aarch64/reloc.c
new file mode 100644
index 0000000..b515a1e
--- /dev/null
+++ b/libexec/rtld-elf/aarch64/reloc.c
@@ -0,0 +1,414 @@
+/*-
+ * Copyright (c) 2014-2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by Andrew Turner
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+
+#include <stdlib.h>
+
+#include "debug.h"
+#include "rtld.h"
+#include "rtld_printf.h"
+
+/*
+ * It is possible for the compiler to emit relocations for unaligned data.
+ * We handle this situation with these inlines.
+ */
+#define	RELOC_ALIGNED_P(x) \
+	(((uintptr_t)(x) & (sizeof(void *) - 1)) == 0)
+
+/*
+ * This is not the correct prototype, but we only need it for
+ * a function pointer to a simple asm function.
+ */
+void *_rtld_tlsdesc(void *);
+void *_rtld_tlsdesc_dynamic(void *);
+
+void _exit(int);
+
+void
+init_pltgot(Obj_Entry *obj)
+{
+
+	if (obj->pltgot != NULL) {
+		obj->pltgot[1] = (Elf_Addr) obj;
+		obj->pltgot[2] = (Elf_Addr) &_rtld_bind_start;
+	}
+}
+
+int
+do_copy_relocations(Obj_Entry *dstobj)
+{
+	const Obj_Entry *srcobj, *defobj;
+	const Elf_Rela *relalim;
+	const Elf_Rela *rela;
+	const Elf_Sym *srcsym;
+	const Elf_Sym *dstsym;
+	const void *srcaddr;
+	const char *name;
+	void *dstaddr;
+	SymLook req;
+	size_t size;
+	int res;
+
+	/*
+	 * COPY relocs are invalid outside of the main program
+	 */
+	assert(dstobj->mainprog);
+
+	relalim = (const Elf_Rela *)((char *)dstobj->rela +
+	    dstobj->relasize);
+	for (rela = dstobj->rela; rela < relalim; rela++) {
+		if (ELF_R_TYPE(rela->r_info) != R_AARCH64_COPY)
+			continue;
+
+		dstaddr = (void *)(dstobj->relocbase + rela->r_offset);
+		dstsym = dstobj->symtab + ELF_R_SYM(rela->r_info);
+		name = dstobj->strtab + dstsym->st_name;
+		size = dstsym->st_size;
+
+		symlook_init(&req, name);
+		req.ventry = fetch_ventry(dstobj, ELF_R_SYM(rela->r_info));
+		req.flags = SYMLOOK_EARLY;
+
+		for (srcobj = dstobj->next; srcobj != NULL;
+		     srcobj = srcobj->next) {
+			res = symlook_obj(&req, srcobj);
+			if (res == 0) {
+				srcsym = req.sym_out;
+				defobj = req.defobj_out;
+				break;
+			}
+		}
+		if (srcobj == NULL) {
+			_rtld_error(
+"Undefined symbol \"%s\" referenced from COPY relocation in %s",
+			    name, dstobj->path);
+			return (-1);
+		}
+
+		srcaddr = (const void *)(defobj->relocbase + srcsym->st_value);
+		memcpy(dstaddr, srcaddr, size);
+	}
+
+	return (0);
+}
+
+struct tls_data {
+	int64_t index;
+	Obj_Entry *obj;
+	const Elf_Rela *rela;
+};
+
+static struct tls_data *
+reloc_tlsdesc_alloc(Obj_Entry *obj, const Elf_Rela *rela)
+{
+	struct tls_data *tlsdesc;
+
+	tlsdesc = xmalloc(sizeof(struct tls_data));
+	tlsdesc->index = -1;
+	tlsdesc->obj = obj;
+	tlsdesc->rela = rela;
+
+	return (tlsdesc);
+}
+
+/*
+ * Look up the symbol to find its tls index
+ */
+static int64_t
+rtld_tlsdesc_handle_locked(struct tls_data *tlsdesc, int flags,
+    RtldLockState *lockstate)
+{
+	const Elf_Rela *rela;
+	const Elf_Sym *def;
+	const Obj_Entry *defobj;
+	Obj_Entry *obj;
+
+	rela = tlsdesc->rela;
+	obj = tlsdesc->obj;
+
+	def = find_symdef(ELF_R_SYM(rela->r_info), obj, &defobj, flags, NULL,
+	    lockstate);
+	if (def == NULL)
+		rtld_die();
+
+	tlsdesc->index = defobj->tlsoffset + def->st_value + rela->r_addend;
+
+	return (tlsdesc->index);
+}
+
+int64_t
+rtld_tlsdesc_handle(struct tls_data *tlsdesc, int flags)
+{
+	RtldLockState lockstate;
+
+	/* We have already found the index, return it */
+	if (tlsdesc->index >= 0)
+		return (tlsdesc->index);
+
+	wlock_acquire(rtld_bind_lock, &lockstate);
+	/* tlsdesc->index may have been set by another thread */
+	if (tlsdesc->index == -1)
+		rtld_tlsdesc_handle_locked(tlsdesc, flags, &lockstate);
+	lock_release(rtld_bind_lock, &lockstate);
+
+	return (tlsdesc->index);
+}
+
+/*
+ * Process the PLT relocations.
+ */
+int
+reloc_plt(Obj_Entry *obj)
+{
+	const Elf_Rela *relalim;
+	const Elf_Rela *rela;
+
+	relalim = (const Elf_Rela *)((char *)obj->pltrela + obj->pltrelasize);
+	for (rela = obj->pltrela; rela < relalim; rela++) {
+		Elf_Addr *where;
+
+		where = (Elf_Addr *)(obj->relocbase + rela->r_offset);
+
+		switch(ELF_R_TYPE(rela->r_info)) {
+		case R_AARCH64_JUMP_SLOT:
+			*where += (Elf_Addr)obj->relocbase;
+			break;
+		case R_AARCH64_TLSDESC:
+			if (ELF_R_SYM(rela->r_info) == 0) {
+				where[0] = (Elf_Addr)_rtld_tlsdesc;
+				where[1] = obj->tlsoffset + rela->r_addend;
+			} else {
+				where[0] = (Elf_Addr)_rtld_tlsdesc_dynamic;
+				where[1] = (Elf_Addr)reloc_tlsdesc_alloc(obj,
+				    rela);
+			}
+			break;
+		default:
+			_rtld_error("Unknown relocation type %u in PLT",
+			    (unsigned int)ELF_R_TYPE(rela->r_info));
+			return (-1);
+		}
+	}
+
+	return (0);
+}
+
+/*
+ * LD_BIND_NOW was set - force relocation for all jump slots
+ */
+int
+reloc_jmpslots(Obj_Entry *obj, int flags, RtldLockState *lockstate)
+{
+	const Obj_Entry *defobj;
+	const Elf_Rela *relalim;
+	const Elf_Rela *rela;
+	const Elf_Sym *def;
+	struct tls_data *tlsdesc;
+
+	relalim = (const Elf_Rela *)((char *)obj->pltrela + obj->pltrelasize);
+	for (rela = obj->pltrela; rela < relalim; rela++) {
+		Elf_Addr *where;
+
+		where = (Elf_Addr *)(obj->relocbase + rela->r_offset);
+		switch(ELF_R_TYPE(rela->r_info)) {
+		case R_AARCH64_JUMP_SLOT:
+			def = find_symdef(ELF_R_SYM(rela->r_info), obj,
+			    &defobj, SYMLOOK_IN_PLT | flags, NULL, lockstate);
+			if (def == NULL) {
+				dbg("reloc_jmpslots: sym not found");
+				return (-1);
+			}
+
+			*where = (Elf_Addr)(defobj->relocbase + def->st_value);
+			break;
+		case R_AARCH64_TLSDESC:
+			if (ELF_R_SYM(rela->r_info) != 0) {
+				tlsdesc = (struct tls_data *)where[1];
+				if (tlsdesc->index == -1)
+					rtld_tlsdesc_handle_locked(tlsdesc,
+					    SYMLOOK_IN_PLT | flags, lockstate);
+			}
+			break;
+		default:
+			_rtld_error("Unknown relocation type %x in jmpslot",
+			    (unsigned int)ELF_R_TYPE(rela->r_info));
+			return (-1);
+		}
+	}
+
+	return (0);
+}
+
+int
+reloc_iresolve(Obj_Entry *obj, struct Struct_RtldLockState *lockstate)
+{
+
+	/* XXX not implemented */
+	return (0);
+}
+
+int
+reloc_gnu_ifunc(Obj_Entry *obj, int flags,
+   struct Struct_RtldLockState *lockstate)
+{
+
+	/* XXX not implemented */
+	return (0);
+}
+
+Elf_Addr
+reloc_jmpslot(Elf_Addr *where, Elf_Addr target, const Obj_Entry *defobj,
+    const Obj_Entry *obj, const Elf_Rel *rel)
+{
+
+	assert(ELF_R_TYPE(rel->r_info) == R_AARCH64_JUMP_SLOT);
+
+	if (*where != target)
+		*where = target;
+
+	return target;
+}
+
+/*
+ * Process non-PLT relocations
+ */
+int
+reloc_non_plt(Obj_Entry *obj, Obj_Entry *obj_rtld, int flags,
+    RtldLockState *lockstate)
+{
+	const Obj_Entry *defobj;
+	const Elf_Rela *relalim;
+	const Elf_Rela *rela;
+	const Elf_Sym *def;
+	SymCache *cache;
+	Elf_Addr *where;
+	unsigned long symnum;
+
+	if ((flags & SYMLOOK_IFUNC) != 0)
+		/* XXX not implemented */
+		return (0);
+
+	/*
+	 * The dynamic loader may be called from a thread, we have
+	 * limited amounts of stack available so we cannot use alloca().
+	 */
+	if (obj == obj_rtld)
+		cache = NULL;
+	else
+		cache = calloc(obj->dynsymcount, sizeof(SymCache));
+		/* No need to check for NULL here */
+
+	relalim = (const Elf_Rela *)((caddr_t)obj->rela + obj->relasize);
+	for (rela = obj->rela; rela < relalim; rela++) {
+		where = (Elf_Addr *)(obj->relocbase + rela->r_offset);
+		symnum = ELF_R_SYM(rela->r_info);
+
+		switch (ELF_R_TYPE(rela->r_info)) {
+		case R_AARCH64_ABS64:
+		case R_AARCH64_GLOB_DAT:
+			def = find_symdef(symnum, obj, &defobj, flags, cache,
+			    lockstate);
+			if (def == NULL)
+				return (-1);
+
+			*where = (Elf_Addr)defobj->relocbase + def->st_value;
+			break;
+		case R_AARCH64_COPY:
+			/*
+			 * These are deferred until all other relocations have
+			 * been done. All we do here is make sure that the
+			 * COPY relocation is not in a shared library. They
+			 * are allowed only in executable files.
+			 */
+			if (!obj->mainprog) {
+				_rtld_error("%s: Unexpected R_AARCH64_COPY "
+				    "relocation in shared library", obj->path);
+				return (-1);
+			}
+			break;
+		case R_AARCH64_TLS_TPREL64:
+			def = find_symdef(symnum, obj, &defobj, flags, cache,
+			    lockstate);
+			if (def == NULL)
+				return (-1);
+
+			/*
+			 * We lazily allocate offsets for static TLS as we
+			 * see the first relocation that references the
+			 * TLS block. This allows us to support (small
+			 * amounts of) static TLS in dynamically loaded
+			 * modules. If we run out of space, we generate an
+			 * error.
+			 */
+			if (!defobj->tls_done) {
+				if (!allocate_tls_offset((Obj_Entry*) defobj)) {
+					_rtld_error(
+					    "%s: No space available for static "
+					    "Thread Local Storage", obj->path);
+					return (-1);
+				}
+			}
+
+			*where = def->st_value + rela->r_addend +
+			    defobj->tlsoffset - TLS_TCB_SIZE;
+			break;
+		case R_AARCH64_RELATIVE:
+			*where = (Elf_Addr)(obj->relocbase + rela->r_addend);
+			break;
+		default:
+			rtld_printf("%s: Unhandled relocation %lu\n",
+			    obj->path, ELF_R_TYPE(rela->r_info));
+			return (-1);
+		}
+	}
+
+	return (0);
+}
+
+void
+allocate_initial_tls(Obj_Entry *objs)
+{
+	Elf_Addr **tp;
+
+	/*
+	* Fix the size of the static TLS block by using the maximum
+	* offset allocated so far and adding a bit for dynamic modules to
+	* use.
+	*/
+	tls_static_space = tls_last_offset + tls_last_size +
+	    RTLD_STATIC_TLS_EXTRA;
+
+	tp = (Elf_Addr **) allocate_tls(objs, NULL, TLS_TCB_SIZE, 16);
+
+	asm volatile("msr	tpidr_el0, %0" : : "r"(tp));
+}
diff --git a/libexec/rtld-elf/aarch64/rtld_machdep.h b/libexec/rtld-elf/aarch64/rtld_machdep.h
new file mode 100644
index 0000000..1cb2029
--- /dev/null
+++ b/libexec/rtld-elf/aarch64/rtld_machdep.h
@@ -0,0 +1,83 @@
+/*-
+ * Copyright (c) 1999, 2000 John D. Polstra.
+ * Copyright (c) 2014 the FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by Andrew Turner
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef RTLD_MACHDEP_H
+#define	RTLD_MACHDEP_H	1
+
+#include <sys/types.h>
+#include <machine/atomic.h>
+
+struct Struct_Obj_Entry;
+
+/* Return the address of the .dynamic section in the dynamic linker. */
+#define	rtld_dynamic(obj)						\
+({									\
+	Elf_Addr _dynamic_addr;						\
+	asm volatile("adr	%0, _DYNAMIC" : "=&r"(_dynamic_addr));	\
+	(const Elf_Dyn *)_dynamic_addr;					\
+})
+#define	RTLD_IS_DYNAMIC() (1)
+
+Elf_Addr reloc_jmpslot(Elf_Addr *where, Elf_Addr target,
+		       const struct Struct_Obj_Entry *defobj,
+		       const struct Struct_Obj_Entry *obj,
+		       const Elf_Rel *rel);
+
+#define	make_function_pointer(def, defobj) \
+	((defobj)->relocbase + (def)->st_value)
+
+#define	call_initfini_pointer(obj, target) \
+	(((InitFunc)(target))())
+
+#define	call_init_pointer(obj, target) \
+	(((InitArrFunc)(target))(main_argc, main_argv, environ))
+
+#define	round(size, align) \
+	(((size) + (align) - 1) & ~((align) - 1))
+#define	calculate_first_tls_offset(size, align) \
+	round(size, align)
+#define	calculate_tls_offset(prev_offset, prev_size, size, align) \
+	round((prev_offset) + (size), align)
+#define	calculate_tls_end(off, size) 	((off) + (size))
+
+#define	TLS_TCB_SIZE	8
+typedef struct {
+    unsigned long ti_module;
+    unsigned long ti_offset;
+} tls_index;
+
+extern void *__tls_get_addr(tls_index *ti);
+
+#define	RTLD_DEFAULT_STACK_PF_EXEC	PF_X
+#define	RTLD_DEFAULT_STACK_EXEC		PROT_EXEC
+
+#endif
diff --git a/libexec/rtld-elf/aarch64/rtld_start.S b/libexec/rtld-elf/aarch64/rtld_start.S
new file mode 100644
index 0000000..bc45ac7
--- /dev/null
+++ b/libexec/rtld-elf/aarch64/rtld_start.S
@@ -0,0 +1,152 @@
+/*-
+ * Copyright (c) 2014 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Andrew Turner under
+ * sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+ENTRY(.rtld_start)
+	mov	x19, x0		/* Put ps_strings in a callee-saved register */
+	mov	x20, sp		/* And the stack pointer */
+
+	sub	x8, x20, #16	/* Make room for obj_main & exit proc */
+	mov	sp, x8		/* Update the stack pointer */
+
+	mov	x0, x20		/* Pass the stack we were given to _rtld */
+	mov	x1, sp		/* exit_proc */
+	add	x2, x1, #8	/* obj_main */
+	bl	_rtld		/* Call the loader */
+	mov	x8, x0		/* Backup the entry point */
+
+	ldr	x2, [sp]	/* Load cleanup */
+	ldr	x1, [sp, #8]	/* Load obj_main */
+	mov	x0, x19		/* Restore ps_strings */
+	mov	sp, x20		/* Restore the stack pointer */
+	br	x8		/* Jump to the entry point */
+END(.rtld_start)
+
+/*
+ * sp + 0 = &GOT[x + 3]
+ * sp + 8 = RA
+ * x16 = &GOT[2]
+ * x17 = &_rtld_bind_start
+ */
+ENTRY(_rtld_bind_start)
+	mov	x17, sp
+
+	/* Save the arguments */
+	stp	x0, x1, [sp, #-16]!
+	stp	x2, x3, [sp, #-16]!
+	stp	x4, x5, [sp, #-16]!
+	stp	x6, x7, [sp, #-16]!
+
+	/* Calculate reloff */
+	ldr	x2, [x17, #0]	/* Get the address of the entry */
+	sub	x1, x2, x16	/* Find its offset */
+	sub	x1, x1, #8	/* Adjust for x16 not being at offset 0 */
+	/* Each rela item has 3 entriesso we need reloff = 3 * index */
+	lsl	x3, x1, #1	/* x3 = 2 * offset */
+	add	x1, x1, x3	/* x1 = x3 + offset = 3 * offset */
+
+	/* Load obj */
+	ldr	x0, [x16, #-8]
+
+	/* Call into rtld */
+	bl	_rtld_bind
+
+	/* Restore the registers saved by the plt code */
+	ldp	xzr, x30, [sp, #(4 * 16)]
+
+	/* Backup the address to branch to */
+	mov	x16, x0
+
+	/* restore the arguments */
+	ldp	x6, x7, [sp], #16
+	ldp	x4, x5, [sp], #16
+	ldp	x2, x3, [sp], #16
+	ldp	x0, x1, [sp], #16
+	/* And the part of the stack the plt entry handled */
+	add	sp, sp, #16
+
+	/* Call into the correct function */
+	br	x16
+END(_rtld_bind_start)
+
+/*
+ * uint64_t _rtld_tlsdesc(struct tlsdesc *);
+ *
+ * struct tlsdesc {
+ *  uint64_t ptr;
+ *  uint64_t data;
+ * };
+ *
+ * Returns the data.
+ */
+ENTRY(_rtld_tlsdesc)
+	ldr	x0, [x0, #8]
+	ret
+END(_rtld_tlsdesc)
+
+/*
+ * uint64_t _rtld_tlsdesc_dynamic(struct tlsdesc *);
+ *
+ * TODO: We could lookup the saved index here to skip saving the entire stack.
+ */
+ENTRY(_rtld_tlsdesc_dynamic)
+	/* Store any registers we may use in rtld_tlsdesc_handle */
+	stp	x29, x30, [sp, #-(10 * 16)]!
+	mov	x29, sp
+	stp	x1, x2,   [sp, #(1 * 16)]
+	stp	x3, x4,   [sp, #(2 * 16)]
+	stp	x5, x6,   [sp, #(3 * 16)]
+	stp	x7, x8,   [sp, #(4 * 16)]
+	stp	x9, x10,  [sp, #(5 * 16)]
+	stp	x11, x12, [sp, #(6 * 16)]
+	stp	x13, x14, [sp, #(7 * 16)]
+	stp	x15, x16, [sp, #(8 * 16)]
+	stp	x17, x18, [sp, #(9 * 16)]
+
+	/* Find the tls offset */
+	ldr	x0, [x0, #8]
+	mov	x1, #1
+	bl	rtld_tlsdesc_handle
+
+	/* Restore the registers */
+	ldp	x17, x18, [sp, #(9 * 16)]
+	ldp	x15, x16, [sp, #(8 * 16)]
+	ldp	x13, x14, [sp, #(7 * 16)]
+	ldp	x11, x12, [sp, #(6 * 16)]
+	ldp	x9, x10,  [sp, #(5 * 16)]
+	ldp	x7, x8,   [sp, #(4 * 16)]
+	ldp	x5, x6,   [sp, #(3 * 16)]
+	ldp	x3, x4,   [sp, #(2 * 16)]
+	ldp	x1, x2,   [sp, #(1 * 16)]
+	ldp	x29, x30, [sp], #(10 * 16)
+
+	ret
+END(_rtld_tlsdesc_dynamic)
diff --git a/libexec/rtld-elf/amd64/reloc.c b/libexec/rtld-elf/amd64/reloc.c
index 35f33cc..53d2bdc 100644
--- a/libexec/rtld-elf/amd64/reloc.c
+++ b/libexec/rtld-elf/amd64/reloc.c
@@ -47,6 +47,7 @@
 
 #include "debug.h"
 #include "rtld.h"
+#include "rtld_tls.h"
 
 /*
  * Process the special R_X86_64_COPY relocations in the main program.  These
diff --git a/libexec/rtld-elf/amd64/rtld_machdep.h b/libexec/rtld-elf/amd64/rtld_machdep.h
index 7b5d4d2..cb5e9a1 100644
--- a/libexec/rtld-elf/amd64/rtld_machdep.h
+++ b/libexec/rtld-elf/amd64/rtld_machdep.h
@@ -74,7 +74,7 @@ typedef struct {
     unsigned long ti_offset;
 } tls_index;
 
-extern void *__tls_get_addr(tls_index *ti);
+void *__tls_get_addr(tls_index *ti) __exported;
 
 #define	RTLD_DEFAULT_STACK_PF_EXEC	PF_X
 #define	RTLD_DEFAULT_STACK_EXEC		PROT_EXEC
diff --git a/libexec/rtld-elf/amd64/rtld_start.S b/libexec/rtld-elf/amd64/rtld_start.S
index 2481f09..387d26c 100644
--- a/libexec/rtld-elf/amd64/rtld_start.S
+++ b/libexec/rtld-elf/amd64/rtld_start.S
@@ -36,7 +36,7 @@
 	movq	%rsp,%rsi		# save address of exit proc
 	movq	%rsp,%rdx		# construct address of obj_main
 	addq	$8,%rdx
-	call	_rtld@PLT		# Call rtld(sp); returns entry point
+	call	_rtld			# Call rtld(sp); returns entry point
 	popq	%rsi			# Get exit procedure address
 	movq	%r12,%rdi		# *ap
 /*
@@ -118,7 +118,7 @@ _rtld_bind_start:
 	leaq	(%rsi,%rsi,2),%rsi	# multiply by 3
 	leaq	(,%rsi,8),%rsi		# now 8, for 24 (sizeof Elf_Rela)
 
-	call	_rtld_bind@PLT		# Transfer control to the binder
+	call	_rtld_bind		# Transfer control to the binder
 	/* Now %rax contains the entry point of the function being called. */
 
 	movq	%rax,0x60(%rsp)		# Store target over reloff argument
diff --git a/libexec/rtld-elf/debug.h b/libexec/rtld-elf/debug.h
index 98fdfb4..ed65227 100644
--- a/libexec/rtld-elf/debug.h
+++ b/libexec/rtld-elf/debug.h
@@ -32,10 +32,6 @@
 #ifndef DEBUG_H
 #define DEBUG_H 1
 
-#ifndef __GNUC__
-#error "This file must be compiled with GCC"
-#endif
-
 #include <sys/cdefs.h>
 
 #include <string.h>
diff --git a/libexec/rtld-elf/i386/reloc.c b/libexec/rtld-elf/i386/reloc.c
index c1e0a39..55b6537 100644
--- a/libexec/rtld-elf/i386/reloc.c
+++ b/libexec/rtld-elf/i386/reloc.c
@@ -48,6 +48,7 @@
 
 #include "debug.h"
 #include "rtld.h"
+#include "rtld_tls.h"
 
 /*
  * Process the special R_386_COPY relocations in the main program.  These
diff --git a/libexec/rtld-elf/i386/rtld_machdep.h b/libexec/rtld-elf/i386/rtld_machdep.h
index dfbe2e1..5c328da 100644
--- a/libexec/rtld-elf/i386/rtld_machdep.h
+++ b/libexec/rtld-elf/i386/rtld_machdep.h
@@ -74,8 +74,8 @@ typedef struct {
     unsigned long ti_offset;
 } tls_index;
 
-extern void *___tls_get_addr(tls_index *ti) __attribute__((__regparm__(1)));
-extern void *__tls_get_addr(tls_index *ti);
+void *___tls_get_addr(tls_index *ti) __attribute__((__regparm__(1))) __exported;
+void *__tls_get_addr(tls_index *ti) __exported;
 
 #define	RTLD_DEFAULT_STACK_PF_EXEC	PF_X
 #define	RTLD_DEFAULT_STACK_EXEC		PROT_EXEC
diff --git a/libexec/rtld-elf/i386/rtld_start.S b/libexec/rtld-elf/i386/rtld_start.S
index e7df748..87dca0e 100644
--- a/libexec/rtld-elf/i386/rtld_start.S
+++ b/libexec/rtld-elf/i386/rtld_start.S
@@ -42,7 +42,7 @@
 	pushl	%ecx			# Pass address of obj_main
 	pushl	%ebx			# Pass address of exit proc
 	pushl	%eax			# Pass initial stack pointer to rtld
-	call	_rtld@PLT		# Call rtld(sp); returns entry point
+	call	_rtld			# Call rtld(sp); returns entry point
 	addl	$16,%esp		# Remove arguments from stack
 	popl	%edx			# Get exit procedure address
 	movl	%esi,%esp		# Ignore obj_main
@@ -78,7 +78,7 @@ _rtld_bind_start:
 	pushl	20(%esp)		# Copy reloff argument
 	pushl	20(%esp)		# Copy obj argument
 
-	call	_rtld_bind@PLT		# Transfer control to the binder
+	call	_rtld_bind		# Transfer control to the binder
 	/* Now %eax contains the entry point of the function being called. */
 
 	addl	$8,%esp			# Discard binder arguments
diff --git a/libexec/rtld-elf/mips/reloc.c b/libexec/rtld-elf/mips/reloc.c
index 4e750d7..809adb8 100644
--- a/libexec/rtld-elf/mips/reloc.c
+++ b/libexec/rtld-elf/mips/reloc.c
@@ -245,7 +245,7 @@ _mips_rtld_bind(Obj_Entry *obj, Elf_Size reloff)
         def = find_symdef(reloff, obj, &defobj, SYMLOOK_IN_PLT, NULL,
 	    NULL);
         if (def == NULL)
-		_rtld_error("bind failed no symbol");
+		rtld_die();
 
         target = (Elf_Addr)(defobj->relocbase + def->st_value);
         dbg("bind now/fixup at %s sym # %jd in %s --> was=%p new=%p",
diff --git a/libexec/rtld-elf/powerpc/reloc.c b/libexec/rtld-elf/powerpc/reloc.c
index 89e5536..1fe9676 100644
--- a/libexec/rtld-elf/powerpc/reloc.c
+++ b/libexec/rtld-elf/powerpc/reloc.c
@@ -483,7 +483,7 @@ reloc_jmpslot(Elf_Addr *wherep, Elf_Addr target, const Obj_Entry *defobj,
 	 */
 	offset = target - (Elf_Addr)wherep;
 
-	if (abs(offset) < 32*1024*1024) {     /* inside 32MB? */
+	if (abs((int)offset) < 32*1024*1024) {     /* inside 32MB? */
 		/* b    value   # branch directly */
 		*wherep = 0x48000000 | (offset & 0x03fffffc);
 		__syncicache(wherep, 4);
@@ -622,8 +622,7 @@ init_pltgot(Obj_Entry *obj)
 void
 allocate_initial_tls(Obj_Entry *list)
 {
-	register Elf_Addr **tp __asm__("r2");
-	Elf_Addr **_tp;
+	Elf_Addr **tp;
 
 	/*
 	* Fix the size of the static TLS block by using the maximum
@@ -633,22 +632,23 @@ allocate_initial_tls(Obj_Entry *list)
 
 	tls_static_space = tls_last_offset + tls_last_size + RTLD_STATIC_TLS_EXTRA;
 
-	_tp = (Elf_Addr **) ((char *) allocate_tls(list, NULL, TLS_TCB_SIZE, 8) 
+	tp = (Elf_Addr **) ((char *) allocate_tls(list, NULL, TLS_TCB_SIZE, 8) 
 	    + TLS_TP_OFFSET + TLS_TCB_SIZE);
 
 	/*
 	 * XXX gcc seems to ignore 'tp = _tp;' 
 	 */
 	 
-	__asm __volatile("mr %0,%1" : "=r"(tp) : "r"(_tp));
+	__asm __volatile("mr 2,%0" :: "r"(tp));
 }
 
 void*
 __tls_get_addr(tls_index* ti)
 {
-	register Elf_Addr **tp __asm__("r2");
+	register Elf_Addr **tp;
 	char *p;
 
+	__asm __volatile("mr %0,2" : "=r"(tp));
 	p = tls_get_addr_common((Elf_Addr**)((Elf_Addr)tp - TLS_TP_OFFSET 
 	    - TLS_TCB_SIZE), ti->ti_module, ti->ti_offset);
 
diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index cc7afda..1d91460 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -34,10 +34,6 @@
  * John Polstra <jdp@polstra.com>.
  */
 
-#ifndef __GNUC__
-#error "GCC is needed to compile this file"
-#endif
-
 #include <sys/param.h>
 #include <sys/mount.h>
 #include <sys/mman.h>
@@ -78,7 +74,6 @@ typedef void * (*path_enum_proc) (const char *path, size_t len, void *arg);
  * Function declarations.
  */
 static const char *basename(const char *);
-static void die(void) __dead2;
 static void digest_dynamic1(Obj_Entry *, int, const Elf_Dyn **,
     const Elf_Dyn **, const Elf_Dyn **);
 static void digest_dynamic2(Obj_Entry *, const Elf_Dyn *, const Elf_Dyn *,
@@ -149,8 +144,10 @@ static void unlink_object(Obj_Entry *);
 static void unload_object(Obj_Entry *);
 static void unref_dag(Obj_Entry *);
 static void ref_dag(Obj_Entry *);
-static char *origin_subst_one(char *, const char *, const char *, bool);
-static char *origin_subst(char *, const char *);
+static char *origin_subst_one(Obj_Entry *, char *, const char *,
+    const char *, bool);
+static char *origin_subst(Obj_Entry *, char *);
+static bool obj_resolve_origin(Obj_Entry *obj);
 static void preinit_main(void);
 static int  rtld_verify_versions(const Objlist *);
 static int  rtld_verify_object_versions(Obj_Entry *);
@@ -163,14 +160,16 @@ static uint32_t gnu_hash(const char *);
 static bool matched_symbol(SymLook *, const Obj_Entry *, Sym_Match_Result *,
     const unsigned long);
 
-void r_debug_state(struct r_debug *, struct link_map *) __noinline;
-void _r_debug_postinit(struct link_map *) __noinline;
+void r_debug_state(struct r_debug *, struct link_map *) __noinline __exported;
+void _r_debug_postinit(struct link_map *) __noinline __exported;
+
+int __sys_openat(int, const char *, int, ...);
 
 /*
  * Data declarations.
  */
 static char *error_message;	/* Message for dlerror(), or NULL */
-struct r_debug r_debug;		/* for GDB; */
+struct r_debug r_debug __exported;	/* for GDB; */
 static bool libmap_disable;	/* Disable libmap */
 static bool ld_loadfltr;	/* Immediate filters processing */
 static char *libmap_override;	/* Maps to use in addition to libmap.conf */
@@ -210,6 +209,23 @@ extern Elf_Dyn _DYNAMIC;
 #define	RTLD_IS_DYNAMIC()	(&_DYNAMIC != NULL)
 #endif
 
+int dlclose(void *) __exported;
+char *dlerror(void) __exported;
+void *dlopen(const char *, int) __exported;
+void *fdlopen(int, int) __exported;
+void *dlsym(void *, const char *) __exported;
+dlfunc_t dlfunc(void *, const char *) __exported;
+void *dlvsym(void *, const char *, const char *) __exported;
+int dladdr(const void *, Dl_info *) __exported;
+void dllockinit(void *, void *(*)(void *), void (*)(void *), void (*)(void *),
+    void (*)(void *), void (*)(void *), void (*)(void *)) __exported;
+int dlinfo(void *, int , void *) __exported;
+int dl_iterate_phdr(__dl_iterate_hdr_callback, void *) __exported;
+int _rtld_addr_phdr(const void *, struct dl_phdr_info *) __exported;
+int _rtld_get_stack_prot(void) __exported;
+int _rtld_is_dlopened(void *) __exported;
+void _rtld_error(const char *, ...) __exported;
+
 int npagesizes, osreldate;
 size_t *pagesizes;
 
@@ -264,6 +280,8 @@ bool ld_library_path_rpath = false;
 #define	UTRACE_PRELOAD_FINISHED		8
 #define	UTRACE_INIT_CALL		9
 #define	UTRACE_FINI_CALL		10
+#define	UTRACE_DLSYM_START		11
+#define	UTRACE_DLSYM_STOP		12
 
 struct utrace_rtld {
 	char sig[4];			/* 'RTLD' */
@@ -409,7 +427,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 	    unsetenv(LD_ "DEBUG") || unsetenv(LD_ "ELF_HINTS_PATH") ||
 	    unsetenv(LD_ "LOADFLTR") || unsetenv(LD_ "LIBRARY_PATH_RPATH")) {
 		_rtld_error("environment corrupt; aborting");
-		die();
+		rtld_die();
 	}
     }
     ld_debug = getenv(LD_ "DEBUG");
@@ -458,7 +476,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 	obj_main = map_object(fd, argv0, NULL);
 	close(fd);
 	if (obj_main == NULL)
-	    die();
+	    rtld_die();
 	max_stack_flags = obj->stack_flags;
     } else {				/* Main program already loaded. */
 	const Elf_Phdr *phdr;
@@ -475,7 +493,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 	assert(aux_info[AT_ENTRY] != NULL);
 	entry = (caddr_t) aux_info[AT_ENTRY]->a_un.a_ptr;
 	if ((obj_main = digest_phdr(phdr, phnum, entry, argv0)) == NULL)
-	    die();
+	    rtld_die();
     }
 
     if (aux_info[AT_EXECPATH] != 0) {
@@ -503,6 +521,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
       aux_info[AT_STACKPROT]->a_un.a_val != 0)
 	    stack_prot = aux_info[AT_STACKPROT]->a_un.a_val;
 
+#ifndef COMPAT_32BIT
     /*
      * Get the actual dynamic linker pathname from the executable if
      * possible.  (It should always be possible.)  That ensures that
@@ -515,6 +534,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 	obj_rtld.path = xstrdup(obj_main->interp);
         __progname = obj_rtld.path;
     }
+#endif
 
     digest_dynamic(obj_main, 0);
     dbg("%s valid_hash_sysv %d valid_hash_gnu %d dynsymcount %d",
@@ -540,12 +560,12 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 
     dbg("loading LD_PRELOAD libraries");
     if (load_preload_objects() == -1)
-	die();
+	rtld_die();
     preload_tail = obj_tail;
 
     dbg("loading needed objects");
     if (load_needed_objects(obj_main, 0) == -1)
-	die();
+	rtld_die();
 
     /* Make a list of all objects loaded at startup. */
     last_interposer = obj_main;
@@ -561,7 +581,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 
     dbg("checking for required versions");
     if (rtld_verify_versions(&list_main) == -1 && !ld_tracing)
-	die();
+	rtld_die();
 
     if (ld_tracing) {		/* We're done */
 	trace_loaded_objects(obj_main);
@@ -590,11 +610,11 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     if (relocate_objects(obj_main,
       ld_bind_now != NULL && *ld_bind_now != '\0',
       &obj_rtld, SYMLOOK_EARLY, NULL) == -1)
-	die();
+	rtld_die();
 
     dbg("doing copy relocations");
     if (do_copy_relocations(obj_main) == -1)
-	die();
+	rtld_die();
 
     if (getenv(LD_ "DUMP_REL_POST") != NULL) {
        dump_relocations(obj_main);
@@ -626,7 +646,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     if (resolve_objects_ifunc(obj_main,
       ld_bind_now != NULL && *ld_bind_now != '\0', SYMLOOK_EARLY,
       NULL) == -1)
-	die();
+	rtld_die();
 
     if (!obj_main->crt_no_init) {
 	/*
@@ -693,7 +713,7 @@ _rtld_bind(Obj_Entry *obj, Elf_Size reloff)
     def = find_symdef(ELF_R_SYM(rel->r_info), obj, &defobj, true, NULL,
 	&lockstate);
     if (def == NULL)
-	die();
+	rtld_die();
     if (ELF_ST_TYPE(def->st_info) == STT_GNU_IFUNC)
 	target = (Elf_Addr)rtld_resolve_ifunc(defobj, def);
     else
@@ -766,8 +786,8 @@ basename(const char *name)
 static struct utsname uts;
 
 static char *
-origin_subst_one(char *real, const char *kw, const char *subst,
-    bool may_free)
+origin_subst_one(Obj_Entry *obj, char *real, const char *kw,
+    const char *subst, bool may_free)
 {
 	char *p, *p1, *res, *resp;
 	int subst_len, kw_len, subst_count, old_len, new_len;
@@ -786,9 +806,15 @@ origin_subst_one(char *real, const char *kw, const char *subst,
 
 	/*
 	 * If the keyword is not found, just return.
+	 *
+	 * Return non-substituted string if resolution failed.  We
+	 * cannot do anything more reasonable, the failure mode of the
+	 * caller is unresolved library anyway.
 	 */
-	if (subst_count == 0)
+	if (subst_count == 0 || (obj != NULL && !obj_resolve_origin(obj)))
 		return (may_free ? real : xstrdup(real));
+	if (obj != NULL)
+		subst = obj->origin_path;
 
 	/*
 	 * There is indeed something to substitute.  Calculate the
@@ -825,25 +851,27 @@ origin_subst_one(char *real, const char *kw, const char *subst,
 }
 
 static char *
-origin_subst(char *real, const char *origin_path)
+origin_subst(Obj_Entry *obj, char *real)
 {
 	char *res1, *res2, *res3, *res4;
 
+	if (obj == NULL || !trust)
+		return (xstrdup(real));
 	if (uts.sysname[0] == '\0') {
 		if (uname(&uts) != 0) {
 			_rtld_error("utsname failed: %d", errno);
 			return (NULL);
 		}
 	}
-	res1 = origin_subst_one(real, "$ORIGIN", origin_path, false);
-	res2 = origin_subst_one(res1, "$OSNAME", uts.sysname, true);
-	res3 = origin_subst_one(res2, "$OSREL", uts.release, true);
-	res4 = origin_subst_one(res3, "$PLATFORM", uts.machine, true);
+	res1 = origin_subst_one(obj, real, "$ORIGIN", NULL, false);
+	res2 = origin_subst_one(NULL, res1, "$OSNAME", uts.sysname, true);
+	res3 = origin_subst_one(NULL, res2, "$OSREL", uts.release, true);
+	res4 = origin_subst_one(NULL, res3, "$PLATFORM", uts.machine, true);
 	return (res4);
 }
 
-static void
-die(void)
+void
+rtld_die(void)
 {
     const char *msg = dlerror();
 
@@ -870,7 +898,6 @@ digest_dynamic1(Obj_Entry *obj, int early, const Elf_Dyn **dyn_rpath,
     const Elf32_Word *hashval;
     Elf32_Word bkt, nmaskwords;
     int bloom_size32;
-    bool nmw_power2;
     int plttype = DT_REL;
 
     *dyn_rpath = NULL;
@@ -980,16 +1007,15 @@ digest_dynamic1(Obj_Entry *obj, int early, const Elf_Dyn **dyn_rpath,
 		obj->symndx_gnu = hashtab[1];
 		nmaskwords = hashtab[2];
 		bloom_size32 = (__ELF_WORD_SIZE / 32) * nmaskwords;
-		/* Number of bitmask words is required to be power of 2 */
-		nmw_power2 = ((nmaskwords & (nmaskwords - 1)) == 0);
 		obj->maskwords_bm_gnu = nmaskwords - 1;
 		obj->shift2_gnu = hashtab[3];
 		obj->bloom_gnu = (Elf_Addr *) (hashtab + 4);
 		obj->buckets_gnu = hashtab + 4 + bloom_size32;
 		obj->chain_zero_gnu = obj->buckets_gnu + obj->nbuckets_gnu -
 		  obj->symndx_gnu;
-		obj->valid_hash_gnu = nmw_power2 && obj->nbuckets_gnu > 0 &&
-		  obj->buckets_gnu != NULL;
+		/* Number of bitmask words is required to be power of 2 */
+		obj->valid_hash_gnu = powerof2(nmaskwords) &&
+		    obj->nbuckets_gnu > 0 && obj->buckets_gnu != NULL;
 	    }
 	    break;
 
@@ -1104,7 +1130,7 @@ digest_dynamic1(Obj_Entry *obj, int early, const Elf_Dyn **dyn_rpath,
 #endif
 
 	case DT_FLAGS:
-		if ((dynp->d_un.d_val & DF_ORIGIN) && trust)
+		if (dynp->d_un.d_val & DF_ORIGIN)
 		    obj->z_origin = true;
 		if (dynp->d_un.d_val & DF_SYMBOLIC)
 		    obj->symbolic = true;
@@ -1136,10 +1162,10 @@ digest_dynamic1(Obj_Entry *obj, int early, const Elf_Dyn **dyn_rpath,
 	case DT_FLAGS_1:
 		if (dynp->d_un.d_val & DF_1_NOOPEN)
 		    obj->z_noopen = true;
-		if ((dynp->d_un.d_val & DF_1_ORIGIN) && trust)
+		if (dynp->d_un.d_val & DF_1_ORIGIN)
 		    obj->z_origin = true;
-		/*if (dynp->d_un.d_val & DF_1_GLOBAL)
-		    XXX ;*/
+		if (dynp->d_un.d_val & DF_1_GLOBAL)
+		    obj->z_global = true;
 		if (dynp->d_un.d_val & DF_1_BIND_NOW)
 		    obj->bind_now = true;
 		if (dynp->d_un.d_val & DF_1_NODELETE)
@@ -1187,30 +1213,33 @@ digest_dynamic1(Obj_Entry *obj, int early, const Elf_Dyn **dyn_rpath,
     }
 }
 
+static bool
+obj_resolve_origin(Obj_Entry *obj)
+{
+
+	if (obj->origin_path != NULL)
+		return (true);
+	obj->origin_path = xmalloc(PATH_MAX);
+	return (rtld_dirname_abs(obj->path, obj->origin_path) != -1);
+}
+
 static void
 digest_dynamic2(Obj_Entry *obj, const Elf_Dyn *dyn_rpath,
     const Elf_Dyn *dyn_soname, const Elf_Dyn *dyn_runpath)
 {
 
-    if (obj->z_origin && obj->origin_path == NULL) {
-	obj->origin_path = xmalloc(PATH_MAX);
-	if (rtld_dirname_abs(obj->path, obj->origin_path) == -1)
-	    die();
-    }
-
-    if (dyn_runpath != NULL) {
-	obj->runpath = (char *)obj->strtab + dyn_runpath->d_un.d_val;
-	if (obj->z_origin)
-	    obj->runpath = origin_subst(obj->runpath, obj->origin_path);
-    }
-    else if (dyn_rpath != NULL) {
-	obj->rpath = (char *)obj->strtab + dyn_rpath->d_un.d_val;
-	if (obj->z_origin)
-	    obj->rpath = origin_subst(obj->rpath, obj->origin_path);
-    }
+	if (obj->z_origin && !obj_resolve_origin(obj))
+		rtld_die();
 
-    if (dyn_soname != NULL)
-	object_add_name(obj, obj->strtab + dyn_soname->d_un.d_val);
+	if (dyn_runpath != NULL) {
+		obj->runpath = (char *)obj->strtab + dyn_runpath->d_un.d_val;
+		obj->runpath = origin_subst(obj, obj->runpath);
+	} else if (dyn_rpath != NULL) {
+		obj->rpath = (char *)obj->strtab + dyn_rpath->d_un.d_val;
+		obj->rpath = origin_subst(obj, obj->rpath);
+	}
+	if (dyn_soname != NULL)
+		object_add_name(obj, obj->strtab + dyn_soname->d_un.d_val);
 }
 
 static void
@@ -1460,12 +1489,8 @@ find_library(const char *xname, const Obj_Entry *refobj, int *fdp)
 	      xname);
 	    return NULL;
 	}
-	if (objgiven && refobj->z_origin) {
-		return (origin_subst(__DECONST(char *, xname),
-		    refobj->origin_path));
-	} else {
-		return (xstrdup(xname));
-	}
+	return (origin_subst(__DECONST(Obj_Entry *, refobj),
+	  __DECONST(char *, xname)));
     }
 
     if (libmap_disable || !objgiven ||
@@ -1770,22 +1795,35 @@ init_dag(Obj_Entry *root)
 }
 
 static void
-process_nodelete(Obj_Entry *root)
+process_z(Obj_Entry *root)
 {
 	const Objlist_Entry *elm;
+	Obj_Entry *obj;
 
 	/*
-	 * Walk over object DAG and process every dependent object that
-	 * is marked as DF_1_NODELETE. They need to grow their own DAG,
-	 * which then should have its reference upped separately.
+	 * Walk over object DAG and process every dependent object
+	 * that is marked as DF_1_NODELETE or DF_1_GLOBAL. They need
+	 * to grow their own DAG.
+	 *
+	 * For DF_1_GLOBAL, DAG is required for symbol lookups in
+	 * symlook_global() to work.
+	 *
+	 * For DF_1_NODELETE, the DAG should have its reference upped.
 	 */
 	STAILQ_FOREACH(elm, &root->dagmembers, link) {
-		if (elm->obj != NULL && elm->obj->z_nodelete &&
-		    !elm->obj->ref_nodel) {
-			dbg("obj %s nodelete", elm->obj->path);
-			init_dag(elm->obj);
-			ref_dag(elm->obj);
-			elm->obj->ref_nodel = true;
+		obj = elm->obj;
+		if (obj == NULL)
+			continue;
+		if (obj->z_nodelete && !obj->ref_nodel) {
+			dbg("obj %s -z nodelete", obj->path);
+			init_dag(obj);
+			ref_dag(obj);
+			obj->ref_nodel = true;
+		}
+		if (obj->z_global && objlist_find(&list_global, obj) == NULL) {
+			dbg("obj %s -z global", obj->path);
+			objlist_push_tail(&list_global, obj);
+			init_dag(obj);
 		}
 	}
 }
@@ -1893,7 +1931,7 @@ init_pagesizes(Elf_Auxinfo **aux_info)
 		}
 		if (sysctl(mib, len, psa, &size, NULL, 0) == -1) {
 			_rtld_error("sysctl for hw.pagesize(s) failed");
-			die();
+			rtld_die();
 		}
 psa_filled:
 		pagesizes = psa;
@@ -2132,7 +2170,7 @@ load_object(const char *name, int fd_u, const Obj_Entry *refobj, int flags)
 	 * To avoid a race, we open the file and use fstat() rather than
 	 * using stat().
 	 */
-	if ((fd = open(path, O_RDONLY | O_CLOEXEC)) == -1) {
+	if ((fd = open(path, O_RDONLY | O_CLOEXEC | O_VERIFY)) == -1) {
 	    _rtld_error("Cannot open \"%s\"", path);
 	    free(path);
 	    return (NULL);
@@ -2220,6 +2258,7 @@ do_load_object(int fd, const char *name, char *path, struct stat *sbp,
 	return (NULL);
     }
 
+    obj->dlopened = (flags & RTLD_LO_DLOPEN) != 0;
     *obj_tail = obj;
     obj_tail = &obj->next;
     obj_count++;
@@ -2821,7 +2860,7 @@ search_library_pathfds(const char *name, const char *path, int *fdp)
 		dirfd = parse_libdir(fdstr);
 		if (dirfd < 0)
 			break;
-		fd = openat(dirfd, name, O_RDONLY | O_CLOEXEC);
+		fd = __sys_openat(dirfd, name, O_RDONLY | O_CLOEXEC | O_VERIFY);
 		if (fd >= 0) {
 			*fdp = fd;
 			len = strlen(fdstr) + strlen(name) + 3;
@@ -2829,7 +2868,7 @@ search_library_pathfds(const char *name, const char *path, int *fdp)
 			if (rtld_snprintf(found, len, "#%d/%s", dirfd, name) < 0) {
 				_rtld_error("error generating '%d/%s'",
 				    dirfd, name);
-				die();
+				rtld_die();
 			}
 			dbg("open('%s') => %d", found, fd);
 			break;
@@ -3023,13 +3062,13 @@ dlopen_object(const char *name, int fd, Obj_Entry *refobj, int lo_flags,
 		initlist_add_objects(obj, &obj->next, &initlist);
 	    }
 	    /*
-	     * Process all no_delete objects here, given them own
-	     * DAGs to prevent their dependencies from being unloaded.
-	     * This has to be done after we have loaded all of the
-	     * dependencies, so that we do not miss any.
+	     * Process all no_delete or global objects here, given
+	     * them own DAGs to prevent their dependencies from being
+	     * unloaded.  This has to be done after we have loaded all
+	     * of the dependencies, so that we do not miss any.
 	     */
 	    if (obj != NULL)
-		process_nodelete(obj);
+		process_z(obj);
 	} else {
 	    /*
 	     * Bump the reference counts for objects on this DAG.  If
@@ -3094,6 +3133,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
     SymLook req;
     RtldLockState lockstate;
     tls_index ti;
+    void *sym;
     int res;
 
     def = NULL;
@@ -3103,6 +3143,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
     req.flags = flags | SYMLOOK_IN_PLT;
     req.lockstate = &lockstate;
 
+    LD_UTRACE(UTRACE_DLSYM_START, handle, NULL, 0, 0, name);
     rlock_acquire(rtld_bind_lock, &lockstate);
     if (sigsetjmp(lockstate.env, 0) != 0)
 	    lock_upgrade(rtld_bind_lock, &lockstate);
@@ -3112,6 +3153,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
 	if ((obj = obj_from_addr(retaddr)) == NULL) {
 	    _rtld_error("Cannot determine caller's shared object");
 	    lock_release(rtld_bind_lock, &lockstate);
+	    LD_UTRACE(UTRACE_DLSYM_STOP, handle, NULL, 0, 0, name);
 	    return NULL;
 	}
 	if (handle == NULL) {	/* Just the caller's shared object. */
@@ -3159,6 +3201,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
     } else {
 	if ((obj = dlcheck(handle)) == NULL) {
 	    lock_release(rtld_bind_lock, &lockstate);
+	    LD_UTRACE(UTRACE_DLSYM_STOP, handle, NULL, 0, 0, name);
 	    return NULL;
 	}
 
@@ -3202,19 +3245,22 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
 	 * symbol.
 	 */
 	if (ELF_ST_TYPE(def->st_info) == STT_FUNC)
-	    return (make_function_pointer(def, defobj));
+	    sym = make_function_pointer(def, defobj);
 	else if (ELF_ST_TYPE(def->st_info) == STT_GNU_IFUNC)
-	    return (rtld_resolve_ifunc(defobj, def));
+	    sym = rtld_resolve_ifunc(defobj, def);
 	else if (ELF_ST_TYPE(def->st_info) == STT_TLS) {
 	    ti.ti_module = defobj->tlsindex;
 	    ti.ti_offset = def->st_value;
-	    return (__tls_get_addr(&ti));
+	    sym = __tls_get_addr(&ti);
 	} else
-	    return (defobj->relocbase + def->st_value);
+	    sym = defobj->relocbase + def->st_value;
+	LD_UTRACE(UTRACE_DLSYM_STOP, handle, sym, 0, 0, name);
+	return (sym);
     }
 
     _rtld_error("Undefined symbol \"%s\"", name);
     lock_release(rtld_bind_lock, &lockstate);
+    LD_UTRACE(UTRACE_DLSYM_STOP, handle, NULL, 0, 0, name);
     return NULL;
 }
 
@@ -3549,17 +3595,16 @@ rtld_dirname(const char *path, char *bname)
 static int
 rtld_dirname_abs(const char *path, char *base)
 {
-	char base_rel[PATH_MAX];
+	char *last;
 
-	if (rtld_dirname(path, base) == -1)
+	if (realpath(path, base) == NULL)
 		return (-1);
-	if (base[0] == '/')
-		return (0);
-	if (getcwd(base_rel, sizeof(base_rel)) == NULL ||
-	    strlcat(base_rel, "/", sizeof(base_rel)) >= sizeof(base_rel) ||
-	    strlcat(base_rel, base, sizeof(base_rel)) >= sizeof(base_rel))
+	dbg("%s -> %s", path, base);
+	last = strrchr(base, '/');
+	if (last == NULL)
 		return (-1);
-	strcpy(base, base_rel);
+	if (last != base)
+		*last = '\0';
 	return (0);
 }
 
@@ -4340,7 +4385,8 @@ tls_get_addr_common(Elf_Addr **dtvp, int index, size_t offset)
 	return (tls_get_addr_slow(dtvp, index, offset));
 }
 
-#if defined(__arm__) || defined(__mips__) || defined(__powerpc__)
+#if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \
+    defined(__powerpc__)
 
 /*
  * Allocate Static TLS using the Variant I method.
@@ -4542,7 +4588,7 @@ allocate_module_tls(int index)
     }
     if (!obj) {
 	_rtld_error("Can't find module with TLS index %d", index);
-	die();
+	rtld_die();
     }
 
     p = malloc_aligned(obj->tlssize, obj->tlsalign);
@@ -4683,7 +4729,7 @@ locate_dependency(const Obj_Entry *obj, const char *name)
     }
     _rtld_error("%s: Unexpected inconsistency: dependency %s not found",
 	obj->path, name);
-    die();
+    rtld_die();
 }
 
 static int
@@ -4882,6 +4928,27 @@ _rtld_get_stack_prot(void)
 	return (stack_prot);
 }
 
+int
+_rtld_is_dlopened(void *arg)
+{
+	Obj_Entry *obj;
+	RtldLockState lockstate;
+	int res;
+
+	rlock_acquire(rtld_bind_lock, &lockstate);
+	obj = dlcheck(arg);
+	if (obj == NULL)
+		obj = obj_from_addr(arg);
+	if (obj == NULL) {
+		_rtld_error("No shared object contains address");
+		lock_release(rtld_bind_lock, &lockstate);
+		return (-1);
+	}
+	res = obj->dlopened ? 1 : 0;
+	lock_release(rtld_bind_lock, &lockstate);
+	return (res);
+}
+
 static void
 map_stacks_exec(RtldLockState *lockstate)
 {
@@ -4999,7 +5066,7 @@ __stack_chk_fail(void)
 {
 
 	_rtld_error("stack overflow detected; terminated");
-	die();
+	rtld_die();
 }
 __weak_reference(__stack_chk_fail, __stack_chk_fail_local);
 
@@ -5008,7 +5075,7 @@ __chk_fail(void)
 {
 
 	_rtld_error("buffer overflow detected; terminated");
-	die();
+	rtld_die();
 }
 
 const char *
diff --git a/libexec/rtld-elf/rtld.h b/libexec/rtld-elf/rtld.h
index ace229f..d75d0ab 100644
--- a/libexec/rtld-elf/rtld.h
+++ b/libexec/rtld-elf/rtld.h
@@ -264,6 +264,7 @@ typedef struct Struct_Obj_Entry {
     bool z_loadfltr : 1;	/* Immediately load filtees */
     bool z_interpose : 1;	/* Interpose all objects but main */
     bool z_nodeflib : 1;	/* Don't search default library path */
+    bool z_global : 1;		/* Make the object global */
     bool ref_nodel : 1;		/* Refcount increased to prevent dlclose */
     bool init_scanned: 1;	/* Object is already on init list. */
     bool on_fini_list: 1;	/* Object is already on fini list. */
@@ -275,6 +276,7 @@ typedef struct Struct_Obj_Entry {
     bool crt_no_init : 1;	/* Object' crt does not call _init/_fini */
     bool valid_hash_sysv : 1;	/* A valid System V hash hash tag is available */
     bool valid_hash_gnu : 1;	/* A valid GNU hash tag is available */
+    bool dlopened : 1;		/* dlopen()-ed (vs. load statically) */
 
     struct link_map linkmap;	/* For GDB and dlinfo() */
     Objlist dldags;		/* Object belongs to these dlopened DAGs (%) */
@@ -352,7 +354,8 @@ typedef struct Struct_SymLook {
     struct Struct_RtldLockState *lockstate;
 } SymLook;
 
-void _rtld_error(const char *, ...) __printflike(1, 2);
+void _rtld_error(const char *, ...) __printflike(1, 2) __exported;
+void rtld_die(void) __dead2;
 const char *rtld_strerror(int);
 Obj_Entry *map_object(int, const char *, const struct stat *);
 void *xcalloc(size_t, size_t);
diff --git a/libexec/rtld-elf/rtld_lock.c b/libexec/rtld-elf/rtld_lock.c
index ea16c4d..f31546c 100644
--- a/libexec/rtld-elf/rtld_lock.c
+++ b/libexec/rtld-elf/rtld_lock.c
@@ -51,6 +51,10 @@
 #include "rtld.h"
 #include "rtld_machdep.h"
 
+void _rtld_thread_init(struct RtldLockInfo *) __exported;
+void _rtld_atfork_pre(int *) __exported;
+void _rtld_atfork_post(int *) __exported;
+
 #define WAFLAG		0x1	/* A writer holds the lock */
 #define RC_INCR		0x2	/* Adjusts count of readers desiring lock */
 
diff --git a/libexec/rtld-elf/rtld_lock.h b/libexec/rtld-elf/rtld_lock.h
index fa63787..3d5aaad 100644
--- a/libexec/rtld-elf/rtld_lock.h
+++ b/libexec/rtld-elf/rtld_lock.h
@@ -44,9 +44,9 @@ struct RtldLockInfo
 	void  (*at_fork)(void);
 };
 
-extern void _rtld_thread_init(struct RtldLockInfo *);
-extern void _rtld_atfork_pre(int *);
-extern void _rtld_atfork_post(int *);
+extern void _rtld_thread_init(struct RtldLockInfo *) __exported;
+extern void _rtld_atfork_pre(int *) __exported;
+extern void _rtld_atfork_post(int *) __exported;
 
 #ifdef IN_RTLD
 
diff --git a/libexec/rtld-elf/rtld_tls.h b/libexec/rtld-elf/rtld_tls.h
index b85db59..40f71c2 100644
--- a/libexec/rtld-elf/rtld_tls.h
+++ b/libexec/rtld-elf/rtld_tls.h
@@ -57,13 +57,14 @@
  * The value returned from this function is suitable for installing
  * directly into the thread pointer register.
  */
-extern void *_rtld_allocate_tls(void* oldtls, size_t tcbsize, size_t tcbalign);
+void *_rtld_allocate_tls(void* oldtls, size_t tcbsize, size_t tcbalign)
+    __exported;
 
 /*
  * Free a TLS block allocated using _rtld_allocate_tls(). The tcbsize
  * and tcbalign parameters must be the same as those used to allocate
  * the block.
  */
-extern void _rtld_free_tls(void *tcb, size_t tcbsize, size_t tcbalign);
+void _rtld_free_tls(void *tcb, size_t tcbsize, size_t tcbalign) __exported;
 
 #endif