MFC r257811 (by markj):

Include rtld itself when iterating over loaded ELF objects in
dl_iterate_phdr(3).

MFC r294373:
Do not call callbacks for dl_iterate_phdr(3) with the rtld bind and
phdr locks locked.

MFC r294470 (by kan):
Fix initlist_add_object invocation parameters.

MFC r294936 (by kan):
Do not unlock rtld_phdr_lock over callback invocations.

9 files changed, 176 insertions, 96 deletions

diff --git a/libexec/rtld-elf/amd64/reloc.c b/libexec/rtld-elf/amd64/reloc.c
index 53d2bdc..96cf93c 100644
--- a/libexec/rtld-elf/amd64/reloc.c
+++ b/libexec/rtld-elf/amd64/reloc.c
@@ -85,7 +85,8 @@ do_copy_relocations(Obj_Entry *dstobj)
 	    req.ventry = fetch_ventry(dstobj, ELF_R_SYM(rela->r_info));
 	    req.flags = SYMLOOK_EARLY;
 
-	    for (srcobj = dstobj->next;  srcobj != NULL;  srcobj = srcobj->next) {
+	    for (srcobj = globallist_next(dstobj); srcobj != NULL;
+	      srcobj = globallist_next(srcobj)) {
 		res = symlook_obj(&req, srcobj);
 		if (res == 0) {
 		    srcsym = req.sym_out;
diff --git a/libexec/rtld-elf/arm/reloc.c b/libexec/rtld-elf/arm/reloc.c
index 9cbdc0e..8b99514 100644
--- a/libexec/rtld-elf/arm/reloc.c
+++ b/libexec/rtld-elf/arm/reloc.c
@@ -56,8 +56,8 @@ do_copy_relocations(Obj_Entry *dstobj)
 			    ELF_R_SYM(rel->r_info));
 			req.flags = SYMLOOK_EARLY;
 
-			for (srcobj = dstobj->next;  srcobj != NULL; 
-			     srcobj = srcobj->next) {
+			for (srcobj = globallist_next(dstobj); srcobj != NULL;
+			    srcobj = globallist_next(srcobj)) {
 				res = symlook_obj(&req, srcobj);
 				if (res == 0) {
 					srcsym = req.sym_out;
diff --git a/libexec/rtld-elf/debug.c b/libexec/rtld-elf/debug.c
index 8f8311c..3c37b7f 100644
--- a/libexec/rtld-elf/debug.c
+++ b/libexec/rtld-elf/debug.c
@@ -62,7 +62,8 @@ dump_relocations (Obj_Entry *obj0)
 {
     Obj_Entry *obj;
 
-    for (obj = obj0; obj != NULL; obj = obj->next) {
+    for (obj = globallist_curr(obj0); obj != NULL;
+      obj = globallist_next(obj)) {
         dump_obj_relocations(obj);
     }
 }
diff --git a/libexec/rtld-elf/i386/reloc.c b/libexec/rtld-elf/i386/reloc.c
index 55b6537..2d6021c 100644
--- a/libexec/rtld-elf/i386/reloc.c
+++ b/libexec/rtld-elf/i386/reloc.c
@@ -86,7 +86,8 @@ do_copy_relocations(Obj_Entry *dstobj)
 	    req.ventry = fetch_ventry(dstobj, ELF_R_SYM(rel->r_info));
 	    req.flags = SYMLOOK_EARLY;
 
-	    for (srcobj = dstobj->next;  srcobj != NULL;  srcobj = srcobj->next) {
+	    for (srcobj = globallist_next(dstobj);  srcobj != NULL;
+	      srcobj = globallist_next(srcobj)) {
 		res = symlook_obj(&req, srcobj);
 		if (res == 0) {
 		    srcsym = req.sym_out;
diff --git a/libexec/rtld-elf/powerpc/reloc.c b/libexec/rtld-elf/powerpc/reloc.c
index 89e5536..a9a16de 100644
--- a/libexec/rtld-elf/powerpc/reloc.c
+++ b/libexec/rtld-elf/powerpc/reloc.c
@@ -94,8 +94,8 @@ do_copy_relocations(Obj_Entry *dstobj)
 		req.ventry = fetch_ventry(dstobj, ELF_R_SYM(rela->r_info));
 		req.flags = SYMLOOK_EARLY;
 
-		for (srcobj = dstobj->next;  srcobj != NULL;
-		     srcobj = srcobj->next) {
+		for (srcobj = globallist_next(dstobj); srcobj != NULL;
+		     srcobj = globallist_next(srcobj)) {
 			res = symlook_obj(&req, srcobj);
 			if (res == 0) {
 				srcsym = req.sym_out;
diff --git a/libexec/rtld-elf/powerpc64/reloc.c b/libexec/rtld-elf/powerpc64/reloc.c
index 65db28f..b33f6fe 100644
--- a/libexec/rtld-elf/powerpc64/reloc.c
+++ b/libexec/rtld-elf/powerpc64/reloc.c
@@ -88,8 +88,8 @@ do_copy_relocations(Obj_Entry *dstobj)
 		req.ventry = fetch_ventry(dstobj, ELF_R_SYM(rela->r_info));
 		req.flags = SYMLOOK_EARLY;
 
-		for (srcobj = dstobj->next;  srcobj != NULL;
-		     srcobj = srcobj->next) {
+		for (srcobj = globallist_next(dstobj); srcobj != NULL;
+		     srcobj = globallist_next(srcobj)) {
 			res = symlook_obj(&req, srcobj);
 			if (res == 0) {
 				srcsym = req.sym_out;
diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 9a0860f..38a6b481 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -95,7 +95,7 @@ static void init_dag(Obj_Entry *);
 static void init_pagesizes(Elf_Auxinfo **aux_info);
 static void init_rtld(caddr_t, Elf_Auxinfo **);
 static void initlist_add_neededs(Needed_Entry *, Objlist *);
-static void initlist_add_objects(Obj_Entry *, Obj_Entry **, Objlist *);
+static void initlist_add_objects(Obj_Entry *, Obj_Entry *, Objlist *);
 static void linkmap_add(Obj_Entry *);
 static void linkmap_delete(Obj_Entry *);
 static void load_filtees(Obj_Entry *, int flags, RtldLockState *);
@@ -180,12 +180,11 @@ static char *ld_preload;	/* Environment variable for libraries to
 static char *ld_elf_hints_path;	/* Environment variable for alternative hints path */
 static char *ld_tracing;	/* Called from ldd to print libs */
 static char *ld_utrace;		/* Use utrace() to log events. */
-static Obj_Entry *obj_list;	/* Head of linked list of shared objects */
-static Obj_Entry **obj_tail;	/* Link field of last object in list */
+static struct obj_entry_q obj_list;	/* Queue of all loaded objects */
 static Obj_Entry *obj_main;	/* The main program shared object */
 static Obj_Entry obj_rtld;	/* The dynamic linker shared object */
 static unsigned int obj_count;	/* Number of objects in obj_list */
-static unsigned int obj_loads;	/* Number of objects in obj_list */
+static unsigned int obj_loads;	/* Number of loads of objects (gen count) */
 
 static Objlist list_global =	/* Objects dlopened with RTLD_GLOBAL */
   STAILQ_HEAD_INITIALIZER(list_global);
@@ -343,7 +342,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     const char *argv0;
     Objlist_Entry *entry;
     Obj_Entry *obj;
-    Obj_Entry **preload_tail;
+    Obj_Entry *preload_tail;
     Obj_Entry *last_interposer;
     Objlist initlist;
     RtldLockState lockstate;
@@ -538,8 +537,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     linkmap_add(&obj_rtld);
 
     /* Link the main program into the list of objects. */
-    *obj_tail = obj_main;
-    obj_tail = &obj_main->next;
+    TAILQ_INSERT_HEAD(&obj_list, obj_main, next);
     obj_count++;
     obj_loads++;
 
@@ -554,7 +552,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     dbg("loading LD_PRELOAD libraries");
     if (load_preload_objects() == -1)
 	rtld_die();
-    preload_tail = obj_tail;
+    preload_tail = globallist_curr(TAILQ_LAST(&obj_list, obj_entry_q));
 
     dbg("loading needed objects");
     if (load_needed_objects(obj_main, 0) == -1)
@@ -562,7 +560,9 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 
     /* Make a list of all objects loaded at startup. */
     last_interposer = obj_main;
-    for (obj = obj_list;  obj != NULL;  obj = obj->next) {
+    TAILQ_FOREACH(obj, &obj_list, next) {
+	if (obj->marker)
+	    continue;
 	if (obj->z_interpose && obj != obj_main) {
 	    objlist_put_after(&list_main, last_interposer, obj);
 	    last_interposer = obj;
@@ -620,7 +620,7 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
      * might be the subject for relocations.
      */
     dbg("initializing initial thread local storage");
-    allocate_initial_tls(obj_list);
+    allocate_initial_tls(globallist_curr(TAILQ_FIRST(&obj_list)));
 
     dbg("initializing key program variables");
     set_program_var("__progname", argv[0] != NULL ? basename(argv[0]) : "");
@@ -629,7 +629,8 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
 
     /* Make a list of init functions to call. */
     objlist_init(&initlist);
-    initlist_add_objects(obj_list, preload_tail, &initlist);
+    initlist_add_objects(globallist_curr(TAILQ_FIRST(&obj_list)),
+      preload_tail, &initlist);
 
     r_debug_state(NULL, &obj_main->linkmap); /* say hello to gdb! */
 
@@ -659,7 +660,9 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     _r_debug_postinit(&obj_main->linkmap);
     objlist_clear(&initlist);
     dbg("loading filtees");
-    for (obj = obj_list->next; obj != NULL; obj = obj->next) {
+    TAILQ_FOREACH(obj, &obj_list, next) {
+	if (obj->marker)
+	    continue;
 	if (ld_loadfltr || obj->z_loadfltr)
 	    load_filtees(obj, 0, &lockstate);
     }
@@ -1373,9 +1376,10 @@ dlcheck(void *handle)
 {
     Obj_Entry *obj;
 
-    for (obj = obj_list;  obj != NULL;  obj = obj->next)
+    TAILQ_FOREACH(obj, &obj_list, next) {
 	if (obj == (Obj_Entry *) handle)
 	    break;
+    }
 
     if (obj == NULL || obj->refcount == 0 || obj->dl_refcount == 0) {
 	_rtld_error("Invalid shared object handle %p", handle);
@@ -1779,6 +1783,32 @@ init_dag(Obj_Entry *root)
     root->dag_inited = true;
 }
 
+Obj_Entry *
+globallist_curr(const Obj_Entry *obj)
+{
+
+	for (;;) {
+		if (obj == NULL)
+			return (NULL);
+		if (!obj->marker)
+			return (__DECONST(Obj_Entry *, obj));
+		obj = TAILQ_PREV(obj, obj_entry_q, next);
+	}
+}
+
+Obj_Entry *
+globallist_next(const Obj_Entry *obj)
+{
+
+	for (;;) {
+		obj = TAILQ_NEXT(obj, next);
+		if (obj == NULL)
+			return (NULL);
+		if (!obj->marker)
+			return (__DECONST(Obj_Entry *, obj));
+	}
+}
+
 static void
 process_z(Obj_Entry *root)
 {
@@ -1861,7 +1891,7 @@ init_rtld(caddr_t mapbase, Elf_Auxinfo **aux_info)
     }
 
     /* Initialize the object list. */
-    obj_tail = &obj_list;
+    TAILQ_INIT(&obj_list);
 
     /* Now that non-local variables can be accesses, copy out obj_rtld. */
     memcpy(&obj_rtld, &objtmp, sizeof(obj_rtld));
@@ -1942,7 +1972,7 @@ initlist_add_neededs(Needed_Entry *needed, Objlist *list)
 
     /* Process the current needed object. */
     if (needed->obj != NULL)
-	initlist_add_objects(needed->obj, &needed->obj->next, list);
+	initlist_add_objects(needed->obj, needed->obj, list);
 }
 
 /*
@@ -1955,16 +1985,18 @@ initlist_add_neededs(Needed_Entry *needed, Objlist *list)
  * held when this function is called.
  */
 static void
-initlist_add_objects(Obj_Entry *obj, Obj_Entry **tail, Objlist *list)
+initlist_add_objects(Obj_Entry *obj, Obj_Entry *tail, Objlist *list)
 {
+    Obj_Entry *nobj;
 
     if (obj->init_scanned || obj->init_done)
 	return;
     obj->init_scanned = true;
 
     /* Recursively process the successor objects. */
-    if (&obj->next != tail)
-	initlist_add_objects(obj->next, tail, list);
+    nobj = globallist_next(obj);
+    if (nobj != NULL && obj != tail)
+	initlist_add_objects(nobj, tail, list);
 
     /* Recursively process the needed objects. */
     if (obj->needed != NULL)
@@ -2067,7 +2099,10 @@ load_needed_objects(Obj_Entry *first, int flags)
 {
     Obj_Entry *obj;
 
-    for (obj = first;  obj != NULL;  obj = obj->next) {
+    obj = first;
+    TAILQ_FOREACH_FROM(obj, &obj_list, next) {
+	if (obj->marker)
+	    continue;
 	if (process_needed(obj, obj->needed, flags) == -1)
 	    return (-1);
     }
@@ -2128,7 +2163,9 @@ load_object(const char *name, int fd_u, const Obj_Entry *refobj, int flags)
     char *path;
 
     if (name != NULL) {
-	for (obj = obj_list->next;  obj != NULL;  obj = obj->next) {
+	TAILQ_FOREACH(obj, &obj_list, next) {
+	    if (obj->marker)
+		continue;
 	    if (object_match_name(obj, name))
 		return (obj);
 	}
@@ -2169,9 +2206,12 @@ load_object(const char *name, int fd_u, const Obj_Entry *refobj, int flags)
 	free(path);
 	return NULL;
     }
-    for (obj = obj_list->next;  obj != NULL;  obj = obj->next)
+    TAILQ_FOREACH(obj, &obj_list, next) {
+	if (obj->marker)
+	    continue;
 	if (obj->ino == sb.st_ino && obj->dev == sb.st_dev)
 	    break;
+    }
     if (obj != NULL && name != NULL) {
 	object_add_name(obj, name);
 	free(path);
@@ -2239,8 +2279,7 @@ do_load_object(int fd, const char *name, char *path, struct stat *sbp,
     }
 
     obj->dlopened = (flags & RTLD_LO_DLOPEN) != 0;
-    *obj_tail = obj;
-    obj_tail = &obj->next;
+    TAILQ_INSERT_TAIL(&obj_list, obj, next);
     obj_count++;
     obj_loads++;
     linkmap_add(obj);	/* for GDB & dlinfo() */
@@ -2261,7 +2300,9 @@ obj_from_addr(const void *addr)
 {
     Obj_Entry *obj;
 
-    for (obj = obj_list;  obj != NULL;  obj = obj->next) {
+    TAILQ_FOREACH(obj, &obj_list, next) {
+	if (obj->marker)
+	    continue;
 	if (addr < (void *) obj->mapbase)
 	    continue;
 	if (addr < (void *) (obj->mapbase + obj->mapsize))
@@ -2387,8 +2428,11 @@ objlist_call_init(Objlist *list, RtldLockState *lockstate)
      * possibly initialized earlier if any of vectors called below
      * cause the change by using dlopen.
      */
-    for (obj = obj_list;  obj != NULL;  obj = obj->next)
+    TAILQ_FOREACH(obj, &obj_list, next) {
+	if (obj->marker)
+	    continue;
 	obj->init_scanned = false;
+    }
 
     /*
      * Preserve the current error message since an init function might
@@ -2632,7 +2676,11 @@ relocate_objects(Obj_Entry *first, bool bind_now, Obj_Entry *rtldobj,
 	Obj_Entry *obj;
 	int error;
 
-	for (error = 0, obj = first;  obj != NULL;  obj = obj->next) {
+	error = 0;
+	obj = first;
+	TAILQ_FOREACH_FROM(obj, &obj_list, next) {
+		if (obj->marker)
+			continue;
 		error = relocate_object(obj, bind_now, rtldobj, flags,
 		    lockstate);
 		if (error == -1)
@@ -2670,7 +2718,10 @@ resolve_objects_ifunc(Obj_Entry *first, bool bind_now, int flags,
 {
 	Obj_Entry *obj;
 
-	for (obj = first;  obj != NULL;  obj = obj->next) {
+	obj = first;
+	TAILQ_FOREACH_FROM(obj, &obj_list, next) {
+		if (obj->marker)
+			continue;
 		if (resolve_object_ifunc(obj, bind_now, flags, lockstate) == -1)
 			return (-1);
 	}
@@ -2921,7 +2972,7 @@ static Obj_Entry *
 dlopen_object(const char *name, int fd, Obj_Entry *refobj, int lo_flags,
     int mode, RtldLockState *lockstate)
 {
-    Obj_Entry **old_obj_tail;
+    Obj_Entry *old_obj_tail;
     Obj_Entry *obj;
     Objlist initlist;
     RtldLockState mlockstate;
@@ -2935,7 +2986,7 @@ dlopen_object(const char *name, int fd, Obj_Entry *refobj, int lo_flags,
     }
     GDB_STATE(RT_ADD,NULL);
 
-    old_obj_tail = obj_tail;
+    old_obj_tail = globallist_curr(TAILQ_LAST(&obj_list, obj_entry_q));
     obj = NULL;
     if (name == NULL && fd == -1) {
 	obj = obj_main;
@@ -2948,8 +2999,9 @@ dlopen_object(const char *name, int fd, Obj_Entry *refobj, int lo_flags,
 	obj->dl_refcount++;
 	if (mode & RTLD_GLOBAL && objlist_find(&list_global, obj) == NULL)
 	    objlist_push_tail(&list_global, obj);
-	if (*old_obj_tail != NULL) {		/* We loaded something new. */
-	    assert(*old_obj_tail == obj);
+	if (globallist_next(old_obj_tail) != NULL) {
+	    /* We loaded something new. */
+	    assert(globallist_next(old_obj_tail) == obj);
 	    result = load_needed_objects(obj,
 		lo_flags & (RTLD_LO_DLOPEN | RTLD_LO_EARLY));
 	    init_dag(obj);
@@ -2976,7 +3028,7 @@ dlopen_object(const char *name, int fd, Obj_Entry *refobj, int lo_flags,
 		 */
 	    } else {
 		/* Make list of init functions to call. */
-		initlist_add_objects(obj, &obj->next, &initlist);
+		initlist_add_objects(obj, obj, &initlist);
 	    }
 	    /*
 	     * Process all no_delete or global objects here, given
@@ -3084,8 +3136,10 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
 	} else if (handle == RTLD_NEXT || /* Objects after caller's */
 		   handle == RTLD_SELF) { /* ... caller included */
 	    if (handle == RTLD_NEXT)
-		obj = obj->next;
-	    for (; obj != NULL; obj = obj->next) {
+		obj = globallist_next(obj);
+	    TAILQ_FOREACH_FROM(obj, &obj_list, next) {
+		if (obj->marker)
+		    continue;
 		res = symlook_obj(&req, obj);
 		if (res == 0) {
 		    if (def == NULL ||
@@ -3361,26 +3415,41 @@ rtld_fill_dl_phdr_info(const Obj_Entry *obj, struct dl_phdr_info *phdr_info)
 int
 dl_iterate_phdr(__dl_iterate_hdr_callback callback, void *param)
 {
-    struct dl_phdr_info phdr_info;
-    const Obj_Entry *obj;
-    RtldLockState bind_lockstate, phdr_lockstate;
-    int error;
-
-    wlock_acquire(rtld_phdr_lock, &phdr_lockstate);
-    rlock_acquire(rtld_bind_lock, &bind_lockstate);
-
-    error = 0;
+	struct dl_phdr_info phdr_info;
+	Obj_Entry *obj, marker;
+	RtldLockState bind_lockstate, phdr_lockstate;
+	int error;
 
-    for (obj = obj_list;  obj != NULL;  obj = obj->next) {
-	rtld_fill_dl_phdr_info(obj, &phdr_info);
-	if ((error = callback(&phdr_info, sizeof phdr_info, param)) != 0)
-		break;
+	bzero(&marker, sizeof(marker));
+	marker.marker = true;
+	error = 0;
 
-    }
-    lock_release(rtld_bind_lock, &bind_lockstate);
-    lock_release(rtld_phdr_lock, &phdr_lockstate);
+	wlock_acquire(rtld_phdr_lock, &phdr_lockstate);
+	rlock_acquire(rtld_bind_lock, &bind_lockstate);
+	for (obj = globallist_curr(TAILQ_FIRST(&obj_list)); obj != NULL;) {
+		TAILQ_INSERT_AFTER(&obj_list, obj, &marker, next);
+		rtld_fill_dl_phdr_info(obj, &phdr_info);
+		lock_release(rtld_bind_lock, &bind_lockstate);
+
+		error = callback(&phdr_info, sizeof phdr_info, param);
+
+		rlock_acquire(rtld_bind_lock, &bind_lockstate);
+		obj = globallist_next(&marker);
+		TAILQ_REMOVE(&obj_list, &marker, next);
+		if (error != 0) {
+			lock_release(rtld_bind_lock, &bind_lockstate);
+			lock_release(rtld_phdr_lock, &phdr_lockstate);
+			return (error);
+		}
+	}
 
-    return (error);
+	if (error == 0) {
+		rtld_fill_dl_phdr_info(&obj_rtld, &phdr_info);
+		lock_release(rtld_bind_lock, &bind_lockstate);
+		error = callback(&phdr_info, sizeof(phdr_info), param);
+	}
+	lock_release(rtld_phdr_lock, &phdr_lockstate);
+	return (error);
 }
 
 static void *
@@ -4100,11 +4169,13 @@ trace_loaded_objects(Obj_Entry *obj)
 
     list_containers = getenv(LD_ "TRACE_LOADED_OBJECTS_ALL");
 
-    for (; obj; obj = obj->next) {
+    TAILQ_FOREACH_FROM(obj, &obj_list, next) {
 	Needed_Entry		*needed;
 	char			*name, *path;
 	bool			is_lib;
 
+	if (obj->marker)
+	    continue;
 	if (list_containers && obj->needed != NULL)
 	    rtld_printf("%s:\n", obj->path);
 	for (needed = obj->needed; needed; needed = needed->next) {
@@ -4187,34 +4258,30 @@ trace_loaded_objects(Obj_Entry *obj)
 static void
 unload_object(Obj_Entry *root)
 {
-    Obj_Entry *obj;
-    Obj_Entry **linkp;
+	Obj_Entry *obj, *obj1;
 
-    assert(root->refcount == 0);
+	assert(root->refcount == 0);
 
-    /*
-     * Pass over the DAG removing unreferenced objects from
-     * appropriate lists.
-     */
-    unlink_object(root);
-
-    /* Unmap all objects that are no longer referenced. */
-    linkp = &obj_list->next;
-    while ((obj = *linkp) != NULL) {
-	if (obj->refcount == 0) {
-	    LD_UTRACE(UTRACE_UNLOAD_OBJECT, obj, obj->mapbase, obj->mapsize, 0,
-		obj->path);
-	    dbg("unloading \"%s\"", obj->path);
-	    unload_filtees(root);
-	    munmap(obj->mapbase, obj->mapsize);
-	    linkmap_delete(obj);
-	    *linkp = obj->next;
-	    obj_count--;
-	    obj_free(obj);
-	} else
-	    linkp = &obj->next;
-    }
-    obj_tail = linkp;
+	/*
+	 * Pass over the DAG removing unreferenced objects from
+	 * appropriate lists.
+	 */
+	unlink_object(root);
+
+	/* Unmap all objects that are no longer referenced. */
+	TAILQ_FOREACH_SAFE(obj, &obj_list, next, obj1) {
+		if (obj->marker || obj->refcount != 0)
+			continue;
+		LD_UTRACE(UTRACE_UNLOAD_OBJECT, obj, obj->mapbase,
+		    obj->mapsize, 0, obj->path);
+		dbg("unloading \"%s\"", obj->path);
+		unload_filtees(root);
+		munmap(obj->mapbase, obj->mapsize);
+		linkmap_delete(obj);
+		TAILQ_REMOVE(&obj_list, obj, next);
+		obj_count--;
+		obj_free(obj);
+	}
 }
 
 static void
@@ -4346,7 +4413,8 @@ allocate_tls(Obj_Entry *objs, void *oldtcb, size_t tcbsize, size_t tcbalign)
 	dtv[0] = tls_dtv_generation;
 	dtv[1] = tls_max_index;
 
-	for (obj = objs; obj; obj = obj->next) {
+	for (obj = globallist_curr(objs); obj != NULL;
+	  obj = globallist_next(obj)) {
 	    if (obj->tlsoffset > 0) {
 		addr = (Elf_Addr)tls + obj->tlsoffset;
 		if (obj->tlsinitsize > 0)
@@ -4445,15 +4513,16 @@ allocate_tls(Obj_Entry *objs, void *oldtls, size_t tcbsize, size_t tcbalign)
 	 */
 	free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr));
     } else {
-	for (obj = objs; obj; obj = obj->next) {
-	    if (obj->tlsoffset) {
+	obj = objs;
+	TAILQ_FOREACH_FROM(obj, &obj_list, next) {
+		if (obj->marker || obj->tlsoffset == 0)
+			continue;
 		addr = segbase - obj->tlsoffset;
 		memset((void*) (addr + obj->tlsinitsize),
 		       0, obj->tlssize - obj->tlsinitsize);
 		if (obj->tlsinit)
 		    memcpy((void*) addr, obj->tlsinit, obj->tlsinitsize);
 		dtv[obj->tlsindex + 1] = addr;
-	    }
 	}
     }
 
@@ -4502,7 +4571,9 @@ allocate_module_tls(int index)
     Obj_Entry* obj;
     char* p;
 
-    for (obj = obj_list; obj; obj = obj->next) {
+    TAILQ_FOREACH(obj, &obj_list, next) {
+	if (obj->marker)
+	    continue;
 	if (obj->tlsindex == index)
 	    break;
     }
@@ -4581,7 +4652,8 @@ _rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign)
     RtldLockState lockstate;
 
     wlock_acquire(rtld_bind_lock, &lockstate);
-    ret = allocate_tls(obj_list, oldtls, tcbsize, tcbalign);
+    ret = allocate_tls(globallist_curr(TAILQ_FIRST(&obj_list)), oldtls,
+      tcbsize, tcbalign);
     lock_release(rtld_bind_lock, &lockstate);
     return (ret);
 }
diff --git a/libexec/rtld-elf/rtld.h b/libexec/rtld-elf/rtld.h
index d75d0ab..99affcb 100644
--- a/libexec/rtld-elf/rtld.h
+++ b/libexec/rtld-elf/rtld.h
@@ -155,7 +155,7 @@ typedef struct Struct_Obj_Entry {
     Elf_Size magic;		/* Magic number (sanity check) */
     Elf_Size version;		/* Version number of struct format */
 
-    struct Struct_Obj_Entry *next;
+    TAILQ_ENTRY(Struct_Obj_Entry) next;
     char *path;			/* Pathname of underlying file (%) */
     char *origin_path;		/* Directory path of origin file */
     int refcount;
@@ -277,6 +277,7 @@ typedef struct Struct_Obj_Entry {
     bool valid_hash_sysv : 1;	/* A valid System V hash hash tag is available */
     bool valid_hash_gnu : 1;	/* A valid GNU hash tag is available */
     bool dlopened : 1;		/* dlopen()-ed (vs. load statically) */
+    bool marker : 1;		/* marker on the global obj list */
 
     struct link_map linkmap;	/* For GDB and dlinfo() */
     Objlist dldags;		/* Object belongs to these dlopened DAGs (%) */
@@ -289,6 +290,8 @@ typedef struct Struct_Obj_Entry {
 #define RTLD_MAGIC	0xd550b87a
 #define RTLD_VERSION	1
 
+TAILQ_HEAD(obj_entry_q, Struct_Obj_Entry);
+
 #define RTLD_STATIC_TLS_EXTRA	128
 
 /* Flags to be passed into symlook_ family of functions. */
@@ -380,6 +383,8 @@ const Elf_Sym *find_symdef(unsigned long, const Obj_Entry *,
 void init_pltgot(Obj_Entry *);
 void lockdflt_init(void);
 void digest_notes(Obj_Entry *, Elf_Addr, Elf_Addr);
+Obj_Entry *globallist_curr(const Obj_Entry *obj);
+Obj_Entry *globallist_next(const Obj_Entry *obj);
 void obj_free(Obj_Entry *);
 Obj_Entry *obj_new(void);
 void _rtld_bind_start(void);
diff --git a/libexec/rtld-elf/sparc64/reloc.c b/libexec/rtld-elf/sparc64/reloc.c
index 738a847..242fb8e 100644
--- a/libexec/rtld-elf/sparc64/reloc.c
+++ b/libexec/rtld-elf/sparc64/reloc.c
@@ -266,8 +266,8 @@ do_copy_relocations(Obj_Entry *dstobj)
 			    ELF_R_SYM(rela->r_info));
 			req.flags = SYMLOOK_EARLY;
 
-			for (srcobj = dstobj->next; srcobj != NULL;
-			    srcobj = srcobj->next) {
+			for (srcobj = globallist_next(dstobj); srcobj != NULL;
+			    srcobj = globallist_next(srcobj)) {
 				res = symlook_obj(&req, srcobj);
 				if (res == 0) {
 					srcsym = req.sym_out;