Move the cleanup code that frees memory allocated for a dead thread from

the thread kernel into a garbage collector thread which is started when
the fisrt thread is created (other than the initial thread). This
removes the window of opportunity where a context switch will cause a
thread that has locked the malloc spinlock, to enter the thread kernel,
find there is a dead thread and try to free memory, therefore trying
to lock the malloc spinlock against itself.

The garbage collector thread acts just like any other thread, so
instead of having a spinlock to control accesses to the dead thread
list, it uses a mutex and a condition variable so that it can happily
wait to be signalled when a thread exists.

6 files changed, 52 insertions, 130 deletions

diff --git a/lib/libkse/thread/Makefile.inc b/lib/libkse/thread/Makefile.inc
index 0c5c582..004982a 100644
--- a/lib/libkse/thread/Makefile.inc
+++ b/lib/libkse/thread/Makefile.inc
@@ -1,4 +1,4 @@
-#	$Id: Makefile.inc,v 1.14 1998/09/07 19:01:43 alex Exp $
+#	$Id: Makefile.inc,v 1.15 1998/09/12 22:03:20 dt Exp $
 
 # uthread sources
 .PATH: ${.CURDIR}/uthread
@@ -40,6 +40,7 @@ SRCS+= \
 	uthread_fstat.c \
 	uthread_fstatfs.c \
 	uthread_fsync.c \
+	uthread_gc.c \
 	uthread_getdirentries.c \
 	uthread_getpeername.c \
 	uthread_getprio.c \
diff --git a/lib/libkse/thread/thr_create.c b/lib/libkse/thread/thr_create.c
index 7603dd3..c7ddeb2 100644
--- a/lib/libkse/thread/thr_create.c
+++ b/lib/libkse/thread/thr_create.c
@@ -46,9 +46,11 @@ int
 pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	       void *(*start_routine) (void *), void *arg)
 {
+	int		f_gc = 0;
 	int             i;
 	int             ret = 0;
 	int             status;
+	pthread_t       gc_thread;
 	pthread_t       new_thread;
 	pthread_attr_t	pattr;
 	void           *stack;
@@ -185,6 +187,12 @@ pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 			/* Lock the thread list: */
 			_lock_thread_list();
 
+			/*
+			 * Check if the garbage collector thread
+			 * needs to be started.
+			 */
+			f_gc = (_thread_link_list == _thread_initial);
+
 			/* Add the thread to the linked list of all threads: */
 			new_thread->nxt = _thread_link_list;
 			_thread_link_list = new_thread;
@@ -197,6 +205,14 @@ pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 
 			/* Schedule the new user thread: */
 			_thread_kern_sched(NULL);
+
+			/*
+			 * Start a garbage collector thread
+			 * if necessary.
+			 */
+			if (f_gc && pthread_create(&gc_thread,NULL,
+				    _thread_gc,NULL) != 0)
+				PANIC("Can't create gc thread");
 		}
 	}
 
diff --git a/lib/libkse/thread/thr_exit.c b/lib/libkse/thread/thr_exit.c
index 0083f2b..a5fc400 100644
--- a/lib/libkse/thread/thr_exit.c
+++ b/lib/libkse/thread/thr_exit.c
@@ -133,56 +133,29 @@ pthread_exit(void *status)
 		PTHREAD_NEW_STATE(pthread,PS_RUNNING);
 	}
 
-	/* Lock the thread list: */
-	_lock_thread_list();
-
-	/* Check if the running thread is at the head of the linked list: */
-	if (_thread_link_list == _thread_run) {
-		/* There is no previous thread: */
-		_thread_link_list = _thread_run->nxt;
-	} else {
-		/* Point to the first thread in the list: */
-		pthread = _thread_link_list;
-
-		/*
-		 * Enter a loop to find the thread in the linked list before
-		 * the running thread: 
-		 */
-		while (pthread != NULL && pthread->nxt != _thread_run) {
-			/* Point to the next thread: */
-			pthread = pthread->nxt;
-		}
-
-		/* Check that a previous thread was found: */
-		if (pthread != NULL) {
-			/*
-			 * Point the previous thread to the one after the
-			 * running thread: 
-			 */
-			pthread->nxt = _thread_run->nxt;
-		}
-	}
-
-	/* Unlock the thread list: */
-	_unlock_thread_list();
-
-	/* Lock the dead thread list: */
-	_lock_dead_thread_list();
-
 	/*
-	 * This thread will never run again. Add it to the list of dead
-	 * threads: 
+	 * Lock the garbage collector mutex to ensure that the garbage
+	 * collector is not using the dead thread list.
 	 */
-	_thread_run->nxt = _thread_dead;
-	_thread_dead = _thread_run;
+	if (pthread_mutex_lock(&_gc_mutex) != 0)
+		PANIC("Cannot lock gc mutex");
 
-	/* Unlock the dead thread list: */
-	_unlock_dead_thread_list();
+	/* Add this thread to the list of dead threads. */
+	_thread_run->nxt_dead = _thread_dead;
+	_thread_dead = _thread_run;
 
 	/*
-	 * The running thread is no longer in the thread link list so it will
-	 * now die: 
+	 * Signal the garbage collector thread that there is something
+	 * to clean up.
 	 */
+	if (pthread_cond_signal(&_gc_cond) != 0)
+		PANIC("Cannot signal gc cond");
+
+	/* Unlock the garbage collector mutex: */
+	if (pthread_mutex_unlock(&_gc_mutex) != 0)
+		PANIC("Cannot lock gc mutex");
+
+	/* This this thread will never be re-scheduled. */
 	_thread_kern_sched_state(PS_DEAD, __FILE__, __LINE__);
 
 	/* This point should not be reached. */
diff --git a/lib/libkse/thread/thr_find_thread.c b/lib/libkse/thread/thr_find_thread.c
index 99e3023..e4a59a0 100644
--- a/lib/libkse/thread/thr_find_thread.c
+++ b/lib/libkse/thread/thr_find_thread.c
@@ -76,8 +76,12 @@ _find_dead_thread(pthread_t pthread)
 		/* Invalid thread: */
 		return(EINVAL);
 
-	/* Lock the dead thread list: */
-	_lock_dead_thread_list();
+	/*
+	 * Lock the garbage collector mutex to ensure that the garbage
+	 * collector is not using the dead thread list.
+	 */
+	if (pthread_mutex_lock(&_gc_mutex) != 0)
+		PANIC("Cannot lock gc mutex");
 
 	/* Point to the first thread in the list: */
 	pthread1 = _thread_dead;
@@ -85,11 +89,12 @@ _find_dead_thread(pthread_t pthread)
 	/* Search for the thread to join to: */
 	while (pthread1 != NULL && pthread1 != pthread) {
 		/* Point to the next thread: */
-		pthread1 = pthread1->nxt;
+		pthread1 = pthread1->nxt_dead;
 	}
 
-	/* Unlock the dead thread list: */
-	_unlock_dead_thread_list();
+	/* Unlock the garbage collector mutex: */
+	if (pthread_mutex_unlock(&_gc_mutex) != 0)
+		PANIC("Cannot lock gc mutex");
 
 	/* Return zero if the thread exists: */
 	return ((pthread1 != NULL) ? 0:ESRCH);
diff --git a/lib/libkse/thread/thr_init.c b/lib/libkse/thread/thr_init.c
index 69b0fef..50f3bef 100644
--- a/lib/libkse/thread/thr_init.c
+++ b/lib/libkse/thread/thr_init.c
@@ -251,6 +251,11 @@ _thread_init(void)
 	__set_dynamic_handler_allocator( dynamic_allocator_handler_fn );
 #endif /* GCC_2_8_MADE_THREAD_AWARE */
 
+	/* Initialise the garbage collector mutex and condition variable. */
+	if (pthread_mutex_init(&_gc_mutex,NULL) != 0 ||
+	    pthread_cond_init(&_gc_cond,NULL) != 0)
+		PANIC("Failed to initialise garbage collector mutex or condvar");
+
 	return;
 }
 
diff --git a/lib/libkse/thread/thr_kern.c b/lib/libkse/thread/thr_kern.c
index ce17cf8..da125cd 100644
--- a/lib/libkse/thread/thr_kern.c
+++ b/lib/libkse/thread/thr_kern.c
@@ -29,7 +29,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $Id: uthread_kern.c,v 1.11 1998/04/30 21:50:29 jb Exp $
+ * $Id: uthread_kern.c,v 1.12 1998/09/07 21:55:01 alex Exp $
  *
  */
 #include <errno.h>
@@ -62,8 +62,6 @@ _thread_kern_sched(struct sigcontext * scp)
 	int             prio = -1;
 	pthread_t       pthread;
 	pthread_t       pthread_h = NULL;
-	pthread_t       pthread_nxt = NULL;
-	pthread_t       pthread_prv = NULL;
 	pthread_t       pthread_s = NULL;
 	struct itimerval itimer;
 	struct timespec ts;
@@ -113,85 +111,9 @@ __asm__("fnsave %0": :"m"(*fdata));
 		 */
 		_dispatch_signals();
 		return;
-	} else {
+	} else
 		/* Flag the jump buffer was the last state saved: */
 		_thread_run->sig_saved = 0;
-	}
-
-	/* Point to the first dead thread (if there are any): */
-	pthread = _thread_dead;
-
-	/* There is no previous dead thread: */
-	pthread_prv = NULL;
-
-	/* Enter a loop to cleanup after dead threads: */
-	while (pthread != NULL) {
-		/* Save a pointer to the next thread: */
-		pthread_nxt = pthread->nxt;
-
-		/* Check if this thread is one which is running: */
-		if (pthread == _thread_run || pthread == _thread_initial) {
-			/*
-			 * Don't destroy the running thread or the initial
-			 * thread. 
-			 */
-			pthread_prv = pthread;
-		}
-		/*
-		 * Check if this thread has detached:
-		 */
-		else if ((pthread->attr.flags & PTHREAD_DETACHED) != 0) {
-			/* Check if there is no previous dead thread: */
-			if (pthread_prv == NULL) {
-				/*
-				 * The dead thread is at the head of the
-				 * list: 
-				 */
-				_thread_dead = pthread_nxt;
-			} else {
-				/*
-				 * The dead thread is not at the head of the
-				 * list: 
-				 */
-				pthread_prv->nxt = pthread->nxt;
-			}
-
-			/*
-			 * Check if the stack was not specified by the caller
-			 * to pthread_create and has not been destroyed yet: 
-			 */
-			if (pthread->attr.stackaddr_attr == NULL && pthread->stack != NULL) {
-				/* Free the stack of the dead thread: */
-				free(pthread->stack);
-			}
-			/* Free the memory allocated to the thread structure: */
-			free(pthread);
-		} else {
-			/*
-			 * This thread has not detached, so do not destroy
-			 * it: 
-			 */
-			pthread_prv = pthread;
-
-			/*
-			 * Check if the stack was not specified by the caller
-			 * to pthread_create and has not been destroyed yet: 
-			 */
-			if (pthread->attr.stackaddr_attr == NULL && pthread->stack != NULL) {
-				/* Free the stack of the dead thread: */
-				free(pthread->stack);
-
-				/*
-				 * NULL the stack pointer now that the memory
-				 * has been freed: 
-				 */
-				pthread->stack = NULL;
-			}
-		}
-
-		/* Point to the next thread: */
-		pthread = pthread_nxt;
-	}
 
 	/*
 	 * Enter a the scheduling loop that finds the next thread that is