Refactor a bunch of scheduler code to give basically the same behaviour

but with slightly cleaned up interfaces.

The KSE structure has become the same as the "per thread scheduler
private data" structure. In order to not make the diffs too great
one is #defined as the other at this time.

The KSE (or td_sched) structure is  now allocated per thread and has no
allocation code of its own.

Concurrency for a KSEGRP is now kept track of via a simple pair of counters
rather than using KSE structures as tokens.

Since the KSE structure is different in each scheduler, kern_switch.c
is now included at the end of each scheduler. Nothing outside the
scheduler knows the contents of the KSE (aka td_sched) structure.

The fields in the ksegrp structure that are to do with the scheduler's
queueing mechanisms are now moved to the kg_sched structure.
(per ksegrp scheduler private data structure). In other words how the
scheduler queues and keeps track of threads is no-one's business except
the scheduler's. This should allow people to write experimental
schedulers with completely different internal structuring.

A scheduler call sched_set_concurrency(kg, N) has been added that
notifies teh scheduler that no more than N threads from that ksegrp
should be allowed to be on concurrently scheduled. This is also
used to enforce 'fainess' at this time so that a ksegrp with
10000 threads can not swamp a the run queue and force out a process
with 1 thread, since the current code will not set the concurrency above
NCPU, and both schedulers will not allow more than that many
onto the system run queue at a time. Each scheduler should eventualy develop
their own methods to do this now that they are effectively separated.

Rejig libthr's kernel interface to follow the same code paths as
linkse for scope system threads. This has slightly hurt libthr's performance
but I will work to recover as much of it as I can.

Thread exit code has been cleaned up greatly.
exit and exec code now transitions a process back to
'standard non-threaded mode' before taking the next step.
Reviewed by:	scottl, peter
MFC after:	1 week

13 files changed, 756 insertions, 733 deletions

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index cb39a82..6acb6b0 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -90,7 +90,6 @@ static struct session session0;
 static struct pgrp pgrp0;
 struct	proc proc0;
 struct	thread thread0;
-struct	kse kse0;
 struct	ksegrp ksegrp0;
 static struct filedesc0 filedesc0;
 struct	vmspace vmspace0;
@@ -320,31 +319,29 @@ proc0_init(void *dummy __unused)
 	register unsigned i;
 	struct thread *td;
 	struct ksegrp *kg;
-	struct kse *ke;
 
 	GIANT_REQUIRED;
 	p = &proc0;
 	td = &thread0;
-	ke = &kse0;
 	kg = &ksegrp0;
 
-	ke->ke_sched = kse0_sched;
-	kg->kg_sched = ksegrp0_sched;
-	p->p_sched = proc0_sched;
-	td->td_sched = thread0_sched;
-
 	/*
 	 * Initialize magic number.
 	 */
 	p->p_magic = P_MAGIC;
 
 	/*
-	 * Initialize thread, process and pgrp structures.
+	 * Initialize thread, process and ksegrp structures.
 	 */
-	procinit();
-	threadinit();
+	procinit();	/* set up proc zone */
+	threadinit();	/* set up thead, upcall and KSEGRP zones */
 
 	/*
+	 * Initialise scheduler resources.
+	 * Add scheduler specific parts to proc, ksegrp, thread as needed.
+	 */
+	schedinit();	/* scheduler gets its house in order */
+	/*
 	 * Initialize sleep queue hash table
 	 */
 	sleepinit();
@@ -371,13 +368,6 @@ proc0_init(void *dummy __unused)
 	session0.s_leader = p;
 
 	p->p_sysent = &null_sysvec;
-
-	/*
-	 * proc_linkup was already done in init_i386() or alphainit() etc.
-	 * because the earlier code needed to follow td->td_proc. Otherwise
-	 * I would have done it here.. maybe this means this should be
-	 * done earlier too.
-	 */
 	p->p_flag = P_SYSTEM;
 	p->p_sflag = PS_INMEM;
 	p->p_state = PRS_NORMAL;
@@ -388,10 +378,7 @@ proc0_init(void *dummy __unused)
 	kg->kg_user_pri = PUSER;
 	td->td_priority = PVM;
 	td->td_base_pri = PUSER;
-	td->td_kse = ke; /* XXXKSE */
 	td->td_oncpu = 0;
-	ke->ke_state = KES_THREAD;
-	ke->ke_thread = td;
 	p->p_peers = 0;
 	p->p_leader = p;
 
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 34c7571..1a79f48 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -254,7 +254,7 @@ kern_execve(td, fname, argv, envv, mac_p)
 	PROC_LOCK(p);
 	KASSERT((p->p_flag & P_INEXEC) == 0,
 	    ("%s(): process already has P_INEXEC flag", __func__));
-	if (p->p_flag & P_SA || p->p_numthreads > 1) {
+	if (p->p_flag & P_HADTHREADS) {
 		if (thread_single(SINGLE_EXIT)) {
 			PROC_UNLOCK(p);
 			mtx_unlock(&Giant);
@@ -262,12 +262,8 @@ kern_execve(td, fname, argv, envv, mac_p)
 		}
 		/*
 		 * If we get here all other threads are dead,
-		 * so unset the associated flags and lose KSE mode.
+		 * and threading mode has been turned off
 		 */
-		p->p_flag &= ~P_SA;
-		td->td_mailbox = NULL;
-		td->td_pflags &= ~TDP_SA;
-		thread_single_end();
 	}
 	p->p_flag |= P_INEXEC;
 	PROC_UNLOCK(p);
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index 96e9cf2..747fb3b 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -134,7 +134,7 @@ exit1(struct thread *td, int rv)
 	 * MUST abort all other threads before proceeding past here.
 	 */
 	PROC_LOCK(p);
-	if (p->p_flag & P_SA || p->p_numthreads > 1) {
+	if (p->p_flag & P_HADTHREADS) {
 retry:
 		/*
 		 * First check if some other thread got here before us..
@@ -164,13 +164,8 @@ retry:
 			goto retry;
 		/*
 		 * All other activity in this process is now stopped.
-		 * Remove excess KSEs and KSEGRPS. XXXKSE (when we have them)
-		 * ...
-		 * Turn off threading support.
+		 * Threading support has been turned off.
 		 */
-		p->p_flag &= ~P_SA;
-		td->td_pflags &= ~TDP_SA;
-		thread_single_end();	/* Don't need this any more. */
 	}
 
 	p->p_flag |= P_WEXIT;
@@ -385,16 +380,6 @@ retry:
 	lim_free(plim);
 
 	/*
-	 * Release this thread's reference to the ucred.  The actual proc
-	 * reference will stay around until the proc is harvested by
-	 * wait().  At this point the ucred is immutable (no other threads
-	 * from this proc are around that can change it) so we leave the
-	 * per-thread ucred pointer intact in case it is needed although
-	 * in theory nothing should be using it at this point.
-	 */
-	crfree(td->td_ucred);
-
-	/*
 	 * Remove proc from allproc queue and pidhash chain.
 	 * Place onto zombproc.  Unlink from parent's child list.
 	 */
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index b5459d8..eeef444 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -203,7 +203,6 @@ fork1(td, flags, pages, procp)
 	struct filedesc *fd;
 	struct filedesc_to_leader *fdtol;
 	struct thread *td2;
-	struct kse *ke2;
 	struct ksegrp *kg2;
 	struct sigacts *newsigacts;
 	int error;
@@ -466,7 +465,6 @@ again:
 	 */
 	td2 = FIRST_THREAD_IN_PROC(p2);
 	kg2 = FIRST_KSEGRP_IN_PROC(p2);
-	ke2 = FIRST_KSE_IN_KSEGRP(kg2);
 
 	/* Allocate and switch to an alternate kstack if specified. */
 	if (pages != 0)
@@ -479,8 +477,6 @@ again:
 
 	bzero(&p2->p_startzero,
 	    (unsigned) RANGEOF(struct proc, p_startzero, p_endzero));
-	bzero(&ke2->ke_startzero,
-	    (unsigned) RANGEOF(struct kse, ke_startzero, ke_endzero));
 	bzero(&td2->td_startzero,
 	    (unsigned) RANGEOF(struct thread, td_startzero, td_endzero));
 	bzero(&kg2->kg_startzero,
@@ -496,11 +492,6 @@ again:
 
 	td2->td_sigstk = td->td_sigstk;
 
-	/* Set up the thread as an active thread (as if runnable). */
-	ke2->ke_state = KES_THREAD;
-	ke2->ke_thread = td2;
-	td2->td_kse = ke2;
-
 	/*
 	 * Duplicate sub-structures as needed.
 	 * Increase reference counts on shared objects.
@@ -515,7 +506,7 @@ again:
 	 * Allow the scheduler to adjust the priority of the child and
 	 * parent while we hold the sched_lock.
 	 */
-	sched_fork(td, p2);
+	sched_fork(td, td2);
 
 	mtx_unlock_spin(&sched_lock);
 	p2->p_ucred = crhold(td->td_ucred);
@@ -792,7 +783,7 @@ fork_exit(callout, arg, frame)
 	mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED);
 	cpu_critical_fork_exit();
 	CTR4(KTR_PROC, "fork_exit: new thread %p (kse %p, pid %d, %s)",
-		td, td->td_kse, p->p_pid, p->p_comm);
+		td, td->td_sched, p->p_pid, p->p_comm);
 
 	/*
 	 * Processes normally resume in mi_switch() after being
diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c
index ca7c206..9d8af71 100644
--- a/sys/kern/kern_intr.c
+++ b/sys/kern/kern_intr.c
@@ -444,6 +444,7 @@ swi_add(struct ithd **ithdp, const char *name, driver_intr_t handler,
 	}
 	return (ithread_add_handler(ithd, name, handler, arg,
 		    (pri * RQ_PPQ) + PI_SOFT, flags, cookiep));
+		    /* XXKSE.. think of a better way to get separate queues */
 }
 
 
diff --git a/sys/kern/kern_kse.c b/sys/kern/kern_kse.c
index 7eca37a..4a89665 100644
--- a/sys/kern/kern_kse.c
+++ b/sys/kern/kern_kse.c
@@ -68,12 +68,6 @@ TAILQ_HEAD(, kse_upcall) zombie_upcalls =
 static int thread_update_usr_ticks(struct thread *td);
 static void thread_alloc_spare(struct thread *td);
 
-/* move to proc.h */
-extern void kse_purge(struct proc *p, struct thread *td);
-extern void kse_purge_group(struct thread *td);
-void kseinit(void);
-void kse_GC(void);
-
 struct kse_upcall *
 upcall_alloc(void)
 {
@@ -277,7 +271,6 @@ kse_exit(struct thread *td, struct kse_exit_args *uap)
 {
 	struct proc *p;
 	struct ksegrp *kg;
-	struct kse *ke;
 	struct kse_upcall *ku, *ku2;
 	int    error, count;
 
@@ -330,22 +323,38 @@ kse_exit(struct thread *td, struct kse_exit_args *uap)
 		psignal(p, SIGSEGV);
 	mtx_lock_spin(&sched_lock);
 	upcall_remove(td);
-	ke = td->td_kse;
-	if (p->p_numthreads == 1) {
-		kse_purge(p, td);
-		p->p_flag &= ~P_SA;
-		mtx_unlock_spin(&sched_lock);
-		PROC_UNLOCK(p);
-	} else {
-		if (kg->kg_numthreads == 1) { /* Shutdown a group */
-			kse_purge_group(td);
-			ke->ke_flags |= KEF_EXIT;
-		}
+	if (p->p_numthreads != 1) {
+		/*
+		 * If we are not the last thread, but we are the last
+		 * thread in this ksegrp, then by definition this is not
+		 * the last group and we need to clean it up as well.
+		 * thread_exit will clean up the kseg as needed.
+		 */
 		thread_stopped(p);
 		thread_exit();
 		/* NOTREACHED */
 	}
+	/*
+	 * This is the last thread. Just return to the user.
+	 * We know that there is only one ksegrp too, as any others
+	 * would have been discarded in previous calls to thread_exit().
+	 * Effectively we have left threading mode..
+	 * The only real thing left to do is ensure that the
+	 * scheduler sets out concurrency back to 1 as that may be a
+	 * resource leak otherwise.
+	 * This is an A[PB]I issue.. what SHOULD we do?
+	 * One possibility is to return to the user. It may not cope well.
+	 * The other possibility would be to let the process exit.
+	 */
+	p->p_flag &= ~(P_SA|P_HADTHREADS);
+	sched_set_concurrency(td->td_ksegrp, 1);
+	mtx_unlock_spin(&sched_lock);
+	PROC_UNLOCK(p);
+#if 1
 	return (0);
+#else
+	exit1(td, 0);
+#endif
 }
 
 /*
@@ -489,6 +498,10 @@ kse_wakeup(struct thread *td, struct kse_wakeup_args *uap)
 /*
  * No new KSEG: first call: use current KSE, don't schedule an upcall
  * All other situations, do allocate max new KSEs and schedule an upcall.
+ *
+ * XXX should be changed so that 'first' behaviour lasts for as long
+ * as you have not made a kse in this ksegrp. i.e. as long as we do not have
+ * a mailbox..
  */
 /* struct kse_create_args {
 	struct kse_mailbox *mbx;
@@ -497,7 +510,6 @@ kse_wakeup(struct thread *td, struct kse_wakeup_args *uap)
 int
 kse_create(struct thread *td, struct kse_create_args *uap)
 {
-	struct kse *newke;
 	struct ksegrp *newkg;
 	struct ksegrp *kg;
 	struct proc *p;
@@ -510,6 +522,13 @@ kse_create(struct thread *td, struct kse_create_args *uap)
 	if ((err = copyin(uap->mbx, &mbx, sizeof(mbx))))
 		return (err);
 
+	/*
+	 * Processes using the other threading model can't
+	 * suddenly start calling this one
+	 */
+	if ((p->p_flag & (P_SA|P_HADTHREADS)) == P_HADTHREADS)
+		 return (EINVAL);
+
 	ncpus = mp_ncpus;
 	if (virtual_cpu != 0)
 		ncpus = virtual_cpu;
@@ -531,7 +550,7 @@ kse_create(struct thread *td, struct kse_create_args *uap)
 	PROC_LOCK(p);
 	if (!(p->p_flag & P_SA)) {
 		first = 1;
-		p->p_flag |= P_SA;
+		p->p_flag |= P_SA|P_HADTHREADS;
 	}
 	PROC_UNLOCK(p);
 	/*
@@ -612,17 +631,7 @@ kse_create(struct thread *td, struct kse_create_args *uap)
 		 * an upcall when blocked.  This simulates pthread system
 		 * scope thread behaviour.
 		 */
-		while (newkg->kg_kses < ncpus) {
-			newke = kse_alloc();
-			bzero(&newke->ke_startzero, RANGEOF(struct kse,
-			      ke_startzero, ke_endzero));
-			mtx_lock_spin(&sched_lock);
-			kse_link(newke, newkg);
-			sched_fork_kse(td, newke);
-			/* Add engine */
-			kse_reassign(newke);
-			mtx_unlock_spin(&sched_lock);
-		}
+		sched_set_concurrency(newkg, ncpus);
 	}
 	/* 
 	 * Even bound LWPs get a mailbox and an upcall to hold it.
@@ -981,7 +990,8 @@ error:
 /*
  * This function is intended to be used to initialize a spare thread
  * for upcall. Initialize thread's large data area outside sched_lock
- * for thread_schedule_upcall().
+ * for thread_schedule_upcall(). The crhold is also here to get it out
+ * from the schedlock as it has a mutex op itself.
  */
 void
 thread_alloc_spare(struct thread *td)
@@ -1037,7 +1047,6 @@ thread_schedule_upcall(struct thread *td, struct kse_upcall *ku)
 	td2->td_upcall = ku;
 	td2->td_flags  = 0;
 	td2->td_pflags = TDP_SA|TDP_UPCALLING;
-	td2->td_kse    = NULL;
 	td2->td_state  = TDS_CAN_RUN;
 	td2->td_inhibitors = 0;
 	SIGFILLSET(td2->td_sigmask);
@@ -1075,9 +1084,9 @@ thread_signal_add(struct thread *td, int sig)
 	PROC_LOCK(p);
 	mtx_lock(&ps->ps_mtx);
 }
-
-void
-thread_switchout(struct thread *td)
+#include "opt_sched.h"
+struct thread *
+thread_switchout(struct thread *td, int flags, struct thread *nextthread)
 {
 	struct kse_upcall *ku;
 	struct thread *td2;
@@ -1113,8 +1122,20 @@ thread_switchout(struct thread *td)
 		td->td_upcall = NULL;
 		td->td_pflags &= ~TDP_CAN_UNBIND;
 		td2 = thread_schedule_upcall(td, ku);
+#ifdef SCHED_4BSD
+		if (flags & SW_INVOL || nextthread) {
+			setrunqueue(td2, SRQ_YIELDING);
+		} else {
+			/* Keep up with reality.. we have one extra thread 
+			 * in the picture.. and it's 'running'.
+			 */
+			return td2;
+		}
+#else
 		setrunqueue(td2, SRQ_YIELDING);
+#endif
 	}
+	return (nextthread);
 }
 
 /*
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index b144543..3f12801 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -100,8 +100,6 @@ int uarea_pages = UAREA_PAGES;
 SYSCTL_INT(_kern, OID_AUTO, kstack_pages, CTLFLAG_RD, &kstack_pages, 0, "");
 SYSCTL_INT(_kern, OID_AUTO, uarea_pages, CTLFLAG_RD, &uarea_pages, 0, "");
 
-#define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start))
-
 CTASSERT(sizeof(struct kinfo_proc) == KINFO_PROC_SIZE);
 
 /*
@@ -145,19 +143,20 @@ proc_dtor(void *mem, int size, void *arg)
 {
 	struct proc *p;
 	struct thread *td;
+#ifdef INVARIANTS
 	struct ksegrp *kg;
-	struct kse *ke;
+#endif
 
 	/* INVARIANTS checks go here */
 	p = (struct proc *)mem;
+        td = FIRST_THREAD_IN_PROC(p);
+#ifdef INVARIANTS
 	KASSERT((p->p_numthreads == 1),
 	    ("bad number of threads in exiting process"));
-        td = FIRST_THREAD_IN_PROC(p);
 	KASSERT((td != NULL), ("proc_dtor: bad thread pointer"));
         kg = FIRST_KSEGRP_IN_PROC(p);
 	KASSERT((kg != NULL), ("proc_dtor: bad kg pointer"));
-        ke = FIRST_KSE_IN_KSEGRP(kg);
-	KASSERT((ke != NULL), ("proc_dtor: bad ke pointer"));
+#endif
 
 	/* Dispose of an alternate kstack, if it exists.
 	 * XXX What if there are more than one thread in the proc?
@@ -166,14 +165,6 @@ proc_dtor(void *mem, int size, void *arg)
 	 */
 	if (((p->p_flag & P_KTHREAD) != 0) && (td->td_altkstack != 0))
 		vm_thread_dispose_altkstack(td);
-
-	/*
-	 * We want to make sure we know the initial linkages.
-	 * so for now tear them down and remake them.
-	 * This is probably un-needed as we can probably rely
-	 * on the state coming in here from wait4().
-	 */
-	proc_linkup(p, kg, ke, td);
 }
 
 /*
@@ -185,17 +176,16 @@ proc_init(void *mem, int size, int flags)
 	struct proc *p;
 	struct thread *td;
 	struct ksegrp *kg;
-	struct kse *ke;
 
 	p = (struct proc *)mem;
 	p->p_sched = (struct p_sched *)&p[1];
 	vm_proc_new(p);
 	td = thread_alloc();
-	ke = kse_alloc();
 	kg = ksegrp_alloc();
-	proc_linkup(p, kg, ke, td);
 	bzero(&p->p_mtx, sizeof(struct mtx));
 	mtx_init(&p->p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK);
+	proc_linkup(p, kg, td);
+	sched_newproc(p, kg, td);
 	return (0);
 }
 
@@ -208,7 +198,6 @@ proc_fini(void *mem, int size)
 	struct proc *p;
 	struct thread *td;
 	struct ksegrp *kg;
-	struct kse *ke;
 
 	p = (struct proc *)mem;
 	KASSERT((p->p_numthreads == 1),
@@ -217,12 +206,10 @@ proc_fini(void *mem, int size)
 	KASSERT((td != NULL), ("proc_fini: bad thread pointer"));
         kg = FIRST_KSEGRP_IN_PROC(p);
 	KASSERT((kg != NULL), ("proc_fini: bad kg pointer"));
-        ke = FIRST_KSE_IN_KSEGRP(kg);
-	KASSERT((ke != NULL), ("proc_fini: bad ke pointer"));
 	vm_proc_dispose(p);
+	sched_destroyproc(p);
 	thread_free(td);
 	ksegrp_free(kg);
-	kse_free(ke);
 	mtx_destroy(&p->p_mtx);
 }
 
@@ -635,7 +622,6 @@ fill_kinfo_thread(struct thread *td, struct kinfo_proc *kp)
 {
 	struct proc *p;
 	struct thread *td0;
-	struct kse *ke;
 	struct ksegrp *kg;
 	struct tty *tp;
 	struct session *sp;
@@ -756,7 +742,6 @@ fill_kinfo_thread(struct thread *td, struct kinfo_proc *kp)
 		}
 
 		kg = td->td_ksegrp;
-		ke = td->td_kse;
 
 		/* things in the KSE GROUP */
 		kp->ki_estcpu = kg->kg_estcpu;
@@ -777,11 +762,8 @@ fill_kinfo_thread(struct thread *td, struct kinfo_proc *kp)
 		kp->ki_kstack = (void *)td->td_kstack;
 		kp->ki_pctcpu = sched_pctcpu(td);
 
-		/* Things in the kse */
-		if (ke)
-			kp->ki_rqindex = ke->ke_rqindex;
-		else
-			kp->ki_rqindex = 0;
+		/* We can't get this anymore but ps etc never used it anyway. */
+		kp->ki_rqindex = 0;
 
 	} else {
 		kp->ki_stat = SZOMB;
diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c
index 9c45ce6..eeed120 100644
--- a/sys/kern/kern_switch.c
+++ b/sys/kern/kern_switch.c
@@ -90,6 +90,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_sched.h"
 
+#ifndef KERN_SWITCH_INCLUDE
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kdb.h>
@@ -100,6 +101,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/queue.h>
 #include <sys/sched.h>
+#else  /* KERN_SWITCH_INCLUDE */
 #if defined(SMP) && (defined(__i386__) || defined(__amd64__))
 #include <sys/smp.h>
 #endif
@@ -116,6 +118,8 @@ __FBSDID("$FreeBSD$");
 
 CTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS);
 
+#define td_kse td_sched
+
 /************************************************************************
  * Functions that manipulate runnability from a thread perspective.	*
  ************************************************************************/
@@ -149,7 +153,7 @@ retry:
 		td = ke->ke_thread;
 		KASSERT((td->td_kse == ke), ("kse/thread mismatch"));
 		kg = ke->ke_ksegrp;
-		if (td->td_proc->p_flag & P_SA) {
+		if (td->td_proc->p_flag & P_HADTHREADS) {
 			if (kg->kg_last_assigned == td) {
 				kg->kg_last_assigned = TAILQ_PREV(td,
 				    threadqueue, td_runq);
@@ -183,51 +187,41 @@ retry:
 }
 
 /*
- * Given a surplus KSE, either assign a new runable thread to it
- * (and put it in the run queue) or put it in the ksegrp's idle KSE list.
- * Assumes that the original thread is not runnable.
+ * Given a surplus system slot, try assign a new runnable thread to it.
+ * Called from:
+ *  sched_thread_exit()  (local)
+ *  sched_switch()  (local)
+ *  sched_thread_exit()  (local)
+ *  remrunqueue()  (local) (commented out)
  */
-void
-kse_reassign(struct kse *ke)
+static void
+slot_fill(struct ksegrp *kg)
 {
-	struct ksegrp *kg;
 	struct thread *td;
-	struct thread *original;
 
 	mtx_assert(&sched_lock, MA_OWNED);
-	original = ke->ke_thread;
-	KASSERT(original == NULL || TD_IS_INHIBITED(original),
-    	    ("reassigning KSE with runnable thread"));
-	kg = ke->ke_ksegrp;
-	if (original)
-		original->td_kse = NULL;
-
-	/*
-	 * Find the first unassigned thread
-	 */
-	if ((td = kg->kg_last_assigned) != NULL)
-		td = TAILQ_NEXT(td, td_runq);
-	else 
-		td = TAILQ_FIRST(&kg->kg_runq);
+	while (kg->kg_avail_opennings > 0) {
+		/*
+		 * Find the first unassigned thread
+		 */
+		if ((td = kg->kg_last_assigned) != NULL)
+			td = TAILQ_NEXT(td, td_runq);
+		else
+			td = TAILQ_FIRST(&kg->kg_runq);
 
-	/*
-	 * If we found one, assign it the kse, otherwise idle the kse.
-	 */
-	if (td) {
-		kg->kg_last_assigned = td;
-		td->td_kse = ke;
-		ke->ke_thread = td;
-		CTR2(KTR_RUNQ, "kse_reassign: ke%p -> td%p", ke, td);
-		sched_add(td, SRQ_BORING);
-		return;
+		/*
+		 * If we found one, send it to the system scheduler.
+		 */
+		if (td) {
+			kg->kg_last_assigned = td;
+			kg->kg_avail_opennings--;
+			sched_add(td, SRQ_BORING);
+			CTR2(KTR_RUNQ, "slot_fill: td%p -> kg%p", td, kg);
+		} else {
+			/* no threads to use up the slots. quit now */
+			break;
+		}
 	}
-
-	ke->ke_state = KES_IDLE;
-	ke->ke_thread = NULL;
-	TAILQ_INSERT_TAIL(&kg->kg_iq, ke, ke_kgrlist);
-	kg->kg_idle_kses++;
-	CTR1(KTR_RUNQ, "kse_reassign: ke%p on idle queue", ke);
-	return;
 }
 
 #if 0
@@ -253,16 +247,17 @@ remrunqueue(struct thread *td)
 	/*
 	 * If it is not a threaded process, take the shortcut.
 	 */
-	if ((td->td_proc->p_flag & P_SA) == 0) {
+	if ((td->td_proc->p_flag & P_HADTHREADS) == 0) {
 		/* Bring its kse with it, leave the thread attached */
 		sched_rem(td);
+		kg->kg_avail_opennings++;
 		ke->ke_state = KES_THREAD; 
 		return;
 	}
    	td3 = TAILQ_PREV(td, threadqueue, td_runq);
 	TAILQ_REMOVE(&kg->kg_runq, td, td_runq);
 	kg->kg_runnable--;
-	if (ke) {
+	if (ke->ke_state == KES_ONRUNQ) {
 		/*
 		 * This thread has been assigned to a KSE.
 		 * We need to dissociate it and try assign the
@@ -270,12 +265,13 @@ remrunqueue(struct thread *td)
 		 * see if we need to move the KSE in the run queues.
 		 */
 		sched_rem(td);
+		kg->kg_avail_opennings++;
 		ke->ke_state = KES_THREAD; 
 		td2 = kg->kg_last_assigned;
 		KASSERT((td2 != NULL), ("last assigned has wrong value"));
 		if (td2 == td) 
 			kg->kg_last_assigned = td3;
-		kse_reassign(ke);
+		slot_fill(kg);
 	}
 }
 #endif
@@ -297,7 +293,7 @@ adjustrunqueue( struct thread *td, int newpri)
 	/*
 	 * If it is not a threaded process, take the shortcut.
 	 */
-	if ((td->td_proc->p_flag & P_SA) == 0) {
+	if ((td->td_proc->p_flag & P_HADTHREADS) == 0) {
 		/* We only care about the kse in the run queue. */
 		td->td_priority = newpri;
 		if (ke->ke_rqindex != (newpri / RQ_PPQ)) {
@@ -310,77 +306,67 @@ adjustrunqueue( struct thread *td, int newpri)
 	/* It is a threaded process */
 	kg = td->td_ksegrp;
 	TD_SET_CAN_RUN(td);
-	if (ke) {
+	if (ke->ke_state == KES_ONRUNQ) {
 		if (kg->kg_last_assigned == td) {
 			kg->kg_last_assigned =
 			    TAILQ_PREV(td, threadqueue, td_runq);
 		}
 		sched_rem(td);
+		kg->kg_avail_opennings++;
 	}
 	TAILQ_REMOVE(&kg->kg_runq, td, td_runq);
 	kg->kg_runnable--;
 	td->td_priority = newpri;
 	setrunqueue(td, SRQ_BORING);
 }
-
+int limitcount;
 void
 setrunqueue(struct thread *td, int flags)
 {
-	struct kse *ke;
 	struct ksegrp *kg;
 	struct thread *td2;
 	struct thread *tda;
 	int count;
 
-	CTR4(KTR_RUNQ, "setrunqueue: td:%p ke:%p kg:%p pid:%d",
-	    td, td->td_kse, td->td_ksegrp, td->td_proc->p_pid);
+	CTR3(KTR_RUNQ, "setrunqueue: td:%p kg:%p pid:%d",
+	    td, td->td_ksegrp, td->td_proc->p_pid);
 	mtx_assert(&sched_lock, MA_OWNED);
 	KASSERT((TD_CAN_RUN(td) || TD_IS_RUNNING(td)),
 	    ("setrunqueue: bad thread state"));
 	TD_SET_RUNQ(td);
 	kg = td->td_ksegrp;
-	if ((td->td_proc->p_flag & P_SA) == 0) {
+	if ((td->td_proc->p_flag & P_HADTHREADS) == 0) {
 		/*
 		 * Common path optimisation: Only one of everything
 		 * and the KSE is always already attached.
 		 * Totally ignore the ksegrp run queue.
 		 */
+		if (kg->kg_avail_opennings != 1) {
+			if (limitcount < 100) {
+				limitcount++;
+				printf("pid %d: bad slot count (%d)\n",
+				    td->td_proc->p_pid, kg->kg_avail_opennings);
+
+			}
+			kg->kg_avail_opennings = 1;
+		}
+		kg->kg_avail_opennings--;
 		sched_add(td, flags);
 		return;
 	}
 
 	tda = kg->kg_last_assigned;
-	if ((ke = td->td_kse) == NULL) {
-		if (kg->kg_idle_kses) {
-			/*
-			 * There is a free one so it's ours for the asking..
-			 */
-			ke = TAILQ_FIRST(&kg->kg_iq);
-			CTR2(KTR_RUNQ, "setrunqueue: kg:%p: Use free ke:%p",
-			    kg, ke);
-			TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist);
-			ke->ke_state = KES_THREAD;
-			kg->kg_idle_kses--;
-		} else if (tda && (tda->td_priority > td->td_priority)) {
-			/*
-			 * None free, but there is one we can commandeer.
-			 */
-			ke = tda->td_kse;
-			CTR3(KTR_RUNQ,
-			    "setrunqueue: kg:%p: take ke:%p from td: %p",
-			    kg, ke, tda);
-			sched_rem(tda);
-			tda->td_kse = NULL;
-			ke->ke_thread = NULL;
-			tda = kg->kg_last_assigned =
-		    	    TAILQ_PREV(tda, threadqueue, td_runq);
-		}
-	} else {
-		/* 
-		 * Temporarily disassociate so it looks like the other cases.
+	if ((kg->kg_avail_opennings <= 0) &&
+	(tda && (tda->td_priority > td->td_priority))) {
+		/*
+		 * None free, but there is one we can commandeer.
 		 */
-		ke->ke_thread = NULL;
-		td->td_kse = NULL;
+		CTR2(KTR_RUNQ,
+		    "setrunqueue: kg:%p: take slot from td: %p", kg, tda);
+		sched_rem(tda);
+		tda = kg->kg_last_assigned =
+		    TAILQ_PREV(tda, threadqueue, td_runq);
+		kg->kg_avail_opennings++;
 	}
 
 	/*
@@ -407,40 +393,30 @@ setrunqueue(struct thread *td, int flags)
 	}
 
 	/*
-	 * If we have a ke to use, then put it on the run queue and
-	 * If needed, readjust the last_assigned pointer.
+	 * If we have a slot to use, then put the thread on the system
+	 * run queue and if needed, readjust the last_assigned pointer.
 	 */
-	if (ke) {
+	if (kg->kg_avail_opennings > 0) {
 		if (tda == NULL) {
 			/*
 			 * No pre-existing last assigned so whoever is first
 			 * gets the KSE we brought in.. (maybe us)
 			 */
 			td2 = TAILQ_FIRST(&kg->kg_runq);
-			KASSERT((td2->td_kse == NULL),
-			    ("unexpected ke present"));
-			td2->td_kse = ke;
-			ke->ke_thread = td2;
 			kg->kg_last_assigned = td2;
 		} else if (tda->td_priority > td->td_priority) {
-			/*
-			 * It's ours, grab it, but last_assigned is past us
-			 * so don't change it.
-			 */
-			td->td_kse = ke;
-			ke->ke_thread = td;
+			td2 = td;
 		} else {
 			/* 
 			 * We are past last_assigned, so 
-			 * put the new kse on whatever is next,
+			 * gave the next slot to whatever is next,
 			 * which may or may not be us.
 			 */
 			td2 = TAILQ_NEXT(tda, td_runq);
 			kg->kg_last_assigned = td2;
-			td2->td_kse = ke;
-			ke->ke_thread = td2;
 		}
-		sched_add(ke->ke_thread, flags);
+		kg->kg_avail_opennings--;
+		sched_add(td2, flags);
 	} else {
 		CTR3(KTR_RUNQ, "setrunqueue: held: td%p kg%p pid%d",
 			td, td->td_ksegrp, td->td_proc->p_pid);
@@ -692,7 +668,6 @@ runq_check(struct runq *rq)
 
 #if defined(SMP) && defined(SCHED_4BSD)
 int runq_fuzz = 1;
-SYSCTL_DECL(_kern_sched);
 SYSCTL_INT(_kern_sched, OID_AUTO, runq_fuzz, CTLFLAG_RW, &runq_fuzz, 0, "");
 #endif
 
@@ -766,3 +741,115 @@ runq_remove(struct runq *rq, struct kse *ke)
 	}
 }
 
+/****** functions that are temporarily here ***********/
+#include <vm/uma.h>
+#define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start))
+extern struct mtx kse_zombie_lock;
+
+/*
+ *  Allocate scheduler specific per-process resources.
+ * The thread and ksegrp have already been linked in.
+ * In this case just set the default concurrency value.
+ *
+ * Called from:
+ *  proc_init() (UMA init method)
+ */
+void
+sched_newproc(struct proc *p, struct ksegrp *kg, struct thread *td)
+{
+
+	/* This can go in sched_fork */
+	sched_init_concurrency(kg);
+}
+
+/*
+ * Called by the uma process fini routine..
+ * undo anything we may have done in the uma_init method.
+ * Panic if it's not all 1:1:1:1
+ * Called from:
+ *  proc_fini() (UMA method)
+ */
+void
+sched_destroyproc(struct proc *p)
+{
+
+	/* this function slated for destruction */
+	KASSERT((p->p_numthreads == 1), ("Cached proc with > 1 thread "));
+	KASSERT((p->p_numksegrps == 1), ("Cached proc with > 1 ksegrp "));
+}
+
+#define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start))
+/*
+ * thread is being either created or recycled.
+ * Fix up the per-scheduler resources associated with it.
+ * Called from:
+ *  sched_fork_thread()
+ *  thread_dtor()  (*may go away)
+ *  thread_init()  (*may go away)
+ */
+void
+sched_newthread(struct thread *td)
+{
+	struct td_sched *ke;
+
+	ke = (struct td_sched *) (td + 1);
+	bzero(ke, sizeof(*ke));
+	td->td_sched     = ke;
+	ke->ke_thread	= td;
+	ke->ke_oncpu	= NOCPU;
+	ke->ke_state	= KES_THREAD;
+}
+
+/*
+ * Set up an initial concurrency of 1
+ * and set the given thread (if given) to be using that
+ * concurrency slot.
+ * May be used "offline"..before the ksegrp is attached to the world
+ * and thus wouldn't need schedlock in that case.
+ * Called from:
+ *  thr_create()
+ *  proc_init() (UMA) via sched_newproc()
+ */
+void
+sched_init_concurrency(struct ksegrp *kg)
+{
+
+	kg->kg_concurrency = 1;
+	kg->kg_avail_opennings = 1;
+}
+
+/*
+ * Change the concurrency of an existing ksegrp to N
+ * Called from:
+ *  kse_create()
+ *  kse_exit()
+ *  thread_exit()
+ *  thread_single()
+ */
+void
+sched_set_concurrency(struct ksegrp *kg, int concurrency)
+{
+
+	/* Handle the case for a declining concurrency */
+	kg->kg_avail_opennings += (concurrency - kg->kg_concurrency);
+	kg->kg_concurrency = concurrency;
+}
+
+/*
+ * Called from thread_exit() for all exiting thread
+ *
+ * Not to be confused with sched_exit_thread()
+ * that is only called from thread_exit() for threads exiting
+ * without the rest of the process exiting because it is also called from
+ * sched_exit() and we wouldn't want to call it twice.
+ * XXX This can probably be fixed.
+ */
+void
+sched_thread_exit(struct thread *td)
+{
+
+	td->td_ksegrp->kg_avail_opennings++;
+	slot_fill(td->td_ksegrp);
+}
+
+#endif /* KERN_SWITCH_INCLUDE */
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index aae9d95..2c97a5b 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -334,13 +334,13 @@ mi_switch(int flags, struct thread *newtd)
 	PCPU_SET(switchtime, new_switchtime);
 	PCPU_SET(switchticks, ticks);
 	CTR4(KTR_PROC, "mi_switch: old thread %p (kse %p, pid %ld, %s)",
-	    (void *)td, td->td_kse, (long)p->p_pid, p->p_comm);
+	    (void *)td, td->td_sched, (long)p->p_pid, p->p_comm);
 	if (td->td_proc->p_flag & P_SA)
-		thread_switchout(td);
+		newtd = thread_switchout(td, flags, newtd);
 	sched_switch(td, newtd);
 
 	CTR4(KTR_PROC, "mi_switch: new thread %p (kse %p, pid %ld, %s)",
-	    (void *)td, td->td_kse, (long)p->p_pid, p->p_comm);
+	    (void *)td, td->td_sched, (long)p->p_pid, p->p_comm);
 
 	/* 
 	 * If the last thread was exiting, finish cleaning it up.
diff --git a/sys/kern/kern_thr.c b/sys/kern/kern_thr.c
index 7be6b6b..e4ec7be 100644
--- a/sys/kern/kern_thr.c
+++ b/sys/kern/kern_thr.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/resourcevar.h>
 #include <sys/sched.h>
+#include <sys/smp.h>
 #include <sys/sysent.h>
 #include <sys/systm.h>
 #include <sys/sysproto.h>
@@ -43,74 +44,13 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/frame.h>
 
+extern int max_threads_per_proc;
+extern int max_groups_per_proc;
+
 /*
  * Back end support functions.
  */
 
-void
-thr_exit1(void)
-{
-	struct ksegrp *kg;
-	struct thread *td;
-	struct kse *ke;
-	struct proc *p;
-
-	td = curthread;
-	p = td->td_proc;
-	kg = td->td_ksegrp;
-	ke = td->td_kse;
-
-	mtx_assert(&sched_lock, MA_OWNED);
-	PROC_LOCK_ASSERT(p, MA_OWNED);
-	KASSERT(!mtx_owned(&Giant), ("dying thread owns giant"));
-
-	/*
-	 * Shutting down last thread in the proc.  This will actually
-	 * call exit() in the trampoline when it returns.
-	 */
-	if (p->p_numthreads == 1) {
-		PROC_UNLOCK(p);
-		return;
-	}
-
-	/*
-	 * XXX Undelivered process wide signals should be reposted to the
-	 * proc.
-	 */
-
-	/* Clean up cpu resources. */
-	cpu_thread_exit(td);
-
-	/* Unlink the thread from the process and kseg. */
-	thread_unlink(td);
-
-	ke->ke_state = KES_UNQUEUED;
-	ke->ke_thread = NULL;
-	kse_unlink(ke);
-	sched_exit_kse(TAILQ_NEXT(ke, ke_kglist), td);
-
-	/*
-	 * If we were stopped while waiting for all threads to exit and this
-	 * is the last thread wakeup the exiting thread.
-	 */
-	if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE)
-		if (p->p_numthreads == 1)
-			thread_unsuspend_one(p->p_singlethread);
-
-	PROC_UNLOCK(p);
-	td->td_kse = NULL;
-	td->td_state = TDS_INACTIVE;
-#if 0
-	td->td_proc = NULL;
-#endif
-	td->td_ksegrp = NULL;
-	td->td_last_kse = NULL;
-	sched_exit_thread(TAILQ_NEXT(td, td_kglist), td);
-	thread_stash(td);
-
-	cpu_throw(td, choosethread());
-}
-
 #define	RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start))
 
 /*
@@ -120,69 +60,79 @@ int
 thr_create(struct thread *td, struct thr_create_args *uap)
     /* ucontext_t *ctx, long *id, int flags */
 {
-	struct kse *ke0;
-	struct thread *td0;
+	struct thread *newtd;
 	ucontext_t ctx;
 	long id;
 	int error;
+	struct ksegrp *kg, *newkg;
+	struct proc *p;
 
+	p = td->td_proc;
+	kg = td->td_ksegrp;
 	if ((error = copyin(uap->ctx, &ctx, sizeof(ctx))))
 		return (error);
 
-	/* Initialize our td. */
-	td0 = thread_alloc();
-
+	/* Have race condition but it is cheap */
+	if ((p->p_numksegrps >= max_groups_per_proc) ||
+	    (p->p_numthreads >= max_threads_per_proc)) {
+		return (EPROCLIM);
+	}
+	/* Initialize our td and new ksegrp.. */
+	newtd = thread_alloc();
+	newkg = ksegrp_alloc();
 	/*
 	 * Try the copyout as soon as we allocate the td so we don't have to
 	 * tear things down in a failure case below.
 	 */
-	id = td0->td_tid;
+	id = newtd->td_tid;
 	if ((error = copyout(&id, uap->id, sizeof(long)))) {
-		thread_free(td0);
+		ksegrp_free(newkg);
+		thread_free(newtd);
 		return (error);
 	}
 
-	bzero(&td0->td_startzero,
-	    (unsigned)RANGEOF(struct thread, td_startzero, td_endzero));
-	bcopy(&td->td_startcopy, &td0->td_startcopy,
+	bzero(&newtd->td_startzero,
+	    (unsigned) RANGEOF(struct thread, td_startzero, td_endzero));
+	bcopy(&td->td_startcopy, &newtd->td_startcopy,
 	    (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy));
 
-	td0->td_proc = td->td_proc;
-	PROC_LOCK(td->td_proc);
-	td0->td_sigmask = td->td_sigmask;
-	PROC_UNLOCK(td->td_proc);
-	td0->td_ucred = crhold(td->td_ucred);
+	bzero(&newkg->kg_startzero,
+	    (unsigned) RANGEOF(struct ksegrp, kg_startzero, kg_endzero));
+	bcopy(&kg->kg_startcopy, &newkg->kg_startcopy,
+	    (unsigned) RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy));
 
-	/* Initialize our kse structure. */
-	ke0 = kse_alloc();
-	bzero(&ke0->ke_startzero,
-	    RANGEOF(struct kse, ke_startzero, ke_endzero));
+	newtd->td_proc = td->td_proc;
+	newtd->td_ucred = crhold(td->td_ucred);
 
 	/* Set up our machine context. */
-	cpu_set_upcall(td0, td);
-	error = set_mcontext(td0, &ctx.uc_mcontext);
+	cpu_set_upcall(newtd, td);
+	error = set_mcontext(newtd, &ctx.uc_mcontext);
 	if (error != 0) {
-		kse_free(ke0);
-		thread_free(td0);
+		ksegrp_free(newkg);
+		thread_free(newtd);
+		crfree(td->td_ucred);
 		goto out;
 	}
 
 	/* Link the thread and kse into the ksegrp and make it runnable. */
+	PROC_LOCK(td->td_proc);
+	td->td_proc->p_flag |= P_HADTHREADS; 
+	newtd->td_sigmask = td->td_sigmask;
 	mtx_lock_spin(&sched_lock);
+	ksegrp_link(newkg, p);
+	thread_link(newtd, newkg);
+	mtx_unlock_spin(&sched_lock);
+	PROC_UNLOCK(p);
+	sched_init_concurrency(newkg);
 
-	thread_link(td0, td->td_ksegrp);
-	kse_link(ke0, td->td_ksegrp);
-
-	/* Bind this thread and kse together. */
-	td0->td_kse = ke0;
-	ke0->ke_thread = td0;
-
-	sched_fork_kse(td, ke0);
-	sched_fork_thread(td, td0);
+	/* let the scheduler know about these things. */
+	mtx_lock_spin(&sched_lock);
+	sched_fork_ksegrp(td, newkg);
+	sched_fork_thread(td, newtd);
 
-	TD_SET_CAN_RUN(td0);
+	TD_SET_CAN_RUN(newtd);
 	if ((uap->flags & THR_SUSPENDED) == 0)
-		setrunqueue(td0, SRQ_BORING);
+		setrunqueue(newtd, SRQ_BORING);
 
 	mtx_unlock_spin(&sched_lock);
 
@@ -216,12 +166,15 @@ thr_exit(struct thread *td, struct thr_exit_args *uap)
 	mtx_lock_spin(&sched_lock);
 
 	/*
-	 * This unlocks proc and doesn't return unless this is the last
-	 * thread.
+	 * Shutting down last thread in the proc.  This will actually
+	 * call exit() in the trampoline when it returns.
 	 */
-	thr_exit1();
+	if (p->p_numthreads != 1) {
+		thread_exit();
+		/* NOTREACHED */
+	}
 	mtx_unlock_spin(&sched_lock);
-
+	PROC_UNLOCK(p);
 	return (0);
 }
 
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index 031b67d..4aaef43 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -50,7 +50,6 @@ __FBSDID("$FreeBSD$");
  * KSEGRP related storage.
  */
 static uma_zone_t ksegrp_zone;
-static uma_zone_t kse_zone;
 static uma_zone_t thread_zone;
 
 /* DEBUG ONLY */
@@ -63,7 +62,7 @@ int max_threads_per_proc = 1500;
 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW,
 	&max_threads_per_proc, 0, "Limit on threads per proc");
 
-int max_groups_per_proc = 500;
+int max_groups_per_proc = 1500;
 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW,
 	&max_groups_per_proc, 0, "Limit on thread groups per proc");
 
@@ -76,19 +75,10 @@ int virtual_cpu;
 #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start))
 
 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads);
-TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses);
 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps);
 struct mtx kse_zombie_lock;
 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN);
 
-void kse_purge(struct proc *p, struct thread *td);
-void kse_purge_group(struct thread *td);
-
-/* move to proc.h */
-extern void	kseinit(void);
-extern void	kse_GC(void);
-
-
 static int
 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS)
 {
@@ -198,6 +188,7 @@ thread_dtor(void *mem, int size, void *arg)
 		/* NOTREACHED */
 	}
 #endif
+	sched_newthread(td);
 }
 
 /*
@@ -252,6 +243,7 @@ thread_init(void *mem, int size, int flags)
 	td->td_sleepqueue = sleepq_alloc();
 	td->td_turnstile = turnstile_alloc();
 	td->td_sched = (struct td_sched *)&td[1];
+	sched_newthread(td);
 	return (0);
 }
 
@@ -287,19 +279,6 @@ thread_fini(void *mem, int size)
 }
 
 /*
- * Initialize type-stable parts of a kse (when newly created).
- */
-static int
-kse_init(void *mem, int size, int flags)
-{
-	struct kse	*ke;
-
-	ke = (struct kse *)mem;
-	ke->ke_sched = (struct ke_sched *)&ke[1];
-	return (0);
-}
-
-/*
  * Initialize type-stable parts of a ksegrp (when newly created).
  */
 static int
@@ -309,46 +288,10 @@ ksegrp_init(void *mem, int size, int flags)
 
 	kg = (struct ksegrp *)mem;
 	kg->kg_sched = (struct kg_sched *)&kg[1];
+	/* sched_newksegrp(kg); */
 	return (0);
 }
 
-/*
- * KSE is linked into kse group.
- */
-void
-kse_link(struct kse *ke, struct ksegrp *kg)
-{
-	struct proc *p = kg->kg_proc;
-
-	TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist);
-	kg->kg_kses++;
-	ke->ke_state	= KES_UNQUEUED;
-	ke->ke_proc	= p;
-	ke->ke_ksegrp	= kg;
-	ke->ke_thread	= NULL;
-	ke->ke_oncpu	= NOCPU;
-	ke->ke_flags	= 0;
-}
-
-void
-kse_unlink(struct kse *ke)
-{
-	struct ksegrp *kg;
-
-	mtx_assert(&sched_lock, MA_OWNED);
-	kg = ke->ke_ksegrp;
-	TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist);
-	if (ke->ke_state == KES_IDLE) {
-		TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist);
-		kg->kg_idle_kses--;
-	}
-	--kg->kg_kses;
-	/*
-	 * Aggregate stats from the KSE
-	 */
-	kse_stash(ke);
-}
-
 void
 ksegrp_link(struct ksegrp *kg, struct proc *p)
 {
@@ -356,8 +299,6 @@ ksegrp_link(struct ksegrp *kg, struct proc *p)
 	TAILQ_INIT(&kg->kg_threads);
 	TAILQ_INIT(&kg->kg_runq);	/* links with td_runq */
 	TAILQ_INIT(&kg->kg_slpq);	/* links with td_runq */
-	TAILQ_INIT(&kg->kg_kseq);	/* all kses in ksegrp */
-	TAILQ_INIT(&kg->kg_iq);		/* all idle kses in ksegrp */
 	TAILQ_INIT(&kg->kg_upcalls);	/* all upcall structure in ksegrp */
 	kg->kg_proc = p;
 	/*
@@ -366,15 +307,16 @@ ksegrp_link(struct ksegrp *kg, struct proc *p)
 	 */
 	kg->kg_numthreads = 0;
 	kg->kg_runnable   = 0;
-	kg->kg_kses       = 0;
-	kg->kg_runq_kses  = 0; /* XXXKSE change name */
-	kg->kg_idle_kses  = 0;
 	kg->kg_numupcalls = 0;
 	/* link it in now that it's consistent */
 	p->p_numksegrps++;
 	TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp);
 }
 
+/*
+ * Called from:
+ *   thread-exit()
+ */
 void
 ksegrp_unlink(struct ksegrp *kg)
 {
@@ -382,7 +324,6 @@ ksegrp_unlink(struct ksegrp *kg)
 
 	mtx_assert(&sched_lock, MA_OWNED);
 	KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads"));
-	KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses"));
 	KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls"));
 
 	p = kg->kg_proc;
@@ -391,16 +332,18 @@ ksegrp_unlink(struct ksegrp *kg)
 	/*
 	 * Aggregate stats from the KSE
 	 */
-	ksegrp_stash(kg);
 }
 
 /*
  * For a newly created process,
  * link up all the structures and its initial threads etc.
+ * called from:
+ * {arch}/{arch}/machdep.c   ia64_init(), init386() etc.
+ * proc_dtor() (should go away)
+ * proc_init()
  */
 void
-proc_linkup(struct proc *p, struct ksegrp *kg,
-	    struct kse *ke, struct thread *td)
+proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td)
 {
 
 	TAILQ_INIT(&p->p_ksegrps);	     /* all ksegrps in proc */
@@ -410,7 +353,6 @@ proc_linkup(struct proc *p, struct ksegrp *kg,
 	p->p_numthreads = 0;
 
 	ksegrp_link(kg, p);
-	kse_link(ke, kg);
 	thread_link(td, kg);
 }
 
@@ -429,10 +371,7 @@ threadinit(void)
 	ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(),
 	    NULL, NULL, ksegrp_init, NULL,
 	    UMA_ALIGN_CACHE, 0);
-	kse_zone = uma_zcreate("KSE", sched_sizeof_kse(),
-	    NULL, NULL, kse_init, NULL,
-	    UMA_ALIGN_CACHE, 0);
-	kseinit();
+	kseinit();	/* set up kse specific stuff  e.g. upcall zone*/
 }
 
 /*
@@ -447,17 +386,6 @@ thread_stash(struct thread *td)
 }
 
 /*
- * Stash an embarasingly extra kse into the zombie kse queue.
- */
-void
-kse_stash(struct kse *ke)
-{
-	mtx_lock_spin(&kse_zombie_lock);
-	TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq);
-	mtx_unlock_spin(&kse_zombie_lock);
-}
-
-/*
  * Stash an embarasingly extra ksegrp into the zombie ksegrp queue.
  */
 void
@@ -475,7 +403,6 @@ void
 thread_reap(void)
 {
 	struct thread *td_first, *td_next;
-	struct kse *ke_first, *ke_next;
 	struct ksegrp *kg_first, * kg_next;
 
 	/*
@@ -483,16 +410,12 @@ thread_reap(void)
 	 * we really don't care about the next instant..
 	 */
 	if ((!TAILQ_EMPTY(&zombie_threads))
-	    || (!TAILQ_EMPTY(&zombie_kses))
 	    || (!TAILQ_EMPTY(&zombie_ksegrps))) {
 		mtx_lock_spin(&kse_zombie_lock);
 		td_first = TAILQ_FIRST(&zombie_threads);
-		ke_first = TAILQ_FIRST(&zombie_kses);
 		kg_first = TAILQ_FIRST(&zombie_ksegrps);
 		if (td_first)
 			TAILQ_INIT(&zombie_threads);
-		if (ke_first)
-			TAILQ_INIT(&zombie_kses);
 		if (kg_first)
 			TAILQ_INIT(&zombie_ksegrps);
 		mtx_unlock_spin(&kse_zombie_lock);
@@ -503,18 +426,17 @@ thread_reap(void)
 			thread_free(td_first);
 			td_first = td_next;
 		}
-		while (ke_first) {
-			ke_next = TAILQ_NEXT(ke_first, ke_procq);
-			kse_free(ke_first);
-			ke_first = ke_next;
-		}
 		while (kg_first) {
 			kg_next = TAILQ_NEXT(kg_first, kg_ksegrp);
 			ksegrp_free(kg_first);
 			kg_first = kg_next;
 		}
+		/*
+		 * there will always be a thread on the list if one of these
+		 * is there.
+		 */
+		kse_GC();
 	}
-	kse_GC();
 }
 
 /*
@@ -527,15 +449,6 @@ ksegrp_alloc(void)
 }
 
 /*
- * Allocate a kse.
- */
-struct kse *
-kse_alloc(void)
-{
-	return (uma_zalloc(kse_zone, M_WAITOK));
-}
-
-/*
  * Allocate a thread.
  */
 struct thread *
@@ -555,15 +468,6 @@ ksegrp_free(struct ksegrp *td)
 }
 
 /*
- * Deallocate a kse.
- */
-void
-kse_free(struct kse *td)
-{
-	uma_zfree(kse_zone, td);
-}
-
-/*
  * Deallocate a thread.
  */
 void
@@ -594,92 +498,150 @@ thread_free(struct thread *td)
  * Of course in the end, they end up coming here through exit1
  * anyhow..  After fixing 'thr' to play by the rules we should be able 
  * to merge these two functions together.
+ *
+ * called from:
+ * exit1()
+ * kse_exit()
+ * thr_exit()
+ * thread_user_enter()
+ * thread_userret()
+ * thread_suspend_check()
  */
 void
 thread_exit(void)
 {
 	struct thread *td;
-	struct kse *ke;
 	struct proc *p;
 	struct ksegrp	*kg;
 
 	td = curthread;
 	kg = td->td_ksegrp;
 	p = td->td_proc;
-	ke = td->td_kse;
 
 	mtx_assert(&sched_lock, MA_OWNED);
+	mtx_assert(&Giant, MA_NOTOWNED);
+	PROC_LOCK_ASSERT(p, MA_OWNED);
 	KASSERT(p != NULL, ("thread exiting without a process"));
-	KASSERT(ke != NULL, ("thread exiting without a kse"));
 	KASSERT(kg != NULL, ("thread exiting without a kse group"));
-	PROC_LOCK_ASSERT(p, MA_OWNED);
 	CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td,
 	    (long)p->p_pid, p->p_comm);
-	mtx_assert(&Giant, MA_NOTOWNED);
 
 	if (td->td_standin != NULL) {
+		/*
+		 * Note that we don't need to free the cred here as it
+		 * is done in thread_reap().
+		 */
 		thread_stash(td->td_standin);
 		td->td_standin = NULL;
 	}
 
+	/*
+	 * drop FPU & debug register state storage, or any other
+	 * architecture specific resources that
+	 * would not be on a new untouched process.
+	 */
 	cpu_thread_exit(td);	/* XXXSMP */
 
 	/*
+	 * The thread is exiting. scheduler can release its stuff
+	 * and collect stats etc.
+	 */
+	sched_thread_exit(td);
+
+	/*
 	 * The last thread is left attached to the process
 	 * So that the whole bundle gets recycled. Skip
-	 * all this stuff.
+	 * all this stuff if we never had threads.
+	 * EXIT clears all sign of other threads when
+	 * it goes to single threading, so the last thread always
+	 * takes the short path.
 	 */
-	if (p->p_numthreads > 1) {
-		thread_unlink(td);
-		if (p->p_maxthrwaits)
-			wakeup(&p->p_numthreads);
-		/*
-		 * The test below is NOT true if we are the
-		 * sole exiting thread. P_STOPPED_SNGL is unset
-		 * in exit1() after it is the only survivor.
-		 */
-		if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
-			if (p->p_numthreads == p->p_suspcount) {
-				thread_unsuspend_one(p->p_singlethread);
+	if (p->p_flag & P_HADTHREADS) {
+		if (p->p_numthreads > 1) {
+			thread_unlink(td);
+
+			/* XXX first arg not used in 4BSD or ULE */
+			sched_exit_thread(FIRST_THREAD_IN_PROC(p), td);
+
+			/*
+			 * as we are exiting there is room for another
+			 * to be created.
+			 */
+			if (p->p_maxthrwaits)
+				wakeup(&p->p_numthreads);
+
+			/*
+			 * The test below is NOT true if we are the
+			 * sole exiting thread. P_STOPPED_SNGL is unset
+			 * in exit1() after it is the only survivor.
+			 */
+			if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) {
+				if (p->p_numthreads == p->p_suspcount) {
+					thread_unsuspend_one(p->p_singlethread);
+				}
 			}
-		}
 
-		/*
-		 * Because each upcall structure has an owner thread,
-		 * owner thread exits only when process is in exiting
-		 * state, so upcall to userland is no longer needed,
-		 * deleting upcall structure is safe here.
-		 * So when all threads in a group is exited, all upcalls
-		 * in the group should be automatically freed.
-		 */
-		if (td->td_upcall)
+			/*
+			 * Because each upcall structure has an owner thread,
+			 * owner thread exits only when process is in exiting
+			 * state, so upcall to userland is no longer needed,
+			 * deleting upcall structure is safe here.
+			 * So when all threads in a group is exited, all upcalls
+			 * in the group should be automatically freed.
+			 *  XXXKSE This is a KSE thing and should be exported
+			 * there somehow.
+			 */
 			upcall_remove(td);
 
-		sched_exit_thread(FIRST_THREAD_IN_PROC(p), td);
-		sched_exit_kse(FIRST_KSE_IN_PROC(p), td);
-		ke->ke_state = KES_UNQUEUED;
-		ke->ke_thread = NULL;
-		/*
-		 * Decide what to do with the KSE attached to this thread.
-		 */
-		if (ke->ke_flags & KEF_EXIT) {
-			kse_unlink(ke);
-			if (kg->kg_kses == 0) {
+			/*
+			 * If the thread we unlinked above was the last one,
+			 * then this ksegrp should go away too.
+			 */
+			if (kg->kg_numthreads == 0) {
+				/*
+				 * let the scheduler know about this in case
+				 * it needs to recover stats or resources.
+				 * Theoretically we could let
+				 * sched_exit_ksegrp()  do the equivalent of
+				 * setting the concurrency to 0
+				 * but don't do it yet to avoid changing
+				 * the existing scheduler code until we
+				 * are ready.
+				 * We supply a random other ksegrp
+				 * as the recipient of any built up
+				 * cpu usage etc. (If the scheduler wants it).
+				 * XXXKSE
+				 * This is probably not fair so think of
+ 				 * a better answer.
+				 */
 				sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td);
+				sched_set_concurrency(kg, 0); /* XXX TEMP */
 				ksegrp_unlink(kg);
+				ksegrp_stash(kg);
 			}
+			PROC_UNLOCK(p);
+			td->td_ksegrp	= NULL;
+			PCPU_SET(deadthread, td);
+		} else {
+			/*
+			 * The last thread is exiting.. but not through exit()
+			 * what should we do?
+			 * Theoretically this can't happen
+ 			 * exit1() - clears threading flags before coming here
+ 			 * kse_exit() - treats last thread specially
+ 			 * thr_exit() - treats last thread specially
+ 			 * thread_user_enter() - only if more exist
+ 			 * thread_userret() - only if more exist
+ 			 * thread_suspend_check() - only if more exist
+			 */
+			panic ("thread_exit: Last thread exiting on its own");
 		}
-		else
-			kse_reassign(ke);
-		PROC_UNLOCK(p);
-		td->td_kse	= NULL;
-#if 0
-		td->td_proc	= NULL;
-#endif
-		td->td_ksegrp	= NULL;
-		td->td_last_kse	= NULL;
-		PCPU_SET(deadthread, td);
 	} else {
+		/*
+		 * non threaded process comes here.
+		 * This includes an EX threaded process that is coming
+		 * here via exit1(). (exit1 dethreads the proc first).
+		 */
 		PROC_UNLOCK(p);
 	}
 	td->td_state = TDS_INACTIVE;
@@ -703,10 +665,13 @@ thread_wait(struct proc *p)
 	KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()"));
 	FOREACH_THREAD_IN_PROC(p, td) {
 		if (td->td_standin != NULL) {
+			crfree(td->td_ucred);
+			td->td_ucred = NULL;
 			thread_free(td->td_standin);
 			td->td_standin = NULL;
 		}
 		cpu_thread_clean(td);
+		crfree(td->td_ucred);
 	}
 	thread_reap();	/* check for zombie threads etc. */
 }
@@ -718,6 +683,10 @@ thread_wait(struct proc *p)
  *
  * Note that we do not link to the proc's ucred here.
  * The thread is linked as if running but no KSE assigned.
+ * Called from:
+ *  proc_linkup()
+ *  thread_schedule_upcall()
+ *  thr_create()
  */
 void
 thread_link(struct thread *td, struct ksegrp *kg)
@@ -728,10 +697,8 @@ thread_link(struct thread *td, struct ksegrp *kg)
 	td->td_state    = TDS_INACTIVE;
 	td->td_proc     = p;
 	td->td_ksegrp   = kg;
-	td->td_last_kse = NULL;
 	td->td_flags    = 0;
 	td->td_kflags	= 0;
-	td->td_kse      = NULL;
 
 	LIST_INIT(&td->td_contested);
 	callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
@@ -741,6 +708,10 @@ thread_link(struct thread *td, struct ksegrp *kg)
 	kg->kg_numthreads++;
 }
 
+/*
+ * Called from:
+ *  thread_exit()
+ */
 void
 thread_unlink(struct thread *td)
 {
@@ -753,73 +724,7 @@ thread_unlink(struct thread *td)
 	TAILQ_REMOVE(&kg->kg_threads, td, td_kglist);
 	kg->kg_numthreads--;
 	/* could clear a few other things here */
-}
-
-/*
- * Purge a ksegrp resource. When a ksegrp is preparing to
- * exit, it calls this function.
- */
-void
-kse_purge_group(struct thread *td)
-{
-	struct ksegrp *kg;
-	struct kse *ke;
-
-	kg = td->td_ksegrp;
- 	KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__));
-	while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) {
-		KASSERT(ke->ke_state == KES_IDLE,
-			("%s: wrong idle KSE state", __func__));
-		kse_unlink(ke);
-	}
-	KASSERT((kg->kg_kses == 1),
-		("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses));
-	KASSERT((kg->kg_numupcalls == 0),
-	        ("%s: ksegrp still has %d upcall datas",
-		__func__, kg->kg_numupcalls));
-}
-
-/*
- * Purge a process's KSE resource. When a process is preparing to
- * exit, it calls kse_purge to release any extra KSE resources in
- * the process.
- */
-void
-kse_purge(struct proc *p, struct thread *td)
-{
-	struct ksegrp *kg;
-	struct kse *ke;
-
- 	KASSERT(p->p_numthreads == 1, ("bad thread number"));
-	while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) {
-		TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp);
-		p->p_numksegrps--;
-		/*
-		 * There is no ownership for KSE, after all threads
-		 * in the group exited, it is possible that some KSEs
-		 * were left in idle queue, gc them now.
-		 */
-		while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) {
-			KASSERT(ke->ke_state == KES_IDLE,
-			   ("%s: wrong idle KSE state", __func__));
-			TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist);
-			kg->kg_idle_kses--;
-			TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist);
-			kg->kg_kses--;
-			kse_stash(ke);
-		}
-		KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) ||
-		        ((kg->kg_kses == 1) && (kg == td->td_ksegrp)),
-		        ("ksegrp has wrong kg_kses: %d", kg->kg_kses));
-		KASSERT((kg->kg_numupcalls == 0),
-		        ("%s: ksegrp still has %d upcall datas",
-			__func__, kg->kg_numupcalls));
-
-		if (kg != td->td_ksegrp)
-			ksegrp_stash(kg);
-	}
-	TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp);
-	p->p_numksegrps++;
+	/* Must  NOT clear links to proc and ksegrp! */
 }
 
 /*
@@ -849,7 +754,7 @@ thread_single(int force_exit)
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 	KASSERT((td != NULL), ("curthread is NULL"));
 
-	if ((p->p_flag & P_SA) == 0 && p->p_numthreads == 1)
+	if ((p->p_flag & P_HADTHREADS) == 0)
 		return (0);
 
 	/* Is someone already single threading? */
@@ -924,11 +829,17 @@ thread_single(int force_exit)
 			remaining = p->p_numthreads - p->p_suspcount;
 	}
 	if (force_exit == SINGLE_EXIT) {
-		if (td->td_upcall)
-			upcall_remove(td);
-		kse_purge(p, td);
+		upcall_remove(td);
+		p->p_flag &= ~(P_SA|P_HADTHREADS);
+		td->td_mailbox = NULL;
+		td->td_pflags &= ~TDP_SA;
+		p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT);
+		p->p_singlethread = NULL;
+		sched_set_concurrency(td->td_ksegrp, 1);
+		mtx_unlock_spin(&sched_lock);
+	} else {
+		mtx_unlock_spin(&sched_lock);
 	}
-	mtx_unlock_spin(&sched_lock);
 	return (0);
 }
 
@@ -1000,10 +911,7 @@ thread_suspend_check(int return_instead)
 		 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE.
 		 */
 		if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) {
-			if (p->p_flag & P_SA)
-				thread_exit();
-			else
-				thr_exit1();
+			thread_exit();
 		}
 
 		/*
@@ -1084,6 +992,10 @@ thread_unsuspend(struct proc *p)
 	}
 }
 
+/*
+ * End the single threading mode..
+ * Part of this is duplicated in thread-single in the SINGLE_EXIT case.
+ */
 void
 thread_single_end(void)
 {
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index 44e34a5..2b1ff36 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -35,6 +35,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#define kse td_sched
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -64,29 +66,69 @@ __FBSDID("$FreeBSD$");
 #endif
 #define	NICE_WEIGHT		1	/* Priorities per nice level. */
 
-struct ke_sched {
-	int		ske_cpticks;	/* (j) Ticks of cpu time. */
-	struct runq	*ske_runq;	/* runq the kse is currently on */
+/*
+ * The schedulable entity that can be given a context to run.
+ * A process may have several of these. Probably one per processor
+ * but posibly a few more. In this universe they are grouped
+ * with a KSEG that contains the priority and niceness
+ * for the group.
+ */
+struct kse {
+	TAILQ_ENTRY(kse) ke_kglist;	/* (*) Queue of KSEs in ke_ksegrp. */
+	TAILQ_ENTRY(kse) ke_kgrlist;	/* (*) Queue of KSEs in this state. */
+	TAILQ_ENTRY(kse) ke_procq;	/* (j/z) Run queue. */
+	struct thread	*ke_thread;	/* (*) Active associated thread. */
+	fixpt_t		ke_pctcpu;	/* (j) %cpu during p_swtime. */
+	u_char		ke_oncpu;	/* (j) Which cpu we are on. */
+	char		ke_rqindex;	/* (j) Run queue index. */
+	enum {
+		KES_THREAD = 0x0,	/* slaved to thread state */
+		KES_ONRUNQ
+	} ke_state;			/* (j) KSE status. */
+	int		ke_cpticks;	/* (j) Ticks of cpu time. */
+	struct runq	*ke_runq;	/* runq the kse is currently on */
+	int		ke_pinned;	/* nested count of pinned to a cpu */
 };
-#define ke_runq 	ke_sched->ske_runq
-#define ke_cpticks 	ke_sched->ske_cpticks
-#define KEF_BOUND	KEF_SCHED1
+
+#define ke_proc		ke_thread->td_proc
+#define ke_ksegrp	ke_thread->td_ksegrp
+
+#define td_kse td_sched
+
+/* flags kept in td_flags */
+#define TDF_DIDRUN	TDF_SCHED0	/* KSE actually ran. */
+#define TDF_EXIT	TDF_SCHED1	/* KSE is being killed. */
+#define TDF_BOUND	TDF_SCHED2
+
+#define ke_flags	ke_thread->td_flags
+#define KEF_DIDRUN	TDF_DIDRUN /* KSE actually ran. */
+#define KEF_EXIT	TDF_EXIT /* KSE is being killed. */
+#define KEF_BOUND	TDF_BOUND /* stuck to one CPU */
 
 #define SKE_RUNQ_PCPU(ke)						\
     ((ke)->ke_runq != 0 && (ke)->ke_runq != &runq)
 
+struct kg_sched {
+	struct thread	*skg_last_assigned; /* (j) Last thread assigned to */
+					   /* the system scheduler. */
+	int	skg_avail_opennings;	/* (j) Num KSEs requested in group. */
+	int	skg_concurrency;	/* (j) Num KSEs requested in group. */
+	int	skg_runq_kses;		/* (j) Num KSEs on runq. */
+};
+#define kg_last_assigned	kg_sched->skg_last_assigned
+#define kg_avail_opennings	kg_sched->skg_avail_opennings
+#define kg_concurrency		kg_sched->skg_concurrency
+#define kg_runq_kses		kg_sched->skg_runq_kses
+
 /*
  * KSE_CAN_MIGRATE macro returns true if the kse can migrate between
  * cpus.
  */
 #define KSE_CAN_MIGRATE(ke)						\
-    ((ke)->ke_thread->td_pinned == 0 && ((ke)->ke_flags & KEF_BOUND) == 0)
-static struct ke_sched ke_sched;
+    ((ke)->ke_pinned == 0 && ((ke)->ke_flags & KEF_BOUND) == 0)
 
-struct ke_sched *kse0_sched = &ke_sched;
-struct kg_sched *ksegrp0_sched = NULL;
-struct p_sched *proc0_sched = NULL;
-struct td_sched *thread0_sched = NULL;
+static struct kse kse0;
+static struct kg_sched kg_sched0;
 
 static int	sched_tdcnt;	/* Total runnable threads in the system. */
 static int	sched_quantum;	/* Roundrobin scheduling quantum in ticks. */
@@ -94,6 +136,9 @@ static int	sched_quantum;	/* Roundrobin scheduling quantum in ticks. */
 
 static struct callout roundrobin_callout;
 
+static void	slot_fill(struct ksegrp *kg);
+static struct kse *sched_choose(void);		/* XXX Should be thread * */
+
 static void	setup_runqs(void);
 static void	roundrobin(void *arg);
 static void	schedcpu(void);
@@ -213,7 +258,7 @@ maybe_resched(struct thread *td)
 {
 
 	mtx_assert(&sched_lock, MA_OWNED);
-	if (td->td_priority < curthread->td_priority && curthread->td_kse)
+	if (td->td_priority < curthread->td_priority)
 		curthread->td_flags |= TDF_NEEDRESCHED;
 }
 
@@ -353,7 +398,8 @@ schedcpu(void)
 		p->p_swtime++;
 		FOREACH_KSEGRP_IN_PROC(p, kg) { 
 			awake = 0;
-			FOREACH_KSE_IN_GROUP(kg, ke) {
+			FOREACH_THREAD_IN_GROUP(kg, td) {
+				ke = td->td_kse;
 				/*
 				 * Increment sleep time (if sleeping).  We
 				 * ignore overflow, as above.
@@ -366,7 +412,7 @@ schedcpu(void)
 					awake = 1;
 					ke->ke_flags &= ~KEF_DIDRUN;
 				} else if ((ke->ke_state == KES_THREAD) &&
-				    (TD_IS_RUNNING(ke->ke_thread))) {
+				    (TD_IS_RUNNING(td))) {
 					awake = 1;
 					/* Do not clear KEF_DIDRUN */
 				} else if (ke->ke_flags & KEF_DIDRUN) {
@@ -517,6 +563,28 @@ sched_setup(void *dummy)
 }
 
 /* External interfaces start here */
+/*
+ * Very early in the boot some setup of scheduler-specific
+ * parts of proc0 and of soem scheduler resources needs to be done.
+ * Called from:
+ *  proc0_init()
+ */
+void
+schedinit(void)
+{
+	/*
+	 * Set up the scheduler specific parts of proc0.
+	 */
+	proc0.p_sched = NULL; /* XXX */
+	ksegrp0.kg_sched = &kg_sched0;
+	thread0.td_sched = &kse0;
+	kse0.ke_thread = &thread0;
+	kse0.ke_oncpu = NOCPU; /* wrong.. can we use PCPU(cpuid) yet? */
+	kse0.ke_state = KES_THREAD;
+	kg_sched0.skg_concurrency = 1;
+	kg_sched0.skg_avail_opennings = 0; /* we are already running */
+}
+
 int
 sched_runnable(void)
 {
@@ -579,17 +647,11 @@ sched_clock(struct thread *td)
 void
 sched_exit(struct proc *p, struct thread *td)
 {
-	sched_exit_kse(FIRST_KSE_IN_PROC(p), td);
 	sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td);
 	sched_exit_thread(FIRST_THREAD_IN_PROC(p), td);
 }
 
 void
-sched_exit_kse(struct kse *ke, struct thread *child)
-{
-}
-
-void
 sched_exit_ksegrp(struct ksegrp *kg, struct thread *childtd)
 {
 
@@ -605,17 +667,10 @@ sched_exit_thread(struct thread *td, struct thread *child)
 }
 
 void
-sched_fork(struct thread *td, struct proc *p1)
-{
-	sched_fork_kse(td, FIRST_KSE_IN_PROC(p1));
-	sched_fork_ksegrp(td, FIRST_KSEGRP_IN_PROC(p1));
-	sched_fork_thread(td, FIRST_THREAD_IN_PROC(p1));
-}
-
-void
-sched_fork_kse(struct thread *td, struct kse *child)
+sched_fork(struct thread *td, struct thread *childtd)
 {
-	child->ke_cpticks = 0;
+	sched_fork_ksegrp(td, childtd->td_ksegrp);
+	sched_fork_thread(td, childtd);
 }
 
 void
@@ -626,8 +681,9 @@ sched_fork_ksegrp(struct thread *td, struct ksegrp *child)
 }
 
 void
-sched_fork_thread(struct thread *td, struct thread *child)
+sched_fork_thread(struct thread *td, struct thread *childtd)
 {
+	sched_newthread(childtd);
 }
 
 void
@@ -687,14 +743,21 @@ sched_switch(struct thread *td, struct thread *newtd)
 	p = td->td_proc;
 
 	mtx_assert(&sched_lock, MA_OWNED);
-	KASSERT((ke->ke_state == KES_THREAD), ("sched_switch: kse state?"));
 
 	if ((p->p_flag & P_NOLOAD) == 0)
 		sched_tdcnt--;
 	if (newtd != NULL && (newtd->td_proc->p_flag & P_NOLOAD) == 0)
 		sched_tdcnt++;
+	/* 
+	 * The thread we are about to run needs to be counted as if it had been 
+	 * added to the run queue and selected.
+	 */
+	if (newtd) {
+		newtd->td_ksegrp->kg_avail_opennings--;
+		newtd->td_kse->ke_flags |= KEF_DIDRUN;
+        	TD_SET_RUNNING(newtd);
+	}
 	td->td_lastcpu = td->td_oncpu;
-	td->td_last_kse = ke;
 	td->td_flags &= ~TDF_NEEDRESCHED;
 	td->td_pflags &= ~TDP_OWEPREEMPT;
 	td->td_oncpu = NOCPU;
@@ -706,16 +769,19 @@ sched_switch(struct thread *td, struct thread *newtd)
 	 */
 	if (td == PCPU_GET(idlethread))
 		TD_SET_CAN_RUN(td);
-	else if (TD_IS_RUNNING(td)) {
-		/* Put us back on the run queue (kse and all). */
-		setrunqueue(td, SRQ_OURSELF|SRQ_YIELDING);
-	} else if (p->p_flag & P_SA) {
-		/*
-		 * We will not be on the run queue. So we must be
-		 * sleeping or similar. As it's available,
-		 * someone else can use the KSE if they need it.
-		 */
-		kse_reassign(ke);
+	else {
+		td->td_ksegrp->kg_avail_opennings++;
+		if (TD_IS_RUNNING(td)) {
+			/* Put us back on the run queue (kse and all). */
+			setrunqueue(td, SRQ_OURSELF|SRQ_YIELDING);
+		} else if (p->p_flag & P_HADTHREADS) {
+			/*
+			 * We will not be on the run queue. So we must be
+			 * sleeping or similar. As it's available,
+			 * someone else can use the KSE if they need it.
+			 */
+			slot_fill(td->td_ksegrp);
+		}
 	}
 	if (newtd == NULL)
 		newtd = choosethread();
@@ -750,7 +816,7 @@ forward_wakeup(int  cpunum)
 
 	mtx_assert(&sched_lock, MA_OWNED);
 
-	CTR0(KTR_SMP, "forward_wakeup()");
+	CTR0(KTR_RUNQ, "forward_wakeup()");
 
 	if ((!forward_wakeup_enabled) ||
 	     (forward_wakeup_use_mask == 0 && forward_wakeup_use_loop == 0))
@@ -838,9 +904,6 @@ sched_add(struct thread *td, int flags)
 
 	ke = td->td_kse;
 	mtx_assert(&sched_lock, MA_OWNED);
-	KASSERT((ke->ke_thread != NULL), ("sched_add: No thread on KSE"));
-	KASSERT((ke->ke_thread->td_kse != NULL),
-	    ("sched_add: No KSE on thread"));
 	KASSERT(ke->ke_state != KES_ONRUNQ,
 	    ("sched_add: kse %p (%s) already in run queue", ke,
 	    ke->ke_proc->p_comm));
@@ -974,10 +1037,6 @@ sched_choose(void)
 		ke->ke_state = KES_THREAD;
 		ke->ke_ksegrp->kg_runq_kses--;
 
-		KASSERT((ke->ke_thread != NULL),
-		    ("sched_choose: No thread on KSE"));
-		KASSERT((ke->ke_thread->td_kse != NULL),
-		    ("sched_choose: No KSE on thread"));
 		KASSERT(ke->ke_proc->p_sflag & PS_INMEM,
 		    ("sched_choose: process swapped out"));
 	}
@@ -1042,14 +1101,9 @@ sched_load(void)
 }
 
 int
-sched_sizeof_kse(void)
-{
-	return (sizeof(struct kse) + sizeof(struct ke_sched));
-}
-int
 sched_sizeof_ksegrp(void)
 {
-	return (sizeof(struct ksegrp));
+	return (sizeof(struct ksegrp) + sizeof(struct kg_sched));
 }
 int
 sched_sizeof_proc(void)
@@ -1059,7 +1113,7 @@ sched_sizeof_proc(void)
 int
 sched_sizeof_thread(void)
 {
-	return (sizeof(struct thread));
+	return (sizeof(struct thread) + sizeof(struct kse));
 }
 
 fixpt_t
@@ -1068,10 +1122,9 @@ sched_pctcpu(struct thread *td)
 	struct kse *ke;
 
 	ke = td->td_kse;
-	if (ke == NULL)
-		ke = td->td_last_kse;
-	if (ke)
-		return (ke->ke_pctcpu);
+	return (ke->ke_pctcpu);
 
 	return (0);
 }
+#define KERN_SWITCH_INCLUDE 1
+#include "kern/kern_switch.c"
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index e235c1f..52349c5 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -29,6 +29,8 @@ __FBSDID("$FreeBSD$");
 
 #include <opt_sched.h>
 
+#define kse td_sched
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kdb.h>
@@ -90,53 +92,82 @@ SYSINIT(preempt_warning, SI_SUB_COPYRIGHT, SI_ORDER_ANY, printf_caddr_t,
 #endif
 
 /*
+ * The schedulable entity that can be given a context to run.
+ * A process may have several of these. Probably one per processor
+ * but posibly a few more. In this universe they are grouped
+ * with a KSEG that contains the priority and niceness
+ * for the group.
+ */
+struct kse {
+	TAILQ_ENTRY(kse) ke_kglist;	/* (*) Queue of threads in ke_ksegrp. */
+	TAILQ_ENTRY(kse) ke_kgrlist;	/* (*) Queue of threads in this state.*/
+	TAILQ_ENTRY(kse) ke_procq;	/* (j/z) Run queue. */
+	int		ke_flags;	/* (j) KEF_* flags. */
+	struct thread	*ke_thread;	/* (*) Active associated thread. */
+	fixpt_t		ke_pctcpu;	/* (j) %cpu during p_swtime. */
+	u_char		ke_oncpu;	/* (j) Which cpu we are on. */
+	char		ke_rqindex;	/* (j) Run queue index. */
+	enum {
+		KES_THREAD = 0x0,	/* slaved to thread state */
+		KES_ONRUNQ
+	} ke_state;			/* (j) thread sched specific status. */
+	int		ke_slptime;
+	int		ke_pinned;
+	int		ke_slice;
+	struct runq	*ke_runq;
+	u_char		ke_cpu;		/* CPU that we have affinity for. */
+	/* The following variables are only used for pctcpu calculation */
+	int		ke_ltick;	/* Last tick that we were running on */
+	int		ke_ftick;	/* First tick that we were running on */
+	int		ke_ticks;	/* Tick count */
+
+};
+
+
+#define td_kse td_sched
+#define	td_slptime		td_kse->ke_slptime
+#define ke_proc			ke_thread->td_proc
+#define ke_ksegrp		ke_thread->td_ksegrp
+
+/* flags kept in ke_flags */
+#define	KEF_SCHED0	0x00001	/* For scheduler-specific use. */
+#define	KEF_SCHED1	0x00002	/* For scheduler-specific use. */
+#define	KEF_SCHED2	0x00004	/* For scheduler-specific use. */
+#define	KEF_SCHED3	0x00008	/* For scheduler-specific use. */
+#define	KEF_DIDRUN	0x02000	/* Thread actually ran. */
+#define	KEF_EXIT	0x04000	/* Thread is being killed. */
+
+/*
  * These datastructures are allocated within their parent datastructure but
  * are scheduler specific.
  */
 
-struct ke_sched {
-	int		ske_slice;
-	struct runq	*ske_runq;
-	/* The following variables are only used for pctcpu calculation */
-	int		ske_ltick;	/* Last tick that we were running on */
-	int		ske_ftick;	/* First tick that we were running on */
-	int		ske_ticks;	/* Tick count */
-	/* CPU that we have affinity for. */
-	u_char		ske_cpu;
-};
-#define	ke_slice	ke_sched->ske_slice
-#define	ke_runq		ke_sched->ske_runq
-#define	ke_ltick	ke_sched->ske_ltick
-#define	ke_ftick	ke_sched->ske_ftick
-#define	ke_ticks	ke_sched->ske_ticks
-#define	ke_cpu		ke_sched->ske_cpu
 #define	ke_assign	ke_procq.tqe_next
 
-#define	KEF_ASSIGNED	KEF_SCHED0	/* KSE is being migrated. */
-#define	KEF_BOUND	KEF_SCHED1	/* KSE can not migrate. */
-#define	KEF_XFERABLE	KEF_SCHED2	/* KSE was added as transferable. */
-#define	KEF_HOLD	KEF_SCHED3	/* KSE is temporarily bound. */
+#define	KEF_ASSIGNED	KEF_SCHED0	/* Thread is being migrated. */
+#define	KEF_BOUND	KEF_SCHED1	/* Thread can not migrate. */
+#define	KEF_XFERABLE	KEF_SCHED2	/* Thread was added as transferable. */
+#define	KEF_HOLD	KEF_SCHED3	/* Thread is temporarily bound. */
 
 struct kg_sched {
+	struct thread	*skg_last_assigned; /* (j) Last thread assigned to */
+					   /* the system scheduler */
 	int	skg_slptime;		/* Number of ticks we vol. slept */
 	int	skg_runtime;		/* Number of ticks we were running */
+	int	skg_avail_opennings;	/* (j) Num unfilled slots in group.*/
+	int	skg_concurrency;	/* (j) Num threads requested in group.*/
+	int	skg_runq_threads;	/* (j) Num KSEs on runq. */
 };
-#define	kg_slptime	kg_sched->skg_slptime
-#define	kg_runtime	kg_sched->skg_runtime
-
-struct td_sched {
-	int	std_slptime;
-};
-#define	td_slptime	td_sched->std_slptime
+#define kg_last_assigned	kg_sched->skg_last_assigned
+#define kg_avail_opennings	kg_sched->skg_avail_opennings
+#define kg_concurrency		kg_sched->skg_concurrency
+#define kg_runq_threads		kg_sched->skg_runq_threads
+#define kg_runtime		kg_sched->skg_runtime
+#define kg_slptime		kg_sched->skg_slptime
 
-struct td_sched td_sched;
-struct ke_sched ke_sched;
-struct kg_sched kg_sched;
 
-struct ke_sched *kse0_sched = &ke_sched;
-struct kg_sched *ksegrp0_sched = &kg_sched;
-struct p_sched *proc0_sched = NULL;
-struct td_sched *thread0_sched = &td_sched;
+static struct kse kse0;
+static struct kg_sched kg_sched0;
 
 /*
  * The priority is primarily determined by the interactivity score.  Thus, we
@@ -191,7 +222,7 @@ struct td_sched *thread0_sched = &td_sched;
     (SCHED_SLICE_MAX - SCHED_SLICE_SCALE((nice), SCHED_SLICE_NTHRESH))
 
 /*
- * This macro determines whether or not the kse belongs on the current or
+ * This macro determines whether or not the thread belongs on the current or
  * next run queue.
  */
 #define	SCHED_INTERACTIVE(kg)						\
@@ -274,6 +305,8 @@ static struct kseq	kseq_cpu;
 #define	KSEQ_CPU(x)	(&kseq_cpu)
 #endif
 
+static void	slot_fill(struct ksegrp *kg);
+static struct kse *sched_choose(void);		/* XXX Should be thread * */
 static void sched_add_internal(struct thread *td, int preemptive);
 static void sched_slice(struct kse *ke);
 static void sched_priority(struct ksegrp *kg);
@@ -1106,6 +1139,28 @@ sched_interact_score(struct ksegrp *kg)
 }
 
 /*
+ * Very early in the boot some setup of scheduler-specific
+ * parts of proc0 and of soem scheduler resources needs to be done.
+ * Called from:
+ *  proc0_init()
+ */
+void
+schedinit(void)
+{
+	/*
+	 * Set up the scheduler specific parts of proc0.
+	 */
+	ksegrp0.kg_sched = &kg_sched0;
+	proc0.p_sched = NULL; /* XXX */
+	thread0.td_kse = &kse0;
+	kse0.ke_thread = &thread0;
+	kse0.ke_oncpu = NOCPU; /* wrong.. can we use PCPU(cpuid) yet? */
+	kse0.ke_state = KES_THREAD;
+	kg_sched0.skg_concurrency = 1;
+	kg_sched0.skg_avail_opennings = 0; /* we are already running */
+}
+
+/*
  * This is only somewhat accurate since given many processes of the same
  * priority they will switch when their slices run out, which will be
  * at most SCHED_SLICE_MAX.
@@ -1178,37 +1233,50 @@ sched_switch(struct thread *td, struct thread *newtd)
 
 	ke = td->td_kse;
 
-	td->td_last_kse = ke;
 	td->td_lastcpu = td->td_oncpu;
 	td->td_oncpu = NOCPU;
 	td->td_flags &= ~TDF_NEEDRESCHED;
 	td->td_pflags &= ~TDP_OWEPREEMPT;
 
 	/*
+	 * If we bring in a thread, 
+	 * then account for it as if it had been added to the run queue and then chosen.
+	 */
+	if (newtd) {
+		newtd->td_ksegrp->kg_avail_opennings--;
+		newtd->td_kse->ke_flags |= KEF_DIDRUN;
+        	TD_SET_RUNNING(newtd);
+	}
+	/*
 	 * If the KSE has been assigned it may be in the process of switching
 	 * to the new cpu.  This is the case in sched_bind().
 	 */
 	if ((ke->ke_flags & KEF_ASSIGNED) == 0) {
 		if (td == PCPU_GET(idlethread)) {
 			TD_SET_CAN_RUN(td);
-		} else if (TD_IS_RUNNING(td)) {
-			kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke);
-			/*
-			 * Don't allow the kse to migrate from a preemption.
-			 */
-			ke->ke_flags |= KEF_HOLD;
-			setrunqueue(td, SRQ_OURSELF|SRQ_YIELDING);
 		} else {
-			if (ke->ke_runq) {
+			/* We are ending our run so make our slot available again */
+			td->td_ksegrp->kg_avail_opennings++;
+			if (TD_IS_RUNNING(td)) {
 				kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke);
-			} else if ((td->td_flags & TDF_IDLETD) == 0)
-				kdb_backtrace();
-			/*
-			 * We will not be on the run queue. So we must be
-			 * sleeping or similar.
-			 */
-			if (td->td_proc->p_flag & P_SA)
-				kse_reassign(ke);
+				/*
+				 * Don't allow the thread to migrate
+				 * from a preemption.
+				 */
+				ke->ke_flags |= KEF_HOLD;
+				setrunqueue(td, SRQ_OURSELF|SRQ_YIELDING);
+			} else {
+				if (ke->ke_runq) {
+					kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke);
+				} else if ((td->td_flags & TDF_IDLETD) == 0)
+					kdb_backtrace();
+				/*
+				 * We will not be on the run queue.
+				 * So we must be sleeping or similar.
+				 */
+				if (td->td_proc->p_flag & P_HADTHREADS)
+					slot_fill(td->td_ksegrp);
+			}
 		}
 	}
 	if (newtd != NULL)
@@ -1237,7 +1305,8 @@ sched_nice(struct proc *p, int nice)
 	 */
 	FOREACH_KSEGRP_IN_PROC(p, kg) {
 		if (kg->kg_pri_class == PRI_TIMESHARE) {
-			FOREACH_KSE_IN_GROUP(kg, ke) {
+			FOREACH_THREAD_IN_GROUP(kg, td) {
+				ke = td->td_kse;
 				if (ke->ke_runq == NULL)
 					continue;
 				kseq = KSEQ_CPU(ke->ke_cpu);
@@ -1262,8 +1331,8 @@ sched_sleep(struct thread *td)
 	td->td_slptime = ticks;
 	td->td_base_pri = td->td_priority;
 
-	CTR2(KTR_ULE, "sleep kse %p (tick: %d)",
-	    td->td_kse, td->td_slptime);
+	CTR2(KTR_ULE, "sleep thread %p (tick: %d)",
+	    td, td->td_slptime);
 }
 
 void
@@ -1289,10 +1358,8 @@ sched_wakeup(struct thread *td)
 			sched_interact_update(kg);
 		}
 		sched_priority(kg);
-		if (td->td_kse)
-			sched_slice(td->td_kse);
-		CTR2(KTR_ULE, "wakeup kse %p (%d ticks)",
-		    td->td_kse, hzticks);
+		sched_slice(td->td_kse);
+		CTR2(KTR_ULE, "wakeup thread %p (%d ticks)", td, hzticks);
 		td->td_slptime = 0;
 	}
 	setrunqueue(td, SRQ_BORING);
@@ -1303,37 +1370,20 @@ sched_wakeup(struct thread *td)
  * priority.
  */
 void
-sched_fork(struct thread *td, struct proc *p1)
+sched_fork(struct thread *td, struct thread *childtd)
 {
 
 	mtx_assert(&sched_lock, MA_OWNED);
 
-	p1->p_nice = td->td_proc->p_nice;
-	sched_fork_ksegrp(td, FIRST_KSEGRP_IN_PROC(p1));
-	sched_fork_kse(td, FIRST_KSE_IN_PROC(p1));
-	sched_fork_thread(td, FIRST_THREAD_IN_PROC(p1));
-}
-
-void
-sched_fork_kse(struct thread *td, struct kse *child)
-{
-	struct kse *ke = td->td_kse;
-
-	child->ke_slice = 1;	/* Attempt to quickly learn interactivity. */
-	child->ke_cpu = ke->ke_cpu;
-	child->ke_runq = NULL;
-
-	/* Grab our parents cpu estimation information. */
-	child->ke_ticks = ke->ke_ticks;
-	child->ke_ltick = ke->ke_ltick;
-	child->ke_ftick = ke->ke_ftick;
+	sched_fork_ksegrp(td, childtd->td_ksegrp);
+	sched_fork_thread(td, childtd);
 }
 
 void
 sched_fork_ksegrp(struct thread *td, struct ksegrp *child)
 {
 	struct ksegrp *kg = td->td_ksegrp;
-	PROC_LOCK_ASSERT(child->kg_proc, MA_OWNED);
+	mtx_assert(&sched_lock, MA_OWNED);
 
 	child->kg_slptime = kg->kg_slptime;
 	child->kg_runtime = kg->kg_runtime;
@@ -1350,6 +1400,20 @@ sched_fork_ksegrp(struct thread *td, struct ksegrp *child)
 void
 sched_fork_thread(struct thread *td, struct thread *child)
 {
+	struct kse *ke;
+	struct kse *ke2;
+
+	sched_newthread(child);
+	ke = td->td_kse;
+	ke2 = child->td_kse;
+	ke2->ke_slice = 1;	/* Attempt to quickly learn interactivity. */
+	ke2->ke_cpu = ke->ke_cpu;
+	ke2->ke_runq = NULL;
+
+	/* Grab our parents cpu estimation information. */
+	ke2->ke_ticks = ke->ke_ticks;
+	ke2->ke_ltick = ke->ke_ltick;
+	ke2->ke_ftick = ke->ke_ftick;
 }
 
 void
@@ -1357,6 +1421,7 @@ sched_class(struct ksegrp *kg, int class)
 {
 	struct kseq *kseq;
 	struct kse *ke;
+	struct thread *td;
 	int nclass;
 	int oclass;
 
@@ -1366,7 +1431,8 @@ sched_class(struct ksegrp *kg, int class)
 
 	nclass = PRI_BASE(class);
 	oclass = PRI_BASE(kg->kg_pri_class);
-	FOREACH_KSE_IN_GROUP(kg, ke) {
+	FOREACH_THREAD_IN_GROUP(kg, td) {
+		ke = td->td_kse;
 		if (ke->ke_state != KES_ONRUNQ &&
 		    ke->ke_state != KES_THREAD)
 			continue;
@@ -1404,19 +1470,15 @@ sched_class(struct ksegrp *kg, int class)
 
 /*
  * Return some of the child's priority and interactivity to the parent.
+ * Avoid using sched_exit_thread to avoid having to decide which
+ * thread in the parent gets the honour since it isn't used.
  */
 void
-sched_exit(struct proc *p, struct thread *td)
+sched_exit(struct proc *p, struct thread *childtd)
 {
 	mtx_assert(&sched_lock, MA_OWNED);
-	sched_exit_kse(FIRST_KSE_IN_PROC(p), td);
-	sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td);
-}
-
-void
-sched_exit_kse(struct kse *ke, struct thread *td)
-{
-	kseq_load_rem(KSEQ_CPU(td->td_kse->ke_cpu), td->td_kse);
+	sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), childtd);
+	kseq_load_rem(KSEQ_CPU(childtd->td_kse->ke_cpu), childtd->td_kse);
 }
 
 void
@@ -1428,8 +1490,9 @@ sched_exit_ksegrp(struct ksegrp *kg, struct thread *td)
 }
 
 void
-sched_exit_thread(struct thread *td, struct thread *child)
+sched_exit_thread(struct thread *td, struct thread *childtd)
 {
+	kseq_load_rem(KSEQ_CPU(childtd->td_kse->ke_cpu), childtd->td_kse);
 }
 
 void
@@ -1483,8 +1546,8 @@ sched_clock(struct thread *td)
 	if (td->td_flags & TDF_IDLETD)
 		return;
 
-	CTR4(KTR_ULE, "Tick kse %p (slice: %d, slptime: %d, runtime: %d)",
-	    ke, ke->ke_slice, kg->kg_slptime >> 10, kg->kg_runtime >> 10);
+	CTR4(KTR_ULE, "Tick thread %p (slice: %d, slptime: %d, runtime: %d)",
+	    td, ke->ke_slice, kg->kg_slptime >> 10, kg->kg_runtime >> 10);
 	/*
 	 * We only do slicing code for TIMESHARE ksegrps.
 	 */
@@ -1581,8 +1644,8 @@ restart:
 		ke->ke_state = KES_THREAD;
 
 		if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) {
-			CTR4(KTR_ULE, "Run kse %p from %p (slice: %d, pri: %d)",
-			    ke, ke->ke_runq, ke->ke_slice,
+			CTR4(KTR_ULE, "Run thread %p from %p (slice: %d, pri: %d)",
+			    ke->ke_thread, ke->ke_runq, ke->ke_slice,
 			    ke->ke_thread->td_priority);
 		}
 		return (ke);
@@ -1627,10 +1690,6 @@ sched_add_internal(struct thread *td, int preemptive)
 	if (ke->ke_flags & KEF_ASSIGNED)
 		return;
 	kseq = KSEQ_SELF();
-	KASSERT((ke->ke_thread != NULL),
-	    ("sched_add: No thread on KSE"));
-	KASSERT((ke->ke_thread->td_kse != NULL),
-	    ("sched_add: No KSE on thread"));
 	KASSERT(ke->ke_state != KES_ONRUNQ,
 	    ("sched_add: kse %p (%s) already in run queue", ke,
 	    ke->ke_proc->p_comm));
@@ -1715,7 +1774,7 @@ sched_add_internal(struct thread *td, int preemptive)
 		curthread->td_flags |= TDF_NEEDRESCHED;
 	if (preemptive && maybe_preempt(td))
 		return;
-	ke->ke_ksegrp->kg_runq_kses++;
+	ke->ke_ksegrp->kg_runq_threads++;
 	ke->ke_state = KES_ONRUNQ;
 
 	kseq_runq_add(kseq, ke);
@@ -1742,7 +1801,7 @@ sched_rem(struct thread *td)
 	    ("sched_rem: KSE not on run queue"));
 
 	ke->ke_state = KES_THREAD;
-	ke->ke_ksegrp->kg_runq_kses--;
+	ke->ke_ksegrp->kg_runq_threads--;
 	kseq = KSEQ_CPU(ke->ke_cpu);
 	kseq_runq_rem(kseq, ke);
 	kseq_load_rem(kseq, ke);
@@ -1795,7 +1854,7 @@ sched_bind(struct thread *td, int cpu)
 		return;
 	/* sched_rem without the runq_remove */
 	ke->ke_state = KES_THREAD;
-	ke->ke_ksegrp->kg_runq_kses--;
+	ke->ke_ksegrp->kg_runq_threads--;
 	kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke);
 	kseq_notify(ke, cpu);
 	/* When we return from mi_switch we'll be on the correct cpu. */
@@ -1827,12 +1886,6 @@ sched_load(void)
 }
 
 int
-sched_sizeof_kse(void)
-{
-	return (sizeof(struct kse) + sizeof(struct ke_sched));
-}
-
-int
 sched_sizeof_ksegrp(void)
 {
 	return (sizeof(struct ksegrp) + sizeof(struct kg_sched));
@@ -1849,3 +1902,5 @@ sched_sizeof_thread(void)
 {
 	return (sizeof(struct thread) + sizeof(struct td_sched));
 }
+#define KERN_SWITCH_INCLUDE 1
+#include "kern/kern_switch.c"