Remove the altkstacks, instead instantiate threads with kernel stack

allocated with the right size from the start. For the thread that has
kernel stack cached, verify that requested stack size is equial to the
actual, and reallocate the stack if sizes differ [1].

This fixes the bug introduced by r173361 that was committed several days
after r173004 and consisted of kthread_add(9) ignoring the non-default
kernel stack size.

Also, r173361 removed the caching of the kernel stacks for a non-first
thread in the process. Introduce separate kernel stack cache that keeps
some limited amount of preallocated kernel stacks to lower the latency
of thread allocation. Add vm_lowmem handler to prune the cache on
low memory condition. This way, system with reasonable amount of the
threads get lower latency of thread creation, while still not exhausting
significant portion of KVA for unused kstacks.

Submitted by:	peter [1]
Discussed with:	jhb, julian, peter
Reviewed by:	jhb
Tested by:	pho
MFC after:	1 week

5 files changed, 30 insertions, 29 deletions

diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 4e2eaa9..b5a4082 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_kdtrace.h"
 #include "opt_ktrace.h"
+#include "opt_kstack_pages.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -276,25 +277,29 @@ norfproc_fail:
 
 	mem_charged = 0;
 	vm2 = NULL;
+	if (pages == 0)
+		pages = KSTACK_PAGES;
 	/* Allocate new proc. */
 	newproc = uma_zalloc(proc_zone, M_WAITOK);
-	if (TAILQ_EMPTY(&newproc->p_threads)) {
-		td2 = thread_alloc();
+	td2 = FIRST_THREAD_IN_PROC(newproc);
+	if (td2 == NULL) {
+		td2 = thread_alloc(pages);
 		if (td2 == NULL) {
 			error = ENOMEM;
 			goto fail1;
 		}
 		proc_linkup(newproc, td2);
-	} else
-		td2 = FIRST_THREAD_IN_PROC(newproc);
-
-	/* Allocate and switch to an alternate kstack if specified. */
-	if (pages != 0) {
-		if (!vm_thread_new_altkstack(td2, pages)) {
-			error = ENOMEM;
-			goto fail1;
+	} else {
+		if (td2->td_kstack == 0 || td2->td_kstack_pages != pages) {
+			if (td2->td_kstack != 0)
+				vm_thread_dispose(td);
+			if (!thread_alloc_stack(td2, pages)) {
+				error = ENOMEM;
+				goto fail1;
+			}
 		}
 	}
+
 	if ((flags & RFMEM) == 0) {
 		vm2 = vmspace_fork(p1->p_vmspace, &mem_charged);
 		if (vm2 == NULL) {
diff --git a/sys/kern/kern_kthread.c b/sys/kern/kern_kthread.c
index 1092832..3c5248e 100644
--- a/sys/kern/kern_kthread.c
+++ b/sys/kern/kern_kthread.c
@@ -256,7 +256,7 @@ kthread_add(void (*func)(void *), void *arg, struct proc *p,
 	}
 
 	/* Initialize our new td  */
-	newtd = thread_alloc();
+	newtd = thread_alloc(pages);
 	if (newtd == NULL)
 		return (ENOMEM);
 
@@ -282,9 +282,6 @@ kthread_add(void (*func)(void *), void *arg, struct proc *p,
 
 	newtd->td_pflags |= TDP_KTHREAD;
 	newtd->td_ucred = crhold(p->p_ucred);
-	/* Allocate and switch to an alternate kstack if specified. */
-	if (pages != 0)
-		vm_thread_new_altkstack(newtd, pages);
 
 	/* this code almost the same as create_thread() in kern_thr.c */
 	PROC_LOCK(p);
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index cdbc012..e012a3e 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -203,14 +203,6 @@ proc_dtor(void *mem, int size, void *arg)
 #endif
 		/* Free all OSD associated to this thread. */
 		osd_thread_exit(td);
-
-		/* Dispose of an alternate kstack, if it exists.
-		 * XXX What if there are more than one thread in the proc?
-		 *     The first thread in the proc is special and not
-		 *     freed, so you gotta do this here.
-		 */
-		if (((p->p_flag & P_KTHREAD) != 0) && (td->td_altkstack != 0))
-			vm_thread_dispose_altkstack(td);
 	}
 	EVENTHANDLER_INVOKE(process_dtor, p);
 	if (p->p_ksi != NULL)
@@ -767,8 +759,6 @@ fill_kinfo_proc_only(struct proc *p, struct kinfo_proc *kp)
 		FOREACH_THREAD_IN_PROC(p, td0) {
 			if (!TD_IS_SWAPPED(td0))
 				kp->ki_rssize += td0->td_kstack_pages;
-			if (td0->td_altkstack_obj != NULL)
-				kp->ki_rssize += td0->td_altkstack_pages;
 		}
 		kp->ki_swrss = vm->vm_swrss;
 		kp->ki_tsize = vm->vm_tsize;
diff --git a/sys/kern/kern_thr.c b/sys/kern/kern_thr.c
index c478c63..630069b 100644
--- a/sys/kern/kern_thr.c
+++ b/sys/kern/kern_thr.c
@@ -176,7 +176,7 @@ create_thread(struct thread *td, mcontext_t *ctx,
 	}
 
 	/* Initialize our td */
-	newtd = thread_alloc();
+	newtd = thread_alloc(0);
 	if (newtd == NULL)
 		return (ENOMEM);
 
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index d47bd8c..4f3b32c 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -283,7 +283,7 @@ thread_reap(void)
  * Allocate a thread.
  */
 struct thread *
-thread_alloc(void)
+thread_alloc(int pages)
 {
 	struct thread *td;
 
@@ -291,7 +291,7 @@ thread_alloc(void)
 
 	td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK);
 	KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack"));
-	if (!vm_thread_new(td, 0)) {
+	if (!vm_thread_new(td, pages)) {
 		uma_zfree(thread_zone, td);
 		return (NULL);
 	}
@@ -299,6 +299,17 @@ thread_alloc(void)
 	return (td);
 }
 
+int
+thread_alloc_stack(struct thread *td, int pages)
+{
+
+	KASSERT(td->td_kstack == 0,
+	    ("thread_alloc_stack called on a thread with kstack"));
+	if (!vm_thread_new(td, pages))
+		return (0);
+	cpu_thread_alloc(td);
+	return (1);
+}
 
 /*
  * Deallocate a thread.
@@ -312,8 +323,6 @@ thread_free(struct thread *td)
 		cpuset_rel(td->td_cpuset);
 	td->td_cpuset = NULL;
 	cpu_thread_free(td);
-	if (td->td_altkstack != 0)
-		vm_thread_dispose_altkstack(td);
 	if (td->td_kstack != 0)
 		vm_thread_dispose(td);
 	uma_zfree(thread_zone, td);