Fix a couple of issues with the stack limit for 32-bit processes on 64-bit

kernels exposed by the recent fixes to resource limits for 32-bit processes
on 64-bit kernels:
- Let ABIs expose their maximum stack size via a new pointer in sysentvec
  and use that in preference to maxssiz during exec() rather than always
  using maxssiz for all processses.
- Apply the ABI's limit fixup to the previous stack size when adjusting
  RLIMIT_STACK to determine if the existing mapping for the stack needs to
  be grown or shrunk (as well as how much it should be grown or shrunk).

Approved by:	re (kensmith)

5 files changed, 33 insertions, 21 deletions

diff --git a/sys/amd64/linux32/linux32_sysvec.c b/sys/amd64/linux32/linux32_sysvec.c
index 89fa72a..ba4adc3 100644
--- a/sys/amd64/linux32/linux32_sysvec.c
+++ b/sys/amd64/linux32/linux32_sysvec.c
@@ -1023,7 +1023,8 @@ struct sysentvec elf_linux_sysvec = {
 	VM_PROT_ALL,
 	linux_copyout_strings,
 	exec_linux_setregs,
-	linux32_fixlimit
+	linux32_fixlimit,
+	&linux32_maxssiz,
 };
 
 static Elf32_Brandinfo linux_brand = {
diff --git a/sys/compat/ia32/ia32_sysvec.c b/sys/compat/ia32/ia32_sysvec.c
index f618934..6277926 100644
--- a/sys/compat/ia32/ia32_sysvec.c
+++ b/sys/compat/ia32/ia32_sysvec.c
@@ -100,6 +100,13 @@ extern struct sysent freebsd32_sysent[];
 
 SYSCTL_NODE(_compat, OID_AUTO, ia32, CTLFLAG_RW, 0, "ia32 mode");
 
+static u_long	ia32_maxdsiz = IA32_MAXDSIZ;
+SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxdsiz, CTLFLAG_RW, &ia32_maxdsiz, 0, "");
+static u_long	ia32_maxssiz = IA32_MAXSSIZ;
+SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxssiz, CTLFLAG_RW, &ia32_maxssiz, 0, "");
+static u_long	ia32_maxvmem = IA32_MAXVMEM;
+SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxvmem, CTLFLAG_RW, &ia32_maxvmem, 0, "");
+
 struct sysentvec ia32_freebsd_sysvec = {
 	FREEBSD32_SYS_MAXSYSCALL,
 	freebsd32_sysent,
@@ -126,7 +133,8 @@ struct sysentvec ia32_freebsd_sysvec = {
 	VM_PROT_ALL,
 	ia32_copyout_strings,
 	ia32_setregs,
-	ia32_fixlimit
+	ia32_fixlimit,
+	&ia32_maxssiz
 };
 
 
@@ -273,13 +281,6 @@ ia32_copyout_strings(struct image_params *imgp)
 	return ((register_t *)stack_base);
 }
 
-static u_long	ia32_maxdsiz = IA32_MAXDSIZ;
-SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxdsiz, CTLFLAG_RW, &ia32_maxdsiz, 0, "");
-static u_long	ia32_maxssiz = IA32_MAXSSIZ;
-SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxssiz, CTLFLAG_RW, &ia32_maxssiz, 0, "");
-static u_long	ia32_maxvmem = IA32_MAXVMEM;
-SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxvmem, CTLFLAG_RW, &ia32_maxvmem, 0, "");
-
 static void
 ia32_fixlimit(struct rlimit *rl, int which)
 {
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 42de9ed..b0c107c 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -894,6 +894,7 @@ exec_new_vmspace(imgp, sv)
 	struct vmspace *vmspace = p->p_vmspace;
 	vm_offset_t stack_addr;
 	vm_map_t map;
+	u_long ssiz;
 
 	imgp->vmspace_destroyed = 1;
 	imgp->sysent = sv;
@@ -919,8 +920,12 @@ exec_new_vmspace(imgp, sv)
 	}
 
 	/* Allocate a new stack */
-	stack_addr = sv->sv_usrstack - maxssiz;
-	error = vm_map_stack(map, stack_addr, (vm_size_t)maxssiz,
+	if (sv->sv_maxssiz != NULL)
+		ssiz = *sv->sv_maxssiz;
+	else
+		ssiz = maxssiz;
+	stack_addr = sv->sv_usrstack - ssiz;
+	error = vm_map_stack(map, stack_addr, (vm_size_t)ssiz,
 	    sv->sv_stackprot, VM_PROT_ALL, MAP_STACK_GROWS_DOWN);
 	if (error)
 		return (error);
@@ -928,7 +933,7 @@ exec_new_vmspace(imgp, sv)
 #ifdef __ia64__
 	/* Allocate a new register stack */
 	stack_addr = IA64_BACKINGSTORE;
-	error = vm_map_stack(map, stack_addr, (vm_size_t)maxssiz,
+	error = vm_map_stack(map, stack_addr, (vm_size_t)ssiz,
 	    sv->sv_stackprot, VM_PROT_ALL, MAP_STACK_GROWS_UP);
 	if (error)
 		return (error);
@@ -939,7 +944,7 @@ exec_new_vmspace(imgp, sv)
 	 * process stack so we can check the stack rlimit.
 	 */
 	vmspace->vm_ssize = sgrowsiz >> PAGE_SHIFT;
-	vmspace->vm_maxsaddr = (char *)sv->sv_usrstack - maxssiz;
+	vmspace->vm_maxsaddr = (char *)sv->sv_usrstack - ssiz;
 
 	return (0);
 }
diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index 7e161cb..8627a14 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -657,7 +657,7 @@ kern_setrlimit(td, which, limp)
 	struct plimit *newlim, *oldlim;
 	struct proc *p;
 	register struct rlimit *alimp;
-	rlim_t oldssiz;
+	struct rlimit oldssiz;
 	int error;
 
 	if (which >= RLIM_NLIMITS)
@@ -671,7 +671,7 @@ kern_setrlimit(td, which, limp)
 	if (limp->rlim_max < 0)
 		limp->rlim_max = RLIM_INFINITY;
 
-	oldssiz = 0;
+	oldssiz.rlim_cur = 0;
 	p = td->td_proc;
 	newlim = lim_alloc();
 	PROC_LOCK(p);
@@ -711,7 +711,10 @@ kern_setrlimit(td, which, limp)
 			limp->rlim_cur = maxssiz;
 		if (limp->rlim_max > maxssiz)
 			limp->rlim_max = maxssiz;
-		oldssiz = alimp->rlim_cur;
+		oldssiz = *alimp;
+		if (td->td_proc->p_sysent->sv_fixlimit != NULL)
+			td->td_proc->p_sysent->sv_fixlimit(&oldssiz,
+			    RLIMIT_STACK);
 		break;
 
 	case RLIMIT_NOFILE:
@@ -745,20 +748,21 @@ kern_setrlimit(td, which, limp)
 		 * "rlim_cur" bytes accessible.  If stack limit is going
 		 * up make more accessible, if going down make inaccessible.
 		 */
-		if (limp->rlim_cur != oldssiz) {
+		if (limp->rlim_cur != oldssiz.rlim_cur) {
 			vm_offset_t addr;
 			vm_size_t size;
 			vm_prot_t prot;
 
-			if (limp->rlim_cur > oldssiz) {
+			if (limp->rlim_cur > oldssiz.rlim_cur) {
 				prot = p->p_sysent->sv_stackprot;
-				size = limp->rlim_cur - oldssiz;
+				size = limp->rlim_cur - oldssiz.rlim_cur;
 				addr = p->p_sysent->sv_usrstack -
 				    limp->rlim_cur;
 			} else {
 				prot = VM_PROT_NONE;
-				size = oldssiz - limp->rlim_cur;
-				addr = p->p_sysent->sv_usrstack - oldssiz;
+				size = oldssiz.rlim_cur - limp->rlim_cur;
+				addr = p->p_sysent->sv_usrstack -
+				    oldssiz.rlim_cur;
 			}
 			addr = trunc_page(addr);
 			size = round_page(size);
diff --git a/sys/sys/sysent.h b/sys/sys/sysent.h
index ba8fc4b..e0ea4b0 100644
--- a/sys/sys/sysent.h
+++ b/sys/sys/sysent.h
@@ -99,6 +99,7 @@ struct sysentvec {
 	register_t	*(*sv_copyout_strings)(struct image_params *);
 	void		(*sv_setregs)(struct thread *, u_long, u_long, u_long);
 	void		(*sv_fixlimit)(struct rlimit *, int);
+	u_long		*sv_maxssiz;
 };
 
 #ifdef _KERNEL