Rework the support for ABIs to override resource limits (used by 32-bit

processes under 64-bit kernels).  Previously, each 32-bit process overwrote
its resource limits at exec() time.  The problem with this approach is that
the new limits affect all child processes of the 32-bit process, including
if the child process forks and execs a 64-bit process.  To fix this, don't
ovewrite the resource limits during exec().  Instead, sv_fixlimits() is
now replaced with a different function sv_fixlimit() which asks the ABI to
sanitize a single resource limit.  We then use this when querying and
setting resource limits.  Thus, if a 32-bit process sets a limit, then
that new limit will be inherited by future children.  However, if the
32-bit process doesn't change a limit, then a future 64-bit child will
see the "full" 64-bit limit rather than the 32-bit limit.

MFC is tentative since it will break the ABI of old linux.ko modules (no
other modules are affected).

MFC after:	1 week

1 files changed, 29 insertions, 31 deletions

diff --git a/sys/compat/ia32/ia32_sysvec.c b/sys/compat/ia32/ia32_sysvec.c
index b31400c..436fda3 100644
--- a/sys/compat/ia32/ia32_sysvec.c
+++ b/sys/compat/ia32/ia32_sysvec.c
@@ -94,7 +94,7 @@ CTASSERT(sizeof(struct ia32_sigframe4) == 408);
 #endif
 
 static register_t *ia32_copyout_strings(struct image_params *imgp);
-static void ia32_fixlimits(struct proc *p);
+static void ia32_fixlimit(struct rlimit *rl, int which);
 
 extern struct sysent freebsd32_sysent[];
 
@@ -126,7 +126,7 @@ struct sysentvec ia32_freebsd_sysvec = {
 	VM_PROT_ALL,
 	ia32_copyout_strings,
 	ia32_setregs,
-	ia32_fixlimits
+	ia32_fixlimit
 };
 
 
@@ -281,35 +281,33 @@ static u_long	ia32_maxvmem = IA32_MAXVMEM;
 SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxvmem, CTLFLAG_RW, &ia32_maxvmem, 0, "");
 
 static void
-ia32_fixlimits(struct proc *p)
+ia32_fixlimit(struct rlimit *rl, int which)
 {
-	struct plimit *oldlim, *newlim;
-
-	if (ia32_maxdsiz == 0 && ia32_maxssiz == 0 && ia32_maxvmem == 0)
-		return;
-	newlim = lim_alloc();
-	PROC_LOCK(p);
-	oldlim = p->p_limit;
-	lim_copy(newlim, oldlim);
-	if (ia32_maxdsiz != 0) {
-		if (newlim->pl_rlimit[RLIMIT_DATA].rlim_cur > ia32_maxdsiz)
-		    newlim->pl_rlimit[RLIMIT_DATA].rlim_cur = ia32_maxdsiz;
-		if (newlim->pl_rlimit[RLIMIT_DATA].rlim_max > ia32_maxdsiz)
-		    newlim->pl_rlimit[RLIMIT_DATA].rlim_max = ia32_maxdsiz;
-	}
-	if (ia32_maxssiz != 0) {
-		if (newlim->pl_rlimit[RLIMIT_STACK].rlim_cur > ia32_maxssiz)
-		    newlim->pl_rlimit[RLIMIT_STACK].rlim_cur = ia32_maxssiz;
-		if (newlim->pl_rlimit[RLIMIT_STACK].rlim_max > ia32_maxssiz)
-		    newlim->pl_rlimit[RLIMIT_STACK].rlim_max = ia32_maxssiz;
-	}
-	if (ia32_maxvmem != 0) {
-		if (newlim->pl_rlimit[RLIMIT_VMEM].rlim_cur > ia32_maxvmem)
-		    newlim->pl_rlimit[RLIMIT_VMEM].rlim_cur = ia32_maxvmem;
-		if (newlim->pl_rlimit[RLIMIT_VMEM].rlim_max > ia32_maxvmem)
-		    newlim->pl_rlimit[RLIMIT_VMEM].rlim_max = ia32_maxvmem;
+
+	switch (which) {
+	case RLIMIT_DATA:
+		if (ia32_maxdsiz != 0) {
+			if (rl->rlim_cur > ia32_maxdsiz)
+				rl->rlim_cur = ia32_maxdsiz;
+			if (rl->rlim_max > ia32_maxdsiz)
+				rl->rlim_max = ia32_maxdsiz;
+		}
+		break;
+	case RLIMIT_STACK:
+		if (ia32_maxssiz != 0) {
+			if (rl->rlim_cur > ia32_maxssiz)
+				rl->rlim_cur = ia32_maxssiz;
+			if (rl->rlim_max > ia32_maxssiz)
+				rl->rlim_max = ia32_maxssiz;
+		}
+		break;
+	case RLIMIT_VMEM:
+		if (ia32_maxvmem != 0) {
+			if (rl->rlim_cur > ia32_maxvmem)
+				rl->rlim_cur = ia32_maxvmem;
+			if (rl->rlim_max > ia32_maxvmem)
+				rl->rlim_max = ia32_maxvmem;
+		}
+		break;
 	}
-	p->p_limit = newlim;
-	PROC_UNLOCK(p);
-	lim_free(oldlim);
 }