Add sysentvec->sv_fixlimits() hook so that we can catch cases on 64 bit

systems where the data/stack/etc limits are too big for a 32 bit process.

Move the 5 or so identical instances of ELF_RTLD_ADDR() into imgact_elf.c.

Supply an ia32_fixlimits function.  Export the clip/default values to
sysctl under the compat.ia32 heirarchy.

Have mmap(0, ...) respect the current p->p_limits[RLIMIT_DATA].rlim_max
value rather than the sysctl tweakable variable.  This allows mmap to
place mappings at sensible locations when limits have been reduced.

Have the imgact_elf.c ld-elf.so.1 placement algorithm use the same
method as mmap(0, ...) now does.

Note that we cannot remove all references to the sysctl tweakable
maxdsiz etc variables because /etc/login.conf specifies a datasize
of 'unlimited'.  And that causes exec etc to fail since it can no
longer find space to mmap things.

2 files changed, 68 insertions, 1 deletions

diff --git a/sys/compat/ia32/ia32_sysvec.c b/sys/compat/ia32/ia32_sysvec.c
index 5efb4d8..bbaa56a 100644
--- a/sys/compat/ia32/ia32_sysvec.c
+++ b/sys/compat/ia32/ia32_sysvec.c
@@ -78,9 +78,12 @@ __FBSDID("$FreeBSD$");
 static register_t *ia32_copyout_strings(struct image_params *imgp);
 static void ia32_setregs(struct thread *td, u_long entry, u_long stack,
     u_long ps_strings);
+static void ia32_fixlimits(struct image_params *imgp);
 
 extern struct sysent freebsd32_sysent[];
 
+SYSCTL_NODE(_compat, OID_AUTO, ia32, CTLFLAG_RW, 0, "ia32 mode");
+
 struct sysentvec ia32_freebsd_sysvec = {
 	SYS_MAXSYSCALL,
 	freebsd32_sysent,
@@ -106,7 +109,8 @@ struct sysentvec ia32_freebsd_sysvec = {
 	FREEBSD32_PS_STRINGS,
 	VM_PROT_ALL,
 	ia32_copyout_strings,
-	ia32_setregs
+	ia32_setregs,
+	ia32_fixlimits
 };
 
 
@@ -283,3 +287,62 @@ ia32_setregs(td, entry, stack, ps_strings)
 	pcb->pcb_flags |= PCB_FULLCTX;
 	td->td_retval[1] = 0;
 }
+
+static u_long	ia32_maxdsiz = IA32_MAXDSIZ;
+SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxdsiz, CTLFLAG_RW, &ia32_maxdsiz, 0, "");
+static u_long	ia32_maxssiz = IA32_MAXSSIZ;
+SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxssiz, CTLFLAG_RW, &ia32_maxssiz, 0, "");
+static u_long	ia32_maxvmem = IA32_MAXVMEM;
+SYSCTL_ULONG(_compat_ia32, OID_AUTO, maxvmem, CTLFLAG_RW, &ia32_maxvmem, 0, "");
+
+static void
+ia32_fixlimits(struct image_params *imgp)
+{
+	struct proc *p = imgp->proc;
+
+	if (ia32_maxdsiz != 0) {
+		if (p->p_rlimit[RLIMIT_DATA].rlim_cur > ia32_maxdsiz ||
+		    p->p_rlimit[RLIMIT_DATA].rlim_max > ia32_maxdsiz) {
+			if (p->p_limit->p_refcnt > 1) {
+				p->p_limit->p_refcnt--;
+				p->p_limit = limcopy(p->p_limit);
+			}
+			if (p->p_rlimit[RLIMIT_DATA].rlim_cur > ia32_maxdsiz)
+				p->p_rlimit[RLIMIT_DATA].rlim_cur =
+				    ia32_maxdsiz;
+			if (p->p_rlimit[RLIMIT_DATA].rlim_max > ia32_maxdsiz)
+				p->p_rlimit[RLIMIT_DATA].rlim_max =
+				    ia32_maxdsiz;
+		}
+	}
+	if (ia32_maxssiz != 0) {
+		if (p->p_rlimit[RLIMIT_STACK].rlim_cur > ia32_maxssiz ||
+		    p->p_rlimit[RLIMIT_STACK].rlim_max > ia32_maxssiz) {
+			if (p->p_limit->p_refcnt > 1) {
+				p->p_limit->p_refcnt--;
+				p->p_limit = limcopy(p->p_limit);
+			}
+			if (p->p_rlimit[RLIMIT_STACK].rlim_cur > ia32_maxssiz)
+				p->p_rlimit[RLIMIT_STACK].rlim_cur =
+				    ia32_maxssiz;
+			if (p->p_rlimit[RLIMIT_STACK].rlim_max > ia32_maxssiz)
+				p->p_rlimit[RLIMIT_STACK].rlim_max =
+				    ia32_maxssiz;
+		}
+	}
+	if (ia32_maxvmem != 0) {
+		if (p->p_rlimit[RLIMIT_VMEM].rlim_cur > ia32_maxvmem ||
+		    p->p_rlimit[RLIMIT_VMEM].rlim_max > ia32_maxvmem) {
+			if (p->p_limit->p_refcnt > 1) {
+				p->p_limit->p_refcnt--;
+				p->p_limit = limcopy(p->p_limit);
+			}
+			if (p->p_rlimit[RLIMIT_VMEM].rlim_cur > ia32_maxvmem)
+				p->p_rlimit[RLIMIT_VMEM].rlim_cur =
+				    ia32_maxvmem;
+			if (p->p_rlimit[RLIMIT_VMEM].rlim_max > ia32_maxvmem)
+				p->p_rlimit[RLIMIT_VMEM].rlim_max =
+				    ia32_maxvmem;
+		}
+	}
+}
diff --git a/sys/compat/ia32/ia32_util.h b/sys/compat/ia32/ia32_util.h
index 06f78cf..f6de375 100644
--- a/sys/compat/ia32/ia32_util.h
+++ b/sys/compat/ia32/ia32_util.h
@@ -38,3 +38,7 @@
 #include <sys/cdefs.h>
 
 #define FREEBSD32_USRSTACK	((1ul << 32) - PAGE_SIZE)
+
+#define	IA32_MAXDSIZ	(512*1024*1024)		/* 512MB */
+#define	IA32_MAXSSIZ	(64*1024*1024)		/* 64MB */
+#define IA32_MAXVMEM	0			/* Unlimited */