Part I of RLIMIT_VMEM implementation. Implement core functionality for

a new resource limit that covers a process's entire VM space, including
mmap()'d space.

(Part II will be additional code to check RLIMIT_VMEM during exec() but it
needs more fleshing out).

PR:		kern/18209
Submitted by:	Andrey Alekseyev <uitm@zenon.net>, Dmitry Kim <jason@nichego.net>
MFC after:	7 days

3 files changed, 15 insertions, 1 deletions

diff --git a/sys/sys/resource.h b/sys/sys/resource.h
index b723781..b29e594 100644
--- a/sys/sys/resource.h
+++ b/sys/sys/resource.h
@@ -91,8 +91,9 @@ struct rusage {
 #define	RLIMIT_NPROC	7		/* number of processes */
 #define	RLIMIT_NOFILE	8		/* number of open files */
 #define	RLIMIT_SBSIZE	9		/* maximum size of all socket buffers */
+#define RLIMIT_VMEM	10		/* virtual process size (inclusive of mmap) */
 
-#define	RLIM_NLIMITS	10		/* number of resource limits */
+#define	RLIM_NLIMITS	11		/* number of resource limits */
 
 #define	RLIM_INFINITY	((rlim_t)(((u_quad_t)1 << 63) - 1))
 
@@ -113,6 +114,7 @@ static char *rlimit_ident[] = {
 	"nproc",
 	"nofile",
 	"sbsize",
+	"vmem",
 };
 #endif
 
diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c
index 4911ef7..698179a 100644
--- a/sys/vm/vm_mmap.c
+++ b/sys/vm/vm_mmap.c
@@ -55,6 +55,8 @@
 #include <sys/sysproto.h>
 #include <sys/filedesc.h>
 #include <sys/proc.h>
+#include <sys/resource.h>
+#include <sys/resourcevar.h>
 #include <sys/vnode.h>
 #include <sys/fcntl.h>
 #include <sys/file.h>
@@ -1124,6 +1126,11 @@ vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
 
 	objsize = size = round_page(size);
 
+	if (td->td_proc->p_vmspace->vm_map.size + size >
+	    td->td_proc->p_rlimit[RLIMIT_VMEM].rlim_cur) {
+		return(ENOMEM);
+	}
+
 	/*
 	 * We currently can only deal with page aligned file offsets.
 	 * The check is here rather than in the syscall because the
diff --git a/sys/vm/vm_unix.c b/sys/vm/vm_unix.c
index bf932fd..1d02f39 100644
--- a/sys/vm/vm_unix.c
+++ b/sys/vm/vm_unix.c
@@ -107,6 +107,11 @@ obreak(td, uap)
 		goto done;
 	}
 	if (new > old) {
+		if (vm->vm_map.size + (new - old) >
+		    td->td_proc->p_rlimit[RLIMIT_VMEM].rlim_cur) {
+			error = ENOMEM;
+			goto done;
+		}
 		rv = vm_map_insert(&vm->vm_map, NULL, 0, old, new,
 		    VM_PROT_ALL, VM_PROT_ALL, 0);
 		if (rv != KERN_SUCCESS) {