Add accounting for most of the memory-related resources.

Sponsored by:	The FreeBSD Foundation
Reviewed by:	kib (earlier version)

11 files changed, 188 insertions, 13 deletions

diff --git a/sys/compat/linux/linux_misc.c b/sys/compat/linux/linux_misc.c
index 7b261d6..bbc805d 100644
--- a/sys/compat/linux/linux_misc.c
+++ b/sys/compat/linux/linux_misc.c
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/priv.h>
 #include <sys/proc.h>
 #include <sys/reboot.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/sched.h>
 #include <sys/signalvar.h>
@@ -357,7 +358,9 @@ linux_uselib(struct thread *td, struct linux_uselib_args *args)
 	 */
 	PROC_LOCK(td->td_proc);
 	if (a_out->a_text > maxtsiz ||
-	    a_out->a_data + bss_size > lim_cur(td->td_proc, RLIMIT_DATA)) {
+	    a_out->a_data + bss_size > lim_cur(td->td_proc, RLIMIT_DATA) ||
+	    racct_set(td->td_proc, RACCT_DATA, a_out->a_data +
+	    bss_size) != 0) {
 		PROC_UNLOCK(td->td_proc);
 		error = ENOMEM;
 		goto cleanup;
diff --git a/sys/compat/svr4/imgact_svr4.c b/sys/compat/svr4/imgact_svr4.c
index 5d80f10..cd8a136 100644
--- a/sys/compat/svr4/imgact_svr4.c
+++ b/sys/compat/svr4/imgact_svr4.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mman.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/vnode.h>
 
@@ -108,7 +109,8 @@ exec_svr4_imgact(imgp)
      */
     PROC_LOCK(imgp->proc);
     if (a_out->a_text > maxtsiz ||
-	a_out->a_data + bss_size > lim_cur(imgp->proc, RLIMIT_DATA)) {
+	a_out->a_data + bss_size > lim_cur(imgp->proc, RLIMIT_DATA) ||
+	racct_set(imgp->proc, RACCT_DATA, a_out->a_data + bss_size) != 0) {;
     	PROC_UNLOCK(imgp->proc);
 	return (ENOMEM);
     }
diff --git a/sys/i386/linux/imgact_linux.c b/sys/i386/linux/imgact_linux.c
index c4e4b66..3a66e87 100644
--- a/sys/i386/linux/imgact_linux.c
+++ b/sys/i386/linux/imgact_linux.c
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mman.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/vnode.h>
 
@@ -107,7 +108,8 @@ exec_linux_imgact(struct image_params *imgp)
      */
     PROC_LOCK(imgp->proc);
     if (a_out->a_text > maxtsiz ||
-	a_out->a_data + bss_size > lim_cur(imgp->proc, RLIMIT_DATA)) {
+	a_out->a_data + bss_size > lim_cur(imgp->proc, RLIMIT_DATA) ||
+	racct_set(imgp->proc, RACCT_DATA, a_out->a_data + bss_size) != 0) {
 	PROC_UNLOCK(imgp->proc);
 	return (ENOMEM);
     }
diff --git a/sys/kern/imgact_aout.c b/sys/kern/imgact_aout.c
index 2f168d8..2f889ca 100644
--- a/sys/kern/imgact_aout.c
+++ b/sys/kern/imgact_aout.c
@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/signalvar.h>
 #include <sys/syscall.h>
@@ -245,7 +246,8 @@ exec_aout_imgact(struct image_params *imgp)
 	    a_out->a_text > maxtsiz ||
 
 	    /* data + bss can't exceed rlimit */
-	    a_out->a_data + bss_size > lim_cur(imgp->proc, RLIMIT_DATA)) {
+	    a_out->a_data + bss_size > lim_cur(imgp->proc, RLIMIT_DATA) ||
+	    racct_set(imgp->proc, RACCT_DATA, a_out->a_data + bss_size) != 0) {
 			PROC_UNLOCK(imgp->proc);
 			return (ENOMEM);
 	}
diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c
index 80bb700..b41741a 100644
--- a/sys/kern/imgact_elf.c
+++ b/sys/kern/imgact_elf.c
@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/pioctl.h>
 #include <sys/proc.h>
 #include <sys/procfs.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/sf_buf.h>
 #include <sys/smp.h>
@@ -874,7 +875,9 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 	PROC_LOCK(imgp->proc);
 	if (data_size > lim_cur(imgp->proc, RLIMIT_DATA) ||
 	    text_size > maxtsiz ||
-	    total_size > lim_cur(imgp->proc, RLIMIT_VMEM)) {
+	    total_size > lim_cur(imgp->proc, RLIMIT_VMEM) ||
+	    racct_set(imgp->proc, RACCT_DATA, data_size) != 0 ||
+	    racct_set(imgp->proc, RACCT_VMEM, total_size) != 0) {
 		PROC_UNLOCK(imgp->proc);
 		return (ENOMEM);
 	}
@@ -1101,6 +1104,13 @@ __elfN(coredump)(struct thread *td, struct vnode *vp, off_t limit, int flags)
 	hdrsize = 0;
 	__elfN(puthdr)(td, (void *)NULL, &hdrsize, seginfo.count);
 
+	PROC_LOCK(td->td_proc);
+	error = racct_add(td->td_proc, RACCT_CORE, hdrsize + seginfo.size);
+	PROC_UNLOCK(td->td_proc);
+	if (error != 0) {
+		error = EFAULT;
+		goto done;
+	}
 	if (hdrsize + seginfo.size >= limit) {
 		error = EFAULT;
 		goto done;
diff --git a/sys/kern/imgact_gzip.c b/sys/kern/imgact_gzip.c
index 66f40f6..c83f36b 100644
--- a/sys/kern/imgact_gzip.c
+++ b/sys/kern/imgact_gzip.c
@@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mman.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/sysent.h>
 #include <sys/systm.h>
@@ -216,7 +217,9 @@ do_aout_hdr(struct imgact_gzip * gz)
 
 	/* data + bss can't exceed rlimit */
 	    gz->a_out.a_data + gz->bss_size >
-	    lim_cur(gz->ip->proc, RLIMIT_DATA)) {
+	    lim_cur(gz->ip->proc, RLIMIT_DATA) ||
+	    racct_set(gz->ip->proc, RACCT_DATA,
+	    gz->a_out.a_data + gz->bss_size) != 0) {
 		PROC_UNLOCK(gz->ip->proc);
 		gz->where = __LINE__;
 		return (ENOMEM);
diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index 39631a6..9a3af95 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -86,6 +86,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/namei.h>
 #include <sys/vnode.h>
 #include <sys/malloc.h>
+#include <sys/racct.h>
 #include <sys/resource.h>
 #include <sys/resourcevar.h>
 #include <sys/sysctl.h>
@@ -192,6 +193,12 @@ swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
 	if (incr & PAGE_MASK)
 		panic("swap_reserve: & PAGE_MASK");
 
+	PROC_LOCK(curproc);
+	error = racct_add(curproc, RACCT_SWAP, incr);
+	PROC_UNLOCK(curproc);
+	if (error != 0)
+		return (0);
+
 	res = 0;
 	mtx_lock(&sw_dev_mtx);
 	r = swap_reserved + incr;
@@ -230,6 +237,12 @@ swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
 		    curproc->p_pid, uip->ui_uid, incr);
 	}
 
+	if (!res) {
+		PROC_LOCK(curproc);
+		racct_sub(curproc, RACCT_SWAP, incr);
+		PROC_UNLOCK(curproc);
+	}
+
 	return (res);
 }
 
@@ -242,6 +255,10 @@ swap_reserve_force(vm_ooffset_t incr)
 	swap_reserved += incr;
 	mtx_unlock(&sw_dev_mtx);
 
+	PROC_LOCK(curproc);
+	racct_add_force(curproc, RACCT_SWAP, incr);
+	PROC_UNLOCK(curproc);
+
 	uip = curthread->td_ucred->cr_ruidinfo;
 	PROC_LOCK(curproc);
 	UIDINFO_VMSIZE_LOCK(uip);
@@ -282,6 +299,8 @@ swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
 		printf("negative vmsize for uid = %d\n", uip->ui_uid);
 	uip->ui_vmsize -= decr;
 	UIDINFO_VMSIZE_UNLOCK(uip);
+
+	racct_sub_cred(cred, RACCT_SWAP, decr);
 }
 
 static void swapdev_strategy(struct buf *, struct swdevt *sw);
diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c
index 69576d5..3ad8fe1 100644
--- a/sys/vm/vm_glue.c
+++ b/sys/vm/vm_glue.c
@@ -69,6 +69,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/sched.h>
 #include <sys/sf_buf.h>
@@ -182,6 +183,7 @@ int
 vslock(void *addr, size_t len)
 {
 	vm_offset_t end, last, start;
+	unsigned long nsize;
 	vm_size_t npages;
 	int error;
 
@@ -194,9 +196,13 @@ vslock(void *addr, size_t len)
 	if (npages > vm_page_max_wired)
 		return (ENOMEM);
 	PROC_LOCK(curproc);
-	if (ptoa(npages +
-	    pmap_wired_count(vm_map_pmap(&curproc->p_vmspace->vm_map))) >
-	    lim_cur(curproc, RLIMIT_MEMLOCK)) {
+	nsize = ptoa(npages +
+	    pmap_wired_count(vm_map_pmap(&curproc->p_vmspace->vm_map)));
+	if (nsize > lim_cur(curproc, RLIMIT_MEMLOCK)) {
+		PROC_UNLOCK(curproc);
+		return (ENOMEM);
+	}
+	if (racct_set(curproc, RACCT_MEMLOCK, nsize)) {
 		PROC_UNLOCK(curproc);
 		return (ENOMEM);
 	}
@@ -216,6 +222,12 @@ vslock(void *addr, size_t len)
 #endif
 	error = vm_map_wire(&curproc->p_vmspace->vm_map, start, end,
 	    VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES);
+	if (error != KERN_SUCCESS) {
+		PROC_LOCK(curproc);
+		racct_set(curproc, RACCT_MEMLOCK, 
+		    ptoa(pmap_wired_count(vm_map_pmap(&curproc->p_vmspace->vm_map))));
+		PROC_UNLOCK(curproc);
+	}
 	/*
 	 * Return EFAULT on error to match copy{in,out}() behaviour
 	 * rather than returning ENOMEM like mlock() would.
@@ -231,6 +243,11 @@ vsunlock(void *addr, size_t len)
 	(void)vm_map_unwire(&curproc->p_vmspace->vm_map,
 	    trunc_page((vm_offset_t)addr), round_page((vm_offset_t)addr + len),
 	    VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES);
+
+	PROC_LOCK(curproc);
+	racct_set(curproc, RACCT_MEMLOCK,
+	    ptoa(pmap_wired_count(vm_map_pmap(&curproc->p_vmspace->vm_map))));
+	PROC_UNLOCK(curproc);
 }
 
 /*
diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index 22fbda1..752354a 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/vmmeter.h>
 #include <sys/mman.h>
 #include <sys/vnode.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/file.h>
 #include <sys/sysctl.h>
@@ -313,6 +314,19 @@ vm_init2(void)
 	    vmspace_zinit, vmspace_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
 }
 
+static void
+vmspace_container_reset(struct proc *p)
+{
+
+	PROC_LOCK(p);
+	racct_set(p, RACCT_DATA, 0);
+	racct_set(p, RACCT_STACK, 0);
+	racct_set(p, RACCT_RSS, 0);
+	racct_set(p, RACCT_MEMLOCK, 0);
+	racct_set(p, RACCT_VMEM, 0);
+	PROC_UNLOCK(p);
+}
+
 static inline void
 vmspace_dofree(struct vmspace *vm)
 {
@@ -410,6 +424,7 @@ vmspace_exit(struct thread *td)
 		pmap_activate(td);
 		vmspace_dofree(vm);
 	}
+	vmspace_container_reset(p);
 }
 
 /* Acquire reference to vmspace owned by another process. */
@@ -3279,6 +3294,10 @@ vm_map_growstack(struct proc *p, vm_offset_t addr)
 	rlim_t stacklim, vmemlim;
 	int is_procstack, rv;
 	struct ucred *cred;
+#ifdef notyet
+	uint64_t limit;
+#endif
+	int error;
 
 Retry:
 	PROC_LOCK(p);
@@ -3377,6 +3396,14 @@ Retry:
 		vm_map_unlock_read(map);
 		return (KERN_NO_SPACE);
 	}
+	PROC_LOCK(p);
+	if (is_procstack &&
+	    racct_set(p, RACCT_STACK, ctob(vm->vm_ssize) + grow_amount)) {
+		PROC_UNLOCK(p);
+		vm_map_unlock_read(map);
+		return (KERN_NO_SPACE);
+	}
+	PROC_UNLOCK(p);
 
 	/* Round up the grow amount modulo SGROWSIZ */
 	grow_amount = roundup (grow_amount, sgrowsiz);
@@ -3386,12 +3413,28 @@ Retry:
 		grow_amount = trunc_page((vm_size_t)stacklim) -
 		    ctob(vm->vm_ssize);
 	}
+#ifdef notyet
+	PROC_LOCK(p);
+	limit = racct_get_available(p, RACCT_STACK);
+	PROC_UNLOCK(p);
+	if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > limit))
+		grow_amount = limit - ctob(vm->vm_ssize);
+#endif
 
 	/* If we would blow our VMEM resource limit, no go */
 	if (map->size + grow_amount > vmemlim) {
 		vm_map_unlock_read(map);
-		return (KERN_NO_SPACE);
+		rv = KERN_NO_SPACE;
+		goto out;
 	}
+	PROC_LOCK(p);
+	if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) {
+		PROC_UNLOCK(p);
+		vm_map_unlock_read(map);
+		rv = KERN_NO_SPACE;
+		goto out;
+	}
+	PROC_UNLOCK(p);
 
 	if (vm_map_lock_upgrade(map))
 		goto Retry;
@@ -3490,6 +3533,16 @@ Retry:
 		    : VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
 	}
 
+out:
+	if (rv != KERN_SUCCESS) {
+		PROC_LOCK(p);
+		error = racct_set(p, RACCT_VMEM, map->size);
+		KASSERT(error == 0, ("decreasing RACCT_VMEM failed"));
+	    	error = racct_set(p, RACCT_STACK, ctob(vm->vm_ssize));
+		KASSERT(error == 0, ("decreasing RACCT_STACK failed"));
+		PROC_UNLOCK(p);
+	}
+
 	return (rv);
 }
 
diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c
index f2b317e..37949a5 100644
--- a/sys/vm/vm_mmap.c
+++ b/sys/vm/vm_mmap.c
@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/filedesc.h>
 #include <sys/priv.h>
 #include <sys/proc.h>
+#include <sys/racct.h>
 #include <sys/resource.h>
 #include <sys/resourcevar.h>
 #include <sys/vnode.h>
@@ -991,6 +992,7 @@ mlock(td, uap)
 	struct proc *proc;
 	vm_offset_t addr, end, last, start;
 	vm_size_t npages, size;
+	unsigned long nsize;
 	int error;
 
 	error = priv_check(td, PRIV_VM_MLOCK);
@@ -1008,17 +1010,28 @@ mlock(td, uap)
 		return (ENOMEM);
 	proc = td->td_proc;
 	PROC_LOCK(proc);
-	if (ptoa(npages +
-	    pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))) >
-	    lim_cur(proc, RLIMIT_MEMLOCK)) {
+	nsize = ptoa(npages +
+	    pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map)));
+	if (nsize > lim_cur(proc, RLIMIT_MEMLOCK)) {
 		PROC_UNLOCK(proc);
 		return (ENOMEM);
 	}
 	PROC_UNLOCK(proc);
 	if (npages + cnt.v_wire_count > vm_page_max_wired)
 		return (EAGAIN);
+	PROC_LOCK(proc);
+	error = racct_set(proc, RACCT_MEMLOCK, nsize);
+	PROC_UNLOCK(proc);
+	if (error != 0)
+		return (ENOMEM);
 	error = vm_map_wire(&proc->p_vmspace->vm_map, start, end,
 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
+	if (error != KERN_SUCCESS) {
+		PROC_LOCK(proc);
+		racct_set(proc, RACCT_MEMLOCK,
+		    ptoa(pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))));
+		PROC_UNLOCK(proc);
+	}
 	return (error == KERN_SUCCESS ? 0 : ENOMEM);
 }
 
@@ -1061,6 +1074,11 @@ mlockall(td, uap)
 	if (error)
 		return (error);
 #endif
+	PROC_LOCK(td->td_proc);
+	error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size);
+	PROC_UNLOCK(td->td_proc);
+	if (error != 0)
+		return (ENOMEM);
 
 	if (uap->how & MCL_FUTURE) {
 		vm_map_lock(map);
@@ -1080,6 +1098,12 @@ mlockall(td, uap)
 		    VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
 		error = (error == KERN_SUCCESS ? 0 : EAGAIN);
 	}
+	if (error != KERN_SUCCESS) {
+		PROC_LOCK(td->td_proc);
+		racct_set(td->td_proc, RACCT_MEMLOCK,
+		    ptoa(pmap_wired_count(vm_map_pmap(&td->td_proc->p_vmspace->vm_map))));
+		PROC_UNLOCK(td->td_proc);
+	}
 
 	return (error);
 }
@@ -1114,6 +1138,11 @@ munlockall(td, uap)
 	/* Forcibly unwire all pages. */
 	error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map),
 	    VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
+	if (error == KERN_SUCCESS) {
+		PROC_LOCK(td->td_proc);
+		racct_set(td->td_proc, RACCT_MEMLOCK, 0);
+		PROC_UNLOCK(td->td_proc);
+	}
 
 	return (error);
 }
@@ -1148,6 +1177,11 @@ munlock(td, uap)
 		return (EINVAL);
 	error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end,
 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
+	if (error == KERN_SUCCESS) {
+		PROC_LOCK(td->td_proc);
+		racct_sub(td->td_proc, RACCT_MEMLOCK, ptoa(end - start));
+		PROC_UNLOCK(td->td_proc);
+	}
 	return (error == KERN_SUCCESS ? 0 : ENOMEM);
 }
 
@@ -1380,6 +1414,11 @@ vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
 		PROC_UNLOCK(td->td_proc);
 		return(ENOMEM);
 	}
+	if (racct_set(td->td_proc, RACCT_VMEM,
+	    td->td_proc->p_vmspace->vm_map.size + size)) {
+		PROC_UNLOCK(td->td_proc);
+		return (ENOMEM);
+	}
 	PROC_UNLOCK(td->td_proc);
 
 	/*
diff --git a/sys/vm/vm_unix.c b/sys/vm/vm_unix.c
index 6bb9917..c07a1c1 100644
--- a/sys/vm/vm_unix.c
+++ b/sys/vm/vm_unix.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/sysproto.h>
 #include <sys/systm.h>
@@ -116,9 +117,29 @@ obreak(td, uap)
 			error = ENOMEM;
 			goto done;
 		}
+		PROC_LOCK(td->td_proc);
+		error = racct_set(td->td_proc, RACCT_DATA, new - base);
+		if (error != 0) {
+			PROC_UNLOCK(td->td_proc);
+			error = ENOMEM;
+			goto done;
+		}
+		error = racct_set(td->td_proc, RACCT_VMEM,
+		    vm->vm_map.size + (new - old));
+		if (error != 0) {
+			racct_set_force(td->td_proc, RACCT_DATA, old - base);
+			PROC_UNLOCK(td->td_proc);
+			error = ENOMEM;
+			goto done;
+		}
+		PROC_UNLOCK(td->td_proc);
 		rv = vm_map_insert(&vm->vm_map, NULL, 0, old, new,
 		    VM_PROT_RW, VM_PROT_ALL, 0);
 		if (rv != KERN_SUCCESS) {
+			PROC_LOCK(td->td_proc);
+			racct_set_force(td->td_proc, RACCT_DATA, old - base);
+			racct_set_force(td->td_proc, RACCT_VMEM, vm->vm_map.size);
+			PROC_UNLOCK(td->td_proc);
 			error = ENOMEM;
 			goto done;
 		}
@@ -144,6 +165,10 @@ obreak(td, uap)
 			goto done;
 		}
 		vm->vm_dsize -= btoc(old - new);
+		PROC_LOCK(td->td_proc);
+		racct_set_force(td->td_proc, RACCT_DATA, new - base);
+		racct_set_force(td->td_proc, RACCT_VMEM, vm->vm_map.size);
+		PROC_UNLOCK(td->td_proc);
 	}
 done:
 	vm_map_unlock(&vm->vm_map);