Embed a struct vmmeter in the per-cpu structure and add a macro,

PCPU_LAZY_INC() which increments elements in it for cases where we
can afford the occassional inaccuracy.  Use of per-cpu stats counters
avoids significant cache stalls in various critical paths that would
otherwise severely limit our cpu scaleability.

Adjust all sysctl's accessing cnt.* elements to now use a procedure
which aggregates the requested field for all cpus and for the global
vmmeter.

The global vmmeter is retained, since some stats counters, like v_free_min,
cannot be made per-cpu.  Also, this allows us to convert counters from
the global vmmeter to the per-cpu vmmeter in a piecemeal fashion, so
have at it!

1 files changed, 129 insertions, 96 deletions

diff --git a/sys/vm/vm_meter.c b/sys/vm/vm_meter.c
index 51f8107..fde1f49 100644
--- a/sys/vm/vm_meter.c
+++ b/sys/vm/vm_meter.c
@@ -43,6 +43,7 @@
 #include <sys/resource.h>
 #include <sys/sx.h>
 #include <sys/vmmeter.h>
+#include <sys/smp.h>
 
 #include <vm/vm.h>
 #include <vm/vm_page.h>
@@ -189,6 +190,36 @@ vmtotal(SYSCTL_HANDLER_ARGS)
 	return (sysctl_handle_opaque(oidp, totalp, sizeof total, req));
 }
 
+/*
+ * vcnt() -	accumulate statistics from all cpus and the global cnt
+ *		structure.
+ *
+ *	The vmmeter structure is now per-cpu as well as global.  Those
+ *	statistics which can be kept on a per-cpu basis (to avoid cache
+ *	stalls between cpus) can be moved to the per-cpu vmmeter.  Remaining
+ *	statistics, such as v_free_reserved, are left in the global
+ *	structure.
+ *
+ * (sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req)
+ */
+static int
+vcnt(SYSCTL_HANDLER_ARGS)
+{
+        int error = 0;
+	int count = *(int *)arg1;
+#ifdef SMP
+	int i;
+	int offset = (char *)arg1 - (char *)&cnt;
+
+	for (i = 0; i < mp_ncpus; ++i) {
+		struct pcpu *pcpu = pcpu_find(i);
+		count += *(int *)((char *)&pcpu->pc_cnt + offset);
+	}
+#endif
+	error = SYSCTL_OUT(req, &count, sizeof(int));
+	return(error);
+}
+
 SYSCTL_PROC(_vm, VM_METER, vmmeter, CTLTYPE_OPAQUE|CTLFLAG_RD,
     0, sizeof(struct vmtotal), vmtotal, "S,vmtotal", 
     "System virtual memory statistics");
@@ -196,104 +227,106 @@ SYSCTL_NODE(_vm, OID_AUTO, stats, CTLFLAG_RW, 0, "VM meter stats");
 SYSCTL_NODE(_vm_stats, OID_AUTO, sys, CTLFLAG_RW, 0, "VM meter sys stats");
 SYSCTL_NODE(_vm_stats, OID_AUTO, vm, CTLFLAG_RW, 0, "VM meter vm stats");
 SYSCTL_NODE(_vm_stats, OID_AUTO, misc, CTLFLAG_RW, 0, "VM meter misc stats");
-SYSCTL_UINT(_vm_stats_sys, OID_AUTO,
-	v_swtch, CTLFLAG_RD, &cnt.v_swtch, 0, "Context switches");
-SYSCTL_UINT(_vm_stats_sys, OID_AUTO,
-	v_trap, CTLFLAG_RD, &cnt.v_trap, 0, "Traps");
-SYSCTL_UINT(_vm_stats_sys, OID_AUTO,
-	v_syscall, CTLFLAG_RD, &cnt.v_syscall, 0, "Syscalls");
-SYSCTL_UINT(_vm_stats_sys, OID_AUTO, v_intr, CTLFLAG_RD,
-    &cnt.v_intr, 0, "Hardware interrupts");
-SYSCTL_UINT(_vm_stats_sys, OID_AUTO, v_soft, CTLFLAG_RD, 
-    &cnt.v_soft, 0, "Software interrupts");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_vm_faults, CTLFLAG_RD, &cnt.v_vm_faults, 0, "VM faults");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_cow_faults, CTLFLAG_RD, &cnt.v_cow_faults, 0, "COW faults");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_cow_optim, CTLFLAG_RD, &cnt.v_cow_optim, 0, "Optimized COW faults");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_zfod, CTLFLAG_RD, &cnt.v_zfod, 0, "Zero fill");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_ozfod, CTLFLAG_RD, &cnt.v_ozfod, 0, "Optimized zero fill");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_swapin, CTLFLAG_RD, &cnt.v_swapin, 0, "Swapin operations");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_swapout, CTLFLAG_RD, &cnt.v_swapout, 0, "Swapout operations");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_swappgsin, CTLFLAG_RD, &cnt.v_swappgsin, 0, "Swapin pages");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_swappgsout, CTLFLAG_RD, &cnt.v_swappgsout, 0, "Swapout pages");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_vnodein, CTLFLAG_RD, &cnt.v_vnodein, 0, "Vnodein operations");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_vnodeout, CTLFLAG_RD, &cnt.v_vnodeout, 0, "Vnodeout operations");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_vnodepgsin, CTLFLAG_RD, &cnt.v_vnodepgsin, 0, "Vnodein pages");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_vnodepgsout, CTLFLAG_RD, &cnt.v_vnodepgsout, 0, "Vnodeout pages");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_intrans, CTLFLAG_RD, &cnt.v_intrans, 0, "In transit page blocking");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_reactivated, CTLFLAG_RD, &cnt.v_reactivated, 0, "Reactivated pages");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_pdwakeups, CTLFLAG_RD, &cnt.v_pdwakeups, 0, "Pagedaemon wakeups");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_pdpages, CTLFLAG_RD, &cnt.v_pdpages, 0, "Pagedaemon page scans");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_dfree, CTLFLAG_RD, &cnt.v_dfree, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_pfree, CTLFLAG_RD, &cnt.v_pfree, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_tfree, CTLFLAG_RD, &cnt.v_tfree, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_page_size, CTLFLAG_RD, &cnt.v_page_size, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_page_count, CTLFLAG_RD, &cnt.v_page_count, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_free_reserved, CTLFLAG_RD, &cnt.v_free_reserved, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_free_target, CTLFLAG_RD, &cnt.v_free_target, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_free_min, CTLFLAG_RD, &cnt.v_free_min, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_free_count, CTLFLAG_RD, &cnt.v_free_count, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_wire_count, CTLFLAG_RD, &cnt.v_wire_count, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_active_count, CTLFLAG_RD, &cnt.v_active_count, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_inactive_target, CTLFLAG_RD, &cnt.v_inactive_target, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_inactive_count, CTLFLAG_RD, &cnt.v_inactive_count, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_cache_count, CTLFLAG_RD, &cnt.v_cache_count, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_cache_min, CTLFLAG_RD, &cnt.v_cache_min, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_cache_max, CTLFLAG_RD, &cnt.v_cache_max, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_pageout_free_min, CTLFLAG_RD, &cnt.v_pageout_free_min, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_interrupt_free_min, CTLFLAG_RD, &cnt.v_interrupt_free_min, 0, "");
+
+SYSCTL_PROC(_vm_stats_sys, OID_AUTO, v_swtch, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_swtch, 0, vcnt, "IU", "Context switches");
+SYSCTL_PROC(_vm_stats_sys, OID_AUTO, v_trap, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_trap, 0, vcnt, "IU", "Traps");
+SYSCTL_PROC(_vm_stats_sys, OID_AUTO, v_syscall, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_syscall, 0, vcnt, "IU", "Syscalls");
+SYSCTL_PROC(_vm_stats_sys, OID_AUTO, v_intr, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_intr, 0, vcnt, "IU", "Hardware interrupts");
+SYSCTL_PROC(_vm_stats_sys, OID_AUTO, v_soft, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_soft, 0, vcnt, "IU", "Software interrupts");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_vm_faults, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_vm_faults, 0, vcnt, "IU", "VM faults");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_cow_faults, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_cow_faults, 0, vcnt, "IU", "COW faults");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_cow_optim, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_cow_optim, 0, vcnt, "IU", "Optimized COW faults");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_zfod, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_zfod, 0, vcnt, "IU", "Zero fill");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_ozfod, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_ozfod, 0, vcnt, "IU", "Optimized zero fill");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_swapin, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_swapin, 0, vcnt, "IU", "Swapin operations");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_swapout, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_swapout, 0, vcnt, "IU", "Swapout operations");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_swappgsin, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_swappgsin, 0, vcnt, "IU", "Swapin pages");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_swappgsout, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_swappgsout, 0, vcnt, "IU", "Swapout pages");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_vnodein, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_vnodein, 0, vcnt, "IU", "Vnodein operations");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_vnodeout, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_vnodeout, 0, vcnt, "IU", "Vnodeout operations");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_vnodepgsin, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_vnodepgsin, 0, vcnt, "IU", "Vnodein pages");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_vnodepgsout, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_vnodepgsout, 0, vcnt, "IU", "Vnodeout pages");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_intrans, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_intrans, 0, vcnt, "IU", "In transit page blocking");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_reactivated, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_reactivated, 0, vcnt, "IU", "Reactivated pages");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_pdwakeups, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_pdwakeups, 0, vcnt, "IU", "Pagedaemon wakeups");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_pdpages, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_pdpages, 0, vcnt, "IU", "Pagedaemon page scans");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_dfree, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_dfree, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_pfree, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_pfree, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_tfree, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_tfree, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_page_size, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_page_size, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_page_count, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_page_count, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_free_reserved, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_free_reserved, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_free_target, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_free_target, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_free_min, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_free_min, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_free_count, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_free_count, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_wire_count, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_wire_count, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_active_count, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_active_count, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_inactive_target, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_inactive_target, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_inactive_count, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_inactive_count, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_cache_count, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_cache_count, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_cache_min, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_cache_min, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_cache_max, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_cache_max, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_pageout_free_min, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_pageout_free_min, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_interrupt_free_min, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_interrupt_free_min, 0, vcnt, "IU", "");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_forks, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_forks, 0, vcnt, "IU", "Number of fork() calls");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_vforks, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_vforks, 0, vcnt, "IU", "Number of vfork() calls");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_rforks, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_rforks, 0, vcnt, "IU", "Number of rfork() calls");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_kthreads, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_kthreads, 0, vcnt, "IU", "Number of fork() calls by kernel");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_forkpages, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_forkpages, 0, vcnt, "IU", "VM pages affected by fork()");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_vforkpages, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_vforkpages, 0, vcnt, "IU", "VM pages affected by vfork()");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_rforkpages, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_rforkpages, 0, vcnt, "IU", "VM pages affected by rfork()");
+SYSCTL_PROC(_vm_stats_vm, OID_AUTO, v_kthreadpages, CTLTYPE_UINT|CTLFLAG_RD,
+	&cnt.v_kthreadpages, 0, vcnt, "IU", "VM pages affected by fork() by kernel");
+
 SYSCTL_INT(_vm_stats_misc, OID_AUTO,
 	zero_page_count, CTLFLAG_RD, &vm_page_zero_count, 0, "");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_forks, CTLFLAG_RD, &cnt.v_forks, 0, "Number of fork() calls");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_vforks, CTLFLAG_RD, &cnt.v_vforks, 0, "Number of vfork() calls");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_rforks, CTLFLAG_RD, &cnt.v_rforks, 0, "Number of rfork() calls");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_kthreads, CTLFLAG_RD, &cnt.v_kthreads, 0, "Number of fork() calls by kernel");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_forkpages, CTLFLAG_RD, &cnt.v_forkpages, 0, "VM pages affected by fork()");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_vforkpages, CTLFLAG_RD, &cnt.v_vforkpages, 0, "VM pages affected by vfork()");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_rforkpages, CTLFLAG_RD, &cnt.v_rforkpages, 0, "VM pages affected by rfork()");
-SYSCTL_UINT(_vm_stats_vm, OID_AUTO,
-	v_kthreadpages, CTLFLAG_RD, &cnt.v_kthreadpages, 0, "VM pages affected by fork() by kernel");
 #if 0
 SYSCTL_INT(_vm_stats_misc, OID_AUTO,
 	page_mask, CTLFLAG_RD, &page_mask, 0, "");