- Use DPCPU for SCHED_STATS. This is somewhat awkward because the

   offset of the stat is not known until link time so we must emit a
   function to call SYSCTL_ADD_PROC rather than using SYSCTL_PROC
   directly.
 - Eliminate the atomic from SCHED_STAT_INC now that it's using per-cpu
   variables.  Sched stats are always incremented while we're holding
   a spinlock so no further protection is required.

Reviewed by:	sam

1 files changed, 18 insertions, 5 deletions

diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index 9d2a34a..d0ebffd 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -162,14 +162,27 @@ sched_unpin(void)
 
 /* Scheduler stats. */
 #ifdef SCHED_STATS
-extern long sched_switch_stats[SWT_COUNT];
+DPCPU_DECLARE(long, sched_switch_stats[SWT_COUNT]);
 
 #define	SCHED_STAT_DEFINE_VAR(name, ptr, descr)				\
-    SYSCTL_LONG(_kern_sched_stats, OID_AUTO, name, CTLFLAG_RD, ptr, 0, descr)
+static void name ## _add_proc(void *dummy __unused)			\
+{									\
+									\
+	SYSCTL_ADD_PROC(NULL,						\
+	    SYSCTL_STATIC_CHILDREN(_kern_sched_stats), OID_AUTO,	\
+	    #name, CTLTYPE_LONG|CTLFLAG_RD|CTLFLAG_MPSAFE,		\
+	    ptr, 0, sysctl_dpcpu_long, "LU", descr);			\
+}									\
+SYSINIT(name, SI_SUB_RUN_SCHEDULER, SI_ORDER_ANY, name ## _add_proc, NULL);
+
 #define	SCHED_STAT_DEFINE(name, descr)					\
-    unsigned long name;							\
-    SCHED_STAT_DEFINE_VAR(name, &name, descr)
-#define SCHED_STAT_INC(var)     atomic_add_long(&(var), 1)
+    DPCPU_DEFINE(unsigned long, name);					\
+    SCHED_STAT_DEFINE_VAR(name, &DPCPU_NAME(name), descr)
+/*
+ * Sched stats are always incremented in critical sections so no atomic
+ * is necesssary to increment them.
+ */
+#define SCHED_STAT_INC(var)     DPCPU_GET(var)++;
 #else
 #define	SCHED_STAT_DEFINE_VAR(name, descr, ptr)
 #define	SCHED_STAT_DEFINE(name, descr)