Add software PMC support.

New kernel events can be added at various location for sampling or counting.
This will for example allow easy system profiling whatever the processor is
with known tools like pmcstat(8).

Simultaneous usage of software PMC and hardware PMC is possible, for example
looking at the lock acquire failure, page fault while sampling on
instructions.

Sponsored by: NETASQ
MFC after:	1 month

7 files changed, 236 insertions, 1 deletions

diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index fdc1302..57dd632 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -74,6 +74,8 @@ __FBSDID("$FreeBSD$");
 
 #ifdef HWPMC_HOOKS
 #include <sys/pmckern.h>
+PMC_SOFT_DEFINE( , , clock, hard);
+PMC_SOFT_DEFINE( , , clock, stat);
 #endif
 
 #ifdef DEVICE_POLLING
@@ -446,9 +448,11 @@ hardclock_cpu(int usermode)
 	td->td_flags |= flags;
 	thread_unlock(td);
 
-#ifdef	HWPMC_HOOKS
+#ifdef HWPMC_HOOKS
 	if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
 		PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
+	if (td->td_intr_frame != NULL)
+		PMC_SOFT_CALL_TF( , , clock, hard, td->td_intr_frame);
 #endif
 	callout_tick();
 }
@@ -537,6 +541,8 @@ hardclock_cnt(int cnt, int usermode)
 #ifdef	HWPMC_HOOKS
 	if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
 		PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
+	if (td->td_intr_frame != NULL)
+		PMC_SOFT_CALL_TF( , , clock, hard, td->td_intr_frame);
 #endif
 	callout_tick();
 	/* We are in charge to handle this tick duty. */
@@ -758,6 +764,10 @@ statclock_cnt(int cnt, int usermode)
 	for ( ; cnt > 0; cnt--)
 		sched_clock(td);
 	thread_unlock(td);
+#ifdef HWPMC_HOOKS
+	if (td->td_intr_frame != NULL)
+		PMC_SOFT_CALL_TF( , , clock, stat, td->td_intr_frame);
+#endif
 }
 
 void
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index 00f1b11..24526b0 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -28,6 +28,7 @@
 
 #include "opt_adaptive_lockmgrs.h"
 #include "opt_ddb.h"
+#include "opt_hwpmc_hooks.h"
 #include "opt_kdtrace.h"
 
 #include <sys/cdefs.h>
@@ -53,6 +54,11 @@ __FBSDID("$FreeBSD$");
 #include <ddb/ddb.h>
 #endif
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+PMC_SOFT_DECLARE( , , lock, failed);
+#endif
+
 CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) ==
     (LK_ADAPTIVE | LK_NOSHARE));
 CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
@@ -514,6 +520,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 					break;
 				continue;
 			}
+#ifdef HWPMC_HOOKS
+			PMC_SOFT_CALL( , , lock, failed);
+#endif
 			lock_profile_obtain_lock_failed(&lk->lock_object,
 			    &contested, &waittime);
 
@@ -744,6 +753,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 
 		while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED,
 		    tid)) {
+#ifdef HWPMC_HOOKS
+			PMC_SOFT_CALL( , , lock, failed);
+#endif
 			lock_profile_obtain_lock_failed(&lk->lock_object,
 			    &contested, &waittime);
 
@@ -1056,6 +1068,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 		}
 
 		while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
+#ifdef HWPMC_HOOKS
+			PMC_SOFT_CALL( , , lock, failed);
+#endif
 			lock_profile_obtain_lock_failed(&lk->lock_object,
 			    &contested, &waittime);
 
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index bfb6547..f718ca0 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_adaptive_mutexes.h"
 #include "opt_ddb.h"
 #include "opt_global.h"
+#include "opt_hwpmc_hooks.h"
 #include "opt_kdtrace.h"
 #include "opt_sched.h"
 
@@ -76,6 +77,11 @@ __FBSDID("$FreeBSD$");
 #define	ADAPTIVE_MUTEXES
 #endif
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+PMC_SOFT_DEFINE( , , lock, failed);
+#endif
+
 /*
  * Internal utility macros.
  */
@@ -364,6 +370,9 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
 		return;
 	}
 
+#ifdef HWPMC_HOOKS
+	PMC_SOFT_CALL( , , lock, failed);
+#endif
 	lock_profile_obtain_lock_failed(&m->lock_object,
 		    &contested, &waittime);
 	if (LOCK_LOG_TEST(&m->lock_object, opts))
@@ -529,6 +538,9 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
 	if (LOCK_LOG_TEST(&m->lock_object, opts))
 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
 
+#ifdef HWPMC_HOOKS
+	PMC_SOFT_CALL( , , lock, failed);
+#endif
 	lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
 	while (!_mtx_obtain_lock(m, tid)) {
 
@@ -600,6 +612,9 @@ retry:
 				m->mtx_recurse++;
 				break;
 			}
+#ifdef HWPMC_HOOKS
+			PMC_SOFT_CALL( , , lock, failed);
+#endif
 			lock_profile_obtain_lock_failed(&m->lock_object,
 			    &contested, &waittime);
 			/* Give interrupts a chance while we spin. */
diff --git a/sys/kern/kern_pmc.c b/sys/kern/kern_pmc.c
index 8d9c7c0..2b50be0 100644
--- a/sys/kern/kern_pmc.c
+++ b/sys/kern/kern_pmc.c
@@ -34,10 +34,17 @@ __FBSDID("$FreeBSD$");
 #include "opt_hwpmc_hooks.h"
 
 #include <sys/types.h>
+#include <sys/ctype.h>
+#include <sys/param.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/pmc.h>
 #include <sys/pmckern.h>
 #include <sys/smp.h>
 #include <sys/sysctl.h>
+#include <sys/systm.h>
 
 #ifdef	HWPMC_HOOKS
 FEATURE(hwpmc_hooks, "Kernel support for HW PMC");
@@ -46,6 +53,9 @@ FEATURE(hwpmc_hooks, "Kernel support for HW PMC");
 #define	PMC_KERNEL_VERSION	0
 #endif
 
+MALLOC_DECLARE(M_PMCHOOKS);
+MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks");
+
 const int pmc_kernel_version = PMC_KERNEL_VERSION;
 
 /* Hook variable. */
@@ -74,6 +84,28 @@ volatile int pmc_ss_count;
  */
 struct sx pmc_sx;
 
+/*
+ * PMC Soft per cpu trapframe.
+ */
+struct trapframe pmc_tf[MAXCPU];
+
+/*
+ * PMC Soft use a global table to store registered events.
+ */
+
+SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
+
+static int pmc_softevents = 16;
+TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "softevents", &pmc_softevents);
+SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_TUN|CTLFLAG_RD,
+    &pmc_softevents, 0, "maximum number of soft events");
+
+struct mtx pmc_softs_mtx;
+int pmc_softs_count;
+struct pmc_soft **pmc_softs;
+
+MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN);
+
 static void
 pmc_init_sx(void)
 {
@@ -182,3 +214,132 @@ pmc_cpu_max_active(void)
 }
 
 #endif
+
+/*
+ * Cleanup event name:
+ * - remove duplicate '_'
+ * - all uppercase
+ */
+static void
+pmc_soft_namecleanup(char *name)
+{
+	char *p, *q;
+
+	p = q = name;
+
+	for ( ; *p == '_' ; p++)
+		;
+	for ( ; *p ; p++) {
+		if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0'))
+			continue;
+		else
+			*q++ = toupper(*p);
+	}
+	*q = '\0';
+}
+
+void
+pmc_soft_ev_register(struct pmc_soft *ps)
+{
+	static int warned = 0;
+	int n;
+
+	ps->ps_running  = 0;
+	ps->ps_ev.pm_ev_code = 0; /* invalid */
+	pmc_soft_namecleanup(ps->ps_ev.pm_ev_name);
+
+	mtx_lock_spin(&pmc_softs_mtx);
+
+	if (pmc_softs_count >= pmc_softevents) {
+		/*
+		 * XXX Reusing events can enter a race condition where
+		 * new allocated event will be used as an old one.
+		 */
+		for (n = 0; n < pmc_softevents; n++)
+			if (pmc_softs[n] == NULL)
+				break;
+		if (n == pmc_softevents) {
+			mtx_unlock_spin(&pmc_softs_mtx);
+			if (!warned) {
+				printf("hwpmc: too many soft events, "
+				    "increase kern.hwpmc.softevents tunable\n");
+				warned = 1;
+			}
+			return;
+		}
+
+		ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n;
+		pmc_softs[n] = ps;
+	} else {
+		ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count;
+		pmc_softs[pmc_softs_count++] = ps;
+	}
+
+	mtx_unlock_spin(&pmc_softs_mtx);
+}
+
+void
+pmc_soft_ev_deregister(struct pmc_soft *ps)
+{
+
+	KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL"));
+
+	mtx_lock_spin(&pmc_softs_mtx);
+
+	if (ps->ps_ev.pm_ev_code != 0 &&
+	    (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) {
+		KASSERT(ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST &&
+		    ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST,
+		    ("pmc_soft_deregister: invalid event value"));
+		pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL;
+	}
+
+	mtx_unlock_spin(&pmc_softs_mtx);
+}
+
+struct pmc_soft *
+pmc_soft_ev_acquire(enum pmc_event ev)
+{
+	struct pmc_soft *ps;
+
+	if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents)
+		return NULL;
+
+	KASSERT(ev >= PMC_EV_SOFT_FIRST &&
+	    ev <= PMC_EV_SOFT_LAST,
+	    ("event out of range"));
+
+	mtx_lock_spin(&pmc_softs_mtx);
+
+	ps = pmc_softs[ev - PMC_EV_SOFT_FIRST];
+	if (ps == NULL)
+		mtx_unlock_spin(&pmc_softs_mtx);
+
+	return ps;
+}
+
+void
+pmc_soft_ev_release(struct pmc_soft *ps)
+{
+
+	mtx_unlock_spin(&pmc_softs_mtx);
+}
+
+/*
+ *  Initialise hwpmc.
+ */
+static void
+init_hwpmc(void *dummy __unused)
+{
+	if (pmc_softevents <= 0 ||
+	    pmc_softevents > PMC_EV_DYN_COUNT) {
+		(void) printf("hwpmc: tunable \"softevents\"=%d out of "
+		    "range.\n", pmc_softevents);
+		pmc_softevents = PMC_EV_DYN_COUNT;
+	}
+	pmc_softs = malloc(pmc_softevents * sizeof(struct pmc_soft *), M_PMCHOOKS, M_NOWAIT|M_ZERO);
+	KASSERT(pmc_softs != NULL, ("cannot allocate soft events table"));
+}
+
+SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL);
+
diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index b571532..c337041 100644
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -35,6 +35,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
+#include "opt_hwpmc_hooks.h"
 #include "opt_kdtrace.h"
 #include "opt_no_adaptive_rwlocks.h"
 
@@ -55,6 +56,11 @@ __FBSDID("$FreeBSD$");
 #define	ADAPTIVE_RWLOCKS
 #endif
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+PMC_SOFT_DECLARE( , , lock, failed);
+#endif
+
 #ifdef ADAPTIVE_RWLOCKS
 static int rowner_retries = 10;
 static int rowner_loops = 10000;
@@ -366,6 +372,9 @@ _rw_rlock(struct rwlock *rw, const char *file, int line)
 			}
 			continue;
 		}
+#ifdef HWPMC_HOOKS
+		PMC_SOFT_CALL( , , lock, failed);
+#endif
 		lock_profile_obtain_lock_failed(&rw->lock_object,
 		    &contested, &waittime);
 
@@ -687,6 +696,9 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
 #ifdef KDTRACE_HOOKS
 		spin_cnt++;
 #endif
+#ifdef HWPMC_HOOKS
+		PMC_SOFT_CALL( , , lock, failed);
+#endif
 		lock_profile_obtain_lock_failed(&rw->lock_object,
 		    &contested, &waittime);
 #ifdef ADAPTIVE_RWLOCKS
diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c
index 9cb9995..bcd7acd 100644
--- a/sys/kern/kern_sx.c
+++ b/sys/kern/kern_sx.c
@@ -37,6 +37,7 @@
  */
 
 #include "opt_ddb.h"
+#include "opt_hwpmc_hooks.h"
 #include "opt_kdtrace.h"
 #include "opt_no_adaptive_sx.h"
 
@@ -67,6 +68,11 @@ __FBSDID("$FreeBSD$");
 
 CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE);
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+PMC_SOFT_DECLARE( , , lock, failed);
+#endif
+
 /* Handy macros for sleep queues. */
 #define	SQ_EXCLUSIVE_QUEUE	0
 #define	SQ_SHARED_QUEUE		1
@@ -524,6 +530,9 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
 #ifdef KDTRACE_HOOKS
 		spin_cnt++;
 #endif
+#ifdef HWPMC_HOOKS
+		PMC_SOFT_CALL( , , lock, failed);
+#endif
 		lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
 		    &waittime);
 #ifdef ADAPTIVE_SX
@@ -811,6 +820,9 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 			}
 			continue;
 		}
+#ifdef HWPMC_HOOKS
+		PMC_SOFT_CALL( , , lock, failed);
+#endif
 		lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
 		    &waittime);
 
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index ba8b66f..ab83906 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -45,6 +45,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_capsicum.h"
+#include "opt_hwpmc_hooks.h"
 #include "opt_ktrace.h"
 #include "opt_kdtrace.h"
 #include "opt_sched.h"
@@ -86,6 +87,10 @@ __FBSDID("$FreeBSD$");
 #include <vm/pmap.h>
 #endif
 
+#ifdef	HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #include <security/mac/mac_framework.h>
 
 /*
@@ -192,6 +197,11 @@ ast(struct trapframe *framep)
 		td->td_profil_ticks = 0;
 		td->td_pflags &= ~TDP_OWEUPC;
 	}
+#ifdef HWPMC_HOOKS
+	/* Handle Software PMC callchain capture. */
+	if (PMC_IS_PENDING_CALLCHAIN(td))
+		PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_USER_CALLCHAIN_SOFT, (void *) framep);
+#endif
 	if (flags & TDF_ALRMPEND) {
 		PROC_LOCK(p);
 		kern_psignal(p, SIGVTALRM);