Add software PMC support.

New kernel events can be added at various location for sampling or counting.
This will for example allow easy system profiling whatever the processor is
with known tools like pmcstat(8).

Simultaneous usage of software PMC and hardware PMC is possible, for example
looking at the lock acquire failure, page fault while sampling on
instructions.

Sponsored by: NETASQ
MFC after:	1 month

1 files changed, 213 insertions, 38 deletions

diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
index 7ca7a47..7eb9f92 100644
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -70,6 +70,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
 
+#include "hwpmc_soft.h"
+
 /*
  * Types
  */
@@ -182,7 +184,7 @@ static int	pmc_attach_one_process(struct proc *p, struct pmc *pm);
 static int	pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
     int cpu);
 static int	pmc_can_attach(struct pmc *pm, struct proc *p);
-static void	pmc_capture_user_callchain(int cpu, struct trapframe *tf);
+static void	pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
 static void	pmc_cleanup(void);
 static int	pmc_detach_process(struct proc *p, struct pmc *pm);
 static int	pmc_detach_one_process(struct proc *p, struct pmc *pm,
@@ -206,7 +208,7 @@ static void	pmc_process_csw_out(struct thread *td);
 static void	pmc_process_exit(void *arg, struct proc *p);
 static void	pmc_process_fork(void *arg, struct proc *p1,
     struct proc *p2, int n);
-static void	pmc_process_samples(int cpu);
+static void	pmc_process_samples(int cpu, int soft);
 static void	pmc_release_pmc_descriptor(struct pmc *pmc);
 static void	pmc_remove_owner(struct pmc_owner *po);
 static void	pmc_remove_process_descriptor(struct pmc_process *pp);
@@ -218,12 +220,16 @@ static int	pmc_stop(struct pmc *pm);
 static int	pmc_syscall_handler(struct thread *td, void *syscall_args);
 static void	pmc_unlink_target_process(struct pmc *pmc,
     struct pmc_process *pp);
+static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp);
+static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp);
+static struct pmc_mdep *pmc_generic_cpu_initialize(void);
+static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
 
 /*
  * Kernel tunables and sysctl(8) interface.
  */
 
-SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
+SYSCTL_DECL(_kern_hwpmc);
 
 static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
 TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "callchaindepth", &pmc_callchaindepth);
@@ -1833,7 +1839,9 @@ const char *pmc_hooknames[] = {
 	"KLDUNLOAD",
 	"MMAP",
 	"MUNMAP",
-	"CALLCHAIN"
+	"CALLCHAIN-NMI",
+	"CALLCHAIN-SOFT",
+	"SOFTSAMPLING"
 };
 #endif
 
@@ -1992,7 +2000,8 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
 		 * lose the interrupt sample.
 		 */
 		CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask);
-		pmc_process_samples(PCPU_GET(cpuid));
+		pmc_process_samples(PCPU_GET(cpuid), PMC_HR);
+		pmc_process_samples(PCPU_GET(cpuid), PMC_SR);
 		break;
 
 
@@ -2022,11 +2031,30 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
 		 */
 		KASSERT(td == curthread, ("[pmc,%d] td != curthread",
 		    __LINE__));
-		pmc_capture_user_callchain(PCPU_GET(cpuid),
+
+		pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
 		    (struct trapframe *) arg);
 		td->td_pflags &= ~TDP_CALLCHAIN;
 		break;
 
+	case PMC_FN_USER_CALLCHAIN_SOFT:
+		/*
+		 * Record a call chain.
+		 */
+		KASSERT(td == curthread, ("[pmc,%d] td != curthread",
+		    __LINE__));
+		pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_SR,
+		    (struct trapframe *) arg);
+		td->td_pflags &= ~TDP_CALLCHAIN;
+		break;
+
+	case PMC_FN_SOFT_SAMPLING:
+		/*
+		 * Call soft PMC sampling intr.
+		 */
+		pmc_soft_intr((struct pmckern_soft *) arg);
+		break;
+
 	default:
 #ifdef	DEBUG
 		KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
@@ -2221,18 +2249,17 @@ pmc_destroy_pmc_descriptor(struct pmc *pm)
 static void
 pmc_wait_for_pmc_idle(struct pmc *pm)
 {
-#ifdef	DEBUG
+#ifdef DEBUG
 	volatile int maxloop;
 
 	maxloop = 100 * pmc_cpu_max();
 #endif
-
 	/*
 	 * Loop (with a forced context switch) till the PMC's runcount
 	 * comes down to zero.
 	 */
 	while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
-#ifdef	DEBUG
+#ifdef DEBUG
 		maxloop--;
 		KASSERT(maxloop > 0,
 		    ("[pmc,%d] (ri%d, rc%d) waiting too long for "
@@ -2972,6 +2999,53 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 	}
 	break;
 
+	/*
+	 * Retrieve soft events list.
+	 */
+	case PMC_OP_GETDYNEVENTINFO:
+	{
+		enum pmc_class			cl;
+		enum pmc_event			ev;
+		struct pmc_op_getdyneventinfo	*gei;
+		struct pmc_dyn_event_descr	dev;
+		struct pmc_soft			*ps;
+		uint32_t			nevent;
+
+		sx_assert(&pmc_sx, SX_LOCKED);
+
+		gei = (struct pmc_op_getdyneventinfo *) arg;
+
+		if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
+			break;
+
+		/* Only SOFT class is dynamic. */
+		if (cl != PMC_CLASS_SOFT) {
+			error = EINVAL;
+			break;
+		}
+
+		nevent = 0;
+		for (ev = PMC_EV_SOFT_FIRST; ev <= PMC_EV_SOFT_LAST; ev++) {
+			ps = pmc_soft_ev_acquire(ev);
+			if (ps == NULL)
+				continue;
+			bcopy(&ps->ps_ev, &dev, sizeof(dev));
+			pmc_soft_ev_release(ps);
+
+			error = copyout(&dev,
+			    &gei->pm_events[nevent],
+			    sizeof(struct pmc_dyn_event_descr));
+			if (error != 0)
+				break;
+			nevent++;
+		}
+		if (error != 0)
+			break;
+
+		error = copyout(&nevent, &gei->pm_nevent,
+		    sizeof(nevent));
+	}
+	break;
 
 	/*
 	 * Get module statistics
@@ -4022,7 +4096,7 @@ pmc_post_callchain_callback(void)
  */
 
 int
-pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
+pmc_process_interrupt(int cpu, int ring, struct pmc *pm, struct trapframe *tf,
     int inuserspace)
 {
 	int error, callchaindepth;
@@ -4035,7 +4109,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
 	/*
 	 * Allocate space for a sample buffer.
 	 */
-	psb = pmc_pcpu[cpu]->pc_sb;
+	psb = pmc_pcpu[cpu]->pc_sb[ring];
 
 	ps = psb->ps_write;
 	if (ps->ps_nsamples) {	/* in use, reader hasn't caught up */
@@ -4061,6 +4135,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
 		pm->pm_runcount));
 
 	atomic_add_rel_int(&pm->pm_runcount, 1);	/* hold onto PMC */
+
 	ps->ps_pmc = pm;
 	if ((td = curthread) && td->td_proc)
 		ps->ps_pid = td->td_proc->p_pid;
@@ -4080,11 +4155,11 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
 		 * Kernel stack traversals can be done immediately,
 		 * while we defer to an AST for user space traversals.
 		 */
-		if (!inuserspace)
+		if (!inuserspace) {
 			callchaindepth =
 			    pmc_save_kernel_callchain(ps->ps_pc,
 				callchaindepth, tf);
-		else {
+		} else {
 			pmc_post_callchain_callback();
 			callchaindepth = PMC_SAMPLE_INUSE;
 		}
@@ -4113,7 +4188,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
  */
 
 static void
-pmc_capture_user_callchain(int cpu, struct trapframe *tf)
+pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
 {
 	int i;
 	struct pmc *pm;
@@ -4124,9 +4199,7 @@ pmc_capture_user_callchain(int cpu, struct trapframe *tf)
 	int ncallchains;
 #endif
 
-	sched_unpin();	/* Can migrate safely now. */
-
-	psb = pmc_pcpu[cpu]->pc_sb;
+	psb = pmc_pcpu[cpu]->pc_sb[ring];
 	td = curthread;
 
 	KASSERT(td->td_pflags & TDP_CALLCHAIN,
@@ -4172,23 +4245,25 @@ pmc_capture_user_callchain(int cpu, struct trapframe *tf)
 #ifdef	INVARIANTS
 		ncallchains++;
 #endif
-
 	}
 
 	KASSERT(ncallchains > 0,
 	    ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
 		cpu));
 
+	KASSERT(td->td_pinned > 0,
+	    ("[pmc,%d] invalid td_pinned value", __LINE__));
+	sched_unpin();	/* Can migrate safely now. */
+
 	return;
 }
 
-
 /*
  * Process saved PC samples.
  */
 
 static void
-pmc_process_samples(int cpu)
+pmc_process_samples(int cpu, int ring)
 {
 	struct pmc *pm;
 	int adjri, n;
@@ -4202,18 +4277,13 @@ pmc_process_samples(int cpu)
 	    ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
 		PCPU_GET(cpuid), cpu));
 
-	psb = pmc_pcpu[cpu]->pc_sb;
+	psb = pmc_pcpu[cpu]->pc_sb[ring];
 
 	for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
 
 		ps = psb->ps_read;
 		if (ps->ps_nsamples == PMC_SAMPLE_FREE)
 			break;
-		if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
-			/* Need a rescan at a later time. */
-			CPU_SET_ATOMIC(cpu, &pmc_cpumask);
-			break;
-		}
 
 		pm = ps->ps_pmc;
 
@@ -4231,6 +4301,13 @@ pmc_process_samples(int cpu)
 		if (pm->pm_state != PMC_STATE_RUNNING)
 			goto entrydone;
 
+		/* If there is a pending AST wait for completion */
+		if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
+			/* Need a rescan at a later time. */
+			CPU_SET_ATOMIC(cpu, &pmc_cpumask);
+			break;
+		}
+
 		PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
 		    pm, ps->ps_nsamples, ps->ps_flags,
 		    (int) (psb->ps_write - psb->ps_samples),
@@ -4256,11 +4333,10 @@ pmc_process_samples(int cpu)
 		 * or a system-wide sampling PMC.  Dispatch a log
 		 * entry to the PMC's owner process.
 		 */
-
 		pmclog_process_callchain(pm, ps);
 
 	entrydone:
-		ps->ps_nsamples = 0;	/* mark entry as free */
+		ps->ps_nsamples = 0; /* mark entry as free */
 		atomic_subtract_rel_int(&pm->pm_runcount, 1);
 
 		/* increment read pointer, modulo sample size */
@@ -4584,6 +4660,76 @@ static const char *pmc_name_of_pmcclass[] = {
 	__PMC_CLASSES()
 };
 
+/*
+ * Base class initializer: allocate structure and set default classes.
+ */
+struct pmc_mdep *
+pmc_mdep_alloc(int nclasses)
+{
+	struct pmc_mdep *md;
+	int	n;
+
+	/* SOFT + md classes */
+	n = 1 + nclasses;
+	md = malloc(sizeof(struct pmc_mdep) + n *
+	    sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
+	if (md != NULL) {
+		md->pmd_nclass = n;
+
+		/* Add base class. */
+		pmc_soft_initialize(md);
+	}
+
+	return md;
+}
+
+void
+pmc_mdep_free(struct pmc_mdep *md)
+{
+	pmc_soft_finalize(md);
+	free(md, M_PMC);
+}
+
+static int
+generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
+{
+	(void) pc; (void) pp;
+
+	return (0);
+}
+
+static int
+generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
+{
+	(void) pc; (void) pp;
+
+	return (0);
+}
+
+static struct pmc_mdep *
+pmc_generic_cpu_initialize(void)
+{
+	struct pmc_mdep *md;
+
+	md = pmc_mdep_alloc(0);
+
+	md->pmd_cputype    = PMC_CPU_GENERIC;
+
+	md->pmd_pcpu_init  = NULL;
+	md->pmd_pcpu_fini  = NULL;
+	md->pmd_switch_in  = generic_switch_in;
+	md->pmd_switch_out = generic_switch_out;
+
+	return (md);
+}
+
+static void
+pmc_generic_cpu_finalize(struct pmc_mdep *md)
+{
+	(void) md;
+}
+
+
 static int
 pmc_initialize(void)
 {
@@ -4643,9 +4789,12 @@ pmc_initialize(void)
 	}
 
 	md = pmc_md_initialize();
-
-	if (md == NULL)
-		return (ENOSYS);
+	if (md == NULL) {
+		/* Default to generic CPU. */
+		md = pmc_generic_cpu_initialize();
+		if (md == NULL)
+			return (ENOSYS);
+        }
 
 	KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
 	    ("[pmc,%d] no classes or pmcs", __LINE__));
@@ -4717,7 +4866,25 @@ pmc_initialize(void)
 			ps->ps_pc = sb->ps_callchains +
 			    (n * pmc_callchaindepth);
 
-		pmc_pcpu[cpu]->pc_sb = sb;
+		pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
+
+		sb = malloc(sizeof(struct pmc_samplebuffer) +
+		    pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
+		    M_WAITOK|M_ZERO);
+		sb->ps_read = sb->ps_write = sb->ps_samples;
+		sb->ps_fence = sb->ps_samples + pmc_nsamples;
+
+		KASSERT(pmc_pcpu[cpu] != NULL,
+		    ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
+
+		sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *
+		    sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO);
+
+		for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
+			ps->ps_pc = sb->ps_callchains +
+			    (n * pmc_callchaindepth);
+
+		pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
 	}
 
 	/* allocate space for the row disposition array */
@@ -4887,9 +5054,12 @@ pmc_cleanup(void)
 				md->pmd_pcpu_fini(md, cpu);
 		}
 
-		pmc_md_finalize(md);
+		if (md->pmd_cputype == PMC_CPU_GENERIC)
+			pmc_generic_cpu_finalize(md);
+		else
+			pmc_md_finalize(md);
 
-		free(md, M_PMC);
+		pmc_mdep_free(md);
 		md = NULL;
 		pmc_restore_cpu_binding(&pb);
 	}
@@ -4898,11 +5068,16 @@ pmc_cleanup(void)
 	for (cpu = 0; cpu < maxcpu; cpu++) {
 		if (!pmc_cpu_is_active(cpu))
 			continue;
-		KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
-		    ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
+		KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
+		    ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
+			cpu));
+		KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
+		    ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
 			cpu));
-		free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC);
-		free(pmc_pcpu[cpu]->pc_sb, M_PMC);
+		free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
+		free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
+		free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
+		free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
 		free(pmc_pcpu[cpu], M_PMC);
 	}