Add an interface for drivers to be notified of changes to CPU frequency.

cpufreq_pre_change is called before the change, giving each driver a chance
to revoke the change.  cpufreq_post_change provides the results of the
change (success or failure).  cpufreq_levels_changed gives the unit number
of the cpufreq device whose number of available levels has changed.  Hook
in all the drivers I could find that needed it.

* TSC: update TSC frequency value.  When the available levels change, take the
highest possible level and notify the timecounter set_cputicker() of that
freq.  This gets rid of the "calcru: runtime went backwards" messages.
* identcpu: updates the sysctl hw.clockrate value
* Profiling: if profiling is active when the clock changes, let the user
know the results may be inaccurate.

Reviewed by:	bde, phk
MFC after:	1 month

3 files changed, 154 insertions, 16 deletions

diff --git a/sys/i386/i386/identcpu.c b/sys/i386/i386/identcpu.c
index 09ff188..c2f533c 100644
--- a/sys/i386/i386/identcpu.c
+++ b/sys/i386/i386/identcpu.c
@@ -45,6 +45,8 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/bus.h>
+#include <sys/cpu.h>
+#include <sys/eventhandler.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/sysctl.h>
@@ -1077,6 +1079,21 @@ identifycyrix(void)
 	write_eflags(eflags);
 }
 
+/* Update TSC freq with the value indicated by the caller. */
+static void
+tsc_freq_changed(void *arg, const struct cf_level *level, int status)
+{
+	/* If there was an error during the transition, don't do anything. */
+	if (status != 0)
+		return;
+
+	/* Total setting for this level gives the new frequency in MHz. */
+	hw_clockrate = level->total_set.freq;
+}
+
+EVENTHANDLER_DEFINE(cpufreq_post_change, tsc_freq_changed, NULL,
+    EVENTHANDLER_PRI_ANY);
+
 /*
  * Final stage of CPU identification. -- Should I check TI?
  */
diff --git a/sys/i386/i386/tsc.c b/sys/i386/i386/tsc.c
index 5c0ca72..b2de9fe 100644
--- a/sys/i386/i386/tsc.c
+++ b/sys/i386/i386/tsc.c
@@ -30,6 +30,9 @@ __FBSDID("$FreeBSD$");
 #include "opt_clock.h"
 
 #include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/cpu.h>
+#include <sys/malloc.h>
 #include <sys/systm.h>
 #include <sys/sysctl.h>
 #include <sys/time.h>
@@ -41,9 +44,12 @@ __FBSDID("$FreeBSD$");
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
 
+#include "cpufreq_if.h"
+
 uint64_t	tsc_freq;
 int		tsc_is_broken;
 u_int		tsc_present;
+static eventhandler_tag tsc_levels_tag, tsc_pre_tag, tsc_post_tag;
 
 #ifdef SMP
 static int	smp_tsc;
@@ -52,14 +58,19 @@ SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc, CTLFLAG_RDTUN, &smp_tsc, 0,
 TUNABLE_INT("kern.timecounter.smp_tsc", &smp_tsc);
 #endif
 
+static void tsc_freq_changed(void *arg, const struct cf_level *level,
+    int status);
+static void tsc_freq_changing(void *arg, const struct cf_level *level,
+    int *status);
 static	unsigned tsc_get_timecount(struct timecounter *tc);
+static void tsc_levels_changed(void *arg, int unit);
 
 static struct timecounter tsc_timecounter = {
 	tsc_get_timecount,	/* get_timecount */
 	0,			/* no poll_pps */
- 	~0u,			/* counter_mask */
+	~0u,			/* counter_mask */
 	0,			/* frequency */
-	 "TSC",			/* name */
+	"TSC",			/* name */
 	800,			/* quality (adjusted in code) */
 };
 
@@ -86,9 +97,23 @@ init_TSC(void)
 	tsc_freq = tscval[1] - tscval[0];
 	if (bootverbose)
 		printf("TSC clock: %ju Hz\n", (intmax_t)tsc_freq);
+
+	/*
+	 * Inform CPU accounting about our boot-time clock rate.  Once the
+	 * system is finished booting, we will get the real max clock rate
+	 * via tsc_freq_max().  This also will be updated if someone loads
+	 * a cpufreq driver after boot that discovers a new max frequency.
+	 */
 	set_cputicker(rdtsc, tsc_freq, 1);
-}
 
+	/* Register to find out about changes in CPU frequency. */
+	tsc_pre_tag = EVENTHANDLER_REGISTER(cpufreq_pre_change,
+	    tsc_freq_changing, NULL, EVENTHANDLER_PRI_FIRST);
+	tsc_post_tag = EVENTHANDLER_REGISTER(cpufreq_post_change,
+	    tsc_freq_changed, NULL, EVENTHANDLER_PRI_FIRST);
+	tsc_levels_tag = EVENTHANDLER_REGISTER(cpufreq_levels_changed,
+	    tsc_levels_changed, NULL, EVENTHANDLER_PRI_ANY);
+}
 
 void
 init_TSC_tc(void)
@@ -128,6 +153,72 @@ init_TSC_tc(void)
 	}
 }
 
+/*
+ * When cpufreq levels change, find out about the (new) max frequency.  We
+ * use this to update CPU accounting in case it got a lower estimate at boot.
+ */
+static void
+tsc_levels_changed(void *arg, int unit)
+{
+	device_t cf_dev;
+	struct cf_level *levels;
+	int count, error;
+	uint64_t max_freq;
+
+	/* Only use values from the first CPU, assuming all are equal. */
+	if (unit != 0)
+		return;
+
+	/* Find the appropriate cpufreq device instance. */
+	cf_dev = devclass_get_device(devclass_find("cpufreq"), unit);
+	if (cf_dev == NULL) {
+		printf("tsc_levels_changed() called but no cpufreq device?\n");
+		return;
+	}
+
+	/* Get settings from the device and find the max frequency. */
+	count = 64;
+	levels = malloc(count * sizeof(*levels), M_TEMP, M_NOWAIT);
+	if (levels == NULL)
+		return;
+	error = CPUFREQ_LEVELS(cf_dev, levels, &count);
+	if (error == 0 && count != 0) {
+		max_freq = (uint64_t)levels[0].total_set.freq * 1000000;
+		set_cputicker(rdtsc, max_freq, 1);
+	} else
+		printf("tsc_levels_changed: no max freq found\n");
+	free(levels, M_TEMP);
+}
+
+/*
+ * If the TSC timecounter is in use, veto the pending change.  It may be
+ * possible in the future to handle a dynamically-changing timecounter rate.
+ */
+static void
+tsc_freq_changing(void *arg, const struct cf_level *level, int *status)
+{
+
+	if (*status != 0 || timecounter != &tsc_timecounter)
+		return;
+
+	printf("timecounter TSC must not be in use when "
+	     "changing frequencies; change denied\n");
+	*status = EBUSY;
+}
+
+/* Update TSC freq with the value indicated by the caller. */
+static void
+tsc_freq_changed(void *arg, const struct cf_level *level, int status)
+{
+	/* If there was an error during the transition, don't do anything. */
+	if (status != 0)
+		return;
+
+	/* Total setting for this level gives the new frequency in MHz. */
+	tsc_freq = (uint64_t)level->total_set.freq * 1000000;
+	tsc_timecounter.tc_frequency = tsc_freq;
+}
+
 static int
 sysctl_machdep_tsc_freq(SYSCTL_HANDLER_ARGS)
 {
diff --git a/sys/i386/isa/prof_machdep.c b/sys/i386/isa/prof_machdep.c
index 2ad00bd..855745f 100644
--- a/sys/i386/isa/prof_machdep.c
+++ b/sys/i386/isa/prof_machdep.c
@@ -33,6 +33,9 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/cpu.h>
+#include <sys/eventhandler.h>
 #include <sys/gmon.h>
 #include <sys/kernel.h>
 #include <sys/smp.h>
@@ -56,6 +59,9 @@ static u_int	cputime_clock_pmc_conf = I586_PMC_GUPROF;
 static int	cputime_clock_pmc_init;
 static struct gmonparam saved_gmp;
 #endif
+#if defined(I586_CPU) || defined(I686_CPU)
+static int	cputime_prof_active;
+#endif
 #endif /* GUPROF */
 
 #ifdef __GNUCLIKE_ASM
@@ -72,19 +78,19 @@ __mcount:							\n\
 	# Check that we are profiling.  Do it early for speed.	\n\
 	#							\n\
 	cmpl	$GMON_PROF_OFF,_gmonparam+GM_STATE		\n\
- 	je	.mcount_exit					\n\
- 	#							\n\
- 	# __mcount is the same as [.]mcount except the caller	\n\
- 	# hasn't changed the stack except to call here, so the	\n\
+	je	.mcount_exit					\n\
+	#							\n\
+	# __mcount is the same as [.]mcount except the caller	\n\
+	# hasn't changed the stack except to call here, so the	\n\
 	# caller's raddr is above our raddr.			\n\
 	#							\n\
- 	movl	4(%esp),%edx					\n\
- 	jmp	.got_frompc					\n\
- 								\n\
- 	.p2align 4,0x90						\n\
- 	.globl	.mcount						\n\
+	movl	4(%esp),%edx					\n\
+	jmp	.got_frompc					\n\
+								\n\
+	.p2align 4,0x90						\n\
+	.globl	.mcount						\n\
 .mcount:							\n\
- 	.globl	__cyg_profile_func_enter			\n\
+	.globl	__cyg_profile_func_enter			\n\
 __cyg_profile_func_enter:					\n\
 	cmpl	$GMON_PROF_OFF,_gmonparam+GM_STATE		\n\
 	je	.mcount_exit					\n\
@@ -139,7 +145,7 @@ GMON_PROF_HIRES	=	4					\n\
 	.p2align 4,0x90						\n\
 	.globl	.mexitcount					\n\
 .mexitcount:							\n\
- 	.globl	__cyg_profile_func_exit				\n\
+	.globl	__cyg_profile_func_exit				\n\
 __cyg_profile_func_exit:					\n\
 	cmpl	$GMON_PROF_HIRES,_gmonparam+GM_STATE		\n\
 	jne	.mexitcount_exit				\n\
@@ -287,14 +293,16 @@ startguprof(gp)
 	if (cputime_clock == CPUTIME_CLOCK_UNINITIALIZED) {
 		cputime_clock = CPUTIME_CLOCK_I8254;
 #if defined(I586_CPU) || defined(I686_CPU)
-		if (tsc_freq != 0 && !tsc_is_broken && mp_ncpus < 2)
+		if (tsc_freq != 0 && !tsc_is_broken && mp_ncpus == 1)
 			cputime_clock = CPUTIME_CLOCK_TSC;
 #endif
 	}
 	gp->profrate = timer_freq << CPUTIME_CLOCK_I8254_SHIFT;
 #if defined(I586_CPU) || defined(I686_CPU)
-	if (cputime_clock == CPUTIME_CLOCK_TSC)
+	if (cputime_clock == CPUTIME_CLOCK_TSC) {
 		gp->profrate = tsc_freq >> 1;
+		cputime_prof_active = 1;
+	}
 #if defined(PERFMON) && defined(I586_PMC_GUPROF)
 	else if (cputime_clock == CPUTIME_CLOCK_I586_PMC) {
 		if (perfmon_avail() &&
@@ -337,5 +345,27 @@ stopguprof(gp)
 		cputime_clock_pmc_init = FALSE;
 	}
 #endif
+#if defined(I586_CPU) || defined(I686_CPU)
+	if (cputime_clock == CPUTIME_CLOCK_TSC)
+		cputime_prof_active = 0;
+#endif
+}
+
+#if defined(I586_CPU) || defined(I686_CPU)
+/* If the cpu frequency changed while profiling, report a warning. */
+static void
+tsc_freq_changed(void *arg, const struct cf_level *level, int status)
+{
+
+	/* If there was an error during the transition, don't do anything. */
+	if (status != 0)
+		return;
+	if (cputime_prof_active && cputime_clock == CPUTIME_CLOCK_TSC)
+		printf("warning: cpu freq changed while profiling active\n");
 }
+
+EVENTHANDLER_DEFINE(cpufreq_post_change, tsc_freq_changed, NULL,
+    EVENTHANDLER_PRI_ANY);
+#endif /* I586_CPU || I686_CPU */
+
 #endif /* GUPROF */