Use atomic load & store for TSC frequency. It may be overkill for amd64 but

safer for i386 because it can be easily over 4 GHz now.  More worse, it can
be easily changed by user with 'machdep.tsc_freq' tunable (directly) or
cpufreq(4) (indirectly).  Note it is intentionally not used in performance
critical paths to avoid performance regression (but we should, in theory).
Alternatively, we may add "virtual TSC" with lower frequency if maximum
frequency overflows 32 bits (and ignore possible incoherency as we do now).

3 files changed, 15 insertions, 11 deletions

diff --git a/sys/x86/cpufreq/est.c b/sys/x86/cpufreq/est.c
index 678efb6..cc9502d 100644
--- a/sys/x86/cpufreq/est.c
+++ b/sys/x86/cpufreq/est.c
@@ -1215,7 +1215,7 @@ est_msr_info(device_t dev, uint64_t msr, freq_info **freqs)
 		return (EOPNOTSUPP);
 
 	/* Figure out the bus clock. */
-	freq = tsc_freq / 1000000;
+	freq = atomic_load_acq_64(&tsc_freq) / 1000000;
 	id = msr >> 32;
 	bus = freq / (id >> 8);
 	device_printf(dev, "Guessed bus clock (high) of %d MHz\n", bus);
diff --git a/sys/x86/isa/clock.c b/sys/x86/isa/clock.c
index f0016b2..87d906a 100644
--- a/sys/x86/isa/clock.c
+++ b/sys/x86/isa/clock.c
@@ -246,13 +246,13 @@ getit(void)
 }
 
 static __inline void
-delay_tsc(int n)
+delay_tsc(int n, uint64_t freq)
 {
 	uint64_t start, end, now;
 
 	sched_pin();
 	start = rdtsc();
-	end = start + (tsc_freq * n) / 1000000;
+	end = start + (freq * n) / 1000000;
 	do {
 		cpu_spinwait();
 		now = rdtsc();
@@ -290,6 +290,7 @@ void
 DELAY(int n)
 {
 	struct timecounter *tc;
+	uint64_t freq;
 	int delta, prev_tick, tick, ticks_left;
 
 #ifdef DELAYDEBUG
@@ -298,8 +299,9 @@ DELAY(int n)
 	static int state = 0;
 #endif
 
-	if (tsc_freq != 0) {
-		delay_tsc(n);
+	freq = atomic_load_acq_64(&tsc_freq);
+	if (freq != 0) {
+		delay_tsc(n, freq);
 		return;
 	}
 	tc = timecounter;
diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
index 1781a78..2b1c89a 100644
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -245,14 +245,16 @@ tsc_freq_changing(void *arg, const struct cf_level *level, int *status)
 static void
 tsc_freq_changed(void *arg, const struct cf_level *level, int status)
 {
+	uint64_t freq;
 
 	/* If there was an error during the transition, don't do anything. */
 	if (tsc_disabled || status != 0)
 		return;
 
 	/* Total setting for this level gives the new frequency in MHz. */
-	tsc_freq = (uint64_t)level->total_set.freq * 1000000;
-	tsc_timecounter.tc_frequency = tsc_freq;
+	freq = (uint64_t)level->total_set.freq * 1000000;
+	atomic_store_rel_64(&tsc_freq, freq);
+	atomic_store_rel_64(&tsc_timecounter.tc_frequency, freq);
 }
 
 static int
@@ -261,13 +263,13 @@ sysctl_machdep_tsc_freq(SYSCTL_HANDLER_ARGS)
 	int error;
 	uint64_t freq;
 
-	if (tsc_timecounter.tc_frequency == 0)
+	freq = atomic_load_acq_64(&tsc_freq);
+	if (freq == 0)
 		return (EOPNOTSUPP);
-	freq = tsc_freq;
 	error = sysctl_handle_64(oidp, &freq, 0, req);
 	if (error == 0 && req->newptr != NULL) {
-		tsc_freq = freq;
-		tsc_timecounter.tc_frequency = tsc_freq;
+		atomic_store_rel_64(&tsc_freq, freq);
+		atomic_store_rel_64(&tsc_timecounter.tc_frequency, freq);
 	}
 	return (error);
 }