1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
/*
* check TSC synchronization.
*
* Copyright (C) 2006, Red Hat, Inc., Ingo Molnar
*
* We check whether all boot CPUs have their TSC's synchronized,
* print a warning if not and turn off the TSC clock-source.
*
* The warp-check is point-to-point between two CPUs, the CPU
* initiating the bootup is the 'source CPU', the freshly booting
* CPU is the 'target CPU'.
*
* Only two CPUs may participate - they can enter in any order.
* ( The serial nature of the boot logic and the CPU hotplug lock
* protects against more than 2 CPUs entering this code. )
*/
#include <linux/spinlock.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/nmi.h>
#include <asm/tsc.h>
/*
* Entry/exit counters that make sure that both CPUs
* run the measurement code at once:
*/
static __cpuinitdata atomic_t start_count;
static __cpuinitdata atomic_t stop_count;
/*
* We use a raw spinlock in this exceptional case, because
* we want to have the fastest, inlined, non-debug version
* of a critical section, to be able to prove TSC time-warps:
*/
static __cpuinitdata raw_spinlock_t sync_lock = __RAW_SPIN_LOCK_UNLOCKED;
static __cpuinitdata cycles_t last_tsc;
static __cpuinitdata cycles_t max_warp;
static __cpuinitdata int nr_warps;
/*
* TSC-warp measurement loop running on both CPUs:
*/
static __cpuinit void check_tsc_warp(void)
{
cycles_t start, now, prev, end;
int i;
rdtsc_barrier();
start = get_cycles();
rdtsc_barrier();
/*
* The measurement runs for 20 msecs:
*/
end = start + tsc_khz * 20ULL;
now = start;
for (i = 0; ; i++) {
/*
* We take the global lock, measure TSC, save the
* previous TSC that was measured (possibly on
* another CPU) and update the previous TSC timestamp.
*/
__raw_spin_lock(&sync_lock);
prev = last_tsc;
rdtsc_barrier();
now = get_cycles();
rdtsc_barrier();
last_tsc = now;
__raw_spin_unlock(&sync_lock);
/*
* Be nice every now and then (and also check whether
* measurement is done [we also insert a 10 million
* loops safety exit, so we dont lock up in case the
* TSC readout is totally broken]):
*/
if (unlikely(!(i & 7))) {
if (now > end || i > 10000000)
break;
cpu_relax();
touch_nmi_watchdog();
}
/*
* Outside the critical section we can now see whether
* we saw a time-warp of the TSC going backwards:
*/
if (unlikely(prev > now)) {
__raw_spin_lock(&sync_lock);
max_warp = max(max_warp, prev - now);
nr_warps++;
__raw_spin_unlock(&sync_lock);
}
}
WARN(!(now-start),
"Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
now-start, end-start);
}
/*
* Source CPU calls into this - it waits for the freshly booted
* target CPU to arrive and then starts the measurement:
*/
void __cpuinit check_tsc_sync_source(int cpu)
{
int cpus = 2;
/*
* No need to check if we already know that the TSC is not
* synchronized:
*/
if (unsynchronized_tsc())
return;
if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE)) {
if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING)
pr_info(
"Skipped synchronization checks as TSC is reliable.\n");
return;
}
/*
* Reset it - in case this is a second bootup:
*/
atomic_set(&stop_count, 0);
/*
* Wait for the target to arrive:
*/
while (atomic_read(&start_count) != cpus-1)
cpu_relax();
/*
* Trigger the target to continue into the measurement too:
*/
atomic_inc(&start_count);
check_tsc_warp();
while (atomic_read(&stop_count) != cpus-1)
cpu_relax();
if (nr_warps) {
pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n",
smp_processor_id(), cpu);
pr_warning("Measured %Ld cycles TSC warp between CPUs, "
"turning off TSC clock.\n", max_warp);
mark_tsc_unstable("check_tsc_sync_source failed");
} else {
pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
smp_processor_id(), cpu);
}
/*
* Reset it - just in case we boot another CPU later:
*/
atomic_set(&start_count, 0);
nr_warps = 0;
max_warp = 0;
last_tsc = 0;
/*
* Let the target continue with the bootup:
*/
atomic_inc(&stop_count);
}
/*
* Freshly booted CPUs call into this:
*/
void __cpuinit check_tsc_sync_target(void)
{
int cpus = 2;
if (unsynchronized_tsc() || boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
return;
/*
* Register this CPU's participation and wait for the
* source CPU to start the measurement:
*/
atomic_inc(&start_count);
while (atomic_read(&start_count) != cpus)
cpu_relax();
check_tsc_warp();
/*
* Ok, we are done:
*/
atomic_inc(&stop_count);
/*
* Wait for the source CPU to print stuff:
*/
while (atomic_read(&stop_count) != cpus)
cpu_relax();
}
|
