Commit | Line | Data |
---|---|---|
c37e7bb5 | 1 | #include <linux/kernel.h> |
2 | #include <linux/sched.h> | |
3 | #include <linux/interrupt.h> | |
4 | #include <linux/init.h> | |
5 | #include <linux/clocksource.h> | |
6 | #include <linux/time.h> | |
7 | #include <linux/acpi.h> | |
8 | #include <linux/cpufreq.h> | |
d371698e | 9 | #include <linux/acpi_pmtmr.h> |
c37e7bb5 | 10 | |
d371698e | 11 | #include <asm/hpet.h> |
c37e7bb5 | 12 | #include <asm/timex.h> |
13 | ||
1489939f | 14 | static int notsc __initdata = 0; |
c37e7bb5 | 15 | |
16 | unsigned int cpu_khz; /* TSC clocks / usec, not used here */ | |
17 | EXPORT_SYMBOL(cpu_khz); | |
6b37f5a2 JR |
18 | unsigned int tsc_khz; |
19 | EXPORT_SYMBOL(tsc_khz); | |
c37e7bb5 | 20 | |
c37e7bb5 | 21 | static unsigned int cyc2ns_scale __read_mostly; |
22 | ||
2f0798a3 | 23 | static inline void set_cyc2ns_scale(unsigned long khz) |
c37e7bb5 | 24 | { |
25 | cyc2ns_scale = (NSEC_PER_MSEC << NS_SCALE) / khz; | |
26 | } | |
27 | ||
1489939f | 28 | static unsigned long long cycles_2_ns(unsigned long long cyc) |
c37e7bb5 | 29 | { |
30 | return (cyc * cyc2ns_scale) >> NS_SCALE; | |
31 | } | |
32 | ||
33 | unsigned long long sched_clock(void) | |
34 | { | |
35 | unsigned long a = 0; | |
36 | ||
37 | /* Could do CPU core sync here. Opteron can execute rdtsc speculatively, | |
38 | * which means it is not completely exact and may not be monotonous | |
39 | * between CPUs. But the errors should be too small to matter for | |
40 | * scheduling purposes. | |
41 | */ | |
42 | ||
43 | rdtscll(a); | |
44 | return cycles_2_ns(a); | |
45 | } | |
46 | ||
1489939f | 47 | static int tsc_unstable; |
48 | ||
d7e28ffe | 49 | inline int check_tsc_unstable(void) |
1489939f | 50 | { |
51 | return tsc_unstable; | |
52 | } | |
c37e7bb5 | 53 | #ifdef CONFIG_CPU_FREQ |
54 | ||
55 | /* Frequency scaling support. Adjust the TSC based timer when the cpu frequency | |
56 | * changes. | |
57 | * | |
58 | * RED-PEN: On SMP we assume all CPUs run with the same frequency. It's | |
59 | * not that important because current Opteron setups do not support | |
60 | * scaling on SMP anyroads. | |
61 | * | |
62 | * Should fix up last_tsc too. Currently gettimeofday in the | |
63 | * first tick after the change will be slightly wrong. | |
64 | */ | |
65 | ||
7ff98478 TG |
66 | static unsigned int ref_freq; |
67 | static unsigned long loops_per_jiffy_ref; | |
68 | static unsigned long tsc_khz_ref; | |
c37e7bb5 | 69 | |
70 | static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, | |
71 | void *data) | |
72 | { | |
73 | struct cpufreq_freqs *freq = data; | |
74 | unsigned long *lpj, dummy; | |
75 | ||
76 | if (cpu_has(&cpu_data[freq->cpu], X86_FEATURE_CONSTANT_TSC)) | |
77 | return 0; | |
78 | ||
79 | lpj = &dummy; | |
80 | if (!(freq->flags & CPUFREQ_CONST_LOOPS)) | |
81 | #ifdef CONFIG_SMP | |
82 | lpj = &cpu_data[freq->cpu].loops_per_jiffy; | |
83 | #else | |
84 | lpj = &boot_cpu_data.loops_per_jiffy; | |
85 | #endif | |
86 | ||
87 | if (!ref_freq) { | |
88 | ref_freq = freq->old; | |
89 | loops_per_jiffy_ref = *lpj; | |
6b37f5a2 | 90 | tsc_khz_ref = tsc_khz; |
c37e7bb5 | 91 | } |
92 | if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || | |
93 | (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || | |
94 | (val == CPUFREQ_RESUMECHANGE)) { | |
95 | *lpj = | |
96 | cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new); | |
97 | ||
6b37f5a2 | 98 | tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new); |
c37e7bb5 | 99 | if (!(freq->flags & CPUFREQ_CONST_LOOPS)) |
5a90cf20 | 100 | mark_tsc_unstable("cpufreq changes"); |
c37e7bb5 | 101 | } |
102 | ||
6b37f5a2 | 103 | set_cyc2ns_scale(tsc_khz_ref); |
c37e7bb5 | 104 | |
105 | return 0; | |
106 | } | |
107 | ||
108 | static struct notifier_block time_cpufreq_notifier_block = { | |
109 | .notifier_call = time_cpufreq_notifier | |
110 | }; | |
111 | ||
112 | static int __init cpufreq_tsc(void) | |
113 | { | |
7ff98478 TG |
114 | cpufreq_register_notifier(&time_cpufreq_notifier_block, |
115 | CPUFREQ_TRANSITION_NOTIFIER); | |
c37e7bb5 | 116 | return 0; |
117 | } | |
118 | ||
119 | core_initcall(cpufreq_tsc); | |
120 | ||
121 | #endif | |
122 | ||
d371698e TG |
123 | #define MAX_RETRIES 5 |
124 | #define SMI_TRESHOLD 50000 | |
125 | ||
126 | /* | |
127 | * Read TSC and the reference counters. Take care of SMI disturbance | |
128 | */ | |
129 | static unsigned long __init tsc_read_refs(unsigned long *pm, | |
130 | unsigned long *hpet) | |
131 | { | |
132 | unsigned long t1, t2; | |
133 | int i; | |
134 | ||
135 | for (i = 0; i < MAX_RETRIES; i++) { | |
136 | t1 = get_cycles_sync(); | |
137 | if (hpet) | |
138 | *hpet = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF; | |
139 | else | |
140 | *pm = acpi_pm_read_early(); | |
141 | t2 = get_cycles_sync(); | |
142 | if ((t2 - t1) < SMI_TRESHOLD) | |
143 | return t2; | |
144 | } | |
145 | return ULONG_MAX; | |
146 | } | |
147 | ||
148 | /** | |
149 | * tsc_calibrate - calibrate the tsc on boot | |
150 | */ | |
151 | void __init tsc_calibrate(void) | |
152 | { | |
153 | unsigned long flags, tsc1, tsc2, tr1, tr2, pm1, pm2, hpet1, hpet2; | |
154 | int hpet = is_hpet_enabled(); | |
155 | ||
156 | local_irq_save(flags); | |
157 | ||
158 | tsc1 = tsc_read_refs(&pm1, hpet ? &hpet1 : NULL); | |
159 | ||
160 | outb((inb(0x61) & ~0x02) | 0x01, 0x61); | |
161 | ||
162 | outb(0xb0, 0x43); | |
163 | outb((CLOCK_TICK_RATE / (1000 / 50)) & 0xff, 0x42); | |
164 | outb((CLOCK_TICK_RATE / (1000 / 50)) >> 8, 0x42); | |
165 | tr1 = get_cycles_sync(); | |
166 | while ((inb(0x61) & 0x20) == 0); | |
167 | tr2 = get_cycles_sync(); | |
168 | ||
169 | tsc2 = tsc_read_refs(&pm2, hpet ? &hpet2 : NULL); | |
170 | ||
171 | local_irq_restore(flags); | |
172 | ||
173 | /* | |
174 | * Preset the result with the raw and inaccurate PIT | |
175 | * calibration value | |
176 | */ | |
177 | tsc_khz = (tr2 - tr1) / 50; | |
178 | ||
179 | /* hpet or pmtimer available ? */ | |
180 | if (!hpet && !pm1 && !pm2) { | |
181 | printk(KERN_INFO "TSC calibrated against PIT\n"); | |
182 | return; | |
183 | } | |
184 | ||
185 | /* Check, whether the sampling was disturbed by an SMI */ | |
186 | if (tsc1 == ULONG_MAX || tsc2 == ULONG_MAX) { | |
187 | printk(KERN_WARNING "TSC calibration disturbed by SMI, " | |
188 | "using PIT calibration result\n"); | |
189 | return; | |
190 | } | |
191 | ||
192 | tsc2 = (tsc2 - tsc1) * 1000000L; | |
193 | ||
194 | if (hpet) { | |
195 | printk(KERN_INFO "TSC calibrated against HPET\n"); | |
196 | if (hpet2 < hpet1) | |
197 | hpet2 += 0x100000000; | |
198 | hpet2 -= hpet1; | |
199 | tsc1 = (hpet2 * hpet_readl(HPET_PERIOD)) / 1000000; | |
200 | } else { | |
201 | printk(KERN_INFO "TSC calibrated against PM_TIMER\n"); | |
202 | if (pm2 < pm1) | |
203 | pm2 += ACPI_PM_OVRRUN; | |
204 | pm2 -= pm1; | |
205 | tsc1 = (pm2 * 1000000000) / PMTMR_TICKS_PER_SEC; | |
206 | } | |
207 | ||
208 | tsc_khz = tsc2 / tsc1; | |
2f0798a3 | 209 | set_cyc2ns_scale(tsc_khz); |
d371698e TG |
210 | } |
211 | ||
c37e7bb5 | 212 | /* |
213 | * Make an educated guess if the TSC is trustworthy and synchronized | |
214 | * over all CPUs. | |
215 | */ | |
216 | __cpuinit int unsynchronized_tsc(void) | |
217 | { | |
218 | if (tsc_unstable) | |
219 | return 1; | |
220 | ||
221 | #ifdef CONFIG_SMP | |
222 | if (apic_is_clustered_box()) | |
223 | return 1; | |
224 | #endif | |
225 | /* Most intel systems have synchronized TSCs except for | |
226 | multi node systems */ | |
7ff98478 | 227 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { |
c37e7bb5 | 228 | #ifdef CONFIG_ACPI |
229 | /* But TSC doesn't tick in C3 so don't use it there */ | |
7ff98478 TG |
230 | if (acpi_gbl_FADT.header.length > 0 && |
231 | acpi_gbl_FADT.C3latency < 1000) | |
c37e7bb5 | 232 | return 1; |
233 | #endif | |
7ff98478 | 234 | return 0; |
c37e7bb5 | 235 | } |
236 | ||
7ff98478 TG |
237 | /* Assume multi socket systems are not synchronized */ |
238 | return num_present_cpus() > 1; | |
c37e7bb5 | 239 | } |
240 | ||
241 | int __init notsc_setup(char *s) | |
242 | { | |
243 | notsc = 1; | |
244 | return 1; | |
245 | } | |
246 | ||
247 | __setup("notsc", notsc_setup); | |
1489939f | 248 | |
249 | ||
250 | /* clock source code: */ | |
251 | static cycle_t read_tsc(void) | |
252 | { | |
253 | cycle_t ret = (cycle_t)get_cycles_sync(); | |
254 | return ret; | |
255 | } | |
256 | ||
7460ed28 | 257 | static cycle_t __vsyscall_fn vread_tsc(void) |
258 | { | |
259 | cycle_t ret = (cycle_t)get_cycles_sync(); | |
260 | return ret; | |
261 | } | |
262 | ||
1489939f | 263 | static struct clocksource clocksource_tsc = { |
264 | .name = "tsc", | |
265 | .rating = 300, | |
266 | .read = read_tsc, | |
267 | .mask = CLOCKSOURCE_MASK(64), | |
268 | .shift = 22, | |
269 | .flags = CLOCK_SOURCE_IS_CONTINUOUS | | |
270 | CLOCK_SOURCE_MUST_VERIFY, | |
7460ed28 | 271 | .vread = vread_tsc, |
1489939f | 272 | }; |
273 | ||
5a90cf20 | 274 | void mark_tsc_unstable(char *reason) |
1489939f | 275 | { |
276 | if (!tsc_unstable) { | |
277 | tsc_unstable = 1; | |
5a90cf20 | 278 | printk("Marking TSC unstable due to %s\n", reason); |
1489939f | 279 | /* Change only the rating, when not registered */ |
280 | if (clocksource_tsc.mult) | |
281 | clocksource_change_rating(&clocksource_tsc, 0); | |
282 | else | |
283 | clocksource_tsc.rating = 0; | |
284 | } | |
285 | } | |
286 | EXPORT_SYMBOL_GPL(mark_tsc_unstable); | |
287 | ||
6bb74df4 | 288 | void __init init_tsc_clocksource(void) |
1489939f | 289 | { |
290 | if (!notsc) { | |
6b37f5a2 | 291 | clocksource_tsc.mult = clocksource_khz2mult(tsc_khz, |
1489939f | 292 | clocksource_tsc.shift); |
293 | if (check_tsc_unstable()) | |
294 | clocksource_tsc.rating = 0; | |
295 | ||
6bb74df4 | 296 | clocksource_register(&clocksource_tsc); |
1489939f | 297 | } |
1489939f | 298 | } |