Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * linux/arch/x86_64/nmi.c | |
3 | * | |
4 | * NMI watchdog support on APIC systems | |
5 | * | |
6 | * Started by Ingo Molnar <mingo@redhat.com> | |
7 | * | |
8 | * Fixes: | |
9 | * Mikael Pettersson : AMD K7 support for local APIC NMI watchdog. | |
10 | * Mikael Pettersson : Power Management for local APIC NMI watchdog. | |
11 | * Pavel Machek and | |
12 | * Mikael Pettersson : PM converted to driver model. Disable/enable API. | |
13 | */ | |
14 | ||
1da177e4 | 15 | #include <linux/mm.h> |
1da177e4 | 16 | #include <linux/delay.h> |
1da177e4 | 17 | #include <linux/interrupt.h> |
1da177e4 LT |
18 | #include <linux/module.h> |
19 | #include <linux/sysdev.h> | |
20 | #include <linux/nmi.h> | |
21 | #include <linux/sysctl.h> | |
eddb6fb9 | 22 | #include <linux/kprobes.h> |
1da177e4 LT |
23 | |
24 | #include <asm/smp.h> | |
1da177e4 | 25 | #include <asm/nmi.h> |
1da177e4 LT |
26 | #include <asm/proto.h> |
27 | #include <asm/kdebug.h> | |
553f265f | 28 | #include <asm/mce.h> |
248dcb2f | 29 | #include <asm/intel_arch_perfmon.h> |
1da177e4 | 30 | |
828f0afd DZ |
31 | /* perfctr_nmi_owner tracks the ownership of the perfctr registers: |
32 | * evtsel_nmi_owner tracks the ownership of the event selection | |
33 | * - different performance counters/ event selection may be reserved for | |
34 | * different subsystems this reservation system just tries to coordinate | |
35 | * things a little | |
36 | */ | |
37 | static DEFINE_PER_CPU(unsigned, perfctr_nmi_owner); | |
38 | static DEFINE_PER_CPU(unsigned, evntsel_nmi_owner[2]); | |
39 | ||
40 | /* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's | |
41 | * offset from MSR_P4_BSU_ESCR0. It will be the max for all platforms (for now) | |
42 | */ | |
43 | #define NMI_MAX_COUNTER_BITS 66 | |
44 | ||
1da177e4 | 45 | /* nmi_active: |
f2802e7f DZ |
46 | * >0: the lapic NMI watchdog is active, but can be disabled |
47 | * <0: the lapic NMI watchdog has not been set up, and cannot | |
1da177e4 | 48 | * be enabled |
f2802e7f | 49 | * 0: the lapic NMI watchdog is disabled, but can be enabled |
1da177e4 | 50 | */ |
f2802e7f | 51 | atomic_t nmi_active = ATOMIC_INIT(0); /* oprofile uses this */ |
1da177e4 LT |
52 | int panic_on_timeout; |
53 | ||
54 | unsigned int nmi_watchdog = NMI_DEFAULT; | |
55 | static unsigned int nmi_hz = HZ; | |
1da177e4 | 56 | |
f2802e7f DZ |
57 | struct nmi_watchdog_ctlblk { |
58 | int enabled; | |
59 | u64 check_bit; | |
60 | unsigned int cccr_msr; | |
61 | unsigned int perfctr_msr; /* the MSR to reset in NMI handler */ | |
62 | unsigned int evntsel_msr; /* the MSR to select the events to handle */ | |
63 | }; | |
64 | static DEFINE_PER_CPU(struct nmi_watchdog_ctlblk, nmi_watchdog_ctlblk); | |
1da177e4 | 65 | |
f2802e7f | 66 | /* local prototypes */ |
f2802e7f | 67 | static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu); |
75152114 | 68 | |
828f0afd DZ |
69 | /* converts an msr to an appropriate reservation bit */ |
70 | static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr) | |
71 | { | |
72 | /* returns the bit offset of the performance counter register */ | |
73 | switch (boot_cpu_data.x86_vendor) { | |
74 | case X86_VENDOR_AMD: | |
75 | return (msr - MSR_K7_PERFCTR0); | |
76 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
77 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
78 | return (msr - MSR_ARCH_PERFMON_PERFCTR0); | |
79 | else | |
80 | return (msr - MSR_P4_BPU_PERFCTR0); | |
828f0afd DZ |
81 | } |
82 | return 0; | |
83 | } | |
84 | ||
85 | /* converts an msr to an appropriate reservation bit */ | |
86 | static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr) | |
87 | { | |
88 | /* returns the bit offset of the event selection register */ | |
89 | switch (boot_cpu_data.x86_vendor) { | |
90 | case X86_VENDOR_AMD: | |
91 | return (msr - MSR_K7_EVNTSEL0); | |
92 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
93 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
94 | return (msr - MSR_ARCH_PERFMON_EVENTSEL0); | |
95 | else | |
96 | return (msr - MSR_P4_BSU_ESCR0); | |
828f0afd DZ |
97 | } |
98 | return 0; | |
99 | } | |
100 | ||
101 | /* checks for a bit availability (hack for oprofile) */ | |
102 | int avail_to_resrv_perfctr_nmi_bit(unsigned int counter) | |
103 | { | |
104 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
105 | ||
106 | return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner))); | |
107 | } | |
108 | ||
109 | /* checks the an msr for availability */ | |
110 | int avail_to_resrv_perfctr_nmi(unsigned int msr) | |
111 | { | |
112 | unsigned int counter; | |
113 | ||
114 | counter = nmi_perfctr_msr_to_bit(msr); | |
115 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
116 | ||
117 | return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner))); | |
118 | } | |
119 | ||
120 | int reserve_perfctr_nmi(unsigned int msr) | |
121 | { | |
122 | unsigned int counter; | |
123 | ||
124 | counter = nmi_perfctr_msr_to_bit(msr); | |
125 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
126 | ||
127 | if (!test_and_set_bit(counter, &__get_cpu_var(perfctr_nmi_owner))) | |
128 | return 1; | |
129 | return 0; | |
130 | } | |
131 | ||
132 | void release_perfctr_nmi(unsigned int msr) | |
133 | { | |
134 | unsigned int counter; | |
135 | ||
136 | counter = nmi_perfctr_msr_to_bit(msr); | |
137 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
138 | ||
139 | clear_bit(counter, &__get_cpu_var(perfctr_nmi_owner)); | |
140 | } | |
141 | ||
142 | int reserve_evntsel_nmi(unsigned int msr) | |
143 | { | |
144 | unsigned int counter; | |
145 | ||
146 | counter = nmi_evntsel_msr_to_bit(msr); | |
147 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
148 | ||
149 | if (!test_and_set_bit(counter, &__get_cpu_var(evntsel_nmi_owner))) | |
150 | return 1; | |
151 | return 0; | |
152 | } | |
153 | ||
154 | void release_evntsel_nmi(unsigned int msr) | |
155 | { | |
156 | unsigned int counter; | |
157 | ||
158 | counter = nmi_evntsel_msr_to_bit(msr); | |
159 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
160 | ||
161 | clear_bit(counter, &__get_cpu_var(evntsel_nmi_owner)); | |
162 | } | |
163 | ||
e6982c67 | 164 | static __cpuinit inline int nmi_known_cpu(void) |
75152114 AK |
165 | { |
166 | switch (boot_cpu_data.x86_vendor) { | |
167 | case X86_VENDOR_AMD: | |
168 | return boot_cpu_data.x86 == 15; | |
169 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
170 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
171 | return 1; | |
172 | else | |
173 | return (boot_cpu_data.x86 == 15); | |
75152114 AK |
174 | } |
175 | return 0; | |
176 | } | |
1da177e4 LT |
177 | |
178 | /* Run after command line and cpu_init init, but before all other checks */ | |
e33e89ab | 179 | void nmi_watchdog_default(void) |
1da177e4 LT |
180 | { |
181 | if (nmi_watchdog != NMI_DEFAULT) | |
182 | return; | |
75152114 AK |
183 | if (nmi_known_cpu()) |
184 | nmi_watchdog = NMI_LOCAL_APIC; | |
185 | else | |
1da177e4 | 186 | nmi_watchdog = NMI_IO_APIC; |
1da177e4 LT |
187 | } |
188 | ||
75152114 AK |
189 | #ifdef CONFIG_SMP |
190 | /* The performance counters used by NMI_LOCAL_APIC don't trigger when | |
191 | * the CPU is idle. To make sure the NMI watchdog really ticks on all | |
192 | * CPUs during the test make them busy. | |
193 | */ | |
194 | static __init void nmi_cpu_busy(void *data) | |
1da177e4 | 195 | { |
75152114 | 196 | volatile int *endflag = data; |
366c7f55 | 197 | local_irq_enable_in_hardirq(); |
75152114 AK |
198 | /* Intentionally don't use cpu_relax here. This is |
199 | to make sure that the performance counter really ticks, | |
200 | even if there is a simulator or similar that catches the | |
201 | pause instruction. On a real HT machine this is fine because | |
202 | all other CPUs are busy with "useless" delay loops and don't | |
203 | care if they get somewhat less cycles. */ | |
204 | while (*endflag == 0) | |
205 | barrier(); | |
1da177e4 | 206 | } |
75152114 | 207 | #endif |
1da177e4 | 208 | |
75152114 | 209 | int __init check_nmi_watchdog (void) |
1da177e4 | 210 | { |
75152114 | 211 | volatile int endflag = 0; |
ac6b931c | 212 | int *counts; |
1da177e4 LT |
213 | int cpu; |
214 | ||
f2802e7f DZ |
215 | if ((nmi_watchdog == NMI_NONE) || (nmi_watchdog == NMI_DEFAULT)) |
216 | return 0; | |
217 | ||
218 | if (!atomic_read(&nmi_active)) | |
219 | return 0; | |
220 | ||
75152114 AK |
221 | counts = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL); |
222 | if (!counts) | |
223 | return -1; | |
1da177e4 | 224 | |
75152114 | 225 | printk(KERN_INFO "testing NMI watchdog ... "); |
ac6b931c | 226 | |
7554c3f0 | 227 | #ifdef CONFIG_SMP |
75152114 AK |
228 | if (nmi_watchdog == NMI_LOCAL_APIC) |
229 | smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0); | |
7554c3f0 | 230 | #endif |
1da177e4 LT |
231 | |
232 | for (cpu = 0; cpu < NR_CPUS; cpu++) | |
df79efde | 233 | counts[cpu] = cpu_pda(cpu)->__nmi_count; |
1da177e4 LT |
234 | local_irq_enable(); |
235 | mdelay((10*1000)/nmi_hz); // wait 10 ticks | |
236 | ||
394e3902 | 237 | for_each_online_cpu(cpu) { |
f2802e7f DZ |
238 | if (!per_cpu(nmi_watchdog_ctlblk, cpu).enabled) |
239 | continue; | |
df79efde | 240 | if (cpu_pda(cpu)->__nmi_count - counts[cpu] <= 5) { |
75152114 | 241 | printk("CPU#%d: NMI appears to be stuck (%d->%d)!\n", |
1da177e4 | 242 | cpu, |
75152114 | 243 | counts[cpu], |
df79efde | 244 | cpu_pda(cpu)->__nmi_count); |
f2802e7f DZ |
245 | per_cpu(nmi_watchdog_ctlblk, cpu).enabled = 0; |
246 | atomic_dec(&nmi_active); | |
1da177e4 LT |
247 | } |
248 | } | |
f2802e7f DZ |
249 | if (!atomic_read(&nmi_active)) { |
250 | kfree(counts); | |
251 | atomic_set(&nmi_active, -1); | |
252 | return -1; | |
253 | } | |
75152114 | 254 | endflag = 1; |
1da177e4 LT |
255 | printk("OK.\n"); |
256 | ||
257 | /* now that we know it works we can reduce NMI frequency to | |
258 | something more reasonable; makes a difference in some configs */ | |
248dcb2f VP |
259 | if (nmi_watchdog == NMI_LOCAL_APIC) { |
260 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
261 | ||
1da177e4 | 262 | nmi_hz = 1; |
248dcb2f VP |
263 | /* |
264 | * On Intel CPUs with ARCH_PERFMON only 32 bits in the counter | |
265 | * are writable, with higher bits sign extending from bit 31. | |
266 | * So, we can only program the counter with 31 bit values and | |
267 | * 32nd bit should be 1, for 33.. to be 1. | |
268 | * Find the appropriate nmi_hz | |
269 | */ | |
270 | if (wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR0 && | |
271 | ((u64)cpu_khz * 1000) > 0x7fffffffULL) { | |
272 | nmi_hz = ((u64)cpu_khz * 1000) / 0x7fffffffUL + 1; | |
273 | } | |
274 | } | |
1da177e4 | 275 | |
ac6b931c | 276 | kfree(counts); |
1da177e4 LT |
277 | return 0; |
278 | } | |
279 | ||
280 | int __init setup_nmi_watchdog(char *str) | |
281 | { | |
282 | int nmi; | |
283 | ||
284 | if (!strncmp(str,"panic",5)) { | |
285 | panic_on_timeout = 1; | |
286 | str = strchr(str, ','); | |
287 | if (!str) | |
288 | return 1; | |
289 | ++str; | |
290 | } | |
291 | ||
292 | get_option(&str, &nmi); | |
293 | ||
f2802e7f | 294 | if ((nmi >= NMI_INVALID) || (nmi < NMI_NONE)) |
1da177e4 | 295 | return 0; |
f2802e7f DZ |
296 | |
297 | if ((nmi == NMI_LOCAL_APIC) && (nmi_known_cpu() == 0)) | |
298 | return 0; /* no lapic support */ | |
75152114 | 299 | nmi_watchdog = nmi; |
1da177e4 LT |
300 | return 1; |
301 | } | |
302 | ||
303 | __setup("nmi_watchdog=", setup_nmi_watchdog); | |
304 | ||
305 | static void disable_lapic_nmi_watchdog(void) | |
306 | { | |
f2802e7f DZ |
307 | BUG_ON(nmi_watchdog != NMI_LOCAL_APIC); |
308 | ||
309 | if (atomic_read(&nmi_active) <= 0) | |
1da177e4 | 310 | return; |
f2802e7f DZ |
311 | |
312 | on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1); | |
313 | ||
314 | BUG_ON(atomic_read(&nmi_active) != 0); | |
1da177e4 LT |
315 | } |
316 | ||
317 | static void enable_lapic_nmi_watchdog(void) | |
318 | { | |
f2802e7f DZ |
319 | BUG_ON(nmi_watchdog != NMI_LOCAL_APIC); |
320 | ||
321 | /* are we already enabled */ | |
322 | if (atomic_read(&nmi_active) != 0) | |
323 | return; | |
324 | ||
325 | /* are we lapic aware */ | |
326 | if (nmi_known_cpu() <= 0) | |
327 | return; | |
328 | ||
329 | on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1); | |
330 | touch_nmi_watchdog(); | |
1da177e4 LT |
331 | } |
332 | ||
1da177e4 LT |
333 | void disable_timer_nmi_watchdog(void) |
334 | { | |
f2802e7f DZ |
335 | BUG_ON(nmi_watchdog != NMI_IO_APIC); |
336 | ||
337 | if (atomic_read(&nmi_active) <= 0) | |
1da177e4 LT |
338 | return; |
339 | ||
340 | disable_irq(0); | |
f2802e7f DZ |
341 | on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1); |
342 | ||
343 | BUG_ON(atomic_read(&nmi_active) != 0); | |
1da177e4 LT |
344 | } |
345 | ||
346 | void enable_timer_nmi_watchdog(void) | |
347 | { | |
f2802e7f DZ |
348 | BUG_ON(nmi_watchdog != NMI_IO_APIC); |
349 | ||
350 | if (atomic_read(&nmi_active) == 0) { | |
1da177e4 | 351 | touch_nmi_watchdog(); |
f2802e7f | 352 | on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1); |
1da177e4 LT |
353 | enable_irq(0); |
354 | } | |
355 | } | |
356 | ||
357 | #ifdef CONFIG_PM | |
358 | ||
359 | static int nmi_pm_active; /* nmi_active before suspend */ | |
360 | ||
829ca9a3 | 361 | static int lapic_nmi_suspend(struct sys_device *dev, pm_message_t state) |
1da177e4 | 362 | { |
4038f901 | 363 | /* only CPU0 goes here, other CPUs should be offline */ |
f2802e7f | 364 | nmi_pm_active = atomic_read(&nmi_active); |
4038f901 SL |
365 | stop_apic_nmi_watchdog(NULL); |
366 | BUG_ON(atomic_read(&nmi_active) != 0); | |
1da177e4 LT |
367 | return 0; |
368 | } | |
369 | ||
370 | static int lapic_nmi_resume(struct sys_device *dev) | |
371 | { | |
4038f901 SL |
372 | /* only CPU0 goes here, other CPUs should be offline */ |
373 | if (nmi_pm_active > 0) { | |
374 | setup_apic_nmi_watchdog(NULL); | |
375 | touch_nmi_watchdog(); | |
376 | } | |
1da177e4 LT |
377 | return 0; |
378 | } | |
379 | ||
380 | static struct sysdev_class nmi_sysclass = { | |
381 | set_kset_name("lapic_nmi"), | |
382 | .resume = lapic_nmi_resume, | |
383 | .suspend = lapic_nmi_suspend, | |
384 | }; | |
385 | ||
386 | static struct sys_device device_lapic_nmi = { | |
387 | .id = 0, | |
388 | .cls = &nmi_sysclass, | |
389 | }; | |
390 | ||
391 | static int __init init_lapic_nmi_sysfs(void) | |
392 | { | |
393 | int error; | |
394 | ||
f2802e7f DZ |
395 | /* should really be a BUG_ON but b/c this is an |
396 | * init call, it just doesn't work. -dcz | |
397 | */ | |
398 | if (nmi_watchdog != NMI_LOCAL_APIC) | |
399 | return 0; | |
400 | ||
401 | if ( atomic_read(&nmi_active) < 0 ) | |
1da177e4 LT |
402 | return 0; |
403 | ||
404 | error = sysdev_class_register(&nmi_sysclass); | |
405 | if (!error) | |
406 | error = sysdev_register(&device_lapic_nmi); | |
407 | return error; | |
408 | } | |
409 | /* must come after the local APIC's device_initcall() */ | |
410 | late_initcall(init_lapic_nmi_sysfs); | |
411 | ||
412 | #endif /* CONFIG_PM */ | |
413 | ||
f2802e7f DZ |
414 | /* |
415 | * Activate the NMI watchdog via the local APIC. | |
416 | * Original code written by Keith Owens. | |
417 | */ | |
418 | ||
419 | /* Note that these events don't tick when the CPU idles. This means | |
420 | the frequency varies with CPU load. */ | |
421 | ||
422 | #define K7_EVNTSEL_ENABLE (1 << 22) | |
423 | #define K7_EVNTSEL_INT (1 << 20) | |
424 | #define K7_EVNTSEL_OS (1 << 17) | |
425 | #define K7_EVNTSEL_USR (1 << 16) | |
426 | #define K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING 0x76 | |
427 | #define K7_NMI_EVENT K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING | |
428 | ||
828f0afd | 429 | static int setup_k7_watchdog(void) |
75152114 | 430 | { |
f2802e7f | 431 | unsigned int perfctr_msr, evntsel_msr; |
1da177e4 | 432 | unsigned int evntsel; |
f2802e7f | 433 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
1da177e4 | 434 | |
f2802e7f DZ |
435 | perfctr_msr = MSR_K7_PERFCTR0; |
436 | evntsel_msr = MSR_K7_EVNTSEL0; | |
437 | if (!reserve_perfctr_nmi(perfctr_msr)) | |
828f0afd DZ |
438 | goto fail; |
439 | ||
f2802e7f | 440 | if (!reserve_evntsel_nmi(evntsel_msr)) |
828f0afd DZ |
441 | goto fail1; |
442 | ||
443 | /* Simulator may not support it */ | |
f2802e7f | 444 | if (checking_wrmsrl(evntsel_msr, 0UL)) |
828f0afd | 445 | goto fail2; |
f2802e7f | 446 | wrmsrl(perfctr_msr, 0UL); |
1da177e4 LT |
447 | |
448 | evntsel = K7_EVNTSEL_INT | |
449 | | K7_EVNTSEL_OS | |
450 | | K7_EVNTSEL_USR | |
451 | | K7_NMI_EVENT; | |
452 | ||
f2802e7f DZ |
453 | /* setup the timer */ |
454 | wrmsr(evntsel_msr, evntsel, 0); | |
455 | wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); | |
1da177e4 LT |
456 | apic_write(APIC_LVTPC, APIC_DM_NMI); |
457 | evntsel |= K7_EVNTSEL_ENABLE; | |
f2802e7f DZ |
458 | wrmsr(evntsel_msr, evntsel, 0); |
459 | ||
460 | wd->perfctr_msr = perfctr_msr; | |
461 | wd->evntsel_msr = evntsel_msr; | |
462 | wd->cccr_msr = 0; //unused | |
463 | wd->check_bit = 1ULL<<63; | |
828f0afd DZ |
464 | return 1; |
465 | fail2: | |
f2802e7f | 466 | release_evntsel_nmi(evntsel_msr); |
828f0afd | 467 | fail1: |
f2802e7f | 468 | release_perfctr_nmi(perfctr_msr); |
828f0afd DZ |
469 | fail: |
470 | return 0; | |
1da177e4 LT |
471 | } |
472 | ||
f2802e7f DZ |
473 | static void stop_k7_watchdog(void) |
474 | { | |
475 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
476 | ||
477 | wrmsr(wd->evntsel_msr, 0, 0); | |
478 | ||
479 | release_evntsel_nmi(wd->evntsel_msr); | |
480 | release_perfctr_nmi(wd->perfctr_msr); | |
481 | } | |
482 | ||
483 | /* Note that these events don't tick when the CPU idles. This means | |
484 | the frequency varies with CPU load. */ | |
485 | ||
486 | #define MSR_P4_MISC_ENABLE_PERF_AVAIL (1<<7) | |
487 | #define P4_ESCR_EVENT_SELECT(N) ((N)<<25) | |
488 | #define P4_ESCR_OS (1<<3) | |
489 | #define P4_ESCR_USR (1<<2) | |
490 | #define P4_CCCR_OVF_PMI0 (1<<26) | |
491 | #define P4_CCCR_OVF_PMI1 (1<<27) | |
492 | #define P4_CCCR_THRESHOLD(N) ((N)<<20) | |
493 | #define P4_CCCR_COMPLEMENT (1<<19) | |
494 | #define P4_CCCR_COMPARE (1<<18) | |
495 | #define P4_CCCR_REQUIRED (3<<16) | |
496 | #define P4_CCCR_ESCR_SELECT(N) ((N)<<13) | |
497 | #define P4_CCCR_ENABLE (1<<12) | |
498 | #define P4_CCCR_OVF (1<<31) | |
499 | /* Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter | |
500 | CRU_ESCR0 (with any non-null event selector) through a complemented | |
501 | max threshold. [IA32-Vol3, Section 14.9.9] */ | |
75152114 AK |
502 | |
503 | static int setup_p4_watchdog(void) | |
504 | { | |
f2802e7f DZ |
505 | unsigned int perfctr_msr, evntsel_msr, cccr_msr; |
506 | unsigned int evntsel, cccr_val; | |
75152114 | 507 | unsigned int misc_enable, dummy; |
f2802e7f DZ |
508 | unsigned int ht_num; |
509 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
75152114 | 510 | |
f2802e7f | 511 | rdmsr(MSR_IA32_MISC_ENABLE, misc_enable, dummy); |
75152114 AK |
512 | if (!(misc_enable & MSR_P4_MISC_ENABLE_PERF_AVAIL)) |
513 | return 0; | |
514 | ||
75152114 | 515 | #ifdef CONFIG_SMP |
f2802e7f DZ |
516 | /* detect which hyperthread we are on */ |
517 | if (smp_num_siblings == 2) { | |
518 | unsigned int ebx, apicid; | |
519 | ||
520 | ebx = cpuid_ebx(1); | |
521 | apicid = (ebx >> 24) & 0xff; | |
522 | ht_num = apicid & 1; | |
523 | } else | |
75152114 | 524 | #endif |
f2802e7f DZ |
525 | ht_num = 0; |
526 | ||
527 | /* performance counters are shared resources | |
528 | * assign each hyperthread its own set | |
529 | * (re-use the ESCR0 register, seems safe | |
530 | * and keeps the cccr_val the same) | |
531 | */ | |
532 | if (!ht_num) { | |
533 | /* logical cpu 0 */ | |
534 | perfctr_msr = MSR_P4_IQ_PERFCTR0; | |
535 | evntsel_msr = MSR_P4_CRU_ESCR0; | |
536 | cccr_msr = MSR_P4_IQ_CCCR0; | |
537 | cccr_val = P4_CCCR_OVF_PMI0 | P4_CCCR_ESCR_SELECT(4); | |
538 | } else { | |
539 | /* logical cpu 1 */ | |
540 | perfctr_msr = MSR_P4_IQ_PERFCTR1; | |
541 | evntsel_msr = MSR_P4_CRU_ESCR0; | |
542 | cccr_msr = MSR_P4_IQ_CCCR1; | |
543 | cccr_val = P4_CCCR_OVF_PMI1 | P4_CCCR_ESCR_SELECT(4); | |
544 | } | |
75152114 | 545 | |
f2802e7f | 546 | if (!reserve_perfctr_nmi(perfctr_msr)) |
828f0afd DZ |
547 | goto fail; |
548 | ||
f2802e7f | 549 | if (!reserve_evntsel_nmi(evntsel_msr)) |
828f0afd | 550 | goto fail1; |
75152114 | 551 | |
f2802e7f DZ |
552 | evntsel = P4_ESCR_EVENT_SELECT(0x3F) |
553 | | P4_ESCR_OS | |
554 | | P4_ESCR_USR; | |
555 | ||
556 | cccr_val |= P4_CCCR_THRESHOLD(15) | |
557 | | P4_CCCR_COMPLEMENT | |
558 | | P4_CCCR_COMPARE | |
559 | | P4_CCCR_REQUIRED; | |
560 | ||
561 | wrmsr(evntsel_msr, evntsel, 0); | |
562 | wrmsr(cccr_msr, cccr_val, 0); | |
563 | wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); | |
75152114 | 564 | apic_write(APIC_LVTPC, APIC_DM_NMI); |
f2802e7f DZ |
565 | cccr_val |= P4_CCCR_ENABLE; |
566 | wrmsr(cccr_msr, cccr_val, 0); | |
567 | ||
568 | wd->perfctr_msr = perfctr_msr; | |
569 | wd->evntsel_msr = evntsel_msr; | |
570 | wd->cccr_msr = cccr_msr; | |
571 | wd->check_bit = 1ULL<<39; | |
75152114 | 572 | return 1; |
828f0afd | 573 | fail1: |
f2802e7f | 574 | release_perfctr_nmi(perfctr_msr); |
828f0afd DZ |
575 | fail: |
576 | return 0; | |
75152114 AK |
577 | } |
578 | ||
f2802e7f | 579 | static void stop_p4_watchdog(void) |
1da177e4 | 580 | { |
f2802e7f DZ |
581 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
582 | ||
583 | wrmsr(wd->cccr_msr, 0, 0); | |
584 | wrmsr(wd->evntsel_msr, 0, 0); | |
585 | ||
586 | release_evntsel_nmi(wd->evntsel_msr); | |
587 | release_perfctr_nmi(wd->perfctr_msr); | |
588 | } | |
589 | ||
248dcb2f VP |
590 | #define ARCH_PERFMON_NMI_EVENT_SEL ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL |
591 | #define ARCH_PERFMON_NMI_EVENT_UMASK ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK | |
592 | ||
593 | static int setup_intel_arch_watchdog(void) | |
594 | { | |
595 | unsigned int ebx; | |
596 | union cpuid10_eax eax; | |
597 | unsigned int unused; | |
598 | unsigned int perfctr_msr, evntsel_msr; | |
599 | unsigned int evntsel; | |
600 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
601 | ||
602 | /* | |
603 | * Check whether the Architectural PerfMon supports | |
604 | * Unhalted Core Cycles Event or not. | |
605 | * NOTE: Corresponding bit = 0 in ebx indicates event present. | |
606 | */ | |
607 | cpuid(10, &(eax.full), &ebx, &unused, &unused); | |
608 | if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) || | |
609 | (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT)) | |
610 | goto fail; | |
611 | ||
612 | perfctr_msr = MSR_ARCH_PERFMON_PERFCTR0; | |
613 | evntsel_msr = MSR_ARCH_PERFMON_EVENTSEL0; | |
614 | ||
615 | if (!reserve_perfctr_nmi(perfctr_msr)) | |
616 | goto fail; | |
617 | ||
618 | if (!reserve_evntsel_nmi(evntsel_msr)) | |
619 | goto fail1; | |
620 | ||
621 | wrmsrl(perfctr_msr, 0UL); | |
622 | ||
623 | evntsel = ARCH_PERFMON_EVENTSEL_INT | |
624 | | ARCH_PERFMON_EVENTSEL_OS | |
625 | | ARCH_PERFMON_EVENTSEL_USR | |
626 | | ARCH_PERFMON_NMI_EVENT_SEL | |
627 | | ARCH_PERFMON_NMI_EVENT_UMASK; | |
628 | ||
629 | /* setup the timer */ | |
630 | wrmsr(evntsel_msr, evntsel, 0); | |
631 | wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); | |
632 | ||
633 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
634 | evntsel |= ARCH_PERFMON_EVENTSEL0_ENABLE; | |
635 | wrmsr(evntsel_msr, evntsel, 0); | |
636 | ||
637 | wd->perfctr_msr = perfctr_msr; | |
638 | wd->evntsel_msr = evntsel_msr; | |
639 | wd->cccr_msr = 0; //unused | |
640 | wd->check_bit = 1ULL << (eax.split.bit_width - 1); | |
641 | return 1; | |
642 | fail1: | |
643 | release_perfctr_nmi(perfctr_msr); | |
644 | fail: | |
645 | return 0; | |
646 | } | |
647 | ||
648 | static void stop_intel_arch_watchdog(void) | |
649 | { | |
650 | unsigned int ebx; | |
651 | union cpuid10_eax eax; | |
652 | unsigned int unused; | |
653 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
654 | ||
655 | /* | |
656 | * Check whether the Architectural PerfMon supports | |
657 | * Unhalted Core Cycles Event or not. | |
658 | * NOTE: Corresponding bit = 0 in ebx indicates event present. | |
659 | */ | |
660 | cpuid(10, &(eax.full), &ebx, &unused, &unused); | |
661 | if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) || | |
662 | (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT)) | |
663 | return; | |
664 | ||
665 | wrmsr(wd->evntsel_msr, 0, 0); | |
666 | ||
667 | release_evntsel_nmi(wd->evntsel_msr); | |
668 | release_perfctr_nmi(wd->perfctr_msr); | |
669 | } | |
670 | ||
f2802e7f DZ |
671 | void setup_apic_nmi_watchdog(void *unused) |
672 | { | |
4038f901 SL |
673 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
674 | ||
f2802e7f DZ |
675 | /* only support LOCAL and IO APICs for now */ |
676 | if ((nmi_watchdog != NMI_LOCAL_APIC) && | |
677 | (nmi_watchdog != NMI_IO_APIC)) | |
678 | return; | |
679 | ||
4038f901 SL |
680 | if (wd->enabled == 1) |
681 | return; | |
682 | ||
683 | /* cheap hack to support suspend/resume */ | |
684 | /* if cpu0 is not active neither should the other cpus */ | |
685 | if ((smp_processor_id() != 0) && (atomic_read(&nmi_active) <= 0)) | |
686 | return; | |
687 | ||
f2802e7f DZ |
688 | if (nmi_watchdog == NMI_LOCAL_APIC) { |
689 | switch (boot_cpu_data.x86_vendor) { | |
690 | case X86_VENDOR_AMD: | |
691 | if (strstr(boot_cpu_data.x86_model_id, "Screwdriver")) | |
692 | return; | |
693 | if (!setup_k7_watchdog()) | |
694 | return; | |
695 | break; | |
696 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
697 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) { |
698 | if (!setup_intel_arch_watchdog()) | |
699 | return; | |
700 | break; | |
701 | } | |
f2802e7f DZ |
702 | if (!setup_p4_watchdog()) |
703 | return; | |
704 | break; | |
705 | default: | |
75152114 | 706 | return; |
f2802e7f DZ |
707 | } |
708 | } | |
4038f901 | 709 | wd->enabled = 1; |
f2802e7f DZ |
710 | atomic_inc(&nmi_active); |
711 | } | |
75152114 | 712 | |
4038f901 | 713 | void stop_apic_nmi_watchdog(void *unused) |
f2802e7f | 714 | { |
4038f901 SL |
715 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
716 | ||
f2802e7f DZ |
717 | /* only support LOCAL and IO APICs for now */ |
718 | if ((nmi_watchdog != NMI_LOCAL_APIC) && | |
719 | (nmi_watchdog != NMI_IO_APIC)) | |
720 | return; | |
721 | ||
4038f901 SL |
722 | if (wd->enabled == 0) |
723 | return; | |
724 | ||
f2802e7f DZ |
725 | if (nmi_watchdog == NMI_LOCAL_APIC) { |
726 | switch (boot_cpu_data.x86_vendor) { | |
727 | case X86_VENDOR_AMD: | |
728 | if (strstr(boot_cpu_data.x86_model_id, "Screwdriver")) | |
729 | return; | |
730 | stop_k7_watchdog(); | |
731 | break; | |
732 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
733 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) { |
734 | stop_intel_arch_watchdog(); | |
735 | break; | |
736 | } | |
f2802e7f DZ |
737 | stop_p4_watchdog(); |
738 | break; | |
739 | default: | |
740 | return; | |
741 | } | |
1da177e4 | 742 | } |
4038f901 | 743 | wd->enabled = 0; |
f2802e7f | 744 | atomic_dec(&nmi_active); |
1da177e4 LT |
745 | } |
746 | ||
747 | /* | |
748 | * the best way to detect whether a CPU has a 'hard lockup' problem | |
749 | * is to check it's local APIC timer IRQ counts. If they are not | |
750 | * changing then that CPU has some problem. | |
751 | * | |
752 | * as these watchdog NMI IRQs are generated on every CPU, we only | |
753 | * have to check the current processor. | |
1da177e4 LT |
754 | */ |
755 | ||
75152114 AK |
756 | static DEFINE_PER_CPU(unsigned, last_irq_sum); |
757 | static DEFINE_PER_CPU(local_t, alert_counter); | |
758 | static DEFINE_PER_CPU(int, nmi_touch); | |
1da177e4 LT |
759 | |
760 | void touch_nmi_watchdog (void) | |
761 | { | |
99019e91 JB |
762 | if (nmi_watchdog > 0) { |
763 | unsigned cpu; | |
1da177e4 | 764 | |
99019e91 JB |
765 | /* |
766 | * Tell other CPUs to reset their alert counters. We cannot | |
767 | * do it ourselves because the alert count increase is not | |
768 | * atomic. | |
769 | */ | |
770 | for_each_present_cpu (cpu) | |
771 | per_cpu(nmi_touch, cpu) = 1; | |
772 | } | |
8446f1d3 IM |
773 | |
774 | touch_softlockup_watchdog(); | |
1da177e4 LT |
775 | } |
776 | ||
3adbbcce | 777 | int __kprobes nmi_watchdog_tick(struct pt_regs * regs, unsigned reason) |
1da177e4 | 778 | { |
75152114 AK |
779 | int sum; |
780 | int touched = 0; | |
f2802e7f DZ |
781 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
782 | u64 dummy; | |
3adbbcce | 783 | int rc=0; |
f2802e7f DZ |
784 | |
785 | /* check for other users first */ | |
786 | if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) | |
787 | == NOTIFY_STOP) { | |
3adbbcce | 788 | rc = 1; |
f2802e7f DZ |
789 | touched = 1; |
790 | } | |
1da177e4 | 791 | |
1da177e4 | 792 | sum = read_pda(apic_timer_irqs); |
75152114 AK |
793 | if (__get_cpu_var(nmi_touch)) { |
794 | __get_cpu_var(nmi_touch) = 0; | |
795 | touched = 1; | |
796 | } | |
f2802e7f | 797 | |
553f265f AK |
798 | #ifdef CONFIG_X86_MCE |
799 | /* Could check oops_in_progress here too, but it's safer | |
800 | not too */ | |
801 | if (atomic_read(&mce_entry) > 0) | |
802 | touched = 1; | |
803 | #endif | |
f2802e7f | 804 | /* if the apic timer isn't firing, this cpu isn't doing much */ |
75152114 | 805 | if (!touched && __get_cpu_var(last_irq_sum) == sum) { |
1da177e4 LT |
806 | /* |
807 | * Ayiee, looks like this CPU is stuck ... | |
808 | * wait a few IRQs (5 seconds) before doing the oops ... | |
809 | */ | |
75152114 | 810 | local_inc(&__get_cpu_var(alert_counter)); |
f2802e7f | 811 | if (local_read(&__get_cpu_var(alert_counter)) == 5*nmi_hz) |
fac58550 AK |
812 | die_nmi("NMI Watchdog detected LOCKUP on CPU %d\n", regs, |
813 | panic_on_timeout); | |
1da177e4 | 814 | } else { |
75152114 AK |
815 | __get_cpu_var(last_irq_sum) = sum; |
816 | local_set(&__get_cpu_var(alert_counter), 0); | |
1da177e4 | 817 | } |
f2802e7f DZ |
818 | |
819 | /* see if the nmi watchdog went off */ | |
820 | if (wd->enabled) { | |
821 | if (nmi_watchdog == NMI_LOCAL_APIC) { | |
822 | rdmsrl(wd->perfctr_msr, dummy); | |
823 | if (dummy & wd->check_bit){ | |
824 | /* this wasn't a watchdog timer interrupt */ | |
825 | goto done; | |
826 | } | |
827 | ||
828 | /* only Intel uses the cccr msr */ | |
829 | if (wd->cccr_msr != 0) { | |
830 | /* | |
831 | * P4 quirks: | |
832 | * - An overflown perfctr will assert its interrupt | |
833 | * until the OVF flag in its CCCR is cleared. | |
834 | * - LVTPC is masked on interrupt and must be | |
835 | * unmasked by the LVTPC handler. | |
836 | */ | |
837 | rdmsrl(wd->cccr_msr, dummy); | |
838 | dummy &= ~P4_CCCR_OVF; | |
839 | wrmsrl(wd->cccr_msr, dummy); | |
840 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
248dcb2f VP |
841 | } else if (wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR0) { |
842 | /* | |
843 | * ArchPerfom/Core Duo needs to re-unmask | |
844 | * the apic vector | |
845 | */ | |
846 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
847 | } | |
f2802e7f DZ |
848 | /* start the cycle over again */ |
849 | wrmsrl(wd->perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); | |
3adbbcce DZ |
850 | rc = 1; |
851 | } else if (nmi_watchdog == NMI_IO_APIC) { | |
852 | /* don't know how to accurately check for this. | |
853 | * just assume it was a watchdog timer interrupt | |
854 | * This matches the old behaviour. | |
855 | */ | |
856 | rc = 1; | |
857 | } else | |
858 | printk(KERN_WARNING "Unknown enabled NMI hardware?!\n"); | |
75152114 | 859 | } |
f2802e7f | 860 | done: |
3adbbcce | 861 | return rc; |
1da177e4 LT |
862 | } |
863 | ||
eddb6fb9 | 864 | asmlinkage __kprobes void do_nmi(struct pt_regs * regs, long error_code) |
1da177e4 | 865 | { |
1da177e4 LT |
866 | nmi_enter(); |
867 | add_pda(__nmi_count,1); | |
3adbbcce | 868 | default_do_nmi(regs); |
1da177e4 LT |
869 | nmi_exit(); |
870 | } | |
871 | ||
3adbbcce DZ |
872 | int do_nmi_callback(struct pt_regs * regs, int cpu) |
873 | { | |
2fbe7b25 DZ |
874 | #ifdef CONFIG_SYSCTL |
875 | if (unknown_nmi_panic) | |
876 | return unknown_nmi_panic_callback(regs, cpu); | |
877 | #endif | |
878 | return 0; | |
1da177e4 LT |
879 | } |
880 | ||
881 | #ifdef CONFIG_SYSCTL | |
882 | ||
883 | static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu) | |
884 | { | |
885 | unsigned char reason = get_nmi_reason(); | |
886 | char buf[64]; | |
887 | ||
2fbe7b25 | 888 | sprintf(buf, "NMI received for unknown reason %02x\n", reason); |
fac58550 | 889 | die_nmi(buf, regs, 1); /* Always panic here */ |
1da177e4 LT |
890 | return 0; |
891 | } | |
892 | ||
407984f1 DZ |
893 | /* |
894 | * proc handler for /proc/sys/kernel/nmi | |
895 | */ | |
896 | int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file, | |
897 | void __user *buffer, size_t *length, loff_t *ppos) | |
898 | { | |
899 | int old_state; | |
900 | ||
901 | nmi_watchdog_enabled = (atomic_read(&nmi_active) > 0) ? 1 : 0; | |
902 | old_state = nmi_watchdog_enabled; | |
903 | proc_dointvec(table, write, file, buffer, length, ppos); | |
904 | if (!!old_state == !!nmi_watchdog_enabled) | |
905 | return 0; | |
906 | ||
907 | if (atomic_read(&nmi_active) < 0) { | |
908 | printk( KERN_WARNING "NMI watchdog is permanently disabled\n"); | |
e33e89ab | 909 | return -EIO; |
407984f1 DZ |
910 | } |
911 | ||
912 | /* if nmi_watchdog is not set yet, then set it */ | |
913 | nmi_watchdog_default(); | |
914 | ||
e33e89ab | 915 | if (nmi_watchdog == NMI_LOCAL_APIC) { |
407984f1 DZ |
916 | if (nmi_watchdog_enabled) |
917 | enable_lapic_nmi_watchdog(); | |
918 | else | |
919 | disable_lapic_nmi_watchdog(); | |
407984f1 | 920 | } else { |
e33e89ab | 921 | printk( KERN_WARNING |
407984f1 DZ |
922 | "NMI watchdog doesn't know what hardware to touch\n"); |
923 | return -EIO; | |
924 | } | |
925 | return 0; | |
926 | } | |
927 | ||
1da177e4 LT |
928 | #endif |
929 | ||
930 | EXPORT_SYMBOL(nmi_active); | |
931 | EXPORT_SYMBOL(nmi_watchdog); | |
828f0afd DZ |
932 | EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi); |
933 | EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit); | |
934 | EXPORT_SYMBOL(reserve_perfctr_nmi); | |
935 | EXPORT_SYMBOL(release_perfctr_nmi); | |
936 | EXPORT_SYMBOL(reserve_evntsel_nmi); | |
937 | EXPORT_SYMBOL(release_evntsel_nmi); | |
1da177e4 LT |
938 | EXPORT_SYMBOL(disable_timer_nmi_watchdog); |
939 | EXPORT_SYMBOL(enable_timer_nmi_watchdog); | |
940 | EXPORT_SYMBOL(touch_nmi_watchdog); |