Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * linux/arch/x86_64/nmi.c | |
3 | * | |
4 | * NMI watchdog support on APIC systems | |
5 | * | |
6 | * Started by Ingo Molnar <mingo@redhat.com> | |
7 | * | |
8 | * Fixes: | |
9 | * Mikael Pettersson : AMD K7 support for local APIC NMI watchdog. | |
10 | * Mikael Pettersson : Power Management for local APIC NMI watchdog. | |
11 | * Pavel Machek and | |
12 | * Mikael Pettersson : PM converted to driver model. Disable/enable API. | |
13 | */ | |
14 | ||
bb81a09e | 15 | #include <linux/nmi.h> |
1da177e4 | 16 | #include <linux/mm.h> |
1da177e4 | 17 | #include <linux/delay.h> |
1da177e4 | 18 | #include <linux/interrupt.h> |
1da177e4 LT |
19 | #include <linux/module.h> |
20 | #include <linux/sysdev.h> | |
1da177e4 | 21 | #include <linux/sysctl.h> |
eddb6fb9 | 22 | #include <linux/kprobes.h> |
bb81a09e | 23 | #include <linux/cpumask.h> |
1da177e4 LT |
24 | |
25 | #include <asm/smp.h> | |
1da177e4 | 26 | #include <asm/nmi.h> |
1da177e4 LT |
27 | #include <asm/proto.h> |
28 | #include <asm/kdebug.h> | |
553f265f | 29 | #include <asm/mce.h> |
248dcb2f | 30 | #include <asm/intel_arch_perfmon.h> |
1da177e4 | 31 | |
29cbc78b AK |
32 | int unknown_nmi_panic; |
33 | int nmi_watchdog_enabled; | |
34 | int panic_on_unrecovered_nmi; | |
35 | ||
828f0afd DZ |
36 | /* perfctr_nmi_owner tracks the ownership of the perfctr registers: |
37 | * evtsel_nmi_owner tracks the ownership of the event selection | |
38 | * - different performance counters/ event selection may be reserved for | |
39 | * different subsystems this reservation system just tries to coordinate | |
40 | * things a little | |
41 | */ | |
42 | static DEFINE_PER_CPU(unsigned, perfctr_nmi_owner); | |
43 | static DEFINE_PER_CPU(unsigned, evntsel_nmi_owner[2]); | |
44 | ||
bb81a09e AM |
45 | static cpumask_t backtrace_mask = CPU_MASK_NONE; |
46 | ||
828f0afd DZ |
47 | /* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's |
48 | * offset from MSR_P4_BSU_ESCR0. It will be the max for all platforms (for now) | |
49 | */ | |
50 | #define NMI_MAX_COUNTER_BITS 66 | |
51 | ||
1da177e4 | 52 | /* nmi_active: |
f2802e7f DZ |
53 | * >0: the lapic NMI watchdog is active, but can be disabled |
54 | * <0: the lapic NMI watchdog has not been set up, and cannot | |
1da177e4 | 55 | * be enabled |
f2802e7f | 56 | * 0: the lapic NMI watchdog is disabled, but can be enabled |
1da177e4 | 57 | */ |
f2802e7f | 58 | atomic_t nmi_active = ATOMIC_INIT(0); /* oprofile uses this */ |
1da177e4 LT |
59 | int panic_on_timeout; |
60 | ||
61 | unsigned int nmi_watchdog = NMI_DEFAULT; | |
62 | static unsigned int nmi_hz = HZ; | |
1da177e4 | 63 | |
f2802e7f DZ |
64 | struct nmi_watchdog_ctlblk { |
65 | int enabled; | |
66 | u64 check_bit; | |
67 | unsigned int cccr_msr; | |
68 | unsigned int perfctr_msr; /* the MSR to reset in NMI handler */ | |
69 | unsigned int evntsel_msr; /* the MSR to select the events to handle */ | |
70 | }; | |
71 | static DEFINE_PER_CPU(struct nmi_watchdog_ctlblk, nmi_watchdog_ctlblk); | |
1da177e4 | 72 | |
f2802e7f | 73 | /* local prototypes */ |
f2802e7f | 74 | static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu); |
75152114 | 75 | |
828f0afd DZ |
76 | /* converts an msr to an appropriate reservation bit */ |
77 | static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr) | |
78 | { | |
79 | /* returns the bit offset of the performance counter register */ | |
80 | switch (boot_cpu_data.x86_vendor) { | |
81 | case X86_VENDOR_AMD: | |
82 | return (msr - MSR_K7_PERFCTR0); | |
83 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
84 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
85 | return (msr - MSR_ARCH_PERFMON_PERFCTR0); | |
86 | else | |
87 | return (msr - MSR_P4_BPU_PERFCTR0); | |
828f0afd DZ |
88 | } |
89 | return 0; | |
90 | } | |
91 | ||
92 | /* converts an msr to an appropriate reservation bit */ | |
93 | static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr) | |
94 | { | |
95 | /* returns the bit offset of the event selection register */ | |
96 | switch (boot_cpu_data.x86_vendor) { | |
97 | case X86_VENDOR_AMD: | |
98 | return (msr - MSR_K7_EVNTSEL0); | |
99 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
100 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
101 | return (msr - MSR_ARCH_PERFMON_EVENTSEL0); | |
102 | else | |
103 | return (msr - MSR_P4_BSU_ESCR0); | |
828f0afd DZ |
104 | } |
105 | return 0; | |
106 | } | |
107 | ||
108 | /* checks for a bit availability (hack for oprofile) */ | |
109 | int avail_to_resrv_perfctr_nmi_bit(unsigned int counter) | |
110 | { | |
111 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
112 | ||
113 | return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner))); | |
114 | } | |
115 | ||
116 | /* checks the an msr for availability */ | |
117 | int avail_to_resrv_perfctr_nmi(unsigned int msr) | |
118 | { | |
119 | unsigned int counter; | |
120 | ||
121 | counter = nmi_perfctr_msr_to_bit(msr); | |
122 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
123 | ||
124 | return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner))); | |
125 | } | |
126 | ||
127 | int reserve_perfctr_nmi(unsigned int msr) | |
128 | { | |
129 | unsigned int counter; | |
130 | ||
131 | counter = nmi_perfctr_msr_to_bit(msr); | |
132 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
133 | ||
134 | if (!test_and_set_bit(counter, &__get_cpu_var(perfctr_nmi_owner))) | |
135 | return 1; | |
136 | return 0; | |
137 | } | |
138 | ||
139 | void release_perfctr_nmi(unsigned int msr) | |
140 | { | |
141 | unsigned int counter; | |
142 | ||
143 | counter = nmi_perfctr_msr_to_bit(msr); | |
144 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
145 | ||
146 | clear_bit(counter, &__get_cpu_var(perfctr_nmi_owner)); | |
147 | } | |
148 | ||
149 | int reserve_evntsel_nmi(unsigned int msr) | |
150 | { | |
151 | unsigned int counter; | |
152 | ||
153 | counter = nmi_evntsel_msr_to_bit(msr); | |
154 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
155 | ||
156 | if (!test_and_set_bit(counter, &__get_cpu_var(evntsel_nmi_owner))) | |
157 | return 1; | |
158 | return 0; | |
159 | } | |
160 | ||
161 | void release_evntsel_nmi(unsigned int msr) | |
162 | { | |
163 | unsigned int counter; | |
164 | ||
165 | counter = nmi_evntsel_msr_to_bit(msr); | |
166 | BUG_ON(counter > NMI_MAX_COUNTER_BITS); | |
167 | ||
168 | clear_bit(counter, &__get_cpu_var(evntsel_nmi_owner)); | |
169 | } | |
170 | ||
e6982c67 | 171 | static __cpuinit inline int nmi_known_cpu(void) |
75152114 AK |
172 | { |
173 | switch (boot_cpu_data.x86_vendor) { | |
174 | case X86_VENDOR_AMD: | |
175 | return boot_cpu_data.x86 == 15; | |
176 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
177 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) |
178 | return 1; | |
179 | else | |
180 | return (boot_cpu_data.x86 == 15); | |
75152114 AK |
181 | } |
182 | return 0; | |
183 | } | |
1da177e4 LT |
184 | |
185 | /* Run after command line and cpu_init init, but before all other checks */ | |
e33e89ab | 186 | void nmi_watchdog_default(void) |
1da177e4 LT |
187 | { |
188 | if (nmi_watchdog != NMI_DEFAULT) | |
189 | return; | |
75152114 AK |
190 | if (nmi_known_cpu()) |
191 | nmi_watchdog = NMI_LOCAL_APIC; | |
192 | else | |
1da177e4 | 193 | nmi_watchdog = NMI_IO_APIC; |
1da177e4 LT |
194 | } |
195 | ||
75152114 AK |
196 | #ifdef CONFIG_SMP |
197 | /* The performance counters used by NMI_LOCAL_APIC don't trigger when | |
198 | * the CPU is idle. To make sure the NMI watchdog really ticks on all | |
199 | * CPUs during the test make them busy. | |
200 | */ | |
201 | static __init void nmi_cpu_busy(void *data) | |
1da177e4 | 202 | { |
75152114 | 203 | volatile int *endflag = data; |
366c7f55 | 204 | local_irq_enable_in_hardirq(); |
75152114 AK |
205 | /* Intentionally don't use cpu_relax here. This is |
206 | to make sure that the performance counter really ticks, | |
207 | even if there is a simulator or similar that catches the | |
208 | pause instruction. On a real HT machine this is fine because | |
209 | all other CPUs are busy with "useless" delay loops and don't | |
210 | care if they get somewhat less cycles. */ | |
211 | while (*endflag == 0) | |
212 | barrier(); | |
1da177e4 | 213 | } |
75152114 | 214 | #endif |
1da177e4 | 215 | |
75152114 | 216 | int __init check_nmi_watchdog (void) |
1da177e4 | 217 | { |
75152114 | 218 | volatile int endflag = 0; |
ac6b931c | 219 | int *counts; |
1da177e4 LT |
220 | int cpu; |
221 | ||
f2802e7f DZ |
222 | if ((nmi_watchdog == NMI_NONE) || (nmi_watchdog == NMI_DEFAULT)) |
223 | return 0; | |
224 | ||
225 | if (!atomic_read(&nmi_active)) | |
226 | return 0; | |
227 | ||
75152114 AK |
228 | counts = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL); |
229 | if (!counts) | |
230 | return -1; | |
1da177e4 | 231 | |
75152114 | 232 | printk(KERN_INFO "testing NMI watchdog ... "); |
ac6b931c | 233 | |
7554c3f0 | 234 | #ifdef CONFIG_SMP |
75152114 AK |
235 | if (nmi_watchdog == NMI_LOCAL_APIC) |
236 | smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0); | |
7554c3f0 | 237 | #endif |
1da177e4 LT |
238 | |
239 | for (cpu = 0; cpu < NR_CPUS; cpu++) | |
df79efde | 240 | counts[cpu] = cpu_pda(cpu)->__nmi_count; |
1da177e4 LT |
241 | local_irq_enable(); |
242 | mdelay((10*1000)/nmi_hz); // wait 10 ticks | |
243 | ||
394e3902 | 244 | for_each_online_cpu(cpu) { |
f2802e7f DZ |
245 | if (!per_cpu(nmi_watchdog_ctlblk, cpu).enabled) |
246 | continue; | |
df79efde | 247 | if (cpu_pda(cpu)->__nmi_count - counts[cpu] <= 5) { |
75152114 | 248 | printk("CPU#%d: NMI appears to be stuck (%d->%d)!\n", |
1da177e4 | 249 | cpu, |
75152114 | 250 | counts[cpu], |
df79efde | 251 | cpu_pda(cpu)->__nmi_count); |
f2802e7f DZ |
252 | per_cpu(nmi_watchdog_ctlblk, cpu).enabled = 0; |
253 | atomic_dec(&nmi_active); | |
1da177e4 LT |
254 | } |
255 | } | |
f2802e7f DZ |
256 | if (!atomic_read(&nmi_active)) { |
257 | kfree(counts); | |
258 | atomic_set(&nmi_active, -1); | |
259 | return -1; | |
260 | } | |
75152114 | 261 | endflag = 1; |
1da177e4 LT |
262 | printk("OK.\n"); |
263 | ||
264 | /* now that we know it works we can reduce NMI frequency to | |
265 | something more reasonable; makes a difference in some configs */ | |
248dcb2f VP |
266 | if (nmi_watchdog == NMI_LOCAL_APIC) { |
267 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
268 | ||
1da177e4 | 269 | nmi_hz = 1; |
248dcb2f VP |
270 | /* |
271 | * On Intel CPUs with ARCH_PERFMON only 32 bits in the counter | |
272 | * are writable, with higher bits sign extending from bit 31. | |
273 | * So, we can only program the counter with 31 bit values and | |
274 | * 32nd bit should be 1, for 33.. to be 1. | |
275 | * Find the appropriate nmi_hz | |
276 | */ | |
277 | if (wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR0 && | |
278 | ((u64)cpu_khz * 1000) > 0x7fffffffULL) { | |
279 | nmi_hz = ((u64)cpu_khz * 1000) / 0x7fffffffUL + 1; | |
280 | } | |
281 | } | |
1da177e4 | 282 | |
ac6b931c | 283 | kfree(counts); |
1da177e4 LT |
284 | return 0; |
285 | } | |
286 | ||
287 | int __init setup_nmi_watchdog(char *str) | |
288 | { | |
289 | int nmi; | |
290 | ||
291 | if (!strncmp(str,"panic",5)) { | |
292 | panic_on_timeout = 1; | |
293 | str = strchr(str, ','); | |
294 | if (!str) | |
295 | return 1; | |
296 | ++str; | |
297 | } | |
298 | ||
299 | get_option(&str, &nmi); | |
300 | ||
f2802e7f | 301 | if ((nmi >= NMI_INVALID) || (nmi < NMI_NONE)) |
1da177e4 | 302 | return 0; |
f2802e7f DZ |
303 | |
304 | if ((nmi == NMI_LOCAL_APIC) && (nmi_known_cpu() == 0)) | |
305 | return 0; /* no lapic support */ | |
75152114 | 306 | nmi_watchdog = nmi; |
1da177e4 LT |
307 | return 1; |
308 | } | |
309 | ||
310 | __setup("nmi_watchdog=", setup_nmi_watchdog); | |
311 | ||
312 | static void disable_lapic_nmi_watchdog(void) | |
313 | { | |
f2802e7f DZ |
314 | BUG_ON(nmi_watchdog != NMI_LOCAL_APIC); |
315 | ||
316 | if (atomic_read(&nmi_active) <= 0) | |
1da177e4 | 317 | return; |
f2802e7f DZ |
318 | |
319 | on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1); | |
320 | ||
321 | BUG_ON(atomic_read(&nmi_active) != 0); | |
1da177e4 LT |
322 | } |
323 | ||
324 | static void enable_lapic_nmi_watchdog(void) | |
325 | { | |
f2802e7f DZ |
326 | BUG_ON(nmi_watchdog != NMI_LOCAL_APIC); |
327 | ||
328 | /* are we already enabled */ | |
329 | if (atomic_read(&nmi_active) != 0) | |
330 | return; | |
331 | ||
332 | /* are we lapic aware */ | |
333 | if (nmi_known_cpu() <= 0) | |
334 | return; | |
335 | ||
336 | on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1); | |
337 | touch_nmi_watchdog(); | |
1da177e4 LT |
338 | } |
339 | ||
1da177e4 LT |
340 | void disable_timer_nmi_watchdog(void) |
341 | { | |
f2802e7f DZ |
342 | BUG_ON(nmi_watchdog != NMI_IO_APIC); |
343 | ||
344 | if (atomic_read(&nmi_active) <= 0) | |
1da177e4 LT |
345 | return; |
346 | ||
347 | disable_irq(0); | |
f2802e7f DZ |
348 | on_each_cpu(stop_apic_nmi_watchdog, NULL, 0, 1); |
349 | ||
350 | BUG_ON(atomic_read(&nmi_active) != 0); | |
1da177e4 LT |
351 | } |
352 | ||
353 | void enable_timer_nmi_watchdog(void) | |
354 | { | |
f2802e7f DZ |
355 | BUG_ON(nmi_watchdog != NMI_IO_APIC); |
356 | ||
357 | if (atomic_read(&nmi_active) == 0) { | |
1da177e4 | 358 | touch_nmi_watchdog(); |
f2802e7f | 359 | on_each_cpu(setup_apic_nmi_watchdog, NULL, 0, 1); |
1da177e4 LT |
360 | enable_irq(0); |
361 | } | |
362 | } | |
363 | ||
364 | #ifdef CONFIG_PM | |
365 | ||
366 | static int nmi_pm_active; /* nmi_active before suspend */ | |
367 | ||
829ca9a3 | 368 | static int lapic_nmi_suspend(struct sys_device *dev, pm_message_t state) |
1da177e4 | 369 | { |
4038f901 | 370 | /* only CPU0 goes here, other CPUs should be offline */ |
f2802e7f | 371 | nmi_pm_active = atomic_read(&nmi_active); |
4038f901 SL |
372 | stop_apic_nmi_watchdog(NULL); |
373 | BUG_ON(atomic_read(&nmi_active) != 0); | |
1da177e4 LT |
374 | return 0; |
375 | } | |
376 | ||
377 | static int lapic_nmi_resume(struct sys_device *dev) | |
378 | { | |
4038f901 SL |
379 | /* only CPU0 goes here, other CPUs should be offline */ |
380 | if (nmi_pm_active > 0) { | |
381 | setup_apic_nmi_watchdog(NULL); | |
382 | touch_nmi_watchdog(); | |
383 | } | |
1da177e4 LT |
384 | return 0; |
385 | } | |
386 | ||
387 | static struct sysdev_class nmi_sysclass = { | |
388 | set_kset_name("lapic_nmi"), | |
389 | .resume = lapic_nmi_resume, | |
390 | .suspend = lapic_nmi_suspend, | |
391 | }; | |
392 | ||
393 | static struct sys_device device_lapic_nmi = { | |
394 | .id = 0, | |
395 | .cls = &nmi_sysclass, | |
396 | }; | |
397 | ||
398 | static int __init init_lapic_nmi_sysfs(void) | |
399 | { | |
400 | int error; | |
401 | ||
f2802e7f DZ |
402 | /* should really be a BUG_ON but b/c this is an |
403 | * init call, it just doesn't work. -dcz | |
404 | */ | |
405 | if (nmi_watchdog != NMI_LOCAL_APIC) | |
406 | return 0; | |
407 | ||
408 | if ( atomic_read(&nmi_active) < 0 ) | |
1da177e4 LT |
409 | return 0; |
410 | ||
411 | error = sysdev_class_register(&nmi_sysclass); | |
412 | if (!error) | |
413 | error = sysdev_register(&device_lapic_nmi); | |
414 | return error; | |
415 | } | |
416 | /* must come after the local APIC's device_initcall() */ | |
417 | late_initcall(init_lapic_nmi_sysfs); | |
418 | ||
419 | #endif /* CONFIG_PM */ | |
420 | ||
f2802e7f DZ |
421 | /* |
422 | * Activate the NMI watchdog via the local APIC. | |
423 | * Original code written by Keith Owens. | |
424 | */ | |
425 | ||
426 | /* Note that these events don't tick when the CPU idles. This means | |
427 | the frequency varies with CPU load. */ | |
428 | ||
429 | #define K7_EVNTSEL_ENABLE (1 << 22) | |
430 | #define K7_EVNTSEL_INT (1 << 20) | |
431 | #define K7_EVNTSEL_OS (1 << 17) | |
432 | #define K7_EVNTSEL_USR (1 << 16) | |
433 | #define K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING 0x76 | |
434 | #define K7_NMI_EVENT K7_EVENT_CYCLES_PROCESSOR_IS_RUNNING | |
435 | ||
828f0afd | 436 | static int setup_k7_watchdog(void) |
75152114 | 437 | { |
f2802e7f | 438 | unsigned int perfctr_msr, evntsel_msr; |
1da177e4 | 439 | unsigned int evntsel; |
f2802e7f | 440 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
1da177e4 | 441 | |
f2802e7f DZ |
442 | perfctr_msr = MSR_K7_PERFCTR0; |
443 | evntsel_msr = MSR_K7_EVNTSEL0; | |
444 | if (!reserve_perfctr_nmi(perfctr_msr)) | |
828f0afd DZ |
445 | goto fail; |
446 | ||
f2802e7f | 447 | if (!reserve_evntsel_nmi(evntsel_msr)) |
828f0afd DZ |
448 | goto fail1; |
449 | ||
450 | /* Simulator may not support it */ | |
f2802e7f | 451 | if (checking_wrmsrl(evntsel_msr, 0UL)) |
828f0afd | 452 | goto fail2; |
f2802e7f | 453 | wrmsrl(perfctr_msr, 0UL); |
1da177e4 LT |
454 | |
455 | evntsel = K7_EVNTSEL_INT | |
456 | | K7_EVNTSEL_OS | |
457 | | K7_EVNTSEL_USR | |
458 | | K7_NMI_EVENT; | |
459 | ||
f2802e7f DZ |
460 | /* setup the timer */ |
461 | wrmsr(evntsel_msr, evntsel, 0); | |
462 | wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); | |
1da177e4 LT |
463 | apic_write(APIC_LVTPC, APIC_DM_NMI); |
464 | evntsel |= K7_EVNTSEL_ENABLE; | |
f2802e7f DZ |
465 | wrmsr(evntsel_msr, evntsel, 0); |
466 | ||
467 | wd->perfctr_msr = perfctr_msr; | |
468 | wd->evntsel_msr = evntsel_msr; | |
469 | wd->cccr_msr = 0; //unused | |
470 | wd->check_bit = 1ULL<<63; | |
828f0afd DZ |
471 | return 1; |
472 | fail2: | |
f2802e7f | 473 | release_evntsel_nmi(evntsel_msr); |
828f0afd | 474 | fail1: |
f2802e7f | 475 | release_perfctr_nmi(perfctr_msr); |
828f0afd DZ |
476 | fail: |
477 | return 0; | |
1da177e4 LT |
478 | } |
479 | ||
f2802e7f DZ |
480 | static void stop_k7_watchdog(void) |
481 | { | |
482 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
483 | ||
484 | wrmsr(wd->evntsel_msr, 0, 0); | |
485 | ||
486 | release_evntsel_nmi(wd->evntsel_msr); | |
487 | release_perfctr_nmi(wd->perfctr_msr); | |
488 | } | |
489 | ||
490 | /* Note that these events don't tick when the CPU idles. This means | |
491 | the frequency varies with CPU load. */ | |
492 | ||
493 | #define MSR_P4_MISC_ENABLE_PERF_AVAIL (1<<7) | |
494 | #define P4_ESCR_EVENT_SELECT(N) ((N)<<25) | |
495 | #define P4_ESCR_OS (1<<3) | |
496 | #define P4_ESCR_USR (1<<2) | |
497 | #define P4_CCCR_OVF_PMI0 (1<<26) | |
498 | #define P4_CCCR_OVF_PMI1 (1<<27) | |
499 | #define P4_CCCR_THRESHOLD(N) ((N)<<20) | |
500 | #define P4_CCCR_COMPLEMENT (1<<19) | |
501 | #define P4_CCCR_COMPARE (1<<18) | |
502 | #define P4_CCCR_REQUIRED (3<<16) | |
503 | #define P4_CCCR_ESCR_SELECT(N) ((N)<<13) | |
504 | #define P4_CCCR_ENABLE (1<<12) | |
505 | #define P4_CCCR_OVF (1<<31) | |
506 | /* Set up IQ_COUNTER0 to behave like a clock, by having IQ_CCCR0 filter | |
507 | CRU_ESCR0 (with any non-null event selector) through a complemented | |
508 | max threshold. [IA32-Vol3, Section 14.9.9] */ | |
75152114 AK |
509 | |
510 | static int setup_p4_watchdog(void) | |
511 | { | |
f2802e7f DZ |
512 | unsigned int perfctr_msr, evntsel_msr, cccr_msr; |
513 | unsigned int evntsel, cccr_val; | |
75152114 | 514 | unsigned int misc_enable, dummy; |
f2802e7f DZ |
515 | unsigned int ht_num; |
516 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
75152114 | 517 | |
f2802e7f | 518 | rdmsr(MSR_IA32_MISC_ENABLE, misc_enable, dummy); |
75152114 AK |
519 | if (!(misc_enable & MSR_P4_MISC_ENABLE_PERF_AVAIL)) |
520 | return 0; | |
521 | ||
75152114 | 522 | #ifdef CONFIG_SMP |
f2802e7f DZ |
523 | /* detect which hyperthread we are on */ |
524 | if (smp_num_siblings == 2) { | |
525 | unsigned int ebx, apicid; | |
526 | ||
527 | ebx = cpuid_ebx(1); | |
528 | apicid = (ebx >> 24) & 0xff; | |
529 | ht_num = apicid & 1; | |
530 | } else | |
75152114 | 531 | #endif |
f2802e7f DZ |
532 | ht_num = 0; |
533 | ||
534 | /* performance counters are shared resources | |
535 | * assign each hyperthread its own set | |
536 | * (re-use the ESCR0 register, seems safe | |
537 | * and keeps the cccr_val the same) | |
538 | */ | |
539 | if (!ht_num) { | |
540 | /* logical cpu 0 */ | |
541 | perfctr_msr = MSR_P4_IQ_PERFCTR0; | |
542 | evntsel_msr = MSR_P4_CRU_ESCR0; | |
543 | cccr_msr = MSR_P4_IQ_CCCR0; | |
544 | cccr_val = P4_CCCR_OVF_PMI0 | P4_CCCR_ESCR_SELECT(4); | |
545 | } else { | |
546 | /* logical cpu 1 */ | |
547 | perfctr_msr = MSR_P4_IQ_PERFCTR1; | |
548 | evntsel_msr = MSR_P4_CRU_ESCR0; | |
549 | cccr_msr = MSR_P4_IQ_CCCR1; | |
550 | cccr_val = P4_CCCR_OVF_PMI1 | P4_CCCR_ESCR_SELECT(4); | |
551 | } | |
75152114 | 552 | |
f2802e7f | 553 | if (!reserve_perfctr_nmi(perfctr_msr)) |
828f0afd DZ |
554 | goto fail; |
555 | ||
f2802e7f | 556 | if (!reserve_evntsel_nmi(evntsel_msr)) |
828f0afd | 557 | goto fail1; |
75152114 | 558 | |
f2802e7f DZ |
559 | evntsel = P4_ESCR_EVENT_SELECT(0x3F) |
560 | | P4_ESCR_OS | |
561 | | P4_ESCR_USR; | |
562 | ||
563 | cccr_val |= P4_CCCR_THRESHOLD(15) | |
564 | | P4_CCCR_COMPLEMENT | |
565 | | P4_CCCR_COMPARE | |
566 | | P4_CCCR_REQUIRED; | |
567 | ||
568 | wrmsr(evntsel_msr, evntsel, 0); | |
569 | wrmsr(cccr_msr, cccr_val, 0); | |
570 | wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); | |
75152114 | 571 | apic_write(APIC_LVTPC, APIC_DM_NMI); |
f2802e7f DZ |
572 | cccr_val |= P4_CCCR_ENABLE; |
573 | wrmsr(cccr_msr, cccr_val, 0); | |
574 | ||
575 | wd->perfctr_msr = perfctr_msr; | |
576 | wd->evntsel_msr = evntsel_msr; | |
577 | wd->cccr_msr = cccr_msr; | |
578 | wd->check_bit = 1ULL<<39; | |
75152114 | 579 | return 1; |
828f0afd | 580 | fail1: |
f2802e7f | 581 | release_perfctr_nmi(perfctr_msr); |
828f0afd DZ |
582 | fail: |
583 | return 0; | |
75152114 AK |
584 | } |
585 | ||
f2802e7f | 586 | static void stop_p4_watchdog(void) |
1da177e4 | 587 | { |
f2802e7f DZ |
588 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
589 | ||
590 | wrmsr(wd->cccr_msr, 0, 0); | |
591 | wrmsr(wd->evntsel_msr, 0, 0); | |
592 | ||
593 | release_evntsel_nmi(wd->evntsel_msr); | |
594 | release_perfctr_nmi(wd->perfctr_msr); | |
595 | } | |
596 | ||
248dcb2f VP |
597 | #define ARCH_PERFMON_NMI_EVENT_SEL ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL |
598 | #define ARCH_PERFMON_NMI_EVENT_UMASK ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK | |
599 | ||
600 | static int setup_intel_arch_watchdog(void) | |
601 | { | |
602 | unsigned int ebx; | |
603 | union cpuid10_eax eax; | |
604 | unsigned int unused; | |
605 | unsigned int perfctr_msr, evntsel_msr; | |
606 | unsigned int evntsel; | |
607 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
608 | ||
609 | /* | |
610 | * Check whether the Architectural PerfMon supports | |
611 | * Unhalted Core Cycles Event or not. | |
612 | * NOTE: Corresponding bit = 0 in ebx indicates event present. | |
613 | */ | |
614 | cpuid(10, &(eax.full), &ebx, &unused, &unused); | |
615 | if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) || | |
616 | (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT)) | |
617 | goto fail; | |
618 | ||
619 | perfctr_msr = MSR_ARCH_PERFMON_PERFCTR0; | |
620 | evntsel_msr = MSR_ARCH_PERFMON_EVENTSEL0; | |
621 | ||
622 | if (!reserve_perfctr_nmi(perfctr_msr)) | |
623 | goto fail; | |
624 | ||
625 | if (!reserve_evntsel_nmi(evntsel_msr)) | |
626 | goto fail1; | |
627 | ||
628 | wrmsrl(perfctr_msr, 0UL); | |
629 | ||
630 | evntsel = ARCH_PERFMON_EVENTSEL_INT | |
631 | | ARCH_PERFMON_EVENTSEL_OS | |
632 | | ARCH_PERFMON_EVENTSEL_USR | |
633 | | ARCH_PERFMON_NMI_EVENT_SEL | |
634 | | ARCH_PERFMON_NMI_EVENT_UMASK; | |
635 | ||
636 | /* setup the timer */ | |
637 | wrmsr(evntsel_msr, evntsel, 0); | |
638 | wrmsrl(perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); | |
639 | ||
640 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
641 | evntsel |= ARCH_PERFMON_EVENTSEL0_ENABLE; | |
642 | wrmsr(evntsel_msr, evntsel, 0); | |
643 | ||
644 | wd->perfctr_msr = perfctr_msr; | |
645 | wd->evntsel_msr = evntsel_msr; | |
646 | wd->cccr_msr = 0; //unused | |
647 | wd->check_bit = 1ULL << (eax.split.bit_width - 1); | |
648 | return 1; | |
649 | fail1: | |
650 | release_perfctr_nmi(perfctr_msr); | |
651 | fail: | |
652 | return 0; | |
653 | } | |
654 | ||
655 | static void stop_intel_arch_watchdog(void) | |
656 | { | |
657 | unsigned int ebx; | |
658 | union cpuid10_eax eax; | |
659 | unsigned int unused; | |
660 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); | |
661 | ||
662 | /* | |
663 | * Check whether the Architectural PerfMon supports | |
664 | * Unhalted Core Cycles Event or not. | |
665 | * NOTE: Corresponding bit = 0 in ebx indicates event present. | |
666 | */ | |
667 | cpuid(10, &(eax.full), &ebx, &unused, &unused); | |
668 | if ((eax.split.mask_length < (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX+1)) || | |
669 | (ebx & ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT)) | |
670 | return; | |
671 | ||
672 | wrmsr(wd->evntsel_msr, 0, 0); | |
673 | ||
674 | release_evntsel_nmi(wd->evntsel_msr); | |
675 | release_perfctr_nmi(wd->perfctr_msr); | |
676 | } | |
677 | ||
f2802e7f DZ |
678 | void setup_apic_nmi_watchdog(void *unused) |
679 | { | |
4038f901 SL |
680 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
681 | ||
f2802e7f DZ |
682 | /* only support LOCAL and IO APICs for now */ |
683 | if ((nmi_watchdog != NMI_LOCAL_APIC) && | |
684 | (nmi_watchdog != NMI_IO_APIC)) | |
685 | return; | |
686 | ||
4038f901 SL |
687 | if (wd->enabled == 1) |
688 | return; | |
689 | ||
690 | /* cheap hack to support suspend/resume */ | |
691 | /* if cpu0 is not active neither should the other cpus */ | |
692 | if ((smp_processor_id() != 0) && (atomic_read(&nmi_active) <= 0)) | |
693 | return; | |
694 | ||
f2802e7f DZ |
695 | if (nmi_watchdog == NMI_LOCAL_APIC) { |
696 | switch (boot_cpu_data.x86_vendor) { | |
697 | case X86_VENDOR_AMD: | |
698 | if (strstr(boot_cpu_data.x86_model_id, "Screwdriver")) | |
699 | return; | |
700 | if (!setup_k7_watchdog()) | |
701 | return; | |
702 | break; | |
703 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
704 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) { |
705 | if (!setup_intel_arch_watchdog()) | |
706 | return; | |
707 | break; | |
708 | } | |
f2802e7f DZ |
709 | if (!setup_p4_watchdog()) |
710 | return; | |
711 | break; | |
712 | default: | |
75152114 | 713 | return; |
f2802e7f DZ |
714 | } |
715 | } | |
4038f901 | 716 | wd->enabled = 1; |
f2802e7f DZ |
717 | atomic_inc(&nmi_active); |
718 | } | |
75152114 | 719 | |
4038f901 | 720 | void stop_apic_nmi_watchdog(void *unused) |
f2802e7f | 721 | { |
4038f901 SL |
722 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
723 | ||
f2802e7f DZ |
724 | /* only support LOCAL and IO APICs for now */ |
725 | if ((nmi_watchdog != NMI_LOCAL_APIC) && | |
726 | (nmi_watchdog != NMI_IO_APIC)) | |
727 | return; | |
728 | ||
4038f901 SL |
729 | if (wd->enabled == 0) |
730 | return; | |
731 | ||
f2802e7f DZ |
732 | if (nmi_watchdog == NMI_LOCAL_APIC) { |
733 | switch (boot_cpu_data.x86_vendor) { | |
734 | case X86_VENDOR_AMD: | |
735 | if (strstr(boot_cpu_data.x86_model_id, "Screwdriver")) | |
736 | return; | |
737 | stop_k7_watchdog(); | |
738 | break; | |
739 | case X86_VENDOR_INTEL: | |
248dcb2f VP |
740 | if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) { |
741 | stop_intel_arch_watchdog(); | |
742 | break; | |
743 | } | |
f2802e7f DZ |
744 | stop_p4_watchdog(); |
745 | break; | |
746 | default: | |
747 | return; | |
748 | } | |
1da177e4 | 749 | } |
4038f901 | 750 | wd->enabled = 0; |
f2802e7f | 751 | atomic_dec(&nmi_active); |
1da177e4 LT |
752 | } |
753 | ||
754 | /* | |
755 | * the best way to detect whether a CPU has a 'hard lockup' problem | |
756 | * is to check it's local APIC timer IRQ counts. If they are not | |
757 | * changing then that CPU has some problem. | |
758 | * | |
759 | * as these watchdog NMI IRQs are generated on every CPU, we only | |
760 | * have to check the current processor. | |
1da177e4 LT |
761 | */ |
762 | ||
75152114 AK |
763 | static DEFINE_PER_CPU(unsigned, last_irq_sum); |
764 | static DEFINE_PER_CPU(local_t, alert_counter); | |
765 | static DEFINE_PER_CPU(int, nmi_touch); | |
1da177e4 LT |
766 | |
767 | void touch_nmi_watchdog (void) | |
768 | { | |
99019e91 JB |
769 | if (nmi_watchdog > 0) { |
770 | unsigned cpu; | |
1da177e4 | 771 | |
99019e91 JB |
772 | /* |
773 | * Tell other CPUs to reset their alert counters. We cannot | |
774 | * do it ourselves because the alert count increase is not | |
775 | * atomic. | |
776 | */ | |
777 | for_each_present_cpu (cpu) | |
778 | per_cpu(nmi_touch, cpu) = 1; | |
779 | } | |
8446f1d3 IM |
780 | |
781 | touch_softlockup_watchdog(); | |
1da177e4 LT |
782 | } |
783 | ||
3adbbcce | 784 | int __kprobes nmi_watchdog_tick(struct pt_regs * regs, unsigned reason) |
1da177e4 | 785 | { |
75152114 AK |
786 | int sum; |
787 | int touched = 0; | |
bb81a09e | 788 | int cpu = smp_processor_id(); |
f2802e7f DZ |
789 | struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); |
790 | u64 dummy; | |
3adbbcce | 791 | int rc=0; |
f2802e7f DZ |
792 | |
793 | /* check for other users first */ | |
794 | if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) | |
795 | == NOTIFY_STOP) { | |
3adbbcce | 796 | rc = 1; |
f2802e7f DZ |
797 | touched = 1; |
798 | } | |
1da177e4 | 799 | |
1da177e4 | 800 | sum = read_pda(apic_timer_irqs); |
75152114 AK |
801 | if (__get_cpu_var(nmi_touch)) { |
802 | __get_cpu_var(nmi_touch) = 0; | |
803 | touched = 1; | |
804 | } | |
f2802e7f | 805 | |
bb81a09e AM |
806 | if (cpu_isset(cpu, backtrace_mask)) { |
807 | static DEFINE_SPINLOCK(lock); /* Serialise the printks */ | |
808 | ||
809 | spin_lock(&lock); | |
810 | printk("NMI backtrace for cpu %d\n", cpu); | |
811 | dump_stack(); | |
812 | spin_unlock(&lock); | |
813 | cpu_clear(cpu, backtrace_mask); | |
814 | } | |
815 | ||
553f265f AK |
816 | #ifdef CONFIG_X86_MCE |
817 | /* Could check oops_in_progress here too, but it's safer | |
818 | not too */ | |
819 | if (atomic_read(&mce_entry) > 0) | |
820 | touched = 1; | |
821 | #endif | |
f2802e7f | 822 | /* if the apic timer isn't firing, this cpu isn't doing much */ |
75152114 | 823 | if (!touched && __get_cpu_var(last_irq_sum) == sum) { |
1da177e4 LT |
824 | /* |
825 | * Ayiee, looks like this CPU is stuck ... | |
826 | * wait a few IRQs (5 seconds) before doing the oops ... | |
827 | */ | |
75152114 | 828 | local_inc(&__get_cpu_var(alert_counter)); |
f2802e7f | 829 | if (local_read(&__get_cpu_var(alert_counter)) == 5*nmi_hz) |
fac58550 AK |
830 | die_nmi("NMI Watchdog detected LOCKUP on CPU %d\n", regs, |
831 | panic_on_timeout); | |
1da177e4 | 832 | } else { |
75152114 AK |
833 | __get_cpu_var(last_irq_sum) = sum; |
834 | local_set(&__get_cpu_var(alert_counter), 0); | |
1da177e4 | 835 | } |
f2802e7f DZ |
836 | |
837 | /* see if the nmi watchdog went off */ | |
838 | if (wd->enabled) { | |
839 | if (nmi_watchdog == NMI_LOCAL_APIC) { | |
840 | rdmsrl(wd->perfctr_msr, dummy); | |
841 | if (dummy & wd->check_bit){ | |
842 | /* this wasn't a watchdog timer interrupt */ | |
843 | goto done; | |
844 | } | |
845 | ||
846 | /* only Intel uses the cccr msr */ | |
847 | if (wd->cccr_msr != 0) { | |
848 | /* | |
849 | * P4 quirks: | |
850 | * - An overflown perfctr will assert its interrupt | |
851 | * until the OVF flag in its CCCR is cleared. | |
852 | * - LVTPC is masked on interrupt and must be | |
853 | * unmasked by the LVTPC handler. | |
854 | */ | |
855 | rdmsrl(wd->cccr_msr, dummy); | |
856 | dummy &= ~P4_CCCR_OVF; | |
857 | wrmsrl(wd->cccr_msr, dummy); | |
858 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
248dcb2f VP |
859 | } else if (wd->perfctr_msr == MSR_ARCH_PERFMON_PERFCTR0) { |
860 | /* | |
861 | * ArchPerfom/Core Duo needs to re-unmask | |
862 | * the apic vector | |
863 | */ | |
864 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
865 | } | |
f2802e7f DZ |
866 | /* start the cycle over again */ |
867 | wrmsrl(wd->perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); | |
3adbbcce DZ |
868 | rc = 1; |
869 | } else if (nmi_watchdog == NMI_IO_APIC) { | |
870 | /* don't know how to accurately check for this. | |
871 | * just assume it was a watchdog timer interrupt | |
872 | * This matches the old behaviour. | |
873 | */ | |
874 | rc = 1; | |
875 | } else | |
876 | printk(KERN_WARNING "Unknown enabled NMI hardware?!\n"); | |
75152114 | 877 | } |
f2802e7f | 878 | done: |
3adbbcce | 879 | return rc; |
1da177e4 LT |
880 | } |
881 | ||
eddb6fb9 | 882 | asmlinkage __kprobes void do_nmi(struct pt_regs * regs, long error_code) |
1da177e4 | 883 | { |
1da177e4 LT |
884 | nmi_enter(); |
885 | add_pda(__nmi_count,1); | |
3adbbcce | 886 | default_do_nmi(regs); |
1da177e4 LT |
887 | nmi_exit(); |
888 | } | |
889 | ||
3adbbcce DZ |
890 | int do_nmi_callback(struct pt_regs * regs, int cpu) |
891 | { | |
2fbe7b25 DZ |
892 | #ifdef CONFIG_SYSCTL |
893 | if (unknown_nmi_panic) | |
894 | return unknown_nmi_panic_callback(regs, cpu); | |
895 | #endif | |
896 | return 0; | |
1da177e4 LT |
897 | } |
898 | ||
899 | #ifdef CONFIG_SYSCTL | |
900 | ||
901 | static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu) | |
902 | { | |
903 | unsigned char reason = get_nmi_reason(); | |
904 | char buf[64]; | |
905 | ||
2fbe7b25 | 906 | sprintf(buf, "NMI received for unknown reason %02x\n", reason); |
fac58550 | 907 | die_nmi(buf, regs, 1); /* Always panic here */ |
1da177e4 LT |
908 | return 0; |
909 | } | |
910 | ||
407984f1 DZ |
911 | /* |
912 | * proc handler for /proc/sys/kernel/nmi | |
913 | */ | |
914 | int proc_nmi_enabled(struct ctl_table *table, int write, struct file *file, | |
915 | void __user *buffer, size_t *length, loff_t *ppos) | |
916 | { | |
917 | int old_state; | |
918 | ||
919 | nmi_watchdog_enabled = (atomic_read(&nmi_active) > 0) ? 1 : 0; | |
920 | old_state = nmi_watchdog_enabled; | |
921 | proc_dointvec(table, write, file, buffer, length, ppos); | |
922 | if (!!old_state == !!nmi_watchdog_enabled) | |
923 | return 0; | |
924 | ||
925 | if (atomic_read(&nmi_active) < 0) { | |
926 | printk( KERN_WARNING "NMI watchdog is permanently disabled\n"); | |
e33e89ab | 927 | return -EIO; |
407984f1 DZ |
928 | } |
929 | ||
930 | /* if nmi_watchdog is not set yet, then set it */ | |
931 | nmi_watchdog_default(); | |
932 | ||
e33e89ab | 933 | if (nmi_watchdog == NMI_LOCAL_APIC) { |
407984f1 DZ |
934 | if (nmi_watchdog_enabled) |
935 | enable_lapic_nmi_watchdog(); | |
936 | else | |
937 | disable_lapic_nmi_watchdog(); | |
407984f1 | 938 | } else { |
e33e89ab | 939 | printk( KERN_WARNING |
407984f1 DZ |
940 | "NMI watchdog doesn't know what hardware to touch\n"); |
941 | return -EIO; | |
942 | } | |
943 | return 0; | |
944 | } | |
945 | ||
1da177e4 LT |
946 | #endif |
947 | ||
bb81a09e AM |
948 | void __trigger_all_cpu_backtrace(void) |
949 | { | |
950 | int i; | |
951 | ||
952 | backtrace_mask = cpu_online_map; | |
953 | /* Wait for up to 10 seconds for all CPUs to do the backtrace */ | |
954 | for (i = 0; i < 10 * 1000; i++) { | |
955 | if (cpus_empty(backtrace_mask)) | |
956 | break; | |
957 | mdelay(1); | |
958 | } | |
959 | } | |
960 | ||
1da177e4 LT |
961 | EXPORT_SYMBOL(nmi_active); |
962 | EXPORT_SYMBOL(nmi_watchdog); | |
828f0afd DZ |
963 | EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi); |
964 | EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit); | |
965 | EXPORT_SYMBOL(reserve_perfctr_nmi); | |
966 | EXPORT_SYMBOL(release_perfctr_nmi); | |
967 | EXPORT_SYMBOL(reserve_evntsel_nmi); | |
968 | EXPORT_SYMBOL(release_evntsel_nmi); | |
1da177e4 LT |
969 | EXPORT_SYMBOL(disable_timer_nmi_watchdog); |
970 | EXPORT_SYMBOL(enable_timer_nmi_watchdog); | |
971 | EXPORT_SYMBOL(touch_nmi_watchdog); |