2 * Detect hard and soft lockups on a system
4 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
6 * this code detects hard lockups: incidents in where on a CPU
7 * the kernel does not respond to anything except NMI.
9 * Note: Most of this code is borrowed heavily from softlockup.c,
10 * so thanks to Ingo for the initial implementation.
11 * Some chunks also taken from arch/x86/kernel/apic/nmi.c, thanks
12 * to those contributors as well.
16 #include <linux/cpu.h>
17 #include <linux/nmi.h>
18 #include <linux/init.h>
19 #include <linux/delay.h>
20 #include <linux/freezer.h>
21 #include <linux/kthread.h>
22 #include <linux/lockdep.h>
23 #include <linux/notifier.h>
24 #include <linux/module.h>
25 #include <linux/sysctl.h>
27 #include <asm/irq_regs.h>
28 #include <linux/perf_event.h>
31 int __read_mostly softlockup_thresh
= 60;
33 static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts
);
34 static DEFINE_PER_CPU(struct task_struct
*, softlockup_watchdog
);
35 static DEFINE_PER_CPU(struct hrtimer
, watchdog_hrtimer
);
36 static DEFINE_PER_CPU(bool, softlockup_touch_sync
);
37 static DEFINE_PER_CPU(bool, soft_watchdog_warn
);
38 #ifdef CONFIG_HARDLOCKUP_DETECTOR
39 static DEFINE_PER_CPU(bool, hard_watchdog_warn
);
40 static DEFINE_PER_CPU(bool, watchdog_nmi_touch
);
41 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts
);
42 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved
);
43 static DEFINE_PER_CPU(struct perf_event
*, watchdog_ev
);
46 static int no_watchdog
;
51 * Should we panic when a soft-lockup or hard-lockup occurs:
53 #ifdef CONFIG_HARDLOCKUP_DETECTOR
54 static int hardlockup_panic
;
56 static int __init
hardlockup_panic_setup(char *str
)
58 if (!strncmp(str
, "panic", 5))
60 else if (!strncmp(str
, "0", 1))
64 __setup("nmi_watchdog=", hardlockup_panic_setup
);
67 unsigned int __read_mostly softlockup_panic
=
68 CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE
;
70 static int __init
softlockup_panic_setup(char *str
)
72 softlockup_panic
= simple_strtoul(str
, NULL
, 0);
76 __setup("softlockup_panic=", softlockup_panic_setup
);
78 static int __init
nowatchdog_setup(char *str
)
83 __setup("nowatchdog", nowatchdog_setup
);
86 static int __init
nosoftlockup_setup(char *str
)
91 __setup("nosoftlockup", nosoftlockup_setup
);
96 * Returns seconds, approximately. We don't need nanosecond
97 * resolution, and we don't need to waste time with a big divide when
100 static unsigned long get_timestamp(int this_cpu
)
102 return cpu_clock(this_cpu
) >> 30LL; /* 2^30 ~= 10^9 */
105 static unsigned long get_sample_period(void)
108 * convert softlockup_thresh from seconds to ns
109 * the divide by 5 is to give hrtimer 5 chances to
110 * increment before the hardlockup detector generates
113 return softlockup_thresh
/ 5 * NSEC_PER_SEC
;
116 /* Commands for resetting the watchdog */
117 static void __touch_watchdog(void)
119 int this_cpu
= smp_processor_id();
121 __get_cpu_var(watchdog_touch_ts
) = get_timestamp(this_cpu
);
124 void touch_softlockup_watchdog(void)
126 __raw_get_cpu_var(watchdog_touch_ts
) = 0;
128 EXPORT_SYMBOL(touch_softlockup_watchdog
);
130 void touch_all_softlockup_watchdogs(void)
135 * this is done lockless
136 * do we care if a 0 races with a timestamp?
137 * all it means is the softlock check starts one cycle later
139 for_each_online_cpu(cpu
)
140 per_cpu(watchdog_touch_ts
, cpu
) = 0;
143 #ifdef CONFIG_HARDLOCKUP_DETECTOR
144 void touch_nmi_watchdog(void)
146 if (watchdog_enabled
) {
149 for_each_present_cpu(cpu
) {
150 if (per_cpu(watchdog_nmi_touch
, cpu
) != true)
151 per_cpu(watchdog_nmi_touch
, cpu
) = true;
154 touch_softlockup_watchdog();
156 EXPORT_SYMBOL(touch_nmi_watchdog
);
160 void touch_softlockup_watchdog_sync(void)
162 __raw_get_cpu_var(softlockup_touch_sync
) = true;
163 __raw_get_cpu_var(watchdog_touch_ts
) = 0;
166 #ifdef CONFIG_HARDLOCKUP_DETECTOR
167 /* watchdog detector functions */
168 static int is_hardlockup(void)
170 unsigned long hrint
= __get_cpu_var(hrtimer_interrupts
);
172 if (__get_cpu_var(hrtimer_interrupts_saved
) == hrint
)
175 __get_cpu_var(hrtimer_interrupts_saved
) = hrint
;
180 static int is_softlockup(unsigned long touch_ts
)
182 unsigned long now
= get_timestamp(smp_processor_id());
184 /* Warn about unreasonable delays: */
185 if (time_after(now
, touch_ts
+ softlockup_thresh
))
186 return now
- touch_ts
;
191 #ifdef CONFIG_HARDLOCKUP_DETECTOR
192 static struct perf_event_attr wd_hw_attr
= {
193 .type
= PERF_TYPE_HARDWARE
,
194 .config
= PERF_COUNT_HW_CPU_CYCLES
,
195 .size
= sizeof(struct perf_event_attr
),
200 /* Callback function for perf event subsystem */
201 static void watchdog_overflow_callback(struct perf_event
*event
, int nmi
,
202 struct perf_sample_data
*data
,
203 struct pt_regs
*regs
)
205 /* Ensure the watchdog never gets throttled */
206 event
->hw
.interrupts
= 0;
208 if (__get_cpu_var(watchdog_nmi_touch
) == true) {
209 __get_cpu_var(watchdog_nmi_touch
) = false;
213 /* check for a hardlockup
214 * This is done by making sure our timer interrupt
215 * is incrementing. The timer interrupt should have
216 * fired multiple times before we overflow'd. If it hasn't
217 * then this is a good indication the cpu is stuck
219 if (is_hardlockup()) {
220 int this_cpu
= smp_processor_id();
222 /* only print hardlockups once */
223 if (__get_cpu_var(hard_watchdog_warn
) == true)
226 if (hardlockup_panic
)
227 panic("Watchdog detected hard LOCKUP on cpu %d", this_cpu
);
229 WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu
);
231 __get_cpu_var(hard_watchdog_warn
) = true;
235 __get_cpu_var(hard_watchdog_warn
) = false;
238 static void watchdog_interrupt_count(void)
240 __get_cpu_var(hrtimer_interrupts
)++;
243 static inline void watchdog_interrupt_count(void) { return; }
244 #endif /* CONFIG_HARDLOCKUP_DETECTOR */
246 /* watchdog kicker functions */
247 static enum hrtimer_restart
watchdog_timer_fn(struct hrtimer
*hrtimer
)
249 unsigned long touch_ts
= __get_cpu_var(watchdog_touch_ts
);
250 struct pt_regs
*regs
= get_irq_regs();
253 /* kick the hardlockup detector */
254 watchdog_interrupt_count();
256 /* kick the softlockup detector */
257 wake_up_process(__get_cpu_var(softlockup_watchdog
));
260 hrtimer_forward_now(hrtimer
, ns_to_ktime(get_sample_period()));
263 if (unlikely(__get_cpu_var(softlockup_touch_sync
))) {
265 * If the time stamp was touched atomically
266 * make sure the scheduler tick is up to date.
268 __get_cpu_var(softlockup_touch_sync
) = false;
272 return HRTIMER_RESTART
;
275 /* check for a softlockup
276 * This is done by making sure a high priority task is
277 * being scheduled. The task touches the watchdog to
278 * indicate it is getting cpu time. If it hasn't then
279 * this is a good indication some task is hogging the cpu
281 duration
= is_softlockup(touch_ts
);
282 if (unlikely(duration
)) {
284 if (__get_cpu_var(soft_watchdog_warn
) == true)
285 return HRTIMER_RESTART
;
287 printk(KERN_ERR
"BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
288 smp_processor_id(), duration
,
289 current
->comm
, task_pid_nr(current
));
291 print_irqtrace_events(current
);
297 if (softlockup_panic
)
298 panic("softlockup: hung tasks");
299 __get_cpu_var(soft_watchdog_warn
) = true;
301 __get_cpu_var(soft_watchdog_warn
) = false;
303 return HRTIMER_RESTART
;
308 * The watchdog thread - touches the timestamp.
310 static int watchdog(void *unused
)
312 struct sched_param param
= { .sched_priority
= MAX_RT_PRIO
-1 };
313 struct hrtimer
*hrtimer
= &__raw_get_cpu_var(watchdog_hrtimer
);
315 sched_setscheduler(current
, SCHED_FIFO
, ¶m
);
317 /* initialize timestamp */
320 /* kick off the timer for the hardlockup detector */
321 /* done here because hrtimer_start can only pin to smp_processor_id() */
322 hrtimer_start(hrtimer
, ns_to_ktime(get_sample_period()),
323 HRTIMER_MODE_REL_PINNED
);
325 set_current_state(TASK_INTERRUPTIBLE
);
327 * Run briefly once per second to reset the softlockup timestamp.
328 * If this gets delayed for more than 60 seconds then the
329 * debug-printout triggers in watchdog_timer_fn().
331 while (!kthread_should_stop()) {
335 if (kthread_should_stop())
338 set_current_state(TASK_INTERRUPTIBLE
);
340 __set_current_state(TASK_RUNNING
);
346 #ifdef CONFIG_HARDLOCKUP_DETECTOR
347 static int watchdog_nmi_enable(int cpu
)
349 struct perf_event_attr
*wd_attr
;
350 struct perf_event
*event
= per_cpu(watchdog_ev
, cpu
);
352 /* is it already setup and enabled? */
353 if (event
&& event
->state
> PERF_EVENT_STATE_OFF
)
356 /* it is setup but not enabled */
360 /* Try to register using hardware perf events */
361 wd_attr
= &wd_hw_attr
;
362 wd_attr
->sample_period
= hw_nmi_get_sample_period();
363 event
= perf_event_create_kernel_counter(wd_attr
, cpu
, NULL
, watchdog_overflow_callback
);
364 if (!IS_ERR(event
)) {
365 printk(KERN_INFO
"NMI watchdog enabled, takes one hw-pmu counter.\n");
369 printk(KERN_ERR
"NMI watchdog failed to create perf event on cpu%i: %p\n", cpu
, event
);
370 return PTR_ERR(event
);
374 per_cpu(watchdog_ev
, cpu
) = event
;
376 perf_event_enable(per_cpu(watchdog_ev
, cpu
));
381 static void watchdog_nmi_disable(int cpu
)
383 struct perf_event
*event
= per_cpu(watchdog_ev
, cpu
);
386 perf_event_disable(event
);
387 per_cpu(watchdog_ev
, cpu
) = NULL
;
389 /* should be in cleanup, but blocks oprofile */
390 perf_event_release_kernel(event
);
395 static int watchdog_nmi_enable(int cpu
) { return 0; }
396 static void watchdog_nmi_disable(int cpu
) { return; }
397 #endif /* CONFIG_HARDLOCKUP_DETECTOR */
399 /* prepare/enable/disable routines */
400 static int watchdog_prepare_cpu(int cpu
)
402 struct hrtimer
*hrtimer
= &per_cpu(watchdog_hrtimer
, cpu
);
404 WARN_ON(per_cpu(softlockup_watchdog
, cpu
));
405 hrtimer_init(hrtimer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
406 hrtimer
->function
= watchdog_timer_fn
;
411 static int watchdog_enable(int cpu
)
413 struct task_struct
*p
= per_cpu(softlockup_watchdog
, cpu
);
416 /* enable the perf event */
417 err
= watchdog_nmi_enable(cpu
);
421 /* create the watchdog thread */
423 p
= kthread_create(watchdog
, (void *)(unsigned long)cpu
, "watchdog/%d", cpu
);
425 printk(KERN_ERR
"softlockup watchdog for %i failed\n", cpu
);
428 kthread_bind(p
, cpu
);
429 per_cpu(watchdog_touch_ts
, cpu
) = 0;
430 per_cpu(softlockup_watchdog
, cpu
) = p
;
434 /* if any cpu succeeds, watchdog is considered enabled for the system */
435 watchdog_enabled
= 1;
440 static void watchdog_disable(int cpu
)
442 struct task_struct
*p
= per_cpu(softlockup_watchdog
, cpu
);
443 struct hrtimer
*hrtimer
= &per_cpu(watchdog_hrtimer
, cpu
);
446 * cancel the timer first to stop incrementing the stats
447 * and waking up the kthread
449 hrtimer_cancel(hrtimer
);
451 /* disable the perf event */
452 watchdog_nmi_disable(cpu
);
454 /* stop the watchdog thread */
456 per_cpu(softlockup_watchdog
, cpu
) = NULL
;
461 static void watchdog_enable_all_cpus(void)
466 for_each_online_cpu(cpu
)
467 result
+= watchdog_enable(cpu
);
470 printk(KERN_ERR
"watchdog: failed to be enabled on some cpus\n");
474 static void watchdog_disable_all_cpus(void)
481 for_each_online_cpu(cpu
)
482 watchdog_disable(cpu
);
484 /* if all watchdogs are disabled, then they are disabled for the system */
485 watchdog_enabled
= 0;
489 /* sysctl functions */
492 * proc handler for /proc/sys/kernel/nmi_watchdog
495 int proc_dowatchdog_enabled(struct ctl_table
*table
, int write
,
496 void __user
*buffer
, size_t *length
, loff_t
*ppos
)
498 proc_dointvec(table
, write
, buffer
, length
, ppos
);
500 if (watchdog_enabled
)
501 watchdog_enable_all_cpus();
503 watchdog_disable_all_cpus();
507 int proc_dowatchdog_thresh(struct ctl_table
*table
, int write
,
509 size_t *lenp
, loff_t
*ppos
)
511 return proc_dointvec_minmax(table
, write
, buffer
, lenp
, ppos
);
513 #endif /* CONFIG_SYSCTL */
517 * Create/destroy watchdog threads as CPUs come and go:
520 cpu_callback(struct notifier_block
*nfb
, unsigned long action
, void *hcpu
)
522 int hotcpu
= (unsigned long)hcpu
;
527 case CPU_UP_PREPARE_FROZEN
:
528 err
= watchdog_prepare_cpu(hotcpu
);
531 case CPU_ONLINE_FROZEN
:
532 err
= watchdog_enable(hotcpu
);
534 #ifdef CONFIG_HOTPLUG_CPU
535 case CPU_UP_CANCELED
:
536 case CPU_UP_CANCELED_FROZEN
:
537 watchdog_disable(hotcpu
);
540 case CPU_DEAD_FROZEN
:
541 watchdog_disable(hotcpu
);
543 #endif /* CONFIG_HOTPLUG_CPU */
545 return notifier_from_errno(err
);
548 static struct notifier_block __cpuinitdata cpu_nfb
= {
549 .notifier_call
= cpu_callback
552 void __init
lockup_detector_init(void)
554 void *cpu
= (void *)(long)smp_processor_id();
560 err
= cpu_callback(&cpu_nfb
, CPU_UP_PREPARE
, cpu
);
561 WARN_ON(notifier_to_errno(err
));
563 cpu_callback(&cpu_nfb
, CPU_ONLINE
, cpu
);
564 register_cpu_notifier(&cpu_nfb
);