Commit | Line | Data |
---|---|---|
352ad25a SR |
1 | /* |
2 | * trace task wakeup timings | |
3 | * | |
4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
5 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> | |
6 | * | |
7 | * Based on code from the latency_tracer, that is: | |
8 | * | |
9 | * Copyright (C) 2004-2006 Ingo Molnar | |
6d49e352 | 10 | * Copyright (C) 2004 Nadia Yvette Chambers |
352ad25a SR |
11 | */ |
12 | #include <linux/module.h> | |
352ad25a SR |
13 | #include <linux/kallsyms.h> |
14 | #include <linux/uaccess.h> | |
15 | #include <linux/ftrace.h> | |
8bd75c77 | 16 | #include <linux/sched/rt.h> |
2d3d891d | 17 | #include <linux/sched/deadline.h> |
ad8d75ff | 18 | #include <trace/events/sched.h> |
352ad25a SR |
19 | #include "trace.h" |
20 | ||
21 | static struct trace_array *wakeup_trace; | |
22 | static int __read_mostly tracer_enabled; | |
23 | ||
24 | static struct task_struct *wakeup_task; | |
25 | static int wakeup_cpu; | |
478142c3 | 26 | static int wakeup_current_cpu; |
352ad25a | 27 | static unsigned wakeup_prio = -1; |
3244351c | 28 | static int wakeup_rt; |
af6ace76 DF |
29 | static int wakeup_dl; |
30 | static int tracing_dl = 0; | |
352ad25a | 31 | |
445c8951 | 32 | static arch_spinlock_t wakeup_lock = |
edc35bd7 | 33 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
352ad25a | 34 | |
7495a5be | 35 | static void wakeup_reset(struct trace_array *tr); |
e309b41d | 36 | static void __wakeup_reset(struct trace_array *tr); |
7495a5be JO |
37 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace); |
38 | static void wakeup_graph_return(struct ftrace_graph_ret *trace); | |
352ad25a | 39 | |
613f04a0 | 40 | static int save_flags; |
328df475 | 41 | static bool function_enabled; |
e9d25fe6 | 42 | |
7495a5be | 43 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 | 44 | static int wakeup_display_graph(struct trace_array *tr, int set); |
729358da | 45 | # define is_graph() (trace_flags & TRACE_ITER_DISPLAY_GRAPH) |
03905582 SRRH |
46 | #else |
47 | static inline int wakeup_display_graph(struct trace_array *tr, int set) | |
48 | { | |
49 | return -EINVAL; | |
50 | } | |
729358da | 51 | # define is_graph() false |
7495a5be | 52 | #endif |
7495a5be | 53 | |
7495a5be | 54 | |
606576ce | 55 | #ifdef CONFIG_FUNCTION_TRACER |
542181d3 | 56 | |
7e18d8e7 | 57 | /* |
542181d3 SR |
58 | * Prologue for the wakeup function tracers. |
59 | * | |
60 | * Returns 1 if it is OK to continue, and preemption | |
61 | * is disabled and data->disabled is incremented. | |
62 | * 0 if the trace is to be ignored, and preemption | |
63 | * is not disabled and data->disabled is | |
64 | * kept the same. | |
65 | * | |
66 | * Note, this function is also used outside this ifdef but | |
67 | * inside the #ifdef of the function graph tracer below. | |
68 | * This is OK, since the function graph tracer is | |
69 | * dependent on the function tracer. | |
7e18d8e7 | 70 | */ |
542181d3 SR |
71 | static int |
72 | func_prolog_preempt_disable(struct trace_array *tr, | |
73 | struct trace_array_cpu **data, | |
74 | int *pc) | |
7e18d8e7 | 75 | { |
7e18d8e7 | 76 | long disabled; |
7e18d8e7 SR |
77 | int cpu; |
78 | ||
79 | if (likely(!wakeup_task)) | |
542181d3 | 80 | return 0; |
7e18d8e7 | 81 | |
542181d3 | 82 | *pc = preempt_count(); |
5168ae50 | 83 | preempt_disable_notrace(); |
7e18d8e7 SR |
84 | |
85 | cpu = raw_smp_processor_id(); | |
478142c3 SR |
86 | if (cpu != wakeup_current_cpu) |
87 | goto out_enable; | |
88 | ||
12883efb | 89 | *data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
542181d3 | 90 | disabled = atomic_inc_return(&(*data)->disabled); |
7e18d8e7 SR |
91 | if (unlikely(disabled != 1)) |
92 | goto out; | |
93 | ||
542181d3 | 94 | return 1; |
7e18d8e7 | 95 | |
542181d3 SR |
96 | out: |
97 | atomic_dec(&(*data)->disabled); | |
7e18d8e7 | 98 | |
542181d3 SR |
99 | out_enable: |
100 | preempt_enable_notrace(); | |
101 | return 0; | |
102 | } | |
103 | ||
104 | /* | |
105 | * wakeup uses its own tracer function to keep the overhead down: | |
106 | */ | |
107 | static void | |
a1e2e31d SR |
108 | wakeup_tracer_call(unsigned long ip, unsigned long parent_ip, |
109 | struct ftrace_ops *op, struct pt_regs *pt_regs) | |
542181d3 SR |
110 | { |
111 | struct trace_array *tr = wakeup_trace; | |
112 | struct trace_array_cpu *data; | |
113 | unsigned long flags; | |
114 | int pc; | |
115 | ||
116 | if (!func_prolog_preempt_disable(tr, &data, &pc)) | |
117 | return; | |
118 | ||
119 | local_irq_save(flags); | |
120 | trace_function(tr, ip, parent_ip, flags, pc); | |
e59494f4 | 121 | local_irq_restore(flags); |
7e18d8e7 | 122 | |
7e18d8e7 | 123 | atomic_dec(&data->disabled); |
5168ae50 | 124 | preempt_enable_notrace(); |
7e18d8e7 | 125 | } |
7e40798f | 126 | #endif /* CONFIG_FUNCTION_TRACER */ |
7495a5be | 127 | |
4104d326 | 128 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) |
7495a5be JO |
129 | { |
130 | int ret; | |
131 | ||
328df475 SRRH |
132 | /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ |
133 | if (function_enabled || (!set && !(trace_flags & TRACE_ITER_FUNCTION))) | |
134 | return 0; | |
135 | ||
136 | if (graph) | |
7495a5be JO |
137 | ret = register_ftrace_graph(&wakeup_graph_return, |
138 | &wakeup_graph_entry); | |
328df475 | 139 | else |
4104d326 | 140 | ret = register_ftrace_function(tr->ops); |
328df475 SRRH |
141 | |
142 | if (!ret) | |
143 | function_enabled = true; | |
144 | ||
145 | return ret; | |
146 | } | |
147 | ||
4104d326 | 148 | static void unregister_wakeup_function(struct trace_array *tr, int graph) |
328df475 SRRH |
149 | { |
150 | if (!function_enabled) | |
151 | return; | |
152 | ||
153 | if (graph) | |
154 | unregister_ftrace_graph(); | |
155 | else | |
4104d326 | 156 | unregister_ftrace_function(tr->ops); |
328df475 SRRH |
157 | |
158 | function_enabled = false; | |
159 | } | |
160 | ||
03905582 | 161 | static int wakeup_function_set(struct trace_array *tr, int set) |
328df475 SRRH |
162 | { |
163 | if (set) | |
4104d326 | 164 | register_wakeup_function(tr, is_graph(), 1); |
328df475 | 165 | else |
4104d326 | 166 | unregister_wakeup_function(tr, is_graph()); |
03905582 | 167 | return 0; |
328df475 SRRH |
168 | } |
169 | ||
bf6065b5 | 170 | static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set) |
328df475 | 171 | { |
bf6065b5 SRRH |
172 | struct tracer *tracer = tr->current_trace; |
173 | ||
328df475 | 174 | if (mask & TRACE_ITER_FUNCTION) |
03905582 SRRH |
175 | return wakeup_function_set(tr, set); |
176 | ||
729358da | 177 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 SRRH |
178 | if (mask & TRACE_ITER_DISPLAY_GRAPH) |
179 | return wakeup_display_graph(tr, set); | |
729358da | 180 | #endif |
328df475 SRRH |
181 | |
182 | return trace_keep_overwrite(tracer, mask, set); | |
183 | } | |
184 | ||
4104d326 | 185 | static int start_func_tracer(struct trace_array *tr, int graph) |
328df475 SRRH |
186 | { |
187 | int ret; | |
188 | ||
4104d326 | 189 | ret = register_wakeup_function(tr, graph, 0); |
7495a5be JO |
190 | |
191 | if (!ret && tracing_is_enabled()) | |
192 | tracer_enabled = 1; | |
193 | else | |
194 | tracer_enabled = 0; | |
195 | ||
196 | return ret; | |
197 | } | |
198 | ||
4104d326 | 199 | static void stop_func_tracer(struct trace_array *tr, int graph) |
7495a5be JO |
200 | { |
201 | tracer_enabled = 0; | |
202 | ||
4104d326 | 203 | unregister_wakeup_function(tr, graph); |
7495a5be JO |
204 | } |
205 | ||
7495a5be | 206 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 | 207 | static int wakeup_display_graph(struct trace_array *tr, int set) |
7495a5be | 208 | { |
7495a5be JO |
209 | if (!(is_graph() ^ set)) |
210 | return 0; | |
211 | ||
4104d326 | 212 | stop_func_tracer(tr, !set); |
7495a5be JO |
213 | |
214 | wakeup_reset(wakeup_trace); | |
6d9b3fa5 | 215 | tr->max_latency = 0; |
7495a5be | 216 | |
4104d326 | 217 | return start_func_tracer(tr, set); |
7495a5be JO |
218 | } |
219 | ||
220 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace) | |
221 | { | |
222 | struct trace_array *tr = wakeup_trace; | |
223 | struct trace_array_cpu *data; | |
224 | unsigned long flags; | |
542181d3 | 225 | int pc, ret = 0; |
7495a5be | 226 | |
542181d3 | 227 | if (!func_prolog_preempt_disable(tr, &data, &pc)) |
7495a5be JO |
228 | return 0; |
229 | ||
7495a5be JO |
230 | local_save_flags(flags); |
231 | ret = __trace_graph_entry(tr, trace, flags, pc); | |
7495a5be | 232 | atomic_dec(&data->disabled); |
7495a5be | 233 | preempt_enable_notrace(); |
542181d3 | 234 | |
7495a5be JO |
235 | return ret; |
236 | } | |
237 | ||
238 | static void wakeup_graph_return(struct ftrace_graph_ret *trace) | |
239 | { | |
240 | struct trace_array *tr = wakeup_trace; | |
241 | struct trace_array_cpu *data; | |
242 | unsigned long flags; | |
542181d3 | 243 | int pc; |
7495a5be | 244 | |
542181d3 | 245 | if (!func_prolog_preempt_disable(tr, &data, &pc)) |
7495a5be JO |
246 | return; |
247 | ||
7495a5be JO |
248 | local_save_flags(flags); |
249 | __trace_graph_return(tr, trace, flags, pc); | |
7495a5be JO |
250 | atomic_dec(&data->disabled); |
251 | ||
7495a5be JO |
252 | preempt_enable_notrace(); |
253 | return; | |
254 | } | |
255 | ||
256 | static void wakeup_trace_open(struct trace_iterator *iter) | |
257 | { | |
258 | if (is_graph()) | |
259 | graph_trace_open(iter); | |
260 | } | |
261 | ||
262 | static void wakeup_trace_close(struct trace_iterator *iter) | |
263 | { | |
264 | if (iter->private) | |
265 | graph_trace_close(iter); | |
266 | } | |
267 | ||
321e68b0 JO |
268 | #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \ |
269 | TRACE_GRAPH_PRINT_ABS_TIME | \ | |
270 | TRACE_GRAPH_PRINT_DURATION) | |
7495a5be JO |
271 | |
272 | static enum print_line_t wakeup_print_line(struct trace_iterator *iter) | |
273 | { | |
274 | /* | |
275 | * In graph mode call the graph tracer output function, | |
276 | * otherwise go with the TRACE_FN event handler | |
277 | */ | |
278 | if (is_graph()) | |
279 | return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS); | |
280 | ||
281 | return TRACE_TYPE_UNHANDLED; | |
282 | } | |
283 | ||
284 | static void wakeup_print_header(struct seq_file *s) | |
285 | { | |
286 | if (is_graph()) | |
287 | print_graph_headers_flags(s, GRAPH_TRACER_FLAGS); | |
288 | else | |
289 | trace_default_header(s); | |
290 | } | |
291 | ||
292 | static void | |
293 | __trace_function(struct trace_array *tr, | |
294 | unsigned long ip, unsigned long parent_ip, | |
295 | unsigned long flags, int pc) | |
296 | { | |
297 | if (is_graph()) | |
298 | trace_graph_function(tr, ip, parent_ip, flags, pc); | |
299 | else | |
300 | trace_function(tr, ip, parent_ip, flags, pc); | |
301 | } | |
302 | #else | |
303 | #define __trace_function trace_function | |
304 | ||
7495a5be JO |
305 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace) |
306 | { | |
307 | return -1; | |
308 | } | |
309 | ||
310 | static enum print_line_t wakeup_print_line(struct trace_iterator *iter) | |
311 | { | |
312 | return TRACE_TYPE_UNHANDLED; | |
313 | } | |
314 | ||
315 | static void wakeup_graph_return(struct ftrace_graph_ret *trace) { } | |
7495a5be JO |
316 | static void wakeup_trace_open(struct trace_iterator *iter) { } |
317 | static void wakeup_trace_close(struct trace_iterator *iter) { } | |
7e9a49ef JO |
318 | |
319 | #ifdef CONFIG_FUNCTION_TRACER | |
320 | static void wakeup_print_header(struct seq_file *s) | |
321 | { | |
322 | trace_default_header(s); | |
323 | } | |
324 | #else | |
325 | static void wakeup_print_header(struct seq_file *s) | |
326 | { | |
327 | trace_latency_header(s); | |
328 | } | |
329 | #endif /* CONFIG_FUNCTION_TRACER */ | |
7495a5be JO |
330 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
331 | ||
352ad25a SR |
332 | /* |
333 | * Should this new latency be reported/recorded? | |
334 | */ | |
6d9b3fa5 | 335 | static int report_latency(struct trace_array *tr, cycle_t delta) |
352ad25a SR |
336 | { |
337 | if (tracing_thresh) { | |
338 | if (delta < tracing_thresh) | |
339 | return 0; | |
340 | } else { | |
6d9b3fa5 | 341 | if (delta <= tr->max_latency) |
352ad25a SR |
342 | return 0; |
343 | } | |
344 | return 1; | |
345 | } | |
346 | ||
38516ab5 SR |
347 | static void |
348 | probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu) | |
478142c3 SR |
349 | { |
350 | if (task != wakeup_task) | |
351 | return; | |
352 | ||
353 | wakeup_current_cpu = cpu; | |
354 | } | |
355 | ||
243f7610 SRRH |
356 | static void |
357 | tracing_sched_switch_trace(struct trace_array *tr, | |
358 | struct task_struct *prev, | |
359 | struct task_struct *next, | |
360 | unsigned long flags, int pc) | |
361 | { | |
2425bcb9 | 362 | struct trace_event_call *call = &event_context_switch; |
243f7610 SRRH |
363 | struct ring_buffer *buffer = tr->trace_buffer.buffer; |
364 | struct ring_buffer_event *event; | |
365 | struct ctx_switch_entry *entry; | |
366 | ||
367 | event = trace_buffer_lock_reserve(buffer, TRACE_CTX, | |
368 | sizeof(*entry), flags, pc); | |
369 | if (!event) | |
370 | return; | |
371 | entry = ring_buffer_event_data(event); | |
372 | entry->prev_pid = prev->pid; | |
373 | entry->prev_prio = prev->prio; | |
374 | entry->prev_state = prev->state; | |
375 | entry->next_pid = next->pid; | |
376 | entry->next_prio = next->prio; | |
377 | entry->next_state = next->state; | |
378 | entry->next_cpu = task_cpu(next); | |
379 | ||
380 | if (!call_filter_check_discard(call, entry, buffer, event)) | |
b7f0c959 | 381 | trace_buffer_unlock_commit(tr, buffer, event, flags, pc); |
243f7610 SRRH |
382 | } |
383 | ||
384 | static void | |
385 | tracing_sched_wakeup_trace(struct trace_array *tr, | |
386 | struct task_struct *wakee, | |
387 | struct task_struct *curr, | |
388 | unsigned long flags, int pc) | |
389 | { | |
2425bcb9 | 390 | struct trace_event_call *call = &event_wakeup; |
243f7610 SRRH |
391 | struct ring_buffer_event *event; |
392 | struct ctx_switch_entry *entry; | |
393 | struct ring_buffer *buffer = tr->trace_buffer.buffer; | |
394 | ||
395 | event = trace_buffer_lock_reserve(buffer, TRACE_WAKE, | |
396 | sizeof(*entry), flags, pc); | |
397 | if (!event) | |
398 | return; | |
399 | entry = ring_buffer_event_data(event); | |
400 | entry->prev_pid = curr->pid; | |
401 | entry->prev_prio = curr->prio; | |
402 | entry->prev_state = curr->state; | |
403 | entry->next_pid = wakee->pid; | |
404 | entry->next_prio = wakee->prio; | |
405 | entry->next_state = wakee->state; | |
406 | entry->next_cpu = task_cpu(wakee); | |
407 | ||
408 | if (!call_filter_check_discard(call, entry, buffer, event)) | |
b7f0c959 | 409 | trace_buffer_unlock_commit(tr, buffer, event, flags, pc); |
243f7610 SRRH |
410 | } |
411 | ||
5b82a1b0 | 412 | static void notrace |
38516ab5 SR |
413 | probe_wakeup_sched_switch(void *ignore, |
414 | struct task_struct *prev, struct task_struct *next) | |
352ad25a | 415 | { |
352ad25a SR |
416 | struct trace_array_cpu *data; |
417 | cycle_t T0, T1, delta; | |
418 | unsigned long flags; | |
419 | long disabled; | |
420 | int cpu; | |
38697053 | 421 | int pc; |
352ad25a | 422 | |
b07c3f19 MD |
423 | tracing_record_cmdline(prev); |
424 | ||
352ad25a SR |
425 | if (unlikely(!tracer_enabled)) |
426 | return; | |
427 | ||
428 | /* | |
429 | * When we start a new trace, we set wakeup_task to NULL | |
430 | * and then set tracer_enabled = 1. We want to make sure | |
431 | * that another CPU does not see the tracer_enabled = 1 | |
432 | * and the wakeup_task with an older task, that might | |
433 | * actually be the same as next. | |
434 | */ | |
435 | smp_rmb(); | |
436 | ||
437 | if (next != wakeup_task) | |
438 | return; | |
439 | ||
38697053 SR |
440 | pc = preempt_count(); |
441 | ||
352ad25a SR |
442 | /* disable local data, not wakeup_cpu data */ |
443 | cpu = raw_smp_processor_id(); | |
12883efb | 444 | disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled); |
352ad25a SR |
445 | if (likely(disabled != 1)) |
446 | goto out; | |
447 | ||
e59494f4 | 448 | local_irq_save(flags); |
0199c4e6 | 449 | arch_spin_lock(&wakeup_lock); |
352ad25a SR |
450 | |
451 | /* We could race with grabbing wakeup_lock */ | |
452 | if (unlikely(!tracer_enabled || next != wakeup_task)) | |
453 | goto out_unlock; | |
454 | ||
9be24414 | 455 | /* The task we are waiting for is waking up */ |
12883efb | 456 | data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu); |
9be24414 | 457 | |
7495a5be | 458 | __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc); |
7be42151 | 459 | tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc); |
352ad25a | 460 | |
352ad25a | 461 | T0 = data->preempt_timestamp; |
750ed1a4 | 462 | T1 = ftrace_now(cpu); |
352ad25a SR |
463 | delta = T1-T0; |
464 | ||
6d9b3fa5 | 465 | if (!report_latency(wakeup_trace, delta)) |
352ad25a SR |
466 | goto out_unlock; |
467 | ||
b5130b1e | 468 | if (likely(!is_tracing_stopped())) { |
6d9b3fa5 | 469 | wakeup_trace->max_latency = delta; |
b5130b1e CE |
470 | update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu); |
471 | } | |
352ad25a | 472 | |
352ad25a | 473 | out_unlock: |
b07c3f19 | 474 | __wakeup_reset(wakeup_trace); |
0199c4e6 | 475 | arch_spin_unlock(&wakeup_lock); |
e59494f4 | 476 | local_irq_restore(flags); |
352ad25a | 477 | out: |
12883efb | 478 | atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled); |
5b82a1b0 MD |
479 | } |
480 | ||
e309b41d | 481 | static void __wakeup_reset(struct trace_array *tr) |
352ad25a | 482 | { |
352ad25a SR |
483 | wakeup_cpu = -1; |
484 | wakeup_prio = -1; | |
af6ace76 | 485 | tracing_dl = 0; |
352ad25a SR |
486 | |
487 | if (wakeup_task) | |
488 | put_task_struct(wakeup_task); | |
489 | ||
490 | wakeup_task = NULL; | |
491 | } | |
492 | ||
e309b41d | 493 | static void wakeup_reset(struct trace_array *tr) |
352ad25a SR |
494 | { |
495 | unsigned long flags; | |
496 | ||
12883efb | 497 | tracing_reset_online_cpus(&tr->trace_buffer); |
2f26ebd5 | 498 | |
e59494f4 | 499 | local_irq_save(flags); |
0199c4e6 | 500 | arch_spin_lock(&wakeup_lock); |
352ad25a | 501 | __wakeup_reset(tr); |
0199c4e6 | 502 | arch_spin_unlock(&wakeup_lock); |
e59494f4 | 503 | local_irq_restore(flags); |
352ad25a SR |
504 | } |
505 | ||
e309b41d | 506 | static void |
fbd705a0 | 507 | probe_wakeup(void *ignore, struct task_struct *p) |
352ad25a | 508 | { |
f8ec1062 | 509 | struct trace_array_cpu *data; |
352ad25a SR |
510 | int cpu = smp_processor_id(); |
511 | unsigned long flags; | |
512 | long disabled; | |
38697053 | 513 | int pc; |
352ad25a | 514 | |
b07c3f19 MD |
515 | if (likely(!tracer_enabled)) |
516 | return; | |
517 | ||
518 | tracing_record_cmdline(p); | |
519 | tracing_record_cmdline(current); | |
520 | ||
af6ace76 DF |
521 | /* |
522 | * Semantic is like this: | |
523 | * - wakeup tracer handles all tasks in the system, independently | |
524 | * from their scheduling class; | |
525 | * - wakeup_rt tracer handles tasks belonging to sched_dl and | |
526 | * sched_rt class; | |
527 | * - wakeup_dl handles tasks belonging to sched_dl class only. | |
528 | */ | |
529 | if (tracing_dl || (wakeup_dl && !dl_task(p)) || | |
530 | (wakeup_rt && !dl_task(p) && !rt_task(p)) || | |
531 | (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio))) | |
352ad25a SR |
532 | return; |
533 | ||
38697053 | 534 | pc = preempt_count(); |
12883efb | 535 | disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled); |
352ad25a SR |
536 | if (unlikely(disabled != 1)) |
537 | goto out; | |
538 | ||
539 | /* interrupts should be off from try_to_wake_up */ | |
0199c4e6 | 540 | arch_spin_lock(&wakeup_lock); |
352ad25a SR |
541 | |
542 | /* check for races. */ | |
af6ace76 DF |
543 | if (!tracer_enabled || tracing_dl || |
544 | (!dl_task(p) && p->prio >= wakeup_prio)) | |
352ad25a SR |
545 | goto out_locked; |
546 | ||
547 | /* reset the trace */ | |
b07c3f19 | 548 | __wakeup_reset(wakeup_trace); |
352ad25a SR |
549 | |
550 | wakeup_cpu = task_cpu(p); | |
478142c3 | 551 | wakeup_current_cpu = wakeup_cpu; |
352ad25a SR |
552 | wakeup_prio = p->prio; |
553 | ||
af6ace76 DF |
554 | /* |
555 | * Once you start tracing a -deadline task, don't bother tracing | |
556 | * another task until the first one wakes up. | |
557 | */ | |
558 | if (dl_task(p)) | |
559 | tracing_dl = 1; | |
560 | else | |
561 | tracing_dl = 0; | |
562 | ||
352ad25a SR |
563 | wakeup_task = p; |
564 | get_task_struct(wakeup_task); | |
565 | ||
566 | local_save_flags(flags); | |
567 | ||
12883efb | 568 | data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu); |
f8ec1062 | 569 | data->preempt_timestamp = ftrace_now(cpu); |
7be42151 | 570 | tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc); |
301fd748 SR |
571 | |
572 | /* | |
573 | * We must be careful in using CALLER_ADDR2. But since wake_up | |
574 | * is not called by an assembly function (where as schedule is) | |
575 | * it should be safe to use it here. | |
576 | */ | |
7495a5be | 577 | __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc); |
352ad25a SR |
578 | |
579 | out_locked: | |
0199c4e6 | 580 | arch_spin_unlock(&wakeup_lock); |
352ad25a | 581 | out: |
12883efb | 582 | atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled); |
352ad25a SR |
583 | } |
584 | ||
e309b41d | 585 | static void start_wakeup_tracer(struct trace_array *tr) |
352ad25a | 586 | { |
5b82a1b0 MD |
587 | int ret; |
588 | ||
38516ab5 | 589 | ret = register_trace_sched_wakeup(probe_wakeup, NULL); |
5b82a1b0 | 590 | if (ret) { |
b07c3f19 | 591 | pr_info("wakeup trace: Couldn't activate tracepoint" |
5b82a1b0 MD |
592 | " probe to kernel_sched_wakeup\n"); |
593 | return; | |
594 | } | |
595 | ||
38516ab5 | 596 | ret = register_trace_sched_wakeup_new(probe_wakeup, NULL); |
5b82a1b0 | 597 | if (ret) { |
b07c3f19 | 598 | pr_info("wakeup trace: Couldn't activate tracepoint" |
5b82a1b0 MD |
599 | " probe to kernel_sched_wakeup_new\n"); |
600 | goto fail_deprobe; | |
601 | } | |
602 | ||
38516ab5 | 603 | ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL); |
5b82a1b0 | 604 | if (ret) { |
b07c3f19 | 605 | pr_info("sched trace: Couldn't activate tracepoint" |
73d8b8bc | 606 | " probe to kernel_sched_switch\n"); |
5b82a1b0 MD |
607 | goto fail_deprobe_wake_new; |
608 | } | |
609 | ||
38516ab5 | 610 | ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL); |
478142c3 SR |
611 | if (ret) { |
612 | pr_info("wakeup trace: Couldn't activate tracepoint" | |
613 | " probe to kernel_sched_migrate_task\n"); | |
614 | return; | |
615 | } | |
616 | ||
352ad25a SR |
617 | wakeup_reset(tr); |
618 | ||
619 | /* | |
620 | * Don't let the tracer_enabled = 1 show up before | |
621 | * the wakeup_task is reset. This may be overkill since | |
622 | * wakeup_reset does a spin_unlock after setting the | |
623 | * wakeup_task to NULL, but I want to be safe. | |
624 | * This is a slow path anyway. | |
625 | */ | |
626 | smp_wmb(); | |
627 | ||
4104d326 | 628 | if (start_func_tracer(tr, is_graph())) |
7495a5be | 629 | printk(KERN_ERR "failed to start wakeup tracer\n"); |
ad591240 | 630 | |
352ad25a | 631 | return; |
5b82a1b0 | 632 | fail_deprobe_wake_new: |
38516ab5 | 633 | unregister_trace_sched_wakeup_new(probe_wakeup, NULL); |
5b82a1b0 | 634 | fail_deprobe: |
38516ab5 | 635 | unregister_trace_sched_wakeup(probe_wakeup, NULL); |
352ad25a SR |
636 | } |
637 | ||
e309b41d | 638 | static void stop_wakeup_tracer(struct trace_array *tr) |
352ad25a SR |
639 | { |
640 | tracer_enabled = 0; | |
4104d326 | 641 | stop_func_tracer(tr, is_graph()); |
38516ab5 SR |
642 | unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL); |
643 | unregister_trace_sched_wakeup_new(probe_wakeup, NULL); | |
644 | unregister_trace_sched_wakeup(probe_wakeup, NULL); | |
645 | unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL); | |
352ad25a SR |
646 | } |
647 | ||
65daaca7 SRRH |
648 | static bool wakeup_busy; |
649 | ||
3244351c | 650 | static int __wakeup_tracer_init(struct trace_array *tr) |
352ad25a | 651 | { |
613f04a0 SRRH |
652 | save_flags = trace_flags; |
653 | ||
654 | /* non overwrite screws up the latency tracers */ | |
2b6080f2 SR |
655 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1); |
656 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1); | |
e9d25fe6 | 657 | |
6d9b3fa5 | 658 | tr->max_latency = 0; |
352ad25a | 659 | wakeup_trace = tr; |
4104d326 | 660 | ftrace_init_array_ops(tr, wakeup_tracer_call); |
c76f0694 | 661 | start_wakeup_tracer(tr); |
65daaca7 SRRH |
662 | |
663 | wakeup_busy = true; | |
1c80025a | 664 | return 0; |
352ad25a SR |
665 | } |
666 | ||
3244351c SR |
667 | static int wakeup_tracer_init(struct trace_array *tr) |
668 | { | |
65daaca7 SRRH |
669 | if (wakeup_busy) |
670 | return -EBUSY; | |
671 | ||
af6ace76 | 672 | wakeup_dl = 0; |
3244351c SR |
673 | wakeup_rt = 0; |
674 | return __wakeup_tracer_init(tr); | |
675 | } | |
676 | ||
677 | static int wakeup_rt_tracer_init(struct trace_array *tr) | |
678 | { | |
65daaca7 SRRH |
679 | if (wakeup_busy) |
680 | return -EBUSY; | |
681 | ||
af6ace76 | 682 | wakeup_dl = 0; |
3244351c SR |
683 | wakeup_rt = 1; |
684 | return __wakeup_tracer_init(tr); | |
685 | } | |
686 | ||
af6ace76 DF |
687 | static int wakeup_dl_tracer_init(struct trace_array *tr) |
688 | { | |
65daaca7 SRRH |
689 | if (wakeup_busy) |
690 | return -EBUSY; | |
691 | ||
af6ace76 DF |
692 | wakeup_dl = 1; |
693 | wakeup_rt = 0; | |
694 | return __wakeup_tracer_init(tr); | |
695 | } | |
696 | ||
e309b41d | 697 | static void wakeup_tracer_reset(struct trace_array *tr) |
352ad25a | 698 | { |
613f04a0 SRRH |
699 | int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT; |
700 | int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE; | |
701 | ||
c76f0694 SR |
702 | stop_wakeup_tracer(tr); |
703 | /* make sure we put back any tasks we are tracing */ | |
704 | wakeup_reset(tr); | |
e9d25fe6 | 705 | |
2b6080f2 SR |
706 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag); |
707 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag); | |
4104d326 | 708 | ftrace_reset_array_ops(tr); |
65daaca7 | 709 | wakeup_busy = false; |
352ad25a SR |
710 | } |
711 | ||
9036990d SR |
712 | static void wakeup_tracer_start(struct trace_array *tr) |
713 | { | |
714 | wakeup_reset(tr); | |
715 | tracer_enabled = 1; | |
9036990d SR |
716 | } |
717 | ||
718 | static void wakeup_tracer_stop(struct trace_array *tr) | |
719 | { | |
720 | tracer_enabled = 0; | |
352ad25a SR |
721 | } |
722 | ||
723 | static struct tracer wakeup_tracer __read_mostly = | |
724 | { | |
725 | .name = "wakeup", | |
726 | .init = wakeup_tracer_init, | |
727 | .reset = wakeup_tracer_reset, | |
9036990d SR |
728 | .start = wakeup_tracer_start, |
729 | .stop = wakeup_tracer_stop, | |
f43c738b | 730 | .print_max = true, |
7495a5be JO |
731 | .print_header = wakeup_print_header, |
732 | .print_line = wakeup_print_line, | |
328df475 | 733 | .flag_changed = wakeup_flag_changed, |
60a11774 SR |
734 | #ifdef CONFIG_FTRACE_SELFTEST |
735 | .selftest = trace_selftest_startup_wakeup, | |
736 | #endif | |
7495a5be JO |
737 | .open = wakeup_trace_open, |
738 | .close = wakeup_trace_close, | |
65daaca7 | 739 | .allow_instances = true, |
f43c738b | 740 | .use_max_tr = true, |
352ad25a SR |
741 | }; |
742 | ||
3244351c SR |
743 | static struct tracer wakeup_rt_tracer __read_mostly = |
744 | { | |
745 | .name = "wakeup_rt", | |
746 | .init = wakeup_rt_tracer_init, | |
747 | .reset = wakeup_tracer_reset, | |
748 | .start = wakeup_tracer_start, | |
749 | .stop = wakeup_tracer_stop, | |
f43c738b | 750 | .print_max = true, |
7495a5be JO |
751 | .print_header = wakeup_print_header, |
752 | .print_line = wakeup_print_line, | |
328df475 | 753 | .flag_changed = wakeup_flag_changed, |
3244351c SR |
754 | #ifdef CONFIG_FTRACE_SELFTEST |
755 | .selftest = trace_selftest_startup_wakeup, | |
756 | #endif | |
7495a5be JO |
757 | .open = wakeup_trace_open, |
758 | .close = wakeup_trace_close, | |
65daaca7 | 759 | .allow_instances = true, |
f43c738b | 760 | .use_max_tr = true, |
3244351c SR |
761 | }; |
762 | ||
af6ace76 DF |
763 | static struct tracer wakeup_dl_tracer __read_mostly = |
764 | { | |
765 | .name = "wakeup_dl", | |
766 | .init = wakeup_dl_tracer_init, | |
767 | .reset = wakeup_tracer_reset, | |
768 | .start = wakeup_tracer_start, | |
769 | .stop = wakeup_tracer_stop, | |
af6ace76 DF |
770 | .print_max = true, |
771 | .print_header = wakeup_print_header, | |
772 | .print_line = wakeup_print_line, | |
af6ace76 DF |
773 | .flag_changed = wakeup_flag_changed, |
774 | #ifdef CONFIG_FTRACE_SELFTEST | |
775 | .selftest = trace_selftest_startup_wakeup, | |
776 | #endif | |
777 | .open = wakeup_trace_open, | |
778 | .close = wakeup_trace_close, | |
779 | .use_max_tr = true, | |
780 | }; | |
781 | ||
352ad25a SR |
782 | __init static int init_wakeup_tracer(void) |
783 | { | |
784 | int ret; | |
785 | ||
786 | ret = register_tracer(&wakeup_tracer); | |
787 | if (ret) | |
788 | return ret; | |
789 | ||
3244351c SR |
790 | ret = register_tracer(&wakeup_rt_tracer); |
791 | if (ret) | |
792 | return ret; | |
793 | ||
af6ace76 DF |
794 | ret = register_tracer(&wakeup_dl_tracer); |
795 | if (ret) | |
796 | return ret; | |
797 | ||
352ad25a SR |
798 | return 0; |
799 | } | |
6f415672 | 800 | core_initcall(init_wakeup_tracer); |