3 * Function graph tracer.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
9 #include <linux/debugfs.h>
10 #include <linux/uaccess.h>
11 #include <linux/ftrace.h>
12 #include <linux/slab.h>
16 #include "trace_output.h"
18 static bool kill_ftrace_graph
;
21 * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called
23 * ftrace_graph_stop() is called when a severe error is detected in
24 * the function graph tracing. This function is called by the critical
25 * paths of function graph to keep those paths from doing any more harm.
27 bool ftrace_graph_is_dead(void)
29 return kill_ftrace_graph
;
33 * ftrace_graph_stop - set to permanently disable function graph tracincg
35 * In case of an error int function graph tracing, this is called
36 * to try to keep function graph tracing from causing any more harm.
37 * Usually this is pretty severe and this is called to try to at least
38 * get a warning out to the user.
40 void ftrace_graph_stop(void)
42 kill_ftrace_graph
= true;
44 * ftrace_stop() will be removed when all archs are updated to
45 * use ftrace_graph_is_dead()
50 /* When set, irq functions will be ignored */
51 static int ftrace_graph_skip_irqs
;
53 struct fgraph_cpu_data
{
58 unsigned long enter_funcs
[FTRACE_RETFUNC_DEPTH
];
62 struct fgraph_cpu_data __percpu
*cpu_data
;
64 /* Place to preserve last processed entry. */
65 struct ftrace_graph_ent_entry ent
;
66 struct ftrace_graph_ret_entry ret
;
71 #define TRACE_GRAPH_INDENT 2
73 static unsigned int max_depth
;
75 static struct tracer_opt trace_opts
[] = {
76 /* Display overruns? (for self-debug purpose) */
77 { TRACER_OPT(funcgraph
-overrun
, TRACE_GRAPH_PRINT_OVERRUN
) },
79 { TRACER_OPT(funcgraph
-cpu
, TRACE_GRAPH_PRINT_CPU
) },
80 /* Display Overhead ? */
81 { TRACER_OPT(funcgraph
-overhead
, TRACE_GRAPH_PRINT_OVERHEAD
) },
82 /* Display proc name/pid */
83 { TRACER_OPT(funcgraph
-proc
, TRACE_GRAPH_PRINT_PROC
) },
84 /* Display duration of execution */
85 { TRACER_OPT(funcgraph
-duration
, TRACE_GRAPH_PRINT_DURATION
) },
86 /* Display absolute time of an entry */
87 { TRACER_OPT(funcgraph
-abstime
, TRACE_GRAPH_PRINT_ABS_TIME
) },
88 /* Display interrupts */
89 { TRACER_OPT(funcgraph
-irqs
, TRACE_GRAPH_PRINT_IRQS
) },
90 /* Display function name after trailing } */
91 { TRACER_OPT(funcgraph
-tail
, TRACE_GRAPH_PRINT_TAIL
) },
95 static struct tracer_flags tracer_flags
= {
96 /* Don't display overruns, proc, or tail by default */
97 .val
= TRACE_GRAPH_PRINT_CPU
| TRACE_GRAPH_PRINT_OVERHEAD
|
98 TRACE_GRAPH_PRINT_DURATION
| TRACE_GRAPH_PRINT_IRQS
,
102 static struct trace_array
*graph_array
;
105 * DURATION column is being also used to display IRQ signs,
106 * following values are used by print_graph_irq and others
107 * to fill in space into DURATION column.
110 FLAGS_FILL_FULL
= 1 << TRACE_GRAPH_PRINT_FILL_SHIFT
,
111 FLAGS_FILL_START
= 2 << TRACE_GRAPH_PRINT_FILL_SHIFT
,
112 FLAGS_FILL_END
= 3 << TRACE_GRAPH_PRINT_FILL_SHIFT
,
115 static enum print_line_t
116 print_graph_duration(unsigned long long duration
, struct trace_seq
*s
,
119 /* Add a function return address to the trace stack on thread info.*/
121 ftrace_push_return_trace(unsigned long ret
, unsigned long func
, int *depth
,
122 unsigned long frame_pointer
)
124 unsigned long long calltime
;
127 if (unlikely(ftrace_graph_is_dead()))
130 if (!current
->ret_stack
)
134 * We must make sure the ret_stack is tested before we read
139 /* The return trace stack is full */
140 if (current
->curr_ret_stack
== FTRACE_RETFUNC_DEPTH
- 1) {
141 atomic_inc(¤t
->trace_overrun
);
146 * The curr_ret_stack is an index to ftrace return stack of
147 * current task. Its value should be in [0, FTRACE_RETFUNC_
148 * DEPTH) when the function graph tracer is used. To support
149 * filtering out specific functions, it makes the index
150 * negative by subtracting huge value (FTRACE_NOTRACE_DEPTH)
151 * so when it sees a negative index the ftrace will ignore
152 * the record. And the index gets recovered when returning
153 * from the filtered function by adding the FTRACE_NOTRACE_
154 * DEPTH and then it'll continue to record functions normally.
156 * The curr_ret_stack is initialized to -1 and get increased
157 * in this function. So it can be less than -1 only if it was
158 * filtered out via ftrace_graph_notrace_addr() which can be
159 * set from set_graph_notrace file in debugfs by user.
161 if (current
->curr_ret_stack
< -1)
164 calltime
= trace_clock_local();
166 index
= ++current
->curr_ret_stack
;
167 if (ftrace_graph_notrace_addr(func
))
168 current
->curr_ret_stack
-= FTRACE_NOTRACE_DEPTH
;
170 current
->ret_stack
[index
].ret
= ret
;
171 current
->ret_stack
[index
].func
= func
;
172 current
->ret_stack
[index
].calltime
= calltime
;
173 current
->ret_stack
[index
].subtime
= 0;
174 current
->ret_stack
[index
].fp
= frame_pointer
;
175 *depth
= current
->curr_ret_stack
;
180 /* Retrieve a function return address to the trace stack on thread info.*/
182 ftrace_pop_return_trace(struct ftrace_graph_ret
*trace
, unsigned long *ret
,
183 unsigned long frame_pointer
)
187 index
= current
->curr_ret_stack
;
190 * A negative index here means that it's just returned from a
191 * notrace'd function. Recover index to get an original
192 * return address. See ftrace_push_return_trace().
194 * TODO: Need to check whether the stack gets corrupted.
197 index
+= FTRACE_NOTRACE_DEPTH
;
199 if (unlikely(index
< 0 || index
>= FTRACE_RETFUNC_DEPTH
)) {
202 /* Might as well panic, otherwise we have no where to go */
203 *ret
= (unsigned long)panic
;
207 #if defined(CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST) && !defined(CC_USING_FENTRY)
209 * The arch may choose to record the frame pointer used
210 * and check it here to make sure that it is what we expect it
211 * to be. If gcc does not set the place holder of the return
212 * address in the frame pointer, and does a copy instead, then
213 * the function graph trace will fail. This test detects this
216 * Currently, x86_32 with optimize for size (-Os) makes the latest
219 * Note, -mfentry does not use frame pointers, and this test
220 * is not needed if CC_USING_FENTRY is set.
222 if (unlikely(current
->ret_stack
[index
].fp
!= frame_pointer
)) {
224 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
225 " from func %ps return to %lx\n",
226 current
->ret_stack
[index
].fp
,
228 (void *)current
->ret_stack
[index
].func
,
229 current
->ret_stack
[index
].ret
);
230 *ret
= (unsigned long)panic
;
235 *ret
= current
->ret_stack
[index
].ret
;
236 trace
->func
= current
->ret_stack
[index
].func
;
237 trace
->calltime
= current
->ret_stack
[index
].calltime
;
238 trace
->overrun
= atomic_read(¤t
->trace_overrun
);
239 trace
->depth
= index
;
243 * Send the trace to the ring-buffer.
244 * @return the original return address.
246 unsigned long ftrace_return_to_handler(unsigned long frame_pointer
)
248 struct ftrace_graph_ret trace
;
251 ftrace_pop_return_trace(&trace
, &ret
, frame_pointer
);
252 trace
.rettime
= trace_clock_local();
254 current
->curr_ret_stack
--;
256 * The curr_ret_stack can be less than -1 only if it was
257 * filtered out and it's about to return from the function.
258 * Recover the index and continue to trace normal functions.
260 if (current
->curr_ret_stack
< -1) {
261 current
->curr_ret_stack
+= FTRACE_NOTRACE_DEPTH
;
266 * The trace should run after decrementing the ret counter
267 * in case an interrupt were to come in. We don't want to
268 * lose the interrupt if max_depth is set.
270 ftrace_graph_return(&trace
);
272 if (unlikely(!ret
)) {
275 /* Might as well panic. What else to do? */
276 ret
= (unsigned long)panic
;
282 int __trace_graph_entry(struct trace_array
*tr
,
283 struct ftrace_graph_ent
*trace
,
287 struct ftrace_event_call
*call
= &event_funcgraph_entry
;
288 struct ring_buffer_event
*event
;
289 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
290 struct ftrace_graph_ent_entry
*entry
;
292 if (unlikely(__this_cpu_read(ftrace_cpu_disabled
)))
295 event
= trace_buffer_lock_reserve(buffer
, TRACE_GRAPH_ENT
,
296 sizeof(*entry
), flags
, pc
);
299 entry
= ring_buffer_event_data(event
);
300 entry
->graph_ent
= *trace
;
301 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
302 __buffer_unlock_commit(buffer
, event
);
307 static inline int ftrace_graph_ignore_irqs(void)
309 if (!ftrace_graph_skip_irqs
|| trace_recursion_test(TRACE_IRQ_BIT
))
315 int trace_graph_entry(struct ftrace_graph_ent
*trace
)
317 struct trace_array
*tr
= graph_array
;
318 struct trace_array_cpu
*data
;
325 if (!ftrace_trace_task(current
))
328 /* trace it when it is-nested-in or is a function enabled. */
329 if ((!(trace
->depth
|| ftrace_graph_addr(trace
->func
)) ||
330 ftrace_graph_ignore_irqs()) || (trace
->depth
< 0) ||
331 (max_depth
&& trace
->depth
>= max_depth
))
335 * Do not trace a function if it's filtered by set_graph_notrace.
336 * Make the index of ret stack negative to indicate that it should
337 * ignore further functions. But it needs its own ret stack entry
338 * to recover the original index in order to continue tracing after
339 * returning from the function.
341 if (ftrace_graph_notrace_addr(trace
->func
))
344 local_irq_save(flags
);
345 cpu
= raw_smp_processor_id();
346 data
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
);
347 disabled
= atomic_inc_return(&data
->disabled
);
348 if (likely(disabled
== 1)) {
349 pc
= preempt_count();
350 ret
= __trace_graph_entry(tr
, trace
, flags
, pc
);
355 atomic_dec(&data
->disabled
);
356 local_irq_restore(flags
);
361 int trace_graph_thresh_entry(struct ftrace_graph_ent
*trace
)
366 return trace_graph_entry(trace
);
370 __trace_graph_function(struct trace_array
*tr
,
371 unsigned long ip
, unsigned long flags
, int pc
)
373 u64 time
= trace_clock_local();
374 struct ftrace_graph_ent ent
= {
378 struct ftrace_graph_ret ret
= {
385 __trace_graph_entry(tr
, &ent
, flags
, pc
);
386 __trace_graph_return(tr
, &ret
, flags
, pc
);
390 trace_graph_function(struct trace_array
*tr
,
391 unsigned long ip
, unsigned long parent_ip
,
392 unsigned long flags
, int pc
)
394 __trace_graph_function(tr
, ip
, flags
, pc
);
397 void __trace_graph_return(struct trace_array
*tr
,
398 struct ftrace_graph_ret
*trace
,
402 struct ftrace_event_call
*call
= &event_funcgraph_exit
;
403 struct ring_buffer_event
*event
;
404 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
405 struct ftrace_graph_ret_entry
*entry
;
407 if (unlikely(__this_cpu_read(ftrace_cpu_disabled
)))
410 event
= trace_buffer_lock_reserve(buffer
, TRACE_GRAPH_RET
,
411 sizeof(*entry
), flags
, pc
);
414 entry
= ring_buffer_event_data(event
);
416 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
417 __buffer_unlock_commit(buffer
, event
);
420 void trace_graph_return(struct ftrace_graph_ret
*trace
)
422 struct trace_array
*tr
= graph_array
;
423 struct trace_array_cpu
*data
;
429 local_irq_save(flags
);
430 cpu
= raw_smp_processor_id();
431 data
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
);
432 disabled
= atomic_inc_return(&data
->disabled
);
433 if (likely(disabled
== 1)) {
434 pc
= preempt_count();
435 __trace_graph_return(tr
, trace
, flags
, pc
);
437 atomic_dec(&data
->disabled
);
438 local_irq_restore(flags
);
441 void set_graph_array(struct trace_array
*tr
)
445 /* Make graph_array visible before we start tracing */
450 void trace_graph_thresh_return(struct ftrace_graph_ret
*trace
)
452 if (tracing_thresh
&&
453 (trace
->rettime
- trace
->calltime
< tracing_thresh
))
456 trace_graph_return(trace
);
459 static int graph_trace_init(struct trace_array
*tr
)
465 ret
= register_ftrace_graph(&trace_graph_thresh_return
,
466 &trace_graph_thresh_entry
);
468 ret
= register_ftrace_graph(&trace_graph_return
,
472 tracing_start_cmdline_record();
477 static void graph_trace_reset(struct trace_array
*tr
)
479 tracing_stop_cmdline_record();
480 unregister_ftrace_graph();
483 static int max_bytes_for_cpu
;
485 static enum print_line_t
486 print_graph_cpu(struct trace_seq
*s
, int cpu
)
491 * Start with a space character - to make it stand out
492 * to the right a bit when trace output is pasted into
495 ret
= trace_seq_printf(s
, " %*d) ", max_bytes_for_cpu
, cpu
);
497 return TRACE_TYPE_PARTIAL_LINE
;
499 return TRACE_TYPE_HANDLED
;
502 #define TRACE_GRAPH_PROCINFO_LENGTH 14
504 static enum print_line_t
505 print_graph_proc(struct trace_seq
*s
, pid_t pid
)
507 char comm
[TASK_COMM_LEN
];
508 /* sign + log10(MAX_INT) + '\0' */
515 trace_find_cmdline(pid
, comm
);
517 sprintf(pid_str
, "%d", pid
);
519 /* 1 stands for the "-" character */
520 len
= strlen(comm
) + strlen(pid_str
) + 1;
522 if (len
< TRACE_GRAPH_PROCINFO_LENGTH
)
523 spaces
= TRACE_GRAPH_PROCINFO_LENGTH
- len
;
525 /* First spaces to align center */
526 for (i
= 0; i
< spaces
/ 2; i
++) {
527 ret
= trace_seq_putc(s
, ' ');
529 return TRACE_TYPE_PARTIAL_LINE
;
532 ret
= trace_seq_printf(s
, "%s-%s", comm
, pid_str
);
534 return TRACE_TYPE_PARTIAL_LINE
;
536 /* Last spaces to align center */
537 for (i
= 0; i
< spaces
- (spaces
/ 2); i
++) {
538 ret
= trace_seq_putc(s
, ' ');
540 return TRACE_TYPE_PARTIAL_LINE
;
542 return TRACE_TYPE_HANDLED
;
546 static enum print_line_t
547 print_graph_lat_fmt(struct trace_seq
*s
, struct trace_entry
*entry
)
549 if (!trace_seq_putc(s
, ' '))
552 return trace_print_lat_fmt(s
, entry
);
555 /* If the pid changed since the last trace, output this event */
556 static enum print_line_t
557 verif_pid(struct trace_seq
*s
, pid_t pid
, int cpu
, struct fgraph_data
*data
)
564 return TRACE_TYPE_HANDLED
;
566 last_pid
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->last_pid
);
568 if (*last_pid
== pid
)
569 return TRACE_TYPE_HANDLED
;
571 prev_pid
= *last_pid
;
575 return TRACE_TYPE_HANDLED
;
577 * Context-switch trace line:
579 ------------------------------------------
580 | 1) migration/0--1 => sshd-1755
581 ------------------------------------------
584 ret
= trace_seq_puts(s
,
585 " ------------------------------------------\n");
587 return TRACE_TYPE_PARTIAL_LINE
;
589 ret
= print_graph_cpu(s
, cpu
);
590 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
591 return TRACE_TYPE_PARTIAL_LINE
;
593 ret
= print_graph_proc(s
, prev_pid
);
594 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
595 return TRACE_TYPE_PARTIAL_LINE
;
597 ret
= trace_seq_puts(s
, " => ");
599 return TRACE_TYPE_PARTIAL_LINE
;
601 ret
= print_graph_proc(s
, pid
);
602 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
603 return TRACE_TYPE_PARTIAL_LINE
;
605 ret
= trace_seq_puts(s
,
606 "\n ------------------------------------------\n\n");
608 return TRACE_TYPE_PARTIAL_LINE
;
610 return TRACE_TYPE_HANDLED
;
613 static struct ftrace_graph_ret_entry
*
614 get_return_for_leaf(struct trace_iterator
*iter
,
615 struct ftrace_graph_ent_entry
*curr
)
617 struct fgraph_data
*data
= iter
->private;
618 struct ring_buffer_iter
*ring_iter
= NULL
;
619 struct ring_buffer_event
*event
;
620 struct ftrace_graph_ret_entry
*next
;
623 * If the previous output failed to write to the seq buffer,
624 * then we just reuse the data from before.
626 if (data
&& data
->failed
) {
631 ring_iter
= trace_buffer_iter(iter
, iter
->cpu
);
633 /* First peek to compare current entry and the next one */
635 event
= ring_buffer_iter_peek(ring_iter
, NULL
);
638 * We need to consume the current entry to see
641 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
,
643 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, iter
->cpu
,
650 next
= ring_buffer_event_data(event
);
654 * Save current and next entries for later reference
655 * if the output fails.
659 * If the next event is not a return type, then
660 * we only care about what type it is. Otherwise we can
661 * safely copy the entire event.
663 if (next
->ent
.type
== TRACE_GRAPH_RET
)
666 data
->ret
.ent
.type
= next
->ent
.type
;
670 if (next
->ent
.type
!= TRACE_GRAPH_RET
)
673 if (curr
->ent
.pid
!= next
->ent
.pid
||
674 curr
->graph_ent
.func
!= next
->ret
.func
)
677 /* this is a leaf, now advance the iterator */
679 ring_buffer_read(ring_iter
, NULL
);
684 static int print_graph_abs_time(u64 t
, struct trace_seq
*s
)
686 unsigned long usecs_rem
;
688 usecs_rem
= do_div(t
, NSEC_PER_SEC
);
691 return trace_seq_printf(s
, "%5lu.%06lu | ",
692 (unsigned long)t
, usecs_rem
);
695 static enum print_line_t
696 print_graph_irq(struct trace_iterator
*iter
, unsigned long addr
,
697 enum trace_type type
, int cpu
, pid_t pid
, u32 flags
)
700 struct trace_seq
*s
= &iter
->seq
;
702 if (addr
< (unsigned long)__irqentry_text_start
||
703 addr
>= (unsigned long)__irqentry_text_end
)
704 return TRACE_TYPE_UNHANDLED
;
706 if (trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
708 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
) {
709 ret
= print_graph_abs_time(iter
->ts
, s
);
711 return TRACE_TYPE_PARTIAL_LINE
;
715 if (flags
& TRACE_GRAPH_PRINT_CPU
) {
716 ret
= print_graph_cpu(s
, cpu
);
717 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
718 return TRACE_TYPE_PARTIAL_LINE
;
722 if (flags
& TRACE_GRAPH_PRINT_PROC
) {
723 ret
= print_graph_proc(s
, pid
);
724 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
725 return TRACE_TYPE_PARTIAL_LINE
;
726 ret
= trace_seq_puts(s
, " | ");
728 return TRACE_TYPE_PARTIAL_LINE
;
733 ret
= print_graph_duration(0, s
, flags
| FLAGS_FILL_START
);
734 if (ret
!= TRACE_TYPE_HANDLED
)
737 if (type
== TRACE_GRAPH_ENT
)
738 ret
= trace_seq_puts(s
, "==========>");
740 ret
= trace_seq_puts(s
, "<==========");
743 return TRACE_TYPE_PARTIAL_LINE
;
745 ret
= print_graph_duration(0, s
, flags
| FLAGS_FILL_END
);
746 if (ret
!= TRACE_TYPE_HANDLED
)
749 ret
= trace_seq_putc(s
, '\n');
752 return TRACE_TYPE_PARTIAL_LINE
;
753 return TRACE_TYPE_HANDLED
;
757 trace_print_graph_duration(unsigned long long duration
, struct trace_seq
*s
)
759 unsigned long nsecs_rem
= do_div(duration
, 1000);
760 /* log10(ULONG_MAX) + '\0' */
766 sprintf(msecs_str
, "%lu", (unsigned long) duration
);
769 ret
= trace_seq_printf(s
, "%s", msecs_str
);
771 return TRACE_TYPE_PARTIAL_LINE
;
773 len
= strlen(msecs_str
);
775 /* Print nsecs (we don't want to exceed 7 numbers) */
777 size_t slen
= min_t(size_t, sizeof(nsecs_str
), 8UL - len
);
779 snprintf(nsecs_str
, slen
, "%03lu", nsecs_rem
);
780 ret
= trace_seq_printf(s
, ".%s", nsecs_str
);
782 return TRACE_TYPE_PARTIAL_LINE
;
783 len
+= strlen(nsecs_str
);
786 ret
= trace_seq_puts(s
, " us ");
788 return TRACE_TYPE_PARTIAL_LINE
;
790 /* Print remaining spaces to fit the row's width */
791 for (i
= len
; i
< 7; i
++) {
792 ret
= trace_seq_putc(s
, ' ');
794 return TRACE_TYPE_PARTIAL_LINE
;
796 return TRACE_TYPE_HANDLED
;
799 static enum print_line_t
800 print_graph_duration(unsigned long long duration
, struct trace_seq
*s
,
805 if (!(flags
& TRACE_GRAPH_PRINT_DURATION
) ||
806 !(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
807 return TRACE_TYPE_HANDLED
;
809 /* No real adata, just filling the column with spaces */
810 switch (flags
& TRACE_GRAPH_PRINT_FILL_MASK
) {
811 case FLAGS_FILL_FULL
:
812 ret
= trace_seq_puts(s
, " | ");
813 return ret
? TRACE_TYPE_HANDLED
: TRACE_TYPE_PARTIAL_LINE
;
814 case FLAGS_FILL_START
:
815 ret
= trace_seq_puts(s
, " ");
816 return ret
? TRACE_TYPE_HANDLED
: TRACE_TYPE_PARTIAL_LINE
;
818 ret
= trace_seq_puts(s
, " |");
819 return ret
? TRACE_TYPE_HANDLED
: TRACE_TYPE_PARTIAL_LINE
;
822 /* Signal a overhead of time execution to the output */
823 if (flags
& TRACE_GRAPH_PRINT_OVERHEAD
) {
824 /* Duration exceeded 100 msecs */
825 if (duration
> 100000ULL)
826 ret
= trace_seq_puts(s
, "! ");
827 /* Duration exceeded 10 msecs */
828 else if (duration
> 10000ULL)
829 ret
= trace_seq_puts(s
, "+ ");
833 * The -1 means we either did not exceed the duration tresholds
834 * or we dont want to print out the overhead. Either way we need
835 * to fill out the space.
838 ret
= trace_seq_puts(s
, " ");
840 /* Catching here any failure happenned above */
842 return TRACE_TYPE_PARTIAL_LINE
;
844 ret
= trace_print_graph_duration(duration
, s
);
845 if (ret
!= TRACE_TYPE_HANDLED
)
848 ret
= trace_seq_puts(s
, "| ");
850 return TRACE_TYPE_PARTIAL_LINE
;
852 return TRACE_TYPE_HANDLED
;
855 /* Case of a leaf function on its call entry */
856 static enum print_line_t
857 print_graph_entry_leaf(struct trace_iterator
*iter
,
858 struct ftrace_graph_ent_entry
*entry
,
859 struct ftrace_graph_ret_entry
*ret_entry
,
860 struct trace_seq
*s
, u32 flags
)
862 struct fgraph_data
*data
= iter
->private;
863 struct ftrace_graph_ret
*graph_ret
;
864 struct ftrace_graph_ent
*call
;
865 unsigned long long duration
;
869 graph_ret
= &ret_entry
->ret
;
870 call
= &entry
->graph_ent
;
871 duration
= graph_ret
->rettime
- graph_ret
->calltime
;
874 struct fgraph_cpu_data
*cpu_data
;
877 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
880 * Comments display at + 1 to depth. Since
881 * this is a leaf function, keep the comments
882 * equal to this depth.
884 cpu_data
->depth
= call
->depth
- 1;
886 /* No need to keep this function around for this depth */
887 if (call
->depth
< FTRACE_RETFUNC_DEPTH
)
888 cpu_data
->enter_funcs
[call
->depth
] = 0;
891 /* Overhead and duration */
892 ret
= print_graph_duration(duration
, s
, flags
);
893 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
894 return TRACE_TYPE_PARTIAL_LINE
;
897 for (i
= 0; i
< call
->depth
* TRACE_GRAPH_INDENT
; i
++) {
898 ret
= trace_seq_putc(s
, ' ');
900 return TRACE_TYPE_PARTIAL_LINE
;
903 ret
= trace_seq_printf(s
, "%ps();\n", (void *)call
->func
);
905 return TRACE_TYPE_PARTIAL_LINE
;
907 return TRACE_TYPE_HANDLED
;
910 static enum print_line_t
911 print_graph_entry_nested(struct trace_iterator
*iter
,
912 struct ftrace_graph_ent_entry
*entry
,
913 struct trace_seq
*s
, int cpu
, u32 flags
)
915 struct ftrace_graph_ent
*call
= &entry
->graph_ent
;
916 struct fgraph_data
*data
= iter
->private;
921 struct fgraph_cpu_data
*cpu_data
;
924 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
925 cpu_data
->depth
= call
->depth
;
927 /* Save this function pointer to see if the exit matches */
928 if (call
->depth
< FTRACE_RETFUNC_DEPTH
)
929 cpu_data
->enter_funcs
[call
->depth
] = call
->func
;
933 ret
= print_graph_duration(0, s
, flags
| FLAGS_FILL_FULL
);
934 if (ret
!= TRACE_TYPE_HANDLED
)
938 for (i
= 0; i
< call
->depth
* TRACE_GRAPH_INDENT
; i
++) {
939 ret
= trace_seq_putc(s
, ' ');
941 return TRACE_TYPE_PARTIAL_LINE
;
944 ret
= trace_seq_printf(s
, "%ps() {\n", (void *)call
->func
);
946 return TRACE_TYPE_PARTIAL_LINE
;
949 * we already consumed the current entry to check the next one
950 * and see if this is a leaf.
952 return TRACE_TYPE_NO_CONSUME
;
955 static enum print_line_t
956 print_graph_prologue(struct trace_iterator
*iter
, struct trace_seq
*s
,
957 int type
, unsigned long addr
, u32 flags
)
959 struct fgraph_data
*data
= iter
->private;
960 struct trace_entry
*ent
= iter
->ent
;
965 if (verif_pid(s
, ent
->pid
, cpu
, data
) == TRACE_TYPE_PARTIAL_LINE
)
966 return TRACE_TYPE_PARTIAL_LINE
;
970 ret
= print_graph_irq(iter
, addr
, type
, cpu
, ent
->pid
, flags
);
971 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
972 return TRACE_TYPE_PARTIAL_LINE
;
975 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
979 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
) {
980 ret
= print_graph_abs_time(iter
->ts
, s
);
982 return TRACE_TYPE_PARTIAL_LINE
;
986 if (flags
& TRACE_GRAPH_PRINT_CPU
) {
987 ret
= print_graph_cpu(s
, cpu
);
988 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
989 return TRACE_TYPE_PARTIAL_LINE
;
993 if (flags
& TRACE_GRAPH_PRINT_PROC
) {
994 ret
= print_graph_proc(s
, ent
->pid
);
995 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
996 return TRACE_TYPE_PARTIAL_LINE
;
998 ret
= trace_seq_puts(s
, " | ");
1000 return TRACE_TYPE_PARTIAL_LINE
;
1003 /* Latency format */
1004 if (trace_flags
& TRACE_ITER_LATENCY_FMT
) {
1005 ret
= print_graph_lat_fmt(s
, ent
);
1006 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
1007 return TRACE_TYPE_PARTIAL_LINE
;
1014 * Entry check for irq code
1017 * - we are inside irq code
1018 * - we just entered irq code
1021 * - funcgraph-interrupts option is set
1022 * - we are not inside irq code
1025 check_irq_entry(struct trace_iterator
*iter
, u32 flags
,
1026 unsigned long addr
, int depth
)
1028 int cpu
= iter
->cpu
;
1030 struct fgraph_data
*data
= iter
->private;
1033 * If we are either displaying irqs, or we got called as
1034 * a graph event and private data does not exist,
1035 * then we bypass the irq check.
1037 if ((flags
& TRACE_GRAPH_PRINT_IRQS
) ||
1041 depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
1044 * We are inside the irq code
1046 if (*depth_irq
>= 0)
1049 if ((addr
< (unsigned long)__irqentry_text_start
) ||
1050 (addr
>= (unsigned long)__irqentry_text_end
))
1054 * We are entering irq code.
1061 * Return check for irq code
1064 * - we are inside irq code
1065 * - we just left irq code
1068 * - funcgraph-interrupts option is set
1069 * - we are not inside irq code
1072 check_irq_return(struct trace_iterator
*iter
, u32 flags
, int depth
)
1074 int cpu
= iter
->cpu
;
1076 struct fgraph_data
*data
= iter
->private;
1079 * If we are either displaying irqs, or we got called as
1080 * a graph event and private data does not exist,
1081 * then we bypass the irq check.
1083 if ((flags
& TRACE_GRAPH_PRINT_IRQS
) ||
1087 depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
1090 * We are not inside the irq code.
1092 if (*depth_irq
== -1)
1096 * We are inside the irq code, and this is returning entry.
1097 * Let's not trace it and clear the entry depth, since
1098 * we are out of irq code.
1100 * This condition ensures that we 'leave the irq code' once
1101 * we are out of the entry depth. Thus protecting us from
1102 * the RETURN entry loss.
1104 if (*depth_irq
>= depth
) {
1110 * We are inside the irq code, and this is not the entry.
1115 static enum print_line_t
1116 print_graph_entry(struct ftrace_graph_ent_entry
*field
, struct trace_seq
*s
,
1117 struct trace_iterator
*iter
, u32 flags
)
1119 struct fgraph_data
*data
= iter
->private;
1120 struct ftrace_graph_ent
*call
= &field
->graph_ent
;
1121 struct ftrace_graph_ret_entry
*leaf_ret
;
1122 static enum print_line_t ret
;
1123 int cpu
= iter
->cpu
;
1125 if (check_irq_entry(iter
, flags
, call
->func
, call
->depth
))
1126 return TRACE_TYPE_HANDLED
;
1128 if (print_graph_prologue(iter
, s
, TRACE_GRAPH_ENT
, call
->func
, flags
))
1129 return TRACE_TYPE_PARTIAL_LINE
;
1131 leaf_ret
= get_return_for_leaf(iter
, field
);
1133 ret
= print_graph_entry_leaf(iter
, field
, leaf_ret
, s
, flags
);
1135 ret
= print_graph_entry_nested(iter
, field
, s
, cpu
, flags
);
1139 * If we failed to write our output, then we need to make
1140 * note of it. Because we already consumed our entry.
1152 static enum print_line_t
1153 print_graph_return(struct ftrace_graph_ret
*trace
, struct trace_seq
*s
,
1154 struct trace_entry
*ent
, struct trace_iterator
*iter
,
1157 unsigned long long duration
= trace
->rettime
- trace
->calltime
;
1158 struct fgraph_data
*data
= iter
->private;
1159 pid_t pid
= ent
->pid
;
1160 int cpu
= iter
->cpu
;
1165 if (check_irq_return(iter
, flags
, trace
->depth
))
1166 return TRACE_TYPE_HANDLED
;
1169 struct fgraph_cpu_data
*cpu_data
;
1170 int cpu
= iter
->cpu
;
1172 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
1175 * Comments display at + 1 to depth. This is the
1176 * return from a function, we now want the comments
1177 * to display at the same level of the bracket.
1179 cpu_data
->depth
= trace
->depth
- 1;
1181 if (trace
->depth
< FTRACE_RETFUNC_DEPTH
) {
1182 if (cpu_data
->enter_funcs
[trace
->depth
] != trace
->func
)
1184 cpu_data
->enter_funcs
[trace
->depth
] = 0;
1188 if (print_graph_prologue(iter
, s
, 0, 0, flags
))
1189 return TRACE_TYPE_PARTIAL_LINE
;
1191 /* Overhead and duration */
1192 ret
= print_graph_duration(duration
, s
, flags
);
1193 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
1194 return TRACE_TYPE_PARTIAL_LINE
;
1197 for (i
= 0; i
< trace
->depth
* TRACE_GRAPH_INDENT
; i
++) {
1198 ret
= trace_seq_putc(s
, ' ');
1200 return TRACE_TYPE_PARTIAL_LINE
;
1204 * If the return function does not have a matching entry,
1205 * then the entry was lost. Instead of just printing
1206 * the '}' and letting the user guess what function this
1207 * belongs to, write out the function name. Always do
1208 * that if the funcgraph-tail option is enabled.
1210 if (func_match
&& !(flags
& TRACE_GRAPH_PRINT_TAIL
)) {
1211 ret
= trace_seq_puts(s
, "}\n");
1213 return TRACE_TYPE_PARTIAL_LINE
;
1215 ret
= trace_seq_printf(s
, "} /* %ps */\n", (void *)trace
->func
);
1217 return TRACE_TYPE_PARTIAL_LINE
;
1221 if (flags
& TRACE_GRAPH_PRINT_OVERRUN
) {
1222 ret
= trace_seq_printf(s
, " (Overruns: %lu)\n",
1225 return TRACE_TYPE_PARTIAL_LINE
;
1228 ret
= print_graph_irq(iter
, trace
->func
, TRACE_GRAPH_RET
,
1230 if (ret
== TRACE_TYPE_PARTIAL_LINE
)
1231 return TRACE_TYPE_PARTIAL_LINE
;
1233 return TRACE_TYPE_HANDLED
;
1236 static enum print_line_t
1237 print_graph_comment(struct trace_seq
*s
, struct trace_entry
*ent
,
1238 struct trace_iterator
*iter
, u32 flags
)
1240 unsigned long sym_flags
= (trace_flags
& TRACE_ITER_SYM_MASK
);
1241 struct fgraph_data
*data
= iter
->private;
1242 struct trace_event
*event
;
1248 depth
= per_cpu_ptr(data
->cpu_data
, iter
->cpu
)->depth
;
1250 if (print_graph_prologue(iter
, s
, 0, 0, flags
))
1251 return TRACE_TYPE_PARTIAL_LINE
;
1254 ret
= print_graph_duration(0, s
, flags
| FLAGS_FILL_FULL
);
1255 if (ret
!= TRACE_TYPE_HANDLED
)
1260 for (i
= 0; i
< (depth
+ 1) * TRACE_GRAPH_INDENT
; i
++) {
1261 ret
= trace_seq_putc(s
, ' ');
1263 return TRACE_TYPE_PARTIAL_LINE
;
1267 ret
= trace_seq_puts(s
, "/* ");
1269 return TRACE_TYPE_PARTIAL_LINE
;
1271 switch (iter
->ent
->type
) {
1273 ret
= trace_print_bprintk_msg_only(iter
);
1274 if (ret
!= TRACE_TYPE_HANDLED
)
1278 ret
= trace_print_printk_msg_only(iter
);
1279 if (ret
!= TRACE_TYPE_HANDLED
)
1283 event
= ftrace_find_event(ent
->type
);
1285 return TRACE_TYPE_UNHANDLED
;
1287 ret
= event
->funcs
->trace(iter
, sym_flags
, event
);
1288 if (ret
!= TRACE_TYPE_HANDLED
)
1292 /* Strip ending newline */
1293 if (s
->buffer
[s
->len
- 1] == '\n') {
1294 s
->buffer
[s
->len
- 1] = '\0';
1298 ret
= trace_seq_puts(s
, " */\n");
1300 return TRACE_TYPE_PARTIAL_LINE
;
1302 return TRACE_TYPE_HANDLED
;
1307 print_graph_function_flags(struct trace_iterator
*iter
, u32 flags
)
1309 struct ftrace_graph_ent_entry
*field
;
1310 struct fgraph_data
*data
= iter
->private;
1311 struct trace_entry
*entry
= iter
->ent
;
1312 struct trace_seq
*s
= &iter
->seq
;
1313 int cpu
= iter
->cpu
;
1316 if (data
&& per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
) {
1317 per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
= 0;
1318 return TRACE_TYPE_HANDLED
;
1322 * If the last output failed, there's a possibility we need
1323 * to print out the missing entry which would never go out.
1325 if (data
&& data
->failed
) {
1327 iter
->cpu
= data
->cpu
;
1328 ret
= print_graph_entry(field
, s
, iter
, flags
);
1329 if (ret
== TRACE_TYPE_HANDLED
&& iter
->cpu
!= cpu
) {
1330 per_cpu_ptr(data
->cpu_data
, iter
->cpu
)->ignore
= 1;
1331 ret
= TRACE_TYPE_NO_CONSUME
;
1337 switch (entry
->type
) {
1338 case TRACE_GRAPH_ENT
: {
1340 * print_graph_entry() may consume the current event,
1341 * thus @field may become invalid, so we need to save it.
1342 * sizeof(struct ftrace_graph_ent_entry) is very small,
1343 * it can be safely saved at the stack.
1345 struct ftrace_graph_ent_entry saved
;
1346 trace_assign_type(field
, entry
);
1348 return print_graph_entry(&saved
, s
, iter
, flags
);
1350 case TRACE_GRAPH_RET
: {
1351 struct ftrace_graph_ret_entry
*field
;
1352 trace_assign_type(field
, entry
);
1353 return print_graph_return(&field
->ret
, s
, entry
, iter
, flags
);
1357 /* dont trace stack and functions as comments */
1358 return TRACE_TYPE_UNHANDLED
;
1361 return print_graph_comment(s
, entry
, iter
, flags
);
1364 return TRACE_TYPE_HANDLED
;
1367 static enum print_line_t
1368 print_graph_function(struct trace_iterator
*iter
)
1370 return print_graph_function_flags(iter
, tracer_flags
.val
);
1373 static enum print_line_t
1374 print_graph_function_event(struct trace_iterator
*iter
, int flags
,
1375 struct trace_event
*event
)
1377 return print_graph_function(iter
);
1380 static void print_lat_header(struct seq_file
*s
, u32 flags
)
1382 static const char spaces
[] = " " /* 16 spaces */
1384 " "; /* 17 spaces */
1387 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1389 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1391 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1394 seq_printf(s
, "#%.*s _-----=> irqs-off \n", size
, spaces
);
1395 seq_printf(s
, "#%.*s / _----=> need-resched \n", size
, spaces
);
1396 seq_printf(s
, "#%.*s| / _---=> hardirq/softirq \n", size
, spaces
);
1397 seq_printf(s
, "#%.*s|| / _--=> preempt-depth \n", size
, spaces
);
1398 seq_printf(s
, "#%.*s||| / \n", size
, spaces
);
1401 static void __print_graph_headers_flags(struct seq_file
*s
, u32 flags
)
1403 int lat
= trace_flags
& TRACE_ITER_LATENCY_FMT
;
1406 print_lat_header(s
, flags
);
1410 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1411 seq_printf(s
, " TIME ");
1412 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1413 seq_printf(s
, " CPU");
1414 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1415 seq_printf(s
, " TASK/PID ");
1417 seq_printf(s
, "||||");
1418 if (flags
& TRACE_GRAPH_PRINT_DURATION
)
1419 seq_printf(s
, " DURATION ");
1420 seq_printf(s
, " FUNCTION CALLS\n");
1424 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1425 seq_printf(s
, " | ");
1426 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1427 seq_printf(s
, " | ");
1428 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1429 seq_printf(s
, " | | ");
1431 seq_printf(s
, "||||");
1432 if (flags
& TRACE_GRAPH_PRINT_DURATION
)
1433 seq_printf(s
, " | | ");
1434 seq_printf(s
, " | | | |\n");
1437 void print_graph_headers(struct seq_file
*s
)
1439 print_graph_headers_flags(s
, tracer_flags
.val
);
1442 void print_graph_headers_flags(struct seq_file
*s
, u32 flags
)
1444 struct trace_iterator
*iter
= s
->private;
1446 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
1449 if (trace_flags
& TRACE_ITER_LATENCY_FMT
) {
1450 /* print nothing if the buffers are empty */
1451 if (trace_empty(iter
))
1454 print_trace_header(s
, iter
);
1457 __print_graph_headers_flags(s
, flags
);
1460 void graph_trace_open(struct trace_iterator
*iter
)
1462 /* pid and depth on the last trace processed */
1463 struct fgraph_data
*data
;
1466 iter
->private = NULL
;
1468 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
1472 data
->cpu_data
= alloc_percpu(struct fgraph_cpu_data
);
1473 if (!data
->cpu_data
)
1476 for_each_possible_cpu(cpu
) {
1477 pid_t
*pid
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->last_pid
);
1478 int *depth
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth
);
1479 int *ignore
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
);
1480 int *depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
1488 iter
->private = data
;
1495 pr_warning("function graph tracer: not enough memory\n");
1498 void graph_trace_close(struct trace_iterator
*iter
)
1500 struct fgraph_data
*data
= iter
->private;
1503 free_percpu(data
->cpu_data
);
1509 func_graph_set_flag(struct trace_array
*tr
, u32 old_flags
, u32 bit
, int set
)
1511 if (bit
== TRACE_GRAPH_PRINT_IRQS
)
1512 ftrace_graph_skip_irqs
= !set
;
1517 static struct trace_event_functions graph_functions
= {
1518 .trace
= print_graph_function_event
,
1521 static struct trace_event graph_trace_entry_event
= {
1522 .type
= TRACE_GRAPH_ENT
,
1523 .funcs
= &graph_functions
,
1526 static struct trace_event graph_trace_ret_event
= {
1527 .type
= TRACE_GRAPH_RET
,
1528 .funcs
= &graph_functions
1531 static struct tracer graph_trace __tracer_data
= {
1532 .name
= "function_graph",
1533 .open
= graph_trace_open
,
1534 .pipe_open
= graph_trace_open
,
1535 .close
= graph_trace_close
,
1536 .pipe_close
= graph_trace_close
,
1537 .init
= graph_trace_init
,
1538 .reset
= graph_trace_reset
,
1539 .print_line
= print_graph_function
,
1540 .print_header
= print_graph_headers
,
1541 .flags
= &tracer_flags
,
1542 .set_flag
= func_graph_set_flag
,
1543 #ifdef CONFIG_FTRACE_SELFTEST
1544 .selftest
= trace_selftest_startup_function_graph
,
1550 graph_depth_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
1556 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
1568 graph_depth_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
1571 char buf
[15]; /* More than enough to hold UINT_MAX + "\n"*/
1574 n
= sprintf(buf
, "%d\n", max_depth
);
1576 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, n
);
1579 static const struct file_operations graph_depth_fops
= {
1580 .open
= tracing_open_generic
,
1581 .write
= graph_depth_write
,
1582 .read
= graph_depth_read
,
1583 .llseek
= generic_file_llseek
,
1586 static __init
int init_graph_debugfs(void)
1588 struct dentry
*d_tracer
;
1590 d_tracer
= tracing_init_dentry();
1594 trace_create_file("max_graph_depth", 0644, d_tracer
,
1595 NULL
, &graph_depth_fops
);
1599 fs_initcall(init_graph_debugfs
);
1601 static __init
int init_graph_trace(void)
1603 max_bytes_for_cpu
= snprintf(NULL
, 0, "%d", nr_cpu_ids
- 1);
1605 if (!register_ftrace_event(&graph_trace_entry_event
)) {
1606 pr_warning("Warning: could not register graph trace events\n");
1610 if (!register_ftrace_event(&graph_trace_ret_event
)) {
1611 pr_warning("Warning: could not register graph trace events\n");
1615 return register_tracer(&graph_trace
);
1618 core_initcall(init_graph_trace
);