2 * Infrastructure for profiling code inserted by 'gcc -pg'.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code in the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30 #include <linux/hash.h>
32 #include <trace/events/sched.h>
34 #include <asm/ftrace.h>
35 #include <asm/setup.h>
37 #include "trace_output.h"
38 #include "trace_stat.h"
40 #define FTRACE_WARN_ON(cond) \
46 #define FTRACE_WARN_ON_ONCE(cond) \
48 if (WARN_ON_ONCE(cond)) \
52 /* hash bits for specific function selection */
53 #define FTRACE_HASH_BITS 7
54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
56 /* ftrace_enabled is a method to turn ftrace on or off */
57 int ftrace_enabled __read_mostly
;
58 static int last_ftrace_enabled
;
60 /* Quick disabling of function tracer. */
61 int function_trace_stop
;
64 * ftrace_disabled is set when an anomaly is discovered.
65 * ftrace_disabled is much stronger than ftrace_enabled.
67 static int ftrace_disabled __read_mostly
;
69 static DEFINE_MUTEX(ftrace_lock
);
71 static struct ftrace_ops ftrace_list_end __read_mostly
=
76 static struct ftrace_ops
*ftrace_list __read_mostly
= &ftrace_list_end
;
77 ftrace_func_t ftrace_trace_function __read_mostly
= ftrace_stub
;
78 ftrace_func_t __ftrace_trace_function __read_mostly
= ftrace_stub
;
79 ftrace_func_t ftrace_pid_function __read_mostly
= ftrace_stub
;
81 static void ftrace_list_func(unsigned long ip
, unsigned long parent_ip
)
83 struct ftrace_ops
*op
= ftrace_list
;
85 /* in case someone actually ports this to alpha! */
86 read_barrier_depends();
88 while (op
!= &ftrace_list_end
) {
90 read_barrier_depends();
91 op
->func(ip
, parent_ip
);
96 static void ftrace_pid_func(unsigned long ip
, unsigned long parent_ip
)
98 if (!test_tsk_trace_trace(current
))
101 ftrace_pid_function(ip
, parent_ip
);
104 static void set_ftrace_pid_function(ftrace_func_t func
)
106 /* do not set ftrace_pid_function to itself! */
107 if (func
!= ftrace_pid_func
)
108 ftrace_pid_function
= func
;
112 * clear_ftrace_function - reset the ftrace function
114 * This NULLs the ftrace function and in essence stops
115 * tracing. There may be lag
117 void clear_ftrace_function(void)
119 ftrace_trace_function
= ftrace_stub
;
120 __ftrace_trace_function
= ftrace_stub
;
121 ftrace_pid_function
= ftrace_stub
;
124 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
126 * For those archs that do not test ftrace_trace_stop in their
127 * mcount call site, we need to do it from C.
129 static void ftrace_test_stop_func(unsigned long ip
, unsigned long parent_ip
)
131 if (function_trace_stop
)
134 __ftrace_trace_function(ip
, parent_ip
);
138 static int __register_ftrace_function(struct ftrace_ops
*ops
)
140 ops
->next
= ftrace_list
;
142 * We are entering ops into the ftrace_list but another
143 * CPU might be walking that list. We need to make sure
144 * the ops->next pointer is valid before another CPU sees
145 * the ops pointer included into the ftrace_list.
150 if (ftrace_enabled
) {
153 if (ops
->next
== &ftrace_list_end
)
156 func
= ftrace_list_func
;
158 if (ftrace_pid_trace
) {
159 set_ftrace_pid_function(func
);
160 func
= ftrace_pid_func
;
164 * For one func, simply call it directly.
165 * For more than one func, call the chain.
167 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
168 ftrace_trace_function
= func
;
170 __ftrace_trace_function
= func
;
171 ftrace_trace_function
= ftrace_test_stop_func
;
178 static int __unregister_ftrace_function(struct ftrace_ops
*ops
)
180 struct ftrace_ops
**p
;
183 * If we are removing the last function, then simply point
184 * to the ftrace_stub.
186 if (ftrace_list
== ops
&& ops
->next
== &ftrace_list_end
) {
187 ftrace_trace_function
= ftrace_stub
;
188 ftrace_list
= &ftrace_list_end
;
192 for (p
= &ftrace_list
; *p
!= &ftrace_list_end
; p
= &(*p
)->next
)
201 if (ftrace_enabled
) {
202 /* If we only have one func left, then call that directly */
203 if (ftrace_list
->next
== &ftrace_list_end
) {
204 ftrace_func_t func
= ftrace_list
->func
;
206 if (ftrace_pid_trace
) {
207 set_ftrace_pid_function(func
);
208 func
= ftrace_pid_func
;
210 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
211 ftrace_trace_function
= func
;
213 __ftrace_trace_function
= func
;
221 static void ftrace_update_pid_func(void)
225 if (ftrace_trace_function
== ftrace_stub
)
228 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
229 func
= ftrace_trace_function
;
231 func
= __ftrace_trace_function
;
234 if (ftrace_pid_trace
) {
235 set_ftrace_pid_function(func
);
236 func
= ftrace_pid_func
;
238 if (func
== ftrace_pid_func
)
239 func
= ftrace_pid_function
;
242 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
243 ftrace_trace_function
= func
;
245 __ftrace_trace_function
= func
;
249 #ifdef CONFIG_FUNCTION_PROFILER
250 struct ftrace_profile
{
251 struct hlist_node node
;
253 unsigned long counter
;
254 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
255 unsigned long long time
;
259 struct ftrace_profile_page
{
260 struct ftrace_profile_page
*next
;
262 struct ftrace_profile records
[];
265 struct ftrace_profile_stat
{
267 struct hlist_head
*hash
;
268 struct ftrace_profile_page
*pages
;
269 struct ftrace_profile_page
*start
;
270 struct tracer_stat stat
;
273 #define PROFILE_RECORDS_SIZE \
274 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
276 #define PROFILES_PER_PAGE \
277 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
279 static int ftrace_profile_bits __read_mostly
;
280 static int ftrace_profile_enabled __read_mostly
;
282 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
283 static DEFINE_MUTEX(ftrace_profile_lock
);
285 static DEFINE_PER_CPU(struct ftrace_profile_stat
, ftrace_profile_stats
);
287 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
290 function_stat_next(void *v
, int idx
)
292 struct ftrace_profile
*rec
= v
;
293 struct ftrace_profile_page
*pg
;
295 pg
= (struct ftrace_profile_page
*)((unsigned long)rec
& PAGE_MASK
);
301 if ((void *)rec
>= (void *)&pg
->records
[pg
->index
]) {
305 rec
= &pg
->records
[0];
313 static void *function_stat_start(struct tracer_stat
*trace
)
315 struct ftrace_profile_stat
*stat
=
316 container_of(trace
, struct ftrace_profile_stat
, stat
);
318 if (!stat
|| !stat
->start
)
321 return function_stat_next(&stat
->start
->records
[0], 0);
324 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
325 /* function graph compares on total time */
326 static int function_stat_cmp(void *p1
, void *p2
)
328 struct ftrace_profile
*a
= p1
;
329 struct ftrace_profile
*b
= p2
;
331 if (a
->time
< b
->time
)
333 if (a
->time
> b
->time
)
339 /* not function graph compares against hits */
340 static int function_stat_cmp(void *p1
, void *p2
)
342 struct ftrace_profile
*a
= p1
;
343 struct ftrace_profile
*b
= p2
;
345 if (a
->counter
< b
->counter
)
347 if (a
->counter
> b
->counter
)
354 static int function_stat_headers(struct seq_file
*m
)
356 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
357 seq_printf(m
, " Function "
362 seq_printf(m
, " Function Hit\n"
368 static int function_stat_show(struct seq_file
*m
, void *v
)
370 struct ftrace_profile
*rec
= v
;
371 char str
[KSYM_SYMBOL_LEN
];
372 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
373 static DEFINE_MUTEX(mutex
);
374 static struct trace_seq s
;
375 unsigned long long avg
;
378 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
379 seq_printf(m
, " %-30.30s %10lu", str
, rec
->counter
);
381 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
384 do_div(avg
, rec
->counter
);
388 trace_print_graph_duration(rec
->time
, &s
);
389 trace_seq_puts(&s
, " ");
390 trace_print_graph_duration(avg
, &s
);
391 trace_print_seq(m
, &s
);
392 mutex_unlock(&mutex
);
399 static void ftrace_profile_reset(struct ftrace_profile_stat
*stat
)
401 struct ftrace_profile_page
*pg
;
403 pg
= stat
->pages
= stat
->start
;
406 memset(pg
->records
, 0, PROFILE_RECORDS_SIZE
);
411 memset(stat
->hash
, 0,
412 FTRACE_PROFILE_HASH_SIZE
* sizeof(struct hlist_head
));
415 int ftrace_profile_pages_init(struct ftrace_profile_stat
*stat
)
417 struct ftrace_profile_page
*pg
;
422 /* If we already allocated, do nothing */
426 stat
->pages
= (void *)get_zeroed_page(GFP_KERNEL
);
430 #ifdef CONFIG_DYNAMIC_FTRACE
431 functions
= ftrace_update_tot_cnt
;
434 * We do not know the number of functions that exist because
435 * dynamic tracing is what counts them. With past experience
436 * we have around 20K functions. That should be more than enough.
437 * It is highly unlikely we will execute every function in
443 pg
= stat
->start
= stat
->pages
;
445 pages
= DIV_ROUND_UP(functions
, PROFILES_PER_PAGE
);
447 for (i
= 0; i
< pages
; i
++) {
448 pg
->next
= (void *)get_zeroed_page(GFP_KERNEL
);
459 unsigned long tmp
= (unsigned long)pg
;
465 free_page((unsigned long)stat
->pages
);
472 static int ftrace_profile_init_cpu(int cpu
)
474 struct ftrace_profile_stat
*stat
;
477 stat
= &per_cpu(ftrace_profile_stats
, cpu
);
480 /* If the profile is already created, simply reset it */
481 ftrace_profile_reset(stat
);
486 * We are profiling all functions, but usually only a few thousand
487 * functions are hit. We'll make a hash of 1024 items.
489 size
= FTRACE_PROFILE_HASH_SIZE
;
491 stat
->hash
= kzalloc(sizeof(struct hlist_head
) * size
, GFP_KERNEL
);
496 if (!ftrace_profile_bits
) {
499 for (; size
; size
>>= 1)
500 ftrace_profile_bits
++;
503 /* Preallocate the function profiling pages */
504 if (ftrace_profile_pages_init(stat
) < 0) {
513 static int ftrace_profile_init(void)
518 for_each_online_cpu(cpu
) {
519 ret
= ftrace_profile_init_cpu(cpu
);
527 /* interrupts must be disabled */
528 static struct ftrace_profile
*
529 ftrace_find_profiled_func(struct ftrace_profile_stat
*stat
, unsigned long ip
)
531 struct ftrace_profile
*rec
;
532 struct hlist_head
*hhd
;
533 struct hlist_node
*n
;
536 key
= hash_long(ip
, ftrace_profile_bits
);
537 hhd
= &stat
->hash
[key
];
539 if (hlist_empty(hhd
))
542 hlist_for_each_entry_rcu(rec
, n
, hhd
, node
) {
550 static void ftrace_add_profile(struct ftrace_profile_stat
*stat
,
551 struct ftrace_profile
*rec
)
555 key
= hash_long(rec
->ip
, ftrace_profile_bits
);
556 hlist_add_head_rcu(&rec
->node
, &stat
->hash
[key
]);
560 * The memory is already allocated, this simply finds a new record to use.
562 static struct ftrace_profile
*
563 ftrace_profile_alloc(struct ftrace_profile_stat
*stat
, unsigned long ip
)
565 struct ftrace_profile
*rec
= NULL
;
567 /* prevent recursion (from NMIs) */
568 if (atomic_inc_return(&stat
->disabled
) != 1)
572 * Try to find the function again since an NMI
573 * could have added it
575 rec
= ftrace_find_profiled_func(stat
, ip
);
579 if (stat
->pages
->index
== PROFILES_PER_PAGE
) {
580 if (!stat
->pages
->next
)
582 stat
->pages
= stat
->pages
->next
;
585 rec
= &stat
->pages
->records
[stat
->pages
->index
++];
587 ftrace_add_profile(stat
, rec
);
590 atomic_dec(&stat
->disabled
);
596 function_profile_call(unsigned long ip
, unsigned long parent_ip
)
598 struct ftrace_profile_stat
*stat
;
599 struct ftrace_profile
*rec
;
602 if (!ftrace_profile_enabled
)
605 local_irq_save(flags
);
607 stat
= &__get_cpu_var(ftrace_profile_stats
);
608 if (!stat
->hash
|| !ftrace_profile_enabled
)
611 rec
= ftrace_find_profiled_func(stat
, ip
);
613 rec
= ftrace_profile_alloc(stat
, ip
);
620 local_irq_restore(flags
);
623 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
624 static int profile_graph_entry(struct ftrace_graph_ent
*trace
)
626 function_profile_call(trace
->func
, 0);
630 static void profile_graph_return(struct ftrace_graph_ret
*trace
)
632 struct ftrace_profile_stat
*stat
;
633 unsigned long long calltime
;
634 struct ftrace_profile
*rec
;
637 local_irq_save(flags
);
638 stat
= &__get_cpu_var(ftrace_profile_stats
);
639 if (!stat
->hash
|| !ftrace_profile_enabled
)
642 calltime
= trace
->rettime
- trace
->calltime
;
644 if (!(trace_flags
& TRACE_ITER_GRAPH_TIME
)) {
647 index
= trace
->depth
;
649 /* Append this call time to the parent time to subtract */
651 current
->ret_stack
[index
- 1].subtime
+= calltime
;
653 if (current
->ret_stack
[index
].subtime
< calltime
)
654 calltime
-= current
->ret_stack
[index
].subtime
;
659 rec
= ftrace_find_profiled_func(stat
, trace
->func
);
661 rec
->time
+= calltime
;
664 local_irq_restore(flags
);
667 static int register_ftrace_profiler(void)
669 return register_ftrace_graph(&profile_graph_return
,
670 &profile_graph_entry
);
673 static void unregister_ftrace_profiler(void)
675 unregister_ftrace_graph();
678 static struct ftrace_ops ftrace_profile_ops __read_mostly
=
680 .func
= function_profile_call
,
683 static int register_ftrace_profiler(void)
685 return register_ftrace_function(&ftrace_profile_ops
);
688 static void unregister_ftrace_profiler(void)
690 unregister_ftrace_function(&ftrace_profile_ops
);
692 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
695 ftrace_profile_write(struct file
*filp
, const char __user
*ubuf
,
696 size_t cnt
, loff_t
*ppos
)
699 char buf
[64]; /* big enough to hold a number */
702 if (cnt
>= sizeof(buf
))
705 if (copy_from_user(&buf
, ubuf
, cnt
))
710 ret
= strict_strtoul(buf
, 10, &val
);
716 mutex_lock(&ftrace_profile_lock
);
717 if (ftrace_profile_enabled
^ val
) {
719 ret
= ftrace_profile_init();
725 ret
= register_ftrace_profiler();
730 ftrace_profile_enabled
= 1;
732 ftrace_profile_enabled
= 0;
734 * unregister_ftrace_profiler calls stop_machine
735 * so this acts like an synchronize_sched.
737 unregister_ftrace_profiler();
741 mutex_unlock(&ftrace_profile_lock
);
749 ftrace_profile_read(struct file
*filp
, char __user
*ubuf
,
750 size_t cnt
, loff_t
*ppos
)
752 char buf
[64]; /* big enough to hold a number */
755 r
= sprintf(buf
, "%u\n", ftrace_profile_enabled
);
756 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
759 static const struct file_operations ftrace_profile_fops
= {
760 .open
= tracing_open_generic
,
761 .read
= ftrace_profile_read
,
762 .write
= ftrace_profile_write
,
765 /* used to initialize the real stat files */
766 static struct tracer_stat function_stats __initdata
= {
768 .stat_start
= function_stat_start
,
769 .stat_next
= function_stat_next
,
770 .stat_cmp
= function_stat_cmp
,
771 .stat_headers
= function_stat_headers
,
772 .stat_show
= function_stat_show
775 static __init
void ftrace_profile_debugfs(struct dentry
*d_tracer
)
777 struct ftrace_profile_stat
*stat
;
778 struct dentry
*entry
;
783 for_each_possible_cpu(cpu
) {
784 stat
= &per_cpu(ftrace_profile_stats
, cpu
);
786 /* allocate enough for function name + cpu number */
787 name
= kmalloc(32, GFP_KERNEL
);
790 * The files created are permanent, if something happens
791 * we still do not free memory.
794 "Could not allocate stat file for cpu %d\n",
798 stat
->stat
= function_stats
;
799 snprintf(name
, 32, "function%d", cpu
);
800 stat
->stat
.name
= name
;
801 ret
= register_stat_tracer(&stat
->stat
);
804 "Could not register function stat for cpu %d\n",
811 entry
= debugfs_create_file("function_profile_enabled", 0644,
812 d_tracer
, NULL
, &ftrace_profile_fops
);
814 pr_warning("Could not create debugfs "
815 "'function_profile_enabled' entry\n");
818 #else /* CONFIG_FUNCTION_PROFILER */
819 static __init
void ftrace_profile_debugfs(struct dentry
*d_tracer
)
822 #endif /* CONFIG_FUNCTION_PROFILER */
824 /* set when tracing only a pid */
825 struct pid
*ftrace_pid_trace
;
826 static struct pid
* const ftrace_swapper_pid
= &init_struct_pid
;
828 #ifdef CONFIG_DYNAMIC_FTRACE
830 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
831 # error Dynamic ftrace depends on MCOUNT_RECORD
834 static struct hlist_head ftrace_func_hash
[FTRACE_FUNC_HASHSIZE
] __read_mostly
;
836 struct ftrace_func_probe
{
837 struct hlist_node node
;
838 struct ftrace_probe_ops
*ops
;
846 FTRACE_ENABLE_CALLS
= (1 << 0),
847 FTRACE_DISABLE_CALLS
= (1 << 1),
848 FTRACE_UPDATE_TRACE_FUNC
= (1 << 2),
849 FTRACE_ENABLE_MCOUNT
= (1 << 3),
850 FTRACE_DISABLE_MCOUNT
= (1 << 4),
851 FTRACE_START_FUNC_RET
= (1 << 5),
852 FTRACE_STOP_FUNC_RET
= (1 << 6),
855 static int ftrace_filtered
;
857 static struct dyn_ftrace
*ftrace_new_addrs
;
859 static DEFINE_MUTEX(ftrace_regex_lock
);
862 struct ftrace_page
*next
;
864 struct dyn_ftrace records
[];
867 #define ENTRIES_PER_PAGE \
868 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
870 /* estimate from running different kernels */
871 #define NR_TO_INIT 10000
873 static struct ftrace_page
*ftrace_pages_start
;
874 static struct ftrace_page
*ftrace_pages
;
876 static struct dyn_ftrace
*ftrace_free_records
;
879 * This is a double for. Do not use 'break' to break out of the loop,
880 * you must use a goto.
882 #define do_for_each_ftrace_rec(pg, rec) \
883 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
885 for (_____i = 0; _____i < pg->index; _____i++) { \
886 rec = &pg->records[_____i];
888 #define while_for_each_ftrace_rec() \
892 #ifdef CONFIG_KPROBES
894 static int frozen_record_count
;
896 static inline void freeze_record(struct dyn_ftrace
*rec
)
898 if (!(rec
->flags
& FTRACE_FL_FROZEN
)) {
899 rec
->flags
|= FTRACE_FL_FROZEN
;
900 frozen_record_count
++;
904 static inline void unfreeze_record(struct dyn_ftrace
*rec
)
906 if (rec
->flags
& FTRACE_FL_FROZEN
) {
907 rec
->flags
&= ~FTRACE_FL_FROZEN
;
908 frozen_record_count
--;
912 static inline int record_frozen(struct dyn_ftrace
*rec
)
914 return rec
->flags
& FTRACE_FL_FROZEN
;
917 # define freeze_record(rec) ({ 0; })
918 # define unfreeze_record(rec) ({ 0; })
919 # define record_frozen(rec) ({ 0; })
920 #endif /* CONFIG_KPROBES */
922 static void ftrace_free_rec(struct dyn_ftrace
*rec
)
924 rec
->freelist
= ftrace_free_records
;
925 ftrace_free_records
= rec
;
926 rec
->flags
|= FTRACE_FL_FREE
;
929 static struct dyn_ftrace
*ftrace_alloc_dyn_node(unsigned long ip
)
931 struct dyn_ftrace
*rec
;
933 /* First check for freed records */
934 if (ftrace_free_records
) {
935 rec
= ftrace_free_records
;
937 if (unlikely(!(rec
->flags
& FTRACE_FL_FREE
))) {
938 FTRACE_WARN_ON_ONCE(1);
939 ftrace_free_records
= NULL
;
943 ftrace_free_records
= rec
->freelist
;
944 memset(rec
, 0, sizeof(*rec
));
948 if (ftrace_pages
->index
== ENTRIES_PER_PAGE
) {
949 if (!ftrace_pages
->next
) {
950 /* allocate another page */
952 (void *)get_zeroed_page(GFP_KERNEL
);
953 if (!ftrace_pages
->next
)
956 ftrace_pages
= ftrace_pages
->next
;
959 return &ftrace_pages
->records
[ftrace_pages
->index
++];
962 static struct dyn_ftrace
*
963 ftrace_record_ip(unsigned long ip
)
965 struct dyn_ftrace
*rec
;
970 rec
= ftrace_alloc_dyn_node(ip
);
975 rec
->newlist
= ftrace_new_addrs
;
976 ftrace_new_addrs
= rec
;
981 static void print_ip_ins(const char *fmt
, unsigned char *p
)
985 printk(KERN_CONT
"%s", fmt
);
987 for (i
= 0; i
< MCOUNT_INSN_SIZE
; i
++)
988 printk(KERN_CONT
"%s%02x", i
? ":" : "", p
[i
]);
991 static void ftrace_bug(int failed
, unsigned long ip
)
995 FTRACE_WARN_ON_ONCE(1);
996 pr_info("ftrace faulted on modifying ");
1000 FTRACE_WARN_ON_ONCE(1);
1001 pr_info("ftrace failed to modify ");
1003 print_ip_ins(" actual: ", (unsigned char *)ip
);
1004 printk(KERN_CONT
"\n");
1007 FTRACE_WARN_ON_ONCE(1);
1008 pr_info("ftrace faulted on writing ");
1012 FTRACE_WARN_ON_ONCE(1);
1013 pr_info("ftrace faulted on unknown error ");
1020 __ftrace_replace_code(struct dyn_ftrace
*rec
, int enable
)
1022 unsigned long ftrace_addr
;
1023 unsigned long flag
= 0UL;
1025 ftrace_addr
= (unsigned long)FTRACE_ADDR
;
1028 * If this record is not to be traced or we want to disable it,
1031 * If we want to enable it and filtering is off, then enable it.
1033 * If we want to enable it and filtering is on, enable it only if
1036 if (enable
&& !(rec
->flags
& FTRACE_FL_NOTRACE
)) {
1037 if (!ftrace_filtered
|| (rec
->flags
& FTRACE_FL_FILTER
))
1038 flag
= FTRACE_FL_ENABLED
;
1041 /* If the state of this record hasn't changed, then do nothing */
1042 if ((rec
->flags
& FTRACE_FL_ENABLED
) == flag
)
1046 rec
->flags
|= FTRACE_FL_ENABLED
;
1047 return ftrace_make_call(rec
, ftrace_addr
);
1050 rec
->flags
&= ~FTRACE_FL_ENABLED
;
1051 return ftrace_make_nop(NULL
, rec
, ftrace_addr
);
1054 static void ftrace_replace_code(int enable
)
1056 struct dyn_ftrace
*rec
;
1057 struct ftrace_page
*pg
;
1060 do_for_each_ftrace_rec(pg
, rec
) {
1062 * Skip over free records, records that have
1063 * failed and not converted.
1065 if (rec
->flags
& FTRACE_FL_FREE
||
1066 rec
->flags
& FTRACE_FL_FAILED
||
1067 !(rec
->flags
& FTRACE_FL_CONVERTED
))
1070 /* ignore updates to this record's mcount site */
1071 if (get_kprobe((void *)rec
->ip
)) {
1075 unfreeze_record(rec
);
1078 failed
= __ftrace_replace_code(rec
, enable
);
1080 rec
->flags
|= FTRACE_FL_FAILED
;
1081 if ((system_state
== SYSTEM_BOOTING
) ||
1082 !core_kernel_text(rec
->ip
)) {
1083 ftrace_free_rec(rec
);
1085 ftrace_bug(failed
, rec
->ip
);
1086 /* Stop processing */
1090 } while_for_each_ftrace_rec();
1094 ftrace_code_disable(struct module
*mod
, struct dyn_ftrace
*rec
)
1101 ret
= ftrace_make_nop(mod
, rec
, MCOUNT_ADDR
);
1103 ftrace_bug(ret
, ip
);
1104 rec
->flags
|= FTRACE_FL_FAILED
;
1111 * archs can override this function if they must do something
1112 * before the modifying code is performed.
1114 int __weak
ftrace_arch_code_modify_prepare(void)
1120 * archs can override this function if they must do something
1121 * after the modifying code is performed.
1123 int __weak
ftrace_arch_code_modify_post_process(void)
1128 static int __ftrace_modify_code(void *data
)
1130 int *command
= data
;
1132 if (*command
& FTRACE_ENABLE_CALLS
)
1133 ftrace_replace_code(1);
1134 else if (*command
& FTRACE_DISABLE_CALLS
)
1135 ftrace_replace_code(0);
1137 if (*command
& FTRACE_UPDATE_TRACE_FUNC
)
1138 ftrace_update_ftrace_func(ftrace_trace_function
);
1140 if (*command
& FTRACE_START_FUNC_RET
)
1141 ftrace_enable_ftrace_graph_caller();
1142 else if (*command
& FTRACE_STOP_FUNC_RET
)
1143 ftrace_disable_ftrace_graph_caller();
1148 static void ftrace_run_update_code(int command
)
1152 ret
= ftrace_arch_code_modify_prepare();
1153 FTRACE_WARN_ON(ret
);
1157 stop_machine(__ftrace_modify_code
, &command
, NULL
);
1159 ret
= ftrace_arch_code_modify_post_process();
1160 FTRACE_WARN_ON(ret
);
1163 static ftrace_func_t saved_ftrace_func
;
1164 static int ftrace_start_up
;
1166 static void ftrace_startup_enable(int command
)
1168 if (saved_ftrace_func
!= ftrace_trace_function
) {
1169 saved_ftrace_func
= ftrace_trace_function
;
1170 command
|= FTRACE_UPDATE_TRACE_FUNC
;
1173 if (!command
|| !ftrace_enabled
)
1176 ftrace_run_update_code(command
);
1179 static void ftrace_startup(int command
)
1181 if (unlikely(ftrace_disabled
))
1185 command
|= FTRACE_ENABLE_CALLS
;
1187 ftrace_startup_enable(command
);
1190 static void ftrace_shutdown(int command
)
1192 if (unlikely(ftrace_disabled
))
1197 * Just warn in case of unbalance, no need to kill ftrace, it's not
1198 * critical but the ftrace_call callers may be never nopped again after
1199 * further ftrace uses.
1201 WARN_ON_ONCE(ftrace_start_up
< 0);
1203 if (!ftrace_start_up
)
1204 command
|= FTRACE_DISABLE_CALLS
;
1206 if (saved_ftrace_func
!= ftrace_trace_function
) {
1207 saved_ftrace_func
= ftrace_trace_function
;
1208 command
|= FTRACE_UPDATE_TRACE_FUNC
;
1211 if (!command
|| !ftrace_enabled
)
1214 ftrace_run_update_code(command
);
1217 static void ftrace_startup_sysctl(void)
1219 int command
= FTRACE_ENABLE_MCOUNT
;
1221 if (unlikely(ftrace_disabled
))
1224 /* Force update next time */
1225 saved_ftrace_func
= NULL
;
1226 /* ftrace_start_up is true if we want ftrace running */
1227 if (ftrace_start_up
)
1228 command
|= FTRACE_ENABLE_CALLS
;
1230 ftrace_run_update_code(command
);
1233 static void ftrace_shutdown_sysctl(void)
1235 int command
= FTRACE_DISABLE_MCOUNT
;
1237 if (unlikely(ftrace_disabled
))
1240 /* ftrace_start_up is true if ftrace is running */
1241 if (ftrace_start_up
)
1242 command
|= FTRACE_DISABLE_CALLS
;
1244 ftrace_run_update_code(command
);
1247 static cycle_t ftrace_update_time
;
1248 static unsigned long ftrace_update_cnt
;
1249 unsigned long ftrace_update_tot_cnt
;
1251 static int ftrace_update_code(struct module
*mod
)
1253 struct dyn_ftrace
*p
;
1254 cycle_t start
, stop
;
1256 start
= ftrace_now(raw_smp_processor_id());
1257 ftrace_update_cnt
= 0;
1259 while (ftrace_new_addrs
) {
1261 /* If something went wrong, bail without enabling anything */
1262 if (unlikely(ftrace_disabled
))
1265 p
= ftrace_new_addrs
;
1266 ftrace_new_addrs
= p
->newlist
;
1269 /* convert record (i.e, patch mcount-call with NOP) */
1270 if (ftrace_code_disable(mod
, p
)) {
1271 p
->flags
|= FTRACE_FL_CONVERTED
;
1272 ftrace_update_cnt
++;
1277 stop
= ftrace_now(raw_smp_processor_id());
1278 ftrace_update_time
= stop
- start
;
1279 ftrace_update_tot_cnt
+= ftrace_update_cnt
;
1284 static int __init
ftrace_dyn_table_alloc(unsigned long num_to_init
)
1286 struct ftrace_page
*pg
;
1290 /* allocate a few pages */
1291 ftrace_pages_start
= (void *)get_zeroed_page(GFP_KERNEL
);
1292 if (!ftrace_pages_start
)
1296 * Allocate a few more pages.
1298 * TODO: have some parser search vmlinux before
1299 * final linking to find all calls to ftrace.
1301 * a) know how many pages to allocate.
1303 * b) set up the table then.
1305 * The dynamic code is still necessary for
1309 pg
= ftrace_pages
= ftrace_pages_start
;
1311 cnt
= num_to_init
/ ENTRIES_PER_PAGE
;
1312 pr_info("ftrace: allocating %ld entries in %d pages\n",
1313 num_to_init
, cnt
+ 1);
1315 for (i
= 0; i
< cnt
; i
++) {
1316 pg
->next
= (void *)get_zeroed_page(GFP_KERNEL
);
1318 /* If we fail, we'll try later anyway */
1329 FTRACE_ITER_FILTER
= (1 << 0),
1330 FTRACE_ITER_NOTRACE
= (1 << 1),
1331 FTRACE_ITER_FAILURES
= (1 << 2),
1332 FTRACE_ITER_PRINTALL
= (1 << 3),
1333 FTRACE_ITER_HASH
= (1 << 4),
1336 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1338 struct ftrace_iterator
{
1339 struct ftrace_page
*pg
;
1343 struct trace_parser parser
;
1347 t_hash_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1349 struct ftrace_iterator
*iter
= m
->private;
1350 struct hlist_node
*hnd
= v
;
1351 struct hlist_head
*hhd
;
1353 WARN_ON(!(iter
->flags
& FTRACE_ITER_HASH
));
1358 if (iter
->hidx
>= FTRACE_FUNC_HASHSIZE
)
1361 hhd
= &ftrace_func_hash
[iter
->hidx
];
1363 if (hlist_empty(hhd
)) {
1382 static void *t_hash_start(struct seq_file
*m
, loff_t
*pos
)
1384 struct ftrace_iterator
*iter
= m
->private;
1388 if (!(iter
->flags
& FTRACE_ITER_HASH
))
1391 iter
->flags
|= FTRACE_ITER_HASH
;
1394 for (l
= 0; l
<= *pos
; ) {
1395 p
= t_hash_next(m
, p
, &l
);
1402 static int t_hash_show(struct seq_file
*m
, void *v
)
1404 struct ftrace_func_probe
*rec
;
1405 struct hlist_node
*hnd
= v
;
1407 rec
= hlist_entry(hnd
, struct ftrace_func_probe
, node
);
1409 if (rec
->ops
->print
)
1410 return rec
->ops
->print(m
, rec
->ip
, rec
->ops
, rec
->data
);
1412 seq_printf(m
, "%ps:%ps", (void *)rec
->ip
, (void *)rec
->ops
->func
);
1415 seq_printf(m
, ":%p", rec
->data
);
1422 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1424 struct ftrace_iterator
*iter
= m
->private;
1425 struct dyn_ftrace
*rec
= NULL
;
1427 if (iter
->flags
& FTRACE_ITER_HASH
)
1428 return t_hash_next(m
, v
, pos
);
1432 if (iter
->flags
& FTRACE_ITER_PRINTALL
)
1436 if (iter
->idx
>= iter
->pg
->index
) {
1437 if (iter
->pg
->next
) {
1438 iter
->pg
= iter
->pg
->next
;
1443 rec
= &iter
->pg
->records
[iter
->idx
++];
1444 if ((rec
->flags
& FTRACE_FL_FREE
) ||
1446 (!(iter
->flags
& FTRACE_ITER_FAILURES
) &&
1447 (rec
->flags
& FTRACE_FL_FAILED
)) ||
1449 ((iter
->flags
& FTRACE_ITER_FAILURES
) &&
1450 !(rec
->flags
& FTRACE_FL_FAILED
)) ||
1452 ((iter
->flags
& FTRACE_ITER_FILTER
) &&
1453 !(rec
->flags
& FTRACE_FL_FILTER
)) ||
1455 ((iter
->flags
& FTRACE_ITER_NOTRACE
) &&
1456 !(rec
->flags
& FTRACE_FL_NOTRACE
))) {
1465 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
1467 struct ftrace_iterator
*iter
= m
->private;
1471 mutex_lock(&ftrace_lock
);
1473 * For set_ftrace_filter reading, if we have the filter
1474 * off, we can short cut and just print out that all
1475 * functions are enabled.
1477 if (iter
->flags
& FTRACE_ITER_FILTER
&& !ftrace_filtered
) {
1479 return t_hash_start(m
, pos
);
1480 iter
->flags
|= FTRACE_ITER_PRINTALL
;
1484 if (iter
->flags
& FTRACE_ITER_HASH
)
1485 return t_hash_start(m
, pos
);
1487 iter
->pg
= ftrace_pages_start
;
1489 for (l
= 0; l
<= *pos
; ) {
1490 p
= t_next(m
, p
, &l
);
1495 if (!p
&& iter
->flags
& FTRACE_ITER_FILTER
)
1496 return t_hash_start(m
, pos
);
1501 static void t_stop(struct seq_file
*m
, void *p
)
1503 mutex_unlock(&ftrace_lock
);
1506 static int t_show(struct seq_file
*m
, void *v
)
1508 struct ftrace_iterator
*iter
= m
->private;
1509 struct dyn_ftrace
*rec
= v
;
1511 if (iter
->flags
& FTRACE_ITER_HASH
)
1512 return t_hash_show(m
, v
);
1514 if (iter
->flags
& FTRACE_ITER_PRINTALL
) {
1515 seq_printf(m
, "#### all functions enabled ####\n");
1522 seq_printf(m
, "%ps\n", (void *)rec
->ip
);
1527 static const struct seq_operations show_ftrace_seq_ops
= {
1535 ftrace_avail_open(struct inode
*inode
, struct file
*file
)
1537 struct ftrace_iterator
*iter
;
1540 if (unlikely(ftrace_disabled
))
1543 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
1547 iter
->pg
= ftrace_pages_start
;
1549 ret
= seq_open(file
, &show_ftrace_seq_ops
);
1551 struct seq_file
*m
= file
->private_data
;
1562 ftrace_failures_open(struct inode
*inode
, struct file
*file
)
1566 struct ftrace_iterator
*iter
;
1568 ret
= ftrace_avail_open(inode
, file
);
1570 m
= (struct seq_file
*)file
->private_data
;
1571 iter
= (struct ftrace_iterator
*)m
->private;
1572 iter
->flags
= FTRACE_ITER_FAILURES
;
1579 static void ftrace_filter_reset(int enable
)
1581 struct ftrace_page
*pg
;
1582 struct dyn_ftrace
*rec
;
1583 unsigned long type
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1585 mutex_lock(&ftrace_lock
);
1587 ftrace_filtered
= 0;
1588 do_for_each_ftrace_rec(pg
, rec
) {
1589 if (rec
->flags
& FTRACE_FL_FAILED
)
1591 rec
->flags
&= ~type
;
1592 } while_for_each_ftrace_rec();
1593 mutex_unlock(&ftrace_lock
);
1597 ftrace_regex_open(struct inode
*inode
, struct file
*file
, int enable
)
1599 struct ftrace_iterator
*iter
;
1602 if (unlikely(ftrace_disabled
))
1605 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
1609 if (trace_parser_get_init(&iter
->parser
, FTRACE_BUFF_MAX
)) {
1614 mutex_lock(&ftrace_regex_lock
);
1615 if ((file
->f_mode
& FMODE_WRITE
) &&
1616 (file
->f_flags
& O_TRUNC
))
1617 ftrace_filter_reset(enable
);
1619 if (file
->f_mode
& FMODE_READ
) {
1620 iter
->pg
= ftrace_pages_start
;
1621 iter
->flags
= enable
? FTRACE_ITER_FILTER
:
1622 FTRACE_ITER_NOTRACE
;
1624 ret
= seq_open(file
, &show_ftrace_seq_ops
);
1626 struct seq_file
*m
= file
->private_data
;
1629 trace_parser_put(&iter
->parser
);
1633 file
->private_data
= iter
;
1634 mutex_unlock(&ftrace_regex_lock
);
1640 ftrace_filter_open(struct inode
*inode
, struct file
*file
)
1642 return ftrace_regex_open(inode
, file
, 1);
1646 ftrace_notrace_open(struct inode
*inode
, struct file
*file
)
1648 return ftrace_regex_open(inode
, file
, 0);
1652 ftrace_regex_lseek(struct file
*file
, loff_t offset
, int origin
)
1656 if (file
->f_mode
& FMODE_READ
)
1657 ret
= seq_lseek(file
, offset
, origin
);
1659 file
->f_pos
= ret
= 1;
1672 * (static function - no need for kernel doc)
1674 * Pass in a buffer containing a glob and this function will
1675 * set search to point to the search part of the buffer and
1676 * return the type of search it is (see enum above).
1677 * This does modify buff.
1679 * Returns enum type.
1680 * search returns the pointer to use for comparison.
1681 * not returns 1 if buff started with a '!'
1685 ftrace_setup_glob(char *buff
, int len
, char **search
, int *not)
1687 int type
= MATCH_FULL
;
1690 if (buff
[0] == '!') {
1699 for (i
= 0; i
< len
; i
++) {
1700 if (buff
[i
] == '*') {
1703 type
= MATCH_END_ONLY
;
1705 if (type
== MATCH_END_ONLY
)
1706 type
= MATCH_MIDDLE_ONLY
;
1708 type
= MATCH_FRONT_ONLY
;
1718 static int ftrace_match(char *str
, char *regex
, int len
, int type
)
1725 if (strcmp(str
, regex
) == 0)
1728 case MATCH_FRONT_ONLY
:
1729 if (strncmp(str
, regex
, len
) == 0)
1732 case MATCH_MIDDLE_ONLY
:
1733 if (strstr(str
, regex
))
1736 case MATCH_END_ONLY
:
1737 ptr
= strstr(str
, regex
);
1738 if (ptr
&& (ptr
[len
] == 0))
1747 ftrace_match_record(struct dyn_ftrace
*rec
, char *regex
, int len
, int type
)
1749 char str
[KSYM_SYMBOL_LEN
];
1751 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
1752 return ftrace_match(str
, regex
, len
, type
);
1755 static void ftrace_match_records(char *buff
, int len
, int enable
)
1757 unsigned int search_len
;
1758 struct ftrace_page
*pg
;
1759 struct dyn_ftrace
*rec
;
1765 flag
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1766 type
= ftrace_setup_glob(buff
, len
, &search
, ¬);
1768 search_len
= strlen(search
);
1770 mutex_lock(&ftrace_lock
);
1771 do_for_each_ftrace_rec(pg
, rec
) {
1773 if (rec
->flags
& FTRACE_FL_FAILED
)
1776 if (ftrace_match_record(rec
, search
, search_len
, type
)) {
1778 rec
->flags
&= ~flag
;
1783 * Only enable filtering if we have a function that
1786 if (enable
&& (rec
->flags
& FTRACE_FL_FILTER
))
1787 ftrace_filtered
= 1;
1788 } while_for_each_ftrace_rec();
1789 mutex_unlock(&ftrace_lock
);
1793 ftrace_match_module_record(struct dyn_ftrace
*rec
, char *mod
,
1794 char *regex
, int len
, int type
)
1796 char str
[KSYM_SYMBOL_LEN
];
1799 kallsyms_lookup(rec
->ip
, NULL
, NULL
, &modname
, str
);
1801 if (!modname
|| strcmp(modname
, mod
))
1804 /* blank search means to match all funcs in the mod */
1806 return ftrace_match(str
, regex
, len
, type
);
1811 static void ftrace_match_module_records(char *buff
, char *mod
, int enable
)
1813 unsigned search_len
= 0;
1814 struct ftrace_page
*pg
;
1815 struct dyn_ftrace
*rec
;
1816 int type
= MATCH_FULL
;
1817 char *search
= buff
;
1821 flag
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1823 /* blank or '*' mean the same */
1824 if (strcmp(buff
, "*") == 0)
1827 /* handle the case of 'dont filter this module' */
1828 if (strcmp(buff
, "!") == 0 || strcmp(buff
, "!*") == 0) {
1834 type
= ftrace_setup_glob(buff
, strlen(buff
), &search
, ¬);
1835 search_len
= strlen(search
);
1838 mutex_lock(&ftrace_lock
);
1839 do_for_each_ftrace_rec(pg
, rec
) {
1841 if (rec
->flags
& FTRACE_FL_FAILED
)
1844 if (ftrace_match_module_record(rec
, mod
,
1845 search
, search_len
, type
)) {
1847 rec
->flags
&= ~flag
;
1851 if (enable
&& (rec
->flags
& FTRACE_FL_FILTER
))
1852 ftrace_filtered
= 1;
1854 } while_for_each_ftrace_rec();
1855 mutex_unlock(&ftrace_lock
);
1859 * We register the module command as a template to show others how
1860 * to register the a command as well.
1864 ftrace_mod_callback(char *func
, char *cmd
, char *param
, int enable
)
1869 * cmd == 'mod' because we only registered this func
1870 * for the 'mod' ftrace_func_command.
1871 * But if you register one func with multiple commands,
1872 * you can tell which command was used by the cmd
1876 /* we must have a module name */
1880 mod
= strsep(¶m
, ":");
1884 ftrace_match_module_records(func
, mod
, enable
);
1888 static struct ftrace_func_command ftrace_mod_cmd
= {
1890 .func
= ftrace_mod_callback
,
1893 static int __init
ftrace_mod_cmd_init(void)
1895 return register_ftrace_command(&ftrace_mod_cmd
);
1897 device_initcall(ftrace_mod_cmd_init
);
1900 function_trace_probe_call(unsigned long ip
, unsigned long parent_ip
)
1902 struct ftrace_func_probe
*entry
;
1903 struct hlist_head
*hhd
;
1904 struct hlist_node
*n
;
1908 key
= hash_long(ip
, FTRACE_HASH_BITS
);
1910 hhd
= &ftrace_func_hash
[key
];
1912 if (hlist_empty(hhd
))
1916 * Disable preemption for these calls to prevent a RCU grace
1917 * period. This syncs the hash iteration and freeing of items
1918 * on the hash. rcu_read_lock is too dangerous here.
1920 resched
= ftrace_preempt_disable();
1921 hlist_for_each_entry_rcu(entry
, n
, hhd
, node
) {
1922 if (entry
->ip
== ip
)
1923 entry
->ops
->func(ip
, parent_ip
, &entry
->data
);
1925 ftrace_preempt_enable(resched
);
1928 static struct ftrace_ops trace_probe_ops __read_mostly
=
1930 .func
= function_trace_probe_call
,
1933 static int ftrace_probe_registered
;
1935 static void __enable_ftrace_function_probe(void)
1939 if (ftrace_probe_registered
)
1942 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
1943 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
1947 /* Nothing registered? */
1948 if (i
== FTRACE_FUNC_HASHSIZE
)
1951 __register_ftrace_function(&trace_probe_ops
);
1953 ftrace_probe_registered
= 1;
1956 static void __disable_ftrace_function_probe(void)
1960 if (!ftrace_probe_registered
)
1963 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
1964 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
1969 /* no more funcs left */
1970 __unregister_ftrace_function(&trace_probe_ops
);
1972 ftrace_probe_registered
= 0;
1976 static void ftrace_free_entry_rcu(struct rcu_head
*rhp
)
1978 struct ftrace_func_probe
*entry
=
1979 container_of(rhp
, struct ftrace_func_probe
, rcu
);
1981 if (entry
->ops
->free
)
1982 entry
->ops
->free(&entry
->data
);
1988 register_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
1991 struct ftrace_func_probe
*entry
;
1992 struct ftrace_page
*pg
;
1993 struct dyn_ftrace
*rec
;
1999 type
= ftrace_setup_glob(glob
, strlen(glob
), &search
, ¬);
2000 len
= strlen(search
);
2002 /* we do not support '!' for function probes */
2006 mutex_lock(&ftrace_lock
);
2007 do_for_each_ftrace_rec(pg
, rec
) {
2009 if (rec
->flags
& FTRACE_FL_FAILED
)
2012 if (!ftrace_match_record(rec
, search
, len
, type
))
2015 entry
= kmalloc(sizeof(*entry
), GFP_KERNEL
);
2017 /* If we did not process any, then return error */
2028 * The caller might want to do something special
2029 * for each function we find. We call the callback
2030 * to give the caller an opportunity to do so.
2032 if (ops
->callback
) {
2033 if (ops
->callback(rec
->ip
, &entry
->data
) < 0) {
2034 /* caller does not like this func */
2041 entry
->ip
= rec
->ip
;
2043 key
= hash_long(entry
->ip
, FTRACE_HASH_BITS
);
2044 hlist_add_head_rcu(&entry
->node
, &ftrace_func_hash
[key
]);
2046 } while_for_each_ftrace_rec();
2047 __enable_ftrace_function_probe();
2050 mutex_unlock(&ftrace_lock
);
2056 PROBE_TEST_FUNC
= 1,
2061 __unregister_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
2062 void *data
, int flags
)
2064 struct ftrace_func_probe
*entry
;
2065 struct hlist_node
*n
, *tmp
;
2066 char str
[KSYM_SYMBOL_LEN
];
2067 int type
= MATCH_FULL
;
2071 if (glob
&& (strcmp(glob
, "*") == 0 || !strlen(glob
)))
2076 type
= ftrace_setup_glob(glob
, strlen(glob
), &search
, ¬);
2077 len
= strlen(search
);
2079 /* we do not support '!' for function probes */
2084 mutex_lock(&ftrace_lock
);
2085 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
2086 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
2088 hlist_for_each_entry_safe(entry
, n
, tmp
, hhd
, node
) {
2090 /* break up if statements for readability */
2091 if ((flags
& PROBE_TEST_FUNC
) && entry
->ops
!= ops
)
2094 if ((flags
& PROBE_TEST_DATA
) && entry
->data
!= data
)
2097 /* do this last, since it is the most expensive */
2099 kallsyms_lookup(entry
->ip
, NULL
, NULL
,
2101 if (!ftrace_match(str
, glob
, len
, type
))
2105 hlist_del(&entry
->node
);
2106 call_rcu(&entry
->rcu
, ftrace_free_entry_rcu
);
2109 __disable_ftrace_function_probe();
2110 mutex_unlock(&ftrace_lock
);
2114 unregister_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
2117 __unregister_ftrace_function_probe(glob
, ops
, data
,
2118 PROBE_TEST_FUNC
| PROBE_TEST_DATA
);
2122 unregister_ftrace_function_probe_func(char *glob
, struct ftrace_probe_ops
*ops
)
2124 __unregister_ftrace_function_probe(glob
, ops
, NULL
, PROBE_TEST_FUNC
);
2127 void unregister_ftrace_function_probe_all(char *glob
)
2129 __unregister_ftrace_function_probe(glob
, NULL
, NULL
, 0);
2132 static LIST_HEAD(ftrace_commands
);
2133 static DEFINE_MUTEX(ftrace_cmd_mutex
);
2135 int register_ftrace_command(struct ftrace_func_command
*cmd
)
2137 struct ftrace_func_command
*p
;
2140 mutex_lock(&ftrace_cmd_mutex
);
2141 list_for_each_entry(p
, &ftrace_commands
, list
) {
2142 if (strcmp(cmd
->name
, p
->name
) == 0) {
2147 list_add(&cmd
->list
, &ftrace_commands
);
2149 mutex_unlock(&ftrace_cmd_mutex
);
2154 int unregister_ftrace_command(struct ftrace_func_command
*cmd
)
2156 struct ftrace_func_command
*p
, *n
;
2159 mutex_lock(&ftrace_cmd_mutex
);
2160 list_for_each_entry_safe(p
, n
, &ftrace_commands
, list
) {
2161 if (strcmp(cmd
->name
, p
->name
) == 0) {
2163 list_del_init(&p
->list
);
2168 mutex_unlock(&ftrace_cmd_mutex
);
2173 static int ftrace_process_regex(char *buff
, int len
, int enable
)
2175 char *func
, *command
, *next
= buff
;
2176 struct ftrace_func_command
*p
;
2179 func
= strsep(&next
, ":");
2182 ftrace_match_records(func
, len
, enable
);
2188 command
= strsep(&next
, ":");
2190 mutex_lock(&ftrace_cmd_mutex
);
2191 list_for_each_entry(p
, &ftrace_commands
, list
) {
2192 if (strcmp(p
->name
, command
) == 0) {
2193 ret
= p
->func(func
, command
, next
, enable
);
2198 mutex_unlock(&ftrace_cmd_mutex
);
2204 ftrace_regex_write(struct file
*file
, const char __user
*ubuf
,
2205 size_t cnt
, loff_t
*ppos
, int enable
)
2207 struct ftrace_iterator
*iter
;
2208 struct trace_parser
*parser
;
2214 mutex_lock(&ftrace_regex_lock
);
2216 if (file
->f_mode
& FMODE_READ
) {
2217 struct seq_file
*m
= file
->private_data
;
2220 iter
= file
->private_data
;
2222 parser
= &iter
->parser
;
2223 read
= trace_get_user(parser
, ubuf
, cnt
, ppos
);
2225 if (read
>= 0 && trace_parser_loaded(parser
) &&
2226 !trace_parser_cont(parser
)) {
2227 ret
= ftrace_process_regex(parser
->buffer
,
2228 parser
->idx
, enable
);
2232 trace_parser_clear(parser
);
2237 mutex_unlock(&ftrace_regex_lock
);
2243 ftrace_filter_write(struct file
*file
, const char __user
*ubuf
,
2244 size_t cnt
, loff_t
*ppos
)
2246 return ftrace_regex_write(file
, ubuf
, cnt
, ppos
, 1);
2250 ftrace_notrace_write(struct file
*file
, const char __user
*ubuf
,
2251 size_t cnt
, loff_t
*ppos
)
2253 return ftrace_regex_write(file
, ubuf
, cnt
, ppos
, 0);
2257 ftrace_set_regex(unsigned char *buf
, int len
, int reset
, int enable
)
2259 if (unlikely(ftrace_disabled
))
2262 mutex_lock(&ftrace_regex_lock
);
2264 ftrace_filter_reset(enable
);
2266 ftrace_match_records(buf
, len
, enable
);
2267 mutex_unlock(&ftrace_regex_lock
);
2271 * ftrace_set_filter - set a function to filter on in ftrace
2272 * @buf - the string that holds the function filter text.
2273 * @len - the length of the string.
2274 * @reset - non zero to reset all filters before applying this filter.
2276 * Filters denote which functions should be enabled when tracing is enabled.
2277 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2279 void ftrace_set_filter(unsigned char *buf
, int len
, int reset
)
2281 ftrace_set_regex(buf
, len
, reset
, 1);
2285 * ftrace_set_notrace - set a function to not trace in ftrace
2286 * @buf - the string that holds the function notrace text.
2287 * @len - the length of the string.
2288 * @reset - non zero to reset all filters before applying this filter.
2290 * Notrace Filters denote which functions should not be enabled when tracing
2291 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2294 void ftrace_set_notrace(unsigned char *buf
, int len
, int reset
)
2296 ftrace_set_regex(buf
, len
, reset
, 0);
2300 * command line interface to allow users to set filters on boot up.
2302 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
2303 static char ftrace_notrace_buf
[FTRACE_FILTER_SIZE
] __initdata
;
2304 static char ftrace_filter_buf
[FTRACE_FILTER_SIZE
] __initdata
;
2306 static int __init
set_ftrace_notrace(char *str
)
2308 strncpy(ftrace_notrace_buf
, str
, FTRACE_FILTER_SIZE
);
2311 __setup("ftrace_notrace=", set_ftrace_notrace
);
2313 static int __init
set_ftrace_filter(char *str
)
2315 strncpy(ftrace_filter_buf
, str
, FTRACE_FILTER_SIZE
);
2318 __setup("ftrace_filter=", set_ftrace_filter
);
2320 static void __init
set_ftrace_early_filter(char *buf
, int enable
)
2325 func
= strsep(&buf
, ",");
2326 ftrace_set_regex(func
, strlen(func
), 0, enable
);
2330 static void __init
set_ftrace_early_filters(void)
2332 if (ftrace_filter_buf
[0])
2333 set_ftrace_early_filter(ftrace_filter_buf
, 1);
2334 if (ftrace_notrace_buf
[0])
2335 set_ftrace_early_filter(ftrace_notrace_buf
, 0);
2339 ftrace_regex_release(struct inode
*inode
, struct file
*file
, int enable
)
2341 struct seq_file
*m
= (struct seq_file
*)file
->private_data
;
2342 struct ftrace_iterator
*iter
;
2343 struct trace_parser
*parser
;
2345 mutex_lock(&ftrace_regex_lock
);
2346 if (file
->f_mode
& FMODE_READ
) {
2349 seq_release(inode
, file
);
2351 iter
= file
->private_data
;
2353 parser
= &iter
->parser
;
2354 if (trace_parser_loaded(parser
)) {
2355 parser
->buffer
[parser
->idx
] = 0;
2356 ftrace_match_records(parser
->buffer
, parser
->idx
, enable
);
2359 mutex_lock(&ftrace_lock
);
2360 if (ftrace_start_up
&& ftrace_enabled
)
2361 ftrace_run_update_code(FTRACE_ENABLE_CALLS
);
2362 mutex_unlock(&ftrace_lock
);
2364 trace_parser_put(parser
);
2367 mutex_unlock(&ftrace_regex_lock
);
2372 ftrace_filter_release(struct inode
*inode
, struct file
*file
)
2374 return ftrace_regex_release(inode
, file
, 1);
2378 ftrace_notrace_release(struct inode
*inode
, struct file
*file
)
2380 return ftrace_regex_release(inode
, file
, 0);
2383 static const struct file_operations ftrace_avail_fops
= {
2384 .open
= ftrace_avail_open
,
2386 .llseek
= seq_lseek
,
2387 .release
= seq_release_private
,
2390 static const struct file_operations ftrace_failures_fops
= {
2391 .open
= ftrace_failures_open
,
2393 .llseek
= seq_lseek
,
2394 .release
= seq_release_private
,
2397 static const struct file_operations ftrace_filter_fops
= {
2398 .open
= ftrace_filter_open
,
2400 .write
= ftrace_filter_write
,
2401 .llseek
= ftrace_regex_lseek
,
2402 .release
= ftrace_filter_release
,
2405 static const struct file_operations ftrace_notrace_fops
= {
2406 .open
= ftrace_notrace_open
,
2408 .write
= ftrace_notrace_write
,
2409 .llseek
= ftrace_regex_lseek
,
2410 .release
= ftrace_notrace_release
,
2413 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2415 static DEFINE_MUTEX(graph_lock
);
2417 int ftrace_graph_count
;
2418 unsigned long ftrace_graph_funcs
[FTRACE_GRAPH_MAX_FUNCS
] __read_mostly
;
2421 __g_next(struct seq_file
*m
, loff_t
*pos
)
2423 if (*pos
>= ftrace_graph_count
)
2425 return &ftrace_graph_funcs
[*pos
];
2429 g_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
2432 return __g_next(m
, pos
);
2435 static void *g_start(struct seq_file
*m
, loff_t
*pos
)
2437 mutex_lock(&graph_lock
);
2439 /* Nothing, tell g_show to print all functions are enabled */
2440 if (!ftrace_graph_count
&& !*pos
)
2443 return __g_next(m
, pos
);
2446 static void g_stop(struct seq_file
*m
, void *p
)
2448 mutex_unlock(&graph_lock
);
2451 static int g_show(struct seq_file
*m
, void *v
)
2453 unsigned long *ptr
= v
;
2458 if (ptr
== (unsigned long *)1) {
2459 seq_printf(m
, "#### all functions enabled ####\n");
2463 seq_printf(m
, "%ps\n", (void *)*ptr
);
2468 static const struct seq_operations ftrace_graph_seq_ops
= {
2476 ftrace_graph_open(struct inode
*inode
, struct file
*file
)
2480 if (unlikely(ftrace_disabled
))
2483 mutex_lock(&graph_lock
);
2484 if ((file
->f_mode
& FMODE_WRITE
) &&
2485 (file
->f_flags
& O_TRUNC
)) {
2486 ftrace_graph_count
= 0;
2487 memset(ftrace_graph_funcs
, 0, sizeof(ftrace_graph_funcs
));
2489 mutex_unlock(&graph_lock
);
2491 if (file
->f_mode
& FMODE_READ
)
2492 ret
= seq_open(file
, &ftrace_graph_seq_ops
);
2498 ftrace_graph_release(struct inode
*inode
, struct file
*file
)
2500 if (file
->f_mode
& FMODE_READ
)
2501 seq_release(inode
, file
);
2506 ftrace_set_func(unsigned long *array
, int *idx
, char *buffer
)
2508 struct dyn_ftrace
*rec
;
2509 struct ftrace_page
*pg
;
2517 if (ftrace_disabled
)
2521 type
= ftrace_setup_glob(buffer
, strlen(buffer
), &search
, ¬);
2525 search_len
= strlen(search
);
2527 mutex_lock(&ftrace_lock
);
2528 do_for_each_ftrace_rec(pg
, rec
) {
2530 if (*idx
>= FTRACE_GRAPH_MAX_FUNCS
)
2533 if (rec
->flags
& (FTRACE_FL_FAILED
| FTRACE_FL_FREE
))
2536 if (ftrace_match_record(rec
, search
, search_len
, type
)) {
2537 /* ensure it is not already in the array */
2539 for (i
= 0; i
< *idx
; i
++)
2540 if (array
[i
] == rec
->ip
) {
2545 array
[(*idx
)++] = rec
->ip
;
2549 } while_for_each_ftrace_rec();
2551 mutex_unlock(&ftrace_lock
);
2553 return found
? 0 : -EINVAL
;
2557 ftrace_graph_write(struct file
*file
, const char __user
*ubuf
,
2558 size_t cnt
, loff_t
*ppos
)
2560 struct trace_parser parser
;
2563 if (!cnt
|| cnt
< 0)
2566 mutex_lock(&graph_lock
);
2568 if (ftrace_graph_count
>= FTRACE_GRAPH_MAX_FUNCS
) {
2573 if (trace_parser_get_init(&parser
, FTRACE_BUFF_MAX
)) {
2578 read
= trace_get_user(&parser
, ubuf
, cnt
, ppos
);
2580 if (read
>= 0 && trace_parser_loaded((&parser
))) {
2581 parser
.buffer
[parser
.idx
] = 0;
2583 /* we allow only one expression at a time */
2584 ret
= ftrace_set_func(ftrace_graph_funcs
, &ftrace_graph_count
,
2593 trace_parser_put(&parser
);
2595 mutex_unlock(&graph_lock
);
2600 static const struct file_operations ftrace_graph_fops
= {
2601 .open
= ftrace_graph_open
,
2603 .write
= ftrace_graph_write
,
2604 .release
= ftrace_graph_release
,
2606 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2608 static __init
int ftrace_init_dyn_debugfs(struct dentry
*d_tracer
)
2611 trace_create_file("available_filter_functions", 0444,
2612 d_tracer
, NULL
, &ftrace_avail_fops
);
2614 trace_create_file("failures", 0444,
2615 d_tracer
, NULL
, &ftrace_failures_fops
);
2617 trace_create_file("set_ftrace_filter", 0644, d_tracer
,
2618 NULL
, &ftrace_filter_fops
);
2620 trace_create_file("set_ftrace_notrace", 0644, d_tracer
,
2621 NULL
, &ftrace_notrace_fops
);
2623 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2624 trace_create_file("set_graph_function", 0444, d_tracer
,
2626 &ftrace_graph_fops
);
2627 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2632 static int ftrace_convert_nops(struct module
*mod
,
2633 unsigned long *start
,
2638 unsigned long flags
;
2640 mutex_lock(&ftrace_lock
);
2643 addr
= ftrace_call_adjust(*p
++);
2645 * Some architecture linkers will pad between
2646 * the different mcount_loc sections of different
2647 * object files to satisfy alignments.
2648 * Skip any NULL pointers.
2652 ftrace_record_ip(addr
);
2655 /* disable interrupts to prevent kstop machine */
2656 local_irq_save(flags
);
2657 ftrace_update_code(mod
);
2658 local_irq_restore(flags
);
2659 mutex_unlock(&ftrace_lock
);
2664 #ifdef CONFIG_MODULES
2665 void ftrace_release(void *start
, void *end
)
2667 struct dyn_ftrace
*rec
;
2668 struct ftrace_page
*pg
;
2669 unsigned long s
= (unsigned long)start
;
2670 unsigned long e
= (unsigned long)end
;
2672 if (ftrace_disabled
|| !start
|| start
== end
)
2675 mutex_lock(&ftrace_lock
);
2676 do_for_each_ftrace_rec(pg
, rec
) {
2677 if ((rec
->ip
>= s
) && (rec
->ip
< e
)) {
2679 * rec->ip is changed in ftrace_free_rec()
2680 * It should not between s and e if record was freed.
2682 FTRACE_WARN_ON(rec
->flags
& FTRACE_FL_FREE
);
2683 ftrace_free_rec(rec
);
2685 } while_for_each_ftrace_rec();
2686 mutex_unlock(&ftrace_lock
);
2689 static void ftrace_init_module(struct module
*mod
,
2690 unsigned long *start
, unsigned long *end
)
2692 if (ftrace_disabled
|| start
== end
)
2694 ftrace_convert_nops(mod
, start
, end
);
2697 static int ftrace_module_notify(struct notifier_block
*self
,
2698 unsigned long val
, void *data
)
2700 struct module
*mod
= data
;
2703 case MODULE_STATE_COMING
:
2704 ftrace_init_module(mod
, mod
->ftrace_callsites
,
2705 mod
->ftrace_callsites
+
2706 mod
->num_ftrace_callsites
);
2708 case MODULE_STATE_GOING
:
2709 ftrace_release(mod
->ftrace_callsites
,
2710 mod
->ftrace_callsites
+
2711 mod
->num_ftrace_callsites
);
2718 static int ftrace_module_notify(struct notifier_block
*self
,
2719 unsigned long val
, void *data
)
2723 #endif /* CONFIG_MODULES */
2725 struct notifier_block ftrace_module_nb
= {
2726 .notifier_call
= ftrace_module_notify
,
2730 extern unsigned long __start_mcount_loc
[];
2731 extern unsigned long __stop_mcount_loc
[];
2733 void __init
ftrace_init(void)
2735 unsigned long count
, addr
, flags
;
2738 /* Keep the ftrace pointer to the stub */
2739 addr
= (unsigned long)ftrace_stub
;
2741 local_irq_save(flags
);
2742 ftrace_dyn_arch_init(&addr
);
2743 local_irq_restore(flags
);
2745 /* ftrace_dyn_arch_init places the return code in addr */
2749 count
= __stop_mcount_loc
- __start_mcount_loc
;
2751 ret
= ftrace_dyn_table_alloc(count
);
2755 last_ftrace_enabled
= ftrace_enabled
= 1;
2757 ret
= ftrace_convert_nops(NULL
,
2761 ret
= register_module_notifier(&ftrace_module_nb
);
2763 pr_warning("Failed to register trace ftrace module notifier\n");
2765 set_ftrace_early_filters();
2769 ftrace_disabled
= 1;
2774 static int __init
ftrace_nodyn_init(void)
2779 device_initcall(ftrace_nodyn_init
);
2781 static inline int ftrace_init_dyn_debugfs(struct dentry
*d_tracer
) { return 0; }
2782 static inline void ftrace_startup_enable(int command
) { }
2783 /* Keep as macros so we do not need to define the commands */
2784 # define ftrace_startup(command) do { } while (0)
2785 # define ftrace_shutdown(command) do { } while (0)
2786 # define ftrace_startup_sysctl() do { } while (0)
2787 # define ftrace_shutdown_sysctl() do { } while (0)
2788 #endif /* CONFIG_DYNAMIC_FTRACE */
2791 ftrace_pid_read(struct file
*file
, char __user
*ubuf
,
2792 size_t cnt
, loff_t
*ppos
)
2797 if (ftrace_pid_trace
== ftrace_swapper_pid
)
2798 r
= sprintf(buf
, "swapper tasks\n");
2799 else if (ftrace_pid_trace
)
2800 r
= sprintf(buf
, "%u\n", pid_vnr(ftrace_pid_trace
));
2802 r
= sprintf(buf
, "no pid\n");
2804 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
2807 static void clear_ftrace_swapper(void)
2809 struct task_struct
*p
;
2813 for_each_online_cpu(cpu
) {
2815 clear_tsk_trace_trace(p
);
2820 static void set_ftrace_swapper(void)
2822 struct task_struct
*p
;
2826 for_each_online_cpu(cpu
) {
2828 set_tsk_trace_trace(p
);
2833 static void clear_ftrace_pid(struct pid
*pid
)
2835 struct task_struct
*p
;
2838 do_each_pid_task(pid
, PIDTYPE_PID
, p
) {
2839 clear_tsk_trace_trace(p
);
2840 } while_each_pid_task(pid
, PIDTYPE_PID
, p
);
2846 static void set_ftrace_pid(struct pid
*pid
)
2848 struct task_struct
*p
;
2851 do_each_pid_task(pid
, PIDTYPE_PID
, p
) {
2852 set_tsk_trace_trace(p
);
2853 } while_each_pid_task(pid
, PIDTYPE_PID
, p
);
2857 static void clear_ftrace_pid_task(struct pid
**pid
)
2859 if (*pid
== ftrace_swapper_pid
)
2860 clear_ftrace_swapper();
2862 clear_ftrace_pid(*pid
);
2867 static void set_ftrace_pid_task(struct pid
*pid
)
2869 if (pid
== ftrace_swapper_pid
)
2870 set_ftrace_swapper();
2872 set_ftrace_pid(pid
);
2876 ftrace_pid_write(struct file
*filp
, const char __user
*ubuf
,
2877 size_t cnt
, loff_t
*ppos
)
2884 if (cnt
>= sizeof(buf
))
2887 if (copy_from_user(&buf
, ubuf
, cnt
))
2892 ret
= strict_strtol(buf
, 10, &val
);
2896 mutex_lock(&ftrace_lock
);
2898 /* disable pid tracing */
2899 if (!ftrace_pid_trace
)
2902 clear_ftrace_pid_task(&ftrace_pid_trace
);
2905 /* swapper task is special */
2907 pid
= ftrace_swapper_pid
;
2908 if (pid
== ftrace_pid_trace
)
2911 pid
= find_get_pid(val
);
2913 if (pid
== ftrace_pid_trace
) {
2919 if (ftrace_pid_trace
)
2920 clear_ftrace_pid_task(&ftrace_pid_trace
);
2925 ftrace_pid_trace
= pid
;
2927 set_ftrace_pid_task(ftrace_pid_trace
);
2930 /* update the function call */
2931 ftrace_update_pid_func();
2932 ftrace_startup_enable(0);
2935 mutex_unlock(&ftrace_lock
);
2940 static const struct file_operations ftrace_pid_fops
= {
2941 .read
= ftrace_pid_read
,
2942 .write
= ftrace_pid_write
,
2945 static __init
int ftrace_init_debugfs(void)
2947 struct dentry
*d_tracer
;
2949 d_tracer
= tracing_init_dentry();
2953 ftrace_init_dyn_debugfs(d_tracer
);
2955 trace_create_file("set_ftrace_pid", 0644, d_tracer
,
2956 NULL
, &ftrace_pid_fops
);
2958 ftrace_profile_debugfs(d_tracer
);
2962 fs_initcall(ftrace_init_debugfs
);
2965 * ftrace_kill - kill ftrace
2967 * This function should be used by panic code. It stops ftrace
2968 * but in a not so nice way. If you need to simply kill ftrace
2969 * from a non-atomic section, use ftrace_kill.
2971 void ftrace_kill(void)
2973 ftrace_disabled
= 1;
2975 clear_ftrace_function();
2979 * register_ftrace_function - register a function for profiling
2980 * @ops - ops structure that holds the function for profiling.
2982 * Register a function to be called by all functions in the
2985 * Note: @ops->func and all the functions it calls must be labeled
2986 * with "notrace", otherwise it will go into a
2989 int register_ftrace_function(struct ftrace_ops
*ops
)
2993 if (unlikely(ftrace_disabled
))
2996 mutex_lock(&ftrace_lock
);
2998 ret
= __register_ftrace_function(ops
);
3001 mutex_unlock(&ftrace_lock
);
3006 * unregister_ftrace_function - unregister a function for profiling.
3007 * @ops - ops structure that holds the function to unregister
3009 * Unregister a function that was added to be called by ftrace profiling.
3011 int unregister_ftrace_function(struct ftrace_ops
*ops
)
3015 mutex_lock(&ftrace_lock
);
3016 ret
= __unregister_ftrace_function(ops
);
3018 mutex_unlock(&ftrace_lock
);
3024 ftrace_enable_sysctl(struct ctl_table
*table
, int write
,
3025 void __user
*buffer
, size_t *lenp
,
3030 if (unlikely(ftrace_disabled
))
3033 mutex_lock(&ftrace_lock
);
3035 ret
= proc_dointvec(table
, write
, buffer
, lenp
, ppos
);
3037 if (ret
|| !write
|| (last_ftrace_enabled
== !!ftrace_enabled
))
3040 last_ftrace_enabled
= !!ftrace_enabled
;
3042 if (ftrace_enabled
) {
3044 ftrace_startup_sysctl();
3046 /* we are starting ftrace again */
3047 if (ftrace_list
!= &ftrace_list_end
) {
3048 if (ftrace_list
->next
== &ftrace_list_end
)
3049 ftrace_trace_function
= ftrace_list
->func
;
3051 ftrace_trace_function
= ftrace_list_func
;
3055 /* stopping ftrace calls (just send to ftrace_stub) */
3056 ftrace_trace_function
= ftrace_stub
;
3058 ftrace_shutdown_sysctl();
3062 mutex_unlock(&ftrace_lock
);
3066 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3068 static int ftrace_graph_active
;
3069 static struct notifier_block ftrace_suspend_notifier
;
3071 int ftrace_graph_entry_stub(struct ftrace_graph_ent
*trace
)
3076 /* The callbacks that hook a function */
3077 trace_func_graph_ret_t ftrace_graph_return
=
3078 (trace_func_graph_ret_t
)ftrace_stub
;
3079 trace_func_graph_ent_t ftrace_graph_entry
= ftrace_graph_entry_stub
;
3081 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3082 static int alloc_retstack_tasklist(struct ftrace_ret_stack
**ret_stack_list
)
3086 unsigned long flags
;
3087 int start
= 0, end
= FTRACE_RETSTACK_ALLOC_SIZE
;
3088 struct task_struct
*g
, *t
;
3090 for (i
= 0; i
< FTRACE_RETSTACK_ALLOC_SIZE
; i
++) {
3091 ret_stack_list
[i
] = kmalloc(FTRACE_RETFUNC_DEPTH
3092 * sizeof(struct ftrace_ret_stack
),
3094 if (!ret_stack_list
[i
]) {
3102 read_lock_irqsave(&tasklist_lock
, flags
);
3103 do_each_thread(g
, t
) {
3109 if (t
->ret_stack
== NULL
) {
3110 atomic_set(&t
->tracing_graph_pause
, 0);
3111 atomic_set(&t
->trace_overrun
, 0);
3112 t
->curr_ret_stack
= -1;
3113 /* Make sure the tasks see the -1 first: */
3115 t
->ret_stack
= ret_stack_list
[start
++];
3117 } while_each_thread(g
, t
);
3120 read_unlock_irqrestore(&tasklist_lock
, flags
);
3122 for (i
= start
; i
< end
; i
++)
3123 kfree(ret_stack_list
[i
]);
3128 ftrace_graph_probe_sched_switch(struct rq
*__rq
, struct task_struct
*prev
,
3129 struct task_struct
*next
)
3131 unsigned long long timestamp
;
3135 * Does the user want to count the time a function was asleep.
3136 * If so, do not update the time stamps.
3138 if (trace_flags
& TRACE_ITER_SLEEP_TIME
)
3141 timestamp
= trace_clock_local();
3143 prev
->ftrace_timestamp
= timestamp
;
3145 /* only process tasks that we timestamped */
3146 if (!next
->ftrace_timestamp
)
3150 * Update all the counters in next to make up for the
3151 * time next was sleeping.
3153 timestamp
-= next
->ftrace_timestamp
;
3155 for (index
= next
->curr_ret_stack
; index
>= 0; index
--)
3156 next
->ret_stack
[index
].calltime
+= timestamp
;
3159 /* Allocate a return stack for each task */
3160 static int start_graph_tracing(void)
3162 struct ftrace_ret_stack
**ret_stack_list
;
3165 ret_stack_list
= kmalloc(FTRACE_RETSTACK_ALLOC_SIZE
*
3166 sizeof(struct ftrace_ret_stack
*),
3169 if (!ret_stack_list
)
3172 /* The cpu_boot init_task->ret_stack will never be freed */
3173 for_each_online_cpu(cpu
) {
3174 if (!idle_task(cpu
)->ret_stack
)
3175 ftrace_graph_init_task(idle_task(cpu
));
3179 ret
= alloc_retstack_tasklist(ret_stack_list
);
3180 } while (ret
== -EAGAIN
);
3183 ret
= register_trace_sched_switch(ftrace_graph_probe_sched_switch
);
3185 pr_info("ftrace_graph: Couldn't activate tracepoint"
3186 " probe to kernel_sched_switch\n");
3189 kfree(ret_stack_list
);
3194 * Hibernation protection.
3195 * The state of the current task is too much unstable during
3196 * suspend/restore to disk. We want to protect against that.
3199 ftrace_suspend_notifier_call(struct notifier_block
*bl
, unsigned long state
,
3203 case PM_HIBERNATION_PREPARE
:
3204 pause_graph_tracing();
3207 case PM_POST_HIBERNATION
:
3208 unpause_graph_tracing();
3214 int register_ftrace_graph(trace_func_graph_ret_t retfunc
,
3215 trace_func_graph_ent_t entryfunc
)
3219 mutex_lock(&ftrace_lock
);
3221 /* we currently allow only one tracer registered at a time */
3222 if (ftrace_graph_active
) {
3227 ftrace_suspend_notifier
.notifier_call
= ftrace_suspend_notifier_call
;
3228 register_pm_notifier(&ftrace_suspend_notifier
);
3230 ftrace_graph_active
++;
3231 ret
= start_graph_tracing();
3233 ftrace_graph_active
--;
3237 ftrace_graph_return
= retfunc
;
3238 ftrace_graph_entry
= entryfunc
;
3240 ftrace_startup(FTRACE_START_FUNC_RET
);
3243 mutex_unlock(&ftrace_lock
);
3247 void unregister_ftrace_graph(void)
3249 mutex_lock(&ftrace_lock
);
3251 if (unlikely(!ftrace_graph_active
))
3254 ftrace_graph_active
--;
3255 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch
);
3256 ftrace_graph_return
= (trace_func_graph_ret_t
)ftrace_stub
;
3257 ftrace_graph_entry
= ftrace_graph_entry_stub
;
3258 ftrace_shutdown(FTRACE_STOP_FUNC_RET
);
3259 unregister_pm_notifier(&ftrace_suspend_notifier
);
3262 mutex_unlock(&ftrace_lock
);
3265 /* Allocate a return stack for newly created task */
3266 void ftrace_graph_init_task(struct task_struct
*t
)
3268 /* Make sure we do not use the parent ret_stack */
3269 t
->ret_stack
= NULL
;
3271 if (ftrace_graph_active
) {
3272 struct ftrace_ret_stack
*ret_stack
;
3274 ret_stack
= kmalloc(FTRACE_RETFUNC_DEPTH
3275 * sizeof(struct ftrace_ret_stack
),
3279 t
->curr_ret_stack
= -1;
3280 atomic_set(&t
->tracing_graph_pause
, 0);
3281 atomic_set(&t
->trace_overrun
, 0);
3282 t
->ftrace_timestamp
= 0;
3283 /* make curr_ret_stack visable before we add the ret_stack */
3285 t
->ret_stack
= ret_stack
;
3289 void ftrace_graph_exit_task(struct task_struct
*t
)
3291 struct ftrace_ret_stack
*ret_stack
= t
->ret_stack
;
3293 t
->ret_stack
= NULL
;
3294 /* NULL must become visible to IRQs before we free it: */
3300 void ftrace_graph_stop(void)