projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'audit.b62' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/audit...
[deliverable/linux.git] / kernel / trace / ftrace.c
1 /*
2 * Infrastructure for profiling code inserted by 'gcc -pg'.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code in the latency_tracer, that is:
11 *
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
14 */
15
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30 #include <linux/hash.h>
31
32 #include <trace/sched.h>
33
34 #include <asm/ftrace.h>
35
36 #include "trace.h"
37
38 #define FTRACE_WARN_ON(cond) \
39 do { \
40 if (WARN_ON(cond)) \
41 ftrace_kill(); \
42 } while (0)
43
44 #define FTRACE_WARN_ON_ONCE(cond) \
45 do { \
46 if (WARN_ON_ONCE(cond)) \
47 ftrace_kill(); \
48 } while (0)
49
50 /* hash bits for specific function selection */
51 #define FTRACE_HASH_BITS 7
52 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
53
54 /* ftrace_enabled is a method to turn ftrace on or off */
55 int ftrace_enabled __read_mostly;
56 static int last_ftrace_enabled;
57
58 /* Quick disabling of function tracer. */
59 int function_trace_stop;
60
61 /*
62 * ftrace_disabled is set when an anomaly is discovered.
63 * ftrace_disabled is much stronger than ftrace_enabled.
64 */
65 static int ftrace_disabled __read_mostly;
66
67 static DEFINE_MUTEX(ftrace_lock);
68
69 static struct ftrace_ops ftrace_list_end __read_mostly =
70 {
71 .func = ftrace_stub,
72 };
73
74 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
75 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
76 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
77 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
78
79 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
80 {
81 struct ftrace_ops *op = ftrace_list;
82
83 /* in case someone actually ports this to alpha! */
84 read_barrier_depends();
85
86 while (op != &ftrace_list_end) {
87 /* silly alpha */
88 read_barrier_depends();
89 op->func(ip, parent_ip);
90 op = op->next;
91 };
92 }
93
94 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
95 {
96 if (!test_tsk_trace_trace(current))
97 return;
98
99 ftrace_pid_function(ip, parent_ip);
100 }
101
102 static void set_ftrace_pid_function(ftrace_func_t func)
103 {
104 /* do not set ftrace_pid_function to itself! */
105 if (func != ftrace_pid_func)
106 ftrace_pid_function = func;
107 }
108
109 /**
110 * clear_ftrace_function - reset the ftrace function
111 *
112 * This NULLs the ftrace function and in essence stops
113 * tracing. There may be lag
114 */
115 void clear_ftrace_function(void)
116 {
117 ftrace_trace_function = ftrace_stub;
118 __ftrace_trace_function = ftrace_stub;
119 ftrace_pid_function = ftrace_stub;
120 }
121
122 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
123 /*
124 * For those archs that do not test ftrace_trace_stop in their
125 * mcount call site, we need to do it from C.
126 */
127 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
128 {
129 if (function_trace_stop)
130 return;
131
132 __ftrace_trace_function(ip, parent_ip);
133 }
134 #endif
135
136 static int __register_ftrace_function(struct ftrace_ops *ops)
137 {
138 ops->next = ftrace_list;
139 /*
140 * We are entering ops into the ftrace_list but another
141 * CPU might be walking that list. We need to make sure
142 * the ops->next pointer is valid before another CPU sees
143 * the ops pointer included into the ftrace_list.
144 */
145 smp_wmb();
146 ftrace_list = ops;
147
148 if (ftrace_enabled) {
149 ftrace_func_t func;
150
151 if (ops->next == &ftrace_list_end)
152 func = ops->func;
153 else
154 func = ftrace_list_func;
155
156 if (ftrace_pid_trace) {
157 set_ftrace_pid_function(func);
158 func = ftrace_pid_func;
159 }
160
161 /*
162 * For one func, simply call it directly.
163 * For more than one func, call the chain.
164 */
165 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
166 ftrace_trace_function = func;
167 #else
168 __ftrace_trace_function = func;
169 ftrace_trace_function = ftrace_test_stop_func;
170 #endif
171 }
172
173 return 0;
174 }
175
176 static int __unregister_ftrace_function(struct ftrace_ops *ops)
177 {
178 struct ftrace_ops **p;
179
180 /*
181 * If we are removing the last function, then simply point
182 * to the ftrace_stub.
183 */
184 if (ftrace_list == ops && ops->next == &ftrace_list_end) {
185 ftrace_trace_function = ftrace_stub;
186 ftrace_list = &ftrace_list_end;
187 return 0;
188 }
189
190 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
191 if (*p == ops)
192 break;
193
194 if (*p != ops)
195 return -1;
196
197 *p = (*p)->next;
198
199 if (ftrace_enabled) {
200 /* If we only have one func left, then call that directly */
201 if (ftrace_list->next == &ftrace_list_end) {
202 ftrace_func_t func = ftrace_list->func;
203
204 if (ftrace_pid_trace) {
205 set_ftrace_pid_function(func);
206 func = ftrace_pid_func;
207 }
208 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
209 ftrace_trace_function = func;
210 #else
211 __ftrace_trace_function = func;
212 #endif
213 }
214 }
215
216 return 0;
217 }
218
219 static void ftrace_update_pid_func(void)
220 {
221 ftrace_func_t func;
222
223 if (ftrace_trace_function == ftrace_stub)
224 return;
225
226 func = ftrace_trace_function;
227
228 if (ftrace_pid_trace) {
229 set_ftrace_pid_function(func);
230 func = ftrace_pid_func;
231 } else {
232 if (func == ftrace_pid_func)
233 func = ftrace_pid_function;
234 }
235
236 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
237 ftrace_trace_function = func;
238 #else
239 __ftrace_trace_function = func;
240 #endif
241 }
242
243 /* set when tracing only a pid */
244 struct pid *ftrace_pid_trace;
245 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
246
247 #ifdef CONFIG_DYNAMIC_FTRACE
248
249 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
250 # error Dynamic ftrace depends on MCOUNT_RECORD
251 #endif
252
253 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
254
255 struct ftrace_func_probe {
256 struct hlist_node node;
257 struct ftrace_probe_ops *ops;
258 unsigned long flags;
259 unsigned long ip;
260 void *data;
261 struct rcu_head rcu;
262 };
263
264
265 enum {
266 FTRACE_ENABLE_CALLS = (1 << 0),
267 FTRACE_DISABLE_CALLS = (1 << 1),
268 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
269 FTRACE_ENABLE_MCOUNT = (1 << 3),
270 FTRACE_DISABLE_MCOUNT = (1 << 4),
271 FTRACE_START_FUNC_RET = (1 << 5),
272 FTRACE_STOP_FUNC_RET = (1 << 6),
273 };
274
275 static int ftrace_filtered;
276
277 static struct dyn_ftrace *ftrace_new_addrs;
278
279 static DEFINE_MUTEX(ftrace_regex_lock);
280
281 struct ftrace_page {
282 struct ftrace_page *next;
283 int index;
284 struct dyn_ftrace records[];
285 };
286
287 #define ENTRIES_PER_PAGE \
288 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
289
290 /* estimate from running different kernels */
291 #define NR_TO_INIT 10000
292
293 static struct ftrace_page *ftrace_pages_start;
294 static struct ftrace_page *ftrace_pages;
295
296 static struct dyn_ftrace *ftrace_free_records;
297
298 /*
299 * This is a double for. Do not use 'break' to break out of the loop,
300 * you must use a goto.
301 */
302 #define do_for_each_ftrace_rec(pg, rec) \
303 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
304 int _____i; \
305 for (_____i = 0; _____i < pg->index; _____i++) { \
306 rec = &pg->records[_____i];
307
308 #define while_for_each_ftrace_rec() \
309 } \
310 }
311
312 #ifdef CONFIG_KPROBES
313
314 static int frozen_record_count;
315
316 static inline void freeze_record(struct dyn_ftrace *rec)
317 {
318 if (!(rec->flags & FTRACE_FL_FROZEN)) {
319 rec->flags |= FTRACE_FL_FROZEN;
320 frozen_record_count++;
321 }
322 }
323
324 static inline void unfreeze_record(struct dyn_ftrace *rec)
325 {
326 if (rec->flags & FTRACE_FL_FROZEN) {
327 rec->flags &= ~FTRACE_FL_FROZEN;
328 frozen_record_count--;
329 }
330 }
331
332 static inline int record_frozen(struct dyn_ftrace *rec)
333 {
334 return rec->flags & FTRACE_FL_FROZEN;
335 }
336 #else
337 # define freeze_record(rec) ({ 0; })
338 # define unfreeze_record(rec) ({ 0; })
339 # define record_frozen(rec) ({ 0; })
340 #endif /* CONFIG_KPROBES */
341
342 static void ftrace_free_rec(struct dyn_ftrace *rec)
343 {
344 rec->freelist = ftrace_free_records;
345 ftrace_free_records = rec;
346 rec->flags |= FTRACE_FL_FREE;
347 }
348
349 void ftrace_release(void *start, unsigned long size)
350 {
351 struct dyn_ftrace *rec;
352 struct ftrace_page *pg;
353 unsigned long s = (unsigned long)start;
354 unsigned long e = s + size;
355
356 if (ftrace_disabled || !start)
357 return;
358
359 mutex_lock(&ftrace_lock);
360 do_for_each_ftrace_rec(pg, rec) {
361 if ((rec->ip >= s) && (rec->ip < e)) {
362 /*
363 * rec->ip is changed in ftrace_free_rec()
364 * It should not between s and e if record was freed.
365 */
366 FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
367 ftrace_free_rec(rec);
368 }
369 } while_for_each_ftrace_rec();
370 mutex_unlock(&ftrace_lock);
371 }
372
373 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
374 {
375 struct dyn_ftrace *rec;
376
377 /* First check for freed records */
378 if (ftrace_free_records) {
379 rec = ftrace_free_records;
380
381 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
382 FTRACE_WARN_ON_ONCE(1);
383 ftrace_free_records = NULL;
384 return NULL;
385 }
386
387 ftrace_free_records = rec->freelist;
388 memset(rec, 0, sizeof(*rec));
389 return rec;
390 }
391
392 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
393 if (!ftrace_pages->next) {
394 /* allocate another page */
395 ftrace_pages->next =
396 (void *)get_zeroed_page(GFP_KERNEL);
397 if (!ftrace_pages->next)
398 return NULL;
399 }
400 ftrace_pages = ftrace_pages->next;
401 }
402
403 return &ftrace_pages->records[ftrace_pages->index++];
404 }
405
406 static struct dyn_ftrace *
407 ftrace_record_ip(unsigned long ip)
408 {
409 struct dyn_ftrace *rec;
410
411 if (ftrace_disabled)
412 return NULL;
413
414 rec = ftrace_alloc_dyn_node(ip);
415 if (!rec)
416 return NULL;
417
418 rec->ip = ip;
419 rec->newlist = ftrace_new_addrs;
420 ftrace_new_addrs = rec;
421
422 return rec;
423 }
424
425 static void print_ip_ins(const char *fmt, unsigned char *p)
426 {
427 int i;
428
429 printk(KERN_CONT "%s", fmt);
430
431 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
432 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
433 }
434
435 static void ftrace_bug(int failed, unsigned long ip)
436 {
437 switch (failed) {
438 case -EFAULT:
439 FTRACE_WARN_ON_ONCE(1);
440 pr_info("ftrace faulted on modifying ");
441 print_ip_sym(ip);
442 break;
443 case -EINVAL:
444 FTRACE_WARN_ON_ONCE(1);
445 pr_info("ftrace failed to modify ");
446 print_ip_sym(ip);
447 print_ip_ins(" actual: ", (unsigned char *)ip);
448 printk(KERN_CONT "\n");
449 break;
450 case -EPERM:
451 FTRACE_WARN_ON_ONCE(1);
452 pr_info("ftrace faulted on writing ");
453 print_ip_sym(ip);
454 break;
455 default:
456 FTRACE_WARN_ON_ONCE(1);
457 pr_info("ftrace faulted on unknown error ");
458 print_ip_sym(ip);
459 }
460 }
461
462
463 static int
464 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
465 {
466 unsigned long ftrace_addr;
467 unsigned long ip, fl;
468
469 ftrace_addr = (unsigned long)FTRACE_ADDR;
470
471 ip = rec->ip;
472
473 /*
474 * If this record is not to be traced and
475 * it is not enabled then do nothing.
476 *
477 * If this record is not to be traced and
478 * it is enabled then disable it.
479 *
480 */
481 if (rec->flags & FTRACE_FL_NOTRACE) {
482 if (rec->flags & FTRACE_FL_ENABLED)
483 rec->flags &= ~FTRACE_FL_ENABLED;
484 else
485 return 0;
486
487 } else if (ftrace_filtered && enable) {
488 /*
489 * Filtering is on:
490 */
491
492 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
493
494 /* Record is filtered and enabled, do nothing */
495 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
496 return 0;
497
498 /* Record is not filtered or enabled, do nothing */
499 if (!fl)
500 return 0;
501
502 /* Record is not filtered but enabled, disable it */
503 if (fl == FTRACE_FL_ENABLED)
504 rec->flags &= ~FTRACE_FL_ENABLED;
505 else
506 /* Otherwise record is filtered but not enabled, enable it */
507 rec->flags |= FTRACE_FL_ENABLED;
508 } else {
509 /* Disable or not filtered */
510
511 if (enable) {
512 /* if record is enabled, do nothing */
513 if (rec->flags & FTRACE_FL_ENABLED)
514 return 0;
515
516 rec->flags |= FTRACE_FL_ENABLED;
517
518 } else {
519
520 /* if record is not enabled, do nothing */
521 if (!(rec->flags & FTRACE_FL_ENABLED))
522 return 0;
523
524 rec->flags &= ~FTRACE_FL_ENABLED;
525 }
526 }
527
528 if (rec->flags & FTRACE_FL_ENABLED)
529 return ftrace_make_call(rec, ftrace_addr);
530 else
531 return ftrace_make_nop(NULL, rec, ftrace_addr);
532 }
533
534 static void ftrace_replace_code(int enable)
535 {
536 struct dyn_ftrace *rec;
537 struct ftrace_page *pg;
538 int failed;
539
540 do_for_each_ftrace_rec(pg, rec) {
541 /*
542 * Skip over free records, records that have
543 * failed and not converted.
544 */
545 if (rec->flags & FTRACE_FL_FREE ||
546 rec->flags & FTRACE_FL_FAILED ||
547 !(rec->flags & FTRACE_FL_CONVERTED))
548 continue;
549
550 /* ignore updates to this record's mcount site */
551 if (get_kprobe((void *)rec->ip)) {
552 freeze_record(rec);
553 continue;
554 } else {
555 unfreeze_record(rec);
556 }
557
558 failed = __ftrace_replace_code(rec, enable);
559 if (failed) {
560 rec->flags |= FTRACE_FL_FAILED;
561 if ((system_state == SYSTEM_BOOTING) ||
562 !core_kernel_text(rec->ip)) {
563 ftrace_free_rec(rec);
564 } else {
565 ftrace_bug(failed, rec->ip);
566 /* Stop processing */
567 return;
568 }
569 }
570 } while_for_each_ftrace_rec();
571 }
572
573 static int
574 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
575 {
576 unsigned long ip;
577 int ret;
578
579 ip = rec->ip;
580
581 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
582 if (ret) {
583 ftrace_bug(ret, ip);
584 rec->flags |= FTRACE_FL_FAILED;
585 return 0;
586 }
587 return 1;
588 }
589
590 /*
591 * archs can override this function if they must do something
592 * before the modifying code is performed.
593 */
594 int __weak ftrace_arch_code_modify_prepare(void)
595 {
596 return 0;
597 }
598
599 /*
600 * archs can override this function if they must do something
601 * after the modifying code is performed.
602 */
603 int __weak ftrace_arch_code_modify_post_process(void)
604 {
605 return 0;
606 }
607
608 static int __ftrace_modify_code(void *data)
609 {
610 int *command = data;
611
612 if (*command & FTRACE_ENABLE_CALLS)
613 ftrace_replace_code(1);
614 else if (*command & FTRACE_DISABLE_CALLS)
615 ftrace_replace_code(0);
616
617 if (*command & FTRACE_UPDATE_TRACE_FUNC)
618 ftrace_update_ftrace_func(ftrace_trace_function);
619
620 if (*command & FTRACE_START_FUNC_RET)
621 ftrace_enable_ftrace_graph_caller();
622 else if (*command & FTRACE_STOP_FUNC_RET)
623 ftrace_disable_ftrace_graph_caller();
624
625 return 0;
626 }
627
628 static void ftrace_run_update_code(int command)
629 {
630 int ret;
631
632 ret = ftrace_arch_code_modify_prepare();
633 FTRACE_WARN_ON(ret);
634 if (ret)
635 return;
636
637 stop_machine(__ftrace_modify_code, &command, NULL);
638
639 ret = ftrace_arch_code_modify_post_process();
640 FTRACE_WARN_ON(ret);
641 }
642
643 static ftrace_func_t saved_ftrace_func;
644 static int ftrace_start_up;
645
646 static void ftrace_startup_enable(int command)
647 {
648 if (saved_ftrace_func != ftrace_trace_function) {
649 saved_ftrace_func = ftrace_trace_function;
650 command |= FTRACE_UPDATE_TRACE_FUNC;
651 }
652
653 if (!command || !ftrace_enabled)
654 return;
655
656 ftrace_run_update_code(command);
657 }
658
659 static void ftrace_startup(int command)
660 {
661 if (unlikely(ftrace_disabled))
662 return;
663
664 ftrace_start_up++;
665 command |= FTRACE_ENABLE_CALLS;
666
667 ftrace_startup_enable(command);
668 }
669
670 static void ftrace_shutdown(int command)
671 {
672 if (unlikely(ftrace_disabled))
673 return;
674
675 ftrace_start_up--;
676 if (!ftrace_start_up)
677 command |= FTRACE_DISABLE_CALLS;
678
679 if (saved_ftrace_func != ftrace_trace_function) {
680 saved_ftrace_func = ftrace_trace_function;
681 command |= FTRACE_UPDATE_TRACE_FUNC;
682 }
683
684 if (!command || !ftrace_enabled)
685 return;
686
687 ftrace_run_update_code(command);
688 }
689
690 static void ftrace_startup_sysctl(void)
691 {
692 int command = FTRACE_ENABLE_MCOUNT;
693
694 if (unlikely(ftrace_disabled))
695 return;
696
697 /* Force update next time */
698 saved_ftrace_func = NULL;
699 /* ftrace_start_up is true if we want ftrace running */
700 if (ftrace_start_up)
701 command |= FTRACE_ENABLE_CALLS;
702
703 ftrace_run_update_code(command);
704 }
705
706 static void ftrace_shutdown_sysctl(void)
707 {
708 int command = FTRACE_DISABLE_MCOUNT;
709
710 if (unlikely(ftrace_disabled))
711 return;
712
713 /* ftrace_start_up is true if ftrace is running */
714 if (ftrace_start_up)
715 command |= FTRACE_DISABLE_CALLS;
716
717 ftrace_run_update_code(command);
718 }
719
720 static cycle_t ftrace_update_time;
721 static unsigned long ftrace_update_cnt;
722 unsigned long ftrace_update_tot_cnt;
723
724 static int ftrace_update_code(struct module *mod)
725 {
726 struct dyn_ftrace *p;
727 cycle_t start, stop;
728
729 start = ftrace_now(raw_smp_processor_id());
730 ftrace_update_cnt = 0;
731
732 while (ftrace_new_addrs) {
733
734 /* If something went wrong, bail without enabling anything */
735 if (unlikely(ftrace_disabled))
736 return -1;
737
738 p = ftrace_new_addrs;
739 ftrace_new_addrs = p->newlist;
740 p->flags = 0L;
741
742 /* convert record (i.e, patch mcount-call with NOP) */
743 if (ftrace_code_disable(mod, p)) {
744 p->flags |= FTRACE_FL_CONVERTED;
745 ftrace_update_cnt++;
746 } else
747 ftrace_free_rec(p);
748 }
749
750 stop = ftrace_now(raw_smp_processor_id());
751 ftrace_update_time = stop - start;
752 ftrace_update_tot_cnt += ftrace_update_cnt;
753
754 return 0;
755 }
756
757 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
758 {
759 struct ftrace_page *pg;
760 int cnt;
761 int i;
762
763 /* allocate a few pages */
764 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
765 if (!ftrace_pages_start)
766 return -1;
767
768 /*
769 * Allocate a few more pages.
770 *
771 * TODO: have some parser search vmlinux before
772 * final linking to find all calls to ftrace.
773 * Then we can:
774 * a) know how many pages to allocate.
775 * and/or
776 * b) set up the table then.
777 *
778 * The dynamic code is still necessary for
779 * modules.
780 */
781
782 pg = ftrace_pages = ftrace_pages_start;
783
784 cnt = num_to_init / ENTRIES_PER_PAGE;
785 pr_info("ftrace: allocating %ld entries in %d pages\n",
786 num_to_init, cnt + 1);
787
788 for (i = 0; i < cnt; i++) {
789 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
790
791 /* If we fail, we'll try later anyway */
792 if (!pg->next)
793 break;
794
795 pg = pg->next;
796 }
797
798 return 0;
799 }
800
801 enum {
802 FTRACE_ITER_FILTER = (1 << 0),
803 FTRACE_ITER_CONT = (1 << 1),
804 FTRACE_ITER_NOTRACE = (1 << 2),
805 FTRACE_ITER_FAILURES = (1 << 3),
806 FTRACE_ITER_PRINTALL = (1 << 4),
807 FTRACE_ITER_HASH = (1 << 5),
808 };
809
810 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
811
812 struct ftrace_iterator {
813 struct ftrace_page *pg;
814 int hidx;
815 int idx;
816 unsigned flags;
817 unsigned char buffer[FTRACE_BUFF_MAX+1];
818 unsigned buffer_idx;
819 unsigned filtered;
820 };
821
822 static void *
823 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
824 {
825 struct ftrace_iterator *iter = m->private;
826 struct hlist_node *hnd = v;
827 struct hlist_head *hhd;
828
829 WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
830
831 (*pos)++;
832
833 retry:
834 if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
835 return NULL;
836
837 hhd = &ftrace_func_hash[iter->hidx];
838
839 if (hlist_empty(hhd)) {
840 iter->hidx++;
841 hnd = NULL;
842 goto retry;
843 }
844
845 if (!hnd)
846 hnd = hhd->first;
847 else {
848 hnd = hnd->next;
849 if (!hnd) {
850 iter->hidx++;
851 goto retry;
852 }
853 }
854
855 return hnd;
856 }
857
858 static void *t_hash_start(struct seq_file *m, loff_t *pos)
859 {
860 struct ftrace_iterator *iter = m->private;
861 void *p = NULL;
862
863 iter->flags |= FTRACE_ITER_HASH;
864
865 return t_hash_next(m, p, pos);
866 }
867
868 static int t_hash_show(struct seq_file *m, void *v)
869 {
870 struct ftrace_func_probe *rec;
871 struct hlist_node *hnd = v;
872 char str[KSYM_SYMBOL_LEN];
873
874 rec = hlist_entry(hnd, struct ftrace_func_probe, node);
875
876 if (rec->ops->print)
877 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
878
879 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
880 seq_printf(m, "%s:", str);
881
882 kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
883 seq_printf(m, "%s", str);
884
885 if (rec->data)
886 seq_printf(m, ":%p", rec->data);
887 seq_putc(m, '\n');
888
889 return 0;
890 }
891
892 static void *
893 t_next(struct seq_file *m, void *v, loff_t *pos)
894 {
895 struct ftrace_iterator *iter = m->private;
896 struct dyn_ftrace *rec = NULL;
897
898 if (iter->flags & FTRACE_ITER_HASH)
899 return t_hash_next(m, v, pos);
900
901 (*pos)++;
902
903 if (iter->flags & FTRACE_ITER_PRINTALL)
904 return NULL;
905
906 retry:
907 if (iter->idx >= iter->pg->index) {
908 if (iter->pg->next) {
909 iter->pg = iter->pg->next;
910 iter->idx = 0;
911 goto retry;
912 } else {
913 iter->idx = -1;
914 }
915 } else {
916 rec = &iter->pg->records[iter->idx++];
917 if ((rec->flags & FTRACE_FL_FREE) ||
918
919 (!(iter->flags & FTRACE_ITER_FAILURES) &&
920 (rec->flags & FTRACE_FL_FAILED)) ||
921
922 ((iter->flags & FTRACE_ITER_FAILURES) &&
923 !(rec->flags & FTRACE_FL_FAILED)) ||
924
925 ((iter->flags & FTRACE_ITER_FILTER) &&
926 !(rec->flags & FTRACE_FL_FILTER)) ||
927
928 ((iter->flags & FTRACE_ITER_NOTRACE) &&
929 !(rec->flags & FTRACE_FL_NOTRACE))) {
930 rec = NULL;
931 goto retry;
932 }
933 }
934
935 return rec;
936 }
937
938 static void *t_start(struct seq_file *m, loff_t *pos)
939 {
940 struct ftrace_iterator *iter = m->private;
941 void *p = NULL;
942
943 mutex_lock(&ftrace_lock);
944 /*
945 * For set_ftrace_filter reading, if we have the filter
946 * off, we can short cut and just print out that all
947 * functions are enabled.
948 */
949 if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
950 if (*pos > 0)
951 return t_hash_start(m, pos);
952 iter->flags |= FTRACE_ITER_PRINTALL;
953 (*pos)++;
954 return iter;
955 }
956
957 if (iter->flags & FTRACE_ITER_HASH)
958 return t_hash_start(m, pos);
959
960 if (*pos > 0) {
961 if (iter->idx < 0)
962 return p;
963 (*pos)--;
964 iter->idx--;
965 }
966
967 p = t_next(m, p, pos);
968
969 if (!p)
970 return t_hash_start(m, pos);
971
972 return p;
973 }
974
975 static void t_stop(struct seq_file *m, void *p)
976 {
977 mutex_unlock(&ftrace_lock);
978 }
979
980 static int t_show(struct seq_file *m, void *v)
981 {
982 struct ftrace_iterator *iter = m->private;
983 struct dyn_ftrace *rec = v;
984 char str[KSYM_SYMBOL_LEN];
985
986 if (iter->flags & FTRACE_ITER_HASH)
987 return t_hash_show(m, v);
988
989 if (iter->flags & FTRACE_ITER_PRINTALL) {
990 seq_printf(m, "#### all functions enabled ####\n");
991 return 0;
992 }
993
994 if (!rec)
995 return 0;
996
997 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
998
999 seq_printf(m, "%s\n", str);
1000
1001 return 0;
1002 }
1003
1004 static struct seq_operations show_ftrace_seq_ops = {
1005 .start = t_start,
1006 .next = t_next,
1007 .stop = t_stop,
1008 .show = t_show,
1009 };
1010
1011 static int
1012 ftrace_avail_open(struct inode *inode, struct file *file)
1013 {
1014 struct ftrace_iterator *iter;
1015 int ret;
1016
1017 if (unlikely(ftrace_disabled))
1018 return -ENODEV;
1019
1020 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1021 if (!iter)
1022 return -ENOMEM;
1023
1024 iter->pg = ftrace_pages_start;
1025
1026 ret = seq_open(file, &show_ftrace_seq_ops);
1027 if (!ret) {
1028 struct seq_file *m = file->private_data;
1029
1030 m->private = iter;
1031 } else {
1032 kfree(iter);
1033 }
1034
1035 return ret;
1036 }
1037
1038 int ftrace_avail_release(struct inode *inode, struct file *file)
1039 {
1040 struct seq_file *m = (struct seq_file *)file->private_data;
1041 struct ftrace_iterator *iter = m->private;
1042
1043 seq_release(inode, file);
1044 kfree(iter);
1045
1046 return 0;
1047 }
1048
1049 static int
1050 ftrace_failures_open(struct inode *inode, struct file *file)
1051 {
1052 int ret;
1053 struct seq_file *m;
1054 struct ftrace_iterator *iter;
1055
1056 ret = ftrace_avail_open(inode, file);
1057 if (!ret) {
1058 m = (struct seq_file *)file->private_data;
1059 iter = (struct ftrace_iterator *)m->private;
1060 iter->flags = FTRACE_ITER_FAILURES;
1061 }
1062
1063 return ret;
1064 }
1065
1066
1067 static void ftrace_filter_reset(int enable)
1068 {
1069 struct ftrace_page *pg;
1070 struct dyn_ftrace *rec;
1071 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1072
1073 mutex_lock(&ftrace_lock);
1074 if (enable)
1075 ftrace_filtered = 0;
1076 do_for_each_ftrace_rec(pg, rec) {
1077 if (rec->flags & FTRACE_FL_FAILED)
1078 continue;
1079 rec->flags &= ~type;
1080 } while_for_each_ftrace_rec();
1081 mutex_unlock(&ftrace_lock);
1082 }
1083
1084 static int
1085 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1086 {
1087 struct ftrace_iterator *iter;
1088 int ret = 0;
1089
1090 if (unlikely(ftrace_disabled))
1091 return -ENODEV;
1092
1093 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1094 if (!iter)
1095 return -ENOMEM;
1096
1097 mutex_lock(&ftrace_regex_lock);
1098 if ((file->f_mode & FMODE_WRITE) &&
1099 !(file->f_flags & O_APPEND))
1100 ftrace_filter_reset(enable);
1101
1102 if (file->f_mode & FMODE_READ) {
1103 iter->pg = ftrace_pages_start;
1104 iter->flags = enable ? FTRACE_ITER_FILTER :
1105 FTRACE_ITER_NOTRACE;
1106
1107 ret = seq_open(file, &show_ftrace_seq_ops);
1108 if (!ret) {
1109 struct seq_file *m = file->private_data;
1110 m->private = iter;
1111 } else
1112 kfree(iter);
1113 } else
1114 file->private_data = iter;
1115 mutex_unlock(&ftrace_regex_lock);
1116
1117 return ret;
1118 }
1119
1120 static int
1121 ftrace_filter_open(struct inode *inode, struct file *file)
1122 {
1123 return ftrace_regex_open(inode, file, 1);
1124 }
1125
1126 static int
1127 ftrace_notrace_open(struct inode *inode, struct file *file)
1128 {
1129 return ftrace_regex_open(inode, file, 0);
1130 }
1131
1132 static loff_t
1133 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1134 {
1135 loff_t ret;
1136
1137 if (file->f_mode & FMODE_READ)
1138 ret = seq_lseek(file, offset, origin);
1139 else
1140 file->f_pos = ret = 1;
1141
1142 return ret;
1143 }
1144
1145 enum {
1146 MATCH_FULL,
1147 MATCH_FRONT_ONLY,
1148 MATCH_MIDDLE_ONLY,
1149 MATCH_END_ONLY,
1150 };
1151
1152 /*
1153 * (static function - no need for kernel doc)
1154 *
1155 * Pass in a buffer containing a glob and this function will
1156 * set search to point to the search part of the buffer and
1157 * return the type of search it is (see enum above).
1158 * This does modify buff.
1159 *
1160 * Returns enum type.
1161 * search returns the pointer to use for comparison.
1162 * not returns 1 if buff started with a '!'
1163 * 0 otherwise.
1164 */
1165 static int
1166 ftrace_setup_glob(char *buff, int len, char **search, int *not)
1167 {
1168 int type = MATCH_FULL;
1169 int i;
1170
1171 if (buff[0] == '!') {
1172 *not = 1;
1173 buff++;
1174 len--;
1175 } else
1176 *not = 0;
1177
1178 *search = buff;
1179
1180 for (i = 0; i < len; i++) {
1181 if (buff[i] == '*') {
1182 if (!i) {
1183 *search = buff + 1;
1184 type = MATCH_END_ONLY;
1185 } else {
1186 if (type == MATCH_END_ONLY)
1187 type = MATCH_MIDDLE_ONLY;
1188 else
1189 type = MATCH_FRONT_ONLY;
1190 buff[i] = 0;
1191 break;
1192 }
1193 }
1194 }
1195
1196 return type;
1197 }
1198
1199 static int ftrace_match(char *str, char *regex, int len, int type)
1200 {
1201 int matched = 0;
1202 char *ptr;
1203
1204 switch (type) {
1205 case MATCH_FULL:
1206 if (strcmp(str, regex) == 0)
1207 matched = 1;
1208 break;
1209 case MATCH_FRONT_ONLY:
1210 if (strncmp(str, regex, len) == 0)
1211 matched = 1;
1212 break;
1213 case MATCH_MIDDLE_ONLY:
1214 if (strstr(str, regex))
1215 matched = 1;
1216 break;
1217 case MATCH_END_ONLY:
1218 ptr = strstr(str, regex);
1219 if (ptr && (ptr[len] == 0))
1220 matched = 1;
1221 break;
1222 }
1223
1224 return matched;
1225 }
1226
1227 static int
1228 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1229 {
1230 char str[KSYM_SYMBOL_LEN];
1231
1232 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1233 return ftrace_match(str, regex, len, type);
1234 }
1235
1236 static void ftrace_match_records(char *buff, int len, int enable)
1237 {
1238 unsigned int search_len;
1239 struct ftrace_page *pg;
1240 struct dyn_ftrace *rec;
1241 unsigned long flag;
1242 char *search;
1243 int type;
1244 int not;
1245
1246 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1247 type = ftrace_setup_glob(buff, len, &search, &not);
1248
1249 search_len = strlen(search);
1250
1251 mutex_lock(&ftrace_lock);
1252 do_for_each_ftrace_rec(pg, rec) {
1253
1254 if (rec->flags & FTRACE_FL_FAILED)
1255 continue;
1256
1257 if (ftrace_match_record(rec, search, search_len, type)) {
1258 if (not)
1259 rec->flags &= ~flag;
1260 else
1261 rec->flags |= flag;
1262 }
1263 /*
1264 * Only enable filtering if we have a function that
1265 * is filtered on.
1266 */
1267 if (enable && (rec->flags & FTRACE_FL_FILTER))
1268 ftrace_filtered = 1;
1269 } while_for_each_ftrace_rec();
1270 mutex_unlock(&ftrace_lock);
1271 }
1272
1273 static int
1274 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1275 char *regex, int len, int type)
1276 {
1277 char str[KSYM_SYMBOL_LEN];
1278 char *modname;
1279
1280 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1281
1282 if (!modname || strcmp(modname, mod))
1283 return 0;
1284
1285 /* blank search means to match all funcs in the mod */
1286 if (len)
1287 return ftrace_match(str, regex, len, type);
1288 else
1289 return 1;
1290 }
1291
1292 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1293 {
1294 unsigned search_len = 0;
1295 struct ftrace_page *pg;
1296 struct dyn_ftrace *rec;
1297 int type = MATCH_FULL;
1298 char *search = buff;
1299 unsigned long flag;
1300 int not = 0;
1301
1302 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1303
1304 /* blank or '*' mean the same */
1305 if (strcmp(buff, "*") == 0)
1306 buff[0] = 0;
1307
1308 /* handle the case of 'dont filter this module' */
1309 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1310 buff[0] = 0;
1311 not = 1;
1312 }
1313
1314 if (strlen(buff)) {
1315 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1316 search_len = strlen(search);
1317 }
1318
1319 mutex_lock(&ftrace_lock);
1320 do_for_each_ftrace_rec(pg, rec) {
1321
1322 if (rec->flags & FTRACE_FL_FAILED)
1323 continue;
1324
1325 if (ftrace_match_module_record(rec, mod,
1326 search, search_len, type)) {
1327 if (not)
1328 rec->flags &= ~flag;
1329 else
1330 rec->flags |= flag;
1331 }
1332 if (enable && (rec->flags & FTRACE_FL_FILTER))
1333 ftrace_filtered = 1;
1334
1335 } while_for_each_ftrace_rec();
1336 mutex_unlock(&ftrace_lock);
1337 }
1338
1339 /*
1340 * We register the module command as a template to show others how
1341 * to register the a command as well.
1342 */
1343
1344 static int
1345 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1346 {
1347 char *mod;
1348
1349 /*
1350 * cmd == 'mod' because we only registered this func
1351 * for the 'mod' ftrace_func_command.
1352 * But if you register one func with multiple commands,
1353 * you can tell which command was used by the cmd
1354 * parameter.
1355 */
1356
1357 /* we must have a module name */
1358 if (!param)
1359 return -EINVAL;
1360
1361 mod = strsep(&param, ":");
1362 if (!strlen(mod))
1363 return -EINVAL;
1364
1365 ftrace_match_module_records(func, mod, enable);
1366 return 0;
1367 }
1368
1369 static struct ftrace_func_command ftrace_mod_cmd = {
1370 .name = "mod",
1371 .func = ftrace_mod_callback,
1372 };
1373
1374 static int __init ftrace_mod_cmd_init(void)
1375 {
1376 return register_ftrace_command(&ftrace_mod_cmd);
1377 }
1378 device_initcall(ftrace_mod_cmd_init);
1379
1380 static void
1381 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1382 {
1383 struct ftrace_func_probe *entry;
1384 struct hlist_head *hhd;
1385 struct hlist_node *n;
1386 unsigned long key;
1387 int resched;
1388
1389 key = hash_long(ip, FTRACE_HASH_BITS);
1390
1391 hhd = &ftrace_func_hash[key];
1392
1393 if (hlist_empty(hhd))
1394 return;
1395
1396 /*
1397 * Disable preemption for these calls to prevent a RCU grace
1398 * period. This syncs the hash iteration and freeing of items
1399 * on the hash. rcu_read_lock is too dangerous here.
1400 */
1401 resched = ftrace_preempt_disable();
1402 hlist_for_each_entry_rcu(entry, n, hhd, node) {
1403 if (entry->ip == ip)
1404 entry->ops->func(ip, parent_ip, &entry->data);
1405 }
1406 ftrace_preempt_enable(resched);
1407 }
1408
1409 static struct ftrace_ops trace_probe_ops __read_mostly =
1410 {
1411 .func = function_trace_probe_call,
1412 };
1413
1414 static int ftrace_probe_registered;
1415
1416 static void __enable_ftrace_function_probe(void)
1417 {
1418 int i;
1419
1420 if (ftrace_probe_registered)
1421 return;
1422
1423 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1424 struct hlist_head *hhd = &ftrace_func_hash[i];
1425 if (hhd->first)
1426 break;
1427 }
1428 /* Nothing registered? */
1429 if (i == FTRACE_FUNC_HASHSIZE)
1430 return;
1431
1432 __register_ftrace_function(&trace_probe_ops);
1433 ftrace_startup(0);
1434 ftrace_probe_registered = 1;
1435 }
1436
1437 static void __disable_ftrace_function_probe(void)
1438 {
1439 int i;
1440
1441 if (!ftrace_probe_registered)
1442 return;
1443
1444 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1445 struct hlist_head *hhd = &ftrace_func_hash[i];
1446 if (hhd->first)
1447 return;
1448 }
1449
1450 /* no more funcs left */
1451 __unregister_ftrace_function(&trace_probe_ops);
1452 ftrace_shutdown(0);
1453 ftrace_probe_registered = 0;
1454 }
1455
1456
1457 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1458 {
1459 struct ftrace_func_probe *entry =
1460 container_of(rhp, struct ftrace_func_probe, rcu);
1461
1462 if (entry->ops->free)
1463 entry->ops->free(&entry->data);
1464 kfree(entry);
1465 }
1466
1467
1468 int
1469 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1470 void *data)
1471 {
1472 struct ftrace_func_probe *entry;
1473 struct ftrace_page *pg;
1474 struct dyn_ftrace *rec;
1475 int type, len, not;
1476 unsigned long key;
1477 int count = 0;
1478 char *search;
1479
1480 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1481 len = strlen(search);
1482
1483 /* we do not support '!' for function probes */
1484 if (WARN_ON(not))
1485 return -EINVAL;
1486
1487 mutex_lock(&ftrace_lock);
1488 do_for_each_ftrace_rec(pg, rec) {
1489
1490 if (rec->flags & FTRACE_FL_FAILED)
1491 continue;
1492
1493 if (!ftrace_match_record(rec, search, len, type))
1494 continue;
1495
1496 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1497 if (!entry) {
1498 /* If we did not process any, then return error */
1499 if (!count)
1500 count = -ENOMEM;
1501 goto out_unlock;
1502 }
1503
1504 count++;
1505
1506 entry->data = data;
1507
1508 /*
1509 * The caller might want to do something special
1510 * for each function we find. We call the callback
1511 * to give the caller an opportunity to do so.
1512 */
1513 if (ops->callback) {
1514 if (ops->callback(rec->ip, &entry->data) < 0) {
1515 /* caller does not like this func */
1516 kfree(entry);
1517 continue;
1518 }
1519 }
1520
1521 entry->ops = ops;
1522 entry->ip = rec->ip;
1523
1524 key = hash_long(entry->ip, FTRACE_HASH_BITS);
1525 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
1526
1527 } while_for_each_ftrace_rec();
1528 __enable_ftrace_function_probe();
1529
1530 out_unlock:
1531 mutex_unlock(&ftrace_lock);
1532
1533 return count;
1534 }
1535
1536 enum {
1537 PROBE_TEST_FUNC = 1,
1538 PROBE_TEST_DATA = 2
1539 };
1540
1541 static void
1542 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1543 void *data, int flags)
1544 {
1545 struct ftrace_func_probe *entry;
1546 struct hlist_node *n, *tmp;
1547 char str[KSYM_SYMBOL_LEN];
1548 int type = MATCH_FULL;
1549 int i, len = 0;
1550 char *search;
1551
1552 if (glob && (strcmp(glob, "*") || !strlen(glob)))
1553 glob = NULL;
1554 else {
1555 int not;
1556
1557 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1558 len = strlen(search);
1559
1560 /* we do not support '!' for function probes */
1561 if (WARN_ON(not))
1562 return;
1563 }
1564
1565 mutex_lock(&ftrace_lock);
1566 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1567 struct hlist_head *hhd = &ftrace_func_hash[i];
1568
1569 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
1570
1571 /* break up if statements for readability */
1572 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
1573 continue;
1574
1575 if ((flags & PROBE_TEST_DATA) && entry->data != data)
1576 continue;
1577
1578 /* do this last, since it is the most expensive */
1579 if (glob) {
1580 kallsyms_lookup(entry->ip, NULL, NULL,
1581 NULL, str);
1582 if (!ftrace_match(str, glob, len, type))
1583 continue;
1584 }
1585
1586 hlist_del(&entry->node);
1587 call_rcu(&entry->rcu, ftrace_free_entry_rcu);
1588 }
1589 }
1590 __disable_ftrace_function_probe();
1591 mutex_unlock(&ftrace_lock);
1592 }
1593
1594 void
1595 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1596 void *data)
1597 {
1598 __unregister_ftrace_function_probe(glob, ops, data,
1599 PROBE_TEST_FUNC | PROBE_TEST_DATA);
1600 }
1601
1602 void
1603 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
1604 {
1605 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
1606 }
1607
1608 void unregister_ftrace_function_probe_all(char *glob)
1609 {
1610 __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
1611 }
1612
1613 static LIST_HEAD(ftrace_commands);
1614 static DEFINE_MUTEX(ftrace_cmd_mutex);
1615
1616 int register_ftrace_command(struct ftrace_func_command *cmd)
1617 {
1618 struct ftrace_func_command *p;
1619 int ret = 0;
1620
1621 mutex_lock(&ftrace_cmd_mutex);
1622 list_for_each_entry(p, &ftrace_commands, list) {
1623 if (strcmp(cmd->name, p->name) == 0) {
1624 ret = -EBUSY;
1625 goto out_unlock;
1626 }
1627 }
1628 list_add(&cmd->list, &ftrace_commands);
1629 out_unlock:
1630 mutex_unlock(&ftrace_cmd_mutex);
1631
1632 return ret;
1633 }
1634
1635 int unregister_ftrace_command(struct ftrace_func_command *cmd)
1636 {
1637 struct ftrace_func_command *p, *n;
1638 int ret = -ENODEV;
1639
1640 mutex_lock(&ftrace_cmd_mutex);
1641 list_for_each_entry_safe(p, n, &ftrace_commands, list) {
1642 if (strcmp(cmd->name, p->name) == 0) {
1643 ret = 0;
1644 list_del_init(&p->list);
1645 goto out_unlock;
1646 }
1647 }
1648 out_unlock:
1649 mutex_unlock(&ftrace_cmd_mutex);
1650
1651 return ret;
1652 }
1653
1654 static int ftrace_process_regex(char *buff, int len, int enable)
1655 {
1656 char *func, *command, *next = buff;
1657 struct ftrace_func_command *p;
1658 int ret = -EINVAL;
1659
1660 func = strsep(&next, ":");
1661
1662 if (!next) {
1663 ftrace_match_records(func, len, enable);
1664 return 0;
1665 }
1666
1667 /* command found */
1668
1669 command = strsep(&next, ":");
1670
1671 mutex_lock(&ftrace_cmd_mutex);
1672 list_for_each_entry(p, &ftrace_commands, list) {
1673 if (strcmp(p->name, command) == 0) {
1674 ret = p->func(func, command, next, enable);
1675 goto out_unlock;
1676 }
1677 }
1678 out_unlock:
1679 mutex_unlock(&ftrace_cmd_mutex);
1680
1681 return ret;
1682 }
1683
1684 static ssize_t
1685 ftrace_regex_write(struct file *file, const char __user *ubuf,
1686 size_t cnt, loff_t *ppos, int enable)
1687 {
1688 struct ftrace_iterator *iter;
1689 char ch;
1690 size_t read = 0;
1691 ssize_t ret;
1692
1693 if (!cnt || cnt < 0)
1694 return 0;
1695
1696 mutex_lock(&ftrace_regex_lock);
1697
1698 if (file->f_mode & FMODE_READ) {
1699 struct seq_file *m = file->private_data;
1700 iter = m->private;
1701 } else
1702 iter = file->private_data;
1703
1704 if (!*ppos) {
1705 iter->flags &= ~FTRACE_ITER_CONT;
1706 iter->buffer_idx = 0;
1707 }
1708
1709 ret = get_user(ch, ubuf++);
1710 if (ret)
1711 goto out;
1712 read++;
1713 cnt--;
1714
1715 if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1716 /* skip white space */
1717 while (cnt && isspace(ch)) {
1718 ret = get_user(ch, ubuf++);
1719 if (ret)
1720 goto out;
1721 read++;
1722 cnt--;
1723 }
1724
1725 if (isspace(ch)) {
1726 file->f_pos += read;
1727 ret = read;
1728 goto out;
1729 }
1730
1731 iter->buffer_idx = 0;
1732 }
1733
1734 while (cnt && !isspace(ch)) {
1735 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1736 iter->buffer[iter->buffer_idx++] = ch;
1737 else {
1738 ret = -EINVAL;
1739 goto out;
1740 }
1741 ret = get_user(ch, ubuf++);
1742 if (ret)
1743 goto out;
1744 read++;
1745 cnt--;
1746 }
1747
1748 if (isspace(ch)) {
1749 iter->filtered++;
1750 iter->buffer[iter->buffer_idx] = 0;
1751 ret = ftrace_process_regex(iter->buffer,
1752 iter->buffer_idx, enable);
1753 if (ret)
1754 goto out;
1755 iter->buffer_idx = 0;
1756 } else
1757 iter->flags |= FTRACE_ITER_CONT;
1758
1759
1760 file->f_pos += read;
1761
1762 ret = read;
1763 out:
1764 mutex_unlock(&ftrace_regex_lock);
1765
1766 return ret;
1767 }
1768
1769 static ssize_t
1770 ftrace_filter_write(struct file *file, const char __user *ubuf,
1771 size_t cnt, loff_t *ppos)
1772 {
1773 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1774 }
1775
1776 static ssize_t
1777 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1778 size_t cnt, loff_t *ppos)
1779 {
1780 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1781 }
1782
1783 static void
1784 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1785 {
1786 if (unlikely(ftrace_disabled))
1787 return;
1788
1789 mutex_lock(&ftrace_regex_lock);
1790 if (reset)
1791 ftrace_filter_reset(enable);
1792 if (buf)
1793 ftrace_match_records(buf, len, enable);
1794 mutex_unlock(&ftrace_regex_lock);
1795 }
1796
1797 /**
1798 * ftrace_set_filter - set a function to filter on in ftrace
1799 * @buf - the string that holds the function filter text.
1800 * @len - the length of the string.
1801 * @reset - non zero to reset all filters before applying this filter.
1802 *
1803 * Filters denote which functions should be enabled when tracing is enabled.
1804 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1805 */
1806 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1807 {
1808 ftrace_set_regex(buf, len, reset, 1);
1809 }
1810
1811 /**
1812 * ftrace_set_notrace - set a function to not trace in ftrace
1813 * @buf - the string that holds the function notrace text.
1814 * @len - the length of the string.
1815 * @reset - non zero to reset all filters before applying this filter.
1816 *
1817 * Notrace Filters denote which functions should not be enabled when tracing
1818 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1819 * for tracing.
1820 */
1821 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1822 {
1823 ftrace_set_regex(buf, len, reset, 0);
1824 }
1825
1826 static int
1827 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1828 {
1829 struct seq_file *m = (struct seq_file *)file->private_data;
1830 struct ftrace_iterator *iter;
1831
1832 mutex_lock(&ftrace_regex_lock);
1833 if (file->f_mode & FMODE_READ) {
1834 iter = m->private;
1835
1836 seq_release(inode, file);
1837 } else
1838 iter = file->private_data;
1839
1840 if (iter->buffer_idx) {
1841 iter->filtered++;
1842 iter->buffer[iter->buffer_idx] = 0;
1843 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
1844 }
1845
1846 mutex_lock(&ftrace_lock);
1847 if (ftrace_start_up && ftrace_enabled)
1848 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1849 mutex_unlock(&ftrace_lock);
1850
1851 kfree(iter);
1852 mutex_unlock(&ftrace_regex_lock);
1853 return 0;
1854 }
1855
1856 static int
1857 ftrace_filter_release(struct inode *inode, struct file *file)
1858 {
1859 return ftrace_regex_release(inode, file, 1);
1860 }
1861
1862 static int
1863 ftrace_notrace_release(struct inode *inode, struct file *file)
1864 {
1865 return ftrace_regex_release(inode, file, 0);
1866 }
1867
1868 static const struct file_operations ftrace_avail_fops = {
1869 .open = ftrace_avail_open,
1870 .read = seq_read,
1871 .llseek = seq_lseek,
1872 .release = ftrace_avail_release,
1873 };
1874
1875 static const struct file_operations ftrace_failures_fops = {
1876 .open = ftrace_failures_open,
1877 .read = seq_read,
1878 .llseek = seq_lseek,
1879 .release = ftrace_avail_release,
1880 };
1881
1882 static const struct file_operations ftrace_filter_fops = {
1883 .open = ftrace_filter_open,
1884 .read = seq_read,
1885 .write = ftrace_filter_write,
1886 .llseek = ftrace_regex_lseek,
1887 .release = ftrace_filter_release,
1888 };
1889
1890 static const struct file_operations ftrace_notrace_fops = {
1891 .open = ftrace_notrace_open,
1892 .read = seq_read,
1893 .write = ftrace_notrace_write,
1894 .llseek = ftrace_regex_lseek,
1895 .release = ftrace_notrace_release,
1896 };
1897
1898 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1899
1900 static DEFINE_MUTEX(graph_lock);
1901
1902 int ftrace_graph_count;
1903 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1904
1905 static void *
1906 g_next(struct seq_file *m, void *v, loff_t *pos)
1907 {
1908 unsigned long *array = m->private;
1909 int index = *pos;
1910
1911 (*pos)++;
1912
1913 if (index >= ftrace_graph_count)
1914 return NULL;
1915
1916 return &array[index];
1917 }
1918
1919 static void *g_start(struct seq_file *m, loff_t *pos)
1920 {
1921 void *p = NULL;
1922
1923 mutex_lock(&graph_lock);
1924
1925 /* Nothing, tell g_show to print all functions are enabled */
1926 if (!ftrace_graph_count && !*pos)
1927 return (void *)1;
1928
1929 p = g_next(m, p, pos);
1930
1931 return p;
1932 }
1933
1934 static void g_stop(struct seq_file *m, void *p)
1935 {
1936 mutex_unlock(&graph_lock);
1937 }
1938
1939 static int g_show(struct seq_file *m, void *v)
1940 {
1941 unsigned long *ptr = v;
1942 char str[KSYM_SYMBOL_LEN];
1943
1944 if (!ptr)
1945 return 0;
1946
1947 if (ptr == (unsigned long *)1) {
1948 seq_printf(m, "#### all functions enabled ####\n");
1949 return 0;
1950 }
1951
1952 kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1953
1954 seq_printf(m, "%s\n", str);
1955
1956 return 0;
1957 }
1958
1959 static struct seq_operations ftrace_graph_seq_ops = {
1960 .start = g_start,
1961 .next = g_next,
1962 .stop = g_stop,
1963 .show = g_show,
1964 };
1965
1966 static int
1967 ftrace_graph_open(struct inode *inode, struct file *file)
1968 {
1969 int ret = 0;
1970
1971 if (unlikely(ftrace_disabled))
1972 return -ENODEV;
1973
1974 mutex_lock(&graph_lock);
1975 if ((file->f_mode & FMODE_WRITE) &&
1976 !(file->f_flags & O_APPEND)) {
1977 ftrace_graph_count = 0;
1978 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1979 }
1980
1981 if (file->f_mode & FMODE_READ) {
1982 ret = seq_open(file, &ftrace_graph_seq_ops);
1983 if (!ret) {
1984 struct seq_file *m = file->private_data;
1985 m->private = ftrace_graph_funcs;
1986 }
1987 } else
1988 file->private_data = ftrace_graph_funcs;
1989 mutex_unlock(&graph_lock);
1990
1991 return ret;
1992 }
1993
1994 static int
1995 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
1996 {
1997 struct dyn_ftrace *rec;
1998 struct ftrace_page *pg;
1999 int search_len;
2000 int found = 0;
2001 int type, not;
2002 char *search;
2003 bool exists;
2004 int i;
2005
2006 if (ftrace_disabled)
2007 return -ENODEV;
2008
2009 /* decode regex */
2010 type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2011 if (not)
2012 return -EINVAL;
2013
2014 search_len = strlen(search);
2015
2016 mutex_lock(&ftrace_lock);
2017 do_for_each_ftrace_rec(pg, rec) {
2018
2019 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2020 break;
2021
2022 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2023 continue;
2024
2025 if (ftrace_match_record(rec, search, search_len, type)) {
2026 /* ensure it is not already in the array */
2027 exists = false;
2028 for (i = 0; i < *idx; i++)
2029 if (array[i] == rec->ip) {
2030 exists = true;
2031 break;
2032 }
2033 if (!exists) {
2034 array[(*idx)++] = rec->ip;
2035 found = 1;
2036 }
2037 }
2038 } while_for_each_ftrace_rec();
2039
2040 mutex_unlock(&ftrace_lock);
2041
2042 return found ? 0 : -EINVAL;
2043 }
2044
2045 static ssize_t
2046 ftrace_graph_write(struct file *file, const char __user *ubuf,
2047 size_t cnt, loff_t *ppos)
2048 {
2049 unsigned char buffer[FTRACE_BUFF_MAX+1];
2050 unsigned long *array;
2051 size_t read = 0;
2052 ssize_t ret;
2053 int index = 0;
2054 char ch;
2055
2056 if (!cnt || cnt < 0)
2057 return 0;
2058
2059 mutex_lock(&graph_lock);
2060
2061 if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2062 ret = -EBUSY;
2063 goto out;
2064 }
2065
2066 if (file->f_mode & FMODE_READ) {
2067 struct seq_file *m = file->private_data;
2068 array = m->private;
2069 } else
2070 array = file->private_data;
2071
2072 ret = get_user(ch, ubuf++);
2073 if (ret)
2074 goto out;
2075 read++;
2076 cnt--;
2077
2078 /* skip white space */
2079 while (cnt && isspace(ch)) {
2080 ret = get_user(ch, ubuf++);
2081 if (ret)
2082 goto out;
2083 read++;
2084 cnt--;
2085 }
2086
2087 if (isspace(ch)) {
2088 *ppos += read;
2089 ret = read;
2090 goto out;
2091 }
2092
2093 while (cnt && !isspace(ch)) {
2094 if (index < FTRACE_BUFF_MAX)
2095 buffer[index++] = ch;
2096 else {
2097 ret = -EINVAL;
2098 goto out;
2099 }
2100 ret = get_user(ch, ubuf++);
2101 if (ret)
2102 goto out;
2103 read++;
2104 cnt--;
2105 }
2106 buffer[index] = 0;
2107
2108 /* we allow only one expression at a time */
2109 ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
2110 if (ret)
2111 goto out;
2112
2113 file->f_pos += read;
2114
2115 ret = read;
2116 out:
2117 mutex_unlock(&graph_lock);
2118
2119 return ret;
2120 }
2121
2122 static const struct file_operations ftrace_graph_fops = {
2123 .open = ftrace_graph_open,
2124 .read = seq_read,
2125 .write = ftrace_graph_write,
2126 };
2127 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2128
2129 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2130 {
2131 struct dentry *entry;
2132
2133 entry = debugfs_create_file("available_filter_functions", 0444,
2134 d_tracer, NULL, &ftrace_avail_fops);
2135 if (!entry)
2136 pr_warning("Could not create debugfs "
2137 "'available_filter_functions' entry\n");
2138
2139 entry = debugfs_create_file("failures", 0444,
2140 d_tracer, NULL, &ftrace_failures_fops);
2141 if (!entry)
2142 pr_warning("Could not create debugfs 'failures' entry\n");
2143
2144 entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
2145 NULL, &ftrace_filter_fops);
2146 if (!entry)
2147 pr_warning("Could not create debugfs "
2148 "'set_ftrace_filter' entry\n");
2149
2150 entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
2151 NULL, &ftrace_notrace_fops);
2152 if (!entry)
2153 pr_warning("Could not create debugfs "
2154 "'set_ftrace_notrace' entry\n");
2155
2156 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2157 entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
2158 NULL,
2159 &ftrace_graph_fops);
2160 if (!entry)
2161 pr_warning("Could not create debugfs "
2162 "'set_graph_function' entry\n");
2163 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2164
2165 return 0;
2166 }
2167
2168 static int ftrace_convert_nops(struct module *mod,
2169 unsigned long *start,
2170 unsigned long *end)
2171 {
2172 unsigned long *p;
2173 unsigned long addr;
2174 unsigned long flags;
2175
2176 mutex_lock(&ftrace_lock);
2177 p = start;
2178 while (p < end) {
2179 addr = ftrace_call_adjust(*p++);
2180 /*
2181 * Some architecture linkers will pad between
2182 * the different mcount_loc sections of different
2183 * object files to satisfy alignments.
2184 * Skip any NULL pointers.
2185 */
2186 if (!addr)
2187 continue;
2188 ftrace_record_ip(addr);
2189 }
2190
2191 /* disable interrupts to prevent kstop machine */
2192 local_irq_save(flags);
2193 ftrace_update_code(mod);
2194 local_irq_restore(flags);
2195 mutex_unlock(&ftrace_lock);
2196
2197 return 0;
2198 }
2199
2200 void ftrace_init_module(struct module *mod,
2201 unsigned long *start, unsigned long *end)
2202 {
2203 if (ftrace_disabled || start == end)
2204 return;
2205 ftrace_convert_nops(mod, start, end);
2206 }
2207
2208 extern unsigned long __start_mcount_loc[];
2209 extern unsigned long __stop_mcount_loc[];
2210
2211 void __init ftrace_init(void)
2212 {
2213 unsigned long count, addr, flags;
2214 int ret;
2215
2216 /* Keep the ftrace pointer to the stub */
2217 addr = (unsigned long)ftrace_stub;
2218
2219 local_irq_save(flags);
2220 ftrace_dyn_arch_init(&addr);
2221 local_irq_restore(flags);
2222
2223 /* ftrace_dyn_arch_init places the return code in addr */
2224 if (addr)
2225 goto failed;
2226
2227 count = __stop_mcount_loc - __start_mcount_loc;
2228
2229 ret = ftrace_dyn_table_alloc(count);
2230 if (ret)
2231 goto failed;
2232
2233 last_ftrace_enabled = ftrace_enabled = 1;
2234
2235 ret = ftrace_convert_nops(NULL,
2236 __start_mcount_loc,
2237 __stop_mcount_loc);
2238
2239 return;
2240 failed:
2241 ftrace_disabled = 1;
2242 }
2243
2244 #else
2245
2246 static int __init ftrace_nodyn_init(void)
2247 {
2248 ftrace_enabled = 1;
2249 return 0;
2250 }
2251 device_initcall(ftrace_nodyn_init);
2252
2253 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2254 static inline void ftrace_startup_enable(int command) { }
2255 /* Keep as macros so we do not need to define the commands */
2256 # define ftrace_startup(command) do { } while (0)
2257 # define ftrace_shutdown(command) do { } while (0)
2258 # define ftrace_startup_sysctl() do { } while (0)
2259 # define ftrace_shutdown_sysctl() do { } while (0)
2260 #endif /* CONFIG_DYNAMIC_FTRACE */
2261
2262 static ssize_t
2263 ftrace_pid_read(struct file *file, char __user *ubuf,
2264 size_t cnt, loff_t *ppos)
2265 {
2266 char buf[64];
2267 int r;
2268
2269 if (ftrace_pid_trace == ftrace_swapper_pid)
2270 r = sprintf(buf, "swapper tasks\n");
2271 else if (ftrace_pid_trace)
2272 r = sprintf(buf, "%u\n", pid_vnr(ftrace_pid_trace));
2273 else
2274 r = sprintf(buf, "no pid\n");
2275
2276 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2277 }
2278
2279 static void clear_ftrace_swapper(void)
2280 {
2281 struct task_struct *p;
2282 int cpu;
2283
2284 get_online_cpus();
2285 for_each_online_cpu(cpu) {
2286 p = idle_task(cpu);
2287 clear_tsk_trace_trace(p);
2288 }
2289 put_online_cpus();
2290 }
2291
2292 static void set_ftrace_swapper(void)
2293 {
2294 struct task_struct *p;
2295 int cpu;
2296
2297 get_online_cpus();
2298 for_each_online_cpu(cpu) {
2299 p = idle_task(cpu);
2300 set_tsk_trace_trace(p);
2301 }
2302 put_online_cpus();
2303 }
2304
2305 static void clear_ftrace_pid(struct pid *pid)
2306 {
2307 struct task_struct *p;
2308
2309 rcu_read_lock();
2310 do_each_pid_task(pid, PIDTYPE_PID, p) {
2311 clear_tsk_trace_trace(p);
2312 } while_each_pid_task(pid, PIDTYPE_PID, p);
2313 rcu_read_unlock();
2314
2315 put_pid(pid);
2316 }
2317
2318 static void set_ftrace_pid(struct pid *pid)
2319 {
2320 struct task_struct *p;
2321
2322 rcu_read_lock();
2323 do_each_pid_task(pid, PIDTYPE_PID, p) {
2324 set_tsk_trace_trace(p);
2325 } while_each_pid_task(pid, PIDTYPE_PID, p);
2326 rcu_read_unlock();
2327 }
2328
2329 static void clear_ftrace_pid_task(struct pid **pid)
2330 {
2331 if (*pid == ftrace_swapper_pid)
2332 clear_ftrace_swapper();
2333 else
2334 clear_ftrace_pid(*pid);
2335
2336 *pid = NULL;
2337 }
2338
2339 static void set_ftrace_pid_task(struct pid *pid)
2340 {
2341 if (pid == ftrace_swapper_pid)
2342 set_ftrace_swapper();
2343 else
2344 set_ftrace_pid(pid);
2345 }
2346
2347 static ssize_t
2348 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2349 size_t cnt, loff_t *ppos)
2350 {
2351 struct pid *pid;
2352 char buf[64];
2353 long val;
2354 int ret;
2355
2356 if (cnt >= sizeof(buf))
2357 return -EINVAL;
2358
2359 if (copy_from_user(&buf, ubuf, cnt))
2360 return -EFAULT;
2361
2362 buf[cnt] = 0;
2363
2364 ret = strict_strtol(buf, 10, &val);
2365 if (ret < 0)
2366 return ret;
2367
2368 mutex_lock(&ftrace_lock);
2369 if (val < 0) {
2370 /* disable pid tracing */
2371 if (!ftrace_pid_trace)
2372 goto out;
2373
2374 clear_ftrace_pid_task(&ftrace_pid_trace);
2375
2376 } else {
2377 /* swapper task is special */
2378 if (!val) {
2379 pid = ftrace_swapper_pid;
2380 if (pid == ftrace_pid_trace)
2381 goto out;
2382 } else {
2383 pid = find_get_pid(val);
2384
2385 if (pid == ftrace_pid_trace) {
2386 put_pid(pid);
2387 goto out;
2388 }
2389 }
2390
2391 if (ftrace_pid_trace)
2392 clear_ftrace_pid_task(&ftrace_pid_trace);
2393
2394 if (!pid)
2395 goto out;
2396
2397 ftrace_pid_trace = pid;
2398
2399 set_ftrace_pid_task(ftrace_pid_trace);
2400 }
2401
2402 /* update the function call */
2403 ftrace_update_pid_func();
2404 ftrace_startup_enable(0);
2405
2406 out:
2407 mutex_unlock(&ftrace_lock);
2408
2409 return cnt;
2410 }
2411
2412 static const struct file_operations ftrace_pid_fops = {
2413 .read = ftrace_pid_read,
2414 .write = ftrace_pid_write,
2415 };
2416
2417 static __init int ftrace_init_debugfs(void)
2418 {
2419 struct dentry *d_tracer;
2420 struct dentry *entry;
2421
2422 d_tracer = tracing_init_dentry();
2423 if (!d_tracer)
2424 return 0;
2425
2426 ftrace_init_dyn_debugfs(d_tracer);
2427
2428 entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
2429 NULL, &ftrace_pid_fops);
2430 if (!entry)
2431 pr_warning("Could not create debugfs "
2432 "'set_ftrace_pid' entry\n");
2433 return 0;
2434 }
2435 fs_initcall(ftrace_init_debugfs);
2436
2437 /**
2438 * ftrace_kill - kill ftrace
2439 *
2440 * This function should be used by panic code. It stops ftrace
2441 * but in a not so nice way. If you need to simply kill ftrace
2442 * from a non-atomic section, use ftrace_kill.
2443 */
2444 void ftrace_kill(void)
2445 {
2446 ftrace_disabled = 1;
2447 ftrace_enabled = 0;
2448 clear_ftrace_function();
2449 }
2450
2451 /**
2452 * register_ftrace_function - register a function for profiling
2453 * @ops - ops structure that holds the function for profiling.
2454 *
2455 * Register a function to be called by all functions in the
2456 * kernel.
2457 *
2458 * Note: @ops->func and all the functions it calls must be labeled
2459 * with "notrace", otherwise it will go into a
2460 * recursive loop.
2461 */
2462 int register_ftrace_function(struct ftrace_ops *ops)
2463 {
2464 int ret;
2465
2466 if (unlikely(ftrace_disabled))
2467 return -1;
2468
2469 mutex_lock(&ftrace_lock);
2470
2471 ret = __register_ftrace_function(ops);
2472 ftrace_startup(0);
2473
2474 mutex_unlock(&ftrace_lock);
2475 return ret;
2476 }
2477
2478 /**
2479 * unregister_ftrace_function - unregister a function for profiling.
2480 * @ops - ops structure that holds the function to unregister
2481 *
2482 * Unregister a function that was added to be called by ftrace profiling.
2483 */
2484 int unregister_ftrace_function(struct ftrace_ops *ops)
2485 {
2486 int ret;
2487
2488 mutex_lock(&ftrace_lock);
2489 ret = __unregister_ftrace_function(ops);
2490 ftrace_shutdown(0);
2491 mutex_unlock(&ftrace_lock);
2492
2493 return ret;
2494 }
2495
2496 int
2497 ftrace_enable_sysctl(struct ctl_table *table, int write,
2498 struct file *file, void __user *buffer, size_t *lenp,
2499 loff_t *ppos)
2500 {
2501 int ret;
2502
2503 if (unlikely(ftrace_disabled))
2504 return -ENODEV;
2505
2506 mutex_lock(&ftrace_lock);
2507
2508 ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
2509
2510 if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
2511 goto out;
2512
2513 last_ftrace_enabled = ftrace_enabled;
2514
2515 if (ftrace_enabled) {
2516
2517 ftrace_startup_sysctl();
2518
2519 /* we are starting ftrace again */
2520 if (ftrace_list != &ftrace_list_end) {
2521 if (ftrace_list->next == &ftrace_list_end)
2522 ftrace_trace_function = ftrace_list->func;
2523 else
2524 ftrace_trace_function = ftrace_list_func;
2525 }
2526
2527 } else {
2528 /* stopping ftrace calls (just send to ftrace_stub) */
2529 ftrace_trace_function = ftrace_stub;
2530
2531 ftrace_shutdown_sysctl();
2532 }
2533
2534 out:
2535 mutex_unlock(&ftrace_lock);
2536 return ret;
2537 }
2538
2539 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2540
2541 static atomic_t ftrace_graph_active;
2542 static struct notifier_block ftrace_suspend_notifier;
2543
2544 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
2545 {
2546 return 0;
2547 }
2548
2549 /* The callbacks that hook a function */
2550 trace_func_graph_ret_t ftrace_graph_return =
2551 (trace_func_graph_ret_t)ftrace_stub;
2552 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
2553
2554 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
2555 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
2556 {
2557 int i;
2558 int ret = 0;
2559 unsigned long flags;
2560 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
2561 struct task_struct *g, *t;
2562
2563 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
2564 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
2565 * sizeof(struct ftrace_ret_stack),
2566 GFP_KERNEL);
2567 if (!ret_stack_list[i]) {
2568 start = 0;
2569 end = i;
2570 ret = -ENOMEM;
2571 goto free;
2572 }
2573 }
2574
2575 read_lock_irqsave(&tasklist_lock, flags);
2576 do_each_thread(g, t) {
2577 if (start == end) {
2578 ret = -EAGAIN;
2579 goto unlock;
2580 }
2581
2582 if (t->ret_stack == NULL) {
2583 t->curr_ret_stack = -1;
2584 /* Make sure IRQs see the -1 first: */
2585 barrier();
2586 t->ret_stack = ret_stack_list[start++];
2587 atomic_set(&t->tracing_graph_pause, 0);
2588 atomic_set(&t->trace_overrun, 0);
2589 }
2590 } while_each_thread(g, t);
2591
2592 unlock:
2593 read_unlock_irqrestore(&tasklist_lock, flags);
2594 free:
2595 for (i = start; i < end; i++)
2596 kfree(ret_stack_list[i]);
2597 return ret;
2598 }
2599
2600 static void
2601 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
2602 struct task_struct *next)
2603 {
2604 unsigned long long timestamp;
2605 int index;
2606
2607 /*
2608 * Does the user want to count the time a function was asleep.
2609 * If so, do not update the time stamps.
2610 */
2611 if (trace_flags & TRACE_ITER_SLEEP_TIME)
2612 return;
2613
2614 timestamp = trace_clock_local();
2615
2616 prev->ftrace_timestamp = timestamp;
2617
2618 /* only process tasks that we timestamped */
2619 if (!next->ftrace_timestamp)
2620 return;
2621
2622 /*
2623 * Update all the counters in next to make up for the
2624 * time next was sleeping.
2625 */
2626 timestamp -= next->ftrace_timestamp;
2627
2628 for (index = next->curr_ret_stack; index >= 0; index--)
2629 next->ret_stack[index].calltime += timestamp;
2630 }
2631
2632 /* Allocate a return stack for each task */
2633 static int start_graph_tracing(void)
2634 {
2635 struct ftrace_ret_stack **ret_stack_list;
2636 int ret, cpu;
2637
2638 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2639 sizeof(struct ftrace_ret_stack *),
2640 GFP_KERNEL);
2641
2642 if (!ret_stack_list)
2643 return -ENOMEM;
2644
2645 /* The cpu_boot init_task->ret_stack will never be freed */
2646 for_each_online_cpu(cpu)
2647 ftrace_graph_init_task(idle_task(cpu));
2648
2649 do {
2650 ret = alloc_retstack_tasklist(ret_stack_list);
2651 } while (ret == -EAGAIN);
2652
2653 if (!ret) {
2654 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
2655 if (ret)
2656 pr_info("ftrace_graph: Couldn't activate tracepoint"
2657 " probe to kernel_sched_switch\n");
2658 }
2659
2660 kfree(ret_stack_list);
2661 return ret;
2662 }
2663
2664 /*
2665 * Hibernation protection.
2666 * The state of the current task is too much unstable during
2667 * suspend/restore to disk. We want to protect against that.
2668 */
2669 static int
2670 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
2671 void *unused)
2672 {
2673 switch (state) {
2674 case PM_HIBERNATION_PREPARE:
2675 pause_graph_tracing();
2676 break;
2677
2678 case PM_POST_HIBERNATION:
2679 unpause_graph_tracing();
2680 break;
2681 }
2682 return NOTIFY_DONE;
2683 }
2684
2685 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2686 trace_func_graph_ent_t entryfunc)
2687 {
2688 int ret = 0;
2689
2690 mutex_lock(&ftrace_lock);
2691
2692 /* we currently allow only one tracer registered at a time */
2693 if (atomic_read(&ftrace_graph_active)) {
2694 ret = -EBUSY;
2695 goto out;
2696 }
2697
2698 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2699 register_pm_notifier(&ftrace_suspend_notifier);
2700
2701 atomic_inc(&ftrace_graph_active);
2702 ret = start_graph_tracing();
2703 if (ret) {
2704 atomic_dec(&ftrace_graph_active);
2705 goto out;
2706 }
2707
2708 ftrace_graph_return = retfunc;
2709 ftrace_graph_entry = entryfunc;
2710
2711 ftrace_startup(FTRACE_START_FUNC_RET);
2712
2713 out:
2714 mutex_unlock(&ftrace_lock);
2715 return ret;
2716 }
2717
2718 void unregister_ftrace_graph(void)
2719 {
2720 mutex_lock(&ftrace_lock);
2721
2722 if (!unlikely(atomic_read(&ftrace_graph_active)))
2723 goto out;
2724
2725 atomic_dec(&ftrace_graph_active);
2726 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
2727 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2728 ftrace_graph_entry = ftrace_graph_entry_stub;
2729 ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2730 unregister_pm_notifier(&ftrace_suspend_notifier);
2731
2732 out:
2733 mutex_unlock(&ftrace_lock);
2734 }
2735
2736 /* Allocate a return stack for newly created task */
2737 void ftrace_graph_init_task(struct task_struct *t)
2738 {
2739 if (atomic_read(&ftrace_graph_active)) {
2740 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2741 * sizeof(struct ftrace_ret_stack),
2742 GFP_KERNEL);
2743 if (!t->ret_stack)
2744 return;
2745 t->curr_ret_stack = -1;
2746 atomic_set(&t->tracing_graph_pause, 0);
2747 atomic_set(&t->trace_overrun, 0);
2748 t->ftrace_timestamp = 0;
2749 } else
2750 t->ret_stack = NULL;
2751 }
2752
2753 void ftrace_graph_exit_task(struct task_struct *t)
2754 {
2755 struct ftrace_ret_stack *ret_stack = t->ret_stack;
2756
2757 t->ret_stack = NULL;
2758 /* NULL must become visible to IRQs before we free it: */
2759 barrier();
2760
2761 kfree(ret_stack);
2762 }
2763
2764 void ftrace_graph_stop(void)
2765 {
2766 ftrace_stop();
2767 }
2768 #endif
2769
Linux kernel - Deliverables
This page took 0.118413 seconds and 6 git commands to generate.