projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge commit 'v2.6.29' into core/header-fixes
[deliverable/linux.git] / kernel / trace / ftrace.c
1 /*
2 * Infrastructure for profiling code inserted by 'gcc -pg'.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code in the latency_tracer, that is:
11 *
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
14 */
15
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30
31 #include <asm/ftrace.h>
32
33 #include "trace.h"
34
35 #define FTRACE_WARN_ON(cond) \
36 do { \
37 if (WARN_ON(cond)) \
38 ftrace_kill(); \
39 } while (0)
40
41 #define FTRACE_WARN_ON_ONCE(cond) \
42 do { \
43 if (WARN_ON_ONCE(cond)) \
44 ftrace_kill(); \
45 } while (0)
46
47 /* ftrace_enabled is a method to turn ftrace on or off */
48 int ftrace_enabled __read_mostly;
49 static int last_ftrace_enabled;
50
51 /* set when tracing only a pid */
52 struct pid *ftrace_pid_trace;
53 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
54
55 /* Quick disabling of function tracer. */
56 int function_trace_stop;
57
58 /*
59 * ftrace_disabled is set when an anomaly is discovered.
60 * ftrace_disabled is much stronger than ftrace_enabled.
61 */
62 static int ftrace_disabled __read_mostly;
63
64 static DEFINE_SPINLOCK(ftrace_lock);
65 static DEFINE_MUTEX(ftrace_sysctl_lock);
66 static DEFINE_MUTEX(ftrace_start_lock);
67
68 static struct ftrace_ops ftrace_list_end __read_mostly =
69 {
70 .func = ftrace_stub,
71 };
72
73 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
74 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
75 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
76 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
77
78 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
79 {
80 struct ftrace_ops *op = ftrace_list;
81
82 /* in case someone actually ports this to alpha! */
83 read_barrier_depends();
84
85 while (op != &ftrace_list_end) {
86 /* silly alpha */
87 read_barrier_depends();
88 op->func(ip, parent_ip);
89 op = op->next;
90 };
91 }
92
93 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
94 {
95 if (!test_tsk_trace_trace(current))
96 return;
97
98 ftrace_pid_function(ip, parent_ip);
99 }
100
101 static void set_ftrace_pid_function(ftrace_func_t func)
102 {
103 /* do not set ftrace_pid_function to itself! */
104 if (func != ftrace_pid_func)
105 ftrace_pid_function = func;
106 }
107
108 /**
109 * clear_ftrace_function - reset the ftrace function
110 *
111 * This NULLs the ftrace function and in essence stops
112 * tracing. There may be lag
113 */
114 void clear_ftrace_function(void)
115 {
116 ftrace_trace_function = ftrace_stub;
117 __ftrace_trace_function = ftrace_stub;
118 ftrace_pid_function = ftrace_stub;
119 }
120
121 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
122 /*
123 * For those archs that do not test ftrace_trace_stop in their
124 * mcount call site, we need to do it from C.
125 */
126 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
127 {
128 if (function_trace_stop)
129 return;
130
131 __ftrace_trace_function(ip, parent_ip);
132 }
133 #endif
134
135 static int __register_ftrace_function(struct ftrace_ops *ops)
136 {
137 /* should not be called from interrupt context */
138 spin_lock(&ftrace_lock);
139
140 ops->next = ftrace_list;
141 /*
142 * We are entering ops into the ftrace_list but another
143 * CPU might be walking that list. We need to make sure
144 * the ops->next pointer is valid before another CPU sees
145 * the ops pointer included into the ftrace_list.
146 */
147 smp_wmb();
148 ftrace_list = ops;
149
150 if (ftrace_enabled) {
151 ftrace_func_t func;
152
153 if (ops->next == &ftrace_list_end)
154 func = ops->func;
155 else
156 func = ftrace_list_func;
157
158 if (ftrace_pid_trace) {
159 set_ftrace_pid_function(func);
160 func = ftrace_pid_func;
161 }
162
163 /*
164 * For one func, simply call it directly.
165 * For more than one func, call the chain.
166 */
167 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
168 ftrace_trace_function = func;
169 #else
170 __ftrace_trace_function = func;
171 ftrace_trace_function = ftrace_test_stop_func;
172 #endif
173 }
174
175 spin_unlock(&ftrace_lock);
176
177 return 0;
178 }
179
180 static int __unregister_ftrace_function(struct ftrace_ops *ops)
181 {
182 struct ftrace_ops **p;
183 int ret = 0;
184
185 /* should not be called from interrupt context */
186 spin_lock(&ftrace_lock);
187
188 /*
189 * If we are removing the last function, then simply point
190 * to the ftrace_stub.
191 */
192 if (ftrace_list == ops && ops->next == &ftrace_list_end) {
193 ftrace_trace_function = ftrace_stub;
194 ftrace_list = &ftrace_list_end;
195 goto out;
196 }
197
198 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
199 if (*p == ops)
200 break;
201
202 if (*p != ops) {
203 ret = -1;
204 goto out;
205 }
206
207 *p = (*p)->next;
208
209 if (ftrace_enabled) {
210 /* If we only have one func left, then call that directly */
211 if (ftrace_list->next == &ftrace_list_end) {
212 ftrace_func_t func = ftrace_list->func;
213
214 if (ftrace_pid_trace) {
215 set_ftrace_pid_function(func);
216 func = ftrace_pid_func;
217 }
218 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
219 ftrace_trace_function = func;
220 #else
221 __ftrace_trace_function = func;
222 #endif
223 }
224 }
225
226 out:
227 spin_unlock(&ftrace_lock);
228
229 return ret;
230 }
231
232 static void ftrace_update_pid_func(void)
233 {
234 ftrace_func_t func;
235
236 /* should not be called from interrupt context */
237 spin_lock(&ftrace_lock);
238
239 if (ftrace_trace_function == ftrace_stub)
240 goto out;
241
242 func = ftrace_trace_function;
243
244 if (ftrace_pid_trace) {
245 set_ftrace_pid_function(func);
246 func = ftrace_pid_func;
247 } else {
248 if (func == ftrace_pid_func)
249 func = ftrace_pid_function;
250 }
251
252 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
253 ftrace_trace_function = func;
254 #else
255 __ftrace_trace_function = func;
256 #endif
257
258 out:
259 spin_unlock(&ftrace_lock);
260 }
261
262 #ifdef CONFIG_DYNAMIC_FTRACE
263 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
264 # error Dynamic ftrace depends on MCOUNT_RECORD
265 #endif
266
267 /*
268 * Since MCOUNT_ADDR may point to mcount itself, we do not want
269 * to get it confused by reading a reference in the code as we
270 * are parsing on objcopy output of text. Use a variable for
271 * it instead.
272 */
273 static unsigned long mcount_addr = MCOUNT_ADDR;
274
275 enum {
276 FTRACE_ENABLE_CALLS = (1 << 0),
277 FTRACE_DISABLE_CALLS = (1 << 1),
278 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
279 FTRACE_ENABLE_MCOUNT = (1 << 3),
280 FTRACE_DISABLE_MCOUNT = (1 << 4),
281 FTRACE_START_FUNC_RET = (1 << 5),
282 FTRACE_STOP_FUNC_RET = (1 << 6),
283 };
284
285 static int ftrace_filtered;
286
287 static LIST_HEAD(ftrace_new_addrs);
288
289 static DEFINE_MUTEX(ftrace_regex_lock);
290
291 struct ftrace_page {
292 struct ftrace_page *next;
293 unsigned long index;
294 struct dyn_ftrace records[];
295 };
296
297 #define ENTRIES_PER_PAGE \
298 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
299
300 /* estimate from running different kernels */
301 #define NR_TO_INIT 10000
302
303 static struct ftrace_page *ftrace_pages_start;
304 static struct ftrace_page *ftrace_pages;
305
306 static struct dyn_ftrace *ftrace_free_records;
307
308
309 #ifdef CONFIG_KPROBES
310
311 static int frozen_record_count;
312
313 static inline void freeze_record(struct dyn_ftrace *rec)
314 {
315 if (!(rec->flags & FTRACE_FL_FROZEN)) {
316 rec->flags |= FTRACE_FL_FROZEN;
317 frozen_record_count++;
318 }
319 }
320
321 static inline void unfreeze_record(struct dyn_ftrace *rec)
322 {
323 if (rec->flags & FTRACE_FL_FROZEN) {
324 rec->flags &= ~FTRACE_FL_FROZEN;
325 frozen_record_count--;
326 }
327 }
328
329 static inline int record_frozen(struct dyn_ftrace *rec)
330 {
331 return rec->flags & FTRACE_FL_FROZEN;
332 }
333 #else
334 # define freeze_record(rec) ({ 0; })
335 # define unfreeze_record(rec) ({ 0; })
336 # define record_frozen(rec) ({ 0; })
337 #endif /* CONFIG_KPROBES */
338
339 static void ftrace_free_rec(struct dyn_ftrace *rec)
340 {
341 rec->ip = (unsigned long)ftrace_free_records;
342 ftrace_free_records = rec;
343 rec->flags |= FTRACE_FL_FREE;
344 }
345
346 void ftrace_release(void *start, unsigned long size)
347 {
348 struct dyn_ftrace *rec;
349 struct ftrace_page *pg;
350 unsigned long s = (unsigned long)start;
351 unsigned long e = s + size;
352 int i;
353
354 if (ftrace_disabled || !start)
355 return;
356
357 /* should not be called from interrupt context */
358 spin_lock(&ftrace_lock);
359
360 for (pg = ftrace_pages_start; pg; pg = pg->next) {
361 for (i = 0; i < pg->index; i++) {
362 rec = &pg->records[i];
363
364 if ((rec->ip >= s) && (rec->ip < e))
365 ftrace_free_rec(rec);
366 }
367 }
368 spin_unlock(&ftrace_lock);
369 }
370
371 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
372 {
373 struct dyn_ftrace *rec;
374
375 /* First check for freed records */
376 if (ftrace_free_records) {
377 rec = ftrace_free_records;
378
379 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
380 FTRACE_WARN_ON_ONCE(1);
381 ftrace_free_records = NULL;
382 return NULL;
383 }
384
385 ftrace_free_records = (void *)rec->ip;
386 memset(rec, 0, sizeof(*rec));
387 return rec;
388 }
389
390 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
391 if (!ftrace_pages->next) {
392 /* allocate another page */
393 ftrace_pages->next =
394 (void *)get_zeroed_page(GFP_KERNEL);
395 if (!ftrace_pages->next)
396 return NULL;
397 }
398 ftrace_pages = ftrace_pages->next;
399 }
400
401 return &ftrace_pages->records[ftrace_pages->index++];
402 }
403
404 static struct dyn_ftrace *
405 ftrace_record_ip(unsigned long ip)
406 {
407 struct dyn_ftrace *rec;
408
409 if (ftrace_disabled)
410 return NULL;
411
412 rec = ftrace_alloc_dyn_node(ip);
413 if (!rec)
414 return NULL;
415
416 rec->ip = ip;
417
418 list_add(&rec->list, &ftrace_new_addrs);
419
420 return rec;
421 }
422
423 static void print_ip_ins(const char *fmt, unsigned char *p)
424 {
425 int i;
426
427 printk(KERN_CONT "%s", fmt);
428
429 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
430 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
431 }
432
433 static void ftrace_bug(int failed, unsigned long ip)
434 {
435 switch (failed) {
436 case -EFAULT:
437 FTRACE_WARN_ON_ONCE(1);
438 pr_info("ftrace faulted on modifying ");
439 print_ip_sym(ip);
440 break;
441 case -EINVAL:
442 FTRACE_WARN_ON_ONCE(1);
443 pr_info("ftrace failed to modify ");
444 print_ip_sym(ip);
445 print_ip_ins(" actual: ", (unsigned char *)ip);
446 printk(KERN_CONT "\n");
447 break;
448 case -EPERM:
449 FTRACE_WARN_ON_ONCE(1);
450 pr_info("ftrace faulted on writing ");
451 print_ip_sym(ip);
452 break;
453 default:
454 FTRACE_WARN_ON_ONCE(1);
455 pr_info("ftrace faulted on unknown error ");
456 print_ip_sym(ip);
457 }
458 }
459
460
461 static int
462 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
463 {
464 unsigned long ip, fl;
465 unsigned long ftrace_addr;
466
467 ftrace_addr = (unsigned long)ftrace_caller;
468
469 ip = rec->ip;
470
471 /*
472 * If this record is not to be traced and
473 * it is not enabled then do nothing.
474 *
475 * If this record is not to be traced and
476 * it is enabled then disabled it.
477 *
478 */
479 if (rec->flags & FTRACE_FL_NOTRACE) {
480 if (rec->flags & FTRACE_FL_ENABLED)
481 rec->flags &= ~FTRACE_FL_ENABLED;
482 else
483 return 0;
484
485 } else if (ftrace_filtered && enable) {
486 /*
487 * Filtering is on:
488 */
489
490 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
491
492 /* Record is filtered and enabled, do nothing */
493 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
494 return 0;
495
496 /* Record is not filtered and is not enabled do nothing */
497 if (!fl)
498 return 0;
499
500 /* Record is not filtered but enabled, disable it */
501 if (fl == FTRACE_FL_ENABLED)
502 rec->flags &= ~FTRACE_FL_ENABLED;
503 else
504 /* Otherwise record is filtered but not enabled, enable it */
505 rec->flags |= FTRACE_FL_ENABLED;
506 } else {
507 /* Disable or not filtered */
508
509 if (enable) {
510 /* if record is enabled, do nothing */
511 if (rec->flags & FTRACE_FL_ENABLED)
512 return 0;
513
514 rec->flags |= FTRACE_FL_ENABLED;
515
516 } else {
517
518 /* if record is not enabled do nothing */
519 if (!(rec->flags & FTRACE_FL_ENABLED))
520 return 0;
521
522 rec->flags &= ~FTRACE_FL_ENABLED;
523 }
524 }
525
526 if (rec->flags & FTRACE_FL_ENABLED)
527 return ftrace_make_call(rec, ftrace_addr);
528 else
529 return ftrace_make_nop(NULL, rec, ftrace_addr);
530 }
531
532 static void ftrace_replace_code(int enable)
533 {
534 int i, failed;
535 struct dyn_ftrace *rec;
536 struct ftrace_page *pg;
537
538 for (pg = ftrace_pages_start; pg; pg = pg->next) {
539 for (i = 0; i < pg->index; i++) {
540 rec = &pg->records[i];
541
542 /*
543 * Skip over free records and records that have
544 * failed.
545 */
546 if (rec->flags & FTRACE_FL_FREE ||
547 rec->flags & FTRACE_FL_FAILED)
548 continue;
549
550 /* ignore updates to this record's mcount site */
551 if (get_kprobe((void *)rec->ip)) {
552 freeze_record(rec);
553 continue;
554 } else {
555 unfreeze_record(rec);
556 }
557
558 failed = __ftrace_replace_code(rec, enable);
559 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
560 rec->flags |= FTRACE_FL_FAILED;
561 if ((system_state == SYSTEM_BOOTING) ||
562 !core_kernel_text(rec->ip)) {
563 ftrace_free_rec(rec);
564 } else
565 ftrace_bug(failed, rec->ip);
566 }
567 }
568 }
569 }
570
571 static int
572 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
573 {
574 unsigned long ip;
575 int ret;
576
577 ip = rec->ip;
578
579 ret = ftrace_make_nop(mod, rec, mcount_addr);
580 if (ret) {
581 ftrace_bug(ret, ip);
582 rec->flags |= FTRACE_FL_FAILED;
583 return 0;
584 }
585 return 1;
586 }
587
588 static int __ftrace_modify_code(void *data)
589 {
590 int *command = data;
591
592 if (*command & FTRACE_ENABLE_CALLS)
593 ftrace_replace_code(1);
594 else if (*command & FTRACE_DISABLE_CALLS)
595 ftrace_replace_code(0);
596
597 if (*command & FTRACE_UPDATE_TRACE_FUNC)
598 ftrace_update_ftrace_func(ftrace_trace_function);
599
600 if (*command & FTRACE_START_FUNC_RET)
601 ftrace_enable_ftrace_graph_caller();
602 else if (*command & FTRACE_STOP_FUNC_RET)
603 ftrace_disable_ftrace_graph_caller();
604
605 return 0;
606 }
607
608 static void ftrace_run_update_code(int command)
609 {
610 stop_machine(__ftrace_modify_code, &command, NULL);
611 }
612
613 static ftrace_func_t saved_ftrace_func;
614 static int ftrace_start_up;
615
616 static void ftrace_startup_enable(int command)
617 {
618 if (saved_ftrace_func != ftrace_trace_function) {
619 saved_ftrace_func = ftrace_trace_function;
620 command |= FTRACE_UPDATE_TRACE_FUNC;
621 }
622
623 if (!command || !ftrace_enabled)
624 return;
625
626 ftrace_run_update_code(command);
627 }
628
629 static void ftrace_startup(int command)
630 {
631 if (unlikely(ftrace_disabled))
632 return;
633
634 mutex_lock(&ftrace_start_lock);
635 ftrace_start_up++;
636 command |= FTRACE_ENABLE_CALLS;
637
638 ftrace_startup_enable(command);
639
640 mutex_unlock(&ftrace_start_lock);
641 }
642
643 static void ftrace_shutdown(int command)
644 {
645 if (unlikely(ftrace_disabled))
646 return;
647
648 mutex_lock(&ftrace_start_lock);
649 ftrace_start_up--;
650 if (!ftrace_start_up)
651 command |= FTRACE_DISABLE_CALLS;
652
653 if (saved_ftrace_func != ftrace_trace_function) {
654 saved_ftrace_func = ftrace_trace_function;
655 command |= FTRACE_UPDATE_TRACE_FUNC;
656 }
657
658 if (!command || !ftrace_enabled)
659 goto out;
660
661 ftrace_run_update_code(command);
662 out:
663 mutex_unlock(&ftrace_start_lock);
664 }
665
666 static void ftrace_startup_sysctl(void)
667 {
668 int command = FTRACE_ENABLE_MCOUNT;
669
670 if (unlikely(ftrace_disabled))
671 return;
672
673 mutex_lock(&ftrace_start_lock);
674 /* Force update next time */
675 saved_ftrace_func = NULL;
676 /* ftrace_start_up is true if we want ftrace running */
677 if (ftrace_start_up)
678 command |= FTRACE_ENABLE_CALLS;
679
680 ftrace_run_update_code(command);
681 mutex_unlock(&ftrace_start_lock);
682 }
683
684 static void ftrace_shutdown_sysctl(void)
685 {
686 int command = FTRACE_DISABLE_MCOUNT;
687
688 if (unlikely(ftrace_disabled))
689 return;
690
691 mutex_lock(&ftrace_start_lock);
692 /* ftrace_start_up is true if ftrace is running */
693 if (ftrace_start_up)
694 command |= FTRACE_DISABLE_CALLS;
695
696 ftrace_run_update_code(command);
697 mutex_unlock(&ftrace_start_lock);
698 }
699
700 static cycle_t ftrace_update_time;
701 static unsigned long ftrace_update_cnt;
702 unsigned long ftrace_update_tot_cnt;
703
704 static int ftrace_update_code(struct module *mod)
705 {
706 struct dyn_ftrace *p, *t;
707 cycle_t start, stop;
708
709 start = ftrace_now(raw_smp_processor_id());
710 ftrace_update_cnt = 0;
711
712 list_for_each_entry_safe(p, t, &ftrace_new_addrs, list) {
713
714 /* If something went wrong, bail without enabling anything */
715 if (unlikely(ftrace_disabled))
716 return -1;
717
718 list_del_init(&p->list);
719
720 /* convert record (i.e, patch mcount-call with NOP) */
721 if (ftrace_code_disable(mod, p)) {
722 p->flags |= FTRACE_FL_CONVERTED;
723 ftrace_update_cnt++;
724 } else
725 ftrace_free_rec(p);
726 }
727
728 stop = ftrace_now(raw_smp_processor_id());
729 ftrace_update_time = stop - start;
730 ftrace_update_tot_cnt += ftrace_update_cnt;
731
732 return 0;
733 }
734
735 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
736 {
737 struct ftrace_page *pg;
738 int cnt;
739 int i;
740
741 /* allocate a few pages */
742 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
743 if (!ftrace_pages_start)
744 return -1;
745
746 /*
747 * Allocate a few more pages.
748 *
749 * TODO: have some parser search vmlinux before
750 * final linking to find all calls to ftrace.
751 * Then we can:
752 * a) know how many pages to allocate.
753 * and/or
754 * b) set up the table then.
755 *
756 * The dynamic code is still necessary for
757 * modules.
758 */
759
760 pg = ftrace_pages = ftrace_pages_start;
761
762 cnt = num_to_init / ENTRIES_PER_PAGE;
763 pr_info("ftrace: allocating %ld entries in %d pages\n",
764 num_to_init, cnt + 1);
765
766 for (i = 0; i < cnt; i++) {
767 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
768
769 /* If we fail, we'll try later anyway */
770 if (!pg->next)
771 break;
772
773 pg = pg->next;
774 }
775
776 return 0;
777 }
778
779 enum {
780 FTRACE_ITER_FILTER = (1 << 0),
781 FTRACE_ITER_CONT = (1 << 1),
782 FTRACE_ITER_NOTRACE = (1 << 2),
783 FTRACE_ITER_FAILURES = (1 << 3),
784 };
785
786 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
787
788 struct ftrace_iterator {
789 struct ftrace_page *pg;
790 unsigned idx;
791 unsigned flags;
792 unsigned char buffer[FTRACE_BUFF_MAX+1];
793 unsigned buffer_idx;
794 unsigned filtered;
795 };
796
797 static void *
798 t_next(struct seq_file *m, void *v, loff_t *pos)
799 {
800 struct ftrace_iterator *iter = m->private;
801 struct dyn_ftrace *rec = NULL;
802
803 (*pos)++;
804
805 /* should not be called from interrupt context */
806 spin_lock(&ftrace_lock);
807 retry:
808 if (iter->idx >= iter->pg->index) {
809 if (iter->pg->next) {
810 iter->pg = iter->pg->next;
811 iter->idx = 0;
812 goto retry;
813 } else {
814 iter->idx = -1;
815 }
816 } else {
817 rec = &iter->pg->records[iter->idx++];
818 if ((rec->flags & FTRACE_FL_FREE) ||
819
820 (!(iter->flags & FTRACE_ITER_FAILURES) &&
821 (rec->flags & FTRACE_FL_FAILED)) ||
822
823 ((iter->flags & FTRACE_ITER_FAILURES) &&
824 !(rec->flags & FTRACE_FL_FAILED)) ||
825
826 ((iter->flags & FTRACE_ITER_FILTER) &&
827 !(rec->flags & FTRACE_FL_FILTER)) ||
828
829 ((iter->flags & FTRACE_ITER_NOTRACE) &&
830 !(rec->flags & FTRACE_FL_NOTRACE))) {
831 rec = NULL;
832 goto retry;
833 }
834 }
835 spin_unlock(&ftrace_lock);
836
837 return rec;
838 }
839
840 static void *t_start(struct seq_file *m, loff_t *pos)
841 {
842 struct ftrace_iterator *iter = m->private;
843 void *p = NULL;
844
845 if (*pos > 0) {
846 if (iter->idx < 0)
847 return p;
848 (*pos)--;
849 iter->idx--;
850 }
851
852 p = t_next(m, p, pos);
853
854 return p;
855 }
856
857 static void t_stop(struct seq_file *m, void *p)
858 {
859 }
860
861 static int t_show(struct seq_file *m, void *v)
862 {
863 struct dyn_ftrace *rec = v;
864 char str[KSYM_SYMBOL_LEN];
865
866 if (!rec)
867 return 0;
868
869 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
870
871 seq_printf(m, "%s\n", str);
872
873 return 0;
874 }
875
876 static struct seq_operations show_ftrace_seq_ops = {
877 .start = t_start,
878 .next = t_next,
879 .stop = t_stop,
880 .show = t_show,
881 };
882
883 static int
884 ftrace_avail_open(struct inode *inode, struct file *file)
885 {
886 struct ftrace_iterator *iter;
887 int ret;
888
889 if (unlikely(ftrace_disabled))
890 return -ENODEV;
891
892 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
893 if (!iter)
894 return -ENOMEM;
895
896 iter->pg = ftrace_pages_start;
897
898 ret = seq_open(file, &show_ftrace_seq_ops);
899 if (!ret) {
900 struct seq_file *m = file->private_data;
901
902 m->private = iter;
903 } else {
904 kfree(iter);
905 }
906
907 return ret;
908 }
909
910 int ftrace_avail_release(struct inode *inode, struct file *file)
911 {
912 struct seq_file *m = (struct seq_file *)file->private_data;
913 struct ftrace_iterator *iter = m->private;
914
915 seq_release(inode, file);
916 kfree(iter);
917
918 return 0;
919 }
920
921 static int
922 ftrace_failures_open(struct inode *inode, struct file *file)
923 {
924 int ret;
925 struct seq_file *m;
926 struct ftrace_iterator *iter;
927
928 ret = ftrace_avail_open(inode, file);
929 if (!ret) {
930 m = (struct seq_file *)file->private_data;
931 iter = (struct ftrace_iterator *)m->private;
932 iter->flags = FTRACE_ITER_FAILURES;
933 }
934
935 return ret;
936 }
937
938
939 static void ftrace_filter_reset(int enable)
940 {
941 struct ftrace_page *pg;
942 struct dyn_ftrace *rec;
943 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
944 unsigned i;
945
946 /* should not be called from interrupt context */
947 spin_lock(&ftrace_lock);
948 if (enable)
949 ftrace_filtered = 0;
950 pg = ftrace_pages_start;
951 while (pg) {
952 for (i = 0; i < pg->index; i++) {
953 rec = &pg->records[i];
954 if (rec->flags & FTRACE_FL_FAILED)
955 continue;
956 rec->flags &= ~type;
957 }
958 pg = pg->next;
959 }
960 spin_unlock(&ftrace_lock);
961 }
962
963 static int
964 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
965 {
966 struct ftrace_iterator *iter;
967 int ret = 0;
968
969 if (unlikely(ftrace_disabled))
970 return -ENODEV;
971
972 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
973 if (!iter)
974 return -ENOMEM;
975
976 mutex_lock(&ftrace_regex_lock);
977 if ((file->f_mode & FMODE_WRITE) &&
978 !(file->f_flags & O_APPEND))
979 ftrace_filter_reset(enable);
980
981 if (file->f_mode & FMODE_READ) {
982 iter->pg = ftrace_pages_start;
983 iter->flags = enable ? FTRACE_ITER_FILTER :
984 FTRACE_ITER_NOTRACE;
985
986 ret = seq_open(file, &show_ftrace_seq_ops);
987 if (!ret) {
988 struct seq_file *m = file->private_data;
989 m->private = iter;
990 } else
991 kfree(iter);
992 } else
993 file->private_data = iter;
994 mutex_unlock(&ftrace_regex_lock);
995
996 return ret;
997 }
998
999 static int
1000 ftrace_filter_open(struct inode *inode, struct file *file)
1001 {
1002 return ftrace_regex_open(inode, file, 1);
1003 }
1004
1005 static int
1006 ftrace_notrace_open(struct inode *inode, struct file *file)
1007 {
1008 return ftrace_regex_open(inode, file, 0);
1009 }
1010
1011 static ssize_t
1012 ftrace_regex_read(struct file *file, char __user *ubuf,
1013 size_t cnt, loff_t *ppos)
1014 {
1015 if (file->f_mode & FMODE_READ)
1016 return seq_read(file, ubuf, cnt, ppos);
1017 else
1018 return -EPERM;
1019 }
1020
1021 static loff_t
1022 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1023 {
1024 loff_t ret;
1025
1026 if (file->f_mode & FMODE_READ)
1027 ret = seq_lseek(file, offset, origin);
1028 else
1029 file->f_pos = ret = 1;
1030
1031 return ret;
1032 }
1033
1034 enum {
1035 MATCH_FULL,
1036 MATCH_FRONT_ONLY,
1037 MATCH_MIDDLE_ONLY,
1038 MATCH_END_ONLY,
1039 };
1040
1041 static void
1042 ftrace_match(unsigned char *buff, int len, int enable)
1043 {
1044 char str[KSYM_SYMBOL_LEN];
1045 char *search = NULL;
1046 struct ftrace_page *pg;
1047 struct dyn_ftrace *rec;
1048 int type = MATCH_FULL;
1049 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1050 unsigned i, match = 0, search_len = 0;
1051 int not = 0;
1052
1053 if (buff[0] == '!') {
1054 not = 1;
1055 buff++;
1056 len--;
1057 }
1058
1059 for (i = 0; i < len; i++) {
1060 if (buff[i] == '*') {
1061 if (!i) {
1062 search = buff + i + 1;
1063 type = MATCH_END_ONLY;
1064 search_len = len - (i + 1);
1065 } else {
1066 if (type == MATCH_END_ONLY) {
1067 type = MATCH_MIDDLE_ONLY;
1068 } else {
1069 match = i;
1070 type = MATCH_FRONT_ONLY;
1071 }
1072 buff[i] = 0;
1073 break;
1074 }
1075 }
1076 }
1077
1078 /* should not be called from interrupt context */
1079 spin_lock(&ftrace_lock);
1080 if (enable)
1081 ftrace_filtered = 1;
1082 pg = ftrace_pages_start;
1083 while (pg) {
1084 for (i = 0; i < pg->index; i++) {
1085 int matched = 0;
1086 char *ptr;
1087
1088 rec = &pg->records[i];
1089 if (rec->flags & FTRACE_FL_FAILED)
1090 continue;
1091 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1092 switch (type) {
1093 case MATCH_FULL:
1094 if (strcmp(str, buff) == 0)
1095 matched = 1;
1096 break;
1097 case MATCH_FRONT_ONLY:
1098 if (memcmp(str, buff, match) == 0)
1099 matched = 1;
1100 break;
1101 case MATCH_MIDDLE_ONLY:
1102 if (strstr(str, search))
1103 matched = 1;
1104 break;
1105 case MATCH_END_ONLY:
1106 ptr = strstr(str, search);
1107 if (ptr && (ptr[search_len] == 0))
1108 matched = 1;
1109 break;
1110 }
1111 if (matched) {
1112 if (not)
1113 rec->flags &= ~flag;
1114 else
1115 rec->flags |= flag;
1116 }
1117 }
1118 pg = pg->next;
1119 }
1120 spin_unlock(&ftrace_lock);
1121 }
1122
1123 static ssize_t
1124 ftrace_regex_write(struct file *file, const char __user *ubuf,
1125 size_t cnt, loff_t *ppos, int enable)
1126 {
1127 struct ftrace_iterator *iter;
1128 char ch;
1129 size_t read = 0;
1130 ssize_t ret;
1131
1132 if (!cnt || cnt < 0)
1133 return 0;
1134
1135 mutex_lock(&ftrace_regex_lock);
1136
1137 if (file->f_mode & FMODE_READ) {
1138 struct seq_file *m = file->private_data;
1139 iter = m->private;
1140 } else
1141 iter = file->private_data;
1142
1143 if (!*ppos) {
1144 iter->flags &= ~FTRACE_ITER_CONT;
1145 iter->buffer_idx = 0;
1146 }
1147
1148 ret = get_user(ch, ubuf++);
1149 if (ret)
1150 goto out;
1151 read++;
1152 cnt--;
1153
1154 if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1155 /* skip white space */
1156 while (cnt && isspace(ch)) {
1157 ret = get_user(ch, ubuf++);
1158 if (ret)
1159 goto out;
1160 read++;
1161 cnt--;
1162 }
1163
1164 if (isspace(ch)) {
1165 file->f_pos += read;
1166 ret = read;
1167 goto out;
1168 }
1169
1170 iter->buffer_idx = 0;
1171 }
1172
1173 while (cnt && !isspace(ch)) {
1174 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1175 iter->buffer[iter->buffer_idx++] = ch;
1176 else {
1177 ret = -EINVAL;
1178 goto out;
1179 }
1180 ret = get_user(ch, ubuf++);
1181 if (ret)
1182 goto out;
1183 read++;
1184 cnt--;
1185 }
1186
1187 if (isspace(ch)) {
1188 iter->filtered++;
1189 iter->buffer[iter->buffer_idx] = 0;
1190 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1191 iter->buffer_idx = 0;
1192 } else
1193 iter->flags |= FTRACE_ITER_CONT;
1194
1195
1196 file->f_pos += read;
1197
1198 ret = read;
1199 out:
1200 mutex_unlock(&ftrace_regex_lock);
1201
1202 return ret;
1203 }
1204
1205 static ssize_t
1206 ftrace_filter_write(struct file *file, const char __user *ubuf,
1207 size_t cnt, loff_t *ppos)
1208 {
1209 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1210 }
1211
1212 static ssize_t
1213 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1214 size_t cnt, loff_t *ppos)
1215 {
1216 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1217 }
1218
1219 static void
1220 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1221 {
1222 if (unlikely(ftrace_disabled))
1223 return;
1224
1225 mutex_lock(&ftrace_regex_lock);
1226 if (reset)
1227 ftrace_filter_reset(enable);
1228 if (buf)
1229 ftrace_match(buf, len, enable);
1230 mutex_unlock(&ftrace_regex_lock);
1231 }
1232
1233 /**
1234 * ftrace_set_filter - set a function to filter on in ftrace
1235 * @buf - the string that holds the function filter text.
1236 * @len - the length of the string.
1237 * @reset - non zero to reset all filters before applying this filter.
1238 *
1239 * Filters denote which functions should be enabled when tracing is enabled.
1240 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1241 */
1242 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1243 {
1244 ftrace_set_regex(buf, len, reset, 1);
1245 }
1246
1247 /**
1248 * ftrace_set_notrace - set a function to not trace in ftrace
1249 * @buf - the string that holds the function notrace text.
1250 * @len - the length of the string.
1251 * @reset - non zero to reset all filters before applying this filter.
1252 *
1253 * Notrace Filters denote which functions should not be enabled when tracing
1254 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1255 * for tracing.
1256 */
1257 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1258 {
1259 ftrace_set_regex(buf, len, reset, 0);
1260 }
1261
1262 static int
1263 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1264 {
1265 struct seq_file *m = (struct seq_file *)file->private_data;
1266 struct ftrace_iterator *iter;
1267
1268 mutex_lock(&ftrace_regex_lock);
1269 if (file->f_mode & FMODE_READ) {
1270 iter = m->private;
1271
1272 seq_release(inode, file);
1273 } else
1274 iter = file->private_data;
1275
1276 if (iter->buffer_idx) {
1277 iter->filtered++;
1278 iter->buffer[iter->buffer_idx] = 0;
1279 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1280 }
1281
1282 mutex_lock(&ftrace_sysctl_lock);
1283 mutex_lock(&ftrace_start_lock);
1284 if (ftrace_start_up && ftrace_enabled)
1285 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1286 mutex_unlock(&ftrace_start_lock);
1287 mutex_unlock(&ftrace_sysctl_lock);
1288
1289 kfree(iter);
1290 mutex_unlock(&ftrace_regex_lock);
1291 return 0;
1292 }
1293
1294 static int
1295 ftrace_filter_release(struct inode *inode, struct file *file)
1296 {
1297 return ftrace_regex_release(inode, file, 1);
1298 }
1299
1300 static int
1301 ftrace_notrace_release(struct inode *inode, struct file *file)
1302 {
1303 return ftrace_regex_release(inode, file, 0);
1304 }
1305
1306 static struct file_operations ftrace_avail_fops = {
1307 .open = ftrace_avail_open,
1308 .read = seq_read,
1309 .llseek = seq_lseek,
1310 .release = ftrace_avail_release,
1311 };
1312
1313 static struct file_operations ftrace_failures_fops = {
1314 .open = ftrace_failures_open,
1315 .read = seq_read,
1316 .llseek = seq_lseek,
1317 .release = ftrace_avail_release,
1318 };
1319
1320 static struct file_operations ftrace_filter_fops = {
1321 .open = ftrace_filter_open,
1322 .read = ftrace_regex_read,
1323 .write = ftrace_filter_write,
1324 .llseek = ftrace_regex_lseek,
1325 .release = ftrace_filter_release,
1326 };
1327
1328 static struct file_operations ftrace_notrace_fops = {
1329 .open = ftrace_notrace_open,
1330 .read = ftrace_regex_read,
1331 .write = ftrace_notrace_write,
1332 .llseek = ftrace_regex_lseek,
1333 .release = ftrace_notrace_release,
1334 };
1335
1336 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1337
1338 static DEFINE_MUTEX(graph_lock);
1339
1340 int ftrace_graph_count;
1341 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1342
1343 static void *
1344 g_next(struct seq_file *m, void *v, loff_t *pos)
1345 {
1346 unsigned long *array = m->private;
1347 int index = *pos;
1348
1349 (*pos)++;
1350
1351 if (index >= ftrace_graph_count)
1352 return NULL;
1353
1354 return &array[index];
1355 }
1356
1357 static void *g_start(struct seq_file *m, loff_t *pos)
1358 {
1359 void *p = NULL;
1360
1361 mutex_lock(&graph_lock);
1362
1363 p = g_next(m, p, pos);
1364
1365 return p;
1366 }
1367
1368 static void g_stop(struct seq_file *m, void *p)
1369 {
1370 mutex_unlock(&graph_lock);
1371 }
1372
1373 static int g_show(struct seq_file *m, void *v)
1374 {
1375 unsigned long *ptr = v;
1376 char str[KSYM_SYMBOL_LEN];
1377
1378 if (!ptr)
1379 return 0;
1380
1381 kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1382
1383 seq_printf(m, "%s\n", str);
1384
1385 return 0;
1386 }
1387
1388 static struct seq_operations ftrace_graph_seq_ops = {
1389 .start = g_start,
1390 .next = g_next,
1391 .stop = g_stop,
1392 .show = g_show,
1393 };
1394
1395 static int
1396 ftrace_graph_open(struct inode *inode, struct file *file)
1397 {
1398 int ret = 0;
1399
1400 if (unlikely(ftrace_disabled))
1401 return -ENODEV;
1402
1403 mutex_lock(&graph_lock);
1404 if ((file->f_mode & FMODE_WRITE) &&
1405 !(file->f_flags & O_APPEND)) {
1406 ftrace_graph_count = 0;
1407 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1408 }
1409
1410 if (file->f_mode & FMODE_READ) {
1411 ret = seq_open(file, &ftrace_graph_seq_ops);
1412 if (!ret) {
1413 struct seq_file *m = file->private_data;
1414 m->private = ftrace_graph_funcs;
1415 }
1416 } else
1417 file->private_data = ftrace_graph_funcs;
1418 mutex_unlock(&graph_lock);
1419
1420 return ret;
1421 }
1422
1423 static ssize_t
1424 ftrace_graph_read(struct file *file, char __user *ubuf,
1425 size_t cnt, loff_t *ppos)
1426 {
1427 if (file->f_mode & FMODE_READ)
1428 return seq_read(file, ubuf, cnt, ppos);
1429 else
1430 return -EPERM;
1431 }
1432
1433 static int
1434 ftrace_set_func(unsigned long *array, int idx, char *buffer)
1435 {
1436 char str[KSYM_SYMBOL_LEN];
1437 struct dyn_ftrace *rec;
1438 struct ftrace_page *pg;
1439 int found = 0;
1440 int i, j;
1441
1442 if (ftrace_disabled)
1443 return -ENODEV;
1444
1445 /* should not be called from interrupt context */
1446 spin_lock(&ftrace_lock);
1447
1448 for (pg = ftrace_pages_start; pg; pg = pg->next) {
1449 for (i = 0; i < pg->index; i++) {
1450 rec = &pg->records[i];
1451
1452 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
1453 continue;
1454
1455 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1456 if (strcmp(str, buffer) == 0) {
1457 found = 1;
1458 for (j = 0; j < idx; j++)
1459 if (array[j] == rec->ip) {
1460 found = 0;
1461 break;
1462 }
1463 if (found)
1464 array[idx] = rec->ip;
1465 break;
1466 }
1467 }
1468 }
1469 spin_unlock(&ftrace_lock);
1470
1471 return found ? 0 : -EINVAL;
1472 }
1473
1474 static ssize_t
1475 ftrace_graph_write(struct file *file, const char __user *ubuf,
1476 size_t cnt, loff_t *ppos)
1477 {
1478 unsigned char buffer[FTRACE_BUFF_MAX+1];
1479 unsigned long *array;
1480 size_t read = 0;
1481 ssize_t ret;
1482 int index = 0;
1483 char ch;
1484
1485 if (!cnt || cnt < 0)
1486 return 0;
1487
1488 mutex_lock(&graph_lock);
1489
1490 if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
1491 ret = -EBUSY;
1492 goto out;
1493 }
1494
1495 if (file->f_mode & FMODE_READ) {
1496 struct seq_file *m = file->private_data;
1497 array = m->private;
1498 } else
1499 array = file->private_data;
1500
1501 ret = get_user(ch, ubuf++);
1502 if (ret)
1503 goto out;
1504 read++;
1505 cnt--;
1506
1507 /* skip white space */
1508 while (cnt && isspace(ch)) {
1509 ret = get_user(ch, ubuf++);
1510 if (ret)
1511 goto out;
1512 read++;
1513 cnt--;
1514 }
1515
1516 if (isspace(ch)) {
1517 *ppos += read;
1518 ret = read;
1519 goto out;
1520 }
1521
1522 while (cnt && !isspace(ch)) {
1523 if (index < FTRACE_BUFF_MAX)
1524 buffer[index++] = ch;
1525 else {
1526 ret = -EINVAL;
1527 goto out;
1528 }
1529 ret = get_user(ch, ubuf++);
1530 if (ret)
1531 goto out;
1532 read++;
1533 cnt--;
1534 }
1535 buffer[index] = 0;
1536
1537 /* we allow only one at a time */
1538 ret = ftrace_set_func(array, ftrace_graph_count, buffer);
1539 if (ret)
1540 goto out;
1541
1542 ftrace_graph_count++;
1543
1544 file->f_pos += read;
1545
1546 ret = read;
1547 out:
1548 mutex_unlock(&graph_lock);
1549
1550 return ret;
1551 }
1552
1553 static const struct file_operations ftrace_graph_fops = {
1554 .open = ftrace_graph_open,
1555 .read = ftrace_graph_read,
1556 .write = ftrace_graph_write,
1557 };
1558 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1559
1560 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
1561 {
1562 struct dentry *entry;
1563
1564 entry = debugfs_create_file("available_filter_functions", 0444,
1565 d_tracer, NULL, &ftrace_avail_fops);
1566 if (!entry)
1567 pr_warning("Could not create debugfs "
1568 "'available_filter_functions' entry\n");
1569
1570 entry = debugfs_create_file("failures", 0444,
1571 d_tracer, NULL, &ftrace_failures_fops);
1572 if (!entry)
1573 pr_warning("Could not create debugfs 'failures' entry\n");
1574
1575 entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1576 NULL, &ftrace_filter_fops);
1577 if (!entry)
1578 pr_warning("Could not create debugfs "
1579 "'set_ftrace_filter' entry\n");
1580
1581 entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1582 NULL, &ftrace_notrace_fops);
1583 if (!entry)
1584 pr_warning("Could not create debugfs "
1585 "'set_ftrace_notrace' entry\n");
1586
1587 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1588 entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
1589 NULL,
1590 &ftrace_graph_fops);
1591 if (!entry)
1592 pr_warning("Could not create debugfs "
1593 "'set_graph_function' entry\n");
1594 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1595
1596 return 0;
1597 }
1598
1599 static int ftrace_convert_nops(struct module *mod,
1600 unsigned long *start,
1601 unsigned long *end)
1602 {
1603 unsigned long *p;
1604 unsigned long addr;
1605 unsigned long flags;
1606
1607 mutex_lock(&ftrace_start_lock);
1608 p = start;
1609 while (p < end) {
1610 addr = ftrace_call_adjust(*p++);
1611 /*
1612 * Some architecture linkers will pad between
1613 * the different mcount_loc sections of different
1614 * object files to satisfy alignments.
1615 * Skip any NULL pointers.
1616 */
1617 if (!addr)
1618 continue;
1619 ftrace_record_ip(addr);
1620 }
1621
1622 /* disable interrupts to prevent kstop machine */
1623 local_irq_save(flags);
1624 ftrace_update_code(mod);
1625 local_irq_restore(flags);
1626 mutex_unlock(&ftrace_start_lock);
1627
1628 return 0;
1629 }
1630
1631 void ftrace_init_module(struct module *mod,
1632 unsigned long *start, unsigned long *end)
1633 {
1634 if (ftrace_disabled || start == end)
1635 return;
1636 ftrace_convert_nops(mod, start, end);
1637 }
1638
1639 extern unsigned long __start_mcount_loc[];
1640 extern unsigned long __stop_mcount_loc[];
1641
1642 void __init ftrace_init(void)
1643 {
1644 unsigned long count, addr, flags;
1645 int ret;
1646
1647 /* Keep the ftrace pointer to the stub */
1648 addr = (unsigned long)ftrace_stub;
1649
1650 local_irq_save(flags);
1651 ftrace_dyn_arch_init(&addr);
1652 local_irq_restore(flags);
1653
1654 /* ftrace_dyn_arch_init places the return code in addr */
1655 if (addr)
1656 goto failed;
1657
1658 count = __stop_mcount_loc - __start_mcount_loc;
1659
1660 ret = ftrace_dyn_table_alloc(count);
1661 if (ret)
1662 goto failed;
1663
1664 last_ftrace_enabled = ftrace_enabled = 1;
1665
1666 ret = ftrace_convert_nops(NULL,
1667 __start_mcount_loc,
1668 __stop_mcount_loc);
1669
1670 return;
1671 failed:
1672 ftrace_disabled = 1;
1673 }
1674
1675 #else
1676
1677 static int __init ftrace_nodyn_init(void)
1678 {
1679 ftrace_enabled = 1;
1680 return 0;
1681 }
1682 device_initcall(ftrace_nodyn_init);
1683
1684 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
1685 static inline void ftrace_startup_enable(int command) { }
1686 /* Keep as macros so we do not need to define the commands */
1687 # define ftrace_startup(command) do { } while (0)
1688 # define ftrace_shutdown(command) do { } while (0)
1689 # define ftrace_startup_sysctl() do { } while (0)
1690 # define ftrace_shutdown_sysctl() do { } while (0)
1691 #endif /* CONFIG_DYNAMIC_FTRACE */
1692
1693 static ssize_t
1694 ftrace_pid_read(struct file *file, char __user *ubuf,
1695 size_t cnt, loff_t *ppos)
1696 {
1697 char buf[64];
1698 int r;
1699
1700 if (ftrace_pid_trace == ftrace_swapper_pid)
1701 r = sprintf(buf, "swapper tasks\n");
1702 else if (ftrace_pid_trace)
1703 r = sprintf(buf, "%u\n", pid_nr(ftrace_pid_trace));
1704 else
1705 r = sprintf(buf, "no pid\n");
1706
1707 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1708 }
1709
1710 static void clear_ftrace_swapper(void)
1711 {
1712 struct task_struct *p;
1713 int cpu;
1714
1715 get_online_cpus();
1716 for_each_online_cpu(cpu) {
1717 p = idle_task(cpu);
1718 clear_tsk_trace_trace(p);
1719 }
1720 put_online_cpus();
1721 }
1722
1723 static void set_ftrace_swapper(void)
1724 {
1725 struct task_struct *p;
1726 int cpu;
1727
1728 get_online_cpus();
1729 for_each_online_cpu(cpu) {
1730 p = idle_task(cpu);
1731 set_tsk_trace_trace(p);
1732 }
1733 put_online_cpus();
1734 }
1735
1736 static void clear_ftrace_pid(struct pid *pid)
1737 {
1738 struct task_struct *p;
1739
1740 rcu_read_lock();
1741 do_each_pid_task(pid, PIDTYPE_PID, p) {
1742 clear_tsk_trace_trace(p);
1743 } while_each_pid_task(pid, PIDTYPE_PID, p);
1744 rcu_read_unlock();
1745
1746 put_pid(pid);
1747 }
1748
1749 static void set_ftrace_pid(struct pid *pid)
1750 {
1751 struct task_struct *p;
1752
1753 rcu_read_lock();
1754 do_each_pid_task(pid, PIDTYPE_PID, p) {
1755 set_tsk_trace_trace(p);
1756 } while_each_pid_task(pid, PIDTYPE_PID, p);
1757 rcu_read_unlock();
1758 }
1759
1760 static void clear_ftrace_pid_task(struct pid **pid)
1761 {
1762 if (*pid == ftrace_swapper_pid)
1763 clear_ftrace_swapper();
1764 else
1765 clear_ftrace_pid(*pid);
1766
1767 *pid = NULL;
1768 }
1769
1770 static void set_ftrace_pid_task(struct pid *pid)
1771 {
1772 if (pid == ftrace_swapper_pid)
1773 set_ftrace_swapper();
1774 else
1775 set_ftrace_pid(pid);
1776 }
1777
1778 static ssize_t
1779 ftrace_pid_write(struct file *filp, const char __user *ubuf,
1780 size_t cnt, loff_t *ppos)
1781 {
1782 struct pid *pid;
1783 char buf[64];
1784 long val;
1785 int ret;
1786
1787 if (cnt >= sizeof(buf))
1788 return -EINVAL;
1789
1790 if (copy_from_user(&buf, ubuf, cnt))
1791 return -EFAULT;
1792
1793 buf[cnt] = 0;
1794
1795 ret = strict_strtol(buf, 10, &val);
1796 if (ret < 0)
1797 return ret;
1798
1799 mutex_lock(&ftrace_start_lock);
1800 if (val < 0) {
1801 /* disable pid tracing */
1802 if (!ftrace_pid_trace)
1803 goto out;
1804
1805 clear_ftrace_pid_task(&ftrace_pid_trace);
1806
1807 } else {
1808 /* swapper task is special */
1809 if (!val) {
1810 pid = ftrace_swapper_pid;
1811 if (pid == ftrace_pid_trace)
1812 goto out;
1813 } else {
1814 pid = find_get_pid(val);
1815
1816 if (pid == ftrace_pid_trace) {
1817 put_pid(pid);
1818 goto out;
1819 }
1820 }
1821
1822 if (ftrace_pid_trace)
1823 clear_ftrace_pid_task(&ftrace_pid_trace);
1824
1825 if (!pid)
1826 goto out;
1827
1828 ftrace_pid_trace = pid;
1829
1830 set_ftrace_pid_task(ftrace_pid_trace);
1831 }
1832
1833 /* update the function call */
1834 ftrace_update_pid_func();
1835 ftrace_startup_enable(0);
1836
1837 out:
1838 mutex_unlock(&ftrace_start_lock);
1839
1840 return cnt;
1841 }
1842
1843 static struct file_operations ftrace_pid_fops = {
1844 .read = ftrace_pid_read,
1845 .write = ftrace_pid_write,
1846 };
1847
1848 static __init int ftrace_init_debugfs(void)
1849 {
1850 struct dentry *d_tracer;
1851 struct dentry *entry;
1852
1853 d_tracer = tracing_init_dentry();
1854 if (!d_tracer)
1855 return 0;
1856
1857 ftrace_init_dyn_debugfs(d_tracer);
1858
1859 entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
1860 NULL, &ftrace_pid_fops);
1861 if (!entry)
1862 pr_warning("Could not create debugfs "
1863 "'set_ftrace_pid' entry\n");
1864 return 0;
1865 }
1866
1867 fs_initcall(ftrace_init_debugfs);
1868
1869 /**
1870 * ftrace_kill - kill ftrace
1871 *
1872 * This function should be used by panic code. It stops ftrace
1873 * but in a not so nice way. If you need to simply kill ftrace
1874 * from a non-atomic section, use ftrace_kill.
1875 */
1876 void ftrace_kill(void)
1877 {
1878 ftrace_disabled = 1;
1879 ftrace_enabled = 0;
1880 clear_ftrace_function();
1881 }
1882
1883 /**
1884 * register_ftrace_function - register a function for profiling
1885 * @ops - ops structure that holds the function for profiling.
1886 *
1887 * Register a function to be called by all functions in the
1888 * kernel.
1889 *
1890 * Note: @ops->func and all the functions it calls must be labeled
1891 * with "notrace", otherwise it will go into a
1892 * recursive loop.
1893 */
1894 int register_ftrace_function(struct ftrace_ops *ops)
1895 {
1896 int ret;
1897
1898 if (unlikely(ftrace_disabled))
1899 return -1;
1900
1901 mutex_lock(&ftrace_sysctl_lock);
1902
1903 ret = __register_ftrace_function(ops);
1904 ftrace_startup(0);
1905
1906 mutex_unlock(&ftrace_sysctl_lock);
1907 return ret;
1908 }
1909
1910 /**
1911 * unregister_ftrace_function - unresgister a function for profiling.
1912 * @ops - ops structure that holds the function to unregister
1913 *
1914 * Unregister a function that was added to be called by ftrace profiling.
1915 */
1916 int unregister_ftrace_function(struct ftrace_ops *ops)
1917 {
1918 int ret;
1919
1920 mutex_lock(&ftrace_sysctl_lock);
1921 ret = __unregister_ftrace_function(ops);
1922 ftrace_shutdown(0);
1923 mutex_unlock(&ftrace_sysctl_lock);
1924
1925 return ret;
1926 }
1927
1928 int
1929 ftrace_enable_sysctl(struct ctl_table *table, int write,
1930 struct file *file, void __user *buffer, size_t *lenp,
1931 loff_t *ppos)
1932 {
1933 int ret;
1934
1935 if (unlikely(ftrace_disabled))
1936 return -ENODEV;
1937
1938 mutex_lock(&ftrace_sysctl_lock);
1939
1940 ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
1941
1942 if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1943 goto out;
1944
1945 last_ftrace_enabled = ftrace_enabled;
1946
1947 if (ftrace_enabled) {
1948
1949 ftrace_startup_sysctl();
1950
1951 /* we are starting ftrace again */
1952 if (ftrace_list != &ftrace_list_end) {
1953 if (ftrace_list->next == &ftrace_list_end)
1954 ftrace_trace_function = ftrace_list->func;
1955 else
1956 ftrace_trace_function = ftrace_list_func;
1957 }
1958
1959 } else {
1960 /* stopping ftrace calls (just send to ftrace_stub) */
1961 ftrace_trace_function = ftrace_stub;
1962
1963 ftrace_shutdown_sysctl();
1964 }
1965
1966 out:
1967 mutex_unlock(&ftrace_sysctl_lock);
1968 return ret;
1969 }
1970
1971 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1972
1973 static atomic_t ftrace_graph_active;
1974 static struct notifier_block ftrace_suspend_notifier;
1975
1976 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
1977 {
1978 return 0;
1979 }
1980
1981 /* The callbacks that hook a function */
1982 trace_func_graph_ret_t ftrace_graph_return =
1983 (trace_func_graph_ret_t)ftrace_stub;
1984 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
1985
1986 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
1987 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
1988 {
1989 int i;
1990 int ret = 0;
1991 unsigned long flags;
1992 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
1993 struct task_struct *g, *t;
1994
1995 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
1996 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
1997 * sizeof(struct ftrace_ret_stack),
1998 GFP_KERNEL);
1999 if (!ret_stack_list[i]) {
2000 start = 0;
2001 end = i;
2002 ret = -ENOMEM;
2003 goto free;
2004 }
2005 }
2006
2007 read_lock_irqsave(&tasklist_lock, flags);
2008 do_each_thread(g, t) {
2009 if (start == end) {
2010 ret = -EAGAIN;
2011 goto unlock;
2012 }
2013
2014 if (t->ret_stack == NULL) {
2015 t->curr_ret_stack = -1;
2016 /* Make sure IRQs see the -1 first: */
2017 barrier();
2018 t->ret_stack = ret_stack_list[start++];
2019 atomic_set(&t->tracing_graph_pause, 0);
2020 atomic_set(&t->trace_overrun, 0);
2021 }
2022 } while_each_thread(g, t);
2023
2024 unlock:
2025 read_unlock_irqrestore(&tasklist_lock, flags);
2026 free:
2027 for (i = start; i < end; i++)
2028 kfree(ret_stack_list[i]);
2029 return ret;
2030 }
2031
2032 /* Allocate a return stack for each task */
2033 static int start_graph_tracing(void)
2034 {
2035 struct ftrace_ret_stack **ret_stack_list;
2036 int ret, cpu;
2037
2038 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2039 sizeof(struct ftrace_ret_stack *),
2040 GFP_KERNEL);
2041
2042 if (!ret_stack_list)
2043 return -ENOMEM;
2044
2045 /* The cpu_boot init_task->ret_stack will never be freed */
2046 for_each_online_cpu(cpu)
2047 ftrace_graph_init_task(idle_task(cpu));
2048
2049 do {
2050 ret = alloc_retstack_tasklist(ret_stack_list);
2051 } while (ret == -EAGAIN);
2052
2053 kfree(ret_stack_list);
2054 return ret;
2055 }
2056
2057 /*
2058 * Hibernation protection.
2059 * The state of the current task is too much unstable during
2060 * suspend/restore to disk. We want to protect against that.
2061 */
2062 static int
2063 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
2064 void *unused)
2065 {
2066 switch (state) {
2067 case PM_HIBERNATION_PREPARE:
2068 pause_graph_tracing();
2069 break;
2070
2071 case PM_POST_HIBERNATION:
2072 unpause_graph_tracing();
2073 break;
2074 }
2075 return NOTIFY_DONE;
2076 }
2077
2078 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2079 trace_func_graph_ent_t entryfunc)
2080 {
2081 int ret = 0;
2082
2083 mutex_lock(&ftrace_sysctl_lock);
2084
2085 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2086 register_pm_notifier(&ftrace_suspend_notifier);
2087
2088 atomic_inc(&ftrace_graph_active);
2089 ret = start_graph_tracing();
2090 if (ret) {
2091 atomic_dec(&ftrace_graph_active);
2092 goto out;
2093 }
2094
2095 ftrace_graph_return = retfunc;
2096 ftrace_graph_entry = entryfunc;
2097
2098 ftrace_startup(FTRACE_START_FUNC_RET);
2099
2100 out:
2101 mutex_unlock(&ftrace_sysctl_lock);
2102 return ret;
2103 }
2104
2105 void unregister_ftrace_graph(void)
2106 {
2107 mutex_lock(&ftrace_sysctl_lock);
2108
2109 atomic_dec(&ftrace_graph_active);
2110 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2111 ftrace_graph_entry = ftrace_graph_entry_stub;
2112 ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2113 unregister_pm_notifier(&ftrace_suspend_notifier);
2114
2115 mutex_unlock(&ftrace_sysctl_lock);
2116 }
2117
2118 /* Allocate a return stack for newly created task */
2119 void ftrace_graph_init_task(struct task_struct *t)
2120 {
2121 if (atomic_read(&ftrace_graph_active)) {
2122 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2123 * sizeof(struct ftrace_ret_stack),
2124 GFP_KERNEL);
2125 if (!t->ret_stack)
2126 return;
2127 t->curr_ret_stack = -1;
2128 atomic_set(&t->tracing_graph_pause, 0);
2129 atomic_set(&t->trace_overrun, 0);
2130 } else
2131 t->ret_stack = NULL;
2132 }
2133
2134 void ftrace_graph_exit_task(struct task_struct *t)
2135 {
2136 struct ftrace_ret_stack *ret_stack = t->ret_stack;
2137
2138 t->ret_stack = NULL;
2139 /* NULL must become visible to IRQs before we free it: */
2140 barrier();
2141
2142 kfree(ret_stack);
2143 }
2144
2145 void ftrace_graph_stop(void)
2146 {
2147 ftrace_stop();
2148 }
2149 #endif
2150
Linux kernel - Deliverables
This page took 0.07185 seconds and 6 git commands to generate.