2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded
;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running
;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled
;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt
[] = {
71 static struct tracer_flags dummy_tracer_flags
= {
73 .opts
= dummy_tracer_opt
77 dummy_set_flag(struct trace_array
*tr
, u32 old_flags
, u32 bit
, int set
)
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
87 static DEFINE_PER_CPU(bool, trace_cmdline_save
);
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
95 static int tracing_disabled
= 1;
97 DEFINE_PER_CPU(int, ftrace_cpu_disabled
);
99 cpumask_var_t __read_mostly tracing_buffer_mask
;
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 enum ftrace_dump_mode ftrace_dump_on_oops
;
119 /* When set, tracing will stop when a WARN*() is hit */
120 int __disable_trace_on_warning
;
122 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
);
124 #define MAX_TRACER_SIZE 100
125 static char bootup_tracer_buf
[MAX_TRACER_SIZE
] __initdata
;
126 static char *default_bootup_tracer
;
128 static bool allocate_snapshot
;
130 static int __init
set_cmdline_ftrace(char *str
)
132 strlcpy(bootup_tracer_buf
, str
, MAX_TRACER_SIZE
);
133 default_bootup_tracer
= bootup_tracer_buf
;
134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded
= true;
138 __setup("ftrace=", set_cmdline_ftrace
);
140 static int __init
set_ftrace_dump_on_oops(char *str
)
142 if (*str
++ != '=' || !*str
) {
143 ftrace_dump_on_oops
= DUMP_ALL
;
147 if (!strcmp("orig_cpu", str
)) {
148 ftrace_dump_on_oops
= DUMP_ORIG
;
154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops
);
156 static int __init
stop_trace_on_warning(char *str
)
158 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
159 __disable_trace_on_warning
= 1;
162 __setup("traceoff_on_warning", stop_trace_on_warning
);
164 static int __init
boot_alloc_snapshot(char *str
)
166 allocate_snapshot
= true;
167 /* We also need the main ring buffer expanded */
168 ring_buffer_expanded
= true;
171 __setup("alloc_snapshot", boot_alloc_snapshot
);
174 static char trace_boot_options_buf
[MAX_TRACER_SIZE
] __initdata
;
175 static char *trace_boot_options __initdata
;
177 static int __init
set_trace_boot_options(char *str
)
179 strlcpy(trace_boot_options_buf
, str
, MAX_TRACER_SIZE
);
180 trace_boot_options
= trace_boot_options_buf
;
183 __setup("trace_options=", set_trace_boot_options
);
185 static char trace_boot_clock_buf
[MAX_TRACER_SIZE
] __initdata
;
186 static char *trace_boot_clock __initdata
;
188 static int __init
set_trace_boot_clock(char *str
)
190 strlcpy(trace_boot_clock_buf
, str
, MAX_TRACER_SIZE
);
191 trace_boot_clock
= trace_boot_clock_buf
;
194 __setup("trace_clock=", set_trace_boot_clock
);
197 unsigned long long ns2usecs(cycle_t nsec
)
205 * The global_trace is the descriptor that holds the tracing
206 * buffers for the live tracing. For each CPU, it contains
207 * a link list of pages that will store trace entries. The
208 * page descriptor of the pages in the memory is used to hold
209 * the link list by linking the lru item in the page descriptor
210 * to each of the pages in the buffer per CPU.
212 * For each active CPU there is a data field that holds the
213 * pages for the buffer for that CPU. Each CPU has the same number
214 * of pages allocated for its buffer.
216 static struct trace_array global_trace
;
218 LIST_HEAD(ftrace_trace_arrays
);
220 int trace_array_get(struct trace_array
*this_tr
)
222 struct trace_array
*tr
;
225 mutex_lock(&trace_types_lock
);
226 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
233 mutex_unlock(&trace_types_lock
);
238 static void __trace_array_put(struct trace_array
*this_tr
)
240 WARN_ON(!this_tr
->ref
);
244 void trace_array_put(struct trace_array
*this_tr
)
246 mutex_lock(&trace_types_lock
);
247 __trace_array_put(this_tr
);
248 mutex_unlock(&trace_types_lock
);
251 int filter_check_discard(struct ftrace_event_file
*file
, void *rec
,
252 struct ring_buffer
*buffer
,
253 struct ring_buffer_event
*event
)
255 if (unlikely(file
->flags
& FTRACE_EVENT_FL_FILTERED
) &&
256 !filter_match_preds(file
->filter
, rec
)) {
257 ring_buffer_discard_commit(buffer
, event
);
263 EXPORT_SYMBOL_GPL(filter_check_discard
);
265 int call_filter_check_discard(struct ftrace_event_call
*call
, void *rec
,
266 struct ring_buffer
*buffer
,
267 struct ring_buffer_event
*event
)
269 if (unlikely(call
->flags
& TRACE_EVENT_FL_FILTERED
) &&
270 !filter_match_preds(call
->filter
, rec
)) {
271 ring_buffer_discard_commit(buffer
, event
);
277 EXPORT_SYMBOL_GPL(call_filter_check_discard
);
279 static cycle_t
buffer_ftrace_now(struct trace_buffer
*buf
, int cpu
)
283 /* Early boot up does not have a buffer yet */
285 return trace_clock_local();
287 ts
= ring_buffer_time_stamp(buf
->buffer
, cpu
);
288 ring_buffer_normalize_time_stamp(buf
->buffer
, cpu
, &ts
);
293 cycle_t
ftrace_now(int cpu
)
295 return buffer_ftrace_now(&global_trace
.trace_buffer
, cpu
);
299 * tracing_is_enabled - Show if global_trace has been disabled
301 * Shows if the global trace has been enabled or not. It uses the
302 * mirror flag "buffer_disabled" to be used in fast paths such as for
303 * the irqsoff tracer. But it may be inaccurate due to races. If you
304 * need to know the accurate state, use tracing_is_on() which is a little
305 * slower, but accurate.
307 int tracing_is_enabled(void)
310 * For quick access (irqsoff uses this in fast path), just
311 * return the mirror variable of the state of the ring buffer.
312 * It's a little racy, but we don't really care.
315 return !global_trace
.buffer_disabled
;
319 * trace_buf_size is the size in bytes that is allocated
320 * for a buffer. Note, the number of bytes is always rounded
323 * This number is purposely set to a low number of 16384.
324 * If the dump on oops happens, it will be much appreciated
325 * to not have to wait for all that output. Anyway this can be
326 * boot time and run time configurable.
328 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
330 static unsigned long trace_buf_size
= TRACE_BUF_SIZE_DEFAULT
;
332 /* trace_types holds a link list of available tracers. */
333 static struct tracer
*trace_types __read_mostly
;
336 * trace_types_lock is used to protect the trace_types list.
338 DEFINE_MUTEX(trace_types_lock
);
341 * serialize the access of the ring buffer
343 * ring buffer serializes readers, but it is low level protection.
344 * The validity of the events (which returns by ring_buffer_peek() ..etc)
345 * are not protected by ring buffer.
347 * The content of events may become garbage if we allow other process consumes
348 * these events concurrently:
349 * A) the page of the consumed events may become a normal page
350 * (not reader page) in ring buffer, and this page will be rewrited
351 * by events producer.
352 * B) The page of the consumed events may become a page for splice_read,
353 * and this page will be returned to system.
355 * These primitives allow multi process access to different cpu ring buffer
358 * These primitives don't distinguish read-only and read-consume access.
359 * Multi read-only access are also serialized.
363 static DECLARE_RWSEM(all_cpu_access_lock
);
364 static DEFINE_PER_CPU(struct mutex
, cpu_access_lock
);
366 static inline void trace_access_lock(int cpu
)
368 if (cpu
== RING_BUFFER_ALL_CPUS
) {
369 /* gain it for accessing the whole ring buffer. */
370 down_write(&all_cpu_access_lock
);
372 /* gain it for accessing a cpu ring buffer. */
374 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
375 down_read(&all_cpu_access_lock
);
377 /* Secondly block other access to this @cpu ring buffer. */
378 mutex_lock(&per_cpu(cpu_access_lock
, cpu
));
382 static inline void trace_access_unlock(int cpu
)
384 if (cpu
== RING_BUFFER_ALL_CPUS
) {
385 up_write(&all_cpu_access_lock
);
387 mutex_unlock(&per_cpu(cpu_access_lock
, cpu
));
388 up_read(&all_cpu_access_lock
);
392 static inline void trace_access_lock_init(void)
396 for_each_possible_cpu(cpu
)
397 mutex_init(&per_cpu(cpu_access_lock
, cpu
));
402 static DEFINE_MUTEX(access_lock
);
404 static inline void trace_access_lock(int cpu
)
407 mutex_lock(&access_lock
);
410 static inline void trace_access_unlock(int cpu
)
413 mutex_unlock(&access_lock
);
416 static inline void trace_access_lock_init(void)
422 /* trace_flags holds trace_options default values */
423 unsigned long trace_flags
= TRACE_ITER_PRINT_PARENT
| TRACE_ITER_PRINTK
|
424 TRACE_ITER_ANNOTATE
| TRACE_ITER_CONTEXT_INFO
| TRACE_ITER_SLEEP_TIME
|
425 TRACE_ITER_GRAPH_TIME
| TRACE_ITER_RECORD_CMD
| TRACE_ITER_OVERWRITE
|
426 TRACE_ITER_IRQ_INFO
| TRACE_ITER_MARKERS
| TRACE_ITER_FUNCTION
;
428 static void tracer_tracing_on(struct trace_array
*tr
)
430 if (tr
->trace_buffer
.buffer
)
431 ring_buffer_record_on(tr
->trace_buffer
.buffer
);
433 * This flag is looked at when buffers haven't been allocated
434 * yet, or by some tracers (like irqsoff), that just want to
435 * know if the ring buffer has been disabled, but it can handle
436 * races of where it gets disabled but we still do a record.
437 * As the check is in the fast path of the tracers, it is more
438 * important to be fast than accurate.
440 tr
->buffer_disabled
= 0;
441 /* Make the flag seen by readers */
446 * tracing_on - enable tracing buffers
448 * This function enables tracing buffers that may have been
449 * disabled with tracing_off.
451 void tracing_on(void)
453 tracer_tracing_on(&global_trace
);
455 EXPORT_SYMBOL_GPL(tracing_on
);
458 * __trace_puts - write a constant string into the trace buffer.
459 * @ip: The address of the caller
460 * @str: The constant string to write
461 * @size: The size of the string.
463 int __trace_puts(unsigned long ip
, const char *str
, int size
)
465 struct ring_buffer_event
*event
;
466 struct ring_buffer
*buffer
;
467 struct print_entry
*entry
;
468 unsigned long irq_flags
;
472 if (!(trace_flags
& TRACE_ITER_PRINTK
))
475 pc
= preempt_count();
477 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
480 alloc
= sizeof(*entry
) + size
+ 2; /* possible \n added */
482 local_save_flags(irq_flags
);
483 buffer
= global_trace
.trace_buffer
.buffer
;
484 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, alloc
,
489 entry
= ring_buffer_event_data(event
);
492 memcpy(&entry
->buf
, str
, size
);
494 /* Add a newline if necessary */
495 if (entry
->buf
[size
- 1] != '\n') {
496 entry
->buf
[size
] = '\n';
497 entry
->buf
[size
+ 1] = '\0';
499 entry
->buf
[size
] = '\0';
501 __buffer_unlock_commit(buffer
, event
);
502 ftrace_trace_stack(buffer
, irq_flags
, 4, pc
);
506 EXPORT_SYMBOL_GPL(__trace_puts
);
509 * __trace_bputs - write the pointer to a constant string into trace buffer
510 * @ip: The address of the caller
511 * @str: The constant string to write to the buffer to
513 int __trace_bputs(unsigned long ip
, const char *str
)
515 struct ring_buffer_event
*event
;
516 struct ring_buffer
*buffer
;
517 struct bputs_entry
*entry
;
518 unsigned long irq_flags
;
519 int size
= sizeof(struct bputs_entry
);
522 if (!(trace_flags
& TRACE_ITER_PRINTK
))
525 pc
= preempt_count();
527 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
530 local_save_flags(irq_flags
);
531 buffer
= global_trace
.trace_buffer
.buffer
;
532 event
= trace_buffer_lock_reserve(buffer
, TRACE_BPUTS
, size
,
537 entry
= ring_buffer_event_data(event
);
541 __buffer_unlock_commit(buffer
, event
);
542 ftrace_trace_stack(buffer
, irq_flags
, 4, pc
);
546 EXPORT_SYMBOL_GPL(__trace_bputs
);
548 #ifdef CONFIG_TRACER_SNAPSHOT
550 * trace_snapshot - take a snapshot of the current buffer.
552 * This causes a swap between the snapshot buffer and the current live
553 * tracing buffer. You can use this to take snapshots of the live
554 * trace when some condition is triggered, but continue to trace.
556 * Note, make sure to allocate the snapshot with either
557 * a tracing_snapshot_alloc(), or by doing it manually
558 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
560 * If the snapshot buffer is not allocated, it will stop tracing.
561 * Basically making a permanent snapshot.
563 void tracing_snapshot(void)
565 struct trace_array
*tr
= &global_trace
;
566 struct tracer
*tracer
= tr
->current_trace
;
570 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
571 internal_trace_puts("*** snapshot is being ignored ***\n");
575 if (!tr
->allocated_snapshot
) {
576 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
577 internal_trace_puts("*** stopping trace here! ***\n");
582 /* Note, snapshot can not be used when the tracer uses it */
583 if (tracer
->use_max_tr
) {
584 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
585 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
589 local_irq_save(flags
);
590 update_max_tr(tr
, current
, smp_processor_id());
591 local_irq_restore(flags
);
593 EXPORT_SYMBOL_GPL(tracing_snapshot
);
595 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
596 struct trace_buffer
*size_buf
, int cpu_id
);
597 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
);
599 static int alloc_snapshot(struct trace_array
*tr
)
603 if (!tr
->allocated_snapshot
) {
605 /* allocate spare buffer */
606 ret
= resize_buffer_duplicate_size(&tr
->max_buffer
,
607 &tr
->trace_buffer
, RING_BUFFER_ALL_CPUS
);
611 tr
->allocated_snapshot
= true;
617 static void free_snapshot(struct trace_array
*tr
)
620 * We don't free the ring buffer. instead, resize it because
621 * The max_tr ring buffer has some state (e.g. ring->clock) and
622 * we want preserve it.
624 ring_buffer_resize(tr
->max_buffer
.buffer
, 1, RING_BUFFER_ALL_CPUS
);
625 set_buffer_entries(&tr
->max_buffer
, 1);
626 tracing_reset_online_cpus(&tr
->max_buffer
);
627 tr
->allocated_snapshot
= false;
631 * tracing_alloc_snapshot - allocate snapshot buffer.
633 * This only allocates the snapshot buffer if it isn't already
634 * allocated - it doesn't also take a snapshot.
636 * This is meant to be used in cases where the snapshot buffer needs
637 * to be set up for events that can't sleep but need to be able to
638 * trigger a snapshot.
640 int tracing_alloc_snapshot(void)
642 struct trace_array
*tr
= &global_trace
;
645 ret
= alloc_snapshot(tr
);
650 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
653 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
655 * This is similar to trace_snapshot(), but it will allocate the
656 * snapshot buffer if it isn't already allocated. Use this only
657 * where it is safe to sleep, as the allocation may sleep.
659 * This causes a swap between the snapshot buffer and the current live
660 * tracing buffer. You can use this to take snapshots of the live
661 * trace when some condition is triggered, but continue to trace.
663 void tracing_snapshot_alloc(void)
667 ret
= tracing_alloc_snapshot();
673 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
675 void tracing_snapshot(void)
677 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
679 EXPORT_SYMBOL_GPL(tracing_snapshot
);
680 int tracing_alloc_snapshot(void)
682 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
685 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
686 void tracing_snapshot_alloc(void)
691 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
692 #endif /* CONFIG_TRACER_SNAPSHOT */
694 static void tracer_tracing_off(struct trace_array
*tr
)
696 if (tr
->trace_buffer
.buffer
)
697 ring_buffer_record_off(tr
->trace_buffer
.buffer
);
699 * This flag is looked at when buffers haven't been allocated
700 * yet, or by some tracers (like irqsoff), that just want to
701 * know if the ring buffer has been disabled, but it can handle
702 * races of where it gets disabled but we still do a record.
703 * As the check is in the fast path of the tracers, it is more
704 * important to be fast than accurate.
706 tr
->buffer_disabled
= 1;
707 /* Make the flag seen by readers */
712 * tracing_off - turn off tracing buffers
714 * This function stops the tracing buffers from recording data.
715 * It does not disable any overhead the tracers themselves may
716 * be causing. This function simply causes all recording to
717 * the ring buffers to fail.
719 void tracing_off(void)
721 tracer_tracing_off(&global_trace
);
723 EXPORT_SYMBOL_GPL(tracing_off
);
725 void disable_trace_on_warning(void)
727 if (__disable_trace_on_warning
)
732 * tracer_tracing_is_on - show real state of ring buffer enabled
733 * @tr : the trace array to know if ring buffer is enabled
735 * Shows real state of the ring buffer if it is enabled or not.
737 static int tracer_tracing_is_on(struct trace_array
*tr
)
739 if (tr
->trace_buffer
.buffer
)
740 return ring_buffer_record_is_on(tr
->trace_buffer
.buffer
);
741 return !tr
->buffer_disabled
;
745 * tracing_is_on - show state of ring buffers enabled
747 int tracing_is_on(void)
749 return tracer_tracing_is_on(&global_trace
);
751 EXPORT_SYMBOL_GPL(tracing_is_on
);
753 static int __init
set_buf_size(char *str
)
755 unsigned long buf_size
;
759 buf_size
= memparse(str
, &str
);
760 /* nr_entries can not be zero */
763 trace_buf_size
= buf_size
;
766 __setup("trace_buf_size=", set_buf_size
);
768 static int __init
set_tracing_thresh(char *str
)
770 unsigned long threshold
;
775 ret
= kstrtoul(str
, 0, &threshold
);
778 tracing_thresh
= threshold
* 1000;
781 __setup("tracing_thresh=", set_tracing_thresh
);
783 unsigned long nsecs_to_usecs(unsigned long nsecs
)
788 /* These must match the bit postions in trace_iterator_flags */
789 static const char *trace_options
[] = {
822 int in_ns
; /* is this clock in nanoseconds? */
824 { trace_clock_local
, "local", 1 },
825 { trace_clock_global
, "global", 1 },
826 { trace_clock_counter
, "counter", 0 },
827 { trace_clock_jiffies
, "uptime", 0 },
828 { trace_clock
, "perf", 1 },
829 { ktime_get_mono_fast_ns
, "mono", 1 },
834 * trace_parser_get_init - gets the buffer for trace parser
836 int trace_parser_get_init(struct trace_parser
*parser
, int size
)
838 memset(parser
, 0, sizeof(*parser
));
840 parser
->buffer
= kmalloc(size
, GFP_KERNEL
);
849 * trace_parser_put - frees the buffer for trace parser
851 void trace_parser_put(struct trace_parser
*parser
)
853 kfree(parser
->buffer
);
857 * trace_get_user - reads the user input string separated by space
858 * (matched by isspace(ch))
860 * For each string found the 'struct trace_parser' is updated,
861 * and the function returns.
863 * Returns number of bytes read.
865 * See kernel/trace/trace.h for 'struct trace_parser' details.
867 int trace_get_user(struct trace_parser
*parser
, const char __user
*ubuf
,
868 size_t cnt
, loff_t
*ppos
)
875 trace_parser_clear(parser
);
877 ret
= get_user(ch
, ubuf
++);
885 * The parser is not finished with the last write,
886 * continue reading the user input without skipping spaces.
889 /* skip white space */
890 while (cnt
&& isspace(ch
)) {
891 ret
= get_user(ch
, ubuf
++);
898 /* only spaces were written */
908 /* read the non-space input */
909 while (cnt
&& !isspace(ch
)) {
910 if (parser
->idx
< parser
->size
- 1)
911 parser
->buffer
[parser
->idx
++] = ch
;
916 ret
= get_user(ch
, ubuf
++);
923 /* We either got finished input or we have to wait for another call. */
925 parser
->buffer
[parser
->idx
] = 0;
926 parser
->cont
= false;
927 } else if (parser
->idx
< parser
->size
- 1) {
929 parser
->buffer
[parser
->idx
++] = ch
;
942 static ssize_t
trace_seq_to_buffer(struct trace_seq
*s
, void *buf
, size_t cnt
)
946 if (s
->len
<= s
->readpos
)
949 len
= s
->len
- s
->readpos
;
952 memcpy(buf
, s
->buffer
+ s
->readpos
, cnt
);
958 unsigned long __read_mostly tracing_thresh
;
960 #ifdef CONFIG_TRACER_MAX_TRACE
962 * Copy the new maximum trace into the separate maximum-trace
963 * structure. (this way the maximum trace is permanently saved,
964 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
967 __update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
969 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
970 struct trace_buffer
*max_buf
= &tr
->max_buffer
;
971 struct trace_array_cpu
*data
= per_cpu_ptr(trace_buf
->data
, cpu
);
972 struct trace_array_cpu
*max_data
= per_cpu_ptr(max_buf
->data
, cpu
);
975 max_buf
->time_start
= data
->preempt_timestamp
;
977 max_data
->saved_latency
= tr
->max_latency
;
978 max_data
->critical_start
= data
->critical_start
;
979 max_data
->critical_end
= data
->critical_end
;
981 memcpy(max_data
->comm
, tsk
->comm
, TASK_COMM_LEN
);
982 max_data
->pid
= tsk
->pid
;
984 * If tsk == current, then use current_uid(), as that does not use
985 * RCU. The irq tracer can be called out of RCU scope.
988 max_data
->uid
= current_uid();
990 max_data
->uid
= task_uid(tsk
);
992 max_data
->nice
= tsk
->static_prio
- 20 - MAX_RT_PRIO
;
993 max_data
->policy
= tsk
->policy
;
994 max_data
->rt_priority
= tsk
->rt_priority
;
996 /* record this tasks comm */
997 tracing_record_cmdline(tsk
);
1001 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1003 * @tsk: the task with the latency
1004 * @cpu: The cpu that initiated the trace.
1006 * Flip the buffers between the @tr and the max_tr and record information
1007 * about which task was the cause of this latency.
1010 update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1012 struct ring_buffer
*buf
;
1017 WARN_ON_ONCE(!irqs_disabled());
1019 if (!tr
->allocated_snapshot
) {
1020 /* Only the nop tracer should hit this when disabling */
1021 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1025 arch_spin_lock(&tr
->max_lock
);
1027 buf
= tr
->trace_buffer
.buffer
;
1028 tr
->trace_buffer
.buffer
= tr
->max_buffer
.buffer
;
1029 tr
->max_buffer
.buffer
= buf
;
1031 __update_max_tr(tr
, tsk
, cpu
);
1032 arch_spin_unlock(&tr
->max_lock
);
1036 * update_max_tr_single - only copy one trace over, and reset the rest
1038 * @tsk - task with the latency
1039 * @cpu - the cpu of the buffer to copy.
1041 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1044 update_max_tr_single(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1051 WARN_ON_ONCE(!irqs_disabled());
1052 if (!tr
->allocated_snapshot
) {
1053 /* Only the nop tracer should hit this when disabling */
1054 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1058 arch_spin_lock(&tr
->max_lock
);
1060 ret
= ring_buffer_swap_cpu(tr
->max_buffer
.buffer
, tr
->trace_buffer
.buffer
, cpu
);
1062 if (ret
== -EBUSY
) {
1064 * We failed to swap the buffer due to a commit taking
1065 * place on this CPU. We fail to record, but we reset
1066 * the max trace buffer (no one writes directly to it)
1067 * and flag that it failed.
1069 trace_array_printk_buf(tr
->max_buffer
.buffer
, _THIS_IP_
,
1070 "Failed to swap buffers due to commit in progress\n");
1073 WARN_ON_ONCE(ret
&& ret
!= -EAGAIN
&& ret
!= -EBUSY
);
1075 __update_max_tr(tr
, tsk
, cpu
);
1076 arch_spin_unlock(&tr
->max_lock
);
1078 #endif /* CONFIG_TRACER_MAX_TRACE */
1080 static int wait_on_pipe(struct trace_iterator
*iter
)
1082 /* Iterators are static, they should be filled or empty */
1083 if (trace_buffer_iter(iter
, iter
->cpu_file
))
1086 return ring_buffer_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
1089 #ifdef CONFIG_FTRACE_STARTUP_TEST
1090 static int run_tracer_selftest(struct tracer
*type
)
1092 struct trace_array
*tr
= &global_trace
;
1093 struct tracer
*saved_tracer
= tr
->current_trace
;
1096 if (!type
->selftest
|| tracing_selftest_disabled
)
1100 * Run a selftest on this tracer.
1101 * Here we reset the trace buffer, and set the current
1102 * tracer to be this tracer. The tracer can then run some
1103 * internal tracing to verify that everything is in order.
1104 * If we fail, we do not register this tracer.
1106 tracing_reset_online_cpus(&tr
->trace_buffer
);
1108 tr
->current_trace
= type
;
1110 #ifdef CONFIG_TRACER_MAX_TRACE
1111 if (type
->use_max_tr
) {
1112 /* If we expanded the buffers, make sure the max is expanded too */
1113 if (ring_buffer_expanded
)
1114 ring_buffer_resize(tr
->max_buffer
.buffer
, trace_buf_size
,
1115 RING_BUFFER_ALL_CPUS
);
1116 tr
->allocated_snapshot
= true;
1120 /* the test is responsible for initializing and enabling */
1121 pr_info("Testing tracer %s: ", type
->name
);
1122 ret
= type
->selftest(type
, tr
);
1123 /* the test is responsible for resetting too */
1124 tr
->current_trace
= saved_tracer
;
1126 printk(KERN_CONT
"FAILED!\n");
1127 /* Add the warning after printing 'FAILED' */
1131 /* Only reset on passing, to avoid touching corrupted buffers */
1132 tracing_reset_online_cpus(&tr
->trace_buffer
);
1134 #ifdef CONFIG_TRACER_MAX_TRACE
1135 if (type
->use_max_tr
) {
1136 tr
->allocated_snapshot
= false;
1138 /* Shrink the max buffer again */
1139 if (ring_buffer_expanded
)
1140 ring_buffer_resize(tr
->max_buffer
.buffer
, 1,
1141 RING_BUFFER_ALL_CPUS
);
1145 printk(KERN_CONT
"PASSED\n");
1149 static inline int run_tracer_selftest(struct tracer
*type
)
1153 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1156 * register_tracer - register a tracer with the ftrace system.
1157 * @type - the plugin for the tracer
1159 * Register a new plugin tracer.
1161 int register_tracer(struct tracer
*type
)
1167 pr_info("Tracer must have a name\n");
1171 if (strlen(type
->name
) >= MAX_TRACER_SIZE
) {
1172 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE
);
1176 mutex_lock(&trace_types_lock
);
1178 tracing_selftest_running
= true;
1180 for (t
= trace_types
; t
; t
= t
->next
) {
1181 if (strcmp(type
->name
, t
->name
) == 0) {
1183 pr_info("Tracer %s already registered\n",
1190 if (!type
->set_flag
)
1191 type
->set_flag
= &dummy_set_flag
;
1193 type
->flags
= &dummy_tracer_flags
;
1195 if (!type
->flags
->opts
)
1196 type
->flags
->opts
= dummy_tracer_opt
;
1198 ret
= run_tracer_selftest(type
);
1202 type
->next
= trace_types
;
1206 tracing_selftest_running
= false;
1207 mutex_unlock(&trace_types_lock
);
1209 if (ret
|| !default_bootup_tracer
)
1212 if (strncmp(default_bootup_tracer
, type
->name
, MAX_TRACER_SIZE
))
1215 printk(KERN_INFO
"Starting tracer '%s'\n", type
->name
);
1216 /* Do we want this tracer to start on bootup? */
1217 tracing_set_tracer(&global_trace
, type
->name
);
1218 default_bootup_tracer
= NULL
;
1219 /* disable other selftests, since this will break it. */
1220 tracing_selftest_disabled
= true;
1221 #ifdef CONFIG_FTRACE_STARTUP_TEST
1222 printk(KERN_INFO
"Disabling FTRACE selftests due to running tracer '%s'\n",
1230 void tracing_reset(struct trace_buffer
*buf
, int cpu
)
1232 struct ring_buffer
*buffer
= buf
->buffer
;
1237 ring_buffer_record_disable(buffer
);
1239 /* Make sure all commits have finished */
1240 synchronize_sched();
1241 ring_buffer_reset_cpu(buffer
, cpu
);
1243 ring_buffer_record_enable(buffer
);
1246 void tracing_reset_online_cpus(struct trace_buffer
*buf
)
1248 struct ring_buffer
*buffer
= buf
->buffer
;
1254 ring_buffer_record_disable(buffer
);
1256 /* Make sure all commits have finished */
1257 synchronize_sched();
1259 buf
->time_start
= buffer_ftrace_now(buf
, buf
->cpu
);
1261 for_each_online_cpu(cpu
)
1262 ring_buffer_reset_cpu(buffer
, cpu
);
1264 ring_buffer_record_enable(buffer
);
1267 /* Must have trace_types_lock held */
1268 void tracing_reset_all_online_cpus(void)
1270 struct trace_array
*tr
;
1272 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
1273 tracing_reset_online_cpus(&tr
->trace_buffer
);
1274 #ifdef CONFIG_TRACER_MAX_TRACE
1275 tracing_reset_online_cpus(&tr
->max_buffer
);
1280 #define SAVED_CMDLINES_DEFAULT 128
1281 #define NO_CMDLINE_MAP UINT_MAX
1282 static arch_spinlock_t trace_cmdline_lock
= __ARCH_SPIN_LOCK_UNLOCKED
;
1283 struct saved_cmdlines_buffer
{
1284 unsigned map_pid_to_cmdline
[PID_MAX_DEFAULT
+1];
1285 unsigned *map_cmdline_to_pid
;
1286 unsigned cmdline_num
;
1288 char *saved_cmdlines
;
1290 static struct saved_cmdlines_buffer
*savedcmd
;
1292 /* temporary disable recording */
1293 static atomic_t trace_record_cmdline_disabled __read_mostly
;
1295 static inline char *get_saved_cmdlines(int idx
)
1297 return &savedcmd
->saved_cmdlines
[idx
* TASK_COMM_LEN
];
1300 static inline void set_cmdline(int idx
, const char *cmdline
)
1302 memcpy(get_saved_cmdlines(idx
), cmdline
, TASK_COMM_LEN
);
1305 static int allocate_cmdlines_buffer(unsigned int val
,
1306 struct saved_cmdlines_buffer
*s
)
1308 s
->map_cmdline_to_pid
= kmalloc(val
* sizeof(*s
->map_cmdline_to_pid
),
1310 if (!s
->map_cmdline_to_pid
)
1313 s
->saved_cmdlines
= kmalloc(val
* TASK_COMM_LEN
, GFP_KERNEL
);
1314 if (!s
->saved_cmdlines
) {
1315 kfree(s
->map_cmdline_to_pid
);
1320 s
->cmdline_num
= val
;
1321 memset(&s
->map_pid_to_cmdline
, NO_CMDLINE_MAP
,
1322 sizeof(s
->map_pid_to_cmdline
));
1323 memset(s
->map_cmdline_to_pid
, NO_CMDLINE_MAP
,
1324 val
* sizeof(*s
->map_cmdline_to_pid
));
1329 static int trace_create_savedcmd(void)
1333 savedcmd
= kmalloc(sizeof(*savedcmd
), GFP_KERNEL
);
1337 ret
= allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT
, savedcmd
);
1347 int is_tracing_stopped(void)
1349 return global_trace
.stop_count
;
1353 * tracing_start - quick start of the tracer
1355 * If tracing is enabled but was stopped by tracing_stop,
1356 * this will start the tracer back up.
1358 void tracing_start(void)
1360 struct ring_buffer
*buffer
;
1361 unsigned long flags
;
1363 if (tracing_disabled
)
1366 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1367 if (--global_trace
.stop_count
) {
1368 if (global_trace
.stop_count
< 0) {
1369 /* Someone screwed up their debugging */
1371 global_trace
.stop_count
= 0;
1376 /* Prevent the buffers from switching */
1377 arch_spin_lock(&global_trace
.max_lock
);
1379 buffer
= global_trace
.trace_buffer
.buffer
;
1381 ring_buffer_record_enable(buffer
);
1383 #ifdef CONFIG_TRACER_MAX_TRACE
1384 buffer
= global_trace
.max_buffer
.buffer
;
1386 ring_buffer_record_enable(buffer
);
1389 arch_spin_unlock(&global_trace
.max_lock
);
1392 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1395 static void tracing_start_tr(struct trace_array
*tr
)
1397 struct ring_buffer
*buffer
;
1398 unsigned long flags
;
1400 if (tracing_disabled
)
1403 /* If global, we need to also start the max tracer */
1404 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1405 return tracing_start();
1407 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1409 if (--tr
->stop_count
) {
1410 if (tr
->stop_count
< 0) {
1411 /* Someone screwed up their debugging */
1418 buffer
= tr
->trace_buffer
.buffer
;
1420 ring_buffer_record_enable(buffer
);
1423 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1427 * tracing_stop - quick stop of the tracer
1429 * Light weight way to stop tracing. Use in conjunction with
1432 void tracing_stop(void)
1434 struct ring_buffer
*buffer
;
1435 unsigned long flags
;
1437 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1438 if (global_trace
.stop_count
++)
1441 /* Prevent the buffers from switching */
1442 arch_spin_lock(&global_trace
.max_lock
);
1444 buffer
= global_trace
.trace_buffer
.buffer
;
1446 ring_buffer_record_disable(buffer
);
1448 #ifdef CONFIG_TRACER_MAX_TRACE
1449 buffer
= global_trace
.max_buffer
.buffer
;
1451 ring_buffer_record_disable(buffer
);
1454 arch_spin_unlock(&global_trace
.max_lock
);
1457 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1460 static void tracing_stop_tr(struct trace_array
*tr
)
1462 struct ring_buffer
*buffer
;
1463 unsigned long flags
;
1465 /* If global, we need to also stop the max tracer */
1466 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1467 return tracing_stop();
1469 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1470 if (tr
->stop_count
++)
1473 buffer
= tr
->trace_buffer
.buffer
;
1475 ring_buffer_record_disable(buffer
);
1478 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1481 void trace_stop_cmdline_recording(void);
1483 static int trace_save_cmdline(struct task_struct
*tsk
)
1487 if (!tsk
->pid
|| unlikely(tsk
->pid
> PID_MAX_DEFAULT
))
1491 * It's not the end of the world if we don't get
1492 * the lock, but we also don't want to spin
1493 * nor do we want to disable interrupts,
1494 * so if we miss here, then better luck next time.
1496 if (!arch_spin_trylock(&trace_cmdline_lock
))
1499 idx
= savedcmd
->map_pid_to_cmdline
[tsk
->pid
];
1500 if (idx
== NO_CMDLINE_MAP
) {
1501 idx
= (savedcmd
->cmdline_idx
+ 1) % savedcmd
->cmdline_num
;
1504 * Check whether the cmdline buffer at idx has a pid
1505 * mapped. We are going to overwrite that entry so we
1506 * need to clear the map_pid_to_cmdline. Otherwise we
1507 * would read the new comm for the old pid.
1509 pid
= savedcmd
->map_cmdline_to_pid
[idx
];
1510 if (pid
!= NO_CMDLINE_MAP
)
1511 savedcmd
->map_pid_to_cmdline
[pid
] = NO_CMDLINE_MAP
;
1513 savedcmd
->map_cmdline_to_pid
[idx
] = tsk
->pid
;
1514 savedcmd
->map_pid_to_cmdline
[tsk
->pid
] = idx
;
1516 savedcmd
->cmdline_idx
= idx
;
1519 set_cmdline(idx
, tsk
->comm
);
1521 arch_spin_unlock(&trace_cmdline_lock
);
1526 static void __trace_find_cmdline(int pid
, char comm
[])
1531 strcpy(comm
, "<idle>");
1535 if (WARN_ON_ONCE(pid
< 0)) {
1536 strcpy(comm
, "<XXX>");
1540 if (pid
> PID_MAX_DEFAULT
) {
1541 strcpy(comm
, "<...>");
1545 map
= savedcmd
->map_pid_to_cmdline
[pid
];
1546 if (map
!= NO_CMDLINE_MAP
)
1547 strcpy(comm
, get_saved_cmdlines(map
));
1549 strcpy(comm
, "<...>");
1552 void trace_find_cmdline(int pid
, char comm
[])
1555 arch_spin_lock(&trace_cmdline_lock
);
1557 __trace_find_cmdline(pid
, comm
);
1559 arch_spin_unlock(&trace_cmdline_lock
);
1563 void tracing_record_cmdline(struct task_struct
*tsk
)
1565 if (atomic_read(&trace_record_cmdline_disabled
) || !tracing_is_on())
1568 if (!__this_cpu_read(trace_cmdline_save
))
1571 if (trace_save_cmdline(tsk
))
1572 __this_cpu_write(trace_cmdline_save
, false);
1576 tracing_generic_entry_update(struct trace_entry
*entry
, unsigned long flags
,
1579 struct task_struct
*tsk
= current
;
1581 entry
->preempt_count
= pc
& 0xff;
1582 entry
->pid
= (tsk
) ? tsk
->pid
: 0;
1584 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1585 (irqs_disabled_flags(flags
) ? TRACE_FLAG_IRQS_OFF
: 0) |
1587 TRACE_FLAG_IRQS_NOSUPPORT
|
1589 ((pc
& HARDIRQ_MASK
) ? TRACE_FLAG_HARDIRQ
: 0) |
1590 ((pc
& SOFTIRQ_MASK
) ? TRACE_FLAG_SOFTIRQ
: 0) |
1591 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED
: 0) |
1592 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED
: 0);
1594 EXPORT_SYMBOL_GPL(tracing_generic_entry_update
);
1596 struct ring_buffer_event
*
1597 trace_buffer_lock_reserve(struct ring_buffer
*buffer
,
1600 unsigned long flags
, int pc
)
1602 struct ring_buffer_event
*event
;
1604 event
= ring_buffer_lock_reserve(buffer
, len
);
1605 if (event
!= NULL
) {
1606 struct trace_entry
*ent
= ring_buffer_event_data(event
);
1608 tracing_generic_entry_update(ent
, flags
, pc
);
1616 __buffer_unlock_commit(struct ring_buffer
*buffer
, struct ring_buffer_event
*event
)
1618 __this_cpu_write(trace_cmdline_save
, true);
1619 ring_buffer_unlock_commit(buffer
, event
);
1623 __trace_buffer_unlock_commit(struct ring_buffer
*buffer
,
1624 struct ring_buffer_event
*event
,
1625 unsigned long flags
, int pc
)
1627 __buffer_unlock_commit(buffer
, event
);
1629 ftrace_trace_stack(buffer
, flags
, 6, pc
);
1630 ftrace_trace_userstack(buffer
, flags
, pc
);
1633 void trace_buffer_unlock_commit(struct ring_buffer
*buffer
,
1634 struct ring_buffer_event
*event
,
1635 unsigned long flags
, int pc
)
1637 __trace_buffer_unlock_commit(buffer
, event
, flags
, pc
);
1639 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit
);
1641 static struct ring_buffer
*temp_buffer
;
1643 struct ring_buffer_event
*
1644 trace_event_buffer_lock_reserve(struct ring_buffer
**current_rb
,
1645 struct ftrace_event_file
*ftrace_file
,
1646 int type
, unsigned long len
,
1647 unsigned long flags
, int pc
)
1649 struct ring_buffer_event
*entry
;
1651 *current_rb
= ftrace_file
->tr
->trace_buffer
.buffer
;
1652 entry
= trace_buffer_lock_reserve(*current_rb
,
1653 type
, len
, flags
, pc
);
1655 * If tracing is off, but we have triggers enabled
1656 * we still need to look at the event data. Use the temp_buffer
1657 * to store the trace event for the tigger to use. It's recusive
1658 * safe and will not be recorded anywhere.
1660 if (!entry
&& ftrace_file
->flags
& FTRACE_EVENT_FL_TRIGGER_COND
) {
1661 *current_rb
= temp_buffer
;
1662 entry
= trace_buffer_lock_reserve(*current_rb
,
1663 type
, len
, flags
, pc
);
1667 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve
);
1669 struct ring_buffer_event
*
1670 trace_current_buffer_lock_reserve(struct ring_buffer
**current_rb
,
1671 int type
, unsigned long len
,
1672 unsigned long flags
, int pc
)
1674 *current_rb
= global_trace
.trace_buffer
.buffer
;
1675 return trace_buffer_lock_reserve(*current_rb
,
1676 type
, len
, flags
, pc
);
1678 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve
);
1680 void trace_current_buffer_unlock_commit(struct ring_buffer
*buffer
,
1681 struct ring_buffer_event
*event
,
1682 unsigned long flags
, int pc
)
1684 __trace_buffer_unlock_commit(buffer
, event
, flags
, pc
);
1686 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit
);
1688 void trace_buffer_unlock_commit_regs(struct ring_buffer
*buffer
,
1689 struct ring_buffer_event
*event
,
1690 unsigned long flags
, int pc
,
1691 struct pt_regs
*regs
)
1693 __buffer_unlock_commit(buffer
, event
);
1695 ftrace_trace_stack_regs(buffer
, flags
, 0, pc
, regs
);
1696 ftrace_trace_userstack(buffer
, flags
, pc
);
1698 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs
);
1700 void trace_current_buffer_discard_commit(struct ring_buffer
*buffer
,
1701 struct ring_buffer_event
*event
)
1703 ring_buffer_discard_commit(buffer
, event
);
1705 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit
);
1708 trace_function(struct trace_array
*tr
,
1709 unsigned long ip
, unsigned long parent_ip
, unsigned long flags
,
1712 struct ftrace_event_call
*call
= &event_function
;
1713 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
1714 struct ring_buffer_event
*event
;
1715 struct ftrace_entry
*entry
;
1717 /* If we are reading the ring buffer, don't trace */
1718 if (unlikely(__this_cpu_read(ftrace_cpu_disabled
)))
1721 event
= trace_buffer_lock_reserve(buffer
, TRACE_FN
, sizeof(*entry
),
1725 entry
= ring_buffer_event_data(event
);
1727 entry
->parent_ip
= parent_ip
;
1729 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1730 __buffer_unlock_commit(buffer
, event
);
1733 #ifdef CONFIG_STACKTRACE
1735 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1736 struct ftrace_stack
{
1737 unsigned long calls
[FTRACE_STACK_MAX_ENTRIES
];
1740 static DEFINE_PER_CPU(struct ftrace_stack
, ftrace_stack
);
1741 static DEFINE_PER_CPU(int, ftrace_stack_reserve
);
1743 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
1744 unsigned long flags
,
1745 int skip
, int pc
, struct pt_regs
*regs
)
1747 struct ftrace_event_call
*call
= &event_kernel_stack
;
1748 struct ring_buffer_event
*event
;
1749 struct stack_entry
*entry
;
1750 struct stack_trace trace
;
1752 int size
= FTRACE_STACK_ENTRIES
;
1754 trace
.nr_entries
= 0;
1758 * Since events can happen in NMIs there's no safe way to
1759 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1760 * or NMI comes in, it will just have to use the default
1761 * FTRACE_STACK_SIZE.
1763 preempt_disable_notrace();
1765 use_stack
= __this_cpu_inc_return(ftrace_stack_reserve
);
1767 * We don't need any atomic variables, just a barrier.
1768 * If an interrupt comes in, we don't care, because it would
1769 * have exited and put the counter back to what we want.
1770 * We just need a barrier to keep gcc from moving things
1774 if (use_stack
== 1) {
1775 trace
.entries
= this_cpu_ptr(ftrace_stack
.calls
);
1776 trace
.max_entries
= FTRACE_STACK_MAX_ENTRIES
;
1779 save_stack_trace_regs(regs
, &trace
);
1781 save_stack_trace(&trace
);
1783 if (trace
.nr_entries
> size
)
1784 size
= trace
.nr_entries
;
1786 /* From now on, use_stack is a boolean */
1789 size
*= sizeof(unsigned long);
1791 event
= trace_buffer_lock_reserve(buffer
, TRACE_STACK
,
1792 sizeof(*entry
) + size
, flags
, pc
);
1795 entry
= ring_buffer_event_data(event
);
1797 memset(&entry
->caller
, 0, size
);
1800 memcpy(&entry
->caller
, trace
.entries
,
1801 trace
.nr_entries
* sizeof(unsigned long));
1803 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
1804 trace
.entries
= entry
->caller
;
1806 save_stack_trace_regs(regs
, &trace
);
1808 save_stack_trace(&trace
);
1811 entry
->size
= trace
.nr_entries
;
1813 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1814 __buffer_unlock_commit(buffer
, event
);
1817 /* Again, don't let gcc optimize things here */
1819 __this_cpu_dec(ftrace_stack_reserve
);
1820 preempt_enable_notrace();
1824 void ftrace_trace_stack_regs(struct ring_buffer
*buffer
, unsigned long flags
,
1825 int skip
, int pc
, struct pt_regs
*regs
)
1827 if (!(trace_flags
& TRACE_ITER_STACKTRACE
))
1830 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, regs
);
1833 void ftrace_trace_stack(struct ring_buffer
*buffer
, unsigned long flags
,
1836 if (!(trace_flags
& TRACE_ITER_STACKTRACE
))
1839 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, NULL
);
1842 void __trace_stack(struct trace_array
*tr
, unsigned long flags
, int skip
,
1845 __ftrace_trace_stack(tr
->trace_buffer
.buffer
, flags
, skip
, pc
, NULL
);
1849 * trace_dump_stack - record a stack back trace in the trace buffer
1850 * @skip: Number of functions to skip (helper handlers)
1852 void trace_dump_stack(int skip
)
1854 unsigned long flags
;
1856 if (tracing_disabled
|| tracing_selftest_running
)
1859 local_save_flags(flags
);
1862 * Skip 3 more, seems to get us at the caller of
1866 __ftrace_trace_stack(global_trace
.trace_buffer
.buffer
,
1867 flags
, skip
, preempt_count(), NULL
);
1870 static DEFINE_PER_CPU(int, user_stack_count
);
1873 ftrace_trace_userstack(struct ring_buffer
*buffer
, unsigned long flags
, int pc
)
1875 struct ftrace_event_call
*call
= &event_user_stack
;
1876 struct ring_buffer_event
*event
;
1877 struct userstack_entry
*entry
;
1878 struct stack_trace trace
;
1880 if (!(trace_flags
& TRACE_ITER_USERSTACKTRACE
))
1884 * NMIs can not handle page faults, even with fix ups.
1885 * The save user stack can (and often does) fault.
1887 if (unlikely(in_nmi()))
1891 * prevent recursion, since the user stack tracing may
1892 * trigger other kernel events.
1895 if (__this_cpu_read(user_stack_count
))
1898 __this_cpu_inc(user_stack_count
);
1900 event
= trace_buffer_lock_reserve(buffer
, TRACE_USER_STACK
,
1901 sizeof(*entry
), flags
, pc
);
1903 goto out_drop_count
;
1904 entry
= ring_buffer_event_data(event
);
1906 entry
->tgid
= current
->tgid
;
1907 memset(&entry
->caller
, 0, sizeof(entry
->caller
));
1909 trace
.nr_entries
= 0;
1910 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
1912 trace
.entries
= entry
->caller
;
1914 save_stack_trace_user(&trace
);
1915 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1916 __buffer_unlock_commit(buffer
, event
);
1919 __this_cpu_dec(user_stack_count
);
1925 static void __trace_userstack(struct trace_array
*tr
, unsigned long flags
)
1927 ftrace_trace_userstack(tr
, flags
, preempt_count());
1931 #endif /* CONFIG_STACKTRACE */
1933 /* created for use with alloc_percpu */
1934 struct trace_buffer_struct
{
1935 char buffer
[TRACE_BUF_SIZE
];
1938 static struct trace_buffer_struct
*trace_percpu_buffer
;
1939 static struct trace_buffer_struct
*trace_percpu_sirq_buffer
;
1940 static struct trace_buffer_struct
*trace_percpu_irq_buffer
;
1941 static struct trace_buffer_struct
*trace_percpu_nmi_buffer
;
1944 * The buffer used is dependent on the context. There is a per cpu
1945 * buffer for normal context, softirq contex, hard irq context and
1946 * for NMI context. Thise allows for lockless recording.
1948 * Note, if the buffers failed to be allocated, then this returns NULL
1950 static char *get_trace_buf(void)
1952 struct trace_buffer_struct
*percpu_buffer
;
1955 * If we have allocated per cpu buffers, then we do not
1956 * need to do any locking.
1959 percpu_buffer
= trace_percpu_nmi_buffer
;
1961 percpu_buffer
= trace_percpu_irq_buffer
;
1962 else if (in_softirq())
1963 percpu_buffer
= trace_percpu_sirq_buffer
;
1965 percpu_buffer
= trace_percpu_buffer
;
1970 return this_cpu_ptr(&percpu_buffer
->buffer
[0]);
1973 static int alloc_percpu_trace_buffer(void)
1975 struct trace_buffer_struct
*buffers
;
1976 struct trace_buffer_struct
*sirq_buffers
;
1977 struct trace_buffer_struct
*irq_buffers
;
1978 struct trace_buffer_struct
*nmi_buffers
;
1980 buffers
= alloc_percpu(struct trace_buffer_struct
);
1984 sirq_buffers
= alloc_percpu(struct trace_buffer_struct
);
1988 irq_buffers
= alloc_percpu(struct trace_buffer_struct
);
1992 nmi_buffers
= alloc_percpu(struct trace_buffer_struct
);
1996 trace_percpu_buffer
= buffers
;
1997 trace_percpu_sirq_buffer
= sirq_buffers
;
1998 trace_percpu_irq_buffer
= irq_buffers
;
1999 trace_percpu_nmi_buffer
= nmi_buffers
;
2004 free_percpu(irq_buffers
);
2006 free_percpu(sirq_buffers
);
2008 free_percpu(buffers
);
2010 WARN(1, "Could not allocate percpu trace_printk buffer");
2014 static int buffers_allocated
;
2016 void trace_printk_init_buffers(void)
2018 if (buffers_allocated
)
2021 if (alloc_percpu_trace_buffer())
2024 /* trace_printk() is for debug use only. Don't use it in production. */
2026 pr_warning("\n**********************************************************\n");
2027 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2028 pr_warning("** **\n");
2029 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2030 pr_warning("** **\n");
2031 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2032 pr_warning("** unsafe for produciton use. **\n");
2033 pr_warning("** **\n");
2034 pr_warning("** If you see this message and you are not debugging **\n");
2035 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2036 pr_warning("** **\n");
2037 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2038 pr_warning("**********************************************************\n");
2040 /* Expand the buffers to set size */
2041 tracing_update_buffers();
2043 buffers_allocated
= 1;
2046 * trace_printk_init_buffers() can be called by modules.
2047 * If that happens, then we need to start cmdline recording
2048 * directly here. If the global_trace.buffer is already
2049 * allocated here, then this was called by module code.
2051 if (global_trace
.trace_buffer
.buffer
)
2052 tracing_start_cmdline_record();
2055 void trace_printk_start_comm(void)
2057 /* Start tracing comms if trace printk is set */
2058 if (!buffers_allocated
)
2060 tracing_start_cmdline_record();
2063 static void trace_printk_start_stop_comm(int enabled
)
2065 if (!buffers_allocated
)
2069 tracing_start_cmdline_record();
2071 tracing_stop_cmdline_record();
2075 * trace_vbprintk - write binary msg to tracing buffer
2078 int trace_vbprintk(unsigned long ip
, const char *fmt
, va_list args
)
2080 struct ftrace_event_call
*call
= &event_bprint
;
2081 struct ring_buffer_event
*event
;
2082 struct ring_buffer
*buffer
;
2083 struct trace_array
*tr
= &global_trace
;
2084 struct bprint_entry
*entry
;
2085 unsigned long flags
;
2087 int len
= 0, size
, pc
;
2089 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
2092 /* Don't pollute graph traces with trace_vprintk internals */
2093 pause_graph_tracing();
2095 pc
= preempt_count();
2096 preempt_disable_notrace();
2098 tbuffer
= get_trace_buf();
2104 len
= vbin_printf((u32
*)tbuffer
, TRACE_BUF_SIZE
/sizeof(int), fmt
, args
);
2106 if (len
> TRACE_BUF_SIZE
/sizeof(int) || len
< 0)
2109 local_save_flags(flags
);
2110 size
= sizeof(*entry
) + sizeof(u32
) * len
;
2111 buffer
= tr
->trace_buffer
.buffer
;
2112 event
= trace_buffer_lock_reserve(buffer
, TRACE_BPRINT
, size
,
2116 entry
= ring_buffer_event_data(event
);
2120 memcpy(entry
->buf
, tbuffer
, sizeof(u32
) * len
);
2121 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2122 __buffer_unlock_commit(buffer
, event
);
2123 ftrace_trace_stack(buffer
, flags
, 6, pc
);
2127 preempt_enable_notrace();
2128 unpause_graph_tracing();
2132 EXPORT_SYMBOL_GPL(trace_vbprintk
);
2135 __trace_array_vprintk(struct ring_buffer
*buffer
,
2136 unsigned long ip
, const char *fmt
, va_list args
)
2138 struct ftrace_event_call
*call
= &event_print
;
2139 struct ring_buffer_event
*event
;
2140 int len
= 0, size
, pc
;
2141 struct print_entry
*entry
;
2142 unsigned long flags
;
2145 if (tracing_disabled
|| tracing_selftest_running
)
2148 /* Don't pollute graph traces with trace_vprintk internals */
2149 pause_graph_tracing();
2151 pc
= preempt_count();
2152 preempt_disable_notrace();
2155 tbuffer
= get_trace_buf();
2161 len
= vscnprintf(tbuffer
, TRACE_BUF_SIZE
, fmt
, args
);
2163 local_save_flags(flags
);
2164 size
= sizeof(*entry
) + len
+ 1;
2165 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
2169 entry
= ring_buffer_event_data(event
);
2172 memcpy(&entry
->buf
, tbuffer
, len
+ 1);
2173 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2174 __buffer_unlock_commit(buffer
, event
);
2175 ftrace_trace_stack(buffer
, flags
, 6, pc
);
2178 preempt_enable_notrace();
2179 unpause_graph_tracing();
2184 int trace_array_vprintk(struct trace_array
*tr
,
2185 unsigned long ip
, const char *fmt
, va_list args
)
2187 return __trace_array_vprintk(tr
->trace_buffer
.buffer
, ip
, fmt
, args
);
2190 int trace_array_printk(struct trace_array
*tr
,
2191 unsigned long ip
, const char *fmt
, ...)
2196 if (!(trace_flags
& TRACE_ITER_PRINTK
))
2200 ret
= trace_array_vprintk(tr
, ip
, fmt
, ap
);
2205 int trace_array_printk_buf(struct ring_buffer
*buffer
,
2206 unsigned long ip
, const char *fmt
, ...)
2211 if (!(trace_flags
& TRACE_ITER_PRINTK
))
2215 ret
= __trace_array_vprintk(buffer
, ip
, fmt
, ap
);
2220 int trace_vprintk(unsigned long ip
, const char *fmt
, va_list args
)
2222 return trace_array_vprintk(&global_trace
, ip
, fmt
, args
);
2224 EXPORT_SYMBOL_GPL(trace_vprintk
);
2226 static void trace_iterator_increment(struct trace_iterator
*iter
)
2228 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, iter
->cpu
);
2232 ring_buffer_read(buf_iter
, NULL
);
2235 static struct trace_entry
*
2236 peek_next_entry(struct trace_iterator
*iter
, int cpu
, u64
*ts
,
2237 unsigned long *lost_events
)
2239 struct ring_buffer_event
*event
;
2240 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, cpu
);
2243 event
= ring_buffer_iter_peek(buf_iter
, ts
);
2245 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, cpu
, ts
,
2249 iter
->ent_size
= ring_buffer_event_length(event
);
2250 return ring_buffer_event_data(event
);
2256 static struct trace_entry
*
2257 __find_next_entry(struct trace_iterator
*iter
, int *ent_cpu
,
2258 unsigned long *missing_events
, u64
*ent_ts
)
2260 struct ring_buffer
*buffer
= iter
->trace_buffer
->buffer
;
2261 struct trace_entry
*ent
, *next
= NULL
;
2262 unsigned long lost_events
= 0, next_lost
= 0;
2263 int cpu_file
= iter
->cpu_file
;
2264 u64 next_ts
= 0, ts
;
2270 * If we are in a per_cpu trace file, don't bother by iterating over
2271 * all cpu and peek directly.
2273 if (cpu_file
> RING_BUFFER_ALL_CPUS
) {
2274 if (ring_buffer_empty_cpu(buffer
, cpu_file
))
2276 ent
= peek_next_entry(iter
, cpu_file
, ent_ts
, missing_events
);
2278 *ent_cpu
= cpu_file
;
2283 for_each_tracing_cpu(cpu
) {
2285 if (ring_buffer_empty_cpu(buffer
, cpu
))
2288 ent
= peek_next_entry(iter
, cpu
, &ts
, &lost_events
);
2291 * Pick the entry with the smallest timestamp:
2293 if (ent
&& (!next
|| ts
< next_ts
)) {
2297 next_lost
= lost_events
;
2298 next_size
= iter
->ent_size
;
2302 iter
->ent_size
= next_size
;
2305 *ent_cpu
= next_cpu
;
2311 *missing_events
= next_lost
;
2316 /* Find the next real entry, without updating the iterator itself */
2317 struct trace_entry
*trace_find_next_entry(struct trace_iterator
*iter
,
2318 int *ent_cpu
, u64
*ent_ts
)
2320 return __find_next_entry(iter
, ent_cpu
, NULL
, ent_ts
);
2323 /* Find the next real entry, and increment the iterator to the next entry */
2324 void *trace_find_next_entry_inc(struct trace_iterator
*iter
)
2326 iter
->ent
= __find_next_entry(iter
, &iter
->cpu
,
2327 &iter
->lost_events
, &iter
->ts
);
2330 trace_iterator_increment(iter
);
2332 return iter
->ent
? iter
: NULL
;
2335 static void trace_consume(struct trace_iterator
*iter
)
2337 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
, &iter
->ts
,
2338 &iter
->lost_events
);
2341 static void *s_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
2343 struct trace_iterator
*iter
= m
->private;
2347 WARN_ON_ONCE(iter
->leftover
);
2351 /* can't go backwards */
2356 ent
= trace_find_next_entry_inc(iter
);
2360 while (ent
&& iter
->idx
< i
)
2361 ent
= trace_find_next_entry_inc(iter
);
2368 void tracing_iter_reset(struct trace_iterator
*iter
, int cpu
)
2370 struct ring_buffer_event
*event
;
2371 struct ring_buffer_iter
*buf_iter
;
2372 unsigned long entries
= 0;
2375 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= 0;
2377 buf_iter
= trace_buffer_iter(iter
, cpu
);
2381 ring_buffer_iter_reset(buf_iter
);
2384 * We could have the case with the max latency tracers
2385 * that a reset never took place on a cpu. This is evident
2386 * by the timestamp being before the start of the buffer.
2388 while ((event
= ring_buffer_iter_peek(buf_iter
, &ts
))) {
2389 if (ts
>= iter
->trace_buffer
->time_start
)
2392 ring_buffer_read(buf_iter
, NULL
);
2395 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= entries
;
2399 * The current tracer is copied to avoid a global locking
2402 static void *s_start(struct seq_file
*m
, loff_t
*pos
)
2404 struct trace_iterator
*iter
= m
->private;
2405 struct trace_array
*tr
= iter
->tr
;
2406 int cpu_file
= iter
->cpu_file
;
2412 * copy the tracer to avoid using a global lock all around.
2413 * iter->trace is a copy of current_trace, the pointer to the
2414 * name may be used instead of a strcmp(), as iter->trace->name
2415 * will point to the same string as current_trace->name.
2417 mutex_lock(&trace_types_lock
);
2418 if (unlikely(tr
->current_trace
&& iter
->trace
->name
!= tr
->current_trace
->name
))
2419 *iter
->trace
= *tr
->current_trace
;
2420 mutex_unlock(&trace_types_lock
);
2422 #ifdef CONFIG_TRACER_MAX_TRACE
2423 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
2424 return ERR_PTR(-EBUSY
);
2427 if (!iter
->snapshot
)
2428 atomic_inc(&trace_record_cmdline_disabled
);
2430 if (*pos
!= iter
->pos
) {
2435 if (cpu_file
== RING_BUFFER_ALL_CPUS
) {
2436 for_each_tracing_cpu(cpu
)
2437 tracing_iter_reset(iter
, cpu
);
2439 tracing_iter_reset(iter
, cpu_file
);
2442 for (p
= iter
; p
&& l
< *pos
; p
= s_next(m
, p
, &l
))
2447 * If we overflowed the seq_file before, then we want
2448 * to just reuse the trace_seq buffer again.
2454 p
= s_next(m
, p
, &l
);
2458 trace_event_read_lock();
2459 trace_access_lock(cpu_file
);
2463 static void s_stop(struct seq_file
*m
, void *p
)
2465 struct trace_iterator
*iter
= m
->private;
2467 #ifdef CONFIG_TRACER_MAX_TRACE
2468 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
2472 if (!iter
->snapshot
)
2473 atomic_dec(&trace_record_cmdline_disabled
);
2475 trace_access_unlock(iter
->cpu_file
);
2476 trace_event_read_unlock();
2480 get_total_entries(struct trace_buffer
*buf
,
2481 unsigned long *total
, unsigned long *entries
)
2483 unsigned long count
;
2489 for_each_tracing_cpu(cpu
) {
2490 count
= ring_buffer_entries_cpu(buf
->buffer
, cpu
);
2492 * If this buffer has skipped entries, then we hold all
2493 * entries for the trace and we need to ignore the
2494 * ones before the time stamp.
2496 if (per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
) {
2497 count
-= per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
;
2498 /* total is the same as the entries */
2502 ring_buffer_overrun_cpu(buf
->buffer
, cpu
);
2507 static void print_lat_help_header(struct seq_file
*m
)
2509 seq_puts(m
, "# _------=> CPU# \n"
2510 "# / _-----=> irqs-off \n"
2511 "# | / _----=> need-resched \n"
2512 "# || / _---=> hardirq/softirq \n"
2513 "# ||| / _--=> preempt-depth \n"
2515 "# cmd pid ||||| time | caller \n"
2516 "# \\ / ||||| \\ | / \n");
2519 static void print_event_info(struct trace_buffer
*buf
, struct seq_file
*m
)
2521 unsigned long total
;
2522 unsigned long entries
;
2524 get_total_entries(buf
, &total
, &entries
);
2525 seq_printf(m
, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2526 entries
, total
, num_online_cpus());
2530 static void print_func_help_header(struct trace_buffer
*buf
, struct seq_file
*m
)
2532 print_event_info(buf
, m
);
2533 seq_puts(m
, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2537 static void print_func_help_header_irq(struct trace_buffer
*buf
, struct seq_file
*m
)
2539 print_event_info(buf
, m
);
2540 seq_puts(m
, "# _-----=> irqs-off\n"
2541 "# / _----=> need-resched\n"
2542 "# | / _---=> hardirq/softirq\n"
2543 "# || / _--=> preempt-depth\n"
2545 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2546 "# | | | |||| | |\n");
2550 print_trace_header(struct seq_file
*m
, struct trace_iterator
*iter
)
2552 unsigned long sym_flags
= (trace_flags
& TRACE_ITER_SYM_MASK
);
2553 struct trace_buffer
*buf
= iter
->trace_buffer
;
2554 struct trace_array_cpu
*data
= per_cpu_ptr(buf
->data
, buf
->cpu
);
2555 struct tracer
*type
= iter
->trace
;
2556 unsigned long entries
;
2557 unsigned long total
;
2558 const char *name
= "preemption";
2562 get_total_entries(buf
, &total
, &entries
);
2564 seq_printf(m
, "# %s latency trace v1.1.5 on %s\n",
2566 seq_puts(m
, "# -----------------------------------"
2567 "---------------------------------\n");
2568 seq_printf(m
, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2569 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2570 nsecs_to_usecs(data
->saved_latency
),
2574 #if defined(CONFIG_PREEMPT_NONE)
2576 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2578 #elif defined(CONFIG_PREEMPT)
2583 /* These are reserved for later use */
2586 seq_printf(m
, " #P:%d)\n", num_online_cpus());
2590 seq_puts(m
, "# -----------------\n");
2591 seq_printf(m
, "# | task: %.16s-%d "
2592 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2593 data
->comm
, data
->pid
,
2594 from_kuid_munged(seq_user_ns(m
), data
->uid
), data
->nice
,
2595 data
->policy
, data
->rt_priority
);
2596 seq_puts(m
, "# -----------------\n");
2598 if (data
->critical_start
) {
2599 seq_puts(m
, "# => started at: ");
2600 seq_print_ip_sym(&iter
->seq
, data
->critical_start
, sym_flags
);
2601 trace_print_seq(m
, &iter
->seq
);
2602 seq_puts(m
, "\n# => ended at: ");
2603 seq_print_ip_sym(&iter
->seq
, data
->critical_end
, sym_flags
);
2604 trace_print_seq(m
, &iter
->seq
);
2605 seq_puts(m
, "\n#\n");
2611 static void test_cpu_buff_start(struct trace_iterator
*iter
)
2613 struct trace_seq
*s
= &iter
->seq
;
2615 if (!(trace_flags
& TRACE_ITER_ANNOTATE
))
2618 if (!(iter
->iter_flags
& TRACE_FILE_ANNOTATE
))
2621 if (cpumask_test_cpu(iter
->cpu
, iter
->started
))
2624 if (per_cpu_ptr(iter
->trace_buffer
->data
, iter
->cpu
)->skipped_entries
)
2627 cpumask_set_cpu(iter
->cpu
, iter
->started
);
2629 /* Don't print started cpu buffer for the first entry of the trace */
2631 trace_seq_printf(s
, "##### CPU %u buffer started ####\n",
2635 static enum print_line_t
print_trace_fmt(struct trace_iterator
*iter
)
2637 struct trace_seq
*s
= &iter
->seq
;
2638 unsigned long sym_flags
= (trace_flags
& TRACE_ITER_SYM_MASK
);
2639 struct trace_entry
*entry
;
2640 struct trace_event
*event
;
2644 test_cpu_buff_start(iter
);
2646 event
= ftrace_find_event(entry
->type
);
2648 if (trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2649 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
2650 trace_print_lat_context(iter
);
2652 trace_print_context(iter
);
2655 if (trace_seq_has_overflowed(s
))
2656 return TRACE_TYPE_PARTIAL_LINE
;
2659 return event
->funcs
->trace(iter
, sym_flags
, event
);
2661 trace_seq_printf(s
, "Unknown type %d\n", entry
->type
);
2663 return trace_handle_return(s
);
2666 static enum print_line_t
print_raw_fmt(struct trace_iterator
*iter
)
2668 struct trace_seq
*s
= &iter
->seq
;
2669 struct trace_entry
*entry
;
2670 struct trace_event
*event
;
2674 if (trace_flags
& TRACE_ITER_CONTEXT_INFO
)
2675 trace_seq_printf(s
, "%d %d %llu ",
2676 entry
->pid
, iter
->cpu
, iter
->ts
);
2678 if (trace_seq_has_overflowed(s
))
2679 return TRACE_TYPE_PARTIAL_LINE
;
2681 event
= ftrace_find_event(entry
->type
);
2683 return event
->funcs
->raw(iter
, 0, event
);
2685 trace_seq_printf(s
, "%d ?\n", entry
->type
);
2687 return trace_handle_return(s
);
2690 static enum print_line_t
print_hex_fmt(struct trace_iterator
*iter
)
2692 struct trace_seq
*s
= &iter
->seq
;
2693 unsigned char newline
= '\n';
2694 struct trace_entry
*entry
;
2695 struct trace_event
*event
;
2699 if (trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2700 SEQ_PUT_HEX_FIELD(s
, entry
->pid
);
2701 SEQ_PUT_HEX_FIELD(s
, iter
->cpu
);
2702 SEQ_PUT_HEX_FIELD(s
, iter
->ts
);
2703 if (trace_seq_has_overflowed(s
))
2704 return TRACE_TYPE_PARTIAL_LINE
;
2707 event
= ftrace_find_event(entry
->type
);
2709 enum print_line_t ret
= event
->funcs
->hex(iter
, 0, event
);
2710 if (ret
!= TRACE_TYPE_HANDLED
)
2714 SEQ_PUT_FIELD(s
, newline
);
2716 return trace_handle_return(s
);
2719 static enum print_line_t
print_bin_fmt(struct trace_iterator
*iter
)
2721 struct trace_seq
*s
= &iter
->seq
;
2722 struct trace_entry
*entry
;
2723 struct trace_event
*event
;
2727 if (trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2728 SEQ_PUT_FIELD(s
, entry
->pid
);
2729 SEQ_PUT_FIELD(s
, iter
->cpu
);
2730 SEQ_PUT_FIELD(s
, iter
->ts
);
2731 if (trace_seq_has_overflowed(s
))
2732 return TRACE_TYPE_PARTIAL_LINE
;
2735 event
= ftrace_find_event(entry
->type
);
2736 return event
? event
->funcs
->binary(iter
, 0, event
) :
2740 int trace_empty(struct trace_iterator
*iter
)
2742 struct ring_buffer_iter
*buf_iter
;
2745 /* If we are looking at one CPU buffer, only check that one */
2746 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
2747 cpu
= iter
->cpu_file
;
2748 buf_iter
= trace_buffer_iter(iter
, cpu
);
2750 if (!ring_buffer_iter_empty(buf_iter
))
2753 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
2759 for_each_tracing_cpu(cpu
) {
2760 buf_iter
= trace_buffer_iter(iter
, cpu
);
2762 if (!ring_buffer_iter_empty(buf_iter
))
2765 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
2773 /* Called with trace_event_read_lock() held. */
2774 enum print_line_t
print_trace_line(struct trace_iterator
*iter
)
2776 enum print_line_t ret
;
2778 if (iter
->lost_events
) {
2779 trace_seq_printf(&iter
->seq
, "CPU:%d [LOST %lu EVENTS]\n",
2780 iter
->cpu
, iter
->lost_events
);
2781 if (trace_seq_has_overflowed(&iter
->seq
))
2782 return TRACE_TYPE_PARTIAL_LINE
;
2785 if (iter
->trace
&& iter
->trace
->print_line
) {
2786 ret
= iter
->trace
->print_line(iter
);
2787 if (ret
!= TRACE_TYPE_UNHANDLED
)
2791 if (iter
->ent
->type
== TRACE_BPUTS
&&
2792 trace_flags
& TRACE_ITER_PRINTK
&&
2793 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2794 return trace_print_bputs_msg_only(iter
);
2796 if (iter
->ent
->type
== TRACE_BPRINT
&&
2797 trace_flags
& TRACE_ITER_PRINTK
&&
2798 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2799 return trace_print_bprintk_msg_only(iter
);
2801 if (iter
->ent
->type
== TRACE_PRINT
&&
2802 trace_flags
& TRACE_ITER_PRINTK
&&
2803 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2804 return trace_print_printk_msg_only(iter
);
2806 if (trace_flags
& TRACE_ITER_BIN
)
2807 return print_bin_fmt(iter
);
2809 if (trace_flags
& TRACE_ITER_HEX
)
2810 return print_hex_fmt(iter
);
2812 if (trace_flags
& TRACE_ITER_RAW
)
2813 return print_raw_fmt(iter
);
2815 return print_trace_fmt(iter
);
2818 void trace_latency_header(struct seq_file
*m
)
2820 struct trace_iterator
*iter
= m
->private;
2822 /* print nothing if the buffers are empty */
2823 if (trace_empty(iter
))
2826 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
2827 print_trace_header(m
, iter
);
2829 if (!(trace_flags
& TRACE_ITER_VERBOSE
))
2830 print_lat_help_header(m
);
2833 void trace_default_header(struct seq_file
*m
)
2835 struct trace_iterator
*iter
= m
->private;
2837 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
2840 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
) {
2841 /* print nothing if the buffers are empty */
2842 if (trace_empty(iter
))
2844 print_trace_header(m
, iter
);
2845 if (!(trace_flags
& TRACE_ITER_VERBOSE
))
2846 print_lat_help_header(m
);
2848 if (!(trace_flags
& TRACE_ITER_VERBOSE
)) {
2849 if (trace_flags
& TRACE_ITER_IRQ_INFO
)
2850 print_func_help_header_irq(iter
->trace_buffer
, m
);
2852 print_func_help_header(iter
->trace_buffer
, m
);
2857 static void test_ftrace_alive(struct seq_file
*m
)
2859 if (!ftrace_is_dead())
2861 seq_puts(m
, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2862 "# MAY BE MISSING FUNCTION EVENTS\n");
2865 #ifdef CONFIG_TRACER_MAX_TRACE
2866 static void show_snapshot_main_help(struct seq_file
*m
)
2868 seq_puts(m
, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2869 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2870 "# Takes a snapshot of the main buffer.\n"
2871 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2872 "# (Doesn't have to be '2' works with any number that\n"
2873 "# is not a '0' or '1')\n");
2876 static void show_snapshot_percpu_help(struct seq_file
*m
)
2878 seq_puts(m
, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2879 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2880 seq_puts(m
, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2881 "# Takes a snapshot of the main buffer for this cpu.\n");
2883 seq_puts(m
, "# echo 1 > snapshot : Not supported with this kernel.\n"
2884 "# Must use main snapshot file to allocate.\n");
2886 seq_puts(m
, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2887 "# (Doesn't have to be '2' works with any number that\n"
2888 "# is not a '0' or '1')\n");
2891 static void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
)
2893 if (iter
->tr
->allocated_snapshot
)
2894 seq_puts(m
, "#\n# * Snapshot is allocated *\n#\n");
2896 seq_puts(m
, "#\n# * Snapshot is freed *\n#\n");
2898 seq_puts(m
, "# Snapshot commands:\n");
2899 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
2900 show_snapshot_main_help(m
);
2902 show_snapshot_percpu_help(m
);
2905 /* Should never be called */
2906 static inline void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
) { }
2909 static int s_show(struct seq_file
*m
, void *v
)
2911 struct trace_iterator
*iter
= v
;
2914 if (iter
->ent
== NULL
) {
2916 seq_printf(m
, "# tracer: %s\n", iter
->trace
->name
);
2918 test_ftrace_alive(m
);
2920 if (iter
->snapshot
&& trace_empty(iter
))
2921 print_snapshot_help(m
, iter
);
2922 else if (iter
->trace
&& iter
->trace
->print_header
)
2923 iter
->trace
->print_header(m
);
2925 trace_default_header(m
);
2927 } else if (iter
->leftover
) {
2929 * If we filled the seq_file buffer earlier, we
2930 * want to just show it now.
2932 ret
= trace_print_seq(m
, &iter
->seq
);
2934 /* ret should this time be zero, but you never know */
2935 iter
->leftover
= ret
;
2938 print_trace_line(iter
);
2939 ret
= trace_print_seq(m
, &iter
->seq
);
2941 * If we overflow the seq_file buffer, then it will
2942 * ask us for this data again at start up.
2944 * ret is 0 if seq_file write succeeded.
2947 iter
->leftover
= ret
;
2954 * Should be used after trace_array_get(), trace_types_lock
2955 * ensures that i_cdev was already initialized.
2957 static inline int tracing_get_cpu(struct inode
*inode
)
2959 if (inode
->i_cdev
) /* See trace_create_cpu_file() */
2960 return (long)inode
->i_cdev
- 1;
2961 return RING_BUFFER_ALL_CPUS
;
2964 static const struct seq_operations tracer_seq_ops
= {
2971 static struct trace_iterator
*
2972 __tracing_open(struct inode
*inode
, struct file
*file
, bool snapshot
)
2974 struct trace_array
*tr
= inode
->i_private
;
2975 struct trace_iterator
*iter
;
2978 if (tracing_disabled
)
2979 return ERR_PTR(-ENODEV
);
2981 iter
= __seq_open_private(file
, &tracer_seq_ops
, sizeof(*iter
));
2983 return ERR_PTR(-ENOMEM
);
2985 iter
->buffer_iter
= kzalloc(sizeof(*iter
->buffer_iter
) * num_possible_cpus(),
2987 if (!iter
->buffer_iter
)
2991 * We make a copy of the current tracer to avoid concurrent
2992 * changes on it while we are reading.
2994 mutex_lock(&trace_types_lock
);
2995 iter
->trace
= kzalloc(sizeof(*iter
->trace
), GFP_KERNEL
);
2999 *iter
->trace
= *tr
->current_trace
;
3001 if (!zalloc_cpumask_var(&iter
->started
, GFP_KERNEL
))
3006 #ifdef CONFIG_TRACER_MAX_TRACE
3007 /* Currently only the top directory has a snapshot */
3008 if (tr
->current_trace
->print_max
|| snapshot
)
3009 iter
->trace_buffer
= &tr
->max_buffer
;
3012 iter
->trace_buffer
= &tr
->trace_buffer
;
3013 iter
->snapshot
= snapshot
;
3015 iter
->cpu_file
= tracing_get_cpu(inode
);
3016 mutex_init(&iter
->mutex
);
3018 /* Notify the tracer early; before we stop tracing. */
3019 if (iter
->trace
&& iter
->trace
->open
)
3020 iter
->trace
->open(iter
);
3022 /* Annotate start of buffers if we had overruns */
3023 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
3024 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
3026 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3027 if (trace_clocks
[tr
->clock_id
].in_ns
)
3028 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
3030 /* stop the trace while dumping if we are not opening "snapshot" */
3031 if (!iter
->snapshot
)
3032 tracing_stop_tr(tr
);
3034 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
) {
3035 for_each_tracing_cpu(cpu
) {
3036 iter
->buffer_iter
[cpu
] =
3037 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3039 ring_buffer_read_prepare_sync();
3040 for_each_tracing_cpu(cpu
) {
3041 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3042 tracing_iter_reset(iter
, cpu
);
3045 cpu
= iter
->cpu_file
;
3046 iter
->buffer_iter
[cpu
] =
3047 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3048 ring_buffer_read_prepare_sync();
3049 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3050 tracing_iter_reset(iter
, cpu
);
3053 mutex_unlock(&trace_types_lock
);
3058 mutex_unlock(&trace_types_lock
);
3060 kfree(iter
->buffer_iter
);
3062 seq_release_private(inode
, file
);
3063 return ERR_PTR(-ENOMEM
);
3066 int tracing_open_generic(struct inode
*inode
, struct file
*filp
)
3068 if (tracing_disabled
)
3071 filp
->private_data
= inode
->i_private
;
3075 bool tracing_is_disabled(void)
3077 return (tracing_disabled
) ? true: false;
3081 * Open and update trace_array ref count.
3082 * Must have the current trace_array passed to it.
3084 static int tracing_open_generic_tr(struct inode
*inode
, struct file
*filp
)
3086 struct trace_array
*tr
= inode
->i_private
;
3088 if (tracing_disabled
)
3091 if (trace_array_get(tr
) < 0)
3094 filp
->private_data
= inode
->i_private
;
3099 static int tracing_release(struct inode
*inode
, struct file
*file
)
3101 struct trace_array
*tr
= inode
->i_private
;
3102 struct seq_file
*m
= file
->private_data
;
3103 struct trace_iterator
*iter
;
3106 if (!(file
->f_mode
& FMODE_READ
)) {
3107 trace_array_put(tr
);
3111 /* Writes do not use seq_file */
3113 mutex_lock(&trace_types_lock
);
3115 for_each_tracing_cpu(cpu
) {
3116 if (iter
->buffer_iter
[cpu
])
3117 ring_buffer_read_finish(iter
->buffer_iter
[cpu
]);
3120 if (iter
->trace
&& iter
->trace
->close
)
3121 iter
->trace
->close(iter
);
3123 if (!iter
->snapshot
)
3124 /* reenable tracing if it was previously enabled */
3125 tracing_start_tr(tr
);
3127 __trace_array_put(tr
);
3129 mutex_unlock(&trace_types_lock
);
3131 mutex_destroy(&iter
->mutex
);
3132 free_cpumask_var(iter
->started
);
3134 kfree(iter
->buffer_iter
);
3135 seq_release_private(inode
, file
);
3140 static int tracing_release_generic_tr(struct inode
*inode
, struct file
*file
)
3142 struct trace_array
*tr
= inode
->i_private
;
3144 trace_array_put(tr
);
3148 static int tracing_single_release_tr(struct inode
*inode
, struct file
*file
)
3150 struct trace_array
*tr
= inode
->i_private
;
3152 trace_array_put(tr
);
3154 return single_release(inode
, file
);
3157 static int tracing_open(struct inode
*inode
, struct file
*file
)
3159 struct trace_array
*tr
= inode
->i_private
;
3160 struct trace_iterator
*iter
;
3163 if (trace_array_get(tr
) < 0)
3166 /* If this file was open for write, then erase contents */
3167 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
3168 int cpu
= tracing_get_cpu(inode
);
3170 if (cpu
== RING_BUFFER_ALL_CPUS
)
3171 tracing_reset_online_cpus(&tr
->trace_buffer
);
3173 tracing_reset(&tr
->trace_buffer
, cpu
);
3176 if (file
->f_mode
& FMODE_READ
) {
3177 iter
= __tracing_open(inode
, file
, false);
3179 ret
= PTR_ERR(iter
);
3180 else if (trace_flags
& TRACE_ITER_LATENCY_FMT
)
3181 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
3185 trace_array_put(tr
);
3191 * Some tracers are not suitable for instance buffers.
3192 * A tracer is always available for the global array (toplevel)
3193 * or if it explicitly states that it is.
3196 trace_ok_for_array(struct tracer
*t
, struct trace_array
*tr
)
3198 return (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) || t
->allow_instances
;
3201 /* Find the next tracer that this trace array may use */
3202 static struct tracer
*
3203 get_tracer_for_array(struct trace_array
*tr
, struct tracer
*t
)
3205 while (t
&& !trace_ok_for_array(t
, tr
))
3212 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3214 struct trace_array
*tr
= m
->private;
3215 struct tracer
*t
= v
;
3220 t
= get_tracer_for_array(tr
, t
->next
);
3225 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
3227 struct trace_array
*tr
= m
->private;
3231 mutex_lock(&trace_types_lock
);
3233 t
= get_tracer_for_array(tr
, trace_types
);
3234 for (; t
&& l
< *pos
; t
= t_next(m
, t
, &l
))
3240 static void t_stop(struct seq_file
*m
, void *p
)
3242 mutex_unlock(&trace_types_lock
);
3245 static int t_show(struct seq_file
*m
, void *v
)
3247 struct tracer
*t
= v
;
3252 seq_puts(m
, t
->name
);
3261 static const struct seq_operations show_traces_seq_ops
= {
3268 static int show_traces_open(struct inode
*inode
, struct file
*file
)
3270 struct trace_array
*tr
= inode
->i_private
;
3274 if (tracing_disabled
)
3277 ret
= seq_open(file
, &show_traces_seq_ops
);
3281 m
= file
->private_data
;
3288 tracing_write_stub(struct file
*filp
, const char __user
*ubuf
,
3289 size_t count
, loff_t
*ppos
)
3294 loff_t
tracing_lseek(struct file
*file
, loff_t offset
, int whence
)
3298 if (file
->f_mode
& FMODE_READ
)
3299 ret
= seq_lseek(file
, offset
, whence
);
3301 file
->f_pos
= ret
= 0;
3306 static const struct file_operations tracing_fops
= {
3307 .open
= tracing_open
,
3309 .write
= tracing_write_stub
,
3310 .llseek
= tracing_lseek
,
3311 .release
= tracing_release
,
3314 static const struct file_operations show_traces_fops
= {
3315 .open
= show_traces_open
,
3317 .release
= seq_release
,
3318 .llseek
= seq_lseek
,
3322 * The tracer itself will not take this lock, but still we want
3323 * to provide a consistent cpumask to user-space:
3325 static DEFINE_MUTEX(tracing_cpumask_update_lock
);
3328 * Temporary storage for the character representation of the
3329 * CPU bitmask (and one more byte for the newline):
3331 static char mask_str
[NR_CPUS
+ 1];
3334 tracing_cpumask_read(struct file
*filp
, char __user
*ubuf
,
3335 size_t count
, loff_t
*ppos
)
3337 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3340 mutex_lock(&tracing_cpumask_update_lock
);
3342 len
= cpumask_scnprintf(mask_str
, count
, tr
->tracing_cpumask
);
3343 if (count
- len
< 2) {
3347 len
+= sprintf(mask_str
+ len
, "\n");
3348 count
= simple_read_from_buffer(ubuf
, count
, ppos
, mask_str
, NR_CPUS
+1);
3351 mutex_unlock(&tracing_cpumask_update_lock
);
3357 tracing_cpumask_write(struct file
*filp
, const char __user
*ubuf
,
3358 size_t count
, loff_t
*ppos
)
3360 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3361 cpumask_var_t tracing_cpumask_new
;
3364 if (!alloc_cpumask_var(&tracing_cpumask_new
, GFP_KERNEL
))
3367 err
= cpumask_parse_user(ubuf
, count
, tracing_cpumask_new
);
3371 mutex_lock(&tracing_cpumask_update_lock
);
3373 local_irq_disable();
3374 arch_spin_lock(&tr
->max_lock
);
3375 for_each_tracing_cpu(cpu
) {
3377 * Increase/decrease the disabled counter if we are
3378 * about to flip a bit in the cpumask:
3380 if (cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
3381 !cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
3382 atomic_inc(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
3383 ring_buffer_record_disable_cpu(tr
->trace_buffer
.buffer
, cpu
);
3385 if (!cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
3386 cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
3387 atomic_dec(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
3388 ring_buffer_record_enable_cpu(tr
->trace_buffer
.buffer
, cpu
);
3391 arch_spin_unlock(&tr
->max_lock
);
3394 cpumask_copy(tr
->tracing_cpumask
, tracing_cpumask_new
);
3396 mutex_unlock(&tracing_cpumask_update_lock
);
3397 free_cpumask_var(tracing_cpumask_new
);
3402 free_cpumask_var(tracing_cpumask_new
);
3407 static const struct file_operations tracing_cpumask_fops
= {
3408 .open
= tracing_open_generic_tr
,
3409 .read
= tracing_cpumask_read
,
3410 .write
= tracing_cpumask_write
,
3411 .release
= tracing_release_generic_tr
,
3412 .llseek
= generic_file_llseek
,
3415 static int tracing_trace_options_show(struct seq_file
*m
, void *v
)
3417 struct tracer_opt
*trace_opts
;
3418 struct trace_array
*tr
= m
->private;
3422 mutex_lock(&trace_types_lock
);
3423 tracer_flags
= tr
->current_trace
->flags
->val
;
3424 trace_opts
= tr
->current_trace
->flags
->opts
;
3426 for (i
= 0; trace_options
[i
]; i
++) {
3427 if (trace_flags
& (1 << i
))
3428 seq_printf(m
, "%s\n", trace_options
[i
]);
3430 seq_printf(m
, "no%s\n", trace_options
[i
]);
3433 for (i
= 0; trace_opts
[i
].name
; i
++) {
3434 if (tracer_flags
& trace_opts
[i
].bit
)
3435 seq_printf(m
, "%s\n", trace_opts
[i
].name
);
3437 seq_printf(m
, "no%s\n", trace_opts
[i
].name
);
3439 mutex_unlock(&trace_types_lock
);
3444 static int __set_tracer_option(struct trace_array
*tr
,
3445 struct tracer_flags
*tracer_flags
,
3446 struct tracer_opt
*opts
, int neg
)
3448 struct tracer
*trace
= tr
->current_trace
;
3451 ret
= trace
->set_flag(tr
, tracer_flags
->val
, opts
->bit
, !neg
);
3456 tracer_flags
->val
&= ~opts
->bit
;
3458 tracer_flags
->val
|= opts
->bit
;
3462 /* Try to assign a tracer specific option */
3463 static int set_tracer_option(struct trace_array
*tr
, char *cmp
, int neg
)
3465 struct tracer
*trace
= tr
->current_trace
;
3466 struct tracer_flags
*tracer_flags
= trace
->flags
;
3467 struct tracer_opt
*opts
= NULL
;
3470 for (i
= 0; tracer_flags
->opts
[i
].name
; i
++) {
3471 opts
= &tracer_flags
->opts
[i
];
3473 if (strcmp(cmp
, opts
->name
) == 0)
3474 return __set_tracer_option(tr
, trace
->flags
, opts
, neg
);
3480 /* Some tracers require overwrite to stay enabled */
3481 int trace_keep_overwrite(struct tracer
*tracer
, u32 mask
, int set
)
3483 if (tracer
->enabled
&& (mask
& TRACE_ITER_OVERWRITE
) && !set
)
3489 int set_tracer_flag(struct trace_array
*tr
, unsigned int mask
, int enabled
)
3491 /* do nothing if flag is already set */
3492 if (!!(trace_flags
& mask
) == !!enabled
)
3495 /* Give the tracer a chance to approve the change */
3496 if (tr
->current_trace
->flag_changed
)
3497 if (tr
->current_trace
->flag_changed(tr
, mask
, !!enabled
))
3501 trace_flags
|= mask
;
3503 trace_flags
&= ~mask
;
3505 if (mask
== TRACE_ITER_RECORD_CMD
)
3506 trace_event_enable_cmd_record(enabled
);
3508 if (mask
== TRACE_ITER_OVERWRITE
) {
3509 ring_buffer_change_overwrite(tr
->trace_buffer
.buffer
, enabled
);
3510 #ifdef CONFIG_TRACER_MAX_TRACE
3511 ring_buffer_change_overwrite(tr
->max_buffer
.buffer
, enabled
);
3515 if (mask
== TRACE_ITER_PRINTK
)
3516 trace_printk_start_stop_comm(enabled
);
3521 static int trace_set_options(struct trace_array
*tr
, char *option
)
3528 cmp
= strstrip(option
);
3530 if (strncmp(cmp
, "no", 2) == 0) {
3535 mutex_lock(&trace_types_lock
);
3537 for (i
= 0; trace_options
[i
]; i
++) {
3538 if (strcmp(cmp
, trace_options
[i
]) == 0) {
3539 ret
= set_tracer_flag(tr
, 1 << i
, !neg
);
3544 /* If no option could be set, test the specific tracer options */
3545 if (!trace_options
[i
])
3546 ret
= set_tracer_option(tr
, cmp
, neg
);
3548 mutex_unlock(&trace_types_lock
);
3554 tracing_trace_options_write(struct file
*filp
, const char __user
*ubuf
,
3555 size_t cnt
, loff_t
*ppos
)
3557 struct seq_file
*m
= filp
->private_data
;
3558 struct trace_array
*tr
= m
->private;
3562 if (cnt
>= sizeof(buf
))
3565 if (copy_from_user(&buf
, ubuf
, cnt
))
3570 ret
= trace_set_options(tr
, buf
);
3579 static int tracing_trace_options_open(struct inode
*inode
, struct file
*file
)
3581 struct trace_array
*tr
= inode
->i_private
;
3584 if (tracing_disabled
)
3587 if (trace_array_get(tr
) < 0)
3590 ret
= single_open(file
, tracing_trace_options_show
, inode
->i_private
);
3592 trace_array_put(tr
);
3597 static const struct file_operations tracing_iter_fops
= {
3598 .open
= tracing_trace_options_open
,
3600 .llseek
= seq_lseek
,
3601 .release
= tracing_single_release_tr
,
3602 .write
= tracing_trace_options_write
,
3605 static const char readme_msg
[] =
3606 "tracing mini-HOWTO:\n\n"
3607 "# echo 0 > tracing_on : quick way to disable tracing\n"
3608 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3609 " Important files:\n"
3610 " trace\t\t\t- The static contents of the buffer\n"
3611 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3612 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3613 " current_tracer\t- function and latency tracers\n"
3614 " available_tracers\t- list of configured tracers for current_tracer\n"
3615 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3616 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3617 " trace_clock\t\t-change the clock used to order events\n"
3618 " local: Per cpu clock but may not be synced across CPUs\n"
3619 " global: Synced across CPUs but slows tracing down.\n"
3620 " counter: Not a clock, but just an increment\n"
3621 " uptime: Jiffy counter from time of boot\n"
3622 " perf: Same clock that perf events use\n"
3623 #ifdef CONFIG_X86_64
3624 " x86-tsc: TSC cycle counter\n"
3626 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3627 " tracing_cpumask\t- Limit which CPUs to trace\n"
3628 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3629 "\t\t\t Remove sub-buffer with rmdir\n"
3630 " trace_options\t\t- Set format or modify how tracing happens\n"
3631 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3632 "\t\t\t option name\n"
3633 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3634 #ifdef CONFIG_DYNAMIC_FTRACE
3635 "\n available_filter_functions - list of functions that can be filtered on\n"
3636 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3637 "\t\t\t functions\n"
3638 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3639 "\t modules: Can select a group via module\n"
3640 "\t Format: :mod:<module-name>\n"
3641 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3642 "\t triggers: a command to perform when function is hit\n"
3643 "\t Format: <function>:<trigger>[:count]\n"
3644 "\t trigger: traceon, traceoff\n"
3645 "\t\t enable_event:<system>:<event>\n"
3646 "\t\t disable_event:<system>:<event>\n"
3647 #ifdef CONFIG_STACKTRACE
3650 #ifdef CONFIG_TRACER_SNAPSHOT
3655 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3656 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3657 "\t The first one will disable tracing every time do_fault is hit\n"
3658 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3659 "\t The first time do trap is hit and it disables tracing, the\n"
3660 "\t counter will decrement to 2. If tracing is already disabled,\n"
3661 "\t the counter will not decrement. It only decrements when the\n"
3662 "\t trigger did work\n"
3663 "\t To remove trigger without count:\n"
3664 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3665 "\t To remove trigger with a count:\n"
3666 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3667 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3668 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3669 "\t modules: Can select a group via module command :mod:\n"
3670 "\t Does not accept triggers\n"
3671 #endif /* CONFIG_DYNAMIC_FTRACE */
3672 #ifdef CONFIG_FUNCTION_TRACER
3673 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3676 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3677 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3678 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3679 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3681 #ifdef CONFIG_TRACER_SNAPSHOT
3682 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3683 "\t\t\t snapshot buffer. Read the contents for more\n"
3684 "\t\t\t information\n"
3686 #ifdef CONFIG_STACK_TRACER
3687 " stack_trace\t\t- Shows the max stack trace when active\n"
3688 " stack_max_size\t- Shows current max stack size that was traced\n"
3689 "\t\t\t Write into this file to reset the max size (trigger a\n"
3690 "\t\t\t new trace)\n"
3691 #ifdef CONFIG_DYNAMIC_FTRACE
3692 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3695 #endif /* CONFIG_STACK_TRACER */
3696 " events/\t\t- Directory containing all trace event subsystems:\n"
3697 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3698 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3699 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3701 " filter\t\t- If set, only events passing filter are traced\n"
3702 " events/<system>/<event>/\t- Directory containing control files for\n"
3704 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3705 " filter\t\t- If set, only events passing filter are traced\n"
3706 " trigger\t\t- If set, a command to perform when event is hit\n"
3707 "\t Format: <trigger>[:count][if <filter>]\n"
3708 "\t trigger: traceon, traceoff\n"
3709 "\t enable_event:<system>:<event>\n"
3710 "\t disable_event:<system>:<event>\n"
3711 #ifdef CONFIG_STACKTRACE
3714 #ifdef CONFIG_TRACER_SNAPSHOT
3717 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3718 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3719 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3720 "\t events/block/block_unplug/trigger\n"
3721 "\t The first disables tracing every time block_unplug is hit.\n"
3722 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3723 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3724 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3725 "\t Like function triggers, the counter is only decremented if it\n"
3726 "\t enabled or disabled tracing.\n"
3727 "\t To remove a trigger without a count:\n"
3728 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3729 "\t To remove a trigger with a count:\n"
3730 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3731 "\t Filters can be ignored when removing a trigger.\n"
3735 tracing_readme_read(struct file
*filp
, char __user
*ubuf
,
3736 size_t cnt
, loff_t
*ppos
)
3738 return simple_read_from_buffer(ubuf
, cnt
, ppos
,
3739 readme_msg
, strlen(readme_msg
));
3742 static const struct file_operations tracing_readme_fops
= {
3743 .open
= tracing_open_generic
,
3744 .read
= tracing_readme_read
,
3745 .llseek
= generic_file_llseek
,
3748 static void *saved_cmdlines_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3750 unsigned int *ptr
= v
;
3752 if (*pos
|| m
->count
)
3757 for (; ptr
< &savedcmd
->map_cmdline_to_pid
[savedcmd
->cmdline_num
];
3759 if (*ptr
== -1 || *ptr
== NO_CMDLINE_MAP
)
3768 static void *saved_cmdlines_start(struct seq_file
*m
, loff_t
*pos
)
3774 arch_spin_lock(&trace_cmdline_lock
);
3776 v
= &savedcmd
->map_cmdline_to_pid
[0];
3778 v
= saved_cmdlines_next(m
, v
, &l
);
3786 static void saved_cmdlines_stop(struct seq_file
*m
, void *v
)
3788 arch_spin_unlock(&trace_cmdline_lock
);
3792 static int saved_cmdlines_show(struct seq_file
*m
, void *v
)
3794 char buf
[TASK_COMM_LEN
];
3795 unsigned int *pid
= v
;
3797 __trace_find_cmdline(*pid
, buf
);
3798 seq_printf(m
, "%d %s\n", *pid
, buf
);
3802 static const struct seq_operations tracing_saved_cmdlines_seq_ops
= {
3803 .start
= saved_cmdlines_start
,
3804 .next
= saved_cmdlines_next
,
3805 .stop
= saved_cmdlines_stop
,
3806 .show
= saved_cmdlines_show
,
3809 static int tracing_saved_cmdlines_open(struct inode
*inode
, struct file
*filp
)
3811 if (tracing_disabled
)
3814 return seq_open(filp
, &tracing_saved_cmdlines_seq_ops
);
3817 static const struct file_operations tracing_saved_cmdlines_fops
= {
3818 .open
= tracing_saved_cmdlines_open
,
3820 .llseek
= seq_lseek
,
3821 .release
= seq_release
,
3825 tracing_saved_cmdlines_size_read(struct file
*filp
, char __user
*ubuf
,
3826 size_t cnt
, loff_t
*ppos
)
3831 arch_spin_lock(&trace_cmdline_lock
);
3832 r
= scnprintf(buf
, sizeof(buf
), "%u\n", savedcmd
->cmdline_num
);
3833 arch_spin_unlock(&trace_cmdline_lock
);
3835 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
3838 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer
*s
)
3840 kfree(s
->saved_cmdlines
);
3841 kfree(s
->map_cmdline_to_pid
);
3845 static int tracing_resize_saved_cmdlines(unsigned int val
)
3847 struct saved_cmdlines_buffer
*s
, *savedcmd_temp
;
3849 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
3853 if (allocate_cmdlines_buffer(val
, s
) < 0) {
3858 arch_spin_lock(&trace_cmdline_lock
);
3859 savedcmd_temp
= savedcmd
;
3861 arch_spin_unlock(&trace_cmdline_lock
);
3862 free_saved_cmdlines_buffer(savedcmd_temp
);
3868 tracing_saved_cmdlines_size_write(struct file
*filp
, const char __user
*ubuf
,
3869 size_t cnt
, loff_t
*ppos
)
3874 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
3878 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3879 if (!val
|| val
> PID_MAX_DEFAULT
)
3882 ret
= tracing_resize_saved_cmdlines((unsigned int)val
);
3891 static const struct file_operations tracing_saved_cmdlines_size_fops
= {
3892 .open
= tracing_open_generic
,
3893 .read
= tracing_saved_cmdlines_size_read
,
3894 .write
= tracing_saved_cmdlines_size_write
,
3898 tracing_set_trace_read(struct file
*filp
, char __user
*ubuf
,
3899 size_t cnt
, loff_t
*ppos
)
3901 struct trace_array
*tr
= filp
->private_data
;
3902 char buf
[MAX_TRACER_SIZE
+2];
3905 mutex_lock(&trace_types_lock
);
3906 r
= sprintf(buf
, "%s\n", tr
->current_trace
->name
);
3907 mutex_unlock(&trace_types_lock
);
3909 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
3912 int tracer_init(struct tracer
*t
, struct trace_array
*tr
)
3914 tracing_reset_online_cpus(&tr
->trace_buffer
);
3918 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
)
3922 for_each_tracing_cpu(cpu
)
3923 per_cpu_ptr(buf
->data
, cpu
)->entries
= val
;
3926 #ifdef CONFIG_TRACER_MAX_TRACE
3927 /* resize @tr's buffer to the size of @size_tr's entries */
3928 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
3929 struct trace_buffer
*size_buf
, int cpu_id
)
3933 if (cpu_id
== RING_BUFFER_ALL_CPUS
) {
3934 for_each_tracing_cpu(cpu
) {
3935 ret
= ring_buffer_resize(trace_buf
->buffer
,
3936 per_cpu_ptr(size_buf
->data
, cpu
)->entries
, cpu
);
3939 per_cpu_ptr(trace_buf
->data
, cpu
)->entries
=
3940 per_cpu_ptr(size_buf
->data
, cpu
)->entries
;
3943 ret
= ring_buffer_resize(trace_buf
->buffer
,
3944 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
, cpu_id
);
3946 per_cpu_ptr(trace_buf
->data
, cpu_id
)->entries
=
3947 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
;
3952 #endif /* CONFIG_TRACER_MAX_TRACE */
3954 static int __tracing_resize_ring_buffer(struct trace_array
*tr
,
3955 unsigned long size
, int cpu
)
3960 * If kernel or user changes the size of the ring buffer
3961 * we use the size that was given, and we can forget about
3962 * expanding it later.
3964 ring_buffer_expanded
= true;
3966 /* May be called before buffers are initialized */
3967 if (!tr
->trace_buffer
.buffer
)
3970 ret
= ring_buffer_resize(tr
->trace_buffer
.buffer
, size
, cpu
);
3974 #ifdef CONFIG_TRACER_MAX_TRACE
3975 if (!(tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) ||
3976 !tr
->current_trace
->use_max_tr
)
3979 ret
= ring_buffer_resize(tr
->max_buffer
.buffer
, size
, cpu
);
3981 int r
= resize_buffer_duplicate_size(&tr
->trace_buffer
,
3982 &tr
->trace_buffer
, cpu
);
3985 * AARGH! We are left with different
3986 * size max buffer!!!!
3987 * The max buffer is our "snapshot" buffer.
3988 * When a tracer needs a snapshot (one of the
3989 * latency tracers), it swaps the max buffer
3990 * with the saved snap shot. We succeeded to
3991 * update the size of the main buffer, but failed to
3992 * update the size of the max buffer. But when we tried
3993 * to reset the main buffer to the original size, we
3994 * failed there too. This is very unlikely to
3995 * happen, but if it does, warn and kill all
3999 tracing_disabled
= 1;
4004 if (cpu
== RING_BUFFER_ALL_CPUS
)
4005 set_buffer_entries(&tr
->max_buffer
, size
);
4007 per_cpu_ptr(tr
->max_buffer
.data
, cpu
)->entries
= size
;
4010 #endif /* CONFIG_TRACER_MAX_TRACE */
4012 if (cpu
== RING_BUFFER_ALL_CPUS
)
4013 set_buffer_entries(&tr
->trace_buffer
, size
);
4015 per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
= size
;
4020 static ssize_t
tracing_resize_ring_buffer(struct trace_array
*tr
,
4021 unsigned long size
, int cpu_id
)
4025 mutex_lock(&trace_types_lock
);
4027 if (cpu_id
!= RING_BUFFER_ALL_CPUS
) {
4028 /* make sure, this cpu is enabled in the mask */
4029 if (!cpumask_test_cpu(cpu_id
, tracing_buffer_mask
)) {
4035 ret
= __tracing_resize_ring_buffer(tr
, size
, cpu_id
);
4040 mutex_unlock(&trace_types_lock
);
4047 * tracing_update_buffers - used by tracing facility to expand ring buffers
4049 * To save on memory when the tracing is never used on a system with it
4050 * configured in. The ring buffers are set to a minimum size. But once
4051 * a user starts to use the tracing facility, then they need to grow
4052 * to their default size.
4054 * This function is to be called when a tracer is about to be used.
4056 int tracing_update_buffers(void)
4060 mutex_lock(&trace_types_lock
);
4061 if (!ring_buffer_expanded
)
4062 ret
= __tracing_resize_ring_buffer(&global_trace
, trace_buf_size
,
4063 RING_BUFFER_ALL_CPUS
);
4064 mutex_unlock(&trace_types_lock
);
4069 struct trace_option_dentry
;
4071 static struct trace_option_dentry
*
4072 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
);
4075 destroy_trace_option_files(struct trace_option_dentry
*topts
);
4078 * Used to clear out the tracer before deletion of an instance.
4079 * Must have trace_types_lock held.
4081 static void tracing_set_nop(struct trace_array
*tr
)
4083 if (tr
->current_trace
== &nop_trace
)
4086 tr
->current_trace
->enabled
--;
4088 if (tr
->current_trace
->reset
)
4089 tr
->current_trace
->reset(tr
);
4091 tr
->current_trace
= &nop_trace
;
4094 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
)
4096 static struct trace_option_dentry
*topts
;
4098 #ifdef CONFIG_TRACER_MAX_TRACE
4103 mutex_lock(&trace_types_lock
);
4105 if (!ring_buffer_expanded
) {
4106 ret
= __tracing_resize_ring_buffer(tr
, trace_buf_size
,
4107 RING_BUFFER_ALL_CPUS
);
4113 for (t
= trace_types
; t
; t
= t
->next
) {
4114 if (strcmp(t
->name
, buf
) == 0)
4121 if (t
== tr
->current_trace
)
4124 /* Some tracers are only allowed for the top level buffer */
4125 if (!trace_ok_for_array(t
, tr
)) {
4130 trace_branch_disable();
4132 tr
->current_trace
->enabled
--;
4134 if (tr
->current_trace
->reset
)
4135 tr
->current_trace
->reset(tr
);
4137 /* Current trace needs to be nop_trace before synchronize_sched */
4138 tr
->current_trace
= &nop_trace
;
4140 #ifdef CONFIG_TRACER_MAX_TRACE
4141 had_max_tr
= tr
->allocated_snapshot
;
4143 if (had_max_tr
&& !t
->use_max_tr
) {
4145 * We need to make sure that the update_max_tr sees that
4146 * current_trace changed to nop_trace to keep it from
4147 * swapping the buffers after we resize it.
4148 * The update_max_tr is called from interrupts disabled
4149 * so a synchronized_sched() is sufficient.
4151 synchronize_sched();
4155 /* Currently, only the top instance has options */
4156 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) {
4157 destroy_trace_option_files(topts
);
4158 topts
= create_trace_option_files(tr
, t
);
4161 #ifdef CONFIG_TRACER_MAX_TRACE
4162 if (t
->use_max_tr
&& !had_max_tr
) {
4163 ret
= alloc_snapshot(tr
);
4170 ret
= tracer_init(t
, tr
);
4175 tr
->current_trace
= t
;
4176 tr
->current_trace
->enabled
++;
4177 trace_branch_enable(tr
);
4179 mutex_unlock(&trace_types_lock
);
4185 tracing_set_trace_write(struct file
*filp
, const char __user
*ubuf
,
4186 size_t cnt
, loff_t
*ppos
)
4188 struct trace_array
*tr
= filp
->private_data
;
4189 char buf
[MAX_TRACER_SIZE
+1];
4196 if (cnt
> MAX_TRACER_SIZE
)
4197 cnt
= MAX_TRACER_SIZE
;
4199 if (copy_from_user(&buf
, ubuf
, cnt
))
4204 /* strip ending whitespace. */
4205 for (i
= cnt
- 1; i
> 0 && isspace(buf
[i
]); i
--)
4208 err
= tracing_set_tracer(tr
, buf
);
4218 tracing_nsecs_read(unsigned long *ptr
, char __user
*ubuf
,
4219 size_t cnt
, loff_t
*ppos
)
4224 r
= snprintf(buf
, sizeof(buf
), "%ld\n",
4225 *ptr
== (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr
));
4226 if (r
> sizeof(buf
))
4228 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4232 tracing_nsecs_write(unsigned long *ptr
, const char __user
*ubuf
,
4233 size_t cnt
, loff_t
*ppos
)
4238 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4248 tracing_thresh_read(struct file
*filp
, char __user
*ubuf
,
4249 size_t cnt
, loff_t
*ppos
)
4251 return tracing_nsecs_read(&tracing_thresh
, ubuf
, cnt
, ppos
);
4255 tracing_thresh_write(struct file
*filp
, const char __user
*ubuf
,
4256 size_t cnt
, loff_t
*ppos
)
4258 struct trace_array
*tr
= filp
->private_data
;
4261 mutex_lock(&trace_types_lock
);
4262 ret
= tracing_nsecs_write(&tracing_thresh
, ubuf
, cnt
, ppos
);
4266 if (tr
->current_trace
->update_thresh
) {
4267 ret
= tr
->current_trace
->update_thresh(tr
);
4274 mutex_unlock(&trace_types_lock
);
4280 tracing_max_lat_read(struct file
*filp
, char __user
*ubuf
,
4281 size_t cnt
, loff_t
*ppos
)
4283 return tracing_nsecs_read(filp
->private_data
, ubuf
, cnt
, ppos
);
4287 tracing_max_lat_write(struct file
*filp
, const char __user
*ubuf
,
4288 size_t cnt
, loff_t
*ppos
)
4290 return tracing_nsecs_write(filp
->private_data
, ubuf
, cnt
, ppos
);
4293 static int tracing_open_pipe(struct inode
*inode
, struct file
*filp
)
4295 struct trace_array
*tr
= inode
->i_private
;
4296 struct trace_iterator
*iter
;
4299 if (tracing_disabled
)
4302 if (trace_array_get(tr
) < 0)
4305 mutex_lock(&trace_types_lock
);
4307 /* create a buffer to store the information to pass to userspace */
4308 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
4311 __trace_array_put(tr
);
4316 * We make a copy of the current tracer to avoid concurrent
4317 * changes on it while we are reading.
4319 iter
->trace
= kmalloc(sizeof(*iter
->trace
), GFP_KERNEL
);
4324 *iter
->trace
= *tr
->current_trace
;
4326 if (!alloc_cpumask_var(&iter
->started
, GFP_KERNEL
)) {
4331 /* trace pipe does not show start of buffer */
4332 cpumask_setall(iter
->started
);
4334 if (trace_flags
& TRACE_ITER_LATENCY_FMT
)
4335 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
4337 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4338 if (trace_clocks
[tr
->clock_id
].in_ns
)
4339 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
4342 iter
->trace_buffer
= &tr
->trace_buffer
;
4343 iter
->cpu_file
= tracing_get_cpu(inode
);
4344 mutex_init(&iter
->mutex
);
4345 filp
->private_data
= iter
;
4347 if (iter
->trace
->pipe_open
)
4348 iter
->trace
->pipe_open(iter
);
4350 nonseekable_open(inode
, filp
);
4352 mutex_unlock(&trace_types_lock
);
4358 __trace_array_put(tr
);
4359 mutex_unlock(&trace_types_lock
);
4363 static int tracing_release_pipe(struct inode
*inode
, struct file
*file
)
4365 struct trace_iterator
*iter
= file
->private_data
;
4366 struct trace_array
*tr
= inode
->i_private
;
4368 mutex_lock(&trace_types_lock
);
4370 if (iter
->trace
->pipe_close
)
4371 iter
->trace
->pipe_close(iter
);
4373 mutex_unlock(&trace_types_lock
);
4375 free_cpumask_var(iter
->started
);
4376 mutex_destroy(&iter
->mutex
);
4380 trace_array_put(tr
);
4386 trace_poll(struct trace_iterator
*iter
, struct file
*filp
, poll_table
*poll_table
)
4388 /* Iterators are static, they should be filled or empty */
4389 if (trace_buffer_iter(iter
, iter
->cpu_file
))
4390 return POLLIN
| POLLRDNORM
;
4392 if (trace_flags
& TRACE_ITER_BLOCK
)
4394 * Always select as readable when in blocking mode
4396 return POLLIN
| POLLRDNORM
;
4398 return ring_buffer_poll_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
4403 tracing_poll_pipe(struct file
*filp
, poll_table
*poll_table
)
4405 struct trace_iterator
*iter
= filp
->private_data
;
4407 return trace_poll(iter
, filp
, poll_table
);
4410 /* Must be called with trace_types_lock mutex held. */
4411 static int tracing_wait_pipe(struct file
*filp
)
4413 struct trace_iterator
*iter
= filp
->private_data
;
4416 while (trace_empty(iter
)) {
4418 if ((filp
->f_flags
& O_NONBLOCK
)) {
4423 * We block until we read something and tracing is disabled.
4424 * We still block if tracing is disabled, but we have never
4425 * read anything. This allows a user to cat this file, and
4426 * then enable tracing. But after we have read something,
4427 * we give an EOF when tracing is again disabled.
4429 * iter->pos will be 0 if we haven't read anything.
4431 if (!tracing_is_on() && iter
->pos
)
4434 mutex_unlock(&iter
->mutex
);
4436 ret
= wait_on_pipe(iter
);
4438 mutex_lock(&iter
->mutex
);
4443 if (signal_pending(current
))
4454 tracing_read_pipe(struct file
*filp
, char __user
*ubuf
,
4455 size_t cnt
, loff_t
*ppos
)
4457 struct trace_iterator
*iter
= filp
->private_data
;
4458 struct trace_array
*tr
= iter
->tr
;
4461 /* return any leftover data */
4462 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
4466 trace_seq_init(&iter
->seq
);
4468 /* copy the tracer to avoid using a global lock all around */
4469 mutex_lock(&trace_types_lock
);
4470 if (unlikely(iter
->trace
->name
!= tr
->current_trace
->name
))
4471 *iter
->trace
= *tr
->current_trace
;
4472 mutex_unlock(&trace_types_lock
);
4475 * Avoid more than one consumer on a single file descriptor
4476 * This is just a matter of traces coherency, the ring buffer itself
4479 mutex_lock(&iter
->mutex
);
4480 if (iter
->trace
->read
) {
4481 sret
= iter
->trace
->read(iter
, filp
, ubuf
, cnt
, ppos
);
4487 sret
= tracing_wait_pipe(filp
);
4491 /* stop when tracing is finished */
4492 if (trace_empty(iter
)) {
4497 if (cnt
>= PAGE_SIZE
)
4498 cnt
= PAGE_SIZE
- 1;
4500 /* reset all but tr, trace, and overruns */
4501 memset(&iter
->seq
, 0,
4502 sizeof(struct trace_iterator
) -
4503 offsetof(struct trace_iterator
, seq
));
4504 cpumask_clear(iter
->started
);
4507 trace_event_read_lock();
4508 trace_access_lock(iter
->cpu_file
);
4509 while (trace_find_next_entry_inc(iter
) != NULL
) {
4510 enum print_line_t ret
;
4511 int len
= iter
->seq
.len
;
4513 ret
= print_trace_line(iter
);
4514 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
4515 /* don't print partial lines */
4516 iter
->seq
.len
= len
;
4519 if (ret
!= TRACE_TYPE_NO_CONSUME
)
4520 trace_consume(iter
);
4522 if (iter
->seq
.len
>= cnt
)
4526 * Setting the full flag means we reached the trace_seq buffer
4527 * size and we should leave by partial output condition above.
4528 * One of the trace_seq_* functions is not used properly.
4530 WARN_ONCE(iter
->seq
.full
, "full flag set for trace type %d",
4533 trace_access_unlock(iter
->cpu_file
);
4534 trace_event_read_unlock();
4536 /* Now copy what we have to the user */
4537 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
4538 if (iter
->seq
.readpos
>= iter
->seq
.len
)
4539 trace_seq_init(&iter
->seq
);
4542 * If there was nothing to send to user, in spite of consuming trace
4543 * entries, go back to wait for more entries.
4549 mutex_unlock(&iter
->mutex
);
4554 static void tracing_spd_release_pipe(struct splice_pipe_desc
*spd
,
4557 __free_page(spd
->pages
[idx
]);
4560 static const struct pipe_buf_operations tracing_pipe_buf_ops
= {
4562 .confirm
= generic_pipe_buf_confirm
,
4563 .release
= generic_pipe_buf_release
,
4564 .steal
= generic_pipe_buf_steal
,
4565 .get
= generic_pipe_buf_get
,
4569 tracing_fill_pipe_page(size_t rem
, struct trace_iterator
*iter
)
4574 /* Seq buffer is page-sized, exactly what we need. */
4576 count
= iter
->seq
.len
;
4577 ret
= print_trace_line(iter
);
4578 count
= iter
->seq
.len
- count
;
4581 iter
->seq
.len
-= count
;
4584 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
4585 iter
->seq
.len
-= count
;
4589 if (ret
!= TRACE_TYPE_NO_CONSUME
)
4590 trace_consume(iter
);
4592 if (!trace_find_next_entry_inc(iter
)) {
4602 static ssize_t
tracing_splice_read_pipe(struct file
*filp
,
4604 struct pipe_inode_info
*pipe
,
4608 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
4609 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
4610 struct trace_iterator
*iter
= filp
->private_data
;
4611 struct splice_pipe_desc spd
= {
4613 .partial
= partial_def
,
4614 .nr_pages
= 0, /* This gets updated below. */
4615 .nr_pages_max
= PIPE_DEF_BUFFERS
,
4617 .ops
= &tracing_pipe_buf_ops
,
4618 .spd_release
= tracing_spd_release_pipe
,
4620 struct trace_array
*tr
= iter
->tr
;
4625 if (splice_grow_spd(pipe
, &spd
))
4628 /* copy the tracer to avoid using a global lock all around */
4629 mutex_lock(&trace_types_lock
);
4630 if (unlikely(iter
->trace
->name
!= tr
->current_trace
->name
))
4631 *iter
->trace
= *tr
->current_trace
;
4632 mutex_unlock(&trace_types_lock
);
4634 mutex_lock(&iter
->mutex
);
4636 if (iter
->trace
->splice_read
) {
4637 ret
= iter
->trace
->splice_read(iter
, filp
,
4638 ppos
, pipe
, len
, flags
);
4643 ret
= tracing_wait_pipe(filp
);
4647 if (!iter
->ent
&& !trace_find_next_entry_inc(iter
)) {
4652 trace_event_read_lock();
4653 trace_access_lock(iter
->cpu_file
);
4655 /* Fill as many pages as possible. */
4656 for (i
= 0, rem
= len
; i
< spd
.nr_pages_max
&& rem
; i
++) {
4657 spd
.pages
[i
] = alloc_page(GFP_KERNEL
);
4661 rem
= tracing_fill_pipe_page(rem
, iter
);
4663 /* Copy the data into the page, so we can start over. */
4664 ret
= trace_seq_to_buffer(&iter
->seq
,
4665 page_address(spd
.pages
[i
]),
4668 __free_page(spd
.pages
[i
]);
4671 spd
.partial
[i
].offset
= 0;
4672 spd
.partial
[i
].len
= iter
->seq
.len
;
4674 trace_seq_init(&iter
->seq
);
4677 trace_access_unlock(iter
->cpu_file
);
4678 trace_event_read_unlock();
4679 mutex_unlock(&iter
->mutex
);
4683 ret
= splice_to_pipe(pipe
, &spd
);
4685 splice_shrink_spd(&spd
);
4689 mutex_unlock(&iter
->mutex
);
4694 tracing_entries_read(struct file
*filp
, char __user
*ubuf
,
4695 size_t cnt
, loff_t
*ppos
)
4697 struct inode
*inode
= file_inode(filp
);
4698 struct trace_array
*tr
= inode
->i_private
;
4699 int cpu
= tracing_get_cpu(inode
);
4704 mutex_lock(&trace_types_lock
);
4706 if (cpu
== RING_BUFFER_ALL_CPUS
) {
4707 int cpu
, buf_size_same
;
4712 /* check if all cpu sizes are same */
4713 for_each_tracing_cpu(cpu
) {
4714 /* fill in the size from first enabled cpu */
4716 size
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
;
4717 if (size
!= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
) {
4723 if (buf_size_same
) {
4724 if (!ring_buffer_expanded
)
4725 r
= sprintf(buf
, "%lu (expanded: %lu)\n",
4727 trace_buf_size
>> 10);
4729 r
= sprintf(buf
, "%lu\n", size
>> 10);
4731 r
= sprintf(buf
, "X\n");
4733 r
= sprintf(buf
, "%lu\n", per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10);
4735 mutex_unlock(&trace_types_lock
);
4737 ret
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4742 tracing_entries_write(struct file
*filp
, const char __user
*ubuf
,
4743 size_t cnt
, loff_t
*ppos
)
4745 struct inode
*inode
= file_inode(filp
);
4746 struct trace_array
*tr
= inode
->i_private
;
4750 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4754 /* must have at least 1 entry */
4758 /* value is in KB */
4760 ret
= tracing_resize_ring_buffer(tr
, val
, tracing_get_cpu(inode
));
4770 tracing_total_entries_read(struct file
*filp
, char __user
*ubuf
,
4771 size_t cnt
, loff_t
*ppos
)
4773 struct trace_array
*tr
= filp
->private_data
;
4776 unsigned long size
= 0, expanded_size
= 0;
4778 mutex_lock(&trace_types_lock
);
4779 for_each_tracing_cpu(cpu
) {
4780 size
+= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10;
4781 if (!ring_buffer_expanded
)
4782 expanded_size
+= trace_buf_size
>> 10;
4784 if (ring_buffer_expanded
)
4785 r
= sprintf(buf
, "%lu\n", size
);
4787 r
= sprintf(buf
, "%lu (expanded: %lu)\n", size
, expanded_size
);
4788 mutex_unlock(&trace_types_lock
);
4790 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4794 tracing_free_buffer_write(struct file
*filp
, const char __user
*ubuf
,
4795 size_t cnt
, loff_t
*ppos
)
4798 * There is no need to read what the user has written, this function
4799 * is just to make sure that there is no error when "echo" is used
4808 tracing_free_buffer_release(struct inode
*inode
, struct file
*filp
)
4810 struct trace_array
*tr
= inode
->i_private
;
4812 /* disable tracing ? */
4813 if (trace_flags
& TRACE_ITER_STOP_ON_FREE
)
4814 tracer_tracing_off(tr
);
4815 /* resize the ring buffer to 0 */
4816 tracing_resize_ring_buffer(tr
, 0, RING_BUFFER_ALL_CPUS
);
4818 trace_array_put(tr
);
4824 tracing_mark_write(struct file
*filp
, const char __user
*ubuf
,
4825 size_t cnt
, loff_t
*fpos
)
4827 unsigned long addr
= (unsigned long)ubuf
;
4828 struct trace_array
*tr
= filp
->private_data
;
4829 struct ring_buffer_event
*event
;
4830 struct ring_buffer
*buffer
;
4831 struct print_entry
*entry
;
4832 unsigned long irq_flags
;
4833 struct page
*pages
[2];
4843 if (tracing_disabled
)
4846 if (!(trace_flags
& TRACE_ITER_MARKERS
))
4849 if (cnt
> TRACE_BUF_SIZE
)
4850 cnt
= TRACE_BUF_SIZE
;
4853 * Userspace is injecting traces into the kernel trace buffer.
4854 * We want to be as non intrusive as possible.
4855 * To do so, we do not want to allocate any special buffers
4856 * or take any locks, but instead write the userspace data
4857 * straight into the ring buffer.
4859 * First we need to pin the userspace buffer into memory,
4860 * which, most likely it is, because it just referenced it.
4861 * But there's no guarantee that it is. By using get_user_pages_fast()
4862 * and kmap_atomic/kunmap_atomic() we can get access to the
4863 * pages directly. We then write the data directly into the
4866 BUILD_BUG_ON(TRACE_BUF_SIZE
>= PAGE_SIZE
);
4868 /* check if we cross pages */
4869 if ((addr
& PAGE_MASK
) != ((addr
+ cnt
) & PAGE_MASK
))
4872 offset
= addr
& (PAGE_SIZE
- 1);
4875 ret
= get_user_pages_fast(addr
, nr_pages
, 0, pages
);
4876 if (ret
< nr_pages
) {
4878 put_page(pages
[ret
]);
4883 for (i
= 0; i
< nr_pages
; i
++)
4884 map_page
[i
] = kmap_atomic(pages
[i
]);
4886 local_save_flags(irq_flags
);
4887 size
= sizeof(*entry
) + cnt
+ 2; /* possible \n added */
4888 buffer
= tr
->trace_buffer
.buffer
;
4889 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
4890 irq_flags
, preempt_count());
4892 /* Ring buffer disabled, return as if not open for write */
4897 entry
= ring_buffer_event_data(event
);
4898 entry
->ip
= _THIS_IP_
;
4900 if (nr_pages
== 2) {
4901 len
= PAGE_SIZE
- offset
;
4902 memcpy(&entry
->buf
, map_page
[0] + offset
, len
);
4903 memcpy(&entry
->buf
[len
], map_page
[1], cnt
- len
);
4905 memcpy(&entry
->buf
, map_page
[0] + offset
, cnt
);
4907 if (entry
->buf
[cnt
- 1] != '\n') {
4908 entry
->buf
[cnt
] = '\n';
4909 entry
->buf
[cnt
+ 1] = '\0';
4911 entry
->buf
[cnt
] = '\0';
4913 __buffer_unlock_commit(buffer
, event
);
4920 for (i
= 0; i
< nr_pages
; i
++){
4921 kunmap_atomic(map_page
[i
]);
4928 static int tracing_clock_show(struct seq_file
*m
, void *v
)
4930 struct trace_array
*tr
= m
->private;
4933 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++)
4935 "%s%s%s%s", i
? " " : "",
4936 i
== tr
->clock_id
? "[" : "", trace_clocks
[i
].name
,
4937 i
== tr
->clock_id
? "]" : "");
4943 static int tracing_set_clock(struct trace_array
*tr
, const char *clockstr
)
4947 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++) {
4948 if (strcmp(trace_clocks
[i
].name
, clockstr
) == 0)
4951 if (i
== ARRAY_SIZE(trace_clocks
))
4954 mutex_lock(&trace_types_lock
);
4958 ring_buffer_set_clock(tr
->trace_buffer
.buffer
, trace_clocks
[i
].func
);
4961 * New clock may not be consistent with the previous clock.
4962 * Reset the buffer so that it doesn't have incomparable timestamps.
4964 tracing_reset_online_cpus(&tr
->trace_buffer
);
4966 #ifdef CONFIG_TRACER_MAX_TRACE
4967 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
&& tr
->max_buffer
.buffer
)
4968 ring_buffer_set_clock(tr
->max_buffer
.buffer
, trace_clocks
[i
].func
);
4969 tracing_reset_online_cpus(&tr
->max_buffer
);
4972 mutex_unlock(&trace_types_lock
);
4977 static ssize_t
tracing_clock_write(struct file
*filp
, const char __user
*ubuf
,
4978 size_t cnt
, loff_t
*fpos
)
4980 struct seq_file
*m
= filp
->private_data
;
4981 struct trace_array
*tr
= m
->private;
4983 const char *clockstr
;
4986 if (cnt
>= sizeof(buf
))
4989 if (copy_from_user(&buf
, ubuf
, cnt
))
4994 clockstr
= strstrip(buf
);
4996 ret
= tracing_set_clock(tr
, clockstr
);
5005 static int tracing_clock_open(struct inode
*inode
, struct file
*file
)
5007 struct trace_array
*tr
= inode
->i_private
;
5010 if (tracing_disabled
)
5013 if (trace_array_get(tr
))
5016 ret
= single_open(file
, tracing_clock_show
, inode
->i_private
);
5018 trace_array_put(tr
);
5023 struct ftrace_buffer_info
{
5024 struct trace_iterator iter
;
5029 #ifdef CONFIG_TRACER_SNAPSHOT
5030 static int tracing_snapshot_open(struct inode
*inode
, struct file
*file
)
5032 struct trace_array
*tr
= inode
->i_private
;
5033 struct trace_iterator
*iter
;
5037 if (trace_array_get(tr
) < 0)
5040 if (file
->f_mode
& FMODE_READ
) {
5041 iter
= __tracing_open(inode
, file
, true);
5043 ret
= PTR_ERR(iter
);
5045 /* Writes still need the seq_file to hold the private data */
5047 m
= kzalloc(sizeof(*m
), GFP_KERNEL
);
5050 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
5058 iter
->trace_buffer
= &tr
->max_buffer
;
5059 iter
->cpu_file
= tracing_get_cpu(inode
);
5061 file
->private_data
= m
;
5065 trace_array_put(tr
);
5071 tracing_snapshot_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
5074 struct seq_file
*m
= filp
->private_data
;
5075 struct trace_iterator
*iter
= m
->private;
5076 struct trace_array
*tr
= iter
->tr
;
5080 ret
= tracing_update_buffers();
5084 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5088 mutex_lock(&trace_types_lock
);
5090 if (tr
->current_trace
->use_max_tr
) {
5097 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
5101 if (tr
->allocated_snapshot
)
5105 /* Only allow per-cpu swap if the ring buffer supports it */
5106 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5107 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
5112 if (!tr
->allocated_snapshot
) {
5113 ret
= alloc_snapshot(tr
);
5117 local_irq_disable();
5118 /* Now, we're going to swap */
5119 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
5120 update_max_tr(tr
, current
, smp_processor_id());
5122 update_max_tr_single(tr
, current
, iter
->cpu_file
);
5126 if (tr
->allocated_snapshot
) {
5127 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
5128 tracing_reset_online_cpus(&tr
->max_buffer
);
5130 tracing_reset(&tr
->max_buffer
, iter
->cpu_file
);
5140 mutex_unlock(&trace_types_lock
);
5144 static int tracing_snapshot_release(struct inode
*inode
, struct file
*file
)
5146 struct seq_file
*m
= file
->private_data
;
5149 ret
= tracing_release(inode
, file
);
5151 if (file
->f_mode
& FMODE_READ
)
5154 /* If write only, the seq_file is just a stub */
5162 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
);
5163 static ssize_t
tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
5164 size_t count
, loff_t
*ppos
);
5165 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
);
5166 static ssize_t
tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
5167 struct pipe_inode_info
*pipe
, size_t len
, unsigned int flags
);
5169 static int snapshot_raw_open(struct inode
*inode
, struct file
*filp
)
5171 struct ftrace_buffer_info
*info
;
5174 ret
= tracing_buffers_open(inode
, filp
);
5178 info
= filp
->private_data
;
5180 if (info
->iter
.trace
->use_max_tr
) {
5181 tracing_buffers_release(inode
, filp
);
5185 info
->iter
.snapshot
= true;
5186 info
->iter
.trace_buffer
= &info
->iter
.tr
->max_buffer
;
5191 #endif /* CONFIG_TRACER_SNAPSHOT */
5194 static const struct file_operations tracing_thresh_fops
= {
5195 .open
= tracing_open_generic
,
5196 .read
= tracing_thresh_read
,
5197 .write
= tracing_thresh_write
,
5198 .llseek
= generic_file_llseek
,
5201 static const struct file_operations tracing_max_lat_fops
= {
5202 .open
= tracing_open_generic
,
5203 .read
= tracing_max_lat_read
,
5204 .write
= tracing_max_lat_write
,
5205 .llseek
= generic_file_llseek
,
5208 static const struct file_operations set_tracer_fops
= {
5209 .open
= tracing_open_generic
,
5210 .read
= tracing_set_trace_read
,
5211 .write
= tracing_set_trace_write
,
5212 .llseek
= generic_file_llseek
,
5215 static const struct file_operations tracing_pipe_fops
= {
5216 .open
= tracing_open_pipe
,
5217 .poll
= tracing_poll_pipe
,
5218 .read
= tracing_read_pipe
,
5219 .splice_read
= tracing_splice_read_pipe
,
5220 .release
= tracing_release_pipe
,
5221 .llseek
= no_llseek
,
5224 static const struct file_operations tracing_entries_fops
= {
5225 .open
= tracing_open_generic_tr
,
5226 .read
= tracing_entries_read
,
5227 .write
= tracing_entries_write
,
5228 .llseek
= generic_file_llseek
,
5229 .release
= tracing_release_generic_tr
,
5232 static const struct file_operations tracing_total_entries_fops
= {
5233 .open
= tracing_open_generic_tr
,
5234 .read
= tracing_total_entries_read
,
5235 .llseek
= generic_file_llseek
,
5236 .release
= tracing_release_generic_tr
,
5239 static const struct file_operations tracing_free_buffer_fops
= {
5240 .open
= tracing_open_generic_tr
,
5241 .write
= tracing_free_buffer_write
,
5242 .release
= tracing_free_buffer_release
,
5245 static const struct file_operations tracing_mark_fops
= {
5246 .open
= tracing_open_generic_tr
,
5247 .write
= tracing_mark_write
,
5248 .llseek
= generic_file_llseek
,
5249 .release
= tracing_release_generic_tr
,
5252 static const struct file_operations trace_clock_fops
= {
5253 .open
= tracing_clock_open
,
5255 .llseek
= seq_lseek
,
5256 .release
= tracing_single_release_tr
,
5257 .write
= tracing_clock_write
,
5260 #ifdef CONFIG_TRACER_SNAPSHOT
5261 static const struct file_operations snapshot_fops
= {
5262 .open
= tracing_snapshot_open
,
5264 .write
= tracing_snapshot_write
,
5265 .llseek
= tracing_lseek
,
5266 .release
= tracing_snapshot_release
,
5269 static const struct file_operations snapshot_raw_fops
= {
5270 .open
= snapshot_raw_open
,
5271 .read
= tracing_buffers_read
,
5272 .release
= tracing_buffers_release
,
5273 .splice_read
= tracing_buffers_splice_read
,
5274 .llseek
= no_llseek
,
5277 #endif /* CONFIG_TRACER_SNAPSHOT */
5279 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
)
5281 struct trace_array
*tr
= inode
->i_private
;
5282 struct ftrace_buffer_info
*info
;
5285 if (tracing_disabled
)
5288 if (trace_array_get(tr
) < 0)
5291 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
5293 trace_array_put(tr
);
5297 mutex_lock(&trace_types_lock
);
5300 info
->iter
.cpu_file
= tracing_get_cpu(inode
);
5301 info
->iter
.trace
= tr
->current_trace
;
5302 info
->iter
.trace_buffer
= &tr
->trace_buffer
;
5304 /* Force reading ring buffer for first read */
5305 info
->read
= (unsigned int)-1;
5307 filp
->private_data
= info
;
5309 mutex_unlock(&trace_types_lock
);
5311 ret
= nonseekable_open(inode
, filp
);
5313 trace_array_put(tr
);
5319 tracing_buffers_poll(struct file
*filp
, poll_table
*poll_table
)
5321 struct ftrace_buffer_info
*info
= filp
->private_data
;
5322 struct trace_iterator
*iter
= &info
->iter
;
5324 return trace_poll(iter
, filp
, poll_table
);
5328 tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
5329 size_t count
, loff_t
*ppos
)
5331 struct ftrace_buffer_info
*info
= filp
->private_data
;
5332 struct trace_iterator
*iter
= &info
->iter
;
5339 mutex_lock(&trace_types_lock
);
5341 #ifdef CONFIG_TRACER_MAX_TRACE
5342 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
) {
5349 info
->spare
= ring_buffer_alloc_read_page(iter
->trace_buffer
->buffer
,
5355 /* Do we have previous read data to read? */
5356 if (info
->read
< PAGE_SIZE
)
5360 trace_access_lock(iter
->cpu_file
);
5361 ret
= ring_buffer_read_page(iter
->trace_buffer
->buffer
,
5365 trace_access_unlock(iter
->cpu_file
);
5368 if (trace_empty(iter
)) {
5369 if ((filp
->f_flags
& O_NONBLOCK
)) {
5373 mutex_unlock(&trace_types_lock
);
5374 ret
= wait_on_pipe(iter
);
5375 mutex_lock(&trace_types_lock
);
5380 if (signal_pending(current
)) {
5392 size
= PAGE_SIZE
- info
->read
;
5396 ret
= copy_to_user(ubuf
, info
->spare
+ info
->read
, size
);
5407 mutex_unlock(&trace_types_lock
);
5412 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
)
5414 struct ftrace_buffer_info
*info
= file
->private_data
;
5415 struct trace_iterator
*iter
= &info
->iter
;
5417 mutex_lock(&trace_types_lock
);
5419 __trace_array_put(iter
->tr
);
5422 ring_buffer_free_read_page(iter
->trace_buffer
->buffer
, info
->spare
);
5425 mutex_unlock(&trace_types_lock
);
5431 struct ring_buffer
*buffer
;
5436 static void buffer_pipe_buf_release(struct pipe_inode_info
*pipe
,
5437 struct pipe_buffer
*buf
)
5439 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
5444 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5449 static void buffer_pipe_buf_get(struct pipe_inode_info
*pipe
,
5450 struct pipe_buffer
*buf
)
5452 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
5457 /* Pipe buffer operations for a buffer. */
5458 static const struct pipe_buf_operations buffer_pipe_buf_ops
= {
5460 .confirm
= generic_pipe_buf_confirm
,
5461 .release
= buffer_pipe_buf_release
,
5462 .steal
= generic_pipe_buf_steal
,
5463 .get
= buffer_pipe_buf_get
,
5467 * Callback from splice_to_pipe(), if we need to release some pages
5468 * at the end of the spd in case we error'ed out in filling the pipe.
5470 static void buffer_spd_release(struct splice_pipe_desc
*spd
, unsigned int i
)
5472 struct buffer_ref
*ref
=
5473 (struct buffer_ref
*)spd
->partial
[i
].private;
5478 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5480 spd
->partial
[i
].private = 0;
5484 tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
5485 struct pipe_inode_info
*pipe
, size_t len
,
5488 struct ftrace_buffer_info
*info
= file
->private_data
;
5489 struct trace_iterator
*iter
= &info
->iter
;
5490 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
5491 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
5492 struct splice_pipe_desc spd
= {
5494 .partial
= partial_def
,
5495 .nr_pages_max
= PIPE_DEF_BUFFERS
,
5497 .ops
= &buffer_pipe_buf_ops
,
5498 .spd_release
= buffer_spd_release
,
5500 struct buffer_ref
*ref
;
5501 int entries
, size
, i
;
5504 mutex_lock(&trace_types_lock
);
5506 #ifdef CONFIG_TRACER_MAX_TRACE
5507 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
) {
5513 if (splice_grow_spd(pipe
, &spd
)) {
5518 if (*ppos
& (PAGE_SIZE
- 1)) {
5523 if (len
& (PAGE_SIZE
- 1)) {
5524 if (len
< PAGE_SIZE
) {
5532 trace_access_lock(iter
->cpu_file
);
5533 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
5535 for (i
= 0; i
< spd
.nr_pages_max
&& len
&& entries
; i
++, len
-= PAGE_SIZE
) {
5539 ref
= kzalloc(sizeof(*ref
), GFP_KERNEL
);
5544 ref
->buffer
= iter
->trace_buffer
->buffer
;
5545 ref
->page
= ring_buffer_alloc_read_page(ref
->buffer
, iter
->cpu_file
);
5551 r
= ring_buffer_read_page(ref
->buffer
, &ref
->page
,
5552 len
, iter
->cpu_file
, 1);
5554 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5560 * zero out any left over data, this is going to
5563 size
= ring_buffer_page_len(ref
->page
);
5564 if (size
< PAGE_SIZE
)
5565 memset(ref
->page
+ size
, 0, PAGE_SIZE
- size
);
5567 page
= virt_to_page(ref
->page
);
5569 spd
.pages
[i
] = page
;
5570 spd
.partial
[i
].len
= PAGE_SIZE
;
5571 spd
.partial
[i
].offset
= 0;
5572 spd
.partial
[i
].private = (unsigned long)ref
;
5576 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
5579 trace_access_unlock(iter
->cpu_file
);
5582 /* did we read anything? */
5583 if (!spd
.nr_pages
) {
5584 if ((file
->f_flags
& O_NONBLOCK
) || (flags
& SPLICE_F_NONBLOCK
)) {
5588 mutex_unlock(&trace_types_lock
);
5589 ret
= wait_on_pipe(iter
);
5590 mutex_lock(&trace_types_lock
);
5593 if (signal_pending(current
)) {
5600 ret
= splice_to_pipe(pipe
, &spd
);
5601 splice_shrink_spd(&spd
);
5603 mutex_unlock(&trace_types_lock
);
5608 static const struct file_operations tracing_buffers_fops
= {
5609 .open
= tracing_buffers_open
,
5610 .read
= tracing_buffers_read
,
5611 .poll
= tracing_buffers_poll
,
5612 .release
= tracing_buffers_release
,
5613 .splice_read
= tracing_buffers_splice_read
,
5614 .llseek
= no_llseek
,
5618 tracing_stats_read(struct file
*filp
, char __user
*ubuf
,
5619 size_t count
, loff_t
*ppos
)
5621 struct inode
*inode
= file_inode(filp
);
5622 struct trace_array
*tr
= inode
->i_private
;
5623 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
5624 int cpu
= tracing_get_cpu(inode
);
5625 struct trace_seq
*s
;
5627 unsigned long long t
;
5628 unsigned long usec_rem
;
5630 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
5636 cnt
= ring_buffer_entries_cpu(trace_buf
->buffer
, cpu
);
5637 trace_seq_printf(s
, "entries: %ld\n", cnt
);
5639 cnt
= ring_buffer_overrun_cpu(trace_buf
->buffer
, cpu
);
5640 trace_seq_printf(s
, "overrun: %ld\n", cnt
);
5642 cnt
= ring_buffer_commit_overrun_cpu(trace_buf
->buffer
, cpu
);
5643 trace_seq_printf(s
, "commit overrun: %ld\n", cnt
);
5645 cnt
= ring_buffer_bytes_cpu(trace_buf
->buffer
, cpu
);
5646 trace_seq_printf(s
, "bytes: %ld\n", cnt
);
5648 if (trace_clocks
[tr
->clock_id
].in_ns
) {
5649 /* local or global for trace_clock */
5650 t
= ns2usecs(ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
5651 usec_rem
= do_div(t
, USEC_PER_SEC
);
5652 trace_seq_printf(s
, "oldest event ts: %5llu.%06lu\n",
5655 t
= ns2usecs(ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
5656 usec_rem
= do_div(t
, USEC_PER_SEC
);
5657 trace_seq_printf(s
, "now ts: %5llu.%06lu\n", t
, usec_rem
);
5659 /* counter or tsc mode for trace_clock */
5660 trace_seq_printf(s
, "oldest event ts: %llu\n",
5661 ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
5663 trace_seq_printf(s
, "now ts: %llu\n",
5664 ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
5667 cnt
= ring_buffer_dropped_events_cpu(trace_buf
->buffer
, cpu
);
5668 trace_seq_printf(s
, "dropped events: %ld\n", cnt
);
5670 cnt
= ring_buffer_read_events_cpu(trace_buf
->buffer
, cpu
);
5671 trace_seq_printf(s
, "read events: %ld\n", cnt
);
5673 count
= simple_read_from_buffer(ubuf
, count
, ppos
, s
->buffer
, s
->len
);
5680 static const struct file_operations tracing_stats_fops
= {
5681 .open
= tracing_open_generic_tr
,
5682 .read
= tracing_stats_read
,
5683 .llseek
= generic_file_llseek
,
5684 .release
= tracing_release_generic_tr
,
5687 #ifdef CONFIG_DYNAMIC_FTRACE
5689 int __weak
ftrace_arch_read_dyn_info(char *buf
, int size
)
5695 tracing_read_dyn_info(struct file
*filp
, char __user
*ubuf
,
5696 size_t cnt
, loff_t
*ppos
)
5698 static char ftrace_dyn_info_buffer
[1024];
5699 static DEFINE_MUTEX(dyn_info_mutex
);
5700 unsigned long *p
= filp
->private_data
;
5701 char *buf
= ftrace_dyn_info_buffer
;
5702 int size
= ARRAY_SIZE(ftrace_dyn_info_buffer
);
5705 mutex_lock(&dyn_info_mutex
);
5706 r
= sprintf(buf
, "%ld ", *p
);
5708 r
+= ftrace_arch_read_dyn_info(buf
+r
, (size
-1)-r
);
5711 r
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5713 mutex_unlock(&dyn_info_mutex
);
5718 static const struct file_operations tracing_dyn_info_fops
= {
5719 .open
= tracing_open_generic
,
5720 .read
= tracing_read_dyn_info
,
5721 .llseek
= generic_file_llseek
,
5723 #endif /* CONFIG_DYNAMIC_FTRACE */
5725 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5727 ftrace_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
5733 ftrace_count_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
5735 unsigned long *count
= (long *)data
;
5747 ftrace_snapshot_print(struct seq_file
*m
, unsigned long ip
,
5748 struct ftrace_probe_ops
*ops
, void *data
)
5750 long count
= (long)data
;
5752 seq_printf(m
, "%ps:", (void *)ip
);
5754 seq_puts(m
, "snapshot");
5757 seq_puts(m
, ":unlimited\n");
5759 seq_printf(m
, ":count=%ld\n", count
);
5764 static struct ftrace_probe_ops snapshot_probe_ops
= {
5765 .func
= ftrace_snapshot
,
5766 .print
= ftrace_snapshot_print
,
5769 static struct ftrace_probe_ops snapshot_count_probe_ops
= {
5770 .func
= ftrace_count_snapshot
,
5771 .print
= ftrace_snapshot_print
,
5775 ftrace_trace_snapshot_callback(struct ftrace_hash
*hash
,
5776 char *glob
, char *cmd
, char *param
, int enable
)
5778 struct ftrace_probe_ops
*ops
;
5779 void *count
= (void *)-1;
5783 /* hash funcs only work with set_ftrace_filter */
5787 ops
= param
? &snapshot_count_probe_ops
: &snapshot_probe_ops
;
5789 if (glob
[0] == '!') {
5790 unregister_ftrace_function_probe_func(glob
+1, ops
);
5797 number
= strsep(¶m
, ":");
5799 if (!strlen(number
))
5803 * We use the callback data field (which is a pointer)
5806 ret
= kstrtoul(number
, 0, (unsigned long *)&count
);
5811 ret
= register_ftrace_function_probe(glob
, ops
, count
);
5814 alloc_snapshot(&global_trace
);
5816 return ret
< 0 ? ret
: 0;
5819 static struct ftrace_func_command ftrace_snapshot_cmd
= {
5821 .func
= ftrace_trace_snapshot_callback
,
5824 static __init
int register_snapshot_cmd(void)
5826 return register_ftrace_command(&ftrace_snapshot_cmd
);
5829 static inline __init
int register_snapshot_cmd(void) { return 0; }
5830 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5832 struct dentry
*tracing_init_dentry_tr(struct trace_array
*tr
)
5837 if (!debugfs_initialized())
5840 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
5841 tr
->dir
= debugfs_create_dir("tracing", NULL
);
5844 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5849 struct dentry
*tracing_init_dentry(void)
5851 return tracing_init_dentry_tr(&global_trace
);
5854 static struct dentry
*tracing_dentry_percpu(struct trace_array
*tr
, int cpu
)
5856 struct dentry
*d_tracer
;
5859 return tr
->percpu_dir
;
5861 d_tracer
= tracing_init_dentry_tr(tr
);
5865 tr
->percpu_dir
= debugfs_create_dir("per_cpu", d_tracer
);
5867 WARN_ONCE(!tr
->percpu_dir
,
5868 "Could not create debugfs directory 'per_cpu/%d'\n", cpu
);
5870 return tr
->percpu_dir
;
5873 static struct dentry
*
5874 trace_create_cpu_file(const char *name
, umode_t mode
, struct dentry
*parent
,
5875 void *data
, long cpu
, const struct file_operations
*fops
)
5877 struct dentry
*ret
= trace_create_file(name
, mode
, parent
, data
, fops
);
5879 if (ret
) /* See tracing_get_cpu() */
5880 ret
->d_inode
->i_cdev
= (void *)(cpu
+ 1);
5885 tracing_init_debugfs_percpu(struct trace_array
*tr
, long cpu
)
5887 struct dentry
*d_percpu
= tracing_dentry_percpu(tr
, cpu
);
5888 struct dentry
*d_cpu
;
5889 char cpu_dir
[30]; /* 30 characters should be more than enough */
5894 snprintf(cpu_dir
, 30, "cpu%ld", cpu
);
5895 d_cpu
= debugfs_create_dir(cpu_dir
, d_percpu
);
5897 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir
);
5901 /* per cpu trace_pipe */
5902 trace_create_cpu_file("trace_pipe", 0444, d_cpu
,
5903 tr
, cpu
, &tracing_pipe_fops
);
5906 trace_create_cpu_file("trace", 0644, d_cpu
,
5907 tr
, cpu
, &tracing_fops
);
5909 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu
,
5910 tr
, cpu
, &tracing_buffers_fops
);
5912 trace_create_cpu_file("stats", 0444, d_cpu
,
5913 tr
, cpu
, &tracing_stats_fops
);
5915 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu
,
5916 tr
, cpu
, &tracing_entries_fops
);
5918 #ifdef CONFIG_TRACER_SNAPSHOT
5919 trace_create_cpu_file("snapshot", 0644, d_cpu
,
5920 tr
, cpu
, &snapshot_fops
);
5922 trace_create_cpu_file("snapshot_raw", 0444, d_cpu
,
5923 tr
, cpu
, &snapshot_raw_fops
);
5927 #ifdef CONFIG_FTRACE_SELFTEST
5928 /* Let selftest have access to static functions in this file */
5929 #include "trace_selftest.c"
5932 struct trace_option_dentry
{
5933 struct tracer_opt
*opt
;
5934 struct tracer_flags
*flags
;
5935 struct trace_array
*tr
;
5936 struct dentry
*entry
;
5940 trace_options_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
5943 struct trace_option_dentry
*topt
= filp
->private_data
;
5946 if (topt
->flags
->val
& topt
->opt
->bit
)
5951 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
5955 trace_options_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
5958 struct trace_option_dentry
*topt
= filp
->private_data
;
5962 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5966 if (val
!= 0 && val
!= 1)
5969 if (!!(topt
->flags
->val
& topt
->opt
->bit
) != val
) {
5970 mutex_lock(&trace_types_lock
);
5971 ret
= __set_tracer_option(topt
->tr
, topt
->flags
,
5973 mutex_unlock(&trace_types_lock
);
5984 static const struct file_operations trace_options_fops
= {
5985 .open
= tracing_open_generic
,
5986 .read
= trace_options_read
,
5987 .write
= trace_options_write
,
5988 .llseek
= generic_file_llseek
,
5992 trace_options_core_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
5995 long index
= (long)filp
->private_data
;
5998 if (trace_flags
& (1 << index
))
6003 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
6007 trace_options_core_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6010 struct trace_array
*tr
= &global_trace
;
6011 long index
= (long)filp
->private_data
;
6015 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6019 if (val
!= 0 && val
!= 1)
6022 mutex_lock(&trace_types_lock
);
6023 ret
= set_tracer_flag(tr
, 1 << index
, val
);
6024 mutex_unlock(&trace_types_lock
);
6034 static const struct file_operations trace_options_core_fops
= {
6035 .open
= tracing_open_generic
,
6036 .read
= trace_options_core_read
,
6037 .write
= trace_options_core_write
,
6038 .llseek
= generic_file_llseek
,
6041 struct dentry
*trace_create_file(const char *name
,
6043 struct dentry
*parent
,
6045 const struct file_operations
*fops
)
6049 ret
= debugfs_create_file(name
, mode
, parent
, data
, fops
);
6051 pr_warning("Could not create debugfs '%s' entry\n", name
);
6057 static struct dentry
*trace_options_init_dentry(struct trace_array
*tr
)
6059 struct dentry
*d_tracer
;
6064 d_tracer
= tracing_init_dentry_tr(tr
);
6068 tr
->options
= debugfs_create_dir("options", d_tracer
);
6070 pr_warning("Could not create debugfs directory 'options'\n");
6078 create_trace_option_file(struct trace_array
*tr
,
6079 struct trace_option_dentry
*topt
,
6080 struct tracer_flags
*flags
,
6081 struct tracer_opt
*opt
)
6083 struct dentry
*t_options
;
6085 t_options
= trace_options_init_dentry(tr
);
6089 topt
->flags
= flags
;
6093 topt
->entry
= trace_create_file(opt
->name
, 0644, t_options
, topt
,
6094 &trace_options_fops
);
6098 static struct trace_option_dentry
*
6099 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
)
6101 struct trace_option_dentry
*topts
;
6102 struct tracer_flags
*flags
;
6103 struct tracer_opt
*opts
;
6109 flags
= tracer
->flags
;
6111 if (!flags
|| !flags
->opts
)
6116 for (cnt
= 0; opts
[cnt
].name
; cnt
++)
6119 topts
= kcalloc(cnt
+ 1, sizeof(*topts
), GFP_KERNEL
);
6123 for (cnt
= 0; opts
[cnt
].name
; cnt
++)
6124 create_trace_option_file(tr
, &topts
[cnt
], flags
,
6131 destroy_trace_option_files(struct trace_option_dentry
*topts
)
6138 for (cnt
= 0; topts
[cnt
].opt
; cnt
++)
6139 debugfs_remove(topts
[cnt
].entry
);
6144 static struct dentry
*
6145 create_trace_option_core_file(struct trace_array
*tr
,
6146 const char *option
, long index
)
6148 struct dentry
*t_options
;
6150 t_options
= trace_options_init_dentry(tr
);
6154 return trace_create_file(option
, 0644, t_options
, (void *)index
,
6155 &trace_options_core_fops
);
6158 static __init
void create_trace_options_dir(struct trace_array
*tr
)
6160 struct dentry
*t_options
;
6163 t_options
= trace_options_init_dentry(tr
);
6167 for (i
= 0; trace_options
[i
]; i
++)
6168 create_trace_option_core_file(tr
, trace_options
[i
], i
);
6172 rb_simple_read(struct file
*filp
, char __user
*ubuf
,
6173 size_t cnt
, loff_t
*ppos
)
6175 struct trace_array
*tr
= filp
->private_data
;
6179 r
= tracer_tracing_is_on(tr
);
6180 r
= sprintf(buf
, "%d\n", r
);
6182 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6186 rb_simple_write(struct file
*filp
, const char __user
*ubuf
,
6187 size_t cnt
, loff_t
*ppos
)
6189 struct trace_array
*tr
= filp
->private_data
;
6190 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
6194 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6199 mutex_lock(&trace_types_lock
);
6201 tracer_tracing_on(tr
);
6202 if (tr
->current_trace
->start
)
6203 tr
->current_trace
->start(tr
);
6205 tracer_tracing_off(tr
);
6206 if (tr
->current_trace
->stop
)
6207 tr
->current_trace
->stop(tr
);
6209 mutex_unlock(&trace_types_lock
);
6217 static const struct file_operations rb_simple_fops
= {
6218 .open
= tracing_open_generic_tr
,
6219 .read
= rb_simple_read
,
6220 .write
= rb_simple_write
,
6221 .release
= tracing_release_generic_tr
,
6222 .llseek
= default_llseek
,
6225 struct dentry
*trace_instance_dir
;
6228 init_tracer_debugfs(struct trace_array
*tr
, struct dentry
*d_tracer
);
6231 allocate_trace_buffer(struct trace_array
*tr
, struct trace_buffer
*buf
, int size
)
6233 enum ring_buffer_flags rb_flags
;
6235 rb_flags
= trace_flags
& TRACE_ITER_OVERWRITE
? RB_FL_OVERWRITE
: 0;
6239 buf
->buffer
= ring_buffer_alloc(size
, rb_flags
);
6243 buf
->data
= alloc_percpu(struct trace_array_cpu
);
6245 ring_buffer_free(buf
->buffer
);
6249 /* Allocate the first page for all buffers */
6250 set_buffer_entries(&tr
->trace_buffer
,
6251 ring_buffer_size(tr
->trace_buffer
.buffer
, 0));
6256 static int allocate_trace_buffers(struct trace_array
*tr
, int size
)
6260 ret
= allocate_trace_buffer(tr
, &tr
->trace_buffer
, size
);
6264 #ifdef CONFIG_TRACER_MAX_TRACE
6265 ret
= allocate_trace_buffer(tr
, &tr
->max_buffer
,
6266 allocate_snapshot
? size
: 1);
6268 ring_buffer_free(tr
->trace_buffer
.buffer
);
6269 free_percpu(tr
->trace_buffer
.data
);
6272 tr
->allocated_snapshot
= allocate_snapshot
;
6275 * Only the top level trace array gets its snapshot allocated
6276 * from the kernel command line.
6278 allocate_snapshot
= false;
6283 static void free_trace_buffer(struct trace_buffer
*buf
)
6286 ring_buffer_free(buf
->buffer
);
6288 free_percpu(buf
->data
);
6293 static void free_trace_buffers(struct trace_array
*tr
)
6298 free_trace_buffer(&tr
->trace_buffer
);
6300 #ifdef CONFIG_TRACER_MAX_TRACE
6301 free_trace_buffer(&tr
->max_buffer
);
6305 static int new_instance_create(const char *name
)
6307 struct trace_array
*tr
;
6310 mutex_lock(&trace_types_lock
);
6313 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
6314 if (tr
->name
&& strcmp(tr
->name
, name
) == 0)
6319 tr
= kzalloc(sizeof(*tr
), GFP_KERNEL
);
6323 tr
->name
= kstrdup(name
, GFP_KERNEL
);
6327 if (!alloc_cpumask_var(&tr
->tracing_cpumask
, GFP_KERNEL
))
6330 cpumask_copy(tr
->tracing_cpumask
, cpu_all_mask
);
6332 raw_spin_lock_init(&tr
->start_lock
);
6334 tr
->max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
6336 tr
->current_trace
= &nop_trace
;
6338 INIT_LIST_HEAD(&tr
->systems
);
6339 INIT_LIST_HEAD(&tr
->events
);
6341 if (allocate_trace_buffers(tr
, trace_buf_size
) < 0)
6344 tr
->dir
= debugfs_create_dir(name
, trace_instance_dir
);
6348 ret
= event_trace_add_tracer(tr
->dir
, tr
);
6350 debugfs_remove_recursive(tr
->dir
);
6354 init_tracer_debugfs(tr
, tr
->dir
);
6356 list_add(&tr
->list
, &ftrace_trace_arrays
);
6358 mutex_unlock(&trace_types_lock
);
6363 free_trace_buffers(tr
);
6364 free_cpumask_var(tr
->tracing_cpumask
);
6369 mutex_unlock(&trace_types_lock
);
6375 static int instance_delete(const char *name
)
6377 struct trace_array
*tr
;
6381 mutex_lock(&trace_types_lock
);
6384 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
6385 if (tr
->name
&& strcmp(tr
->name
, name
) == 0) {
6397 list_del(&tr
->list
);
6399 tracing_set_nop(tr
);
6400 event_trace_del_tracer(tr
);
6401 ftrace_destroy_function_files(tr
);
6402 debugfs_remove_recursive(tr
->dir
);
6403 free_trace_buffers(tr
);
6411 mutex_unlock(&trace_types_lock
);
6416 static int instance_mkdir (struct inode
*inode
, struct dentry
*dentry
, umode_t mode
)
6418 struct dentry
*parent
;
6421 /* Paranoid: Make sure the parent is the "instances" directory */
6422 parent
= hlist_entry(inode
->i_dentry
.first
, struct dentry
, d_alias
);
6423 if (WARN_ON_ONCE(parent
!= trace_instance_dir
))
6427 * The inode mutex is locked, but debugfs_create_dir() will also
6428 * take the mutex. As the instances directory can not be destroyed
6429 * or changed in any other way, it is safe to unlock it, and
6430 * let the dentry try. If two users try to make the same dir at
6431 * the same time, then the new_instance_create() will determine the
6434 mutex_unlock(&inode
->i_mutex
);
6436 ret
= new_instance_create(dentry
->d_iname
);
6438 mutex_lock(&inode
->i_mutex
);
6443 static int instance_rmdir(struct inode
*inode
, struct dentry
*dentry
)
6445 struct dentry
*parent
;
6448 /* Paranoid: Make sure the parent is the "instances" directory */
6449 parent
= hlist_entry(inode
->i_dentry
.first
, struct dentry
, d_alias
);
6450 if (WARN_ON_ONCE(parent
!= trace_instance_dir
))
6453 /* The caller did a dget() on dentry */
6454 mutex_unlock(&dentry
->d_inode
->i_mutex
);
6457 * The inode mutex is locked, but debugfs_create_dir() will also
6458 * take the mutex. As the instances directory can not be destroyed
6459 * or changed in any other way, it is safe to unlock it, and
6460 * let the dentry try. If two users try to make the same dir at
6461 * the same time, then the instance_delete() will determine the
6464 mutex_unlock(&inode
->i_mutex
);
6466 ret
= instance_delete(dentry
->d_iname
);
6468 mutex_lock_nested(&inode
->i_mutex
, I_MUTEX_PARENT
);
6469 mutex_lock(&dentry
->d_inode
->i_mutex
);
6474 static const struct inode_operations instance_dir_inode_operations
= {
6475 .lookup
= simple_lookup
,
6476 .mkdir
= instance_mkdir
,
6477 .rmdir
= instance_rmdir
,
6480 static __init
void create_trace_instances(struct dentry
*d_tracer
)
6482 trace_instance_dir
= debugfs_create_dir("instances", d_tracer
);
6483 if (WARN_ON(!trace_instance_dir
))
6486 /* Hijack the dir inode operations, to allow mkdir */
6487 trace_instance_dir
->d_inode
->i_op
= &instance_dir_inode_operations
;
6491 init_tracer_debugfs(struct trace_array
*tr
, struct dentry
*d_tracer
)
6495 trace_create_file("available_tracers", 0444, d_tracer
,
6496 tr
, &show_traces_fops
);
6498 trace_create_file("current_tracer", 0644, d_tracer
,
6499 tr
, &set_tracer_fops
);
6501 trace_create_file("tracing_cpumask", 0644, d_tracer
,
6502 tr
, &tracing_cpumask_fops
);
6504 trace_create_file("trace_options", 0644, d_tracer
,
6505 tr
, &tracing_iter_fops
);
6507 trace_create_file("trace", 0644, d_tracer
,
6510 trace_create_file("trace_pipe", 0444, d_tracer
,
6511 tr
, &tracing_pipe_fops
);
6513 trace_create_file("buffer_size_kb", 0644, d_tracer
,
6514 tr
, &tracing_entries_fops
);
6516 trace_create_file("buffer_total_size_kb", 0444, d_tracer
,
6517 tr
, &tracing_total_entries_fops
);
6519 trace_create_file("free_buffer", 0200, d_tracer
,
6520 tr
, &tracing_free_buffer_fops
);
6522 trace_create_file("trace_marker", 0220, d_tracer
,
6523 tr
, &tracing_mark_fops
);
6525 trace_create_file("trace_clock", 0644, d_tracer
, tr
,
6528 trace_create_file("tracing_on", 0644, d_tracer
,
6529 tr
, &rb_simple_fops
);
6531 #ifdef CONFIG_TRACER_MAX_TRACE
6532 trace_create_file("tracing_max_latency", 0644, d_tracer
,
6533 &tr
->max_latency
, &tracing_max_lat_fops
);
6536 if (ftrace_create_function_files(tr
, d_tracer
))
6537 WARN(1, "Could not allocate function filter files");
6539 #ifdef CONFIG_TRACER_SNAPSHOT
6540 trace_create_file("snapshot", 0644, d_tracer
,
6541 tr
, &snapshot_fops
);
6544 for_each_tracing_cpu(cpu
)
6545 tracing_init_debugfs_percpu(tr
, cpu
);
6549 static __init
int tracer_init_debugfs(void)
6551 struct dentry
*d_tracer
;
6553 trace_access_lock_init();
6555 d_tracer
= tracing_init_dentry();
6559 init_tracer_debugfs(&global_trace
, d_tracer
);
6561 trace_create_file("tracing_thresh", 0644, d_tracer
,
6562 &global_trace
, &tracing_thresh_fops
);
6564 trace_create_file("README", 0444, d_tracer
,
6565 NULL
, &tracing_readme_fops
);
6567 trace_create_file("saved_cmdlines", 0444, d_tracer
,
6568 NULL
, &tracing_saved_cmdlines_fops
);
6570 trace_create_file("saved_cmdlines_size", 0644, d_tracer
,
6571 NULL
, &tracing_saved_cmdlines_size_fops
);
6573 #ifdef CONFIG_DYNAMIC_FTRACE
6574 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer
,
6575 &ftrace_update_tot_cnt
, &tracing_dyn_info_fops
);
6578 create_trace_instances(d_tracer
);
6580 create_trace_options_dir(&global_trace
);
6585 static int trace_panic_handler(struct notifier_block
*this,
6586 unsigned long event
, void *unused
)
6588 if (ftrace_dump_on_oops
)
6589 ftrace_dump(ftrace_dump_on_oops
);
6593 static struct notifier_block trace_panic_notifier
= {
6594 .notifier_call
= trace_panic_handler
,
6596 .priority
= 150 /* priority: INT_MAX >= x >= 0 */
6599 static int trace_die_handler(struct notifier_block
*self
,
6605 if (ftrace_dump_on_oops
)
6606 ftrace_dump(ftrace_dump_on_oops
);
6614 static struct notifier_block trace_die_notifier
= {
6615 .notifier_call
= trace_die_handler
,
6620 * printk is set to max of 1024, we really don't need it that big.
6621 * Nothing should be printing 1000 characters anyway.
6623 #define TRACE_MAX_PRINT 1000
6626 * Define here KERN_TRACE so that we have one place to modify
6627 * it if we decide to change what log level the ftrace dump
6630 #define KERN_TRACE KERN_EMERG
6633 trace_printk_seq(struct trace_seq
*s
)
6635 /* Probably should print a warning here. */
6636 if (s
->len
>= TRACE_MAX_PRINT
)
6637 s
->len
= TRACE_MAX_PRINT
;
6639 /* should be zero ended, but we are paranoid. */
6640 s
->buffer
[s
->len
] = 0;
6642 printk(KERN_TRACE
"%s", s
->buffer
);
6647 void trace_init_global_iter(struct trace_iterator
*iter
)
6649 iter
->tr
= &global_trace
;
6650 iter
->trace
= iter
->tr
->current_trace
;
6651 iter
->cpu_file
= RING_BUFFER_ALL_CPUS
;
6652 iter
->trace_buffer
= &global_trace
.trace_buffer
;
6654 if (iter
->trace
&& iter
->trace
->open
)
6655 iter
->trace
->open(iter
);
6657 /* Annotate start of buffers if we had overruns */
6658 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
6659 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
6661 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6662 if (trace_clocks
[iter
->tr
->clock_id
].in_ns
)
6663 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
6666 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode
)
6668 /* use static because iter can be a bit big for the stack */
6669 static struct trace_iterator iter
;
6670 static atomic_t dump_running
;
6671 unsigned int old_userobj
;
6672 unsigned long flags
;
6675 /* Only allow one dump user at a time. */
6676 if (atomic_inc_return(&dump_running
) != 1) {
6677 atomic_dec(&dump_running
);
6682 * Always turn off tracing when we dump.
6683 * We don't need to show trace output of what happens
6684 * between multiple crashes.
6686 * If the user does a sysrq-z, then they can re-enable
6687 * tracing with echo 1 > tracing_on.
6691 local_irq_save(flags
);
6693 /* Simulate the iterator */
6694 trace_init_global_iter(&iter
);
6696 for_each_tracing_cpu(cpu
) {
6697 atomic_inc(&per_cpu_ptr(iter
.tr
->trace_buffer
.data
, cpu
)->disabled
);
6700 old_userobj
= trace_flags
& TRACE_ITER_SYM_USEROBJ
;
6702 /* don't look at user memory in panic mode */
6703 trace_flags
&= ~TRACE_ITER_SYM_USEROBJ
;
6705 switch (oops_dump_mode
) {
6707 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
6710 iter
.cpu_file
= raw_smp_processor_id();
6715 printk(KERN_TRACE
"Bad dumping mode, switching to all CPUs dump\n");
6716 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
6719 printk(KERN_TRACE
"Dumping ftrace buffer:\n");
6721 /* Did function tracer already get disabled? */
6722 if (ftrace_is_dead()) {
6723 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6724 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6728 * We need to stop all tracing on all CPUS to read the
6729 * the next buffer. This is a bit expensive, but is
6730 * not done often. We fill all what we can read,
6731 * and then release the locks again.
6734 while (!trace_empty(&iter
)) {
6737 printk(KERN_TRACE
"---------------------------------\n");
6741 /* reset all but tr, trace, and overruns */
6742 memset(&iter
.seq
, 0,
6743 sizeof(struct trace_iterator
) -
6744 offsetof(struct trace_iterator
, seq
));
6745 iter
.iter_flags
|= TRACE_FILE_LAT_FMT
;
6748 if (trace_find_next_entry_inc(&iter
) != NULL
) {
6751 ret
= print_trace_line(&iter
);
6752 if (ret
!= TRACE_TYPE_NO_CONSUME
)
6753 trace_consume(&iter
);
6755 touch_nmi_watchdog();
6757 trace_printk_seq(&iter
.seq
);
6761 printk(KERN_TRACE
" (ftrace buffer empty)\n");
6763 printk(KERN_TRACE
"---------------------------------\n");
6766 trace_flags
|= old_userobj
;
6768 for_each_tracing_cpu(cpu
) {
6769 atomic_dec(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
6771 atomic_dec(&dump_running
);
6772 local_irq_restore(flags
);
6774 EXPORT_SYMBOL_GPL(ftrace_dump
);
6776 __init
static int tracer_alloc_buffers(void)
6782 if (!alloc_cpumask_var(&tracing_buffer_mask
, GFP_KERNEL
))
6785 if (!alloc_cpumask_var(&global_trace
.tracing_cpumask
, GFP_KERNEL
))
6786 goto out_free_buffer_mask
;
6788 /* Only allocate trace_printk buffers if a trace_printk exists */
6789 if (__stop___trace_bprintk_fmt
!= __start___trace_bprintk_fmt
)
6790 /* Must be called before global_trace.buffer is allocated */
6791 trace_printk_init_buffers();
6793 /* To save memory, keep the ring buffer size to its minimum */
6794 if (ring_buffer_expanded
)
6795 ring_buf_size
= trace_buf_size
;
6799 cpumask_copy(tracing_buffer_mask
, cpu_possible_mask
);
6800 cpumask_copy(global_trace
.tracing_cpumask
, cpu_all_mask
);
6802 raw_spin_lock_init(&global_trace
.start_lock
);
6804 /* Used for event triggers */
6805 temp_buffer
= ring_buffer_alloc(PAGE_SIZE
, RB_FL_OVERWRITE
);
6807 goto out_free_cpumask
;
6809 if (trace_create_savedcmd() < 0)
6810 goto out_free_temp_buffer
;
6812 /* TODO: make the number of buffers hot pluggable with CPUS */
6813 if (allocate_trace_buffers(&global_trace
, ring_buf_size
) < 0) {
6814 printk(KERN_ERR
"tracer: failed to allocate ring buffer!\n");
6816 goto out_free_savedcmd
;
6819 if (global_trace
.buffer_disabled
)
6822 if (trace_boot_clock
) {
6823 ret
= tracing_set_clock(&global_trace
, trace_boot_clock
);
6825 pr_warning("Trace clock %s not defined, going back to default\n",
6830 * register_tracer() might reference current_trace, so it
6831 * needs to be set before we register anything. This is
6832 * just a bootstrap of current_trace anyway.
6834 global_trace
.current_trace
= &nop_trace
;
6836 global_trace
.max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
6838 ftrace_init_global_array_ops(&global_trace
);
6840 register_tracer(&nop_trace
);
6842 /* All seems OK, enable tracing */
6843 tracing_disabled
= 0;
6845 atomic_notifier_chain_register(&panic_notifier_list
,
6846 &trace_panic_notifier
);
6848 register_die_notifier(&trace_die_notifier
);
6850 global_trace
.flags
= TRACE_ARRAY_FL_GLOBAL
;
6852 INIT_LIST_HEAD(&global_trace
.systems
);
6853 INIT_LIST_HEAD(&global_trace
.events
);
6854 list_add(&global_trace
.list
, &ftrace_trace_arrays
);
6856 while (trace_boot_options
) {
6859 option
= strsep(&trace_boot_options
, ",");
6860 trace_set_options(&global_trace
, option
);
6863 register_snapshot_cmd();
6868 free_saved_cmdlines_buffer(savedcmd
);
6869 out_free_temp_buffer
:
6870 ring_buffer_free(temp_buffer
);
6872 free_cpumask_var(global_trace
.tracing_cpumask
);
6873 out_free_buffer_mask
:
6874 free_cpumask_var(tracing_buffer_mask
);
6879 void __init
trace_init(void)
6881 tracer_alloc_buffers();
6882 init_ftrace_syscalls();
6886 __init
static int clear_boot_tracer(void)
6889 * The default tracer at boot buffer is an init section.
6890 * This function is called in lateinit. If we did not
6891 * find the boot tracer, then clear it out, to prevent
6892 * later registration from accessing the buffer that is
6893 * about to be freed.
6895 if (!default_bootup_tracer
)
6898 printk(KERN_INFO
"ftrace bootup tracer '%s' not registered.\n",
6899 default_bootup_tracer
);
6900 default_bootup_tracer
= NULL
;
6905 fs_initcall(tracer_init_debugfs
);
6906 late_initcall(clear_boot_tracer
);