2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded
;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running
;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled
;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator
*tracepoint_print_iter
;
70 int tracepoint_printk
;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt
[] = {
77 static struct tracer_flags dummy_tracer_flags
= {
79 .opts
= dummy_tracer_opt
83 dummy_set_flag(struct trace_array
*tr
, u32 old_flags
, u32 bit
, int set
)
89 * To prevent the comm cache from being overwritten when no
90 * tracing is active, only save the comm when a trace event
93 static DEFINE_PER_CPU(bool, trace_cmdline_save
);
96 * Kill all tracing for good (never come back).
97 * It is initialized to 1 but will turn to zero if the initialization
98 * of the tracer is successful. But that is the only place that sets
101 static int tracing_disabled
= 1;
103 DEFINE_PER_CPU(int, ftrace_cpu_disabled
);
105 cpumask_var_t __read_mostly tracing_buffer_mask
;
108 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
110 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
111 * is set, then ftrace_dump is called. This will output the contents
112 * of the ftrace buffers to the console. This is very useful for
113 * capturing traces that lead to crashes and outputing it to a
116 * It is default off, but you can enable it with either specifying
117 * "ftrace_dump_on_oops" in the kernel command line, or setting
118 * /proc/sys/kernel/ftrace_dump_on_oops
119 * Set 1 if you want to dump buffers of all CPUs
120 * Set 2 if you want to dump the buffer of the CPU that triggered oops
123 enum ftrace_dump_mode ftrace_dump_on_oops
;
125 /* When set, tracing will stop when a WARN*() is hit */
126 int __disable_trace_on_warning
;
128 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
129 /* Map of enums to their values, for "enum_map" file */
130 struct trace_enum_map_head
{
132 unsigned long length
;
135 union trace_enum_map_item
;
137 struct trace_enum_map_tail
{
139 * "end" is first and points to NULL as it must be different
140 * than "mod" or "enum_string"
142 union trace_enum_map_item
*next
;
143 const char *end
; /* points to NULL */
146 static DEFINE_MUTEX(trace_enum_mutex
);
149 * The trace_enum_maps are saved in an array with two extra elements,
150 * one at the beginning, and one at the end. The beginning item contains
151 * the count of the saved maps (head.length), and the module they
152 * belong to if not built in (head.mod). The ending item contains a
153 * pointer to the next array of saved enum_map items.
155 union trace_enum_map_item
{
156 struct trace_enum_map map
;
157 struct trace_enum_map_head head
;
158 struct trace_enum_map_tail tail
;
161 static union trace_enum_map_item
*trace_enum_maps
;
162 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
164 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
);
166 #define MAX_TRACER_SIZE 100
167 static char bootup_tracer_buf
[MAX_TRACER_SIZE
] __initdata
;
168 static char *default_bootup_tracer
;
170 static bool allocate_snapshot
;
172 static int __init
set_cmdline_ftrace(char *str
)
174 strlcpy(bootup_tracer_buf
, str
, MAX_TRACER_SIZE
);
175 default_bootup_tracer
= bootup_tracer_buf
;
176 /* We are using ftrace early, expand it */
177 ring_buffer_expanded
= true;
180 __setup("ftrace=", set_cmdline_ftrace
);
182 static int __init
set_ftrace_dump_on_oops(char *str
)
184 if (*str
++ != '=' || !*str
) {
185 ftrace_dump_on_oops
= DUMP_ALL
;
189 if (!strcmp("orig_cpu", str
)) {
190 ftrace_dump_on_oops
= DUMP_ORIG
;
196 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops
);
198 static int __init
stop_trace_on_warning(char *str
)
200 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
201 __disable_trace_on_warning
= 1;
204 __setup("traceoff_on_warning", stop_trace_on_warning
);
206 static int __init
boot_alloc_snapshot(char *str
)
208 allocate_snapshot
= true;
209 /* We also need the main ring buffer expanded */
210 ring_buffer_expanded
= true;
213 __setup("alloc_snapshot", boot_alloc_snapshot
);
216 static char trace_boot_options_buf
[MAX_TRACER_SIZE
] __initdata
;
217 static char *trace_boot_options __initdata
;
219 static int __init
set_trace_boot_options(char *str
)
221 strlcpy(trace_boot_options_buf
, str
, MAX_TRACER_SIZE
);
222 trace_boot_options
= trace_boot_options_buf
;
225 __setup("trace_options=", set_trace_boot_options
);
227 static char trace_boot_clock_buf
[MAX_TRACER_SIZE
] __initdata
;
228 static char *trace_boot_clock __initdata
;
230 static int __init
set_trace_boot_clock(char *str
)
232 strlcpy(trace_boot_clock_buf
, str
, MAX_TRACER_SIZE
);
233 trace_boot_clock
= trace_boot_clock_buf
;
236 __setup("trace_clock=", set_trace_boot_clock
);
238 static int __init
set_tracepoint_printk(char *str
)
240 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
241 tracepoint_printk
= 1;
244 __setup("tp_printk", set_tracepoint_printk
);
246 unsigned long long ns2usecs(cycle_t nsec
)
253 /* trace_flags holds trace_options default values */
254 #define TRACE_DEFAULT_FLAGS \
255 (FUNCTION_DEFAULT_FLAGS | \
256 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
257 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
258 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
259 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
262 * The global_trace is the descriptor that holds the tracing
263 * buffers for the live tracing. For each CPU, it contains
264 * a link list of pages that will store trace entries. The
265 * page descriptor of the pages in the memory is used to hold
266 * the link list by linking the lru item in the page descriptor
267 * to each of the pages in the buffer per CPU.
269 * For each active CPU there is a data field that holds the
270 * pages for the buffer for that CPU. Each CPU has the same number
271 * of pages allocated for its buffer.
273 static struct trace_array global_trace
= {
274 .trace_flags
= TRACE_DEFAULT_FLAGS
,
277 LIST_HEAD(ftrace_trace_arrays
);
279 int trace_array_get(struct trace_array
*this_tr
)
281 struct trace_array
*tr
;
284 mutex_lock(&trace_types_lock
);
285 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
292 mutex_unlock(&trace_types_lock
);
297 static void __trace_array_put(struct trace_array
*this_tr
)
299 WARN_ON(!this_tr
->ref
);
303 void trace_array_put(struct trace_array
*this_tr
)
305 mutex_lock(&trace_types_lock
);
306 __trace_array_put(this_tr
);
307 mutex_unlock(&trace_types_lock
);
310 int filter_check_discard(struct trace_event_file
*file
, void *rec
,
311 struct ring_buffer
*buffer
,
312 struct ring_buffer_event
*event
)
314 if (unlikely(file
->flags
& EVENT_FILE_FL_FILTERED
) &&
315 !filter_match_preds(file
->filter
, rec
)) {
316 ring_buffer_discard_commit(buffer
, event
);
322 EXPORT_SYMBOL_GPL(filter_check_discard
);
324 int call_filter_check_discard(struct trace_event_call
*call
, void *rec
,
325 struct ring_buffer
*buffer
,
326 struct ring_buffer_event
*event
)
328 if (unlikely(call
->flags
& TRACE_EVENT_FL_FILTERED
) &&
329 !filter_match_preds(call
->filter
, rec
)) {
330 ring_buffer_discard_commit(buffer
, event
);
336 EXPORT_SYMBOL_GPL(call_filter_check_discard
);
338 static cycle_t
buffer_ftrace_now(struct trace_buffer
*buf
, int cpu
)
342 /* Early boot up does not have a buffer yet */
344 return trace_clock_local();
346 ts
= ring_buffer_time_stamp(buf
->buffer
, cpu
);
347 ring_buffer_normalize_time_stamp(buf
->buffer
, cpu
, &ts
);
352 cycle_t
ftrace_now(int cpu
)
354 return buffer_ftrace_now(&global_trace
.trace_buffer
, cpu
);
358 * tracing_is_enabled - Show if global_trace has been disabled
360 * Shows if the global trace has been enabled or not. It uses the
361 * mirror flag "buffer_disabled" to be used in fast paths such as for
362 * the irqsoff tracer. But it may be inaccurate due to races. If you
363 * need to know the accurate state, use tracing_is_on() which is a little
364 * slower, but accurate.
366 int tracing_is_enabled(void)
369 * For quick access (irqsoff uses this in fast path), just
370 * return the mirror variable of the state of the ring buffer.
371 * It's a little racy, but we don't really care.
374 return !global_trace
.buffer_disabled
;
378 * trace_buf_size is the size in bytes that is allocated
379 * for a buffer. Note, the number of bytes is always rounded
382 * This number is purposely set to a low number of 16384.
383 * If the dump on oops happens, it will be much appreciated
384 * to not have to wait for all that output. Anyway this can be
385 * boot time and run time configurable.
387 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
389 static unsigned long trace_buf_size
= TRACE_BUF_SIZE_DEFAULT
;
391 /* trace_types holds a link list of available tracers. */
392 static struct tracer
*trace_types __read_mostly
;
395 * trace_types_lock is used to protect the trace_types list.
397 DEFINE_MUTEX(trace_types_lock
);
400 * serialize the access of the ring buffer
402 * ring buffer serializes readers, but it is low level protection.
403 * The validity of the events (which returns by ring_buffer_peek() ..etc)
404 * are not protected by ring buffer.
406 * The content of events may become garbage if we allow other process consumes
407 * these events concurrently:
408 * A) the page of the consumed events may become a normal page
409 * (not reader page) in ring buffer, and this page will be rewrited
410 * by events producer.
411 * B) The page of the consumed events may become a page for splice_read,
412 * and this page will be returned to system.
414 * These primitives allow multi process access to different cpu ring buffer
417 * These primitives don't distinguish read-only and read-consume access.
418 * Multi read-only access are also serialized.
422 static DECLARE_RWSEM(all_cpu_access_lock
);
423 static DEFINE_PER_CPU(struct mutex
, cpu_access_lock
);
425 static inline void trace_access_lock(int cpu
)
427 if (cpu
== RING_BUFFER_ALL_CPUS
) {
428 /* gain it for accessing the whole ring buffer. */
429 down_write(&all_cpu_access_lock
);
431 /* gain it for accessing a cpu ring buffer. */
433 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
434 down_read(&all_cpu_access_lock
);
436 /* Secondly block other access to this @cpu ring buffer. */
437 mutex_lock(&per_cpu(cpu_access_lock
, cpu
));
441 static inline void trace_access_unlock(int cpu
)
443 if (cpu
== RING_BUFFER_ALL_CPUS
) {
444 up_write(&all_cpu_access_lock
);
446 mutex_unlock(&per_cpu(cpu_access_lock
, cpu
));
447 up_read(&all_cpu_access_lock
);
451 static inline void trace_access_lock_init(void)
455 for_each_possible_cpu(cpu
)
456 mutex_init(&per_cpu(cpu_access_lock
, cpu
));
461 static DEFINE_MUTEX(access_lock
);
463 static inline void trace_access_lock(int cpu
)
466 mutex_lock(&access_lock
);
469 static inline void trace_access_unlock(int cpu
)
472 mutex_unlock(&access_lock
);
475 static inline void trace_access_lock_init(void)
481 #ifdef CONFIG_STACKTRACE
482 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
484 int skip
, int pc
, struct pt_regs
*regs
);
485 static inline void ftrace_trace_stack(struct ring_buffer
*buffer
,
487 int skip
, int pc
, struct pt_regs
*regs
);
490 static inline void __ftrace_trace_stack(struct ring_buffer
*buffer
,
492 int skip
, int pc
, struct pt_regs
*regs
)
495 static inline void ftrace_trace_stack(struct ring_buffer
*buffer
,
497 int skip
, int pc
, struct pt_regs
*regs
)
503 static void tracer_tracing_on(struct trace_array
*tr
)
505 if (tr
->trace_buffer
.buffer
)
506 ring_buffer_record_on(tr
->trace_buffer
.buffer
);
508 * This flag is looked at when buffers haven't been allocated
509 * yet, or by some tracers (like irqsoff), that just want to
510 * know if the ring buffer has been disabled, but it can handle
511 * races of where it gets disabled but we still do a record.
512 * As the check is in the fast path of the tracers, it is more
513 * important to be fast than accurate.
515 tr
->buffer_disabled
= 0;
516 /* Make the flag seen by readers */
521 * tracing_on - enable tracing buffers
523 * This function enables tracing buffers that may have been
524 * disabled with tracing_off.
526 void tracing_on(void)
528 tracer_tracing_on(&global_trace
);
530 EXPORT_SYMBOL_GPL(tracing_on
);
533 * __trace_puts - write a constant string into the trace buffer.
534 * @ip: The address of the caller
535 * @str: The constant string to write
536 * @size: The size of the string.
538 int __trace_puts(unsigned long ip
, const char *str
, int size
)
540 struct ring_buffer_event
*event
;
541 struct ring_buffer
*buffer
;
542 struct print_entry
*entry
;
543 unsigned long irq_flags
;
547 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
550 pc
= preempt_count();
552 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
555 alloc
= sizeof(*entry
) + size
+ 2; /* possible \n added */
557 local_save_flags(irq_flags
);
558 buffer
= global_trace
.trace_buffer
.buffer
;
559 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, alloc
,
564 entry
= ring_buffer_event_data(event
);
567 memcpy(&entry
->buf
, str
, size
);
569 /* Add a newline if necessary */
570 if (entry
->buf
[size
- 1] != '\n') {
571 entry
->buf
[size
] = '\n';
572 entry
->buf
[size
+ 1] = '\0';
574 entry
->buf
[size
] = '\0';
576 __buffer_unlock_commit(buffer
, event
);
577 ftrace_trace_stack(buffer
, irq_flags
, 4, pc
, NULL
);
581 EXPORT_SYMBOL_GPL(__trace_puts
);
584 * __trace_bputs - write the pointer to a constant string into trace buffer
585 * @ip: The address of the caller
586 * @str: The constant string to write to the buffer to
588 int __trace_bputs(unsigned long ip
, const char *str
)
590 struct ring_buffer_event
*event
;
591 struct ring_buffer
*buffer
;
592 struct bputs_entry
*entry
;
593 unsigned long irq_flags
;
594 int size
= sizeof(struct bputs_entry
);
597 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
600 pc
= preempt_count();
602 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
605 local_save_flags(irq_flags
);
606 buffer
= global_trace
.trace_buffer
.buffer
;
607 event
= trace_buffer_lock_reserve(buffer
, TRACE_BPUTS
, size
,
612 entry
= ring_buffer_event_data(event
);
616 __buffer_unlock_commit(buffer
, event
);
617 ftrace_trace_stack(buffer
, irq_flags
, 4, pc
, NULL
);
621 EXPORT_SYMBOL_GPL(__trace_bputs
);
623 #ifdef CONFIG_TRACER_SNAPSHOT
625 * trace_snapshot - take a snapshot of the current buffer.
627 * This causes a swap between the snapshot buffer and the current live
628 * tracing buffer. You can use this to take snapshots of the live
629 * trace when some condition is triggered, but continue to trace.
631 * Note, make sure to allocate the snapshot with either
632 * a tracing_snapshot_alloc(), or by doing it manually
633 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
635 * If the snapshot buffer is not allocated, it will stop tracing.
636 * Basically making a permanent snapshot.
638 void tracing_snapshot(void)
640 struct trace_array
*tr
= &global_trace
;
641 struct tracer
*tracer
= tr
->current_trace
;
645 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
646 internal_trace_puts("*** snapshot is being ignored ***\n");
650 if (!tr
->allocated_snapshot
) {
651 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
652 internal_trace_puts("*** stopping trace here! ***\n");
657 /* Note, snapshot can not be used when the tracer uses it */
658 if (tracer
->use_max_tr
) {
659 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
660 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
664 local_irq_save(flags
);
665 update_max_tr(tr
, current
, smp_processor_id());
666 local_irq_restore(flags
);
668 EXPORT_SYMBOL_GPL(tracing_snapshot
);
670 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
671 struct trace_buffer
*size_buf
, int cpu_id
);
672 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
);
674 static int alloc_snapshot(struct trace_array
*tr
)
678 if (!tr
->allocated_snapshot
) {
680 /* allocate spare buffer */
681 ret
= resize_buffer_duplicate_size(&tr
->max_buffer
,
682 &tr
->trace_buffer
, RING_BUFFER_ALL_CPUS
);
686 tr
->allocated_snapshot
= true;
692 static void free_snapshot(struct trace_array
*tr
)
695 * We don't free the ring buffer. instead, resize it because
696 * The max_tr ring buffer has some state (e.g. ring->clock) and
697 * we want preserve it.
699 ring_buffer_resize(tr
->max_buffer
.buffer
, 1, RING_BUFFER_ALL_CPUS
);
700 set_buffer_entries(&tr
->max_buffer
, 1);
701 tracing_reset_online_cpus(&tr
->max_buffer
);
702 tr
->allocated_snapshot
= false;
706 * tracing_alloc_snapshot - allocate snapshot buffer.
708 * This only allocates the snapshot buffer if it isn't already
709 * allocated - it doesn't also take a snapshot.
711 * This is meant to be used in cases where the snapshot buffer needs
712 * to be set up for events that can't sleep but need to be able to
713 * trigger a snapshot.
715 int tracing_alloc_snapshot(void)
717 struct trace_array
*tr
= &global_trace
;
720 ret
= alloc_snapshot(tr
);
725 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
728 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
730 * This is similar to trace_snapshot(), but it will allocate the
731 * snapshot buffer if it isn't already allocated. Use this only
732 * where it is safe to sleep, as the allocation may sleep.
734 * This causes a swap between the snapshot buffer and the current live
735 * tracing buffer. You can use this to take snapshots of the live
736 * trace when some condition is triggered, but continue to trace.
738 void tracing_snapshot_alloc(void)
742 ret
= tracing_alloc_snapshot();
748 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
750 void tracing_snapshot(void)
752 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
754 EXPORT_SYMBOL_GPL(tracing_snapshot
);
755 int tracing_alloc_snapshot(void)
757 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
760 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
761 void tracing_snapshot_alloc(void)
766 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
767 #endif /* CONFIG_TRACER_SNAPSHOT */
769 static void tracer_tracing_off(struct trace_array
*tr
)
771 if (tr
->trace_buffer
.buffer
)
772 ring_buffer_record_off(tr
->trace_buffer
.buffer
);
774 * This flag is looked at when buffers haven't been allocated
775 * yet, or by some tracers (like irqsoff), that just want to
776 * know if the ring buffer has been disabled, but it can handle
777 * races of where it gets disabled but we still do a record.
778 * As the check is in the fast path of the tracers, it is more
779 * important to be fast than accurate.
781 tr
->buffer_disabled
= 1;
782 /* Make the flag seen by readers */
787 * tracing_off - turn off tracing buffers
789 * This function stops the tracing buffers from recording data.
790 * It does not disable any overhead the tracers themselves may
791 * be causing. This function simply causes all recording to
792 * the ring buffers to fail.
794 void tracing_off(void)
796 tracer_tracing_off(&global_trace
);
798 EXPORT_SYMBOL_GPL(tracing_off
);
800 void disable_trace_on_warning(void)
802 if (__disable_trace_on_warning
)
807 * tracer_tracing_is_on - show real state of ring buffer enabled
808 * @tr : the trace array to know if ring buffer is enabled
810 * Shows real state of the ring buffer if it is enabled or not.
812 static int tracer_tracing_is_on(struct trace_array
*tr
)
814 if (tr
->trace_buffer
.buffer
)
815 return ring_buffer_record_is_on(tr
->trace_buffer
.buffer
);
816 return !tr
->buffer_disabled
;
820 * tracing_is_on - show state of ring buffers enabled
822 int tracing_is_on(void)
824 return tracer_tracing_is_on(&global_trace
);
826 EXPORT_SYMBOL_GPL(tracing_is_on
);
828 static int __init
set_buf_size(char *str
)
830 unsigned long buf_size
;
834 buf_size
= memparse(str
, &str
);
835 /* nr_entries can not be zero */
838 trace_buf_size
= buf_size
;
841 __setup("trace_buf_size=", set_buf_size
);
843 static int __init
set_tracing_thresh(char *str
)
845 unsigned long threshold
;
850 ret
= kstrtoul(str
, 0, &threshold
);
853 tracing_thresh
= threshold
* 1000;
856 __setup("tracing_thresh=", set_tracing_thresh
);
858 unsigned long nsecs_to_usecs(unsigned long nsecs
)
864 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
865 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
866 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
867 * of strings in the order that the enums were defined.
872 /* These must match the bit postions in trace_iterator_flags */
873 static const char *trace_options
[] = {
881 int in_ns
; /* is this clock in nanoseconds? */
883 { trace_clock_local
, "local", 1 },
884 { trace_clock_global
, "global", 1 },
885 { trace_clock_counter
, "counter", 0 },
886 { trace_clock_jiffies
, "uptime", 0 },
887 { trace_clock
, "perf", 1 },
888 { ktime_get_mono_fast_ns
, "mono", 1 },
889 { ktime_get_raw_fast_ns
, "mono_raw", 1 },
894 * trace_parser_get_init - gets the buffer for trace parser
896 int trace_parser_get_init(struct trace_parser
*parser
, int size
)
898 memset(parser
, 0, sizeof(*parser
));
900 parser
->buffer
= kmalloc(size
, GFP_KERNEL
);
909 * trace_parser_put - frees the buffer for trace parser
911 void trace_parser_put(struct trace_parser
*parser
)
913 kfree(parser
->buffer
);
917 * trace_get_user - reads the user input string separated by space
918 * (matched by isspace(ch))
920 * For each string found the 'struct trace_parser' is updated,
921 * and the function returns.
923 * Returns number of bytes read.
925 * See kernel/trace/trace.h for 'struct trace_parser' details.
927 int trace_get_user(struct trace_parser
*parser
, const char __user
*ubuf
,
928 size_t cnt
, loff_t
*ppos
)
935 trace_parser_clear(parser
);
937 ret
= get_user(ch
, ubuf
++);
945 * The parser is not finished with the last write,
946 * continue reading the user input without skipping spaces.
949 /* skip white space */
950 while (cnt
&& isspace(ch
)) {
951 ret
= get_user(ch
, ubuf
++);
958 /* only spaces were written */
968 /* read the non-space input */
969 while (cnt
&& !isspace(ch
)) {
970 if (parser
->idx
< parser
->size
- 1)
971 parser
->buffer
[parser
->idx
++] = ch
;
976 ret
= get_user(ch
, ubuf
++);
983 /* We either got finished input or we have to wait for another call. */
985 parser
->buffer
[parser
->idx
] = 0;
986 parser
->cont
= false;
987 } else if (parser
->idx
< parser
->size
- 1) {
989 parser
->buffer
[parser
->idx
++] = ch
;
1002 /* TODO add a seq_buf_to_buffer() */
1003 static ssize_t
trace_seq_to_buffer(struct trace_seq
*s
, void *buf
, size_t cnt
)
1007 if (trace_seq_used(s
) <= s
->seq
.readpos
)
1010 len
= trace_seq_used(s
) - s
->seq
.readpos
;
1013 memcpy(buf
, s
->buffer
+ s
->seq
.readpos
, cnt
);
1015 s
->seq
.readpos
+= cnt
;
1019 unsigned long __read_mostly tracing_thresh
;
1021 #ifdef CONFIG_TRACER_MAX_TRACE
1023 * Copy the new maximum trace into the separate maximum-trace
1024 * structure. (this way the maximum trace is permanently saved,
1025 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1028 __update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1030 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
1031 struct trace_buffer
*max_buf
= &tr
->max_buffer
;
1032 struct trace_array_cpu
*data
= per_cpu_ptr(trace_buf
->data
, cpu
);
1033 struct trace_array_cpu
*max_data
= per_cpu_ptr(max_buf
->data
, cpu
);
1036 max_buf
->time_start
= data
->preempt_timestamp
;
1038 max_data
->saved_latency
= tr
->max_latency
;
1039 max_data
->critical_start
= data
->critical_start
;
1040 max_data
->critical_end
= data
->critical_end
;
1042 memcpy(max_data
->comm
, tsk
->comm
, TASK_COMM_LEN
);
1043 max_data
->pid
= tsk
->pid
;
1045 * If tsk == current, then use current_uid(), as that does not use
1046 * RCU. The irq tracer can be called out of RCU scope.
1049 max_data
->uid
= current_uid();
1051 max_data
->uid
= task_uid(tsk
);
1053 max_data
->nice
= tsk
->static_prio
- 20 - MAX_RT_PRIO
;
1054 max_data
->policy
= tsk
->policy
;
1055 max_data
->rt_priority
= tsk
->rt_priority
;
1057 /* record this tasks comm */
1058 tracing_record_cmdline(tsk
);
1062 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1064 * @tsk: the task with the latency
1065 * @cpu: The cpu that initiated the trace.
1067 * Flip the buffers between the @tr and the max_tr and record information
1068 * about which task was the cause of this latency.
1071 update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1073 struct ring_buffer
*buf
;
1078 WARN_ON_ONCE(!irqs_disabled());
1080 if (!tr
->allocated_snapshot
) {
1081 /* Only the nop tracer should hit this when disabling */
1082 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1086 arch_spin_lock(&tr
->max_lock
);
1088 buf
= tr
->trace_buffer
.buffer
;
1089 tr
->trace_buffer
.buffer
= tr
->max_buffer
.buffer
;
1090 tr
->max_buffer
.buffer
= buf
;
1092 __update_max_tr(tr
, tsk
, cpu
);
1093 arch_spin_unlock(&tr
->max_lock
);
1097 * update_max_tr_single - only copy one trace over, and reset the rest
1099 * @tsk - task with the latency
1100 * @cpu - the cpu of the buffer to copy.
1102 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1105 update_max_tr_single(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1112 WARN_ON_ONCE(!irqs_disabled());
1113 if (!tr
->allocated_snapshot
) {
1114 /* Only the nop tracer should hit this when disabling */
1115 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1119 arch_spin_lock(&tr
->max_lock
);
1121 ret
= ring_buffer_swap_cpu(tr
->max_buffer
.buffer
, tr
->trace_buffer
.buffer
, cpu
);
1123 if (ret
== -EBUSY
) {
1125 * We failed to swap the buffer due to a commit taking
1126 * place on this CPU. We fail to record, but we reset
1127 * the max trace buffer (no one writes directly to it)
1128 * and flag that it failed.
1130 trace_array_printk_buf(tr
->max_buffer
.buffer
, _THIS_IP_
,
1131 "Failed to swap buffers due to commit in progress\n");
1134 WARN_ON_ONCE(ret
&& ret
!= -EAGAIN
&& ret
!= -EBUSY
);
1136 __update_max_tr(tr
, tsk
, cpu
);
1137 arch_spin_unlock(&tr
->max_lock
);
1139 #endif /* CONFIG_TRACER_MAX_TRACE */
1141 static int wait_on_pipe(struct trace_iterator
*iter
, bool full
)
1143 /* Iterators are static, they should be filled or empty */
1144 if (trace_buffer_iter(iter
, iter
->cpu_file
))
1147 return ring_buffer_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
1151 #ifdef CONFIG_FTRACE_STARTUP_TEST
1152 static int run_tracer_selftest(struct tracer
*type
)
1154 struct trace_array
*tr
= &global_trace
;
1155 struct tracer
*saved_tracer
= tr
->current_trace
;
1158 if (!type
->selftest
|| tracing_selftest_disabled
)
1162 * Run a selftest on this tracer.
1163 * Here we reset the trace buffer, and set the current
1164 * tracer to be this tracer. The tracer can then run some
1165 * internal tracing to verify that everything is in order.
1166 * If we fail, we do not register this tracer.
1168 tracing_reset_online_cpus(&tr
->trace_buffer
);
1170 tr
->current_trace
= type
;
1172 #ifdef CONFIG_TRACER_MAX_TRACE
1173 if (type
->use_max_tr
) {
1174 /* If we expanded the buffers, make sure the max is expanded too */
1175 if (ring_buffer_expanded
)
1176 ring_buffer_resize(tr
->max_buffer
.buffer
, trace_buf_size
,
1177 RING_BUFFER_ALL_CPUS
);
1178 tr
->allocated_snapshot
= true;
1182 /* the test is responsible for initializing and enabling */
1183 pr_info("Testing tracer %s: ", type
->name
);
1184 ret
= type
->selftest(type
, tr
);
1185 /* the test is responsible for resetting too */
1186 tr
->current_trace
= saved_tracer
;
1188 printk(KERN_CONT
"FAILED!\n");
1189 /* Add the warning after printing 'FAILED' */
1193 /* Only reset on passing, to avoid touching corrupted buffers */
1194 tracing_reset_online_cpus(&tr
->trace_buffer
);
1196 #ifdef CONFIG_TRACER_MAX_TRACE
1197 if (type
->use_max_tr
) {
1198 tr
->allocated_snapshot
= false;
1200 /* Shrink the max buffer again */
1201 if (ring_buffer_expanded
)
1202 ring_buffer_resize(tr
->max_buffer
.buffer
, 1,
1203 RING_BUFFER_ALL_CPUS
);
1207 printk(KERN_CONT
"PASSED\n");
1211 static inline int run_tracer_selftest(struct tracer
*type
)
1215 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1217 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
);
1220 * register_tracer - register a tracer with the ftrace system.
1221 * @type - the plugin for the tracer
1223 * Register a new plugin tracer.
1225 int register_tracer(struct tracer
*type
)
1231 pr_info("Tracer must have a name\n");
1235 if (strlen(type
->name
) >= MAX_TRACER_SIZE
) {
1236 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE
);
1240 mutex_lock(&trace_types_lock
);
1242 tracing_selftest_running
= true;
1244 for (t
= trace_types
; t
; t
= t
->next
) {
1245 if (strcmp(type
->name
, t
->name
) == 0) {
1247 pr_info("Tracer %s already registered\n",
1254 if (!type
->set_flag
)
1255 type
->set_flag
= &dummy_set_flag
;
1257 type
->flags
= &dummy_tracer_flags
;
1259 if (!type
->flags
->opts
)
1260 type
->flags
->opts
= dummy_tracer_opt
;
1262 ret
= run_tracer_selftest(type
);
1266 type
->next
= trace_types
;
1268 add_tracer_options(&global_trace
, type
);
1271 tracing_selftest_running
= false;
1272 mutex_unlock(&trace_types_lock
);
1274 if (ret
|| !default_bootup_tracer
)
1277 if (strncmp(default_bootup_tracer
, type
->name
, MAX_TRACER_SIZE
))
1280 printk(KERN_INFO
"Starting tracer '%s'\n", type
->name
);
1281 /* Do we want this tracer to start on bootup? */
1282 tracing_set_tracer(&global_trace
, type
->name
);
1283 default_bootup_tracer
= NULL
;
1284 /* disable other selftests, since this will break it. */
1285 tracing_selftest_disabled
= true;
1286 #ifdef CONFIG_FTRACE_STARTUP_TEST
1287 printk(KERN_INFO
"Disabling FTRACE selftests due to running tracer '%s'\n",
1295 void tracing_reset(struct trace_buffer
*buf
, int cpu
)
1297 struct ring_buffer
*buffer
= buf
->buffer
;
1302 ring_buffer_record_disable(buffer
);
1304 /* Make sure all commits have finished */
1305 synchronize_sched();
1306 ring_buffer_reset_cpu(buffer
, cpu
);
1308 ring_buffer_record_enable(buffer
);
1311 void tracing_reset_online_cpus(struct trace_buffer
*buf
)
1313 struct ring_buffer
*buffer
= buf
->buffer
;
1319 ring_buffer_record_disable(buffer
);
1321 /* Make sure all commits have finished */
1322 synchronize_sched();
1324 buf
->time_start
= buffer_ftrace_now(buf
, buf
->cpu
);
1326 for_each_online_cpu(cpu
)
1327 ring_buffer_reset_cpu(buffer
, cpu
);
1329 ring_buffer_record_enable(buffer
);
1332 /* Must have trace_types_lock held */
1333 void tracing_reset_all_online_cpus(void)
1335 struct trace_array
*tr
;
1337 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
1338 tracing_reset_online_cpus(&tr
->trace_buffer
);
1339 #ifdef CONFIG_TRACER_MAX_TRACE
1340 tracing_reset_online_cpus(&tr
->max_buffer
);
1345 #define SAVED_CMDLINES_DEFAULT 128
1346 #define NO_CMDLINE_MAP UINT_MAX
1347 static arch_spinlock_t trace_cmdline_lock
= __ARCH_SPIN_LOCK_UNLOCKED
;
1348 struct saved_cmdlines_buffer
{
1349 unsigned map_pid_to_cmdline
[PID_MAX_DEFAULT
+1];
1350 unsigned *map_cmdline_to_pid
;
1351 unsigned cmdline_num
;
1353 char *saved_cmdlines
;
1355 static struct saved_cmdlines_buffer
*savedcmd
;
1357 /* temporary disable recording */
1358 static atomic_t trace_record_cmdline_disabled __read_mostly
;
1360 static inline char *get_saved_cmdlines(int idx
)
1362 return &savedcmd
->saved_cmdlines
[idx
* TASK_COMM_LEN
];
1365 static inline void set_cmdline(int idx
, const char *cmdline
)
1367 memcpy(get_saved_cmdlines(idx
), cmdline
, TASK_COMM_LEN
);
1370 static int allocate_cmdlines_buffer(unsigned int val
,
1371 struct saved_cmdlines_buffer
*s
)
1373 s
->map_cmdline_to_pid
= kmalloc(val
* sizeof(*s
->map_cmdline_to_pid
),
1375 if (!s
->map_cmdline_to_pid
)
1378 s
->saved_cmdlines
= kmalloc(val
* TASK_COMM_LEN
, GFP_KERNEL
);
1379 if (!s
->saved_cmdlines
) {
1380 kfree(s
->map_cmdline_to_pid
);
1385 s
->cmdline_num
= val
;
1386 memset(&s
->map_pid_to_cmdline
, NO_CMDLINE_MAP
,
1387 sizeof(s
->map_pid_to_cmdline
));
1388 memset(s
->map_cmdline_to_pid
, NO_CMDLINE_MAP
,
1389 val
* sizeof(*s
->map_cmdline_to_pid
));
1394 static int trace_create_savedcmd(void)
1398 savedcmd
= kmalloc(sizeof(*savedcmd
), GFP_KERNEL
);
1402 ret
= allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT
, savedcmd
);
1412 int is_tracing_stopped(void)
1414 return global_trace
.stop_count
;
1418 * tracing_start - quick start of the tracer
1420 * If tracing is enabled but was stopped by tracing_stop,
1421 * this will start the tracer back up.
1423 void tracing_start(void)
1425 struct ring_buffer
*buffer
;
1426 unsigned long flags
;
1428 if (tracing_disabled
)
1431 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1432 if (--global_trace
.stop_count
) {
1433 if (global_trace
.stop_count
< 0) {
1434 /* Someone screwed up their debugging */
1436 global_trace
.stop_count
= 0;
1441 /* Prevent the buffers from switching */
1442 arch_spin_lock(&global_trace
.max_lock
);
1444 buffer
= global_trace
.trace_buffer
.buffer
;
1446 ring_buffer_record_enable(buffer
);
1448 #ifdef CONFIG_TRACER_MAX_TRACE
1449 buffer
= global_trace
.max_buffer
.buffer
;
1451 ring_buffer_record_enable(buffer
);
1454 arch_spin_unlock(&global_trace
.max_lock
);
1457 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1460 static void tracing_start_tr(struct trace_array
*tr
)
1462 struct ring_buffer
*buffer
;
1463 unsigned long flags
;
1465 if (tracing_disabled
)
1468 /* If global, we need to also start the max tracer */
1469 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1470 return tracing_start();
1472 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1474 if (--tr
->stop_count
) {
1475 if (tr
->stop_count
< 0) {
1476 /* Someone screwed up their debugging */
1483 buffer
= tr
->trace_buffer
.buffer
;
1485 ring_buffer_record_enable(buffer
);
1488 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1492 * tracing_stop - quick stop of the tracer
1494 * Light weight way to stop tracing. Use in conjunction with
1497 void tracing_stop(void)
1499 struct ring_buffer
*buffer
;
1500 unsigned long flags
;
1502 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1503 if (global_trace
.stop_count
++)
1506 /* Prevent the buffers from switching */
1507 arch_spin_lock(&global_trace
.max_lock
);
1509 buffer
= global_trace
.trace_buffer
.buffer
;
1511 ring_buffer_record_disable(buffer
);
1513 #ifdef CONFIG_TRACER_MAX_TRACE
1514 buffer
= global_trace
.max_buffer
.buffer
;
1516 ring_buffer_record_disable(buffer
);
1519 arch_spin_unlock(&global_trace
.max_lock
);
1522 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1525 static void tracing_stop_tr(struct trace_array
*tr
)
1527 struct ring_buffer
*buffer
;
1528 unsigned long flags
;
1530 /* If global, we need to also stop the max tracer */
1531 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1532 return tracing_stop();
1534 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1535 if (tr
->stop_count
++)
1538 buffer
= tr
->trace_buffer
.buffer
;
1540 ring_buffer_record_disable(buffer
);
1543 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1546 void trace_stop_cmdline_recording(void);
1548 static int trace_save_cmdline(struct task_struct
*tsk
)
1552 if (!tsk
->pid
|| unlikely(tsk
->pid
> PID_MAX_DEFAULT
))
1556 * It's not the end of the world if we don't get
1557 * the lock, but we also don't want to spin
1558 * nor do we want to disable interrupts,
1559 * so if we miss here, then better luck next time.
1561 if (!arch_spin_trylock(&trace_cmdline_lock
))
1564 idx
= savedcmd
->map_pid_to_cmdline
[tsk
->pid
];
1565 if (idx
== NO_CMDLINE_MAP
) {
1566 idx
= (savedcmd
->cmdline_idx
+ 1) % savedcmd
->cmdline_num
;
1569 * Check whether the cmdline buffer at idx has a pid
1570 * mapped. We are going to overwrite that entry so we
1571 * need to clear the map_pid_to_cmdline. Otherwise we
1572 * would read the new comm for the old pid.
1574 pid
= savedcmd
->map_cmdline_to_pid
[idx
];
1575 if (pid
!= NO_CMDLINE_MAP
)
1576 savedcmd
->map_pid_to_cmdline
[pid
] = NO_CMDLINE_MAP
;
1578 savedcmd
->map_cmdline_to_pid
[idx
] = tsk
->pid
;
1579 savedcmd
->map_pid_to_cmdline
[tsk
->pid
] = idx
;
1581 savedcmd
->cmdline_idx
= idx
;
1584 set_cmdline(idx
, tsk
->comm
);
1586 arch_spin_unlock(&trace_cmdline_lock
);
1591 static void __trace_find_cmdline(int pid
, char comm
[])
1596 strcpy(comm
, "<idle>");
1600 if (WARN_ON_ONCE(pid
< 0)) {
1601 strcpy(comm
, "<XXX>");
1605 if (pid
> PID_MAX_DEFAULT
) {
1606 strcpy(comm
, "<...>");
1610 map
= savedcmd
->map_pid_to_cmdline
[pid
];
1611 if (map
!= NO_CMDLINE_MAP
)
1612 strcpy(comm
, get_saved_cmdlines(map
));
1614 strcpy(comm
, "<...>");
1617 void trace_find_cmdline(int pid
, char comm
[])
1620 arch_spin_lock(&trace_cmdline_lock
);
1622 __trace_find_cmdline(pid
, comm
);
1624 arch_spin_unlock(&trace_cmdline_lock
);
1628 void tracing_record_cmdline(struct task_struct
*tsk
)
1630 if (atomic_read(&trace_record_cmdline_disabled
) || !tracing_is_on())
1633 if (!__this_cpu_read(trace_cmdline_save
))
1636 if (trace_save_cmdline(tsk
))
1637 __this_cpu_write(trace_cmdline_save
, false);
1641 tracing_generic_entry_update(struct trace_entry
*entry
, unsigned long flags
,
1644 struct task_struct
*tsk
= current
;
1646 entry
->preempt_count
= pc
& 0xff;
1647 entry
->pid
= (tsk
) ? tsk
->pid
: 0;
1649 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1650 (irqs_disabled_flags(flags
) ? TRACE_FLAG_IRQS_OFF
: 0) |
1652 TRACE_FLAG_IRQS_NOSUPPORT
|
1654 ((pc
& HARDIRQ_MASK
) ? TRACE_FLAG_HARDIRQ
: 0) |
1655 ((pc
& SOFTIRQ_MASK
) ? TRACE_FLAG_SOFTIRQ
: 0) |
1656 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED
: 0) |
1657 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED
: 0);
1659 EXPORT_SYMBOL_GPL(tracing_generic_entry_update
);
1661 struct ring_buffer_event
*
1662 trace_buffer_lock_reserve(struct ring_buffer
*buffer
,
1665 unsigned long flags
, int pc
)
1667 struct ring_buffer_event
*event
;
1669 event
= ring_buffer_lock_reserve(buffer
, len
);
1670 if (event
!= NULL
) {
1671 struct trace_entry
*ent
= ring_buffer_event_data(event
);
1673 tracing_generic_entry_update(ent
, flags
, pc
);
1681 __buffer_unlock_commit(struct ring_buffer
*buffer
, struct ring_buffer_event
*event
)
1683 __this_cpu_write(trace_cmdline_save
, true);
1684 ring_buffer_unlock_commit(buffer
, event
);
1687 void trace_buffer_unlock_commit(struct trace_array
*tr
,
1688 struct ring_buffer
*buffer
,
1689 struct ring_buffer_event
*event
,
1690 unsigned long flags
, int pc
)
1692 __buffer_unlock_commit(buffer
, event
);
1694 ftrace_trace_stack(buffer
, flags
, 6, pc
, NULL
);
1695 ftrace_trace_userstack(buffer
, flags
, pc
);
1697 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit
);
1699 static struct ring_buffer
*temp_buffer
;
1701 struct ring_buffer_event
*
1702 trace_event_buffer_lock_reserve(struct ring_buffer
**current_rb
,
1703 struct trace_event_file
*trace_file
,
1704 int type
, unsigned long len
,
1705 unsigned long flags
, int pc
)
1707 struct ring_buffer_event
*entry
;
1709 *current_rb
= trace_file
->tr
->trace_buffer
.buffer
;
1710 entry
= trace_buffer_lock_reserve(*current_rb
,
1711 type
, len
, flags
, pc
);
1713 * If tracing is off, but we have triggers enabled
1714 * we still need to look at the event data. Use the temp_buffer
1715 * to store the trace event for the tigger to use. It's recusive
1716 * safe and will not be recorded anywhere.
1718 if (!entry
&& trace_file
->flags
& EVENT_FILE_FL_TRIGGER_COND
) {
1719 *current_rb
= temp_buffer
;
1720 entry
= trace_buffer_lock_reserve(*current_rb
,
1721 type
, len
, flags
, pc
);
1725 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve
);
1727 struct ring_buffer_event
*
1728 trace_current_buffer_lock_reserve(struct ring_buffer
**current_rb
,
1729 int type
, unsigned long len
,
1730 unsigned long flags
, int pc
)
1732 *current_rb
= global_trace
.trace_buffer
.buffer
;
1733 return trace_buffer_lock_reserve(*current_rb
,
1734 type
, len
, flags
, pc
);
1736 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve
);
1738 void trace_buffer_unlock_commit_regs(struct trace_array
*tr
,
1739 struct ring_buffer
*buffer
,
1740 struct ring_buffer_event
*event
,
1741 unsigned long flags
, int pc
,
1742 struct pt_regs
*regs
)
1744 __buffer_unlock_commit(buffer
, event
);
1746 ftrace_trace_stack(buffer
, flags
, 6, pc
, regs
);
1747 ftrace_trace_userstack(buffer
, flags
, pc
);
1749 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs
);
1751 void trace_current_buffer_discard_commit(struct ring_buffer
*buffer
,
1752 struct ring_buffer_event
*event
)
1754 ring_buffer_discard_commit(buffer
, event
);
1756 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit
);
1759 trace_function(struct trace_array
*tr
,
1760 unsigned long ip
, unsigned long parent_ip
, unsigned long flags
,
1763 struct trace_event_call
*call
= &event_function
;
1764 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
1765 struct ring_buffer_event
*event
;
1766 struct ftrace_entry
*entry
;
1768 /* If we are reading the ring buffer, don't trace */
1769 if (unlikely(__this_cpu_read(ftrace_cpu_disabled
)))
1772 event
= trace_buffer_lock_reserve(buffer
, TRACE_FN
, sizeof(*entry
),
1776 entry
= ring_buffer_event_data(event
);
1778 entry
->parent_ip
= parent_ip
;
1780 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1781 __buffer_unlock_commit(buffer
, event
);
1784 #ifdef CONFIG_STACKTRACE
1786 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1787 struct ftrace_stack
{
1788 unsigned long calls
[FTRACE_STACK_MAX_ENTRIES
];
1791 static DEFINE_PER_CPU(struct ftrace_stack
, ftrace_stack
);
1792 static DEFINE_PER_CPU(int, ftrace_stack_reserve
);
1794 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
1795 unsigned long flags
,
1796 int skip
, int pc
, struct pt_regs
*regs
)
1798 struct trace_event_call
*call
= &event_kernel_stack
;
1799 struct ring_buffer_event
*event
;
1800 struct stack_entry
*entry
;
1801 struct stack_trace trace
;
1803 int size
= FTRACE_STACK_ENTRIES
;
1805 trace
.nr_entries
= 0;
1809 * Since events can happen in NMIs there's no safe way to
1810 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1811 * or NMI comes in, it will just have to use the default
1812 * FTRACE_STACK_SIZE.
1814 preempt_disable_notrace();
1816 use_stack
= __this_cpu_inc_return(ftrace_stack_reserve
);
1818 * We don't need any atomic variables, just a barrier.
1819 * If an interrupt comes in, we don't care, because it would
1820 * have exited and put the counter back to what we want.
1821 * We just need a barrier to keep gcc from moving things
1825 if (use_stack
== 1) {
1826 trace
.entries
= this_cpu_ptr(ftrace_stack
.calls
);
1827 trace
.max_entries
= FTRACE_STACK_MAX_ENTRIES
;
1830 save_stack_trace_regs(regs
, &trace
);
1832 save_stack_trace(&trace
);
1834 if (trace
.nr_entries
> size
)
1835 size
= trace
.nr_entries
;
1837 /* From now on, use_stack is a boolean */
1840 size
*= sizeof(unsigned long);
1842 event
= trace_buffer_lock_reserve(buffer
, TRACE_STACK
,
1843 sizeof(*entry
) + size
, flags
, pc
);
1846 entry
= ring_buffer_event_data(event
);
1848 memset(&entry
->caller
, 0, size
);
1851 memcpy(&entry
->caller
, trace
.entries
,
1852 trace
.nr_entries
* sizeof(unsigned long));
1854 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
1855 trace
.entries
= entry
->caller
;
1857 save_stack_trace_regs(regs
, &trace
);
1859 save_stack_trace(&trace
);
1862 entry
->size
= trace
.nr_entries
;
1864 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1865 __buffer_unlock_commit(buffer
, event
);
1868 /* Again, don't let gcc optimize things here */
1870 __this_cpu_dec(ftrace_stack_reserve
);
1871 preempt_enable_notrace();
1875 static inline void ftrace_trace_stack(struct ring_buffer
*buffer
,
1876 unsigned long flags
,
1877 int skip
, int pc
, struct pt_regs
*regs
)
1879 if (!(global_trace
.trace_flags
& TRACE_ITER_STACKTRACE
))
1882 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, regs
);
1885 void __trace_stack(struct trace_array
*tr
, unsigned long flags
, int skip
,
1888 __ftrace_trace_stack(tr
->trace_buffer
.buffer
, flags
, skip
, pc
, NULL
);
1892 * trace_dump_stack - record a stack back trace in the trace buffer
1893 * @skip: Number of functions to skip (helper handlers)
1895 void trace_dump_stack(int skip
)
1897 unsigned long flags
;
1899 if (tracing_disabled
|| tracing_selftest_running
)
1902 local_save_flags(flags
);
1905 * Skip 3 more, seems to get us at the caller of
1909 __ftrace_trace_stack(global_trace
.trace_buffer
.buffer
,
1910 flags
, skip
, preempt_count(), NULL
);
1913 static DEFINE_PER_CPU(int, user_stack_count
);
1916 ftrace_trace_userstack(struct ring_buffer
*buffer
, unsigned long flags
, int pc
)
1918 struct trace_event_call
*call
= &event_user_stack
;
1919 struct ring_buffer_event
*event
;
1920 struct userstack_entry
*entry
;
1921 struct stack_trace trace
;
1923 if (!(global_trace
.trace_flags
& TRACE_ITER_USERSTACKTRACE
))
1927 * NMIs can not handle page faults, even with fix ups.
1928 * The save user stack can (and often does) fault.
1930 if (unlikely(in_nmi()))
1934 * prevent recursion, since the user stack tracing may
1935 * trigger other kernel events.
1938 if (__this_cpu_read(user_stack_count
))
1941 __this_cpu_inc(user_stack_count
);
1943 event
= trace_buffer_lock_reserve(buffer
, TRACE_USER_STACK
,
1944 sizeof(*entry
), flags
, pc
);
1946 goto out_drop_count
;
1947 entry
= ring_buffer_event_data(event
);
1949 entry
->tgid
= current
->tgid
;
1950 memset(&entry
->caller
, 0, sizeof(entry
->caller
));
1952 trace
.nr_entries
= 0;
1953 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
1955 trace
.entries
= entry
->caller
;
1957 save_stack_trace_user(&trace
);
1958 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1959 __buffer_unlock_commit(buffer
, event
);
1962 __this_cpu_dec(user_stack_count
);
1968 static void __trace_userstack(struct trace_array
*tr
, unsigned long flags
)
1970 ftrace_trace_userstack(tr
, flags
, preempt_count());
1974 #endif /* CONFIG_STACKTRACE */
1976 /* created for use with alloc_percpu */
1977 struct trace_buffer_struct
{
1978 char buffer
[TRACE_BUF_SIZE
];
1981 static struct trace_buffer_struct
*trace_percpu_buffer
;
1982 static struct trace_buffer_struct
*trace_percpu_sirq_buffer
;
1983 static struct trace_buffer_struct
*trace_percpu_irq_buffer
;
1984 static struct trace_buffer_struct
*trace_percpu_nmi_buffer
;
1987 * The buffer used is dependent on the context. There is a per cpu
1988 * buffer for normal context, softirq contex, hard irq context and
1989 * for NMI context. Thise allows for lockless recording.
1991 * Note, if the buffers failed to be allocated, then this returns NULL
1993 static char *get_trace_buf(void)
1995 struct trace_buffer_struct
*percpu_buffer
;
1998 * If we have allocated per cpu buffers, then we do not
1999 * need to do any locking.
2002 percpu_buffer
= trace_percpu_nmi_buffer
;
2004 percpu_buffer
= trace_percpu_irq_buffer
;
2005 else if (in_softirq())
2006 percpu_buffer
= trace_percpu_sirq_buffer
;
2008 percpu_buffer
= trace_percpu_buffer
;
2013 return this_cpu_ptr(&percpu_buffer
->buffer
[0]);
2016 static int alloc_percpu_trace_buffer(void)
2018 struct trace_buffer_struct
*buffers
;
2019 struct trace_buffer_struct
*sirq_buffers
;
2020 struct trace_buffer_struct
*irq_buffers
;
2021 struct trace_buffer_struct
*nmi_buffers
;
2023 buffers
= alloc_percpu(struct trace_buffer_struct
);
2027 sirq_buffers
= alloc_percpu(struct trace_buffer_struct
);
2031 irq_buffers
= alloc_percpu(struct trace_buffer_struct
);
2035 nmi_buffers
= alloc_percpu(struct trace_buffer_struct
);
2039 trace_percpu_buffer
= buffers
;
2040 trace_percpu_sirq_buffer
= sirq_buffers
;
2041 trace_percpu_irq_buffer
= irq_buffers
;
2042 trace_percpu_nmi_buffer
= nmi_buffers
;
2047 free_percpu(irq_buffers
);
2049 free_percpu(sirq_buffers
);
2051 free_percpu(buffers
);
2053 WARN(1, "Could not allocate percpu trace_printk buffer");
2057 static int buffers_allocated
;
2059 void trace_printk_init_buffers(void)
2061 if (buffers_allocated
)
2064 if (alloc_percpu_trace_buffer())
2067 /* trace_printk() is for debug use only. Don't use it in production. */
2070 pr_warning("**********************************************************\n");
2071 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2072 pr_warning("** **\n");
2073 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2074 pr_warning("** **\n");
2075 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2076 pr_warning("** unsafe for production use. **\n");
2077 pr_warning("** **\n");
2078 pr_warning("** If you see this message and you are not debugging **\n");
2079 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2080 pr_warning("** **\n");
2081 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2082 pr_warning("**********************************************************\n");
2084 /* Expand the buffers to set size */
2085 tracing_update_buffers();
2087 buffers_allocated
= 1;
2090 * trace_printk_init_buffers() can be called by modules.
2091 * If that happens, then we need to start cmdline recording
2092 * directly here. If the global_trace.buffer is already
2093 * allocated here, then this was called by module code.
2095 if (global_trace
.trace_buffer
.buffer
)
2096 tracing_start_cmdline_record();
2099 void trace_printk_start_comm(void)
2101 /* Start tracing comms if trace printk is set */
2102 if (!buffers_allocated
)
2104 tracing_start_cmdline_record();
2107 static void trace_printk_start_stop_comm(int enabled
)
2109 if (!buffers_allocated
)
2113 tracing_start_cmdline_record();
2115 tracing_stop_cmdline_record();
2119 * trace_vbprintk - write binary msg to tracing buffer
2122 int trace_vbprintk(unsigned long ip
, const char *fmt
, va_list args
)
2124 struct trace_event_call
*call
= &event_bprint
;
2125 struct ring_buffer_event
*event
;
2126 struct ring_buffer
*buffer
;
2127 struct trace_array
*tr
= &global_trace
;
2128 struct bprint_entry
*entry
;
2129 unsigned long flags
;
2131 int len
= 0, size
, pc
;
2133 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
2136 /* Don't pollute graph traces with trace_vprintk internals */
2137 pause_graph_tracing();
2139 pc
= preempt_count();
2140 preempt_disable_notrace();
2142 tbuffer
= get_trace_buf();
2148 len
= vbin_printf((u32
*)tbuffer
, TRACE_BUF_SIZE
/sizeof(int), fmt
, args
);
2150 if (len
> TRACE_BUF_SIZE
/sizeof(int) || len
< 0)
2153 local_save_flags(flags
);
2154 size
= sizeof(*entry
) + sizeof(u32
) * len
;
2155 buffer
= tr
->trace_buffer
.buffer
;
2156 event
= trace_buffer_lock_reserve(buffer
, TRACE_BPRINT
, size
,
2160 entry
= ring_buffer_event_data(event
);
2164 memcpy(entry
->buf
, tbuffer
, sizeof(u32
) * len
);
2165 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2166 __buffer_unlock_commit(buffer
, event
);
2167 ftrace_trace_stack(buffer
, flags
, 6, pc
, NULL
);
2171 preempt_enable_notrace();
2172 unpause_graph_tracing();
2176 EXPORT_SYMBOL_GPL(trace_vbprintk
);
2179 __trace_array_vprintk(struct ring_buffer
*buffer
,
2180 unsigned long ip
, const char *fmt
, va_list args
)
2182 struct trace_event_call
*call
= &event_print
;
2183 struct ring_buffer_event
*event
;
2184 int len
= 0, size
, pc
;
2185 struct print_entry
*entry
;
2186 unsigned long flags
;
2189 if (tracing_disabled
|| tracing_selftest_running
)
2192 /* Don't pollute graph traces with trace_vprintk internals */
2193 pause_graph_tracing();
2195 pc
= preempt_count();
2196 preempt_disable_notrace();
2199 tbuffer
= get_trace_buf();
2205 len
= vscnprintf(tbuffer
, TRACE_BUF_SIZE
, fmt
, args
);
2207 local_save_flags(flags
);
2208 size
= sizeof(*entry
) + len
+ 1;
2209 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
2213 entry
= ring_buffer_event_data(event
);
2216 memcpy(&entry
->buf
, tbuffer
, len
+ 1);
2217 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2218 __buffer_unlock_commit(buffer
, event
);
2219 ftrace_trace_stack(buffer
, flags
, 6, pc
, NULL
);
2222 preempt_enable_notrace();
2223 unpause_graph_tracing();
2228 int trace_array_vprintk(struct trace_array
*tr
,
2229 unsigned long ip
, const char *fmt
, va_list args
)
2231 return __trace_array_vprintk(tr
->trace_buffer
.buffer
, ip
, fmt
, args
);
2234 int trace_array_printk(struct trace_array
*tr
,
2235 unsigned long ip
, const char *fmt
, ...)
2240 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
2244 ret
= trace_array_vprintk(tr
, ip
, fmt
, ap
);
2249 int trace_array_printk_buf(struct ring_buffer
*buffer
,
2250 unsigned long ip
, const char *fmt
, ...)
2255 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
2259 ret
= __trace_array_vprintk(buffer
, ip
, fmt
, ap
);
2264 int trace_vprintk(unsigned long ip
, const char *fmt
, va_list args
)
2266 return trace_array_vprintk(&global_trace
, ip
, fmt
, args
);
2268 EXPORT_SYMBOL_GPL(trace_vprintk
);
2270 static void trace_iterator_increment(struct trace_iterator
*iter
)
2272 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, iter
->cpu
);
2276 ring_buffer_read(buf_iter
, NULL
);
2279 static struct trace_entry
*
2280 peek_next_entry(struct trace_iterator
*iter
, int cpu
, u64
*ts
,
2281 unsigned long *lost_events
)
2283 struct ring_buffer_event
*event
;
2284 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, cpu
);
2287 event
= ring_buffer_iter_peek(buf_iter
, ts
);
2289 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, cpu
, ts
,
2293 iter
->ent_size
= ring_buffer_event_length(event
);
2294 return ring_buffer_event_data(event
);
2300 static struct trace_entry
*
2301 __find_next_entry(struct trace_iterator
*iter
, int *ent_cpu
,
2302 unsigned long *missing_events
, u64
*ent_ts
)
2304 struct ring_buffer
*buffer
= iter
->trace_buffer
->buffer
;
2305 struct trace_entry
*ent
, *next
= NULL
;
2306 unsigned long lost_events
= 0, next_lost
= 0;
2307 int cpu_file
= iter
->cpu_file
;
2308 u64 next_ts
= 0, ts
;
2314 * If we are in a per_cpu trace file, don't bother by iterating over
2315 * all cpu and peek directly.
2317 if (cpu_file
> RING_BUFFER_ALL_CPUS
) {
2318 if (ring_buffer_empty_cpu(buffer
, cpu_file
))
2320 ent
= peek_next_entry(iter
, cpu_file
, ent_ts
, missing_events
);
2322 *ent_cpu
= cpu_file
;
2327 for_each_tracing_cpu(cpu
) {
2329 if (ring_buffer_empty_cpu(buffer
, cpu
))
2332 ent
= peek_next_entry(iter
, cpu
, &ts
, &lost_events
);
2335 * Pick the entry with the smallest timestamp:
2337 if (ent
&& (!next
|| ts
< next_ts
)) {
2341 next_lost
= lost_events
;
2342 next_size
= iter
->ent_size
;
2346 iter
->ent_size
= next_size
;
2349 *ent_cpu
= next_cpu
;
2355 *missing_events
= next_lost
;
2360 /* Find the next real entry, without updating the iterator itself */
2361 struct trace_entry
*trace_find_next_entry(struct trace_iterator
*iter
,
2362 int *ent_cpu
, u64
*ent_ts
)
2364 return __find_next_entry(iter
, ent_cpu
, NULL
, ent_ts
);
2367 /* Find the next real entry, and increment the iterator to the next entry */
2368 void *trace_find_next_entry_inc(struct trace_iterator
*iter
)
2370 iter
->ent
= __find_next_entry(iter
, &iter
->cpu
,
2371 &iter
->lost_events
, &iter
->ts
);
2374 trace_iterator_increment(iter
);
2376 return iter
->ent
? iter
: NULL
;
2379 static void trace_consume(struct trace_iterator
*iter
)
2381 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
, &iter
->ts
,
2382 &iter
->lost_events
);
2385 static void *s_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
2387 struct trace_iterator
*iter
= m
->private;
2391 WARN_ON_ONCE(iter
->leftover
);
2395 /* can't go backwards */
2400 ent
= trace_find_next_entry_inc(iter
);
2404 while (ent
&& iter
->idx
< i
)
2405 ent
= trace_find_next_entry_inc(iter
);
2412 void tracing_iter_reset(struct trace_iterator
*iter
, int cpu
)
2414 struct ring_buffer_event
*event
;
2415 struct ring_buffer_iter
*buf_iter
;
2416 unsigned long entries
= 0;
2419 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= 0;
2421 buf_iter
= trace_buffer_iter(iter
, cpu
);
2425 ring_buffer_iter_reset(buf_iter
);
2428 * We could have the case with the max latency tracers
2429 * that a reset never took place on a cpu. This is evident
2430 * by the timestamp being before the start of the buffer.
2432 while ((event
= ring_buffer_iter_peek(buf_iter
, &ts
))) {
2433 if (ts
>= iter
->trace_buffer
->time_start
)
2436 ring_buffer_read(buf_iter
, NULL
);
2439 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= entries
;
2443 * The current tracer is copied to avoid a global locking
2446 static void *s_start(struct seq_file
*m
, loff_t
*pos
)
2448 struct trace_iterator
*iter
= m
->private;
2449 struct trace_array
*tr
= iter
->tr
;
2450 int cpu_file
= iter
->cpu_file
;
2456 * copy the tracer to avoid using a global lock all around.
2457 * iter->trace is a copy of current_trace, the pointer to the
2458 * name may be used instead of a strcmp(), as iter->trace->name
2459 * will point to the same string as current_trace->name.
2461 mutex_lock(&trace_types_lock
);
2462 if (unlikely(tr
->current_trace
&& iter
->trace
->name
!= tr
->current_trace
->name
))
2463 *iter
->trace
= *tr
->current_trace
;
2464 mutex_unlock(&trace_types_lock
);
2466 #ifdef CONFIG_TRACER_MAX_TRACE
2467 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
2468 return ERR_PTR(-EBUSY
);
2471 if (!iter
->snapshot
)
2472 atomic_inc(&trace_record_cmdline_disabled
);
2474 if (*pos
!= iter
->pos
) {
2479 if (cpu_file
== RING_BUFFER_ALL_CPUS
) {
2480 for_each_tracing_cpu(cpu
)
2481 tracing_iter_reset(iter
, cpu
);
2483 tracing_iter_reset(iter
, cpu_file
);
2486 for (p
= iter
; p
&& l
< *pos
; p
= s_next(m
, p
, &l
))
2491 * If we overflowed the seq_file before, then we want
2492 * to just reuse the trace_seq buffer again.
2498 p
= s_next(m
, p
, &l
);
2502 trace_event_read_lock();
2503 trace_access_lock(cpu_file
);
2507 static void s_stop(struct seq_file
*m
, void *p
)
2509 struct trace_iterator
*iter
= m
->private;
2511 #ifdef CONFIG_TRACER_MAX_TRACE
2512 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
2516 if (!iter
->snapshot
)
2517 atomic_dec(&trace_record_cmdline_disabled
);
2519 trace_access_unlock(iter
->cpu_file
);
2520 trace_event_read_unlock();
2524 get_total_entries(struct trace_buffer
*buf
,
2525 unsigned long *total
, unsigned long *entries
)
2527 unsigned long count
;
2533 for_each_tracing_cpu(cpu
) {
2534 count
= ring_buffer_entries_cpu(buf
->buffer
, cpu
);
2536 * If this buffer has skipped entries, then we hold all
2537 * entries for the trace and we need to ignore the
2538 * ones before the time stamp.
2540 if (per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
) {
2541 count
-= per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
;
2542 /* total is the same as the entries */
2546 ring_buffer_overrun_cpu(buf
->buffer
, cpu
);
2551 static void print_lat_help_header(struct seq_file
*m
)
2553 seq_puts(m
, "# _------=> CPU# \n"
2554 "# / _-----=> irqs-off \n"
2555 "# | / _----=> need-resched \n"
2556 "# || / _---=> hardirq/softirq \n"
2557 "# ||| / _--=> preempt-depth \n"
2559 "# cmd pid ||||| time | caller \n"
2560 "# \\ / ||||| \\ | / \n");
2563 static void print_event_info(struct trace_buffer
*buf
, struct seq_file
*m
)
2565 unsigned long total
;
2566 unsigned long entries
;
2568 get_total_entries(buf
, &total
, &entries
);
2569 seq_printf(m
, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2570 entries
, total
, num_online_cpus());
2574 static void print_func_help_header(struct trace_buffer
*buf
, struct seq_file
*m
)
2576 print_event_info(buf
, m
);
2577 seq_puts(m
, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2581 static void print_func_help_header_irq(struct trace_buffer
*buf
, struct seq_file
*m
)
2583 print_event_info(buf
, m
);
2584 seq_puts(m
, "# _-----=> irqs-off\n"
2585 "# / _----=> need-resched\n"
2586 "# | / _---=> hardirq/softirq\n"
2587 "# || / _--=> preempt-depth\n"
2589 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2590 "# | | | |||| | |\n");
2594 print_trace_header(struct seq_file
*m
, struct trace_iterator
*iter
)
2596 unsigned long sym_flags
= (global_trace
.trace_flags
& TRACE_ITER_SYM_MASK
);
2597 struct trace_buffer
*buf
= iter
->trace_buffer
;
2598 struct trace_array_cpu
*data
= per_cpu_ptr(buf
->data
, buf
->cpu
);
2599 struct tracer
*type
= iter
->trace
;
2600 unsigned long entries
;
2601 unsigned long total
;
2602 const char *name
= "preemption";
2606 get_total_entries(buf
, &total
, &entries
);
2608 seq_printf(m
, "# %s latency trace v1.1.5 on %s\n",
2610 seq_puts(m
, "# -----------------------------------"
2611 "---------------------------------\n");
2612 seq_printf(m
, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2613 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2614 nsecs_to_usecs(data
->saved_latency
),
2618 #if defined(CONFIG_PREEMPT_NONE)
2620 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2622 #elif defined(CONFIG_PREEMPT)
2627 /* These are reserved for later use */
2630 seq_printf(m
, " #P:%d)\n", num_online_cpus());
2634 seq_puts(m
, "# -----------------\n");
2635 seq_printf(m
, "# | task: %.16s-%d "
2636 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2637 data
->comm
, data
->pid
,
2638 from_kuid_munged(seq_user_ns(m
), data
->uid
), data
->nice
,
2639 data
->policy
, data
->rt_priority
);
2640 seq_puts(m
, "# -----------------\n");
2642 if (data
->critical_start
) {
2643 seq_puts(m
, "# => started at: ");
2644 seq_print_ip_sym(&iter
->seq
, data
->critical_start
, sym_flags
);
2645 trace_print_seq(m
, &iter
->seq
);
2646 seq_puts(m
, "\n# => ended at: ");
2647 seq_print_ip_sym(&iter
->seq
, data
->critical_end
, sym_flags
);
2648 trace_print_seq(m
, &iter
->seq
);
2649 seq_puts(m
, "\n#\n");
2655 static void test_cpu_buff_start(struct trace_iterator
*iter
)
2657 struct trace_seq
*s
= &iter
->seq
;
2658 struct trace_array
*tr
= iter
->tr
;
2660 if (!(tr
->trace_flags
& TRACE_ITER_ANNOTATE
))
2663 if (!(iter
->iter_flags
& TRACE_FILE_ANNOTATE
))
2666 if (cpumask_test_cpu(iter
->cpu
, iter
->started
))
2669 if (per_cpu_ptr(iter
->trace_buffer
->data
, iter
->cpu
)->skipped_entries
)
2672 cpumask_set_cpu(iter
->cpu
, iter
->started
);
2674 /* Don't print started cpu buffer for the first entry of the trace */
2676 trace_seq_printf(s
, "##### CPU %u buffer started ####\n",
2680 static enum print_line_t
print_trace_fmt(struct trace_iterator
*iter
)
2682 struct trace_array
*tr
= iter
->tr
;
2683 struct trace_seq
*s
= &iter
->seq
;
2684 unsigned long sym_flags
= (tr
->trace_flags
& TRACE_ITER_SYM_MASK
);
2685 struct trace_entry
*entry
;
2686 struct trace_event
*event
;
2690 test_cpu_buff_start(iter
);
2692 event
= ftrace_find_event(entry
->type
);
2694 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2695 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
2696 trace_print_lat_context(iter
);
2698 trace_print_context(iter
);
2701 if (trace_seq_has_overflowed(s
))
2702 return TRACE_TYPE_PARTIAL_LINE
;
2705 return event
->funcs
->trace(iter
, sym_flags
, event
);
2707 trace_seq_printf(s
, "Unknown type %d\n", entry
->type
);
2709 return trace_handle_return(s
);
2712 static enum print_line_t
print_raw_fmt(struct trace_iterator
*iter
)
2714 struct trace_array
*tr
= iter
->tr
;
2715 struct trace_seq
*s
= &iter
->seq
;
2716 struct trace_entry
*entry
;
2717 struct trace_event
*event
;
2721 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
)
2722 trace_seq_printf(s
, "%d %d %llu ",
2723 entry
->pid
, iter
->cpu
, iter
->ts
);
2725 if (trace_seq_has_overflowed(s
))
2726 return TRACE_TYPE_PARTIAL_LINE
;
2728 event
= ftrace_find_event(entry
->type
);
2730 return event
->funcs
->raw(iter
, 0, event
);
2732 trace_seq_printf(s
, "%d ?\n", entry
->type
);
2734 return trace_handle_return(s
);
2737 static enum print_line_t
print_hex_fmt(struct trace_iterator
*iter
)
2739 struct trace_array
*tr
= iter
->tr
;
2740 struct trace_seq
*s
= &iter
->seq
;
2741 unsigned char newline
= '\n';
2742 struct trace_entry
*entry
;
2743 struct trace_event
*event
;
2747 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2748 SEQ_PUT_HEX_FIELD(s
, entry
->pid
);
2749 SEQ_PUT_HEX_FIELD(s
, iter
->cpu
);
2750 SEQ_PUT_HEX_FIELD(s
, iter
->ts
);
2751 if (trace_seq_has_overflowed(s
))
2752 return TRACE_TYPE_PARTIAL_LINE
;
2755 event
= ftrace_find_event(entry
->type
);
2757 enum print_line_t ret
= event
->funcs
->hex(iter
, 0, event
);
2758 if (ret
!= TRACE_TYPE_HANDLED
)
2762 SEQ_PUT_FIELD(s
, newline
);
2764 return trace_handle_return(s
);
2767 static enum print_line_t
print_bin_fmt(struct trace_iterator
*iter
)
2769 struct trace_array
*tr
= iter
->tr
;
2770 struct trace_seq
*s
= &iter
->seq
;
2771 struct trace_entry
*entry
;
2772 struct trace_event
*event
;
2776 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2777 SEQ_PUT_FIELD(s
, entry
->pid
);
2778 SEQ_PUT_FIELD(s
, iter
->cpu
);
2779 SEQ_PUT_FIELD(s
, iter
->ts
);
2780 if (trace_seq_has_overflowed(s
))
2781 return TRACE_TYPE_PARTIAL_LINE
;
2784 event
= ftrace_find_event(entry
->type
);
2785 return event
? event
->funcs
->binary(iter
, 0, event
) :
2789 int trace_empty(struct trace_iterator
*iter
)
2791 struct ring_buffer_iter
*buf_iter
;
2794 /* If we are looking at one CPU buffer, only check that one */
2795 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
2796 cpu
= iter
->cpu_file
;
2797 buf_iter
= trace_buffer_iter(iter
, cpu
);
2799 if (!ring_buffer_iter_empty(buf_iter
))
2802 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
2808 for_each_tracing_cpu(cpu
) {
2809 buf_iter
= trace_buffer_iter(iter
, cpu
);
2811 if (!ring_buffer_iter_empty(buf_iter
))
2814 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
2822 /* Called with trace_event_read_lock() held. */
2823 enum print_line_t
print_trace_line(struct trace_iterator
*iter
)
2825 struct trace_array
*tr
= iter
->tr
;
2826 unsigned long trace_flags
= tr
->trace_flags
;
2827 enum print_line_t ret
;
2829 if (iter
->lost_events
) {
2830 trace_seq_printf(&iter
->seq
, "CPU:%d [LOST %lu EVENTS]\n",
2831 iter
->cpu
, iter
->lost_events
);
2832 if (trace_seq_has_overflowed(&iter
->seq
))
2833 return TRACE_TYPE_PARTIAL_LINE
;
2836 if (iter
->trace
&& iter
->trace
->print_line
) {
2837 ret
= iter
->trace
->print_line(iter
);
2838 if (ret
!= TRACE_TYPE_UNHANDLED
)
2842 if (iter
->ent
->type
== TRACE_BPUTS
&&
2843 trace_flags
& TRACE_ITER_PRINTK
&&
2844 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2845 return trace_print_bputs_msg_only(iter
);
2847 if (iter
->ent
->type
== TRACE_BPRINT
&&
2848 trace_flags
& TRACE_ITER_PRINTK
&&
2849 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2850 return trace_print_bprintk_msg_only(iter
);
2852 if (iter
->ent
->type
== TRACE_PRINT
&&
2853 trace_flags
& TRACE_ITER_PRINTK
&&
2854 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2855 return trace_print_printk_msg_only(iter
);
2857 if (trace_flags
& TRACE_ITER_BIN
)
2858 return print_bin_fmt(iter
);
2860 if (trace_flags
& TRACE_ITER_HEX
)
2861 return print_hex_fmt(iter
);
2863 if (trace_flags
& TRACE_ITER_RAW
)
2864 return print_raw_fmt(iter
);
2866 return print_trace_fmt(iter
);
2869 void trace_latency_header(struct seq_file
*m
)
2871 struct trace_iterator
*iter
= m
->private;
2872 struct trace_array
*tr
= iter
->tr
;
2874 /* print nothing if the buffers are empty */
2875 if (trace_empty(iter
))
2878 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
2879 print_trace_header(m
, iter
);
2881 if (!(tr
->trace_flags
& TRACE_ITER_VERBOSE
))
2882 print_lat_help_header(m
);
2885 void trace_default_header(struct seq_file
*m
)
2887 struct trace_iterator
*iter
= m
->private;
2888 struct trace_array
*tr
= iter
->tr
;
2889 unsigned long trace_flags
= tr
->trace_flags
;
2891 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
2894 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
) {
2895 /* print nothing if the buffers are empty */
2896 if (trace_empty(iter
))
2898 print_trace_header(m
, iter
);
2899 if (!(trace_flags
& TRACE_ITER_VERBOSE
))
2900 print_lat_help_header(m
);
2902 if (!(trace_flags
& TRACE_ITER_VERBOSE
)) {
2903 if (trace_flags
& TRACE_ITER_IRQ_INFO
)
2904 print_func_help_header_irq(iter
->trace_buffer
, m
);
2906 print_func_help_header(iter
->trace_buffer
, m
);
2911 static void test_ftrace_alive(struct seq_file
*m
)
2913 if (!ftrace_is_dead())
2915 seq_puts(m
, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2916 "# MAY BE MISSING FUNCTION EVENTS\n");
2919 #ifdef CONFIG_TRACER_MAX_TRACE
2920 static void show_snapshot_main_help(struct seq_file
*m
)
2922 seq_puts(m
, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2923 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2924 "# Takes a snapshot of the main buffer.\n"
2925 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2926 "# (Doesn't have to be '2' works with any number that\n"
2927 "# is not a '0' or '1')\n");
2930 static void show_snapshot_percpu_help(struct seq_file
*m
)
2932 seq_puts(m
, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2933 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2934 seq_puts(m
, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2935 "# Takes a snapshot of the main buffer for this cpu.\n");
2937 seq_puts(m
, "# echo 1 > snapshot : Not supported with this kernel.\n"
2938 "# Must use main snapshot file to allocate.\n");
2940 seq_puts(m
, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2941 "# (Doesn't have to be '2' works with any number that\n"
2942 "# is not a '0' or '1')\n");
2945 static void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
)
2947 if (iter
->tr
->allocated_snapshot
)
2948 seq_puts(m
, "#\n# * Snapshot is allocated *\n#\n");
2950 seq_puts(m
, "#\n# * Snapshot is freed *\n#\n");
2952 seq_puts(m
, "# Snapshot commands:\n");
2953 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
2954 show_snapshot_main_help(m
);
2956 show_snapshot_percpu_help(m
);
2959 /* Should never be called */
2960 static inline void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
) { }
2963 static int s_show(struct seq_file
*m
, void *v
)
2965 struct trace_iterator
*iter
= v
;
2968 if (iter
->ent
== NULL
) {
2970 seq_printf(m
, "# tracer: %s\n", iter
->trace
->name
);
2972 test_ftrace_alive(m
);
2974 if (iter
->snapshot
&& trace_empty(iter
))
2975 print_snapshot_help(m
, iter
);
2976 else if (iter
->trace
&& iter
->trace
->print_header
)
2977 iter
->trace
->print_header(m
);
2979 trace_default_header(m
);
2981 } else if (iter
->leftover
) {
2983 * If we filled the seq_file buffer earlier, we
2984 * want to just show it now.
2986 ret
= trace_print_seq(m
, &iter
->seq
);
2988 /* ret should this time be zero, but you never know */
2989 iter
->leftover
= ret
;
2992 print_trace_line(iter
);
2993 ret
= trace_print_seq(m
, &iter
->seq
);
2995 * If we overflow the seq_file buffer, then it will
2996 * ask us for this data again at start up.
2998 * ret is 0 if seq_file write succeeded.
3001 iter
->leftover
= ret
;
3008 * Should be used after trace_array_get(), trace_types_lock
3009 * ensures that i_cdev was already initialized.
3011 static inline int tracing_get_cpu(struct inode
*inode
)
3013 if (inode
->i_cdev
) /* See trace_create_cpu_file() */
3014 return (long)inode
->i_cdev
- 1;
3015 return RING_BUFFER_ALL_CPUS
;
3018 static const struct seq_operations tracer_seq_ops
= {
3025 static struct trace_iterator
*
3026 __tracing_open(struct inode
*inode
, struct file
*file
, bool snapshot
)
3028 struct trace_array
*tr
= inode
->i_private
;
3029 struct trace_iterator
*iter
;
3032 if (tracing_disabled
)
3033 return ERR_PTR(-ENODEV
);
3035 iter
= __seq_open_private(file
, &tracer_seq_ops
, sizeof(*iter
));
3037 return ERR_PTR(-ENOMEM
);
3039 iter
->buffer_iter
= kcalloc(nr_cpu_ids
, sizeof(*iter
->buffer_iter
),
3041 if (!iter
->buffer_iter
)
3045 * We make a copy of the current tracer to avoid concurrent
3046 * changes on it while we are reading.
3048 mutex_lock(&trace_types_lock
);
3049 iter
->trace
= kzalloc(sizeof(*iter
->trace
), GFP_KERNEL
);
3053 *iter
->trace
= *tr
->current_trace
;
3055 if (!zalloc_cpumask_var(&iter
->started
, GFP_KERNEL
))
3060 #ifdef CONFIG_TRACER_MAX_TRACE
3061 /* Currently only the top directory has a snapshot */
3062 if (tr
->current_trace
->print_max
|| snapshot
)
3063 iter
->trace_buffer
= &tr
->max_buffer
;
3066 iter
->trace_buffer
= &tr
->trace_buffer
;
3067 iter
->snapshot
= snapshot
;
3069 iter
->cpu_file
= tracing_get_cpu(inode
);
3070 mutex_init(&iter
->mutex
);
3072 /* Notify the tracer early; before we stop tracing. */
3073 if (iter
->trace
&& iter
->trace
->open
)
3074 iter
->trace
->open(iter
);
3076 /* Annotate start of buffers if we had overruns */
3077 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
3078 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
3080 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3081 if (trace_clocks
[tr
->clock_id
].in_ns
)
3082 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
3084 /* stop the trace while dumping if we are not opening "snapshot" */
3085 if (!iter
->snapshot
)
3086 tracing_stop_tr(tr
);
3088 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
) {
3089 for_each_tracing_cpu(cpu
) {
3090 iter
->buffer_iter
[cpu
] =
3091 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3093 ring_buffer_read_prepare_sync();
3094 for_each_tracing_cpu(cpu
) {
3095 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3096 tracing_iter_reset(iter
, cpu
);
3099 cpu
= iter
->cpu_file
;
3100 iter
->buffer_iter
[cpu
] =
3101 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3102 ring_buffer_read_prepare_sync();
3103 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3104 tracing_iter_reset(iter
, cpu
);
3107 mutex_unlock(&trace_types_lock
);
3112 mutex_unlock(&trace_types_lock
);
3114 kfree(iter
->buffer_iter
);
3116 seq_release_private(inode
, file
);
3117 return ERR_PTR(-ENOMEM
);
3120 int tracing_open_generic(struct inode
*inode
, struct file
*filp
)
3122 if (tracing_disabled
)
3125 filp
->private_data
= inode
->i_private
;
3129 bool tracing_is_disabled(void)
3131 return (tracing_disabled
) ? true: false;
3135 * Open and update trace_array ref count.
3136 * Must have the current trace_array passed to it.
3138 static int tracing_open_generic_tr(struct inode
*inode
, struct file
*filp
)
3140 struct trace_array
*tr
= inode
->i_private
;
3142 if (tracing_disabled
)
3145 if (trace_array_get(tr
) < 0)
3148 filp
->private_data
= inode
->i_private
;
3153 static int tracing_release(struct inode
*inode
, struct file
*file
)
3155 struct trace_array
*tr
= inode
->i_private
;
3156 struct seq_file
*m
= file
->private_data
;
3157 struct trace_iterator
*iter
;
3160 if (!(file
->f_mode
& FMODE_READ
)) {
3161 trace_array_put(tr
);
3165 /* Writes do not use seq_file */
3167 mutex_lock(&trace_types_lock
);
3169 for_each_tracing_cpu(cpu
) {
3170 if (iter
->buffer_iter
[cpu
])
3171 ring_buffer_read_finish(iter
->buffer_iter
[cpu
]);
3174 if (iter
->trace
&& iter
->trace
->close
)
3175 iter
->trace
->close(iter
);
3177 if (!iter
->snapshot
)
3178 /* reenable tracing if it was previously enabled */
3179 tracing_start_tr(tr
);
3181 __trace_array_put(tr
);
3183 mutex_unlock(&trace_types_lock
);
3185 mutex_destroy(&iter
->mutex
);
3186 free_cpumask_var(iter
->started
);
3188 kfree(iter
->buffer_iter
);
3189 seq_release_private(inode
, file
);
3194 static int tracing_release_generic_tr(struct inode
*inode
, struct file
*file
)
3196 struct trace_array
*tr
= inode
->i_private
;
3198 trace_array_put(tr
);
3202 static int tracing_single_release_tr(struct inode
*inode
, struct file
*file
)
3204 struct trace_array
*tr
= inode
->i_private
;
3206 trace_array_put(tr
);
3208 return single_release(inode
, file
);
3211 static int tracing_open(struct inode
*inode
, struct file
*file
)
3213 struct trace_array
*tr
= inode
->i_private
;
3214 struct trace_iterator
*iter
;
3217 if (trace_array_get(tr
) < 0)
3220 /* If this file was open for write, then erase contents */
3221 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
3222 int cpu
= tracing_get_cpu(inode
);
3224 if (cpu
== RING_BUFFER_ALL_CPUS
)
3225 tracing_reset_online_cpus(&tr
->trace_buffer
);
3227 tracing_reset(&tr
->trace_buffer
, cpu
);
3230 if (file
->f_mode
& FMODE_READ
) {
3231 iter
= __tracing_open(inode
, file
, false);
3233 ret
= PTR_ERR(iter
);
3234 else if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
3235 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
3239 trace_array_put(tr
);
3245 * Some tracers are not suitable for instance buffers.
3246 * A tracer is always available for the global array (toplevel)
3247 * or if it explicitly states that it is.
3250 trace_ok_for_array(struct tracer
*t
, struct trace_array
*tr
)
3252 return (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) || t
->allow_instances
;
3255 /* Find the next tracer that this trace array may use */
3256 static struct tracer
*
3257 get_tracer_for_array(struct trace_array
*tr
, struct tracer
*t
)
3259 while (t
&& !trace_ok_for_array(t
, tr
))
3266 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3268 struct trace_array
*tr
= m
->private;
3269 struct tracer
*t
= v
;
3274 t
= get_tracer_for_array(tr
, t
->next
);
3279 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
3281 struct trace_array
*tr
= m
->private;
3285 mutex_lock(&trace_types_lock
);
3287 t
= get_tracer_for_array(tr
, trace_types
);
3288 for (; t
&& l
< *pos
; t
= t_next(m
, t
, &l
))
3294 static void t_stop(struct seq_file
*m
, void *p
)
3296 mutex_unlock(&trace_types_lock
);
3299 static int t_show(struct seq_file
*m
, void *v
)
3301 struct tracer
*t
= v
;
3306 seq_puts(m
, t
->name
);
3315 static const struct seq_operations show_traces_seq_ops
= {
3322 static int show_traces_open(struct inode
*inode
, struct file
*file
)
3324 struct trace_array
*tr
= inode
->i_private
;
3328 if (tracing_disabled
)
3331 ret
= seq_open(file
, &show_traces_seq_ops
);
3335 m
= file
->private_data
;
3342 tracing_write_stub(struct file
*filp
, const char __user
*ubuf
,
3343 size_t count
, loff_t
*ppos
)
3348 loff_t
tracing_lseek(struct file
*file
, loff_t offset
, int whence
)
3352 if (file
->f_mode
& FMODE_READ
)
3353 ret
= seq_lseek(file
, offset
, whence
);
3355 file
->f_pos
= ret
= 0;
3360 static const struct file_operations tracing_fops
= {
3361 .open
= tracing_open
,
3363 .write
= tracing_write_stub
,
3364 .llseek
= tracing_lseek
,
3365 .release
= tracing_release
,
3368 static const struct file_operations show_traces_fops
= {
3369 .open
= show_traces_open
,
3371 .release
= seq_release
,
3372 .llseek
= seq_lseek
,
3376 * The tracer itself will not take this lock, but still we want
3377 * to provide a consistent cpumask to user-space:
3379 static DEFINE_MUTEX(tracing_cpumask_update_lock
);
3382 * Temporary storage for the character representation of the
3383 * CPU bitmask (and one more byte for the newline):
3385 static char mask_str
[NR_CPUS
+ 1];
3388 tracing_cpumask_read(struct file
*filp
, char __user
*ubuf
,
3389 size_t count
, loff_t
*ppos
)
3391 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3394 mutex_lock(&tracing_cpumask_update_lock
);
3396 len
= snprintf(mask_str
, count
, "%*pb\n",
3397 cpumask_pr_args(tr
->tracing_cpumask
));
3402 count
= simple_read_from_buffer(ubuf
, count
, ppos
, mask_str
, NR_CPUS
+1);
3405 mutex_unlock(&tracing_cpumask_update_lock
);
3411 tracing_cpumask_write(struct file
*filp
, const char __user
*ubuf
,
3412 size_t count
, loff_t
*ppos
)
3414 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3415 cpumask_var_t tracing_cpumask_new
;
3418 if (!alloc_cpumask_var(&tracing_cpumask_new
, GFP_KERNEL
))
3421 err
= cpumask_parse_user(ubuf
, count
, tracing_cpumask_new
);
3425 mutex_lock(&tracing_cpumask_update_lock
);
3427 local_irq_disable();
3428 arch_spin_lock(&tr
->max_lock
);
3429 for_each_tracing_cpu(cpu
) {
3431 * Increase/decrease the disabled counter if we are
3432 * about to flip a bit in the cpumask:
3434 if (cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
3435 !cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
3436 atomic_inc(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
3437 ring_buffer_record_disable_cpu(tr
->trace_buffer
.buffer
, cpu
);
3439 if (!cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
3440 cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
3441 atomic_dec(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
3442 ring_buffer_record_enable_cpu(tr
->trace_buffer
.buffer
, cpu
);
3445 arch_spin_unlock(&tr
->max_lock
);
3448 cpumask_copy(tr
->tracing_cpumask
, tracing_cpumask_new
);
3450 mutex_unlock(&tracing_cpumask_update_lock
);
3451 free_cpumask_var(tracing_cpumask_new
);
3456 free_cpumask_var(tracing_cpumask_new
);
3461 static const struct file_operations tracing_cpumask_fops
= {
3462 .open
= tracing_open_generic_tr
,
3463 .read
= tracing_cpumask_read
,
3464 .write
= tracing_cpumask_write
,
3465 .release
= tracing_release_generic_tr
,
3466 .llseek
= generic_file_llseek
,
3469 static int tracing_trace_options_show(struct seq_file
*m
, void *v
)
3471 struct tracer_opt
*trace_opts
;
3472 struct trace_array
*tr
= m
->private;
3476 mutex_lock(&trace_types_lock
);
3477 tracer_flags
= tr
->current_trace
->flags
->val
;
3478 trace_opts
= tr
->current_trace
->flags
->opts
;
3480 for (i
= 0; trace_options
[i
]; i
++) {
3481 if (tr
->trace_flags
& (1 << i
))
3482 seq_printf(m
, "%s\n", trace_options
[i
]);
3484 seq_printf(m
, "no%s\n", trace_options
[i
]);
3487 for (i
= 0; trace_opts
[i
].name
; i
++) {
3488 if (tracer_flags
& trace_opts
[i
].bit
)
3489 seq_printf(m
, "%s\n", trace_opts
[i
].name
);
3491 seq_printf(m
, "no%s\n", trace_opts
[i
].name
);
3493 mutex_unlock(&trace_types_lock
);
3498 static int __set_tracer_option(struct trace_array
*tr
,
3499 struct tracer_flags
*tracer_flags
,
3500 struct tracer_opt
*opts
, int neg
)
3502 struct tracer
*trace
= tr
->current_trace
;
3505 ret
= trace
->set_flag(tr
, tracer_flags
->val
, opts
->bit
, !neg
);
3510 tracer_flags
->val
&= ~opts
->bit
;
3512 tracer_flags
->val
|= opts
->bit
;
3516 /* Try to assign a tracer specific option */
3517 static int set_tracer_option(struct trace_array
*tr
, char *cmp
, int neg
)
3519 struct tracer
*trace
= tr
->current_trace
;
3520 struct tracer_flags
*tracer_flags
= trace
->flags
;
3521 struct tracer_opt
*opts
= NULL
;
3524 for (i
= 0; tracer_flags
->opts
[i
].name
; i
++) {
3525 opts
= &tracer_flags
->opts
[i
];
3527 if (strcmp(cmp
, opts
->name
) == 0)
3528 return __set_tracer_option(tr
, trace
->flags
, opts
, neg
);
3534 /* Some tracers require overwrite to stay enabled */
3535 int trace_keep_overwrite(struct tracer
*tracer
, u32 mask
, int set
)
3537 if (tracer
->enabled
&& (mask
& TRACE_ITER_OVERWRITE
) && !set
)
3543 int set_tracer_flag(struct trace_array
*tr
, unsigned int mask
, int enabled
)
3545 /* do nothing if flag is already set */
3546 if (!!(tr
->trace_flags
& mask
) == !!enabled
)
3549 /* Give the tracer a chance to approve the change */
3550 if (tr
->current_trace
->flag_changed
)
3551 if (tr
->current_trace
->flag_changed(tr
, mask
, !!enabled
))
3555 tr
->trace_flags
|= mask
;
3557 tr
->trace_flags
&= ~mask
;
3559 if (mask
== TRACE_ITER_RECORD_CMD
)
3560 trace_event_enable_cmd_record(enabled
);
3562 if (mask
== TRACE_ITER_OVERWRITE
) {
3563 ring_buffer_change_overwrite(tr
->trace_buffer
.buffer
, enabled
);
3564 #ifdef CONFIG_TRACER_MAX_TRACE
3565 ring_buffer_change_overwrite(tr
->max_buffer
.buffer
, enabled
);
3569 if (mask
== TRACE_ITER_PRINTK
) {
3570 trace_printk_start_stop_comm(enabled
);
3571 trace_printk_control(enabled
);
3577 static int trace_set_options(struct trace_array
*tr
, char *option
)
3584 cmp
= strstrip(option
);
3586 if (strncmp(cmp
, "no", 2) == 0) {
3591 mutex_lock(&trace_types_lock
);
3593 for (i
= 0; trace_options
[i
]; i
++) {
3594 if (strcmp(cmp
, trace_options
[i
]) == 0) {
3595 ret
= set_tracer_flag(tr
, 1 << i
, !neg
);
3600 /* If no option could be set, test the specific tracer options */
3601 if (!trace_options
[i
])
3602 ret
= set_tracer_option(tr
, cmp
, neg
);
3604 mutex_unlock(&trace_types_lock
);
3610 tracing_trace_options_write(struct file
*filp
, const char __user
*ubuf
,
3611 size_t cnt
, loff_t
*ppos
)
3613 struct seq_file
*m
= filp
->private_data
;
3614 struct trace_array
*tr
= m
->private;
3618 if (cnt
>= sizeof(buf
))
3621 if (copy_from_user(&buf
, ubuf
, cnt
))
3626 ret
= trace_set_options(tr
, buf
);
3635 static int tracing_trace_options_open(struct inode
*inode
, struct file
*file
)
3637 struct trace_array
*tr
= inode
->i_private
;
3640 if (tracing_disabled
)
3643 if (trace_array_get(tr
) < 0)
3646 ret
= single_open(file
, tracing_trace_options_show
, inode
->i_private
);
3648 trace_array_put(tr
);
3653 static const struct file_operations tracing_iter_fops
= {
3654 .open
= tracing_trace_options_open
,
3656 .llseek
= seq_lseek
,
3657 .release
= tracing_single_release_tr
,
3658 .write
= tracing_trace_options_write
,
3661 static const char readme_msg
[] =
3662 "tracing mini-HOWTO:\n\n"
3663 "# echo 0 > tracing_on : quick way to disable tracing\n"
3664 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3665 " Important files:\n"
3666 " trace\t\t\t- The static contents of the buffer\n"
3667 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3668 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3669 " current_tracer\t- function and latency tracers\n"
3670 " available_tracers\t- list of configured tracers for current_tracer\n"
3671 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3672 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3673 " trace_clock\t\t-change the clock used to order events\n"
3674 " local: Per cpu clock but may not be synced across CPUs\n"
3675 " global: Synced across CPUs but slows tracing down.\n"
3676 " counter: Not a clock, but just an increment\n"
3677 " uptime: Jiffy counter from time of boot\n"
3678 " perf: Same clock that perf events use\n"
3679 #ifdef CONFIG_X86_64
3680 " x86-tsc: TSC cycle counter\n"
3682 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3683 " tracing_cpumask\t- Limit which CPUs to trace\n"
3684 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3685 "\t\t\t Remove sub-buffer with rmdir\n"
3686 " trace_options\t\t- Set format or modify how tracing happens\n"
3687 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3688 "\t\t\t option name\n"
3689 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3690 #ifdef CONFIG_DYNAMIC_FTRACE
3691 "\n available_filter_functions - list of functions that can be filtered on\n"
3692 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3693 "\t\t\t functions\n"
3694 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3695 "\t modules: Can select a group via module\n"
3696 "\t Format: :mod:<module-name>\n"
3697 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3698 "\t triggers: a command to perform when function is hit\n"
3699 "\t Format: <function>:<trigger>[:count]\n"
3700 "\t trigger: traceon, traceoff\n"
3701 "\t\t enable_event:<system>:<event>\n"
3702 "\t\t disable_event:<system>:<event>\n"
3703 #ifdef CONFIG_STACKTRACE
3706 #ifdef CONFIG_TRACER_SNAPSHOT
3711 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3712 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3713 "\t The first one will disable tracing every time do_fault is hit\n"
3714 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3715 "\t The first time do trap is hit and it disables tracing, the\n"
3716 "\t counter will decrement to 2. If tracing is already disabled,\n"
3717 "\t the counter will not decrement. It only decrements when the\n"
3718 "\t trigger did work\n"
3719 "\t To remove trigger without count:\n"
3720 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3721 "\t To remove trigger with a count:\n"
3722 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3723 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3724 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3725 "\t modules: Can select a group via module command :mod:\n"
3726 "\t Does not accept triggers\n"
3727 #endif /* CONFIG_DYNAMIC_FTRACE */
3728 #ifdef CONFIG_FUNCTION_TRACER
3729 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3732 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3733 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3734 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3735 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3737 #ifdef CONFIG_TRACER_SNAPSHOT
3738 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3739 "\t\t\t snapshot buffer. Read the contents for more\n"
3740 "\t\t\t information\n"
3742 #ifdef CONFIG_STACK_TRACER
3743 " stack_trace\t\t- Shows the max stack trace when active\n"
3744 " stack_max_size\t- Shows current max stack size that was traced\n"
3745 "\t\t\t Write into this file to reset the max size (trigger a\n"
3746 "\t\t\t new trace)\n"
3747 #ifdef CONFIG_DYNAMIC_FTRACE
3748 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3751 #endif /* CONFIG_STACK_TRACER */
3752 " events/\t\t- Directory containing all trace event subsystems:\n"
3753 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3754 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3755 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3757 " filter\t\t- If set, only events passing filter are traced\n"
3758 " events/<system>/<event>/\t- Directory containing control files for\n"
3760 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3761 " filter\t\t- If set, only events passing filter are traced\n"
3762 " trigger\t\t- If set, a command to perform when event is hit\n"
3763 "\t Format: <trigger>[:count][if <filter>]\n"
3764 "\t trigger: traceon, traceoff\n"
3765 "\t enable_event:<system>:<event>\n"
3766 "\t disable_event:<system>:<event>\n"
3767 #ifdef CONFIG_STACKTRACE
3770 #ifdef CONFIG_TRACER_SNAPSHOT
3773 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3774 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3775 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3776 "\t events/block/block_unplug/trigger\n"
3777 "\t The first disables tracing every time block_unplug is hit.\n"
3778 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3779 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3780 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3781 "\t Like function triggers, the counter is only decremented if it\n"
3782 "\t enabled or disabled tracing.\n"
3783 "\t To remove a trigger without a count:\n"
3784 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3785 "\t To remove a trigger with a count:\n"
3786 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3787 "\t Filters can be ignored when removing a trigger.\n"
3791 tracing_readme_read(struct file
*filp
, char __user
*ubuf
,
3792 size_t cnt
, loff_t
*ppos
)
3794 return simple_read_from_buffer(ubuf
, cnt
, ppos
,
3795 readme_msg
, strlen(readme_msg
));
3798 static const struct file_operations tracing_readme_fops
= {
3799 .open
= tracing_open_generic
,
3800 .read
= tracing_readme_read
,
3801 .llseek
= generic_file_llseek
,
3804 static void *saved_cmdlines_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3806 unsigned int *ptr
= v
;
3808 if (*pos
|| m
->count
)
3813 for (; ptr
< &savedcmd
->map_cmdline_to_pid
[savedcmd
->cmdline_num
];
3815 if (*ptr
== -1 || *ptr
== NO_CMDLINE_MAP
)
3824 static void *saved_cmdlines_start(struct seq_file
*m
, loff_t
*pos
)
3830 arch_spin_lock(&trace_cmdline_lock
);
3832 v
= &savedcmd
->map_cmdline_to_pid
[0];
3834 v
= saved_cmdlines_next(m
, v
, &l
);
3842 static void saved_cmdlines_stop(struct seq_file
*m
, void *v
)
3844 arch_spin_unlock(&trace_cmdline_lock
);
3848 static int saved_cmdlines_show(struct seq_file
*m
, void *v
)
3850 char buf
[TASK_COMM_LEN
];
3851 unsigned int *pid
= v
;
3853 __trace_find_cmdline(*pid
, buf
);
3854 seq_printf(m
, "%d %s\n", *pid
, buf
);
3858 static const struct seq_operations tracing_saved_cmdlines_seq_ops
= {
3859 .start
= saved_cmdlines_start
,
3860 .next
= saved_cmdlines_next
,
3861 .stop
= saved_cmdlines_stop
,
3862 .show
= saved_cmdlines_show
,
3865 static int tracing_saved_cmdlines_open(struct inode
*inode
, struct file
*filp
)
3867 if (tracing_disabled
)
3870 return seq_open(filp
, &tracing_saved_cmdlines_seq_ops
);
3873 static const struct file_operations tracing_saved_cmdlines_fops
= {
3874 .open
= tracing_saved_cmdlines_open
,
3876 .llseek
= seq_lseek
,
3877 .release
= seq_release
,
3881 tracing_saved_cmdlines_size_read(struct file
*filp
, char __user
*ubuf
,
3882 size_t cnt
, loff_t
*ppos
)
3887 arch_spin_lock(&trace_cmdline_lock
);
3888 r
= scnprintf(buf
, sizeof(buf
), "%u\n", savedcmd
->cmdline_num
);
3889 arch_spin_unlock(&trace_cmdline_lock
);
3891 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
3894 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer
*s
)
3896 kfree(s
->saved_cmdlines
);
3897 kfree(s
->map_cmdline_to_pid
);
3901 static int tracing_resize_saved_cmdlines(unsigned int val
)
3903 struct saved_cmdlines_buffer
*s
, *savedcmd_temp
;
3905 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
3909 if (allocate_cmdlines_buffer(val
, s
) < 0) {
3914 arch_spin_lock(&trace_cmdline_lock
);
3915 savedcmd_temp
= savedcmd
;
3917 arch_spin_unlock(&trace_cmdline_lock
);
3918 free_saved_cmdlines_buffer(savedcmd_temp
);
3924 tracing_saved_cmdlines_size_write(struct file
*filp
, const char __user
*ubuf
,
3925 size_t cnt
, loff_t
*ppos
)
3930 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
3934 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3935 if (!val
|| val
> PID_MAX_DEFAULT
)
3938 ret
= tracing_resize_saved_cmdlines((unsigned int)val
);
3947 static const struct file_operations tracing_saved_cmdlines_size_fops
= {
3948 .open
= tracing_open_generic
,
3949 .read
= tracing_saved_cmdlines_size_read
,
3950 .write
= tracing_saved_cmdlines_size_write
,
3953 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
3954 static union trace_enum_map_item
*
3955 update_enum_map(union trace_enum_map_item
*ptr
)
3957 if (!ptr
->map
.enum_string
) {
3958 if (ptr
->tail
.next
) {
3959 ptr
= ptr
->tail
.next
;
3960 /* Set ptr to the next real item (skip head) */
3968 static void *enum_map_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3970 union trace_enum_map_item
*ptr
= v
;
3973 * Paranoid! If ptr points to end, we don't want to increment past it.
3974 * This really should never happen.
3976 ptr
= update_enum_map(ptr
);
3977 if (WARN_ON_ONCE(!ptr
))
3984 ptr
= update_enum_map(ptr
);
3989 static void *enum_map_start(struct seq_file
*m
, loff_t
*pos
)
3991 union trace_enum_map_item
*v
;
3994 mutex_lock(&trace_enum_mutex
);
3996 v
= trace_enum_maps
;
4000 while (v
&& l
< *pos
) {
4001 v
= enum_map_next(m
, v
, &l
);
4007 static void enum_map_stop(struct seq_file
*m
, void *v
)
4009 mutex_unlock(&trace_enum_mutex
);
4012 static int enum_map_show(struct seq_file
*m
, void *v
)
4014 union trace_enum_map_item
*ptr
= v
;
4016 seq_printf(m
, "%s %ld (%s)\n",
4017 ptr
->map
.enum_string
, ptr
->map
.enum_value
,
4023 static const struct seq_operations tracing_enum_map_seq_ops
= {
4024 .start
= enum_map_start
,
4025 .next
= enum_map_next
,
4026 .stop
= enum_map_stop
,
4027 .show
= enum_map_show
,
4030 static int tracing_enum_map_open(struct inode
*inode
, struct file
*filp
)
4032 if (tracing_disabled
)
4035 return seq_open(filp
, &tracing_enum_map_seq_ops
);
4038 static const struct file_operations tracing_enum_map_fops
= {
4039 .open
= tracing_enum_map_open
,
4041 .llseek
= seq_lseek
,
4042 .release
= seq_release
,
4045 static inline union trace_enum_map_item
*
4046 trace_enum_jmp_to_tail(union trace_enum_map_item
*ptr
)
4048 /* Return tail of array given the head */
4049 return ptr
+ ptr
->head
.length
+ 1;
4053 trace_insert_enum_map_file(struct module
*mod
, struct trace_enum_map
**start
,
4056 struct trace_enum_map
**stop
;
4057 struct trace_enum_map
**map
;
4058 union trace_enum_map_item
*map_array
;
4059 union trace_enum_map_item
*ptr
;
4064 * The trace_enum_maps contains the map plus a head and tail item,
4065 * where the head holds the module and length of array, and the
4066 * tail holds a pointer to the next list.
4068 map_array
= kmalloc(sizeof(*map_array
) * (len
+ 2), GFP_KERNEL
);
4070 pr_warning("Unable to allocate trace enum mapping\n");
4074 mutex_lock(&trace_enum_mutex
);
4076 if (!trace_enum_maps
)
4077 trace_enum_maps
= map_array
;
4079 ptr
= trace_enum_maps
;
4081 ptr
= trace_enum_jmp_to_tail(ptr
);
4082 if (!ptr
->tail
.next
)
4084 ptr
= ptr
->tail
.next
;
4087 ptr
->tail
.next
= map_array
;
4089 map_array
->head
.mod
= mod
;
4090 map_array
->head
.length
= len
;
4093 for (map
= start
; (unsigned long)map
< (unsigned long)stop
; map
++) {
4094 map_array
->map
= **map
;
4097 memset(map_array
, 0, sizeof(*map_array
));
4099 mutex_unlock(&trace_enum_mutex
);
4102 static void trace_create_enum_file(struct dentry
*d_tracer
)
4104 trace_create_file("enum_map", 0444, d_tracer
,
4105 NULL
, &tracing_enum_map_fops
);
4108 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4109 static inline void trace_create_enum_file(struct dentry
*d_tracer
) { }
4110 static inline void trace_insert_enum_map_file(struct module
*mod
,
4111 struct trace_enum_map
**start
, int len
) { }
4112 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4114 static void trace_insert_enum_map(struct module
*mod
,
4115 struct trace_enum_map
**start
, int len
)
4117 struct trace_enum_map
**map
;
4124 trace_event_enum_update(map
, len
);
4126 trace_insert_enum_map_file(mod
, start
, len
);
4130 tracing_set_trace_read(struct file
*filp
, char __user
*ubuf
,
4131 size_t cnt
, loff_t
*ppos
)
4133 struct trace_array
*tr
= filp
->private_data
;
4134 char buf
[MAX_TRACER_SIZE
+2];
4137 mutex_lock(&trace_types_lock
);
4138 r
= sprintf(buf
, "%s\n", tr
->current_trace
->name
);
4139 mutex_unlock(&trace_types_lock
);
4141 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4144 int tracer_init(struct tracer
*t
, struct trace_array
*tr
)
4146 tracing_reset_online_cpus(&tr
->trace_buffer
);
4150 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
)
4154 for_each_tracing_cpu(cpu
)
4155 per_cpu_ptr(buf
->data
, cpu
)->entries
= val
;
4158 #ifdef CONFIG_TRACER_MAX_TRACE
4159 /* resize @tr's buffer to the size of @size_tr's entries */
4160 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
4161 struct trace_buffer
*size_buf
, int cpu_id
)
4165 if (cpu_id
== RING_BUFFER_ALL_CPUS
) {
4166 for_each_tracing_cpu(cpu
) {
4167 ret
= ring_buffer_resize(trace_buf
->buffer
,
4168 per_cpu_ptr(size_buf
->data
, cpu
)->entries
, cpu
);
4171 per_cpu_ptr(trace_buf
->data
, cpu
)->entries
=
4172 per_cpu_ptr(size_buf
->data
, cpu
)->entries
;
4175 ret
= ring_buffer_resize(trace_buf
->buffer
,
4176 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
, cpu_id
);
4178 per_cpu_ptr(trace_buf
->data
, cpu_id
)->entries
=
4179 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
;
4184 #endif /* CONFIG_TRACER_MAX_TRACE */
4186 static int __tracing_resize_ring_buffer(struct trace_array
*tr
,
4187 unsigned long size
, int cpu
)
4192 * If kernel or user changes the size of the ring buffer
4193 * we use the size that was given, and we can forget about
4194 * expanding it later.
4196 ring_buffer_expanded
= true;
4198 /* May be called before buffers are initialized */
4199 if (!tr
->trace_buffer
.buffer
)
4202 ret
= ring_buffer_resize(tr
->trace_buffer
.buffer
, size
, cpu
);
4206 #ifdef CONFIG_TRACER_MAX_TRACE
4207 if (!(tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) ||
4208 !tr
->current_trace
->use_max_tr
)
4211 ret
= ring_buffer_resize(tr
->max_buffer
.buffer
, size
, cpu
);
4213 int r
= resize_buffer_duplicate_size(&tr
->trace_buffer
,
4214 &tr
->trace_buffer
, cpu
);
4217 * AARGH! We are left with different
4218 * size max buffer!!!!
4219 * The max buffer is our "snapshot" buffer.
4220 * When a tracer needs a snapshot (one of the
4221 * latency tracers), it swaps the max buffer
4222 * with the saved snap shot. We succeeded to
4223 * update the size of the main buffer, but failed to
4224 * update the size of the max buffer. But when we tried
4225 * to reset the main buffer to the original size, we
4226 * failed there too. This is very unlikely to
4227 * happen, but if it does, warn and kill all
4231 tracing_disabled
= 1;
4236 if (cpu
== RING_BUFFER_ALL_CPUS
)
4237 set_buffer_entries(&tr
->max_buffer
, size
);
4239 per_cpu_ptr(tr
->max_buffer
.data
, cpu
)->entries
= size
;
4242 #endif /* CONFIG_TRACER_MAX_TRACE */
4244 if (cpu
== RING_BUFFER_ALL_CPUS
)
4245 set_buffer_entries(&tr
->trace_buffer
, size
);
4247 per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
= size
;
4252 static ssize_t
tracing_resize_ring_buffer(struct trace_array
*tr
,
4253 unsigned long size
, int cpu_id
)
4257 mutex_lock(&trace_types_lock
);
4259 if (cpu_id
!= RING_BUFFER_ALL_CPUS
) {
4260 /* make sure, this cpu is enabled in the mask */
4261 if (!cpumask_test_cpu(cpu_id
, tracing_buffer_mask
)) {
4267 ret
= __tracing_resize_ring_buffer(tr
, size
, cpu_id
);
4272 mutex_unlock(&trace_types_lock
);
4279 * tracing_update_buffers - used by tracing facility to expand ring buffers
4281 * To save on memory when the tracing is never used on a system with it
4282 * configured in. The ring buffers are set to a minimum size. But once
4283 * a user starts to use the tracing facility, then they need to grow
4284 * to their default size.
4286 * This function is to be called when a tracer is about to be used.
4288 int tracing_update_buffers(void)
4292 mutex_lock(&trace_types_lock
);
4293 if (!ring_buffer_expanded
)
4294 ret
= __tracing_resize_ring_buffer(&global_trace
, trace_buf_size
,
4295 RING_BUFFER_ALL_CPUS
);
4296 mutex_unlock(&trace_types_lock
);
4301 struct trace_option_dentry
;
4303 static struct trace_option_dentry
*
4304 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
);
4307 * Used to clear out the tracer before deletion of an instance.
4308 * Must have trace_types_lock held.
4310 static void tracing_set_nop(struct trace_array
*tr
)
4312 if (tr
->current_trace
== &nop_trace
)
4315 tr
->current_trace
->enabled
--;
4317 if (tr
->current_trace
->reset
)
4318 tr
->current_trace
->reset(tr
);
4320 tr
->current_trace
= &nop_trace
;
4323 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
)
4325 /* Only enable if the directory has been created already. */
4329 /* Currently, only the top instance has options */
4330 if (!(tr
->flags
& TRACE_ARRAY_FL_GLOBAL
))
4333 /* Ignore if they were already created */
4337 t
->topts
= create_trace_option_files(tr
, t
);
4340 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
)
4343 #ifdef CONFIG_TRACER_MAX_TRACE
4348 mutex_lock(&trace_types_lock
);
4350 if (!ring_buffer_expanded
) {
4351 ret
= __tracing_resize_ring_buffer(tr
, trace_buf_size
,
4352 RING_BUFFER_ALL_CPUS
);
4358 for (t
= trace_types
; t
; t
= t
->next
) {
4359 if (strcmp(t
->name
, buf
) == 0)
4366 if (t
== tr
->current_trace
)
4369 /* Some tracers are only allowed for the top level buffer */
4370 if (!trace_ok_for_array(t
, tr
)) {
4375 /* If trace pipe files are being read, we can't change the tracer */
4376 if (tr
->current_trace
->ref
) {
4381 trace_branch_disable();
4383 tr
->current_trace
->enabled
--;
4385 if (tr
->current_trace
->reset
)
4386 tr
->current_trace
->reset(tr
);
4388 /* Current trace needs to be nop_trace before synchronize_sched */
4389 tr
->current_trace
= &nop_trace
;
4391 #ifdef CONFIG_TRACER_MAX_TRACE
4392 had_max_tr
= tr
->allocated_snapshot
;
4394 if (had_max_tr
&& !t
->use_max_tr
) {
4396 * We need to make sure that the update_max_tr sees that
4397 * current_trace changed to nop_trace to keep it from
4398 * swapping the buffers after we resize it.
4399 * The update_max_tr is called from interrupts disabled
4400 * so a synchronized_sched() is sufficient.
4402 synchronize_sched();
4407 #ifdef CONFIG_TRACER_MAX_TRACE
4408 if (t
->use_max_tr
&& !had_max_tr
) {
4409 ret
= alloc_snapshot(tr
);
4416 ret
= tracer_init(t
, tr
);
4421 tr
->current_trace
= t
;
4422 tr
->current_trace
->enabled
++;
4423 trace_branch_enable(tr
);
4425 mutex_unlock(&trace_types_lock
);
4431 tracing_set_trace_write(struct file
*filp
, const char __user
*ubuf
,
4432 size_t cnt
, loff_t
*ppos
)
4434 struct trace_array
*tr
= filp
->private_data
;
4435 char buf
[MAX_TRACER_SIZE
+1];
4442 if (cnt
> MAX_TRACER_SIZE
)
4443 cnt
= MAX_TRACER_SIZE
;
4445 if (copy_from_user(&buf
, ubuf
, cnt
))
4450 /* strip ending whitespace. */
4451 for (i
= cnt
- 1; i
> 0 && isspace(buf
[i
]); i
--)
4454 err
= tracing_set_tracer(tr
, buf
);
4464 tracing_nsecs_read(unsigned long *ptr
, char __user
*ubuf
,
4465 size_t cnt
, loff_t
*ppos
)
4470 r
= snprintf(buf
, sizeof(buf
), "%ld\n",
4471 *ptr
== (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr
));
4472 if (r
> sizeof(buf
))
4474 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4478 tracing_nsecs_write(unsigned long *ptr
, const char __user
*ubuf
,
4479 size_t cnt
, loff_t
*ppos
)
4484 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4494 tracing_thresh_read(struct file
*filp
, char __user
*ubuf
,
4495 size_t cnt
, loff_t
*ppos
)
4497 return tracing_nsecs_read(&tracing_thresh
, ubuf
, cnt
, ppos
);
4501 tracing_thresh_write(struct file
*filp
, const char __user
*ubuf
,
4502 size_t cnt
, loff_t
*ppos
)
4504 struct trace_array
*tr
= filp
->private_data
;
4507 mutex_lock(&trace_types_lock
);
4508 ret
= tracing_nsecs_write(&tracing_thresh
, ubuf
, cnt
, ppos
);
4512 if (tr
->current_trace
->update_thresh
) {
4513 ret
= tr
->current_trace
->update_thresh(tr
);
4520 mutex_unlock(&trace_types_lock
);
4526 tracing_max_lat_read(struct file
*filp
, char __user
*ubuf
,
4527 size_t cnt
, loff_t
*ppos
)
4529 return tracing_nsecs_read(filp
->private_data
, ubuf
, cnt
, ppos
);
4533 tracing_max_lat_write(struct file
*filp
, const char __user
*ubuf
,
4534 size_t cnt
, loff_t
*ppos
)
4536 return tracing_nsecs_write(filp
->private_data
, ubuf
, cnt
, ppos
);
4539 static int tracing_open_pipe(struct inode
*inode
, struct file
*filp
)
4541 struct trace_array
*tr
= inode
->i_private
;
4542 struct trace_iterator
*iter
;
4545 if (tracing_disabled
)
4548 if (trace_array_get(tr
) < 0)
4551 mutex_lock(&trace_types_lock
);
4553 /* create a buffer to store the information to pass to userspace */
4554 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
4557 __trace_array_put(tr
);
4561 trace_seq_init(&iter
->seq
);
4562 iter
->trace
= tr
->current_trace
;
4564 if (!alloc_cpumask_var(&iter
->started
, GFP_KERNEL
)) {
4569 /* trace pipe does not show start of buffer */
4570 cpumask_setall(iter
->started
);
4572 if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
4573 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
4575 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4576 if (trace_clocks
[tr
->clock_id
].in_ns
)
4577 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
4580 iter
->trace_buffer
= &tr
->trace_buffer
;
4581 iter
->cpu_file
= tracing_get_cpu(inode
);
4582 mutex_init(&iter
->mutex
);
4583 filp
->private_data
= iter
;
4585 if (iter
->trace
->pipe_open
)
4586 iter
->trace
->pipe_open(iter
);
4588 nonseekable_open(inode
, filp
);
4590 tr
->current_trace
->ref
++;
4592 mutex_unlock(&trace_types_lock
);
4598 __trace_array_put(tr
);
4599 mutex_unlock(&trace_types_lock
);
4603 static int tracing_release_pipe(struct inode
*inode
, struct file
*file
)
4605 struct trace_iterator
*iter
= file
->private_data
;
4606 struct trace_array
*tr
= inode
->i_private
;
4608 mutex_lock(&trace_types_lock
);
4610 tr
->current_trace
->ref
--;
4612 if (iter
->trace
->pipe_close
)
4613 iter
->trace
->pipe_close(iter
);
4615 mutex_unlock(&trace_types_lock
);
4617 free_cpumask_var(iter
->started
);
4618 mutex_destroy(&iter
->mutex
);
4621 trace_array_put(tr
);
4627 trace_poll(struct trace_iterator
*iter
, struct file
*filp
, poll_table
*poll_table
)
4629 struct trace_array
*tr
= iter
->tr
;
4631 /* Iterators are static, they should be filled or empty */
4632 if (trace_buffer_iter(iter
, iter
->cpu_file
))
4633 return POLLIN
| POLLRDNORM
;
4635 if (tr
->trace_flags
& TRACE_ITER_BLOCK
)
4637 * Always select as readable when in blocking mode
4639 return POLLIN
| POLLRDNORM
;
4641 return ring_buffer_poll_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
4646 tracing_poll_pipe(struct file
*filp
, poll_table
*poll_table
)
4648 struct trace_iterator
*iter
= filp
->private_data
;
4650 return trace_poll(iter
, filp
, poll_table
);
4653 /* Must be called with iter->mutex held. */
4654 static int tracing_wait_pipe(struct file
*filp
)
4656 struct trace_iterator
*iter
= filp
->private_data
;
4659 while (trace_empty(iter
)) {
4661 if ((filp
->f_flags
& O_NONBLOCK
)) {
4666 * We block until we read something and tracing is disabled.
4667 * We still block if tracing is disabled, but we have never
4668 * read anything. This allows a user to cat this file, and
4669 * then enable tracing. But after we have read something,
4670 * we give an EOF when tracing is again disabled.
4672 * iter->pos will be 0 if we haven't read anything.
4674 if (!tracing_is_on() && iter
->pos
)
4677 mutex_unlock(&iter
->mutex
);
4679 ret
= wait_on_pipe(iter
, false);
4681 mutex_lock(&iter
->mutex
);
4694 tracing_read_pipe(struct file
*filp
, char __user
*ubuf
,
4695 size_t cnt
, loff_t
*ppos
)
4697 struct trace_iterator
*iter
= filp
->private_data
;
4700 /* return any leftover data */
4701 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
4705 trace_seq_init(&iter
->seq
);
4708 * Avoid more than one consumer on a single file descriptor
4709 * This is just a matter of traces coherency, the ring buffer itself
4712 mutex_lock(&iter
->mutex
);
4713 if (iter
->trace
->read
) {
4714 sret
= iter
->trace
->read(iter
, filp
, ubuf
, cnt
, ppos
);
4720 sret
= tracing_wait_pipe(filp
);
4724 /* stop when tracing is finished */
4725 if (trace_empty(iter
)) {
4730 if (cnt
>= PAGE_SIZE
)
4731 cnt
= PAGE_SIZE
- 1;
4733 /* reset all but tr, trace, and overruns */
4734 memset(&iter
->seq
, 0,
4735 sizeof(struct trace_iterator
) -
4736 offsetof(struct trace_iterator
, seq
));
4737 cpumask_clear(iter
->started
);
4740 trace_event_read_lock();
4741 trace_access_lock(iter
->cpu_file
);
4742 while (trace_find_next_entry_inc(iter
) != NULL
) {
4743 enum print_line_t ret
;
4744 int save_len
= iter
->seq
.seq
.len
;
4746 ret
= print_trace_line(iter
);
4747 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
4748 /* don't print partial lines */
4749 iter
->seq
.seq
.len
= save_len
;
4752 if (ret
!= TRACE_TYPE_NO_CONSUME
)
4753 trace_consume(iter
);
4755 if (trace_seq_used(&iter
->seq
) >= cnt
)
4759 * Setting the full flag means we reached the trace_seq buffer
4760 * size and we should leave by partial output condition above.
4761 * One of the trace_seq_* functions is not used properly.
4763 WARN_ONCE(iter
->seq
.full
, "full flag set for trace type %d",
4766 trace_access_unlock(iter
->cpu_file
);
4767 trace_event_read_unlock();
4769 /* Now copy what we have to the user */
4770 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
4771 if (iter
->seq
.seq
.readpos
>= trace_seq_used(&iter
->seq
))
4772 trace_seq_init(&iter
->seq
);
4775 * If there was nothing to send to user, in spite of consuming trace
4776 * entries, go back to wait for more entries.
4782 mutex_unlock(&iter
->mutex
);
4787 static void tracing_spd_release_pipe(struct splice_pipe_desc
*spd
,
4790 __free_page(spd
->pages
[idx
]);
4793 static const struct pipe_buf_operations tracing_pipe_buf_ops
= {
4795 .confirm
= generic_pipe_buf_confirm
,
4796 .release
= generic_pipe_buf_release
,
4797 .steal
= generic_pipe_buf_steal
,
4798 .get
= generic_pipe_buf_get
,
4802 tracing_fill_pipe_page(size_t rem
, struct trace_iterator
*iter
)
4808 /* Seq buffer is page-sized, exactly what we need. */
4810 save_len
= iter
->seq
.seq
.len
;
4811 ret
= print_trace_line(iter
);
4813 if (trace_seq_has_overflowed(&iter
->seq
)) {
4814 iter
->seq
.seq
.len
= save_len
;
4819 * This should not be hit, because it should only
4820 * be set if the iter->seq overflowed. But check it
4821 * anyway to be safe.
4823 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
4824 iter
->seq
.seq
.len
= save_len
;
4828 count
= trace_seq_used(&iter
->seq
) - save_len
;
4831 iter
->seq
.seq
.len
= save_len
;
4835 if (ret
!= TRACE_TYPE_NO_CONSUME
)
4836 trace_consume(iter
);
4838 if (!trace_find_next_entry_inc(iter
)) {
4848 static ssize_t
tracing_splice_read_pipe(struct file
*filp
,
4850 struct pipe_inode_info
*pipe
,
4854 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
4855 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
4856 struct trace_iterator
*iter
= filp
->private_data
;
4857 struct splice_pipe_desc spd
= {
4859 .partial
= partial_def
,
4860 .nr_pages
= 0, /* This gets updated below. */
4861 .nr_pages_max
= PIPE_DEF_BUFFERS
,
4863 .ops
= &tracing_pipe_buf_ops
,
4864 .spd_release
= tracing_spd_release_pipe
,
4870 if (splice_grow_spd(pipe
, &spd
))
4873 mutex_lock(&iter
->mutex
);
4875 if (iter
->trace
->splice_read
) {
4876 ret
= iter
->trace
->splice_read(iter
, filp
,
4877 ppos
, pipe
, len
, flags
);
4882 ret
= tracing_wait_pipe(filp
);
4886 if (!iter
->ent
&& !trace_find_next_entry_inc(iter
)) {
4891 trace_event_read_lock();
4892 trace_access_lock(iter
->cpu_file
);
4894 /* Fill as many pages as possible. */
4895 for (i
= 0, rem
= len
; i
< spd
.nr_pages_max
&& rem
; i
++) {
4896 spd
.pages
[i
] = alloc_page(GFP_KERNEL
);
4900 rem
= tracing_fill_pipe_page(rem
, iter
);
4902 /* Copy the data into the page, so we can start over. */
4903 ret
= trace_seq_to_buffer(&iter
->seq
,
4904 page_address(spd
.pages
[i
]),
4905 trace_seq_used(&iter
->seq
));
4907 __free_page(spd
.pages
[i
]);
4910 spd
.partial
[i
].offset
= 0;
4911 spd
.partial
[i
].len
= trace_seq_used(&iter
->seq
);
4913 trace_seq_init(&iter
->seq
);
4916 trace_access_unlock(iter
->cpu_file
);
4917 trace_event_read_unlock();
4918 mutex_unlock(&iter
->mutex
);
4922 ret
= splice_to_pipe(pipe
, &spd
);
4924 splice_shrink_spd(&spd
);
4928 mutex_unlock(&iter
->mutex
);
4933 tracing_entries_read(struct file
*filp
, char __user
*ubuf
,
4934 size_t cnt
, loff_t
*ppos
)
4936 struct inode
*inode
= file_inode(filp
);
4937 struct trace_array
*tr
= inode
->i_private
;
4938 int cpu
= tracing_get_cpu(inode
);
4943 mutex_lock(&trace_types_lock
);
4945 if (cpu
== RING_BUFFER_ALL_CPUS
) {
4946 int cpu
, buf_size_same
;
4951 /* check if all cpu sizes are same */
4952 for_each_tracing_cpu(cpu
) {
4953 /* fill in the size from first enabled cpu */
4955 size
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
;
4956 if (size
!= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
) {
4962 if (buf_size_same
) {
4963 if (!ring_buffer_expanded
)
4964 r
= sprintf(buf
, "%lu (expanded: %lu)\n",
4966 trace_buf_size
>> 10);
4968 r
= sprintf(buf
, "%lu\n", size
>> 10);
4970 r
= sprintf(buf
, "X\n");
4972 r
= sprintf(buf
, "%lu\n", per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10);
4974 mutex_unlock(&trace_types_lock
);
4976 ret
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4981 tracing_entries_write(struct file
*filp
, const char __user
*ubuf
,
4982 size_t cnt
, loff_t
*ppos
)
4984 struct inode
*inode
= file_inode(filp
);
4985 struct trace_array
*tr
= inode
->i_private
;
4989 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4993 /* must have at least 1 entry */
4997 /* value is in KB */
4999 ret
= tracing_resize_ring_buffer(tr
, val
, tracing_get_cpu(inode
));
5009 tracing_total_entries_read(struct file
*filp
, char __user
*ubuf
,
5010 size_t cnt
, loff_t
*ppos
)
5012 struct trace_array
*tr
= filp
->private_data
;
5015 unsigned long size
= 0, expanded_size
= 0;
5017 mutex_lock(&trace_types_lock
);
5018 for_each_tracing_cpu(cpu
) {
5019 size
+= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10;
5020 if (!ring_buffer_expanded
)
5021 expanded_size
+= trace_buf_size
>> 10;
5023 if (ring_buffer_expanded
)
5024 r
= sprintf(buf
, "%lu\n", size
);
5026 r
= sprintf(buf
, "%lu (expanded: %lu)\n", size
, expanded_size
);
5027 mutex_unlock(&trace_types_lock
);
5029 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5033 tracing_free_buffer_write(struct file
*filp
, const char __user
*ubuf
,
5034 size_t cnt
, loff_t
*ppos
)
5037 * There is no need to read what the user has written, this function
5038 * is just to make sure that there is no error when "echo" is used
5047 tracing_free_buffer_release(struct inode
*inode
, struct file
*filp
)
5049 struct trace_array
*tr
= inode
->i_private
;
5051 /* disable tracing ? */
5052 if (tr
->trace_flags
& TRACE_ITER_STOP_ON_FREE
)
5053 tracer_tracing_off(tr
);
5054 /* resize the ring buffer to 0 */
5055 tracing_resize_ring_buffer(tr
, 0, RING_BUFFER_ALL_CPUS
);
5057 trace_array_put(tr
);
5063 tracing_mark_write(struct file
*filp
, const char __user
*ubuf
,
5064 size_t cnt
, loff_t
*fpos
)
5066 unsigned long addr
= (unsigned long)ubuf
;
5067 struct trace_array
*tr
= filp
->private_data
;
5068 struct ring_buffer_event
*event
;
5069 struct ring_buffer
*buffer
;
5070 struct print_entry
*entry
;
5071 unsigned long irq_flags
;
5072 struct page
*pages
[2];
5082 if (tracing_disabled
)
5085 if (!(tr
->trace_flags
& TRACE_ITER_MARKERS
))
5088 if (cnt
> TRACE_BUF_SIZE
)
5089 cnt
= TRACE_BUF_SIZE
;
5092 * Userspace is injecting traces into the kernel trace buffer.
5093 * We want to be as non intrusive as possible.
5094 * To do so, we do not want to allocate any special buffers
5095 * or take any locks, but instead write the userspace data
5096 * straight into the ring buffer.
5098 * First we need to pin the userspace buffer into memory,
5099 * which, most likely it is, because it just referenced it.
5100 * But there's no guarantee that it is. By using get_user_pages_fast()
5101 * and kmap_atomic/kunmap_atomic() we can get access to the
5102 * pages directly. We then write the data directly into the
5105 BUILD_BUG_ON(TRACE_BUF_SIZE
>= PAGE_SIZE
);
5107 /* check if we cross pages */
5108 if ((addr
& PAGE_MASK
) != ((addr
+ cnt
) & PAGE_MASK
))
5111 offset
= addr
& (PAGE_SIZE
- 1);
5114 ret
= get_user_pages_fast(addr
, nr_pages
, 0, pages
);
5115 if (ret
< nr_pages
) {
5117 put_page(pages
[ret
]);
5122 for (i
= 0; i
< nr_pages
; i
++)
5123 map_page
[i
] = kmap_atomic(pages
[i
]);
5125 local_save_flags(irq_flags
);
5126 size
= sizeof(*entry
) + cnt
+ 2; /* possible \n added */
5127 buffer
= tr
->trace_buffer
.buffer
;
5128 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
5129 irq_flags
, preempt_count());
5131 /* Ring buffer disabled, return as if not open for write */
5136 entry
= ring_buffer_event_data(event
);
5137 entry
->ip
= _THIS_IP_
;
5139 if (nr_pages
== 2) {
5140 len
= PAGE_SIZE
- offset
;
5141 memcpy(&entry
->buf
, map_page
[0] + offset
, len
);
5142 memcpy(&entry
->buf
[len
], map_page
[1], cnt
- len
);
5144 memcpy(&entry
->buf
, map_page
[0] + offset
, cnt
);
5146 if (entry
->buf
[cnt
- 1] != '\n') {
5147 entry
->buf
[cnt
] = '\n';
5148 entry
->buf
[cnt
+ 1] = '\0';
5150 entry
->buf
[cnt
] = '\0';
5152 __buffer_unlock_commit(buffer
, event
);
5159 for (i
= nr_pages
- 1; i
>= 0; i
--) {
5160 kunmap_atomic(map_page
[i
]);
5167 static int tracing_clock_show(struct seq_file
*m
, void *v
)
5169 struct trace_array
*tr
= m
->private;
5172 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++)
5174 "%s%s%s%s", i
? " " : "",
5175 i
== tr
->clock_id
? "[" : "", trace_clocks
[i
].name
,
5176 i
== tr
->clock_id
? "]" : "");
5182 static int tracing_set_clock(struct trace_array
*tr
, const char *clockstr
)
5186 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++) {
5187 if (strcmp(trace_clocks
[i
].name
, clockstr
) == 0)
5190 if (i
== ARRAY_SIZE(trace_clocks
))
5193 mutex_lock(&trace_types_lock
);
5197 ring_buffer_set_clock(tr
->trace_buffer
.buffer
, trace_clocks
[i
].func
);
5200 * New clock may not be consistent with the previous clock.
5201 * Reset the buffer so that it doesn't have incomparable timestamps.
5203 tracing_reset_online_cpus(&tr
->trace_buffer
);
5205 #ifdef CONFIG_TRACER_MAX_TRACE
5206 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
&& tr
->max_buffer
.buffer
)
5207 ring_buffer_set_clock(tr
->max_buffer
.buffer
, trace_clocks
[i
].func
);
5208 tracing_reset_online_cpus(&tr
->max_buffer
);
5211 mutex_unlock(&trace_types_lock
);
5216 static ssize_t
tracing_clock_write(struct file
*filp
, const char __user
*ubuf
,
5217 size_t cnt
, loff_t
*fpos
)
5219 struct seq_file
*m
= filp
->private_data
;
5220 struct trace_array
*tr
= m
->private;
5222 const char *clockstr
;
5225 if (cnt
>= sizeof(buf
))
5228 if (copy_from_user(&buf
, ubuf
, cnt
))
5233 clockstr
= strstrip(buf
);
5235 ret
= tracing_set_clock(tr
, clockstr
);
5244 static int tracing_clock_open(struct inode
*inode
, struct file
*file
)
5246 struct trace_array
*tr
= inode
->i_private
;
5249 if (tracing_disabled
)
5252 if (trace_array_get(tr
))
5255 ret
= single_open(file
, tracing_clock_show
, inode
->i_private
);
5257 trace_array_put(tr
);
5262 struct ftrace_buffer_info
{
5263 struct trace_iterator iter
;
5268 #ifdef CONFIG_TRACER_SNAPSHOT
5269 static int tracing_snapshot_open(struct inode
*inode
, struct file
*file
)
5271 struct trace_array
*tr
= inode
->i_private
;
5272 struct trace_iterator
*iter
;
5276 if (trace_array_get(tr
) < 0)
5279 if (file
->f_mode
& FMODE_READ
) {
5280 iter
= __tracing_open(inode
, file
, true);
5282 ret
= PTR_ERR(iter
);
5284 /* Writes still need the seq_file to hold the private data */
5286 m
= kzalloc(sizeof(*m
), GFP_KERNEL
);
5289 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
5297 iter
->trace_buffer
= &tr
->max_buffer
;
5298 iter
->cpu_file
= tracing_get_cpu(inode
);
5300 file
->private_data
= m
;
5304 trace_array_put(tr
);
5310 tracing_snapshot_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
5313 struct seq_file
*m
= filp
->private_data
;
5314 struct trace_iterator
*iter
= m
->private;
5315 struct trace_array
*tr
= iter
->tr
;
5319 ret
= tracing_update_buffers();
5323 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5327 mutex_lock(&trace_types_lock
);
5329 if (tr
->current_trace
->use_max_tr
) {
5336 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
5340 if (tr
->allocated_snapshot
)
5344 /* Only allow per-cpu swap if the ring buffer supports it */
5345 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5346 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
5351 if (!tr
->allocated_snapshot
) {
5352 ret
= alloc_snapshot(tr
);
5356 local_irq_disable();
5357 /* Now, we're going to swap */
5358 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
5359 update_max_tr(tr
, current
, smp_processor_id());
5361 update_max_tr_single(tr
, current
, iter
->cpu_file
);
5365 if (tr
->allocated_snapshot
) {
5366 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
5367 tracing_reset_online_cpus(&tr
->max_buffer
);
5369 tracing_reset(&tr
->max_buffer
, iter
->cpu_file
);
5379 mutex_unlock(&trace_types_lock
);
5383 static int tracing_snapshot_release(struct inode
*inode
, struct file
*file
)
5385 struct seq_file
*m
= file
->private_data
;
5388 ret
= tracing_release(inode
, file
);
5390 if (file
->f_mode
& FMODE_READ
)
5393 /* If write only, the seq_file is just a stub */
5401 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
);
5402 static ssize_t
tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
5403 size_t count
, loff_t
*ppos
);
5404 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
);
5405 static ssize_t
tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
5406 struct pipe_inode_info
*pipe
, size_t len
, unsigned int flags
);
5408 static int snapshot_raw_open(struct inode
*inode
, struct file
*filp
)
5410 struct ftrace_buffer_info
*info
;
5413 ret
= tracing_buffers_open(inode
, filp
);
5417 info
= filp
->private_data
;
5419 if (info
->iter
.trace
->use_max_tr
) {
5420 tracing_buffers_release(inode
, filp
);
5424 info
->iter
.snapshot
= true;
5425 info
->iter
.trace_buffer
= &info
->iter
.tr
->max_buffer
;
5430 #endif /* CONFIG_TRACER_SNAPSHOT */
5433 static const struct file_operations tracing_thresh_fops
= {
5434 .open
= tracing_open_generic
,
5435 .read
= tracing_thresh_read
,
5436 .write
= tracing_thresh_write
,
5437 .llseek
= generic_file_llseek
,
5440 static const struct file_operations tracing_max_lat_fops
= {
5441 .open
= tracing_open_generic
,
5442 .read
= tracing_max_lat_read
,
5443 .write
= tracing_max_lat_write
,
5444 .llseek
= generic_file_llseek
,
5447 static const struct file_operations set_tracer_fops
= {
5448 .open
= tracing_open_generic
,
5449 .read
= tracing_set_trace_read
,
5450 .write
= tracing_set_trace_write
,
5451 .llseek
= generic_file_llseek
,
5454 static const struct file_operations tracing_pipe_fops
= {
5455 .open
= tracing_open_pipe
,
5456 .poll
= tracing_poll_pipe
,
5457 .read
= tracing_read_pipe
,
5458 .splice_read
= tracing_splice_read_pipe
,
5459 .release
= tracing_release_pipe
,
5460 .llseek
= no_llseek
,
5463 static const struct file_operations tracing_entries_fops
= {
5464 .open
= tracing_open_generic_tr
,
5465 .read
= tracing_entries_read
,
5466 .write
= tracing_entries_write
,
5467 .llseek
= generic_file_llseek
,
5468 .release
= tracing_release_generic_tr
,
5471 static const struct file_operations tracing_total_entries_fops
= {
5472 .open
= tracing_open_generic_tr
,
5473 .read
= tracing_total_entries_read
,
5474 .llseek
= generic_file_llseek
,
5475 .release
= tracing_release_generic_tr
,
5478 static const struct file_operations tracing_free_buffer_fops
= {
5479 .open
= tracing_open_generic_tr
,
5480 .write
= tracing_free_buffer_write
,
5481 .release
= tracing_free_buffer_release
,
5484 static const struct file_operations tracing_mark_fops
= {
5485 .open
= tracing_open_generic_tr
,
5486 .write
= tracing_mark_write
,
5487 .llseek
= generic_file_llseek
,
5488 .release
= tracing_release_generic_tr
,
5491 static const struct file_operations trace_clock_fops
= {
5492 .open
= tracing_clock_open
,
5494 .llseek
= seq_lseek
,
5495 .release
= tracing_single_release_tr
,
5496 .write
= tracing_clock_write
,
5499 #ifdef CONFIG_TRACER_SNAPSHOT
5500 static const struct file_operations snapshot_fops
= {
5501 .open
= tracing_snapshot_open
,
5503 .write
= tracing_snapshot_write
,
5504 .llseek
= tracing_lseek
,
5505 .release
= tracing_snapshot_release
,
5508 static const struct file_operations snapshot_raw_fops
= {
5509 .open
= snapshot_raw_open
,
5510 .read
= tracing_buffers_read
,
5511 .release
= tracing_buffers_release
,
5512 .splice_read
= tracing_buffers_splice_read
,
5513 .llseek
= no_llseek
,
5516 #endif /* CONFIG_TRACER_SNAPSHOT */
5518 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
)
5520 struct trace_array
*tr
= inode
->i_private
;
5521 struct ftrace_buffer_info
*info
;
5524 if (tracing_disabled
)
5527 if (trace_array_get(tr
) < 0)
5530 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
5532 trace_array_put(tr
);
5536 mutex_lock(&trace_types_lock
);
5539 info
->iter
.cpu_file
= tracing_get_cpu(inode
);
5540 info
->iter
.trace
= tr
->current_trace
;
5541 info
->iter
.trace_buffer
= &tr
->trace_buffer
;
5543 /* Force reading ring buffer for first read */
5544 info
->read
= (unsigned int)-1;
5546 filp
->private_data
= info
;
5548 tr
->current_trace
->ref
++;
5550 mutex_unlock(&trace_types_lock
);
5552 ret
= nonseekable_open(inode
, filp
);
5554 trace_array_put(tr
);
5560 tracing_buffers_poll(struct file
*filp
, poll_table
*poll_table
)
5562 struct ftrace_buffer_info
*info
= filp
->private_data
;
5563 struct trace_iterator
*iter
= &info
->iter
;
5565 return trace_poll(iter
, filp
, poll_table
);
5569 tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
5570 size_t count
, loff_t
*ppos
)
5572 struct ftrace_buffer_info
*info
= filp
->private_data
;
5573 struct trace_iterator
*iter
= &info
->iter
;
5580 #ifdef CONFIG_TRACER_MAX_TRACE
5581 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
5586 info
->spare
= ring_buffer_alloc_read_page(iter
->trace_buffer
->buffer
,
5591 /* Do we have previous read data to read? */
5592 if (info
->read
< PAGE_SIZE
)
5596 trace_access_lock(iter
->cpu_file
);
5597 ret
= ring_buffer_read_page(iter
->trace_buffer
->buffer
,
5601 trace_access_unlock(iter
->cpu_file
);
5604 if (trace_empty(iter
)) {
5605 if ((filp
->f_flags
& O_NONBLOCK
))
5608 ret
= wait_on_pipe(iter
, false);
5619 size
= PAGE_SIZE
- info
->read
;
5623 ret
= copy_to_user(ubuf
, info
->spare
+ info
->read
, size
);
5635 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
)
5637 struct ftrace_buffer_info
*info
= file
->private_data
;
5638 struct trace_iterator
*iter
= &info
->iter
;
5640 mutex_lock(&trace_types_lock
);
5642 iter
->tr
->current_trace
->ref
--;
5644 __trace_array_put(iter
->tr
);
5647 ring_buffer_free_read_page(iter
->trace_buffer
->buffer
, info
->spare
);
5650 mutex_unlock(&trace_types_lock
);
5656 struct ring_buffer
*buffer
;
5661 static void buffer_pipe_buf_release(struct pipe_inode_info
*pipe
,
5662 struct pipe_buffer
*buf
)
5664 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
5669 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5674 static void buffer_pipe_buf_get(struct pipe_inode_info
*pipe
,
5675 struct pipe_buffer
*buf
)
5677 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
5682 /* Pipe buffer operations for a buffer. */
5683 static const struct pipe_buf_operations buffer_pipe_buf_ops
= {
5685 .confirm
= generic_pipe_buf_confirm
,
5686 .release
= buffer_pipe_buf_release
,
5687 .steal
= generic_pipe_buf_steal
,
5688 .get
= buffer_pipe_buf_get
,
5692 * Callback from splice_to_pipe(), if we need to release some pages
5693 * at the end of the spd in case we error'ed out in filling the pipe.
5695 static void buffer_spd_release(struct splice_pipe_desc
*spd
, unsigned int i
)
5697 struct buffer_ref
*ref
=
5698 (struct buffer_ref
*)spd
->partial
[i
].private;
5703 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5705 spd
->partial
[i
].private = 0;
5709 tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
5710 struct pipe_inode_info
*pipe
, size_t len
,
5713 struct ftrace_buffer_info
*info
= file
->private_data
;
5714 struct trace_iterator
*iter
= &info
->iter
;
5715 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
5716 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
5717 struct splice_pipe_desc spd
= {
5719 .partial
= partial_def
,
5720 .nr_pages_max
= PIPE_DEF_BUFFERS
,
5722 .ops
= &buffer_pipe_buf_ops
,
5723 .spd_release
= buffer_spd_release
,
5725 struct buffer_ref
*ref
;
5726 int entries
, size
, i
;
5729 #ifdef CONFIG_TRACER_MAX_TRACE
5730 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
5734 if (splice_grow_spd(pipe
, &spd
))
5737 if (*ppos
& (PAGE_SIZE
- 1))
5740 if (len
& (PAGE_SIZE
- 1)) {
5741 if (len
< PAGE_SIZE
)
5747 trace_access_lock(iter
->cpu_file
);
5748 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
5750 for (i
= 0; i
< spd
.nr_pages_max
&& len
&& entries
; i
++, len
-= PAGE_SIZE
) {
5754 ref
= kzalloc(sizeof(*ref
), GFP_KERNEL
);
5761 ref
->buffer
= iter
->trace_buffer
->buffer
;
5762 ref
->page
= ring_buffer_alloc_read_page(ref
->buffer
, iter
->cpu_file
);
5769 r
= ring_buffer_read_page(ref
->buffer
, &ref
->page
,
5770 len
, iter
->cpu_file
, 1);
5772 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5778 * zero out any left over data, this is going to
5781 size
= ring_buffer_page_len(ref
->page
);
5782 if (size
< PAGE_SIZE
)
5783 memset(ref
->page
+ size
, 0, PAGE_SIZE
- size
);
5785 page
= virt_to_page(ref
->page
);
5787 spd
.pages
[i
] = page
;
5788 spd
.partial
[i
].len
= PAGE_SIZE
;
5789 spd
.partial
[i
].offset
= 0;
5790 spd
.partial
[i
].private = (unsigned long)ref
;
5794 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
5797 trace_access_unlock(iter
->cpu_file
);
5800 /* did we read anything? */
5801 if (!spd
.nr_pages
) {
5805 if ((file
->f_flags
& O_NONBLOCK
) || (flags
& SPLICE_F_NONBLOCK
))
5808 ret
= wait_on_pipe(iter
, true);
5815 ret
= splice_to_pipe(pipe
, &spd
);
5816 splice_shrink_spd(&spd
);
5821 static const struct file_operations tracing_buffers_fops
= {
5822 .open
= tracing_buffers_open
,
5823 .read
= tracing_buffers_read
,
5824 .poll
= tracing_buffers_poll
,
5825 .release
= tracing_buffers_release
,
5826 .splice_read
= tracing_buffers_splice_read
,
5827 .llseek
= no_llseek
,
5831 tracing_stats_read(struct file
*filp
, char __user
*ubuf
,
5832 size_t count
, loff_t
*ppos
)
5834 struct inode
*inode
= file_inode(filp
);
5835 struct trace_array
*tr
= inode
->i_private
;
5836 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
5837 int cpu
= tracing_get_cpu(inode
);
5838 struct trace_seq
*s
;
5840 unsigned long long t
;
5841 unsigned long usec_rem
;
5843 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
5849 cnt
= ring_buffer_entries_cpu(trace_buf
->buffer
, cpu
);
5850 trace_seq_printf(s
, "entries: %ld\n", cnt
);
5852 cnt
= ring_buffer_overrun_cpu(trace_buf
->buffer
, cpu
);
5853 trace_seq_printf(s
, "overrun: %ld\n", cnt
);
5855 cnt
= ring_buffer_commit_overrun_cpu(trace_buf
->buffer
, cpu
);
5856 trace_seq_printf(s
, "commit overrun: %ld\n", cnt
);
5858 cnt
= ring_buffer_bytes_cpu(trace_buf
->buffer
, cpu
);
5859 trace_seq_printf(s
, "bytes: %ld\n", cnt
);
5861 if (trace_clocks
[tr
->clock_id
].in_ns
) {
5862 /* local or global for trace_clock */
5863 t
= ns2usecs(ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
5864 usec_rem
= do_div(t
, USEC_PER_SEC
);
5865 trace_seq_printf(s
, "oldest event ts: %5llu.%06lu\n",
5868 t
= ns2usecs(ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
5869 usec_rem
= do_div(t
, USEC_PER_SEC
);
5870 trace_seq_printf(s
, "now ts: %5llu.%06lu\n", t
, usec_rem
);
5872 /* counter or tsc mode for trace_clock */
5873 trace_seq_printf(s
, "oldest event ts: %llu\n",
5874 ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
5876 trace_seq_printf(s
, "now ts: %llu\n",
5877 ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
5880 cnt
= ring_buffer_dropped_events_cpu(trace_buf
->buffer
, cpu
);
5881 trace_seq_printf(s
, "dropped events: %ld\n", cnt
);
5883 cnt
= ring_buffer_read_events_cpu(trace_buf
->buffer
, cpu
);
5884 trace_seq_printf(s
, "read events: %ld\n", cnt
);
5886 count
= simple_read_from_buffer(ubuf
, count
, ppos
,
5887 s
->buffer
, trace_seq_used(s
));
5894 static const struct file_operations tracing_stats_fops
= {
5895 .open
= tracing_open_generic_tr
,
5896 .read
= tracing_stats_read
,
5897 .llseek
= generic_file_llseek
,
5898 .release
= tracing_release_generic_tr
,
5901 #ifdef CONFIG_DYNAMIC_FTRACE
5903 int __weak
ftrace_arch_read_dyn_info(char *buf
, int size
)
5909 tracing_read_dyn_info(struct file
*filp
, char __user
*ubuf
,
5910 size_t cnt
, loff_t
*ppos
)
5912 static char ftrace_dyn_info_buffer
[1024];
5913 static DEFINE_MUTEX(dyn_info_mutex
);
5914 unsigned long *p
= filp
->private_data
;
5915 char *buf
= ftrace_dyn_info_buffer
;
5916 int size
= ARRAY_SIZE(ftrace_dyn_info_buffer
);
5919 mutex_lock(&dyn_info_mutex
);
5920 r
= sprintf(buf
, "%ld ", *p
);
5922 r
+= ftrace_arch_read_dyn_info(buf
+r
, (size
-1)-r
);
5925 r
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5927 mutex_unlock(&dyn_info_mutex
);
5932 static const struct file_operations tracing_dyn_info_fops
= {
5933 .open
= tracing_open_generic
,
5934 .read
= tracing_read_dyn_info
,
5935 .llseek
= generic_file_llseek
,
5937 #endif /* CONFIG_DYNAMIC_FTRACE */
5939 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5941 ftrace_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
5947 ftrace_count_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
5949 unsigned long *count
= (long *)data
;
5961 ftrace_snapshot_print(struct seq_file
*m
, unsigned long ip
,
5962 struct ftrace_probe_ops
*ops
, void *data
)
5964 long count
= (long)data
;
5966 seq_printf(m
, "%ps:", (void *)ip
);
5968 seq_puts(m
, "snapshot");
5971 seq_puts(m
, ":unlimited\n");
5973 seq_printf(m
, ":count=%ld\n", count
);
5978 static struct ftrace_probe_ops snapshot_probe_ops
= {
5979 .func
= ftrace_snapshot
,
5980 .print
= ftrace_snapshot_print
,
5983 static struct ftrace_probe_ops snapshot_count_probe_ops
= {
5984 .func
= ftrace_count_snapshot
,
5985 .print
= ftrace_snapshot_print
,
5989 ftrace_trace_snapshot_callback(struct ftrace_hash
*hash
,
5990 char *glob
, char *cmd
, char *param
, int enable
)
5992 struct ftrace_probe_ops
*ops
;
5993 void *count
= (void *)-1;
5997 /* hash funcs only work with set_ftrace_filter */
6001 ops
= param
? &snapshot_count_probe_ops
: &snapshot_probe_ops
;
6003 if (glob
[0] == '!') {
6004 unregister_ftrace_function_probe_func(glob
+1, ops
);
6011 number
= strsep(¶m
, ":");
6013 if (!strlen(number
))
6017 * We use the callback data field (which is a pointer)
6020 ret
= kstrtoul(number
, 0, (unsigned long *)&count
);
6025 ret
= register_ftrace_function_probe(glob
, ops
, count
);
6028 alloc_snapshot(&global_trace
);
6030 return ret
< 0 ? ret
: 0;
6033 static struct ftrace_func_command ftrace_snapshot_cmd
= {
6035 .func
= ftrace_trace_snapshot_callback
,
6038 static __init
int register_snapshot_cmd(void)
6040 return register_ftrace_command(&ftrace_snapshot_cmd
);
6043 static inline __init
int register_snapshot_cmd(void) { return 0; }
6044 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6046 static struct dentry
*tracing_get_dentry(struct trace_array
*tr
)
6048 if (WARN_ON(!tr
->dir
))
6049 return ERR_PTR(-ENODEV
);
6051 /* Top directory uses NULL as the parent */
6052 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
6055 /* All sub buffers have a descriptor */
6059 static struct dentry
*tracing_dentry_percpu(struct trace_array
*tr
, int cpu
)
6061 struct dentry
*d_tracer
;
6064 return tr
->percpu_dir
;
6066 d_tracer
= tracing_get_dentry(tr
);
6067 if (IS_ERR(d_tracer
))
6070 tr
->percpu_dir
= tracefs_create_dir("per_cpu", d_tracer
);
6072 WARN_ONCE(!tr
->percpu_dir
,
6073 "Could not create tracefs directory 'per_cpu/%d'\n", cpu
);
6075 return tr
->percpu_dir
;
6078 static struct dentry
*
6079 trace_create_cpu_file(const char *name
, umode_t mode
, struct dentry
*parent
,
6080 void *data
, long cpu
, const struct file_operations
*fops
)
6082 struct dentry
*ret
= trace_create_file(name
, mode
, parent
, data
, fops
);
6084 if (ret
) /* See tracing_get_cpu() */
6085 d_inode(ret
)->i_cdev
= (void *)(cpu
+ 1);
6090 tracing_init_tracefs_percpu(struct trace_array
*tr
, long cpu
)
6092 struct dentry
*d_percpu
= tracing_dentry_percpu(tr
, cpu
);
6093 struct dentry
*d_cpu
;
6094 char cpu_dir
[30]; /* 30 characters should be more than enough */
6099 snprintf(cpu_dir
, 30, "cpu%ld", cpu
);
6100 d_cpu
= tracefs_create_dir(cpu_dir
, d_percpu
);
6102 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir
);
6106 /* per cpu trace_pipe */
6107 trace_create_cpu_file("trace_pipe", 0444, d_cpu
,
6108 tr
, cpu
, &tracing_pipe_fops
);
6111 trace_create_cpu_file("trace", 0644, d_cpu
,
6112 tr
, cpu
, &tracing_fops
);
6114 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu
,
6115 tr
, cpu
, &tracing_buffers_fops
);
6117 trace_create_cpu_file("stats", 0444, d_cpu
,
6118 tr
, cpu
, &tracing_stats_fops
);
6120 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu
,
6121 tr
, cpu
, &tracing_entries_fops
);
6123 #ifdef CONFIG_TRACER_SNAPSHOT
6124 trace_create_cpu_file("snapshot", 0644, d_cpu
,
6125 tr
, cpu
, &snapshot_fops
);
6127 trace_create_cpu_file("snapshot_raw", 0444, d_cpu
,
6128 tr
, cpu
, &snapshot_raw_fops
);
6132 #ifdef CONFIG_FTRACE_SELFTEST
6133 /* Let selftest have access to static functions in this file */
6134 #include "trace_selftest.c"
6138 trace_options_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
6141 struct trace_option_dentry
*topt
= filp
->private_data
;
6144 if (topt
->flags
->val
& topt
->opt
->bit
)
6149 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
6153 trace_options_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6156 struct trace_option_dentry
*topt
= filp
->private_data
;
6160 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6164 if (val
!= 0 && val
!= 1)
6167 if (!!(topt
->flags
->val
& topt
->opt
->bit
) != val
) {
6168 mutex_lock(&trace_types_lock
);
6169 ret
= __set_tracer_option(topt
->tr
, topt
->flags
,
6171 mutex_unlock(&trace_types_lock
);
6182 static const struct file_operations trace_options_fops
= {
6183 .open
= tracing_open_generic
,
6184 .read
= trace_options_read
,
6185 .write
= trace_options_write
,
6186 .llseek
= generic_file_llseek
,
6190 * In order to pass in both the trace_array descriptor as well as the index
6191 * to the flag that the trace option file represents, the trace_array
6192 * has a character array of trace_flags_index[], which holds the index
6193 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6194 * The address of this character array is passed to the flag option file
6195 * read/write callbacks.
6197 * In order to extract both the index and the trace_array descriptor,
6198 * get_tr_index() uses the following algorithm.
6202 * As the pointer itself contains the address of the index (remember
6205 * Then to get the trace_array descriptor, by subtracting that index
6206 * from the ptr, we get to the start of the index itself.
6208 * ptr - idx == &index[0]
6210 * Then a simple container_of() from that pointer gets us to the
6211 * trace_array descriptor.
6213 static void get_tr_index(void *data
, struct trace_array
**ptr
,
6214 unsigned int *pindex
)
6216 *pindex
= *(unsigned char *)data
;
6218 *ptr
= container_of(data
- *pindex
, struct trace_array
,
6223 trace_options_core_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
6226 void *tr_index
= filp
->private_data
;
6227 struct trace_array
*tr
;
6231 get_tr_index(tr_index
, &tr
, &index
);
6233 if (tr
->trace_flags
& (1 << index
))
6238 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
6242 trace_options_core_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6245 void *tr_index
= filp
->private_data
;
6246 struct trace_array
*tr
;
6251 get_tr_index(tr_index
, &tr
, &index
);
6253 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6257 if (val
!= 0 && val
!= 1)
6260 mutex_lock(&trace_types_lock
);
6261 ret
= set_tracer_flag(tr
, 1 << index
, val
);
6262 mutex_unlock(&trace_types_lock
);
6272 static const struct file_operations trace_options_core_fops
= {
6273 .open
= tracing_open_generic
,
6274 .read
= trace_options_core_read
,
6275 .write
= trace_options_core_write
,
6276 .llseek
= generic_file_llseek
,
6279 struct dentry
*trace_create_file(const char *name
,
6281 struct dentry
*parent
,
6283 const struct file_operations
*fops
)
6287 ret
= tracefs_create_file(name
, mode
, parent
, data
, fops
);
6289 pr_warning("Could not create tracefs '%s' entry\n", name
);
6295 static struct dentry
*trace_options_init_dentry(struct trace_array
*tr
)
6297 struct dentry
*d_tracer
;
6302 d_tracer
= tracing_get_dentry(tr
);
6303 if (IS_ERR(d_tracer
))
6306 tr
->options
= tracefs_create_dir("options", d_tracer
);
6308 pr_warning("Could not create tracefs directory 'options'\n");
6316 create_trace_option_file(struct trace_array
*tr
,
6317 struct trace_option_dentry
*topt
,
6318 struct tracer_flags
*flags
,
6319 struct tracer_opt
*opt
)
6321 struct dentry
*t_options
;
6323 t_options
= trace_options_init_dentry(tr
);
6327 topt
->flags
= flags
;
6331 topt
->entry
= trace_create_file(opt
->name
, 0644, t_options
, topt
,
6332 &trace_options_fops
);
6336 static struct trace_option_dentry
*
6337 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
)
6339 struct trace_option_dentry
*topts
;
6340 struct tracer_flags
*flags
;
6341 struct tracer_opt
*opts
;
6347 flags
= tracer
->flags
;
6349 if (!flags
|| !flags
->opts
)
6354 for (cnt
= 0; opts
[cnt
].name
; cnt
++)
6357 topts
= kcalloc(cnt
+ 1, sizeof(*topts
), GFP_KERNEL
);
6361 for (cnt
= 0; opts
[cnt
].name
; cnt
++) {
6362 create_trace_option_file(tr
, &topts
[cnt
], flags
,
6364 WARN_ONCE(topts
[cnt
].entry
== NULL
,
6365 "Failed to create trace option: %s",
6372 static struct dentry
*
6373 create_trace_option_core_file(struct trace_array
*tr
,
6374 const char *option
, long index
)
6376 struct dentry
*t_options
;
6378 t_options
= trace_options_init_dentry(tr
);
6382 return trace_create_file(option
, 0644, t_options
,
6383 (void *)&tr
->trace_flags_index
[index
],
6384 &trace_options_core_fops
);
6387 static __init
void create_trace_options_dir(struct trace_array
*tr
)
6389 struct dentry
*t_options
;
6392 t_options
= trace_options_init_dentry(tr
);
6396 for (i
= 0; trace_options
[i
]; i
++)
6397 create_trace_option_core_file(tr
, trace_options
[i
], i
);
6401 rb_simple_read(struct file
*filp
, char __user
*ubuf
,
6402 size_t cnt
, loff_t
*ppos
)
6404 struct trace_array
*tr
= filp
->private_data
;
6408 r
= tracer_tracing_is_on(tr
);
6409 r
= sprintf(buf
, "%d\n", r
);
6411 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6415 rb_simple_write(struct file
*filp
, const char __user
*ubuf
,
6416 size_t cnt
, loff_t
*ppos
)
6418 struct trace_array
*tr
= filp
->private_data
;
6419 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
6423 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6428 mutex_lock(&trace_types_lock
);
6430 tracer_tracing_on(tr
);
6431 if (tr
->current_trace
->start
)
6432 tr
->current_trace
->start(tr
);
6434 tracer_tracing_off(tr
);
6435 if (tr
->current_trace
->stop
)
6436 tr
->current_trace
->stop(tr
);
6438 mutex_unlock(&trace_types_lock
);
6446 static const struct file_operations rb_simple_fops
= {
6447 .open
= tracing_open_generic_tr
,
6448 .read
= rb_simple_read
,
6449 .write
= rb_simple_write
,
6450 .release
= tracing_release_generic_tr
,
6451 .llseek
= default_llseek
,
6454 struct dentry
*trace_instance_dir
;
6457 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
);
6460 allocate_trace_buffer(struct trace_array
*tr
, struct trace_buffer
*buf
, int size
)
6462 enum ring_buffer_flags rb_flags
;
6464 rb_flags
= tr
->trace_flags
& TRACE_ITER_OVERWRITE
? RB_FL_OVERWRITE
: 0;
6468 buf
->buffer
= ring_buffer_alloc(size
, rb_flags
);
6472 buf
->data
= alloc_percpu(struct trace_array_cpu
);
6474 ring_buffer_free(buf
->buffer
);
6478 /* Allocate the first page for all buffers */
6479 set_buffer_entries(&tr
->trace_buffer
,
6480 ring_buffer_size(tr
->trace_buffer
.buffer
, 0));
6485 static int allocate_trace_buffers(struct trace_array
*tr
, int size
)
6489 ret
= allocate_trace_buffer(tr
, &tr
->trace_buffer
, size
);
6493 #ifdef CONFIG_TRACER_MAX_TRACE
6494 ret
= allocate_trace_buffer(tr
, &tr
->max_buffer
,
6495 allocate_snapshot
? size
: 1);
6497 ring_buffer_free(tr
->trace_buffer
.buffer
);
6498 free_percpu(tr
->trace_buffer
.data
);
6501 tr
->allocated_snapshot
= allocate_snapshot
;
6504 * Only the top level trace array gets its snapshot allocated
6505 * from the kernel command line.
6507 allocate_snapshot
= false;
6512 static void free_trace_buffer(struct trace_buffer
*buf
)
6515 ring_buffer_free(buf
->buffer
);
6517 free_percpu(buf
->data
);
6522 static void free_trace_buffers(struct trace_array
*tr
)
6527 free_trace_buffer(&tr
->trace_buffer
);
6529 #ifdef CONFIG_TRACER_MAX_TRACE
6530 free_trace_buffer(&tr
->max_buffer
);
6534 static void init_trace_flags_index(struct trace_array
*tr
)
6538 /* Used by the trace options files */
6539 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++)
6540 tr
->trace_flags_index
[i
] = i
;
6543 static int instance_mkdir(const char *name
)
6545 struct trace_array
*tr
;
6548 mutex_lock(&trace_types_lock
);
6551 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
6552 if (tr
->name
&& strcmp(tr
->name
, name
) == 0)
6557 tr
= kzalloc(sizeof(*tr
), GFP_KERNEL
);
6561 tr
->name
= kstrdup(name
, GFP_KERNEL
);
6565 if (!alloc_cpumask_var(&tr
->tracing_cpumask
, GFP_KERNEL
))
6568 tr
->trace_flags
= global_trace
.trace_flags
;
6570 cpumask_copy(tr
->tracing_cpumask
, cpu_all_mask
);
6572 raw_spin_lock_init(&tr
->start_lock
);
6574 tr
->max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
6576 tr
->current_trace
= &nop_trace
;
6578 INIT_LIST_HEAD(&tr
->systems
);
6579 INIT_LIST_HEAD(&tr
->events
);
6581 if (allocate_trace_buffers(tr
, trace_buf_size
) < 0)
6584 tr
->dir
= tracefs_create_dir(name
, trace_instance_dir
);
6588 ret
= event_trace_add_tracer(tr
->dir
, tr
);
6590 tracefs_remove_recursive(tr
->dir
);
6594 init_tracer_tracefs(tr
, tr
->dir
);
6595 init_trace_flags_index(tr
);
6597 list_add(&tr
->list
, &ftrace_trace_arrays
);
6599 mutex_unlock(&trace_types_lock
);
6604 free_trace_buffers(tr
);
6605 free_cpumask_var(tr
->tracing_cpumask
);
6610 mutex_unlock(&trace_types_lock
);
6616 static int instance_rmdir(const char *name
)
6618 struct trace_array
*tr
;
6622 mutex_lock(&trace_types_lock
);
6625 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
6626 if (tr
->name
&& strcmp(tr
->name
, name
) == 0) {
6635 if (tr
->ref
|| (tr
->current_trace
&& tr
->current_trace
->ref
))
6638 list_del(&tr
->list
);
6640 tracing_set_nop(tr
);
6641 event_trace_del_tracer(tr
);
6642 ftrace_destroy_function_files(tr
);
6643 debugfs_remove_recursive(tr
->dir
);
6644 free_trace_buffers(tr
);
6652 mutex_unlock(&trace_types_lock
);
6657 static __init
void create_trace_instances(struct dentry
*d_tracer
)
6659 trace_instance_dir
= tracefs_create_instance_dir("instances", d_tracer
,
6662 if (WARN_ON(!trace_instance_dir
))
6667 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
)
6671 trace_create_file("available_tracers", 0444, d_tracer
,
6672 tr
, &show_traces_fops
);
6674 trace_create_file("current_tracer", 0644, d_tracer
,
6675 tr
, &set_tracer_fops
);
6677 trace_create_file("tracing_cpumask", 0644, d_tracer
,
6678 tr
, &tracing_cpumask_fops
);
6680 trace_create_file("trace_options", 0644, d_tracer
,
6681 tr
, &tracing_iter_fops
);
6683 trace_create_file("trace", 0644, d_tracer
,
6686 trace_create_file("trace_pipe", 0444, d_tracer
,
6687 tr
, &tracing_pipe_fops
);
6689 trace_create_file("buffer_size_kb", 0644, d_tracer
,
6690 tr
, &tracing_entries_fops
);
6692 trace_create_file("buffer_total_size_kb", 0444, d_tracer
,
6693 tr
, &tracing_total_entries_fops
);
6695 trace_create_file("free_buffer", 0200, d_tracer
,
6696 tr
, &tracing_free_buffer_fops
);
6698 trace_create_file("trace_marker", 0220, d_tracer
,
6699 tr
, &tracing_mark_fops
);
6701 trace_create_file("trace_clock", 0644, d_tracer
, tr
,
6704 trace_create_file("tracing_on", 0644, d_tracer
,
6705 tr
, &rb_simple_fops
);
6707 #ifdef CONFIG_TRACER_MAX_TRACE
6708 trace_create_file("tracing_max_latency", 0644, d_tracer
,
6709 &tr
->max_latency
, &tracing_max_lat_fops
);
6712 if (ftrace_create_function_files(tr
, d_tracer
))
6713 WARN(1, "Could not allocate function filter files");
6715 #ifdef CONFIG_TRACER_SNAPSHOT
6716 trace_create_file("snapshot", 0644, d_tracer
,
6717 tr
, &snapshot_fops
);
6720 for_each_tracing_cpu(cpu
)
6721 tracing_init_tracefs_percpu(tr
, cpu
);
6725 static struct vfsmount
*trace_automount(void *ingore
)
6727 struct vfsmount
*mnt
;
6728 struct file_system_type
*type
;
6731 * To maintain backward compatibility for tools that mount
6732 * debugfs to get to the tracing facility, tracefs is automatically
6733 * mounted to the debugfs/tracing directory.
6735 type
= get_fs_type("tracefs");
6738 mnt
= vfs_kern_mount(type
, 0, "tracefs", NULL
);
6739 put_filesystem(type
);
6748 * tracing_init_dentry - initialize top level trace array
6750 * This is called when creating files or directories in the tracing
6751 * directory. It is called via fs_initcall() by any of the boot up code
6752 * and expects to return the dentry of the top level tracing directory.
6754 struct dentry
*tracing_init_dentry(void)
6756 struct trace_array
*tr
= &global_trace
;
6758 /* The top level trace array uses NULL as parent */
6762 if (WARN_ON(!debugfs_initialized()))
6763 return ERR_PTR(-ENODEV
);
6766 * As there may still be users that expect the tracing
6767 * files to exist in debugfs/tracing, we must automount
6768 * the tracefs file system there, so older tools still
6769 * work with the newer kerenl.
6771 tr
->dir
= debugfs_create_automount("tracing", NULL
,
6772 trace_automount
, NULL
);
6774 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6775 return ERR_PTR(-ENOMEM
);
6781 extern struct trace_enum_map
*__start_ftrace_enum_maps
[];
6782 extern struct trace_enum_map
*__stop_ftrace_enum_maps
[];
6784 static void __init
trace_enum_init(void)
6788 len
= __stop_ftrace_enum_maps
- __start_ftrace_enum_maps
;
6789 trace_insert_enum_map(NULL
, __start_ftrace_enum_maps
, len
);
6792 #ifdef CONFIG_MODULES
6793 static void trace_module_add_enums(struct module
*mod
)
6795 if (!mod
->num_trace_enums
)
6799 * Modules with bad taint do not have events created, do
6800 * not bother with enums either.
6802 if (trace_module_has_bad_taint(mod
))
6805 trace_insert_enum_map(mod
, mod
->trace_enums
, mod
->num_trace_enums
);
6808 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
6809 static void trace_module_remove_enums(struct module
*mod
)
6811 union trace_enum_map_item
*map
;
6812 union trace_enum_map_item
**last
= &trace_enum_maps
;
6814 if (!mod
->num_trace_enums
)
6817 mutex_lock(&trace_enum_mutex
);
6819 map
= trace_enum_maps
;
6822 if (map
->head
.mod
== mod
)
6824 map
= trace_enum_jmp_to_tail(map
);
6825 last
= &map
->tail
.next
;
6826 map
= map
->tail
.next
;
6831 *last
= trace_enum_jmp_to_tail(map
)->tail
.next
;
6834 mutex_unlock(&trace_enum_mutex
);
6837 static inline void trace_module_remove_enums(struct module
*mod
) { }
6838 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6840 static int trace_module_notify(struct notifier_block
*self
,
6841 unsigned long val
, void *data
)
6843 struct module
*mod
= data
;
6846 case MODULE_STATE_COMING
:
6847 trace_module_add_enums(mod
);
6849 case MODULE_STATE_GOING
:
6850 trace_module_remove_enums(mod
);
6857 static struct notifier_block trace_module_nb
= {
6858 .notifier_call
= trace_module_notify
,
6861 #endif /* CONFIG_MODULES */
6863 static __init
int tracer_init_tracefs(void)
6865 struct dentry
*d_tracer
;
6868 trace_access_lock_init();
6870 d_tracer
= tracing_init_dentry();
6871 if (IS_ERR(d_tracer
))
6874 init_tracer_tracefs(&global_trace
, d_tracer
);
6876 trace_create_file("tracing_thresh", 0644, d_tracer
,
6877 &global_trace
, &tracing_thresh_fops
);
6879 trace_create_file("README", 0444, d_tracer
,
6880 NULL
, &tracing_readme_fops
);
6882 trace_create_file("saved_cmdlines", 0444, d_tracer
,
6883 NULL
, &tracing_saved_cmdlines_fops
);
6885 trace_create_file("saved_cmdlines_size", 0644, d_tracer
,
6886 NULL
, &tracing_saved_cmdlines_size_fops
);
6890 trace_create_enum_file(d_tracer
);
6892 #ifdef CONFIG_MODULES
6893 register_module_notifier(&trace_module_nb
);
6896 #ifdef CONFIG_DYNAMIC_FTRACE
6897 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer
,
6898 &ftrace_update_tot_cnt
, &tracing_dyn_info_fops
);
6901 create_trace_instances(d_tracer
);
6903 create_trace_options_dir(&global_trace
);
6905 mutex_lock(&trace_types_lock
);
6906 for (t
= trace_types
; t
; t
= t
->next
)
6907 add_tracer_options(&global_trace
, t
);
6908 mutex_unlock(&trace_types_lock
);
6913 static int trace_panic_handler(struct notifier_block
*this,
6914 unsigned long event
, void *unused
)
6916 if (ftrace_dump_on_oops
)
6917 ftrace_dump(ftrace_dump_on_oops
);
6921 static struct notifier_block trace_panic_notifier
= {
6922 .notifier_call
= trace_panic_handler
,
6924 .priority
= 150 /* priority: INT_MAX >= x >= 0 */
6927 static int trace_die_handler(struct notifier_block
*self
,
6933 if (ftrace_dump_on_oops
)
6934 ftrace_dump(ftrace_dump_on_oops
);
6942 static struct notifier_block trace_die_notifier
= {
6943 .notifier_call
= trace_die_handler
,
6948 * printk is set to max of 1024, we really don't need it that big.
6949 * Nothing should be printing 1000 characters anyway.
6951 #define TRACE_MAX_PRINT 1000
6954 * Define here KERN_TRACE so that we have one place to modify
6955 * it if we decide to change what log level the ftrace dump
6958 #define KERN_TRACE KERN_EMERG
6961 trace_printk_seq(struct trace_seq
*s
)
6963 /* Probably should print a warning here. */
6964 if (s
->seq
.len
>= TRACE_MAX_PRINT
)
6965 s
->seq
.len
= TRACE_MAX_PRINT
;
6968 * More paranoid code. Although the buffer size is set to
6969 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
6970 * an extra layer of protection.
6972 if (WARN_ON_ONCE(s
->seq
.len
>= s
->seq
.size
))
6973 s
->seq
.len
= s
->seq
.size
- 1;
6975 /* should be zero ended, but we are paranoid. */
6976 s
->buffer
[s
->seq
.len
] = 0;
6978 printk(KERN_TRACE
"%s", s
->buffer
);
6983 void trace_init_global_iter(struct trace_iterator
*iter
)
6985 iter
->tr
= &global_trace
;
6986 iter
->trace
= iter
->tr
->current_trace
;
6987 iter
->cpu_file
= RING_BUFFER_ALL_CPUS
;
6988 iter
->trace_buffer
= &global_trace
.trace_buffer
;
6990 if (iter
->trace
&& iter
->trace
->open
)
6991 iter
->trace
->open(iter
);
6993 /* Annotate start of buffers if we had overruns */
6994 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
6995 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
6997 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6998 if (trace_clocks
[iter
->tr
->clock_id
].in_ns
)
6999 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
7002 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode
)
7004 /* use static because iter can be a bit big for the stack */
7005 static struct trace_iterator iter
;
7006 static atomic_t dump_running
;
7007 struct trace_array
*tr
= &global_trace
;
7008 unsigned int old_userobj
;
7009 unsigned long flags
;
7012 /* Only allow one dump user at a time. */
7013 if (atomic_inc_return(&dump_running
) != 1) {
7014 atomic_dec(&dump_running
);
7019 * Always turn off tracing when we dump.
7020 * We don't need to show trace output of what happens
7021 * between multiple crashes.
7023 * If the user does a sysrq-z, then they can re-enable
7024 * tracing with echo 1 > tracing_on.
7028 local_irq_save(flags
);
7030 /* Simulate the iterator */
7031 trace_init_global_iter(&iter
);
7033 for_each_tracing_cpu(cpu
) {
7034 atomic_inc(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
7037 old_userobj
= tr
->trace_flags
& TRACE_ITER_SYM_USEROBJ
;
7039 /* don't look at user memory in panic mode */
7040 tr
->trace_flags
&= ~TRACE_ITER_SYM_USEROBJ
;
7042 switch (oops_dump_mode
) {
7044 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
7047 iter
.cpu_file
= raw_smp_processor_id();
7052 printk(KERN_TRACE
"Bad dumping mode, switching to all CPUs dump\n");
7053 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
7056 printk(KERN_TRACE
"Dumping ftrace buffer:\n");
7058 /* Did function tracer already get disabled? */
7059 if (ftrace_is_dead()) {
7060 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7061 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7065 * We need to stop all tracing on all CPUS to read the
7066 * the next buffer. This is a bit expensive, but is
7067 * not done often. We fill all what we can read,
7068 * and then release the locks again.
7071 while (!trace_empty(&iter
)) {
7074 printk(KERN_TRACE
"---------------------------------\n");
7078 /* reset all but tr, trace, and overruns */
7079 memset(&iter
.seq
, 0,
7080 sizeof(struct trace_iterator
) -
7081 offsetof(struct trace_iterator
, seq
));
7082 iter
.iter_flags
|= TRACE_FILE_LAT_FMT
;
7085 if (trace_find_next_entry_inc(&iter
) != NULL
) {
7088 ret
= print_trace_line(&iter
);
7089 if (ret
!= TRACE_TYPE_NO_CONSUME
)
7090 trace_consume(&iter
);
7092 touch_nmi_watchdog();
7094 trace_printk_seq(&iter
.seq
);
7098 printk(KERN_TRACE
" (ftrace buffer empty)\n");
7100 printk(KERN_TRACE
"---------------------------------\n");
7103 tr
->trace_flags
|= old_userobj
;
7105 for_each_tracing_cpu(cpu
) {
7106 atomic_dec(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
7108 atomic_dec(&dump_running
);
7109 local_irq_restore(flags
);
7111 EXPORT_SYMBOL_GPL(ftrace_dump
);
7113 __init
static int tracer_alloc_buffers(void)
7119 * Make sure we don't accidently add more trace options
7120 * than we have bits for.
7122 BUILD_BUG_ON(TRACE_ITER_LAST_BIT
> TRACE_FLAGS_MAX_SIZE
);
7124 if (!alloc_cpumask_var(&tracing_buffer_mask
, GFP_KERNEL
))
7127 if (!alloc_cpumask_var(&global_trace
.tracing_cpumask
, GFP_KERNEL
))
7128 goto out_free_buffer_mask
;
7130 /* Only allocate trace_printk buffers if a trace_printk exists */
7131 if (__stop___trace_bprintk_fmt
!= __start___trace_bprintk_fmt
)
7132 /* Must be called before global_trace.buffer is allocated */
7133 trace_printk_init_buffers();
7135 /* To save memory, keep the ring buffer size to its minimum */
7136 if (ring_buffer_expanded
)
7137 ring_buf_size
= trace_buf_size
;
7141 cpumask_copy(tracing_buffer_mask
, cpu_possible_mask
);
7142 cpumask_copy(global_trace
.tracing_cpumask
, cpu_all_mask
);
7144 raw_spin_lock_init(&global_trace
.start_lock
);
7146 /* Used for event triggers */
7147 temp_buffer
= ring_buffer_alloc(PAGE_SIZE
, RB_FL_OVERWRITE
);
7149 goto out_free_cpumask
;
7151 if (trace_create_savedcmd() < 0)
7152 goto out_free_temp_buffer
;
7154 /* TODO: make the number of buffers hot pluggable with CPUS */
7155 if (allocate_trace_buffers(&global_trace
, ring_buf_size
) < 0) {
7156 printk(KERN_ERR
"tracer: failed to allocate ring buffer!\n");
7158 goto out_free_savedcmd
;
7161 if (global_trace
.buffer_disabled
)
7164 if (trace_boot_clock
) {
7165 ret
= tracing_set_clock(&global_trace
, trace_boot_clock
);
7167 pr_warning("Trace clock %s not defined, going back to default\n",
7172 * register_tracer() might reference current_trace, so it
7173 * needs to be set before we register anything. This is
7174 * just a bootstrap of current_trace anyway.
7176 global_trace
.current_trace
= &nop_trace
;
7178 global_trace
.max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
7180 ftrace_init_global_array_ops(&global_trace
);
7182 init_trace_flags_index(&global_trace
);
7184 register_tracer(&nop_trace
);
7186 /* All seems OK, enable tracing */
7187 tracing_disabled
= 0;
7189 atomic_notifier_chain_register(&panic_notifier_list
,
7190 &trace_panic_notifier
);
7192 register_die_notifier(&trace_die_notifier
);
7194 global_trace
.flags
= TRACE_ARRAY_FL_GLOBAL
;
7196 INIT_LIST_HEAD(&global_trace
.systems
);
7197 INIT_LIST_HEAD(&global_trace
.events
);
7198 list_add(&global_trace
.list
, &ftrace_trace_arrays
);
7200 while (trace_boot_options
) {
7203 option
= strsep(&trace_boot_options
, ",");
7204 trace_set_options(&global_trace
, option
);
7207 register_snapshot_cmd();
7212 free_saved_cmdlines_buffer(savedcmd
);
7213 out_free_temp_buffer
:
7214 ring_buffer_free(temp_buffer
);
7216 free_cpumask_var(global_trace
.tracing_cpumask
);
7217 out_free_buffer_mask
:
7218 free_cpumask_var(tracing_buffer_mask
);
7223 void __init
trace_init(void)
7225 if (tracepoint_printk
) {
7226 tracepoint_print_iter
=
7227 kmalloc(sizeof(*tracepoint_print_iter
), GFP_KERNEL
);
7228 if (WARN_ON(!tracepoint_print_iter
))
7229 tracepoint_printk
= 0;
7231 tracer_alloc_buffers();
7235 __init
static int clear_boot_tracer(void)
7238 * The default tracer at boot buffer is an init section.
7239 * This function is called in lateinit. If we did not
7240 * find the boot tracer, then clear it out, to prevent
7241 * later registration from accessing the buffer that is
7242 * about to be freed.
7244 if (!default_bootup_tracer
)
7247 printk(KERN_INFO
"ftrace bootup tracer '%s' not registered.\n",
7248 default_bootup_tracer
);
7249 default_bootup_tracer
= NULL
;
7254 fs_initcall(tracer_init_tracefs
);
7255 late_initcall(clear_boot_tracer
);