projects / deliverable / linux.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
ftrace: make filtered functions effective on setting
[deliverable/linux.git] / kernel / trace / trace.c
... / ...
CommitLineData
1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
14#include <linux/utsrelease.h>
15#include <linux/kallsyms.h>
16#include <linux/seq_file.h>
17#include <linux/notifier.h>
18#include <linux/debugfs.h>
19#include <linux/pagemap.h>
20#include <linux/hardirq.h>
21#include <linux/linkage.h>
22#include <linux/uaccess.h>
23#include <linux/ftrace.h>
24#include <linux/module.h>
25#include <linux/percpu.h>
26#include <linux/kdebug.h>
27#include <linux/ctype.h>
28#include <linux/init.h>
29#include <linux/poll.h>
30#include <linux/gfp.h>
31#include <linux/fs.h>
32#include <linux/kprobes.h>
33#include <linux/writeback.h>
34
35#include <linux/stacktrace.h>
36#include <linux/ring_buffer.h>
37#include <linux/irqflags.h>
38
39#include "trace.h"
40
41#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
42
43unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX;
44unsigned long __read_mostly tracing_thresh;
45
46
47/*
48 * Kill all tracing for good (never come back).
49 * It is initialized to 1 but will turn to zero if the initialization
50 * of the tracer is successful. But that is the only place that sets
51 * this back to zero.
52 */
53int tracing_disabled = 1;
54
55static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
56
57static inline void ftrace_disable_cpu(void)
58{
59 preempt_disable();
60 local_inc(&__get_cpu_var(ftrace_cpu_disabled));
61}
62
63static inline void ftrace_enable_cpu(void)
64{
65 local_dec(&__get_cpu_var(ftrace_cpu_disabled));
66 preempt_enable();
67}
68
69static cpumask_t __read_mostly tracing_buffer_mask;
70
71#define for_each_tracing_cpu(cpu) \
72 for_each_cpu_mask(cpu, tracing_buffer_mask)
73
74/*
75 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
76 *
77 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
78 * is set, then ftrace_dump is called. This will output the contents
79 * of the ftrace buffers to the console. This is very useful for
80 * capturing traces that lead to crashes and outputing it to a
81 * serial console.
82 *
83 * It is default off, but you can enable it with either specifying
84 * "ftrace_dump_on_oops" in the kernel command line, or setting
85 * /proc/sys/kernel/ftrace_dump_on_oops to true.
86 */
87int ftrace_dump_on_oops;
88
89static int tracing_set_tracer(char *buf);
90
91static int __init set_ftrace(char *str)
92{
93 tracing_set_tracer(str);
94 return 1;
95}
96__setup("ftrace", set_ftrace);
97
98static int __init set_ftrace_dump_on_oops(char *str)
99{
100 ftrace_dump_on_oops = 1;
101 return 1;
102}
103__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
104
105long
106ns2usecs(cycle_t nsec)
107{
108 nsec += 500;
109 do_div(nsec, 1000);
110 return nsec;
111}
112
113cycle_t ftrace_now(int cpu)
114{
115 u64 ts = ring_buffer_time_stamp(cpu);
116 ring_buffer_normalize_time_stamp(cpu, &ts);
117 return ts;
118}
119
120/*
121 * The global_trace is the descriptor that holds the tracing
122 * buffers for the live tracing. For each CPU, it contains
123 * a link list of pages that will store trace entries. The
124 * page descriptor of the pages in the memory is used to hold
125 * the link list by linking the lru item in the page descriptor
126 * to each of the pages in the buffer per CPU.
127 *
128 * For each active CPU there is a data field that holds the
129 * pages for the buffer for that CPU. Each CPU has the same number
130 * of pages allocated for its buffer.
131 */
132static struct trace_array global_trace;
133
134static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
135
136/*
137 * The max_tr is used to snapshot the global_trace when a maximum
138 * latency is reached. Some tracers will use this to store a maximum
139 * trace while it continues examining live traces.
140 *
141 * The buffers for the max_tr are set up the same as the global_trace.
142 * When a snapshot is taken, the link list of the max_tr is swapped
143 * with the link list of the global_trace and the buffers are reset for
144 * the global_trace so the tracing can continue.
145 */
146static struct trace_array max_tr;
147
148static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
149
150/* tracer_enabled is used to toggle activation of a tracer */
151static int tracer_enabled = 1;
152
153/**
154 * tracing_is_enabled - return tracer_enabled status
155 *
156 * This function is used by other tracers to know the status
157 * of the tracer_enabled flag. Tracers may use this function
158 * to know if it should enable their features when starting
159 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
160 */
161int tracing_is_enabled(void)
162{
163 return tracer_enabled;
164}
165
166/* function tracing enabled */
167int ftrace_function_enabled;
168
169/*
170 * trace_buf_size is the size in bytes that is allocated
171 * for a buffer. Note, the number of bytes is always rounded
172 * to page size.
173 *
174 * This number is purposely set to a low number of 16384.
175 * If the dump on oops happens, it will be much appreciated
176 * to not have to wait for all that output. Anyway this can be
177 * boot time and run time configurable.
178 */
179#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
180
181static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
182
183/* trace_types holds a link list of available tracers. */
184static struct tracer *trace_types __read_mostly;
185
186/* current_trace points to the tracer that is currently active */
187static struct tracer *current_trace __read_mostly;
188
189/*
190 * max_tracer_type_len is used to simplify the allocating of
191 * buffers to read userspace tracer names. We keep track of
192 * the longest tracer name registered.
193 */
194static int max_tracer_type_len;
195
196/*
197 * trace_types_lock is used to protect the trace_types list.
198 * This lock is also used to keep user access serialized.
199 * Accesses from userspace will grab this lock while userspace
200 * activities happen inside the kernel.
201 */
202static DEFINE_MUTEX(trace_types_lock);
203
204/* trace_wait is a waitqueue for tasks blocked on trace_poll */
205static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
206
207/* trace_flags holds trace_options default values */
208unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
209 TRACE_ITER_ANNOTATE;
210
211/**
212 * trace_wake_up - wake up tasks waiting for trace input
213 *
214 * Simply wakes up any task that is blocked on the trace_wait
215 * queue. These is used with trace_poll for tasks polling the trace.
216 */
217void trace_wake_up(void)
218{
219 /*
220 * The runqueue_is_locked() can fail, but this is the best we
221 * have for now:
222 */
223 if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
224 wake_up(&trace_wait);
225}
226
227static int __init set_buf_size(char *str)
228{
229 unsigned long buf_size;
230 int ret;
231
232 if (!str)
233 return 0;
234 ret = strict_strtoul(str, 0, &buf_size);
235 /* nr_entries can not be zero */
236 if (ret < 0 || buf_size == 0)
237 return 0;
238 trace_buf_size = buf_size;
239 return 1;
240}
241__setup("trace_buf_size=", set_buf_size);
242
243unsigned long nsecs_to_usecs(unsigned long nsecs)
244{
245 return nsecs / 1000;
246}
247
248/* These must match the bit postions in trace_iterator_flags */
249static const char *trace_options[] = {
250 "print-parent",
251 "sym-offset",
252 "sym-addr",
253 "verbose",
254 "raw",
255 "hex",
256 "bin",
257 "block",
258 "stacktrace",
259 "sched-tree",
260 "ftrace_printk",
261 "ftrace_preempt",
262#ifdef CONFIG_BRANCH_TRACER
263 "branch",
264#endif
265 "annotate",
266 NULL
267};
268
269/*
270 * ftrace_max_lock is used to protect the swapping of buffers
271 * when taking a max snapshot. The buffers themselves are
272 * protected by per_cpu spinlocks. But the action of the swap
273 * needs its own lock.
274 *
275 * This is defined as a raw_spinlock_t in order to help
276 * with performance when lockdep debugging is enabled.
277 */
278static raw_spinlock_t ftrace_max_lock =
279 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
280
281/*
282 * Copy the new maximum trace into the separate maximum-trace
283 * structure. (this way the maximum trace is permanently saved,
284 * for later retrieval via /debugfs/tracing/latency_trace)
285 */
286static void
287__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
288{
289 struct trace_array_cpu *data = tr->data[cpu];
290
291 max_tr.cpu = cpu;
292 max_tr.time_start = data->preempt_timestamp;
293
294 data = max_tr.data[cpu];
295 data->saved_latency = tracing_max_latency;
296
297 memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
298 data->pid = tsk->pid;
299 data->uid = tsk->uid;
300 data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
301 data->policy = tsk->policy;
302 data->rt_priority = tsk->rt_priority;
303
304 /* record this tasks comm */
305 tracing_record_cmdline(current);
306}
307
308/**
309 * trace_seq_printf - sequence printing of trace information
310 * @s: trace sequence descriptor
311 * @fmt: printf format string
312 *
313 * The tracer may use either sequence operations or its own
314 * copy to user routines. To simplify formating of a trace
315 * trace_seq_printf is used to store strings into a special
316 * buffer (@s). Then the output may be either used by
317 * the sequencer or pulled into another buffer.
318 */
319int
320trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
321{
322 int len = (PAGE_SIZE - 1) - s->len;
323 va_list ap;
324 int ret;
325
326 if (!len)
327 return 0;
328
329 va_start(ap, fmt);
330 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
331 va_end(ap);
332
333 /* If we can't write it all, don't bother writing anything */
334 if (ret >= len)
335 return 0;
336
337 s->len += ret;
338
339 return len;
340}
341
342/**
343 * trace_seq_puts - trace sequence printing of simple string
344 * @s: trace sequence descriptor
345 * @str: simple string to record
346 *
347 * The tracer may use either the sequence operations or its own
348 * copy to user routines. This function records a simple string
349 * into a special buffer (@s) for later retrieval by a sequencer
350 * or other mechanism.
351 */
352static int
353trace_seq_puts(struct trace_seq *s, const char *str)
354{
355 int len = strlen(str);
356
357 if (len > ((PAGE_SIZE - 1) - s->len))
358 return 0;
359
360 memcpy(s->buffer + s->len, str, len);
361 s->len += len;
362
363 return len;
364}
365
366static int
367trace_seq_putc(struct trace_seq *s, unsigned char c)
368{
369 if (s->len >= (PAGE_SIZE - 1))
370 return 0;
371
372 s->buffer[s->len++] = c;
373
374 return 1;
375}
376
377static int
378trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
379{
380 if (len > ((PAGE_SIZE - 1) - s->len))
381 return 0;
382
383 memcpy(s->buffer + s->len, mem, len);
384 s->len += len;
385
386 return len;
387}
388
389#define MAX_MEMHEX_BYTES 8
390#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
391
392static int
393trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
394{
395 unsigned char hex[HEX_CHARS];
396 unsigned char *data = mem;
397 int i, j;
398
399#ifdef __BIG_ENDIAN
400 for (i = 0, j = 0; i < len; i++) {
401#else
402 for (i = len-1, j = 0; i >= 0; i--) {
403#endif
404 hex[j++] = hex_asc_hi(data[i]);
405 hex[j++] = hex_asc_lo(data[i]);
406 }
407 hex[j++] = ' ';
408
409 return trace_seq_putmem(s, hex, j);
410}
411
412static void
413trace_seq_reset(struct trace_seq *s)
414{
415 s->len = 0;
416 s->readpos = 0;
417}
418
419ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
420{
421 int len;
422 int ret;
423
424 if (s->len <= s->readpos)
425 return -EBUSY;
426
427 len = s->len - s->readpos;
428 if (cnt > len)
429 cnt = len;
430 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
431 if (ret)
432 return -EFAULT;
433
434 s->readpos += len;
435 return cnt;
436}
437
438static void
439trace_print_seq(struct seq_file *m, struct trace_seq *s)
440{
441 int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
442
443 s->buffer[len] = 0;
444 seq_puts(m, s->buffer);
445
446 trace_seq_reset(s);
447}
448
449/**
450 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
451 * @tr: tracer
452 * @tsk: the task with the latency
453 * @cpu: The cpu that initiated the trace.
454 *
455 * Flip the buffers between the @tr and the max_tr and record information
456 * about which task was the cause of this latency.
457 */
458void
459update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
460{
461 struct ring_buffer *buf = tr->buffer;
462
463 WARN_ON_ONCE(!irqs_disabled());
464 __raw_spin_lock(&ftrace_max_lock);
465
466 tr->buffer = max_tr.buffer;
467 max_tr.buffer = buf;
468
469 ftrace_disable_cpu();
470 ring_buffer_reset(tr->buffer);
471 ftrace_enable_cpu();
472
473 __update_max_tr(tr, tsk, cpu);
474 __raw_spin_unlock(&ftrace_max_lock);
475}
476
477/**
478 * update_max_tr_single - only copy one trace over, and reset the rest
479 * @tr - tracer
480 * @tsk - task with the latency
481 * @cpu - the cpu of the buffer to copy.
482 *
483 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
484 */
485void
486update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
487{
488 int ret;
489
490 WARN_ON_ONCE(!irqs_disabled());
491 __raw_spin_lock(&ftrace_max_lock);
492
493 ftrace_disable_cpu();
494
495 ring_buffer_reset(max_tr.buffer);
496 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
497
498 ftrace_enable_cpu();
499
500 WARN_ON_ONCE(ret);
501
502 __update_max_tr(tr, tsk, cpu);
503 __raw_spin_unlock(&ftrace_max_lock);
504}
505
506/**
507 * register_tracer - register a tracer with the ftrace system.
508 * @type - the plugin for the tracer
509 *
510 * Register a new plugin tracer.
511 */
512int register_tracer(struct tracer *type)
513{
514 struct tracer *t;
515 int len;
516 int ret = 0;
517
518 if (!type->name) {
519 pr_info("Tracer must have a name\n");
520 return -1;
521 }
522
523 mutex_lock(&trace_types_lock);
524 for (t = trace_types; t; t = t->next) {
525 if (strcmp(type->name, t->name) == 0) {
526 /* already found */
527 pr_info("Trace %s already registered\n",
528 type->name);
529 ret = -1;
530 goto out;
531 }
532 }
533
534#ifdef CONFIG_FTRACE_STARTUP_TEST
535 if (type->selftest) {
536 struct tracer *saved_tracer = current_trace;
537 struct trace_array *tr = &global_trace;
538 int i;
539 /*
540 * Run a selftest on this tracer.
541 * Here we reset the trace buffer, and set the current
542 * tracer to be this tracer. The tracer can then run some
543 * internal tracing to verify that everything is in order.
544 * If we fail, we do not register this tracer.
545 */
546 for_each_tracing_cpu(i) {
547 tracing_reset(tr, i);
548 }
549 current_trace = type;
550 /* the test is responsible for initializing and enabling */
551 pr_info("Testing tracer %s: ", type->name);
552 ret = type->selftest(type, tr);
553 /* the test is responsible for resetting too */
554 current_trace = saved_tracer;
555 if (ret) {
556 printk(KERN_CONT "FAILED!\n");
557 goto out;
558 }
559 /* Only reset on passing, to avoid touching corrupted buffers */
560 for_each_tracing_cpu(i) {
561 tracing_reset(tr, i);
562 }
563 printk(KERN_CONT "PASSED\n");
564 }
565#endif
566
567 type->next = trace_types;
568 trace_types = type;
569 len = strlen(type->name);
570 if (len > max_tracer_type_len)
571 max_tracer_type_len = len;
572
573 out:
574 mutex_unlock(&trace_types_lock);
575
576 return ret;
577}
578
579void unregister_tracer(struct tracer *type)
580{
581 struct tracer **t;
582 int len;
583
584 mutex_lock(&trace_types_lock);
585 for (t = &trace_types; *t; t = &(*t)->next) {
586 if (*t == type)
587 goto found;
588 }
589 pr_info("Trace %s not registered\n", type->name);
590 goto out;
591
592 found:
593 *t = (*t)->next;
594 if (strlen(type->name) != max_tracer_type_len)
595 goto out;
596
597 max_tracer_type_len = 0;
598 for (t = &trace_types; *t; t = &(*t)->next) {
599 len = strlen((*t)->name);
600 if (len > max_tracer_type_len)
601 max_tracer_type_len = len;
602 }
603 out:
604 mutex_unlock(&trace_types_lock);
605}
606
607void tracing_reset(struct trace_array *tr, int cpu)
608{
609 ftrace_disable_cpu();
610 ring_buffer_reset_cpu(tr->buffer, cpu);
611 ftrace_enable_cpu();
612}
613
614#define SAVED_CMDLINES 128
615static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
616static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
617static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
618static int cmdline_idx;
619static DEFINE_SPINLOCK(trace_cmdline_lock);
620
621/* temporary disable recording */
622atomic_t trace_record_cmdline_disabled __read_mostly;
623
624static void trace_init_cmdlines(void)
625{
626 memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
627 memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
628 cmdline_idx = 0;
629}
630
631static int trace_stop_count;
632static DEFINE_SPINLOCK(tracing_start_lock);
633
634/**
635 * tracing_start - quick start of the tracer
636 *
637 * If tracing is enabled but was stopped by tracing_stop,
638 * this will start the tracer back up.
639 */
640void tracing_start(void)
641{
642 struct ring_buffer *buffer;
643 unsigned long flags;
644
645 if (tracing_disabled)
646 return;
647
648 spin_lock_irqsave(&tracing_start_lock, flags);
649 if (--trace_stop_count)
650 goto out;
651
652 if (trace_stop_count < 0) {
653 /* Someone screwed up their debugging */
654 WARN_ON_ONCE(1);
655 trace_stop_count = 0;
656 goto out;
657 }
658
659
660 buffer = global_trace.buffer;
661 if (buffer)
662 ring_buffer_record_enable(buffer);
663
664 buffer = max_tr.buffer;
665 if (buffer)
666 ring_buffer_record_enable(buffer);
667
668 ftrace_start();
669 out:
670 spin_unlock_irqrestore(&tracing_start_lock, flags);
671}
672
673/**
674 * tracing_stop - quick stop of the tracer
675 *
676 * Light weight way to stop tracing. Use in conjunction with
677 * tracing_start.
678 */
679void tracing_stop(void)
680{
681 struct ring_buffer *buffer;
682 unsigned long flags;
683
684 ftrace_stop();
685 spin_lock_irqsave(&tracing_start_lock, flags);
686 if (trace_stop_count++)
687 goto out;
688
689 buffer = global_trace.buffer;
690 if (buffer)
691 ring_buffer_record_disable(buffer);
692
693 buffer = max_tr.buffer;
694 if (buffer)
695 ring_buffer_record_disable(buffer);
696
697 out:
698 spin_unlock_irqrestore(&tracing_start_lock, flags);
699}
700
701void trace_stop_cmdline_recording(void);
702
703static void trace_save_cmdline(struct task_struct *tsk)
704{
705 unsigned map;
706 unsigned idx;
707
708 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
709 return;
710
711 /*
712 * It's not the end of the world if we don't get
713 * the lock, but we also don't want to spin
714 * nor do we want to disable interrupts,
715 * so if we miss here, then better luck next time.
716 */
717 if (!spin_trylock(&trace_cmdline_lock))
718 return;
719
720 idx = map_pid_to_cmdline[tsk->pid];
721 if (idx >= SAVED_CMDLINES) {
722 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
723
724 map = map_cmdline_to_pid[idx];
725 if (map <= PID_MAX_DEFAULT)
726 map_pid_to_cmdline[map] = (unsigned)-1;
727
728 map_pid_to_cmdline[tsk->pid] = idx;
729
730 cmdline_idx = idx;
731 }
732
733 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
734
735 spin_unlock(&trace_cmdline_lock);
736}
737
738static char *trace_find_cmdline(int pid)
739{
740 char *cmdline = "<...>";
741 unsigned map;
742
743 if (!pid)
744 return "<idle>";
745
746 if (pid > PID_MAX_DEFAULT)
747 goto out;
748
749 map = map_pid_to_cmdline[pid];
750 if (map >= SAVED_CMDLINES)
751 goto out;
752
753 cmdline = saved_cmdlines[map];
754
755 out:
756 return cmdline;
757}
758
759void tracing_record_cmdline(struct task_struct *tsk)
760{
761 if (atomic_read(&trace_record_cmdline_disabled))
762 return;
763
764 trace_save_cmdline(tsk);
765}
766
767void
768tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
769 int pc)
770{
771 struct task_struct *tsk = current;
772
773 entry->preempt_count = pc & 0xff;
774 entry->pid = (tsk) ? tsk->pid : 0;
775 entry->flags =
776#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
777 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
778#else
779 TRACE_FLAG_IRQS_NOSUPPORT |
780#endif
781 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
782 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
783 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
784}
785
786void
787trace_function(struct trace_array *tr, struct trace_array_cpu *data,
788 unsigned long ip, unsigned long parent_ip, unsigned long flags,
789 int pc)
790{
791 struct ring_buffer_event *event;
792 struct ftrace_entry *entry;
793 unsigned long irq_flags;
794
795 /* If we are reading the ring buffer, don't trace */
796 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
797 return;
798
799 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
800 &irq_flags);
801 if (!event)
802 return;
803 entry = ring_buffer_event_data(event);
804 tracing_generic_entry_update(&entry->ent, flags, pc);
805 entry->ent.type = TRACE_FN;
806 entry->ip = ip;
807 entry->parent_ip = parent_ip;
808 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
809}
810
811#ifdef CONFIG_FUNCTION_RET_TRACER
812static void __trace_function_return(struct trace_array *tr,
813 struct trace_array_cpu *data,
814 struct ftrace_retfunc *trace,
815 unsigned long flags,
816 int pc)
817{
818 struct ring_buffer_event *event;
819 struct ftrace_ret_entry *entry;
820 unsigned long irq_flags;
821
822 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
823 return;
824
825 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry),
826 &irq_flags);
827 if (!event)
828 return;
829 entry = ring_buffer_event_data(event);
830 tracing_generic_entry_update(&entry->ent, flags, pc);
831 entry->ent.type = TRACE_FN_RET;
832 entry->ip = trace->func;
833 entry->parent_ip = trace->ret;
834 entry->rettime = trace->rettime;
835 entry->calltime = trace->calltime;
836 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags);
837}
838#endif
839
840void
841ftrace(struct trace_array *tr, struct trace_array_cpu *data,
842 unsigned long ip, unsigned long parent_ip, unsigned long flags,
843 int pc)
844{
845 if (likely(!atomic_read(&data->disabled)))
846 trace_function(tr, data, ip, parent_ip, flags, pc);
847}
848
849static void ftrace_trace_stack(struct trace_array *tr,
850 struct trace_array_cpu *data,
851 unsigned long flags,
852 int skip, int pc)
853{
854#ifdef CONFIG_STACKTRACE
855 struct ring_buffer_event *event;
856 struct stack_entry *entry;
857 struct stack_trace trace;
858 unsigned long irq_flags;
859
860 if (!(trace_flags & TRACE_ITER_STACKTRACE))
861 return;
862
863 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
864 &irq_flags);
865 if (!event)
866 return;
867 entry = ring_buffer_event_data(event);
868 tracing_generic_entry_update(&entry->ent, flags, pc);
869 entry->ent.type = TRACE_STACK;
870
871 memset(&entry->caller, 0, sizeof(entry->caller));
872
873 trace.nr_entries = 0;
874 trace.max_entries = FTRACE_STACK_ENTRIES;
875 trace.skip = skip;
876 trace.entries = entry->caller;
877
878 save_stack_trace(&trace);
879 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
880#endif
881}
882
883void __trace_stack(struct trace_array *tr,
884 struct trace_array_cpu *data,
885 unsigned long flags,
886 int skip)
887{
888 ftrace_trace_stack(tr, data, flags, skip, preempt_count());
889}
890
891static void
892ftrace_trace_special(void *__tr, void *__data,
893 unsigned long arg1, unsigned long arg2, unsigned long arg3,
894 int pc)
895{
896 struct ring_buffer_event *event;
897 struct trace_array_cpu *data = __data;
898 struct trace_array *tr = __tr;
899 struct special_entry *entry;
900 unsigned long irq_flags;
901
902 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
903 &irq_flags);
904 if (!event)
905 return;
906 entry = ring_buffer_event_data(event);
907 tracing_generic_entry_update(&entry->ent, 0, pc);
908 entry->ent.type = TRACE_SPECIAL;
909 entry->arg1 = arg1;
910 entry->arg2 = arg2;
911 entry->arg3 = arg3;
912 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
913 ftrace_trace_stack(tr, data, irq_flags, 4, pc);
914
915 trace_wake_up();
916}
917
918void
919__trace_special(void *__tr, void *__data,
920 unsigned long arg1, unsigned long arg2, unsigned long arg3)
921{
922 ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count());
923}
924
925void
926tracing_sched_switch_trace(struct trace_array *tr,
927 struct trace_array_cpu *data,
928 struct task_struct *prev,
929 struct task_struct *next,
930 unsigned long flags, int pc)
931{
932 struct ring_buffer_event *event;
933 struct ctx_switch_entry *entry;
934 unsigned long irq_flags;
935
936 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
937 &irq_flags);
938 if (!event)
939 return;
940 entry = ring_buffer_event_data(event);
941 tracing_generic_entry_update(&entry->ent, flags, pc);
942 entry->ent.type = TRACE_CTX;
943 entry->prev_pid = prev->pid;
944 entry->prev_prio = prev->prio;
945 entry->prev_state = prev->state;
946 entry->next_pid = next->pid;
947 entry->next_prio = next->prio;
948 entry->next_state = next->state;
949 entry->next_cpu = task_cpu(next);
950 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
951 ftrace_trace_stack(tr, data, flags, 5, pc);
952}
953
954void
955tracing_sched_wakeup_trace(struct trace_array *tr,
956 struct trace_array_cpu *data,
957 struct task_struct *wakee,
958 struct task_struct *curr,
959 unsigned long flags, int pc)
960{
961 struct ring_buffer_event *event;
962 struct ctx_switch_entry *entry;
963 unsigned long irq_flags;
964
965 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
966 &irq_flags);
967 if (!event)
968 return;
969 entry = ring_buffer_event_data(event);
970 tracing_generic_entry_update(&entry->ent, flags, pc);
971 entry->ent.type = TRACE_WAKE;
972 entry->prev_pid = curr->pid;
973 entry->prev_prio = curr->prio;
974 entry->prev_state = curr->state;
975 entry->next_pid = wakee->pid;
976 entry->next_prio = wakee->prio;
977 entry->next_state = wakee->state;
978 entry->next_cpu = task_cpu(wakee);
979 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
980 ftrace_trace_stack(tr, data, flags, 6, pc);
981
982 trace_wake_up();
983}
984
985void
986ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
987{
988 struct trace_array *tr = &global_trace;
989 struct trace_array_cpu *data;
990 unsigned long flags;
991 int cpu;
992 int pc;
993
994 if (tracing_disabled)
995 return;
996
997 pc = preempt_count();
998 local_irq_save(flags);
999 cpu = raw_smp_processor_id();
1000 data = tr->data[cpu];
1001
1002 if (likely(atomic_inc_return(&data->disabled) == 1))
1003 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc);
1004
1005 atomic_dec(&data->disabled);
1006 local_irq_restore(flags);
1007}
1008
1009#ifdef CONFIG_FUNCTION_TRACER
1010static void
1011function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
1012{
1013 struct trace_array *tr = &global_trace;
1014 struct trace_array_cpu *data;
1015 unsigned long flags;
1016 long disabled;
1017 int cpu, resched;
1018 int pc;
1019
1020 if (unlikely(!ftrace_function_enabled))
1021 return;
1022
1023 pc = preempt_count();
1024 resched = ftrace_preempt_disable();
1025 local_save_flags(flags);
1026 cpu = raw_smp_processor_id();
1027 data = tr->data[cpu];
1028 disabled = atomic_inc_return(&data->disabled);
1029
1030 if (likely(disabled == 1))
1031 trace_function(tr, data, ip, parent_ip, flags, pc);
1032
1033 atomic_dec(&data->disabled);
1034 ftrace_preempt_enable(resched);
1035}
1036
1037static void
1038function_trace_call(unsigned long ip, unsigned long parent_ip)
1039{
1040 struct trace_array *tr = &global_trace;
1041 struct trace_array_cpu *data;
1042 unsigned long flags;
1043 long disabled;
1044 int cpu;
1045 int pc;
1046
1047 if (unlikely(!ftrace_function_enabled))
1048 return;
1049
1050 /*
1051 * Need to use raw, since this must be called before the
1052 * recursive protection is performed.
1053 */
1054 local_irq_save(flags);
1055 cpu = raw_smp_processor_id();
1056 data = tr->data[cpu];
1057 disabled = atomic_inc_return(&data->disabled);
1058
1059 if (likely(disabled == 1)) {
1060 pc = preempt_count();
1061 trace_function(tr, data, ip, parent_ip, flags, pc);
1062 }
1063
1064 atomic_dec(&data->disabled);
1065 local_irq_restore(flags);
1066}
1067
1068#ifdef CONFIG_FUNCTION_RET_TRACER
1069void trace_function_return(struct ftrace_retfunc *trace)
1070{
1071 struct trace_array *tr = &global_trace;
1072 struct trace_array_cpu *data;
1073 unsigned long flags;
1074 long disabled;
1075 int cpu;
1076 int pc;
1077
1078 raw_local_irq_save(flags);
1079 cpu = raw_smp_processor_id();
1080 data = tr->data[cpu];
1081 disabled = atomic_inc_return(&data->disabled);
1082 if (likely(disabled == 1)) {
1083 pc = preempt_count();
1084 __trace_function_return(tr, data, trace, flags, pc);
1085 }
1086 atomic_dec(&data->disabled);
1087 raw_local_irq_restore(flags);
1088}
1089#endif /* CONFIG_FUNCTION_RET_TRACER */
1090
1091static struct ftrace_ops trace_ops __read_mostly =
1092{
1093 .func = function_trace_call,
1094};
1095
1096void tracing_start_function_trace(void)
1097{
1098 ftrace_function_enabled = 0;
1099
1100 if (trace_flags & TRACE_ITER_PREEMPTONLY)
1101 trace_ops.func = function_trace_call_preempt_only;
1102 else
1103 trace_ops.func = function_trace_call;
1104
1105 register_ftrace_function(&trace_ops);
1106 ftrace_function_enabled = 1;
1107}
1108
1109void tracing_stop_function_trace(void)
1110{
1111 ftrace_function_enabled = 0;
1112 unregister_ftrace_function(&trace_ops);
1113}
1114#endif
1115
1116enum trace_file_type {
1117 TRACE_FILE_LAT_FMT = 1,
1118 TRACE_FILE_ANNOTATE = 2,
1119};
1120
1121static void trace_iterator_increment(struct trace_iterator *iter, int cpu)
1122{
1123 /* Don't allow ftrace to trace into the ring buffers */
1124 ftrace_disable_cpu();
1125
1126 iter->idx++;
1127 if (iter->buffer_iter[iter->cpu])
1128 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1129
1130 ftrace_enable_cpu();
1131}
1132
1133static struct trace_entry *
1134peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
1135{
1136 struct ring_buffer_event *event;
1137 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1138
1139 /* Don't allow ftrace to trace into the ring buffers */
1140 ftrace_disable_cpu();
1141
1142 if (buf_iter)
1143 event = ring_buffer_iter_peek(buf_iter, ts);
1144 else
1145 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1146
1147 ftrace_enable_cpu();
1148
1149 return event ? ring_buffer_event_data(event) : NULL;
1150}
1151
1152static struct trace_entry *
1153__find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1154{
1155 struct ring_buffer *buffer = iter->tr->buffer;
1156 struct trace_entry *ent, *next = NULL;
1157 u64 next_ts = 0, ts;
1158 int next_cpu = -1;
1159 int cpu;
1160
1161 for_each_tracing_cpu(cpu) {
1162
1163 if (ring_buffer_empty_cpu(buffer, cpu))
1164 continue;
1165
1166 ent = peek_next_entry(iter, cpu, &ts);
1167
1168 /*
1169 * Pick the entry with the smallest timestamp:
1170 */
1171 if (ent && (!next || ts < next_ts)) {
1172 next = ent;
1173 next_cpu = cpu;
1174 next_ts = ts;
1175 }
1176 }
1177
1178 if (ent_cpu)
1179 *ent_cpu = next_cpu;
1180
1181 if (ent_ts)
1182 *ent_ts = next_ts;
1183
1184 return next;
1185}
1186
1187/* Find the next real entry, without updating the iterator itself */
1188static struct trace_entry *
1189find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1190{
1191 return __find_next_entry(iter, ent_cpu, ent_ts);
1192}
1193
1194/* Find the next real entry, and increment the iterator to the next entry */
1195static void *find_next_entry_inc(struct trace_iterator *iter)
1196{
1197 iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1198
1199 if (iter->ent)
1200 trace_iterator_increment(iter, iter->cpu);
1201
1202 return iter->ent ? iter : NULL;
1203}
1204
1205static void trace_consume(struct trace_iterator *iter)
1206{
1207 /* Don't allow ftrace to trace into the ring buffers */
1208 ftrace_disable_cpu();
1209 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1210 ftrace_enable_cpu();
1211}
1212
1213static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1214{
1215 struct trace_iterator *iter = m->private;
1216 int i = (int)*pos;
1217 void *ent;
1218
1219 (*pos)++;
1220
1221 /* can't go backwards */
1222 if (iter->idx > i)
1223 return NULL;
1224
1225 if (iter->idx < 0)
1226 ent = find_next_entry_inc(iter);
1227 else
1228 ent = iter;
1229
1230 while (ent && iter->idx < i)
1231 ent = find_next_entry_inc(iter);
1232
1233 iter->pos = *pos;
1234
1235 return ent;
1236}
1237
1238static void *s_start(struct seq_file *m, loff_t *pos)
1239{
1240 struct trace_iterator *iter = m->private;
1241 void *p = NULL;
1242 loff_t l = 0;
1243 int cpu;
1244
1245 mutex_lock(&trace_types_lock);
1246
1247 if (!current_trace || current_trace != iter->trace) {
1248 mutex_unlock(&trace_types_lock);
1249 return NULL;
1250 }
1251
1252 atomic_inc(&trace_record_cmdline_disabled);
1253
1254 if (*pos != iter->pos) {
1255 iter->ent = NULL;
1256 iter->cpu = 0;
1257 iter->idx = -1;
1258
1259 ftrace_disable_cpu();
1260
1261 for_each_tracing_cpu(cpu) {
1262 ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1263 }
1264
1265 ftrace_enable_cpu();
1266
1267 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1268 ;
1269
1270 } else {
1271 l = *pos - 1;
1272 p = s_next(m, p, &l);
1273 }
1274
1275 return p;
1276}
1277
1278static void s_stop(struct seq_file *m, void *p)
1279{
1280 atomic_dec(&trace_record_cmdline_disabled);
1281 mutex_unlock(&trace_types_lock);
1282}
1283
1284#ifdef CONFIG_KRETPROBES
1285static inline const char *kretprobed(const char *name)
1286{
1287 static const char tramp_name[] = "kretprobe_trampoline";
1288 int size = sizeof(tramp_name);
1289
1290 if (strncmp(tramp_name, name, size) == 0)
1291 return "[unknown/kretprobe'd]";
1292 return name;
1293}
1294#else
1295static inline const char *kretprobed(const char *name)
1296{
1297 return name;
1298}
1299#endif /* CONFIG_KRETPROBES */
1300
1301static int
1302seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1303{
1304#ifdef CONFIG_KALLSYMS
1305 char str[KSYM_SYMBOL_LEN];
1306 const char *name;
1307
1308 kallsyms_lookup(address, NULL, NULL, NULL, str);
1309
1310 name = kretprobed(str);
1311
1312 return trace_seq_printf(s, fmt, name);
1313#endif
1314 return 1;
1315}
1316
1317static int
1318seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1319 unsigned long address)
1320{
1321#ifdef CONFIG_KALLSYMS
1322 char str[KSYM_SYMBOL_LEN];
1323 const char *name;
1324
1325 sprint_symbol(str, address);
1326 name = kretprobed(str);
1327
1328 return trace_seq_printf(s, fmt, name);
1329#endif
1330 return 1;
1331}
1332
1333#ifndef CONFIG_64BIT
1334# define IP_FMT "%08lx"
1335#else
1336# define IP_FMT "%016lx"
1337#endif
1338
1339int
1340seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1341{
1342 int ret;
1343
1344 if (!ip)
1345 return trace_seq_printf(s, "0");
1346
1347 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1348 ret = seq_print_sym_offset(s, "%s", ip);
1349 else
1350 ret = seq_print_sym_short(s, "%s", ip);
1351
1352 if (!ret)
1353 return 0;
1354
1355 if (sym_flags & TRACE_ITER_SYM_ADDR)
1356 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1357 return ret;
1358}
1359
1360static void print_lat_help_header(struct seq_file *m)
1361{
1362 seq_puts(m, "# _------=> CPU# \n");
1363 seq_puts(m, "# / _-----=> irqs-off \n");
1364 seq_puts(m, "# | / _----=> need-resched \n");
1365 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1366 seq_puts(m, "# ||| / _--=> preempt-depth \n");
1367 seq_puts(m, "# |||| / \n");
1368 seq_puts(m, "# ||||| delay \n");
1369 seq_puts(m, "# cmd pid ||||| time | caller \n");
1370 seq_puts(m, "# \\ / ||||| \\ | / \n");
1371}
1372
1373static void print_func_help_header(struct seq_file *m)
1374{
1375 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1376 seq_puts(m, "# | | | | |\n");
1377}
1378
1379
1380static void
1381print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1382{
1383 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1384 struct trace_array *tr = iter->tr;
1385 struct trace_array_cpu *data = tr->data[tr->cpu];
1386 struct tracer *type = current_trace;
1387 unsigned long total;
1388 unsigned long entries;
1389 const char *name = "preemption";
1390
1391 if (type)
1392 name = type->name;
1393
1394 entries = ring_buffer_entries(iter->tr->buffer);
1395 total = entries +
1396 ring_buffer_overruns(iter->tr->buffer);
1397
1398 seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1399 name, UTS_RELEASE);
1400 seq_puts(m, "-----------------------------------"
1401 "---------------------------------\n");
1402 seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1403 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1404 nsecs_to_usecs(data->saved_latency),
1405 entries,
1406 total,
1407 tr->cpu,
1408#if defined(CONFIG_PREEMPT_NONE)
1409 "server",
1410#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1411 "desktop",
1412#elif defined(CONFIG_PREEMPT)
1413 "preempt",
1414#else
1415 "unknown",
1416#endif
1417 /* These are reserved for later use */
1418 0, 0, 0, 0);
1419#ifdef CONFIG_SMP
1420 seq_printf(m, " #P:%d)\n", num_online_cpus());
1421#else
1422 seq_puts(m, ")\n");
1423#endif
1424 seq_puts(m, " -----------------\n");
1425 seq_printf(m, " | task: %.16s-%d "
1426 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1427 data->comm, data->pid, data->uid, data->nice,
1428 data->policy, data->rt_priority);
1429 seq_puts(m, " -----------------\n");
1430
1431 if (data->critical_start) {
1432 seq_puts(m, " => started at: ");
1433 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1434 trace_print_seq(m, &iter->seq);
1435 seq_puts(m, "\n => ended at: ");
1436 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1437 trace_print_seq(m, &iter->seq);
1438 seq_puts(m, "\n");
1439 }
1440
1441 seq_puts(m, "\n");
1442}
1443
1444static void
1445lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1446{
1447 int hardirq, softirq;
1448 char *comm;
1449
1450 comm = trace_find_cmdline(entry->pid);
1451
1452 trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1453 trace_seq_printf(s, "%3d", cpu);
1454 trace_seq_printf(s, "%c%c",
1455 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
1456 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.',
1457 ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1458
1459 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1460 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1461 if (hardirq && softirq) {
1462 trace_seq_putc(s, 'H');
1463 } else {
1464 if (hardirq) {
1465 trace_seq_putc(s, 'h');
1466 } else {
1467 if (softirq)
1468 trace_seq_putc(s, 's');
1469 else
1470 trace_seq_putc(s, '.');
1471 }
1472 }
1473
1474 if (entry->preempt_count)
1475 trace_seq_printf(s, "%x", entry->preempt_count);
1476 else
1477 trace_seq_puts(s, ".");
1478}
1479
1480unsigned long preempt_mark_thresh = 100;
1481
1482static void
1483lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1484 unsigned long rel_usecs)
1485{
1486 trace_seq_printf(s, " %4lldus", abs_usecs);
1487 if (rel_usecs > preempt_mark_thresh)
1488 trace_seq_puts(s, "!: ");
1489 else if (rel_usecs > 1)
1490 trace_seq_puts(s, "+: ");
1491 else
1492 trace_seq_puts(s, " : ");
1493}
1494
1495static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1496
1497/*
1498 * The message is supposed to contain an ending newline.
1499 * If the printing stops prematurely, try to add a newline of our own.
1500 */
1501void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1502{
1503 struct trace_entry *ent;
1504 struct trace_field_cont *cont;
1505 bool ok = true;
1506
1507 ent = peek_next_entry(iter, iter->cpu, NULL);
1508 if (!ent || ent->type != TRACE_CONT) {
1509 trace_seq_putc(s, '\n');
1510 return;
1511 }
1512
1513 do {
1514 cont = (struct trace_field_cont *)ent;
1515 if (ok)
1516 ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1517
1518 ftrace_disable_cpu();
1519
1520 if (iter->buffer_iter[iter->cpu])
1521 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1522 else
1523 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1524
1525 ftrace_enable_cpu();
1526
1527 ent = peek_next_entry(iter, iter->cpu, NULL);
1528 } while (ent && ent->type == TRACE_CONT);
1529
1530 if (!ok)
1531 trace_seq_putc(s, '\n');
1532}
1533
1534static void test_cpu_buff_start(struct trace_iterator *iter)
1535{
1536 struct trace_seq *s = &iter->seq;
1537
1538 if (!(trace_flags & TRACE_ITER_ANNOTATE))
1539 return;
1540
1541 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1542 return;
1543
1544 if (cpu_isset(iter->cpu, iter->started))
1545 return;
1546
1547 cpu_set(iter->cpu, iter->started);
1548 trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu);
1549}
1550
1551static enum print_line_t
1552print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1553{
1554 struct trace_seq *s = &iter->seq;
1555 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1556 struct trace_entry *next_entry;
1557 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1558 struct trace_entry *entry = iter->ent;
1559 unsigned long abs_usecs;
1560 unsigned long rel_usecs;
1561 u64 next_ts;
1562 char *comm;
1563 int S, T;
1564 int i;
1565 unsigned state;
1566
1567 if (entry->type == TRACE_CONT)
1568 return TRACE_TYPE_HANDLED;
1569
1570 test_cpu_buff_start(iter);
1571
1572 next_entry = find_next_entry(iter, NULL, &next_ts);
1573 if (!next_entry)
1574 next_ts = iter->ts;
1575 rel_usecs = ns2usecs(next_ts - iter->ts);
1576 abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1577
1578 if (verbose) {
1579 comm = trace_find_cmdline(entry->pid);
1580 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1581 " %ld.%03ldms (+%ld.%03ldms): ",
1582 comm,
1583 entry->pid, cpu, entry->flags,
1584 entry->preempt_count, trace_idx,
1585 ns2usecs(iter->ts),
1586 abs_usecs/1000,
1587 abs_usecs % 1000, rel_usecs/1000,
1588 rel_usecs % 1000);
1589 } else {
1590 lat_print_generic(s, entry, cpu);
1591 lat_print_timestamp(s, abs_usecs, rel_usecs);
1592 }
1593 switch (entry->type) {
1594 case TRACE_FN: {
1595 struct ftrace_entry *field;
1596
1597 trace_assign_type(field, entry);
1598
1599 seq_print_ip_sym(s, field->ip, sym_flags);
1600 trace_seq_puts(s, " (");
1601 seq_print_ip_sym(s, field->parent_ip, sym_flags);
1602 trace_seq_puts(s, ")\n");
1603 break;
1604 }
1605 case TRACE_CTX:
1606 case TRACE_WAKE: {
1607 struct ctx_switch_entry *field;
1608
1609 trace_assign_type(field, entry);
1610
1611 T = field->next_state < sizeof(state_to_char) ?
1612 state_to_char[field->next_state] : 'X';
1613
1614 state = field->prev_state ?
1615 __ffs(field->prev_state) + 1 : 0;
1616 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1617 comm = trace_find_cmdline(field->next_pid);
1618 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1619 field->prev_pid,
1620 field->prev_prio,
1621 S, entry->type == TRACE_CTX ? "==>" : " +",
1622 field->next_cpu,
1623 field->next_pid,
1624 field->next_prio,
1625 T, comm);
1626 break;
1627 }
1628 case TRACE_SPECIAL: {
1629 struct special_entry *field;
1630
1631 trace_assign_type(field, entry);
1632
1633 trace_seq_printf(s, "# %ld %ld %ld\n",
1634 field->arg1,
1635 field->arg2,
1636 field->arg3);
1637 break;
1638 }
1639 case TRACE_STACK: {
1640 struct stack_entry *field;
1641
1642 trace_assign_type(field, entry);
1643
1644 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1645 if (i)
1646 trace_seq_puts(s, " <= ");
1647 seq_print_ip_sym(s, field->caller[i], sym_flags);
1648 }
1649 trace_seq_puts(s, "\n");
1650 break;
1651 }
1652 case TRACE_PRINT: {
1653 struct print_entry *field;
1654
1655 trace_assign_type(field, entry);
1656
1657 seq_print_ip_sym(s, field->ip, sym_flags);
1658 trace_seq_printf(s, ": %s", field->buf);
1659 if (entry->flags & TRACE_FLAG_CONT)
1660 trace_seq_print_cont(s, iter);
1661 break;
1662 }
1663 case TRACE_BRANCH: {
1664 struct trace_branch *field;
1665
1666 trace_assign_type(field, entry);
1667
1668 trace_seq_printf(s, "[%s] %s:%s:%d\n",
1669 field->correct ? " ok " : " MISS ",
1670 field->func,
1671 field->file,
1672 field->line);
1673 break;
1674 }
1675 default:
1676 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1677 }
1678 return TRACE_TYPE_HANDLED;
1679}
1680
1681static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1682{
1683 struct trace_seq *s = &iter->seq;
1684 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1685 struct trace_entry *entry;
1686 unsigned long usec_rem;
1687 unsigned long long t;
1688 unsigned long secs;
1689 char *comm;
1690 int ret;
1691 int S, T;
1692 int i;
1693
1694 entry = iter->ent;
1695
1696 if (entry->type == TRACE_CONT)
1697 return TRACE_TYPE_HANDLED;
1698
1699 test_cpu_buff_start(iter);
1700
1701 comm = trace_find_cmdline(iter->ent->pid);
1702
1703 t = ns2usecs(iter->ts);
1704 usec_rem = do_div(t, 1000000ULL);
1705 secs = (unsigned long)t;
1706
1707 ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1708 if (!ret)
1709 return TRACE_TYPE_PARTIAL_LINE;
1710 ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1711 if (!ret)
1712 return TRACE_TYPE_PARTIAL_LINE;
1713 ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1714 if (!ret)
1715 return TRACE_TYPE_PARTIAL_LINE;
1716
1717 switch (entry->type) {
1718 case TRACE_FN: {
1719 struct ftrace_entry *field;
1720
1721 trace_assign_type(field, entry);
1722
1723 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1724 if (!ret)
1725 return TRACE_TYPE_PARTIAL_LINE;
1726 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1727 field->parent_ip) {
1728 ret = trace_seq_printf(s, " <-");
1729 if (!ret)
1730 return TRACE_TYPE_PARTIAL_LINE;
1731 ret = seq_print_ip_sym(s,
1732 field->parent_ip,
1733 sym_flags);
1734 if (!ret)
1735 return TRACE_TYPE_PARTIAL_LINE;
1736 }
1737 ret = trace_seq_printf(s, "\n");
1738 if (!ret)
1739 return TRACE_TYPE_PARTIAL_LINE;
1740 break;
1741 }
1742 case TRACE_CTX:
1743 case TRACE_WAKE: {
1744 struct ctx_switch_entry *field;
1745
1746 trace_assign_type(field, entry);
1747
1748 S = field->prev_state < sizeof(state_to_char) ?
1749 state_to_char[field->prev_state] : 'X';
1750 T = field->next_state < sizeof(state_to_char) ?
1751 state_to_char[field->next_state] : 'X';
1752 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
1753 field->prev_pid,
1754 field->prev_prio,
1755 S,
1756 entry->type == TRACE_CTX ? "==>" : " +",
1757 field->next_cpu,
1758 field->next_pid,
1759 field->next_prio,
1760 T);
1761 if (!ret)
1762 return TRACE_TYPE_PARTIAL_LINE;
1763 break;
1764 }
1765 case TRACE_SPECIAL: {
1766 struct special_entry *field;
1767
1768 trace_assign_type(field, entry);
1769
1770 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1771 field->arg1,
1772 field->arg2,
1773 field->arg3);
1774 if (!ret)
1775 return TRACE_TYPE_PARTIAL_LINE;
1776 break;
1777 }
1778 case TRACE_STACK: {
1779 struct stack_entry *field;
1780
1781 trace_assign_type(field, entry);
1782
1783 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1784 if (i) {
1785 ret = trace_seq_puts(s, " <= ");
1786 if (!ret)
1787 return TRACE_TYPE_PARTIAL_LINE;
1788 }
1789 ret = seq_print_ip_sym(s, field->caller[i],
1790 sym_flags);
1791 if (!ret)
1792 return TRACE_TYPE_PARTIAL_LINE;
1793 }
1794 ret = trace_seq_puts(s, "\n");
1795 if (!ret)
1796 return TRACE_TYPE_PARTIAL_LINE;
1797 break;
1798 }
1799 case TRACE_PRINT: {
1800 struct print_entry *field;
1801
1802 trace_assign_type(field, entry);
1803
1804 seq_print_ip_sym(s, field->ip, sym_flags);
1805 trace_seq_printf(s, ": %s", field->buf);
1806 if (entry->flags & TRACE_FLAG_CONT)
1807 trace_seq_print_cont(s, iter);
1808 break;
1809 }
1810 case TRACE_FN_RET: {
1811 return print_return_function(iter);
1812 break;
1813 }
1814 case TRACE_BRANCH: {
1815 struct trace_branch *field;
1816
1817 trace_assign_type(field, entry);
1818
1819 trace_seq_printf(s, "[%s] %s:%s:%d\n",
1820 field->correct ? " ok " : " MISS ",
1821 field->func,
1822 field->file,
1823 field->line);
1824 break;
1825 }
1826 }
1827 return TRACE_TYPE_HANDLED;
1828}
1829
1830static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1831{
1832 struct trace_seq *s = &iter->seq;
1833 struct trace_entry *entry;
1834 int ret;
1835 int S, T;
1836
1837 entry = iter->ent;
1838
1839 if (entry->type == TRACE_CONT)
1840 return TRACE_TYPE_HANDLED;
1841
1842 ret = trace_seq_printf(s, "%d %d %llu ",
1843 entry->pid, iter->cpu, iter->ts);
1844 if (!ret)
1845 return TRACE_TYPE_PARTIAL_LINE;
1846
1847 switch (entry->type) {
1848 case TRACE_FN: {
1849 struct ftrace_entry *field;
1850
1851 trace_assign_type(field, entry);
1852
1853 ret = trace_seq_printf(s, "%x %x\n",
1854 field->ip,
1855 field->parent_ip);
1856 if (!ret)
1857 return TRACE_TYPE_PARTIAL_LINE;
1858 break;
1859 }
1860 case TRACE_CTX:
1861 case TRACE_WAKE: {
1862 struct ctx_switch_entry *field;
1863
1864 trace_assign_type(field, entry);
1865
1866 S = field->prev_state < sizeof(state_to_char) ?
1867 state_to_char[field->prev_state] : 'X';
1868 T = field->next_state < sizeof(state_to_char) ?
1869 state_to_char[field->next_state] : 'X';
1870 if (entry->type == TRACE_WAKE)
1871 S = '+';
1872 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
1873 field->prev_pid,
1874 field->prev_prio,
1875 S,
1876 field->next_cpu,
1877 field->next_pid,
1878 field->next_prio,
1879 T);
1880 if (!ret)
1881 return TRACE_TYPE_PARTIAL_LINE;
1882 break;
1883 }
1884 case TRACE_SPECIAL:
1885 case TRACE_STACK: {
1886 struct special_entry *field;
1887
1888 trace_assign_type(field, entry);
1889
1890 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1891 field->arg1,
1892 field->arg2,
1893 field->arg3);
1894 if (!ret)
1895 return TRACE_TYPE_PARTIAL_LINE;
1896 break;
1897 }
1898 case TRACE_PRINT: {
1899 struct print_entry *field;
1900
1901 trace_assign_type(field, entry);
1902
1903 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
1904 if (entry->flags & TRACE_FLAG_CONT)
1905 trace_seq_print_cont(s, iter);
1906 break;
1907 }
1908 }
1909 return TRACE_TYPE_HANDLED;
1910}
1911
1912#define SEQ_PUT_FIELD_RET(s, x) \
1913do { \
1914 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
1915 return 0; \
1916} while (0)
1917
1918#define SEQ_PUT_HEX_FIELD_RET(s, x) \
1919do { \
1920 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
1921 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
1922 return 0; \
1923} while (0)
1924
1925static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1926{
1927 struct trace_seq *s = &iter->seq;
1928 unsigned char newline = '\n';
1929 struct trace_entry *entry;
1930 int S, T;
1931
1932 entry = iter->ent;
1933
1934 if (entry->type == TRACE_CONT)
1935 return TRACE_TYPE_HANDLED;
1936
1937 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1938 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1939 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1940
1941 switch (entry->type) {
1942 case TRACE_FN: {
1943 struct ftrace_entry *field;
1944
1945 trace_assign_type(field, entry);
1946
1947 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
1948 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
1949 break;
1950 }
1951 case TRACE_CTX:
1952 case TRACE_WAKE: {
1953 struct ctx_switch_entry *field;
1954
1955 trace_assign_type(field, entry);
1956
1957 S = field->prev_state < sizeof(state_to_char) ?
1958 state_to_char[field->prev_state] : 'X';
1959 T = field->next_state < sizeof(state_to_char) ?
1960 state_to_char[field->next_state] : 'X';
1961 if (entry->type == TRACE_WAKE)
1962 S = '+';
1963 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
1964 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
1965 SEQ_PUT_HEX_FIELD_RET(s, S);
1966 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
1967 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
1968 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
1969 SEQ_PUT_HEX_FIELD_RET(s, T);
1970 break;
1971 }
1972 case TRACE_SPECIAL:
1973 case TRACE_STACK: {
1974 struct special_entry *field;
1975
1976 trace_assign_type(field, entry);
1977
1978 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
1979 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
1980 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
1981 break;
1982 }
1983 }
1984 SEQ_PUT_FIELD_RET(s, newline);
1985
1986 return TRACE_TYPE_HANDLED;
1987}
1988
1989static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1990{
1991 struct trace_seq *s = &iter->seq;
1992 struct trace_entry *entry;
1993
1994 entry = iter->ent;
1995
1996 if (entry->type == TRACE_CONT)
1997 return TRACE_TYPE_HANDLED;
1998
1999 SEQ_PUT_FIELD_RET(s, entry->pid);
2000 SEQ_PUT_FIELD_RET(s, entry->cpu);
2001 SEQ_PUT_FIELD_RET(s, iter->ts);
2002
2003 switch (entry->type) {
2004 case TRACE_FN: {
2005 struct ftrace_entry *field;
2006
2007 trace_assign_type(field, entry);
2008
2009 SEQ_PUT_FIELD_RET(s, field->ip);
2010 SEQ_PUT_FIELD_RET(s, field->parent_ip);
2011 break;
2012 }
2013 case TRACE_CTX: {
2014 struct ctx_switch_entry *field;
2015
2016 trace_assign_type(field, entry);
2017
2018 SEQ_PUT_FIELD_RET(s, field->prev_pid);
2019 SEQ_PUT_FIELD_RET(s, field->prev_prio);
2020 SEQ_PUT_FIELD_RET(s, field->prev_state);
2021 SEQ_PUT_FIELD_RET(s, field->next_pid);
2022 SEQ_PUT_FIELD_RET(s, field->next_prio);
2023 SEQ_PUT_FIELD_RET(s, field->next_state);
2024 break;
2025 }
2026 case TRACE_SPECIAL:
2027 case TRACE_STACK: {
2028 struct special_entry *field;
2029
2030 trace_assign_type(field, entry);
2031
2032 SEQ_PUT_FIELD_RET(s, field->arg1);
2033 SEQ_PUT_FIELD_RET(s, field->arg2);
2034 SEQ_PUT_FIELD_RET(s, field->arg3);
2035 break;
2036 }
2037 }
2038 return 1;
2039}
2040
2041static int trace_empty(struct trace_iterator *iter)
2042{
2043 int cpu;
2044
2045 for_each_tracing_cpu(cpu) {
2046 if (iter->buffer_iter[cpu]) {
2047 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2048 return 0;
2049 } else {
2050 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2051 return 0;
2052 }
2053 }
2054
2055 return 1;
2056}
2057
2058static enum print_line_t print_trace_line(struct trace_iterator *iter)
2059{
2060 enum print_line_t ret;
2061
2062 if (iter->trace && iter->trace->print_line) {
2063 ret = iter->trace->print_line(iter);
2064 if (ret != TRACE_TYPE_UNHANDLED)
2065 return ret;
2066 }
2067
2068 if (trace_flags & TRACE_ITER_BIN)
2069 return print_bin_fmt(iter);
2070
2071 if (trace_flags & TRACE_ITER_HEX)
2072 return print_hex_fmt(iter);
2073
2074 if (trace_flags & TRACE_ITER_RAW)
2075 return print_raw_fmt(iter);
2076
2077 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2078 return print_lat_fmt(iter, iter->idx, iter->cpu);
2079
2080 return print_trace_fmt(iter);
2081}
2082
2083static int s_show(struct seq_file *m, void *v)
2084{
2085 struct trace_iterator *iter = v;
2086
2087 if (iter->ent == NULL) {
2088 if (iter->tr) {
2089 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2090 seq_puts(m, "#\n");
2091 }
2092 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2093 /* print nothing if the buffers are empty */
2094 if (trace_empty(iter))
2095 return 0;
2096 print_trace_header(m, iter);
2097 if (!(trace_flags & TRACE_ITER_VERBOSE))
2098 print_lat_help_header(m);
2099 } else {
2100 if (!(trace_flags & TRACE_ITER_VERBOSE))
2101 print_func_help_header(m);
2102 }
2103 } else {
2104 print_trace_line(iter);
2105 trace_print_seq(m, &iter->seq);
2106 }
2107
2108 return 0;
2109}
2110
2111static struct seq_operations tracer_seq_ops = {
2112 .start = s_start,
2113 .next = s_next,
2114 .stop = s_stop,
2115 .show = s_show,
2116};
2117
2118static struct trace_iterator *
2119__tracing_open(struct inode *inode, struct file *file, int *ret)
2120{
2121 struct trace_iterator *iter;
2122 struct seq_file *m;
2123 int cpu;
2124
2125 if (tracing_disabled) {
2126 *ret = -ENODEV;
2127 return NULL;
2128 }
2129
2130 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2131 if (!iter) {
2132 *ret = -ENOMEM;
2133 goto out;
2134 }
2135
2136 mutex_lock(&trace_types_lock);
2137 if (current_trace && current_trace->print_max)
2138 iter->tr = &max_tr;
2139 else
2140 iter->tr = inode->i_private;
2141 iter->trace = current_trace;
2142 iter->pos = -1;
2143
2144 /* Annotate start of buffers if we had overruns */
2145 if (ring_buffer_overruns(iter->tr->buffer))
2146 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2147
2148
2149 for_each_tracing_cpu(cpu) {
2150
2151 iter->buffer_iter[cpu] =
2152 ring_buffer_read_start(iter->tr->buffer, cpu);
2153
2154 if (!iter->buffer_iter[cpu])
2155 goto fail_buffer;
2156 }
2157
2158 /* TODO stop tracer */
2159 *ret = seq_open(file, &tracer_seq_ops);
2160 if (*ret)
2161 goto fail_buffer;
2162
2163 m = file->private_data;
2164 m->private = iter;
2165
2166 /* stop the trace while dumping */
2167 tracing_stop();
2168
2169 if (iter->trace && iter->trace->open)
2170 iter->trace->open(iter);
2171
2172 mutex_unlock(&trace_types_lock);
2173
2174 out:
2175 return iter;
2176
2177 fail_buffer:
2178 for_each_tracing_cpu(cpu) {
2179 if (iter->buffer_iter[cpu])
2180 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2181 }
2182 mutex_unlock(&trace_types_lock);
2183
2184 return ERR_PTR(-ENOMEM);
2185}
2186
2187int tracing_open_generic(struct inode *inode, struct file *filp)
2188{
2189 if (tracing_disabled)
2190 return -ENODEV;
2191
2192 filp->private_data = inode->i_private;
2193 return 0;
2194}
2195
2196int tracing_release(struct inode *inode, struct file *file)
2197{
2198 struct seq_file *m = (struct seq_file *)file->private_data;
2199 struct trace_iterator *iter = m->private;
2200 int cpu;
2201
2202 mutex_lock(&trace_types_lock);
2203 for_each_tracing_cpu(cpu) {
2204 if (iter->buffer_iter[cpu])
2205 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2206 }
2207
2208 if (iter->trace && iter->trace->close)
2209 iter->trace->close(iter);
2210
2211 /* reenable tracing if it was previously enabled */
2212 tracing_start();
2213 mutex_unlock(&trace_types_lock);
2214
2215 seq_release(inode, file);
2216 kfree(iter);
2217 return 0;
2218}
2219
2220static int tracing_open(struct inode *inode, struct file *file)
2221{
2222 int ret;
2223
2224 __tracing_open(inode, file, &ret);
2225
2226 return ret;
2227}
2228
2229static int tracing_lt_open(struct inode *inode, struct file *file)
2230{
2231 struct trace_iterator *iter;
2232 int ret;
2233
2234 iter = __tracing_open(inode, file, &ret);
2235
2236 if (!ret)
2237 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2238
2239 return ret;
2240}
2241
2242
2243static void *
2244t_next(struct seq_file *m, void *v, loff_t *pos)
2245{
2246 struct tracer *t = m->private;
2247
2248 (*pos)++;
2249
2250 if (t)
2251 t = t->next;
2252
2253 m->private = t;
2254
2255 return t;
2256}
2257
2258static void *t_start(struct seq_file *m, loff_t *pos)
2259{
2260 struct tracer *t = m->private;
2261 loff_t l = 0;
2262
2263 mutex_lock(&trace_types_lock);
2264 for (; t && l < *pos; t = t_next(m, t, &l))
2265 ;
2266
2267 return t;
2268}
2269
2270static void t_stop(struct seq_file *m, void *p)
2271{
2272 mutex_unlock(&trace_types_lock);
2273}
2274
2275static int t_show(struct seq_file *m, void *v)
2276{
2277 struct tracer *t = v;
2278
2279 if (!t)
2280 return 0;
2281
2282 seq_printf(m, "%s", t->name);
2283 if (t->next)
2284 seq_putc(m, ' ');
2285 else
2286 seq_putc(m, '\n');
2287
2288 return 0;
2289}
2290
2291static struct seq_operations show_traces_seq_ops = {
2292 .start = t_start,
2293 .next = t_next,
2294 .stop = t_stop,
2295 .show = t_show,
2296};
2297
2298static int show_traces_open(struct inode *inode, struct file *file)
2299{
2300 int ret;
2301
2302 if (tracing_disabled)
2303 return -ENODEV;
2304
2305 ret = seq_open(file, &show_traces_seq_ops);
2306 if (!ret) {
2307 struct seq_file *m = file->private_data;
2308 m->private = trace_types;
2309 }
2310
2311 return ret;
2312}
2313
2314static struct file_operations tracing_fops = {
2315 .open = tracing_open,
2316 .read = seq_read,
2317 .llseek = seq_lseek,
2318 .release = tracing_release,
2319};
2320
2321static struct file_operations tracing_lt_fops = {
2322 .open = tracing_lt_open,
2323 .read = seq_read,
2324 .llseek = seq_lseek,
2325 .release = tracing_release,
2326};
2327
2328static struct file_operations show_traces_fops = {
2329 .open = show_traces_open,
2330 .read = seq_read,
2331 .release = seq_release,
2332};
2333
2334/*
2335 * Only trace on a CPU if the bitmask is set:
2336 */
2337static cpumask_t tracing_cpumask = CPU_MASK_ALL;
2338
2339/*
2340 * When tracing/tracing_cpu_mask is modified then this holds
2341 * the new bitmask we are about to install:
2342 */
2343static cpumask_t tracing_cpumask_new;
2344
2345/*
2346 * The tracer itself will not take this lock, but still we want
2347 * to provide a consistent cpumask to user-space:
2348 */
2349static DEFINE_MUTEX(tracing_cpumask_update_lock);
2350
2351/*
2352 * Temporary storage for the character representation of the
2353 * CPU bitmask (and one more byte for the newline):
2354 */
2355static char mask_str[NR_CPUS + 1];
2356
2357static ssize_t
2358tracing_cpumask_read(struct file *filp, char __user *ubuf,
2359 size_t count, loff_t *ppos)
2360{
2361 int len;
2362
2363 mutex_lock(&tracing_cpumask_update_lock);
2364
2365 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2366 if (count - len < 2) {
2367 count = -EINVAL;
2368 goto out_err;
2369 }
2370 len += sprintf(mask_str + len, "\n");
2371 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2372
2373out_err:
2374 mutex_unlock(&tracing_cpumask_update_lock);
2375
2376 return count;
2377}
2378
2379static ssize_t
2380tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2381 size_t count, loff_t *ppos)
2382{
2383 int err, cpu;
2384
2385 mutex_lock(&tracing_cpumask_update_lock);
2386 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2387 if (err)
2388 goto err_unlock;
2389
2390 raw_local_irq_disable();
2391 __raw_spin_lock(&ftrace_max_lock);
2392 for_each_tracing_cpu(cpu) {
2393 /*
2394 * Increase/decrease the disabled counter if we are
2395 * about to flip a bit in the cpumask:
2396 */
2397 if (cpu_isset(cpu, tracing_cpumask) &&
2398 !cpu_isset(cpu, tracing_cpumask_new)) {
2399 atomic_inc(&global_trace.data[cpu]->disabled);
2400 }
2401 if (!cpu_isset(cpu, tracing_cpumask) &&
2402 cpu_isset(cpu, tracing_cpumask_new)) {
2403 atomic_dec(&global_trace.data[cpu]->disabled);
2404 }
2405 }
2406 __raw_spin_unlock(&ftrace_max_lock);
2407 raw_local_irq_enable();
2408
2409 tracing_cpumask = tracing_cpumask_new;
2410
2411 mutex_unlock(&tracing_cpumask_update_lock);
2412
2413 return count;
2414
2415err_unlock:
2416 mutex_unlock(&tracing_cpumask_update_lock);
2417
2418 return err;
2419}
2420
2421static struct file_operations tracing_cpumask_fops = {
2422 .open = tracing_open_generic,
2423 .read = tracing_cpumask_read,
2424 .write = tracing_cpumask_write,
2425};
2426
2427static ssize_t
2428tracing_trace_options_read(struct file *filp, char __user *ubuf,
2429 size_t cnt, loff_t *ppos)
2430{
2431 char *buf;
2432 int r = 0;
2433 int len = 0;
2434 int i;
2435
2436 /* calulate max size */
2437 for (i = 0; trace_options[i]; i++) {
2438 len += strlen(trace_options[i]);
2439 len += 3; /* "no" and space */
2440 }
2441
2442 /* +2 for \n and \0 */
2443 buf = kmalloc(len + 2, GFP_KERNEL);
2444 if (!buf)
2445 return -ENOMEM;
2446
2447 for (i = 0; trace_options[i]; i++) {
2448 if (trace_flags & (1 << i))
2449 r += sprintf(buf + r, "%s ", trace_options[i]);
2450 else
2451 r += sprintf(buf + r, "no%s ", trace_options[i]);
2452 }
2453
2454 r += sprintf(buf + r, "\n");
2455 WARN_ON(r >= len + 2);
2456
2457 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2458
2459 kfree(buf);
2460
2461 return r;
2462}
2463
2464static ssize_t
2465tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2466 size_t cnt, loff_t *ppos)
2467{
2468 char buf[64];
2469 char *cmp = buf;
2470 int neg = 0;
2471 int i;
2472
2473 if (cnt >= sizeof(buf))
2474 return -EINVAL;
2475
2476 if (copy_from_user(&buf, ubuf, cnt))
2477 return -EFAULT;
2478
2479 buf[cnt] = 0;
2480
2481 if (strncmp(buf, "no", 2) == 0) {
2482 neg = 1;
2483 cmp += 2;
2484 }
2485
2486 for (i = 0; trace_options[i]; i++) {
2487 int len = strlen(trace_options[i]);
2488
2489 if (strncmp(cmp, trace_options[i], len) == 0) {
2490 if (neg)
2491 trace_flags &= ~(1 << i);
2492 else
2493 trace_flags |= (1 << i);
2494 break;
2495 }
2496 }
2497 /*
2498 * If no option could be set, return an error:
2499 */
2500 if (!trace_options[i])
2501 return -EINVAL;
2502
2503 filp->f_pos += cnt;
2504
2505 return cnt;
2506}
2507
2508static struct file_operations tracing_iter_fops = {
2509 .open = tracing_open_generic,
2510 .read = tracing_trace_options_read,
2511 .write = tracing_trace_options_write,
2512};
2513
2514static const char readme_msg[] =
2515 "tracing mini-HOWTO:\n\n"
2516 "# mkdir /debug\n"
2517 "# mount -t debugfs nodev /debug\n\n"
2518 "# cat /debug/tracing/available_tracers\n"
2519 "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2520 "# cat /debug/tracing/current_tracer\n"
2521 "none\n"
2522 "# echo sched_switch > /debug/tracing/current_tracer\n"
2523 "# cat /debug/tracing/current_tracer\n"
2524 "sched_switch\n"
2525 "# cat /debug/tracing/trace_options\n"
2526 "noprint-parent nosym-offset nosym-addr noverbose\n"
2527 "# echo print-parent > /debug/tracing/trace_options\n"
2528 "# echo 1 > /debug/tracing/tracing_enabled\n"
2529 "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2530 "echo 0 > /debug/tracing/tracing_enabled\n"
2531;
2532
2533static ssize_t
2534tracing_readme_read(struct file *filp, char __user *ubuf,
2535 size_t cnt, loff_t *ppos)
2536{
2537 return simple_read_from_buffer(ubuf, cnt, ppos,
2538 readme_msg, strlen(readme_msg));
2539}
2540
2541static struct file_operations tracing_readme_fops = {
2542 .open = tracing_open_generic,
2543 .read = tracing_readme_read,
2544};
2545
2546static ssize_t
2547tracing_ctrl_read(struct file *filp, char __user *ubuf,
2548 size_t cnt, loff_t *ppos)
2549{
2550 char buf[64];
2551 int r;
2552
2553 r = sprintf(buf, "%u\n", tracer_enabled);
2554 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2555}
2556
2557static ssize_t
2558tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2559 size_t cnt, loff_t *ppos)
2560{
2561 struct trace_array *tr = filp->private_data;
2562 char buf[64];
2563 long val;
2564 int ret;
2565
2566 if (cnt >= sizeof(buf))
2567 return -EINVAL;
2568
2569 if (copy_from_user(&buf, ubuf, cnt))
2570 return -EFAULT;
2571
2572 buf[cnt] = 0;
2573
2574 ret = strict_strtoul(buf, 10, &val);
2575 if (ret < 0)
2576 return ret;
2577
2578 val = !!val;
2579
2580 mutex_lock(&trace_types_lock);
2581 if (tracer_enabled ^ val) {
2582 if (val) {
2583 tracer_enabled = 1;
2584 if (current_trace->start)
2585 current_trace->start(tr);
2586 tracing_start();
2587 } else {
2588 tracer_enabled = 0;
2589 tracing_stop();
2590 if (current_trace->stop)
2591 current_trace->stop(tr);
2592 }
2593 }
2594 mutex_unlock(&trace_types_lock);
2595
2596 filp->f_pos += cnt;
2597
2598 return cnt;
2599}
2600
2601static ssize_t
2602tracing_set_trace_read(struct file *filp, char __user *ubuf,
2603 size_t cnt, loff_t *ppos)
2604{
2605 char buf[max_tracer_type_len+2];
2606 int r;
2607
2608 mutex_lock(&trace_types_lock);
2609 if (current_trace)
2610 r = sprintf(buf, "%s\n", current_trace->name);
2611 else
2612 r = sprintf(buf, "\n");
2613 mutex_unlock(&trace_types_lock);
2614
2615 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2616}
2617
2618static int tracing_set_tracer(char *buf)
2619{
2620 struct trace_array *tr = &global_trace;
2621 struct tracer *t;
2622 int ret = 0;
2623
2624 mutex_lock(&trace_types_lock);
2625 for (t = trace_types; t; t = t->next) {
2626 if (strcmp(t->name, buf) == 0)
2627 break;
2628 }
2629 if (!t) {
2630 ret = -EINVAL;
2631 goto out;
2632 }
2633 if (t == current_trace)
2634 goto out;
2635
2636 trace_branch_disable();
2637 if (current_trace && current_trace->reset)
2638 current_trace->reset(tr);
2639
2640 current_trace = t;
2641 if (t->init)
2642 t->init(tr);
2643
2644 trace_branch_enable(tr);
2645 out:
2646 mutex_unlock(&trace_types_lock);
2647
2648 return ret;
2649}
2650
2651static ssize_t
2652tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2653 size_t cnt, loff_t *ppos)
2654{
2655 char buf[max_tracer_type_len+1];
2656 int i;
2657 size_t ret;
2658
2659 if (cnt > max_tracer_type_len)
2660 cnt = max_tracer_type_len;
2661
2662 if (copy_from_user(&buf, ubuf, cnt))
2663 return -EFAULT;
2664
2665 buf[cnt] = 0;
2666
2667 /* strip ending whitespace. */
2668 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2669 buf[i] = 0;
2670
2671 ret = tracing_set_tracer(buf);
2672 if (!ret)
2673 ret = cnt;
2674
2675 if (ret > 0)
2676 filp->f_pos += ret;
2677
2678 return ret;
2679}
2680
2681static ssize_t
2682tracing_max_lat_read(struct file *filp, char __user *ubuf,
2683 size_t cnt, loff_t *ppos)
2684{
2685 unsigned long *ptr = filp->private_data;
2686 char buf[64];
2687 int r;
2688
2689 r = snprintf(buf, sizeof(buf), "%ld\n",
2690 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2691 if (r > sizeof(buf))
2692 r = sizeof(buf);
2693 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2694}
2695
2696static ssize_t
2697tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2698 size_t cnt, loff_t *ppos)
2699{
2700 long *ptr = filp->private_data;
2701 char buf[64];
2702 long val;
2703 int ret;
2704
2705 if (cnt >= sizeof(buf))
2706 return -EINVAL;
2707
2708 if (copy_from_user(&buf, ubuf, cnt))
2709 return -EFAULT;
2710
2711 buf[cnt] = 0;
2712
2713 ret = strict_strtoul(buf, 10, &val);
2714 if (ret < 0)
2715 return ret;
2716
2717 *ptr = val * 1000;
2718
2719 return cnt;
2720}
2721
2722static atomic_t tracing_reader;
2723
2724static int tracing_open_pipe(struct inode *inode, struct file *filp)
2725{
2726 struct trace_iterator *iter;
2727
2728 if (tracing_disabled)
2729 return -ENODEV;
2730
2731 /* We only allow for reader of the pipe */
2732 if (atomic_inc_return(&tracing_reader) != 1) {
2733 atomic_dec(&tracing_reader);
2734 return -EBUSY;
2735 }
2736
2737 /* create a buffer to store the information to pass to userspace */
2738 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2739 if (!iter)
2740 return -ENOMEM;
2741
2742 mutex_lock(&trace_types_lock);
2743
2744 /* trace pipe does not show start of buffer */
2745 cpus_setall(iter->started);
2746
2747 iter->tr = &global_trace;
2748 iter->trace = current_trace;
2749 filp->private_data = iter;
2750
2751 if (iter->trace->pipe_open)
2752 iter->trace->pipe_open(iter);
2753 mutex_unlock(&trace_types_lock);
2754
2755 return 0;
2756}
2757
2758static int tracing_release_pipe(struct inode *inode, struct file *file)
2759{
2760 struct trace_iterator *iter = file->private_data;
2761
2762 kfree(iter);
2763 atomic_dec(&tracing_reader);
2764
2765 return 0;
2766}
2767
2768static unsigned int
2769tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2770{
2771 struct trace_iterator *iter = filp->private_data;
2772
2773 if (trace_flags & TRACE_ITER_BLOCK) {
2774 /*
2775 * Always select as readable when in blocking mode
2776 */
2777 return POLLIN | POLLRDNORM;
2778 } else {
2779 if (!trace_empty(iter))
2780 return POLLIN | POLLRDNORM;
2781 poll_wait(filp, &trace_wait, poll_table);
2782 if (!trace_empty(iter))
2783 return POLLIN | POLLRDNORM;
2784
2785 return 0;
2786 }
2787}
2788
2789/*
2790 * Consumer reader.
2791 */
2792static ssize_t
2793tracing_read_pipe(struct file *filp, char __user *ubuf,
2794 size_t cnt, loff_t *ppos)
2795{
2796 struct trace_iterator *iter = filp->private_data;
2797 ssize_t sret;
2798
2799 /* return any leftover data */
2800 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2801 if (sret != -EBUSY)
2802 return sret;
2803
2804 trace_seq_reset(&iter->seq);
2805
2806 mutex_lock(&trace_types_lock);
2807 if (iter->trace->read) {
2808 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2809 if (sret)
2810 goto out;
2811 }
2812
2813waitagain:
2814 sret = 0;
2815 while (trace_empty(iter)) {
2816
2817 if ((filp->f_flags & O_NONBLOCK)) {
2818 sret = -EAGAIN;
2819 goto out;
2820 }
2821
2822 /*
2823 * This is a make-shift waitqueue. The reason we don't use
2824 * an actual wait queue is because:
2825 * 1) we only ever have one waiter
2826 * 2) the tracing, traces all functions, we don't want
2827 * the overhead of calling wake_up and friends
2828 * (and tracing them too)
2829 * Anyway, this is really very primitive wakeup.
2830 */
2831 set_current_state(TASK_INTERRUPTIBLE);
2832 iter->tr->waiter = current;
2833
2834 mutex_unlock(&trace_types_lock);
2835
2836 /* sleep for 100 msecs, and try again. */
2837 schedule_timeout(HZ/10);
2838
2839 mutex_lock(&trace_types_lock);
2840
2841 iter->tr->waiter = NULL;
2842
2843 if (signal_pending(current)) {
2844 sret = -EINTR;
2845 goto out;
2846 }
2847
2848 if (iter->trace != current_trace)
2849 goto out;
2850
2851 /*
2852 * We block until we read something and tracing is disabled.
2853 * We still block if tracing is disabled, but we have never
2854 * read anything. This allows a user to cat this file, and
2855 * then enable tracing. But after we have read something,
2856 * we give an EOF when tracing is again disabled.
2857 *
2858 * iter->pos will be 0 if we haven't read anything.
2859 */
2860 if (!tracer_enabled && iter->pos)
2861 break;
2862
2863 continue;
2864 }
2865
2866 /* stop when tracing is finished */
2867 if (trace_empty(iter))
2868 goto out;
2869
2870 if (cnt >= PAGE_SIZE)
2871 cnt = PAGE_SIZE - 1;
2872
2873 /* reset all but tr, trace, and overruns */
2874 memset(&iter->seq, 0,
2875 sizeof(struct trace_iterator) -
2876 offsetof(struct trace_iterator, seq));
2877 iter->pos = -1;
2878
2879 while (find_next_entry_inc(iter) != NULL) {
2880 enum print_line_t ret;
2881 int len = iter->seq.len;
2882
2883 ret = print_trace_line(iter);
2884 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2885 /* don't print partial lines */
2886 iter->seq.len = len;
2887 break;
2888 }
2889
2890 trace_consume(iter);
2891
2892 if (iter->seq.len >= cnt)
2893 break;
2894 }
2895
2896 /* Now copy what we have to the user */
2897 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2898 if (iter->seq.readpos >= iter->seq.len)
2899 trace_seq_reset(&iter->seq);
2900
2901 /*
2902 * If there was nothing to send to user, inspite of consuming trace
2903 * entries, go back to wait for more entries.
2904 */
2905 if (sret == -EBUSY)
2906 goto waitagain;
2907
2908out:
2909 mutex_unlock(&trace_types_lock);
2910
2911 return sret;
2912}
2913
2914static ssize_t
2915tracing_entries_read(struct file *filp, char __user *ubuf,
2916 size_t cnt, loff_t *ppos)
2917{
2918 struct trace_array *tr = filp->private_data;
2919 char buf[64];
2920 int r;
2921
2922 r = sprintf(buf, "%lu\n", tr->entries >> 10);
2923 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2924}
2925
2926static ssize_t
2927tracing_entries_write(struct file *filp, const char __user *ubuf,
2928 size_t cnt, loff_t *ppos)
2929{
2930 unsigned long val;
2931 char buf[64];
2932 int ret, cpu;
2933
2934 if (cnt >= sizeof(buf))
2935 return -EINVAL;
2936
2937 if (copy_from_user(&buf, ubuf, cnt))
2938 return -EFAULT;
2939
2940 buf[cnt] = 0;
2941
2942 ret = strict_strtoul(buf, 10, &val);
2943 if (ret < 0)
2944 return ret;
2945
2946 /* must have at least 1 entry */
2947 if (!val)
2948 return -EINVAL;
2949
2950 mutex_lock(&trace_types_lock);
2951
2952 tracing_stop();
2953
2954 /* disable all cpu buffers */
2955 for_each_tracing_cpu(cpu) {
2956 if (global_trace.data[cpu])
2957 atomic_inc(&global_trace.data[cpu]->disabled);
2958 if (max_tr.data[cpu])
2959 atomic_inc(&max_tr.data[cpu]->disabled);
2960 }
2961
2962 /* value is in KB */
2963 val <<= 10;
2964
2965 if (val != global_trace.entries) {
2966 ret = ring_buffer_resize(global_trace.buffer, val);
2967 if (ret < 0) {
2968 cnt = ret;
2969 goto out;
2970 }
2971
2972 ret = ring_buffer_resize(max_tr.buffer, val);
2973 if (ret < 0) {
2974 int r;
2975 cnt = ret;
2976 r = ring_buffer_resize(global_trace.buffer,
2977 global_trace.entries);
2978 if (r < 0) {
2979 /* AARGH! We are left with different
2980 * size max buffer!!!! */
2981 WARN_ON(1);
2982 tracing_disabled = 1;
2983 }
2984 goto out;
2985 }
2986
2987 global_trace.entries = val;
2988 }
2989
2990 filp->f_pos += cnt;
2991
2992 /* If check pages failed, return ENOMEM */
2993 if (tracing_disabled)
2994 cnt = -ENOMEM;
2995 out:
2996 for_each_tracing_cpu(cpu) {
2997 if (global_trace.data[cpu])
2998 atomic_dec(&global_trace.data[cpu]->disabled);
2999 if (max_tr.data[cpu])
3000 atomic_dec(&max_tr.data[cpu]->disabled);
3001 }
3002
3003 tracing_start();
3004 max_tr.entries = global_trace.entries;
3005 mutex_unlock(&trace_types_lock);
3006
3007 return cnt;
3008}
3009
3010static int mark_printk(const char *fmt, ...)
3011{
3012 int ret;
3013 va_list args;
3014 va_start(args, fmt);
3015 ret = trace_vprintk(0, fmt, args);
3016 va_end(args);
3017 return ret;
3018}
3019
3020static ssize_t
3021tracing_mark_write(struct file *filp, const char __user *ubuf,
3022 size_t cnt, loff_t *fpos)
3023{
3024 char *buf;
3025 char *end;
3026
3027 if (tracing_disabled)
3028 return -EINVAL;
3029
3030 if (cnt > TRACE_BUF_SIZE)
3031 cnt = TRACE_BUF_SIZE;
3032
3033 buf = kmalloc(cnt + 1, GFP_KERNEL);
3034 if (buf == NULL)
3035 return -ENOMEM;
3036
3037 if (copy_from_user(buf, ubuf, cnt)) {
3038 kfree(buf);
3039 return -EFAULT;
3040 }
3041
3042 /* Cut from the first nil or newline. */
3043 buf[cnt] = '\0';
3044 end = strchr(buf, '\n');
3045 if (end)
3046 *end = '\0';
3047
3048 cnt = mark_printk("%s\n", buf);
3049 kfree(buf);
3050 *fpos += cnt;
3051
3052 return cnt;
3053}
3054
3055static struct file_operations tracing_max_lat_fops = {
3056 .open = tracing_open_generic,
3057 .read = tracing_max_lat_read,
3058 .write = tracing_max_lat_write,
3059};
3060
3061static struct file_operations tracing_ctrl_fops = {
3062 .open = tracing_open_generic,
3063 .read = tracing_ctrl_read,
3064 .write = tracing_ctrl_write,
3065};
3066
3067static struct file_operations set_tracer_fops = {
3068 .open = tracing_open_generic,
3069 .read = tracing_set_trace_read,
3070 .write = tracing_set_trace_write,
3071};
3072
3073static struct file_operations tracing_pipe_fops = {
3074 .open = tracing_open_pipe,
3075 .poll = tracing_poll_pipe,
3076 .read = tracing_read_pipe,
3077 .release = tracing_release_pipe,
3078};
3079
3080static struct file_operations tracing_entries_fops = {
3081 .open = tracing_open_generic,
3082 .read = tracing_entries_read,
3083 .write = tracing_entries_write,
3084};
3085
3086static struct file_operations tracing_mark_fops = {
3087 .open = tracing_open_generic,
3088 .write = tracing_mark_write,
3089};
3090
3091#ifdef CONFIG_DYNAMIC_FTRACE
3092
3093int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3094{
3095 return 0;
3096}
3097
3098static ssize_t
3099tracing_read_dyn_info(struct file *filp, char __user *ubuf,
3100 size_t cnt, loff_t *ppos)
3101{
3102 static char ftrace_dyn_info_buffer[1024];
3103 static DEFINE_MUTEX(dyn_info_mutex);
3104 unsigned long *p = filp->private_data;
3105 char *buf = ftrace_dyn_info_buffer;
3106 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
3107 int r;
3108
3109 mutex_lock(&dyn_info_mutex);
3110 r = sprintf(buf, "%ld ", *p);
3111
3112 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3113 buf[r++] = '\n';
3114
3115 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3116
3117 mutex_unlock(&dyn_info_mutex);
3118
3119 return r;
3120}
3121
3122static struct file_operations tracing_dyn_info_fops = {
3123 .open = tracing_open_generic,
3124 .read = tracing_read_dyn_info,
3125};
3126#endif
3127
3128static struct dentry *d_tracer;
3129
3130struct dentry *tracing_init_dentry(void)
3131{
3132 static int once;
3133
3134 if (d_tracer)
3135 return d_tracer;
3136
3137 d_tracer = debugfs_create_dir("tracing", NULL);
3138
3139 if (!d_tracer && !once) {
3140 once = 1;
3141 pr_warning("Could not create debugfs directory 'tracing'\n");
3142 return NULL;
3143 }
3144
3145 return d_tracer;
3146}
3147
3148#ifdef CONFIG_FTRACE_SELFTEST
3149/* Let selftest have access to static functions in this file */
3150#include "trace_selftest.c"
3151#endif
3152
3153static __init int tracer_init_debugfs(void)
3154{
3155 struct dentry *d_tracer;
3156 struct dentry *entry;
3157
3158 d_tracer = tracing_init_dentry();
3159
3160 entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
3161 &global_trace, &tracing_ctrl_fops);
3162 if (!entry)
3163 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
3164
3165 entry = debugfs_create_file("trace_options", 0644, d_tracer,
3166 NULL, &tracing_iter_fops);
3167 if (!entry)
3168 pr_warning("Could not create debugfs 'trace_options' entry\n");
3169
3170 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
3171 NULL, &tracing_cpumask_fops);
3172 if (!entry)
3173 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
3174
3175 entry = debugfs_create_file("latency_trace", 0444, d_tracer,
3176 &global_trace, &tracing_lt_fops);
3177 if (!entry)
3178 pr_warning("Could not create debugfs 'latency_trace' entry\n");
3179
3180 entry = debugfs_create_file("trace", 0444, d_tracer,
3181 &global_trace, &tracing_fops);
3182 if (!entry)
3183 pr_warning("Could not create debugfs 'trace' entry\n");
3184
3185 entry = debugfs_create_file("available_tracers", 0444, d_tracer,
3186 &global_trace, &show_traces_fops);
3187 if (!entry)
3188 pr_warning("Could not create debugfs 'available_tracers' entry\n");
3189
3190 entry = debugfs_create_file("current_tracer", 0444, d_tracer,
3191 &global_trace, &set_tracer_fops);
3192 if (!entry)
3193 pr_warning("Could not create debugfs 'current_tracer' entry\n");
3194
3195 entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
3196 &tracing_max_latency,
3197 &tracing_max_lat_fops);
3198 if (!entry)
3199 pr_warning("Could not create debugfs "
3200 "'tracing_max_latency' entry\n");
3201
3202 entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
3203 &tracing_thresh, &tracing_max_lat_fops);
3204 if (!entry)
3205 pr_warning("Could not create debugfs "
3206 "'tracing_thresh' entry\n");
3207 entry = debugfs_create_file("README", 0644, d_tracer,
3208 NULL, &tracing_readme_fops);
3209 if (!entry)
3210 pr_warning("Could not create debugfs 'README' entry\n");
3211
3212 entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
3213 NULL, &tracing_pipe_fops);
3214 if (!entry)
3215 pr_warning("Could not create debugfs "
3216 "'trace_pipe' entry\n");
3217
3218 entry = debugfs_create_file("buffer_size_kb", 0644, d_tracer,
3219 &global_trace, &tracing_entries_fops);
3220 if (!entry)
3221 pr_warning("Could not create debugfs "
3222 "'buffer_size_kb' entry\n");
3223
3224 entry = debugfs_create_file("trace_marker", 0220, d_tracer,
3225 NULL, &tracing_mark_fops);
3226 if (!entry)
3227 pr_warning("Could not create debugfs "
3228 "'trace_marker' entry\n");
3229
3230#ifdef CONFIG_DYNAMIC_FTRACE
3231 entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
3232 &ftrace_update_tot_cnt,
3233 &tracing_dyn_info_fops);
3234 if (!entry)
3235 pr_warning("Could not create debugfs "
3236 "'dyn_ftrace_total_info' entry\n");
3237#endif
3238#ifdef CONFIG_SYSPROF_TRACER
3239 init_tracer_sysprof_debugfs(d_tracer);
3240#endif
3241 return 0;
3242}
3243
3244int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3245{
3246 static DEFINE_SPINLOCK(trace_buf_lock);
3247 static char trace_buf[TRACE_BUF_SIZE];
3248
3249 struct ring_buffer_event *event;
3250 struct trace_array *tr = &global_trace;
3251 struct trace_array_cpu *data;
3252 struct print_entry *entry;
3253 unsigned long flags, irq_flags;
3254 int cpu, len = 0, size, pc;
3255
3256 if (tracing_disabled)
3257 return 0;
3258
3259 pc = preempt_count();
3260 preempt_disable_notrace();
3261 cpu = raw_smp_processor_id();
3262 data = tr->data[cpu];
3263
3264 if (unlikely(atomic_read(&data->disabled)))
3265 goto out;
3266
3267 spin_lock_irqsave(&trace_buf_lock, flags);
3268 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
3269
3270 len = min(len, TRACE_BUF_SIZE-1);
3271 trace_buf[len] = 0;
3272
3273 size = sizeof(*entry) + len + 1;
3274 event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
3275 if (!event)
3276 goto out_unlock;
3277 entry = ring_buffer_event_data(event);
3278 tracing_generic_entry_update(&entry->ent, flags, pc);
3279 entry->ent.type = TRACE_PRINT;
3280 entry->ip = ip;
3281
3282 memcpy(&entry->buf, trace_buf, len);
3283 entry->buf[len] = 0;
3284 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
3285
3286 out_unlock:
3287 spin_unlock_irqrestore(&trace_buf_lock, flags);
3288
3289 out:
3290 preempt_enable_notrace();
3291
3292 return len;
3293}
3294EXPORT_SYMBOL_GPL(trace_vprintk);
3295
3296int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3297{
3298 int ret;
3299 va_list ap;
3300
3301 if (!(trace_flags & TRACE_ITER_PRINTK))
3302 return 0;
3303
3304 va_start(ap, fmt);
3305 ret = trace_vprintk(ip, fmt, ap);
3306 va_end(ap);
3307 return ret;
3308}
3309EXPORT_SYMBOL_GPL(__ftrace_printk);
3310
3311static int trace_panic_handler(struct notifier_block *this,
3312 unsigned long event, void *unused)
3313{
3314 if (ftrace_dump_on_oops)
3315 ftrace_dump();
3316 return NOTIFY_OK;
3317}
3318
3319static struct notifier_block trace_panic_notifier = {
3320 .notifier_call = trace_panic_handler,
3321 .next = NULL,
3322 .priority = 150 /* priority: INT_MAX >= x >= 0 */
3323};
3324
3325static int trace_die_handler(struct notifier_block *self,
3326 unsigned long val,
3327 void *data)
3328{
3329 switch (val) {
3330 case DIE_OOPS:
3331 if (ftrace_dump_on_oops)
3332 ftrace_dump();
3333 break;
3334 default:
3335 break;
3336 }
3337 return NOTIFY_OK;
3338}
3339
3340static struct notifier_block trace_die_notifier = {
3341 .notifier_call = trace_die_handler,
3342 .priority = 200
3343};
3344
3345/*
3346 * printk is set to max of 1024, we really don't need it that big.
3347 * Nothing should be printing 1000 characters anyway.
3348 */
3349#define TRACE_MAX_PRINT 1000
3350
3351/*
3352 * Define here KERN_TRACE so that we have one place to modify
3353 * it if we decide to change what log level the ftrace dump
3354 * should be at.
3355 */
3356#define KERN_TRACE KERN_INFO
3357
3358static void
3359trace_printk_seq(struct trace_seq *s)
3360{
3361 /* Probably should print a warning here. */
3362 if (s->len >= 1000)
3363 s->len = 1000;
3364
3365 /* should be zero ended, but we are paranoid. */
3366 s->buffer[s->len] = 0;
3367
3368 printk(KERN_TRACE "%s", s->buffer);
3369
3370 trace_seq_reset(s);
3371}
3372
3373void ftrace_dump(void)
3374{
3375 static DEFINE_SPINLOCK(ftrace_dump_lock);
3376 /* use static because iter can be a bit big for the stack */
3377 static struct trace_iterator iter;
3378 static cpumask_t mask;
3379 static int dump_ran;
3380 unsigned long flags;
3381 int cnt = 0, cpu;
3382
3383 /* only one dump */
3384 spin_lock_irqsave(&ftrace_dump_lock, flags);
3385 if (dump_ran)
3386 goto out;
3387
3388 dump_ran = 1;
3389
3390 /* No turning back! */
3391 ftrace_kill();
3392
3393 for_each_tracing_cpu(cpu) {
3394 atomic_inc(&global_trace.data[cpu]->disabled);
3395 }
3396
3397 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3398
3399 iter.tr = &global_trace;
3400 iter.trace = current_trace;
3401
3402 /*
3403 * We need to stop all tracing on all CPUS to read the
3404 * the next buffer. This is a bit expensive, but is
3405 * not done often. We fill all what we can read,
3406 * and then release the locks again.
3407 */
3408
3409 cpus_clear(mask);
3410
3411 while (!trace_empty(&iter)) {
3412
3413 if (!cnt)
3414 printk(KERN_TRACE "---------------------------------\n");
3415
3416 cnt++;
3417
3418 /* reset all but tr, trace, and overruns */
3419 memset(&iter.seq, 0,
3420 sizeof(struct trace_iterator) -
3421 offsetof(struct trace_iterator, seq));
3422 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3423 iter.pos = -1;
3424
3425 if (find_next_entry_inc(&iter) != NULL) {
3426 print_trace_line(&iter);
3427 trace_consume(&iter);
3428 }
3429
3430 trace_printk_seq(&iter.seq);
3431 }
3432
3433 if (!cnt)
3434 printk(KERN_TRACE " (ftrace buffer empty)\n");
3435 else
3436 printk(KERN_TRACE "---------------------------------\n");
3437
3438 out:
3439 spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3440}
3441
3442__init static int tracer_alloc_buffers(void)
3443{
3444 struct trace_array_cpu *data;
3445 int i;
3446
3447 /* TODO: make the number of buffers hot pluggable with CPUS */
3448 tracing_buffer_mask = cpu_possible_map;
3449
3450 global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3451 TRACE_BUFFER_FLAGS);
3452 if (!global_trace.buffer) {
3453 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3454 WARN_ON(1);
3455 return 0;
3456 }
3457 global_trace.entries = ring_buffer_size(global_trace.buffer);
3458
3459#ifdef CONFIG_TRACER_MAX_TRACE
3460 max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3461 TRACE_BUFFER_FLAGS);
3462 if (!max_tr.buffer) {
3463 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3464 WARN_ON(1);
3465 ring_buffer_free(global_trace.buffer);
3466 return 0;
3467 }
3468 max_tr.entries = ring_buffer_size(max_tr.buffer);
3469 WARN_ON(max_tr.entries != global_trace.entries);
3470#endif
3471
3472 /* Allocate the first page for all buffers */
3473 for_each_tracing_cpu(i) {
3474 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3475 max_tr.data[i] = &per_cpu(max_data, i);
3476 }
3477
3478 trace_init_cmdlines();
3479
3480 register_tracer(&nop_trace);
3481#ifdef CONFIG_BOOT_TRACER
3482 register_tracer(&boot_tracer);
3483 current_trace = &boot_tracer;
3484 current_trace->init(&global_trace);
3485#else
3486 current_trace = &nop_trace;
3487#endif
3488
3489 /* All seems OK, enable tracing */
3490 tracing_disabled = 0;
3491
3492 atomic_notifier_chain_register(&panic_notifier_list,
3493 &trace_panic_notifier);
3494
3495 register_die_notifier(&trace_die_notifier);
3496
3497 return 0;
3498}
3499early_initcall(tracer_alloc_buffers);
3500fs_initcall(tracer_init_debugfs);
Linux kernel - Deliverables
This page took 0.091797 seconds and 5 git commands to generate.