2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
22 #include <linux/kprobes.h>
23 #include <linux/seq_file.h>
24 #include <linux/slab.h>
25 #include <linux/smp.h>
26 #include <linux/debugfs.h>
27 #include <linux/types.h>
28 #include <linux/string.h>
29 #include <linux/ctype.h>
30 #include <linux/ptrace.h>
31 #include <linux/perf_event.h>
34 #include "trace_output.h"
36 #define MAX_TRACE_ARGS 128
37 #define MAX_ARGSTR_LEN 63
38 #define MAX_EVENT_NAME_LEN 64
39 #define KPROBE_EVENT_SYSTEM "kprobes"
41 /* Reserved field names */
42 #define FIELD_STRING_IP "__probe_ip"
43 #define FIELD_STRING_NARGS "__probe_nargs"
44 #define FIELD_STRING_RETIP "__probe_ret_ip"
45 #define FIELD_STRING_FUNC "__probe_func"
47 const char *reserved_field_names
[] = {
50 "common_preempt_count",
61 unsigned long (*func
)(struct pt_regs
*, void *);
65 static __kprobes
unsigned long call_fetch(struct fetch_func
*f
,
68 return f
->func(regs
, f
->data
);
72 static __kprobes
unsigned long fetch_register(struct pt_regs
*regs
,
75 return regs_get_register(regs
, (unsigned int)((unsigned long)offset
));
78 static __kprobes
unsigned long fetch_stack(struct pt_regs
*regs
,
81 return regs_get_kernel_stack_nth(regs
,
82 (unsigned int)((unsigned long)num
));
85 static __kprobes
unsigned long fetch_memory(struct pt_regs
*regs
, void *addr
)
89 if (probe_kernel_address(addr
, retval
))
94 static __kprobes
unsigned long fetch_retvalue(struct pt_regs
*regs
,
97 return regs_return_value(regs
);
100 static __kprobes
unsigned long fetch_stack_address(struct pt_regs
*regs
,
103 return kernel_stack_pointer(regs
);
106 /* Memory fetching by symbol */
107 struct symbol_cache
{
113 static unsigned long update_symbol_cache(struct symbol_cache
*sc
)
115 sc
->addr
= (unsigned long)kallsyms_lookup_name(sc
->symbol
);
117 sc
->addr
+= sc
->offset
;
121 static void free_symbol_cache(struct symbol_cache
*sc
)
127 static struct symbol_cache
*alloc_symbol_cache(const char *sym
, long offset
)
129 struct symbol_cache
*sc
;
131 if (!sym
|| strlen(sym
) == 0)
133 sc
= kzalloc(sizeof(struct symbol_cache
), GFP_KERNEL
);
137 sc
->symbol
= kstrdup(sym
, GFP_KERNEL
);
144 update_symbol_cache(sc
);
148 static __kprobes
unsigned long fetch_symbol(struct pt_regs
*regs
, void *data
)
150 struct symbol_cache
*sc
= data
;
153 return fetch_memory(regs
, (void *)sc
->addr
);
158 /* Special indirect memory access interface */
159 struct indirect_fetch_data
{
160 struct fetch_func orig
;
164 static __kprobes
unsigned long fetch_indirect(struct pt_regs
*regs
, void *data
)
166 struct indirect_fetch_data
*ind
= data
;
169 addr
= call_fetch(&ind
->orig
, regs
);
172 return fetch_memory(regs
, (void *)addr
);
177 static __kprobes
void free_indirect_fetch_data(struct indirect_fetch_data
*data
)
179 if (data
->orig
.func
== fetch_indirect
)
180 free_indirect_fetch_data(data
->orig
.data
);
181 else if (data
->orig
.func
== fetch_symbol
)
182 free_symbol_cache(data
->orig
.data
);
187 * Kprobe event core functions
191 struct fetch_func fetch
;
195 /* Flags for trace_probe */
196 #define TP_FLAG_TRACE 1
197 #define TP_FLAG_PROFILE 2
200 struct list_head list
;
201 struct kretprobe rp
; /* Use rp.kp for kprobe use */
203 unsigned int flags
; /* For TP_FLAG_* */
204 const char *symbol
; /* symbol name */
205 struct ftrace_event_class
class;
206 struct ftrace_event_call call
;
207 unsigned int nr_args
;
208 struct probe_arg args
[];
211 #define SIZEOF_TRACE_PROBE(n) \
212 (offsetof(struct trace_probe, args) + \
213 (sizeof(struct probe_arg) * (n)))
215 static __kprobes
int probe_is_return(struct trace_probe
*tp
)
217 return tp
->rp
.handler
!= NULL
;
220 static __kprobes
const char *probe_symbol(struct trace_probe
*tp
)
222 return tp
->symbol
? tp
->symbol
: "unknown";
225 static int probe_arg_string(char *buf
, size_t n
, struct fetch_func
*ff
)
229 if (ff
->func
== fetch_register
) {
231 name
= regs_query_register_name((unsigned int)((long)ff
->data
));
232 ret
= snprintf(buf
, n
, "%%%s", name
);
233 } else if (ff
->func
== fetch_stack
)
234 ret
= snprintf(buf
, n
, "$stack%lu", (unsigned long)ff
->data
);
235 else if (ff
->func
== fetch_memory
)
236 ret
= snprintf(buf
, n
, "@0x%p", ff
->data
);
237 else if (ff
->func
== fetch_symbol
) {
238 struct symbol_cache
*sc
= ff
->data
;
240 ret
= snprintf(buf
, n
, "@%s%+ld", sc
->symbol
,
243 ret
= snprintf(buf
, n
, "@%s", sc
->symbol
);
244 } else if (ff
->func
== fetch_retvalue
)
245 ret
= snprintf(buf
, n
, "$retval");
246 else if (ff
->func
== fetch_stack_address
)
247 ret
= snprintf(buf
, n
, "$stack");
248 else if (ff
->func
== fetch_indirect
) {
249 struct indirect_fetch_data
*id
= ff
->data
;
251 ret
= snprintf(buf
, n
, "%+ld(", id
->offset
);
255 ret
= probe_arg_string(buf
+ l
, n
- l
, &id
->orig
);
259 ret
= snprintf(buf
+ l
, n
- l
, ")");
268 static int register_probe_event(struct trace_probe
*tp
);
269 static void unregister_probe_event(struct trace_probe
*tp
);
271 static DEFINE_MUTEX(probe_lock
);
272 static LIST_HEAD(probe_list
);
274 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
);
275 static int kretprobe_dispatcher(struct kretprobe_instance
*ri
,
276 struct pt_regs
*regs
);
278 /* Check the name is good for event/group */
279 static int check_event_name(const char *name
)
281 if (!isalpha(*name
) && *name
!= '_')
283 while (*++name
!= '\0') {
284 if (!isalpha(*name
) && !isdigit(*name
) && *name
!= '_')
291 * Allocate new trace_probe and initialize it (including kprobes).
293 static struct trace_probe
*alloc_trace_probe(const char *group
,
298 int nargs
, int is_return
)
300 struct trace_probe
*tp
;
303 tp
= kzalloc(SIZEOF_TRACE_PROBE(nargs
), GFP_KERNEL
);
308 tp
->symbol
= kstrdup(symbol
, GFP_KERNEL
);
311 tp
->rp
.kp
.symbol_name
= tp
->symbol
;
312 tp
->rp
.kp
.offset
= offs
;
314 tp
->rp
.kp
.addr
= addr
;
317 tp
->rp
.handler
= kretprobe_dispatcher
;
319 tp
->rp
.kp
.pre_handler
= kprobe_dispatcher
;
321 if (!event
|| !check_event_name(event
)) {
326 tp
->call
.class = &tp
->class;
327 tp
->call
.name
= kstrdup(event
, GFP_KERNEL
);
331 if (!group
|| !check_event_name(group
)) {
336 tp
->class.system
= kstrdup(group
, GFP_KERNEL
);
337 if (!tp
->class.system
)
340 INIT_LIST_HEAD(&tp
->list
);
343 kfree(tp
->call
.name
);
349 static void free_probe_arg(struct probe_arg
*arg
)
351 if (arg
->fetch
.func
== fetch_symbol
)
352 free_symbol_cache(arg
->fetch
.data
);
353 else if (arg
->fetch
.func
== fetch_indirect
)
354 free_indirect_fetch_data(arg
->fetch
.data
);
358 static void free_trace_probe(struct trace_probe
*tp
)
362 for (i
= 0; i
< tp
->nr_args
; i
++)
363 free_probe_arg(&tp
->args
[i
]);
365 kfree(tp
->call
.class->system
);
366 kfree(tp
->call
.name
);
371 static struct trace_probe
*find_probe_event(const char *event
,
374 struct trace_probe
*tp
;
376 list_for_each_entry(tp
, &probe_list
, list
)
377 if (strcmp(tp
->call
.name
, event
) == 0 &&
378 strcmp(tp
->call
.class->system
, group
) == 0)
383 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
384 static void unregister_trace_probe(struct trace_probe
*tp
)
386 if (probe_is_return(tp
))
387 unregister_kretprobe(&tp
->rp
);
389 unregister_kprobe(&tp
->rp
.kp
);
391 unregister_probe_event(tp
);
394 /* Register a trace_probe and probe_event */
395 static int register_trace_probe(struct trace_probe
*tp
)
397 struct trace_probe
*old_tp
;
400 mutex_lock(&probe_lock
);
402 /* register as an event */
403 old_tp
= find_probe_event(tp
->call
.name
, tp
->call
.class->system
);
405 /* delete old event */
406 unregister_trace_probe(old_tp
);
407 free_trace_probe(old_tp
);
409 ret
= register_probe_event(tp
);
411 pr_warning("Faild to register probe event(%d)\n", ret
);
415 tp
->rp
.kp
.flags
|= KPROBE_FLAG_DISABLED
;
416 if (probe_is_return(tp
))
417 ret
= register_kretprobe(&tp
->rp
);
419 ret
= register_kprobe(&tp
->rp
.kp
);
422 pr_warning("Could not insert probe(%d)\n", ret
);
423 if (ret
== -EILSEQ
) {
424 pr_warning("Probing address(0x%p) is not an "
425 "instruction boundary.\n",
429 unregister_probe_event(tp
);
431 list_add_tail(&tp
->list
, &probe_list
);
433 mutex_unlock(&probe_lock
);
437 /* Split symbol and offset. */
438 static int split_symbol_offset(char *symbol
, unsigned long *offset
)
446 tmp
= strchr(symbol
, '+');
448 /* skip sign because strict_strtol doesn't accept '+' */
449 ret
= strict_strtoul(tmp
+ 1, 0, offset
);
458 #define PARAM_MAX_ARGS 16
459 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
461 static int parse_probe_vars(char *arg
, struct fetch_func
*ff
, int is_return
)
466 if (strcmp(arg
, "retval") == 0) {
468 ff
->func
= fetch_retvalue
;
472 } else if (strncmp(arg
, "stack", 5) == 0) {
473 if (arg
[5] == '\0') {
474 ff
->func
= fetch_stack_address
;
476 } else if (isdigit(arg
[5])) {
477 ret
= strict_strtoul(arg
+ 5, 10, ¶m
);
478 if (ret
|| param
> PARAM_MAX_STACK
)
481 ff
->func
= fetch_stack
;
482 ff
->data
= (void *)param
;
491 /* Recursive argument parser */
492 static int __parse_probe_arg(char *arg
, struct fetch_func
*ff
, int is_return
)
501 ret
= parse_probe_vars(arg
+ 1, ff
, is_return
);
503 case '%': /* named register */
504 ret
= regs_query_register_offset(arg
+ 1);
506 ff
->func
= fetch_register
;
507 ff
->data
= (void *)(unsigned long)ret
;
511 case '@': /* memory or symbol */
512 if (isdigit(arg
[1])) {
513 ret
= strict_strtoul(arg
+ 1, 0, ¶m
);
516 ff
->func
= fetch_memory
;
517 ff
->data
= (void *)param
;
519 ret
= split_symbol_offset(arg
+ 1, &offset
);
522 ff
->data
= alloc_symbol_cache(arg
+ 1, offset
);
524 ff
->func
= fetch_symbol
;
529 case '+': /* indirect memory */
531 tmp
= strchr(arg
, '(');
537 ret
= strict_strtol(arg
+ 1, 0, &offset
);
543 tmp
= strrchr(arg
, ')');
545 struct indirect_fetch_data
*id
;
547 id
= kzalloc(sizeof(struct indirect_fetch_data
),
552 ret
= __parse_probe_arg(arg
, &id
->orig
, is_return
);
556 ff
->func
= fetch_indirect
;
557 ff
->data
= (void *)id
;
563 /* TODO: support custom handler */
569 /* String length checking wrapper */
570 static int parse_probe_arg(char *arg
, struct fetch_func
*ff
, int is_return
)
572 if (strlen(arg
) > MAX_ARGSTR_LEN
) {
573 pr_info("Argument is too long.: %s\n", arg
);
576 return __parse_probe_arg(arg
, ff
, is_return
);
579 /* Return 1 if name is reserved or already used by another argument */
580 static int conflict_field_name(const char *name
,
581 struct probe_arg
*args
, int narg
)
584 for (i
= 0; i
< ARRAY_SIZE(reserved_field_names
); i
++)
585 if (strcmp(reserved_field_names
[i
], name
) == 0)
587 for (i
= 0; i
< narg
; i
++)
588 if (strcmp(args
[i
].name
, name
) == 0)
593 static int create_trace_probe(int argc
, char **argv
)
597 * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS]
598 * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
600 * $retval : fetch return value
601 * $stack : fetch stack address
602 * $stackN : fetch Nth of stack (N:0-)
603 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
604 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
605 * %REG : fetch register REG
606 * Indirect memory fetch:
607 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
608 * Alias name of args:
609 * NAME=FETCHARG : set NAME as alias of FETCHARG.
611 struct trace_probe
*tp
;
613 int is_return
= 0, is_delete
= 0;
614 char *symbol
= NULL
, *event
= NULL
, *arg
= NULL
, *group
= NULL
;
615 unsigned long offset
= 0;
617 char buf
[MAX_EVENT_NAME_LEN
];
619 /* argc must be >= 1 */
620 if (argv
[0][0] == 'p')
622 else if (argv
[0][0] == 'r')
624 else if (argv
[0][0] == '-')
627 pr_info("Probe definition must be started with 'p', 'r' or"
632 if (argv
[0][1] == ':') {
634 if (strchr(event
, '/')) {
636 event
= strchr(group
, '/') + 1;
638 if (strlen(group
) == 0) {
639 pr_info("Group name is not specified\n");
643 if (strlen(event
) == 0) {
644 pr_info("Event name is not specified\n");
649 group
= KPROBE_EVENT_SYSTEM
;
653 pr_info("Delete command needs an event name.\n");
656 tp
= find_probe_event(event
, group
);
658 pr_info("Event %s/%s doesn't exist.\n", group
, event
);
661 /* delete an event */
662 unregister_trace_probe(tp
);
663 free_trace_probe(tp
);
668 pr_info("Probe point is not specified.\n");
671 if (isdigit(argv
[1][0])) {
673 pr_info("Return probe point must be a symbol.\n");
676 /* an address specified */
677 ret
= strict_strtoul(&argv
[1][0], 0, (unsigned long *)&addr
);
679 pr_info("Failed to parse address.\n");
683 /* a symbol specified */
685 /* TODO: support .init module functions */
686 ret
= split_symbol_offset(symbol
, &offset
);
688 pr_info("Failed to parse symbol.\n");
691 if (offset
&& is_return
) {
692 pr_info("Return probe must be used without offset.\n");
696 argc
-= 2; argv
+= 2;
700 /* Make a new event name */
702 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_%s_%ld",
703 is_return
? 'r' : 'p', symbol
, offset
);
705 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_0x%p",
706 is_return
? 'r' : 'p', addr
);
709 tp
= alloc_trace_probe(group
, event
, addr
, symbol
, offset
, argc
,
712 pr_info("Failed to allocate trace_probe.(%d)\n",
717 /* parse arguments */
719 for (i
= 0; i
< argc
&& i
< MAX_TRACE_ARGS
; i
++) {
720 /* Parse argument name */
721 arg
= strchr(argv
[i
], '=');
727 if (conflict_field_name(argv
[i
], tp
->args
, i
)) {
728 pr_info("Argument%d name '%s' conflicts with "
729 "another field.\n", i
, argv
[i
]);
734 tp
->args
[i
].name
= kstrdup(argv
[i
], GFP_KERNEL
);
735 if (!tp
->args
[i
].name
) {
736 pr_info("Failed to allocate argument%d name '%s'.\n",
742 /* Parse fetch argument */
743 ret
= parse_probe_arg(arg
, &tp
->args
[i
].fetch
, is_return
);
745 pr_info("Parse error at argument%d. (%d)\n", i
, ret
);
746 kfree(tp
->args
[i
].name
);
753 ret
= register_trace_probe(tp
);
759 free_trace_probe(tp
);
763 static void cleanup_all_probes(void)
765 struct trace_probe
*tp
;
767 mutex_lock(&probe_lock
);
768 /* TODO: Use batch unregistration */
769 while (!list_empty(&probe_list
)) {
770 tp
= list_entry(probe_list
.next
, struct trace_probe
, list
);
771 unregister_trace_probe(tp
);
772 free_trace_probe(tp
);
774 mutex_unlock(&probe_lock
);
778 /* Probes listing interfaces */
779 static void *probes_seq_start(struct seq_file
*m
, loff_t
*pos
)
781 mutex_lock(&probe_lock
);
782 return seq_list_start(&probe_list
, *pos
);
785 static void *probes_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
787 return seq_list_next(v
, &probe_list
, pos
);
790 static void probes_seq_stop(struct seq_file
*m
, void *v
)
792 mutex_unlock(&probe_lock
);
795 static int probes_seq_show(struct seq_file
*m
, void *v
)
797 struct trace_probe
*tp
= v
;
799 char buf
[MAX_ARGSTR_LEN
+ 1];
801 seq_printf(m
, "%c", probe_is_return(tp
) ? 'r' : 'p');
802 seq_printf(m
, ":%s/%s", tp
->call
.class->system
, tp
->call
.name
);
805 seq_printf(m
, " 0x%p", tp
->rp
.kp
.addr
);
806 else if (tp
->rp
.kp
.offset
)
807 seq_printf(m
, " %s+%u", probe_symbol(tp
), tp
->rp
.kp
.offset
);
809 seq_printf(m
, " %s", probe_symbol(tp
));
811 for (i
= 0; i
< tp
->nr_args
; i
++) {
812 ret
= probe_arg_string(buf
, MAX_ARGSTR_LEN
, &tp
->args
[i
].fetch
);
814 pr_warning("Argument%d decoding error(%d).\n", i
, ret
);
817 seq_printf(m
, " %s=%s", tp
->args
[i
].name
, buf
);
823 static const struct seq_operations probes_seq_op
= {
824 .start
= probes_seq_start
,
825 .next
= probes_seq_next
,
826 .stop
= probes_seq_stop
,
827 .show
= probes_seq_show
830 static int probes_open(struct inode
*inode
, struct file
*file
)
832 if ((file
->f_mode
& FMODE_WRITE
) &&
833 (file
->f_flags
& O_TRUNC
))
834 cleanup_all_probes();
836 return seq_open(file
, &probes_seq_op
);
839 static int command_trace_probe(const char *buf
)
842 int argc
= 0, ret
= 0;
844 argv
= argv_split(GFP_KERNEL
, buf
, &argc
);
849 ret
= create_trace_probe(argc
, argv
);
855 #define WRITE_BUFSIZE 128
857 static ssize_t
probes_write(struct file
*file
, const char __user
*buffer
,
858 size_t count
, loff_t
*ppos
)
865 kbuf
= kmalloc(WRITE_BUFSIZE
, GFP_KERNEL
);
870 while (done
< count
) {
872 if (size
>= WRITE_BUFSIZE
)
873 size
= WRITE_BUFSIZE
- 1;
874 if (copy_from_user(kbuf
, buffer
+ done
, size
)) {
879 tmp
= strchr(kbuf
, '\n');
882 size
= tmp
- kbuf
+ 1;
883 } else if (done
+ size
< count
) {
884 pr_warning("Line length is too long: "
885 "Should be less than %d.", WRITE_BUFSIZE
);
890 /* Remove comments */
891 tmp
= strchr(kbuf
, '#');
895 ret
= command_trace_probe(kbuf
);
905 static const struct file_operations kprobe_events_ops
= {
906 .owner
= THIS_MODULE
,
910 .release
= seq_release
,
911 .write
= probes_write
,
914 /* Probes profiling interfaces */
915 static int probes_profile_seq_show(struct seq_file
*m
, void *v
)
917 struct trace_probe
*tp
= v
;
919 seq_printf(m
, " %-44s %15lu %15lu\n", tp
->call
.name
, tp
->nhit
,
925 static const struct seq_operations profile_seq_op
= {
926 .start
= probes_seq_start
,
927 .next
= probes_seq_next
,
928 .stop
= probes_seq_stop
,
929 .show
= probes_profile_seq_show
932 static int profile_open(struct inode
*inode
, struct file
*file
)
934 return seq_open(file
, &profile_seq_op
);
937 static const struct file_operations kprobe_profile_ops
= {
938 .owner
= THIS_MODULE
,
939 .open
= profile_open
,
942 .release
= seq_release
,
946 static __kprobes
void kprobe_trace_func(struct kprobe
*kp
, struct pt_regs
*regs
)
948 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
949 struct kprobe_trace_entry
*entry
;
950 struct ring_buffer_event
*event
;
951 struct ring_buffer
*buffer
;
953 unsigned long irq_flags
;
954 struct ftrace_event_call
*call
= &tp
->call
;
958 local_save_flags(irq_flags
);
959 pc
= preempt_count();
961 size
= SIZEOF_KPROBE_TRACE_ENTRY(tp
->nr_args
);
963 event
= trace_current_buffer_lock_reserve(&buffer
, call
->event
.type
,
964 size
, irq_flags
, pc
);
968 entry
= ring_buffer_event_data(event
);
969 entry
->nargs
= tp
->nr_args
;
970 entry
->ip
= (unsigned long)kp
->addr
;
971 for (i
= 0; i
< tp
->nr_args
; i
++)
972 entry
->args
[i
] = call_fetch(&tp
->args
[i
].fetch
, regs
);
974 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
975 trace_nowake_buffer_unlock_commit(buffer
, event
, irq_flags
, pc
);
978 /* Kretprobe handler */
979 static __kprobes
void kretprobe_trace_func(struct kretprobe_instance
*ri
,
980 struct pt_regs
*regs
)
982 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
983 struct kretprobe_trace_entry
*entry
;
984 struct ring_buffer_event
*event
;
985 struct ring_buffer
*buffer
;
987 unsigned long irq_flags
;
988 struct ftrace_event_call
*call
= &tp
->call
;
990 local_save_flags(irq_flags
);
991 pc
= preempt_count();
993 size
= SIZEOF_KRETPROBE_TRACE_ENTRY(tp
->nr_args
);
995 event
= trace_current_buffer_lock_reserve(&buffer
, call
->event
.type
,
996 size
, irq_flags
, pc
);
1000 entry
= ring_buffer_event_data(event
);
1001 entry
->nargs
= tp
->nr_args
;
1002 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1003 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1004 for (i
= 0; i
< tp
->nr_args
; i
++)
1005 entry
->args
[i
] = call_fetch(&tp
->args
[i
].fetch
, regs
);
1007 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
1008 trace_nowake_buffer_unlock_commit(buffer
, event
, irq_flags
, pc
);
1011 /* Event entry printers */
1013 print_kprobe_event(struct trace_iterator
*iter
, int flags
,
1014 struct trace_event
*event
)
1016 struct kprobe_trace_entry
*field
;
1017 struct trace_seq
*s
= &iter
->seq
;
1018 struct trace_probe
*tp
;
1021 field
= (struct kprobe_trace_entry
*)iter
->ent
;
1022 tp
= container_of(event
, struct trace_probe
, call
.event
);
1024 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
1027 if (!seq_print_ip_sym(s
, field
->ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1030 if (!trace_seq_puts(s
, ")"))
1033 for (i
= 0; i
< field
->nargs
; i
++)
1034 if (!trace_seq_printf(s
, " %s=%lx",
1035 tp
->args
[i
].name
, field
->args
[i
]))
1038 if (!trace_seq_puts(s
, "\n"))
1041 return TRACE_TYPE_HANDLED
;
1043 return TRACE_TYPE_PARTIAL_LINE
;
1047 print_kretprobe_event(struct trace_iterator
*iter
, int flags
,
1048 struct trace_event
*event
)
1050 struct kretprobe_trace_entry
*field
;
1051 struct trace_seq
*s
= &iter
->seq
;
1052 struct trace_probe
*tp
;
1055 field
= (struct kretprobe_trace_entry
*)iter
->ent
;
1056 tp
= container_of(event
, struct trace_probe
, call
.event
);
1058 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
1061 if (!seq_print_ip_sym(s
, field
->ret_ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1064 if (!trace_seq_puts(s
, " <- "))
1067 if (!seq_print_ip_sym(s
, field
->func
, flags
& ~TRACE_ITER_SYM_OFFSET
))
1070 if (!trace_seq_puts(s
, ")"))
1073 for (i
= 0; i
< field
->nargs
; i
++)
1074 if (!trace_seq_printf(s
, " %s=%lx",
1075 tp
->args
[i
].name
, field
->args
[i
]))
1078 if (!trace_seq_puts(s
, "\n"))
1081 return TRACE_TYPE_HANDLED
;
1083 return TRACE_TYPE_PARTIAL_LINE
;
1086 static int probe_event_enable(struct ftrace_event_call
*call
)
1088 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1090 tp
->flags
|= TP_FLAG_TRACE
;
1091 if (probe_is_return(tp
))
1092 return enable_kretprobe(&tp
->rp
);
1094 return enable_kprobe(&tp
->rp
.kp
);
1097 static void probe_event_disable(struct ftrace_event_call
*call
)
1099 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1101 tp
->flags
&= ~TP_FLAG_TRACE
;
1102 if (!(tp
->flags
& (TP_FLAG_TRACE
| TP_FLAG_PROFILE
))) {
1103 if (probe_is_return(tp
))
1104 disable_kretprobe(&tp
->rp
);
1106 disable_kprobe(&tp
->rp
.kp
);
1110 static int probe_event_raw_init(struct ftrace_event_call
*event_call
)
1116 #define DEFINE_FIELD(type, item, name, is_signed) \
1118 ret = trace_define_field(event_call, #type, name, \
1119 offsetof(typeof(field), item), \
1120 sizeof(field.item), is_signed, \
1126 static int kprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1129 struct kprobe_trace_entry field
;
1130 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1132 DEFINE_FIELD(unsigned long, ip
, FIELD_STRING_IP
, 0);
1133 DEFINE_FIELD(int, nargs
, FIELD_STRING_NARGS
, 1);
1134 /* Set argument names as fields */
1135 for (i
= 0; i
< tp
->nr_args
; i
++)
1136 DEFINE_FIELD(unsigned long, args
[i
], tp
->args
[i
].name
, 0);
1140 static int kretprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1143 struct kretprobe_trace_entry field
;
1144 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1146 DEFINE_FIELD(unsigned long, func
, FIELD_STRING_FUNC
, 0);
1147 DEFINE_FIELD(unsigned long, ret_ip
, FIELD_STRING_RETIP
, 0);
1148 DEFINE_FIELD(int, nargs
, FIELD_STRING_NARGS
, 1);
1149 /* Set argument names as fields */
1150 for (i
= 0; i
< tp
->nr_args
; i
++)
1151 DEFINE_FIELD(unsigned long, args
[i
], tp
->args
[i
].name
, 0);
1155 static int __set_print_fmt(struct trace_probe
*tp
, char *buf
, int len
)
1160 const char *fmt
, *arg
;
1162 if (!probe_is_return(tp
)) {
1164 arg
= "REC->" FIELD_STRING_IP
;
1166 fmt
= "(%lx <- %lx)";
1167 arg
= "REC->" FIELD_STRING_FUNC
", REC->" FIELD_STRING_RETIP
;
1170 /* When len=0, we just calculate the needed length */
1171 #define LEN_OR_ZERO (len ? len - pos : 0)
1173 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\"%s", fmt
);
1175 for (i
= 0; i
< tp
->nr_args
; i
++) {
1176 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, " %s=%%lx",
1180 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\", %s", arg
);
1182 for (i
= 0; i
< tp
->nr_args
; i
++) {
1183 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, ", REC->%s",
1189 /* return the length of print_fmt */
1193 static int set_print_fmt(struct trace_probe
*tp
)
1198 /* First: called with 0 length to calculate the needed length */
1199 len
= __set_print_fmt(tp
, NULL
, 0);
1200 print_fmt
= kmalloc(len
+ 1, GFP_KERNEL
);
1204 /* Second: actually write the @print_fmt */
1205 __set_print_fmt(tp
, print_fmt
, len
+ 1);
1206 tp
->call
.print_fmt
= print_fmt
;
1211 #ifdef CONFIG_PERF_EVENTS
1213 /* Kprobe profile handler */
1214 static __kprobes
void kprobe_perf_func(struct kprobe
*kp
,
1215 struct pt_regs
*regs
)
1217 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1218 struct ftrace_event_call
*call
= &tp
->call
;
1219 struct kprobe_trace_entry
*entry
;
1220 int size
, __size
, i
;
1221 unsigned long irq_flags
;
1224 __size
= SIZEOF_KPROBE_TRACE_ENTRY(tp
->nr_args
);
1225 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1226 size
-= sizeof(u32
);
1227 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1228 "profile buffer not large enough"))
1231 entry
= perf_trace_buf_prepare(size
, call
->event
.type
,
1236 entry
->nargs
= tp
->nr_args
;
1237 entry
->ip
= (unsigned long)kp
->addr
;
1238 for (i
= 0; i
< tp
->nr_args
; i
++)
1239 entry
->args
[i
] = call_fetch(&tp
->args
[i
].fetch
, regs
);
1241 perf_trace_buf_submit(entry
, size
, rctx
, entry
->ip
, 1, irq_flags
, regs
);
1244 /* Kretprobe profile handler */
1245 static __kprobes
void kretprobe_perf_func(struct kretprobe_instance
*ri
,
1246 struct pt_regs
*regs
)
1248 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1249 struct ftrace_event_call
*call
= &tp
->call
;
1250 struct kretprobe_trace_entry
*entry
;
1251 int size
, __size
, i
;
1252 unsigned long irq_flags
;
1255 __size
= SIZEOF_KRETPROBE_TRACE_ENTRY(tp
->nr_args
);
1256 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1257 size
-= sizeof(u32
);
1258 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1259 "profile buffer not large enough"))
1262 entry
= perf_trace_buf_prepare(size
, call
->event
.type
,
1267 entry
->nargs
= tp
->nr_args
;
1268 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1269 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1270 for (i
= 0; i
< tp
->nr_args
; i
++)
1271 entry
->args
[i
] = call_fetch(&tp
->args
[i
].fetch
, regs
);
1273 perf_trace_buf_submit(entry
, size
, rctx
, entry
->ret_ip
, 1,
1277 static int probe_perf_enable(struct ftrace_event_call
*call
)
1279 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1281 tp
->flags
|= TP_FLAG_PROFILE
;
1283 if (probe_is_return(tp
))
1284 return enable_kretprobe(&tp
->rp
);
1286 return enable_kprobe(&tp
->rp
.kp
);
1289 static void probe_perf_disable(struct ftrace_event_call
*call
)
1291 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1293 tp
->flags
&= ~TP_FLAG_PROFILE
;
1295 if (!(tp
->flags
& TP_FLAG_TRACE
)) {
1296 if (probe_is_return(tp
))
1297 disable_kretprobe(&tp
->rp
);
1299 disable_kprobe(&tp
->rp
.kp
);
1302 #endif /* CONFIG_PERF_EVENTS */
1305 int kprobe_register(struct ftrace_event_call
*event
, enum trace_reg type
)
1308 case TRACE_REG_REGISTER
:
1309 return probe_event_enable(event
);
1310 case TRACE_REG_UNREGISTER
:
1311 probe_event_disable(event
);
1314 #ifdef CONFIG_PERF_EVENTS
1315 case TRACE_REG_PERF_REGISTER
:
1316 return probe_perf_enable(event
);
1317 case TRACE_REG_PERF_UNREGISTER
:
1318 probe_perf_disable(event
);
1326 int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
)
1328 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1330 if (tp
->flags
& TP_FLAG_TRACE
)
1331 kprobe_trace_func(kp
, regs
);
1332 #ifdef CONFIG_PERF_EVENTS
1333 if (tp
->flags
& TP_FLAG_PROFILE
)
1334 kprobe_perf_func(kp
, regs
);
1336 return 0; /* We don't tweek kernel, so just return 0 */
1340 int kretprobe_dispatcher(struct kretprobe_instance
*ri
, struct pt_regs
*regs
)
1342 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1344 if (tp
->flags
& TP_FLAG_TRACE
)
1345 kretprobe_trace_func(ri
, regs
);
1346 #ifdef CONFIG_PERF_EVENTS
1347 if (tp
->flags
& TP_FLAG_PROFILE
)
1348 kretprobe_perf_func(ri
, regs
);
1350 return 0; /* We don't tweek kernel, so just return 0 */
1353 static struct trace_event_functions kretprobe_funcs
= {
1354 .trace
= print_kretprobe_event
1357 static struct trace_event_functions kprobe_funcs
= {
1358 .trace
= print_kprobe_event
1361 static int register_probe_event(struct trace_probe
*tp
)
1363 struct ftrace_event_call
*call
= &tp
->call
;
1366 /* Initialize ftrace_event_call */
1367 if (probe_is_return(tp
)) {
1368 INIT_LIST_HEAD(&call
->class->fields
);
1369 call
->event
.funcs
= &kretprobe_funcs
;
1370 call
->class->raw_init
= probe_event_raw_init
;
1371 call
->class->define_fields
= kretprobe_event_define_fields
;
1373 INIT_LIST_HEAD(&call
->class->fields
);
1374 call
->event
.funcs
= &kprobe_funcs
;
1375 call
->class->raw_init
= probe_event_raw_init
;
1376 call
->class->define_fields
= kprobe_event_define_fields
;
1378 if (set_print_fmt(tp
) < 0)
1380 ret
= register_ftrace_event(&call
->event
);
1382 kfree(call
->print_fmt
);
1386 call
->class->reg
= kprobe_register
;
1388 ret
= trace_add_event_call(call
);
1390 pr_info("Failed to register kprobe event: %s\n", call
->name
);
1391 kfree(call
->print_fmt
);
1392 unregister_ftrace_event(&call
->event
);
1397 static void unregister_probe_event(struct trace_probe
*tp
)
1399 /* tp->event is unregistered in trace_remove_event_call() */
1400 trace_remove_event_call(&tp
->call
);
1401 kfree(tp
->call
.print_fmt
);
1404 /* Make a debugfs interface for controling probe points */
1405 static __init
int init_kprobe_trace(void)
1407 struct dentry
*d_tracer
;
1408 struct dentry
*entry
;
1410 d_tracer
= tracing_init_dentry();
1414 entry
= debugfs_create_file("kprobe_events", 0644, d_tracer
,
1415 NULL
, &kprobe_events_ops
);
1417 /* Event list interface */
1419 pr_warning("Could not create debugfs "
1420 "'kprobe_events' entry\n");
1422 /* Profile interface */
1423 entry
= debugfs_create_file("kprobe_profile", 0444, d_tracer
,
1424 NULL
, &kprobe_profile_ops
);
1427 pr_warning("Could not create debugfs "
1428 "'kprobe_profile' entry\n");
1431 fs_initcall(init_kprobe_trace
);
1434 #ifdef CONFIG_FTRACE_STARTUP_TEST
1436 static int kprobe_trace_selftest_target(int a1
, int a2
, int a3
,
1437 int a4
, int a5
, int a6
)
1439 return a1
+ a2
+ a3
+ a4
+ a5
+ a6
;
1442 static __init
int kprobe_trace_self_tests_init(void)
1445 int (*target
)(int, int, int, int, int, int);
1446 struct trace_probe
*tp
;
1448 target
= kprobe_trace_selftest_target
;
1450 pr_info("Testing kprobe tracing: ");
1452 ret
= command_trace_probe("p:testprobe kprobe_trace_selftest_target "
1453 "$stack $stack0 +0($stack)");
1454 if (WARN_ON_ONCE(ret
)) {
1455 pr_warning("error on probing function entry.\n");
1458 /* Enable trace point */
1459 tp
= find_probe_event("testprobe", KPROBE_EVENT_SYSTEM
);
1460 if (WARN_ON_ONCE(tp
== NULL
)) {
1461 pr_warning("error on getting new probe.\n");
1464 probe_event_enable(&tp
->call
);
1467 ret
= command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
1469 if (WARN_ON_ONCE(ret
)) {
1470 pr_warning("error on probing function return.\n");
1473 /* Enable trace point */
1474 tp
= find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM
);
1475 if (WARN_ON_ONCE(tp
== NULL
)) {
1476 pr_warning("error on getting new probe.\n");
1479 probe_event_enable(&tp
->call
);
1485 ret
= target(1, 2, 3, 4, 5, 6);
1487 ret
= command_trace_probe("-:testprobe");
1488 if (WARN_ON_ONCE(ret
)) {
1489 pr_warning("error on deleting a probe.\n");
1493 ret
= command_trace_probe("-:testprobe2");
1494 if (WARN_ON_ONCE(ret
)) {
1495 pr_warning("error on deleting a probe.\n");
1500 cleanup_all_probes();
1502 pr_cont("NG: Some tests are failed. Please check them.\n");
1508 late_initcall(kprobe_trace_self_tests_init
);