Commit | Line | Data |
---|---|---|
014c257c AS |
1 | /* |
2 | * Dynamic function tracing support. | |
3 | * | |
4 | * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> | |
3b6c223b | 5 | * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> |
014c257c AS |
6 | * |
7 | * For licencing details, see COPYING. | |
8 | * | |
9 | * Defines low-level handling of mcount calls when the kernel | |
10 | * is compiled with the -pg flag. When using dynamic ftrace, the | |
3b6c223b RV |
11 | * mcount call-sites get patched with NOP till they are enabled. |
12 | * All code mutation routines here are called under stop_machine(). | |
014c257c AS |
13 | */ |
14 | ||
15 | #include <linux/ftrace.h> | |
3b6c223b | 16 | #include <linux/uaccess.h> |
a672917a | 17 | #include <linux/module.h> |
80d6b0c2 | 18 | #include <linux/stop_machine.h> |
395a59d0 | 19 | |
014c257c | 20 | #include <asm/cacheflush.h> |
4394e282 | 21 | #include <asm/opcodes.h> |
395a59d0 | 22 | #include <asm/ftrace.h> |
0dc016db | 23 | #include <asm/insn.h> |
d82227cf | 24 | |
72dc43a9 | 25 | #ifdef CONFIG_THUMB2_KERNEL |
4394e282 | 26 | #define NOP 0xf85deb04 /* pop.w {lr} */ |
72dc43a9 | 27 | #else |
3b6c223b | 28 | #define NOP 0xe8bd4000 /* pop {lr} */ |
72dc43a9 | 29 | #endif |
014c257c | 30 | |
376cfa87 | 31 | #ifdef CONFIG_DYNAMIC_FTRACE |
3b6c223b RV |
32 | #ifdef CONFIG_OLD_MCOUNT |
33 | #define OLD_MCOUNT_ADDR ((unsigned long) mcount) | |
34 | #define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) | |
014c257c | 35 | |
3b6c223b RV |
36 | #define OLD_NOP 0xe1a00000 /* mov r0, r0 */ |
37 | ||
80d6b0c2 KC |
38 | static int __ftrace_modify_code(void *data) |
39 | { | |
40 | int *command = data; | |
41 | ||
42 | set_kernel_text_rw(); | |
43 | ftrace_modify_all_code(*command); | |
44 | set_kernel_text_ro(); | |
45 | ||
46 | return 0; | |
47 | } | |
48 | ||
49 | void arch_ftrace_update_code(int command) | |
50 | { | |
51 | stop_machine(__ftrace_modify_code, &command, NULL); | |
52 | } | |
53 | ||
3b6c223b RV |
54 | static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) |
55 | { | |
56 | return rec->arch.old_mcount ? OLD_NOP : NOP; | |
57 | } | |
58 | ||
59 | static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) | |
60 | { | |
61 | if (!rec->arch.old_mcount) | |
62 | return addr; | |
63 | ||
64 | if (addr == MCOUNT_ADDR) | |
65 | addr = OLD_MCOUNT_ADDR; | |
66 | else if (addr == FTRACE_ADDR) | |
67 | addr = OLD_FTRACE_ADDR; | |
68 | ||
69 | return addr; | |
70 | } | |
71 | #else | |
72 | static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) | |
73 | { | |
74 | return NOP; | |
75 | } | |
76 | ||
77 | static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) | |
014c257c | 78 | { |
3b6c223b | 79 | return addr; |
014c257c | 80 | } |
3b6c223b | 81 | #endif |
014c257c | 82 | |
a672917a RV |
83 | int ftrace_arch_code_modify_prepare(void) |
84 | { | |
85 | set_all_modules_text_rw(); | |
86 | return 0; | |
87 | } | |
88 | ||
89 | int ftrace_arch_code_modify_post_process(void) | |
90 | { | |
91 | set_all_modules_text_ro(); | |
80d6b0c2 KC |
92 | /* Make sure any TLB misses during machine stop are cleared. */ |
93 | flush_tlb_all(); | |
a672917a RV |
94 | return 0; |
95 | } | |
96 | ||
dd686eb1 RV |
97 | static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) |
98 | { | |
d82227cf | 99 | return arm_gen_branch_link(pc, addr); |
dd686eb1 RV |
100 | } |
101 | ||
3b6c223b | 102 | static int ftrace_modify_code(unsigned long pc, unsigned long old, |
dc283d70 | 103 | unsigned long new, bool validate) |
3b6c223b RV |
104 | { |
105 | unsigned long replaced; | |
014c257c | 106 | |
4394e282 RV |
107 | if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) { |
108 | old = __opcode_to_mem_thumb32(old); | |
109 | new = __opcode_to_mem_thumb32(new); | |
110 | } else { | |
111 | old = __opcode_to_mem_arm(old); | |
112 | new = __opcode_to_mem_arm(new); | |
113 | } | |
114 | ||
dc283d70 RV |
115 | if (validate) { |
116 | if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) | |
117 | return -EFAULT; | |
014c257c | 118 | |
dc283d70 RV |
119 | if (replaced != old) |
120 | return -EINVAL; | |
121 | } | |
014c257c | 122 | |
3b6c223b RV |
123 | if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) |
124 | return -EPERM; | |
014c257c | 125 | |
3b6c223b | 126 | flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); |
014c257c | 127 | |
3b6c223b | 128 | return 0; |
014c257c AS |
129 | } |
130 | ||
131 | int ftrace_update_ftrace_func(ftrace_func_t func) | |
132 | { | |
dc283d70 | 133 | unsigned long pc; |
3b6c223b RV |
134 | unsigned long new; |
135 | int ret; | |
014c257c AS |
136 | |
137 | pc = (unsigned long)&ftrace_call; | |
014c257c | 138 | new = ftrace_call_replace(pc, (unsigned long)func); |
3b6c223b | 139 | |
dc283d70 | 140 | ret = ftrace_modify_code(pc, 0, new, false); |
3b6c223b RV |
141 | |
142 | #ifdef CONFIG_OLD_MCOUNT | |
143 | if (!ret) { | |
144 | pc = (unsigned long)&ftrace_call_old; | |
3b6c223b RV |
145 | new = ftrace_call_replace(pc, (unsigned long)func); |
146 | ||
dc283d70 | 147 | ret = ftrace_modify_code(pc, 0, new, false); |
3b6c223b RV |
148 | } |
149 | #endif | |
150 | ||
151 | return ret; | |
152 | } | |
153 | ||
154 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) | |
155 | { | |
156 | unsigned long new, old; | |
157 | unsigned long ip = rec->ip; | |
158 | ||
159 | old = ftrace_nop_replace(rec); | |
160 | new = ftrace_call_replace(ip, adjust_address(rec, addr)); | |
161 | ||
dc283d70 | 162 | return ftrace_modify_code(rec->ip, old, new, true); |
3b6c223b RV |
163 | } |
164 | ||
165 | int ftrace_make_nop(struct module *mod, | |
166 | struct dyn_ftrace *rec, unsigned long addr) | |
167 | { | |
168 | unsigned long ip = rec->ip; | |
169 | unsigned long old; | |
170 | unsigned long new; | |
171 | int ret; | |
172 | ||
173 | old = ftrace_call_replace(ip, adjust_address(rec, addr)); | |
174 | new = ftrace_nop_replace(rec); | |
dc283d70 | 175 | ret = ftrace_modify_code(ip, old, new, true); |
3b6c223b RV |
176 | |
177 | #ifdef CONFIG_OLD_MCOUNT | |
178 | if (ret == -EINVAL && addr == MCOUNT_ADDR) { | |
179 | rec->arch.old_mcount = true; | |
180 | ||
181 | old = ftrace_call_replace(ip, adjust_address(rec, addr)); | |
182 | new = ftrace_nop_replace(rec); | |
dc283d70 | 183 | ret = ftrace_modify_code(ip, old, new, true); |
3b6c223b RV |
184 | } |
185 | #endif | |
186 | ||
014c257c AS |
187 | return ret; |
188 | } | |
189 | ||
3a36cb11 | 190 | int __init ftrace_dyn_arch_init(void) |
014c257c | 191 | { |
014c257c AS |
192 | return 0; |
193 | } | |
376cfa87 TB |
194 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
195 | ||
196 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
197 | void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, | |
198 | unsigned long frame_pointer) | |
199 | { | |
200 | unsigned long return_hooker = (unsigned long) &return_to_handler; | |
201 | struct ftrace_graph_ent trace; | |
202 | unsigned long old; | |
203 | int err; | |
204 | ||
205 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) | |
206 | return; | |
207 | ||
208 | old = *parent; | |
209 | *parent = return_hooker; | |
210 | ||
376cfa87 | 211 | trace.func = self_addr; |
4c36595e | 212 | trace.depth = current->curr_ret_stack + 1; |
376cfa87 TB |
213 | |
214 | /* Only trace if the calling function expects to */ | |
215 | if (!ftrace_graph_entry(&trace)) { | |
376cfa87 | 216 | *parent = old; |
4c36595e CC |
217 | return; |
218 | } | |
219 | ||
220 | err = ftrace_push_return_trace(old, self_addr, &trace.depth, | |
221 | frame_pointer); | |
222 | if (err == -EBUSY) { | |
223 | *parent = old; | |
224 | return; | |
376cfa87 TB |
225 | } |
226 | } | |
dd686eb1 RV |
227 | |
228 | #ifdef CONFIG_DYNAMIC_FTRACE | |
229 | extern unsigned long ftrace_graph_call; | |
230 | extern unsigned long ftrace_graph_call_old; | |
231 | extern void ftrace_graph_caller_old(void); | |
232 | ||
233 | static int __ftrace_modify_caller(unsigned long *callsite, | |
234 | void (*func) (void), bool enable) | |
235 | { | |
236 | unsigned long caller_fn = (unsigned long) func; | |
237 | unsigned long pc = (unsigned long) callsite; | |
d82227cf | 238 | unsigned long branch = arm_gen_branch(pc, caller_fn); |
dd686eb1 RV |
239 | unsigned long nop = 0xe1a00000; /* mov r0, r0 */ |
240 | unsigned long old = enable ? nop : branch; | |
241 | unsigned long new = enable ? branch : nop; | |
242 | ||
dc283d70 | 243 | return ftrace_modify_code(pc, old, new, true); |
dd686eb1 RV |
244 | } |
245 | ||
246 | static int ftrace_modify_graph_caller(bool enable) | |
247 | { | |
248 | int ret; | |
249 | ||
250 | ret = __ftrace_modify_caller(&ftrace_graph_call, | |
251 | ftrace_graph_caller, | |
252 | enable); | |
253 | ||
254 | #ifdef CONFIG_OLD_MCOUNT | |
255 | if (!ret) | |
256 | ret = __ftrace_modify_caller(&ftrace_graph_call_old, | |
257 | ftrace_graph_caller_old, | |
258 | enable); | |
259 | #endif | |
260 | ||
261 | return ret; | |
262 | } | |
263 | ||
264 | int ftrace_enable_ftrace_graph_caller(void) | |
265 | { | |
266 | return ftrace_modify_graph_caller(true); | |
267 | } | |
268 | ||
269 | int ftrace_disable_ftrace_graph_caller(void) | |
270 | { | |
271 | return ftrace_modify_graph_caller(false); | |
272 | } | |
273 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
376cfa87 | 274 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |