Commit | Line | Data |
---|---|---|
538f1952 WZ |
1 | /* |
2 | * Code for replacing ftrace calls with jumps. | |
3 | * | |
4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
e4240540 | 5 | * Copyright (C) 2009, 2010 DSLab, Lanzhou University, China |
f7a904df | 6 | * Author: Wu Zhangjin <wuzhangjin@gmail.com> |
538f1952 WZ |
7 | * |
8 | * Thanks goes to Steven Rostedt for writing the original x86 version. | |
9 | */ | |
10 | ||
11 | #include <linux/uaccess.h> | |
12 | #include <linux/init.h> | |
13 | #include <linux/ftrace.h> | |
19e2e172 | 14 | #include <linux/syscalls.h> |
538f1952 | 15 | |
29c5d346 WZ |
16 | #include <asm/asm.h> |
17 | #include <asm/asm-offsets.h> | |
e4240540 | 18 | #include <asm/cacheflush.h> |
19e2e172 | 19 | #include <asm/syscall.h> |
e4240540 | 20 | #include <asm/uasm.h> |
19e2e172 | 21 | #include <asm/unistd.h> |
538f1952 | 22 | |
d9cdb2f1 | 23 | #include <asm-generic/sections.h> |
c9f84873 | 24 | |
49de830a TG |
25 | #if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) |
26 | #define MCOUNT_OFFSET_INSNS 5 | |
27 | #else | |
28 | #define MCOUNT_OFFSET_INSNS 4 | |
29 | #endif | |
30 | ||
cb2f9938 MC |
31 | #ifdef CONFIG_DYNAMIC_FTRACE |
32 | ||
58b69401 AC |
33 | /* Arch override because MIPS doesn't need to run this from stop_machine() */ |
34 | void arch_ftrace_update_code(int command) | |
35 | { | |
36 | ftrace_modify_all_code(command); | |
37 | } | |
38 | ||
cb2f9938 MC |
39 | #endif |
40 | ||
49de830a TG |
41 | /* |
42 | * Check if the address is in kernel space | |
43 | * | |
44 | * Clone core_kernel_text() from kernel/extable.c, but doesn't call | |
45 | * init_kernel_text() for Ftrace doesn't trace functions in init sections. | |
46 | */ | |
47 | static inline int in_kernel_space(unsigned long ip) | |
48 | { | |
49 | if (ip >= (unsigned long)_stext && | |
50 | ip <= (unsigned long)_etext) | |
51 | return 1; | |
52 | return 0; | |
53 | } | |
54 | ||
538f1952 WZ |
55 | #ifdef CONFIG_DYNAMIC_FTRACE |
56 | ||
57 | #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ | |
58 | #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ | |
c54794d1 | 59 | #define JUMP_RANGE_MASK ((1UL << 28) - 1) |
538f1952 | 60 | |
4d6829f9 | 61 | #define INSN_NOP 0x00000000 /* nop */ |
e4240540 WZ |
62 | #define INSN_JAL(addr) \ |
63 | ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK))) | |
64 | ||
65 | static unsigned int insn_jal_ftrace_caller __read_mostly; | |
dce0e7d5 | 66 | static unsigned int insn_la_mcount[2] __read_mostly; |
e4240540 WZ |
67 | static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly; |
68 | ||
69 | static inline void ftrace_dyn_arch_init_insns(void) | |
70 | { | |
71 | u32 *buf; | |
72 | unsigned int v1; | |
73 | ||
dce0e7d5 | 74 | /* la v1, _mcount */ |
e4240540 | 75 | v1 = 3; |
dce0e7d5 PG |
76 | buf = (u32 *)&insn_la_mcount[0]; |
77 | UASM_i_LA(&buf, v1, MCOUNT_ADDR); | |
e4240540 WZ |
78 | |
79 | /* jal (ftrace_caller + 8), jump over the first two instruction */ | |
80 | buf = (u32 *)&insn_jal_ftrace_caller; | |
c54794d1 | 81 | uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK); |
e4240540 WZ |
82 | |
83 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
84 | /* j ftrace_graph_caller */ | |
85 | buf = (u32 *)&insn_j_ftrace_graph_caller; | |
c54794d1 | 86 | uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK); |
e4240540 WZ |
87 | #endif |
88 | } | |
538f1952 WZ |
89 | |
90 | static int ftrace_modify_code(unsigned long ip, unsigned int new_code) | |
91 | { | |
046199ca | 92 | int faulted; |
6ebda44f | 93 | mm_segment_t old_fs; |
046199ca WZ |
94 | |
95 | /* *(unsigned int *)ip = new_code; */ | |
96 | safe_store_code(new_code, ip, faulted); | |
97 | ||
98 | if (unlikely(faulted)) | |
99 | return -EFAULT; | |
538f1952 | 100 | |
6ebda44f LY |
101 | old_fs = get_fs(); |
102 | set_fs(get_ds()); | |
538f1952 | 103 | flush_icache_range(ip, ip + 8); |
6ebda44f | 104 | set_fs(old_fs); |
538f1952 WZ |
105 | |
106 | return 0; | |
107 | } | |
108 | ||
58b69401 AC |
109 | #ifndef CONFIG_64BIT |
110 | static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, | |
111 | unsigned int new_code2) | |
112 | { | |
113 | int faulted; | |
dce0e7d5 | 114 | mm_segment_t old_fs; |
58b69401 AC |
115 | |
116 | safe_store_code(new_code1, ip, faulted); | |
117 | if (unlikely(faulted)) | |
118 | return -EFAULT; | |
dce0e7d5 PG |
119 | |
120 | ip += 4; | |
121 | safe_store_code(new_code2, ip, faulted); | |
58b69401 AC |
122 | if (unlikely(faulted)) |
123 | return -EFAULT; | |
dce0e7d5 PG |
124 | |
125 | ip -= 4; | |
126 | old_fs = get_fs(); | |
127 | set_fs(get_ds()); | |
a4671094 | 128 | flush_icache_range(ip, ip + 8); |
dce0e7d5 PG |
129 | set_fs(old_fs); |
130 | ||
131 | return 0; | |
132 | } | |
133 | ||
134 | static int ftrace_modify_code_2r(unsigned long ip, unsigned int new_code1, | |
135 | unsigned int new_code2) | |
136 | { | |
137 | int faulted; | |
138 | mm_segment_t old_fs; | |
139 | ||
140 | ip += 4; | |
141 | safe_store_code(new_code2, ip, faulted); | |
142 | if (unlikely(faulted)) | |
143 | return -EFAULT; | |
144 | ||
145 | ip -= 4; | |
146 | safe_store_code(new_code1, ip, faulted); | |
147 | if (unlikely(faulted)) | |
148 | return -EFAULT; | |
149 | ||
150 | old_fs = get_fs(); | |
151 | set_fs(get_ds()); | |
152 | flush_icache_range(ip, ip + 8); | |
153 | set_fs(old_fs); | |
154 | ||
58b69401 AC |
155 | return 0; |
156 | } | |
157 | #endif | |
158 | ||
7f21a609 WZ |
159 | /* |
160 | * The details about the calling site of mcount on MIPS | |
161 | * | |
162 | * 1. For kernel: | |
163 | * | |
164 | * move at, ra | |
165 | * jal _mcount --> nop | |
dce0e7d5 | 166 | * sub sp, sp, 8 --> nop (CONFIG_32BIT) |
7f21a609 WZ |
167 | * |
168 | * 2. For modules: | |
169 | * | |
170 | * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT | |
171 | * | |
70342287 | 172 | * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) |
dce0e7d5 | 173 | * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) |
7f21a609 WZ |
174 | * move at, ra |
175 | * move $12, ra_address | |
176 | * jalr v1 | |
177 | * sub sp, sp, 8 | |
70342287 | 178 | * 1: offset = 5 instructions |
7f21a609 WZ |
179 | * 2.2 For the Other situations |
180 | * | |
70342287 | 181 | * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) |
dce0e7d5 | 182 | * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) |
7f21a609 WZ |
183 | * move at, ra |
184 | * jalr v1 | |
185 | * nop | move $12, ra_address | sub sp, sp, 8 | |
70342287 | 186 | * 1: offset = 4 instructions |
7f21a609 WZ |
187 | */ |
188 | ||
7f21a609 WZ |
189 | #define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS) |
190 | ||
538f1952 WZ |
191 | int ftrace_make_nop(struct module *mod, |
192 | struct dyn_ftrace *rec, unsigned long addr) | |
193 | { | |
194 | unsigned int new; | |
195 | unsigned long ip = rec->ip; | |
196 | ||
e4240540 | 197 | /* |
d9cdb2f1 WZ |
198 | * If ip is in kernel space, no long call, otherwise, long call is |
199 | * needed. | |
e4240540 | 200 | */ |
7f21a609 | 201 | new = in_kernel_space(ip) ? INSN_NOP : INSN_B_1F; |
58b69401 | 202 | #ifdef CONFIG_64BIT |
538f1952 | 203 | return ftrace_modify_code(ip, new); |
58b69401 AC |
204 | #else |
205 | /* | |
206 | * On 32 bit MIPS platforms, gcc adds a stack adjust | |
207 | * instruction in the delay slot after the branch to | |
208 | * mcount and expects mcount to restore the sp on return. | |
209 | * This is based on a legacy API and does nothing but | |
210 | * waste instructions so it's being removed at runtime. | |
211 | */ | |
212 | return ftrace_modify_code_2(ip, new, INSN_NOP); | |
213 | #endif | |
538f1952 WZ |
214 | } |
215 | ||
538f1952 WZ |
216 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
217 | { | |
218 | unsigned int new; | |
219 | unsigned long ip = rec->ip; | |
220 | ||
dce0e7d5 | 221 | new = in_kernel_space(ip) ? insn_jal_ftrace_caller : insn_la_mcount[0]; |
538f1952 | 222 | |
dce0e7d5 | 223 | #ifdef CONFIG_64BIT |
538f1952 | 224 | return ftrace_modify_code(ip, new); |
dce0e7d5 PG |
225 | #else |
226 | return ftrace_modify_code_2r(ip, new, in_kernel_space(ip) ? | |
227 | INSN_NOP : insn_la_mcount[1]); | |
228 | #endif | |
538f1952 WZ |
229 | } |
230 | ||
231 | #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) | |
232 | ||
233 | int ftrace_update_ftrace_func(ftrace_func_t func) | |
234 | { | |
235 | unsigned int new; | |
236 | ||
4d6829f9 | 237 | new = INSN_JAL((unsigned long)func); |
538f1952 WZ |
238 | |
239 | return ftrace_modify_code(FTRACE_CALL_IP, new); | |
240 | } | |
241 | ||
3a36cb11 | 242 | int __init ftrace_dyn_arch_init(void) |
538f1952 | 243 | { |
e4240540 WZ |
244 | /* Encode the instructions when booting */ |
245 | ftrace_dyn_arch_init_insns(); | |
246 | ||
247 | /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */ | |
248 | ftrace_modify_code(MCOUNT_ADDR, INSN_NOP); | |
249 | ||
538f1952 WZ |
250 | return 0; |
251 | } | |
68ccf752 | 252 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
29c5d346 WZ |
253 | |
254 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
255 | ||
e17ff5fe WZ |
256 | #ifdef CONFIG_DYNAMIC_FTRACE |
257 | ||
258 | extern void ftrace_graph_call(void); | |
e17ff5fe WZ |
259 | #define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call)) |
260 | ||
261 | int ftrace_enable_ftrace_graph_caller(void) | |
262 | { | |
263 | return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, | |
e4240540 | 264 | insn_j_ftrace_graph_caller); |
e17ff5fe WZ |
265 | } |
266 | ||
267 | int ftrace_disable_ftrace_graph_caller(void) | |
268 | { | |
4d6829f9 | 269 | return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP); |
e17ff5fe WZ |
270 | } |
271 | ||
68ccf752 | 272 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
e17ff5fe | 273 | |
7326c4e5 | 274 | #ifndef KBUILD_MCOUNT_RA_ADDRESS |
68ccf752 | 275 | |
70342287 RB |
276 | #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ |
277 | #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ | |
29c5d346 WZ |
278 | #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ |
279 | ||
2816e325 WZ |
280 | unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long |
281 | old_parent_ra, unsigned long parent_ra_addr, unsigned long fp) | |
29c5d346 | 282 | { |
2816e325 | 283 | unsigned long sp, ip, tmp; |
29c5d346 | 284 | unsigned int code; |
046199ca | 285 | int faulted; |
29c5d346 | 286 | |
68ccf752 | 287 | /* |
2816e325 | 288 | * For module, move the ip from the return address after the |
9a620a55 WZ |
289 | * instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for |
290 | * kernel, move after the instruction "move ra, at"(offset is 16) | |
68ccf752 | 291 | */ |
2816e325 | 292 | ip = self_ra - (in_kernel_space(self_ra) ? 16 : 24); |
29c5d346 | 293 | |
68ccf752 WZ |
294 | /* |
295 | * search the text until finding the non-store instruction or "s{d,w} | |
296 | * ra, offset(sp)" instruction | |
297 | */ | |
29c5d346 | 298 | do { |
046199ca WZ |
299 | /* get the code at "ip": code = *(unsigned int *)ip; */ |
300 | safe_load_code(code, ip, faulted); | |
301 | ||
302 | if (unlikely(faulted)) | |
303 | return 0; | |
68ccf752 WZ |
304 | /* |
305 | * If we hit the non-store instruction before finding where the | |
29c5d346 | 306 | * ra is stored, then this is a leaf function and it does not |
68ccf752 WZ |
307 | * store the ra on the stack |
308 | */ | |
29c5d346 | 309 | if ((code & S_R_SP) != S_R_SP) |
2816e325 | 310 | return parent_ra_addr; |
29c5d346 | 311 | |
9a620a55 WZ |
312 | /* Move to the next instruction */ |
313 | ip -= 4; | |
314 | } while ((code & S_RA_SP) != S_RA_SP); | |
29c5d346 WZ |
315 | |
316 | sp = fp + (code & OFFSET_MASK); | |
046199ca | 317 | |
2816e325 WZ |
318 | /* tmp = *(unsigned long *)sp; */ |
319 | safe_load_stack(tmp, sp, faulted); | |
046199ca WZ |
320 | if (unlikely(faulted)) |
321 | return 0; | |
29c5d346 | 322 | |
2816e325 | 323 | if (tmp == old_parent_ra) |
29c5d346 | 324 | return sp; |
29c5d346 WZ |
325 | return 0; |
326 | } | |
327 | ||
68ccf752 | 328 | #endif /* !KBUILD_MCOUNT_RA_ADDRESS */ |
7326c4e5 | 329 | |
29c5d346 WZ |
330 | /* |
331 | * Hook the return address and push it in the stack of return addrs | |
332 | * in current thread info. | |
333 | */ | |
2816e325 | 334 | void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, |
29c5d346 WZ |
335 | unsigned long fp) |
336 | { | |
2816e325 | 337 | unsigned long old_parent_ra; |
29c5d346 WZ |
338 | struct ftrace_graph_ent trace; |
339 | unsigned long return_hooker = (unsigned long) | |
340 | &return_to_handler; | |
b9f07eb2 | 341 | int faulted, insns; |
29c5d346 | 342 | |
6a8a5051 SRRH |
343 | if (unlikely(ftrace_graph_is_dead())) |
344 | return; | |
345 | ||
29c5d346 WZ |
346 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
347 | return; | |
348 | ||
68ccf752 | 349 | /* |
2816e325 WZ |
350 | * "parent_ra_addr" is the stack address saved the return address of |
351 | * the caller of _mcount. | |
7326c4e5 WZ |
352 | * |
353 | * if the gcc < 4.5, a leaf function does not save the return address | |
354 | * in the stack address, so, we "emulate" one in _mcount's stack space, | |
355 | * and hijack it directly, but for a non-leaf function, it save the | |
356 | * return address to the its own stack space, we can not hijack it | |
357 | * directly, but need to find the real stack address, | |
29c5d346 | 358 | * ftrace_get_parent_addr() does it! |
7326c4e5 WZ |
359 | * |
360 | * if gcc>= 4.5, with the new -mmcount-ra-address option, for a | |
361 | * non-leaf function, the location of the return address will be saved | |
362 | * to $12 for us, and for a leaf function, only put a zero into $12. we | |
363 | * do it in ftrace_graph_caller of mcount.S. | |
29c5d346 WZ |
364 | */ |
365 | ||
2816e325 WZ |
366 | /* old_parent_ra = *parent_ra_addr; */ |
367 | safe_load_stack(old_parent_ra, parent_ra_addr, faulted); | |
046199ca WZ |
368 | if (unlikely(faulted)) |
369 | goto out; | |
7326c4e5 | 370 | #ifndef KBUILD_MCOUNT_RA_ADDRESS |
2816e325 WZ |
371 | parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra, |
372 | old_parent_ra, (unsigned long)parent_ra_addr, fp); | |
68ccf752 WZ |
373 | /* |
374 | * If fails when getting the stack address of the non-leaf function's | |
375 | * ra, stop function graph tracer and return | |
376 | */ | |
2816e325 | 377 | if (parent_ra_addr == 0) |
046199ca | 378 | goto out; |
7326c4e5 | 379 | #endif |
2816e325 WZ |
380 | /* *parent_ra_addr = return_hooker; */ |
381 | safe_store_stack(return_hooker, parent_ra_addr, faulted); | |
046199ca WZ |
382 | if (unlikely(faulted)) |
383 | goto out; | |
29c5d346 | 384 | |
2816e325 WZ |
385 | if (ftrace_push_return_trace(old_parent_ra, self_ra, &trace.depth, fp) |
386 | == -EBUSY) { | |
387 | *parent_ra_addr = old_parent_ra; | |
29c5d346 WZ |
388 | return; |
389 | } | |
390 | ||
b9f07eb2 WZ |
391 | /* |
392 | * Get the recorded ip of the current mcount calling site in the | |
393 | * __mcount_loc section, which will be used to filter the function | |
394 | * entries configured through the tracing/set_graph_function interface. | |
395 | */ | |
396 | ||
397 | insns = in_kernel_space(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1; | |
398 | trace.func = self_ra - (MCOUNT_INSN_SIZE * insns); | |
29c5d346 WZ |
399 | |
400 | /* Only trace if the calling function expects to */ | |
401 | if (!ftrace_graph_entry(&trace)) { | |
402 | current->curr_ret_stack--; | |
2816e325 | 403 | *parent_ra_addr = old_parent_ra; |
29c5d346 | 404 | } |
046199ca WZ |
405 | return; |
406 | out: | |
407 | ftrace_graph_stop(); | |
408 | WARN_ON(1); | |
29c5d346 | 409 | } |
68ccf752 | 410 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
19e2e172 RB |
411 | |
412 | #ifdef CONFIG_FTRACE_SYSCALLS | |
413 | ||
414 | #ifdef CONFIG_32BIT | |
415 | unsigned long __init arch_syscall_addr(int nr) | |
416 | { | |
417 | return (unsigned long)sys_call_table[nr - __NR_O32_Linux]; | |
418 | } | |
419 | #endif | |
420 | ||
421 | #ifdef CONFIG_64BIT | |
422 | ||
423 | unsigned long __init arch_syscall_addr(int nr) | |
424 | { | |
425 | #ifdef CONFIG_MIPS32_N32 | |
426 | if (nr >= __NR_N32_Linux && nr <= __NR_N32_Linux + __NR_N32_Linux_syscalls) | |
46e12c07 | 427 | return (unsigned long)sysn32_call_table[nr - __NR_N32_Linux]; |
19e2e172 RB |
428 | #endif |
429 | if (nr >= __NR_64_Linux && nr <= __NR_64_Linux + __NR_64_Linux_syscalls) | |
430 | return (unsigned long)sys_call_table[nr - __NR_64_Linux]; | |
431 | #ifdef CONFIG_MIPS32_O32 | |
432 | if (nr >= __NR_O32_Linux && nr <= __NR_O32_Linux + __NR_O32_Linux_syscalls) | |
433 | return (unsigned long)sys32_call_table[nr - __NR_O32_Linux]; | |
434 | #endif | |
435 | ||
436 | return (unsigned long) &sys_ni_syscall; | |
437 | } | |
438 | #endif | |
439 | ||
440 | #endif /* CONFIG_FTRACE_SYSCALLS */ |