[deliverable/linux.git] / arch / x86 / kernel / ftrace.c

/*
 * Code for replacing ftrace calls with jumps.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 *
 * Thanks goes to Ingo Molnar, for suggesting the idea.
 * Mathieu Desnoyers, for suggesting postponing the modifications.
 * Arjan van de Ven, for keeping me straight, and explaining to me
 * the dangers of modifying code on the run.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/spinlock.h>
#include <linux/hardirq.h>
#include <linux/uaccess.h>
#include <linux/ftrace.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/list.h>

#include <trace/syscall.h>

#include <asm/cacheflush.h>
#include <asm/ftrace.h>
#include <asm/nops.h>
#include <asm/nmi.h>


#ifdef CONFIG_DYNAMIC_FTRACE

/*
 * modifying_code is set to notify NMIs that they need to use
 * memory barriers when entering or exiting. But we don't want
 * to burden NMIs with unnecessary memory barriers when code
 * modification is not being done (which is most of the time).
 *
 * A mutex is already held when ftrace_arch_code_modify_prepare
 * and post_process are called. No locks need to be taken here.
 *
 * Stop machine will make sure currently running NMIs are done
 * and new NMIs will see the updated variable before we need
 * to worry about NMIs doing memory barriers.
 */
static int modifying_code __read_mostly;
static DEFINE_PER_CPU(int, save_modifying_code);

int ftrace_arch_code_modify_prepare(void)
{
	set_kernel_text_rw();
	modifying_code = 1;
	return 0;
}

int ftrace_arch_code_modify_post_process(void)
{
	modifying_code = 0;
	set_kernel_text_ro();
	return 0;
}

union ftrace_code_union {
	char code[MCOUNT_INSN_SIZE];
	struct {
		char e8;
		int offset;
	} __attribute__((packed));
};

static int ftrace_calc_offset(long ip, long addr)
{
	return (int)(addr - ip);
}

static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
{
	static union ftrace_code_union calc;

	calc.e8		= 0xe8;
	calc.offset	= ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);

	/*
	 * No locking needed, this must be called via kstop_machine
	 * which in essence is like running on a uniprocessor machine.
	 */
	return calc.code;
}

/*
 * Modifying code must take extra care. On an SMP machine, if
 * the code being modified is also being executed on another CPU
 * that CPU will have undefined results and possibly take a GPF.
 * We use kstop_machine to stop other CPUS from exectuing code.
 * But this does not stop NMIs from happening. We still need
 * to protect against that. We separate out the modification of
 * the code to take care of this.
 *
 * Two buffers are added: An IP buffer and a "code" buffer.
 *
 * 1) Put the instruction pointer into the IP buffer
 *    and the new code into the "code" buffer.
 * 2) Wait for any running NMIs to finish and set a flag that says
 *    we are modifying code, it is done in an atomic operation.
 * 3) Write the code
 * 4) clear the flag.
 * 5) Wait for any running NMIs to finish.
 *
 * If an NMI is executed, the first thing it does is to call
 * "ftrace_nmi_enter". This will check if the flag is set to write
 * and if it is, it will write what is in the IP and "code" buffers.
 *
 * The trick is, it does not matter if everyone is writing the same
 * content to the code location. Also, if a CPU is executing code
 * it is OK to write to that code location if the contents being written
 * are the same as what exists.
 */

#define MOD_CODE_WRITE_FLAG (1 << 31)	/* set when NMI should do the write */
static atomic_t nmi_running = ATOMIC_INIT(0);
static int mod_code_status;		/* holds return value of text write */
static void *mod_code_ip;		/* holds the IP to write to */
static void *mod_code_newcode;		/* holds the text to write to the IP */

static unsigned nmi_wait_count;
static atomic_t nmi_update_count = ATOMIC_INIT(0);

int ftrace_arch_read_dyn_info(char *buf, int size)
{
	int r;

	r = snprintf(buf, size, "%u %u",
		     nmi_wait_count,
		     atomic_read(&nmi_update_count));
	return r;
}

static void clear_mod_flag(void)
{
	int old = atomic_read(&nmi_running);

	for (;;) {
		int new = old & ~MOD_CODE_WRITE_FLAG;

		if (old == new)
			break;

		old = atomic_cmpxchg(&nmi_running, old, new);
	}
}

static void ftrace_mod_code(void)
{
	/*
	 * Yes, more than one CPU process can be writing to mod_code_status.
	 *    (and the code itself)
	 * But if one were to fail, then they all should, and if one were
	 * to succeed, then they all should.
	 */
	mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
					     MCOUNT_INSN_SIZE);

	/* if we fail, then kill any new writers */
	if (mod_code_status)
		clear_mod_flag();
}

void ftrace_nmi_enter(void)
{
	__get_cpu_var(save_modifying_code) = modifying_code;

	if (!__get_cpu_var(save_modifying_code))
		return;

	if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
		smp_rmb();
		ftrace_mod_code();
		atomic_inc(&nmi_update_count);
	}
	/* Must have previous changes seen before executions */
	smp_mb();
}

void ftrace_nmi_exit(void)
{
	if (!__get_cpu_var(save_modifying_code))
		return;

	/* Finish all executions before clearing nmi_running */
	smp_mb();
	atomic_dec(&nmi_running);
}

static void wait_for_nmi_and_set_mod_flag(void)
{
	if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
		return;

	do {
		cpu_relax();
	} while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));

	nmi_wait_count++;
}

static void wait_for_nmi(void)
{
	if (!atomic_read(&nmi_running))
		return;

	do {
		cpu_relax();
	} while (atomic_read(&nmi_running));

	nmi_wait_count++;
}

static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
	return addr >= start && addr < end;
}

static int
do_ftrace_mod_code(unsigned long ip, void *new_code)
{
	/*
	 * On x86_64, kernel text mappings are mapped read-only with
	 * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
	 * of the kernel text mapping to modify the kernel text.
	 *
	 * For 32bit kernels, these mappings are same and we can use
	 * kernel identity mapping to modify code.
	 */
	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
		ip = (unsigned long)__va(__pa(ip));

	mod_code_ip = (void *)ip;
	mod_code_newcode = new_code;

	/* The buffers need to be visible before we let NMIs write them */
	smp_mb();

	wait_for_nmi_and_set_mod_flag();

	/* Make sure all running NMIs have finished before we write the code */
	smp_mb();

	ftrace_mod_code();

	/* Make sure the write happens before clearing the bit */
	smp_mb();

	clear_mod_flag();
	wait_for_nmi();

	return mod_code_status;
}

static unsigned char *ftrace_nop_replace(void)
{
	return ideal_nop5;
}

static int
ftrace_modify_code(unsigned long ip, unsigned char *old_code,
		   unsigned char *new_code)
{
	unsigned char replaced[MCOUNT_INSN_SIZE];

	/*
	 * Note: Due to modules and __init, code can
	 *  disappear and change, we need to protect against faulting
	 *  as well as code changing. We do this by using the
	 *  probe_kernel_* functions.
	 *
	 * No real locking needed, this code is run through
	 * kstop_machine, or before SMP starts.
	 */

	/* read the text we want to modify */
	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	/* Make sure it is what we expect it to be */
	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
		return -EINVAL;

	/* replace the text with the new text */
	if (do_ftrace_mod_code(ip, new_code))
		return -EPERM;

	sync_core();

	return 0;
}

int ftrace_make_nop(struct module *mod,
		    struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned char *new, *old;
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);
	new = ftrace_nop_replace();

	return ftrace_modify_code(rec->ip, old, new);
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned char *new, *old;
	unsigned long ip = rec->ip;

	old = ftrace_nop_replace();
	new = ftrace_call_replace(ip, addr);

	return ftrace_modify_code(rec->ip, old, new);
}

int ftrace_update_ftrace_func(ftrace_func_t func)
{
	unsigned long ip = (unsigned long)(&ftrace_call);
	unsigned char old[MCOUNT_INSN_SIZE], *new;
	int ret;

	memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
	new = ftrace_call_replace(ip, (unsigned long)func);
	ret = ftrace_modify_code(ip, old, new);

	return ret;
}

int __init ftrace_dyn_arch_init(void *data)
{
	/* The return code is retured via data */
	*(unsigned long *)data = 0;

	return 0;
}
#endif

#ifdef CONFIG_FUNCTION_GRAPH_TRACER

#ifdef CONFIG_DYNAMIC_FTRACE
extern void ftrace_graph_call(void);

static int ftrace_mod_jmp(unsigned long ip,
			  int old_offset, int new_offset)
{
	unsigned char code[MCOUNT_INSN_SIZE];

	if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	if (code[0] != 0xe9 || old_offset != *(int *)(&code[1]))
		return -EINVAL;

	*(int *)(&code[1]) = new_offset;

	if (do_ftrace_mod_code(ip, &code))
		return -EPERM;

	return 0;
}

int ftrace_enable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);
	int old_offset, new_offset;

	old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
	new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);

	return ftrace_mod_jmp(ip, old_offset, new_offset);
}

int ftrace_disable_ftrace_graph_caller(void)
{
	unsigned long ip = (unsigned long)(&ftrace_graph_call);
	int old_offset, new_offset;

	old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
	new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);

	return ftrace_mod_jmp(ip, old_offset, new_offset);
}

#endif /* !CONFIG_DYNAMIC_FTRACE */

/*
 * Hook the return address and push it in the stack of return addrs
 * in current thread info.
 */
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
			   unsigned long frame_pointer)
{
	unsigned long old;
	int faulted;
	struct ftrace_graph_ent trace;
	unsigned long return_hooker = (unsigned long)
				&return_to_handler;

	if (unlikely(atomic_read(&current->tracing_graph_pause)))
		return;

	/*
	 * Protect against fault, even if it shouldn't
	 * happen. This tool is too much intrusive to
	 * ignore such a protection.
	 */
	asm volatile(
		"1: " _ASM_MOV " (%[parent]), %[old]\n"
		"2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
		"   movl $0, %[faulted]\n"
		"3:\n"

		".section .fixup, \"ax\"\n"
		"4: movl $1, %[faulted]\n"
		"   jmp 3b\n"
		".previous\n"

		_ASM_EXTABLE(1b, 4b)
		_ASM_EXTABLE(2b, 4b)

		: [old] "=&r" (old), [faulted] "=r" (faulted)
		: [parent] "r" (parent), [return_hooker] "r" (return_hooker)
		: "memory"
	);

	if (unlikely(faulted)) {
		ftrace_graph_stop();
		WARN_ON(1);
		return;
	}

	if (ftrace_push_return_trace(old, self_addr, &trace.depth,
		    frame_pointer) == -EBUSY) {
		*parent = old;
		return;
	}

	trace.func = self_addr;

	/* Only trace if the calling function expects to */
	if (!ftrace_graph_entry(&trace)) {
		current->curr_ret_stack--;
		*parent = old;
	}
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
Commit	Line	Data
3d083395 SR	1	/*
	2	* Code for replacing ftrace calls with jumps.
	3	*
	4	* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
	5	*
	6	* Thanks goes to Ingo Molnar, for suggesting the idea.
	7	* Mathieu Desnoyers, for suggesting postponing the modifications.
	8	* Arjan van de Ven, for keeping me straight, and explaining to me
	9	* the dangers of modifying code on the run.
	10	*/
	11
3bb258bf JP	12	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3bb258bf JP	13
3d083395 SR	14	#include <linux/spinlock.h>
3d083395 SR	15	#include <linux/hardirq.h>
6f93fc07	16	#include <linux/uaccess.h>
3d083395 SR	17	#include <linux/ftrace.h>
3d083395 SR	18	#include <linux/percpu.h>
19b3e967	19	#include <linux/sched.h>
3d083395 SR	20	#include <linux/init.h>
	21	#include <linux/list.h>
	22
47788c58 FW	23	#include <trace/syscall.h>
47788c58 FW	24
16239630	25	#include <asm/cacheflush.h>
395a59d0	26	#include <asm/ftrace.h>
732f3ca7	27	#include <asm/nops.h>
caf4b323	28	#include <asm/nmi.h>
3d083395	29
3d083395	30
caf4b323	31	#ifdef CONFIG_DYNAMIC_FTRACE
3d083395	32
0c54dd34 SR	33	/*
	34	* modifying_code is set to notify NMIs that they need to use
	35	* memory barriers when entering or exiting. But we don't want
	36	* to burden NMIs with unnecessary memory barriers when code
	37	* modification is not being done (which is most of the time).
	38	*
	39	* A mutex is already held when ftrace_arch_code_modify_prepare
	40	* and post_process are called. No locks need to be taken here.
	41	*
	42	* Stop machine will make sure currently running NMIs are done
	43	* and new NMIs will see the updated variable before we need
	44	* to worry about NMIs doing memory barriers.
	45	*/
	46	static int modifying_code __read_mostly;
	47	static DEFINE_PER_CPU(int, save_modifying_code);
	48
16239630 SR	49	int ftrace_arch_code_modify_prepare(void)
	50	{
	51	set_kernel_text_rw();
0c54dd34	52	modifying_code = 1;
16239630 SR	53	return 0;
	54	}
	55
	56	int ftrace_arch_code_modify_post_process(void)
	57	{
0c54dd34	58	modifying_code = 0;
16239630 SR	59	set_kernel_text_ro();
	60	return 0;
	61	}
	62
3d083395	63	union ftrace_code_union {
395a59d0	64	char code[MCOUNT_INSN_SIZE];
3d083395 SR	65	struct {
	66	char e8;
	67	int offset;
	68	} __attribute__((packed));
	69	};
	70
15adc048	71	static int ftrace_calc_offset(long ip, long addr)
3c1720f0 SR	72	{
	73	return (int)(addr - ip);
	74	}
3d083395	75
31e88909	76	static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
3c1720f0 SR	77	{
3c1720f0 SR	78	static union ftrace_code_union calc;
3d083395	79
3c1720f0	80	calc.e8 = 0xe8;
395a59d0	81	calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
3c1720f0 SR	82
	83	/*
	84	* No locking needed, this must be called via kstop_machine
	85	* which in essence is like running on a uniprocessor machine.
	86	*/
	87	return calc.code;
3d083395 SR	88	}
3d083395 SR	89
17666f02 SR	90	/*
	91	* Modifying code must take extra care. On an SMP machine, if
	92	* the code being modified is also being executed on another CPU
	93	* that CPU will have undefined results and possibly take a GPF.
	94	* We use kstop_machine to stop other CPUS from exectuing code.
	95	* But this does not stop NMIs from happening. We still need
	96	* to protect against that. We separate out the modification of
	97	* the code to take care of this.
	98	*
	99	* Two buffers are added: An IP buffer and a "code" buffer.
	100	*
a26a2a27	101	* 1) Put the instruction pointer into the IP buffer
17666f02	102	* and the new code into the "code" buffer.
e9d9df44 LJ	103	* 2) Wait for any running NMIs to finish and set a flag that says
	104	* we are modifying code, it is done in an atomic operation.
	105	* 3) Write the code
	106	* 4) clear the flag.
	107	* 5) Wait for any running NMIs to finish.
17666f02 SR	108	*
	109	* If an NMI is executed, the first thing it does is to call
	110	* "ftrace_nmi_enter". This will check if the flag is set to write
	111	* and if it is, it will write what is in the IP and "code" buffers.
	112	*
	113	* The trick is, it does not matter if everyone is writing the same
	114	* content to the code location. Also, if a CPU is executing code
	115	* it is OK to write to that code location if the contents being written
	116	* are the same as what exists.
	117	*/
	118
e9d9df44	119	#define MOD_CODE_WRITE_FLAG (1 << 31) /* set when NMI should do the write */
4e6ea144	120	static atomic_t nmi_running = ATOMIC_INIT(0);
a26a2a27	121	static int mod_code_status; /* holds return value of text write */
a26a2a27 SR	122	static void mod_code_ip; / holds the IP to write to */
a26a2a27 SR	123	static void mod_code_newcode; / holds the text to write to the IP */
17666f02	124
a26a2a27 SR	125	static unsigned nmi_wait_count;
a26a2a27 SR	126	static atomic_t nmi_update_count = ATOMIC_INIT(0);
b807c3d0 SR	127
	128	int ftrace_arch_read_dyn_info(char *buf, int size)
	129	{
	130	int r;
	131
	132	r = snprintf(buf, size, "%u %u",
	133	nmi_wait_count,
	134	atomic_read(&nmi_update_count));
	135	return r;
	136	}
	137
e9d9df44 LJ	138	static void clear_mod_flag(void)
	139	{
	140	int old = atomic_read(&nmi_running);
	141
	142	for (;;) {
	143	int new = old & ~MOD_CODE_WRITE_FLAG;
	144
	145	if (old == new)
	146	break;
	147
	148	old = atomic_cmpxchg(&nmi_running, old, new);
	149	}
	150	}
	151
17666f02 SR	152	static void ftrace_mod_code(void)
	153	{
	154	/*
	155	* Yes, more than one CPU process can be writing to mod_code_status.
	156	* (and the code itself)
	157	* But if one were to fail, then they all should, and if one were
	158	* to succeed, then they all should.
	159	*/
	160	mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
	161	MCOUNT_INSN_SIZE);
90c7ac49 SR	162
	163	/* if we fail, then kill any new writers */
	164	if (mod_code_status)
e9d9df44	165	clear_mod_flag();
17666f02 SR	166	}
17666f02 SR	167
a81bd80a	168	void ftrace_nmi_enter(void)
17666f02	169	{
0c54dd34 SR	170	__get_cpu_var(save_modifying_code) = modifying_code;
	171
	172	if (!__get_cpu_var(save_modifying_code))
	173	return;
	174
e9d9df44 LJ	175	if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
e9d9df44 LJ	176	smp_rmb();
17666f02	177	ftrace_mod_code();
b807c3d0 SR	178	atomic_inc(&nmi_update_count);
b807c3d0 SR	179	}
e9d9df44 LJ	180	/* Must have previous changes seen before executions */
e9d9df44 LJ	181	smp_mb();
17666f02 SR	182	}
17666f02 SR	183
a81bd80a	184	void ftrace_nmi_exit(void)
17666f02	185	{
0c54dd34 SR	186	if (!__get_cpu_var(save_modifying_code))
	187	return;
	188
4e6ea144	189	/* Finish all executions before clearing nmi_running */
e9d9df44	190	smp_mb();
4e6ea144	191	atomic_dec(&nmi_running);
17666f02 SR	192	}
17666f02 SR	193
e9d9df44 LJ	194	static void wait_for_nmi_and_set_mod_flag(void)
	195	{
	196	if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
	197	return;
	198
	199	do {
	200	cpu_relax();
	201	} while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));
	202
	203	nmi_wait_count++;
	204	}
	205
17666f02 SR	206	static void wait_for_nmi(void)
17666f02 SR	207	{
4e6ea144	208	if (!atomic_read(&nmi_running))
89025282	209	return;
b807c3d0	210
89025282	211	do {
17666f02	212	cpu_relax();
4e6ea144	213	} while (atomic_read(&nmi_running));
b807c3d0	214
89025282	215	nmi_wait_count++;
17666f02 SR	216	}
17666f02 SR	217
55ca3cc1 SS	218	static inline int
	219	within(unsigned long addr, unsigned long start, unsigned long end)
	220	{
	221	return addr >= start && addr < end;
	222	}
	223
17666f02 SR	224	static int
	225	do_ftrace_mod_code(unsigned long ip, void *new_code)
	226	{
55ca3cc1 SS	227	/*
	228	* On x86_64, kernel text mappings are mapped read-only with
	229	* CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
	230	* of the kernel text mapping to modify the kernel text.
	231	*
	232	* For 32bit kernels, these mappings are same and we can use
	233	* kernel identity mapping to modify code.
	234	*/
	235	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
	236	ip = (unsigned long)__va(__pa(ip));
	237
17666f02 SR	238	mod_code_ip = (void *)ip;
	239	mod_code_newcode = new_code;
	240
	241	/* The buffers need to be visible before we let NMIs write them */
17666f02 SR	242	smp_mb();
17666f02 SR	243
e9d9df44	244	wait_for_nmi_and_set_mod_flag();
17666f02 SR	245
	246	/* Make sure all running NMIs have finished before we write the code */
	247	smp_mb();
	248
	249	ftrace_mod_code();
	250
	251	/* Make sure the write happens before clearing the bit */
17666f02 SR	252	smp_mb();
17666f02 SR	253
e9d9df44	254	clear_mod_flag();
17666f02 SR	255	wait_for_nmi();
	256
	257	return mod_code_status;
	258	}
	259
31e88909	260	static unsigned char *ftrace_nop_replace(void)
caf4b323	261	{
f49aa448	262	return ideal_nop5;
caf4b323 FW	263	}
caf4b323 FW	264
31e88909	265	static int
3d083395 SR	266	ftrace_modify_code(unsigned long ip, unsigned char *old_code,
	267	unsigned char *new_code)
	268	{
6f93fc07	269	unsigned char replaced[MCOUNT_INSN_SIZE];
3d083395 SR	270
	271	/*
	272	* Note: Due to modules and __init, code can
	273	* disappear and change, we need to protect against faulting
76aefee5	274	* as well as code changing. We do this by using the
ab9a0918	275	* probe_kernel_* functions.
3d083395 SR	276	*
3d083395 SR	277	* No real locking needed, this code is run through
6f93fc07	278	* kstop_machine, or before SMP starts.
3d083395	279	*/
76aefee5 SR	280
76aefee5 SR	281	/* read the text we want to modify */
ab9a0918	282	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
593eb8a2	283	return -EFAULT;
6f93fc07	284
76aefee5	285	/* Make sure it is what we expect it to be */
6f93fc07	286	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
593eb8a2	287	return -EINVAL;
3d083395	288
76aefee5	289	/* replace the text with the new text */
17666f02	290	if (do_ftrace_mod_code(ip, new_code))
593eb8a2	291	return -EPERM;
6f93fc07 SR	292
6f93fc07 SR	293	sync_core();
3d083395	294
6f93fc07	295	return 0;
3d083395 SR	296	}
3d083395 SR	297
31e88909 SR	298	int ftrace_make_nop(struct module *mod,
	299	struct dyn_ftrace *rec, unsigned long addr)
	300	{
	301	unsigned char new, old;
	302	unsigned long ip = rec->ip;
	303
	304	old = ftrace_call_replace(ip, addr);
	305	new = ftrace_nop_replace();
	306
	307	return ftrace_modify_code(rec->ip, old, new);
	308	}
	309
	310	int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
	311	{
	312	unsigned char new, old;
	313	unsigned long ip = rec->ip;
	314
	315	old = ftrace_nop_replace();
	316	new = ftrace_call_replace(ip, addr);
	317
	318	return ftrace_modify_code(rec->ip, old, new);
	319	}
	320
15adc048	321	int ftrace_update_ftrace_func(ftrace_func_t func)
d61f82d0 SR	322	{
d61f82d0 SR	323	unsigned long ip = (unsigned long)(&ftrace_call);
395a59d0	324	unsigned char old[MCOUNT_INSN_SIZE], *new;
d61f82d0 SR	325	int ret;
d61f82d0 SR	326
395a59d0	327	memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
d61f82d0 SR	328	new = ftrace_call_replace(ip, (unsigned long)func);
	329	ret = ftrace_modify_code(ip, old, new);
	330
	331	return ret;
	332	}
	333
d61f82d0	334	int __init ftrace_dyn_arch_init(void *data)
3d083395	335	{
732f3ca7 SR	336	/* The return code is retured via data */
732f3ca7 SR	337	(unsigned long )data = 0;
dfa60aba	338
3d083395 SR	339	return 0;
3d083395 SR	340	}
caf4b323	341	#endif
e7d3737e	342
fb52607a	343	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
e7d3737e	344
5a45cfe1 SR	345	#ifdef CONFIG_DYNAMIC_FTRACE
	346	extern void ftrace_graph_call(void);
	347
	348	static int ftrace_mod_jmp(unsigned long ip,
	349	int old_offset, int new_offset)
	350	{
	351	unsigned char code[MCOUNT_INSN_SIZE];
	352
	353	if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
	354	return -EFAULT;
	355
	356	if (code[0] != 0xe9 \|\| old_offset != (int )(&code[1]))
	357	return -EINVAL;
	358
	359	(int )(&code[1]) = new_offset;
	360
	361	if (do_ftrace_mod_code(ip, &code))
	362	return -EPERM;
	363
	364	return 0;
	365	}
	366
	367	int ftrace_enable_ftrace_graph_caller(void)
	368	{
	369	unsigned long ip = (unsigned long)(&ftrace_graph_call);
	370	int old_offset, new_offset;
	371
	372	old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
	373	new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
	374
	375	return ftrace_mod_jmp(ip, old_offset, new_offset);
	376	}
	377
	378	int ftrace_disable_ftrace_graph_caller(void)
	379	{
	380	unsigned long ip = (unsigned long)(&ftrace_graph_call);
	381	int old_offset, new_offset;
	382
	383	old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
	384	new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
	385
	386	return ftrace_mod_jmp(ip, old_offset, new_offset);
	387	}
	388
e7d3737e FW	389	#endif /* !CONFIG_DYNAMIC_FTRACE */
e7d3737e FW	390
e7d3737e FW	391	/*
	392	* Hook the return address and push it in the stack of return addrs
	393	* in current thread info.
	394	*/
71e308a2 SR	395	void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
71e308a2 SR	396	unsigned long frame_pointer)
e7d3737e FW	397	{
e7d3737e FW	398	unsigned long old;
e7d3737e	399	int faulted;
287b6e68	400	struct ftrace_graph_ent trace;
e7d3737e FW	401	unsigned long return_hooker = (unsigned long)
	402	&return_to_handler;
	403
380c4b14	404	if (unlikely(atomic_read(&current->tracing_graph_pause)))
e7d3737e FW	405	return;
	406
	407	/*
	408	* Protect against fault, even if it shouldn't
	409	* happen. This tool is too much intrusive to
	410	* ignore such a protection.
	411	*/
	412	asm volatile(
96665788 SR	413	"1: " _ASM_MOV " (%[parent]), %[old]\n"
96665788 SR	414	"2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
e7d3737e	415	" movl $0, %[faulted]\n"
e3944bfa	416	"3:\n"
e7d3737e FW	417
e7d3737e FW	418	".section .fixup, \"ax\"\n"
e3944bfa SR	419	"4: movl $1, %[faulted]\n"
e3944bfa SR	420	" jmp 3b\n"
e7d3737e FW	421	".previous\n"
e7d3737e FW	422
e3944bfa SR	423	_ASM_EXTABLE(1b, 4b)
e3944bfa SR	424	_ASM_EXTABLE(2b, 4b)
e7d3737e	425
aa512a27	426	: [old] "=&r" (old), [faulted] "=r" (faulted)
96665788	427	: [parent] "r" (parent), [return_hooker] "r" (return_hooker)
e7d3737e FW	428	: "memory"
	429	);
	430
14a866c5 SR	431	if (unlikely(faulted)) {
	432	ftrace_graph_stop();
	433	WARN_ON(1);
e7d3737e FW	434	return;
	435	}
	436
71e308a2 SR	437	if (ftrace_push_return_trace(old, self_addr, &trace.depth,
71e308a2 SR	438	frame_pointer) == -EBUSY) {
e7d3737e	439	*parent = old;
287b6e68 FW	440	return;
	441	}
	442
	443	trace.func = self_addr;
287b6e68	444
e49dc19c SR	445	/* Only trace if the calling function expects to */
	446	if (!ftrace_graph_entry(&trace)) {
	447	current->curr_ret_stack--;
	448	*parent = old;
	449	}
e7d3737e	450	}
fb52607a	451	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */