[deliverable/linux.git] / arch / x86 / kernel / vsyscall_64.c

/*
 *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
 *  Copyright 2003 Andi Kleen, SuSE Labs.
 *
 *  [ NOTE: this mechanism is now deprecated in favor of the vDSO. ]
 *
 *  Thanks to hpa@transmeta.com for some useful hint.
 *  Special thanks to Ingo Molnar for his early experience with
 *  a different vsyscall implementation for Linux/IA32 and for the name.
 *
 *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
 *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
 *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
 *  jumping out of line if necessary. We cannot add more with this
 *  mechanism because older kernels won't return -ENOSYS.
 *
 *  Note: the concept clashes with user mode linux.  UML users should
 *  use the vDSO.
 */

#include <linux/time.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/seqlock.h>
#include <linux/jiffies.h>
#include <linux/sysctl.h>
#include <linux/topology.h>
#include <linux/clocksource.h>
#include <linux/getcpu.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/notifier.h>
#include <linux/syscalls.h>
#include <linux/ratelimit.h>

#include <asm/vsyscall.h>
#include <asm/pgtable.h>
#include <asm/compat.h>
#include <asm/page.h>
#include <asm/unistd.h>
#include <asm/fixmap.h>
#include <asm/errno.h>
#include <asm/io.h>
#include <asm/segment.h>
#include <asm/desc.h>
#include <asm/topology.h>
#include <asm/vgtod.h>
#include <asm/traps.h>

#define CREATE_TRACE_POINTS
#include "vsyscall_trace.h"

DEFINE_VVAR(int, vgetcpu_mode);
DEFINE_VVAR(struct vsyscall_gtod_data, vsyscall_gtod_data) =
{
	.lock = __SEQLOCK_UNLOCKED(__vsyscall_gtod_data.lock),
};

static enum { EMULATE, NATIVE, NONE } vsyscall_mode = NATIVE;

static int __init vsyscall_setup(char *str)
{
	if (str) {
		if (!strcmp("emulate", str))
			vsyscall_mode = EMULATE;
		else if (!strcmp("native", str))
			vsyscall_mode = NATIVE;
		else if (!strcmp("none", str))
			vsyscall_mode = NONE;
		else
			return -EINVAL;

		return 0;
	}

	return -EINVAL;
}
early_param("vsyscall", vsyscall_setup);

void update_vsyscall_tz(void)
{
	unsigned long flags;

	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
	/* sys_tz has changed */
	vsyscall_gtod_data.sys_tz = sys_tz;
	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
}

void update_vsyscall(struct timespec *wall_time, struct timespec *wtm,
			struct clocksource *clock, u32 mult)
{
	unsigned long flags;

	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);

	/* copy vsyscall data */
	vsyscall_gtod_data.clock.vclock_mode	= clock->archdata.vclock_mode;
	vsyscall_gtod_data.clock.cycle_last	= clock->cycle_last;
	vsyscall_gtod_data.clock.mask		= clock->mask;
	vsyscall_gtod_data.clock.mult		= mult;
	vsyscall_gtod_data.clock.shift		= clock->shift;
	vsyscall_gtod_data.wall_time_sec	= wall_time->tv_sec;
	vsyscall_gtod_data.wall_time_nsec	= wall_time->tv_nsec;
	vsyscall_gtod_data.wall_to_monotonic	= *wtm;
	vsyscall_gtod_data.wall_time_coarse	= __current_kernel_time();

	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
}

static void warn_bad_vsyscall(const char *level, struct pt_regs *regs,
			      const char *message)
{
	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
	struct task_struct *tsk;

	if (!show_unhandled_signals || !__ratelimit(&rs))
		return;

	tsk = current;

	printk("%s%s[%d] %s ip:%lx cs:%lx sp:%lx ax:%lx si:%lx di:%lx\n",
	       level, tsk->comm, task_pid_nr(tsk),
	       message, regs->ip, regs->cs,
	       regs->sp, regs->ax, regs->si, regs->di);
}

static int addr_to_vsyscall_nr(unsigned long addr)
{
	int nr;

	if ((addr & ~0xC00UL) != VSYSCALL_START)
		return -EINVAL;

	nr = (addr & 0xC00UL) >> 10;
	if (nr >= 3)
		return -EINVAL;

	return nr;
}

static bool write_ok_or_segv(unsigned long ptr, size_t size)
{
	/*
	 * XXX: if access_ok, get_user, and put_user handled
	 * sig_on_uaccess_error, this could go away.
	 */

	if (!access_ok(VERIFY_WRITE, (void __user *)ptr, size)) {
		siginfo_t info;
		struct thread_struct *thread = &current->thread;

		thread->error_code	= 6;  /* user fault, no page, write */
		thread->cr2		= ptr;
		thread->trap_no		= 14;

		memset(&info, 0, sizeof(info));
		info.si_signo		= SIGSEGV;
		info.si_errno		= 0;
		info.si_code		= SEGV_MAPERR;
		info.si_addr		= (void __user *)ptr;

		force_sig_info(SIGSEGV, &info, current);
		return false;
	} else {
		return true;
	}
}

bool emulate_vsyscall(struct pt_regs *regs, unsigned long address)
{
	struct task_struct *tsk;
	unsigned long caller;
	int vsyscall_nr;
	int prev_sig_on_uaccess_error;
	long ret;

	/*
	 * No point in checking CS -- the only way to get here is a user mode
	 * trap to a high address, which means that we're in 64-bit user code.
	 */

	WARN_ON_ONCE(address != regs->ip);

	if (vsyscall_mode == NONE) {
		warn_bad_vsyscall(KERN_INFO, regs,
				  "vsyscall attempted with vsyscall=none");
		return false;
	}

	vsyscall_nr = addr_to_vsyscall_nr(address);

	trace_emulate_vsyscall(vsyscall_nr);

	if (vsyscall_nr < 0) {
		warn_bad_vsyscall(KERN_WARNING, regs,
				  "misaligned vsyscall (exploit attempt or buggy program) -- look up the vsyscall kernel parameter if you need a workaround");
		goto sigsegv;
	}

	if (get_user(caller, (unsigned long __user *)regs->sp) != 0) {
		warn_bad_vsyscall(KERN_WARNING, regs,
				  "vsyscall with bad stack (exploit attempt?)");
		goto sigsegv;
	}

	tsk = current;
	if (seccomp_mode(&tsk->seccomp))
		do_exit(SIGKILL);

	/*
	 * With a real vsyscall, page faults cause SIGSEGV.  We want to
	 * preserve that behavior to make writing exploits harder.
	 */
	prev_sig_on_uaccess_error = current_thread_info()->sig_on_uaccess_error;
	current_thread_info()->sig_on_uaccess_error = 1;

	/*
	 * 0 is a valid user pointer (in the access_ok sense) on 32-bit and
	 * 64-bit, so we don't need to special-case it here.  For all the
	 * vsyscalls, 0 means "don't write anything" not "write it at
	 * address 0".
	 */
	ret = -EFAULT;
	switch (vsyscall_nr) {
	case 0:
		if (!write_ok_or_segv(regs->di, sizeof(struct timeval)) ||
		    !write_ok_or_segv(regs->si, sizeof(struct timezone)))
			break;

		ret = sys_gettimeofday(
			(struct timeval __user *)regs->di,
			(struct timezone __user *)regs->si);
		break;

	case 1:
		if (!write_ok_or_segv(regs->di, sizeof(time_t)))
			break;

		ret = sys_time((time_t __user *)regs->di);
		break;

	case 2:
		if (!write_ok_or_segv(regs->di, sizeof(unsigned)) ||
		    !write_ok_or_segv(regs->si, sizeof(unsigned)))
			break;

		ret = sys_getcpu((unsigned __user *)regs->di,
				 (unsigned __user *)regs->si,
				 0);
		break;
	}

	current_thread_info()->sig_on_uaccess_error = prev_sig_on_uaccess_error;

	if (ret == -EFAULT) {
		/* Bad news -- userspace fed a bad pointer to a vsyscall. */
		warn_bad_vsyscall(KERN_INFO, regs,
				  "vsyscall fault (exploit attempt?)");

		/*
		 * If we failed to generate a signal for any reason,
		 * generate one here.  (This should be impossible.)
		 */
		if (WARN_ON_ONCE(!sigismember(&tsk->pending.signal, SIGBUS) &&
				 !sigismember(&tsk->pending.signal, SIGSEGV)))
			goto sigsegv;

		return true;  /* Don't emulate the ret. */
	}

	regs->ax = ret;

	/* Emulate a ret instruction. */
	regs->ip = caller;
	regs->sp += 8;

	return true;

sigsegv:
	force_sig(SIGSEGV, current);
	return true;
}

/*
 * Assume __initcall executes before all user space. Hopefully kmod
 * doesn't violate that. We'll find out if it does.
 */
static void __cpuinit vsyscall_set_cpu(int cpu)
{
	unsigned long d;
	unsigned long node = 0;
#ifdef CONFIG_NUMA
	node = cpu_to_node(cpu);
#endif
	if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
		write_rdtscp_aux((node << 12) | cpu);

	/*
	 * Store cpu number in limit so that it can be loaded quickly
	 * in user space in vgetcpu. (12 bits for the CPU and 8 bits for the node)
	 */
	d = 0x0f40000000000ULL;
	d |= cpu;
	d |= (node & 0xf) << 12;
	d |= (node >> 4) << 48;

	write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
}

static void __cpuinit cpu_vsyscall_init(void *arg)
{
	/* preemption should be already off */
	vsyscall_set_cpu(raw_smp_processor_id());
}

static int __cpuinit
cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
{
	long cpu = (long)arg;

	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
		smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);

	return NOTIFY_DONE;
}

void __init map_vsyscall(void)
{
	extern char __vsyscall_page;
	unsigned long physaddr_vsyscall = __pa_symbol(&__vsyscall_page);
	extern char __vvar_page;
	unsigned long physaddr_vvar_page = __pa_symbol(&__vvar_page);

	__set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_vsyscall,
		     vsyscall_mode == NATIVE
		     ? PAGE_KERNEL_VSYSCALL
		     : PAGE_KERNEL_VVAR);
	BUILD_BUG_ON((unsigned long)__fix_to_virt(VSYSCALL_FIRST_PAGE) !=
		     (unsigned long)VSYSCALL_START);

	__set_fixmap(VVAR_PAGE, physaddr_vvar_page, PAGE_KERNEL_VVAR);
	BUILD_BUG_ON((unsigned long)__fix_to_virt(VVAR_PAGE) !=
		     (unsigned long)VVAR_ADDRESS);
}

static int __init vsyscall_init(void)
{
	BUG_ON(VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE));

	on_each_cpu(cpu_vsyscall_init, NULL, 1);
	/* notifier priority > KVM */
	hotcpu_notifier(cpu_vsyscall_notifier, 30);

	return 0;
}
__initcall(vsyscall_init);
Commit	Line	Data
1da177e4	1	/*
1da177e4 LT	2	* Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
	3	* Copyright 2003 Andi Kleen, SuSE Labs.
	4	*
5cec93c2 AL	5	* [ NOTE: this mechanism is now deprecated in favor of the vDSO. ]
5cec93c2 AL	6	*
1da177e4 LT	7	* Thanks to hpa@transmeta.com for some useful hint.
	8	* Special thanks to Ingo Molnar for his early experience with
	9	* a different vsyscall implementation for Linux/IA32 and for the name.
	10	*
	11	* vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
	12	* at virtual address -10Mbyte+1024bytes etc... There are at max 4
	13	* vsyscalls. One vsyscall can reserve more than 1 slot to avoid
	14	* jumping out of line if necessary. We cannot add more with this
	15	* mechanism because older kernels won't return -ENOSYS.
1da177e4	16	*
5cec93c2 AL	17	* Note: the concept clashes with user mode linux. UML users should
5cec93c2 AL	18	* use the vDSO.
1da177e4 LT	19	*/
	20
	21	#include <linux/time.h>
	22	#include <linux/init.h>
	23	#include <linux/kernel.h>
	24	#include <linux/timer.h>
	25	#include <linux/seqlock.h>
	26	#include <linux/jiffies.h>
	27	#include <linux/sysctl.h>
29574022	28	#include <linux/topology.h>
7460ed28	29	#include <linux/clocksource.h>
c08c8205	30	#include <linux/getcpu.h>
8c131af1 AK	31	#include <linux/cpu.h>
	32	#include <linux/smp.h>
	33	#include <linux/notifier.h>
5cec93c2 AL	34	#include <linux/syscalls.h>
5cec93c2 AL	35	#include <linux/ratelimit.h>
1da177e4 LT	36
	37	#include <asm/vsyscall.h>
	38	#include <asm/pgtable.h>
c9712944	39	#include <asm/compat.h>
1da177e4	40	#include <asm/page.h>
7460ed28	41	#include <asm/unistd.h>
1da177e4 LT	42	#include <asm/fixmap.h>
	43	#include <asm/errno.h>
	44	#include <asm/io.h>
c08c8205 VP	45	#include <asm/segment.h>
	46	#include <asm/desc.h>
	47	#include <asm/topology.h>
2aae950b	48	#include <asm/vgtod.h>
5cec93c2	49	#include <asm/traps.h>
1da177e4	50
c149a665 AL	51	#define CREATE_TRACE_POINTS
	52	#include "vsyscall_trace.h"
	53
8c49d9a7 AL	54	DEFINE_VVAR(int, vgetcpu_mode);
8c49d9a7 AL	55	DEFINE_VVAR(struct vsyscall_gtod_data, vsyscall_gtod_data) =
1da177e4	56	{
c4dbe54e	57	.lock = __SEQLOCK_UNLOCKED(__vsyscall_gtod_data.lock),
7460ed28	58	};
1da177e4	59
2b666859	60	static enum { EMULATE, NATIVE, NONE } vsyscall_mode = NATIVE;
3ae36655 AL	61
	62	static int __init vsyscall_setup(char *str)
	63	{
	64	if (str) {
	65	if (!strcmp("emulate", str))
	66	vsyscall_mode = EMULATE;
	67	else if (!strcmp("native", str))
	68	vsyscall_mode = NATIVE;
	69	else if (!strcmp("none", str))
	70	vsyscall_mode = NONE;
	71	else
	72	return -EINVAL;
	73
	74	return 0;
	75	}
	76
	77	return -EINVAL;
	78	}
	79	early_param("vsyscall", vsyscall_setup);
	80
2c622148 TB	81	void update_vsyscall_tz(void)
	82	{
	83	unsigned long flags;
	84
	85	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
	86	/* sys_tz has changed */
	87	vsyscall_gtod_data.sys_tz = sys_tz;
	88	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
	89	}
	90
7615856e JS	91	void update_vsyscall(struct timespec wall_time, struct timespec wtm,
7615856e JS	92	struct clocksource *clock, u32 mult)
1da177e4	93	{
7460ed28	94	unsigned long flags;
1da177e4	95
7460ed28	96	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
5cec93c2	97
7460ed28	98	/* copy vsyscall data */
98d0ac38	99	vsyscall_gtod_data.clock.vclock_mode = clock->archdata.vclock_mode;
5cec93c2 AL	100	vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
	101	vsyscall_gtod_data.clock.mask = clock->mask;
	102	vsyscall_gtod_data.clock.mult = mult;
	103	vsyscall_gtod_data.clock.shift = clock->shift;
	104	vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
	105	vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
	106	vsyscall_gtod_data.wall_to_monotonic = *wtm;
	107	vsyscall_gtod_data.wall_time_coarse = __current_kernel_time();
	108
7460ed28	109	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
1da177e4 LT	110	}
1da177e4 LT	111
5cec93c2 AL	112	static void warn_bad_vsyscall(const char level, struct pt_regs regs,
5cec93c2 AL	113	const char *message)
1da177e4	114	{
5cec93c2 AL	115	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
5cec93c2 AL	116	struct task_struct *tsk;
1da177e4	117
5cec93c2 AL	118	if (!show_unhandled_signals \|\| !__ratelimit(&rs))
5cec93c2 AL	119	return;
1da177e4	120
5cec93c2	121	tsk = current;
7460ed28	122
c9712944	123	printk("%s%s[%d] %s ip:%lx cs:%lx sp:%lx ax:%lx si:%lx di:%lx\n",
5cec93c2	124	level, tsk->comm, task_pid_nr(tsk),
3ae36655	125	message, regs->ip, regs->cs,
c9712944 AL	126	regs->sp, regs->ax, regs->si, regs->di);
	127	}
	128
	129	static int addr_to_vsyscall_nr(unsigned long addr)
	130	{
	131	int nr;
	132
	133	if ((addr & ~0xC00UL) != VSYSCALL_START)
	134	return -EINVAL;
	135
	136	nr = (addr & 0xC00UL) >> 10;
	137	if (nr >= 3)
	138	return -EINVAL;
	139
	140	return nr;
1da177e4 LT	141	}
1da177e4 LT	142
4fc34901 AL	143	static bool write_ok_or_segv(unsigned long ptr, size_t size)
	144	{
	145	/*
	146	* XXX: if access_ok, get_user, and put_user handled
	147	* sig_on_uaccess_error, this could go away.
	148	*/
	149
	150	if (!access_ok(VERIFY_WRITE, (void __user *)ptr, size)) {
	151	siginfo_t info;
	152	struct thread_struct *thread = &current->thread;
	153
	154	thread->error_code = 6; /* user fault, no page, write */
	155	thread->cr2 = ptr;
	156	thread->trap_no = 14;
	157
	158	memset(&info, 0, sizeof(info));
	159	info.si_signo = SIGSEGV;
	160	info.si_errno = 0;
	161	info.si_code = SEGV_MAPERR;
	162	info.si_addr = (void __user *)ptr;
	163
	164	force_sig_info(SIGSEGV, &info, current);
	165	return false;
	166	} else {
	167	return true;
	168	}
	169	}
	170
3ae36655	171	bool emulate_vsyscall(struct pt_regs *regs, unsigned long address)
1da177e4	172	{
5cec93c2 AL	173	struct task_struct *tsk;
	174	unsigned long caller;
	175	int vsyscall_nr;
4fc34901	176	int prev_sig_on_uaccess_error;
5cec93c2 AL	177	long ret;
5cec93c2 AL	178
3ae36655 AL	179	/*
	180	* No point in checking CS -- the only way to get here is a user mode
	181	* trap to a high address, which means that we're in 64-bit user code.
	182	*/
5cec93c2	183
3ae36655	184	WARN_ON_ONCE(address != regs->ip);
c9712944	185
3ae36655 AL	186	if (vsyscall_mode == NONE) {
	187	warn_bad_vsyscall(KERN_INFO, regs,
	188	"vsyscall attempted with vsyscall=none");
	189	return false;
c9712944 AL	190	}
c9712944 AL	191
3ae36655	192	vsyscall_nr = addr_to_vsyscall_nr(address);
c149a665 AL	193
	194	trace_emulate_vsyscall(vsyscall_nr);
	195
c9712944 AL	196	if (vsyscall_nr < 0) {
c9712944 AL	197	warn_bad_vsyscall(KERN_WARNING, regs,
3ae36655	198	"misaligned vsyscall (exploit attempt or buggy program) -- look up the vsyscall kernel parameter if you need a workaround");
5cec93c2 AL	199	goto sigsegv;
5cec93c2 AL	200	}
d0aff6e6	201
5cec93c2	202	if (get_user(caller, (unsigned long __user *)regs->sp) != 0) {
3ae36655 AL	203	warn_bad_vsyscall(KERN_WARNING, regs,
3ae36655 AL	204	"vsyscall with bad stack (exploit attempt?)");
5cec93c2 AL	205	goto sigsegv;
5cec93c2 AL	206	}
8c73626a	207
5cec93c2 AL	208	tsk = current;
	209	if (seccomp_mode(&tsk->seccomp))
	210	do_exit(SIGKILL);
	211
4fc34901 AL	212	/*
	213	* With a real vsyscall, page faults cause SIGSEGV. We want to
	214	* preserve that behavior to make writing exploits harder.
	215	*/
	216	prev_sig_on_uaccess_error = current_thread_info()->sig_on_uaccess_error;
	217	current_thread_info()->sig_on_uaccess_error = 1;
	218
	219	/*
	220	* 0 is a valid user pointer (in the access_ok sense) on 32-bit and
	221	* 64-bit, so we don't need to special-case it here. For all the
	222	* vsyscalls, 0 means "don't write anything" not "write it at
	223	* address 0".
	224	*/
	225	ret = -EFAULT;
5cec93c2 AL	226	switch (vsyscall_nr) {
5cec93c2 AL	227	case 0:
4fc34901 AL	228	if (!write_ok_or_segv(regs->di, sizeof(struct timeval)) \|\|
	229	!write_ok_or_segv(regs->si, sizeof(struct timezone)))
	230	break;
	231
5cec93c2 AL	232	ret = sys_gettimeofday(
	233	(struct timeval __user *)regs->di,
	234	(struct timezone __user *)regs->si);
	235	break;
	236
	237	case 1:
4fc34901 AL	238	if (!write_ok_or_segv(regs->di, sizeof(time_t)))
	239	break;
	240
5cec93c2 AL	241	ret = sys_time((time_t __user *)regs->di);
	242	break;
	243
	244	case 2:
4fc34901 AL	245	if (!write_ok_or_segv(regs->di, sizeof(unsigned)) \|\|
	246	!write_ok_or_segv(regs->si, sizeof(unsigned)))
	247	break;
	248
5cec93c2 AL	249	ret = sys_getcpu((unsigned __user *)regs->di,
	250	(unsigned __user *)regs->si,
	251	0);
	252	break;
5cec93c2	253	}
8c73626a	254
4fc34901 AL	255	current_thread_info()->sig_on_uaccess_error = prev_sig_on_uaccess_error;
4fc34901 AL	256
5cec93c2	257	if (ret == -EFAULT) {
4fc34901	258	/* Bad news -- userspace fed a bad pointer to a vsyscall. */
5cec93c2 AL	259	warn_bad_vsyscall(KERN_INFO, regs,
5cec93c2 AL	260	"vsyscall fault (exploit attempt?)");
4fc34901 AL	261
	262	/*
	263	* If we failed to generate a signal for any reason,
	264	* generate one here. (This should be impossible.)
	265	*/
	266	if (WARN_ON_ONCE(!sigismember(&tsk->pending.signal, SIGBUS) &&
	267	!sigismember(&tsk->pending.signal, SIGSEGV)))
	268	goto sigsegv;
	269
	270	return true; /* Don't emulate the ret. */
5cec93c2	271	}
8c73626a	272
5cec93c2	273	regs->ax = ret;
1da177e4	274
5cec93c2 AL	275	/* Emulate a ret instruction. */
	276	regs->ip = caller;
	277	regs->sp += 8;
c08c8205	278
3ae36655	279	return true;
5cec93c2 AL	280
5cec93c2 AL	281	sigsegv:
5cec93c2	282	force_sig(SIGSEGV, current);
3ae36655	283	return true;
1da177e4 LT	284	}
1da177e4 LT	285
5cec93c2 AL	286	/*
	287	* Assume __initcall executes before all user space. Hopefully kmod
	288	* doesn't violate that. We'll find out if it does.
	289	*/
8c131af1	290	static void __cpuinit vsyscall_set_cpu(int cpu)
c08c8205	291	{
fc8b8a60	292	unsigned long d;
c08c8205 VP	293	unsigned long node = 0;
c08c8205 VP	294	#ifdef CONFIG_NUMA
98c9e27a	295	node = cpu_to_node(cpu);
c08c8205	296	#endif
92cb7612	297	if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
8c131af1	298	write_rdtscp_aux((node << 12) \| cpu);
c08c8205	299
5cec93c2 AL	300	/*
	301	* Store cpu number in limit so that it can be loaded quickly
	302	* in user space in vgetcpu. (12 bits for the CPU and 8 bits for the node)
	303	*/
fc8b8a60 JF	304	d = 0x0f40000000000ULL;
	305	d \|= cpu;
	306	d \|= (node & 0xf) << 12;
	307	d \|= (node >> 4) << 48;
5cec93c2	308
fc8b8a60	309	write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
c08c8205 VP	310	}
c08c8205 VP	311
8c131af1 AK	312	static void __cpuinit cpu_vsyscall_init(void *arg)
	313	{
	314	/* preemption should be already off */
	315	vsyscall_set_cpu(raw_smp_processor_id());
	316	}
	317
	318	static int __cpuinit
	319	cpu_vsyscall_notifier(struct notifier_block n, unsigned long action, void arg)
	320	{
	321	long cpu = (long)arg;
5cec93c2	322
8bb78442	323	if (action == CPU_ONLINE \|\| action == CPU_ONLINE_FROZEN)
8691e5a8	324	smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
5cec93c2	325
8c131af1 AK	326	return NOTIFY_DONE;
	327	}
	328
e4026440	329	void __init map_vsyscall(void)
1da177e4	330	{
3ae36655 AL	331	extern char __vsyscall_page;
3ae36655 AL	332	unsigned long physaddr_vsyscall = __pa_symbol(&__vsyscall_page);
9fd67b4e AL	333	extern char __vvar_page;
9fd67b4e AL	334	unsigned long physaddr_vvar_page = __pa_symbol(&__vvar_page);
1da177e4	335
3ae36655 AL	336	__set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_vsyscall,
	337	vsyscall_mode == NATIVE
	338	? PAGE_KERNEL_VSYSCALL
	339	: PAGE_KERNEL_VVAR);
	340	BUILD_BUG_ON((unsigned long)__fix_to_virt(VSYSCALL_FIRST_PAGE) !=
	341	(unsigned long)VSYSCALL_START);
	342
9fd67b4e	343	__set_fixmap(VVAR_PAGE, physaddr_vvar_page, PAGE_KERNEL_VVAR);
3ae36655 AL	344	BUILD_BUG_ON((unsigned long)__fix_to_virt(VVAR_PAGE) !=
3ae36655 AL	345	(unsigned long)VVAR_ADDRESS);
1da177e4 LT	346	}
	347
	348	static int __init vsyscall_init(void)
	349	{
5cec93c2 AL	350	BUG_ON(VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE));
5cec93c2 AL	351
15c8b6c1	352	on_each_cpu(cpu_vsyscall_init, NULL, 1);
be43f83d SY	353	/* notifier priority > KVM */
be43f83d SY	354	hotcpu_notifier(cpu_vsyscall_notifier, 30);
5cec93c2	355
1da177e4 LT	356	return 0;
1da177e4 LT	357	}
1da177e4	358	__initcall(vsyscall_init);