[deliverable/linux.git] / arch / x86 / xen / xen-asm_64.S

/*
	Asm versions of Xen pv-ops, suitable for either direct use or inlining.
	The inline versions are the same as the direct-use versions, with the
	pre- and post-amble chopped off.

	This code is encoded for size rather than absolute efficiency,
	with a view to being able to inline as much as possible.

	We only bother with direct forms (ie, vcpu in pda) of the operations
	here; the indirect forms are better handled in C, since they're
	generally too large to inline anyway.
 */

#include <asm/errno.h>
#include <asm/percpu.h>
#include <asm/processor-flags.h>
#include <asm/segment.h>

#include <xen/interface/xen.h>

#include "xen-asm.h"

ENTRY(xen_adjust_exception_frame)
	mov 8+0(%rsp),%rcx
	mov 8+8(%rsp),%r11
	ret $16

hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
/*
	Xen64 iret frame:

	ss
	rsp
	rflags
	cs
	rip		<-- standard iret frame

	flags

	rcx		}
	r11		}<-- pushed by hypercall page
rsp ->	rax		}
 */
ENTRY(xen_iret)
	pushq $0
1:	jmp hypercall_iret
ENDPATCH(xen_iret)
RELOC(xen_iret, 1b+1)

/*
	sysexit is not used for 64-bit processes, so it's
	only ever used to return to 32-bit compat userspace.
 */
ENTRY(xen_sysexit)
	pushq $__USER32_DS
	pushq %rcx
	pushq $X86_EFLAGS_IF
	pushq $__USER32_CS
	pushq %rdx

	pushq $0
1:	jmp hypercall_iret
ENDPATCH(xen_sysexit)
RELOC(xen_sysexit, 1b+1)

ENTRY(xen_sysret64)
	/* We're already on the usermode stack at this point, but still
	   with the kernel gs, so we can easily switch back */
	movq %rsp, PER_CPU_VAR(old_rsp)
	movq PER_CPU_VAR(kernel_stack),%rsp

	pushq $__USER_DS
	pushq PER_CPU_VAR(old_rsp)
	pushq %r11
	pushq $__USER_CS
	pushq %rcx

	pushq $VGCF_in_syscall
1:	jmp hypercall_iret
ENDPATCH(xen_sysret64)
RELOC(xen_sysret64, 1b+1)

ENTRY(xen_sysret32)
	/* We're already on the usermode stack at this point, but still
	   with the kernel gs, so we can easily switch back */
	movq %rsp, PER_CPU_VAR(old_rsp)
	movq PER_CPU_VAR(kernel_stack), %rsp

	pushq $__USER32_DS
	pushq PER_CPU_VAR(old_rsp)
	pushq %r11
	pushq $__USER32_CS
	pushq %rcx

	pushq $VGCF_in_syscall
1:	jmp hypercall_iret
ENDPATCH(xen_sysret32)
RELOC(xen_sysret32, 1b+1)

/*
	Xen handles syscall callbacks much like ordinary exceptions,
	which means we have:
	 - kernel gs
	 - kernel rsp
	 - an iret-like stack frame on the stack (including rcx and r11):
		ss
		rsp
		rflags
		cs
		rip
		r11
	rsp->	rcx

	In all the entrypoints, we undo all that to make it look
	like a CPU-generated syscall/sysenter and jump to the normal
	entrypoint.
 */

.macro undo_xen_syscall
	mov 0*8(%rsp),%rcx
	mov 1*8(%rsp),%r11
	mov 5*8(%rsp),%rsp
.endm

/* Normal 64-bit system call target */
ENTRY(xen_syscall_target)
	undo_xen_syscall
	jmp system_call_after_swapgs
ENDPROC(xen_syscall_target)

#ifdef CONFIG_IA32_EMULATION

/* 32-bit compat syscall target */
ENTRY(xen_syscall32_target)
	undo_xen_syscall
	jmp ia32_cstar_target
ENDPROC(xen_syscall32_target)

/* 32-bit compat sysenter target */
ENTRY(xen_sysenter_target)
	undo_xen_syscall
	jmp ia32_sysenter_target
ENDPROC(xen_sysenter_target)

#else /* !CONFIG_IA32_EMULATION */

ENTRY(xen_syscall32_target)
ENTRY(xen_sysenter_target)
	lea 16(%rsp), %rsp	/* strip %rcx,%r11 */
	mov $-ENOSYS, %rax
	pushq $VGCF_in_syscall
	jmp hypercall_iret
ENDPROC(xen_syscall32_target)
ENDPROC(xen_sysenter_target)

#endif	/* CONFIG_IA32_EMULATION */
Commit	Line	Data
cdacc127 JF	1	/*
	2	Asm versions of Xen pv-ops, suitable for either direct use or inlining.
	3	The inline versions are the same as the direct-use versions, with the
	4	pre- and post-amble chopped off.
	5
	6	This code is encoded for size rather than absolute efficiency,
	7	with a view to being able to inline as much as possible.
	8
	9	We only bother with direct forms (ie, vcpu in pda) of the operations
	10	here; the indirect forms are better handled in C, since they're
	11	generally too large to inline anyway.
	12	*/
	13
6fcac6d3	14	#include <asm/errno.h>
9af45651	15	#include <asm/percpu.h>
5393744b JF	16	#include <asm/processor-flags.h>
5393744b JF	17	#include <asm/segment.h>
cdacc127 JF	18
	19	#include <xen/interface/xen.h>
	20
5393744b	21	#include "xen-asm.h"
cdacc127	22
997409d3 JF	23	ENTRY(xen_adjust_exception_frame)
	24	mov 8+0(%rsp),%rcx
	25	mov 8+8(%rsp),%r11
	26	ret $16
	27
6fcac6d3 JF	28	hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
	29	/*
	30	Xen64 iret frame:
	31
	32	ss
	33	rsp
	34	rflags
	35	cs
	36	rip <-- standard iret frame
	37
	38	flags
	39
	40	rcx }
	41	r11 }<-- pushed by hypercall page
	42	rsp -> rax }
	43	*/
cdacc127 JF	44	ENTRY(xen_iret)
cdacc127 JF	45	pushq $0
6fcac6d3 JF	46	1: jmp hypercall_iret
	47	ENDPATCH(xen_iret)
	48	RELOC(xen_iret, 1b+1)
cdacc127	49
6fcac6d3 JF	50	/*
	51	sysexit is not used for 64-bit processes, so it's
	52	only ever used to return to 32-bit compat userspace.
	53	*/
cdacc127	54	ENTRY(xen_sysexit)
6fcac6d3 JF	55	pushq $__USER32_DS
	56	pushq %rcx
	57	pushq $X86_EFLAGS_IF
	58	pushq $__USER32_CS
	59	pushq %rdx
	60
2dc1697e	61	pushq $0
6fcac6d3 JF	62	1: jmp hypercall_iret
	63	ENDPATCH(xen_sysexit)
	64	RELOC(xen_sysexit, 1b+1)
	65
	66	ENTRY(xen_sysret64)
	67	/* We're already on the usermode stack at this point, but still
	68	with the kernel gs, so we can easily switch back */
3d1e42a7	69	movq %rsp, PER_CPU_VAR(old_rsp)
9af45651	70	movq PER_CPU_VAR(kernel_stack),%rsp
6fcac6d3 JF	71
6fcac6d3 JF	72	pushq $__USER_DS
3d1e42a7	73	pushq PER_CPU_VAR(old_rsp)
6fcac6d3 JF	74	pushq %r11
	75	pushq $__USER_CS
	76	pushq %rcx
	77
	78	pushq $VGCF_in_syscall
	79	1: jmp hypercall_iret
	80	ENDPATCH(xen_sysret64)
	81	RELOC(xen_sysret64, 1b+1)
	82
	83	ENTRY(xen_sysret32)
	84	/* We're already on the usermode stack at this point, but still
	85	with the kernel gs, so we can easily switch back */
3d1e42a7	86	movq %rsp, PER_CPU_VAR(old_rsp)
9af45651	87	movq PER_CPU_VAR(kernel_stack), %rsp
6fcac6d3 JF	88
6fcac6d3 JF	89	pushq $__USER32_DS
3d1e42a7	90	pushq PER_CPU_VAR(old_rsp)
6fcac6d3 JF	91	pushq %r11
	92	pushq $__USER32_CS
	93	pushq %rcx
	94
	95	pushq $VGCF_in_syscall
	96	1: jmp hypercall_iret
	97	ENDPATCH(xen_sysret32)
	98	RELOC(xen_sysret32, 1b+1)
	99
	100	/*
	101	Xen handles syscall callbacks much like ordinary exceptions,
	102	which means we have:
	103	- kernel gs
	104	- kernel rsp
	105	- an iret-like stack frame on the stack (including rcx and r11):
	106	ss
	107	rsp
	108	rflags
	109	cs
	110	rip
	111	r11
	112	rsp-> rcx
	113
	114	In all the entrypoints, we undo all that to make it look
	115	like a CPU-generated syscall/sysenter and jump to the normal
	116	entrypoint.
	117	*/
	118
	119	.macro undo_xen_syscall
	120	mov 0*8(%rsp),%rcx
	121	mov 1*8(%rsp),%r11
	122	mov 5*8(%rsp),%rsp
	123	.endm
	124
	125	/* Normal 64-bit system call target */
	126	ENTRY(xen_syscall_target)
	127	undo_xen_syscall
	128	jmp system_call_after_swapgs
	129	ENDPROC(xen_syscall_target)
	130
	131	#ifdef CONFIG_IA32_EMULATION
	132
	133	/* 32-bit compat syscall target */
	134	ENTRY(xen_syscall32_target)
	135	undo_xen_syscall
	136	jmp ia32_cstar_target
	137	ENDPROC(xen_syscall32_target)
	138
	139	/* 32-bit compat sysenter target */
	140	ENTRY(xen_sysenter_target)
	141	undo_xen_syscall
	142	jmp ia32_sysenter_target
	143	ENDPROC(xen_sysenter_target)
	144
	145	#else /* !CONFIG_IA32_EMULATION */
	146
	147	ENTRY(xen_syscall32_target)
	148	ENTRY(xen_sysenter_target)
	149	lea 16(%rsp), %rsp /* strip %rcx,%r11 */
	150	mov $-ENOSYS, %rax
	151	pushq $VGCF_in_syscall
	152	jmp hypercall_iret
	153	ENDPROC(xen_syscall32_target)
	154	ENDPROC(xen_sysenter_target)
155
156	#endif /* CONFIG_IA32_EMULATION */