[deliverable/linux.git] / arch / x86 / ia32 / ia32entry.S

/*
 * Compatibility mode system call entry point for x86-64. 
 * 		
 * Copyright 2000-2002 Andi Kleen, SuSE Labs.
 */		 

#include <asm/dwarf2.h>
#include <asm/calling.h>
#include <asm/asm-offsets.h>
#include <asm/current.h>
#include <asm/errno.h>
#include <asm/ia32_unistd.h>	
#include <asm/thread_info.h>	
#include <asm/segment.h>
#include <asm/irqflags.h>
#include <asm/asm.h>
#include <asm/smap.h>
#include <linux/linkage.h>
#include <linux/err.h>

/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
#include <linux/elf-em.h>
#define AUDIT_ARCH_I386		(EM_386|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_LE	   0x40000000

#ifndef CONFIG_AUDITSYSCALL
#define sysexit_audit ia32_ret_from_sys_call
#define sysretl_audit ia32_ret_from_sys_call
#endif

	.section .entry.text, "ax"

	/* clobbers %rax */
	.macro  CLEAR_RREGS _r9=rax
	xorl 	%eax,%eax
	movq	%rax,R11(%rsp)
	movq	%rax,R10(%rsp)
	movq	%\_r9,R9(%rsp)
	movq	%rax,R8(%rsp)
	.endm

	/*
	 * Reload arg registers from stack in case ptrace changed them.
	 * We don't reload %eax because syscall_trace_enter() returned
	 * the %rax value we should see.  Instead, we just truncate that
	 * value to 32 bits again as we did on entry from user mode.
	 * If it's a new value set by user_regset during entry tracing,
	 * this matches the normal truncation of the user-mode value.
	 * If it's -1 to make us punt the syscall, then (u32)-1 is still
	 * an appropriately invalid value.
	 */
	.macro LOAD_ARGS32 _r9=0
	.if \_r9
	movl R9(%rsp),%r9d
	.endif
	movl RCX(%rsp),%ecx
	movl RDX(%rsp),%edx
	movl RSI(%rsp),%esi
	movl RDI(%rsp),%edi
	movl %eax,%eax			/* zero extension */
	.endm
	
	.macro CFI_STARTPROC32 simple
	CFI_STARTPROC	\simple
	CFI_UNDEFINED	r8
	CFI_UNDEFINED	r9
	CFI_UNDEFINED	r10
	CFI_UNDEFINED	r11
	CFI_UNDEFINED	r12
	CFI_UNDEFINED	r13
	CFI_UNDEFINED	r14
	CFI_UNDEFINED	r15
	.endm

#ifdef CONFIG_PARAVIRT
ENTRY(native_usergs_sysret32)
	swapgs
	sysretl
ENDPROC(native_usergs_sysret32)

ENTRY(native_irq_enable_sysexit)
	swapgs
	sti
	sysexit
ENDPROC(native_irq_enable_sysexit)
#endif

/*
 * 32bit SYSENTER instruction entry.
 *
 * SYSENTER loads ss, rsp, cs, and rip from previously programmed MSRs.
 * IF and VM in rflags are cleared (IOW: interrupts are off).
 * SYSENTER does not save anything on the stack,
 * and does not save old rip (!!!) and rflags.
 *
 * Arguments:
 * eax  system call number
 * ebx  arg1
 * ecx  arg2
 * edx  arg3
 * esi  arg4
 * edi  arg5
 * ebp  user stack
 * 0(%ebp) arg6
 *
 * This is purely a fast path. For anything complicated we use the int 0x80
 * path below. We set up a complete hardware stack frame to share code
 * with the int 0x80 path.
 */
ENTRY(ia32_sysenter_target)
	CFI_STARTPROC32	simple
	CFI_SIGNAL_FRAME
	CFI_DEF_CFA	rsp,0
	CFI_REGISTER	rsp,rbp

	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
	SWAPGS_UNSAFE_STACK
	movq	PER_CPU_VAR(cpu_tss + TSS_sp0), %rsp
	ENABLE_INTERRUPTS(CLBR_NONE)

	/* Zero-extending 32-bit regs, do not remove */
	movl	%ebp, %ebp
	movl	%eax, %eax

	movl	ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
	CFI_REGISTER rip,r10

	/* Construct struct pt_regs on stack */
	pushq_cfi	$__USER32_DS		/* pt_regs->ss */
	pushq_cfi	%rbp			/* pt_regs->sp */
	CFI_REL_OFFSET	rsp,0
	pushfq_cfi				/* pt_regs->flags */
	pushq_cfi	$__USER32_CS		/* pt_regs->cs */
	pushq_cfi	%r10 /* pt_regs->ip = thread_info->sysenter_return */
	CFI_REL_OFFSET	rip,0
	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
	pushq_cfi_reg	rdi			/* pt_regs->di */
	pushq_cfi_reg	rsi			/* pt_regs->si */
	pushq_cfi_reg	rdx			/* pt_regs->dx */
	pushq_cfi_reg	rcx			/* pt_regs->cx */
	pushq_cfi_reg	rax			/* pt_regs->ax */
	cld
	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
	CFI_ADJUST_CFA_OFFSET 10*8

	/*
	 * no need to do an access_ok check here because rbp has been
	 * 32bit zero extended
	 */
	ASM_STAC
1:	movl	(%rbp),%ebp
	_ASM_EXTABLE(1b,ia32_badarg)
	ASM_CLAC

	/*
	 * Sysenter doesn't filter flags, so we need to clear NT
	 * ourselves.  To save a few cycles, we can check whether
	 * NT was set instead of doing an unconditional popfq.
	 */
	testl $X86_EFLAGS_NT,EFLAGS(%rsp)
	jnz sysenter_fix_flags
sysenter_flags_fixed:

	orl     $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
	testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	CFI_REMEMBER_STATE
	jnz  sysenter_tracesys
	cmpq	$(IA32_NR_syscalls-1),%rax
	ja	ia32_badsys
sysenter_do_call:
	/* 32bit syscall -> 64bit C ABI argument conversion */
	movl	%edi,%r8d	/* arg5 */
	movl	%ebp,%r9d	/* arg6 */
	xchg	%ecx,%esi	/* rsi:arg2, rcx:arg4 */
	movl	%ebx,%edi	/* arg1 */
	movl	%edx,%edx	/* arg3 (zero extension) */
sysenter_dispatch:
	call	*ia32_sys_call_table(,%rax,8)
	movq	%rax,RAX(%rsp)
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
	testl	$_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz	sysexit_audit
sysexit_from_sys_call:
	andl    $~TS_COMPAT,ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
	/* clear IF, that popfq doesn't enable interrupts early */
	andl	$~0x200,EFLAGS(%rsp)
	movl	RIP(%rsp),%edx		/* User %eip */
	CFI_REGISTER rip,rdx
	RESTORE_RSI_RDI
	/* pop everything except ss,rsp,rflags slots */
	REMOVE_PT_GPREGS_FROM_STACK 3*8
	xorq	%r8,%r8
	xorq	%r9,%r9
	xorq	%r10,%r10
	xorq	%r11,%r11
	popfq_cfi
	/*CFI_RESTORE rflags*/
	popq_cfi %rcx				/* User %esp */
	CFI_REGISTER rsp,rcx
	TRACE_IRQS_ON
	/*
	 * 32bit SYSEXIT restores eip from edx, esp from ecx.
	 * cs and ss are loaded from MSRs.
	 */
	ENABLE_INTERRUPTS_SYSEXIT32

	CFI_RESTORE_STATE

#ifdef CONFIG_AUDITSYSCALL
	.macro auditsys_entry_common
	movl %esi,%r8d			/* 5th arg: 4th syscall arg */
	movl %ecx,%r9d			/*swap with edx*/
	movl %edx,%ecx			/* 4th arg: 3rd syscall arg */
	movl %r9d,%edx			/* 3rd arg: 2nd syscall arg */
	movl %ebx,%esi			/* 2nd arg: 1st syscall arg */
	movl %eax,%edi			/* 1st arg: syscall number */
	call __audit_syscall_entry
	movl RAX(%rsp),%eax	/* reload syscall number */
	cmpq $(IA32_NR_syscalls-1),%rax
	ja ia32_badsys
	movl %ebx,%edi			/* reload 1st syscall arg */
	movl RCX(%rsp),%esi	/* reload 2nd syscall arg */
	movl RDX(%rsp),%edx	/* reload 3rd syscall arg */
	movl RSI(%rsp),%ecx	/* reload 4th syscall arg */
	movl RDI(%rsp),%r8d	/* reload 5th syscall arg */
	.endm

	.macro auditsys_exit exit
	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz ia32_ret_from_sys_call
	TRACE_IRQS_ON
	ENABLE_INTERRUPTS(CLBR_NONE)
	movl %eax,%esi		/* second arg, syscall return value */
	cmpl $-MAX_ERRNO,%eax	/* is it an error ? */
	jbe 1f
	movslq %eax, %rsi	/* if error sign extend to 64 bits */
1:	setbe %al		/* 1 if error, 0 if not */
	movzbl %al,%edi		/* zero-extend that into %edi */
	call __audit_syscall_exit
	movq RAX(%rsp),%rax	/* reload syscall return value */
	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
	testl %edi, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jz \exit
	CLEAR_RREGS
	jmp int_with_check
	.endm

sysenter_auditsys:
	auditsys_entry_common
	movl %ebp,%r9d			/* reload 6th syscall arg */
	jmp sysenter_dispatch

sysexit_audit:
	auditsys_exit sysexit_from_sys_call
#endif

sysenter_fix_flags:
	pushq_cfi $(X86_EFLAGS_IF|X86_EFLAGS_FIXED)
	popfq_cfi
	jmp sysenter_flags_fixed

sysenter_tracesys:
#ifdef CONFIG_AUDITSYSCALL
	testl	$(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jz	sysenter_auditsys
#endif
	SAVE_EXTRA_REGS
	CLEAR_RREGS
	movq	$-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
	movq	%rsp,%rdi        /* &pt_regs -> arg1 */
	call	syscall_trace_enter
	LOAD_ARGS32  /* reload args from stack in case ptrace changed it */
	RESTORE_EXTRA_REGS
	cmpq	$(IA32_NR_syscalls-1),%rax
	ja	int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
	jmp	sysenter_do_call
	CFI_ENDPROC
ENDPROC(ia32_sysenter_target)

/*
 * 32bit SYSCALL instruction entry.
 *
 * 32bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
 * then loads new ss, cs, and rip from previously programmed MSRs.
 * rflags gets masked by a value from another MSR (so CLD and CLAC
 * are not needed). SYSCALL does not save anything on the stack
 * and does not change rsp.
 *
 * Note: rflags saving+masking-with-MSR happens only in Long mode
 * (in legacy 32bit mode, IF, RF and VM bits are cleared and that's it).
 * Don't get confused: rflags saving+masking depends on Long Mode Active bit
 * (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
 * or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
 *
 * Arguments:
 * eax  system call number
 * ecx  return address
 * ebx  arg1
 * ebp  arg2	(note: not saved in the stack frame, should not be touched)
 * edx  arg3
 * esi  arg4
 * edi  arg5
 * esp  user stack
 * 0(%esp) arg6
 *
 * This is purely a fast path. For anything complicated we use the int 0x80
 * path below. We set up a complete hardware stack frame to share code
 * with the int 0x80 path.
 */
ENTRY(ia32_cstar_target)
	CFI_STARTPROC32	simple
	CFI_SIGNAL_FRAME
	CFI_DEF_CFA	rsp,0
	CFI_REGISTER	rip,rcx
	/*CFI_REGISTER	rflags,r11*/

	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
	SWAPGS_UNSAFE_STACK
	movl	%esp,%r8d
	CFI_REGISTER	rsp,r8
	movq	PER_CPU_VAR(kernel_stack),%rsp
	ENABLE_INTERRUPTS(CLBR_NONE)

	/* Zero-extending 32-bit regs, do not remove */
	movl	%eax,%eax

	/* Construct struct pt_regs on stack */
	pushq_cfi	$__USER32_DS		/* pt_regs->ss */
	pushq_cfi	%r8			/* pt_regs->sp */
	CFI_REL_OFFSET rsp,0
	pushq_cfi	%r11			/* pt_regs->flags */
	pushq_cfi	$__USER32_CS		/* pt_regs->cs */
	pushq_cfi	%rcx			/* pt_regs->ip */
	CFI_REL_OFFSET rip,0
	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
	pushq_cfi_reg	rdi			/* pt_regs->di */
	pushq_cfi_reg	rsi			/* pt_regs->si */
	pushq_cfi_reg	rdx			/* pt_regs->dx */
	pushq_cfi_reg	rbp			/* pt_regs->cx */
	movl	%ebp,%ecx
	pushq_cfi_reg	rax			/* pt_regs->ax */
	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
	CFI_ADJUST_CFA_OFFSET 10*8

	/*
	 * no need to do an access_ok check here because r8 has been
	 * 32bit zero extended
	 */
	ASM_STAC
1:	movl	(%r8),%r9d
	_ASM_EXTABLE(1b,ia32_badarg)
	ASM_CLAC
	orl     $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
	testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	CFI_REMEMBER_STATE
	jnz   cstar_tracesys
	cmpq $IA32_NR_syscalls-1,%rax
	ja  ia32_badsys
cstar_do_call:
	/* 32bit syscall -> 64bit C ABI argument conversion */
	movl	%edi,%r8d	/* arg5 */
	/* r9 already loaded */	/* arg6 */
	xchg	%ecx,%esi	/* rsi:arg2, rcx:arg4 */
	movl	%ebx,%edi	/* arg1 */
	movl	%edx,%edx	/* arg3 (zero extension) */
cstar_dispatch:
	call *ia32_sys_call_table(,%rax,8)
	movq %rax,RAX(%rsp)
	DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF
	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz sysretl_audit
sysretl_from_sys_call:
	andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
	RESTORE_RSI_RDI_RDX
	movl RIP(%rsp),%ecx
	CFI_REGISTER rip,rcx
	movl EFLAGS(%rsp),%r11d
	/*CFI_REGISTER rflags,r11*/
	xorq	%r10,%r10
	xorq	%r9,%r9
	xorq	%r8,%r8
	TRACE_IRQS_ON
	movl RSP(%rsp),%esp
	CFI_RESTORE rsp
	/*
	 * 64bit->32bit SYSRET restores eip from ecx,
	 * eflags from r11 (but RF and VM bits are forced to 0),
	 * cs and ss are loaded from MSRs.
	 * (Note: 32bit->32bit SYSRET is different: since r11
	 * does not exist, it merely sets eflags.IF=1).
	 */
	USERGS_SYSRET32

#ifdef CONFIG_AUDITSYSCALL
cstar_auditsys:
	CFI_RESTORE_STATE
	movl %r9d,R9(%rsp)	/* register to be clobbered by call */
	auditsys_entry_common
	movl R9(%rsp),%r9d	/* reload 6th syscall arg */
	jmp cstar_dispatch

sysretl_audit:
	auditsys_exit sysretl_from_sys_call
#endif

cstar_tracesys:
#ifdef CONFIG_AUDITSYSCALL
	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jz cstar_auditsys
#endif
	xchgl %r9d,%ebp
	SAVE_EXTRA_REGS
	CLEAR_RREGS r9
	movq $-ENOSYS,RAX(%rsp)	/* ptrace can change this for a bad syscall */
	movq %rsp,%rdi        /* &pt_regs -> arg1 */
	call syscall_trace_enter
	LOAD_ARGS32 1	/* reload args from stack in case ptrace changed it */
	RESTORE_EXTRA_REGS
	xchgl %ebp,%r9d
	cmpq $(IA32_NR_syscalls-1),%rax
	ja int_ret_from_sys_call /* cstar_tracesys has set RAX(%rsp) */
	jmp cstar_do_call
END(ia32_cstar_target)
				
ia32_badarg:
	ASM_CLAC
	movq $-EFAULT,%rax
	jmp ia32_sysret
	CFI_ENDPROC

/*
 * Emulated IA32 system calls via int 0x80.
 *
 * Arguments:
 * eax  system call number
 * ebx  arg1
 * ecx  arg2
 * edx  arg3
 * esi  arg4
 * edi  arg5
 * ebp  arg6	(note: not saved in the stack frame, should not be touched)
 *
 * Notes:
 * Uses the same stack frame as the x86-64 version.
 * All registers except eax must be saved (but ptrace may violate that).
 * Arguments are zero extended. For system calls that want sign extension and
 * take long arguments a wrapper is needed. Most calls can just be called
 * directly.
 * Assumes it is only called from user space and entered with interrupts off.
 */

ENTRY(ia32_syscall)
	CFI_STARTPROC32	simple
	CFI_SIGNAL_FRAME
	CFI_DEF_CFA	rsp,5*8
	/*CFI_REL_OFFSET	ss,4*8 */
	CFI_REL_OFFSET	rsp,3*8
	/*CFI_REL_OFFSET	rflags,2*8 */
	/*CFI_REL_OFFSET	cs,1*8 */
	CFI_REL_OFFSET	rip,0*8

	/*
	 * Interrupts are off on entry.
	 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	 * it is too small to ever cause noticeable irq latency.
	 */
	PARAVIRT_ADJUST_EXCEPTION_FRAME
	SWAPGS
	ENABLE_INTERRUPTS(CLBR_NONE)

	/* Zero-extending 32-bit regs, do not remove */
	movl	%eax,%eax

	/* Construct struct pt_regs on stack (iret frame is already on stack) */
	pushq_cfi_reg	rax			/* pt_regs->orig_ax */
	pushq_cfi_reg	rdi			/* pt_regs->di */
	pushq_cfi_reg	rsi			/* pt_regs->si */
	pushq_cfi_reg	rdx			/* pt_regs->dx */
	pushq_cfi_reg	rcx			/* pt_regs->cx */
	pushq_cfi_reg	rax			/* pt_regs->ax */
	cld
	sub	$(10*8),%rsp /* pt_regs->r8-11,bp,bx,r12-15 not saved */
	CFI_ADJUST_CFA_OFFSET 10*8

	orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
	jnz ia32_tracesys
	cmpq $(IA32_NR_syscalls-1),%rax
	ja ia32_badsys
ia32_do_call:
	/* 32bit syscall -> 64bit C ABI argument conversion */
	movl %edi,%r8d	/* arg5 */
	movl %ebp,%r9d	/* arg6 */
	xchg %ecx,%esi	/* rsi:arg2, rcx:arg4 */
	movl %ebx,%edi	/* arg1 */
	movl %edx,%edx	/* arg3 (zero extension) */
	call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
ia32_sysret:
	movq %rax,RAX(%rsp)
ia32_ret_from_sys_call:
	CLEAR_RREGS
	jmp int_ret_from_sys_call

ia32_tracesys:
	SAVE_EXTRA_REGS
	CLEAR_RREGS
	movq $-ENOSYS,RAX(%rsp)	/* ptrace can change this for a bad syscall */
	movq %rsp,%rdi        /* &pt_regs -> arg1 */
	call syscall_trace_enter
	LOAD_ARGS32	/* reload args from stack in case ptrace changed it */
	RESTORE_EXTRA_REGS
	cmpq $(IA32_NR_syscalls-1),%rax
	ja  int_ret_from_sys_call	/* ia32_tracesys has set RAX(%rsp) */
	jmp ia32_do_call
END(ia32_syscall)

ia32_badsys:
	movq $0,ORIG_RAX(%rsp)
	movq $-ENOSYS,%rax
	jmp ia32_sysret

	CFI_ENDPROC
	
	.macro PTREGSCALL label, func
	ALIGN
GLOBAL(\label)
	leaq \func(%rip),%rax
	jmp  ia32_ptregs_common	
	.endm

	CFI_STARTPROC32

	PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn
	PTREGSCALL stub32_sigreturn, sys32_sigreturn
	PTREGSCALL stub32_execve, compat_sys_execve
	PTREGSCALL stub32_execveat, compat_sys_execveat
	PTREGSCALL stub32_fork, sys_fork
	PTREGSCALL stub32_vfork, sys_vfork

	ALIGN
GLOBAL(stub32_clone)
	leaq sys_clone(%rip),%rax
	mov	%r8, %rcx
	jmp  ia32_ptregs_common	

	ALIGN
ia32_ptregs_common:
	CFI_ENDPROC
	CFI_STARTPROC32	simple
	CFI_SIGNAL_FRAME
	CFI_DEF_CFA	rsp,SIZEOF_PTREGS
	CFI_REL_OFFSET	rax,RAX
	CFI_REL_OFFSET	rcx,RCX
	CFI_REL_OFFSET	rdx,RDX
	CFI_REL_OFFSET	rsi,RSI
	CFI_REL_OFFSET	rdi,RDI
	CFI_REL_OFFSET	rip,RIP
/*	CFI_REL_OFFSET	cs,CS*/
/*	CFI_REL_OFFSET	rflags,EFLAGS*/
	CFI_REL_OFFSET	rsp,RSP
/*	CFI_REL_OFFSET	ss,SS*/
	SAVE_EXTRA_REGS 8
	call *%rax
	RESTORE_EXTRA_REGS 8
	ret
	CFI_ENDPROC
END(ia32_ptregs_common)
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Compatibility mode system call entry point for x86-64.
	3	*
	4	* Copyright 2000-2002 Andi Kleen, SuSE Labs.
	5	*/
	6
	7	#include <asm/dwarf2.h>
	8	#include <asm/calling.h>
e2d5df93	9	#include <asm/asm-offsets.h>
1da177e4 LT	10	#include <asm/current.h>
	11	#include <asm/errno.h>
	12	#include <asm/ia32_unistd.h>
	13	#include <asm/thread_info.h>
	14	#include <asm/segment.h>
2601e64d	15	#include <asm/irqflags.h>
1ce6f868	16	#include <asm/asm.h>
63bcff2a	17	#include <asm/smap.h>
1da177e4	18	#include <linux/linkage.h>
d7e7528b	19	#include <linux/err.h>
1da177e4	20
5cbf1565 RM	21	/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
	22	#include <linux/elf-em.h>
	23	#define AUDIT_ARCH_I386 (EM_386\|__AUDIT_ARCH_LE)
	24	#define __AUDIT_ARCH_LE 0x40000000
	25
	26	#ifndef CONFIG_AUDITSYSCALL
24e35800 JB	27	#define sysexit_audit ia32_ret_from_sys_call
24e35800 JB	28	#define sysretl_audit ia32_ret_from_sys_call
5cbf1565 RM	29	#endif
5cbf1565 RM	30
ea714547 JO	31	.section .entry.text, "ax"
ea714547 JO	32
f2db9382 DV	33	/* clobbers %rax */
f2db9382 DV	34	.macro CLEAR_RREGS _r9=rax
1da177e4	35	xorl %eax,%eax
f2db9382 DV	36	movq %rax,R11(%rsp)
	37	movq %rax,R10(%rsp)
	38	movq %\_r9,R9(%rsp)
	39	movq %rax,R8(%rsp)
1da177e4 LT	40	.endm
1da177e4 LT	41
d4d67150 RM	42	/*
	43	* Reload arg registers from stack in case ptrace changed them.
	44	* We don't reload %eax because syscall_trace_enter() returned
eefdca04 RM	45	* the %rax value we should see. Instead, we just truncate that
	46	* value to 32 bits again as we did on entry from user mode.
	47	* If it's a new value set by user_regset during entry tracing,
	48	* this matches the normal truncation of the user-mode value.
	49	* If it's -1 to make us punt the syscall, then (u32)-1 is still
	50	* an appropriately invalid value.
d4d67150	51	*/
f2db9382	52	.macro LOAD_ARGS32 _r9=0
295286a8	53	.if \_r9
f2db9382	54	movl R9(%rsp),%r9d
295286a8	55	.endif
f2db9382 DV	56	movl RCX(%rsp),%ecx
	57	movl RDX(%rsp),%edx
	58	movl RSI(%rsp),%esi
	59	movl RDI(%rsp),%edi
eefdca04	60	movl %eax,%eax /* zero extension */
176df245 AK	61	.endm
176df245 AK	62
2765130b JB	63	.macro CFI_STARTPROC32 simple
	64	CFI_STARTPROC \simple
	65	CFI_UNDEFINED r8
	66	CFI_UNDEFINED r9
	67	CFI_UNDEFINED r10
	68	CFI_UNDEFINED r11
	69	CFI_UNDEFINED r12
	70	CFI_UNDEFINED r13
	71	CFI_UNDEFINED r14
	72	CFI_UNDEFINED r15
	73	.endm
	74
2be29982 JF	75	#ifdef CONFIG_PARAVIRT
	76	ENTRY(native_usergs_sysret32)
	77	swapgs
	78	sysretl
	79	ENDPROC(native_usergs_sysret32)
	80
	81	ENTRY(native_irq_enable_sysexit)
	82	swapgs
	83	sti
	84	sysexit
	85	ENDPROC(native_irq_enable_sysexit)
	86	#endif
	87
1da177e4 LT	88	/*
	89	* 32bit SYSENTER instruction entry.
	90	*
b87cf63e DV	91	* SYSENTER loads ss, rsp, cs, and rip from previously programmed MSRs.
	92	* IF and VM in rflags are cleared (IOW: interrupts are off).
	93	* SYSENTER does not save anything on the stack,
	94	* and does not save old rip (!!!) and rflags.
	95	*
1da177e4	96	* Arguments:
b87cf63e DV	97	* eax system call number
	98	* ebx arg1
	99	* ecx arg2
	100	* edx arg3
	101	* esi arg4
	102	* edi arg5
	103	* ebp user stack
	104	* 0(%ebp) arg6
	105	*
1da177e4	106	* This is purely a fast path. For anything complicated we use the int 0x80
b87cf63e	107	* path below. We set up a complete hardware stack frame to share code
1da177e4	108	* with the int 0x80 path.
b87cf63e	109	*/
1da177e4	110	ENTRY(ia32_sysenter_target)
2765130b	111	CFI_STARTPROC32 simple
adf14236	112	CFI_SIGNAL_FRAME
7effaa88 JB	113	CFI_DEF_CFA rsp,0
7effaa88 JB	114	CFI_REGISTER rsp,rbp
a232e3d5	115
2601e64d	116	/*
a232e3d5 DV	117	* Interrupts are off on entry.
	118	* We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	119	* it is too small to ever cause noticeable irq latency.
2601e64d	120	*/
a232e3d5 DV	121	SWAPGS_UNSAFE_STACK
a232e3d5 DV	122	movq PER_CPU_VAR(cpu_tss + TSS_sp0), %rsp
66804154	123	ENABLE_INTERRUPTS(CLBR_NONE)
a232e3d5	124
4ee8ec17 DV	125	/* Zero-extending 32-bit regs, do not remove */
	126	movl %ebp, %ebp
	127	movl %eax, %eax
	128
4c9c0e91	129	movl ASM_THREAD_INFO(TI_sysenter_return, %rsp, 0), %r10d
7effaa88	130	CFI_REGISTER rip,r10
4c9c0e91 DV	131
	132	/* Construct struct pt_regs on stack */
	133	pushq_cfi $__USER32_DS /* pt_regs->ss */
	134	pushq_cfi %rbp /* pt_regs->sp */
	135	CFI_REL_OFFSET rsp,0
	136	pushfq_cfi /* pt_regs->flags */
	137	pushq_cfi $__USER32_CS /* pt_regs->cs */
	138	pushq_cfi %r10 /* pt_regs->ip = thread_info->sysenter_return */
	139	CFI_REL_OFFSET rip,0
	140	pushq_cfi_reg rax /* pt_regs->orig_ax */
	141	pushq_cfi_reg rdi /* pt_regs->di */
	142	pushq_cfi_reg rsi /* pt_regs->si */
	143	pushq_cfi_reg rdx /* pt_regs->dx */
	144	pushq_cfi_reg rcx /* pt_regs->cx */
	145	pushq_cfi_reg rax /* pt_regs->ax */
1da177e4	146	cld
4c9c0e91 DV	147	sub $(108),%rsp / pt_regs->r8-11,bp,bx,r12-15 not saved */
	148	CFI_ADJUST_CFA_OFFSET 10*8
	149
b87cf63e DV	150	/*
	151	* no need to do an access_ok check here because rbp has been
	152	* 32bit zero extended
	153	*/
63bcff2a	154	ASM_STAC
295286a8	155	1: movl (%rbp),%ebp
1ce6f868	156	_ASM_EXTABLE(1b,ia32_badarg)
63bcff2a	157	ASM_CLAC
8c7aa698 AL	158
	159	/*
	160	* Sysenter doesn't filter flags, so we need to clear NT
	161	* ourselves. To save a few cycles, we can check whether
	162	* NT was set instead of doing an unconditional popfq.
	163	*/
f2db9382	164	testl $X86_EFLAGS_NT,EFLAGS(%rsp)
8c7aa698 AL	165	jnz sysenter_fix_flags
	166	sysenter_flags_fixed:
	167
dca5b52a IM	168	orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
dca5b52a IM	169	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
7effaa88	170	CFI_REMEMBER_STATE
1da177e4	171	jnz sysenter_tracesys
36d001c7	172	cmpq $(IA32_NR_syscalls-1),%rax
67d53ea5	173	ja ia32_badsys
d4d67150	174	sysenter_do_call:
14f6e953 DV	175	/* 32bit syscall -> 64bit C ABI argument conversion */
	176	movl %edi,%r8d /* arg5 */
	177	movl %ebp,%r9d /* arg6 */
	178	xchg %ecx,%esi /* rsi:arg2, rcx:arg4 */
	179	movl %ebx,%edi /* arg1 */
	180	movl %edx,%edx /* arg3 (zero extension) */
5cbf1565	181	sysenter_dispatch:
1da177e4	182	call *ia32_sys_call_table(,%rax,8)
f2db9382	183	movq %rax,RAX(%rsp)
66804154	184	DISABLE_INTERRUPTS(CLBR_NONE)
2601e64d	185	TRACE_IRQS_OFF
dca5b52a	186	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
5cbf1565 RM	187	jnz sysexit_audit
5cbf1565 RM	188	sysexit_from_sys_call:
dca5b52a	189	andl $~TS_COMPAT,ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
1da177e4	190	/* clear IF, that popfq doesn't enable interrupts early */
f2db9382 DV	191	andl $~0x200,EFLAGS(%rsp)
f2db9382 DV	192	movl RIP(%rsp),%edx /* User %eip */
36197c92	193	CFI_REGISTER rip,rdx
76f5df43	194	RESTORE_RSI_RDI
b87cf63e	195	/* pop everything except ss,rsp,rflags slots */
76f5df43	196	REMOVE_PT_GPREGS_FROM_STACK 3*8
24e35800 JB	197	xorq %r8,%r8
	198	xorq %r9,%r9
	199	xorq %r10,%r10
	200	xorq %r11,%r11
60cf637a	201	popfq_cfi
7effaa88	202	/CFI_RESTORE rflags/
60cf637a	203	popq_cfi %rcx /* User %esp */
7effaa88	204	CFI_REGISTER rsp,rcx
2601e64d	205	TRACE_IRQS_ON
b87cf63e DV	206	/*
	207	* 32bit SYSEXIT restores eip from edx, esp from ecx.
	208	* cs and ss are loaded from MSRs.
	209	*/
2be29982	210	ENABLE_INTERRUPTS_SYSEXIT32
1da177e4	211
8c7aa698 AL	212	CFI_RESTORE_STATE
8c7aa698 AL	213
5cbf1565 RM	214	#ifdef CONFIG_AUDITSYSCALL
5cbf1565 RM	215	.macro auditsys_entry_common
b4f0d375 RGB	216	movl %esi,%r8d /* 5th arg: 4th syscall arg */
	217	movl %ecx,%r9d /swap with edx/
	218	movl %edx,%ecx /* 4th arg: 3rd syscall arg */
	219	movl %r9d,%edx /* 3rd arg: 2nd syscall arg */
	220	movl %ebx,%esi /* 2nd arg: 1st syscall arg */
	221	movl %eax,%edi /* 1st arg: syscall number */
b05d8447	222	call __audit_syscall_entry
f2db9382	223	movl RAX(%rsp),%eax /* reload syscall number */
36d001c7	224	cmpq $(IA32_NR_syscalls-1),%rax
5cbf1565 RM	225	ja ia32_badsys
5cbf1565 RM	226	movl %ebx,%edi /* reload 1st syscall arg */
f2db9382 DV	227	movl RCX(%rsp),%esi /* reload 2nd syscall arg */
	228	movl RDX(%rsp),%edx /* reload 3rd syscall arg */
	229	movl RSI(%rsp),%ecx /* reload 4th syscall arg */
	230	movl RDI(%rsp),%r8d /* reload 5th syscall arg */
5cbf1565 RM	231	.endm
5cbf1565 RM	232
81766741	233	.macro auditsys_exit exit
dca5b52a	234	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
24e35800	235	jnz ia32_ret_from_sys_call
5cbf1565	236	TRACE_IRQS_ON
40a1ef95	237	ENABLE_INTERRUPTS(CLBR_NONE)
5cbf1565	238	movl %eax,%esi /* second arg, syscall return value */
d7e7528b	239	cmpl $-MAX_ERRNO,%eax /* is it an error ? */
f031cd25 EP	240	jbe 1f
	241	movslq %eax, %rsi /* if error sign extend to 64 bits */
	242	1: setbe %al /* 1 if error, 0 if not */
5cbf1565	243	movzbl %al,%edi /* zero-extend that into %edi */
d7e7528b	244	call __audit_syscall_exit
f2db9382	245	movq RAX(%rsp),%rax /* reload syscall return value */
5cbf1565	246	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
40a1ef95	247	DISABLE_INTERRUPTS(CLBR_NONE)
5cbf1565	248	TRACE_IRQS_OFF
dca5b52a	249	testl %edi, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
24e35800	250	jz \exit
f2db9382	251	CLEAR_RREGS
24e35800	252	jmp int_with_check
5cbf1565 RM	253	.endm
	254
	255	sysenter_auditsys:
5cbf1565 RM	256	auditsys_entry_common
	257	movl %ebp,%r9d /* reload 6th syscall arg */
	258	jmp sysenter_dispatch
	259
	260	sysexit_audit:
	261	auditsys_exit sysexit_from_sys_call
	262	#endif
	263
8c7aa698 AL	264	sysenter_fix_flags:
	265	pushq_cfi $(X86_EFLAGS_IF\|X86_EFLAGS_FIXED)
	266	popfq_cfi
	267	jmp sysenter_flags_fixed
	268
5cbf1565	269	sysenter_tracesys:
5cbf1565	270	#ifdef CONFIG_AUDITSYSCALL
dca5b52a	271	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
5cbf1565 RM	272	jz sysenter_auditsys
5cbf1565 RM	273	#endif
76f5df43	274	SAVE_EXTRA_REGS
1da177e4	275	CLEAR_RREGS
48ee679a	276	movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
1da177e4 LT	277	movq %rsp,%rdi /* &pt_regs -> arg1 */
1da177e4 LT	278	call syscall_trace_enter
f2db9382	279	LOAD_ARGS32 /* reload args from stack in case ptrace changed it */
76f5df43	280	RESTORE_EXTRA_REGS
36d001c7	281	cmpq $(IA32_NR_syscalls-1),%rax
48ee679a	282	ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
1da177e4 LT	283	jmp sysenter_do_call
1da177e4 LT	284	CFI_ENDPROC
4b787e0b	285	ENDPROC(ia32_sysenter_target)
1da177e4 LT	286
	287	/*
	288	* 32bit SYSCALL instruction entry.
	289	*
b87cf63e DV	290	* 32bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
	291	* then loads new ss, cs, and rip from previously programmed MSRs.
	292	* rflags gets masked by a value from another MSR (so CLD and CLAC
	293	* are not needed). SYSCALL does not save anything on the stack
	294	* and does not change rsp.
	295	*
	296	* Note: rflags saving+masking-with-MSR happens only in Long mode
	297	* (in legacy 32bit mode, IF, RF and VM bits are cleared and that's it).
	298	* Don't get confused: rflags saving+masking depends on Long Mode Active bit
	299	* (EFER.LMA=1), NOT on bitness of userspace where SYSCALL executes
	300	* or target CS descriptor's L bit (SYSCALL does not read segment descriptors).
	301	*
1da177e4	302	* Arguments:
b87cf63e DV	303	* eax system call number
	304	* ecx return address
	305	* ebx arg1
	306	* ebp arg2 (note: not saved in the stack frame, should not be touched)
	307	* edx arg3
	308	* esi arg4
	309	* edi arg5
	310	* esp user stack
	311	* 0(%esp) arg6
	312	*
1da177e4	313	* This is purely a fast path. For anything complicated we use the int 0x80
b87cf63e DV	314	* path below. We set up a complete hardware stack frame to share code
	315	* with the int 0x80 path.
	316	*/
1da177e4	317	ENTRY(ia32_cstar_target)
2765130b	318	CFI_STARTPROC32 simple
adf14236	319	CFI_SIGNAL_FRAME
ef593260	320	CFI_DEF_CFA rsp,0
7effaa88 JB	321	CFI_REGISTER rip,rcx
7effaa88 JB	322	/CFI_REGISTER rflags,r11/
a232e3d5 DV	323
	324	/*
	325	* Interrupts are off on entry.
	326	* We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	327	* it is too small to ever cause noticeable irq latency.
	328	*/
457da70e	329	SWAPGS_UNSAFE_STACK
1da177e4	330	movl %esp,%r8d
7effaa88	331	CFI_REGISTER rsp,r8
9af45651	332	movq PER_CPU_VAR(kernel_stack),%rsp
66804154	333	ENABLE_INTERRUPTS(CLBR_NONE)
a232e3d5	334
4ee8ec17 DV	335	/* Zero-extending 32-bit regs, do not remove */
	336	movl %eax,%eax
	337
4c9c0e91 DV	338	/* Construct struct pt_regs on stack */
	339	pushq_cfi $__USER32_DS /* pt_regs->ss */
	340	pushq_cfi %r8 /* pt_regs->sp */
	341	CFI_REL_OFFSET rsp,0
	342	pushq_cfi %r11 /* pt_regs->flags */
	343	pushq_cfi $__USER32_CS /* pt_regs->cs */
	344	pushq_cfi %rcx /* pt_regs->ip */
	345	CFI_REL_OFFSET rip,0
	346	pushq_cfi_reg rax /* pt_regs->orig_ax */
	347	pushq_cfi_reg rdi /* pt_regs->di */
	348	pushq_cfi_reg rsi /* pt_regs->si */
	349	pushq_cfi_reg rdx /* pt_regs->dx */
	350	pushq_cfi_reg rbp /* pt_regs->cx */
1da177e4	351	movl %ebp,%ecx
4c9c0e91 DV	352	pushq_cfi_reg rax /* pt_regs->ax */
	353	sub $(108),%rsp / pt_regs->r8-11,bp,bx,r12-15 not saved */
	354	CFI_ADJUST_CFA_OFFSET 10*8
	355
b87cf63e DV	356	/*
	357	* no need to do an access_ok check here because r8 has been
	358	* 32bit zero extended
	359	*/
63bcff2a	360	ASM_STAC
1da177e4	361	1: movl (%r8),%r9d
a3e859fe	362	_ASM_EXTABLE(1b,ia32_badarg)
63bcff2a	363	ASM_CLAC
dca5b52a IM	364	orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
dca5b52a IM	365	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
7effaa88	366	CFI_REMEMBER_STATE
1da177e4	367	jnz cstar_tracesys
36d001c7	368	cmpq $IA32_NR_syscalls-1,%rax
67d53ea5	369	ja ia32_badsys
295286a8	370	cstar_do_call:
14f6e953 DV	371	/* 32bit syscall -> 64bit C ABI argument conversion */
	372	movl %edi,%r8d /* arg5 */
	373	/* r9 already loaded / / arg6 */
	374	xchg %ecx,%esi /* rsi:arg2, rcx:arg4 */
	375	movl %ebx,%edi /* arg1 */
	376	movl %edx,%edx /* arg3 (zero extension) */
5cbf1565	377	cstar_dispatch:
1da177e4	378	call *ia32_sys_call_table(,%rax,8)
f2db9382	379	movq %rax,RAX(%rsp)
66804154	380	DISABLE_INTERRUPTS(CLBR_NONE)
2601e64d	381	TRACE_IRQS_OFF
dca5b52a	382	testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
5cbf1565 RM	383	jnz sysretl_audit
5cbf1565 RM	384	sysretl_from_sys_call:
dca5b52a	385	andl $~TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
76f5df43	386	RESTORE_RSI_RDI_RDX
f2db9382	387	movl RIP(%rsp),%ecx
7effaa88	388	CFI_REGISTER rip,rcx
f2db9382	389	movl EFLAGS(%rsp),%r11d
7effaa88	390	/CFI_REGISTER rflags,r11/
24e35800 JB	391	xorq %r10,%r10
	392	xorq %r9,%r9
	393	xorq %r8,%r8
2601e64d	394	TRACE_IRQS_ON
f2db9382	395	movl RSP(%rsp),%esp
7effaa88	396	CFI_RESTORE rsp
b87cf63e DV	397	/*
	398	* 64bit->32bit SYSRET restores eip from ecx,
	399	* eflags from r11 (but RF and VM bits are forced to 0),
	400	* cs and ss are loaded from MSRs.
	401	* (Note: 32bit->32bit SYSRET is different: since r11
	402	* does not exist, it merely sets eflags.IF=1).
	403	*/
2be29982	404	USERGS_SYSRET32
b87cf63e	405
5cbf1565 RM	406	#ifdef CONFIG_AUDITSYSCALL
5cbf1565 RM	407	cstar_auditsys:
7effaa88	408	CFI_RESTORE_STATE
f2db9382	409	movl %r9d,R9(%rsp) /* register to be clobbered by call */
5cbf1565	410	auditsys_entry_common
f2db9382	411	movl R9(%rsp),%r9d /* reload 6th syscall arg */
5cbf1565 RM	412	jmp cstar_dispatch
	413
	414	sysretl_audit:
81766741	415	auditsys_exit sysretl_from_sys_call
5cbf1565 RM	416	#endif
	417
	418	cstar_tracesys:
	419	#ifdef CONFIG_AUDITSYSCALL
dca5b52a	420	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT), ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
5cbf1565 RM	421	jz cstar_auditsys
5cbf1565 RM	422	#endif
ecd744ee	423	xchgl %r9d,%ebp
76f5df43	424	SAVE_EXTRA_REGS
f2db9382	425	CLEAR_RREGS r9
48ee679a	426	movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
1da177e4 LT	427	movq %rsp,%rdi /* &pt_regs -> arg1 */
1da177e4 LT	428	call syscall_trace_enter
f2db9382	429	LOAD_ARGS32 1 /* reload args from stack in case ptrace changed it */
76f5df43	430	RESTORE_EXTRA_REGS
ecd744ee	431	xchgl %ebp,%r9d
36d001c7	432	cmpq $(IA32_NR_syscalls-1),%rax
48ee679a	433	ja int_ret_from_sys_call /* cstar_tracesys has set RAX(%rsp) */
1da177e4	434	jmp cstar_do_call
4b787e0b	435	END(ia32_cstar_target)
1da177e4 LT	436
1da177e4 LT	437	ia32_badarg:
63bcff2a	438	ASM_CLAC
1da177e4 LT	439	movq $-EFAULT,%rax
	440	jmp ia32_sysret
	441	CFI_ENDPROC
	442
b87cf63e DV	443	/*
b87cf63e DV	444	* Emulated IA32 system calls via int 0x80.
1da177e4	445	*
b87cf63e DV	446	* Arguments:
	447	* eax system call number
	448	* ebx arg1
	449	* ecx arg2
	450	* edx arg3
	451	* esi arg4
	452	* edi arg5
	453	* ebp arg6 (note: not saved in the stack frame, should not be touched)
1da177e4 LT	454	*
1da177e4 LT	455	* Notes:
b87cf63e DV	456	* Uses the same stack frame as the x86-64 version.
b87cf63e DV	457	* All registers except eax must be saved (but ptrace may violate that).
1da177e4 LT	458	* Arguments are zero extended. For system calls that want sign extension and
	459	* take long arguments a wrapper is needed. Most calls can just be called
	460	* directly.
b87cf63e DV	461	* Assumes it is only called from user space and entered with interrupts off.
b87cf63e DV	462	*/
1da177e4 LT	463
1da177e4 LT	464	ENTRY(ia32_syscall)
cdc1793e	465	CFI_STARTPROC32 simple
adf14236	466	CFI_SIGNAL_FRAME
a232e3d5 DV	467	CFI_DEF_CFA rsp,5*8
	468	/CFI_REL_OFFSET ss,48 */
	469	CFI_REL_OFFSET rsp,3*8
	470	/CFI_REL_OFFSET rflags,28 */
	471	/CFI_REL_OFFSET cs,18 */
	472	CFI_REL_OFFSET rip,0*8
	473
2601e64d	474	/*
a232e3d5 DV	475	* Interrupts are off on entry.
	476	* We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
	477	* it is too small to ever cause noticeable irq latency.
2601e64d	478	*/
a232e3d5 DV	479	PARAVIRT_ADJUST_EXCEPTION_FRAME
a232e3d5 DV	480	SWAPGS
66804154	481	ENABLE_INTERRUPTS(CLBR_NONE)
a232e3d5	482
4ee8ec17 DV	483	/* Zero-extending 32-bit regs, do not remove */
	484	movl %eax,%eax
	485
4c9c0e91 DV	486	/* Construct struct pt_regs on stack (iret frame is already on stack) */
	487	pushq_cfi_reg rax /* pt_regs->orig_ax */
	488	pushq_cfi_reg rdi /* pt_regs->di */
	489	pushq_cfi_reg rsi /* pt_regs->si */
	490	pushq_cfi_reg rdx /* pt_regs->dx */
	491	pushq_cfi_reg rcx /* pt_regs->cx */
	492	pushq_cfi_reg rax /* pt_regs->ax */
1da177e4	493	cld
4c9c0e91 DV	494	sub $(108),%rsp / pt_regs->r8-11,bp,bx,r12-15 not saved */
	495	CFI_ADJUST_CFA_OFFSET 10*8
	496
dca5b52a IM	497	orl $TS_COMPAT, ASM_THREAD_INFO(TI_status, %rsp, SIZEOF_PTREGS)
dca5b52a IM	498	testl $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
1da177e4	499	jnz ia32_tracesys
36d001c7	500	cmpq $(IA32_NR_syscalls-1),%rax
c09249f8 RM	501	ja ia32_badsys
c09249f8 RM	502	ia32_do_call:
14f6e953 DV	503	/* 32bit syscall -> 64bit C ABI argument conversion */
	504	movl %edi,%r8d /* arg5 */
	505	movl %ebp,%r9d /* arg6 */
	506	xchg %ecx,%esi /* rsi:arg2, rcx:arg4 */
	507	movl %ebx,%edi /* arg1 */
	508	movl %edx,%edx /* arg3 (zero extension) */
1da177e4 LT	509	call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
1da177e4 LT	510	ia32_sysret:
f2db9382	511	movq %rax,RAX(%rsp)
24e35800	512	ia32_ret_from_sys_call:
f2db9382	513	CLEAR_RREGS
76f5df43	514	jmp int_ret_from_sys_call
1da177e4	515
76f5df43 DV	516	ia32_tracesys:
76f5df43 DV	517	SAVE_EXTRA_REGS
cdc1793e	518	CLEAR_RREGS
8ab32bb8	519	movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
1da177e4 LT	520	movq %rsp,%rdi /* &pt_regs -> arg1 */
1da177e4 LT	521	call syscall_trace_enter
f2db9382	522	LOAD_ARGS32 /* reload args from stack in case ptrace changed it */
76f5df43	523	RESTORE_EXTRA_REGS
36d001c7	524	cmpq $(IA32_NR_syscalls-1),%rax
c09249f8 RM	525	ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */
c09249f8 RM	526	jmp ia32_do_call
4b787e0b	527	END(ia32_syscall)
1da177e4 LT	528
1da177e4 LT	529	ia32_badsys:
f2db9382	530	movq $0,ORIG_RAX(%rsp)
24e35800 JB	531	movq $-ENOSYS,%rax
24e35800 JB	532	jmp ia32_sysret
1da177e4	533
1da177e4 LT	534	CFI_ENDPROC
1da177e4 LT	535
d2475b8f	536	.macro PTREGSCALL label, func
f6b2bc84 JB	537	ALIGN
f6b2bc84 JB	538	GLOBAL(\label)
1da177e4	539	leaq \func(%rip),%rax
1da177e4 LT	540	jmp ia32_ptregs_common
	541	.endm
	542
2765130b	543	CFI_STARTPROC32
7effaa88	544
3fe26fa3 AV	545	PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn
	546	PTREGSCALL stub32_sigreturn, sys32_sigreturn
	547	PTREGSCALL stub32_execve, compat_sys_execve
27d6ec7a	548	PTREGSCALL stub32_execveat, compat_sys_execveat
3fe26fa3 AV	549	PTREGSCALL stub32_fork, sys_fork
3fe26fa3 AV	550	PTREGSCALL stub32_vfork, sys_vfork
1da177e4	551
1d4b4b29 AV	552	ALIGN
	553	GLOBAL(stub32_clone)
	554	leaq sys_clone(%rip),%rax
	555	mov %r8, %rcx
	556	jmp ia32_ptregs_common
	557
f6b2bc84 JB	558	ALIGN
f6b2bc84 JB	559	ia32_ptregs_common:
2765130b JB	560	CFI_ENDPROC
2765130b JB	561	CFI_STARTPROC32 simple
adf14236	562	CFI_SIGNAL_FRAME
b3ab90b3	563	CFI_DEF_CFA rsp,SIZEOF_PTREGS
f2db9382 DV	564	CFI_REL_OFFSET rax,RAX
	565	CFI_REL_OFFSET rcx,RCX
	566	CFI_REL_OFFSET rdx,RDX
	567	CFI_REL_OFFSET rsi,RSI
	568	CFI_REL_OFFSET rdi,RDI
	569	CFI_REL_OFFSET rip,RIP
	570	/* CFI_REL_OFFSET cs,CS*/
	571	/* CFI_REL_OFFSET rflags,EFLAGS*/
	572	CFI_REL_OFFSET rsp,RSP
	573	/* CFI_REL_OFFSET ss,SS*/
76f5df43	574	SAVE_EXTRA_REGS 8
1da177e4	575	call *%rax
76f5df43 DV	576	RESTORE_EXTRA_REGS 8
76f5df43 DV	577	ret
1da177e4	578	CFI_ENDPROC
4b787e0b	579	END(ia32_ptregs_common)