arch/x86_64/kernel/entry.S

   1 /*
   2  *  linux/arch/x86_64/entry.S
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
   6  *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
   7  *
   8  *  $Id$
   9  */
  10
  11 /*
  12  * entry.S contains the system-call and fault low-level handling routines.
  13  *
  14  * NOTE: This code handles signal-recognition, which happens every time
  15  * after an interrupt and after each system call.
  16  *
  17  * Normal syscalls and interrupts don't save a full stack frame, this is
  18  * only done for syscall tracing, signals or fork/exec et.al.
  19  *
  20  * A note on terminology:
  21  * - top of stack: Architecture defined interrupt frame from SS to RIP
  22  * at the top of the kernel process stack.
  23  * - partial stack frame: partially saved registers upto R11.
  24  * - full stack frame: Like partial stack frame, but all register saved.
  25  *
  26  * TODO:
  27  * - schedule it carefully for the final hardware.
  28  */
  29
  30 #define ASSEMBLY 1
  31 #include <linux/config.h>
  32 #include <linux/linkage.h>
  33 #include <asm/segment.h>
  34 #include <asm/smp.h>
  35 #include <asm/cache.h>
  36 #include <asm/errno.h>
  37 #include <asm/dwarf2.h>
  38 #include <asm/calling.h>
  39 #include <asm/offset.h>
  40 #include <asm/msr.h>
  41 #include <asm/unistd.h>
  42 #include <asm/thread_info.h>
  43 #include <asm/hw_irq.h>
  44
  45         .code64
  46
  47 #ifdef CONFIG_PREEMPT
  48 #define preempt_stop cli
  49 #else
  50 #define preempt_stop
  51 #define retint_kernel retint_restore_args
  52 #endif
  53
  54 /*
  55  * C code is not supposed to know about undefined top of stack. Every time
  56  * a C function with an pt_regs argument is called from the SYSCALL based
  57  * fast path FIXUP_TOP_OF_STACK is needed.
  58  * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
  59  * manipulation.
  60  */
  61
  62         /* %rsp:at FRAMEEND */
  63         .macro FIXUP_TOP_OF_STACK tmp
  64         movq    %gs:pda_oldrsp,\tmp
  65         movq    \tmp,RSP(%rsp)
  66         movq    $__USER_DS,SS(%rsp)
  67         movq    $__USER_CS,CS(%rsp)
  68         movq    $-1,RCX(%rsp)
  69         movq    R11(%rsp),\tmp  /* get eflags */
  70         movq    \tmp,EFLAGS(%rsp)
  71         .endm
  72
  73         .macro RESTORE_TOP_OF_STACK tmp,offset=0
  74         movq   RSP-\offset(%rsp),\tmp
  75         movq   \tmp,%gs:pda_oldrsp
  76         movq   EFLAGS-\offset(%rsp),\tmp
  77         movq   \tmp,R11-\offset(%rsp)
  78         .endm
  79
  80         .macro FAKE_STACK_FRAME child_rip
  81         /* push in order ss, rsp, eflags, cs, rip */
  82         xorq %rax, %rax
  83         pushq %rax /* ss */
  84         CFI_ADJUST_CFA_OFFSET   8
  85         pushq %rax /* rsp */
  86         CFI_ADJUST_CFA_OFFSET   8
  87         CFI_OFFSET      rip,0
  88         pushq $(1<<9) /* eflags - interrupts on */
  89         CFI_ADJUST_CFA_OFFSET   8
  90         pushq $__KERNEL_CS /* cs */
  91         CFI_ADJUST_CFA_OFFSET   8
  92         pushq \child_rip /* rip */
  93         CFI_ADJUST_CFA_OFFSET   8
  94         CFI_OFFSET      rip,0
  95         pushq   %rax /* orig rax */
  96         CFI_ADJUST_CFA_OFFSET   8
  97         .endm
  98
  99         .macro UNFAKE_STACK_FRAME
 100         addq $8*6, %rsp
 101         CFI_ADJUST_CFA_OFFSET   -(6*8)
 102         .endm
 103
 104         .macro  CFI_DEFAULT_STACK
 105         CFI_ADJUST_CFA_OFFSET  (SS)
 106         CFI_OFFSET      r15,R15-SS
 107         CFI_OFFSET      r14,R14-SS
 108         CFI_OFFSET      r13,R13-SS
 109         CFI_OFFSET      r12,R12-SS
 110         CFI_OFFSET      rbp,RBP-SS
 111         CFI_OFFSET      rbx,RBX-SS
 112         CFI_OFFSET      r11,R11-SS
 113         CFI_OFFSET      r10,R10-SS
 114         CFI_OFFSET      r9,R9-SS
 115         CFI_OFFSET      r8,R8-SS
 116         CFI_OFFSET      rax,RAX-SS
 117         CFI_OFFSET      rcx,RCX-SS
 118         CFI_OFFSET      rdx,RDX-SS
 119         CFI_OFFSET      rsi,RSI-SS
 120         CFI_OFFSET      rdi,RDI-SS
 121         CFI_OFFSET      rsp,RSP-SS
 122         CFI_OFFSET      rip,RIP-SS
 123         .endm
 124 /*
 125  * A newly forked process directly context switches into this.
 126  */
 127 /* rdi: prev */
 128 ENTRY(ret_from_fork)
 129         CFI_STARTPROC
 130         CFI_DEFAULT_STACK
 131         call schedule_tail
 132         GET_THREAD_INFO(%rcx)
 133         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
 134         jnz rff_trace
 135 rff_action:
 136         RESTORE_REST
 137         testl $3,CS-ARGOFFSET(%rsp)     # from kernel_thread?
 138         je   int_ret_from_sys_call
 139         testl $_TIF_IA32,threadinfo_flags(%rcx)
 140         jnz  int_ret_from_sys_call
 141         RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
 142         jmp ret_from_sys_call
 143 rff_trace:
 144         movq %rsp,%rdi
 145         call syscall_trace_leave
 146         GET_THREAD_INFO(%rcx)
 147         jmp rff_action
 148         CFI_ENDPROC
 149
 150 /*
 151  * System call entry. Upto 6 arguments in registers are supported.
 152  *
 153  * SYSCALL does not save anything on the stack and does not change the
 154  * stack pointer.
 155  */
 156
 157 /*
 158  * Register setup:
 159  * rax  system call number
 160  * rdi  arg0
 161  * rcx  return address for syscall/sysret, C arg3
 162  * rsi  arg1
 163  * rdx  arg2
 164  * r10  arg3    (--> moved to rcx for C)
 165  * r8   arg4
 166  * r9   arg5
 167  * r11  eflags for syscall/sysret, temporary for C
 168  * r12-r15,rbp,rbx saved by C code, not touched.
 169  *
 170  * Interrupts are off on entry.
 171  * Only called from user space.
 172  *
 173  * XXX  if we had a free scratch register we could save the RSP into the stack frame
 174  *      and report it properly in ps. Unfortunately we haven't.
 175  */
 176
 177 ENTRY(system_call)
 178         CFI_STARTPROC
 179         swapgs
 180         movq    %rsp,%gs:pda_oldrsp
 181         movq    %gs:pda_kernelstack,%rsp
 182         sti
 183         SAVE_ARGS 8,1
 184         movq  %rax,ORIG_RAX-ARGOFFSET(%rsp)
 185         movq  %rcx,RIP-ARGOFFSET(%rsp)
 186         GET_THREAD_INFO(%rcx)
 187         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
 188         jnz tracesys
 189         cmpq $__NR_syscall_max,%rax
 190         ja badsys
 191         movq %r10,%rcx
 192         call *sys_call_table(,%rax,8)  # XXX:    rip relative
 193         movq %rax,RAX-ARGOFFSET(%rsp)
 194 /*
 195  * Syscall return path ending with SYSRET (fast path)
 196  * Has incomplete stack frame and undefined top of stack.
 197  */
 198         .globl ret_from_sys_call
 199 ret_from_sys_call:
 200         movl $_TIF_ALLWORK_MASK,%edi
 201         /* edi: flagmask */
 202 sysret_check:
 203         GET_THREAD_INFO(%rcx)
 204         cli
 205         movl threadinfo_flags(%rcx),%edx
 206         andl %edi,%edx
 207         jnz  sysret_careful
 208         movq RIP-ARGOFFSET(%rsp),%rcx
 209         RESTORE_ARGS 0,-ARG_SKIP,1
 210         movq    %gs:pda_oldrsp,%rsp
 211         swapgs
 212         sysretq
 213
 214         /* Handle reschedules */
 215         /* edx: work, edi: workmask */
 216 sysret_careful:
 217         bt $TIF_NEED_RESCHED,%edx
 218         jnc sysret_signal
 219         sti
 220         pushq %rdi
 221         call schedule
 222         popq  %rdi
 223         jmp sysret_check
 224
 225         /* Handle a signal */
 226         /* edx: work flags (arg3) */
 227 sysret_signal:
 228         sti
 229         leaq do_notify_resume(%rip),%rax
 230         leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
 231         xorl %esi,%esi # oldset -> arg2
 232         call ptregscall_common
 233         jmp sysret_check
 234
 235         /* Do syscall tracing */
 236 tracesys:
 237         SAVE_REST
 238         movq $-ENOSYS,RAX(%rsp)
 239         FIXUP_TOP_OF_STACK %rdi
 240         movq %rsp,%rdi
 241         call syscall_trace_enter
 242         LOAD_ARGS ARGOFFSET  /* reload args from stack in case ptrace changed it */
 243         RESTORE_REST
 244         cmpq $__NR_syscall_max,%rax
 245         ja  1f
 246         movq %r10,%rcx  /* fixup for C */
 247         call *sys_call_table(,%rax,8)
 248         movq %rax,RAX-ARGOFFSET(%rsp)
 249 1:      SAVE_REST
 250         movq %rsp,%rdi
 251         call syscall_trace_leave
 252         RESTORE_TOP_OF_STACK %rbx
 253         RESTORE_REST
 254         jmp ret_from_sys_call
 255
 256 badsys:
 257         movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
 258         jmp ret_from_sys_call
 259
 260 /*
 261  * Syscall return path ending with IRET.
 262  * Has correct top of stack, but partial stack frame.
 263  */
 264 ENTRY(int_ret_from_sys_call)
 265         cli
 266         testl $3,CS-ARGOFFSET(%rsp)
 267         je retint_restore_args
 268         movl $_TIF_ALLWORK_MASK,%edi
 269         /* edi: mask to check */
 270 int_with_check:
 271         GET_THREAD_INFO(%rcx)
 272         movl threadinfo_flags(%rcx),%edx
 273         andl %edi,%edx
 274         jnz   int_careful
 275         jmp   retint_swapgs
 276
 277         /* Either reschedule or signal or syscall exit tracking needed. */
 278         /* First do a reschedule test. */
 279         /* edx: work, edi: workmask */
 280 int_careful:
 281         bt $TIF_NEED_RESCHED,%edx
 282         jnc  int_very_careful
 283         sti
 284         pushq %rdi
 285         call schedule
 286         popq %rdi
 287         jmp int_with_check
 288
 289         /* handle signals and tracing -- both require a full stack frame */
 290 int_very_careful:
 291         sti
 292         SAVE_REST
 293         /* Check for syscall exit trace */
 294         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
 295         jz int_signal
 296         pushq %rdi
 297         leaq 8(%rsp),%rdi       # &ptregs -> arg1
 298         call syscall_trace_leave
 299         popq %rdi
 300         andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
 301         jmp int_restore_rest
 302
 303 int_signal:
 304         testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
 305         jz 1f
 306         movq %rsp,%rdi          # &ptregs -> arg1
 307         xorl %esi,%esi          # oldset -> arg2
 308         call do_notify_resume
 309 1:      movl $_TIF_NEED_RESCHED,%edi
 310 int_restore_rest:
 311         RESTORE_REST
 312         jmp int_with_check
 313         CFI_ENDPROC
 314
 315 /*
 316  * Certain special system calls that need to save a complete full stack frame.
 317  */
 318
 319         .macro PTREGSCALL label,func,arg
 320         .globl \label
 321 \label:
 322         leaq    \func(%rip),%rax
 323         leaq    -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
 324         jmp     ptregscall_common
 325         .endm
 326
 327         PTREGSCALL stub_clone, sys_clone, %r8
 328         PTREGSCALL stub_fork, sys_fork, %rdi
 329         PTREGSCALL stub_vfork, sys_vfork, %rdi
 330         PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
 331         PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
 332         PTREGSCALL stub_iopl, sys_iopl, %rsi
 333
 334 ENTRY(ptregscall_common)
 335         CFI_STARTPROC
 336         popq %r11
 337         CFI_ADJUST_CFA_OFFSET   -8
 338         SAVE_REST
 339         movq %r11, %r15
 340         FIXUP_TOP_OF_STACK %r11
 341         call *%rax
 342         RESTORE_TOP_OF_STACK %r11
 343         movq %r15, %r11
 344         RESTORE_REST
 345         pushq %r11
 346         CFI_ADJUST_CFA_OFFSET   8
 347         ret
 348         CFI_ENDPROC
 349
 350 ENTRY(stub_execve)
 351         CFI_STARTPROC
 352         popq %r11
 353         CFI_ADJUST_CFA_OFFSET   -8
 354         SAVE_REST
 355         movq %r11, %r15
 356         FIXUP_TOP_OF_STACK %r11
 357         call sys_execve
 358         GET_THREAD_INFO(%rcx)
 359         bt $TIF_IA32,threadinfo_flags(%rcx)
 360         jc exec_32bit
 361         RESTORE_TOP_OF_STACK %r11
 362         movq %r15, %r11
 363         RESTORE_REST
 364         push %r11
 365         ret
 366
 367 exec_32bit:
 368         CFI_ADJUST_CFA_OFFSET   REST_SKIP
 369         movq %rax,RAX(%rsp)
 370         RESTORE_REST
 371         jmp int_ret_from_sys_call
 372         CFI_ENDPROC
 373
 374 /*
 375  * sigreturn is special because it needs to restore all registers on return.
 376  * This cannot be done with SYSRET, so use the IRET return path instead.
 377  */
 378 ENTRY(stub_rt_sigreturn)
 379         CFI_STARTPROC
 380         addq $8, %rsp
 381         SAVE_REST
 382         movq %rsp,%rdi
 383         FIXUP_TOP_OF_STACK %r11
 384         call sys_rt_sigreturn
 385         movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
 386         RESTORE_REST
 387         jmp int_ret_from_sys_call
 388         CFI_ENDPROC
 389
 390 /*
 391  * Interrupt entry/exit.
 392  *
 393  * Interrupt entry points save only callee clobbered registers in fast path.
 394  *
 395  * Entry runs with interrupts off.
 396  */
 397
 398 /* 0(%rsp): interrupt number */
 399         .macro interrupt func
 400         CFI_STARTPROC   simple
 401         CFI_DEF_CFA     rsp,(SS-RDI)
 402         CFI_REL_OFFSET  rsp,(RSP-ORIG_RAX)
 403         CFI_REL_OFFSET  rip,(RIP-ORIG_RAX)
 404         cld
 405 #ifdef CONFIG_DEBUG_INFO
 406         SAVE_ALL
 407         movq %rsp,%rdi
 408         /*
 409          * Setup a stack frame pointer.  This allows gdb to trace
 410          * back to the original stack.
 411          */
 412         movq %rsp,%rbp
 413         CFI_DEF_CFA_REGISTER    rbp
 414 #else
 415         SAVE_ARGS
 416         leaq -ARGOFFSET(%rsp),%rdi      # arg1 for handler
 417 #endif
 418         testl $3,CS(%rdi)
 419         je 1f
 420         swapgs
 421 1:      addl $1,%gs:pda_irqcount        # RED-PEN should check preempt count
 422         movq %gs:pda_irqstackptr,%rax
 423         cmoveq %rax,%rsp
 424         pushq %rdi                      # save old stack
 425         call \func
 426         .endm
 427
 428 ENTRY(common_interrupt)
 429         interrupt do_IRQ
 430         /* 0(%rsp): oldrsp-ARGOFFSET */
 431 ret_from_intr:
 432         popq  %rdi
 433         cli
 434         subl $1,%gs:pda_irqcount
 435 #ifdef CONFIG_DEBUG_INFO
 436         movq RBP(%rdi),%rbp
 437 #endif
 438         leaq ARGOFFSET(%rdi),%rsp
 439 exit_intr:
 440         GET_THREAD_INFO(%rcx)
 441         testl $3,CS-ARGOFFSET(%rsp)
 442         je retint_kernel
 443
 444         /* Interrupt came from user space */
 445         /*
 446          * Has a correct top of stack, but a partial stack frame
 447          * %rcx: thread info. Interrupts off.
 448          */
 449 retint_with_reschedule:
 450         movl $_TIF_WORK_MASK,%edi
 451 retint_check:
 452         movl threadinfo_flags(%rcx),%edx
 453         andl %edi,%edx
 454         jnz  retint_careful
 455 retint_swapgs:
 456         cli
 457         swapgs
 458 retint_restore_args:
 459         cli
 460         RESTORE_ARGS 0,8,0
 461 iret_label:
 462         iretq
 463
 464         .section __ex_table,"a"
 465         .quad iret_label,bad_iret
 466         .previous
 467         .section .fixup,"ax"
 468         /* force a signal here? this matches i386 behaviour */
 469         /* running with kernel gs */
 470 bad_iret:
 471         movq $-9999,%rdi        /* better code? */
 472         jmp do_exit
 473         .previous
 474
 475         /* edi: workmask, edx: work */
 476 retint_careful:
 477         bt    $TIF_NEED_RESCHED,%edx
 478         jnc   retint_signal
 479         sti
 480         pushq %rdi
 481         call  schedule
 482         popq %rdi
 483         GET_THREAD_INFO(%rcx)
 484         cli
 485         jmp retint_check
 486
 487 retint_signal:
 488         sti
 489         SAVE_REST
 490         movq $-1,ORIG_RAX(%rsp)
 491         xorq %rsi,%rsi          # oldset
 492         movq %rsp,%rdi          # &pt_regs
 493         call do_notify_resume
 494         RESTORE_REST
 495         cli
 496         GET_THREAD_INFO(%rcx)
 497         jmp retint_check
 498
 499 #ifdef CONFIG_PREEMPT
 500         /* Returning to kernel space. Check if we need preemption */
 501         /* rcx:  threadinfo. interrupts off. */
 502         .p2align
 503 retint_kernel:
 504         cmpl $0,threadinfo_preempt_count(%rcx)
 505         jnz  retint_restore_args
 506         bt  $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
 507         jnc  retint_restore_args
 508         bt   $9,EFLAGS-ARGOFFSET(%rsp)  /* interrupts off? */
 509         jnc  retint_restore_args
 510         call preempt_schedule_irq
 511         jmp exit_intr
 512 #endif
 513         CFI_ENDPROC
 514
 515 /*
 516  * APIC interrupts.
 517  */
 518         .macro apicinterrupt num,func
 519         pushq $\num-256
 520         interrupt \func
 521         jmp ret_from_intr
 522         CFI_ENDPROC
 523         .endm
 524
 525 ENTRY(thermal_interrupt)
 526         apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
 527
 528 #ifdef CONFIG_SMP
 529 ENTRY(reschedule_interrupt)
 530         apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
 531
 532 ENTRY(invalidate_interrupt)
 533         apicinterrupt INVALIDATE_TLB_VECTOR,smp_invalidate_interrupt
 534
 535 ENTRY(call_function_interrupt)
 536         apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
 537 #endif
 538
 539 #ifdef CONFIG_X86_LOCAL_APIC
 540 ENTRY(apic_timer_interrupt)
 541         apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
 542
 543 ENTRY(error_interrupt)
 544         apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
 545
 546 ENTRY(spurious_interrupt)
 547         apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
 548 #endif
 549
 550 /*
 551  * Exception entry points.
 552  */
 553         .macro zeroentry sym
 554         pushq $0        /* push error code/oldrax */
 555         pushq %rax      /* push real oldrax to the rdi slot */
 556         leaq  \sym(%rip),%rax
 557         jmp error_entry
 558         .endm
 559
 560         .macro errorentry sym
 561         pushq %rax
 562         leaq  \sym(%rip),%rax
 563         jmp error_entry
 564         .endm
 565
 566         /* error code is on the stack already */
 567         /* handle NMI like exceptions that can happen everywhere */
 568         .macro paranoidentry sym
 569         SAVE_ALL
 570         cld
 571         movl $1,%ebx
 572         movl  $MSR_GS_BASE,%ecx
 573         rdmsr
 574         testl %edx,%edx
 575         js    1f
 576         swapgs
 577         xorl  %ebx,%ebx
 578 1:      movq %rsp,%rdi
 579         movq ORIG_RAX(%rsp),%rsi
 580         movq $-1,ORIG_RAX(%rsp)
 581         call \sym
 582         cli
 583         .endm
 584
 585 /*
 586  * Exception entry point. This expects an error code/orig_rax on the stack
 587  * and the exception handler in %rax.
 588  */
 589 ENTRY(error_entry)
 590         CFI_STARTPROC   simple
 591         CFI_DEF_CFA     rsp,(SS-RDI)
 592         CFI_REL_OFFSET  rsp,(RSP-RDI)
 593         CFI_REL_OFFSET  rip,(RIP-RDI)
 594         /* rdi slot contains rax, oldrax contains error code */
 595         cld
 596         subq  $14*8,%rsp
 597         CFI_ADJUST_CFA_OFFSET   (14*8)
 598         movq %rsi,13*8(%rsp)
 599         CFI_REL_OFFSET  rsi,RSI
 600         movq 14*8(%rsp),%rsi    /* load rax from rdi slot */
 601         movq %rdx,12*8(%rsp)
 602         CFI_REL_OFFSET  rdx,RDX
 603         movq %rcx,11*8(%rsp)
 604         CFI_REL_OFFSET  rcx,RCX
 605         movq %rsi,10*8(%rsp)    /* store rax */
 606         CFI_REL_OFFSET  rax,RAX
 607         movq %r8, 9*8(%rsp)
 608         CFI_REL_OFFSET  r8,R8
 609         movq %r9, 8*8(%rsp)
 610         CFI_REL_OFFSET  r9,R9
 611         movq %r10,7*8(%rsp)
 612         CFI_REL_OFFSET  r10,R10
 613         movq %r11,6*8(%rsp)
 614         CFI_REL_OFFSET  r11,R11
 615         movq %rbx,5*8(%rsp)
 616         CFI_REL_OFFSET  rbx,RBX
 617         movq %rbp,4*8(%rsp)
 618         CFI_REL_OFFSET  rbp,RBP
 619         movq %r12,3*8(%rsp)
 620         CFI_REL_OFFSET  r12,R12
 621         movq %r13,2*8(%rsp)
 622         CFI_REL_OFFSET  r13,R13
 623         movq %r14,1*8(%rsp)
 624         CFI_REL_OFFSET  r14,R14
 625         movq %r15,(%rsp)
 626         CFI_REL_OFFSET  r15,R15
 627         xorl %ebx,%ebx
 628         testl $3,CS(%rsp)
 629         je  error_kernelspace
 630 error_swapgs:
 631         swapgs
 632 error_sti:
 633         movq %rdi,RDI(%rsp)
 634         movq %rsp,%rdi
 635         movq ORIG_RAX(%rsp),%rsi        /* get error code */
 636         movq $-1,ORIG_RAX(%rsp)
 637         call *%rax
 638         /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
 639 error_exit:
 640         movl %ebx,%eax
 641         RESTORE_REST
 642         cli
 643         GET_THREAD_INFO(%rcx)
 644         testl %eax,%eax
 645         jne  retint_kernel
 646         movl  threadinfo_flags(%rcx),%edx
 647         movl  $_TIF_WORK_MASK,%edi
 648         andl  %edi,%edx
 649         jnz  retint_careful
 650         swapgs
 651         RESTORE_ARGS 0,8,0
 652         iretq
 653         CFI_ENDPROC
 654
 655 error_kernelspace:
 656         incl %ebx
 657        /* There are two places in the kernel that can potentially fault with
 658           usergs. Handle them here. The exception handlers after
 659            iret run with kernel gs again, so don't set the user space flag.
 660            B stepping K8s sometimes report an truncated RIP for IRET
 661            exceptions returning to compat mode. Check for these here too. */
 662         leaq iret_label(%rip),%rbp
 663         cmpq %rbp,RIP(%rsp)
 664         je   error_swapgs
 665         movl %ebp,%ebp  /* zero extend */
 666         cmpq %rbp,RIP(%rsp)
 667         je   error_swapgs
 668         cmpq $gs_change,RIP(%rsp)
 669         je   error_swapgs
 670         jmp  error_sti
 671
 672        /* Reload gs selector with exception handling */
 673        /* edi:  new selector */
 674 ENTRY(load_gs_index)
 675         pushf
 676         cli
 677         swapgs
 678 gs_change:
 679         movl %edi,%gs
 680 2:      mfence          /* workaround */
 681         swapgs
 682         popf
 683         ret
 684
 685         .section __ex_table,"a"
 686         .align 8
 687         .quad gs_change,bad_gs
 688         .previous
 689         .section .fixup,"ax"
 690         /* running with kernelgs */
 691 bad_gs:
 692         swapgs                  /* switch back to user gs */
 693         xorl %eax,%eax
 694         movl %eax,%gs
 695         jmp  2b
 696         .previous
 697
 698 /*
 699  * Create a kernel thread.
 700  *
 701  * C extern interface:
 702  *      extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
 703  *
 704  * asm input arguments:
 705  *      rdi: fn, rsi: arg, rdx: flags
 706  */
 707 ENTRY(kernel_thread)
 708         CFI_STARTPROC
 709         FAKE_STACK_FRAME $child_rip
 710         SAVE_ALL
 711
 712         # rdi: flags, rsi: usp, rdx: will be &pt_regs
 713         movq %rdx,%rdi
 714         orq  kernel_thread_flags(%rip),%rdi
 715         movq $-1, %rsi
 716         movq %rsp, %rdx
 717
 718         xorl %r8d,%r8d
 719         xorl %r9d,%r9d
 720
 721         # clone now
 722         call do_fork
 723         movq %rax,RAX(%rsp)
 724         xorl %edi,%edi
 725
 726         /*
 727          * It isn't worth to check for reschedule here,
 728          * so internally to the x86_64 port you can rely on kernel_thread()
 729          * not to reschedule the child before returning, this avoids the need
 730          * of hacks for example to fork off the per-CPU idle tasks.
 731          * [Hopefully no generic code relies on the reschedule -AK]
 732          */
 733         RESTORE_ALL
 734         UNFAKE_STACK_FRAME
 735         ret
 736         CFI_ENDPROC
 737
 738
 739 child_rip:
 740         /*
 741          * Here we are in the child and the registers are set as they were
 742          * at kernel_thread() invocation in the parent.
 743          */
 744         movq %rdi, %rax
 745         movq %rsi, %rdi
 746         call *%rax
 747         # exit
 748         xorq %rdi, %rdi
 749         call do_exit
 750
 751 /*
 752  * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
 753  *
 754  * C extern interface:
 755  *       extern long execve(char *name, char **argv, char **envp)
 756  *
 757  * asm input arguments:
 758  *      rdi: name, rsi: argv, rdx: envp
 759  *
 760  * We want to fallback into:
 761  *      extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
 762  *
 763  * do_sys_execve asm fallback arguments:
 764  *      rdi: name, rsi: argv, rdx: envp, fake frame on the stack
 765  */
 766 ENTRY(execve)
 767         CFI_STARTPROC
 768         FAKE_STACK_FRAME $0
 769         SAVE_ALL
 770         call sys_execve
 771         movq %rax, RAX(%rsp)
 772         RESTORE_REST
 773         testq %rax,%rax
 774         je int_ret_from_sys_call
 775         RESTORE_ARGS
 776         UNFAKE_STACK_FRAME
 777         ret
 778         CFI_ENDPROC
 779
 780 ENTRY(page_fault)
 781         errorentry do_page_fault
 782
 783 ENTRY(coprocessor_error)
 784         zeroentry do_coprocessor_error
 785
 786 ENTRY(simd_coprocessor_error)
 787         zeroentry do_simd_coprocessor_error
 788
 789 ENTRY(device_not_available)
 790         zeroentry math_state_restore
 791
 792         /* runs on exception stack */
 793 ENTRY(debug)
 794         CFI_STARTPROC
 795         pushq $0
 796         CFI_ADJUST_CFA_OFFSET 8
 797         paranoidentry do_debug
 798         jmp paranoid_exit
 799         CFI_ENDPROC
 800
 801         /* runs on exception stack */
 802 ENTRY(nmi)
 803         CFI_STARTPROC
 804         pushq $-1
 805         CFI_ADJUST_CFA_OFFSET 8
 806         paranoidentry do_nmi
 807         /*
 808          * "Paranoid" exit path from exception stack.
 809          * Paranoid because this is used by NMIs and cannot take
 810          * any kernel state for granted.
 811          * We don't do kernel preemption checks here, because only
 812          * NMI should be common and it does not enable IRQs and
 813          * cannot get reschedule ticks.
 814          */
 815         /* ebx: no swapgs flag */
 816 paranoid_exit:
 817         testl %ebx,%ebx                         /* swapgs needed? */
 818         jnz paranoid_restore
 819         testl $3,CS(%rsp)
 820         jnz   paranoid_userspace
 821 paranoid_swapgs:
 822         swapgs
 823 paranoid_restore:
 824         RESTORE_ALL 8
 825         iretq
 826 paranoid_userspace:
 827         GET_THREAD_INFO(%rcx)
 828         movl threadinfo_flags(%rcx),%ebx
 829         andl $_TIF_WORK_MASK,%ebx
 830         jz paranoid_swapgs
 831         movq %rsp,%rdi                  /* &pt_regs */
 832         call sync_regs
 833         movq %rax,%rsp                  /* switch stack for scheduling */
 834         testl $_TIF_NEED_RESCHED,%ebx
 835         jnz paranoid_schedule
 836         movl %ebx,%edx                  /* arg3: thread flags */
 837         sti
 838         xorl %esi,%esi                  /* arg2: oldset */
 839         movq %rsp,%rdi                  /* arg1: &pt_regs */
 840         call do_notify_resume
 841         cli
 842         jmp paranoid_userspace
 843 paranoid_schedule:
 844         sti
 845         call schedule
 846         cli
 847         jmp paranoid_userspace
 848         CFI_ENDPROC
 849
 850 ENTRY(int3)
 851         zeroentry do_int3
 852
 853 ENTRY(overflow)
 854         zeroentry do_overflow
 855
 856 ENTRY(bounds)
 857         zeroentry do_bounds
 858
 859 ENTRY(invalid_op)
 860         zeroentry do_invalid_op
 861
 862 ENTRY(coprocessor_segment_overrun)
 863         zeroentry do_coprocessor_segment_overrun
 864
 865 ENTRY(reserved)
 866         zeroentry do_reserved
 867
 868         /* runs on exception stack */
 869 ENTRY(double_fault)
 870         CFI_STARTPROC
 871         paranoidentry do_double_fault
 872         jmp paranoid_exit
 873         CFI_ENDPROC
 874
 875 ENTRY(invalid_TSS)
 876         errorentry do_invalid_TSS
 877
 878 ENTRY(segment_not_present)
 879         errorentry do_segment_not_present
 880
 881         /* runs on exception stack */
 882 ENTRY(stack_segment)
 883         CFI_STARTPROC
 884         paranoidentry do_stack_segment
 885         jmp paranoid_exit
 886         CFI_ENDPROC
 887
 888 ENTRY(general_protection)
 889         errorentry do_general_protection
 890
 891 ENTRY(alignment_check)
 892         errorentry do_alignment_check
 893
 894 ENTRY(divide_error)
 895         zeroentry do_divide_error
 896
 897 ENTRY(spurious_interrupt_bug)
 898         zeroentry do_spurious_interrupt_bug
 899
 900 #ifdef CONFIG_X86_MCE
 901         /* runs on exception stack */
 902 ENTRY(machine_check)
 903         CFI_STARTPROC
 904         pushq $0
 905         CFI_ADJUST_CFA_OFFSET 8
 906         paranoidentry do_machine_check
 907         jmp paranoid_exit
 908         CFI_ENDPROC
 909 #endif
 910
 911 ENTRY(call_debug)
 912        zeroentry do_call_debug
 913