arch/x86_64/kernel/entry.S

   1 /*
   2  *  linux/arch/x86_64/entry.S
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
   6  *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
   7  *
   8  *  $Id$
   9  */
  10
  11 /*
  12  * entry.S contains the system-call and fault low-level handling routines.
  13  *
  14  * NOTE: This code handles signal-recognition, which happens every time
  15  * after an interrupt and after each system call.
  16  *
  17  * Normal syscalls and interrupts don't save a full stack frame, this is
  18  * only done for syscall tracing, signals or fork/exec et.al.
  19  *
  20  * A note on terminology:
  21  * - top of stack: Architecture defined interrupt frame from SS to RIP
  22  * at the top of the kernel process stack.
  23  * - partial stack frame: partially saved registers upto R11.
  24  * - full stack frame: Like partial stack frame, but all register saved.
  25  *
  26  * TODO:
  27  * - schedule it carefully for the final hardware.
  28  */
  29
  30 #define ASSEMBLY 1
  31 #include <linux/config.h>
  32 #include <linux/linkage.h>
  33 #include <asm/segment.h>
  34 #include <asm/smp.h>
  35 #include <asm/cache.h>
  36 #include <asm/errno.h>
  37 #include <asm/dwarf2.h>
  38 #include <asm/calling.h>
  39 #include <asm/asm-offsets.h>
  40 #include <asm/msr.h>
  41 #include <asm/unistd.h>
  42 #include <asm/thread_info.h>
  43 #include <asm/hw_irq.h>
  44
  45         .code64
  46
  47 #ifndef CONFIG_PREEMPT
  48 #define retint_kernel retint_restore_args
  49 #endif
  50
  51 /*
  52  * C code is not supposed to know about undefined top of stack. Every time
  53  * a C function with an pt_regs argument is called from the SYSCALL based
  54  * fast path FIXUP_TOP_OF_STACK is needed.
  55  * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
  56  * manipulation.
  57  */
  58
  59         /* %rsp:at FRAMEEND */
  60         .macro FIXUP_TOP_OF_STACK tmp
  61         movq    %gs:pda_oldrsp,\tmp
  62         movq    \tmp,RSP(%rsp)
  63         movq    $__USER_DS,SS(%rsp)
  64         movq    $__USER_CS,CS(%rsp)
  65         movq    $-1,RCX(%rsp)
  66         movq    R11(%rsp),\tmp  /* get eflags */
  67         movq    \tmp,EFLAGS(%rsp)
  68         .endm
  69
  70         .macro RESTORE_TOP_OF_STACK tmp,offset=0
  71         movq   RSP-\offset(%rsp),\tmp
  72         movq   \tmp,%gs:pda_oldrsp
  73         movq   EFLAGS-\offset(%rsp),\tmp
  74         movq   \tmp,R11-\offset(%rsp)
  75         .endm
  76
  77         .macro FAKE_STACK_FRAME child_rip
  78         /* push in order ss, rsp, eflags, cs, rip */
  79         xorl %eax, %eax
  80         pushq %rax /* ss */
  81         CFI_ADJUST_CFA_OFFSET   8
  82         /*CFI_REL_OFFSET        ss,0*/
  83         pushq %rax /* rsp */
  84         CFI_ADJUST_CFA_OFFSET   8
  85         CFI_REL_OFFSET  rsp,0
  86         pushq $(1<<9) /* eflags - interrupts on */
  87         CFI_ADJUST_CFA_OFFSET   8
  88         /*CFI_REL_OFFSET        rflags,0*/
  89         pushq $__KERNEL_CS /* cs */
  90         CFI_ADJUST_CFA_OFFSET   8
  91         /*CFI_REL_OFFSET        cs,0*/
  92         pushq \child_rip /* rip */
  93         CFI_ADJUST_CFA_OFFSET   8
  94         CFI_REL_OFFSET  rip,0
  95         pushq   %rax /* orig rax */
  96         CFI_ADJUST_CFA_OFFSET   8
  97         .endm
  98
  99         .macro UNFAKE_STACK_FRAME
 100         addq $8*6, %rsp
 101         CFI_ADJUST_CFA_OFFSET   -(6*8)
 102         .endm
 103
 104         .macro  CFI_DEFAULT_STACK start=1
 105         .if \start
 106         CFI_STARTPROC   simple
 107         CFI_DEF_CFA     rsp,SS+8
 108         .else
 109         CFI_DEF_CFA_OFFSET SS+8
 110         .endif
 111         CFI_REL_OFFSET  r15,R15
 112         CFI_REL_OFFSET  r14,R14
 113         CFI_REL_OFFSET  r13,R13
 114         CFI_REL_OFFSET  r12,R12
 115         CFI_REL_OFFSET  rbp,RBP
 116         CFI_REL_OFFSET  rbx,RBX
 117         CFI_REL_OFFSET  r11,R11
 118         CFI_REL_OFFSET  r10,R10
 119         CFI_REL_OFFSET  r9,R9
 120         CFI_REL_OFFSET  r8,R8
 121         CFI_REL_OFFSET  rax,RAX
 122         CFI_REL_OFFSET  rcx,RCX
 123         CFI_REL_OFFSET  rdx,RDX
 124         CFI_REL_OFFSET  rsi,RSI
 125         CFI_REL_OFFSET  rdi,RDI
 126         CFI_REL_OFFSET  rip,RIP
 127         /*CFI_REL_OFFSET        cs,CS*/
 128         /*CFI_REL_OFFSET        rflags,EFLAGS*/
 129         CFI_REL_OFFSET  rsp,RSP
 130         /*CFI_REL_OFFSET        ss,SS*/
 131         .endm
 132 /*
 133  * A newly forked process directly context switches into this.
 134  */
 135 /* rdi: prev */
 136 ENTRY(ret_from_fork)
 137         CFI_DEFAULT_STACK
 138         call schedule_tail
 139         GET_THREAD_INFO(%rcx)
 140         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
 141         jnz rff_trace
 142 rff_action:
 143         RESTORE_REST
 144         testl $3,CS-ARGOFFSET(%rsp)     # from kernel_thread?
 145         je   int_ret_from_sys_call
 146         testl $_TIF_IA32,threadinfo_flags(%rcx)
 147         jnz  int_ret_from_sys_call
 148         RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
 149         jmp ret_from_sys_call
 150 rff_trace:
 151         movq %rsp,%rdi
 152         call syscall_trace_leave
 153         GET_THREAD_INFO(%rcx)
 154         jmp rff_action
 155         CFI_ENDPROC
 156
 157 /*
 158  * System call entry. Upto 6 arguments in registers are supported.
 159  *
 160  * SYSCALL does not save anything on the stack and does not change the
 161  * stack pointer.
 162  */
 163
 164 /*
 165  * Register setup:
 166  * rax  system call number
 167  * rdi  arg0
 168  * rcx  return address for syscall/sysret, C arg3
 169  * rsi  arg1
 170  * rdx  arg2
 171  * r10  arg3    (--> moved to rcx for C)
 172  * r8   arg4
 173  * r9   arg5
 174  * r11  eflags for syscall/sysret, temporary for C
 175  * r12-r15,rbp,rbx saved by C code, not touched.
 176  *
 177  * Interrupts are off on entry.
 178  * Only called from user space.
 179  *
 180  * XXX  if we had a free scratch register we could save the RSP into the stack frame
 181  *      and report it properly in ps. Unfortunately we haven't.
 182  */
 183
 184 ENTRY(system_call)
 185         CFI_STARTPROC   simple
 186         CFI_DEF_CFA     rsp,0
 187         CFI_REGISTER    rip,rcx
 188         /*CFI_REGISTER  rflags,r11*/
 189         swapgs
 190         movq    %rsp,%gs:pda_oldrsp
 191         movq    %gs:pda_kernelstack,%rsp
 192         sti
 193         SAVE_ARGS 8,1
 194         movq  %rax,ORIG_RAX-ARGOFFSET(%rsp)
 195         movq  %rcx,RIP-ARGOFFSET(%rsp)
 196         CFI_REL_OFFSET rip,RIP-ARGOFFSET
 197         GET_THREAD_INFO(%rcx)
 198         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
 199         CFI_REMEMBER_STATE
 200         jnz tracesys
 201         cmpq $__NR_syscall_max,%rax
 202         ja badsys
 203         movq %r10,%rcx
 204         call *sys_call_table(,%rax,8)  # XXX:    rip relative
 205         movq %rax,RAX-ARGOFFSET(%rsp)
 206 /*
 207  * Syscall return path ending with SYSRET (fast path)
 208  * Has incomplete stack frame and undefined top of stack.
 209  */
 210         .globl ret_from_sys_call
 211 ret_from_sys_call:
 212         movl $_TIF_ALLWORK_MASK,%edi
 213         /* edi: flagmask */
 214 sysret_check:
 215         GET_THREAD_INFO(%rcx)
 216         cli
 217         movl threadinfo_flags(%rcx),%edx
 218         andl %edi,%edx
 219         CFI_REMEMBER_STATE
 220         jnz  sysret_careful
 221         movq RIP-ARGOFFSET(%rsp),%rcx
 222         CFI_REGISTER    rip,rcx
 223         RESTORE_ARGS 0,-ARG_SKIP,1
 224         /*CFI_REGISTER  rflags,r11*/
 225         movq    %gs:pda_oldrsp,%rsp
 226         swapgs
 227         sysretq
 228
 229         /* Handle reschedules */
 230         /* edx: work, edi: workmask */
 231 sysret_careful:
 232         CFI_RESTORE_STATE
 233         bt $TIF_NEED_RESCHED,%edx
 234         jnc sysret_signal
 235         sti
 236         pushq %rdi
 237         CFI_ADJUST_CFA_OFFSET 8
 238         call schedule
 239         popq  %rdi
 240         CFI_ADJUST_CFA_OFFSET -8
 241         jmp sysret_check
 242
 243         /* Handle a signal */
 244 sysret_signal:
 245         sti
 246         testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
 247         jz    1f
 248
 249         /* Really a signal */
 250         /* edx: work flags (arg3) */
 251         leaq do_notify_resume(%rip),%rax
 252         leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
 253         xorl %esi,%esi # oldset -> arg2
 254         call ptregscall_common
 255 1:      movl $_TIF_NEED_RESCHED,%edi
 256         jmp sysret_check
 257
 258 badsys:
 259         movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
 260         jmp ret_from_sys_call
 261
 262         /* Do syscall tracing */
 263 tracesys:
 264         CFI_RESTORE_STATE
 265         SAVE_REST
 266         movq $-ENOSYS,RAX(%rsp)
 267         FIXUP_TOP_OF_STACK %rdi
 268         movq %rsp,%rdi
 269         call syscall_trace_enter
 270         LOAD_ARGS ARGOFFSET  /* reload args from stack in case ptrace changed it */
 271         RESTORE_REST
 272         cmpq $__NR_syscall_max,%rax
 273         ja  1f
 274         movq %r10,%rcx  /* fixup for C */
 275         call *sys_call_table(,%rax,8)
 276         movq %rax,RAX-ARGOFFSET(%rsp)
 277 1:      SAVE_REST
 278         movq %rsp,%rdi
 279         call syscall_trace_leave
 280         RESTORE_TOP_OF_STACK %rbx
 281         RESTORE_REST
 282         jmp ret_from_sys_call
 283         CFI_ENDPROC
 284
 285 /*
 286  * Syscall return path ending with IRET.
 287  * Has correct top of stack, but partial stack frame.
 288  */
 289 ENTRY(int_ret_from_sys_call)
 290         CFI_STARTPROC   simple
 291         CFI_DEF_CFA     rsp,SS+8-ARGOFFSET
 292         /*CFI_REL_OFFSET        ss,SS-ARGOFFSET*/
 293         CFI_REL_OFFSET  rsp,RSP-ARGOFFSET
 294         /*CFI_REL_OFFSET        rflags,EFLAGS-ARGOFFSET*/
 295         /*CFI_REL_OFFSET        cs,CS-ARGOFFSET*/
 296         CFI_REL_OFFSET  rip,RIP-ARGOFFSET
 297         CFI_REL_OFFSET  rdx,RDX-ARGOFFSET
 298         CFI_REL_OFFSET  rcx,RCX-ARGOFFSET
 299         CFI_REL_OFFSET  rax,RAX-ARGOFFSET
 300         CFI_REL_OFFSET  rdi,RDI-ARGOFFSET
 301         CFI_REL_OFFSET  rsi,RSI-ARGOFFSET
 302         CFI_REL_OFFSET  r8,R8-ARGOFFSET
 303         CFI_REL_OFFSET  r9,R9-ARGOFFSET
 304         CFI_REL_OFFSET  r10,R10-ARGOFFSET
 305         CFI_REL_OFFSET  r11,R11-ARGOFFSET
 306         cli
 307         testl $3,CS-ARGOFFSET(%rsp)
 308         je retint_restore_args
 309         movl $_TIF_ALLWORK_MASK,%edi
 310         /* edi: mask to check */
 311 int_with_check:
 312         GET_THREAD_INFO(%rcx)
 313         movl threadinfo_flags(%rcx),%edx
 314         andl %edi,%edx
 315         jnz   int_careful
 316         jmp   retint_swapgs
 317
 318         /* Either reschedule or signal or syscall exit tracking needed. */
 319         /* First do a reschedule test. */
 320         /* edx: work, edi: workmask */
 321 int_careful:
 322         bt $TIF_NEED_RESCHED,%edx
 323         jnc  int_very_careful
 324         sti
 325         pushq %rdi
 326         CFI_ADJUST_CFA_OFFSET 8
 327         call schedule
 328         popq %rdi
 329         CFI_ADJUST_CFA_OFFSET -8
 330         cli
 331         jmp int_with_check
 332
 333         /* handle signals and tracing -- both require a full stack frame */
 334 int_very_careful:
 335         sti
 336         SAVE_REST
 337         /* Check for syscall exit trace */
 338         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
 339         jz int_signal
 340         pushq %rdi
 341         CFI_ADJUST_CFA_OFFSET 8
 342         leaq 8(%rsp),%rdi       # &ptregs -> arg1
 343         call syscall_trace_leave
 344         popq %rdi
 345         CFI_ADJUST_CFA_OFFSET -8
 346         andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
 347         cli
 348         jmp int_restore_rest
 349
 350 int_signal:
 351         testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
 352         jz 1f
 353         movq %rsp,%rdi          # &ptregs -> arg1
 354         xorl %esi,%esi          # oldset -> arg2
 355         call do_notify_resume
 356 1:      movl $_TIF_NEED_RESCHED,%edi
 357 int_restore_rest:
 358         RESTORE_REST
 359         cli
 360         jmp int_with_check
 361         CFI_ENDPROC
 362
 363 /*
 364  * Certain special system calls that need to save a complete full stack frame.
 365  */
 366
 367         .macro PTREGSCALL label,func,arg
 368         .globl \label
 369 \label:
 370         leaq    \func(%rip),%rax
 371         leaq    -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
 372         jmp     ptregscall_common
 373         .endm
 374
 375         CFI_STARTPROC
 376
 377         PTREGSCALL stub_clone, sys_clone, %r8
 378         PTREGSCALL stub_fork, sys_fork, %rdi
 379         PTREGSCALL stub_vfork, sys_vfork, %rdi
 380         PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
 381         PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
 382         PTREGSCALL stub_iopl, sys_iopl, %rsi
 383
 384 ENTRY(ptregscall_common)
 385         popq %r11
 386         CFI_ADJUST_CFA_OFFSET -8
 387         CFI_REGISTER rip, r11
 388         SAVE_REST
 389         movq %r11, %r15
 390         CFI_REGISTER rip, r15
 391         FIXUP_TOP_OF_STACK %r11
 392         call *%rax
 393         RESTORE_TOP_OF_STACK %r11
 394         movq %r15, %r11
 395         CFI_REGISTER rip, r11
 396         RESTORE_REST
 397         pushq %r11
 398         CFI_ADJUST_CFA_OFFSET 8
 399         CFI_REL_OFFSET rip, 0
 400         ret
 401         CFI_ENDPROC
 402
 403 ENTRY(stub_execve)
 404         CFI_STARTPROC
 405         popq %r11
 406         CFI_ADJUST_CFA_OFFSET -8
 407         CFI_REGISTER rip, r11
 408         SAVE_REST
 409         movq %r11, %r15
 410         CFI_REGISTER rip, r15
 411         FIXUP_TOP_OF_STACK %r11
 412         call sys_execve
 413         GET_THREAD_INFO(%rcx)
 414         bt $TIF_IA32,threadinfo_flags(%rcx)
 415         CFI_REMEMBER_STATE
 416         jc exec_32bit
 417         RESTORE_TOP_OF_STACK %r11
 418         movq %r15, %r11
 419         CFI_REGISTER rip, r11
 420         RESTORE_REST
 421         pushq %r11
 422         CFI_ADJUST_CFA_OFFSET 8
 423         CFI_REL_OFFSET rip, 0
 424         ret
 425
 426 exec_32bit:
 427         CFI_RESTORE_STATE
 428         movq %rax,RAX(%rsp)
 429         RESTORE_REST
 430         jmp int_ret_from_sys_call
 431         CFI_ENDPROC
 432
 433 /*
 434  * sigreturn is special because it needs to restore all registers on return.
 435  * This cannot be done with SYSRET, so use the IRET return path instead.
 436  */
 437 ENTRY(stub_rt_sigreturn)
 438         CFI_STARTPROC
 439         addq $8, %rsp
 440         CFI_ADJUST_CFA_OFFSET   -8
 441         SAVE_REST
 442         movq %rsp,%rdi
 443         FIXUP_TOP_OF_STACK %r11
 444         call sys_rt_sigreturn
 445         movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
 446         RESTORE_REST
 447         jmp int_ret_from_sys_call
 448         CFI_ENDPROC
 449
 450 /*
 451  * initial frame state for interrupts and exceptions
 452  */
 453         .macro _frame ref
 454         CFI_STARTPROC simple
 455         CFI_DEF_CFA rsp,SS+8-\ref
 456         /*CFI_REL_OFFSET ss,SS-\ref*/
 457         CFI_REL_OFFSET rsp,RSP-\ref
 458         /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
 459         /*CFI_REL_OFFSET cs,CS-\ref*/
 460         CFI_REL_OFFSET rip,RIP-\ref
 461         .endm
 462
 463 /* initial frame state for interrupts (and exceptions without error code) */
 464 #define INTR_FRAME _frame RIP
 465 /* initial frame state for exceptions with error code (and interrupts with
 466    vector already pushed) */
 467 #define XCPT_FRAME _frame ORIG_RAX
 468
 469 /*
 470  * Interrupt entry/exit.
 471  *
 472  * Interrupt entry points save only callee clobbered registers in fast path.
 473  *
 474  * Entry runs with interrupts off.
 475  */
 476
 477 /* 0(%rsp): interrupt number */
 478         .macro interrupt func
 479         cld
 480 #ifdef CONFIG_DEBUG_INFO
 481         SAVE_ALL
 482         movq %rsp,%rdi
 483         /*
 484          * Setup a stack frame pointer.  This allows gdb to trace
 485          * back to the original stack.
 486          */
 487         movq %rsp,%rbp
 488         CFI_DEF_CFA_REGISTER    rbp
 489 #else
 490         SAVE_ARGS
 491         leaq -ARGOFFSET(%rsp),%rdi      # arg1 for handler
 492 #endif
 493         testl $3,CS(%rdi)
 494         je 1f
 495         swapgs
 496 1:      incl    %gs:pda_irqcount        # RED-PEN should check preempt count
 497         movq %gs:pda_irqstackptr,%rax
 498         cmoveq %rax,%rsp /*todo This needs CFI annotation! */
 499         pushq %rdi                      # save old stack
 500         CFI_ADJUST_CFA_OFFSET   8
 501         call \func
 502         .endm
 503
 504 ENTRY(common_interrupt)
 505         XCPT_FRAME
 506         interrupt do_IRQ
 507         /* 0(%rsp): oldrsp-ARGOFFSET */
 508 ret_from_intr:
 509         popq  %rdi
 510         CFI_ADJUST_CFA_OFFSET   -8
 511         cli
 512         decl %gs:pda_irqcount
 513 #ifdef CONFIG_DEBUG_INFO
 514         movq RBP(%rdi),%rbp
 515         CFI_DEF_CFA_REGISTER    rsp
 516 #endif
 517         leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
 518 exit_intr:
 519         GET_THREAD_INFO(%rcx)
 520         testl $3,CS-ARGOFFSET(%rsp)
 521         je retint_kernel
 522
 523         /* Interrupt came from user space */
 524         /*
 525          * Has a correct top of stack, but a partial stack frame
 526          * %rcx: thread info. Interrupts off.
 527          */
 528 retint_with_reschedule:
 529         movl $_TIF_WORK_MASK,%edi
 530 retint_check:
 531         movl threadinfo_flags(%rcx),%edx
 532         andl %edi,%edx
 533         CFI_REMEMBER_STATE
 534         jnz  retint_careful
 535 retint_swapgs:
 536         swapgs
 537 retint_restore_args:
 538         cli
 539         RESTORE_ARGS 0,8,0
 540 iret_label:
 541         iretq
 542
 543         .section __ex_table,"a"
 544         .quad iret_label,bad_iret
 545         .previous
 546         .section .fixup,"ax"
 547         /* force a signal here? this matches i386 behaviour */
 548         /* running with kernel gs */
 549 bad_iret:
 550         movq $-9999,%rdi        /* better code? */
 551         jmp do_exit
 552         .previous
 553
 554         /* edi: workmask, edx: work */
 555 retint_careful:
 556         CFI_RESTORE_STATE
 557         bt    $TIF_NEED_RESCHED,%edx
 558         jnc   retint_signal
 559         sti
 560         pushq %rdi
 561         CFI_ADJUST_CFA_OFFSET   8
 562         call  schedule
 563         popq %rdi
 564         CFI_ADJUST_CFA_OFFSET   -8
 565         GET_THREAD_INFO(%rcx)
 566         cli
 567         jmp retint_check
 568
 569 retint_signal:
 570         testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
 571         jz    retint_swapgs
 572         sti
 573         SAVE_REST
 574         movq $-1,ORIG_RAX(%rsp)
 575         xorl %esi,%esi          # oldset
 576         movq %rsp,%rdi          # &pt_regs
 577         call do_notify_resume
 578         RESTORE_REST
 579         cli
 580         movl $_TIF_NEED_RESCHED,%edi
 581         GET_THREAD_INFO(%rcx)
 582         jmp retint_check
 583
 584 #ifdef CONFIG_PREEMPT
 585         /* Returning to kernel space. Check if we need preemption */
 586         /* rcx:  threadinfo. interrupts off. */
 587         .p2align
 588 retint_kernel:
 589         cmpl $0,threadinfo_preempt_count(%rcx)
 590         jnz  retint_restore_args
 591         bt  $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
 592         jnc  retint_restore_args
 593         bt   $9,EFLAGS-ARGOFFSET(%rsp)  /* interrupts off? */
 594         jnc  retint_restore_args
 595         call preempt_schedule_irq
 596         jmp exit_intr
 597 #endif
 598         CFI_ENDPROC
 599
 600 /*
 601  * APIC interrupts.
 602  */
 603         .macro apicinterrupt num,func
 604         INTR_FRAME
 605         pushq $\num-256
 606         CFI_ADJUST_CFA_OFFSET 8
 607         interrupt \func
 608         jmp ret_from_intr
 609         CFI_ENDPROC
 610         .endm
 611
 612 ENTRY(thermal_interrupt)
 613         apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
 614
 615 ENTRY(threshold_interrupt)
 616         apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
 617
 618 #ifdef CONFIG_SMP
 619 ENTRY(reschedule_interrupt)
 620         apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
 621
 622         .macro INVALIDATE_ENTRY num
 623 ENTRY(invalidate_interrupt\num)
 624         apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
 625         .endm
 626
 627         INVALIDATE_ENTRY 0
 628         INVALIDATE_ENTRY 1
 629         INVALIDATE_ENTRY 2
 630         INVALIDATE_ENTRY 3
 631         INVALIDATE_ENTRY 4
 632         INVALIDATE_ENTRY 5
 633         INVALIDATE_ENTRY 6
 634         INVALIDATE_ENTRY 7
 635
 636 ENTRY(call_function_interrupt)
 637         apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
 638 #endif
 639
 640 #ifdef CONFIG_X86_LOCAL_APIC
 641 ENTRY(apic_timer_interrupt)
 642         apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
 643
 644 ENTRY(error_interrupt)
 645         apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
 646
 647 ENTRY(spurious_interrupt)
 648         apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
 649 #endif
 650
 651 /*
 652  * Exception entry points.
 653  */
 654         .macro zeroentry sym
 655         INTR_FRAME
 656         pushq $0        /* push error code/oldrax */
 657         CFI_ADJUST_CFA_OFFSET 8
 658         pushq %rax      /* push real oldrax to the rdi slot */
 659         CFI_ADJUST_CFA_OFFSET 8
 660         leaq  \sym(%rip),%rax
 661         jmp error_entry
 662         CFI_ENDPROC
 663         .endm
 664
 665         .macro errorentry sym
 666         XCPT_FRAME
 667         pushq %rax
 668         CFI_ADJUST_CFA_OFFSET 8
 669         leaq  \sym(%rip),%rax
 670         jmp error_entry
 671         CFI_ENDPROC
 672         .endm
 673
 674         /* error code is on the stack already */
 675         /* handle NMI like exceptions that can happen everywhere */
 676 #ifndef DEBUG_IST
 677 # define DEBUG_IST 0
 678 #endif
 679         .macro paranoidentry sym, ist=0
 680         SAVE_ALL
 681         cld
 682         movl $1,%ebx
 683         movl  $MSR_GS_BASE,%ecx
 684         rdmsr
 685         testl %edx,%edx
 686         js    1f
 687         swapgs
 688         xorl  %ebx,%ebx
 689 1:
 690         .if \ist
 691         movq    %gs:pda_data_offset, %rbp
 692         .endif
 693         movq %rsp,%rdi
 694         movq ORIG_RAX(%rsp),%rsi
 695         movq $-1,ORIG_RAX(%rsp)
 696         .if \ist
 697         subq    $EXCEPTION_STACK_SIZE, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
 698         .endif
 699         call \sym
 700         .if \ist
 701         addq    $EXCEPTION_STACK_SIZE, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
 702         .endif
 703         cli
 704         .endm
 705
 706 /*
 707  * Exception entry point. This expects an error code/orig_rax on the stack
 708  * and the exception handler in %rax.
 709  */
 710 ENTRY(error_entry)
 711         _frame RDI
 712         /* rdi slot contains rax, oldrax contains error code */
 713         cld
 714         subq  $14*8,%rsp
 715         CFI_ADJUST_CFA_OFFSET   (14*8)
 716         movq %rsi,13*8(%rsp)
 717         CFI_REL_OFFSET  rsi,RSI
 718         movq 14*8(%rsp),%rsi    /* load rax from rdi slot */
 719         movq %rdx,12*8(%rsp)
 720         CFI_REL_OFFSET  rdx,RDX
 721         movq %rcx,11*8(%rsp)
 722         CFI_REL_OFFSET  rcx,RCX
 723         movq %rsi,10*8(%rsp)    /* store rax */
 724         CFI_REL_OFFSET  rax,RAX
 725         movq %r8, 9*8(%rsp)
 726         CFI_REL_OFFSET  r8,R8
 727         movq %r9, 8*8(%rsp)
 728         CFI_REL_OFFSET  r9,R9
 729         movq %r10,7*8(%rsp)
 730         CFI_REL_OFFSET  r10,R10
 731         movq %r11,6*8(%rsp)
 732         CFI_REL_OFFSET  r11,R11
 733         movq %rbx,5*8(%rsp)
 734         CFI_REL_OFFSET  rbx,RBX
 735         movq %rbp,4*8(%rsp)
 736         CFI_REL_OFFSET  rbp,RBP
 737         movq %r12,3*8(%rsp)
 738         CFI_REL_OFFSET  r12,R12
 739         movq %r13,2*8(%rsp)
 740         CFI_REL_OFFSET  r13,R13
 741         movq %r14,1*8(%rsp)
 742         CFI_REL_OFFSET  r14,R14
 743         movq %r15,(%rsp)
 744         CFI_REL_OFFSET  r15,R15
 745         xorl %ebx,%ebx
 746         testl $3,CS(%rsp)
 747         je  error_kernelspace
 748 error_swapgs:
 749         swapgs
 750 error_sti:
 751         movq %rdi,RDI(%rsp)
 752         movq %rsp,%rdi
 753         movq ORIG_RAX(%rsp),%rsi        /* get error code */
 754         movq $-1,ORIG_RAX(%rsp)
 755         call *%rax
 756         /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
 757 error_exit:
 758         movl %ebx,%eax
 759         RESTORE_REST
 760         cli
 761         GET_THREAD_INFO(%rcx)
 762         testl %eax,%eax
 763         jne  retint_kernel
 764         movl  threadinfo_flags(%rcx),%edx
 765         movl  $_TIF_WORK_MASK,%edi
 766         andl  %edi,%edx
 767         jnz  retint_careful
 768         swapgs
 769         RESTORE_ARGS 0,8,0
 770         jmp iret_label
 771         CFI_ENDPROC
 772
 773 error_kernelspace:
 774         incl %ebx
 775        /* There are two places in the kernel that can potentially fault with
 776           usergs. Handle them here. The exception handlers after
 777            iret run with kernel gs again, so don't set the user space flag.
 778            B stepping K8s sometimes report an truncated RIP for IRET
 779            exceptions returning to compat mode. Check for these here too. */
 780         leaq iret_label(%rip),%rbp
 781         cmpq %rbp,RIP(%rsp)
 782         je   error_swapgs
 783         movl %ebp,%ebp  /* zero extend */
 784         cmpq %rbp,RIP(%rsp)
 785         je   error_swapgs
 786         cmpq $gs_change,RIP(%rsp)
 787         je   error_swapgs
 788         jmp  error_sti
 789
 790        /* Reload gs selector with exception handling */
 791        /* edi:  new selector */
 792 ENTRY(load_gs_index)
 793         CFI_STARTPROC
 794         pushf
 795         CFI_ADJUST_CFA_OFFSET 8
 796         cli
 797         swapgs
 798 gs_change:
 799         movl %edi,%gs
 800 2:      mfence          /* workaround */
 801         swapgs
 802         popf
 803         CFI_ADJUST_CFA_OFFSET -8
 804         ret
 805         CFI_ENDPROC
 806
 807         .section __ex_table,"a"
 808         .align 8
 809         .quad gs_change,bad_gs
 810         .previous
 811         .section .fixup,"ax"
 812         /* running with kernelgs */
 813 bad_gs:
 814         swapgs                  /* switch back to user gs */
 815         xorl %eax,%eax
 816         movl %eax,%gs
 817         jmp  2b
 818         .previous
 819
 820 /*
 821  * Create a kernel thread.
 822  *
 823  * C extern interface:
 824  *      extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
 825  *
 826  * asm input arguments:
 827  *      rdi: fn, rsi: arg, rdx: flags
 828  */
 829 ENTRY(kernel_thread)
 830         CFI_STARTPROC
 831         FAKE_STACK_FRAME $child_rip
 832         SAVE_ALL
 833
 834         # rdi: flags, rsi: usp, rdx: will be &pt_regs
 835         movq %rdx,%rdi
 836         orq  kernel_thread_flags(%rip),%rdi
 837         movq $-1, %rsi
 838         movq %rsp, %rdx
 839
 840         xorl %r8d,%r8d
 841         xorl %r9d,%r9d
 842
 843         # clone now
 844         call do_fork
 845         movq %rax,RAX(%rsp)
 846         xorl %edi,%edi
 847
 848         /*
 849          * It isn't worth to check for reschedule here,
 850          * so internally to the x86_64 port you can rely on kernel_thread()
 851          * not to reschedule the child before returning, this avoids the need
 852          * of hacks for example to fork off the per-CPU idle tasks.
 853          * [Hopefully no generic code relies on the reschedule -AK]
 854          */
 855         RESTORE_ALL
 856         UNFAKE_STACK_FRAME
 857         ret
 858         CFI_ENDPROC
 859
 860
 861 child_rip:
 862         /*
 863          * Here we are in the child and the registers are set as they were
 864          * at kernel_thread() invocation in the parent.
 865          */
 866         movq %rdi, %rax
 867         movq %rsi, %rdi
 868         call *%rax
 869         # exit
 870         xorl %edi, %edi
 871         call do_exit
 872
 873 /*
 874  * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
 875  *
 876  * C extern interface:
 877  *       extern long execve(char *name, char **argv, char **envp)
 878  *
 879  * asm input arguments:
 880  *      rdi: name, rsi: argv, rdx: envp
 881  *
 882  * We want to fallback into:
 883  *      extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
 884  *
 885  * do_sys_execve asm fallback arguments:
 886  *      rdi: name, rsi: argv, rdx: envp, fake frame on the stack
 887  */
 888 ENTRY(execve)
 889         CFI_STARTPROC
 890         FAKE_STACK_FRAME $0
 891         SAVE_ALL
 892         call sys_execve
 893         movq %rax, RAX(%rsp)
 894         RESTORE_REST
 895         testq %rax,%rax
 896         je int_ret_from_sys_call
 897         RESTORE_ARGS
 898         UNFAKE_STACK_FRAME
 899         ret
 900         CFI_ENDPROC
 901
 902 KPROBE_ENTRY(page_fault)
 903         errorentry do_page_fault
 904         .previous .text
 905
 906 ENTRY(coprocessor_error)
 907         zeroentry do_coprocessor_error
 908
 909 ENTRY(simd_coprocessor_error)
 910         zeroentry do_simd_coprocessor_error
 911
 912 ENTRY(device_not_available)
 913         zeroentry math_state_restore
 914
 915         /* runs on exception stack */
 916 KPROBE_ENTRY(debug)
 917         INTR_FRAME
 918         pushq $0
 919         CFI_ADJUST_CFA_OFFSET 8
 920         paranoidentry do_debug, DEBUG_IST
 921         jmp paranoid_exit
 922         CFI_ENDPROC
 923         .previous .text
 924
 925         /* runs on exception stack */
 926 ENTRY(nmi)
 927         INTR_FRAME
 928         pushq $-1
 929         CFI_ADJUST_CFA_OFFSET 8
 930         paranoidentry do_nmi
 931         /*
 932          * "Paranoid" exit path from exception stack.
 933          * Paranoid because this is used by NMIs and cannot take
 934          * any kernel state for granted.
 935          * We don't do kernel preemption checks here, because only
 936          * NMI should be common and it does not enable IRQs and
 937          * cannot get reschedule ticks.
 938          */
 939         /* ebx: no swapgs flag */
 940 paranoid_exit:
 941         testl %ebx,%ebx                         /* swapgs needed? */
 942         jnz paranoid_restore
 943         testl $3,CS(%rsp)
 944         jnz   paranoid_userspace
 945 paranoid_swapgs:
 946         swapgs
 947 paranoid_restore:
 948         RESTORE_ALL 8
 949         iretq
 950 paranoid_userspace:
 951         GET_THREAD_INFO(%rcx)
 952         movl threadinfo_flags(%rcx),%ebx
 953         andl $_TIF_WORK_MASK,%ebx
 954         jz paranoid_swapgs
 955         movq %rsp,%rdi                  /* &pt_regs */
 956         call sync_regs
 957         movq %rax,%rsp                  /* switch stack for scheduling */
 958         testl $_TIF_NEED_RESCHED,%ebx
 959         jnz paranoid_schedule
 960         movl %ebx,%edx                  /* arg3: thread flags */
 961         sti
 962         xorl %esi,%esi                  /* arg2: oldset */
 963         movq %rsp,%rdi                  /* arg1: &pt_regs */
 964         call do_notify_resume
 965         cli
 966         jmp paranoid_userspace
 967 paranoid_schedule:
 968         sti
 969         call schedule
 970         cli
 971         jmp paranoid_userspace
 972         CFI_ENDPROC
 973
 974 KPROBE_ENTRY(int3)
 975         INTR_FRAME
 976         pushq $0
 977         CFI_ADJUST_CFA_OFFSET 8
 978         paranoidentry do_int3, DEBUG_IST
 979         jmp paranoid_exit
 980         CFI_ENDPROC
 981         .previous .text
 982
 983 ENTRY(overflow)
 984         zeroentry do_overflow
 985
 986 ENTRY(bounds)
 987         zeroentry do_bounds
 988
 989 ENTRY(invalid_op)
 990         zeroentry do_invalid_op
 991
 992 ENTRY(coprocessor_segment_overrun)
 993         zeroentry do_coprocessor_segment_overrun
 994
 995 ENTRY(reserved)
 996         zeroentry do_reserved
 997
 998         /* runs on exception stack */
 999 ENTRY(double_fault)
1000         XCPT_FRAME
1001         paranoidentry do_double_fault
1002         jmp paranoid_exit
1003         CFI_ENDPROC
1004
1005 ENTRY(invalid_TSS)
1006         errorentry do_invalid_TSS
1007
1008 ENTRY(segment_not_present)
1009         errorentry do_segment_not_present
1010
1011         /* runs on exception stack */
1012 ENTRY(stack_segment)
1013         XCPT_FRAME
1014         paranoidentry do_stack_segment
1015         jmp paranoid_exit
1016         CFI_ENDPROC
1017
1018 KPROBE_ENTRY(general_protection)
1019         errorentry do_general_protection
1020         .previous .text
1021
1022 ENTRY(alignment_check)
1023         errorentry do_alignment_check
1024
1025 ENTRY(divide_error)
1026         zeroentry do_divide_error
1027
1028 ENTRY(spurious_interrupt_bug)
1029         zeroentry do_spurious_interrupt_bug
1030
1031 #ifdef CONFIG_X86_MCE
1032         /* runs on exception stack */
1033 ENTRY(machine_check)
1034         INTR_FRAME
1035         pushq $0
1036         CFI_ADJUST_CFA_OFFSET 8
1037         paranoidentry do_machine_check
1038         jmp paranoid_exit
1039         CFI_ENDPROC
1040 #endif
1041
1042 ENTRY(call_debug)
1043        zeroentry do_call_debug
1044
1045 ENTRY(call_softirq)
1046         CFI_STARTPROC
1047         movq %gs:pda_irqstackptr,%rax
1048         movq %rsp,%rdx
1049         CFI_DEF_CFA_REGISTER    rdx
1050         incl %gs:pda_irqcount
1051         cmove %rax,%rsp
1052         pushq %rdx
1053         /*todo CFI_DEF_CFA_EXPRESSION ...*/
1054         call __do_softirq
1055         popq %rsp
1056         CFI_DEF_CFA_REGISTER    rsp
1057         decl %gs:pda_irqcount
1058         ret
1059         CFI_ENDPROC