arch/arm64/kernel/entry.S

   1 /*
   2  * Low-level exception handling code
   3  *
   4  * Copyright (C) 2012 ARM Ltd.
   5  * Authors:     Catalin Marinas <catalin.marinas@arm.com>
   6  *              Will Deacon <will.deacon@arm.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #include <linux/init.h>
  22 #include <linux/linkage.h>
  23
  24 #include <asm/assembler.h>
  25 #include <asm/asm-offsets.h>
  26 #include <asm/errno.h>
  27 #include <asm/esr.h>
  28 #include <asm/thread_info.h>
  29 #include <asm/unistd.h>
  30
  31 /*
  32  * Context tracking subsystem.  Used to instrument transitions
  33  * between user and kernel mode.
  34  */
  35         .macro ct_user_exit, syscall = 0
  36 #ifdef CONFIG_CONTEXT_TRACKING
  37         bl      context_tracking_user_exit
  38         .if \syscall == 1
  39         /*
  40          * Save/restore needed during syscalls.  Restore syscall arguments from
  41          * the values already saved on stack during kernel_entry.
  42          */
  43         ldp     x0, x1, [sp]
  44         ldp     x2, x3, [sp, #S_X2]
  45         ldp     x4, x5, [sp, #S_X4]
  46         ldp     x6, x7, [sp, #S_X6]
  47         .endif
  48 #endif
  49         .endm
  50
  51         .macro ct_user_enter
  52 #ifdef CONFIG_CONTEXT_TRACKING
  53         bl      context_tracking_user_enter
  54 #endif
  55         .endm
  56
  57 /*
  58  * Bad Abort numbers
  59  *-----------------
  60  */
  61 #define BAD_SYNC        0
  62 #define BAD_IRQ         1
  63 #define BAD_FIQ         2
  64 #define BAD_ERROR       3
  65
  66         .macro  kernel_entry, el, regsize = 64
  67         sub     sp, sp, #S_FRAME_SIZE - S_LR    // room for LR, SP, SPSR, ELR
  68         .if     \regsize == 32
  69         mov     w0, w0                          // zero upper 32 bits of x0
  70         .endif
  71         push    x28, x29
  72         push    x26, x27
  73         push    x24, x25
  74         push    x22, x23
  75         push    x20, x21
  76         push    x18, x19
  77         push    x16, x17
  78         push    x14, x15
  79         push    x12, x13
  80         push    x10, x11
  81         push    x8, x9
  82         push    x6, x7
  83         push    x4, x5
  84         push    x2, x3
  85         push    x0, x1
  86         .if     \el == 0
  87         mrs     x21, sp_el0
  88         get_thread_info tsk                     // Ensure MDSCR_EL1.SS is clear,
  89         ldr     x19, [tsk, #TI_FLAGS]           // since we can unmask debug
  90         disable_step_tsk x19, x20               // exceptions when scheduling.
  91         .else
  92         add     x21, sp, #S_FRAME_SIZE
  93         .endif
  94         mrs     x22, elr_el1
  95         mrs     x23, spsr_el1
  96         stp     lr, x21, [sp, #S_LR]
  97         stp     x22, x23, [sp, #S_PC]
  98
  99         /*
 100          * Set syscallno to -1 by default (overridden later if real syscall).
 101          */
 102         .if     \el == 0
 103         mvn     x21, xzr
 104         str     x21, [sp, #S_SYSCALLNO]
 105         .endif
 106
 107         /*
 108          * Registers that may be useful after this macro is invoked:
 109          *
 110          * x21 - aborted SP
 111          * x22 - aborted PC
 112          * x23 - aborted PSTATE
 113         */
 114         .endm
 115
 116         .macro  kernel_exit, el, ret = 0
 117         ldp     x21, x22, [sp, #S_PC]           // load ELR, SPSR
 118         .if     \el == 0
 119         ct_user_enter
 120         ldr     x23, [sp, #S_SP]                // load return stack pointer
 121         .endif
 122         .if     \ret
 123         ldr     x1, [sp, #S_X1]                 // preserve x0 (syscall return)
 124         add     sp, sp, S_X2
 125         .else
 126         pop     x0, x1
 127         .endif
 128         pop     x2, x3                          // load the rest of the registers
 129         pop     x4, x5
 130         pop     x6, x7
 131         pop     x8, x9
 132         msr     elr_el1, x21                    // set up the return data
 133         msr     spsr_el1, x22
 134         .if     \el == 0
 135         msr     sp_el0, x23
 136         .endif
 137         pop     x10, x11
 138         pop     x12, x13
 139         pop     x14, x15
 140         pop     x16, x17
 141         pop     x18, x19
 142         pop     x20, x21
 143         pop     x22, x23
 144         pop     x24, x25
 145         pop     x26, x27
 146         pop     x28, x29
 147         ldr     lr, [sp], #S_FRAME_SIZE - S_LR  // load LR and restore SP
 148         eret                                    // return to kernel
 149         .endm
 150
 151         .macro  get_thread_info, rd
 152         mov     \rd, sp
 153         and     \rd, \rd, #~(THREAD_SIZE - 1)   // top of stack
 154         .endm
 155
 156 /*
 157  * These are the registers used in the syscall handler, and allow us to
 158  * have in theory up to 7 arguments to a function - x0 to x6.
 159  *
 160  * x7 is reserved for the system call number in 32-bit mode.
 161  */
 162 sc_nr   .req    x25             // number of system calls
 163 scno    .req    x26             // syscall number
 164 stbl    .req    x27             // syscall table pointer
 165 tsk     .req    x28             // current thread_info
 166
 167 /*
 168  * Interrupt handling.
 169  */
 170         .macro  irq_handler
 171         ldr     x1, handle_arch_irq
 172         mov     x0, sp
 173         blr     x1
 174         .endm
 175
 176         .text
 177
 178 /*
 179  * Exception vectors.
 180  */
 181
 182         .align  11
 183 ENTRY(vectors)
 184         ventry  el1_sync_invalid                // Synchronous EL1t
 185         ventry  el1_irq_invalid                 // IRQ EL1t
 186         ventry  el1_fiq_invalid                 // FIQ EL1t
 187         ventry  el1_error_invalid               // Error EL1t
 188
 189         ventry  el1_sync                        // Synchronous EL1h
 190         ventry  el1_irq                         // IRQ EL1h
 191         ventry  el1_fiq_invalid                 // FIQ EL1h
 192         ventry  el1_error_invalid               // Error EL1h
 193
 194         ventry  el0_sync                        // Synchronous 64-bit EL0
 195         ventry  el0_irq                         // IRQ 64-bit EL0
 196         ventry  el0_fiq_invalid                 // FIQ 64-bit EL0
 197         ventry  el0_error_invalid               // Error 64-bit EL0
 198
 199 #ifdef CONFIG_COMPAT
 200         ventry  el0_sync_compat                 // Synchronous 32-bit EL0
 201         ventry  el0_irq_compat                  // IRQ 32-bit EL0
 202         ventry  el0_fiq_invalid_compat          // FIQ 32-bit EL0
 203         ventry  el0_error_invalid_compat        // Error 32-bit EL0
 204 #else
 205         ventry  el0_sync_invalid                // Synchronous 32-bit EL0
 206         ventry  el0_irq_invalid                 // IRQ 32-bit EL0
 207         ventry  el0_fiq_invalid                 // FIQ 32-bit EL0
 208         ventry  el0_error_invalid               // Error 32-bit EL0
 209 #endif
 210 END(vectors)
 211
 212 /*
 213  * Invalid mode handlers
 214  */
 215         .macro  inv_entry, el, reason, regsize = 64
 216         kernel_entry el, \regsize
 217         mov     x0, sp
 218         mov     x1, #\reason
 219         mrs     x2, esr_el1
 220         b       bad_mode
 221         .endm
 222
 223 el0_sync_invalid:
 224         inv_entry 0, BAD_SYNC
 225 ENDPROC(el0_sync_invalid)
 226
 227 el0_irq_invalid:
 228         inv_entry 0, BAD_IRQ
 229 ENDPROC(el0_irq_invalid)
 230
 231 el0_fiq_invalid:
 232         inv_entry 0, BAD_FIQ
 233 ENDPROC(el0_fiq_invalid)
 234
 235 el0_error_invalid:
 236         inv_entry 0, BAD_ERROR
 237 ENDPROC(el0_error_invalid)
 238
 239 #ifdef CONFIG_COMPAT
 240 el0_fiq_invalid_compat:
 241         inv_entry 0, BAD_FIQ, 32
 242 ENDPROC(el0_fiq_invalid_compat)
 243
 244 el0_error_invalid_compat:
 245         inv_entry 0, BAD_ERROR, 32
 246 ENDPROC(el0_error_invalid_compat)
 247 #endif
 248
 249 el1_sync_invalid:
 250         inv_entry 1, BAD_SYNC
 251 ENDPROC(el1_sync_invalid)
 252
 253 el1_irq_invalid:
 254         inv_entry 1, BAD_IRQ
 255 ENDPROC(el1_irq_invalid)
 256
 257 el1_fiq_invalid:
 258         inv_entry 1, BAD_FIQ
 259 ENDPROC(el1_fiq_invalid)
 260
 261 el1_error_invalid:
 262         inv_entry 1, BAD_ERROR
 263 ENDPROC(el1_error_invalid)
 264
 265 /*
 266  * EL1 mode handlers.
 267  */
 268         .align  6
 269 el1_sync:
 270         kernel_entry 1
 271         mrs     x1, esr_el1                     // read the syndrome register
 272         lsr     x24, x1, #ESR_EL1_EC_SHIFT      // exception class
 273         cmp     x24, #ESR_EL1_EC_DABT_EL1       // data abort in EL1
 274         b.eq    el1_da
 275         cmp     x24, #ESR_EL1_EC_SYS64          // configurable trap
 276         b.eq    el1_undef
 277         cmp     x24, #ESR_EL1_EC_SP_ALIGN       // stack alignment exception
 278         b.eq    el1_sp_pc
 279         cmp     x24, #ESR_EL1_EC_PC_ALIGN       // pc alignment exception
 280         b.eq    el1_sp_pc
 281         cmp     x24, #ESR_EL1_EC_UNKNOWN        // unknown exception in EL1
 282         b.eq    el1_undef
 283         cmp     x24, #ESR_EL1_EC_BREAKPT_EL1    // debug exception in EL1
 284         b.ge    el1_dbg
 285         b       el1_inv
 286 el1_da:
 287         /*
 288          * Data abort handling
 289          */
 290         mrs     x0, far_el1
 291         enable_dbg
 292         // re-enable interrupts if they were enabled in the aborted context
 293         tbnz    x23, #7, 1f                     // PSR_I_BIT
 294         enable_irq
 295 1:
 296         mov     x2, sp                          // struct pt_regs
 297         bl      do_mem_abort
 298
 299         // disable interrupts before pulling preserved data off the stack
 300         disable_irq
 301         kernel_exit 1
 302 el1_sp_pc:
 303         /*
 304          * Stack or PC alignment exception handling
 305          */
 306         mrs     x0, far_el1
 307         enable_dbg
 308         mov     x2, sp
 309         b       do_sp_pc_abort
 310 el1_undef:
 311         /*
 312          * Undefined instruction
 313          */
 314         enable_dbg
 315         mov     x0, sp
 316         b       do_undefinstr
 317 el1_dbg:
 318         /*
 319          * Debug exception handling
 320          */
 321         cmp     x24, #ESR_EL1_EC_BRK64          // if BRK64
 322         cinc    x24, x24, eq                    // set bit '0'
 323         tbz     x24, #0, el1_inv                // EL1 only
 324         mrs     x0, far_el1
 325         mov     x2, sp                          // struct pt_regs
 326         bl      do_debug_exception
 327         enable_dbg
 328         kernel_exit 1
 329 el1_inv:
 330         // TODO: add support for undefined instructions in kernel mode
 331         enable_dbg
 332         mov     x0, sp
 333         mov     x1, #BAD_SYNC
 334         mrs     x2, esr_el1
 335         b       bad_mode
 336 ENDPROC(el1_sync)
 337
 338         .align  6
 339 el1_irq:
 340         kernel_entry 1
 341         enable_dbg
 342 #ifdef CONFIG_TRACE_IRQFLAGS
 343         bl      trace_hardirqs_off
 344 #endif
 345
 346         irq_handler
 347
 348 #ifdef CONFIG_PREEMPT
 349         get_thread_info tsk
 350         ldr     w24, [tsk, #TI_PREEMPT]         // get preempt count
 351         cbnz    w24, 1f                         // preempt count != 0
 352         ldr     x0, [tsk, #TI_FLAGS]            // get flags
 353         tbz     x0, #TIF_NEED_RESCHED, 1f       // needs rescheduling?
 354         bl      el1_preempt
 355 1:
 356 #endif
 357 #ifdef CONFIG_TRACE_IRQFLAGS
 358         bl      trace_hardirqs_on
 359 #endif
 360         kernel_exit 1
 361 ENDPROC(el1_irq)
 362
 363 #ifdef CONFIG_PREEMPT
 364 el1_preempt:
 365         mov     x24, lr
 366 1:      bl      preempt_schedule_irq            // irq en/disable is done inside
 367         ldr     x0, [tsk, #TI_FLAGS]            // get new tasks TI_FLAGS
 368         tbnz    x0, #TIF_NEED_RESCHED, 1b       // needs rescheduling?
 369         ret     x24
 370 #endif
 371
 372 /*
 373  * EL0 mode handlers.
 374  */
 375         .align  6
 376 el0_sync:
 377         kernel_entry 0
 378         mrs     x25, esr_el1                    // read the syndrome register
 379         lsr     x24, x25, #ESR_EL1_EC_SHIFT     // exception class
 380         cmp     x24, #ESR_EL1_EC_SVC64          // SVC in 64-bit state
 381         b.eq    el0_svc
 382         cmp     x24, #ESR_EL1_EC_DABT_EL0       // data abort in EL0
 383         b.eq    el0_da
 384         cmp     x24, #ESR_EL1_EC_IABT_EL0       // instruction abort in EL0
 385         b.eq    el0_ia
 386         cmp     x24, #ESR_EL1_EC_FP_ASIMD       // FP/ASIMD access
 387         b.eq    el0_fpsimd_acc
 388         cmp     x24, #ESR_EL1_EC_FP_EXC64       // FP/ASIMD exception
 389         b.eq    el0_fpsimd_exc
 390         cmp     x24, #ESR_EL1_EC_SYS64          // configurable trap
 391         b.eq    el0_undef
 392         cmp     x24, #ESR_EL1_EC_SP_ALIGN       // stack alignment exception
 393         b.eq    el0_sp_pc
 394         cmp     x24, #ESR_EL1_EC_PC_ALIGN       // pc alignment exception
 395         b.eq    el0_sp_pc
 396         cmp     x24, #ESR_EL1_EC_UNKNOWN        // unknown exception in EL0
 397         b.eq    el0_undef
 398         cmp     x24, #ESR_EL1_EC_BREAKPT_EL0    // debug exception in EL0
 399         b.ge    el0_dbg
 400         b       el0_inv
 401
 402 #ifdef CONFIG_COMPAT
 403         .align  6
 404 el0_sync_compat:
 405         kernel_entry 0, 32
 406         mrs     x25, esr_el1                    // read the syndrome register
 407         lsr     x24, x25, #ESR_EL1_EC_SHIFT     // exception class
 408         cmp     x24, #ESR_EL1_EC_SVC32          // SVC in 32-bit state
 409         b.eq    el0_svc_compat
 410         cmp     x24, #ESR_EL1_EC_DABT_EL0       // data abort in EL0
 411         b.eq    el0_da
 412         cmp     x24, #ESR_EL1_EC_IABT_EL0       // instruction abort in EL0
 413         b.eq    el0_ia
 414         cmp     x24, #ESR_EL1_EC_FP_ASIMD       // FP/ASIMD access
 415         b.eq    el0_fpsimd_acc
 416         cmp     x24, #ESR_EL1_EC_FP_EXC32       // FP/ASIMD exception
 417         b.eq    el0_fpsimd_exc
 418         cmp     x24, #ESR_EL1_EC_UNKNOWN        // unknown exception in EL0
 419         b.eq    el0_undef
 420         cmp     x24, #ESR_EL1_EC_CP15_32        // CP15 MRC/MCR trap
 421         b.eq    el0_undef
 422         cmp     x24, #ESR_EL1_EC_CP15_64        // CP15 MRRC/MCRR trap
 423         b.eq    el0_undef
 424         cmp     x24, #ESR_EL1_EC_CP14_MR        // CP14 MRC/MCR trap
 425         b.eq    el0_undef
 426         cmp     x24, #ESR_EL1_EC_CP14_LS        // CP14 LDC/STC trap
 427         b.eq    el0_undef
 428         cmp     x24, #ESR_EL1_EC_CP14_64        // CP14 MRRC/MCRR trap
 429         b.eq    el0_undef
 430         cmp     x24, #ESR_EL1_EC_BREAKPT_EL0    // debug exception in EL0
 431         b.ge    el0_dbg
 432         b       el0_inv
 433 el0_svc_compat:
 434         /*
 435          * AArch32 syscall handling
 436          */
 437         adr     stbl, compat_sys_call_table     // load compat syscall table pointer
 438         uxtw    scno, w7                        // syscall number in w7 (r7)
 439         mov     sc_nr, #__NR_compat_syscalls
 440         b       el0_svc_naked
 441
 442         .align  6
 443 el0_irq_compat:
 444         kernel_entry 0, 32
 445         b       el0_irq_naked
 446 #endif
 447
 448 el0_da:
 449         /*
 450          * Data abort handling
 451          */
 452         mrs     x26, far_el1
 453         // enable interrupts before calling the main handler
 454         enable_dbg_and_irq
 455         ct_user_exit
 456         bic     x0, x26, #(0xff << 56)
 457         mov     x1, x25
 458         mov     x2, sp
 459         adr     lr, ret_to_user
 460         b       do_mem_abort
 461 el0_ia:
 462         /*
 463          * Instruction abort handling
 464          */
 465         mrs     x26, far_el1
 466         // enable interrupts before calling the main handler
 467         enable_dbg_and_irq
 468         ct_user_exit
 469         mov     x0, x26
 470         orr     x1, x25, #1 << 24               // use reserved ISS bit for instruction aborts
 471         mov     x2, sp
 472         adr     lr, ret_to_user
 473         b       do_mem_abort
 474 el0_fpsimd_acc:
 475         /*
 476          * Floating Point or Advanced SIMD access
 477          */
 478         enable_dbg
 479         ct_user_exit
 480         mov     x0, x25
 481         mov     x1, sp
 482         adr     lr, ret_to_user
 483         b       do_fpsimd_acc
 484 el0_fpsimd_exc:
 485         /*
 486          * Floating Point or Advanced SIMD exception
 487          */
 488         enable_dbg
 489         ct_user_exit
 490         mov     x0, x25
 491         mov     x1, sp
 492         adr     lr, ret_to_user
 493         b       do_fpsimd_exc
 494 el0_sp_pc:
 495         /*
 496          * Stack or PC alignment exception handling
 497          */
 498         mrs     x26, far_el1
 499         // enable interrupts before calling the main handler
 500         enable_dbg_and_irq
 501         mov     x0, x26
 502         mov     x1, x25
 503         mov     x2, sp
 504         adr     lr, ret_to_user
 505         b       do_sp_pc_abort
 506 el0_undef:
 507         /*
 508          * Undefined instruction
 509          */
 510         // enable interrupts before calling the main handler
 511         enable_dbg_and_irq
 512         ct_user_exit
 513         mov     x0, sp
 514         adr     lr, ret_to_user
 515         b       do_undefinstr
 516 el0_dbg:
 517         /*
 518          * Debug exception handling
 519          */
 520         tbnz    x24, #0, el0_inv                // EL0 only
 521         mrs     x0, far_el1
 522         mov     x1, x25
 523         mov     x2, sp
 524         bl      do_debug_exception
 525         enable_dbg
 526         ct_user_exit
 527         b       ret_to_user
 528 el0_inv:
 529         enable_dbg
 530         ct_user_exit
 531         mov     x0, sp
 532         mov     x1, #BAD_SYNC
 533         mrs     x2, esr_el1
 534         adr     lr, ret_to_user
 535         b       bad_mode
 536 ENDPROC(el0_sync)
 537
 538         .align  6
 539 el0_irq:
 540         kernel_entry 0
 541 el0_irq_naked:
 542         enable_dbg
 543 #ifdef CONFIG_TRACE_IRQFLAGS
 544         bl      trace_hardirqs_off
 545 #endif
 546
 547         ct_user_exit
 548         irq_handler
 549
 550 #ifdef CONFIG_TRACE_IRQFLAGS
 551         bl      trace_hardirqs_on
 552 #endif
 553         b       ret_to_user
 554 ENDPROC(el0_irq)
 555
 556 /*
 557  * Register switch for AArch64. The callee-saved registers need to be saved
 558  * and restored. On entry:
 559  *   x0 = previous task_struct (must be preserved across the switch)
 560  *   x1 = next task_struct
 561  * Previous and next are guaranteed not to be the same.
 562  *
 563  */
 564 ENTRY(cpu_switch_to)
 565         add     x8, x0, #THREAD_CPU_CONTEXT
 566         mov     x9, sp
 567         stp     x19, x20, [x8], #16             // store callee-saved registers
 568         stp     x21, x22, [x8], #16
 569         stp     x23, x24, [x8], #16
 570         stp     x25, x26, [x8], #16
 571         stp     x27, x28, [x8], #16
 572         stp     x29, x9, [x8], #16
 573         str     lr, [x8]
 574         add     x8, x1, #THREAD_CPU_CONTEXT
 575         ldp     x19, x20, [x8], #16             // restore callee-saved registers
 576         ldp     x21, x22, [x8], #16
 577         ldp     x23, x24, [x8], #16
 578         ldp     x25, x26, [x8], #16
 579         ldp     x27, x28, [x8], #16
 580         ldp     x29, x9, [x8], #16
 581         ldr     lr, [x8]
 582         mov     sp, x9
 583         ret
 584 ENDPROC(cpu_switch_to)
 585
 586 /*
 587  * This is the fast syscall return path.  We do as little as possible here,
 588  * and this includes saving x0 back into the kernel stack.
 589  */
 590 ret_fast_syscall:
 591         disable_irq                             // disable interrupts
 592         ldr     x1, [tsk, #TI_FLAGS]
 593         and     x2, x1, #_TIF_WORK_MASK
 594         cbnz    x2, fast_work_pending
 595         enable_step_tsk x1, x2
 596         kernel_exit 0, ret = 1
 597
 598 /*
 599  * Ok, we need to do extra processing, enter the slow path.
 600  */
 601 fast_work_pending:
 602         str     x0, [sp, #S_X0]                 // returned x0
 603 work_pending:
 604         tbnz    x1, #TIF_NEED_RESCHED, work_resched
 605         /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
 606         ldr     x2, [sp, #S_PSTATE]
 607         mov     x0, sp                          // 'regs'
 608         tst     x2, #PSR_MODE_MASK              // user mode regs?
 609         b.ne    no_work_pending                 // returning to kernel
 610         enable_irq                              // enable interrupts for do_notify_resume()
 611         bl      do_notify_resume
 612         b       ret_to_user
 613 work_resched:
 614         bl      schedule
 615
 616 /*
 617  * "slow" syscall return path.
 618  */
 619 ret_to_user:
 620         disable_irq                             // disable interrupts
 621         ldr     x1, [tsk, #TI_FLAGS]
 622         and     x2, x1, #_TIF_WORK_MASK
 623         cbnz    x2, work_pending
 624         enable_step_tsk x1, x2
 625 no_work_pending:
 626         kernel_exit 0, ret = 0
 627 ENDPROC(ret_to_user)
 628
 629 /*
 630  * This is how we return from a fork.
 631  */
 632 ENTRY(ret_from_fork)
 633         bl      schedule_tail
 634         cbz     x19, 1f                         // not a kernel thread
 635         mov     x0, x20
 636         blr     x19
 637 1:      get_thread_info tsk
 638         b       ret_to_user
 639 ENDPROC(ret_from_fork)
 640
 641 /*
 642  * SVC handler.
 643  */
 644         .align  6
 645 el0_svc:
 646         adrp    stbl, sys_call_table            // load syscall table pointer
 647         uxtw    scno, w8                        // syscall number in w8
 648         mov     sc_nr, #__NR_syscalls
 649 el0_svc_naked:                                  // compat entry point
 650         stp     x0, scno, [sp, #S_ORIG_X0]      // save the original x0 and syscall number
 651         enable_dbg_and_irq
 652         ct_user_exit 1
 653
 654         ldr     x16, [tsk, #TI_FLAGS]           // check for syscall hooks
 655         tst     x16, #_TIF_SYSCALL_WORK
 656         b.ne    __sys_trace
 657         adr     lr, ret_fast_syscall            // return address
 658         cmp     scno, sc_nr                     // check upper syscall limit
 659         b.hs    ni_sys
 660         ldr     x16, [stbl, scno, lsl #3]       // address in the syscall table
 661         br      x16                             // call sys_* routine
 662 ni_sys:
 663         mov     x0, sp
 664         b       do_ni_syscall
 665 ENDPROC(el0_svc)
 666
 667         /*
 668          * This is the really slow path.  We're going to be doing context
 669          * switches, and waiting for our parent to respond.
 670          */
 671 __sys_trace:
 672         mov     x0, sp
 673         bl      syscall_trace_enter
 674         adr     lr, __sys_trace_return          // return address
 675         uxtw    scno, w0                        // syscall number (possibly new)
 676         mov     x1, sp                          // pointer to regs
 677         cmp     scno, sc_nr                     // check upper syscall limit
 678         b.hs    ni_sys
 679         ldp     x0, x1, [sp]                    // restore the syscall args
 680         ldp     x2, x3, [sp, #S_X2]
 681         ldp     x4, x5, [sp, #S_X4]
 682         ldp     x6, x7, [sp, #S_X6]
 683         ldr     x16, [stbl, scno, lsl #3]       // address in the syscall table
 684         br      x16                             // call sys_* routine
 685
 686 __sys_trace_return:
 687         str     x0, [sp]                        // save returned x0
 688         mov     x0, sp
 689         bl      syscall_trace_exit
 690         b       ret_to_user
 691
 692 /*
 693  * Special system call wrappers.
 694  */
 695 ENTRY(sys_rt_sigreturn_wrapper)
 696         mov     x0, sp
 697         b       sys_rt_sigreturn
 698 ENDPROC(sys_rt_sigreturn_wrapper)
 699
 700 ENTRY(handle_arch_irq)
 701         .quad   0