arch/arm/kvm/interrupts.S

   1 /*
   2  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
   3  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License, version 2, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  17  */
  18
  19 #include <linux/linkage.h>
  20 #include <linux/const.h>
  21 #include <asm/unified.h>
  22 #include <asm/page.h>
  23 #include <asm/ptrace.h>
  24 #include <asm/asm-offsets.h>
  25 #include <asm/kvm_asm.h>
  26 #include <asm/kvm_arm.h>
  27 #include <asm/vfpmacros.h>
  28 #include "interrupts_head.S"
  29
  30         .text
  31         .pushsection    .hyp.text, "ax"
  32
  33 /********************************************************************
  34  * Flush per-VMID TLBs
  35  *
  36  * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
  37  *
  38  * We rely on the hardware to broadcast the TLB invalidation to all CPUs
  39  * inside the inner-shareable domain (which is the case for all v7
  40  * implementations).  If we come across a non-IS SMP implementation, we'll
  41  * have to use an IPI based mechanism. Until then, we stick to the simple
  42  * hardware assisted version.
  43  *
  44  * As v7 does not support flushing per IPA, just nuke the whole TLB
  45  * instead, ignoring the ipa value.
  46  */
  47 ENTRY(__kvm_tlb_flush_vmid_ipa)
  48         push    {r2, r3}
  49
  50         dsb     ishst
  51         add     r0, r0, #KVM_VTTBR
  52         ldrd    r2, r3, [r0]
  53         mcrr    p15, 6, rr_lo_hi(r2, r3), c2    @ Write VTTBR
  54         isb
  55         mcr     p15, 0, r0, c8, c3, 0   @ TLBIALLIS (rt ignored)
  56         dsb     ish
  57         isb
  58         mov     r2, #0
  59         mov     r3, #0
  60         mcrr    p15, 6, r2, r3, c2      @ Back to VMID #0
  61         isb                             @ Not necessary if followed by eret
  62
  63         pop     {r2, r3}
  64         bx      lr
  65 ENDPROC(__kvm_tlb_flush_vmid_ipa)
  66
  67 /**
  68  * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
  69  *
  70  * Reuses __kvm_tlb_flush_vmid_ipa() for ARMv7, without passing address
  71  * parameter
  72  */
  73
  74 ENTRY(__kvm_tlb_flush_vmid)
  75         b       __kvm_tlb_flush_vmid_ipa
  76 ENDPROC(__kvm_tlb_flush_vmid)
  77
  78 /********************************************************************
  79  * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
  80  * domain, for all VMIDs
  81  *
  82  * void __kvm_flush_vm_context(void);
  83  */
  84 ENTRY(__kvm_flush_vm_context)
  85         mov     r0, #0                  @ rn parameter for c15 flushes is SBZ
  86
  87         /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
  88         mcr     p15, 4, r0, c8, c3, 4
  89         /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
  90         mcr     p15, 0, r0, c7, c1, 0
  91         dsb     ish
  92         isb                             @ Not necessary if followed by eret
  93
  94         bx      lr
  95 ENDPROC(__kvm_flush_vm_context)
  96
  97
  98 /********************************************************************
  99  *  Hypervisor world-switch code
 100  *
 101  *
 102  * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
 103  */
 104 ENTRY(__kvm_vcpu_run)
 105         @ Save the vcpu pointer
 106         mcr     p15, 4, vcpu, c13, c0, 2        @ HTPIDR
 107
 108         save_host_regs
 109
 110         restore_vgic_state
 111         restore_timer_state
 112
 113         @ Store hardware CP15 state and load guest state
 114         read_cp15_state store_to_vcpu = 0
 115         write_cp15_state read_from_vcpu = 1
 116
 117         @ If the host kernel has not been configured with VFPv3 support,
 118         @ then it is safer if we deny guests from using it as well.
 119 #ifdef CONFIG_VFPv3
 120         @ Set FPEXC_EN so the guest doesn't trap floating point instructions
 121         VFPFMRX r2, FPEXC               @ VMRS
 122         push    {r2}
 123         orr     r2, r2, #FPEXC_EN
 124         VFPFMXR FPEXC, r2               @ VMSR
 125 #endif
 126
 127         @ Configure Hyp-role
 128         configure_hyp_role vmentry
 129
 130         @ Trap coprocessor CRx accesses
 131         set_hstr vmentry
 132         set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
 133         set_hdcr vmentry
 134
 135         @ Write configured ID register into MIDR alias
 136         ldr     r1, [vcpu, #VCPU_MIDR]
 137         mcr     p15, 4, r1, c0, c0, 0
 138
 139         @ Write guest view of MPIDR into VMPIDR
 140         ldr     r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
 141         mcr     p15, 4, r1, c0, c0, 5
 142
 143         @ Set up guest memory translation
 144         ldr     r1, [vcpu, #VCPU_KVM]
 145         add     r1, r1, #KVM_VTTBR
 146         ldrd    r2, r3, [r1]
 147         mcrr    p15, 6, rr_lo_hi(r2, r3), c2    @ Write VTTBR
 148
 149         @ We're all done, just restore the GPRs and go to the guest
 150         restore_guest_regs
 151         clrex                           @ Clear exclusive monitor
 152         eret
 153
 154 __kvm_vcpu_return:
 155         /*
 156          * return convention:
 157          * guest r0, r1, r2 saved on the stack
 158          * r0: vcpu pointer
 159          * r1: exception code
 160          */
 161         save_guest_regs
 162
 163         @ Set VMID == 0
 164         mov     r2, #0
 165         mov     r3, #0
 166         mcrr    p15, 6, r2, r3, c2      @ Write VTTBR
 167
 168         @ Don't trap coprocessor accesses for host kernel
 169         set_hstr vmexit
 170         set_hdcr vmexit
 171         set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)), after_vfp_restore
 172
 173 #ifdef CONFIG_VFPv3
 174         @ Switch VFP/NEON hardware state to the host's
 175         add     r7, vcpu, #(VCPU_GUEST_CTXT + CPU_CTXT_VFP)
 176         store_vfp_state r7
 177         add     r7, vcpu, #VCPU_HOST_CTXT
 178         ldr     r7, [r7]
 179         add     r7, r7, #CPU_CTXT_VFP
 180         restore_vfp_state r7
 181
 182 after_vfp_restore:
 183         @ Restore FPEXC_EN which we clobbered on entry
 184         pop     {r2}
 185         VFPFMXR FPEXC, r2
 186 #else
 187 after_vfp_restore:
 188 #endif
 189
 190         @ Reset Hyp-role
 191         configure_hyp_role vmexit
 192
 193         @ Let host read hardware MIDR
 194         mrc     p15, 0, r2, c0, c0, 0
 195         mcr     p15, 4, r2, c0, c0, 0
 196
 197         @ Back to hardware MPIDR
 198         mrc     p15, 0, r2, c0, c0, 5
 199         mcr     p15, 4, r2, c0, c0, 5
 200
 201         @ Store guest CP15 state and restore host state
 202         read_cp15_state store_to_vcpu = 1
 203         write_cp15_state read_from_vcpu = 0
 204
 205         save_timer_state
 206         save_vgic_state
 207
 208         restore_host_regs
 209         clrex                           @ Clear exclusive monitor
 210 #ifndef CONFIG_CPU_ENDIAN_BE8
 211         mov     r0, r1                  @ Return the return code
 212         mov     r1, #0                  @ Clear upper bits in return value
 213 #else
 214         @ r1 already has return code
 215         mov     r0, #0                  @ Clear upper bits in return value
 216 #endif /* CONFIG_CPU_ENDIAN_BE8 */
 217         bx      lr                      @ return to IOCTL
 218
 219 /********************************************************************
 220  *  Call function in Hyp mode
 221  *
 222  *
 223  * u64 kvm_call_hyp(void *hypfn, ...);
 224  *
 225  * This is not really a variadic function in the classic C-way and care must
 226  * be taken when calling this to ensure parameters are passed in registers
 227  * only, since the stack will change between the caller and the callee.
 228  *
 229  * Call the function with the first argument containing a pointer to the
 230  * function you wish to call in Hyp mode, and subsequent arguments will be
 231  * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
 232  * function pointer can be passed).  The function being called must be mapped
 233  * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c).  Return values are
 234  * passed in r0 and r1.
 235  *
 236  * A function pointer with a value of 0xffffffff has a special meaning,
 237  * and is used to implement __hyp_get_vectors in the same way as in
 238  * arch/arm/kernel/hyp_stub.S.
 239  *
 240  * The calling convention follows the standard AAPCS:
 241  *   r0 - r3: caller save
 242  *   r12:     caller save
 243  *   rest:    callee save
 244  */
 245 ENTRY(kvm_call_hyp)
 246         hvc     #0
 247         bx      lr
 248
 249 /********************************************************************
 250  * Hypervisor exception vector and handlers
 251  *
 252  *
 253  * The KVM/ARM Hypervisor ABI is defined as follows:
 254  *
 255  * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
 256  * instruction is issued since all traps are disabled when running the host
 257  * kernel as per the Hyp-mode initialization at boot time.
 258  *
 259  * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
 260  * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
 261  * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
 262  * instructions are called from within Hyp-mode.
 263  *
 264  * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
 265  *    Switching to Hyp mode is done through a simple HVC #0 instruction. The
 266  *    exception vector code will check that the HVC comes from VMID==0 and if
 267  *    so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
 268  *    - r0 contains a pointer to a HYP function
 269  *    - r1, r2, and r3 contain arguments to the above function.
 270  *    - The HYP function will be called with its arguments in r0, r1 and r2.
 271  *    On HYP function return, we return directly to SVC.
 272  *
 273  * Note that the above is used to execute code in Hyp-mode from a host-kernel
 274  * point of view, and is a different concept from performing a world-switch and
 275  * executing guest code SVC mode (with a VMID != 0).
 276  */
 277
 278 /* Handle undef, svc, pabt, or dabt by crashing with a user notice */
 279 .macro bad_exception exception_code, panic_str
 280         push    {r0-r2}
 281         mrrc    p15, 6, r0, r1, c2      @ Read VTTBR
 282         lsr     r1, r1, #16
 283         ands    r1, r1, #0xff
 284         beq     99f
 285
 286         load_vcpu                       @ Load VCPU pointer
 287         .if \exception_code == ARM_EXCEPTION_DATA_ABORT
 288         mrc     p15, 4, r2, c5, c2, 0   @ HSR
 289         mrc     p15, 4, r1, c6, c0, 0   @ HDFAR
 290         str     r2, [vcpu, #VCPU_HSR]
 291         str     r1, [vcpu, #VCPU_HxFAR]
 292         .endif
 293         .if \exception_code == ARM_EXCEPTION_PREF_ABORT
 294         mrc     p15, 4, r2, c5, c2, 0   @ HSR
 295         mrc     p15, 4, r1, c6, c0, 2   @ HIFAR
 296         str     r2, [vcpu, #VCPU_HSR]
 297         str     r1, [vcpu, #VCPU_HxFAR]
 298         .endif
 299         mov     r1, #\exception_code
 300         b       __kvm_vcpu_return
 301
 302         @ We were in the host already. Let's craft a panic-ing return to SVC.
 303 99:     mrs     r2, cpsr
 304         bic     r2, r2, #MODE_MASK
 305         orr     r2, r2, #SVC_MODE
 306 THUMB(  orr     r2, r2, #PSR_T_BIT      )
 307         msr     spsr_cxsf, r2
 308         mrs     r1, ELR_hyp
 309         ldr     r2, =panic
 310         msr     ELR_hyp, r2
 311         ldr     r0, =\panic_str
 312         clrex                           @ Clear exclusive monitor
 313         eret
 314 .endm
 315
 316         .align 5
 317 __kvm_hyp_vector:
 318         .globl __kvm_hyp_vector
 319
 320         @ Hyp-mode exception vector
 321         W(b)    hyp_reset
 322         W(b)    hyp_undef
 323         W(b)    hyp_svc
 324         W(b)    hyp_pabt
 325         W(b)    hyp_dabt
 326         W(b)    hyp_hvc
 327         W(b)    hyp_irq
 328         W(b)    hyp_fiq
 329
 330         .align
 331 hyp_reset:
 332         b       hyp_reset
 333
 334         .align
 335 hyp_undef:
 336         bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
 337
 338         .align
 339 hyp_svc:
 340         bad_exception ARM_EXCEPTION_HVC, svc_die_str
 341
 342         .align
 343 hyp_pabt:
 344         bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
 345
 346         .align
 347 hyp_dabt:
 348         bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
 349
 350         .align
 351 hyp_hvc:
 352         /*
 353          * Getting here is either becuase of a trap from a guest or from calling
 354          * HVC from the host kernel, which means "switch to Hyp mode".
 355          */
 356         push    {r0, r1, r2}
 357
 358         @ Check syndrome register
 359         mrc     p15, 4, r1, c5, c2, 0   @ HSR
 360         lsr     r0, r1, #HSR_EC_SHIFT
 361         cmp     r0, #HSR_EC_HVC
 362         bne     guest_trap              @ Not HVC instr.
 363
 364         /*
 365          * Let's check if the HVC came from VMID 0 and allow simple
 366          * switch to Hyp mode
 367          */
 368         mrrc    p15, 6, r0, r2, c2
 369         lsr     r2, r2, #16
 370         and     r2, r2, #0xff
 371         cmp     r2, #0
 372         bne     guest_trap              @ Guest called HVC
 373
 374         /*
 375          * Getting here means host called HVC, we shift parameters and branch
 376          * to Hyp function.
 377          */
 378         pop     {r0, r1, r2}
 379
 380         /* Check for __hyp_get_vectors */
 381         cmp     r0, #-1
 382         mrceq   p15, 4, r0, c12, c0, 0  @ get HVBAR
 383         beq     1f
 384
 385         push    {lr}
 386         mrs     lr, SPSR
 387         push    {lr}
 388
 389         mov     lr, r0
 390         mov     r0, r1
 391         mov     r1, r2
 392         mov     r2, r3
 393
 394 THUMB(  orr     lr, #1)
 395         blx     lr                      @ Call the HYP function
 396
 397         pop     {lr}
 398         msr     SPSR_csxf, lr
 399         pop     {lr}
 400 1:      eret
 401
 402 guest_trap:
 403         load_vcpu                       @ Load VCPU pointer to r0
 404         str     r1, [vcpu, #VCPU_HSR]
 405
 406         @ Check if we need the fault information
 407         lsr     r1, r1, #HSR_EC_SHIFT
 408 #ifdef CONFIG_VFPv3
 409         cmp     r1, #HSR_EC_CP_0_13
 410         beq     switch_to_guest_vfp
 411 #endif
 412         cmp     r1, #HSR_EC_IABT
 413         mrceq   p15, 4, r2, c6, c0, 2   @ HIFAR
 414         beq     2f
 415         cmp     r1, #HSR_EC_DABT
 416         bne     1f
 417         mrc     p15, 4, r2, c6, c0, 0   @ HDFAR
 418
 419 2:      str     r2, [vcpu, #VCPU_HxFAR]
 420
 421         /*
 422          * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
 423          *
 424          * Abort on the stage 2 translation for a memory access from a
 425          * Non-secure PL1 or PL0 mode:
 426          *
 427          * For any Access flag fault or Translation fault, and also for any
 428          * Permission fault on the stage 2 translation of a memory access
 429          * made as part of a translation table walk for a stage 1 translation,
 430          * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
 431          * is UNKNOWN.
 432          */
 433
 434         /* Check for permission fault, and S1PTW */
 435         mrc     p15, 4, r1, c5, c2, 0   @ HSR
 436         and     r0, r1, #HSR_FSC_TYPE
 437         cmp     r0, #FSC_PERM
 438         tsteq   r1, #(1 << 7)           @ S1PTW
 439         mrcne   p15, 4, r2, c6, c0, 4   @ HPFAR
 440         bne     3f
 441
 442         /* Preserve PAR */
 443         mrrc    p15, 0, r0, r1, c7      @ PAR
 444         push    {r0, r1}
 445
 446         /* Resolve IPA using the xFAR */
 447         mcr     p15, 0, r2, c7, c8, 0   @ ATS1CPR
 448         isb
 449         mrrc    p15, 0, r0, r1, c7      @ PAR
 450         tst     r0, #1
 451         bne     4f                      @ Failed translation
 452         ubfx    r2, r0, #12, #20
 453         lsl     r2, r2, #4
 454         orr     r2, r2, r1, lsl #24
 455
 456         /* Restore PAR */
 457         pop     {r0, r1}
 458         mcrr    p15, 0, r0, r1, c7      @ PAR
 459
 460 3:      load_vcpu                       @ Load VCPU pointer to r0
 461         str     r2, [r0, #VCPU_HPFAR]
 462
 463 1:      mov     r1, #ARM_EXCEPTION_HVC
 464         b       __kvm_vcpu_return
 465
 466 4:      pop     {r0, r1}                @ Failed translation, return to guest
 467         mcrr    p15, 0, r0, r1, c7      @ PAR
 468         clrex
 469         pop     {r0, r1, r2}
 470         eret
 471
 472 /*
 473  * If VFPv3 support is not available, then we will not switch the VFP
 474  * registers; however cp10 and cp11 accesses will still trap and fallback
 475  * to the regular coprocessor emulation code, which currently will
 476  * inject an undefined exception to the guest.
 477  */
 478 #ifdef CONFIG_VFPv3
 479 switch_to_guest_vfp:
 480         push    {r3-r7}
 481
 482         @ NEON/VFP used.  Turn on VFP access.
 483         set_hcptr vmtrap, (HCPTR_TCP(10) | HCPTR_TCP(11))
 484
 485         @ Switch VFP/NEON hardware state to the guest's
 486         add     r7, r0, #VCPU_HOST_CTXT
 487         ldr     r7, [r7]
 488         add     r7, r7, #CPU_CTXT_VFP
 489         store_vfp_state r7
 490         add     r7, r0, #(VCPU_GUEST_CTXT + CPU_CTXT_VFP)
 491         restore_vfp_state r7
 492
 493         pop     {r3-r7}
 494         pop     {r0-r2}
 495         clrex
 496         eret
 497 #endif
 498
 499         .align
 500 hyp_irq:
 501         push    {r0, r1, r2}
 502         mov     r1, #ARM_EXCEPTION_IRQ
 503         load_vcpu                       @ Load VCPU pointer to r0
 504         b       __kvm_vcpu_return
 505
 506         .align
 507 hyp_fiq:
 508         b       hyp_fiq
 509
 510         .ltorg
 511
 512         .popsection
 513
 514         .pushsection ".rodata"
 515
 516 und_die_str:
 517         .ascii  "unexpected undefined exception in Hyp mode at: %#08x\n"
 518 pabt_die_str:
 519         .ascii  "unexpected prefetch abort in Hyp mode at: %#08x\n"
 520 dabt_die_str:
 521         .ascii  "unexpected data abort in Hyp mode at: %#08x\n"
 522 svc_die_str:
 523         .ascii  "unexpected HVC/SVC trap in Hyp mode at: %#08x\n"
 524
 525         .popsection