[deliverable/linux.git] / arch / arm64 / kvm / hyp / entry.S

/*
 * Copyright (C) 2015 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/linkage.h>

#include <asm/asm-offsets.h>
#include <asm/assembler.h>
#include <asm/fpsimdmacros.h>
#include <asm/kvm.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_mmu.h>

#define CPU_GP_REG_OFFSET(x)	(CPU_GP_REGS + x)
#define CPU_XREG_OFFSET(x)	CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
#define CPU_SYSREG_OFFSET(x)	(CPU_SYSREGS + 8*x)

	.text
	.pushsection	.hyp.text, "ax"

.macro save_callee_saved_regs ctxt
	stp	x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
	stp	x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
	stp	x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
	stp	x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
	stp	x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
	stp	x29, lr,  [\ctxt, #CPU_XREG_OFFSET(29)]
.endm

.macro restore_callee_saved_regs ctxt
	ldp	x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
	ldp	x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
	ldp	x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
	ldp	x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
	ldp	x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
	ldp	x29, lr,  [\ctxt, #CPU_XREG_OFFSET(29)]
.endm

/*
 * u64 __guest_enter(struct kvm_vcpu *vcpu,
 *		     struct kvm_cpu_context *host_ctxt);
 */
ENTRY(__guest_enter)
	// x0: vcpu
	// x1: host/guest context
	// x2-x18: clobbered by macros

	// Store the host regs
	save_callee_saved_regs x1

	// Preserve vcpu & host_ctxt for use at exit time
	stp	x0, x1, [sp, #-16]!

	add	x1, x0, #VCPU_CONTEXT

	// Prepare x0-x1 for later restore by pushing them onto the stack
	ldp	x2, x3, [x1, #CPU_XREG_OFFSET(0)]
	stp	x2, x3, [sp, #-16]!

	// x2-x18
	ldp	x2, x3,   [x1, #CPU_XREG_OFFSET(2)]
	ldp	x4, x5,   [x1, #CPU_XREG_OFFSET(4)]
	ldp	x6, x7,   [x1, #CPU_XREG_OFFSET(6)]
	ldp	x8, x9,   [x1, #CPU_XREG_OFFSET(8)]
	ldp	x10, x11, [x1, #CPU_XREG_OFFSET(10)]
	ldp	x12, x13, [x1, #CPU_XREG_OFFSET(12)]
	ldp	x14, x15, [x1, #CPU_XREG_OFFSET(14)]
	ldp	x16, x17, [x1, #CPU_XREG_OFFSET(16)]
	ldr	x18,      [x1, #CPU_XREG_OFFSET(18)]

	// x19-x29, lr
	restore_callee_saved_regs x1

	// Last bits of the 64bit state
	ldp	x0, x1, [sp], #16

	// Do not touch any register after this!
	eret
ENDPROC(__guest_enter)

ENTRY(__guest_exit)
	// x0: vcpu
	// x1: return code
	// x2-x3: free
	// x4-x29,lr: vcpu regs
	// vcpu x0-x3 on the stack

	add	x2, x0, #VCPU_CONTEXT

	stp	x4, x5,   [x2, #CPU_XREG_OFFSET(4)]
	stp	x6, x7,   [x2, #CPU_XREG_OFFSET(6)]
	stp	x8, x9,   [x2, #CPU_XREG_OFFSET(8)]
	stp	x10, x11, [x2, #CPU_XREG_OFFSET(10)]
	stp	x12, x13, [x2, #CPU_XREG_OFFSET(12)]
	stp	x14, x15, [x2, #CPU_XREG_OFFSET(14)]
	stp	x16, x17, [x2, #CPU_XREG_OFFSET(16)]
	str	x18,      [x2, #CPU_XREG_OFFSET(18)]

	ldp	x6, x7, [sp], #16	// x2, x3
	ldp	x4, x5, [sp], #16	// x0, x1

	stp	x4, x5, [x2, #CPU_XREG_OFFSET(0)]
	stp	x6, x7, [x2, #CPU_XREG_OFFSET(2)]

	save_callee_saved_regs x2

	// Restore vcpu & host_ctxt from the stack
	// (preserving return code in x1)
	ldp	x0, x2, [sp], #16
	// Now restore the host regs
	restore_callee_saved_regs x2

	mov	x0, x1
	ret
ENDPROC(__guest_exit)

ENTRY(__fpsimd_guest_restore)
	stp	x4, lr, [sp, #-16]!

	mrs	x2, cptr_el2
	bic	x2, x2, #CPTR_EL2_TFP
	msr	cptr_el2, x2
	isb

	mrs	x3, tpidr_el2

	ldr	x0, [x3, #VCPU_HOST_CONTEXT]
	kern_hyp_va x0
	add	x0, x0, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
	bl	__fpsimd_save_state

	add	x2, x3, #VCPU_CONTEXT
	add	x0, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
	bl	__fpsimd_restore_state

	// Skip restoring fpexc32 for AArch64 guests
	mrs	x1, hcr_el2
	tbnz	x1, #HCR_RW_SHIFT, 1f
	ldr	x4, [x2, #CPU_SYSREG_OFFSET(FPEXC32_EL2)]
	msr	fpexc32_el2, x4
1:
	ldp	x4, lr, [sp], #16
	ldp	x2, x3, [sp], #16
	ldp	x0, x1, [sp], #16

	eret
ENDPROC(__fpsimd_guest_restore)
Commit	Line	Data
b97b66c1 MZ	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/linkage.h>
	19
	20	#include <asm/asm-offsets.h>
	21	#include <asm/assembler.h>
	22	#include <asm/fpsimdmacros.h>
	23	#include <asm/kvm.h>
	24	#include <asm/kvm_arm.h>
	25	#include <asm/kvm_asm.h>
	26	#include <asm/kvm_mmu.h>
	27
	28	#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
	29	#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
c13d1683	30	#define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x)
b97b66c1 MZ	31
	32	.text
	33	.pushsection .hyp.text, "ax"
	34
	35	.macro save_callee_saved_regs ctxt
	36	stp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
	37	stp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
	38	stp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
	39	stp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
	40	stp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
	41	stp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
	42	.endm
	43
	44	.macro restore_callee_saved_regs ctxt
	45	ldp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
	46	ldp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
	47	ldp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
	48	ldp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
	49	ldp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
	50	ldp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
	51	.endm
	52
	53	/*
	54	* u64 __guest_enter(struct kvm_vcpu *vcpu,
	55	* struct kvm_cpu_context *host_ctxt);
	56	*/
	57	ENTRY(__guest_enter)
	58	// x0: vcpu
	59	// x1: host/guest context
	60	// x2-x18: clobbered by macros
	61
	62	// Store the host regs
	63	save_callee_saved_regs x1
	64
	65	// Preserve vcpu & host_ctxt for use at exit time
	66	stp x0, x1, [sp, #-16]!
	67
	68	add x1, x0, #VCPU_CONTEXT
	69
	70	// Prepare x0-x1 for later restore by pushing them onto the stack
	71	ldp x2, x3, [x1, #CPU_XREG_OFFSET(0)]
	72	stp x2, x3, [sp, #-16]!
	73
	74	// x2-x18
	75	ldp x2, x3, [x1, #CPU_XREG_OFFSET(2)]
	76	ldp x4, x5, [x1, #CPU_XREG_OFFSET(4)]
	77	ldp x6, x7, [x1, #CPU_XREG_OFFSET(6)]
	78	ldp x8, x9, [x1, #CPU_XREG_OFFSET(8)]
	79	ldp x10, x11, [x1, #CPU_XREG_OFFSET(10)]
	80	ldp x12, x13, [x1, #CPU_XREG_OFFSET(12)]
	81	ldp x14, x15, [x1, #CPU_XREG_OFFSET(14)]
	82	ldp x16, x17, [x1, #CPU_XREG_OFFSET(16)]
	83	ldr x18, [x1, #CPU_XREG_OFFSET(18)]
	84
	85	// x19-x29, lr
	86	restore_callee_saved_regs x1
	87
	88	// Last bits of the 64bit state
	89	ldp x0, x1, [sp], #16
	90
	91	// Do not touch any register after this!
	92	eret
	93	ENDPROC(__guest_enter)
	94
95	ENTRY(__guest_exit)
96	// x0: vcpu
97	// x1: return code
98	// x2-x3: free
99	// x4-x29,lr: vcpu regs
100	// vcpu x0-x3 on the stack
101
102	add x2, x0, #VCPU_CONTEXT
103
104	stp x4, x5, [x2, #CPU_XREG_OFFSET(4)]
105	stp x6, x7, [x2, #CPU_XREG_OFFSET(6)]
106	stp x8, x9, [x2, #CPU_XREG_OFFSET(8)]
107	stp x10, x11, [x2, #CPU_XREG_OFFSET(10)]
108	stp x12, x13, [x2, #CPU_XREG_OFFSET(12)]
109	stp x14, x15, [x2, #CPU_XREG_OFFSET(14)]
110	stp x16, x17, [x2, #CPU_XREG_OFFSET(16)]
111	str x18, [x2, #CPU_XREG_OFFSET(18)]
112
113	ldp x6, x7, [sp], #16 // x2, x3
114	ldp x4, x5, [sp], #16 // x0, x1
115
116	stp x4, x5, [x2, #CPU_XREG_OFFSET(0)]
117	stp x6, x7, [x2, #CPU_XREG_OFFSET(2)]
118
119	save_callee_saved_regs x2
120
121	// Restore vcpu & host_ctxt from the stack
122	// (preserving return code in x1)
123	ldp x0, x2, [sp], #16
124	// Now restore the host regs
125	restore_callee_saved_regs x2
126
127	mov x0, x1
128	ret
129	ENDPROC(__guest_exit)
130
c13d1683 MZ	131	ENTRY(__fpsimd_guest_restore)
	132	stp x4, lr, [sp, #-16]!
	133
	134	mrs x2, cptr_el2
	135	bic x2, x2, #CPTR_EL2_TFP
	136	msr cptr_el2, x2
	137	isb
	138
	139	mrs x3, tpidr_el2
	140
	141	ldr x0, [x3, #VCPU_HOST_CONTEXT]
	142	kern_hyp_va x0
	143	add x0, x0, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
	144	bl __fpsimd_save_state
	145
	146	add x2, x3, #VCPU_CONTEXT
	147	add x0, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
	148	bl __fpsimd_restore_state
	149
5eec0a91	150	// Skip restoring fpexc32 for AArch64 guests
c13d1683 MZ	151	mrs x1, hcr_el2
	152	tbnz x1, #HCR_RW_SHIFT, 1f
	153	ldr x4, [x2, #CPU_SYSREG_OFFSET(FPEXC32_EL2)]
	154	msr fpexc32_el2, x4
	155	1:
	156	ldp x4, lr, [sp], #16
	157	ldp x2, x3, [sp], #16
	158	ldp x0, x1, [sp], #16
	159
	160	eret
	161	ENDPROC(__fpsimd_guest_restore)