Commit | Line | Data |
---|---|---|
749cf76c CD |
1 | /* |
2 | * Copyright (C) 2012 - Virtual Open Systems and Columbia University | |
3 | * Author: Christoffer Dall <c.dall@virtualopensystems.com> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License, version 2, as | |
7 | * published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |
17 | */ | |
18 | ||
5b3e5e5b CD |
19 | #include <linux/mm.h> |
20 | #include <linux/kvm_host.h> | |
21 | #include <asm/kvm_arm.h> | |
749cf76c | 22 | #include <asm/kvm_emulate.h> |
c5997563 | 23 | #include <asm/opcodes.h> |
5b3e5e5b CD |
24 | #include <trace/events/kvm.h> |
25 | ||
26 | #include "trace.h" | |
749cf76c CD |
27 | |
28 | #define VCPU_NR_MODES 6 | |
29 | #define VCPU_REG_OFFSET_USR 0 | |
30 | #define VCPU_REG_OFFSET_FIQ 1 | |
31 | #define VCPU_REG_OFFSET_IRQ 2 | |
32 | #define VCPU_REG_OFFSET_SVC 3 | |
33 | #define VCPU_REG_OFFSET_ABT 4 | |
34 | #define VCPU_REG_OFFSET_UND 5 | |
35 | #define REG_OFFSET(_reg) \ | |
36 | (offsetof(struct kvm_regs, _reg) / sizeof(u32)) | |
37 | ||
38 | #define USR_REG_OFFSET(_num) REG_OFFSET(usr_regs.uregs[_num]) | |
39 | ||
40 | static const unsigned long vcpu_reg_offsets[VCPU_NR_MODES][15] = { | |
41 | /* USR/SYS Registers */ | |
42 | [VCPU_REG_OFFSET_USR] = { | |
43 | USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), | |
44 | USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), | |
45 | USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), | |
46 | USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), | |
47 | USR_REG_OFFSET(12), USR_REG_OFFSET(13), USR_REG_OFFSET(14), | |
48 | }, | |
49 | ||
50 | /* FIQ Registers */ | |
51 | [VCPU_REG_OFFSET_FIQ] = { | |
52 | USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), | |
53 | USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), | |
54 | USR_REG_OFFSET(6), USR_REG_OFFSET(7), | |
55 | REG_OFFSET(fiq_regs[0]), /* r8 */ | |
56 | REG_OFFSET(fiq_regs[1]), /* r9 */ | |
57 | REG_OFFSET(fiq_regs[2]), /* r10 */ | |
58 | REG_OFFSET(fiq_regs[3]), /* r11 */ | |
59 | REG_OFFSET(fiq_regs[4]), /* r12 */ | |
60 | REG_OFFSET(fiq_regs[5]), /* r13 */ | |
61 | REG_OFFSET(fiq_regs[6]), /* r14 */ | |
62 | }, | |
63 | ||
64 | /* IRQ Registers */ | |
65 | [VCPU_REG_OFFSET_IRQ] = { | |
66 | USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), | |
67 | USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), | |
68 | USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), | |
69 | USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), | |
70 | USR_REG_OFFSET(12), | |
71 | REG_OFFSET(irq_regs[0]), /* r13 */ | |
72 | REG_OFFSET(irq_regs[1]), /* r14 */ | |
73 | }, | |
74 | ||
75 | /* SVC Registers */ | |
76 | [VCPU_REG_OFFSET_SVC] = { | |
77 | USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), | |
78 | USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), | |
79 | USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), | |
80 | USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), | |
81 | USR_REG_OFFSET(12), | |
82 | REG_OFFSET(svc_regs[0]), /* r13 */ | |
83 | REG_OFFSET(svc_regs[1]), /* r14 */ | |
84 | }, | |
85 | ||
86 | /* ABT Registers */ | |
87 | [VCPU_REG_OFFSET_ABT] = { | |
88 | USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), | |
89 | USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), | |
90 | USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), | |
91 | USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), | |
92 | USR_REG_OFFSET(12), | |
93 | REG_OFFSET(abt_regs[0]), /* r13 */ | |
94 | REG_OFFSET(abt_regs[1]), /* r14 */ | |
95 | }, | |
96 | ||
97 | /* UND Registers */ | |
98 | [VCPU_REG_OFFSET_UND] = { | |
99 | USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2), | |
100 | USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5), | |
101 | USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8), | |
102 | USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11), | |
103 | USR_REG_OFFSET(12), | |
104 | REG_OFFSET(und_regs[0]), /* r13 */ | |
105 | REG_OFFSET(und_regs[1]), /* r14 */ | |
106 | }, | |
107 | }; | |
108 | ||
109 | /* | |
110 | * Return a pointer to the register number valid in the current mode of | |
111 | * the virtual CPU. | |
112 | */ | |
db730d8d | 113 | unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num) |
749cf76c | 114 | { |
c2a8dab5 | 115 | unsigned long *reg_array = (unsigned long *)&vcpu->arch.ctxt.gp_regs; |
db730d8d | 116 | unsigned long mode = *vcpu_cpsr(vcpu) & MODE_MASK; |
749cf76c CD |
117 | |
118 | switch (mode) { | |
119 | case USR_MODE...SVC_MODE: | |
120 | mode &= ~MODE32_BIT; /* 0 ... 3 */ | |
121 | break; | |
122 | ||
123 | case ABT_MODE: | |
124 | mode = VCPU_REG_OFFSET_ABT; | |
125 | break; | |
126 | ||
127 | case UND_MODE: | |
128 | mode = VCPU_REG_OFFSET_UND; | |
129 | break; | |
130 | ||
131 | case SYSTEM_MODE: | |
132 | mode = VCPU_REG_OFFSET_USR; | |
133 | break; | |
134 | ||
135 | default: | |
136 | BUG(); | |
137 | } | |
138 | ||
139 | return reg_array + vcpu_reg_offsets[mode][reg_num]; | |
140 | } | |
141 | ||
142 | /* | |
143 | * Return the SPSR for the current mode of the virtual CPU. | |
144 | */ | |
db730d8d | 145 | unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu) |
749cf76c | 146 | { |
db730d8d | 147 | unsigned long mode = *vcpu_cpsr(vcpu) & MODE_MASK; |
749cf76c CD |
148 | switch (mode) { |
149 | case SVC_MODE: | |
c2a8dab5 | 150 | return &vcpu->arch.ctxt.gp_regs.KVM_ARM_SVC_spsr; |
749cf76c | 151 | case ABT_MODE: |
c2a8dab5 | 152 | return &vcpu->arch.ctxt.gp_regs.KVM_ARM_ABT_spsr; |
749cf76c | 153 | case UND_MODE: |
c2a8dab5 | 154 | return &vcpu->arch.ctxt.gp_regs.KVM_ARM_UND_spsr; |
749cf76c | 155 | case IRQ_MODE: |
c2a8dab5 | 156 | return &vcpu->arch.ctxt.gp_regs.KVM_ARM_IRQ_spsr; |
749cf76c | 157 | case FIQ_MODE: |
c2a8dab5 | 158 | return &vcpu->arch.ctxt.gp_regs.KVM_ARM_FIQ_spsr; |
749cf76c CD |
159 | default: |
160 | BUG(); | |
161 | } | |
162 | } | |
5b3e5e5b | 163 | |
c5997563 MZ |
164 | /* |
165 | * A conditional instruction is allowed to trap, even though it | |
166 | * wouldn't be executed. So let's re-implement the hardware, in | |
167 | * software! | |
168 | */ | |
169 | bool kvm_condition_valid(struct kvm_vcpu *vcpu) | |
170 | { | |
171 | unsigned long cpsr, cond, insn; | |
172 | ||
173 | /* | |
174 | * Exception Code 0 can only happen if we set HCR.TGE to 1, to | |
175 | * catch undefined instructions, and then we won't get past | |
176 | * the arm_exit_handlers test anyway. | |
177 | */ | |
178 | BUG_ON(!kvm_vcpu_trap_get_class(vcpu)); | |
179 | ||
180 | /* Top two bits non-zero? Unconditional. */ | |
181 | if (kvm_vcpu_get_hsr(vcpu) >> 30) | |
182 | return true; | |
183 | ||
184 | cpsr = *vcpu_cpsr(vcpu); | |
185 | ||
186 | /* Is condition field valid? */ | |
187 | if ((kvm_vcpu_get_hsr(vcpu) & HSR_CV) >> HSR_CV_SHIFT) | |
188 | cond = (kvm_vcpu_get_hsr(vcpu) & HSR_COND) >> HSR_COND_SHIFT; | |
189 | else { | |
190 | /* This can happen in Thumb mode: examine IT state. */ | |
191 | unsigned long it; | |
192 | ||
193 | it = ((cpsr >> 8) & 0xFC) | ((cpsr >> 25) & 0x3); | |
194 | ||
195 | /* it == 0 => unconditional. */ | |
196 | if (it == 0) | |
197 | return true; | |
198 | ||
199 | /* The cond for this insn works out as the top 4 bits. */ | |
200 | cond = (it >> 4); | |
201 | } | |
202 | ||
203 | /* Shift makes it look like an ARM-mode instruction */ | |
204 | insn = cond << 28; | |
205 | return arm_check_condition(insn, cpsr) != ARM_OPCODE_CONDTEST_FAIL; | |
206 | } | |
207 | ||
5b3e5e5b CD |
208 | /** |
209 | * adjust_itstate - adjust ITSTATE when emulating instructions in IT-block | |
210 | * @vcpu: The VCPU pointer | |
211 | * | |
212 | * When exceptions occur while instructions are executed in Thumb IF-THEN | |
213 | * blocks, the ITSTATE field of the CPSR is not advanved (updated), so we have | |
214 | * to do this little bit of work manually. The fields map like this: | |
215 | * | |
216 | * IT[7:0] -> CPSR[26:25],CPSR[15:10] | |
217 | */ | |
218 | static void kvm_adjust_itstate(struct kvm_vcpu *vcpu) | |
219 | { | |
220 | unsigned long itbits, cond; | |
221 | unsigned long cpsr = *vcpu_cpsr(vcpu); | |
222 | bool is_arm = !(cpsr & PSR_T_BIT); | |
223 | ||
224 | BUG_ON(is_arm && (cpsr & PSR_IT_MASK)); | |
225 | ||
226 | if (!(cpsr & PSR_IT_MASK)) | |
227 | return; | |
228 | ||
229 | cond = (cpsr & 0xe000) >> 13; | |
230 | itbits = (cpsr & 0x1c00) >> (10 - 2); | |
231 | itbits |= (cpsr & (0x3 << 25)) >> 25; | |
232 | ||
233 | /* Perform ITAdvance (see page A-52 in ARM DDI 0406C) */ | |
234 | if ((itbits & 0x7) == 0) | |
235 | itbits = cond = 0; | |
236 | else | |
237 | itbits = (itbits << 1) & 0x1f; | |
238 | ||
239 | cpsr &= ~PSR_IT_MASK; | |
240 | cpsr |= cond << 13; | |
241 | cpsr |= (itbits & 0x1c) << (10 - 2); | |
242 | cpsr |= (itbits & 0x3) << 25; | |
243 | *vcpu_cpsr(vcpu) = cpsr; | |
244 | } | |
245 | ||
246 | /** | |
247 | * kvm_skip_instr - skip a trapped instruction and proceed to the next | |
248 | * @vcpu: The vcpu pointer | |
249 | */ | |
250 | void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr) | |
251 | { | |
252 | bool is_thumb; | |
253 | ||
254 | is_thumb = !!(*vcpu_cpsr(vcpu) & PSR_T_BIT); | |
255 | if (is_thumb && !is_wide_instr) | |
256 | *vcpu_pc(vcpu) += 2; | |
257 | else | |
258 | *vcpu_pc(vcpu) += 4; | |
259 | kvm_adjust_itstate(vcpu); | |
260 | } | |
261 | ||
262 | ||
263 | /****************************************************************************** | |
264 | * Inject exceptions into the guest | |
265 | */ | |
266 | ||
267 | static u32 exc_vector_base(struct kvm_vcpu *vcpu) | |
268 | { | |
fb32a52a MZ |
269 | u32 sctlr = vcpu_cp15(vcpu, c1_SCTLR); |
270 | u32 vbar = vcpu_cp15(vcpu, c12_VBAR); | |
5b3e5e5b CD |
271 | |
272 | if (sctlr & SCTLR_V) | |
273 | return 0xffff0000; | |
274 | else /* always have security exceptions */ | |
275 | return vbar; | |
276 | } | |
277 | ||
e078ef81 MZ |
278 | /* |
279 | * Switch to an exception mode, updating both CPSR and SPSR. Follow | |
280 | * the logic described in AArch32.EnterMode() from the ARMv8 ARM. | |
281 | */ | |
282 | static void kvm_update_psr(struct kvm_vcpu *vcpu, unsigned long mode) | |
283 | { | |
284 | unsigned long cpsr = *vcpu_cpsr(vcpu); | |
fb32a52a | 285 | u32 sctlr = vcpu_cp15(vcpu, c1_SCTLR); |
e078ef81 MZ |
286 | |
287 | *vcpu_cpsr(vcpu) = (cpsr & ~MODE_MASK) | mode; | |
288 | ||
289 | switch (mode) { | |
290 | case FIQ_MODE: | |
291 | *vcpu_cpsr(vcpu) |= PSR_F_BIT; | |
292 | /* Fall through */ | |
293 | case ABT_MODE: | |
294 | case IRQ_MODE: | |
295 | *vcpu_cpsr(vcpu) |= PSR_A_BIT; | |
296 | /* Fall through */ | |
297 | default: | |
298 | *vcpu_cpsr(vcpu) |= PSR_I_BIT; | |
299 | } | |
300 | ||
301 | *vcpu_cpsr(vcpu) &= ~(PSR_IT_MASK | PSR_J_BIT | PSR_E_BIT | PSR_T_BIT); | |
302 | ||
303 | if (sctlr & SCTLR_TE) | |
304 | *vcpu_cpsr(vcpu) |= PSR_T_BIT; | |
305 | if (sctlr & SCTLR_EE) | |
306 | *vcpu_cpsr(vcpu) |= PSR_E_BIT; | |
307 | ||
308 | /* Note: These now point to the mode banked copies */ | |
309 | *vcpu_spsr(vcpu) = cpsr; | |
310 | } | |
311 | ||
5b3e5e5b CD |
312 | /** |
313 | * kvm_inject_undefined - inject an undefined exception into the guest | |
314 | * @vcpu: The VCPU to receive the undefined exception | |
315 | * | |
316 | * It is assumed that this code is called from the VCPU thread and that the | |
317 | * VCPU therefore is not currently executing guest code. | |
318 | * | |
319 | * Modelled after TakeUndefInstrException() pseudocode. | |
320 | */ | |
321 | void kvm_inject_undefined(struct kvm_vcpu *vcpu) | |
322 | { | |
db730d8d | 323 | unsigned long cpsr = *vcpu_cpsr(vcpu); |
5b3e5e5b CD |
324 | bool is_thumb = (cpsr & PSR_T_BIT); |
325 | u32 vect_offset = 4; | |
326 | u32 return_offset = (is_thumb) ? 2 : 4; | |
327 | ||
e078ef81 MZ |
328 | kvm_update_psr(vcpu, UND_MODE); |
329 | *vcpu_reg(vcpu, 14) = *vcpu_pc(vcpu) - return_offset; | |
5b3e5e5b CD |
330 | |
331 | /* Branch to exception vector */ | |
332 | *vcpu_pc(vcpu) = exc_vector_base(vcpu) + vect_offset; | |
333 | } | |
334 | ||
335 | /* | |
336 | * Modelled after TakeDataAbortException() and TakePrefetchAbortException | |
337 | * pseudocode. | |
338 | */ | |
339 | static void inject_abt(struct kvm_vcpu *vcpu, bool is_pabt, unsigned long addr) | |
340 | { | |
db730d8d | 341 | unsigned long cpsr = *vcpu_cpsr(vcpu); |
5b3e5e5b CD |
342 | bool is_thumb = (cpsr & PSR_T_BIT); |
343 | u32 vect_offset; | |
344 | u32 return_offset = (is_thumb) ? 4 : 0; | |
345 | bool is_lpae; | |
346 | ||
e078ef81 MZ |
347 | kvm_update_psr(vcpu, ABT_MODE); |
348 | *vcpu_reg(vcpu, 14) = *vcpu_pc(vcpu) + return_offset; | |
5b3e5e5b CD |
349 | |
350 | if (is_pabt) | |
351 | vect_offset = 12; | |
352 | else | |
353 | vect_offset = 16; | |
354 | ||
355 | /* Branch to exception vector */ | |
356 | *vcpu_pc(vcpu) = exc_vector_base(vcpu) + vect_offset; | |
357 | ||
358 | if (is_pabt) { | |
b373e492 | 359 | /* Set IFAR and IFSR */ |
fb32a52a MZ |
360 | vcpu_cp15(vcpu, c6_IFAR) = addr; |
361 | is_lpae = (vcpu_cp15(vcpu, c2_TTBCR) >> 31); | |
5b3e5e5b CD |
362 | /* Always give debug fault for now - should give guest a clue */ |
363 | if (is_lpae) | |
fb32a52a | 364 | vcpu_cp15(vcpu, c5_IFSR) = 1 << 9 | 0x22; |
5b3e5e5b | 365 | else |
fb32a52a | 366 | vcpu_cp15(vcpu, c5_IFSR) = 2; |
5b3e5e5b CD |
367 | } else { /* !iabt */ |
368 | /* Set DFAR and DFSR */ | |
fb32a52a MZ |
369 | vcpu_cp15(vcpu, c6_DFAR) = addr; |
370 | is_lpae = (vcpu_cp15(vcpu, c2_TTBCR) >> 31); | |
5b3e5e5b CD |
371 | /* Always give debug fault for now - should give guest a clue */ |
372 | if (is_lpae) | |
fb32a52a | 373 | vcpu_cp15(vcpu, c5_DFSR) = 1 << 9 | 0x22; |
5b3e5e5b | 374 | else |
fb32a52a | 375 | vcpu_cp15(vcpu, c5_DFSR) = 2; |
5b3e5e5b CD |
376 | } |
377 | ||
378 | } | |
379 | ||
380 | /** | |
381 | * kvm_inject_dabt - inject a data abort into the guest | |
382 | * @vcpu: The VCPU to receive the undefined exception | |
383 | * @addr: The address to report in the DFAR | |
384 | * | |
385 | * It is assumed that this code is called from the VCPU thread and that the | |
386 | * VCPU therefore is not currently executing guest code. | |
387 | */ | |
388 | void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr) | |
389 | { | |
390 | inject_abt(vcpu, false, addr); | |
391 | } | |
392 | ||
393 | /** | |
394 | * kvm_inject_pabt - inject a prefetch abort into the guest | |
395 | * @vcpu: The VCPU to receive the undefined exception | |
396 | * @addr: The address to report in the DFAR | |
397 | * | |
398 | * It is assumed that this code is called from the VCPU thread and that the | |
399 | * VCPU therefore is not currently executing guest code. | |
400 | */ | |
401 | void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr) | |
402 | { | |
403 | inject_abt(vcpu, true, addr); | |
404 | } |