projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'acpi-4.7-rc1-more' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael...
[deliverable/linux.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
31 #include <asm/etr.h>
32 #include <asm/pgtable.h>
33 #include <asm/gmap.h>
34 #include <asm/nmi.h>
35 #include <asm/switch_to.h>
36 #include <asm/isc.h>
37 #include <asm/sclp.h>
38 #include "kvm-s390.h"
39 #include "gaccess.h"
40
41 #define KMSG_COMPONENT "kvm-s390"
42 #undef pr_fmt
43 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
44
45 #define CREATE_TRACE_POINTS
46 #include "trace.h"
47 #include "trace-s390.h"
48
49 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
50 #define LOCAL_IRQS 32
51 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
52 (KVM_MAX_VCPUS + LOCAL_IRQS))
53
54 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
55
56 struct kvm_stats_debugfs_item debugfs_entries[] = {
57 { "userspace_handled", VCPU_STAT(exit_userspace) },
58 { "exit_null", VCPU_STAT(exit_null) },
59 { "exit_validity", VCPU_STAT(exit_validity) },
60 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
61 { "exit_external_request", VCPU_STAT(exit_external_request) },
62 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
63 { "exit_instruction", VCPU_STAT(exit_instruction) },
64 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
65 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
66 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
67 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
68 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
69 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
70 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
71 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
72 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
73 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
74 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
75 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
76 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
77 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
78 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
79 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
80 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
81 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
82 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
83 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
84 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
85 { "instruction_spx", VCPU_STAT(instruction_spx) },
86 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
87 { "instruction_stap", VCPU_STAT(instruction_stap) },
88 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
89 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
90 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
91 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
92 { "instruction_essa", VCPU_STAT(instruction_essa) },
93 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
94 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
95 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
96 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
97 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
98 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
99 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
100 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
101 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
102 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
103 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
104 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
105 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
106 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
107 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
108 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
109 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
110 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
111 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
112 { "diagnose_10", VCPU_STAT(diagnose_10) },
113 { "diagnose_44", VCPU_STAT(diagnose_44) },
114 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
115 { "diagnose_258", VCPU_STAT(diagnose_258) },
116 { "diagnose_308", VCPU_STAT(diagnose_308) },
117 { "diagnose_500", VCPU_STAT(diagnose_500) },
118 { NULL }
119 };
120
121 /* upper facilities limit for kvm */
122 unsigned long kvm_s390_fac_list_mask[16] = {
123 0xffe6000000000000UL,
124 0x005e000000000000UL,
125 };
126
127 unsigned long kvm_s390_fac_list_mask_size(void)
128 {
129 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
130 return ARRAY_SIZE(kvm_s390_fac_list_mask);
131 }
132
133 static struct gmap_notifier gmap_notifier;
134 debug_info_t *kvm_s390_dbf;
135
136 /* Section: not file related */
137 int kvm_arch_hardware_enable(void)
138 {
139 /* every s390 is virtualization enabled ;-) */
140 return 0;
141 }
142
143 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
144
145 /*
146 * This callback is executed during stop_machine(). All CPUs are therefore
147 * temporarily stopped. In order not to change guest behavior, we have to
148 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
149 * so a CPU won't be stopped while calculating with the epoch.
150 */
151 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
152 void *v)
153 {
154 struct kvm *kvm;
155 struct kvm_vcpu *vcpu;
156 int i;
157 unsigned long long *delta = v;
158
159 list_for_each_entry(kvm, &vm_list, vm_list) {
160 kvm->arch.epoch -= *delta;
161 kvm_for_each_vcpu(i, vcpu, kvm) {
162 vcpu->arch.sie_block->epoch -= *delta;
163 if (vcpu->arch.cputm_enabled)
164 vcpu->arch.cputm_start += *delta;
165 }
166 }
167 return NOTIFY_OK;
168 }
169
170 static struct notifier_block kvm_clock_notifier = {
171 .notifier_call = kvm_clock_sync,
172 };
173
174 int kvm_arch_hardware_setup(void)
175 {
176 gmap_notifier.notifier_call = kvm_gmap_notifier;
177 gmap_register_ipte_notifier(&gmap_notifier);
178 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
179 &kvm_clock_notifier);
180 return 0;
181 }
182
183 void kvm_arch_hardware_unsetup(void)
184 {
185 gmap_unregister_ipte_notifier(&gmap_notifier);
186 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
187 &kvm_clock_notifier);
188 }
189
190 int kvm_arch_init(void *opaque)
191 {
192 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
193 if (!kvm_s390_dbf)
194 return -ENOMEM;
195
196 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
197 debug_unregister(kvm_s390_dbf);
198 return -ENOMEM;
199 }
200
201 /* Register floating interrupt controller interface. */
202 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
203 }
204
205 void kvm_arch_exit(void)
206 {
207 debug_unregister(kvm_s390_dbf);
208 }
209
210 /* Section: device related */
211 long kvm_arch_dev_ioctl(struct file *filp,
212 unsigned int ioctl, unsigned long arg)
213 {
214 if (ioctl == KVM_S390_ENABLE_SIE)
215 return s390_enable_sie();
216 return -EINVAL;
217 }
218
219 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
220 {
221 int r;
222
223 switch (ext) {
224 case KVM_CAP_S390_PSW:
225 case KVM_CAP_S390_GMAP:
226 case KVM_CAP_SYNC_MMU:
227 #ifdef CONFIG_KVM_S390_UCONTROL
228 case KVM_CAP_S390_UCONTROL:
229 #endif
230 case KVM_CAP_ASYNC_PF:
231 case KVM_CAP_SYNC_REGS:
232 case KVM_CAP_ONE_REG:
233 case KVM_CAP_ENABLE_CAP:
234 case KVM_CAP_S390_CSS_SUPPORT:
235 case KVM_CAP_IOEVENTFD:
236 case KVM_CAP_DEVICE_CTRL:
237 case KVM_CAP_ENABLE_CAP_VM:
238 case KVM_CAP_S390_IRQCHIP:
239 case KVM_CAP_VM_ATTRIBUTES:
240 case KVM_CAP_MP_STATE:
241 case KVM_CAP_S390_INJECT_IRQ:
242 case KVM_CAP_S390_USER_SIGP:
243 case KVM_CAP_S390_USER_STSI:
244 case KVM_CAP_S390_SKEYS:
245 case KVM_CAP_S390_IRQ_STATE:
246 r = 1;
247 break;
248 case KVM_CAP_S390_MEM_OP:
249 r = MEM_OP_MAX_SIZE;
250 break;
251 case KVM_CAP_NR_VCPUS:
252 case KVM_CAP_MAX_VCPUS:
253 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
254 : KVM_S390_BSCA_CPU_SLOTS;
255 break;
256 case KVM_CAP_NR_MEMSLOTS:
257 r = KVM_USER_MEM_SLOTS;
258 break;
259 case KVM_CAP_S390_COW:
260 r = MACHINE_HAS_ESOP;
261 break;
262 case KVM_CAP_S390_VECTOR_REGISTERS:
263 r = MACHINE_HAS_VX;
264 break;
265 case KVM_CAP_S390_RI:
266 r = test_facility(64);
267 break;
268 default:
269 r = 0;
270 }
271 return r;
272 }
273
274 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
275 struct kvm_memory_slot *memslot)
276 {
277 gfn_t cur_gfn, last_gfn;
278 unsigned long address;
279 struct gmap *gmap = kvm->arch.gmap;
280
281 /* Loop over all guest pages */
282 last_gfn = memslot->base_gfn + memslot->npages;
283 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
284 address = gfn_to_hva_memslot(memslot, cur_gfn);
285
286 if (test_and_clear_guest_dirty(gmap->mm, address))
287 mark_page_dirty(kvm, cur_gfn);
288 if (fatal_signal_pending(current))
289 return;
290 cond_resched();
291 }
292 }
293
294 /* Section: vm related */
295 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
296
297 /*
298 * Get (and clear) the dirty memory log for a memory slot.
299 */
300 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
301 struct kvm_dirty_log *log)
302 {
303 int r;
304 unsigned long n;
305 struct kvm_memslots *slots;
306 struct kvm_memory_slot *memslot;
307 int is_dirty = 0;
308
309 mutex_lock(&kvm->slots_lock);
310
311 r = -EINVAL;
312 if (log->slot >= KVM_USER_MEM_SLOTS)
313 goto out;
314
315 slots = kvm_memslots(kvm);
316 memslot = id_to_memslot(slots, log->slot);
317 r = -ENOENT;
318 if (!memslot->dirty_bitmap)
319 goto out;
320
321 kvm_s390_sync_dirty_log(kvm, memslot);
322 r = kvm_get_dirty_log(kvm, log, &is_dirty);
323 if (r)
324 goto out;
325
326 /* Clear the dirty log */
327 if (is_dirty) {
328 n = kvm_dirty_bitmap_bytes(memslot);
329 memset(memslot->dirty_bitmap, 0, n);
330 }
331 r = 0;
332 out:
333 mutex_unlock(&kvm->slots_lock);
334 return r;
335 }
336
337 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
338 {
339 int r;
340
341 if (cap->flags)
342 return -EINVAL;
343
344 switch (cap->cap) {
345 case KVM_CAP_S390_IRQCHIP:
346 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
347 kvm->arch.use_irqchip = 1;
348 r = 0;
349 break;
350 case KVM_CAP_S390_USER_SIGP:
351 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
352 kvm->arch.user_sigp = 1;
353 r = 0;
354 break;
355 case KVM_CAP_S390_VECTOR_REGISTERS:
356 mutex_lock(&kvm->lock);
357 if (atomic_read(&kvm->online_vcpus)) {
358 r = -EBUSY;
359 } else if (MACHINE_HAS_VX) {
360 set_kvm_facility(kvm->arch.model.fac_mask, 129);
361 set_kvm_facility(kvm->arch.model.fac_list, 129);
362 r = 0;
363 } else
364 r = -EINVAL;
365 mutex_unlock(&kvm->lock);
366 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
367 r ? "(not available)" : "(success)");
368 break;
369 case KVM_CAP_S390_RI:
370 r = -EINVAL;
371 mutex_lock(&kvm->lock);
372 if (atomic_read(&kvm->online_vcpus)) {
373 r = -EBUSY;
374 } else if (test_facility(64)) {
375 set_kvm_facility(kvm->arch.model.fac_mask, 64);
376 set_kvm_facility(kvm->arch.model.fac_list, 64);
377 r = 0;
378 }
379 mutex_unlock(&kvm->lock);
380 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
381 r ? "(not available)" : "(success)");
382 break;
383 case KVM_CAP_S390_USER_STSI:
384 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
385 kvm->arch.user_stsi = 1;
386 r = 0;
387 break;
388 default:
389 r = -EINVAL;
390 break;
391 }
392 return r;
393 }
394
395 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
396 {
397 int ret;
398
399 switch (attr->attr) {
400 case KVM_S390_VM_MEM_LIMIT_SIZE:
401 ret = 0;
402 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
403 kvm->arch.mem_limit);
404 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
405 ret = -EFAULT;
406 break;
407 default:
408 ret = -ENXIO;
409 break;
410 }
411 return ret;
412 }
413
414 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
415 {
416 int ret;
417 unsigned int idx;
418 switch (attr->attr) {
419 case KVM_S390_VM_MEM_ENABLE_CMMA:
420 /* enable CMMA only for z10 and later (EDAT_1) */
421 ret = -EINVAL;
422 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
423 break;
424
425 ret = -EBUSY;
426 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
427 mutex_lock(&kvm->lock);
428 if (atomic_read(&kvm->online_vcpus) == 0) {
429 kvm->arch.use_cmma = 1;
430 ret = 0;
431 }
432 mutex_unlock(&kvm->lock);
433 break;
434 case KVM_S390_VM_MEM_CLR_CMMA:
435 ret = -EINVAL;
436 if (!kvm->arch.use_cmma)
437 break;
438
439 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
440 mutex_lock(&kvm->lock);
441 idx = srcu_read_lock(&kvm->srcu);
442 s390_reset_cmma(kvm->arch.gmap->mm);
443 srcu_read_unlock(&kvm->srcu, idx);
444 mutex_unlock(&kvm->lock);
445 ret = 0;
446 break;
447 case KVM_S390_VM_MEM_LIMIT_SIZE: {
448 unsigned long new_limit;
449
450 if (kvm_is_ucontrol(kvm))
451 return -EINVAL;
452
453 if (get_user(new_limit, (u64 __user *)attr->addr))
454 return -EFAULT;
455
456 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
457 new_limit > kvm->arch.mem_limit)
458 return -E2BIG;
459
460 if (!new_limit)
461 return -EINVAL;
462
463 /* gmap_alloc takes last usable address */
464 if (new_limit != KVM_S390_NO_MEM_LIMIT)
465 new_limit -= 1;
466
467 ret = -EBUSY;
468 mutex_lock(&kvm->lock);
469 if (atomic_read(&kvm->online_vcpus) == 0) {
470 /* gmap_alloc will round the limit up */
471 struct gmap *new = gmap_alloc(current->mm, new_limit);
472
473 if (!new) {
474 ret = -ENOMEM;
475 } else {
476 gmap_free(kvm->arch.gmap);
477 new->private = kvm;
478 kvm->arch.gmap = new;
479 ret = 0;
480 }
481 }
482 mutex_unlock(&kvm->lock);
483 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
484 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
485 (void *) kvm->arch.gmap->asce);
486 break;
487 }
488 default:
489 ret = -ENXIO;
490 break;
491 }
492 return ret;
493 }
494
495 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
496
497 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
498 {
499 struct kvm_vcpu *vcpu;
500 int i;
501
502 if (!test_kvm_facility(kvm, 76))
503 return -EINVAL;
504
505 mutex_lock(&kvm->lock);
506 switch (attr->attr) {
507 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
508 get_random_bytes(
509 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
510 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
511 kvm->arch.crypto.aes_kw = 1;
512 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
513 break;
514 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
515 get_random_bytes(
516 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
517 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
518 kvm->arch.crypto.dea_kw = 1;
519 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
520 break;
521 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
522 kvm->arch.crypto.aes_kw = 0;
523 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
524 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
525 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
526 break;
527 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
528 kvm->arch.crypto.dea_kw = 0;
529 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
530 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
531 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
532 break;
533 default:
534 mutex_unlock(&kvm->lock);
535 return -ENXIO;
536 }
537
538 kvm_for_each_vcpu(i, vcpu, kvm) {
539 kvm_s390_vcpu_crypto_setup(vcpu);
540 exit_sie(vcpu);
541 }
542 mutex_unlock(&kvm->lock);
543 return 0;
544 }
545
546 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
547 {
548 u8 gtod_high;
549
550 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
551 sizeof(gtod_high)))
552 return -EFAULT;
553
554 if (gtod_high != 0)
555 return -EINVAL;
556 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
557
558 return 0;
559 }
560
561 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
562 {
563 u64 gtod;
564
565 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
566 return -EFAULT;
567
568 kvm_s390_set_tod_clock(kvm, gtod);
569 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
570 return 0;
571 }
572
573 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
574 {
575 int ret;
576
577 if (attr->flags)
578 return -EINVAL;
579
580 switch (attr->attr) {
581 case KVM_S390_VM_TOD_HIGH:
582 ret = kvm_s390_set_tod_high(kvm, attr);
583 break;
584 case KVM_S390_VM_TOD_LOW:
585 ret = kvm_s390_set_tod_low(kvm, attr);
586 break;
587 default:
588 ret = -ENXIO;
589 break;
590 }
591 return ret;
592 }
593
594 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
595 {
596 u8 gtod_high = 0;
597
598 if (copy_to_user((void __user *)attr->addr, &gtod_high,
599 sizeof(gtod_high)))
600 return -EFAULT;
601 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
602
603 return 0;
604 }
605
606 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
607 {
608 u64 gtod;
609
610 gtod = kvm_s390_get_tod_clock_fast(kvm);
611 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
612 return -EFAULT;
613 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
614
615 return 0;
616 }
617
618 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
619 {
620 int ret;
621
622 if (attr->flags)
623 return -EINVAL;
624
625 switch (attr->attr) {
626 case KVM_S390_VM_TOD_HIGH:
627 ret = kvm_s390_get_tod_high(kvm, attr);
628 break;
629 case KVM_S390_VM_TOD_LOW:
630 ret = kvm_s390_get_tod_low(kvm, attr);
631 break;
632 default:
633 ret = -ENXIO;
634 break;
635 }
636 return ret;
637 }
638
639 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
640 {
641 struct kvm_s390_vm_cpu_processor *proc;
642 u16 lowest_ibc, unblocked_ibc;
643 int ret = 0;
644
645 mutex_lock(&kvm->lock);
646 if (atomic_read(&kvm->online_vcpus)) {
647 ret = -EBUSY;
648 goto out;
649 }
650 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
651 if (!proc) {
652 ret = -ENOMEM;
653 goto out;
654 }
655 if (!copy_from_user(proc, (void __user *)attr->addr,
656 sizeof(*proc))) {
657 kvm->arch.model.cpuid = proc->cpuid;
658 lowest_ibc = sclp.ibc >> 16 & 0xfff;
659 unblocked_ibc = sclp.ibc & 0xfff;
660 if (lowest_ibc) {
661 if (proc->ibc > unblocked_ibc)
662 kvm->arch.model.ibc = unblocked_ibc;
663 else if (proc->ibc < lowest_ibc)
664 kvm->arch.model.ibc = lowest_ibc;
665 else
666 kvm->arch.model.ibc = proc->ibc;
667 }
668 memcpy(kvm->arch.model.fac_list, proc->fac_list,
669 S390_ARCH_FAC_LIST_SIZE_BYTE);
670 } else
671 ret = -EFAULT;
672 kfree(proc);
673 out:
674 mutex_unlock(&kvm->lock);
675 return ret;
676 }
677
678 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
679 {
680 int ret = -ENXIO;
681
682 switch (attr->attr) {
683 case KVM_S390_VM_CPU_PROCESSOR:
684 ret = kvm_s390_set_processor(kvm, attr);
685 break;
686 }
687 return ret;
688 }
689
690 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
691 {
692 struct kvm_s390_vm_cpu_processor *proc;
693 int ret = 0;
694
695 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
696 if (!proc) {
697 ret = -ENOMEM;
698 goto out;
699 }
700 proc->cpuid = kvm->arch.model.cpuid;
701 proc->ibc = kvm->arch.model.ibc;
702 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
703 S390_ARCH_FAC_LIST_SIZE_BYTE);
704 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
705 ret = -EFAULT;
706 kfree(proc);
707 out:
708 return ret;
709 }
710
711 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
712 {
713 struct kvm_s390_vm_cpu_machine *mach;
714 int ret = 0;
715
716 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
717 if (!mach) {
718 ret = -ENOMEM;
719 goto out;
720 }
721 get_cpu_id((struct cpuid *) &mach->cpuid);
722 mach->ibc = sclp.ibc;
723 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
724 S390_ARCH_FAC_LIST_SIZE_BYTE);
725 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
726 S390_ARCH_FAC_LIST_SIZE_BYTE);
727 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
728 ret = -EFAULT;
729 kfree(mach);
730 out:
731 return ret;
732 }
733
734 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
735 {
736 int ret = -ENXIO;
737
738 switch (attr->attr) {
739 case KVM_S390_VM_CPU_PROCESSOR:
740 ret = kvm_s390_get_processor(kvm, attr);
741 break;
742 case KVM_S390_VM_CPU_MACHINE:
743 ret = kvm_s390_get_machine(kvm, attr);
744 break;
745 }
746 return ret;
747 }
748
749 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
750 {
751 int ret;
752
753 switch (attr->group) {
754 case KVM_S390_VM_MEM_CTRL:
755 ret = kvm_s390_set_mem_control(kvm, attr);
756 break;
757 case KVM_S390_VM_TOD:
758 ret = kvm_s390_set_tod(kvm, attr);
759 break;
760 case KVM_S390_VM_CPU_MODEL:
761 ret = kvm_s390_set_cpu_model(kvm, attr);
762 break;
763 case KVM_S390_VM_CRYPTO:
764 ret = kvm_s390_vm_set_crypto(kvm, attr);
765 break;
766 default:
767 ret = -ENXIO;
768 break;
769 }
770
771 return ret;
772 }
773
774 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
775 {
776 int ret;
777
778 switch (attr->group) {
779 case KVM_S390_VM_MEM_CTRL:
780 ret = kvm_s390_get_mem_control(kvm, attr);
781 break;
782 case KVM_S390_VM_TOD:
783 ret = kvm_s390_get_tod(kvm, attr);
784 break;
785 case KVM_S390_VM_CPU_MODEL:
786 ret = kvm_s390_get_cpu_model(kvm, attr);
787 break;
788 default:
789 ret = -ENXIO;
790 break;
791 }
792
793 return ret;
794 }
795
796 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
797 {
798 int ret;
799
800 switch (attr->group) {
801 case KVM_S390_VM_MEM_CTRL:
802 switch (attr->attr) {
803 case KVM_S390_VM_MEM_ENABLE_CMMA:
804 case KVM_S390_VM_MEM_CLR_CMMA:
805 case KVM_S390_VM_MEM_LIMIT_SIZE:
806 ret = 0;
807 break;
808 default:
809 ret = -ENXIO;
810 break;
811 }
812 break;
813 case KVM_S390_VM_TOD:
814 switch (attr->attr) {
815 case KVM_S390_VM_TOD_LOW:
816 case KVM_S390_VM_TOD_HIGH:
817 ret = 0;
818 break;
819 default:
820 ret = -ENXIO;
821 break;
822 }
823 break;
824 case KVM_S390_VM_CPU_MODEL:
825 switch (attr->attr) {
826 case KVM_S390_VM_CPU_PROCESSOR:
827 case KVM_S390_VM_CPU_MACHINE:
828 ret = 0;
829 break;
830 default:
831 ret = -ENXIO;
832 break;
833 }
834 break;
835 case KVM_S390_VM_CRYPTO:
836 switch (attr->attr) {
837 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
838 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
839 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
840 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
841 ret = 0;
842 break;
843 default:
844 ret = -ENXIO;
845 break;
846 }
847 break;
848 default:
849 ret = -ENXIO;
850 break;
851 }
852
853 return ret;
854 }
855
856 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
857 {
858 uint8_t *keys;
859 uint64_t hva;
860 unsigned long curkey;
861 int i, r = 0;
862
863 if (args->flags != 0)
864 return -EINVAL;
865
866 /* Is this guest using storage keys? */
867 if (!mm_use_skey(current->mm))
868 return KVM_S390_GET_SKEYS_NONE;
869
870 /* Enforce sane limit on memory allocation */
871 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
872 return -EINVAL;
873
874 keys = kmalloc_array(args->count, sizeof(uint8_t),
875 GFP_KERNEL | __GFP_NOWARN);
876 if (!keys)
877 keys = vmalloc(sizeof(uint8_t) * args->count);
878 if (!keys)
879 return -ENOMEM;
880
881 for (i = 0; i < args->count; i++) {
882 hva = gfn_to_hva(kvm, args->start_gfn + i);
883 if (kvm_is_error_hva(hva)) {
884 r = -EFAULT;
885 goto out;
886 }
887
888 curkey = get_guest_storage_key(current->mm, hva);
889 if (IS_ERR_VALUE(curkey)) {
890 r = curkey;
891 goto out;
892 }
893 keys[i] = curkey;
894 }
895
896 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
897 sizeof(uint8_t) * args->count);
898 if (r)
899 r = -EFAULT;
900 out:
901 kvfree(keys);
902 return r;
903 }
904
905 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
906 {
907 uint8_t *keys;
908 uint64_t hva;
909 int i, r = 0;
910
911 if (args->flags != 0)
912 return -EINVAL;
913
914 /* Enforce sane limit on memory allocation */
915 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
916 return -EINVAL;
917
918 keys = kmalloc_array(args->count, sizeof(uint8_t),
919 GFP_KERNEL | __GFP_NOWARN);
920 if (!keys)
921 keys = vmalloc(sizeof(uint8_t) * args->count);
922 if (!keys)
923 return -ENOMEM;
924
925 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
926 sizeof(uint8_t) * args->count);
927 if (r) {
928 r = -EFAULT;
929 goto out;
930 }
931
932 /* Enable storage key handling for the guest */
933 r = s390_enable_skey();
934 if (r)
935 goto out;
936
937 for (i = 0; i < args->count; i++) {
938 hva = gfn_to_hva(kvm, args->start_gfn + i);
939 if (kvm_is_error_hva(hva)) {
940 r = -EFAULT;
941 goto out;
942 }
943
944 /* Lowest order bit is reserved */
945 if (keys[i] & 0x01) {
946 r = -EINVAL;
947 goto out;
948 }
949
950 r = set_guest_storage_key(current->mm, hva,
951 (unsigned long)keys[i], 0);
952 if (r)
953 goto out;
954 }
955 out:
956 kvfree(keys);
957 return r;
958 }
959
960 long kvm_arch_vm_ioctl(struct file *filp,
961 unsigned int ioctl, unsigned long arg)
962 {
963 struct kvm *kvm = filp->private_data;
964 void __user *argp = (void __user *)arg;
965 struct kvm_device_attr attr;
966 int r;
967
968 switch (ioctl) {
969 case KVM_S390_INTERRUPT: {
970 struct kvm_s390_interrupt s390int;
971
972 r = -EFAULT;
973 if (copy_from_user(&s390int, argp, sizeof(s390int)))
974 break;
975 r = kvm_s390_inject_vm(kvm, &s390int);
976 break;
977 }
978 case KVM_ENABLE_CAP: {
979 struct kvm_enable_cap cap;
980 r = -EFAULT;
981 if (copy_from_user(&cap, argp, sizeof(cap)))
982 break;
983 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
984 break;
985 }
986 case KVM_CREATE_IRQCHIP: {
987 struct kvm_irq_routing_entry routing;
988
989 r = -EINVAL;
990 if (kvm->arch.use_irqchip) {
991 /* Set up dummy routing. */
992 memset(&routing, 0, sizeof(routing));
993 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
994 }
995 break;
996 }
997 case KVM_SET_DEVICE_ATTR: {
998 r = -EFAULT;
999 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1000 break;
1001 r = kvm_s390_vm_set_attr(kvm, &attr);
1002 break;
1003 }
1004 case KVM_GET_DEVICE_ATTR: {
1005 r = -EFAULT;
1006 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1007 break;
1008 r = kvm_s390_vm_get_attr(kvm, &attr);
1009 break;
1010 }
1011 case KVM_HAS_DEVICE_ATTR: {
1012 r = -EFAULT;
1013 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1014 break;
1015 r = kvm_s390_vm_has_attr(kvm, &attr);
1016 break;
1017 }
1018 case KVM_S390_GET_SKEYS: {
1019 struct kvm_s390_skeys args;
1020
1021 r = -EFAULT;
1022 if (copy_from_user(&args, argp,
1023 sizeof(struct kvm_s390_skeys)))
1024 break;
1025 r = kvm_s390_get_skeys(kvm, &args);
1026 break;
1027 }
1028 case KVM_S390_SET_SKEYS: {
1029 struct kvm_s390_skeys args;
1030
1031 r = -EFAULT;
1032 if (copy_from_user(&args, argp,
1033 sizeof(struct kvm_s390_skeys)))
1034 break;
1035 r = kvm_s390_set_skeys(kvm, &args);
1036 break;
1037 }
1038 default:
1039 r = -ENOTTY;
1040 }
1041
1042 return r;
1043 }
1044
1045 static int kvm_s390_query_ap_config(u8 *config)
1046 {
1047 u32 fcn_code = 0x04000000UL;
1048 u32 cc = 0;
1049
1050 memset(config, 0, 128);
1051 asm volatile(
1052 "lgr 0,%1\n"
1053 "lgr 2,%2\n"
1054 ".long 0xb2af0000\n" /* PQAP(QCI) */
1055 "0: ipm %0\n"
1056 "srl %0,28\n"
1057 "1:\n"
1058 EX_TABLE(0b, 1b)
1059 : "+r" (cc)
1060 : "r" (fcn_code), "r" (config)
1061 : "cc", "0", "2", "memory"
1062 );
1063
1064 return cc;
1065 }
1066
1067 static int kvm_s390_apxa_installed(void)
1068 {
1069 u8 config[128];
1070 int cc;
1071
1072 if (test_facility(12)) {
1073 cc = kvm_s390_query_ap_config(config);
1074
1075 if (cc)
1076 pr_err("PQAP(QCI) failed with cc=%d", cc);
1077 else
1078 return config[0] & 0x40;
1079 }
1080
1081 return 0;
1082 }
1083
1084 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1085 {
1086 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1087
1088 if (kvm_s390_apxa_installed())
1089 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1090 else
1091 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1092 }
1093
1094 static u64 kvm_s390_get_initial_cpuid(void)
1095 {
1096 struct cpuid cpuid;
1097
1098 get_cpu_id(&cpuid);
1099 cpuid.version = 0xff;
1100 return *((u64 *) &cpuid);
1101 }
1102
1103 static void kvm_s390_crypto_init(struct kvm *kvm)
1104 {
1105 if (!test_kvm_facility(kvm, 76))
1106 return;
1107
1108 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1109 kvm_s390_set_crycb_format(kvm);
1110
1111 /* Enable AES/DEA protected key functions by default */
1112 kvm->arch.crypto.aes_kw = 1;
1113 kvm->arch.crypto.dea_kw = 1;
1114 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1115 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1116 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1117 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1118 }
1119
1120 static void sca_dispose(struct kvm *kvm)
1121 {
1122 if (kvm->arch.use_esca)
1123 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1124 else
1125 free_page((unsigned long)(kvm->arch.sca));
1126 kvm->arch.sca = NULL;
1127 }
1128
1129 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1130 {
1131 int i, rc;
1132 char debug_name[16];
1133 static unsigned long sca_offset;
1134
1135 rc = -EINVAL;
1136 #ifdef CONFIG_KVM_S390_UCONTROL
1137 if (type & ~KVM_VM_S390_UCONTROL)
1138 goto out_err;
1139 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1140 goto out_err;
1141 #else
1142 if (type)
1143 goto out_err;
1144 #endif
1145
1146 rc = s390_enable_sie();
1147 if (rc)
1148 goto out_err;
1149
1150 rc = -ENOMEM;
1151
1152 kvm->arch.use_esca = 0; /* start with basic SCA */
1153 rwlock_init(&kvm->arch.sca_lock);
1154 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
1155 if (!kvm->arch.sca)
1156 goto out_err;
1157 spin_lock(&kvm_lock);
1158 sca_offset += 16;
1159 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1160 sca_offset = 0;
1161 kvm->arch.sca = (struct bsca_block *)
1162 ((char *) kvm->arch.sca + sca_offset);
1163 spin_unlock(&kvm_lock);
1164
1165 sprintf(debug_name, "kvm-%u", current->pid);
1166
1167 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1168 if (!kvm->arch.dbf)
1169 goto out_err;
1170
1171 kvm->arch.sie_page2 =
1172 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1173 if (!kvm->arch.sie_page2)
1174 goto out_err;
1175
1176 /* Populate the facility mask initially. */
1177 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1178 S390_ARCH_FAC_LIST_SIZE_BYTE);
1179 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1180 if (i < kvm_s390_fac_list_mask_size())
1181 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1182 else
1183 kvm->arch.model.fac_mask[i] = 0UL;
1184 }
1185
1186 /* Populate the facility list initially. */
1187 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1188 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1189 S390_ARCH_FAC_LIST_SIZE_BYTE);
1190
1191 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1192 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1193
1194 kvm_s390_crypto_init(kvm);
1195
1196 spin_lock_init(&kvm->arch.float_int.lock);
1197 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1198 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1199 init_waitqueue_head(&kvm->arch.ipte_wq);
1200 mutex_init(&kvm->arch.ipte_mutex);
1201
1202 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1203 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1204
1205 if (type & KVM_VM_S390_UCONTROL) {
1206 kvm->arch.gmap = NULL;
1207 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1208 } else {
1209 if (sclp.hamax == U64_MAX)
1210 kvm->arch.mem_limit = TASK_MAX_SIZE;
1211 else
1212 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1213 sclp.hamax + 1);
1214 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
1215 if (!kvm->arch.gmap)
1216 goto out_err;
1217 kvm->arch.gmap->private = kvm;
1218 kvm->arch.gmap->pfault_enabled = 0;
1219 }
1220
1221 kvm->arch.css_support = 0;
1222 kvm->arch.use_irqchip = 0;
1223 kvm->arch.epoch = 0;
1224
1225 spin_lock_init(&kvm->arch.start_stop_lock);
1226 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1227
1228 return 0;
1229 out_err:
1230 free_page((unsigned long)kvm->arch.sie_page2);
1231 debug_unregister(kvm->arch.dbf);
1232 sca_dispose(kvm);
1233 KVM_EVENT(3, "creation of vm failed: %d", rc);
1234 return rc;
1235 }
1236
1237 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1238 {
1239 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1240 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1241 kvm_s390_clear_local_irqs(vcpu);
1242 kvm_clear_async_pf_completion_queue(vcpu);
1243 if (!kvm_is_ucontrol(vcpu->kvm))
1244 sca_del_vcpu(vcpu);
1245
1246 if (kvm_is_ucontrol(vcpu->kvm))
1247 gmap_free(vcpu->arch.gmap);
1248
1249 if (vcpu->kvm->arch.use_cmma)
1250 kvm_s390_vcpu_unsetup_cmma(vcpu);
1251 free_page((unsigned long)(vcpu->arch.sie_block));
1252
1253 kvm_vcpu_uninit(vcpu);
1254 kmem_cache_free(kvm_vcpu_cache, vcpu);
1255 }
1256
1257 static void kvm_free_vcpus(struct kvm *kvm)
1258 {
1259 unsigned int i;
1260 struct kvm_vcpu *vcpu;
1261
1262 kvm_for_each_vcpu(i, vcpu, kvm)
1263 kvm_arch_vcpu_destroy(vcpu);
1264
1265 mutex_lock(&kvm->lock);
1266 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1267 kvm->vcpus[i] = NULL;
1268
1269 atomic_set(&kvm->online_vcpus, 0);
1270 mutex_unlock(&kvm->lock);
1271 }
1272
1273 void kvm_arch_destroy_vm(struct kvm *kvm)
1274 {
1275 kvm_free_vcpus(kvm);
1276 sca_dispose(kvm);
1277 debug_unregister(kvm->arch.dbf);
1278 free_page((unsigned long)kvm->arch.sie_page2);
1279 if (!kvm_is_ucontrol(kvm))
1280 gmap_free(kvm->arch.gmap);
1281 kvm_s390_destroy_adapters(kvm);
1282 kvm_s390_clear_float_irqs(kvm);
1283 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1284 }
1285
1286 /* Section: vcpu related */
1287 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1288 {
1289 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1290 if (!vcpu->arch.gmap)
1291 return -ENOMEM;
1292 vcpu->arch.gmap->private = vcpu->kvm;
1293
1294 return 0;
1295 }
1296
1297 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1298 {
1299 read_lock(&vcpu->kvm->arch.sca_lock);
1300 if (vcpu->kvm->arch.use_esca) {
1301 struct esca_block *sca = vcpu->kvm->arch.sca;
1302
1303 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1304 sca->cpu[vcpu->vcpu_id].sda = 0;
1305 } else {
1306 struct bsca_block *sca = vcpu->kvm->arch.sca;
1307
1308 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1309 sca->cpu[vcpu->vcpu_id].sda = 0;
1310 }
1311 read_unlock(&vcpu->kvm->arch.sca_lock);
1312 }
1313
1314 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1315 {
1316 read_lock(&vcpu->kvm->arch.sca_lock);
1317 if (vcpu->kvm->arch.use_esca) {
1318 struct esca_block *sca = vcpu->kvm->arch.sca;
1319
1320 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1321 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1322 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1323 vcpu->arch.sie_block->ecb2 |= 0x04U;
1324 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1325 } else {
1326 struct bsca_block *sca = vcpu->kvm->arch.sca;
1327
1328 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1329 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1330 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1331 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1332 }
1333 read_unlock(&vcpu->kvm->arch.sca_lock);
1334 }
1335
1336 /* Basic SCA to Extended SCA data copy routines */
1337 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1338 {
1339 d->sda = s->sda;
1340 d->sigp_ctrl.c = s->sigp_ctrl.c;
1341 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1342 }
1343
1344 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1345 {
1346 int i;
1347
1348 d->ipte_control = s->ipte_control;
1349 d->mcn[0] = s->mcn;
1350 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1351 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1352 }
1353
1354 static int sca_switch_to_extended(struct kvm *kvm)
1355 {
1356 struct bsca_block *old_sca = kvm->arch.sca;
1357 struct esca_block *new_sca;
1358 struct kvm_vcpu *vcpu;
1359 unsigned int vcpu_idx;
1360 u32 scaol, scaoh;
1361
1362 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1363 if (!new_sca)
1364 return -ENOMEM;
1365
1366 scaoh = (u32)((u64)(new_sca) >> 32);
1367 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1368
1369 kvm_s390_vcpu_block_all(kvm);
1370 write_lock(&kvm->arch.sca_lock);
1371
1372 sca_copy_b_to_e(new_sca, old_sca);
1373
1374 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1375 vcpu->arch.sie_block->scaoh = scaoh;
1376 vcpu->arch.sie_block->scaol = scaol;
1377 vcpu->arch.sie_block->ecb2 |= 0x04U;
1378 }
1379 kvm->arch.sca = new_sca;
1380 kvm->arch.use_esca = 1;
1381
1382 write_unlock(&kvm->arch.sca_lock);
1383 kvm_s390_vcpu_unblock_all(kvm);
1384
1385 free_page((unsigned long)old_sca);
1386
1387 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1388 old_sca, kvm->arch.sca);
1389 return 0;
1390 }
1391
1392 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1393 {
1394 int rc;
1395
1396 if (id < KVM_S390_BSCA_CPU_SLOTS)
1397 return true;
1398 if (!sclp.has_esca)
1399 return false;
1400
1401 mutex_lock(&kvm->lock);
1402 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1403 mutex_unlock(&kvm->lock);
1404
1405 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1406 }
1407
1408 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1409 {
1410 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1411 kvm_clear_async_pf_completion_queue(vcpu);
1412 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1413 KVM_SYNC_GPRS |
1414 KVM_SYNC_ACRS |
1415 KVM_SYNC_CRS |
1416 KVM_SYNC_ARCH0 |
1417 KVM_SYNC_PFAULT;
1418 if (test_kvm_facility(vcpu->kvm, 64))
1419 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1420 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1421 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1422 */
1423 if (MACHINE_HAS_VX)
1424 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1425 else
1426 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1427
1428 if (kvm_is_ucontrol(vcpu->kvm))
1429 return __kvm_ucontrol_vcpu_init(vcpu);
1430
1431 return 0;
1432 }
1433
1434 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1435 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1436 {
1437 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1438 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1439 vcpu->arch.cputm_start = get_tod_clock_fast();
1440 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1441 }
1442
1443 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1444 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1445 {
1446 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1447 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1448 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1449 vcpu->arch.cputm_start = 0;
1450 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1451 }
1452
1453 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1454 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1455 {
1456 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1457 vcpu->arch.cputm_enabled = true;
1458 __start_cpu_timer_accounting(vcpu);
1459 }
1460
1461 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1462 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1463 {
1464 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1465 __stop_cpu_timer_accounting(vcpu);
1466 vcpu->arch.cputm_enabled = false;
1467 }
1468
1469 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1470 {
1471 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1472 __enable_cpu_timer_accounting(vcpu);
1473 preempt_enable();
1474 }
1475
1476 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1477 {
1478 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1479 __disable_cpu_timer_accounting(vcpu);
1480 preempt_enable();
1481 }
1482
1483 /* set the cpu timer - may only be called from the VCPU thread itself */
1484 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1485 {
1486 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1487 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1488 if (vcpu->arch.cputm_enabled)
1489 vcpu->arch.cputm_start = get_tod_clock_fast();
1490 vcpu->arch.sie_block->cputm = cputm;
1491 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1492 preempt_enable();
1493 }
1494
1495 /* update and get the cpu timer - can also be called from other VCPU threads */
1496 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1497 {
1498 unsigned int seq;
1499 __u64 value;
1500
1501 if (unlikely(!vcpu->arch.cputm_enabled))
1502 return vcpu->arch.sie_block->cputm;
1503
1504 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1505 do {
1506 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1507 /*
1508 * If the writer would ever execute a read in the critical
1509 * section, e.g. in irq context, we have a deadlock.
1510 */
1511 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1512 value = vcpu->arch.sie_block->cputm;
1513 /* if cputm_start is 0, accounting is being started/stopped */
1514 if (likely(vcpu->arch.cputm_start))
1515 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1516 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1517 preempt_enable();
1518 return value;
1519 }
1520
1521 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1522 {
1523 /* Save host register state */
1524 save_fpu_regs();
1525 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1526 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
1527
1528 if (MACHINE_HAS_VX)
1529 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1530 else
1531 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
1532 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1533 if (test_fp_ctl(current->thread.fpu.fpc))
1534 /* User space provided an invalid FPC, let's clear it */
1535 current->thread.fpu.fpc = 0;
1536
1537 save_access_regs(vcpu->arch.host_acrs);
1538 restore_access_regs(vcpu->run->s.regs.acrs);
1539 gmap_enable(vcpu->arch.gmap);
1540 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1541 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1542 __start_cpu_timer_accounting(vcpu);
1543 vcpu->cpu = cpu;
1544 }
1545
1546 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1547 {
1548 vcpu->cpu = -1;
1549 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1550 __stop_cpu_timer_accounting(vcpu);
1551 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1552 gmap_disable(vcpu->arch.gmap);
1553
1554 /* Save guest register state */
1555 save_fpu_regs();
1556 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1557
1558 /* Restore host register state */
1559 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1560 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
1561
1562 save_access_regs(vcpu->run->s.regs.acrs);
1563 restore_access_regs(vcpu->arch.host_acrs);
1564 }
1565
1566 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1567 {
1568 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1569 vcpu->arch.sie_block->gpsw.mask = 0UL;
1570 vcpu->arch.sie_block->gpsw.addr = 0UL;
1571 kvm_s390_set_prefix(vcpu, 0);
1572 kvm_s390_set_cpu_timer(vcpu, 0);
1573 vcpu->arch.sie_block->ckc = 0UL;
1574 vcpu->arch.sie_block->todpr = 0;
1575 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1576 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1577 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1578 /* make sure the new fpc will be lazily loaded */
1579 save_fpu_regs();
1580 current->thread.fpu.fpc = 0;
1581 vcpu->arch.sie_block->gbea = 1;
1582 vcpu->arch.sie_block->pp = 0;
1583 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1584 kvm_clear_async_pf_completion_queue(vcpu);
1585 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1586 kvm_s390_vcpu_stop(vcpu);
1587 kvm_s390_clear_local_irqs(vcpu);
1588 }
1589
1590 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1591 {
1592 mutex_lock(&vcpu->kvm->lock);
1593 preempt_disable();
1594 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1595 preempt_enable();
1596 mutex_unlock(&vcpu->kvm->lock);
1597 if (!kvm_is_ucontrol(vcpu->kvm)) {
1598 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1599 sca_add_vcpu(vcpu);
1600 }
1601
1602 }
1603
1604 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1605 {
1606 if (!test_kvm_facility(vcpu->kvm, 76))
1607 return;
1608
1609 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1610
1611 if (vcpu->kvm->arch.crypto.aes_kw)
1612 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1613 if (vcpu->kvm->arch.crypto.dea_kw)
1614 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1615
1616 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1617 }
1618
1619 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1620 {
1621 free_page(vcpu->arch.sie_block->cbrlo);
1622 vcpu->arch.sie_block->cbrlo = 0;
1623 }
1624
1625 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1626 {
1627 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1628 if (!vcpu->arch.sie_block->cbrlo)
1629 return -ENOMEM;
1630
1631 vcpu->arch.sie_block->ecb2 |= 0x80;
1632 vcpu->arch.sie_block->ecb2 &= ~0x08;
1633 return 0;
1634 }
1635
1636 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1637 {
1638 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1639
1640 vcpu->arch.sie_block->ibc = model->ibc;
1641 if (test_kvm_facility(vcpu->kvm, 7))
1642 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1643 }
1644
1645 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1646 {
1647 int rc = 0;
1648
1649 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1650 CPUSTAT_SM |
1651 CPUSTAT_STOPPED);
1652
1653 if (test_kvm_facility(vcpu->kvm, 78))
1654 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1655 else if (test_kvm_facility(vcpu->kvm, 8))
1656 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1657
1658 kvm_s390_vcpu_setup_model(vcpu);
1659
1660 vcpu->arch.sie_block->ecb = 0x02;
1661 if (test_kvm_facility(vcpu->kvm, 9))
1662 vcpu->arch.sie_block->ecb |= 0x04;
1663 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1664 vcpu->arch.sie_block->ecb |= 0x10;
1665
1666 if (test_kvm_facility(vcpu->kvm, 8))
1667 vcpu->arch.sie_block->ecb2 |= 0x08;
1668 vcpu->arch.sie_block->eca = 0xC1002000U;
1669 if (sclp.has_siif)
1670 vcpu->arch.sie_block->eca |= 1;
1671 if (sclp.has_sigpif)
1672 vcpu->arch.sie_block->eca |= 0x10000000U;
1673 if (test_kvm_facility(vcpu->kvm, 64))
1674 vcpu->arch.sie_block->ecb3 |= 0x01;
1675 if (test_kvm_facility(vcpu->kvm, 129)) {
1676 vcpu->arch.sie_block->eca |= 0x00020000;
1677 vcpu->arch.sie_block->ecd |= 0x20000000;
1678 }
1679 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1680 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1681
1682 if (vcpu->kvm->arch.use_cmma) {
1683 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1684 if (rc)
1685 return rc;
1686 }
1687 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1688 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1689
1690 kvm_s390_vcpu_crypto_setup(vcpu);
1691
1692 return rc;
1693 }
1694
1695 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1696 unsigned int id)
1697 {
1698 struct kvm_vcpu *vcpu;
1699 struct sie_page *sie_page;
1700 int rc = -EINVAL;
1701
1702 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1703 goto out;
1704
1705 rc = -ENOMEM;
1706
1707 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1708 if (!vcpu)
1709 goto out;
1710
1711 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1712 if (!sie_page)
1713 goto out_free_cpu;
1714
1715 vcpu->arch.sie_block = &sie_page->sie_block;
1716 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1717
1718 vcpu->arch.sie_block->icpua = id;
1719 spin_lock_init(&vcpu->arch.local_int.lock);
1720 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1721 vcpu->arch.local_int.wq = &vcpu->wq;
1722 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1723 seqcount_init(&vcpu->arch.cputm_seqcount);
1724
1725 rc = kvm_vcpu_init(vcpu, kvm, id);
1726 if (rc)
1727 goto out_free_sie_block;
1728 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
1729 vcpu->arch.sie_block);
1730 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1731
1732 return vcpu;
1733 out_free_sie_block:
1734 free_page((unsigned long)(vcpu->arch.sie_block));
1735 out_free_cpu:
1736 kmem_cache_free(kvm_vcpu_cache, vcpu);
1737 out:
1738 return ERR_PTR(rc);
1739 }
1740
1741 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1742 {
1743 return kvm_s390_vcpu_has_irq(vcpu, 0);
1744 }
1745
1746 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1747 {
1748 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1749 exit_sie(vcpu);
1750 }
1751
1752 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1753 {
1754 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1755 }
1756
1757 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1758 {
1759 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1760 exit_sie(vcpu);
1761 }
1762
1763 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1764 {
1765 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1766 }
1767
1768 /*
1769 * Kick a guest cpu out of SIE and wait until SIE is not running.
1770 * If the CPU is not running (e.g. waiting as idle) the function will
1771 * return immediately. */
1772 void exit_sie(struct kvm_vcpu *vcpu)
1773 {
1774 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1775 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1776 cpu_relax();
1777 }
1778
1779 /* Kick a guest cpu out of SIE to process a request synchronously */
1780 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1781 {
1782 kvm_make_request(req, vcpu);
1783 kvm_s390_vcpu_request(vcpu);
1784 }
1785
1786 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1787 {
1788 int i;
1789 struct kvm *kvm = gmap->private;
1790 struct kvm_vcpu *vcpu;
1791
1792 kvm_for_each_vcpu(i, vcpu, kvm) {
1793 /* match against both prefix pages */
1794 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1795 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1796 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1797 }
1798 }
1799 }
1800
1801 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1802 {
1803 /* kvm common code refers to this, but never calls it */
1804 BUG();
1805 return 0;
1806 }
1807
1808 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1809 struct kvm_one_reg *reg)
1810 {
1811 int r = -EINVAL;
1812
1813 switch (reg->id) {
1814 case KVM_REG_S390_TODPR:
1815 r = put_user(vcpu->arch.sie_block->todpr,
1816 (u32 __user *)reg->addr);
1817 break;
1818 case KVM_REG_S390_EPOCHDIFF:
1819 r = put_user(vcpu->arch.sie_block->epoch,
1820 (u64 __user *)reg->addr);
1821 break;
1822 case KVM_REG_S390_CPU_TIMER:
1823 r = put_user(kvm_s390_get_cpu_timer(vcpu),
1824 (u64 __user *)reg->addr);
1825 break;
1826 case KVM_REG_S390_CLOCK_COMP:
1827 r = put_user(vcpu->arch.sie_block->ckc,
1828 (u64 __user *)reg->addr);
1829 break;
1830 case KVM_REG_S390_PFTOKEN:
1831 r = put_user(vcpu->arch.pfault_token,
1832 (u64 __user *)reg->addr);
1833 break;
1834 case KVM_REG_S390_PFCOMPARE:
1835 r = put_user(vcpu->arch.pfault_compare,
1836 (u64 __user *)reg->addr);
1837 break;
1838 case KVM_REG_S390_PFSELECT:
1839 r = put_user(vcpu->arch.pfault_select,
1840 (u64 __user *)reg->addr);
1841 break;
1842 case KVM_REG_S390_PP:
1843 r = put_user(vcpu->arch.sie_block->pp,
1844 (u64 __user *)reg->addr);
1845 break;
1846 case KVM_REG_S390_GBEA:
1847 r = put_user(vcpu->arch.sie_block->gbea,
1848 (u64 __user *)reg->addr);
1849 break;
1850 default:
1851 break;
1852 }
1853
1854 return r;
1855 }
1856
1857 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1858 struct kvm_one_reg *reg)
1859 {
1860 int r = -EINVAL;
1861 __u64 val;
1862
1863 switch (reg->id) {
1864 case KVM_REG_S390_TODPR:
1865 r = get_user(vcpu->arch.sie_block->todpr,
1866 (u32 __user *)reg->addr);
1867 break;
1868 case KVM_REG_S390_EPOCHDIFF:
1869 r = get_user(vcpu->arch.sie_block->epoch,
1870 (u64 __user *)reg->addr);
1871 break;
1872 case KVM_REG_S390_CPU_TIMER:
1873 r = get_user(val, (u64 __user *)reg->addr);
1874 if (!r)
1875 kvm_s390_set_cpu_timer(vcpu, val);
1876 break;
1877 case KVM_REG_S390_CLOCK_COMP:
1878 r = get_user(vcpu->arch.sie_block->ckc,
1879 (u64 __user *)reg->addr);
1880 break;
1881 case KVM_REG_S390_PFTOKEN:
1882 r = get_user(vcpu->arch.pfault_token,
1883 (u64 __user *)reg->addr);
1884 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1885 kvm_clear_async_pf_completion_queue(vcpu);
1886 break;
1887 case KVM_REG_S390_PFCOMPARE:
1888 r = get_user(vcpu->arch.pfault_compare,
1889 (u64 __user *)reg->addr);
1890 break;
1891 case KVM_REG_S390_PFSELECT:
1892 r = get_user(vcpu->arch.pfault_select,
1893 (u64 __user *)reg->addr);
1894 break;
1895 case KVM_REG_S390_PP:
1896 r = get_user(vcpu->arch.sie_block->pp,
1897 (u64 __user *)reg->addr);
1898 break;
1899 case KVM_REG_S390_GBEA:
1900 r = get_user(vcpu->arch.sie_block->gbea,
1901 (u64 __user *)reg->addr);
1902 break;
1903 default:
1904 break;
1905 }
1906
1907 return r;
1908 }
1909
1910 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1911 {
1912 kvm_s390_vcpu_initial_reset(vcpu);
1913 return 0;
1914 }
1915
1916 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1917 {
1918 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1919 return 0;
1920 }
1921
1922 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1923 {
1924 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1925 return 0;
1926 }
1927
1928 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1929 struct kvm_sregs *sregs)
1930 {
1931 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1932 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1933 restore_access_regs(vcpu->run->s.regs.acrs);
1934 return 0;
1935 }
1936
1937 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1938 struct kvm_sregs *sregs)
1939 {
1940 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1941 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1942 return 0;
1943 }
1944
1945 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1946 {
1947 /* make sure the new values will be lazily loaded */
1948 save_fpu_regs();
1949 if (test_fp_ctl(fpu->fpc))
1950 return -EINVAL;
1951 current->thread.fpu.fpc = fpu->fpc;
1952 if (MACHINE_HAS_VX)
1953 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
1954 else
1955 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
1956 return 0;
1957 }
1958
1959 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1960 {
1961 /* make sure we have the latest values */
1962 save_fpu_regs();
1963 if (MACHINE_HAS_VX)
1964 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
1965 else
1966 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
1967 fpu->fpc = current->thread.fpu.fpc;
1968 return 0;
1969 }
1970
1971 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1972 {
1973 int rc = 0;
1974
1975 if (!is_vcpu_stopped(vcpu))
1976 rc = -EBUSY;
1977 else {
1978 vcpu->run->psw_mask = psw.mask;
1979 vcpu->run->psw_addr = psw.addr;
1980 }
1981 return rc;
1982 }
1983
1984 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1985 struct kvm_translation *tr)
1986 {
1987 return -EINVAL; /* not implemented yet */
1988 }
1989
1990 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1991 KVM_GUESTDBG_USE_HW_BP | \
1992 KVM_GUESTDBG_ENABLE)
1993
1994 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1995 struct kvm_guest_debug *dbg)
1996 {
1997 int rc = 0;
1998
1999 vcpu->guest_debug = 0;
2000 kvm_s390_clear_bp_data(vcpu);
2001
2002 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2003 return -EINVAL;
2004
2005 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2006 vcpu->guest_debug = dbg->control;
2007 /* enforce guest PER */
2008 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2009
2010 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2011 rc = kvm_s390_import_bp_data(vcpu, dbg);
2012 } else {
2013 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2014 vcpu->arch.guestdbg.last_bp = 0;
2015 }
2016
2017 if (rc) {
2018 vcpu->guest_debug = 0;
2019 kvm_s390_clear_bp_data(vcpu);
2020 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2021 }
2022
2023 return rc;
2024 }
2025
2026 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2027 struct kvm_mp_state *mp_state)
2028 {
2029 /* CHECK_STOP and LOAD are not supported yet */
2030 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2031 KVM_MP_STATE_OPERATING;
2032 }
2033
2034 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2035 struct kvm_mp_state *mp_state)
2036 {
2037 int rc = 0;
2038
2039 /* user space knows about this interface - let it control the state */
2040 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2041
2042 switch (mp_state->mp_state) {
2043 case KVM_MP_STATE_STOPPED:
2044 kvm_s390_vcpu_stop(vcpu);
2045 break;
2046 case KVM_MP_STATE_OPERATING:
2047 kvm_s390_vcpu_start(vcpu);
2048 break;
2049 case KVM_MP_STATE_LOAD:
2050 case KVM_MP_STATE_CHECK_STOP:
2051 /* fall through - CHECK_STOP and LOAD are not supported yet */
2052 default:
2053 rc = -ENXIO;
2054 }
2055
2056 return rc;
2057 }
2058
2059 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2060 {
2061 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2062 }
2063
2064 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2065 {
2066 retry:
2067 kvm_s390_vcpu_request_handled(vcpu);
2068 if (!vcpu->requests)
2069 return 0;
2070 /*
2071 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2072 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2073 * This ensures that the ipte instruction for this request has
2074 * already finished. We might race against a second unmapper that
2075 * wants to set the blocking bit. Lets just retry the request loop.
2076 */
2077 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2078 int rc;
2079 rc = gmap_ipte_notify(vcpu->arch.gmap,
2080 kvm_s390_get_prefix(vcpu),
2081 PAGE_SIZE * 2);
2082 if (rc)
2083 return rc;
2084 goto retry;
2085 }
2086
2087 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2088 vcpu->arch.sie_block->ihcpu = 0xffff;
2089 goto retry;
2090 }
2091
2092 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2093 if (!ibs_enabled(vcpu)) {
2094 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2095 atomic_or(CPUSTAT_IBS,
2096 &vcpu->arch.sie_block->cpuflags);
2097 }
2098 goto retry;
2099 }
2100
2101 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2102 if (ibs_enabled(vcpu)) {
2103 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2104 atomic_andnot(CPUSTAT_IBS,
2105 &vcpu->arch.sie_block->cpuflags);
2106 }
2107 goto retry;
2108 }
2109
2110 /* nothing to do, just clear the request */
2111 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2112
2113 return 0;
2114 }
2115
2116 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2117 {
2118 struct kvm_vcpu *vcpu;
2119 int i;
2120
2121 mutex_lock(&kvm->lock);
2122 preempt_disable();
2123 kvm->arch.epoch = tod - get_tod_clock();
2124 kvm_s390_vcpu_block_all(kvm);
2125 kvm_for_each_vcpu(i, vcpu, kvm)
2126 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2127 kvm_s390_vcpu_unblock_all(kvm);
2128 preempt_enable();
2129 mutex_unlock(&kvm->lock);
2130 }
2131
2132 /**
2133 * kvm_arch_fault_in_page - fault-in guest page if necessary
2134 * @vcpu: The corresponding virtual cpu
2135 * @gpa: Guest physical address
2136 * @writable: Whether the page should be writable or not
2137 *
2138 * Make sure that a guest page has been faulted-in on the host.
2139 *
2140 * Return: Zero on success, negative error code otherwise.
2141 */
2142 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2143 {
2144 return gmap_fault(vcpu->arch.gmap, gpa,
2145 writable ? FAULT_FLAG_WRITE : 0);
2146 }
2147
2148 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2149 unsigned long token)
2150 {
2151 struct kvm_s390_interrupt inti;
2152 struct kvm_s390_irq irq;
2153
2154 if (start_token) {
2155 irq.u.ext.ext_params2 = token;
2156 irq.type = KVM_S390_INT_PFAULT_INIT;
2157 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2158 } else {
2159 inti.type = KVM_S390_INT_PFAULT_DONE;
2160 inti.parm64 = token;
2161 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2162 }
2163 }
2164
2165 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2166 struct kvm_async_pf *work)
2167 {
2168 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2169 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2170 }
2171
2172 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2173 struct kvm_async_pf *work)
2174 {
2175 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2176 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2177 }
2178
2179 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2180 struct kvm_async_pf *work)
2181 {
2182 /* s390 will always inject the page directly */
2183 }
2184
2185 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2186 {
2187 /*
2188 * s390 will always inject the page directly,
2189 * but we still want check_async_completion to cleanup
2190 */
2191 return true;
2192 }
2193
2194 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2195 {
2196 hva_t hva;
2197 struct kvm_arch_async_pf arch;
2198 int rc;
2199
2200 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2201 return 0;
2202 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2203 vcpu->arch.pfault_compare)
2204 return 0;
2205 if (psw_extint_disabled(vcpu))
2206 return 0;
2207 if (kvm_s390_vcpu_has_irq(vcpu, 0))
2208 return 0;
2209 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2210 return 0;
2211 if (!vcpu->arch.gmap->pfault_enabled)
2212 return 0;
2213
2214 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2215 hva += current->thread.gmap_addr & ~PAGE_MASK;
2216 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2217 return 0;
2218
2219 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2220 return rc;
2221 }
2222
2223 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2224 {
2225 int rc, cpuflags;
2226
2227 /*
2228 * On s390 notifications for arriving pages will be delivered directly
2229 * to the guest but the house keeping for completed pfaults is
2230 * handled outside the worker.
2231 */
2232 kvm_check_async_pf_completion(vcpu);
2233
2234 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2235 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2236
2237 if (need_resched())
2238 schedule();
2239
2240 if (test_cpu_flag(CIF_MCCK_PENDING))
2241 s390_handle_mcck();
2242
2243 if (!kvm_is_ucontrol(vcpu->kvm)) {
2244 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2245 if (rc)
2246 return rc;
2247 }
2248
2249 rc = kvm_s390_handle_requests(vcpu);
2250 if (rc)
2251 return rc;
2252
2253 if (guestdbg_enabled(vcpu)) {
2254 kvm_s390_backup_guest_per_regs(vcpu);
2255 kvm_s390_patch_guest_per_regs(vcpu);
2256 }
2257
2258 vcpu->arch.sie_block->icptcode = 0;
2259 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2260 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2261 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2262
2263 return 0;
2264 }
2265
2266 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2267 {
2268 struct kvm_s390_pgm_info pgm_info = {
2269 .code = PGM_ADDRESSING,
2270 };
2271 u8 opcode, ilen;
2272 int rc;
2273
2274 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2275 trace_kvm_s390_sie_fault(vcpu);
2276
2277 /*
2278 * We want to inject an addressing exception, which is defined as a
2279 * suppressing or terminating exception. However, since we came here
2280 * by a DAT access exception, the PSW still points to the faulting
2281 * instruction since DAT exceptions are nullifying. So we've got
2282 * to look up the current opcode to get the length of the instruction
2283 * to be able to forward the PSW.
2284 */
2285 rc = read_guest_instr(vcpu, &opcode, 1);
2286 ilen = insn_length(opcode);
2287 if (rc < 0) {
2288 return rc;
2289 } else if (rc) {
2290 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2291 * Forward by arbitrary ilc, injection will take care of
2292 * nullification if necessary.
2293 */
2294 pgm_info = vcpu->arch.pgm;
2295 ilen = 4;
2296 }
2297 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2298 kvm_s390_forward_psw(vcpu, ilen);
2299 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2300 }
2301
2302 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2303 {
2304 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2305 vcpu->arch.sie_block->icptcode);
2306 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2307
2308 if (guestdbg_enabled(vcpu))
2309 kvm_s390_restore_guest_per_regs(vcpu);
2310
2311 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2312 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2313
2314 if (vcpu->arch.sie_block->icptcode > 0) {
2315 int rc = kvm_handle_sie_intercept(vcpu);
2316
2317 if (rc != -EOPNOTSUPP)
2318 return rc;
2319 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2320 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2321 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2322 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2323 return -EREMOTE;
2324 } else if (exit_reason != -EFAULT) {
2325 vcpu->stat.exit_null++;
2326 return 0;
2327 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2328 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2329 vcpu->run->s390_ucontrol.trans_exc_code =
2330 current->thread.gmap_addr;
2331 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2332 return -EREMOTE;
2333 } else if (current->thread.gmap_pfault) {
2334 trace_kvm_s390_major_guest_pfault(vcpu);
2335 current->thread.gmap_pfault = 0;
2336 if (kvm_arch_setup_async_pf(vcpu))
2337 return 0;
2338 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2339 }
2340 return vcpu_post_run_fault_in_sie(vcpu);
2341 }
2342
2343 static int __vcpu_run(struct kvm_vcpu *vcpu)
2344 {
2345 int rc, exit_reason;
2346
2347 /*
2348 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2349 * ning the guest), so that memslots (and other stuff) are protected
2350 */
2351 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2352
2353 do {
2354 rc = vcpu_pre_run(vcpu);
2355 if (rc)
2356 break;
2357
2358 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2359 /*
2360 * As PF_VCPU will be used in fault handler, between
2361 * guest_enter and guest_exit should be no uaccess.
2362 */
2363 local_irq_disable();
2364 __kvm_guest_enter();
2365 __disable_cpu_timer_accounting(vcpu);
2366 local_irq_enable();
2367 exit_reason = sie64a(vcpu->arch.sie_block,
2368 vcpu->run->s.regs.gprs);
2369 local_irq_disable();
2370 __enable_cpu_timer_accounting(vcpu);
2371 __kvm_guest_exit();
2372 local_irq_enable();
2373 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2374
2375 rc = vcpu_post_run(vcpu, exit_reason);
2376 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2377
2378 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2379 return rc;
2380 }
2381
2382 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2383 {
2384 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2385 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2386 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2387 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2388 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2389 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2390 /* some control register changes require a tlb flush */
2391 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2392 }
2393 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2394 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2395 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2396 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2397 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2398 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2399 }
2400 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2401 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2402 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2403 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2404 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2405 kvm_clear_async_pf_completion_queue(vcpu);
2406 }
2407 kvm_run->kvm_dirty_regs = 0;
2408 }
2409
2410 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2411 {
2412 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2413 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2414 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2415 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2416 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2417 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2418 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2419 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2420 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2421 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2422 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2423 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2424 }
2425
2426 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2427 {
2428 int rc;
2429 sigset_t sigsaved;
2430
2431 if (guestdbg_exit_pending(vcpu)) {
2432 kvm_s390_prepare_debug_exit(vcpu);
2433 return 0;
2434 }
2435
2436 if (vcpu->sigset_active)
2437 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2438
2439 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2440 kvm_s390_vcpu_start(vcpu);
2441 } else if (is_vcpu_stopped(vcpu)) {
2442 pr_err_ratelimited("can't run stopped vcpu %d\n",
2443 vcpu->vcpu_id);
2444 return -EINVAL;
2445 }
2446
2447 sync_regs(vcpu, kvm_run);
2448 enable_cpu_timer_accounting(vcpu);
2449
2450 might_fault();
2451 rc = __vcpu_run(vcpu);
2452
2453 if (signal_pending(current) && !rc) {
2454 kvm_run->exit_reason = KVM_EXIT_INTR;
2455 rc = -EINTR;
2456 }
2457
2458 if (guestdbg_exit_pending(vcpu) && !rc) {
2459 kvm_s390_prepare_debug_exit(vcpu);
2460 rc = 0;
2461 }
2462
2463 if (rc == -EREMOTE) {
2464 /* userspace support is needed, kvm_run has been prepared */
2465 rc = 0;
2466 }
2467
2468 disable_cpu_timer_accounting(vcpu);
2469 store_regs(vcpu, kvm_run);
2470
2471 if (vcpu->sigset_active)
2472 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2473
2474 vcpu->stat.exit_userspace++;
2475 return rc;
2476 }
2477
2478 /*
2479 * store status at address
2480 * we use have two special cases:
2481 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2482 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2483 */
2484 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2485 {
2486 unsigned char archmode = 1;
2487 freg_t fprs[NUM_FPRS];
2488 unsigned int px;
2489 u64 clkcomp, cputm;
2490 int rc;
2491
2492 px = kvm_s390_get_prefix(vcpu);
2493 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2494 if (write_guest_abs(vcpu, 163, &archmode, 1))
2495 return -EFAULT;
2496 gpa = 0;
2497 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2498 if (write_guest_real(vcpu, 163, &archmode, 1))
2499 return -EFAULT;
2500 gpa = px;
2501 } else
2502 gpa -= __LC_FPREGS_SAVE_AREA;
2503
2504 /* manually convert vector registers if necessary */
2505 if (MACHINE_HAS_VX) {
2506 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2507 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2508 fprs, 128);
2509 } else {
2510 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2511 vcpu->run->s.regs.fprs, 128);
2512 }
2513 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2514 vcpu->run->s.regs.gprs, 128);
2515 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2516 &vcpu->arch.sie_block->gpsw, 16);
2517 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2518 &px, 4);
2519 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2520 &vcpu->run->s.regs.fpc, 4);
2521 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2522 &vcpu->arch.sie_block->todpr, 4);
2523 cputm = kvm_s390_get_cpu_timer(vcpu);
2524 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2525 &cputm, 8);
2526 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2527 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2528 &clkcomp, 8);
2529 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2530 &vcpu->run->s.regs.acrs, 64);
2531 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2532 &vcpu->arch.sie_block->gcr, 128);
2533 return rc ? -EFAULT : 0;
2534 }
2535
2536 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2537 {
2538 /*
2539 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2540 * copying in vcpu load/put. Lets update our copies before we save
2541 * it into the save area
2542 */
2543 save_fpu_regs();
2544 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2545 save_access_regs(vcpu->run->s.regs.acrs);
2546
2547 return kvm_s390_store_status_unloaded(vcpu, addr);
2548 }
2549
2550 /*
2551 * store additional status at address
2552 */
2553 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2554 unsigned long gpa)
2555 {
2556 /* Only bits 0-53 are used for address formation */
2557 if (!(gpa & ~0x3ff))
2558 return 0;
2559
2560 return write_guest_abs(vcpu, gpa & ~0x3ff,
2561 (void *)&vcpu->run->s.regs.vrs, 512);
2562 }
2563
2564 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2565 {
2566 if (!test_kvm_facility(vcpu->kvm, 129))
2567 return 0;
2568
2569 /*
2570 * The guest VXRS are in the host VXRs due to the lazy
2571 * copying in vcpu load/put. We can simply call save_fpu_regs()
2572 * to save the current register state because we are in the
2573 * middle of a load/put cycle.
2574 *
2575 * Let's update our copies before we save it into the save area.
2576 */
2577 save_fpu_regs();
2578
2579 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2580 }
2581
2582 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2583 {
2584 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2585 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2586 }
2587
2588 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2589 {
2590 unsigned int i;
2591 struct kvm_vcpu *vcpu;
2592
2593 kvm_for_each_vcpu(i, vcpu, kvm) {
2594 __disable_ibs_on_vcpu(vcpu);
2595 }
2596 }
2597
2598 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2599 {
2600 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2601 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2602 }
2603
2604 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2605 {
2606 int i, online_vcpus, started_vcpus = 0;
2607
2608 if (!is_vcpu_stopped(vcpu))
2609 return;
2610
2611 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2612 /* Only one cpu at a time may enter/leave the STOPPED state. */
2613 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2614 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2615
2616 for (i = 0; i < online_vcpus; i++) {
2617 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2618 started_vcpus++;
2619 }
2620
2621 if (started_vcpus == 0) {
2622 /* we're the only active VCPU -> speed it up */
2623 __enable_ibs_on_vcpu(vcpu);
2624 } else if (started_vcpus == 1) {
2625 /*
2626 * As we are starting a second VCPU, we have to disable
2627 * the IBS facility on all VCPUs to remove potentially
2628 * oustanding ENABLE requests.
2629 */
2630 __disable_ibs_on_all_vcpus(vcpu->kvm);
2631 }
2632
2633 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2634 /*
2635 * Another VCPU might have used IBS while we were offline.
2636 * Let's play safe and flush the VCPU at startup.
2637 */
2638 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2639 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2640 return;
2641 }
2642
2643 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2644 {
2645 int i, online_vcpus, started_vcpus = 0;
2646 struct kvm_vcpu *started_vcpu = NULL;
2647
2648 if (is_vcpu_stopped(vcpu))
2649 return;
2650
2651 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2652 /* Only one cpu at a time may enter/leave the STOPPED state. */
2653 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2654 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2655
2656 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2657 kvm_s390_clear_stop_irq(vcpu);
2658
2659 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2660 __disable_ibs_on_vcpu(vcpu);
2661
2662 for (i = 0; i < online_vcpus; i++) {
2663 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2664 started_vcpus++;
2665 started_vcpu = vcpu->kvm->vcpus[i];
2666 }
2667 }
2668
2669 if (started_vcpus == 1) {
2670 /*
2671 * As we only have one VCPU left, we want to enable the
2672 * IBS facility for that VCPU to speed it up.
2673 */
2674 __enable_ibs_on_vcpu(started_vcpu);
2675 }
2676
2677 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2678 return;
2679 }
2680
2681 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2682 struct kvm_enable_cap *cap)
2683 {
2684 int r;
2685
2686 if (cap->flags)
2687 return -EINVAL;
2688
2689 switch (cap->cap) {
2690 case KVM_CAP_S390_CSS_SUPPORT:
2691 if (!vcpu->kvm->arch.css_support) {
2692 vcpu->kvm->arch.css_support = 1;
2693 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2694 trace_kvm_s390_enable_css(vcpu->kvm);
2695 }
2696 r = 0;
2697 break;
2698 default:
2699 r = -EINVAL;
2700 break;
2701 }
2702 return r;
2703 }
2704
2705 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2706 struct kvm_s390_mem_op *mop)
2707 {
2708 void __user *uaddr = (void __user *)mop->buf;
2709 void *tmpbuf = NULL;
2710 int r, srcu_idx;
2711 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2712 | KVM_S390_MEMOP_F_CHECK_ONLY;
2713
2714 if (mop->flags & ~supported_flags)
2715 return -EINVAL;
2716
2717 if (mop->size > MEM_OP_MAX_SIZE)
2718 return -E2BIG;
2719
2720 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2721 tmpbuf = vmalloc(mop->size);
2722 if (!tmpbuf)
2723 return -ENOMEM;
2724 }
2725
2726 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2727
2728 switch (mop->op) {
2729 case KVM_S390_MEMOP_LOGICAL_READ:
2730 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2731 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2732 mop->size, GACC_FETCH);
2733 break;
2734 }
2735 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2736 if (r == 0) {
2737 if (copy_to_user(uaddr, tmpbuf, mop->size))
2738 r = -EFAULT;
2739 }
2740 break;
2741 case KVM_S390_MEMOP_LOGICAL_WRITE:
2742 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2743 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2744 mop->size, GACC_STORE);
2745 break;
2746 }
2747 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2748 r = -EFAULT;
2749 break;
2750 }
2751 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2752 break;
2753 default:
2754 r = -EINVAL;
2755 }
2756
2757 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2758
2759 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2760 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2761
2762 vfree(tmpbuf);
2763 return r;
2764 }
2765
2766 long kvm_arch_vcpu_ioctl(struct file *filp,
2767 unsigned int ioctl, unsigned long arg)
2768 {
2769 struct kvm_vcpu *vcpu = filp->private_data;
2770 void __user *argp = (void __user *)arg;
2771 int idx;
2772 long r;
2773
2774 switch (ioctl) {
2775 case KVM_S390_IRQ: {
2776 struct kvm_s390_irq s390irq;
2777
2778 r = -EFAULT;
2779 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2780 break;
2781 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2782 break;
2783 }
2784 case KVM_S390_INTERRUPT: {
2785 struct kvm_s390_interrupt s390int;
2786 struct kvm_s390_irq s390irq;
2787
2788 r = -EFAULT;
2789 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2790 break;
2791 if (s390int_to_s390irq(&s390int, &s390irq))
2792 return -EINVAL;
2793 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2794 break;
2795 }
2796 case KVM_S390_STORE_STATUS:
2797 idx = srcu_read_lock(&vcpu->kvm->srcu);
2798 r = kvm_s390_vcpu_store_status(vcpu, arg);
2799 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2800 break;
2801 case KVM_S390_SET_INITIAL_PSW: {
2802 psw_t psw;
2803
2804 r = -EFAULT;
2805 if (copy_from_user(&psw, argp, sizeof(psw)))
2806 break;
2807 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2808 break;
2809 }
2810 case KVM_S390_INITIAL_RESET:
2811 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2812 break;
2813 case KVM_SET_ONE_REG:
2814 case KVM_GET_ONE_REG: {
2815 struct kvm_one_reg reg;
2816 r = -EFAULT;
2817 if (copy_from_user(&reg, argp, sizeof(reg)))
2818 break;
2819 if (ioctl == KVM_SET_ONE_REG)
2820 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2821 else
2822 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2823 break;
2824 }
2825 #ifdef CONFIG_KVM_S390_UCONTROL
2826 case KVM_S390_UCAS_MAP: {
2827 struct kvm_s390_ucas_mapping ucasmap;
2828
2829 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2830 r = -EFAULT;
2831 break;
2832 }
2833
2834 if (!kvm_is_ucontrol(vcpu->kvm)) {
2835 r = -EINVAL;
2836 break;
2837 }
2838
2839 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2840 ucasmap.vcpu_addr, ucasmap.length);
2841 break;
2842 }
2843 case KVM_S390_UCAS_UNMAP: {
2844 struct kvm_s390_ucas_mapping ucasmap;
2845
2846 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2847 r = -EFAULT;
2848 break;
2849 }
2850
2851 if (!kvm_is_ucontrol(vcpu->kvm)) {
2852 r = -EINVAL;
2853 break;
2854 }
2855
2856 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2857 ucasmap.length);
2858 break;
2859 }
2860 #endif
2861 case KVM_S390_VCPU_FAULT: {
2862 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2863 break;
2864 }
2865 case KVM_ENABLE_CAP:
2866 {
2867 struct kvm_enable_cap cap;
2868 r = -EFAULT;
2869 if (copy_from_user(&cap, argp, sizeof(cap)))
2870 break;
2871 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2872 break;
2873 }
2874 case KVM_S390_MEM_OP: {
2875 struct kvm_s390_mem_op mem_op;
2876
2877 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2878 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2879 else
2880 r = -EFAULT;
2881 break;
2882 }
2883 case KVM_S390_SET_IRQ_STATE: {
2884 struct kvm_s390_irq_state irq_state;
2885
2886 r = -EFAULT;
2887 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2888 break;
2889 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2890 irq_state.len == 0 ||
2891 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2892 r = -EINVAL;
2893 break;
2894 }
2895 r = kvm_s390_set_irq_state(vcpu,
2896 (void __user *) irq_state.buf,
2897 irq_state.len);
2898 break;
2899 }
2900 case KVM_S390_GET_IRQ_STATE: {
2901 struct kvm_s390_irq_state irq_state;
2902
2903 r = -EFAULT;
2904 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2905 break;
2906 if (irq_state.len == 0) {
2907 r = -EINVAL;
2908 break;
2909 }
2910 r = kvm_s390_get_irq_state(vcpu,
2911 (__u8 __user *) irq_state.buf,
2912 irq_state.len);
2913 break;
2914 }
2915 default:
2916 r = -ENOTTY;
2917 }
2918 return r;
2919 }
2920
2921 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2922 {
2923 #ifdef CONFIG_KVM_S390_UCONTROL
2924 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2925 && (kvm_is_ucontrol(vcpu->kvm))) {
2926 vmf->page = virt_to_page(vcpu->arch.sie_block);
2927 get_page(vmf->page);
2928 return 0;
2929 }
2930 #endif
2931 return VM_FAULT_SIGBUS;
2932 }
2933
2934 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2935 unsigned long npages)
2936 {
2937 return 0;
2938 }
2939
2940 /* Section: memory related */
2941 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2942 struct kvm_memory_slot *memslot,
2943 const struct kvm_userspace_memory_region *mem,
2944 enum kvm_mr_change change)
2945 {
2946 /* A few sanity checks. We can have memory slots which have to be
2947 located/ended at a segment boundary (1MB). The memory in userland is
2948 ok to be fragmented into various different vmas. It is okay to mmap()
2949 and munmap() stuff in this slot after doing this call at any time */
2950
2951 if (mem->userspace_addr & 0xffffful)
2952 return -EINVAL;
2953
2954 if (mem->memory_size & 0xffffful)
2955 return -EINVAL;
2956
2957 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
2958 return -EINVAL;
2959
2960 return 0;
2961 }
2962
2963 void kvm_arch_commit_memory_region(struct kvm *kvm,
2964 const struct kvm_userspace_memory_region *mem,
2965 const struct kvm_memory_slot *old,
2966 const struct kvm_memory_slot *new,
2967 enum kvm_mr_change change)
2968 {
2969 int rc;
2970
2971 /* If the basics of the memslot do not change, we do not want
2972 * to update the gmap. Every update causes several unnecessary
2973 * segment translation exceptions. This is usually handled just
2974 * fine by the normal fault handler + gmap, but it will also
2975 * cause faults on the prefix page of running guest CPUs.
2976 */
2977 if (old->userspace_addr == mem->userspace_addr &&
2978 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2979 old->npages * PAGE_SIZE == mem->memory_size)
2980 return;
2981
2982 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2983 mem->guest_phys_addr, mem->memory_size);
2984 if (rc)
2985 pr_warn("failed to commit memory region\n");
2986 return;
2987 }
2988
2989 static inline unsigned long nonhyp_mask(int i)
2990 {
2991 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
2992
2993 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
2994 }
2995
2996 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
2997 {
2998 vcpu->valid_wakeup = false;
2999 }
3000
3001 static int __init kvm_s390_init(void)
3002 {
3003 int i;
3004
3005 if (!sclp.has_sief2) {
3006 pr_info("SIE not available\n");
3007 return -ENODEV;
3008 }
3009
3010 for (i = 0; i < 16; i++)
3011 kvm_s390_fac_list_mask[i] |=
3012 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3013
3014 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3015 }
3016
3017 static void __exit kvm_s390_exit(void)
3018 {
3019 kvm_exit();
3020 }
3021
3022 module_init(kvm_s390_init);
3023 module_exit(kvm_s390_exit);
3024
3025 /*
3026 * Enable autoloading of the kvm module.
3027 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3028 * since x86 takes a different approach.
3029 */
3030 #include <linux/miscdevice.h>
3031 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3032 MODULE_ALIAS("devname:kvm");
Linux kernel - Deliverables
This page took 0.139894 seconds and 6 git commands to generate.