projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
KVM: s390: factor out reading of the guest TOD clock
[deliverable/linux.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
31 #include <asm/etr.h>
32 #include <asm/pgtable.h>
33 #include <asm/nmi.h>
34 #include <asm/switch_to.h>
35 #include <asm/isc.h>
36 #include <asm/sclp.h>
37 #include "kvm-s390.h"
38 #include "gaccess.h"
39
40 #define KMSG_COMPONENT "kvm-s390"
41 #undef pr_fmt
42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
43
44 #define CREATE_TRACE_POINTS
45 #include "trace.h"
46 #include "trace-s390.h"
47
48 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
49 #define LOCAL_IRQS 32
50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
51 (KVM_MAX_VCPUS + LOCAL_IRQS))
52
53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
54
55 struct kvm_stats_debugfs_item debugfs_entries[] = {
56 { "userspace_handled", VCPU_STAT(exit_userspace) },
57 { "exit_null", VCPU_STAT(exit_null) },
58 { "exit_validity", VCPU_STAT(exit_validity) },
59 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
60 { "exit_external_request", VCPU_STAT(exit_external_request) },
61 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
62 { "exit_instruction", VCPU_STAT(exit_instruction) },
63 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
64 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
65 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
66 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
67 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
68 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
69 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
70 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
71 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
72 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
73 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
74 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
75 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
76 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
77 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
78 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
79 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
80 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
81 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
82 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
83 { "instruction_spx", VCPU_STAT(instruction_spx) },
84 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
85 { "instruction_stap", VCPU_STAT(instruction_stap) },
86 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
87 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
88 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
89 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
90 { "instruction_essa", VCPU_STAT(instruction_essa) },
91 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
92 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
93 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
94 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
95 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
96 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
97 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
98 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
99 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
100 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
101 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
102 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
103 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
104 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
105 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
106 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
107 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
108 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
109 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
110 { "diagnose_10", VCPU_STAT(diagnose_10) },
111 { "diagnose_44", VCPU_STAT(diagnose_44) },
112 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
113 { "diagnose_258", VCPU_STAT(diagnose_258) },
114 { "diagnose_308", VCPU_STAT(diagnose_308) },
115 { "diagnose_500", VCPU_STAT(diagnose_500) },
116 { NULL }
117 };
118
119 /* upper facilities limit for kvm */
120 unsigned long kvm_s390_fac_list_mask[] = {
121 0xffe6fffbfcfdfc40UL,
122 0x005e800000000000UL,
123 };
124
125 unsigned long kvm_s390_fac_list_mask_size(void)
126 {
127 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
128 return ARRAY_SIZE(kvm_s390_fac_list_mask);
129 }
130
131 static struct gmap_notifier gmap_notifier;
132 debug_info_t *kvm_s390_dbf;
133
134 /* Section: not file related */
135 int kvm_arch_hardware_enable(void)
136 {
137 /* every s390 is virtualization enabled ;-) */
138 return 0;
139 }
140
141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
142
143 /*
144 * This callback is executed during stop_machine(). All CPUs are therefore
145 * temporarily stopped. In order not to change guest behavior, we have to
146 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
147 * so a CPU won't be stopped while calculating with the epoch.
148 */
149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
150 void *v)
151 {
152 struct kvm *kvm;
153 struct kvm_vcpu *vcpu;
154 int i;
155 unsigned long long *delta = v;
156
157 list_for_each_entry(kvm, &vm_list, vm_list) {
158 kvm->arch.epoch -= *delta;
159 kvm_for_each_vcpu(i, vcpu, kvm) {
160 vcpu->arch.sie_block->epoch -= *delta;
161 }
162 }
163 return NOTIFY_OK;
164 }
165
166 static struct notifier_block kvm_clock_notifier = {
167 .notifier_call = kvm_clock_sync,
168 };
169
170 int kvm_arch_hardware_setup(void)
171 {
172 gmap_notifier.notifier_call = kvm_gmap_notifier;
173 gmap_register_ipte_notifier(&gmap_notifier);
174 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
175 &kvm_clock_notifier);
176 return 0;
177 }
178
179 void kvm_arch_hardware_unsetup(void)
180 {
181 gmap_unregister_ipte_notifier(&gmap_notifier);
182 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
183 &kvm_clock_notifier);
184 }
185
186 int kvm_arch_init(void *opaque)
187 {
188 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
189 if (!kvm_s390_dbf)
190 return -ENOMEM;
191
192 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
193 debug_unregister(kvm_s390_dbf);
194 return -ENOMEM;
195 }
196
197 /* Register floating interrupt controller interface. */
198 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
199 }
200
201 void kvm_arch_exit(void)
202 {
203 debug_unregister(kvm_s390_dbf);
204 }
205
206 /* Section: device related */
207 long kvm_arch_dev_ioctl(struct file *filp,
208 unsigned int ioctl, unsigned long arg)
209 {
210 if (ioctl == KVM_S390_ENABLE_SIE)
211 return s390_enable_sie();
212 return -EINVAL;
213 }
214
215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
216 {
217 int r;
218
219 switch (ext) {
220 case KVM_CAP_S390_PSW:
221 case KVM_CAP_S390_GMAP:
222 case KVM_CAP_SYNC_MMU:
223 #ifdef CONFIG_KVM_S390_UCONTROL
224 case KVM_CAP_S390_UCONTROL:
225 #endif
226 case KVM_CAP_ASYNC_PF:
227 case KVM_CAP_SYNC_REGS:
228 case KVM_CAP_ONE_REG:
229 case KVM_CAP_ENABLE_CAP:
230 case KVM_CAP_S390_CSS_SUPPORT:
231 case KVM_CAP_IOEVENTFD:
232 case KVM_CAP_DEVICE_CTRL:
233 case KVM_CAP_ENABLE_CAP_VM:
234 case KVM_CAP_S390_IRQCHIP:
235 case KVM_CAP_VM_ATTRIBUTES:
236 case KVM_CAP_MP_STATE:
237 case KVM_CAP_S390_INJECT_IRQ:
238 case KVM_CAP_S390_USER_SIGP:
239 case KVM_CAP_S390_USER_STSI:
240 case KVM_CAP_S390_SKEYS:
241 case KVM_CAP_S390_IRQ_STATE:
242 r = 1;
243 break;
244 case KVM_CAP_S390_MEM_OP:
245 r = MEM_OP_MAX_SIZE;
246 break;
247 case KVM_CAP_NR_VCPUS:
248 case KVM_CAP_MAX_VCPUS:
249 r = KVM_MAX_VCPUS;
250 break;
251 case KVM_CAP_NR_MEMSLOTS:
252 r = KVM_USER_MEM_SLOTS;
253 break;
254 case KVM_CAP_S390_COW:
255 r = MACHINE_HAS_ESOP;
256 break;
257 case KVM_CAP_S390_VECTOR_REGISTERS:
258 r = MACHINE_HAS_VX;
259 break;
260 default:
261 r = 0;
262 }
263 return r;
264 }
265
266 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
267 struct kvm_memory_slot *memslot)
268 {
269 gfn_t cur_gfn, last_gfn;
270 unsigned long address;
271 struct gmap *gmap = kvm->arch.gmap;
272
273 down_read(&gmap->mm->mmap_sem);
274 /* Loop over all guest pages */
275 last_gfn = memslot->base_gfn + memslot->npages;
276 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
277 address = gfn_to_hva_memslot(memslot, cur_gfn);
278
279 if (gmap_test_and_clear_dirty(address, gmap))
280 mark_page_dirty(kvm, cur_gfn);
281 }
282 up_read(&gmap->mm->mmap_sem);
283 }
284
285 /* Section: vm related */
286 /*
287 * Get (and clear) the dirty memory log for a memory slot.
288 */
289 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
290 struct kvm_dirty_log *log)
291 {
292 int r;
293 unsigned long n;
294 struct kvm_memslots *slots;
295 struct kvm_memory_slot *memslot;
296 int is_dirty = 0;
297
298 mutex_lock(&kvm->slots_lock);
299
300 r = -EINVAL;
301 if (log->slot >= KVM_USER_MEM_SLOTS)
302 goto out;
303
304 slots = kvm_memslots(kvm);
305 memslot = id_to_memslot(slots, log->slot);
306 r = -ENOENT;
307 if (!memslot->dirty_bitmap)
308 goto out;
309
310 kvm_s390_sync_dirty_log(kvm, memslot);
311 r = kvm_get_dirty_log(kvm, log, &is_dirty);
312 if (r)
313 goto out;
314
315 /* Clear the dirty log */
316 if (is_dirty) {
317 n = kvm_dirty_bitmap_bytes(memslot);
318 memset(memslot->dirty_bitmap, 0, n);
319 }
320 r = 0;
321 out:
322 mutex_unlock(&kvm->slots_lock);
323 return r;
324 }
325
326 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
327 {
328 int r;
329
330 if (cap->flags)
331 return -EINVAL;
332
333 switch (cap->cap) {
334 case KVM_CAP_S390_IRQCHIP:
335 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
336 kvm->arch.use_irqchip = 1;
337 r = 0;
338 break;
339 case KVM_CAP_S390_USER_SIGP:
340 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
341 kvm->arch.user_sigp = 1;
342 r = 0;
343 break;
344 case KVM_CAP_S390_VECTOR_REGISTERS:
345 if (MACHINE_HAS_VX) {
346 set_kvm_facility(kvm->arch.model.fac->mask, 129);
347 set_kvm_facility(kvm->arch.model.fac->list, 129);
348 r = 0;
349 } else
350 r = -EINVAL;
351 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
352 r ? "(not available)" : "(success)");
353 break;
354 case KVM_CAP_S390_USER_STSI:
355 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
356 kvm->arch.user_stsi = 1;
357 r = 0;
358 break;
359 default:
360 r = -EINVAL;
361 break;
362 }
363 return r;
364 }
365
366 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
367 {
368 int ret;
369
370 switch (attr->attr) {
371 case KVM_S390_VM_MEM_LIMIT_SIZE:
372 ret = 0;
373 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
374 kvm->arch.gmap->asce_end);
375 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
376 ret = -EFAULT;
377 break;
378 default:
379 ret = -ENXIO;
380 break;
381 }
382 return ret;
383 }
384
385 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
386 {
387 int ret;
388 unsigned int idx;
389 switch (attr->attr) {
390 case KVM_S390_VM_MEM_ENABLE_CMMA:
391 /* enable CMMA only for z10 and later (EDAT_1) */
392 ret = -EINVAL;
393 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
394 break;
395
396 ret = -EBUSY;
397 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
398 mutex_lock(&kvm->lock);
399 if (atomic_read(&kvm->online_vcpus) == 0) {
400 kvm->arch.use_cmma = 1;
401 ret = 0;
402 }
403 mutex_unlock(&kvm->lock);
404 break;
405 case KVM_S390_VM_MEM_CLR_CMMA:
406 ret = -EINVAL;
407 if (!kvm->arch.use_cmma)
408 break;
409
410 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
411 mutex_lock(&kvm->lock);
412 idx = srcu_read_lock(&kvm->srcu);
413 s390_reset_cmma(kvm->arch.gmap->mm);
414 srcu_read_unlock(&kvm->srcu, idx);
415 mutex_unlock(&kvm->lock);
416 ret = 0;
417 break;
418 case KVM_S390_VM_MEM_LIMIT_SIZE: {
419 unsigned long new_limit;
420
421 if (kvm_is_ucontrol(kvm))
422 return -EINVAL;
423
424 if (get_user(new_limit, (u64 __user *)attr->addr))
425 return -EFAULT;
426
427 if (new_limit > kvm->arch.gmap->asce_end)
428 return -E2BIG;
429
430 ret = -EBUSY;
431 mutex_lock(&kvm->lock);
432 if (atomic_read(&kvm->online_vcpus) == 0) {
433 /* gmap_alloc will round the limit up */
434 struct gmap *new = gmap_alloc(current->mm, new_limit);
435
436 if (!new) {
437 ret = -ENOMEM;
438 } else {
439 gmap_free(kvm->arch.gmap);
440 new->private = kvm;
441 kvm->arch.gmap = new;
442 ret = 0;
443 }
444 }
445 mutex_unlock(&kvm->lock);
446 VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
447 break;
448 }
449 default:
450 ret = -ENXIO;
451 break;
452 }
453 return ret;
454 }
455
456 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
457
458 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
459 {
460 struct kvm_vcpu *vcpu;
461 int i;
462
463 if (!test_kvm_facility(kvm, 76))
464 return -EINVAL;
465
466 mutex_lock(&kvm->lock);
467 switch (attr->attr) {
468 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
469 get_random_bytes(
470 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
471 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
472 kvm->arch.crypto.aes_kw = 1;
473 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
474 break;
475 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
476 get_random_bytes(
477 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
478 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
479 kvm->arch.crypto.dea_kw = 1;
480 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
481 break;
482 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
483 kvm->arch.crypto.aes_kw = 0;
484 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
485 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
486 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
487 break;
488 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
489 kvm->arch.crypto.dea_kw = 0;
490 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
491 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
492 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
493 break;
494 default:
495 mutex_unlock(&kvm->lock);
496 return -ENXIO;
497 }
498
499 kvm_for_each_vcpu(i, vcpu, kvm) {
500 kvm_s390_vcpu_crypto_setup(vcpu);
501 exit_sie(vcpu);
502 }
503 mutex_unlock(&kvm->lock);
504 return 0;
505 }
506
507 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
508 {
509 u8 gtod_high;
510
511 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
512 sizeof(gtod_high)))
513 return -EFAULT;
514
515 if (gtod_high != 0)
516 return -EINVAL;
517 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x\n", gtod_high);
518
519 return 0;
520 }
521
522 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
523 {
524 u64 gtod;
525
526 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
527 return -EFAULT;
528
529 kvm_s390_set_tod_clock(kvm, gtod);
530 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx\n", gtod);
531 return 0;
532 }
533
534 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
535 {
536 int ret;
537
538 if (attr->flags)
539 return -EINVAL;
540
541 switch (attr->attr) {
542 case KVM_S390_VM_TOD_HIGH:
543 ret = kvm_s390_set_tod_high(kvm, attr);
544 break;
545 case KVM_S390_VM_TOD_LOW:
546 ret = kvm_s390_set_tod_low(kvm, attr);
547 break;
548 default:
549 ret = -ENXIO;
550 break;
551 }
552 return ret;
553 }
554
555 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
556 {
557 u8 gtod_high = 0;
558
559 if (copy_to_user((void __user *)attr->addr, &gtod_high,
560 sizeof(gtod_high)))
561 return -EFAULT;
562 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x\n", gtod_high);
563
564 return 0;
565 }
566
567 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
568 {
569 u64 gtod;
570
571 gtod = kvm_s390_get_tod_clock_fast(kvm);
572 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
573 return -EFAULT;
574 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx\n", gtod);
575
576 return 0;
577 }
578
579 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
580 {
581 int ret;
582
583 if (attr->flags)
584 return -EINVAL;
585
586 switch (attr->attr) {
587 case KVM_S390_VM_TOD_HIGH:
588 ret = kvm_s390_get_tod_high(kvm, attr);
589 break;
590 case KVM_S390_VM_TOD_LOW:
591 ret = kvm_s390_get_tod_low(kvm, attr);
592 break;
593 default:
594 ret = -ENXIO;
595 break;
596 }
597 return ret;
598 }
599
600 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
601 {
602 struct kvm_s390_vm_cpu_processor *proc;
603 int ret = 0;
604
605 mutex_lock(&kvm->lock);
606 if (atomic_read(&kvm->online_vcpus)) {
607 ret = -EBUSY;
608 goto out;
609 }
610 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
611 if (!proc) {
612 ret = -ENOMEM;
613 goto out;
614 }
615 if (!copy_from_user(proc, (void __user *)attr->addr,
616 sizeof(*proc))) {
617 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
618 sizeof(struct cpuid));
619 kvm->arch.model.ibc = proc->ibc;
620 memcpy(kvm->arch.model.fac->list, proc->fac_list,
621 S390_ARCH_FAC_LIST_SIZE_BYTE);
622 } else
623 ret = -EFAULT;
624 kfree(proc);
625 out:
626 mutex_unlock(&kvm->lock);
627 return ret;
628 }
629
630 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
631 {
632 int ret = -ENXIO;
633
634 switch (attr->attr) {
635 case KVM_S390_VM_CPU_PROCESSOR:
636 ret = kvm_s390_set_processor(kvm, attr);
637 break;
638 }
639 return ret;
640 }
641
642 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
643 {
644 struct kvm_s390_vm_cpu_processor *proc;
645 int ret = 0;
646
647 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
648 if (!proc) {
649 ret = -ENOMEM;
650 goto out;
651 }
652 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
653 proc->ibc = kvm->arch.model.ibc;
654 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
655 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
656 ret = -EFAULT;
657 kfree(proc);
658 out:
659 return ret;
660 }
661
662 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
663 {
664 struct kvm_s390_vm_cpu_machine *mach;
665 int ret = 0;
666
667 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
668 if (!mach) {
669 ret = -ENOMEM;
670 goto out;
671 }
672 get_cpu_id((struct cpuid *) &mach->cpuid);
673 mach->ibc = sclp.ibc;
674 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
675 S390_ARCH_FAC_LIST_SIZE_BYTE);
676 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
677 S390_ARCH_FAC_LIST_SIZE_BYTE);
678 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
679 ret = -EFAULT;
680 kfree(mach);
681 out:
682 return ret;
683 }
684
685 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
686 {
687 int ret = -ENXIO;
688
689 switch (attr->attr) {
690 case KVM_S390_VM_CPU_PROCESSOR:
691 ret = kvm_s390_get_processor(kvm, attr);
692 break;
693 case KVM_S390_VM_CPU_MACHINE:
694 ret = kvm_s390_get_machine(kvm, attr);
695 break;
696 }
697 return ret;
698 }
699
700 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
701 {
702 int ret;
703
704 switch (attr->group) {
705 case KVM_S390_VM_MEM_CTRL:
706 ret = kvm_s390_set_mem_control(kvm, attr);
707 break;
708 case KVM_S390_VM_TOD:
709 ret = kvm_s390_set_tod(kvm, attr);
710 break;
711 case KVM_S390_VM_CPU_MODEL:
712 ret = kvm_s390_set_cpu_model(kvm, attr);
713 break;
714 case KVM_S390_VM_CRYPTO:
715 ret = kvm_s390_vm_set_crypto(kvm, attr);
716 break;
717 default:
718 ret = -ENXIO;
719 break;
720 }
721
722 return ret;
723 }
724
725 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
726 {
727 int ret;
728
729 switch (attr->group) {
730 case KVM_S390_VM_MEM_CTRL:
731 ret = kvm_s390_get_mem_control(kvm, attr);
732 break;
733 case KVM_S390_VM_TOD:
734 ret = kvm_s390_get_tod(kvm, attr);
735 break;
736 case KVM_S390_VM_CPU_MODEL:
737 ret = kvm_s390_get_cpu_model(kvm, attr);
738 break;
739 default:
740 ret = -ENXIO;
741 break;
742 }
743
744 return ret;
745 }
746
747 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
748 {
749 int ret;
750
751 switch (attr->group) {
752 case KVM_S390_VM_MEM_CTRL:
753 switch (attr->attr) {
754 case KVM_S390_VM_MEM_ENABLE_CMMA:
755 case KVM_S390_VM_MEM_CLR_CMMA:
756 case KVM_S390_VM_MEM_LIMIT_SIZE:
757 ret = 0;
758 break;
759 default:
760 ret = -ENXIO;
761 break;
762 }
763 break;
764 case KVM_S390_VM_TOD:
765 switch (attr->attr) {
766 case KVM_S390_VM_TOD_LOW:
767 case KVM_S390_VM_TOD_HIGH:
768 ret = 0;
769 break;
770 default:
771 ret = -ENXIO;
772 break;
773 }
774 break;
775 case KVM_S390_VM_CPU_MODEL:
776 switch (attr->attr) {
777 case KVM_S390_VM_CPU_PROCESSOR:
778 case KVM_S390_VM_CPU_MACHINE:
779 ret = 0;
780 break;
781 default:
782 ret = -ENXIO;
783 break;
784 }
785 break;
786 case KVM_S390_VM_CRYPTO:
787 switch (attr->attr) {
788 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
789 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
790 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
791 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
792 ret = 0;
793 break;
794 default:
795 ret = -ENXIO;
796 break;
797 }
798 break;
799 default:
800 ret = -ENXIO;
801 break;
802 }
803
804 return ret;
805 }
806
807 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
808 {
809 uint8_t *keys;
810 uint64_t hva;
811 unsigned long curkey;
812 int i, r = 0;
813
814 if (args->flags != 0)
815 return -EINVAL;
816
817 /* Is this guest using storage keys? */
818 if (!mm_use_skey(current->mm))
819 return KVM_S390_GET_SKEYS_NONE;
820
821 /* Enforce sane limit on memory allocation */
822 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
823 return -EINVAL;
824
825 keys = kmalloc_array(args->count, sizeof(uint8_t),
826 GFP_KERNEL | __GFP_NOWARN);
827 if (!keys)
828 keys = vmalloc(sizeof(uint8_t) * args->count);
829 if (!keys)
830 return -ENOMEM;
831
832 for (i = 0; i < args->count; i++) {
833 hva = gfn_to_hva(kvm, args->start_gfn + i);
834 if (kvm_is_error_hva(hva)) {
835 r = -EFAULT;
836 goto out;
837 }
838
839 curkey = get_guest_storage_key(current->mm, hva);
840 if (IS_ERR_VALUE(curkey)) {
841 r = curkey;
842 goto out;
843 }
844 keys[i] = curkey;
845 }
846
847 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
848 sizeof(uint8_t) * args->count);
849 if (r)
850 r = -EFAULT;
851 out:
852 kvfree(keys);
853 return r;
854 }
855
856 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
857 {
858 uint8_t *keys;
859 uint64_t hva;
860 int i, r = 0;
861
862 if (args->flags != 0)
863 return -EINVAL;
864
865 /* Enforce sane limit on memory allocation */
866 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
867 return -EINVAL;
868
869 keys = kmalloc_array(args->count, sizeof(uint8_t),
870 GFP_KERNEL | __GFP_NOWARN);
871 if (!keys)
872 keys = vmalloc(sizeof(uint8_t) * args->count);
873 if (!keys)
874 return -ENOMEM;
875
876 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
877 sizeof(uint8_t) * args->count);
878 if (r) {
879 r = -EFAULT;
880 goto out;
881 }
882
883 /* Enable storage key handling for the guest */
884 r = s390_enable_skey();
885 if (r)
886 goto out;
887
888 for (i = 0; i < args->count; i++) {
889 hva = gfn_to_hva(kvm, args->start_gfn + i);
890 if (kvm_is_error_hva(hva)) {
891 r = -EFAULT;
892 goto out;
893 }
894
895 /* Lowest order bit is reserved */
896 if (keys[i] & 0x01) {
897 r = -EINVAL;
898 goto out;
899 }
900
901 r = set_guest_storage_key(current->mm, hva,
902 (unsigned long)keys[i], 0);
903 if (r)
904 goto out;
905 }
906 out:
907 kvfree(keys);
908 return r;
909 }
910
911 long kvm_arch_vm_ioctl(struct file *filp,
912 unsigned int ioctl, unsigned long arg)
913 {
914 struct kvm *kvm = filp->private_data;
915 void __user *argp = (void __user *)arg;
916 struct kvm_device_attr attr;
917 int r;
918
919 switch (ioctl) {
920 case KVM_S390_INTERRUPT: {
921 struct kvm_s390_interrupt s390int;
922
923 r = -EFAULT;
924 if (copy_from_user(&s390int, argp, sizeof(s390int)))
925 break;
926 r = kvm_s390_inject_vm(kvm, &s390int);
927 break;
928 }
929 case KVM_ENABLE_CAP: {
930 struct kvm_enable_cap cap;
931 r = -EFAULT;
932 if (copy_from_user(&cap, argp, sizeof(cap)))
933 break;
934 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
935 break;
936 }
937 case KVM_CREATE_IRQCHIP: {
938 struct kvm_irq_routing_entry routing;
939
940 r = -EINVAL;
941 if (kvm->arch.use_irqchip) {
942 /* Set up dummy routing. */
943 memset(&routing, 0, sizeof(routing));
944 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
945 }
946 break;
947 }
948 case KVM_SET_DEVICE_ATTR: {
949 r = -EFAULT;
950 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
951 break;
952 r = kvm_s390_vm_set_attr(kvm, &attr);
953 break;
954 }
955 case KVM_GET_DEVICE_ATTR: {
956 r = -EFAULT;
957 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
958 break;
959 r = kvm_s390_vm_get_attr(kvm, &attr);
960 break;
961 }
962 case KVM_HAS_DEVICE_ATTR: {
963 r = -EFAULT;
964 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
965 break;
966 r = kvm_s390_vm_has_attr(kvm, &attr);
967 break;
968 }
969 case KVM_S390_GET_SKEYS: {
970 struct kvm_s390_skeys args;
971
972 r = -EFAULT;
973 if (copy_from_user(&args, argp,
974 sizeof(struct kvm_s390_skeys)))
975 break;
976 r = kvm_s390_get_skeys(kvm, &args);
977 break;
978 }
979 case KVM_S390_SET_SKEYS: {
980 struct kvm_s390_skeys args;
981
982 r = -EFAULT;
983 if (copy_from_user(&args, argp,
984 sizeof(struct kvm_s390_skeys)))
985 break;
986 r = kvm_s390_set_skeys(kvm, &args);
987 break;
988 }
989 default:
990 r = -ENOTTY;
991 }
992
993 return r;
994 }
995
996 static int kvm_s390_query_ap_config(u8 *config)
997 {
998 u32 fcn_code = 0x04000000UL;
999 u32 cc = 0;
1000
1001 memset(config, 0, 128);
1002 asm volatile(
1003 "lgr 0,%1\n"
1004 "lgr 2,%2\n"
1005 ".long 0xb2af0000\n" /* PQAP(QCI) */
1006 "0: ipm %0\n"
1007 "srl %0,28\n"
1008 "1:\n"
1009 EX_TABLE(0b, 1b)
1010 : "+r" (cc)
1011 : "r" (fcn_code), "r" (config)
1012 : "cc", "0", "2", "memory"
1013 );
1014
1015 return cc;
1016 }
1017
1018 static int kvm_s390_apxa_installed(void)
1019 {
1020 u8 config[128];
1021 int cc;
1022
1023 if (test_facility(2) && test_facility(12)) {
1024 cc = kvm_s390_query_ap_config(config);
1025
1026 if (cc)
1027 pr_err("PQAP(QCI) failed with cc=%d", cc);
1028 else
1029 return config[0] & 0x40;
1030 }
1031
1032 return 0;
1033 }
1034
1035 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1036 {
1037 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1038
1039 if (kvm_s390_apxa_installed())
1040 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1041 else
1042 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1043 }
1044
1045 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1046 {
1047 get_cpu_id(cpu_id);
1048 cpu_id->version = 0xff;
1049 }
1050
1051 static int kvm_s390_crypto_init(struct kvm *kvm)
1052 {
1053 if (!test_kvm_facility(kvm, 76))
1054 return 0;
1055
1056 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1057 GFP_KERNEL | GFP_DMA);
1058 if (!kvm->arch.crypto.crycb)
1059 return -ENOMEM;
1060
1061 kvm_s390_set_crycb_format(kvm);
1062
1063 /* Enable AES/DEA protected key functions by default */
1064 kvm->arch.crypto.aes_kw = 1;
1065 kvm->arch.crypto.dea_kw = 1;
1066 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1067 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1068 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1069 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1070
1071 return 0;
1072 }
1073
1074 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1075 {
1076 int i, rc;
1077 char debug_name[16];
1078 static unsigned long sca_offset;
1079
1080 rc = -EINVAL;
1081 #ifdef CONFIG_KVM_S390_UCONTROL
1082 if (type & ~KVM_VM_S390_UCONTROL)
1083 goto out_err;
1084 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1085 goto out_err;
1086 #else
1087 if (type)
1088 goto out_err;
1089 #endif
1090
1091 rc = s390_enable_sie();
1092 if (rc)
1093 goto out_err;
1094
1095 rc = -ENOMEM;
1096
1097 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1098 if (!kvm->arch.sca)
1099 goto out_err;
1100 spin_lock(&kvm_lock);
1101 sca_offset = (sca_offset + 16) & 0x7f0;
1102 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1103 spin_unlock(&kvm_lock);
1104
1105 sprintf(debug_name, "kvm-%u", current->pid);
1106
1107 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1108 if (!kvm->arch.dbf)
1109 goto out_err;
1110
1111 /*
1112 * The architectural maximum amount of facilities is 16 kbit. To store
1113 * this amount, 2 kbyte of memory is required. Thus we need a full
1114 * page to hold the guest facility list (arch.model.fac->list) and the
1115 * facility mask (arch.model.fac->mask). Its address size has to be
1116 * 31 bits and word aligned.
1117 */
1118 kvm->arch.model.fac =
1119 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1120 if (!kvm->arch.model.fac)
1121 goto out_err;
1122
1123 /* Populate the facility mask initially. */
1124 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1125 S390_ARCH_FAC_LIST_SIZE_BYTE);
1126 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1127 if (i < kvm_s390_fac_list_mask_size())
1128 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1129 else
1130 kvm->arch.model.fac->mask[i] = 0UL;
1131 }
1132
1133 /* Populate the facility list initially. */
1134 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1135 S390_ARCH_FAC_LIST_SIZE_BYTE);
1136
1137 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1138 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1139
1140 if (kvm_s390_crypto_init(kvm) < 0)
1141 goto out_err;
1142
1143 spin_lock_init(&kvm->arch.float_int.lock);
1144 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1145 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1146 init_waitqueue_head(&kvm->arch.ipte_wq);
1147 mutex_init(&kvm->arch.ipte_mutex);
1148
1149 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1150 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1151
1152 if (type & KVM_VM_S390_UCONTROL) {
1153 kvm->arch.gmap = NULL;
1154 } else {
1155 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1156 if (!kvm->arch.gmap)
1157 goto out_err;
1158 kvm->arch.gmap->private = kvm;
1159 kvm->arch.gmap->pfault_enabled = 0;
1160 }
1161
1162 kvm->arch.css_support = 0;
1163 kvm->arch.use_irqchip = 0;
1164 kvm->arch.epoch = 0;
1165
1166 spin_lock_init(&kvm->arch.start_stop_lock);
1167 KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
1168
1169 return 0;
1170 out_err:
1171 kfree(kvm->arch.crypto.crycb);
1172 free_page((unsigned long)kvm->arch.model.fac);
1173 debug_unregister(kvm->arch.dbf);
1174 free_page((unsigned long)(kvm->arch.sca));
1175 KVM_EVENT(3, "creation of vm failed: %d", rc);
1176 return rc;
1177 }
1178
1179 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1180 {
1181 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1182 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1183 kvm_s390_clear_local_irqs(vcpu);
1184 kvm_clear_async_pf_completion_queue(vcpu);
1185 if (!kvm_is_ucontrol(vcpu->kvm)) {
1186 clear_bit(63 - vcpu->vcpu_id,
1187 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1188 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1189 (__u64) vcpu->arch.sie_block)
1190 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1191 }
1192 smp_mb();
1193
1194 if (kvm_is_ucontrol(vcpu->kvm))
1195 gmap_free(vcpu->arch.gmap);
1196
1197 if (vcpu->kvm->arch.use_cmma)
1198 kvm_s390_vcpu_unsetup_cmma(vcpu);
1199 free_page((unsigned long)(vcpu->arch.sie_block));
1200
1201 kvm_vcpu_uninit(vcpu);
1202 kmem_cache_free(kvm_vcpu_cache, vcpu);
1203 }
1204
1205 static void kvm_free_vcpus(struct kvm *kvm)
1206 {
1207 unsigned int i;
1208 struct kvm_vcpu *vcpu;
1209
1210 kvm_for_each_vcpu(i, vcpu, kvm)
1211 kvm_arch_vcpu_destroy(vcpu);
1212
1213 mutex_lock(&kvm->lock);
1214 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1215 kvm->vcpus[i] = NULL;
1216
1217 atomic_set(&kvm->online_vcpus, 0);
1218 mutex_unlock(&kvm->lock);
1219 }
1220
1221 void kvm_arch_destroy_vm(struct kvm *kvm)
1222 {
1223 kvm_free_vcpus(kvm);
1224 free_page((unsigned long)kvm->arch.model.fac);
1225 free_page((unsigned long)(kvm->arch.sca));
1226 debug_unregister(kvm->arch.dbf);
1227 kfree(kvm->arch.crypto.crycb);
1228 if (!kvm_is_ucontrol(kvm))
1229 gmap_free(kvm->arch.gmap);
1230 kvm_s390_destroy_adapters(kvm);
1231 kvm_s390_clear_float_irqs(kvm);
1232 KVM_EVENT(3, "vm 0x%p destroyed", kvm);
1233 }
1234
1235 /* Section: vcpu related */
1236 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1237 {
1238 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1239 if (!vcpu->arch.gmap)
1240 return -ENOMEM;
1241 vcpu->arch.gmap->private = vcpu->kvm;
1242
1243 return 0;
1244 }
1245
1246 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1247 {
1248 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1249 kvm_clear_async_pf_completion_queue(vcpu);
1250 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1251 KVM_SYNC_GPRS |
1252 KVM_SYNC_ACRS |
1253 KVM_SYNC_CRS |
1254 KVM_SYNC_ARCH0 |
1255 KVM_SYNC_PFAULT;
1256 if (test_kvm_facility(vcpu->kvm, 129))
1257 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1258
1259 if (kvm_is_ucontrol(vcpu->kvm))
1260 return __kvm_ucontrol_vcpu_init(vcpu);
1261
1262 return 0;
1263 }
1264
1265 /*
1266 * Backs up the current FP/VX register save area on a particular
1267 * destination. Used to switch between different register save
1268 * areas.
1269 */
1270 static inline void save_fpu_to(struct fpu *dst)
1271 {
1272 dst->fpc = current->thread.fpu.fpc;
1273 dst->flags = current->thread.fpu.flags;
1274 dst->regs = current->thread.fpu.regs;
1275 }
1276
1277 /*
1278 * Switches the FP/VX register save area from which to lazy
1279 * restore register contents.
1280 */
1281 static inline void load_fpu_from(struct fpu *from)
1282 {
1283 current->thread.fpu.fpc = from->fpc;
1284 current->thread.fpu.flags = from->flags;
1285 current->thread.fpu.regs = from->regs;
1286 }
1287
1288 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1289 {
1290 /* Save host register state */
1291 save_fpu_regs();
1292 save_fpu_to(&vcpu->arch.host_fpregs);
1293
1294 if (test_kvm_facility(vcpu->kvm, 129)) {
1295 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1296 current->thread.fpu.flags = FPU_USE_VX;
1297 /*
1298 * Use the register save area in the SIE-control block
1299 * for register restore and save in kvm_arch_vcpu_put()
1300 */
1301 current->thread.fpu.vxrs =
1302 (__vector128 *)&vcpu->run->s.regs.vrs;
1303 /* Always enable the vector extension for KVM */
1304 __ctl_set_vx();
1305 } else
1306 load_fpu_from(&vcpu->arch.guest_fpregs);
1307
1308 if (test_fp_ctl(current->thread.fpu.fpc))
1309 /* User space provided an invalid FPC, let's clear it */
1310 current->thread.fpu.fpc = 0;
1311
1312 save_access_regs(vcpu->arch.host_acrs);
1313 restore_access_regs(vcpu->run->s.regs.acrs);
1314 gmap_enable(vcpu->arch.gmap);
1315 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1316 }
1317
1318 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1319 {
1320 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1321 gmap_disable(vcpu->arch.gmap);
1322
1323 save_fpu_regs();
1324
1325 if (test_kvm_facility(vcpu->kvm, 129))
1326 /*
1327 * kvm_arch_vcpu_load() set up the register save area to
1328 * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1329 * are already saved. Only the floating-point control must be
1330 * copied.
1331 */
1332 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1333 else
1334 save_fpu_to(&vcpu->arch.guest_fpregs);
1335 load_fpu_from(&vcpu->arch.host_fpregs);
1336
1337 save_access_regs(vcpu->run->s.regs.acrs);
1338 restore_access_regs(vcpu->arch.host_acrs);
1339 }
1340
1341 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1342 {
1343 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1344 vcpu->arch.sie_block->gpsw.mask = 0UL;
1345 vcpu->arch.sie_block->gpsw.addr = 0UL;
1346 kvm_s390_set_prefix(vcpu, 0);
1347 vcpu->arch.sie_block->cputm = 0UL;
1348 vcpu->arch.sie_block->ckc = 0UL;
1349 vcpu->arch.sie_block->todpr = 0;
1350 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1351 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1352 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1353 vcpu->arch.guest_fpregs.fpc = 0;
1354 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1355 vcpu->arch.sie_block->gbea = 1;
1356 vcpu->arch.sie_block->pp = 0;
1357 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1358 kvm_clear_async_pf_completion_queue(vcpu);
1359 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1360 kvm_s390_vcpu_stop(vcpu);
1361 kvm_s390_clear_local_irqs(vcpu);
1362 }
1363
1364 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1365 {
1366 mutex_lock(&vcpu->kvm->lock);
1367 preempt_disable();
1368 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1369 preempt_enable();
1370 mutex_unlock(&vcpu->kvm->lock);
1371 if (!kvm_is_ucontrol(vcpu->kvm))
1372 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1373 }
1374
1375 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1376 {
1377 if (!test_kvm_facility(vcpu->kvm, 76))
1378 return;
1379
1380 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1381
1382 if (vcpu->kvm->arch.crypto.aes_kw)
1383 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1384 if (vcpu->kvm->arch.crypto.dea_kw)
1385 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1386
1387 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1388 }
1389
1390 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1391 {
1392 free_page(vcpu->arch.sie_block->cbrlo);
1393 vcpu->arch.sie_block->cbrlo = 0;
1394 }
1395
1396 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1397 {
1398 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1399 if (!vcpu->arch.sie_block->cbrlo)
1400 return -ENOMEM;
1401
1402 vcpu->arch.sie_block->ecb2 |= 0x80;
1403 vcpu->arch.sie_block->ecb2 &= ~0x08;
1404 return 0;
1405 }
1406
1407 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1408 {
1409 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1410
1411 vcpu->arch.cpu_id = model->cpu_id;
1412 vcpu->arch.sie_block->ibc = model->ibc;
1413 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1414 }
1415
1416 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1417 {
1418 int rc = 0;
1419
1420 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1421 CPUSTAT_SM |
1422 CPUSTAT_STOPPED);
1423
1424 if (test_kvm_facility(vcpu->kvm, 78))
1425 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1426 else if (test_kvm_facility(vcpu->kvm, 8))
1427 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1428
1429 kvm_s390_vcpu_setup_model(vcpu);
1430
1431 vcpu->arch.sie_block->ecb = 6;
1432 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1433 vcpu->arch.sie_block->ecb |= 0x10;
1434
1435 vcpu->arch.sie_block->ecb2 = 8;
1436 vcpu->arch.sie_block->eca = 0xC1002000U;
1437 if (sclp.has_siif)
1438 vcpu->arch.sie_block->eca |= 1;
1439 if (sclp.has_sigpif)
1440 vcpu->arch.sie_block->eca |= 0x10000000U;
1441 if (test_kvm_facility(vcpu->kvm, 129)) {
1442 vcpu->arch.sie_block->eca |= 0x00020000;
1443 vcpu->arch.sie_block->ecd |= 0x20000000;
1444 }
1445 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1446
1447 if (vcpu->kvm->arch.use_cmma) {
1448 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1449 if (rc)
1450 return rc;
1451 }
1452 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1453 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1454
1455 kvm_s390_vcpu_crypto_setup(vcpu);
1456
1457 return rc;
1458 }
1459
1460 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1461 unsigned int id)
1462 {
1463 struct kvm_vcpu *vcpu;
1464 struct sie_page *sie_page;
1465 int rc = -EINVAL;
1466
1467 if (id >= KVM_MAX_VCPUS)
1468 goto out;
1469
1470 rc = -ENOMEM;
1471
1472 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1473 if (!vcpu)
1474 goto out;
1475
1476 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1477 if (!sie_page)
1478 goto out_free_cpu;
1479
1480 vcpu->arch.sie_block = &sie_page->sie_block;
1481 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1482
1483 vcpu->arch.sie_block->icpua = id;
1484 if (!kvm_is_ucontrol(kvm)) {
1485 if (!kvm->arch.sca) {
1486 WARN_ON_ONCE(1);
1487 goto out_free_cpu;
1488 }
1489 if (!kvm->arch.sca->cpu[id].sda)
1490 kvm->arch.sca->cpu[id].sda =
1491 (__u64) vcpu->arch.sie_block;
1492 vcpu->arch.sie_block->scaoh =
1493 (__u32)(((__u64)kvm->arch.sca) >> 32);
1494 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1495 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1496 }
1497
1498 spin_lock_init(&vcpu->arch.local_int.lock);
1499 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1500 vcpu->arch.local_int.wq = &vcpu->wq;
1501 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1502
1503 /*
1504 * Allocate a save area for floating-point registers. If the vector
1505 * extension is available, register contents are saved in the SIE
1506 * control block. The allocated save area is still required in
1507 * particular places, for example, in kvm_s390_vcpu_store_status().
1508 */
1509 vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1510 GFP_KERNEL);
1511 if (!vcpu->arch.guest_fpregs.fprs) {
1512 rc = -ENOMEM;
1513 goto out_free_sie_block;
1514 }
1515
1516 rc = kvm_vcpu_init(vcpu, kvm, id);
1517 if (rc)
1518 goto out_free_sie_block;
1519 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1520 vcpu->arch.sie_block);
1521 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1522
1523 return vcpu;
1524 out_free_sie_block:
1525 free_page((unsigned long)(vcpu->arch.sie_block));
1526 out_free_cpu:
1527 kmem_cache_free(kvm_vcpu_cache, vcpu);
1528 out:
1529 return ERR_PTR(rc);
1530 }
1531
1532 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1533 {
1534 return kvm_s390_vcpu_has_irq(vcpu, 0);
1535 }
1536
1537 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1538 {
1539 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1540 exit_sie(vcpu);
1541 }
1542
1543 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1544 {
1545 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1546 }
1547
1548 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1549 {
1550 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1551 exit_sie(vcpu);
1552 }
1553
1554 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1555 {
1556 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1557 }
1558
1559 /*
1560 * Kick a guest cpu out of SIE and wait until SIE is not running.
1561 * If the CPU is not running (e.g. waiting as idle) the function will
1562 * return immediately. */
1563 void exit_sie(struct kvm_vcpu *vcpu)
1564 {
1565 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1566 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1567 cpu_relax();
1568 }
1569
1570 /* Kick a guest cpu out of SIE to process a request synchronously */
1571 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1572 {
1573 kvm_make_request(req, vcpu);
1574 kvm_s390_vcpu_request(vcpu);
1575 }
1576
1577 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1578 {
1579 int i;
1580 struct kvm *kvm = gmap->private;
1581 struct kvm_vcpu *vcpu;
1582
1583 kvm_for_each_vcpu(i, vcpu, kvm) {
1584 /* match against both prefix pages */
1585 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1586 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1587 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1588 }
1589 }
1590 }
1591
1592 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1593 {
1594 /* kvm common code refers to this, but never calls it */
1595 BUG();
1596 return 0;
1597 }
1598
1599 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1600 struct kvm_one_reg *reg)
1601 {
1602 int r = -EINVAL;
1603
1604 switch (reg->id) {
1605 case KVM_REG_S390_TODPR:
1606 r = put_user(vcpu->arch.sie_block->todpr,
1607 (u32 __user *)reg->addr);
1608 break;
1609 case KVM_REG_S390_EPOCHDIFF:
1610 r = put_user(vcpu->arch.sie_block->epoch,
1611 (u64 __user *)reg->addr);
1612 break;
1613 case KVM_REG_S390_CPU_TIMER:
1614 r = put_user(vcpu->arch.sie_block->cputm,
1615 (u64 __user *)reg->addr);
1616 break;
1617 case KVM_REG_S390_CLOCK_COMP:
1618 r = put_user(vcpu->arch.sie_block->ckc,
1619 (u64 __user *)reg->addr);
1620 break;
1621 case KVM_REG_S390_PFTOKEN:
1622 r = put_user(vcpu->arch.pfault_token,
1623 (u64 __user *)reg->addr);
1624 break;
1625 case KVM_REG_S390_PFCOMPARE:
1626 r = put_user(vcpu->arch.pfault_compare,
1627 (u64 __user *)reg->addr);
1628 break;
1629 case KVM_REG_S390_PFSELECT:
1630 r = put_user(vcpu->arch.pfault_select,
1631 (u64 __user *)reg->addr);
1632 break;
1633 case KVM_REG_S390_PP:
1634 r = put_user(vcpu->arch.sie_block->pp,
1635 (u64 __user *)reg->addr);
1636 break;
1637 case KVM_REG_S390_GBEA:
1638 r = put_user(vcpu->arch.sie_block->gbea,
1639 (u64 __user *)reg->addr);
1640 break;
1641 default:
1642 break;
1643 }
1644
1645 return r;
1646 }
1647
1648 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1649 struct kvm_one_reg *reg)
1650 {
1651 int r = -EINVAL;
1652
1653 switch (reg->id) {
1654 case KVM_REG_S390_TODPR:
1655 r = get_user(vcpu->arch.sie_block->todpr,
1656 (u32 __user *)reg->addr);
1657 break;
1658 case KVM_REG_S390_EPOCHDIFF:
1659 r = get_user(vcpu->arch.sie_block->epoch,
1660 (u64 __user *)reg->addr);
1661 break;
1662 case KVM_REG_S390_CPU_TIMER:
1663 r = get_user(vcpu->arch.sie_block->cputm,
1664 (u64 __user *)reg->addr);
1665 break;
1666 case KVM_REG_S390_CLOCK_COMP:
1667 r = get_user(vcpu->arch.sie_block->ckc,
1668 (u64 __user *)reg->addr);
1669 break;
1670 case KVM_REG_S390_PFTOKEN:
1671 r = get_user(vcpu->arch.pfault_token,
1672 (u64 __user *)reg->addr);
1673 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1674 kvm_clear_async_pf_completion_queue(vcpu);
1675 break;
1676 case KVM_REG_S390_PFCOMPARE:
1677 r = get_user(vcpu->arch.pfault_compare,
1678 (u64 __user *)reg->addr);
1679 break;
1680 case KVM_REG_S390_PFSELECT:
1681 r = get_user(vcpu->arch.pfault_select,
1682 (u64 __user *)reg->addr);
1683 break;
1684 case KVM_REG_S390_PP:
1685 r = get_user(vcpu->arch.sie_block->pp,
1686 (u64 __user *)reg->addr);
1687 break;
1688 case KVM_REG_S390_GBEA:
1689 r = get_user(vcpu->arch.sie_block->gbea,
1690 (u64 __user *)reg->addr);
1691 break;
1692 default:
1693 break;
1694 }
1695
1696 return r;
1697 }
1698
1699 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1700 {
1701 kvm_s390_vcpu_initial_reset(vcpu);
1702 return 0;
1703 }
1704
1705 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1706 {
1707 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1708 return 0;
1709 }
1710
1711 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1712 {
1713 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1714 return 0;
1715 }
1716
1717 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1718 struct kvm_sregs *sregs)
1719 {
1720 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1721 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1722 restore_access_regs(vcpu->run->s.regs.acrs);
1723 return 0;
1724 }
1725
1726 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1727 struct kvm_sregs *sregs)
1728 {
1729 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1730 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1731 return 0;
1732 }
1733
1734 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1735 {
1736 if (test_fp_ctl(fpu->fpc))
1737 return -EINVAL;
1738 memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1739 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1740 save_fpu_regs();
1741 load_fpu_from(&vcpu->arch.guest_fpregs);
1742 return 0;
1743 }
1744
1745 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1746 {
1747 memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1748 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1749 return 0;
1750 }
1751
1752 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1753 {
1754 int rc = 0;
1755
1756 if (!is_vcpu_stopped(vcpu))
1757 rc = -EBUSY;
1758 else {
1759 vcpu->run->psw_mask = psw.mask;
1760 vcpu->run->psw_addr = psw.addr;
1761 }
1762 return rc;
1763 }
1764
1765 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1766 struct kvm_translation *tr)
1767 {
1768 return -EINVAL; /* not implemented yet */
1769 }
1770
1771 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1772 KVM_GUESTDBG_USE_HW_BP | \
1773 KVM_GUESTDBG_ENABLE)
1774
1775 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1776 struct kvm_guest_debug *dbg)
1777 {
1778 int rc = 0;
1779
1780 vcpu->guest_debug = 0;
1781 kvm_s390_clear_bp_data(vcpu);
1782
1783 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1784 return -EINVAL;
1785
1786 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1787 vcpu->guest_debug = dbg->control;
1788 /* enforce guest PER */
1789 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1790
1791 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1792 rc = kvm_s390_import_bp_data(vcpu, dbg);
1793 } else {
1794 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1795 vcpu->arch.guestdbg.last_bp = 0;
1796 }
1797
1798 if (rc) {
1799 vcpu->guest_debug = 0;
1800 kvm_s390_clear_bp_data(vcpu);
1801 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1802 }
1803
1804 return rc;
1805 }
1806
1807 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1808 struct kvm_mp_state *mp_state)
1809 {
1810 /* CHECK_STOP and LOAD are not supported yet */
1811 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1812 KVM_MP_STATE_OPERATING;
1813 }
1814
1815 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1816 struct kvm_mp_state *mp_state)
1817 {
1818 int rc = 0;
1819
1820 /* user space knows about this interface - let it control the state */
1821 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1822
1823 switch (mp_state->mp_state) {
1824 case KVM_MP_STATE_STOPPED:
1825 kvm_s390_vcpu_stop(vcpu);
1826 break;
1827 case KVM_MP_STATE_OPERATING:
1828 kvm_s390_vcpu_start(vcpu);
1829 break;
1830 case KVM_MP_STATE_LOAD:
1831 case KVM_MP_STATE_CHECK_STOP:
1832 /* fall through - CHECK_STOP and LOAD are not supported yet */
1833 default:
1834 rc = -ENXIO;
1835 }
1836
1837 return rc;
1838 }
1839
1840 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1841 {
1842 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1843 }
1844
1845 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1846 {
1847 retry:
1848 kvm_s390_vcpu_request_handled(vcpu);
1849 if (!vcpu->requests)
1850 return 0;
1851 /*
1852 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1853 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1854 * This ensures that the ipte instruction for this request has
1855 * already finished. We might race against a second unmapper that
1856 * wants to set the blocking bit. Lets just retry the request loop.
1857 */
1858 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1859 int rc;
1860 rc = gmap_ipte_notify(vcpu->arch.gmap,
1861 kvm_s390_get_prefix(vcpu),
1862 PAGE_SIZE * 2);
1863 if (rc)
1864 return rc;
1865 goto retry;
1866 }
1867
1868 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1869 vcpu->arch.sie_block->ihcpu = 0xffff;
1870 goto retry;
1871 }
1872
1873 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1874 if (!ibs_enabled(vcpu)) {
1875 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1876 atomic_or(CPUSTAT_IBS,
1877 &vcpu->arch.sie_block->cpuflags);
1878 }
1879 goto retry;
1880 }
1881
1882 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1883 if (ibs_enabled(vcpu)) {
1884 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1885 atomic_andnot(CPUSTAT_IBS,
1886 &vcpu->arch.sie_block->cpuflags);
1887 }
1888 goto retry;
1889 }
1890
1891 /* nothing to do, just clear the request */
1892 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1893
1894 return 0;
1895 }
1896
1897 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
1898 {
1899 struct kvm_vcpu *vcpu;
1900 int i;
1901
1902 mutex_lock(&kvm->lock);
1903 preempt_disable();
1904 kvm->arch.epoch = tod - get_tod_clock();
1905 kvm_s390_vcpu_block_all(kvm);
1906 kvm_for_each_vcpu(i, vcpu, kvm)
1907 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
1908 kvm_s390_vcpu_unblock_all(kvm);
1909 preempt_enable();
1910 mutex_unlock(&kvm->lock);
1911 }
1912
1913 /**
1914 * kvm_arch_fault_in_page - fault-in guest page if necessary
1915 * @vcpu: The corresponding virtual cpu
1916 * @gpa: Guest physical address
1917 * @writable: Whether the page should be writable or not
1918 *
1919 * Make sure that a guest page has been faulted-in on the host.
1920 *
1921 * Return: Zero on success, negative error code otherwise.
1922 */
1923 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1924 {
1925 return gmap_fault(vcpu->arch.gmap, gpa,
1926 writable ? FAULT_FLAG_WRITE : 0);
1927 }
1928
1929 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1930 unsigned long token)
1931 {
1932 struct kvm_s390_interrupt inti;
1933 struct kvm_s390_irq irq;
1934
1935 if (start_token) {
1936 irq.u.ext.ext_params2 = token;
1937 irq.type = KVM_S390_INT_PFAULT_INIT;
1938 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1939 } else {
1940 inti.type = KVM_S390_INT_PFAULT_DONE;
1941 inti.parm64 = token;
1942 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1943 }
1944 }
1945
1946 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1947 struct kvm_async_pf *work)
1948 {
1949 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1950 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1951 }
1952
1953 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1954 struct kvm_async_pf *work)
1955 {
1956 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1957 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1958 }
1959
1960 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1961 struct kvm_async_pf *work)
1962 {
1963 /* s390 will always inject the page directly */
1964 }
1965
1966 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1967 {
1968 /*
1969 * s390 will always inject the page directly,
1970 * but we still want check_async_completion to cleanup
1971 */
1972 return true;
1973 }
1974
1975 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1976 {
1977 hva_t hva;
1978 struct kvm_arch_async_pf arch;
1979 int rc;
1980
1981 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1982 return 0;
1983 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1984 vcpu->arch.pfault_compare)
1985 return 0;
1986 if (psw_extint_disabled(vcpu))
1987 return 0;
1988 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1989 return 0;
1990 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1991 return 0;
1992 if (!vcpu->arch.gmap->pfault_enabled)
1993 return 0;
1994
1995 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1996 hva += current->thread.gmap_addr & ~PAGE_MASK;
1997 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1998 return 0;
1999
2000 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2001 return rc;
2002 }
2003
2004 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2005 {
2006 int rc, cpuflags;
2007
2008 /*
2009 * On s390 notifications for arriving pages will be delivered directly
2010 * to the guest but the house keeping for completed pfaults is
2011 * handled outside the worker.
2012 */
2013 kvm_check_async_pf_completion(vcpu);
2014
2015 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
2016
2017 if (need_resched())
2018 schedule();
2019
2020 if (test_cpu_flag(CIF_MCCK_PENDING))
2021 s390_handle_mcck();
2022
2023 if (!kvm_is_ucontrol(vcpu->kvm)) {
2024 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2025 if (rc)
2026 return rc;
2027 }
2028
2029 rc = kvm_s390_handle_requests(vcpu);
2030 if (rc)
2031 return rc;
2032
2033 if (guestdbg_enabled(vcpu)) {
2034 kvm_s390_backup_guest_per_regs(vcpu);
2035 kvm_s390_patch_guest_per_regs(vcpu);
2036 }
2037
2038 vcpu->arch.sie_block->icptcode = 0;
2039 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2040 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2041 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2042
2043 return 0;
2044 }
2045
2046 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2047 {
2048 psw_t *psw = &vcpu->arch.sie_block->gpsw;
2049 u8 opcode;
2050 int rc;
2051
2052 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2053 trace_kvm_s390_sie_fault(vcpu);
2054
2055 /*
2056 * We want to inject an addressing exception, which is defined as a
2057 * suppressing or terminating exception. However, since we came here
2058 * by a DAT access exception, the PSW still points to the faulting
2059 * instruction since DAT exceptions are nullifying. So we've got
2060 * to look up the current opcode to get the length of the instruction
2061 * to be able to forward the PSW.
2062 */
2063 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
2064 if (rc)
2065 return kvm_s390_inject_prog_cond(vcpu, rc);
2066 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2067
2068 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2069 }
2070
2071 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2072 {
2073 int rc = -1;
2074
2075 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2076 vcpu->arch.sie_block->icptcode);
2077 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2078
2079 if (guestdbg_enabled(vcpu))
2080 kvm_s390_restore_guest_per_regs(vcpu);
2081
2082 if (exit_reason >= 0) {
2083 rc = 0;
2084 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2085 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2086 vcpu->run->s390_ucontrol.trans_exc_code =
2087 current->thread.gmap_addr;
2088 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2089 rc = -EREMOTE;
2090
2091 } else if (current->thread.gmap_pfault) {
2092 trace_kvm_s390_major_guest_pfault(vcpu);
2093 current->thread.gmap_pfault = 0;
2094 if (kvm_arch_setup_async_pf(vcpu)) {
2095 rc = 0;
2096 } else {
2097 gpa_t gpa = current->thread.gmap_addr;
2098 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
2099 }
2100 }
2101
2102 if (rc == -1)
2103 rc = vcpu_post_run_fault_in_sie(vcpu);
2104
2105 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
2106
2107 if (rc == 0) {
2108 if (kvm_is_ucontrol(vcpu->kvm))
2109 /* Don't exit for host interrupts. */
2110 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
2111 else
2112 rc = kvm_handle_sie_intercept(vcpu);
2113 }
2114
2115 return rc;
2116 }
2117
2118 static int __vcpu_run(struct kvm_vcpu *vcpu)
2119 {
2120 int rc, exit_reason;
2121
2122 /*
2123 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2124 * ning the guest), so that memslots (and other stuff) are protected
2125 */
2126 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2127
2128 do {
2129 rc = vcpu_pre_run(vcpu);
2130 if (rc)
2131 break;
2132
2133 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2134 /*
2135 * As PF_VCPU will be used in fault handler, between
2136 * guest_enter and guest_exit should be no uaccess.
2137 */
2138 local_irq_disable();
2139 __kvm_guest_enter();
2140 local_irq_enable();
2141 exit_reason = sie64a(vcpu->arch.sie_block,
2142 vcpu->run->s.regs.gprs);
2143 local_irq_disable();
2144 __kvm_guest_exit();
2145 local_irq_enable();
2146 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2147
2148 rc = vcpu_post_run(vcpu, exit_reason);
2149 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2150
2151 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2152 return rc;
2153 }
2154
2155 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2156 {
2157 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2158 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2159 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2160 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2161 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2162 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2163 /* some control register changes require a tlb flush */
2164 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2165 }
2166 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2167 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2168 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2169 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2170 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2171 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2172 }
2173 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2174 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2175 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2176 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2177 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2178 kvm_clear_async_pf_completion_queue(vcpu);
2179 }
2180 kvm_run->kvm_dirty_regs = 0;
2181 }
2182
2183 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2184 {
2185 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2186 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2187 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2188 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2189 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2190 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2191 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2192 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2193 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2194 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2195 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2196 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2197 }
2198
2199 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2200 {
2201 int rc;
2202 sigset_t sigsaved;
2203
2204 if (guestdbg_exit_pending(vcpu)) {
2205 kvm_s390_prepare_debug_exit(vcpu);
2206 return 0;
2207 }
2208
2209 if (vcpu->sigset_active)
2210 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2211
2212 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2213 kvm_s390_vcpu_start(vcpu);
2214 } else if (is_vcpu_stopped(vcpu)) {
2215 pr_err_ratelimited("can't run stopped vcpu %d\n",
2216 vcpu->vcpu_id);
2217 return -EINVAL;
2218 }
2219
2220 sync_regs(vcpu, kvm_run);
2221
2222 might_fault();
2223 rc = __vcpu_run(vcpu);
2224
2225 if (signal_pending(current) && !rc) {
2226 kvm_run->exit_reason = KVM_EXIT_INTR;
2227 rc = -EINTR;
2228 }
2229
2230 if (guestdbg_exit_pending(vcpu) && !rc) {
2231 kvm_s390_prepare_debug_exit(vcpu);
2232 rc = 0;
2233 }
2234
2235 if (rc == -EOPNOTSUPP) {
2236 /* intercept cannot be handled in-kernel, prepare kvm-run */
2237 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
2238 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2239 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2240 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2241 rc = 0;
2242 }
2243
2244 if (rc == -EREMOTE) {
2245 /* intercept was handled, but userspace support is needed
2246 * kvm_run has been prepared by the handler */
2247 rc = 0;
2248 }
2249
2250 store_regs(vcpu, kvm_run);
2251
2252 if (vcpu->sigset_active)
2253 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2254
2255 vcpu->stat.exit_userspace++;
2256 return rc;
2257 }
2258
2259 /*
2260 * store status at address
2261 * we use have two special cases:
2262 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2263 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2264 */
2265 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2266 {
2267 unsigned char archmode = 1;
2268 unsigned int px;
2269 u64 clkcomp;
2270 int rc;
2271
2272 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2273 if (write_guest_abs(vcpu, 163, &archmode, 1))
2274 return -EFAULT;
2275 gpa = SAVE_AREA_BASE;
2276 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2277 if (write_guest_real(vcpu, 163, &archmode, 1))
2278 return -EFAULT;
2279 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2280 }
2281 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2282 vcpu->arch.guest_fpregs.fprs, 128);
2283 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2284 vcpu->run->s.regs.gprs, 128);
2285 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2286 &vcpu->arch.sie_block->gpsw, 16);
2287 px = kvm_s390_get_prefix(vcpu);
2288 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2289 &px, 4);
2290 rc |= write_guest_abs(vcpu,
2291 gpa + offsetof(struct save_area, fp_ctrl_reg),
2292 &vcpu->arch.guest_fpregs.fpc, 4);
2293 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2294 &vcpu->arch.sie_block->todpr, 4);
2295 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2296 &vcpu->arch.sie_block->cputm, 8);
2297 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2298 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2299 &clkcomp, 8);
2300 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2301 &vcpu->run->s.regs.acrs, 64);
2302 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2303 &vcpu->arch.sie_block->gcr, 128);
2304 return rc ? -EFAULT : 0;
2305 }
2306
2307 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2308 {
2309 /*
2310 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2311 * copying in vcpu load/put. Lets update our copies before we save
2312 * it into the save area
2313 */
2314 save_fpu_regs();
2315 if (test_kvm_facility(vcpu->kvm, 129)) {
2316 /*
2317 * If the vector extension is available, the vector registers
2318 * which overlaps with floating-point registers are saved in
2319 * the SIE-control block. Hence, extract the floating-point
2320 * registers and the FPC value and store them in the
2321 * guest_fpregs structure.
2322 */
2323 WARN_ON(!is_vx_task(current)); /* XXX remove later */
2324 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2325 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2326 current->thread.fpu.vxrs);
2327 } else
2328 save_fpu_to(&vcpu->arch.guest_fpregs);
2329 save_access_regs(vcpu->run->s.regs.acrs);
2330
2331 return kvm_s390_store_status_unloaded(vcpu, addr);
2332 }
2333
2334 /*
2335 * store additional status at address
2336 */
2337 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2338 unsigned long gpa)
2339 {
2340 /* Only bits 0-53 are used for address formation */
2341 if (!(gpa & ~0x3ff))
2342 return 0;
2343
2344 return write_guest_abs(vcpu, gpa & ~0x3ff,
2345 (void *)&vcpu->run->s.regs.vrs, 512);
2346 }
2347
2348 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2349 {
2350 if (!test_kvm_facility(vcpu->kvm, 129))
2351 return 0;
2352
2353 /*
2354 * The guest VXRS are in the host VXRs due to the lazy
2355 * copying in vcpu load/put. We can simply call save_fpu_regs()
2356 * to save the current register state because we are in the
2357 * middle of a load/put cycle.
2358 *
2359 * Let's update our copies before we save it into the save area.
2360 */
2361 save_fpu_regs();
2362
2363 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2364 }
2365
2366 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2367 {
2368 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2369 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2370 }
2371
2372 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2373 {
2374 unsigned int i;
2375 struct kvm_vcpu *vcpu;
2376
2377 kvm_for_each_vcpu(i, vcpu, kvm) {
2378 __disable_ibs_on_vcpu(vcpu);
2379 }
2380 }
2381
2382 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2383 {
2384 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2385 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2386 }
2387
2388 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2389 {
2390 int i, online_vcpus, started_vcpus = 0;
2391
2392 if (!is_vcpu_stopped(vcpu))
2393 return;
2394
2395 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2396 /* Only one cpu at a time may enter/leave the STOPPED state. */
2397 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2398 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2399
2400 for (i = 0; i < online_vcpus; i++) {
2401 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2402 started_vcpus++;
2403 }
2404
2405 if (started_vcpus == 0) {
2406 /* we're the only active VCPU -> speed it up */
2407 __enable_ibs_on_vcpu(vcpu);
2408 } else if (started_vcpus == 1) {
2409 /*
2410 * As we are starting a second VCPU, we have to disable
2411 * the IBS facility on all VCPUs to remove potentially
2412 * oustanding ENABLE requests.
2413 */
2414 __disable_ibs_on_all_vcpus(vcpu->kvm);
2415 }
2416
2417 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2418 /*
2419 * Another VCPU might have used IBS while we were offline.
2420 * Let's play safe and flush the VCPU at startup.
2421 */
2422 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2423 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2424 return;
2425 }
2426
2427 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2428 {
2429 int i, online_vcpus, started_vcpus = 0;
2430 struct kvm_vcpu *started_vcpu = NULL;
2431
2432 if (is_vcpu_stopped(vcpu))
2433 return;
2434
2435 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2436 /* Only one cpu at a time may enter/leave the STOPPED state. */
2437 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2438 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2439
2440 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2441 kvm_s390_clear_stop_irq(vcpu);
2442
2443 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2444 __disable_ibs_on_vcpu(vcpu);
2445
2446 for (i = 0; i < online_vcpus; i++) {
2447 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2448 started_vcpus++;
2449 started_vcpu = vcpu->kvm->vcpus[i];
2450 }
2451 }
2452
2453 if (started_vcpus == 1) {
2454 /*
2455 * As we only have one VCPU left, we want to enable the
2456 * IBS facility for that VCPU to speed it up.
2457 */
2458 __enable_ibs_on_vcpu(started_vcpu);
2459 }
2460
2461 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2462 return;
2463 }
2464
2465 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2466 struct kvm_enable_cap *cap)
2467 {
2468 int r;
2469
2470 if (cap->flags)
2471 return -EINVAL;
2472
2473 switch (cap->cap) {
2474 case KVM_CAP_S390_CSS_SUPPORT:
2475 if (!vcpu->kvm->arch.css_support) {
2476 vcpu->kvm->arch.css_support = 1;
2477 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2478 trace_kvm_s390_enable_css(vcpu->kvm);
2479 }
2480 r = 0;
2481 break;
2482 default:
2483 r = -EINVAL;
2484 break;
2485 }
2486 return r;
2487 }
2488
2489 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2490 struct kvm_s390_mem_op *mop)
2491 {
2492 void __user *uaddr = (void __user *)mop->buf;
2493 void *tmpbuf = NULL;
2494 int r, srcu_idx;
2495 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2496 | KVM_S390_MEMOP_F_CHECK_ONLY;
2497
2498 if (mop->flags & ~supported_flags)
2499 return -EINVAL;
2500
2501 if (mop->size > MEM_OP_MAX_SIZE)
2502 return -E2BIG;
2503
2504 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2505 tmpbuf = vmalloc(mop->size);
2506 if (!tmpbuf)
2507 return -ENOMEM;
2508 }
2509
2510 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2511
2512 switch (mop->op) {
2513 case KVM_S390_MEMOP_LOGICAL_READ:
2514 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2515 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2516 break;
2517 }
2518 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2519 if (r == 0) {
2520 if (copy_to_user(uaddr, tmpbuf, mop->size))
2521 r = -EFAULT;
2522 }
2523 break;
2524 case KVM_S390_MEMOP_LOGICAL_WRITE:
2525 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2526 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2527 break;
2528 }
2529 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2530 r = -EFAULT;
2531 break;
2532 }
2533 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2534 break;
2535 default:
2536 r = -EINVAL;
2537 }
2538
2539 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2540
2541 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2542 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2543
2544 vfree(tmpbuf);
2545 return r;
2546 }
2547
2548 long kvm_arch_vcpu_ioctl(struct file *filp,
2549 unsigned int ioctl, unsigned long arg)
2550 {
2551 struct kvm_vcpu *vcpu = filp->private_data;
2552 void __user *argp = (void __user *)arg;
2553 int idx;
2554 long r;
2555
2556 switch (ioctl) {
2557 case KVM_S390_IRQ: {
2558 struct kvm_s390_irq s390irq;
2559
2560 r = -EFAULT;
2561 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2562 break;
2563 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2564 break;
2565 }
2566 case KVM_S390_INTERRUPT: {
2567 struct kvm_s390_interrupt s390int;
2568 struct kvm_s390_irq s390irq;
2569
2570 r = -EFAULT;
2571 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2572 break;
2573 if (s390int_to_s390irq(&s390int, &s390irq))
2574 return -EINVAL;
2575 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2576 break;
2577 }
2578 case KVM_S390_STORE_STATUS:
2579 idx = srcu_read_lock(&vcpu->kvm->srcu);
2580 r = kvm_s390_vcpu_store_status(vcpu, arg);
2581 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2582 break;
2583 case KVM_S390_SET_INITIAL_PSW: {
2584 psw_t psw;
2585
2586 r = -EFAULT;
2587 if (copy_from_user(&psw, argp, sizeof(psw)))
2588 break;
2589 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2590 break;
2591 }
2592 case KVM_S390_INITIAL_RESET:
2593 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2594 break;
2595 case KVM_SET_ONE_REG:
2596 case KVM_GET_ONE_REG: {
2597 struct kvm_one_reg reg;
2598 r = -EFAULT;
2599 if (copy_from_user(&reg, argp, sizeof(reg)))
2600 break;
2601 if (ioctl == KVM_SET_ONE_REG)
2602 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2603 else
2604 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2605 break;
2606 }
2607 #ifdef CONFIG_KVM_S390_UCONTROL
2608 case KVM_S390_UCAS_MAP: {
2609 struct kvm_s390_ucas_mapping ucasmap;
2610
2611 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2612 r = -EFAULT;
2613 break;
2614 }
2615
2616 if (!kvm_is_ucontrol(vcpu->kvm)) {
2617 r = -EINVAL;
2618 break;
2619 }
2620
2621 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2622 ucasmap.vcpu_addr, ucasmap.length);
2623 break;
2624 }
2625 case KVM_S390_UCAS_UNMAP: {
2626 struct kvm_s390_ucas_mapping ucasmap;
2627
2628 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2629 r = -EFAULT;
2630 break;
2631 }
2632
2633 if (!kvm_is_ucontrol(vcpu->kvm)) {
2634 r = -EINVAL;
2635 break;
2636 }
2637
2638 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2639 ucasmap.length);
2640 break;
2641 }
2642 #endif
2643 case KVM_S390_VCPU_FAULT: {
2644 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2645 break;
2646 }
2647 case KVM_ENABLE_CAP:
2648 {
2649 struct kvm_enable_cap cap;
2650 r = -EFAULT;
2651 if (copy_from_user(&cap, argp, sizeof(cap)))
2652 break;
2653 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2654 break;
2655 }
2656 case KVM_S390_MEM_OP: {
2657 struct kvm_s390_mem_op mem_op;
2658
2659 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2660 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2661 else
2662 r = -EFAULT;
2663 break;
2664 }
2665 case KVM_S390_SET_IRQ_STATE: {
2666 struct kvm_s390_irq_state irq_state;
2667
2668 r = -EFAULT;
2669 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2670 break;
2671 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2672 irq_state.len == 0 ||
2673 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2674 r = -EINVAL;
2675 break;
2676 }
2677 r = kvm_s390_set_irq_state(vcpu,
2678 (void __user *) irq_state.buf,
2679 irq_state.len);
2680 break;
2681 }
2682 case KVM_S390_GET_IRQ_STATE: {
2683 struct kvm_s390_irq_state irq_state;
2684
2685 r = -EFAULT;
2686 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2687 break;
2688 if (irq_state.len == 0) {
2689 r = -EINVAL;
2690 break;
2691 }
2692 r = kvm_s390_get_irq_state(vcpu,
2693 (__u8 __user *) irq_state.buf,
2694 irq_state.len);
2695 break;
2696 }
2697 default:
2698 r = -ENOTTY;
2699 }
2700 return r;
2701 }
2702
2703 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2704 {
2705 #ifdef CONFIG_KVM_S390_UCONTROL
2706 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2707 && (kvm_is_ucontrol(vcpu->kvm))) {
2708 vmf->page = virt_to_page(vcpu->arch.sie_block);
2709 get_page(vmf->page);
2710 return 0;
2711 }
2712 #endif
2713 return VM_FAULT_SIGBUS;
2714 }
2715
2716 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2717 unsigned long npages)
2718 {
2719 return 0;
2720 }
2721
2722 /* Section: memory related */
2723 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2724 struct kvm_memory_slot *memslot,
2725 const struct kvm_userspace_memory_region *mem,
2726 enum kvm_mr_change change)
2727 {
2728 /* A few sanity checks. We can have memory slots which have to be
2729 located/ended at a segment boundary (1MB). The memory in userland is
2730 ok to be fragmented into various different vmas. It is okay to mmap()
2731 and munmap() stuff in this slot after doing this call at any time */
2732
2733 if (mem->userspace_addr & 0xffffful)
2734 return -EINVAL;
2735
2736 if (mem->memory_size & 0xffffful)
2737 return -EINVAL;
2738
2739 return 0;
2740 }
2741
2742 void kvm_arch_commit_memory_region(struct kvm *kvm,
2743 const struct kvm_userspace_memory_region *mem,
2744 const struct kvm_memory_slot *old,
2745 const struct kvm_memory_slot *new,
2746 enum kvm_mr_change change)
2747 {
2748 int rc;
2749
2750 /* If the basics of the memslot do not change, we do not want
2751 * to update the gmap. Every update causes several unnecessary
2752 * segment translation exceptions. This is usually handled just
2753 * fine by the normal fault handler + gmap, but it will also
2754 * cause faults on the prefix page of running guest CPUs.
2755 */
2756 if (old->userspace_addr == mem->userspace_addr &&
2757 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2758 old->npages * PAGE_SIZE == mem->memory_size)
2759 return;
2760
2761 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2762 mem->guest_phys_addr, mem->memory_size);
2763 if (rc)
2764 pr_warn("failed to commit memory region\n");
2765 return;
2766 }
2767
2768 static int __init kvm_s390_init(void)
2769 {
2770 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2771 }
2772
2773 static void __exit kvm_s390_exit(void)
2774 {
2775 kvm_exit();
2776 }
2777
2778 module_init(kvm_s390_init);
2779 module_exit(kvm_s390_exit);
2780
2781 /*
2782 * Enable autoloading of the kvm module.
2783 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2784 * since x86 takes a different approach.
2785 */
2786 #include <linux/miscdevice.h>
2787 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2788 MODULE_ALIAS("devname:kvm");
Linux kernel - Deliverables
This page took 0.101183 seconds and 5 git commands to generate.