projects / deliverable / linux.git / blob
? search:


1dbab2340a6623732ad8bed18327cadb7408983d
[deliverable/linux.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/lowcore.h>
30 #include <asm/pgtable.h>
31 #include <asm/nmi.h>
32 #include <asm/switch_to.h>
33 #include <asm/facility.h>
34 #include <asm/sclp.h>
35 #include "kvm-s390.h"
36 #include "gaccess.h"
37
38 #define CREATE_TRACE_POINTS
39 #include "trace.h"
40 #include "trace-s390.h"
41
42 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
43
44 struct kvm_stats_debugfs_item debugfs_entries[] = {
45 { "userspace_handled", VCPU_STAT(exit_userspace) },
46 { "exit_null", VCPU_STAT(exit_null) },
47 { "exit_validity", VCPU_STAT(exit_validity) },
48 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
49 { "exit_external_request", VCPU_STAT(exit_external_request) },
50 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
51 { "exit_instruction", VCPU_STAT(exit_instruction) },
52 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
53 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
54 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
55 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
56 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
57 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
58 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
59 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
60 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
61 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
62 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
63 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
64 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
65 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
66 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
67 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
68 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
70 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
71 { "instruction_spx", VCPU_STAT(instruction_spx) },
72 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
73 { "instruction_stap", VCPU_STAT(instruction_stap) },
74 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
75 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
76 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
77 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
78 { "instruction_essa", VCPU_STAT(instruction_essa) },
79 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
80 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
81 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
82 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
83 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
84 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
85 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
86 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
87 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
88 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
89 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
90 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
91 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
92 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
93 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
94 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
95 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
96 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
97 { "diagnose_10", VCPU_STAT(diagnose_10) },
98 { "diagnose_44", VCPU_STAT(diagnose_44) },
99 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
100 { NULL }
101 };
102
103 unsigned long *vfacilities;
104 static struct gmap_notifier gmap_notifier;
105
106 /* test availability of vfacility */
107 int test_vfacility(unsigned long nr)
108 {
109 return __test_facility(nr, (void *) vfacilities);
110 }
111
112 /* Section: not file related */
113 int kvm_arch_hardware_enable(void)
114 {
115 /* every s390 is virtualization enabled ;-) */
116 return 0;
117 }
118
119 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
120
121 int kvm_arch_hardware_setup(void)
122 {
123 gmap_notifier.notifier_call = kvm_gmap_notifier;
124 gmap_register_ipte_notifier(&gmap_notifier);
125 return 0;
126 }
127
128 void kvm_arch_hardware_unsetup(void)
129 {
130 gmap_unregister_ipte_notifier(&gmap_notifier);
131 }
132
133 int kvm_arch_init(void *opaque)
134 {
135 /* Register floating interrupt controller interface. */
136 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
137 }
138
139 /* Section: device related */
140 long kvm_arch_dev_ioctl(struct file *filp,
141 unsigned int ioctl, unsigned long arg)
142 {
143 if (ioctl == KVM_S390_ENABLE_SIE)
144 return s390_enable_sie();
145 return -EINVAL;
146 }
147
148 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
149 {
150 int r;
151
152 switch (ext) {
153 case KVM_CAP_S390_PSW:
154 case KVM_CAP_S390_GMAP:
155 case KVM_CAP_SYNC_MMU:
156 #ifdef CONFIG_KVM_S390_UCONTROL
157 case KVM_CAP_S390_UCONTROL:
158 #endif
159 case KVM_CAP_ASYNC_PF:
160 case KVM_CAP_SYNC_REGS:
161 case KVM_CAP_ONE_REG:
162 case KVM_CAP_ENABLE_CAP:
163 case KVM_CAP_S390_CSS_SUPPORT:
164 case KVM_CAP_IRQFD:
165 case KVM_CAP_IOEVENTFD:
166 case KVM_CAP_DEVICE_CTRL:
167 case KVM_CAP_ENABLE_CAP_VM:
168 case KVM_CAP_S390_IRQCHIP:
169 case KVM_CAP_VM_ATTRIBUTES:
170 case KVM_CAP_MP_STATE:
171 case KVM_CAP_S390_USER_SIGP:
172 r = 1;
173 break;
174 case KVM_CAP_NR_VCPUS:
175 case KVM_CAP_MAX_VCPUS:
176 r = KVM_MAX_VCPUS;
177 break;
178 case KVM_CAP_NR_MEMSLOTS:
179 r = KVM_USER_MEM_SLOTS;
180 break;
181 case KVM_CAP_S390_COW:
182 r = MACHINE_HAS_ESOP;
183 break;
184 default:
185 r = 0;
186 }
187 return r;
188 }
189
190 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
191 struct kvm_memory_slot *memslot)
192 {
193 gfn_t cur_gfn, last_gfn;
194 unsigned long address;
195 struct gmap *gmap = kvm->arch.gmap;
196
197 down_read(&gmap->mm->mmap_sem);
198 /* Loop over all guest pages */
199 last_gfn = memslot->base_gfn + memslot->npages;
200 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
201 address = gfn_to_hva_memslot(memslot, cur_gfn);
202
203 if (gmap_test_and_clear_dirty(address, gmap))
204 mark_page_dirty(kvm, cur_gfn);
205 }
206 up_read(&gmap->mm->mmap_sem);
207 }
208
209 /* Section: vm related */
210 /*
211 * Get (and clear) the dirty memory log for a memory slot.
212 */
213 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
214 struct kvm_dirty_log *log)
215 {
216 int r;
217 unsigned long n;
218 struct kvm_memory_slot *memslot;
219 int is_dirty = 0;
220
221 mutex_lock(&kvm->slots_lock);
222
223 r = -EINVAL;
224 if (log->slot >= KVM_USER_MEM_SLOTS)
225 goto out;
226
227 memslot = id_to_memslot(kvm->memslots, log->slot);
228 r = -ENOENT;
229 if (!memslot->dirty_bitmap)
230 goto out;
231
232 kvm_s390_sync_dirty_log(kvm, memslot);
233 r = kvm_get_dirty_log(kvm, log, &is_dirty);
234 if (r)
235 goto out;
236
237 /* Clear the dirty log */
238 if (is_dirty) {
239 n = kvm_dirty_bitmap_bytes(memslot);
240 memset(memslot->dirty_bitmap, 0, n);
241 }
242 r = 0;
243 out:
244 mutex_unlock(&kvm->slots_lock);
245 return r;
246 }
247
248 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
249 {
250 int r;
251
252 if (cap->flags)
253 return -EINVAL;
254
255 switch (cap->cap) {
256 case KVM_CAP_S390_IRQCHIP:
257 kvm->arch.use_irqchip = 1;
258 r = 0;
259 break;
260 case KVM_CAP_S390_USER_SIGP:
261 kvm->arch.user_sigp = 1;
262 r = 0;
263 break;
264 default:
265 r = -EINVAL;
266 break;
267 }
268 return r;
269 }
270
271 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
272 {
273 int ret;
274
275 switch (attr->attr) {
276 case KVM_S390_VM_MEM_LIMIT_SIZE:
277 ret = 0;
278 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
279 ret = -EFAULT;
280 break;
281 default:
282 ret = -ENXIO;
283 break;
284 }
285 return ret;
286 }
287
288 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
289 {
290 int ret;
291 unsigned int idx;
292 switch (attr->attr) {
293 case KVM_S390_VM_MEM_ENABLE_CMMA:
294 ret = -EBUSY;
295 mutex_lock(&kvm->lock);
296 if (atomic_read(&kvm->online_vcpus) == 0) {
297 kvm->arch.use_cmma = 1;
298 ret = 0;
299 }
300 mutex_unlock(&kvm->lock);
301 break;
302 case KVM_S390_VM_MEM_CLR_CMMA:
303 mutex_lock(&kvm->lock);
304 idx = srcu_read_lock(&kvm->srcu);
305 s390_reset_cmma(kvm->arch.gmap->mm);
306 srcu_read_unlock(&kvm->srcu, idx);
307 mutex_unlock(&kvm->lock);
308 ret = 0;
309 break;
310 case KVM_S390_VM_MEM_LIMIT_SIZE: {
311 unsigned long new_limit;
312
313 if (kvm_is_ucontrol(kvm))
314 return -EINVAL;
315
316 if (get_user(new_limit, (u64 __user *)attr->addr))
317 return -EFAULT;
318
319 if (new_limit > kvm->arch.gmap->asce_end)
320 return -E2BIG;
321
322 ret = -EBUSY;
323 mutex_lock(&kvm->lock);
324 if (atomic_read(&kvm->online_vcpus) == 0) {
325 /* gmap_alloc will round the limit up */
326 struct gmap *new = gmap_alloc(current->mm, new_limit);
327
328 if (!new) {
329 ret = -ENOMEM;
330 } else {
331 gmap_free(kvm->arch.gmap);
332 new->private = kvm;
333 kvm->arch.gmap = new;
334 ret = 0;
335 }
336 }
337 mutex_unlock(&kvm->lock);
338 break;
339 }
340 default:
341 ret = -ENXIO;
342 break;
343 }
344 return ret;
345 }
346
347 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
348
349 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
350 {
351 struct kvm_vcpu *vcpu;
352 int i;
353
354 if (!test_vfacility(76))
355 return -EINVAL;
356
357 mutex_lock(&kvm->lock);
358 switch (attr->attr) {
359 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
360 get_random_bytes(
361 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
362 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
363 kvm->arch.crypto.aes_kw = 1;
364 break;
365 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
366 get_random_bytes(
367 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
368 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
369 kvm->arch.crypto.dea_kw = 1;
370 break;
371 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
372 kvm->arch.crypto.aes_kw = 0;
373 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
374 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
375 break;
376 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
377 kvm->arch.crypto.dea_kw = 0;
378 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
379 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
380 break;
381 default:
382 mutex_unlock(&kvm->lock);
383 return -ENXIO;
384 }
385
386 kvm_for_each_vcpu(i, vcpu, kvm) {
387 kvm_s390_vcpu_crypto_setup(vcpu);
388 exit_sie(vcpu);
389 }
390 mutex_unlock(&kvm->lock);
391 return 0;
392 }
393
394 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
395 {
396 u8 gtod_high;
397
398 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
399 sizeof(gtod_high)))
400 return -EFAULT;
401
402 if (gtod_high != 0)
403 return -EINVAL;
404
405 return 0;
406 }
407
408 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
409 {
410 struct kvm_vcpu *cur_vcpu;
411 unsigned int vcpu_idx;
412 u64 host_tod, gtod;
413 int r;
414
415 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
416 return -EFAULT;
417
418 r = store_tod_clock(&host_tod);
419 if (r)
420 return r;
421
422 mutex_lock(&kvm->lock);
423 kvm->arch.epoch = gtod - host_tod;
424 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
425 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
426 exit_sie(cur_vcpu);
427 }
428 mutex_unlock(&kvm->lock);
429 return 0;
430 }
431
432 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
433 {
434 int ret;
435
436 if (attr->flags)
437 return -EINVAL;
438
439 switch (attr->attr) {
440 case KVM_S390_VM_TOD_HIGH:
441 ret = kvm_s390_set_tod_high(kvm, attr);
442 break;
443 case KVM_S390_VM_TOD_LOW:
444 ret = kvm_s390_set_tod_low(kvm, attr);
445 break;
446 default:
447 ret = -ENXIO;
448 break;
449 }
450 return ret;
451 }
452
453 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
454 {
455 u8 gtod_high = 0;
456
457 if (copy_to_user((void __user *)attr->addr, &gtod_high,
458 sizeof(gtod_high)))
459 return -EFAULT;
460
461 return 0;
462 }
463
464 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
465 {
466 u64 host_tod, gtod;
467 int r;
468
469 r = store_tod_clock(&host_tod);
470 if (r)
471 return r;
472
473 gtod = host_tod + kvm->arch.epoch;
474 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
475 return -EFAULT;
476
477 return 0;
478 }
479
480 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
481 {
482 int ret;
483
484 if (attr->flags)
485 return -EINVAL;
486
487 switch (attr->attr) {
488 case KVM_S390_VM_TOD_HIGH:
489 ret = kvm_s390_get_tod_high(kvm, attr);
490 break;
491 case KVM_S390_VM_TOD_LOW:
492 ret = kvm_s390_get_tod_low(kvm, attr);
493 break;
494 default:
495 ret = -ENXIO;
496 break;
497 }
498 return ret;
499 }
500
501 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
502 {
503 int ret;
504
505 switch (attr->group) {
506 case KVM_S390_VM_MEM_CTRL:
507 ret = kvm_s390_set_mem_control(kvm, attr);
508 break;
509 case KVM_S390_VM_TOD:
510 ret = kvm_s390_set_tod(kvm, attr);
511 break;
512 case KVM_S390_VM_CRYPTO:
513 ret = kvm_s390_vm_set_crypto(kvm, attr);
514 break;
515 default:
516 ret = -ENXIO;
517 break;
518 }
519
520 return ret;
521 }
522
523 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
524 {
525 int ret;
526
527 switch (attr->group) {
528 case KVM_S390_VM_MEM_CTRL:
529 ret = kvm_s390_get_mem_control(kvm, attr);
530 break;
531 case KVM_S390_VM_TOD:
532 ret = kvm_s390_get_tod(kvm, attr);
533 break;
534 default:
535 ret = -ENXIO;
536 break;
537 }
538
539 return ret;
540 }
541
542 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
543 {
544 int ret;
545
546 switch (attr->group) {
547 case KVM_S390_VM_MEM_CTRL:
548 switch (attr->attr) {
549 case KVM_S390_VM_MEM_ENABLE_CMMA:
550 case KVM_S390_VM_MEM_CLR_CMMA:
551 case KVM_S390_VM_MEM_LIMIT_SIZE:
552 ret = 0;
553 break;
554 default:
555 ret = -ENXIO;
556 break;
557 }
558 break;
559 case KVM_S390_VM_TOD:
560 switch (attr->attr) {
561 case KVM_S390_VM_TOD_LOW:
562 case KVM_S390_VM_TOD_HIGH:
563 ret = 0;
564 break;
565 default:
566 ret = -ENXIO;
567 break;
568 }
569 break;
570 case KVM_S390_VM_CRYPTO:
571 switch (attr->attr) {
572 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
573 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
574 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
575 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
576 ret = 0;
577 break;
578 default:
579 ret = -ENXIO;
580 break;
581 }
582 break;
583 default:
584 ret = -ENXIO;
585 break;
586 }
587
588 return ret;
589 }
590
591 long kvm_arch_vm_ioctl(struct file *filp,
592 unsigned int ioctl, unsigned long arg)
593 {
594 struct kvm *kvm = filp->private_data;
595 void __user *argp = (void __user *)arg;
596 struct kvm_device_attr attr;
597 int r;
598
599 switch (ioctl) {
600 case KVM_S390_INTERRUPT: {
601 struct kvm_s390_interrupt s390int;
602
603 r = -EFAULT;
604 if (copy_from_user(&s390int, argp, sizeof(s390int)))
605 break;
606 r = kvm_s390_inject_vm(kvm, &s390int);
607 break;
608 }
609 case KVM_ENABLE_CAP: {
610 struct kvm_enable_cap cap;
611 r = -EFAULT;
612 if (copy_from_user(&cap, argp, sizeof(cap)))
613 break;
614 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
615 break;
616 }
617 case KVM_CREATE_IRQCHIP: {
618 struct kvm_irq_routing_entry routing;
619
620 r = -EINVAL;
621 if (kvm->arch.use_irqchip) {
622 /* Set up dummy routing. */
623 memset(&routing, 0, sizeof(routing));
624 kvm_set_irq_routing(kvm, &routing, 0, 0);
625 r = 0;
626 }
627 break;
628 }
629 case KVM_SET_DEVICE_ATTR: {
630 r = -EFAULT;
631 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
632 break;
633 r = kvm_s390_vm_set_attr(kvm, &attr);
634 break;
635 }
636 case KVM_GET_DEVICE_ATTR: {
637 r = -EFAULT;
638 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
639 break;
640 r = kvm_s390_vm_get_attr(kvm, &attr);
641 break;
642 }
643 case KVM_HAS_DEVICE_ATTR: {
644 r = -EFAULT;
645 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
646 break;
647 r = kvm_s390_vm_has_attr(kvm, &attr);
648 break;
649 }
650 default:
651 r = -ENOTTY;
652 }
653
654 return r;
655 }
656
657 static int kvm_s390_crypto_init(struct kvm *kvm)
658 {
659 if (!test_vfacility(76))
660 return 0;
661
662 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
663 GFP_KERNEL | GFP_DMA);
664 if (!kvm->arch.crypto.crycb)
665 return -ENOMEM;
666
667 kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
668 CRYCB_FORMAT1;
669
670 /* Disable AES/DEA protected key functions by default */
671 kvm->arch.crypto.aes_kw = 0;
672 kvm->arch.crypto.dea_kw = 0;
673
674 return 0;
675 }
676
677 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
678 {
679 int rc;
680 char debug_name[16];
681 static unsigned long sca_offset;
682
683 rc = -EINVAL;
684 #ifdef CONFIG_KVM_S390_UCONTROL
685 if (type & ~KVM_VM_S390_UCONTROL)
686 goto out_err;
687 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
688 goto out_err;
689 #else
690 if (type)
691 goto out_err;
692 #endif
693
694 rc = s390_enable_sie();
695 if (rc)
696 goto out_err;
697
698 rc = -ENOMEM;
699
700 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
701 if (!kvm->arch.sca)
702 goto out_err;
703 spin_lock(&kvm_lock);
704 sca_offset = (sca_offset + 16) & 0x7f0;
705 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
706 spin_unlock(&kvm_lock);
707
708 sprintf(debug_name, "kvm-%u", current->pid);
709
710 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
711 if (!kvm->arch.dbf)
712 goto out_nodbf;
713
714 if (kvm_s390_crypto_init(kvm) < 0)
715 goto out_crypto;
716
717 spin_lock_init(&kvm->arch.float_int.lock);
718 INIT_LIST_HEAD(&kvm->arch.float_int.list);
719 init_waitqueue_head(&kvm->arch.ipte_wq);
720 mutex_init(&kvm->arch.ipte_mutex);
721
722 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
723 VM_EVENT(kvm, 3, "%s", "vm created");
724
725 if (type & KVM_VM_S390_UCONTROL) {
726 kvm->arch.gmap = NULL;
727 } else {
728 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
729 if (!kvm->arch.gmap)
730 goto out_nogmap;
731 kvm->arch.gmap->private = kvm;
732 kvm->arch.gmap->pfault_enabled = 0;
733 }
734
735 kvm->arch.css_support = 0;
736 kvm->arch.use_irqchip = 0;
737 kvm->arch.epoch = 0;
738
739 spin_lock_init(&kvm->arch.start_stop_lock);
740
741 return 0;
742 out_nogmap:
743 kfree(kvm->arch.crypto.crycb);
744 out_crypto:
745 debug_unregister(kvm->arch.dbf);
746 out_nodbf:
747 free_page((unsigned long)(kvm->arch.sca));
748 out_err:
749 return rc;
750 }
751
752 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
753 {
754 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
755 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
756 kvm_s390_clear_local_irqs(vcpu);
757 kvm_clear_async_pf_completion_queue(vcpu);
758 if (!kvm_is_ucontrol(vcpu->kvm)) {
759 clear_bit(63 - vcpu->vcpu_id,
760 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
761 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
762 (__u64) vcpu->arch.sie_block)
763 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
764 }
765 smp_mb();
766
767 if (kvm_is_ucontrol(vcpu->kvm))
768 gmap_free(vcpu->arch.gmap);
769
770 if (kvm_s390_cmma_enabled(vcpu->kvm))
771 kvm_s390_vcpu_unsetup_cmma(vcpu);
772 free_page((unsigned long)(vcpu->arch.sie_block));
773
774 kvm_vcpu_uninit(vcpu);
775 kmem_cache_free(kvm_vcpu_cache, vcpu);
776 }
777
778 static void kvm_free_vcpus(struct kvm *kvm)
779 {
780 unsigned int i;
781 struct kvm_vcpu *vcpu;
782
783 kvm_for_each_vcpu(i, vcpu, kvm)
784 kvm_arch_vcpu_destroy(vcpu);
785
786 mutex_lock(&kvm->lock);
787 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
788 kvm->vcpus[i] = NULL;
789
790 atomic_set(&kvm->online_vcpus, 0);
791 mutex_unlock(&kvm->lock);
792 }
793
794 void kvm_arch_destroy_vm(struct kvm *kvm)
795 {
796 kvm_free_vcpus(kvm);
797 free_page((unsigned long)(kvm->arch.sca));
798 debug_unregister(kvm->arch.dbf);
799 kfree(kvm->arch.crypto.crycb);
800 if (!kvm_is_ucontrol(kvm))
801 gmap_free(kvm->arch.gmap);
802 kvm_s390_destroy_adapters(kvm);
803 kvm_s390_clear_float_irqs(kvm);
804 }
805
806 /* Section: vcpu related */
807 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
808 {
809 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
810 if (!vcpu->arch.gmap)
811 return -ENOMEM;
812 vcpu->arch.gmap->private = vcpu->kvm;
813
814 return 0;
815 }
816
817 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
818 {
819 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
820 kvm_clear_async_pf_completion_queue(vcpu);
821 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
822 KVM_SYNC_GPRS |
823 KVM_SYNC_ACRS |
824 KVM_SYNC_CRS |
825 KVM_SYNC_ARCH0 |
826 KVM_SYNC_PFAULT;
827
828 if (kvm_is_ucontrol(vcpu->kvm))
829 return __kvm_ucontrol_vcpu_init(vcpu);
830
831 return 0;
832 }
833
834 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
835 {
836 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
837 save_fp_regs(vcpu->arch.host_fpregs.fprs);
838 save_access_regs(vcpu->arch.host_acrs);
839 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
840 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
841 restore_access_regs(vcpu->run->s.regs.acrs);
842 gmap_enable(vcpu->arch.gmap);
843 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
844 }
845
846 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
847 {
848 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
849 gmap_disable(vcpu->arch.gmap);
850 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
851 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
852 save_access_regs(vcpu->run->s.regs.acrs);
853 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
854 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
855 restore_access_regs(vcpu->arch.host_acrs);
856 }
857
858 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
859 {
860 /* this equals initial cpu reset in pop, but we don't switch to ESA */
861 vcpu->arch.sie_block->gpsw.mask = 0UL;
862 vcpu->arch.sie_block->gpsw.addr = 0UL;
863 kvm_s390_set_prefix(vcpu, 0);
864 vcpu->arch.sie_block->cputm = 0UL;
865 vcpu->arch.sie_block->ckc = 0UL;
866 vcpu->arch.sie_block->todpr = 0;
867 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
868 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
869 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
870 vcpu->arch.guest_fpregs.fpc = 0;
871 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
872 vcpu->arch.sie_block->gbea = 1;
873 vcpu->arch.sie_block->pp = 0;
874 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
875 kvm_clear_async_pf_completion_queue(vcpu);
876 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
877 kvm_s390_vcpu_stop(vcpu);
878 kvm_s390_clear_local_irqs(vcpu);
879 }
880
881 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
882 {
883 mutex_lock(&vcpu->kvm->lock);
884 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
885 mutex_unlock(&vcpu->kvm->lock);
886 if (!kvm_is_ucontrol(vcpu->kvm))
887 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
888 }
889
890 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
891 {
892 if (!test_vfacility(76))
893 return;
894
895 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
896
897 if (vcpu->kvm->arch.crypto.aes_kw)
898 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
899 if (vcpu->kvm->arch.crypto.dea_kw)
900 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
901
902 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
903 }
904
905 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
906 {
907 free_page(vcpu->arch.sie_block->cbrlo);
908 vcpu->arch.sie_block->cbrlo = 0;
909 }
910
911 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
912 {
913 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
914 if (!vcpu->arch.sie_block->cbrlo)
915 return -ENOMEM;
916
917 vcpu->arch.sie_block->ecb2 |= 0x80;
918 vcpu->arch.sie_block->ecb2 &= ~0x08;
919 return 0;
920 }
921
922 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
923 {
924 int rc = 0;
925
926 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
927 CPUSTAT_SM |
928 CPUSTAT_STOPPED |
929 CPUSTAT_GED);
930 vcpu->arch.sie_block->ecb = 6;
931 if (test_vfacility(50) && test_vfacility(73))
932 vcpu->arch.sie_block->ecb |= 0x10;
933
934 vcpu->arch.sie_block->ecb2 = 8;
935 vcpu->arch.sie_block->eca = 0xC1002000U;
936 if (sclp_has_siif())
937 vcpu->arch.sie_block->eca |= 1;
938 if (sclp_has_sigpif())
939 vcpu->arch.sie_block->eca |= 0x10000000U;
940 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
941 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
942 ICTL_TPROT;
943
944 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
945 rc = kvm_s390_vcpu_setup_cmma(vcpu);
946 if (rc)
947 return rc;
948 }
949 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
950 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
951 get_cpu_id(&vcpu->arch.cpu_id);
952 vcpu->arch.cpu_id.version = 0xff;
953
954 kvm_s390_vcpu_crypto_setup(vcpu);
955
956 return rc;
957 }
958
959 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
960 unsigned int id)
961 {
962 struct kvm_vcpu *vcpu;
963 struct sie_page *sie_page;
964 int rc = -EINVAL;
965
966 if (id >= KVM_MAX_VCPUS)
967 goto out;
968
969 rc = -ENOMEM;
970
971 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
972 if (!vcpu)
973 goto out;
974
975 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
976 if (!sie_page)
977 goto out_free_cpu;
978
979 vcpu->arch.sie_block = &sie_page->sie_block;
980 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
981
982 vcpu->arch.sie_block->icpua = id;
983 if (!kvm_is_ucontrol(kvm)) {
984 if (!kvm->arch.sca) {
985 WARN_ON_ONCE(1);
986 goto out_free_cpu;
987 }
988 if (!kvm->arch.sca->cpu[id].sda)
989 kvm->arch.sca->cpu[id].sda =
990 (__u64) vcpu->arch.sie_block;
991 vcpu->arch.sie_block->scaoh =
992 (__u32)(((__u64)kvm->arch.sca) >> 32);
993 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
994 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
995 }
996
997 spin_lock_init(&vcpu->arch.local_int.lock);
998 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
999 vcpu->arch.local_int.wq = &vcpu->wq;
1000 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1001
1002 rc = kvm_vcpu_init(vcpu, kvm, id);
1003 if (rc)
1004 goto out_free_sie_block;
1005 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1006 vcpu->arch.sie_block);
1007 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1008
1009 return vcpu;
1010 out_free_sie_block:
1011 free_page((unsigned long)(vcpu->arch.sie_block));
1012 out_free_cpu:
1013 kmem_cache_free(kvm_vcpu_cache, vcpu);
1014 out:
1015 return ERR_PTR(rc);
1016 }
1017
1018 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1019 {
1020 return kvm_s390_vcpu_has_irq(vcpu, 0);
1021 }
1022
1023 void s390_vcpu_block(struct kvm_vcpu *vcpu)
1024 {
1025 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1026 }
1027
1028 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1029 {
1030 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1031 }
1032
1033 /*
1034 * Kick a guest cpu out of SIE and wait until SIE is not running.
1035 * If the CPU is not running (e.g. waiting as idle) the function will
1036 * return immediately. */
1037 void exit_sie(struct kvm_vcpu *vcpu)
1038 {
1039 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1040 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1041 cpu_relax();
1042 }
1043
1044 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
1045 void exit_sie_sync(struct kvm_vcpu *vcpu)
1046 {
1047 s390_vcpu_block(vcpu);
1048 exit_sie(vcpu);
1049 }
1050
1051 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1052 {
1053 int i;
1054 struct kvm *kvm = gmap->private;
1055 struct kvm_vcpu *vcpu;
1056
1057 kvm_for_each_vcpu(i, vcpu, kvm) {
1058 /* match against both prefix pages */
1059 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1060 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1061 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1062 exit_sie_sync(vcpu);
1063 }
1064 }
1065 }
1066
1067 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1068 {
1069 /* kvm common code refers to this, but never calls it */
1070 BUG();
1071 return 0;
1072 }
1073
1074 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1075 struct kvm_one_reg *reg)
1076 {
1077 int r = -EINVAL;
1078
1079 switch (reg->id) {
1080 case KVM_REG_S390_TODPR:
1081 r = put_user(vcpu->arch.sie_block->todpr,
1082 (u32 __user *)reg->addr);
1083 break;
1084 case KVM_REG_S390_EPOCHDIFF:
1085 r = put_user(vcpu->arch.sie_block->epoch,
1086 (u64 __user *)reg->addr);
1087 break;
1088 case KVM_REG_S390_CPU_TIMER:
1089 r = put_user(vcpu->arch.sie_block->cputm,
1090 (u64 __user *)reg->addr);
1091 break;
1092 case KVM_REG_S390_CLOCK_COMP:
1093 r = put_user(vcpu->arch.sie_block->ckc,
1094 (u64 __user *)reg->addr);
1095 break;
1096 case KVM_REG_S390_PFTOKEN:
1097 r = put_user(vcpu->arch.pfault_token,
1098 (u64 __user *)reg->addr);
1099 break;
1100 case KVM_REG_S390_PFCOMPARE:
1101 r = put_user(vcpu->arch.pfault_compare,
1102 (u64 __user *)reg->addr);
1103 break;
1104 case KVM_REG_S390_PFSELECT:
1105 r = put_user(vcpu->arch.pfault_select,
1106 (u64 __user *)reg->addr);
1107 break;
1108 case KVM_REG_S390_PP:
1109 r = put_user(vcpu->arch.sie_block->pp,
1110 (u64 __user *)reg->addr);
1111 break;
1112 case KVM_REG_S390_GBEA:
1113 r = put_user(vcpu->arch.sie_block->gbea,
1114 (u64 __user *)reg->addr);
1115 break;
1116 default:
1117 break;
1118 }
1119
1120 return r;
1121 }
1122
1123 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1124 struct kvm_one_reg *reg)
1125 {
1126 int r = -EINVAL;
1127
1128 switch (reg->id) {
1129 case KVM_REG_S390_TODPR:
1130 r = get_user(vcpu->arch.sie_block->todpr,
1131 (u32 __user *)reg->addr);
1132 break;
1133 case KVM_REG_S390_EPOCHDIFF:
1134 r = get_user(vcpu->arch.sie_block->epoch,
1135 (u64 __user *)reg->addr);
1136 break;
1137 case KVM_REG_S390_CPU_TIMER:
1138 r = get_user(vcpu->arch.sie_block->cputm,
1139 (u64 __user *)reg->addr);
1140 break;
1141 case KVM_REG_S390_CLOCK_COMP:
1142 r = get_user(vcpu->arch.sie_block->ckc,
1143 (u64 __user *)reg->addr);
1144 break;
1145 case KVM_REG_S390_PFTOKEN:
1146 r = get_user(vcpu->arch.pfault_token,
1147 (u64 __user *)reg->addr);
1148 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1149 kvm_clear_async_pf_completion_queue(vcpu);
1150 break;
1151 case KVM_REG_S390_PFCOMPARE:
1152 r = get_user(vcpu->arch.pfault_compare,
1153 (u64 __user *)reg->addr);
1154 break;
1155 case KVM_REG_S390_PFSELECT:
1156 r = get_user(vcpu->arch.pfault_select,
1157 (u64 __user *)reg->addr);
1158 break;
1159 case KVM_REG_S390_PP:
1160 r = get_user(vcpu->arch.sie_block->pp,
1161 (u64 __user *)reg->addr);
1162 break;
1163 case KVM_REG_S390_GBEA:
1164 r = get_user(vcpu->arch.sie_block->gbea,
1165 (u64 __user *)reg->addr);
1166 break;
1167 default:
1168 break;
1169 }
1170
1171 return r;
1172 }
1173
1174 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1175 {
1176 kvm_s390_vcpu_initial_reset(vcpu);
1177 return 0;
1178 }
1179
1180 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1181 {
1182 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1183 return 0;
1184 }
1185
1186 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1187 {
1188 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1189 return 0;
1190 }
1191
1192 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1193 struct kvm_sregs *sregs)
1194 {
1195 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1196 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1197 restore_access_regs(vcpu->run->s.regs.acrs);
1198 return 0;
1199 }
1200
1201 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1202 struct kvm_sregs *sregs)
1203 {
1204 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1205 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1206 return 0;
1207 }
1208
1209 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1210 {
1211 if (test_fp_ctl(fpu->fpc))
1212 return -EINVAL;
1213 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1214 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1215 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1216 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1217 return 0;
1218 }
1219
1220 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1221 {
1222 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1223 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1224 return 0;
1225 }
1226
1227 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1228 {
1229 int rc = 0;
1230
1231 if (!is_vcpu_stopped(vcpu))
1232 rc = -EBUSY;
1233 else {
1234 vcpu->run->psw_mask = psw.mask;
1235 vcpu->run->psw_addr = psw.addr;
1236 }
1237 return rc;
1238 }
1239
1240 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1241 struct kvm_translation *tr)
1242 {
1243 return -EINVAL; /* not implemented yet */
1244 }
1245
1246 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1247 KVM_GUESTDBG_USE_HW_BP | \
1248 KVM_GUESTDBG_ENABLE)
1249
1250 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1251 struct kvm_guest_debug *dbg)
1252 {
1253 int rc = 0;
1254
1255 vcpu->guest_debug = 0;
1256 kvm_s390_clear_bp_data(vcpu);
1257
1258 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1259 return -EINVAL;
1260
1261 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1262 vcpu->guest_debug = dbg->control;
1263 /* enforce guest PER */
1264 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1265
1266 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1267 rc = kvm_s390_import_bp_data(vcpu, dbg);
1268 } else {
1269 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1270 vcpu->arch.guestdbg.last_bp = 0;
1271 }
1272
1273 if (rc) {
1274 vcpu->guest_debug = 0;
1275 kvm_s390_clear_bp_data(vcpu);
1276 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1277 }
1278
1279 return rc;
1280 }
1281
1282 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1283 struct kvm_mp_state *mp_state)
1284 {
1285 /* CHECK_STOP and LOAD are not supported yet */
1286 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1287 KVM_MP_STATE_OPERATING;
1288 }
1289
1290 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1291 struct kvm_mp_state *mp_state)
1292 {
1293 int rc = 0;
1294
1295 /* user space knows about this interface - let it control the state */
1296 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1297
1298 switch (mp_state->mp_state) {
1299 case KVM_MP_STATE_STOPPED:
1300 kvm_s390_vcpu_stop(vcpu);
1301 break;
1302 case KVM_MP_STATE_OPERATING:
1303 kvm_s390_vcpu_start(vcpu);
1304 break;
1305 case KVM_MP_STATE_LOAD:
1306 case KVM_MP_STATE_CHECK_STOP:
1307 /* fall through - CHECK_STOP and LOAD are not supported yet */
1308 default:
1309 rc = -ENXIO;
1310 }
1311
1312 return rc;
1313 }
1314
1315 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1316 {
1317 if (!MACHINE_IS_LPAR)
1318 return false;
1319 /* only enable for z10 and later */
1320 if (!MACHINE_HAS_EDAT1)
1321 return false;
1322 if (!kvm->arch.use_cmma)
1323 return false;
1324 return true;
1325 }
1326
1327 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1328 {
1329 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1330 }
1331
1332 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1333 {
1334 retry:
1335 s390_vcpu_unblock(vcpu);
1336 /*
1337 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1338 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1339 * This ensures that the ipte instruction for this request has
1340 * already finished. We might race against a second unmapper that
1341 * wants to set the blocking bit. Lets just retry the request loop.
1342 */
1343 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1344 int rc;
1345 rc = gmap_ipte_notify(vcpu->arch.gmap,
1346 kvm_s390_get_prefix(vcpu),
1347 PAGE_SIZE * 2);
1348 if (rc)
1349 return rc;
1350 goto retry;
1351 }
1352
1353 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1354 vcpu->arch.sie_block->ihcpu = 0xffff;
1355 goto retry;
1356 }
1357
1358 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1359 if (!ibs_enabled(vcpu)) {
1360 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1361 atomic_set_mask(CPUSTAT_IBS,
1362 &vcpu->arch.sie_block->cpuflags);
1363 }
1364 goto retry;
1365 }
1366
1367 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1368 if (ibs_enabled(vcpu)) {
1369 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1370 atomic_clear_mask(CPUSTAT_IBS,
1371 &vcpu->arch.sie_block->cpuflags);
1372 }
1373 goto retry;
1374 }
1375
1376 /* nothing to do, just clear the request */
1377 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1378
1379 return 0;
1380 }
1381
1382 /**
1383 * kvm_arch_fault_in_page - fault-in guest page if necessary
1384 * @vcpu: The corresponding virtual cpu
1385 * @gpa: Guest physical address
1386 * @writable: Whether the page should be writable or not
1387 *
1388 * Make sure that a guest page has been faulted-in on the host.
1389 *
1390 * Return: Zero on success, negative error code otherwise.
1391 */
1392 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1393 {
1394 return gmap_fault(vcpu->arch.gmap, gpa,
1395 writable ? FAULT_FLAG_WRITE : 0);
1396 }
1397
1398 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1399 unsigned long token)
1400 {
1401 struct kvm_s390_interrupt inti;
1402 struct kvm_s390_irq irq;
1403
1404 if (start_token) {
1405 irq.u.ext.ext_params2 = token;
1406 irq.type = KVM_S390_INT_PFAULT_INIT;
1407 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1408 } else {
1409 inti.type = KVM_S390_INT_PFAULT_DONE;
1410 inti.parm64 = token;
1411 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1412 }
1413 }
1414
1415 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1416 struct kvm_async_pf *work)
1417 {
1418 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1419 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1420 }
1421
1422 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1423 struct kvm_async_pf *work)
1424 {
1425 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1426 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1427 }
1428
1429 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1430 struct kvm_async_pf *work)
1431 {
1432 /* s390 will always inject the page directly */
1433 }
1434
1435 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1436 {
1437 /*
1438 * s390 will always inject the page directly,
1439 * but we still want check_async_completion to cleanup
1440 */
1441 return true;
1442 }
1443
1444 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1445 {
1446 hva_t hva;
1447 struct kvm_arch_async_pf arch;
1448 int rc;
1449
1450 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1451 return 0;
1452 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1453 vcpu->arch.pfault_compare)
1454 return 0;
1455 if (psw_extint_disabled(vcpu))
1456 return 0;
1457 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1458 return 0;
1459 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1460 return 0;
1461 if (!vcpu->arch.gmap->pfault_enabled)
1462 return 0;
1463
1464 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1465 hva += current->thread.gmap_addr & ~PAGE_MASK;
1466 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1467 return 0;
1468
1469 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1470 return rc;
1471 }
1472
1473 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1474 {
1475 int rc, cpuflags;
1476
1477 /*
1478 * On s390 notifications for arriving pages will be delivered directly
1479 * to the guest but the house keeping for completed pfaults is
1480 * handled outside the worker.
1481 */
1482 kvm_check_async_pf_completion(vcpu);
1483
1484 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1485
1486 if (need_resched())
1487 schedule();
1488
1489 if (test_cpu_flag(CIF_MCCK_PENDING))
1490 s390_handle_mcck();
1491
1492 if (!kvm_is_ucontrol(vcpu->kvm)) {
1493 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1494 if (rc)
1495 return rc;
1496 }
1497
1498 rc = kvm_s390_handle_requests(vcpu);
1499 if (rc)
1500 return rc;
1501
1502 if (guestdbg_enabled(vcpu)) {
1503 kvm_s390_backup_guest_per_regs(vcpu);
1504 kvm_s390_patch_guest_per_regs(vcpu);
1505 }
1506
1507 vcpu->arch.sie_block->icptcode = 0;
1508 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1509 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1510 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1511
1512 return 0;
1513 }
1514
1515 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1516 {
1517 int rc = -1;
1518
1519 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1520 vcpu->arch.sie_block->icptcode);
1521 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1522
1523 if (guestdbg_enabled(vcpu))
1524 kvm_s390_restore_guest_per_regs(vcpu);
1525
1526 if (exit_reason >= 0) {
1527 rc = 0;
1528 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1529 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1530 vcpu->run->s390_ucontrol.trans_exc_code =
1531 current->thread.gmap_addr;
1532 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1533 rc = -EREMOTE;
1534
1535 } else if (current->thread.gmap_pfault) {
1536 trace_kvm_s390_major_guest_pfault(vcpu);
1537 current->thread.gmap_pfault = 0;
1538 if (kvm_arch_setup_async_pf(vcpu)) {
1539 rc = 0;
1540 } else {
1541 gpa_t gpa = current->thread.gmap_addr;
1542 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1543 }
1544 }
1545
1546 if (rc == -1) {
1547 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1548 trace_kvm_s390_sie_fault(vcpu);
1549 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1550 }
1551
1552 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1553
1554 if (rc == 0) {
1555 if (kvm_is_ucontrol(vcpu->kvm))
1556 /* Don't exit for host interrupts. */
1557 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1558 else
1559 rc = kvm_handle_sie_intercept(vcpu);
1560 }
1561
1562 return rc;
1563 }
1564
1565 static int __vcpu_run(struct kvm_vcpu *vcpu)
1566 {
1567 int rc, exit_reason;
1568
1569 /*
1570 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1571 * ning the guest), so that memslots (and other stuff) are protected
1572 */
1573 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1574
1575 do {
1576 rc = vcpu_pre_run(vcpu);
1577 if (rc)
1578 break;
1579
1580 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1581 /*
1582 * As PF_VCPU will be used in fault handler, between
1583 * guest_enter and guest_exit should be no uaccess.
1584 */
1585 preempt_disable();
1586 kvm_guest_enter();
1587 preempt_enable();
1588 exit_reason = sie64a(vcpu->arch.sie_block,
1589 vcpu->run->s.regs.gprs);
1590 kvm_guest_exit();
1591 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1592
1593 rc = vcpu_post_run(vcpu, exit_reason);
1594 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1595
1596 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1597 return rc;
1598 }
1599
1600 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1601 {
1602 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1603 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1604 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1605 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1606 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1607 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1608 /* some control register changes require a tlb flush */
1609 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1610 }
1611 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1612 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1613 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1614 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1615 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1616 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1617 }
1618 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1619 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1620 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1621 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1622 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1623 kvm_clear_async_pf_completion_queue(vcpu);
1624 }
1625 kvm_run->kvm_dirty_regs = 0;
1626 }
1627
1628 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1629 {
1630 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1631 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1632 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1633 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1634 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1635 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1636 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1637 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1638 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1639 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1640 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1641 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1642 }
1643
1644 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1645 {
1646 int rc;
1647 sigset_t sigsaved;
1648
1649 if (guestdbg_exit_pending(vcpu)) {
1650 kvm_s390_prepare_debug_exit(vcpu);
1651 return 0;
1652 }
1653
1654 if (vcpu->sigset_active)
1655 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1656
1657 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1658 kvm_s390_vcpu_start(vcpu);
1659 } else if (is_vcpu_stopped(vcpu)) {
1660 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1661 vcpu->vcpu_id);
1662 return -EINVAL;
1663 }
1664
1665 sync_regs(vcpu, kvm_run);
1666
1667 might_fault();
1668 rc = __vcpu_run(vcpu);
1669
1670 if (signal_pending(current) && !rc) {
1671 kvm_run->exit_reason = KVM_EXIT_INTR;
1672 rc = -EINTR;
1673 }
1674
1675 if (guestdbg_exit_pending(vcpu) && !rc) {
1676 kvm_s390_prepare_debug_exit(vcpu);
1677 rc = 0;
1678 }
1679
1680 if (rc == -EOPNOTSUPP) {
1681 /* intercept cannot be handled in-kernel, prepare kvm-run */
1682 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1683 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1684 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1685 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1686 rc = 0;
1687 }
1688
1689 if (rc == -EREMOTE) {
1690 /* intercept was handled, but userspace support is needed
1691 * kvm_run has been prepared by the handler */
1692 rc = 0;
1693 }
1694
1695 store_regs(vcpu, kvm_run);
1696
1697 if (vcpu->sigset_active)
1698 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1699
1700 vcpu->stat.exit_userspace++;
1701 return rc;
1702 }
1703
1704 /*
1705 * store status at address
1706 * we use have two special cases:
1707 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1708 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1709 */
1710 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1711 {
1712 unsigned char archmode = 1;
1713 unsigned int px;
1714 u64 clkcomp;
1715 int rc;
1716
1717 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1718 if (write_guest_abs(vcpu, 163, &archmode, 1))
1719 return -EFAULT;
1720 gpa = SAVE_AREA_BASE;
1721 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1722 if (write_guest_real(vcpu, 163, &archmode, 1))
1723 return -EFAULT;
1724 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1725 }
1726 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1727 vcpu->arch.guest_fpregs.fprs, 128);
1728 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1729 vcpu->run->s.regs.gprs, 128);
1730 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1731 &vcpu->arch.sie_block->gpsw, 16);
1732 px = kvm_s390_get_prefix(vcpu);
1733 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1734 &px, 4);
1735 rc |= write_guest_abs(vcpu,
1736 gpa + offsetof(struct save_area, fp_ctrl_reg),
1737 &vcpu->arch.guest_fpregs.fpc, 4);
1738 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1739 &vcpu->arch.sie_block->todpr, 4);
1740 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1741 &vcpu->arch.sie_block->cputm, 8);
1742 clkcomp = vcpu->arch.sie_block->ckc >> 8;
1743 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1744 &clkcomp, 8);
1745 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1746 &vcpu->run->s.regs.acrs, 64);
1747 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1748 &vcpu->arch.sie_block->gcr, 128);
1749 return rc ? -EFAULT : 0;
1750 }
1751
1752 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1753 {
1754 /*
1755 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1756 * copying in vcpu load/put. Lets update our copies before we save
1757 * it into the save area
1758 */
1759 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1760 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1761 save_access_regs(vcpu->run->s.regs.acrs);
1762
1763 return kvm_s390_store_status_unloaded(vcpu, addr);
1764 }
1765
1766 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1767 {
1768 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1769 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1770 exit_sie_sync(vcpu);
1771 }
1772
1773 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1774 {
1775 unsigned int i;
1776 struct kvm_vcpu *vcpu;
1777
1778 kvm_for_each_vcpu(i, vcpu, kvm) {
1779 __disable_ibs_on_vcpu(vcpu);
1780 }
1781 }
1782
1783 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1784 {
1785 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1786 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1787 exit_sie_sync(vcpu);
1788 }
1789
1790 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1791 {
1792 int i, online_vcpus, started_vcpus = 0;
1793
1794 if (!is_vcpu_stopped(vcpu))
1795 return;
1796
1797 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
1798 /* Only one cpu at a time may enter/leave the STOPPED state. */
1799 spin_lock(&vcpu->kvm->arch.start_stop_lock);
1800 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1801
1802 for (i = 0; i < online_vcpus; i++) {
1803 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1804 started_vcpus++;
1805 }
1806
1807 if (started_vcpus == 0) {
1808 /* we're the only active VCPU -> speed it up */
1809 __enable_ibs_on_vcpu(vcpu);
1810 } else if (started_vcpus == 1) {
1811 /*
1812 * As we are starting a second VCPU, we have to disable
1813 * the IBS facility on all VCPUs to remove potentially
1814 * oustanding ENABLE requests.
1815 */
1816 __disable_ibs_on_all_vcpus(vcpu->kvm);
1817 }
1818
1819 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1820 /*
1821 * Another VCPU might have used IBS while we were offline.
1822 * Let's play safe and flush the VCPU at startup.
1823 */
1824 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1825 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1826 return;
1827 }
1828
1829 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1830 {
1831 int i, online_vcpus, started_vcpus = 0;
1832 struct kvm_vcpu *started_vcpu = NULL;
1833
1834 if (is_vcpu_stopped(vcpu))
1835 return;
1836
1837 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
1838 /* Only one cpu at a time may enter/leave the STOPPED state. */
1839 spin_lock(&vcpu->kvm->arch.start_stop_lock);
1840 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1841
1842 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1843 kvm_s390_clear_stop_irq(vcpu);
1844
1845 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1846 __disable_ibs_on_vcpu(vcpu);
1847
1848 for (i = 0; i < online_vcpus; i++) {
1849 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1850 started_vcpus++;
1851 started_vcpu = vcpu->kvm->vcpus[i];
1852 }
1853 }
1854
1855 if (started_vcpus == 1) {
1856 /*
1857 * As we only have one VCPU left, we want to enable the
1858 * IBS facility for that VCPU to speed it up.
1859 */
1860 __enable_ibs_on_vcpu(started_vcpu);
1861 }
1862
1863 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1864 return;
1865 }
1866
1867 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1868 struct kvm_enable_cap *cap)
1869 {
1870 int r;
1871
1872 if (cap->flags)
1873 return -EINVAL;
1874
1875 switch (cap->cap) {
1876 case KVM_CAP_S390_CSS_SUPPORT:
1877 if (!vcpu->kvm->arch.css_support) {
1878 vcpu->kvm->arch.css_support = 1;
1879 trace_kvm_s390_enable_css(vcpu->kvm);
1880 }
1881 r = 0;
1882 break;
1883 default:
1884 r = -EINVAL;
1885 break;
1886 }
1887 return r;
1888 }
1889
1890 long kvm_arch_vcpu_ioctl(struct file *filp,
1891 unsigned int ioctl, unsigned long arg)
1892 {
1893 struct kvm_vcpu *vcpu = filp->private_data;
1894 void __user *argp = (void __user *)arg;
1895 int idx;
1896 long r;
1897
1898 switch (ioctl) {
1899 case KVM_S390_INTERRUPT: {
1900 struct kvm_s390_interrupt s390int;
1901 struct kvm_s390_irq s390irq;
1902
1903 r = -EFAULT;
1904 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1905 break;
1906 if (s390int_to_s390irq(&s390int, &s390irq))
1907 return -EINVAL;
1908 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
1909 break;
1910 }
1911 case KVM_S390_STORE_STATUS:
1912 idx = srcu_read_lock(&vcpu->kvm->srcu);
1913 r = kvm_s390_vcpu_store_status(vcpu, arg);
1914 srcu_read_unlock(&vcpu->kvm->srcu, idx);
1915 break;
1916 case KVM_S390_SET_INITIAL_PSW: {
1917 psw_t psw;
1918
1919 r = -EFAULT;
1920 if (copy_from_user(&psw, argp, sizeof(psw)))
1921 break;
1922 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1923 break;
1924 }
1925 case KVM_S390_INITIAL_RESET:
1926 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1927 break;
1928 case KVM_SET_ONE_REG:
1929 case KVM_GET_ONE_REG: {
1930 struct kvm_one_reg reg;
1931 r = -EFAULT;
1932 if (copy_from_user(&reg, argp, sizeof(reg)))
1933 break;
1934 if (ioctl == KVM_SET_ONE_REG)
1935 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1936 else
1937 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1938 break;
1939 }
1940 #ifdef CONFIG_KVM_S390_UCONTROL
1941 case KVM_S390_UCAS_MAP: {
1942 struct kvm_s390_ucas_mapping ucasmap;
1943
1944 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1945 r = -EFAULT;
1946 break;
1947 }
1948
1949 if (!kvm_is_ucontrol(vcpu->kvm)) {
1950 r = -EINVAL;
1951 break;
1952 }
1953
1954 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1955 ucasmap.vcpu_addr, ucasmap.length);
1956 break;
1957 }
1958 case KVM_S390_UCAS_UNMAP: {
1959 struct kvm_s390_ucas_mapping ucasmap;
1960
1961 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1962 r = -EFAULT;
1963 break;
1964 }
1965
1966 if (!kvm_is_ucontrol(vcpu->kvm)) {
1967 r = -EINVAL;
1968 break;
1969 }
1970
1971 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1972 ucasmap.length);
1973 break;
1974 }
1975 #endif
1976 case KVM_S390_VCPU_FAULT: {
1977 r = gmap_fault(vcpu->arch.gmap, arg, 0);
1978 break;
1979 }
1980 case KVM_ENABLE_CAP:
1981 {
1982 struct kvm_enable_cap cap;
1983 r = -EFAULT;
1984 if (copy_from_user(&cap, argp, sizeof(cap)))
1985 break;
1986 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1987 break;
1988 }
1989 default:
1990 r = -ENOTTY;
1991 }
1992 return r;
1993 }
1994
1995 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1996 {
1997 #ifdef CONFIG_KVM_S390_UCONTROL
1998 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1999 && (kvm_is_ucontrol(vcpu->kvm))) {
2000 vmf->page = virt_to_page(vcpu->arch.sie_block);
2001 get_page(vmf->page);
2002 return 0;
2003 }
2004 #endif
2005 return VM_FAULT_SIGBUS;
2006 }
2007
2008 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2009 unsigned long npages)
2010 {
2011 return 0;
2012 }
2013
2014 /* Section: memory related */
2015 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2016 struct kvm_memory_slot *memslot,
2017 struct kvm_userspace_memory_region *mem,
2018 enum kvm_mr_change change)
2019 {
2020 /* A few sanity checks. We can have memory slots which have to be
2021 located/ended at a segment boundary (1MB). The memory in userland is
2022 ok to be fragmented into various different vmas. It is okay to mmap()
2023 and munmap() stuff in this slot after doing this call at any time */
2024
2025 if (mem->userspace_addr & 0xffffful)
2026 return -EINVAL;
2027
2028 if (mem->memory_size & 0xffffful)
2029 return -EINVAL;
2030
2031 return 0;
2032 }
2033
2034 void kvm_arch_commit_memory_region(struct kvm *kvm,
2035 struct kvm_userspace_memory_region *mem,
2036 const struct kvm_memory_slot *old,
2037 enum kvm_mr_change change)
2038 {
2039 int rc;
2040
2041 /* If the basics of the memslot do not change, we do not want
2042 * to update the gmap. Every update causes several unnecessary
2043 * segment translation exceptions. This is usually handled just
2044 * fine by the normal fault handler + gmap, but it will also
2045 * cause faults on the prefix page of running guest CPUs.
2046 */
2047 if (old->userspace_addr == mem->userspace_addr &&
2048 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2049 old->npages * PAGE_SIZE == mem->memory_size)
2050 return;
2051
2052 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2053 mem->guest_phys_addr, mem->memory_size);
2054 if (rc)
2055 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
2056 return;
2057 }
2058
2059 static int __init kvm_s390_init(void)
2060 {
2061 int ret;
2062 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2063 if (ret)
2064 return ret;
2065
2066 /*
2067 * guests can ask for up to 255+1 double words, we need a full page
2068 * to hold the maximum amount of facilities. On the other hand, we
2069 * only set facilities that are known to work in KVM.
2070 */
2071 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
2072 if (!vfacilities) {
2073 kvm_exit();
2074 return -ENOMEM;
2075 }
2076 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
2077 vfacilities[0] &= 0xff82fffbf47c2000UL;
2078 vfacilities[1] &= 0x005c000000000000UL;
2079 return 0;
2080 }
2081
2082 static void __exit kvm_s390_exit(void)
2083 {
2084 free_page((unsigned long) vfacilities);
2085 kvm_exit();
2086 }
2087
2088 module_init(kvm_s390_init);
2089 module_exit(kvm_s390_exit);
2090
2091 /*
2092 * Enable autoloading of the kvm module.
2093 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2094 * since x86 takes a different approach.
2095 */
2096 #include <linux/miscdevice.h>
2097 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2098 MODULE_ALIAS("devname:kvm");
Linux kernel - Deliverables
This page took 0.569029 seconds and 5 git commands to generate.