projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branches 'acpi-video' and 'acpi-cppc'
[deliverable/linux.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
31 #include <asm/etr.h>
32 #include <asm/pgtable.h>
33 #include <asm/nmi.h>
34 #include <asm/switch_to.h>
35 #include <asm/isc.h>
36 #include <asm/sclp.h>
37 #include "kvm-s390.h"
38 #include "gaccess.h"
39
40 #define KMSG_COMPONENT "kvm-s390"
41 #undef pr_fmt
42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
43
44 #define CREATE_TRACE_POINTS
45 #include "trace.h"
46 #include "trace-s390.h"
47
48 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
49 #define LOCAL_IRQS 32
50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
51 (KVM_MAX_VCPUS + LOCAL_IRQS))
52
53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
54
55 struct kvm_stats_debugfs_item debugfs_entries[] = {
56 { "userspace_handled", VCPU_STAT(exit_userspace) },
57 { "exit_null", VCPU_STAT(exit_null) },
58 { "exit_validity", VCPU_STAT(exit_validity) },
59 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
60 { "exit_external_request", VCPU_STAT(exit_external_request) },
61 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
62 { "exit_instruction", VCPU_STAT(exit_instruction) },
63 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
64 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
65 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
66 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
67 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
68 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
69 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
70 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
71 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
72 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
73 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
74 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
75 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
76 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
77 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
78 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
79 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
80 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
81 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
82 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
83 { "instruction_spx", VCPU_STAT(instruction_spx) },
84 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
85 { "instruction_stap", VCPU_STAT(instruction_stap) },
86 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
87 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
88 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
89 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
90 { "instruction_essa", VCPU_STAT(instruction_essa) },
91 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
92 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
93 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
94 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
95 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
96 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
97 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
98 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
99 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
100 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
101 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
102 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
103 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
104 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
105 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
106 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
107 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
108 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
109 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
110 { "diagnose_10", VCPU_STAT(diagnose_10) },
111 { "diagnose_44", VCPU_STAT(diagnose_44) },
112 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
113 { "diagnose_258", VCPU_STAT(diagnose_258) },
114 { "diagnose_308", VCPU_STAT(diagnose_308) },
115 { "diagnose_500", VCPU_STAT(diagnose_500) },
116 { NULL }
117 };
118
119 /* upper facilities limit for kvm */
120 unsigned long kvm_s390_fac_list_mask[] = {
121 0xffe6fffbfcfdfc40UL,
122 0x005e800000000000UL,
123 };
124
125 unsigned long kvm_s390_fac_list_mask_size(void)
126 {
127 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
128 return ARRAY_SIZE(kvm_s390_fac_list_mask);
129 }
130
131 static struct gmap_notifier gmap_notifier;
132 debug_info_t *kvm_s390_dbf;
133
134 /* Section: not file related */
135 int kvm_arch_hardware_enable(void)
136 {
137 /* every s390 is virtualization enabled ;-) */
138 return 0;
139 }
140
141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
142
143 /*
144 * This callback is executed during stop_machine(). All CPUs are therefore
145 * temporarily stopped. In order not to change guest behavior, we have to
146 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
147 * so a CPU won't be stopped while calculating with the epoch.
148 */
149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
150 void *v)
151 {
152 struct kvm *kvm;
153 struct kvm_vcpu *vcpu;
154 int i;
155 unsigned long long *delta = v;
156
157 list_for_each_entry(kvm, &vm_list, vm_list) {
158 kvm->arch.epoch -= *delta;
159 kvm_for_each_vcpu(i, vcpu, kvm) {
160 vcpu->arch.sie_block->epoch -= *delta;
161 }
162 }
163 return NOTIFY_OK;
164 }
165
166 static struct notifier_block kvm_clock_notifier = {
167 .notifier_call = kvm_clock_sync,
168 };
169
170 int kvm_arch_hardware_setup(void)
171 {
172 gmap_notifier.notifier_call = kvm_gmap_notifier;
173 gmap_register_ipte_notifier(&gmap_notifier);
174 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
175 &kvm_clock_notifier);
176 return 0;
177 }
178
179 void kvm_arch_hardware_unsetup(void)
180 {
181 gmap_unregister_ipte_notifier(&gmap_notifier);
182 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
183 &kvm_clock_notifier);
184 }
185
186 int kvm_arch_init(void *opaque)
187 {
188 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
189 if (!kvm_s390_dbf)
190 return -ENOMEM;
191
192 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
193 debug_unregister(kvm_s390_dbf);
194 return -ENOMEM;
195 }
196
197 /* Register floating interrupt controller interface. */
198 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
199 }
200
201 void kvm_arch_exit(void)
202 {
203 debug_unregister(kvm_s390_dbf);
204 }
205
206 /* Section: device related */
207 long kvm_arch_dev_ioctl(struct file *filp,
208 unsigned int ioctl, unsigned long arg)
209 {
210 if (ioctl == KVM_S390_ENABLE_SIE)
211 return s390_enable_sie();
212 return -EINVAL;
213 }
214
215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
216 {
217 int r;
218
219 switch (ext) {
220 case KVM_CAP_S390_PSW:
221 case KVM_CAP_S390_GMAP:
222 case KVM_CAP_SYNC_MMU:
223 #ifdef CONFIG_KVM_S390_UCONTROL
224 case KVM_CAP_S390_UCONTROL:
225 #endif
226 case KVM_CAP_ASYNC_PF:
227 case KVM_CAP_SYNC_REGS:
228 case KVM_CAP_ONE_REG:
229 case KVM_CAP_ENABLE_CAP:
230 case KVM_CAP_S390_CSS_SUPPORT:
231 case KVM_CAP_IOEVENTFD:
232 case KVM_CAP_DEVICE_CTRL:
233 case KVM_CAP_ENABLE_CAP_VM:
234 case KVM_CAP_S390_IRQCHIP:
235 case KVM_CAP_VM_ATTRIBUTES:
236 case KVM_CAP_MP_STATE:
237 case KVM_CAP_S390_INJECT_IRQ:
238 case KVM_CAP_S390_USER_SIGP:
239 case KVM_CAP_S390_USER_STSI:
240 case KVM_CAP_S390_SKEYS:
241 case KVM_CAP_S390_IRQ_STATE:
242 r = 1;
243 break;
244 case KVM_CAP_S390_MEM_OP:
245 r = MEM_OP_MAX_SIZE;
246 break;
247 case KVM_CAP_NR_VCPUS:
248 case KVM_CAP_MAX_VCPUS:
249 r = KVM_MAX_VCPUS;
250 break;
251 case KVM_CAP_NR_MEMSLOTS:
252 r = KVM_USER_MEM_SLOTS;
253 break;
254 case KVM_CAP_S390_COW:
255 r = MACHINE_HAS_ESOP;
256 break;
257 case KVM_CAP_S390_VECTOR_REGISTERS:
258 r = MACHINE_HAS_VX;
259 break;
260 default:
261 r = 0;
262 }
263 return r;
264 }
265
266 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
267 struct kvm_memory_slot *memslot)
268 {
269 gfn_t cur_gfn, last_gfn;
270 unsigned long address;
271 struct gmap *gmap = kvm->arch.gmap;
272
273 down_read(&gmap->mm->mmap_sem);
274 /* Loop over all guest pages */
275 last_gfn = memslot->base_gfn + memslot->npages;
276 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
277 address = gfn_to_hva_memslot(memslot, cur_gfn);
278
279 if (gmap_test_and_clear_dirty(address, gmap))
280 mark_page_dirty(kvm, cur_gfn);
281 }
282 up_read(&gmap->mm->mmap_sem);
283 }
284
285 /* Section: vm related */
286 /*
287 * Get (and clear) the dirty memory log for a memory slot.
288 */
289 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
290 struct kvm_dirty_log *log)
291 {
292 int r;
293 unsigned long n;
294 struct kvm_memslots *slots;
295 struct kvm_memory_slot *memslot;
296 int is_dirty = 0;
297
298 mutex_lock(&kvm->slots_lock);
299
300 r = -EINVAL;
301 if (log->slot >= KVM_USER_MEM_SLOTS)
302 goto out;
303
304 slots = kvm_memslots(kvm);
305 memslot = id_to_memslot(slots, log->slot);
306 r = -ENOENT;
307 if (!memslot->dirty_bitmap)
308 goto out;
309
310 kvm_s390_sync_dirty_log(kvm, memslot);
311 r = kvm_get_dirty_log(kvm, log, &is_dirty);
312 if (r)
313 goto out;
314
315 /* Clear the dirty log */
316 if (is_dirty) {
317 n = kvm_dirty_bitmap_bytes(memslot);
318 memset(memslot->dirty_bitmap, 0, n);
319 }
320 r = 0;
321 out:
322 mutex_unlock(&kvm->slots_lock);
323 return r;
324 }
325
326 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
327 {
328 int r;
329
330 if (cap->flags)
331 return -EINVAL;
332
333 switch (cap->cap) {
334 case KVM_CAP_S390_IRQCHIP:
335 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
336 kvm->arch.use_irqchip = 1;
337 r = 0;
338 break;
339 case KVM_CAP_S390_USER_SIGP:
340 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
341 kvm->arch.user_sigp = 1;
342 r = 0;
343 break;
344 case KVM_CAP_S390_VECTOR_REGISTERS:
345 if (MACHINE_HAS_VX) {
346 set_kvm_facility(kvm->arch.model.fac->mask, 129);
347 set_kvm_facility(kvm->arch.model.fac->list, 129);
348 r = 0;
349 } else
350 r = -EINVAL;
351 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
352 r ? "(not available)" : "(success)");
353 break;
354 case KVM_CAP_S390_USER_STSI:
355 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
356 kvm->arch.user_stsi = 1;
357 r = 0;
358 break;
359 default:
360 r = -EINVAL;
361 break;
362 }
363 return r;
364 }
365
366 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
367 {
368 int ret;
369
370 switch (attr->attr) {
371 case KVM_S390_VM_MEM_LIMIT_SIZE:
372 ret = 0;
373 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
374 kvm->arch.gmap->asce_end);
375 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
376 ret = -EFAULT;
377 break;
378 default:
379 ret = -ENXIO;
380 break;
381 }
382 return ret;
383 }
384
385 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
386 {
387 int ret;
388 unsigned int idx;
389 switch (attr->attr) {
390 case KVM_S390_VM_MEM_ENABLE_CMMA:
391 /* enable CMMA only for z10 and later (EDAT_1) */
392 ret = -EINVAL;
393 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
394 break;
395
396 ret = -EBUSY;
397 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
398 mutex_lock(&kvm->lock);
399 if (atomic_read(&kvm->online_vcpus) == 0) {
400 kvm->arch.use_cmma = 1;
401 ret = 0;
402 }
403 mutex_unlock(&kvm->lock);
404 break;
405 case KVM_S390_VM_MEM_CLR_CMMA:
406 ret = -EINVAL;
407 if (!kvm->arch.use_cmma)
408 break;
409
410 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
411 mutex_lock(&kvm->lock);
412 idx = srcu_read_lock(&kvm->srcu);
413 s390_reset_cmma(kvm->arch.gmap->mm);
414 srcu_read_unlock(&kvm->srcu, idx);
415 mutex_unlock(&kvm->lock);
416 ret = 0;
417 break;
418 case KVM_S390_VM_MEM_LIMIT_SIZE: {
419 unsigned long new_limit;
420
421 if (kvm_is_ucontrol(kvm))
422 return -EINVAL;
423
424 if (get_user(new_limit, (u64 __user *)attr->addr))
425 return -EFAULT;
426
427 if (new_limit > kvm->arch.gmap->asce_end)
428 return -E2BIG;
429
430 ret = -EBUSY;
431 mutex_lock(&kvm->lock);
432 if (atomic_read(&kvm->online_vcpus) == 0) {
433 /* gmap_alloc will round the limit up */
434 struct gmap *new = gmap_alloc(current->mm, new_limit);
435
436 if (!new) {
437 ret = -ENOMEM;
438 } else {
439 gmap_free(kvm->arch.gmap);
440 new->private = kvm;
441 kvm->arch.gmap = new;
442 ret = 0;
443 }
444 }
445 mutex_unlock(&kvm->lock);
446 VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
447 break;
448 }
449 default:
450 ret = -ENXIO;
451 break;
452 }
453 return ret;
454 }
455
456 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
457
458 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
459 {
460 struct kvm_vcpu *vcpu;
461 int i;
462
463 if (!test_kvm_facility(kvm, 76))
464 return -EINVAL;
465
466 mutex_lock(&kvm->lock);
467 switch (attr->attr) {
468 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
469 get_random_bytes(
470 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
471 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
472 kvm->arch.crypto.aes_kw = 1;
473 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
474 break;
475 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
476 get_random_bytes(
477 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
478 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
479 kvm->arch.crypto.dea_kw = 1;
480 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
481 break;
482 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
483 kvm->arch.crypto.aes_kw = 0;
484 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
485 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
486 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
487 break;
488 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
489 kvm->arch.crypto.dea_kw = 0;
490 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
491 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
492 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
493 break;
494 default:
495 mutex_unlock(&kvm->lock);
496 return -ENXIO;
497 }
498
499 kvm_for_each_vcpu(i, vcpu, kvm) {
500 kvm_s390_vcpu_crypto_setup(vcpu);
501 exit_sie(vcpu);
502 }
503 mutex_unlock(&kvm->lock);
504 return 0;
505 }
506
507 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
508 {
509 u8 gtod_high;
510
511 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
512 sizeof(gtod_high)))
513 return -EFAULT;
514
515 if (gtod_high != 0)
516 return -EINVAL;
517 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x\n", gtod_high);
518
519 return 0;
520 }
521
522 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
523 {
524 struct kvm_vcpu *cur_vcpu;
525 unsigned int vcpu_idx;
526 u64 host_tod, gtod;
527 int r;
528
529 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
530 return -EFAULT;
531
532 r = store_tod_clock(&host_tod);
533 if (r)
534 return r;
535
536 mutex_lock(&kvm->lock);
537 preempt_disable();
538 kvm->arch.epoch = gtod - host_tod;
539 kvm_s390_vcpu_block_all(kvm);
540 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm)
541 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
542 kvm_s390_vcpu_unblock_all(kvm);
543 preempt_enable();
544 mutex_unlock(&kvm->lock);
545 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx\n", gtod);
546 return 0;
547 }
548
549 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
550 {
551 int ret;
552
553 if (attr->flags)
554 return -EINVAL;
555
556 switch (attr->attr) {
557 case KVM_S390_VM_TOD_HIGH:
558 ret = kvm_s390_set_tod_high(kvm, attr);
559 break;
560 case KVM_S390_VM_TOD_LOW:
561 ret = kvm_s390_set_tod_low(kvm, attr);
562 break;
563 default:
564 ret = -ENXIO;
565 break;
566 }
567 return ret;
568 }
569
570 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
571 {
572 u8 gtod_high = 0;
573
574 if (copy_to_user((void __user *)attr->addr, &gtod_high,
575 sizeof(gtod_high)))
576 return -EFAULT;
577 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x\n", gtod_high);
578
579 return 0;
580 }
581
582 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
583 {
584 u64 host_tod, gtod;
585 int r;
586
587 r = store_tod_clock(&host_tod);
588 if (r)
589 return r;
590
591 preempt_disable();
592 gtod = host_tod + kvm->arch.epoch;
593 preempt_enable();
594 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
595 return -EFAULT;
596 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx\n", gtod);
597
598 return 0;
599 }
600
601 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
602 {
603 int ret;
604
605 if (attr->flags)
606 return -EINVAL;
607
608 switch (attr->attr) {
609 case KVM_S390_VM_TOD_HIGH:
610 ret = kvm_s390_get_tod_high(kvm, attr);
611 break;
612 case KVM_S390_VM_TOD_LOW:
613 ret = kvm_s390_get_tod_low(kvm, attr);
614 break;
615 default:
616 ret = -ENXIO;
617 break;
618 }
619 return ret;
620 }
621
622 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
623 {
624 struct kvm_s390_vm_cpu_processor *proc;
625 int ret = 0;
626
627 mutex_lock(&kvm->lock);
628 if (atomic_read(&kvm->online_vcpus)) {
629 ret = -EBUSY;
630 goto out;
631 }
632 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
633 if (!proc) {
634 ret = -ENOMEM;
635 goto out;
636 }
637 if (!copy_from_user(proc, (void __user *)attr->addr,
638 sizeof(*proc))) {
639 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
640 sizeof(struct cpuid));
641 kvm->arch.model.ibc = proc->ibc;
642 memcpy(kvm->arch.model.fac->list, proc->fac_list,
643 S390_ARCH_FAC_LIST_SIZE_BYTE);
644 } else
645 ret = -EFAULT;
646 kfree(proc);
647 out:
648 mutex_unlock(&kvm->lock);
649 return ret;
650 }
651
652 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
653 {
654 int ret = -ENXIO;
655
656 switch (attr->attr) {
657 case KVM_S390_VM_CPU_PROCESSOR:
658 ret = kvm_s390_set_processor(kvm, attr);
659 break;
660 }
661 return ret;
662 }
663
664 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
665 {
666 struct kvm_s390_vm_cpu_processor *proc;
667 int ret = 0;
668
669 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
670 if (!proc) {
671 ret = -ENOMEM;
672 goto out;
673 }
674 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
675 proc->ibc = kvm->arch.model.ibc;
676 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
677 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
678 ret = -EFAULT;
679 kfree(proc);
680 out:
681 return ret;
682 }
683
684 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
685 {
686 struct kvm_s390_vm_cpu_machine *mach;
687 int ret = 0;
688
689 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
690 if (!mach) {
691 ret = -ENOMEM;
692 goto out;
693 }
694 get_cpu_id((struct cpuid *) &mach->cpuid);
695 mach->ibc = sclp.ibc;
696 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
697 S390_ARCH_FAC_LIST_SIZE_BYTE);
698 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
699 S390_ARCH_FAC_LIST_SIZE_BYTE);
700 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
701 ret = -EFAULT;
702 kfree(mach);
703 out:
704 return ret;
705 }
706
707 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
708 {
709 int ret = -ENXIO;
710
711 switch (attr->attr) {
712 case KVM_S390_VM_CPU_PROCESSOR:
713 ret = kvm_s390_get_processor(kvm, attr);
714 break;
715 case KVM_S390_VM_CPU_MACHINE:
716 ret = kvm_s390_get_machine(kvm, attr);
717 break;
718 }
719 return ret;
720 }
721
722 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
723 {
724 int ret;
725
726 switch (attr->group) {
727 case KVM_S390_VM_MEM_CTRL:
728 ret = kvm_s390_set_mem_control(kvm, attr);
729 break;
730 case KVM_S390_VM_TOD:
731 ret = kvm_s390_set_tod(kvm, attr);
732 break;
733 case KVM_S390_VM_CPU_MODEL:
734 ret = kvm_s390_set_cpu_model(kvm, attr);
735 break;
736 case KVM_S390_VM_CRYPTO:
737 ret = kvm_s390_vm_set_crypto(kvm, attr);
738 break;
739 default:
740 ret = -ENXIO;
741 break;
742 }
743
744 return ret;
745 }
746
747 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
748 {
749 int ret;
750
751 switch (attr->group) {
752 case KVM_S390_VM_MEM_CTRL:
753 ret = kvm_s390_get_mem_control(kvm, attr);
754 break;
755 case KVM_S390_VM_TOD:
756 ret = kvm_s390_get_tod(kvm, attr);
757 break;
758 case KVM_S390_VM_CPU_MODEL:
759 ret = kvm_s390_get_cpu_model(kvm, attr);
760 break;
761 default:
762 ret = -ENXIO;
763 break;
764 }
765
766 return ret;
767 }
768
769 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
770 {
771 int ret;
772
773 switch (attr->group) {
774 case KVM_S390_VM_MEM_CTRL:
775 switch (attr->attr) {
776 case KVM_S390_VM_MEM_ENABLE_CMMA:
777 case KVM_S390_VM_MEM_CLR_CMMA:
778 case KVM_S390_VM_MEM_LIMIT_SIZE:
779 ret = 0;
780 break;
781 default:
782 ret = -ENXIO;
783 break;
784 }
785 break;
786 case KVM_S390_VM_TOD:
787 switch (attr->attr) {
788 case KVM_S390_VM_TOD_LOW:
789 case KVM_S390_VM_TOD_HIGH:
790 ret = 0;
791 break;
792 default:
793 ret = -ENXIO;
794 break;
795 }
796 break;
797 case KVM_S390_VM_CPU_MODEL:
798 switch (attr->attr) {
799 case KVM_S390_VM_CPU_PROCESSOR:
800 case KVM_S390_VM_CPU_MACHINE:
801 ret = 0;
802 break;
803 default:
804 ret = -ENXIO;
805 break;
806 }
807 break;
808 case KVM_S390_VM_CRYPTO:
809 switch (attr->attr) {
810 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
811 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
812 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
813 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
814 ret = 0;
815 break;
816 default:
817 ret = -ENXIO;
818 break;
819 }
820 break;
821 default:
822 ret = -ENXIO;
823 break;
824 }
825
826 return ret;
827 }
828
829 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
830 {
831 uint8_t *keys;
832 uint64_t hva;
833 unsigned long curkey;
834 int i, r = 0;
835
836 if (args->flags != 0)
837 return -EINVAL;
838
839 /* Is this guest using storage keys? */
840 if (!mm_use_skey(current->mm))
841 return KVM_S390_GET_SKEYS_NONE;
842
843 /* Enforce sane limit on memory allocation */
844 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
845 return -EINVAL;
846
847 keys = kmalloc_array(args->count, sizeof(uint8_t),
848 GFP_KERNEL | __GFP_NOWARN);
849 if (!keys)
850 keys = vmalloc(sizeof(uint8_t) * args->count);
851 if (!keys)
852 return -ENOMEM;
853
854 for (i = 0; i < args->count; i++) {
855 hva = gfn_to_hva(kvm, args->start_gfn + i);
856 if (kvm_is_error_hva(hva)) {
857 r = -EFAULT;
858 goto out;
859 }
860
861 curkey = get_guest_storage_key(current->mm, hva);
862 if (IS_ERR_VALUE(curkey)) {
863 r = curkey;
864 goto out;
865 }
866 keys[i] = curkey;
867 }
868
869 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
870 sizeof(uint8_t) * args->count);
871 if (r)
872 r = -EFAULT;
873 out:
874 kvfree(keys);
875 return r;
876 }
877
878 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
879 {
880 uint8_t *keys;
881 uint64_t hva;
882 int i, r = 0;
883
884 if (args->flags != 0)
885 return -EINVAL;
886
887 /* Enforce sane limit on memory allocation */
888 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
889 return -EINVAL;
890
891 keys = kmalloc_array(args->count, sizeof(uint8_t),
892 GFP_KERNEL | __GFP_NOWARN);
893 if (!keys)
894 keys = vmalloc(sizeof(uint8_t) * args->count);
895 if (!keys)
896 return -ENOMEM;
897
898 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
899 sizeof(uint8_t) * args->count);
900 if (r) {
901 r = -EFAULT;
902 goto out;
903 }
904
905 /* Enable storage key handling for the guest */
906 r = s390_enable_skey();
907 if (r)
908 goto out;
909
910 for (i = 0; i < args->count; i++) {
911 hva = gfn_to_hva(kvm, args->start_gfn + i);
912 if (kvm_is_error_hva(hva)) {
913 r = -EFAULT;
914 goto out;
915 }
916
917 /* Lowest order bit is reserved */
918 if (keys[i] & 0x01) {
919 r = -EINVAL;
920 goto out;
921 }
922
923 r = set_guest_storage_key(current->mm, hva,
924 (unsigned long)keys[i], 0);
925 if (r)
926 goto out;
927 }
928 out:
929 kvfree(keys);
930 return r;
931 }
932
933 long kvm_arch_vm_ioctl(struct file *filp,
934 unsigned int ioctl, unsigned long arg)
935 {
936 struct kvm *kvm = filp->private_data;
937 void __user *argp = (void __user *)arg;
938 struct kvm_device_attr attr;
939 int r;
940
941 switch (ioctl) {
942 case KVM_S390_INTERRUPT: {
943 struct kvm_s390_interrupt s390int;
944
945 r = -EFAULT;
946 if (copy_from_user(&s390int, argp, sizeof(s390int)))
947 break;
948 r = kvm_s390_inject_vm(kvm, &s390int);
949 break;
950 }
951 case KVM_ENABLE_CAP: {
952 struct kvm_enable_cap cap;
953 r = -EFAULT;
954 if (copy_from_user(&cap, argp, sizeof(cap)))
955 break;
956 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
957 break;
958 }
959 case KVM_CREATE_IRQCHIP: {
960 struct kvm_irq_routing_entry routing;
961
962 r = -EINVAL;
963 if (kvm->arch.use_irqchip) {
964 /* Set up dummy routing. */
965 memset(&routing, 0, sizeof(routing));
966 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
967 }
968 break;
969 }
970 case KVM_SET_DEVICE_ATTR: {
971 r = -EFAULT;
972 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
973 break;
974 r = kvm_s390_vm_set_attr(kvm, &attr);
975 break;
976 }
977 case KVM_GET_DEVICE_ATTR: {
978 r = -EFAULT;
979 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
980 break;
981 r = kvm_s390_vm_get_attr(kvm, &attr);
982 break;
983 }
984 case KVM_HAS_DEVICE_ATTR: {
985 r = -EFAULT;
986 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
987 break;
988 r = kvm_s390_vm_has_attr(kvm, &attr);
989 break;
990 }
991 case KVM_S390_GET_SKEYS: {
992 struct kvm_s390_skeys args;
993
994 r = -EFAULT;
995 if (copy_from_user(&args, argp,
996 sizeof(struct kvm_s390_skeys)))
997 break;
998 r = kvm_s390_get_skeys(kvm, &args);
999 break;
1000 }
1001 case KVM_S390_SET_SKEYS: {
1002 struct kvm_s390_skeys args;
1003
1004 r = -EFAULT;
1005 if (copy_from_user(&args, argp,
1006 sizeof(struct kvm_s390_skeys)))
1007 break;
1008 r = kvm_s390_set_skeys(kvm, &args);
1009 break;
1010 }
1011 default:
1012 r = -ENOTTY;
1013 }
1014
1015 return r;
1016 }
1017
1018 static int kvm_s390_query_ap_config(u8 *config)
1019 {
1020 u32 fcn_code = 0x04000000UL;
1021 u32 cc = 0;
1022
1023 memset(config, 0, 128);
1024 asm volatile(
1025 "lgr 0,%1\n"
1026 "lgr 2,%2\n"
1027 ".long 0xb2af0000\n" /* PQAP(QCI) */
1028 "0: ipm %0\n"
1029 "srl %0,28\n"
1030 "1:\n"
1031 EX_TABLE(0b, 1b)
1032 : "+r" (cc)
1033 : "r" (fcn_code), "r" (config)
1034 : "cc", "0", "2", "memory"
1035 );
1036
1037 return cc;
1038 }
1039
1040 static int kvm_s390_apxa_installed(void)
1041 {
1042 u8 config[128];
1043 int cc;
1044
1045 if (test_facility(2) && test_facility(12)) {
1046 cc = kvm_s390_query_ap_config(config);
1047
1048 if (cc)
1049 pr_err("PQAP(QCI) failed with cc=%d", cc);
1050 else
1051 return config[0] & 0x40;
1052 }
1053
1054 return 0;
1055 }
1056
1057 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1058 {
1059 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1060
1061 if (kvm_s390_apxa_installed())
1062 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1063 else
1064 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1065 }
1066
1067 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1068 {
1069 get_cpu_id(cpu_id);
1070 cpu_id->version = 0xff;
1071 }
1072
1073 static int kvm_s390_crypto_init(struct kvm *kvm)
1074 {
1075 if (!test_kvm_facility(kvm, 76))
1076 return 0;
1077
1078 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1079 GFP_KERNEL | GFP_DMA);
1080 if (!kvm->arch.crypto.crycb)
1081 return -ENOMEM;
1082
1083 kvm_s390_set_crycb_format(kvm);
1084
1085 /* Enable AES/DEA protected key functions by default */
1086 kvm->arch.crypto.aes_kw = 1;
1087 kvm->arch.crypto.dea_kw = 1;
1088 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1089 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1090 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1091 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1092
1093 return 0;
1094 }
1095
1096 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1097 {
1098 int i, rc;
1099 char debug_name[16];
1100 static unsigned long sca_offset;
1101
1102 rc = -EINVAL;
1103 #ifdef CONFIG_KVM_S390_UCONTROL
1104 if (type & ~KVM_VM_S390_UCONTROL)
1105 goto out_err;
1106 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1107 goto out_err;
1108 #else
1109 if (type)
1110 goto out_err;
1111 #endif
1112
1113 rc = s390_enable_sie();
1114 if (rc)
1115 goto out_err;
1116
1117 rc = -ENOMEM;
1118
1119 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1120 if (!kvm->arch.sca)
1121 goto out_err;
1122 spin_lock(&kvm_lock);
1123 sca_offset = (sca_offset + 16) & 0x7f0;
1124 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1125 spin_unlock(&kvm_lock);
1126
1127 sprintf(debug_name, "kvm-%u", current->pid);
1128
1129 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1130 if (!kvm->arch.dbf)
1131 goto out_err;
1132
1133 /*
1134 * The architectural maximum amount of facilities is 16 kbit. To store
1135 * this amount, 2 kbyte of memory is required. Thus we need a full
1136 * page to hold the guest facility list (arch.model.fac->list) and the
1137 * facility mask (arch.model.fac->mask). Its address size has to be
1138 * 31 bits and word aligned.
1139 */
1140 kvm->arch.model.fac =
1141 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1142 if (!kvm->arch.model.fac)
1143 goto out_err;
1144
1145 /* Populate the facility mask initially. */
1146 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1147 S390_ARCH_FAC_LIST_SIZE_BYTE);
1148 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1149 if (i < kvm_s390_fac_list_mask_size())
1150 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1151 else
1152 kvm->arch.model.fac->mask[i] = 0UL;
1153 }
1154
1155 /* Populate the facility list initially. */
1156 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1157 S390_ARCH_FAC_LIST_SIZE_BYTE);
1158
1159 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1160 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1161
1162 if (kvm_s390_crypto_init(kvm) < 0)
1163 goto out_err;
1164
1165 spin_lock_init(&kvm->arch.float_int.lock);
1166 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1167 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1168 init_waitqueue_head(&kvm->arch.ipte_wq);
1169 mutex_init(&kvm->arch.ipte_mutex);
1170
1171 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1172 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1173
1174 if (type & KVM_VM_S390_UCONTROL) {
1175 kvm->arch.gmap = NULL;
1176 } else {
1177 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1178 if (!kvm->arch.gmap)
1179 goto out_err;
1180 kvm->arch.gmap->private = kvm;
1181 kvm->arch.gmap->pfault_enabled = 0;
1182 }
1183
1184 kvm->arch.css_support = 0;
1185 kvm->arch.use_irqchip = 0;
1186 kvm->arch.epoch = 0;
1187
1188 spin_lock_init(&kvm->arch.start_stop_lock);
1189 KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
1190
1191 return 0;
1192 out_err:
1193 kfree(kvm->arch.crypto.crycb);
1194 free_page((unsigned long)kvm->arch.model.fac);
1195 debug_unregister(kvm->arch.dbf);
1196 free_page((unsigned long)(kvm->arch.sca));
1197 KVM_EVENT(3, "creation of vm failed: %d", rc);
1198 return rc;
1199 }
1200
1201 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1202 {
1203 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1204 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1205 kvm_s390_clear_local_irqs(vcpu);
1206 kvm_clear_async_pf_completion_queue(vcpu);
1207 if (!kvm_is_ucontrol(vcpu->kvm)) {
1208 clear_bit(63 - vcpu->vcpu_id,
1209 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1210 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1211 (__u64) vcpu->arch.sie_block)
1212 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1213 }
1214 smp_mb();
1215
1216 if (kvm_is_ucontrol(vcpu->kvm))
1217 gmap_free(vcpu->arch.gmap);
1218
1219 if (vcpu->kvm->arch.use_cmma)
1220 kvm_s390_vcpu_unsetup_cmma(vcpu);
1221 free_page((unsigned long)(vcpu->arch.sie_block));
1222
1223 kvm_vcpu_uninit(vcpu);
1224 kmem_cache_free(kvm_vcpu_cache, vcpu);
1225 }
1226
1227 static void kvm_free_vcpus(struct kvm *kvm)
1228 {
1229 unsigned int i;
1230 struct kvm_vcpu *vcpu;
1231
1232 kvm_for_each_vcpu(i, vcpu, kvm)
1233 kvm_arch_vcpu_destroy(vcpu);
1234
1235 mutex_lock(&kvm->lock);
1236 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1237 kvm->vcpus[i] = NULL;
1238
1239 atomic_set(&kvm->online_vcpus, 0);
1240 mutex_unlock(&kvm->lock);
1241 }
1242
1243 void kvm_arch_destroy_vm(struct kvm *kvm)
1244 {
1245 kvm_free_vcpus(kvm);
1246 free_page((unsigned long)kvm->arch.model.fac);
1247 free_page((unsigned long)(kvm->arch.sca));
1248 debug_unregister(kvm->arch.dbf);
1249 kfree(kvm->arch.crypto.crycb);
1250 if (!kvm_is_ucontrol(kvm))
1251 gmap_free(kvm->arch.gmap);
1252 kvm_s390_destroy_adapters(kvm);
1253 kvm_s390_clear_float_irqs(kvm);
1254 KVM_EVENT(3, "vm 0x%p destroyed", kvm);
1255 }
1256
1257 /* Section: vcpu related */
1258 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1259 {
1260 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1261 if (!vcpu->arch.gmap)
1262 return -ENOMEM;
1263 vcpu->arch.gmap->private = vcpu->kvm;
1264
1265 return 0;
1266 }
1267
1268 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1269 {
1270 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1271 kvm_clear_async_pf_completion_queue(vcpu);
1272 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1273 KVM_SYNC_GPRS |
1274 KVM_SYNC_ACRS |
1275 KVM_SYNC_CRS |
1276 KVM_SYNC_ARCH0 |
1277 KVM_SYNC_PFAULT;
1278 if (test_kvm_facility(vcpu->kvm, 129))
1279 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1280
1281 if (kvm_is_ucontrol(vcpu->kvm))
1282 return __kvm_ucontrol_vcpu_init(vcpu);
1283
1284 return 0;
1285 }
1286
1287 /*
1288 * Backs up the current FP/VX register save area on a particular
1289 * destination. Used to switch between different register save
1290 * areas.
1291 */
1292 static inline void save_fpu_to(struct fpu *dst)
1293 {
1294 dst->fpc = current->thread.fpu.fpc;
1295 dst->regs = current->thread.fpu.regs;
1296 }
1297
1298 /*
1299 * Switches the FP/VX register save area from which to lazy
1300 * restore register contents.
1301 */
1302 static inline void load_fpu_from(struct fpu *from)
1303 {
1304 current->thread.fpu.fpc = from->fpc;
1305 current->thread.fpu.regs = from->regs;
1306 }
1307
1308 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1309 {
1310 /* Save host register state */
1311 save_fpu_regs();
1312 save_fpu_to(&vcpu->arch.host_fpregs);
1313
1314 if (test_kvm_facility(vcpu->kvm, 129)) {
1315 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1316 /*
1317 * Use the register save area in the SIE-control block
1318 * for register restore and save in kvm_arch_vcpu_put()
1319 */
1320 current->thread.fpu.vxrs =
1321 (__vector128 *)&vcpu->run->s.regs.vrs;
1322 } else
1323 load_fpu_from(&vcpu->arch.guest_fpregs);
1324
1325 if (test_fp_ctl(current->thread.fpu.fpc))
1326 /* User space provided an invalid FPC, let's clear it */
1327 current->thread.fpu.fpc = 0;
1328
1329 save_access_regs(vcpu->arch.host_acrs);
1330 restore_access_regs(vcpu->run->s.regs.acrs);
1331 gmap_enable(vcpu->arch.gmap);
1332 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1333 }
1334
1335 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1336 {
1337 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1338 gmap_disable(vcpu->arch.gmap);
1339
1340 save_fpu_regs();
1341
1342 if (test_kvm_facility(vcpu->kvm, 129))
1343 /*
1344 * kvm_arch_vcpu_load() set up the register save area to
1345 * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1346 * are already saved. Only the floating-point control must be
1347 * copied.
1348 */
1349 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1350 else
1351 save_fpu_to(&vcpu->arch.guest_fpregs);
1352 load_fpu_from(&vcpu->arch.host_fpregs);
1353
1354 save_access_regs(vcpu->run->s.regs.acrs);
1355 restore_access_regs(vcpu->arch.host_acrs);
1356 }
1357
1358 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1359 {
1360 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1361 vcpu->arch.sie_block->gpsw.mask = 0UL;
1362 vcpu->arch.sie_block->gpsw.addr = 0UL;
1363 kvm_s390_set_prefix(vcpu, 0);
1364 vcpu->arch.sie_block->cputm = 0UL;
1365 vcpu->arch.sie_block->ckc = 0UL;
1366 vcpu->arch.sie_block->todpr = 0;
1367 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1368 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1369 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1370 vcpu->arch.guest_fpregs.fpc = 0;
1371 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1372 vcpu->arch.sie_block->gbea = 1;
1373 vcpu->arch.sie_block->pp = 0;
1374 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1375 kvm_clear_async_pf_completion_queue(vcpu);
1376 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1377 kvm_s390_vcpu_stop(vcpu);
1378 kvm_s390_clear_local_irqs(vcpu);
1379 }
1380
1381 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1382 {
1383 mutex_lock(&vcpu->kvm->lock);
1384 preempt_disable();
1385 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1386 preempt_enable();
1387 mutex_unlock(&vcpu->kvm->lock);
1388 if (!kvm_is_ucontrol(vcpu->kvm))
1389 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1390 }
1391
1392 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1393 {
1394 if (!test_kvm_facility(vcpu->kvm, 76))
1395 return;
1396
1397 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1398
1399 if (vcpu->kvm->arch.crypto.aes_kw)
1400 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1401 if (vcpu->kvm->arch.crypto.dea_kw)
1402 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1403
1404 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1405 }
1406
1407 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1408 {
1409 free_page(vcpu->arch.sie_block->cbrlo);
1410 vcpu->arch.sie_block->cbrlo = 0;
1411 }
1412
1413 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1414 {
1415 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1416 if (!vcpu->arch.sie_block->cbrlo)
1417 return -ENOMEM;
1418
1419 vcpu->arch.sie_block->ecb2 |= 0x80;
1420 vcpu->arch.sie_block->ecb2 &= ~0x08;
1421 return 0;
1422 }
1423
1424 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1425 {
1426 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1427
1428 vcpu->arch.cpu_id = model->cpu_id;
1429 vcpu->arch.sie_block->ibc = model->ibc;
1430 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1431 }
1432
1433 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1434 {
1435 int rc = 0;
1436
1437 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1438 CPUSTAT_SM |
1439 CPUSTAT_STOPPED);
1440
1441 if (test_kvm_facility(vcpu->kvm, 78))
1442 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1443 else if (test_kvm_facility(vcpu->kvm, 8))
1444 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1445
1446 kvm_s390_vcpu_setup_model(vcpu);
1447
1448 vcpu->arch.sie_block->ecb = 6;
1449 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1450 vcpu->arch.sie_block->ecb |= 0x10;
1451
1452 vcpu->arch.sie_block->ecb2 = 8;
1453 vcpu->arch.sie_block->eca = 0xC1002000U;
1454 if (sclp.has_siif)
1455 vcpu->arch.sie_block->eca |= 1;
1456 if (sclp.has_sigpif)
1457 vcpu->arch.sie_block->eca |= 0x10000000U;
1458 if (test_kvm_facility(vcpu->kvm, 129)) {
1459 vcpu->arch.sie_block->eca |= 0x00020000;
1460 vcpu->arch.sie_block->ecd |= 0x20000000;
1461 }
1462 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1463
1464 if (vcpu->kvm->arch.use_cmma) {
1465 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1466 if (rc)
1467 return rc;
1468 }
1469 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1470 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1471
1472 kvm_s390_vcpu_crypto_setup(vcpu);
1473
1474 return rc;
1475 }
1476
1477 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1478 unsigned int id)
1479 {
1480 struct kvm_vcpu *vcpu;
1481 struct sie_page *sie_page;
1482 int rc = -EINVAL;
1483
1484 if (id >= KVM_MAX_VCPUS)
1485 goto out;
1486
1487 rc = -ENOMEM;
1488
1489 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1490 if (!vcpu)
1491 goto out;
1492
1493 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1494 if (!sie_page)
1495 goto out_free_cpu;
1496
1497 vcpu->arch.sie_block = &sie_page->sie_block;
1498 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1499
1500 vcpu->arch.sie_block->icpua = id;
1501 if (!kvm_is_ucontrol(kvm)) {
1502 if (!kvm->arch.sca) {
1503 WARN_ON_ONCE(1);
1504 goto out_free_cpu;
1505 }
1506 if (!kvm->arch.sca->cpu[id].sda)
1507 kvm->arch.sca->cpu[id].sda =
1508 (__u64) vcpu->arch.sie_block;
1509 vcpu->arch.sie_block->scaoh =
1510 (__u32)(((__u64)kvm->arch.sca) >> 32);
1511 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1512 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1513 }
1514
1515 spin_lock_init(&vcpu->arch.local_int.lock);
1516 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1517 vcpu->arch.local_int.wq = &vcpu->wq;
1518 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1519
1520 /*
1521 * Allocate a save area for floating-point registers. If the vector
1522 * extension is available, register contents are saved in the SIE
1523 * control block. The allocated save area is still required in
1524 * particular places, for example, in kvm_s390_vcpu_store_status().
1525 */
1526 vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1527 GFP_KERNEL);
1528 if (!vcpu->arch.guest_fpregs.fprs) {
1529 rc = -ENOMEM;
1530 goto out_free_sie_block;
1531 }
1532
1533 rc = kvm_vcpu_init(vcpu, kvm, id);
1534 if (rc)
1535 goto out_free_sie_block;
1536 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1537 vcpu->arch.sie_block);
1538 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1539
1540 return vcpu;
1541 out_free_sie_block:
1542 free_page((unsigned long)(vcpu->arch.sie_block));
1543 out_free_cpu:
1544 kmem_cache_free(kvm_vcpu_cache, vcpu);
1545 out:
1546 return ERR_PTR(rc);
1547 }
1548
1549 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1550 {
1551 return kvm_s390_vcpu_has_irq(vcpu, 0);
1552 }
1553
1554 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1555 {
1556 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1557 exit_sie(vcpu);
1558 }
1559
1560 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1561 {
1562 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1563 }
1564
1565 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1566 {
1567 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1568 exit_sie(vcpu);
1569 }
1570
1571 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1572 {
1573 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1574 }
1575
1576 /*
1577 * Kick a guest cpu out of SIE and wait until SIE is not running.
1578 * If the CPU is not running (e.g. waiting as idle) the function will
1579 * return immediately. */
1580 void exit_sie(struct kvm_vcpu *vcpu)
1581 {
1582 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1583 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1584 cpu_relax();
1585 }
1586
1587 /* Kick a guest cpu out of SIE to process a request synchronously */
1588 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1589 {
1590 kvm_make_request(req, vcpu);
1591 kvm_s390_vcpu_request(vcpu);
1592 }
1593
1594 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1595 {
1596 int i;
1597 struct kvm *kvm = gmap->private;
1598 struct kvm_vcpu *vcpu;
1599
1600 kvm_for_each_vcpu(i, vcpu, kvm) {
1601 /* match against both prefix pages */
1602 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1603 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1604 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1605 }
1606 }
1607 }
1608
1609 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1610 {
1611 /* kvm common code refers to this, but never calls it */
1612 BUG();
1613 return 0;
1614 }
1615
1616 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1617 struct kvm_one_reg *reg)
1618 {
1619 int r = -EINVAL;
1620
1621 switch (reg->id) {
1622 case KVM_REG_S390_TODPR:
1623 r = put_user(vcpu->arch.sie_block->todpr,
1624 (u32 __user *)reg->addr);
1625 break;
1626 case KVM_REG_S390_EPOCHDIFF:
1627 r = put_user(vcpu->arch.sie_block->epoch,
1628 (u64 __user *)reg->addr);
1629 break;
1630 case KVM_REG_S390_CPU_TIMER:
1631 r = put_user(vcpu->arch.sie_block->cputm,
1632 (u64 __user *)reg->addr);
1633 break;
1634 case KVM_REG_S390_CLOCK_COMP:
1635 r = put_user(vcpu->arch.sie_block->ckc,
1636 (u64 __user *)reg->addr);
1637 break;
1638 case KVM_REG_S390_PFTOKEN:
1639 r = put_user(vcpu->arch.pfault_token,
1640 (u64 __user *)reg->addr);
1641 break;
1642 case KVM_REG_S390_PFCOMPARE:
1643 r = put_user(vcpu->arch.pfault_compare,
1644 (u64 __user *)reg->addr);
1645 break;
1646 case KVM_REG_S390_PFSELECT:
1647 r = put_user(vcpu->arch.pfault_select,
1648 (u64 __user *)reg->addr);
1649 break;
1650 case KVM_REG_S390_PP:
1651 r = put_user(vcpu->arch.sie_block->pp,
1652 (u64 __user *)reg->addr);
1653 break;
1654 case KVM_REG_S390_GBEA:
1655 r = put_user(vcpu->arch.sie_block->gbea,
1656 (u64 __user *)reg->addr);
1657 break;
1658 default:
1659 break;
1660 }
1661
1662 return r;
1663 }
1664
1665 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1666 struct kvm_one_reg *reg)
1667 {
1668 int r = -EINVAL;
1669
1670 switch (reg->id) {
1671 case KVM_REG_S390_TODPR:
1672 r = get_user(vcpu->arch.sie_block->todpr,
1673 (u32 __user *)reg->addr);
1674 break;
1675 case KVM_REG_S390_EPOCHDIFF:
1676 r = get_user(vcpu->arch.sie_block->epoch,
1677 (u64 __user *)reg->addr);
1678 break;
1679 case KVM_REG_S390_CPU_TIMER:
1680 r = get_user(vcpu->arch.sie_block->cputm,
1681 (u64 __user *)reg->addr);
1682 break;
1683 case KVM_REG_S390_CLOCK_COMP:
1684 r = get_user(vcpu->arch.sie_block->ckc,
1685 (u64 __user *)reg->addr);
1686 break;
1687 case KVM_REG_S390_PFTOKEN:
1688 r = get_user(vcpu->arch.pfault_token,
1689 (u64 __user *)reg->addr);
1690 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1691 kvm_clear_async_pf_completion_queue(vcpu);
1692 break;
1693 case KVM_REG_S390_PFCOMPARE:
1694 r = get_user(vcpu->arch.pfault_compare,
1695 (u64 __user *)reg->addr);
1696 break;
1697 case KVM_REG_S390_PFSELECT:
1698 r = get_user(vcpu->arch.pfault_select,
1699 (u64 __user *)reg->addr);
1700 break;
1701 case KVM_REG_S390_PP:
1702 r = get_user(vcpu->arch.sie_block->pp,
1703 (u64 __user *)reg->addr);
1704 break;
1705 case KVM_REG_S390_GBEA:
1706 r = get_user(vcpu->arch.sie_block->gbea,
1707 (u64 __user *)reg->addr);
1708 break;
1709 default:
1710 break;
1711 }
1712
1713 return r;
1714 }
1715
1716 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1717 {
1718 kvm_s390_vcpu_initial_reset(vcpu);
1719 return 0;
1720 }
1721
1722 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1723 {
1724 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1725 return 0;
1726 }
1727
1728 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1729 {
1730 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1731 return 0;
1732 }
1733
1734 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1735 struct kvm_sregs *sregs)
1736 {
1737 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1738 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1739 restore_access_regs(vcpu->run->s.regs.acrs);
1740 return 0;
1741 }
1742
1743 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1744 struct kvm_sregs *sregs)
1745 {
1746 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1747 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1748 return 0;
1749 }
1750
1751 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1752 {
1753 if (test_fp_ctl(fpu->fpc))
1754 return -EINVAL;
1755 memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1756 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1757 save_fpu_regs();
1758 load_fpu_from(&vcpu->arch.guest_fpregs);
1759 return 0;
1760 }
1761
1762 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1763 {
1764 memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1765 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1766 return 0;
1767 }
1768
1769 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1770 {
1771 int rc = 0;
1772
1773 if (!is_vcpu_stopped(vcpu))
1774 rc = -EBUSY;
1775 else {
1776 vcpu->run->psw_mask = psw.mask;
1777 vcpu->run->psw_addr = psw.addr;
1778 }
1779 return rc;
1780 }
1781
1782 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1783 struct kvm_translation *tr)
1784 {
1785 return -EINVAL; /* not implemented yet */
1786 }
1787
1788 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1789 KVM_GUESTDBG_USE_HW_BP | \
1790 KVM_GUESTDBG_ENABLE)
1791
1792 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1793 struct kvm_guest_debug *dbg)
1794 {
1795 int rc = 0;
1796
1797 vcpu->guest_debug = 0;
1798 kvm_s390_clear_bp_data(vcpu);
1799
1800 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1801 return -EINVAL;
1802
1803 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1804 vcpu->guest_debug = dbg->control;
1805 /* enforce guest PER */
1806 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1807
1808 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1809 rc = kvm_s390_import_bp_data(vcpu, dbg);
1810 } else {
1811 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1812 vcpu->arch.guestdbg.last_bp = 0;
1813 }
1814
1815 if (rc) {
1816 vcpu->guest_debug = 0;
1817 kvm_s390_clear_bp_data(vcpu);
1818 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1819 }
1820
1821 return rc;
1822 }
1823
1824 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1825 struct kvm_mp_state *mp_state)
1826 {
1827 /* CHECK_STOP and LOAD are not supported yet */
1828 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1829 KVM_MP_STATE_OPERATING;
1830 }
1831
1832 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1833 struct kvm_mp_state *mp_state)
1834 {
1835 int rc = 0;
1836
1837 /* user space knows about this interface - let it control the state */
1838 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1839
1840 switch (mp_state->mp_state) {
1841 case KVM_MP_STATE_STOPPED:
1842 kvm_s390_vcpu_stop(vcpu);
1843 break;
1844 case KVM_MP_STATE_OPERATING:
1845 kvm_s390_vcpu_start(vcpu);
1846 break;
1847 case KVM_MP_STATE_LOAD:
1848 case KVM_MP_STATE_CHECK_STOP:
1849 /* fall through - CHECK_STOP and LOAD are not supported yet */
1850 default:
1851 rc = -ENXIO;
1852 }
1853
1854 return rc;
1855 }
1856
1857 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1858 {
1859 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1860 }
1861
1862 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1863 {
1864 retry:
1865 kvm_s390_vcpu_request_handled(vcpu);
1866 if (!vcpu->requests)
1867 return 0;
1868 /*
1869 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1870 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1871 * This ensures that the ipte instruction for this request has
1872 * already finished. We might race against a second unmapper that
1873 * wants to set the blocking bit. Lets just retry the request loop.
1874 */
1875 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1876 int rc;
1877 rc = gmap_ipte_notify(vcpu->arch.gmap,
1878 kvm_s390_get_prefix(vcpu),
1879 PAGE_SIZE * 2);
1880 if (rc)
1881 return rc;
1882 goto retry;
1883 }
1884
1885 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1886 vcpu->arch.sie_block->ihcpu = 0xffff;
1887 goto retry;
1888 }
1889
1890 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1891 if (!ibs_enabled(vcpu)) {
1892 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1893 atomic_or(CPUSTAT_IBS,
1894 &vcpu->arch.sie_block->cpuflags);
1895 }
1896 goto retry;
1897 }
1898
1899 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1900 if (ibs_enabled(vcpu)) {
1901 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1902 atomic_andnot(CPUSTAT_IBS,
1903 &vcpu->arch.sie_block->cpuflags);
1904 }
1905 goto retry;
1906 }
1907
1908 /* nothing to do, just clear the request */
1909 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1910
1911 return 0;
1912 }
1913
1914 /**
1915 * kvm_arch_fault_in_page - fault-in guest page if necessary
1916 * @vcpu: The corresponding virtual cpu
1917 * @gpa: Guest physical address
1918 * @writable: Whether the page should be writable or not
1919 *
1920 * Make sure that a guest page has been faulted-in on the host.
1921 *
1922 * Return: Zero on success, negative error code otherwise.
1923 */
1924 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1925 {
1926 return gmap_fault(vcpu->arch.gmap, gpa,
1927 writable ? FAULT_FLAG_WRITE : 0);
1928 }
1929
1930 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1931 unsigned long token)
1932 {
1933 struct kvm_s390_interrupt inti;
1934 struct kvm_s390_irq irq;
1935
1936 if (start_token) {
1937 irq.u.ext.ext_params2 = token;
1938 irq.type = KVM_S390_INT_PFAULT_INIT;
1939 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1940 } else {
1941 inti.type = KVM_S390_INT_PFAULT_DONE;
1942 inti.parm64 = token;
1943 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1944 }
1945 }
1946
1947 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1948 struct kvm_async_pf *work)
1949 {
1950 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1951 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1952 }
1953
1954 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1955 struct kvm_async_pf *work)
1956 {
1957 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1958 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1959 }
1960
1961 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1962 struct kvm_async_pf *work)
1963 {
1964 /* s390 will always inject the page directly */
1965 }
1966
1967 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1968 {
1969 /*
1970 * s390 will always inject the page directly,
1971 * but we still want check_async_completion to cleanup
1972 */
1973 return true;
1974 }
1975
1976 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1977 {
1978 hva_t hva;
1979 struct kvm_arch_async_pf arch;
1980 int rc;
1981
1982 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1983 return 0;
1984 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1985 vcpu->arch.pfault_compare)
1986 return 0;
1987 if (psw_extint_disabled(vcpu))
1988 return 0;
1989 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1990 return 0;
1991 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1992 return 0;
1993 if (!vcpu->arch.gmap->pfault_enabled)
1994 return 0;
1995
1996 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1997 hva += current->thread.gmap_addr & ~PAGE_MASK;
1998 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1999 return 0;
2000
2001 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2002 return rc;
2003 }
2004
2005 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2006 {
2007 int rc, cpuflags;
2008
2009 /*
2010 * On s390 notifications for arriving pages will be delivered directly
2011 * to the guest but the house keeping for completed pfaults is
2012 * handled outside the worker.
2013 */
2014 kvm_check_async_pf_completion(vcpu);
2015
2016 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
2017
2018 if (need_resched())
2019 schedule();
2020
2021 if (test_cpu_flag(CIF_MCCK_PENDING))
2022 s390_handle_mcck();
2023
2024 if (!kvm_is_ucontrol(vcpu->kvm)) {
2025 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2026 if (rc)
2027 return rc;
2028 }
2029
2030 rc = kvm_s390_handle_requests(vcpu);
2031 if (rc)
2032 return rc;
2033
2034 if (guestdbg_enabled(vcpu)) {
2035 kvm_s390_backup_guest_per_regs(vcpu);
2036 kvm_s390_patch_guest_per_regs(vcpu);
2037 }
2038
2039 vcpu->arch.sie_block->icptcode = 0;
2040 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2041 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2042 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2043
2044 return 0;
2045 }
2046
2047 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2048 {
2049 psw_t *psw = &vcpu->arch.sie_block->gpsw;
2050 u8 opcode;
2051 int rc;
2052
2053 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2054 trace_kvm_s390_sie_fault(vcpu);
2055
2056 /*
2057 * We want to inject an addressing exception, which is defined as a
2058 * suppressing or terminating exception. However, since we came here
2059 * by a DAT access exception, the PSW still points to the faulting
2060 * instruction since DAT exceptions are nullifying. So we've got
2061 * to look up the current opcode to get the length of the instruction
2062 * to be able to forward the PSW.
2063 */
2064 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
2065 if (rc)
2066 return kvm_s390_inject_prog_cond(vcpu, rc);
2067 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2068
2069 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2070 }
2071
2072 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2073 {
2074 int rc = -1;
2075
2076 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2077 vcpu->arch.sie_block->icptcode);
2078 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2079
2080 if (guestdbg_enabled(vcpu))
2081 kvm_s390_restore_guest_per_regs(vcpu);
2082
2083 if (exit_reason >= 0) {
2084 rc = 0;
2085 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2086 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2087 vcpu->run->s390_ucontrol.trans_exc_code =
2088 current->thread.gmap_addr;
2089 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2090 rc = -EREMOTE;
2091
2092 } else if (current->thread.gmap_pfault) {
2093 trace_kvm_s390_major_guest_pfault(vcpu);
2094 current->thread.gmap_pfault = 0;
2095 if (kvm_arch_setup_async_pf(vcpu)) {
2096 rc = 0;
2097 } else {
2098 gpa_t gpa = current->thread.gmap_addr;
2099 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
2100 }
2101 }
2102
2103 if (rc == -1)
2104 rc = vcpu_post_run_fault_in_sie(vcpu);
2105
2106 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
2107
2108 if (rc == 0) {
2109 if (kvm_is_ucontrol(vcpu->kvm))
2110 /* Don't exit for host interrupts. */
2111 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
2112 else
2113 rc = kvm_handle_sie_intercept(vcpu);
2114 }
2115
2116 return rc;
2117 }
2118
2119 static int __vcpu_run(struct kvm_vcpu *vcpu)
2120 {
2121 int rc, exit_reason;
2122
2123 /*
2124 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2125 * ning the guest), so that memslots (and other stuff) are protected
2126 */
2127 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2128
2129 do {
2130 rc = vcpu_pre_run(vcpu);
2131 if (rc)
2132 break;
2133
2134 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2135 /*
2136 * As PF_VCPU will be used in fault handler, between
2137 * guest_enter and guest_exit should be no uaccess.
2138 */
2139 local_irq_disable();
2140 __kvm_guest_enter();
2141 local_irq_enable();
2142 exit_reason = sie64a(vcpu->arch.sie_block,
2143 vcpu->run->s.regs.gprs);
2144 local_irq_disable();
2145 __kvm_guest_exit();
2146 local_irq_enable();
2147 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2148
2149 rc = vcpu_post_run(vcpu, exit_reason);
2150 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2151
2152 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2153 return rc;
2154 }
2155
2156 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2157 {
2158 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2159 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2160 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2161 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2162 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2163 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2164 /* some control register changes require a tlb flush */
2165 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2166 }
2167 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2168 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2169 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2170 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2171 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2172 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2173 }
2174 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2175 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2176 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2177 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2178 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2179 kvm_clear_async_pf_completion_queue(vcpu);
2180 }
2181 kvm_run->kvm_dirty_regs = 0;
2182 }
2183
2184 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2185 {
2186 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2187 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2188 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2189 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2190 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2191 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2192 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2193 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2194 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2195 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2196 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2197 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2198 }
2199
2200 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2201 {
2202 int rc;
2203 sigset_t sigsaved;
2204
2205 if (guestdbg_exit_pending(vcpu)) {
2206 kvm_s390_prepare_debug_exit(vcpu);
2207 return 0;
2208 }
2209
2210 if (vcpu->sigset_active)
2211 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2212
2213 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2214 kvm_s390_vcpu_start(vcpu);
2215 } else if (is_vcpu_stopped(vcpu)) {
2216 pr_err_ratelimited("can't run stopped vcpu %d\n",
2217 vcpu->vcpu_id);
2218 return -EINVAL;
2219 }
2220
2221 sync_regs(vcpu, kvm_run);
2222
2223 might_fault();
2224 rc = __vcpu_run(vcpu);
2225
2226 if (signal_pending(current) && !rc) {
2227 kvm_run->exit_reason = KVM_EXIT_INTR;
2228 rc = -EINTR;
2229 }
2230
2231 if (guestdbg_exit_pending(vcpu) && !rc) {
2232 kvm_s390_prepare_debug_exit(vcpu);
2233 rc = 0;
2234 }
2235
2236 if (rc == -EOPNOTSUPP) {
2237 /* intercept cannot be handled in-kernel, prepare kvm-run */
2238 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
2239 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2240 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2241 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2242 rc = 0;
2243 }
2244
2245 if (rc == -EREMOTE) {
2246 /* intercept was handled, but userspace support is needed
2247 * kvm_run has been prepared by the handler */
2248 rc = 0;
2249 }
2250
2251 store_regs(vcpu, kvm_run);
2252
2253 if (vcpu->sigset_active)
2254 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2255
2256 vcpu->stat.exit_userspace++;
2257 return rc;
2258 }
2259
2260 /*
2261 * store status at address
2262 * we use have two special cases:
2263 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2264 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2265 */
2266 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2267 {
2268 unsigned char archmode = 1;
2269 unsigned int px;
2270 u64 clkcomp;
2271 int rc;
2272
2273 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2274 if (write_guest_abs(vcpu, 163, &archmode, 1))
2275 return -EFAULT;
2276 gpa = SAVE_AREA_BASE;
2277 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2278 if (write_guest_real(vcpu, 163, &archmode, 1))
2279 return -EFAULT;
2280 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2281 }
2282 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2283 vcpu->arch.guest_fpregs.fprs, 128);
2284 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2285 vcpu->run->s.regs.gprs, 128);
2286 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2287 &vcpu->arch.sie_block->gpsw, 16);
2288 px = kvm_s390_get_prefix(vcpu);
2289 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2290 &px, 4);
2291 rc |= write_guest_abs(vcpu,
2292 gpa + offsetof(struct save_area, fp_ctrl_reg),
2293 &vcpu->arch.guest_fpregs.fpc, 4);
2294 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2295 &vcpu->arch.sie_block->todpr, 4);
2296 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2297 &vcpu->arch.sie_block->cputm, 8);
2298 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2299 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2300 &clkcomp, 8);
2301 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2302 &vcpu->run->s.regs.acrs, 64);
2303 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2304 &vcpu->arch.sie_block->gcr, 128);
2305 return rc ? -EFAULT : 0;
2306 }
2307
2308 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2309 {
2310 /*
2311 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2312 * copying in vcpu load/put. Lets update our copies before we save
2313 * it into the save area
2314 */
2315 save_fpu_regs();
2316 if (test_kvm_facility(vcpu->kvm, 129)) {
2317 /*
2318 * If the vector extension is available, the vector registers
2319 * which overlaps with floating-point registers are saved in
2320 * the SIE-control block. Hence, extract the floating-point
2321 * registers and the FPC value and store them in the
2322 * guest_fpregs structure.
2323 */
2324 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2325 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2326 current->thread.fpu.vxrs);
2327 } else
2328 save_fpu_to(&vcpu->arch.guest_fpregs);
2329 save_access_regs(vcpu->run->s.regs.acrs);
2330
2331 return kvm_s390_store_status_unloaded(vcpu, addr);
2332 }
2333
2334 /*
2335 * store additional status at address
2336 */
2337 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2338 unsigned long gpa)
2339 {
2340 /* Only bits 0-53 are used for address formation */
2341 if (!(gpa & ~0x3ff))
2342 return 0;
2343
2344 return write_guest_abs(vcpu, gpa & ~0x3ff,
2345 (void *)&vcpu->run->s.regs.vrs, 512);
2346 }
2347
2348 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2349 {
2350 if (!test_kvm_facility(vcpu->kvm, 129))
2351 return 0;
2352
2353 /*
2354 * The guest VXRS are in the host VXRs due to the lazy
2355 * copying in vcpu load/put. We can simply call save_fpu_regs()
2356 * to save the current register state because we are in the
2357 * middle of a load/put cycle.
2358 *
2359 * Let's update our copies before we save it into the save area.
2360 */
2361 save_fpu_regs();
2362
2363 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2364 }
2365
2366 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2367 {
2368 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2369 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2370 }
2371
2372 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2373 {
2374 unsigned int i;
2375 struct kvm_vcpu *vcpu;
2376
2377 kvm_for_each_vcpu(i, vcpu, kvm) {
2378 __disable_ibs_on_vcpu(vcpu);
2379 }
2380 }
2381
2382 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2383 {
2384 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2385 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2386 }
2387
2388 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2389 {
2390 int i, online_vcpus, started_vcpus = 0;
2391
2392 if (!is_vcpu_stopped(vcpu))
2393 return;
2394
2395 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2396 /* Only one cpu at a time may enter/leave the STOPPED state. */
2397 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2398 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2399
2400 for (i = 0; i < online_vcpus; i++) {
2401 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2402 started_vcpus++;
2403 }
2404
2405 if (started_vcpus == 0) {
2406 /* we're the only active VCPU -> speed it up */
2407 __enable_ibs_on_vcpu(vcpu);
2408 } else if (started_vcpus == 1) {
2409 /*
2410 * As we are starting a second VCPU, we have to disable
2411 * the IBS facility on all VCPUs to remove potentially
2412 * oustanding ENABLE requests.
2413 */
2414 __disable_ibs_on_all_vcpus(vcpu->kvm);
2415 }
2416
2417 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2418 /*
2419 * Another VCPU might have used IBS while we were offline.
2420 * Let's play safe and flush the VCPU at startup.
2421 */
2422 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2423 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2424 return;
2425 }
2426
2427 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2428 {
2429 int i, online_vcpus, started_vcpus = 0;
2430 struct kvm_vcpu *started_vcpu = NULL;
2431
2432 if (is_vcpu_stopped(vcpu))
2433 return;
2434
2435 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2436 /* Only one cpu at a time may enter/leave the STOPPED state. */
2437 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2438 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2439
2440 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2441 kvm_s390_clear_stop_irq(vcpu);
2442
2443 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2444 __disable_ibs_on_vcpu(vcpu);
2445
2446 for (i = 0; i < online_vcpus; i++) {
2447 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2448 started_vcpus++;
2449 started_vcpu = vcpu->kvm->vcpus[i];
2450 }
2451 }
2452
2453 if (started_vcpus == 1) {
2454 /*
2455 * As we only have one VCPU left, we want to enable the
2456 * IBS facility for that VCPU to speed it up.
2457 */
2458 __enable_ibs_on_vcpu(started_vcpu);
2459 }
2460
2461 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2462 return;
2463 }
2464
2465 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2466 struct kvm_enable_cap *cap)
2467 {
2468 int r;
2469
2470 if (cap->flags)
2471 return -EINVAL;
2472
2473 switch (cap->cap) {
2474 case KVM_CAP_S390_CSS_SUPPORT:
2475 if (!vcpu->kvm->arch.css_support) {
2476 vcpu->kvm->arch.css_support = 1;
2477 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2478 trace_kvm_s390_enable_css(vcpu->kvm);
2479 }
2480 r = 0;
2481 break;
2482 default:
2483 r = -EINVAL;
2484 break;
2485 }
2486 return r;
2487 }
2488
2489 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2490 struct kvm_s390_mem_op *mop)
2491 {
2492 void __user *uaddr = (void __user *)mop->buf;
2493 void *tmpbuf = NULL;
2494 int r, srcu_idx;
2495 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2496 | KVM_S390_MEMOP_F_CHECK_ONLY;
2497
2498 if (mop->flags & ~supported_flags)
2499 return -EINVAL;
2500
2501 if (mop->size > MEM_OP_MAX_SIZE)
2502 return -E2BIG;
2503
2504 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2505 tmpbuf = vmalloc(mop->size);
2506 if (!tmpbuf)
2507 return -ENOMEM;
2508 }
2509
2510 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2511
2512 switch (mop->op) {
2513 case KVM_S390_MEMOP_LOGICAL_READ:
2514 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2515 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2516 break;
2517 }
2518 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2519 if (r == 0) {
2520 if (copy_to_user(uaddr, tmpbuf, mop->size))
2521 r = -EFAULT;
2522 }
2523 break;
2524 case KVM_S390_MEMOP_LOGICAL_WRITE:
2525 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2526 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2527 break;
2528 }
2529 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2530 r = -EFAULT;
2531 break;
2532 }
2533 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2534 break;
2535 default:
2536 r = -EINVAL;
2537 }
2538
2539 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2540
2541 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2542 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2543
2544 vfree(tmpbuf);
2545 return r;
2546 }
2547
2548 long kvm_arch_vcpu_ioctl(struct file *filp,
2549 unsigned int ioctl, unsigned long arg)
2550 {
2551 struct kvm_vcpu *vcpu = filp->private_data;
2552 void __user *argp = (void __user *)arg;
2553 int idx;
2554 long r;
2555
2556 switch (ioctl) {
2557 case KVM_S390_IRQ: {
2558 struct kvm_s390_irq s390irq;
2559
2560 r = -EFAULT;
2561 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2562 break;
2563 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2564 break;
2565 }
2566 case KVM_S390_INTERRUPT: {
2567 struct kvm_s390_interrupt s390int;
2568 struct kvm_s390_irq s390irq;
2569
2570 r = -EFAULT;
2571 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2572 break;
2573 if (s390int_to_s390irq(&s390int, &s390irq))
2574 return -EINVAL;
2575 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2576 break;
2577 }
2578 case KVM_S390_STORE_STATUS:
2579 idx = srcu_read_lock(&vcpu->kvm->srcu);
2580 r = kvm_s390_vcpu_store_status(vcpu, arg);
2581 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2582 break;
2583 case KVM_S390_SET_INITIAL_PSW: {
2584 psw_t psw;
2585
2586 r = -EFAULT;
2587 if (copy_from_user(&psw, argp, sizeof(psw)))
2588 break;
2589 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2590 break;
2591 }
2592 case KVM_S390_INITIAL_RESET:
2593 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2594 break;
2595 case KVM_SET_ONE_REG:
2596 case KVM_GET_ONE_REG: {
2597 struct kvm_one_reg reg;
2598 r = -EFAULT;
2599 if (copy_from_user(&reg, argp, sizeof(reg)))
2600 break;
2601 if (ioctl == KVM_SET_ONE_REG)
2602 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2603 else
2604 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2605 break;
2606 }
2607 #ifdef CONFIG_KVM_S390_UCONTROL
2608 case KVM_S390_UCAS_MAP: {
2609 struct kvm_s390_ucas_mapping ucasmap;
2610
2611 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2612 r = -EFAULT;
2613 break;
2614 }
2615
2616 if (!kvm_is_ucontrol(vcpu->kvm)) {
2617 r = -EINVAL;
2618 break;
2619 }
2620
2621 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2622 ucasmap.vcpu_addr, ucasmap.length);
2623 break;
2624 }
2625 case KVM_S390_UCAS_UNMAP: {
2626 struct kvm_s390_ucas_mapping ucasmap;
2627
2628 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2629 r = -EFAULT;
2630 break;
2631 }
2632
2633 if (!kvm_is_ucontrol(vcpu->kvm)) {
2634 r = -EINVAL;
2635 break;
2636 }
2637
2638 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2639 ucasmap.length);
2640 break;
2641 }
2642 #endif
2643 case KVM_S390_VCPU_FAULT: {
2644 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2645 break;
2646 }
2647 case KVM_ENABLE_CAP:
2648 {
2649 struct kvm_enable_cap cap;
2650 r = -EFAULT;
2651 if (copy_from_user(&cap, argp, sizeof(cap)))
2652 break;
2653 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2654 break;
2655 }
2656 case KVM_S390_MEM_OP: {
2657 struct kvm_s390_mem_op mem_op;
2658
2659 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2660 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2661 else
2662 r = -EFAULT;
2663 break;
2664 }
2665 case KVM_S390_SET_IRQ_STATE: {
2666 struct kvm_s390_irq_state irq_state;
2667
2668 r = -EFAULT;
2669 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2670 break;
2671 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2672 irq_state.len == 0 ||
2673 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2674 r = -EINVAL;
2675 break;
2676 }
2677 r = kvm_s390_set_irq_state(vcpu,
2678 (void __user *) irq_state.buf,
2679 irq_state.len);
2680 break;
2681 }
2682 case KVM_S390_GET_IRQ_STATE: {
2683 struct kvm_s390_irq_state irq_state;
2684
2685 r = -EFAULT;
2686 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2687 break;
2688 if (irq_state.len == 0) {
2689 r = -EINVAL;
2690 break;
2691 }
2692 r = kvm_s390_get_irq_state(vcpu,
2693 (__u8 __user *) irq_state.buf,
2694 irq_state.len);
2695 break;
2696 }
2697 default:
2698 r = -ENOTTY;
2699 }
2700 return r;
2701 }
2702
2703 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2704 {
2705 #ifdef CONFIG_KVM_S390_UCONTROL
2706 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2707 && (kvm_is_ucontrol(vcpu->kvm))) {
2708 vmf->page = virt_to_page(vcpu->arch.sie_block);
2709 get_page(vmf->page);
2710 return 0;
2711 }
2712 #endif
2713 return VM_FAULT_SIGBUS;
2714 }
2715
2716 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2717 unsigned long npages)
2718 {
2719 return 0;
2720 }
2721
2722 /* Section: memory related */
2723 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2724 struct kvm_memory_slot *memslot,
2725 const struct kvm_userspace_memory_region *mem,
2726 enum kvm_mr_change change)
2727 {
2728 /* A few sanity checks. We can have memory slots which have to be
2729 located/ended at a segment boundary (1MB). The memory in userland is
2730 ok to be fragmented into various different vmas. It is okay to mmap()
2731 and munmap() stuff in this slot after doing this call at any time */
2732
2733 if (mem->userspace_addr & 0xffffful)
2734 return -EINVAL;
2735
2736 if (mem->memory_size & 0xffffful)
2737 return -EINVAL;
2738
2739 return 0;
2740 }
2741
2742 void kvm_arch_commit_memory_region(struct kvm *kvm,
2743 const struct kvm_userspace_memory_region *mem,
2744 const struct kvm_memory_slot *old,
2745 const struct kvm_memory_slot *new,
2746 enum kvm_mr_change change)
2747 {
2748 int rc;
2749
2750 /* If the basics of the memslot do not change, we do not want
2751 * to update the gmap. Every update causes several unnecessary
2752 * segment translation exceptions. This is usually handled just
2753 * fine by the normal fault handler + gmap, but it will also
2754 * cause faults on the prefix page of running guest CPUs.
2755 */
2756 if (old->userspace_addr == mem->userspace_addr &&
2757 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2758 old->npages * PAGE_SIZE == mem->memory_size)
2759 return;
2760
2761 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2762 mem->guest_phys_addr, mem->memory_size);
2763 if (rc)
2764 pr_warn("failed to commit memory region\n");
2765 return;
2766 }
2767
2768 static int __init kvm_s390_init(void)
2769 {
2770 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2771 }
2772
2773 static void __exit kvm_s390_exit(void)
2774 {
2775 kvm_exit();
2776 }
2777
2778 module_init(kvm_s390_init);
2779 module_exit(kvm_s390_exit);
2780
2781 /*
2782 * Enable autoloading of the kvm module.
2783 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2784 * since x86 takes a different approach.
2785 */
2786 #include <linux/miscdevice.h>
2787 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2788 MODULE_ALIAS("devname:kvm");
Linux kernel - Deliverables
This page took 0.135255 seconds and 6 git commands to generate.