projects / deliverable / linux.git / blob
? search:


5a02be4628f1ecd059d4cbd38b73d31ddf5777ea
[deliverable/linux.git] / arch / s390 / kvm / kvm-s390.c
1 /*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17 #include <linux/compiler.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/lowcore.h>
30 #include <asm/pgtable.h>
31 #include <asm/nmi.h>
32 #include <asm/switch_to.h>
33 #include <asm/sclp.h>
34 #include "kvm-s390.h"
35 #include "gaccess.h"
36
37 #define CREATE_TRACE_POINTS
38 #include "trace.h"
39 #include "trace-s390.h"
40
41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43 struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
45 { "exit_null", VCPU_STAT(exit_null) },
46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
53 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
54 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
55 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
56 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
57 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
58 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
59 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
60 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
61 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
62 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
63 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
64 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
65 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
66 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
67 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
68 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
69 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
70 { "instruction_spx", VCPU_STAT(instruction_spx) },
71 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
72 { "instruction_stap", VCPU_STAT(instruction_stap) },
73 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
74 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
75 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
76 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
77 { "instruction_essa", VCPU_STAT(instruction_essa) },
78 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
79 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
80 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
81 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
82 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
83 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
84 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
85 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
86 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
87 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
88 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
89 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
90 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
91 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
92 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
93 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
94 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
95 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
96 { "diagnose_10", VCPU_STAT(diagnose_10) },
97 { "diagnose_44", VCPU_STAT(diagnose_44) },
98 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
99 { NULL }
100 };
101
102 /* upper facilities limit for kvm */
103 unsigned long kvm_s390_fac_list_mask[] = {
104 0xff82fffbf4fc2000UL,
105 0x005c000000000000UL,
106 };
107
108 unsigned long kvm_s390_fac_list_mask_size(void)
109 {
110 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
111 return ARRAY_SIZE(kvm_s390_fac_list_mask);
112 }
113
114 static struct gmap_notifier gmap_notifier;
115
116 /* Section: not file related */
117 int kvm_arch_hardware_enable(void)
118 {
119 /* every s390 is virtualization enabled ;-) */
120 return 0;
121 }
122
123 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
124
125 int kvm_arch_hardware_setup(void)
126 {
127 gmap_notifier.notifier_call = kvm_gmap_notifier;
128 gmap_register_ipte_notifier(&gmap_notifier);
129 return 0;
130 }
131
132 void kvm_arch_hardware_unsetup(void)
133 {
134 gmap_unregister_ipte_notifier(&gmap_notifier);
135 }
136
137 int kvm_arch_init(void *opaque)
138 {
139 /* Register floating interrupt controller interface. */
140 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
141 }
142
143 /* Section: device related */
144 long kvm_arch_dev_ioctl(struct file *filp,
145 unsigned int ioctl, unsigned long arg)
146 {
147 if (ioctl == KVM_S390_ENABLE_SIE)
148 return s390_enable_sie();
149 return -EINVAL;
150 }
151
152 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
153 {
154 int r;
155
156 switch (ext) {
157 case KVM_CAP_S390_PSW:
158 case KVM_CAP_S390_GMAP:
159 case KVM_CAP_SYNC_MMU:
160 #ifdef CONFIG_KVM_S390_UCONTROL
161 case KVM_CAP_S390_UCONTROL:
162 #endif
163 case KVM_CAP_ASYNC_PF:
164 case KVM_CAP_SYNC_REGS:
165 case KVM_CAP_ONE_REG:
166 case KVM_CAP_ENABLE_CAP:
167 case KVM_CAP_S390_CSS_SUPPORT:
168 case KVM_CAP_IRQFD:
169 case KVM_CAP_IOEVENTFD:
170 case KVM_CAP_DEVICE_CTRL:
171 case KVM_CAP_ENABLE_CAP_VM:
172 case KVM_CAP_S390_IRQCHIP:
173 case KVM_CAP_VM_ATTRIBUTES:
174 case KVM_CAP_MP_STATE:
175 case KVM_CAP_S390_USER_SIGP:
176 r = 1;
177 break;
178 case KVM_CAP_NR_VCPUS:
179 case KVM_CAP_MAX_VCPUS:
180 r = KVM_MAX_VCPUS;
181 break;
182 case KVM_CAP_NR_MEMSLOTS:
183 r = KVM_USER_MEM_SLOTS;
184 break;
185 case KVM_CAP_S390_COW:
186 r = MACHINE_HAS_ESOP;
187 break;
188 default:
189 r = 0;
190 }
191 return r;
192 }
193
194 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
195 struct kvm_memory_slot *memslot)
196 {
197 gfn_t cur_gfn, last_gfn;
198 unsigned long address;
199 struct gmap *gmap = kvm->arch.gmap;
200
201 down_read(&gmap->mm->mmap_sem);
202 /* Loop over all guest pages */
203 last_gfn = memslot->base_gfn + memslot->npages;
204 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
205 address = gfn_to_hva_memslot(memslot, cur_gfn);
206
207 if (gmap_test_and_clear_dirty(address, gmap))
208 mark_page_dirty(kvm, cur_gfn);
209 }
210 up_read(&gmap->mm->mmap_sem);
211 }
212
213 /* Section: vm related */
214 /*
215 * Get (and clear) the dirty memory log for a memory slot.
216 */
217 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
218 struct kvm_dirty_log *log)
219 {
220 int r;
221 unsigned long n;
222 struct kvm_memory_slot *memslot;
223 int is_dirty = 0;
224
225 mutex_lock(&kvm->slots_lock);
226
227 r = -EINVAL;
228 if (log->slot >= KVM_USER_MEM_SLOTS)
229 goto out;
230
231 memslot = id_to_memslot(kvm->memslots, log->slot);
232 r = -ENOENT;
233 if (!memslot->dirty_bitmap)
234 goto out;
235
236 kvm_s390_sync_dirty_log(kvm, memslot);
237 r = kvm_get_dirty_log(kvm, log, &is_dirty);
238 if (r)
239 goto out;
240
241 /* Clear the dirty log */
242 if (is_dirty) {
243 n = kvm_dirty_bitmap_bytes(memslot);
244 memset(memslot->dirty_bitmap, 0, n);
245 }
246 r = 0;
247 out:
248 mutex_unlock(&kvm->slots_lock);
249 return r;
250 }
251
252 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
253 {
254 int r;
255
256 if (cap->flags)
257 return -EINVAL;
258
259 switch (cap->cap) {
260 case KVM_CAP_S390_IRQCHIP:
261 kvm->arch.use_irqchip = 1;
262 r = 0;
263 break;
264 case KVM_CAP_S390_USER_SIGP:
265 kvm->arch.user_sigp = 1;
266 r = 0;
267 break;
268 default:
269 r = -EINVAL;
270 break;
271 }
272 return r;
273 }
274
275 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
276 {
277 int ret;
278
279 switch (attr->attr) {
280 case KVM_S390_VM_MEM_LIMIT_SIZE:
281 ret = 0;
282 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
283 ret = -EFAULT;
284 break;
285 default:
286 ret = -ENXIO;
287 break;
288 }
289 return ret;
290 }
291
292 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
293 {
294 int ret;
295 unsigned int idx;
296 switch (attr->attr) {
297 case KVM_S390_VM_MEM_ENABLE_CMMA:
298 ret = -EBUSY;
299 mutex_lock(&kvm->lock);
300 if (atomic_read(&kvm->online_vcpus) == 0) {
301 kvm->arch.use_cmma = 1;
302 ret = 0;
303 }
304 mutex_unlock(&kvm->lock);
305 break;
306 case KVM_S390_VM_MEM_CLR_CMMA:
307 mutex_lock(&kvm->lock);
308 idx = srcu_read_lock(&kvm->srcu);
309 s390_reset_cmma(kvm->arch.gmap->mm);
310 srcu_read_unlock(&kvm->srcu, idx);
311 mutex_unlock(&kvm->lock);
312 ret = 0;
313 break;
314 case KVM_S390_VM_MEM_LIMIT_SIZE: {
315 unsigned long new_limit;
316
317 if (kvm_is_ucontrol(kvm))
318 return -EINVAL;
319
320 if (get_user(new_limit, (u64 __user *)attr->addr))
321 return -EFAULT;
322
323 if (new_limit > kvm->arch.gmap->asce_end)
324 return -E2BIG;
325
326 ret = -EBUSY;
327 mutex_lock(&kvm->lock);
328 if (atomic_read(&kvm->online_vcpus) == 0) {
329 /* gmap_alloc will round the limit up */
330 struct gmap *new = gmap_alloc(current->mm, new_limit);
331
332 if (!new) {
333 ret = -ENOMEM;
334 } else {
335 gmap_free(kvm->arch.gmap);
336 new->private = kvm;
337 kvm->arch.gmap = new;
338 ret = 0;
339 }
340 }
341 mutex_unlock(&kvm->lock);
342 break;
343 }
344 default:
345 ret = -ENXIO;
346 break;
347 }
348 return ret;
349 }
350
351 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
352
353 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
354 {
355 struct kvm_vcpu *vcpu;
356 int i;
357
358 if (!test_kvm_facility(kvm, 76))
359 return -EINVAL;
360
361 mutex_lock(&kvm->lock);
362 switch (attr->attr) {
363 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
364 get_random_bytes(
365 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
366 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
367 kvm->arch.crypto.aes_kw = 1;
368 break;
369 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
370 get_random_bytes(
371 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
372 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
373 kvm->arch.crypto.dea_kw = 1;
374 break;
375 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
376 kvm->arch.crypto.aes_kw = 0;
377 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
378 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
379 break;
380 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
381 kvm->arch.crypto.dea_kw = 0;
382 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
383 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
384 break;
385 default:
386 mutex_unlock(&kvm->lock);
387 return -ENXIO;
388 }
389
390 kvm_for_each_vcpu(i, vcpu, kvm) {
391 kvm_s390_vcpu_crypto_setup(vcpu);
392 exit_sie(vcpu);
393 }
394 mutex_unlock(&kvm->lock);
395 return 0;
396 }
397
398 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
399 {
400 u8 gtod_high;
401
402 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
403 sizeof(gtod_high)))
404 return -EFAULT;
405
406 if (gtod_high != 0)
407 return -EINVAL;
408
409 return 0;
410 }
411
412 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
413 {
414 struct kvm_vcpu *cur_vcpu;
415 unsigned int vcpu_idx;
416 u64 host_tod, gtod;
417 int r;
418
419 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
420 return -EFAULT;
421
422 r = store_tod_clock(&host_tod);
423 if (r)
424 return r;
425
426 mutex_lock(&kvm->lock);
427 kvm->arch.epoch = gtod - host_tod;
428 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
429 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
430 exit_sie(cur_vcpu);
431 }
432 mutex_unlock(&kvm->lock);
433 return 0;
434 }
435
436 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
437 {
438 int ret;
439
440 if (attr->flags)
441 return -EINVAL;
442
443 switch (attr->attr) {
444 case KVM_S390_VM_TOD_HIGH:
445 ret = kvm_s390_set_tod_high(kvm, attr);
446 break;
447 case KVM_S390_VM_TOD_LOW:
448 ret = kvm_s390_set_tod_low(kvm, attr);
449 break;
450 default:
451 ret = -ENXIO;
452 break;
453 }
454 return ret;
455 }
456
457 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
458 {
459 u8 gtod_high = 0;
460
461 if (copy_to_user((void __user *)attr->addr, &gtod_high,
462 sizeof(gtod_high)))
463 return -EFAULT;
464
465 return 0;
466 }
467
468 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
469 {
470 u64 host_tod, gtod;
471 int r;
472
473 r = store_tod_clock(&host_tod);
474 if (r)
475 return r;
476
477 gtod = host_tod + kvm->arch.epoch;
478 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
479 return -EFAULT;
480
481 return 0;
482 }
483
484 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
485 {
486 int ret;
487
488 if (attr->flags)
489 return -EINVAL;
490
491 switch (attr->attr) {
492 case KVM_S390_VM_TOD_HIGH:
493 ret = kvm_s390_get_tod_high(kvm, attr);
494 break;
495 case KVM_S390_VM_TOD_LOW:
496 ret = kvm_s390_get_tod_low(kvm, attr);
497 break;
498 default:
499 ret = -ENXIO;
500 break;
501 }
502 return ret;
503 }
504
505 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
506 {
507 struct kvm_s390_vm_cpu_processor *proc;
508 int ret = 0;
509
510 mutex_lock(&kvm->lock);
511 if (atomic_read(&kvm->online_vcpus)) {
512 ret = -EBUSY;
513 goto out;
514 }
515 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
516 if (!proc) {
517 ret = -ENOMEM;
518 goto out;
519 }
520 if (!copy_from_user(proc, (void __user *)attr->addr,
521 sizeof(*proc))) {
522 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
523 sizeof(struct cpuid));
524 kvm->arch.model.ibc = proc->ibc;
525 memcpy(kvm->arch.model.fac->list, proc->fac_list,
526 S390_ARCH_FAC_LIST_SIZE_BYTE);
527 } else
528 ret = -EFAULT;
529 kfree(proc);
530 out:
531 mutex_unlock(&kvm->lock);
532 return ret;
533 }
534
535 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
536 {
537 int ret = -ENXIO;
538
539 switch (attr->attr) {
540 case KVM_S390_VM_CPU_PROCESSOR:
541 ret = kvm_s390_set_processor(kvm, attr);
542 break;
543 }
544 return ret;
545 }
546
547 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
548 {
549 struct kvm_s390_vm_cpu_processor *proc;
550 int ret = 0;
551
552 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
553 if (!proc) {
554 ret = -ENOMEM;
555 goto out;
556 }
557 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
558 proc->ibc = kvm->arch.model.ibc;
559 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
560 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
561 ret = -EFAULT;
562 kfree(proc);
563 out:
564 return ret;
565 }
566
567 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
568 {
569 struct kvm_s390_vm_cpu_machine *mach;
570 int ret = 0;
571
572 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
573 if (!mach) {
574 ret = -ENOMEM;
575 goto out;
576 }
577 get_cpu_id((struct cpuid *) &mach->cpuid);
578 mach->ibc = sclp_get_ibc();
579 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
580 S390_ARCH_FAC_LIST_SIZE_BYTE);
581 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
582 S390_ARCH_FAC_LIST_SIZE_BYTE);
583 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
584 ret = -EFAULT;
585 kfree(mach);
586 out:
587 return ret;
588 }
589
590 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
591 {
592 int ret = -ENXIO;
593
594 switch (attr->attr) {
595 case KVM_S390_VM_CPU_PROCESSOR:
596 ret = kvm_s390_get_processor(kvm, attr);
597 break;
598 case KVM_S390_VM_CPU_MACHINE:
599 ret = kvm_s390_get_machine(kvm, attr);
600 break;
601 }
602 return ret;
603 }
604
605 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
606 {
607 int ret;
608
609 switch (attr->group) {
610 case KVM_S390_VM_MEM_CTRL:
611 ret = kvm_s390_set_mem_control(kvm, attr);
612 break;
613 case KVM_S390_VM_TOD:
614 ret = kvm_s390_set_tod(kvm, attr);
615 break;
616 case KVM_S390_VM_CPU_MODEL:
617 ret = kvm_s390_set_cpu_model(kvm, attr);
618 break;
619 case KVM_S390_VM_CRYPTO:
620 ret = kvm_s390_vm_set_crypto(kvm, attr);
621 break;
622 default:
623 ret = -ENXIO;
624 break;
625 }
626
627 return ret;
628 }
629
630 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
631 {
632 int ret;
633
634 switch (attr->group) {
635 case KVM_S390_VM_MEM_CTRL:
636 ret = kvm_s390_get_mem_control(kvm, attr);
637 break;
638 case KVM_S390_VM_TOD:
639 ret = kvm_s390_get_tod(kvm, attr);
640 break;
641 case KVM_S390_VM_CPU_MODEL:
642 ret = kvm_s390_get_cpu_model(kvm, attr);
643 break;
644 default:
645 ret = -ENXIO;
646 break;
647 }
648
649 return ret;
650 }
651
652 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
653 {
654 int ret;
655
656 switch (attr->group) {
657 case KVM_S390_VM_MEM_CTRL:
658 switch (attr->attr) {
659 case KVM_S390_VM_MEM_ENABLE_CMMA:
660 case KVM_S390_VM_MEM_CLR_CMMA:
661 case KVM_S390_VM_MEM_LIMIT_SIZE:
662 ret = 0;
663 break;
664 default:
665 ret = -ENXIO;
666 break;
667 }
668 break;
669 case KVM_S390_VM_TOD:
670 switch (attr->attr) {
671 case KVM_S390_VM_TOD_LOW:
672 case KVM_S390_VM_TOD_HIGH:
673 ret = 0;
674 break;
675 default:
676 ret = -ENXIO;
677 break;
678 }
679 break;
680 case KVM_S390_VM_CPU_MODEL:
681 switch (attr->attr) {
682 case KVM_S390_VM_CPU_PROCESSOR:
683 case KVM_S390_VM_CPU_MACHINE:
684 ret = 0;
685 break;
686 default:
687 ret = -ENXIO;
688 break;
689 }
690 break;
691 case KVM_S390_VM_CRYPTO:
692 switch (attr->attr) {
693 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
694 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
695 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
696 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
697 ret = 0;
698 break;
699 default:
700 ret = -ENXIO;
701 break;
702 }
703 break;
704 default:
705 ret = -ENXIO;
706 break;
707 }
708
709 return ret;
710 }
711
712 long kvm_arch_vm_ioctl(struct file *filp,
713 unsigned int ioctl, unsigned long arg)
714 {
715 struct kvm *kvm = filp->private_data;
716 void __user *argp = (void __user *)arg;
717 struct kvm_device_attr attr;
718 int r;
719
720 switch (ioctl) {
721 case KVM_S390_INTERRUPT: {
722 struct kvm_s390_interrupt s390int;
723
724 r = -EFAULT;
725 if (copy_from_user(&s390int, argp, sizeof(s390int)))
726 break;
727 r = kvm_s390_inject_vm(kvm, &s390int);
728 break;
729 }
730 case KVM_ENABLE_CAP: {
731 struct kvm_enable_cap cap;
732 r = -EFAULT;
733 if (copy_from_user(&cap, argp, sizeof(cap)))
734 break;
735 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
736 break;
737 }
738 case KVM_CREATE_IRQCHIP: {
739 struct kvm_irq_routing_entry routing;
740
741 r = -EINVAL;
742 if (kvm->arch.use_irqchip) {
743 /* Set up dummy routing. */
744 memset(&routing, 0, sizeof(routing));
745 kvm_set_irq_routing(kvm, &routing, 0, 0);
746 r = 0;
747 }
748 break;
749 }
750 case KVM_SET_DEVICE_ATTR: {
751 r = -EFAULT;
752 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
753 break;
754 r = kvm_s390_vm_set_attr(kvm, &attr);
755 break;
756 }
757 case KVM_GET_DEVICE_ATTR: {
758 r = -EFAULT;
759 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
760 break;
761 r = kvm_s390_vm_get_attr(kvm, &attr);
762 break;
763 }
764 case KVM_HAS_DEVICE_ATTR: {
765 r = -EFAULT;
766 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
767 break;
768 r = kvm_s390_vm_has_attr(kvm, &attr);
769 break;
770 }
771 default:
772 r = -ENOTTY;
773 }
774
775 return r;
776 }
777
778 static int kvm_s390_query_ap_config(u8 *config)
779 {
780 u32 fcn_code = 0x04000000UL;
781 u32 cc = 0;
782
783 memset(config, 0, 128);
784 asm volatile(
785 "lgr 0,%1\n"
786 "lgr 2,%2\n"
787 ".long 0xb2af0000\n" /* PQAP(QCI) */
788 "0: ipm %0\n"
789 "srl %0,28\n"
790 "1:\n"
791 EX_TABLE(0b, 1b)
792 : "+r" (cc)
793 : "r" (fcn_code), "r" (config)
794 : "cc", "0", "2", "memory"
795 );
796
797 return cc;
798 }
799
800 static int kvm_s390_apxa_installed(void)
801 {
802 u8 config[128];
803 int cc;
804
805 if (test_facility(2) && test_facility(12)) {
806 cc = kvm_s390_query_ap_config(config);
807
808 if (cc)
809 pr_err("PQAP(QCI) failed with cc=%d", cc);
810 else
811 return config[0] & 0x40;
812 }
813
814 return 0;
815 }
816
817 static void kvm_s390_set_crycb_format(struct kvm *kvm)
818 {
819 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
820
821 if (kvm_s390_apxa_installed())
822 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
823 else
824 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
825 }
826
827 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
828 {
829 get_cpu_id(cpu_id);
830 cpu_id->version = 0xff;
831 }
832
833 static int kvm_s390_crypto_init(struct kvm *kvm)
834 {
835 if (!test_kvm_facility(kvm, 76))
836 return 0;
837
838 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
839 GFP_KERNEL | GFP_DMA);
840 if (!kvm->arch.crypto.crycb)
841 return -ENOMEM;
842
843 kvm_s390_set_crycb_format(kvm);
844
845 /* Enable AES/DEA protected key functions by default */
846 kvm->arch.crypto.aes_kw = 1;
847 kvm->arch.crypto.dea_kw = 1;
848 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
849 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
850 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
851 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
852
853 return 0;
854 }
855
856 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
857 {
858 int i, rc;
859 char debug_name[16];
860 static unsigned long sca_offset;
861
862 rc = -EINVAL;
863 #ifdef CONFIG_KVM_S390_UCONTROL
864 if (type & ~KVM_VM_S390_UCONTROL)
865 goto out_err;
866 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
867 goto out_err;
868 #else
869 if (type)
870 goto out_err;
871 #endif
872
873 rc = s390_enable_sie();
874 if (rc)
875 goto out_err;
876
877 rc = -ENOMEM;
878
879 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
880 if (!kvm->arch.sca)
881 goto out_err;
882 spin_lock(&kvm_lock);
883 sca_offset = (sca_offset + 16) & 0x7f0;
884 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
885 spin_unlock(&kvm_lock);
886
887 sprintf(debug_name, "kvm-%u", current->pid);
888
889 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
890 if (!kvm->arch.dbf)
891 goto out_nodbf;
892
893 /*
894 * The architectural maximum amount of facilities is 16 kbit. To store
895 * this amount, 2 kbyte of memory is required. Thus we need a full
896 * page to hold the guest facility list (arch.model.fac->list) and the
897 * facility mask (arch.model.fac->mask). Its address size has to be
898 * 31 bits and word aligned.
899 */
900 kvm->arch.model.fac =
901 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
902 if (!kvm->arch.model.fac)
903 goto out_nofac;
904
905 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
906 S390_ARCH_FAC_LIST_SIZE_BYTE);
907
908 /*
909 * If this KVM host runs *not* in a LPAR, relax the facility bits
910 * of the kvm facility mask by all missing facilities. This will allow
911 * to determine the right CPU model by means of the remaining facilities.
912 * Live guest migration must prohibit the migration of KVMs running in
913 * a LPAR to non LPAR hosts.
914 */
915 if (!MACHINE_IS_LPAR)
916 for (i = 0; i < kvm_s390_fac_list_mask_size(); i++)
917 kvm_s390_fac_list_mask[i] &= kvm->arch.model.fac->mask[i];
918
919 /*
920 * Apply the kvm facility mask to limit the kvm supported/tolerated
921 * facility list.
922 */
923 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
924 if (i < kvm_s390_fac_list_mask_size())
925 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
926 else
927 kvm->arch.model.fac->mask[i] = 0UL;
928 }
929
930 /* Populate the facility list initially. */
931 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
932 S390_ARCH_FAC_LIST_SIZE_BYTE);
933
934 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
935 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
936
937 if (kvm_s390_crypto_init(kvm) < 0)
938 goto out_crypto;
939
940 spin_lock_init(&kvm->arch.float_int.lock);
941 INIT_LIST_HEAD(&kvm->arch.float_int.list);
942 init_waitqueue_head(&kvm->arch.ipte_wq);
943 mutex_init(&kvm->arch.ipte_mutex);
944
945 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
946 VM_EVENT(kvm, 3, "%s", "vm created");
947
948 if (type & KVM_VM_S390_UCONTROL) {
949 kvm->arch.gmap = NULL;
950 } else {
951 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
952 if (!kvm->arch.gmap)
953 goto out_nogmap;
954 kvm->arch.gmap->private = kvm;
955 kvm->arch.gmap->pfault_enabled = 0;
956 }
957
958 kvm->arch.css_support = 0;
959 kvm->arch.use_irqchip = 0;
960 kvm->arch.epoch = 0;
961
962 spin_lock_init(&kvm->arch.start_stop_lock);
963
964 return 0;
965 out_nogmap:
966 kfree(kvm->arch.crypto.crycb);
967 out_crypto:
968 free_page((unsigned long)kvm->arch.model.fac);
969 out_nofac:
970 debug_unregister(kvm->arch.dbf);
971 out_nodbf:
972 free_page((unsigned long)(kvm->arch.sca));
973 out_err:
974 return rc;
975 }
976
977 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
978 {
979 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
980 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
981 kvm_s390_clear_local_irqs(vcpu);
982 kvm_clear_async_pf_completion_queue(vcpu);
983 if (!kvm_is_ucontrol(vcpu->kvm)) {
984 clear_bit(63 - vcpu->vcpu_id,
985 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
986 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
987 (__u64) vcpu->arch.sie_block)
988 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
989 }
990 smp_mb();
991
992 if (kvm_is_ucontrol(vcpu->kvm))
993 gmap_free(vcpu->arch.gmap);
994
995 if (kvm_s390_cmma_enabled(vcpu->kvm))
996 kvm_s390_vcpu_unsetup_cmma(vcpu);
997 free_page((unsigned long)(vcpu->arch.sie_block));
998
999 kvm_vcpu_uninit(vcpu);
1000 kmem_cache_free(kvm_vcpu_cache, vcpu);
1001 }
1002
1003 static void kvm_free_vcpus(struct kvm *kvm)
1004 {
1005 unsigned int i;
1006 struct kvm_vcpu *vcpu;
1007
1008 kvm_for_each_vcpu(i, vcpu, kvm)
1009 kvm_arch_vcpu_destroy(vcpu);
1010
1011 mutex_lock(&kvm->lock);
1012 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1013 kvm->vcpus[i] = NULL;
1014
1015 atomic_set(&kvm->online_vcpus, 0);
1016 mutex_unlock(&kvm->lock);
1017 }
1018
1019 void kvm_arch_destroy_vm(struct kvm *kvm)
1020 {
1021 kvm_free_vcpus(kvm);
1022 free_page((unsigned long)kvm->arch.model.fac);
1023 free_page((unsigned long)(kvm->arch.sca));
1024 debug_unregister(kvm->arch.dbf);
1025 kfree(kvm->arch.crypto.crycb);
1026 if (!kvm_is_ucontrol(kvm))
1027 gmap_free(kvm->arch.gmap);
1028 kvm_s390_destroy_adapters(kvm);
1029 kvm_s390_clear_float_irqs(kvm);
1030 }
1031
1032 /* Section: vcpu related */
1033 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1034 {
1035 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1036 if (!vcpu->arch.gmap)
1037 return -ENOMEM;
1038 vcpu->arch.gmap->private = vcpu->kvm;
1039
1040 return 0;
1041 }
1042
1043 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1044 {
1045 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1046 kvm_clear_async_pf_completion_queue(vcpu);
1047 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1048 KVM_SYNC_GPRS |
1049 KVM_SYNC_ACRS |
1050 KVM_SYNC_CRS |
1051 KVM_SYNC_ARCH0 |
1052 KVM_SYNC_PFAULT;
1053
1054 if (kvm_is_ucontrol(vcpu->kvm))
1055 return __kvm_ucontrol_vcpu_init(vcpu);
1056
1057 return 0;
1058 }
1059
1060 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1061 {
1062 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1063 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1064 save_access_regs(vcpu->arch.host_acrs);
1065 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1066 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1067 restore_access_regs(vcpu->run->s.regs.acrs);
1068 gmap_enable(vcpu->arch.gmap);
1069 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1070 }
1071
1072 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1073 {
1074 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1075 gmap_disable(vcpu->arch.gmap);
1076 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1077 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1078 save_access_regs(vcpu->run->s.regs.acrs);
1079 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1080 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1081 restore_access_regs(vcpu->arch.host_acrs);
1082 }
1083
1084 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1085 {
1086 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1087 vcpu->arch.sie_block->gpsw.mask = 0UL;
1088 vcpu->arch.sie_block->gpsw.addr = 0UL;
1089 kvm_s390_set_prefix(vcpu, 0);
1090 vcpu->arch.sie_block->cputm = 0UL;
1091 vcpu->arch.sie_block->ckc = 0UL;
1092 vcpu->arch.sie_block->todpr = 0;
1093 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1094 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1095 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1096 vcpu->arch.guest_fpregs.fpc = 0;
1097 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1098 vcpu->arch.sie_block->gbea = 1;
1099 vcpu->arch.sie_block->pp = 0;
1100 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1101 kvm_clear_async_pf_completion_queue(vcpu);
1102 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1103 kvm_s390_vcpu_stop(vcpu);
1104 kvm_s390_clear_local_irqs(vcpu);
1105 }
1106
1107 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1108 {
1109 mutex_lock(&vcpu->kvm->lock);
1110 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1111 mutex_unlock(&vcpu->kvm->lock);
1112 if (!kvm_is_ucontrol(vcpu->kvm))
1113 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1114 }
1115
1116 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1117 {
1118 if (!test_kvm_facility(vcpu->kvm, 76))
1119 return;
1120
1121 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1122
1123 if (vcpu->kvm->arch.crypto.aes_kw)
1124 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1125 if (vcpu->kvm->arch.crypto.dea_kw)
1126 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1127
1128 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1129 }
1130
1131 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1132 {
1133 free_page(vcpu->arch.sie_block->cbrlo);
1134 vcpu->arch.sie_block->cbrlo = 0;
1135 }
1136
1137 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1138 {
1139 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1140 if (!vcpu->arch.sie_block->cbrlo)
1141 return -ENOMEM;
1142
1143 vcpu->arch.sie_block->ecb2 |= 0x80;
1144 vcpu->arch.sie_block->ecb2 &= ~0x08;
1145 return 0;
1146 }
1147
1148 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1149 {
1150 int rc = 0;
1151
1152 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1153 CPUSTAT_SM |
1154 CPUSTAT_STOPPED |
1155 CPUSTAT_GED);
1156 vcpu->arch.sie_block->ecb = 6;
1157 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1158 vcpu->arch.sie_block->ecb |= 0x10;
1159
1160 vcpu->arch.sie_block->ecb2 = 8;
1161 vcpu->arch.sie_block->eca = 0xC1002000U;
1162 if (sclp_has_siif())
1163 vcpu->arch.sie_block->eca |= 1;
1164 if (sclp_has_sigpif())
1165 vcpu->arch.sie_block->eca |= 0x10000000U;
1166 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
1167 ICTL_TPROT;
1168
1169 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1170 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1171 if (rc)
1172 return rc;
1173 }
1174 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1175 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1176
1177 mutex_lock(&vcpu->kvm->lock);
1178 vcpu->arch.cpu_id = vcpu->kvm->arch.model.cpu_id;
1179 vcpu->arch.sie_block->ibc = vcpu->kvm->arch.model.ibc;
1180 mutex_unlock(&vcpu->kvm->lock);
1181
1182 kvm_s390_vcpu_crypto_setup(vcpu);
1183
1184 return rc;
1185 }
1186
1187 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1188 unsigned int id)
1189 {
1190 struct kvm_vcpu *vcpu;
1191 struct sie_page *sie_page;
1192 int rc = -EINVAL;
1193
1194 if (id >= KVM_MAX_VCPUS)
1195 goto out;
1196
1197 rc = -ENOMEM;
1198
1199 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1200 if (!vcpu)
1201 goto out;
1202
1203 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1204 if (!sie_page)
1205 goto out_free_cpu;
1206
1207 vcpu->arch.sie_block = &sie_page->sie_block;
1208 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1209
1210 vcpu->arch.sie_block->icpua = id;
1211 if (!kvm_is_ucontrol(kvm)) {
1212 if (!kvm->arch.sca) {
1213 WARN_ON_ONCE(1);
1214 goto out_free_cpu;
1215 }
1216 if (!kvm->arch.sca->cpu[id].sda)
1217 kvm->arch.sca->cpu[id].sda =
1218 (__u64) vcpu->arch.sie_block;
1219 vcpu->arch.sie_block->scaoh =
1220 (__u32)(((__u64)kvm->arch.sca) >> 32);
1221 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1222 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1223 }
1224 vcpu->arch.sie_block->fac = (int) (long) kvm->arch.model.fac->list;
1225
1226 spin_lock_init(&vcpu->arch.local_int.lock);
1227 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1228 vcpu->arch.local_int.wq = &vcpu->wq;
1229 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1230
1231 rc = kvm_vcpu_init(vcpu, kvm, id);
1232 if (rc)
1233 goto out_free_sie_block;
1234 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1235 vcpu->arch.sie_block);
1236 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1237
1238 return vcpu;
1239 out_free_sie_block:
1240 free_page((unsigned long)(vcpu->arch.sie_block));
1241 out_free_cpu:
1242 kmem_cache_free(kvm_vcpu_cache, vcpu);
1243 out:
1244 return ERR_PTR(rc);
1245 }
1246
1247 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1248 {
1249 return kvm_s390_vcpu_has_irq(vcpu, 0);
1250 }
1251
1252 void s390_vcpu_block(struct kvm_vcpu *vcpu)
1253 {
1254 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1255 }
1256
1257 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1258 {
1259 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1260 }
1261
1262 /*
1263 * Kick a guest cpu out of SIE and wait until SIE is not running.
1264 * If the CPU is not running (e.g. waiting as idle) the function will
1265 * return immediately. */
1266 void exit_sie(struct kvm_vcpu *vcpu)
1267 {
1268 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1269 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1270 cpu_relax();
1271 }
1272
1273 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
1274 void exit_sie_sync(struct kvm_vcpu *vcpu)
1275 {
1276 s390_vcpu_block(vcpu);
1277 exit_sie(vcpu);
1278 }
1279
1280 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1281 {
1282 int i;
1283 struct kvm *kvm = gmap->private;
1284 struct kvm_vcpu *vcpu;
1285
1286 kvm_for_each_vcpu(i, vcpu, kvm) {
1287 /* match against both prefix pages */
1288 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1289 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1290 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1291 exit_sie_sync(vcpu);
1292 }
1293 }
1294 }
1295
1296 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1297 {
1298 /* kvm common code refers to this, but never calls it */
1299 BUG();
1300 return 0;
1301 }
1302
1303 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1304 struct kvm_one_reg *reg)
1305 {
1306 int r = -EINVAL;
1307
1308 switch (reg->id) {
1309 case KVM_REG_S390_TODPR:
1310 r = put_user(vcpu->arch.sie_block->todpr,
1311 (u32 __user *)reg->addr);
1312 break;
1313 case KVM_REG_S390_EPOCHDIFF:
1314 r = put_user(vcpu->arch.sie_block->epoch,
1315 (u64 __user *)reg->addr);
1316 break;
1317 case KVM_REG_S390_CPU_TIMER:
1318 r = put_user(vcpu->arch.sie_block->cputm,
1319 (u64 __user *)reg->addr);
1320 break;
1321 case KVM_REG_S390_CLOCK_COMP:
1322 r = put_user(vcpu->arch.sie_block->ckc,
1323 (u64 __user *)reg->addr);
1324 break;
1325 case KVM_REG_S390_PFTOKEN:
1326 r = put_user(vcpu->arch.pfault_token,
1327 (u64 __user *)reg->addr);
1328 break;
1329 case KVM_REG_S390_PFCOMPARE:
1330 r = put_user(vcpu->arch.pfault_compare,
1331 (u64 __user *)reg->addr);
1332 break;
1333 case KVM_REG_S390_PFSELECT:
1334 r = put_user(vcpu->arch.pfault_select,
1335 (u64 __user *)reg->addr);
1336 break;
1337 case KVM_REG_S390_PP:
1338 r = put_user(vcpu->arch.sie_block->pp,
1339 (u64 __user *)reg->addr);
1340 break;
1341 case KVM_REG_S390_GBEA:
1342 r = put_user(vcpu->arch.sie_block->gbea,
1343 (u64 __user *)reg->addr);
1344 break;
1345 default:
1346 break;
1347 }
1348
1349 return r;
1350 }
1351
1352 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1353 struct kvm_one_reg *reg)
1354 {
1355 int r = -EINVAL;
1356
1357 switch (reg->id) {
1358 case KVM_REG_S390_TODPR:
1359 r = get_user(vcpu->arch.sie_block->todpr,
1360 (u32 __user *)reg->addr);
1361 break;
1362 case KVM_REG_S390_EPOCHDIFF:
1363 r = get_user(vcpu->arch.sie_block->epoch,
1364 (u64 __user *)reg->addr);
1365 break;
1366 case KVM_REG_S390_CPU_TIMER:
1367 r = get_user(vcpu->arch.sie_block->cputm,
1368 (u64 __user *)reg->addr);
1369 break;
1370 case KVM_REG_S390_CLOCK_COMP:
1371 r = get_user(vcpu->arch.sie_block->ckc,
1372 (u64 __user *)reg->addr);
1373 break;
1374 case KVM_REG_S390_PFTOKEN:
1375 r = get_user(vcpu->arch.pfault_token,
1376 (u64 __user *)reg->addr);
1377 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1378 kvm_clear_async_pf_completion_queue(vcpu);
1379 break;
1380 case KVM_REG_S390_PFCOMPARE:
1381 r = get_user(vcpu->arch.pfault_compare,
1382 (u64 __user *)reg->addr);
1383 break;
1384 case KVM_REG_S390_PFSELECT:
1385 r = get_user(vcpu->arch.pfault_select,
1386 (u64 __user *)reg->addr);
1387 break;
1388 case KVM_REG_S390_PP:
1389 r = get_user(vcpu->arch.sie_block->pp,
1390 (u64 __user *)reg->addr);
1391 break;
1392 case KVM_REG_S390_GBEA:
1393 r = get_user(vcpu->arch.sie_block->gbea,
1394 (u64 __user *)reg->addr);
1395 break;
1396 default:
1397 break;
1398 }
1399
1400 return r;
1401 }
1402
1403 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1404 {
1405 kvm_s390_vcpu_initial_reset(vcpu);
1406 return 0;
1407 }
1408
1409 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1410 {
1411 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1412 return 0;
1413 }
1414
1415 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1416 {
1417 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1418 return 0;
1419 }
1420
1421 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1422 struct kvm_sregs *sregs)
1423 {
1424 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1425 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1426 restore_access_regs(vcpu->run->s.regs.acrs);
1427 return 0;
1428 }
1429
1430 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1431 struct kvm_sregs *sregs)
1432 {
1433 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1434 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1435 return 0;
1436 }
1437
1438 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1439 {
1440 if (test_fp_ctl(fpu->fpc))
1441 return -EINVAL;
1442 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1443 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1444 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1445 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1446 return 0;
1447 }
1448
1449 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1450 {
1451 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1452 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1453 return 0;
1454 }
1455
1456 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1457 {
1458 int rc = 0;
1459
1460 if (!is_vcpu_stopped(vcpu))
1461 rc = -EBUSY;
1462 else {
1463 vcpu->run->psw_mask = psw.mask;
1464 vcpu->run->psw_addr = psw.addr;
1465 }
1466 return rc;
1467 }
1468
1469 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1470 struct kvm_translation *tr)
1471 {
1472 return -EINVAL; /* not implemented yet */
1473 }
1474
1475 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1476 KVM_GUESTDBG_USE_HW_BP | \
1477 KVM_GUESTDBG_ENABLE)
1478
1479 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1480 struct kvm_guest_debug *dbg)
1481 {
1482 int rc = 0;
1483
1484 vcpu->guest_debug = 0;
1485 kvm_s390_clear_bp_data(vcpu);
1486
1487 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1488 return -EINVAL;
1489
1490 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1491 vcpu->guest_debug = dbg->control;
1492 /* enforce guest PER */
1493 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1494
1495 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1496 rc = kvm_s390_import_bp_data(vcpu, dbg);
1497 } else {
1498 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1499 vcpu->arch.guestdbg.last_bp = 0;
1500 }
1501
1502 if (rc) {
1503 vcpu->guest_debug = 0;
1504 kvm_s390_clear_bp_data(vcpu);
1505 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1506 }
1507
1508 return rc;
1509 }
1510
1511 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1512 struct kvm_mp_state *mp_state)
1513 {
1514 /* CHECK_STOP and LOAD are not supported yet */
1515 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1516 KVM_MP_STATE_OPERATING;
1517 }
1518
1519 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1520 struct kvm_mp_state *mp_state)
1521 {
1522 int rc = 0;
1523
1524 /* user space knows about this interface - let it control the state */
1525 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1526
1527 switch (mp_state->mp_state) {
1528 case KVM_MP_STATE_STOPPED:
1529 kvm_s390_vcpu_stop(vcpu);
1530 break;
1531 case KVM_MP_STATE_OPERATING:
1532 kvm_s390_vcpu_start(vcpu);
1533 break;
1534 case KVM_MP_STATE_LOAD:
1535 case KVM_MP_STATE_CHECK_STOP:
1536 /* fall through - CHECK_STOP and LOAD are not supported yet */
1537 default:
1538 rc = -ENXIO;
1539 }
1540
1541 return rc;
1542 }
1543
1544 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1545 {
1546 if (!MACHINE_IS_LPAR)
1547 return false;
1548 /* only enable for z10 and later */
1549 if (!MACHINE_HAS_EDAT1)
1550 return false;
1551 if (!kvm->arch.use_cmma)
1552 return false;
1553 return true;
1554 }
1555
1556 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1557 {
1558 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1559 }
1560
1561 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1562 {
1563 retry:
1564 s390_vcpu_unblock(vcpu);
1565 /*
1566 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1567 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1568 * This ensures that the ipte instruction for this request has
1569 * already finished. We might race against a second unmapper that
1570 * wants to set the blocking bit. Lets just retry the request loop.
1571 */
1572 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1573 int rc;
1574 rc = gmap_ipte_notify(vcpu->arch.gmap,
1575 kvm_s390_get_prefix(vcpu),
1576 PAGE_SIZE * 2);
1577 if (rc)
1578 return rc;
1579 goto retry;
1580 }
1581
1582 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1583 vcpu->arch.sie_block->ihcpu = 0xffff;
1584 goto retry;
1585 }
1586
1587 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1588 if (!ibs_enabled(vcpu)) {
1589 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1590 atomic_set_mask(CPUSTAT_IBS,
1591 &vcpu->arch.sie_block->cpuflags);
1592 }
1593 goto retry;
1594 }
1595
1596 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1597 if (ibs_enabled(vcpu)) {
1598 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1599 atomic_clear_mask(CPUSTAT_IBS,
1600 &vcpu->arch.sie_block->cpuflags);
1601 }
1602 goto retry;
1603 }
1604
1605 /* nothing to do, just clear the request */
1606 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1607
1608 return 0;
1609 }
1610
1611 /**
1612 * kvm_arch_fault_in_page - fault-in guest page if necessary
1613 * @vcpu: The corresponding virtual cpu
1614 * @gpa: Guest physical address
1615 * @writable: Whether the page should be writable or not
1616 *
1617 * Make sure that a guest page has been faulted-in on the host.
1618 *
1619 * Return: Zero on success, negative error code otherwise.
1620 */
1621 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1622 {
1623 return gmap_fault(vcpu->arch.gmap, gpa,
1624 writable ? FAULT_FLAG_WRITE : 0);
1625 }
1626
1627 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1628 unsigned long token)
1629 {
1630 struct kvm_s390_interrupt inti;
1631 struct kvm_s390_irq irq;
1632
1633 if (start_token) {
1634 irq.u.ext.ext_params2 = token;
1635 irq.type = KVM_S390_INT_PFAULT_INIT;
1636 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1637 } else {
1638 inti.type = KVM_S390_INT_PFAULT_DONE;
1639 inti.parm64 = token;
1640 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1641 }
1642 }
1643
1644 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1645 struct kvm_async_pf *work)
1646 {
1647 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1648 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1649 }
1650
1651 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1652 struct kvm_async_pf *work)
1653 {
1654 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1655 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1656 }
1657
1658 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1659 struct kvm_async_pf *work)
1660 {
1661 /* s390 will always inject the page directly */
1662 }
1663
1664 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1665 {
1666 /*
1667 * s390 will always inject the page directly,
1668 * but we still want check_async_completion to cleanup
1669 */
1670 return true;
1671 }
1672
1673 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1674 {
1675 hva_t hva;
1676 struct kvm_arch_async_pf arch;
1677 int rc;
1678
1679 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1680 return 0;
1681 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1682 vcpu->arch.pfault_compare)
1683 return 0;
1684 if (psw_extint_disabled(vcpu))
1685 return 0;
1686 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1687 return 0;
1688 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1689 return 0;
1690 if (!vcpu->arch.gmap->pfault_enabled)
1691 return 0;
1692
1693 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1694 hva += current->thread.gmap_addr & ~PAGE_MASK;
1695 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1696 return 0;
1697
1698 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1699 return rc;
1700 }
1701
1702 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1703 {
1704 int rc, cpuflags;
1705
1706 /*
1707 * On s390 notifications for arriving pages will be delivered directly
1708 * to the guest but the house keeping for completed pfaults is
1709 * handled outside the worker.
1710 */
1711 kvm_check_async_pf_completion(vcpu);
1712
1713 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1714
1715 if (need_resched())
1716 schedule();
1717
1718 if (test_cpu_flag(CIF_MCCK_PENDING))
1719 s390_handle_mcck();
1720
1721 if (!kvm_is_ucontrol(vcpu->kvm)) {
1722 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1723 if (rc)
1724 return rc;
1725 }
1726
1727 rc = kvm_s390_handle_requests(vcpu);
1728 if (rc)
1729 return rc;
1730
1731 if (guestdbg_enabled(vcpu)) {
1732 kvm_s390_backup_guest_per_regs(vcpu);
1733 kvm_s390_patch_guest_per_regs(vcpu);
1734 }
1735
1736 vcpu->arch.sie_block->icptcode = 0;
1737 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1738 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1739 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1740
1741 return 0;
1742 }
1743
1744 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1745 {
1746 int rc = -1;
1747
1748 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1749 vcpu->arch.sie_block->icptcode);
1750 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1751
1752 if (guestdbg_enabled(vcpu))
1753 kvm_s390_restore_guest_per_regs(vcpu);
1754
1755 if (exit_reason >= 0) {
1756 rc = 0;
1757 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1758 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1759 vcpu->run->s390_ucontrol.trans_exc_code =
1760 current->thread.gmap_addr;
1761 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1762 rc = -EREMOTE;
1763
1764 } else if (current->thread.gmap_pfault) {
1765 trace_kvm_s390_major_guest_pfault(vcpu);
1766 current->thread.gmap_pfault = 0;
1767 if (kvm_arch_setup_async_pf(vcpu)) {
1768 rc = 0;
1769 } else {
1770 gpa_t gpa = current->thread.gmap_addr;
1771 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1772 }
1773 }
1774
1775 if (rc == -1) {
1776 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1777 trace_kvm_s390_sie_fault(vcpu);
1778 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1779 }
1780
1781 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1782
1783 if (rc == 0) {
1784 if (kvm_is_ucontrol(vcpu->kvm))
1785 /* Don't exit for host interrupts. */
1786 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1787 else
1788 rc = kvm_handle_sie_intercept(vcpu);
1789 }
1790
1791 return rc;
1792 }
1793
1794 static int __vcpu_run(struct kvm_vcpu *vcpu)
1795 {
1796 int rc, exit_reason;
1797
1798 /*
1799 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1800 * ning the guest), so that memslots (and other stuff) are protected
1801 */
1802 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1803
1804 do {
1805 rc = vcpu_pre_run(vcpu);
1806 if (rc)
1807 break;
1808
1809 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1810 /*
1811 * As PF_VCPU will be used in fault handler, between
1812 * guest_enter and guest_exit should be no uaccess.
1813 */
1814 preempt_disable();
1815 kvm_guest_enter();
1816 preempt_enable();
1817 exit_reason = sie64a(vcpu->arch.sie_block,
1818 vcpu->run->s.regs.gprs);
1819 kvm_guest_exit();
1820 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1821
1822 rc = vcpu_post_run(vcpu, exit_reason);
1823 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1824
1825 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1826 return rc;
1827 }
1828
1829 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1830 {
1831 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1832 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1833 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1834 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1835 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1836 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1837 /* some control register changes require a tlb flush */
1838 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1839 }
1840 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1841 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1842 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1843 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1844 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1845 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1846 }
1847 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1848 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1849 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1850 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1851 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1852 kvm_clear_async_pf_completion_queue(vcpu);
1853 }
1854 kvm_run->kvm_dirty_regs = 0;
1855 }
1856
1857 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1858 {
1859 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1860 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1861 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1862 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1863 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1864 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1865 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1866 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1867 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1868 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1869 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1870 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1871 }
1872
1873 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1874 {
1875 int rc;
1876 sigset_t sigsaved;
1877
1878 if (guestdbg_exit_pending(vcpu)) {
1879 kvm_s390_prepare_debug_exit(vcpu);
1880 return 0;
1881 }
1882
1883 if (vcpu->sigset_active)
1884 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1885
1886 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1887 kvm_s390_vcpu_start(vcpu);
1888 } else if (is_vcpu_stopped(vcpu)) {
1889 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1890 vcpu->vcpu_id);
1891 return -EINVAL;
1892 }
1893
1894 sync_regs(vcpu, kvm_run);
1895
1896 might_fault();
1897 rc = __vcpu_run(vcpu);
1898
1899 if (signal_pending(current) && !rc) {
1900 kvm_run->exit_reason = KVM_EXIT_INTR;
1901 rc = -EINTR;
1902 }
1903
1904 if (guestdbg_exit_pending(vcpu) && !rc) {
1905 kvm_s390_prepare_debug_exit(vcpu);
1906 rc = 0;
1907 }
1908
1909 if (rc == -EOPNOTSUPP) {
1910 /* intercept cannot be handled in-kernel, prepare kvm-run */
1911 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1912 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1913 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1914 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1915 rc = 0;
1916 }
1917
1918 if (rc == -EREMOTE) {
1919 /* intercept was handled, but userspace support is needed
1920 * kvm_run has been prepared by the handler */
1921 rc = 0;
1922 }
1923
1924 store_regs(vcpu, kvm_run);
1925
1926 if (vcpu->sigset_active)
1927 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1928
1929 vcpu->stat.exit_userspace++;
1930 return rc;
1931 }
1932
1933 /*
1934 * store status at address
1935 * we use have two special cases:
1936 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1937 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1938 */
1939 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1940 {
1941 unsigned char archmode = 1;
1942 unsigned int px;
1943 u64 clkcomp;
1944 int rc;
1945
1946 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1947 if (write_guest_abs(vcpu, 163, &archmode, 1))
1948 return -EFAULT;
1949 gpa = SAVE_AREA_BASE;
1950 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1951 if (write_guest_real(vcpu, 163, &archmode, 1))
1952 return -EFAULT;
1953 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1954 }
1955 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1956 vcpu->arch.guest_fpregs.fprs, 128);
1957 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1958 vcpu->run->s.regs.gprs, 128);
1959 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1960 &vcpu->arch.sie_block->gpsw, 16);
1961 px = kvm_s390_get_prefix(vcpu);
1962 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1963 &px, 4);
1964 rc |= write_guest_abs(vcpu,
1965 gpa + offsetof(struct save_area, fp_ctrl_reg),
1966 &vcpu->arch.guest_fpregs.fpc, 4);
1967 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1968 &vcpu->arch.sie_block->todpr, 4);
1969 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1970 &vcpu->arch.sie_block->cputm, 8);
1971 clkcomp = vcpu->arch.sie_block->ckc >> 8;
1972 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1973 &clkcomp, 8);
1974 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1975 &vcpu->run->s.regs.acrs, 64);
1976 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1977 &vcpu->arch.sie_block->gcr, 128);
1978 return rc ? -EFAULT : 0;
1979 }
1980
1981 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1982 {
1983 /*
1984 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1985 * copying in vcpu load/put. Lets update our copies before we save
1986 * it into the save area
1987 */
1988 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1989 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1990 save_access_regs(vcpu->run->s.regs.acrs);
1991
1992 return kvm_s390_store_status_unloaded(vcpu, addr);
1993 }
1994
1995 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1996 {
1997 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1998 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1999 exit_sie_sync(vcpu);
2000 }
2001
2002 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2003 {
2004 unsigned int i;
2005 struct kvm_vcpu *vcpu;
2006
2007 kvm_for_each_vcpu(i, vcpu, kvm) {
2008 __disable_ibs_on_vcpu(vcpu);
2009 }
2010 }
2011
2012 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2013 {
2014 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2015 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
2016 exit_sie_sync(vcpu);
2017 }
2018
2019 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2020 {
2021 int i, online_vcpus, started_vcpus = 0;
2022
2023 if (!is_vcpu_stopped(vcpu))
2024 return;
2025
2026 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2027 /* Only one cpu at a time may enter/leave the STOPPED state. */
2028 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2029 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2030
2031 for (i = 0; i < online_vcpus; i++) {
2032 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2033 started_vcpus++;
2034 }
2035
2036 if (started_vcpus == 0) {
2037 /* we're the only active VCPU -> speed it up */
2038 __enable_ibs_on_vcpu(vcpu);
2039 } else if (started_vcpus == 1) {
2040 /*
2041 * As we are starting a second VCPU, we have to disable
2042 * the IBS facility on all VCPUs to remove potentially
2043 * oustanding ENABLE requests.
2044 */
2045 __disable_ibs_on_all_vcpus(vcpu->kvm);
2046 }
2047
2048 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2049 /*
2050 * Another VCPU might have used IBS while we were offline.
2051 * Let's play safe and flush the VCPU at startup.
2052 */
2053 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2054 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2055 return;
2056 }
2057
2058 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2059 {
2060 int i, online_vcpus, started_vcpus = 0;
2061 struct kvm_vcpu *started_vcpu = NULL;
2062
2063 if (is_vcpu_stopped(vcpu))
2064 return;
2065
2066 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2067 /* Only one cpu at a time may enter/leave the STOPPED state. */
2068 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2069 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2070
2071 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2072 kvm_s390_clear_stop_irq(vcpu);
2073
2074 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2075 __disable_ibs_on_vcpu(vcpu);
2076
2077 for (i = 0; i < online_vcpus; i++) {
2078 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2079 started_vcpus++;
2080 started_vcpu = vcpu->kvm->vcpus[i];
2081 }
2082 }
2083
2084 if (started_vcpus == 1) {
2085 /*
2086 * As we only have one VCPU left, we want to enable the
2087 * IBS facility for that VCPU to speed it up.
2088 */
2089 __enable_ibs_on_vcpu(started_vcpu);
2090 }
2091
2092 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2093 return;
2094 }
2095
2096 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2097 struct kvm_enable_cap *cap)
2098 {
2099 int r;
2100
2101 if (cap->flags)
2102 return -EINVAL;
2103
2104 switch (cap->cap) {
2105 case KVM_CAP_S390_CSS_SUPPORT:
2106 if (!vcpu->kvm->arch.css_support) {
2107 vcpu->kvm->arch.css_support = 1;
2108 trace_kvm_s390_enable_css(vcpu->kvm);
2109 }
2110 r = 0;
2111 break;
2112 default:
2113 r = -EINVAL;
2114 break;
2115 }
2116 return r;
2117 }
2118
2119 long kvm_arch_vcpu_ioctl(struct file *filp,
2120 unsigned int ioctl, unsigned long arg)
2121 {
2122 struct kvm_vcpu *vcpu = filp->private_data;
2123 void __user *argp = (void __user *)arg;
2124 int idx;
2125 long r;
2126
2127 switch (ioctl) {
2128 case KVM_S390_INTERRUPT: {
2129 struct kvm_s390_interrupt s390int;
2130 struct kvm_s390_irq s390irq;
2131
2132 r = -EFAULT;
2133 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2134 break;
2135 if (s390int_to_s390irq(&s390int, &s390irq))
2136 return -EINVAL;
2137 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2138 break;
2139 }
2140 case KVM_S390_STORE_STATUS:
2141 idx = srcu_read_lock(&vcpu->kvm->srcu);
2142 r = kvm_s390_vcpu_store_status(vcpu, arg);
2143 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2144 break;
2145 case KVM_S390_SET_INITIAL_PSW: {
2146 psw_t psw;
2147
2148 r = -EFAULT;
2149 if (copy_from_user(&psw, argp, sizeof(psw)))
2150 break;
2151 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2152 break;
2153 }
2154 case KVM_S390_INITIAL_RESET:
2155 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2156 break;
2157 case KVM_SET_ONE_REG:
2158 case KVM_GET_ONE_REG: {
2159 struct kvm_one_reg reg;
2160 r = -EFAULT;
2161 if (copy_from_user(&reg, argp, sizeof(reg)))
2162 break;
2163 if (ioctl == KVM_SET_ONE_REG)
2164 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2165 else
2166 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2167 break;
2168 }
2169 #ifdef CONFIG_KVM_S390_UCONTROL
2170 case KVM_S390_UCAS_MAP: {
2171 struct kvm_s390_ucas_mapping ucasmap;
2172
2173 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2174 r = -EFAULT;
2175 break;
2176 }
2177
2178 if (!kvm_is_ucontrol(vcpu->kvm)) {
2179 r = -EINVAL;
2180 break;
2181 }
2182
2183 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2184 ucasmap.vcpu_addr, ucasmap.length);
2185 break;
2186 }
2187 case KVM_S390_UCAS_UNMAP: {
2188 struct kvm_s390_ucas_mapping ucasmap;
2189
2190 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2191 r = -EFAULT;
2192 break;
2193 }
2194
2195 if (!kvm_is_ucontrol(vcpu->kvm)) {
2196 r = -EINVAL;
2197 break;
2198 }
2199
2200 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2201 ucasmap.length);
2202 break;
2203 }
2204 #endif
2205 case KVM_S390_VCPU_FAULT: {
2206 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2207 break;
2208 }
2209 case KVM_ENABLE_CAP:
2210 {
2211 struct kvm_enable_cap cap;
2212 r = -EFAULT;
2213 if (copy_from_user(&cap, argp, sizeof(cap)))
2214 break;
2215 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2216 break;
2217 }
2218 default:
2219 r = -ENOTTY;
2220 }
2221 return r;
2222 }
2223
2224 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2225 {
2226 #ifdef CONFIG_KVM_S390_UCONTROL
2227 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2228 && (kvm_is_ucontrol(vcpu->kvm))) {
2229 vmf->page = virt_to_page(vcpu->arch.sie_block);
2230 get_page(vmf->page);
2231 return 0;
2232 }
2233 #endif
2234 return VM_FAULT_SIGBUS;
2235 }
2236
2237 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2238 unsigned long npages)
2239 {
2240 return 0;
2241 }
2242
2243 /* Section: memory related */
2244 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2245 struct kvm_memory_slot *memslot,
2246 struct kvm_userspace_memory_region *mem,
2247 enum kvm_mr_change change)
2248 {
2249 /* A few sanity checks. We can have memory slots which have to be
2250 located/ended at a segment boundary (1MB). The memory in userland is
2251 ok to be fragmented into various different vmas. It is okay to mmap()
2252 and munmap() stuff in this slot after doing this call at any time */
2253
2254 if (mem->userspace_addr & 0xffffful)
2255 return -EINVAL;
2256
2257 if (mem->memory_size & 0xffffful)
2258 return -EINVAL;
2259
2260 return 0;
2261 }
2262
2263 void kvm_arch_commit_memory_region(struct kvm *kvm,
2264 struct kvm_userspace_memory_region *mem,
2265 const struct kvm_memory_slot *old,
2266 enum kvm_mr_change change)
2267 {
2268 int rc;
2269
2270 /* If the basics of the memslot do not change, we do not want
2271 * to update the gmap. Every update causes several unnecessary
2272 * segment translation exceptions. This is usually handled just
2273 * fine by the normal fault handler + gmap, but it will also
2274 * cause faults on the prefix page of running guest CPUs.
2275 */
2276 if (old->userspace_addr == mem->userspace_addr &&
2277 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2278 old->npages * PAGE_SIZE == mem->memory_size)
2279 return;
2280
2281 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2282 mem->guest_phys_addr, mem->memory_size);
2283 if (rc)
2284 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
2285 return;
2286 }
2287
2288 static int __init kvm_s390_init(void)
2289 {
2290 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2291 }
2292
2293 static void __exit kvm_s390_exit(void)
2294 {
2295 kvm_exit();
2296 }
2297
2298 module_init(kvm_s390_init);
2299 module_exit(kvm_s390_exit);
2300
2301 /*
2302 * Enable autoloading of the kvm module.
2303 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2304 * since x86 takes a different approach.
2305 */
2306 #include <linux/miscdevice.h>
2307 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2308 MODULE_ALIAS("devname:kvm");
Linux kernel - Deliverables
This page took 0.124061 seconds and 4 git commands to generate.