2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/slab.h>
26 #include <linux/timer.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/lowcore.h>
29 #include <asm/pgtable.h>
31 #include <asm/switch_to.h>
32 #include <asm/facility.h>
37 #define CREATE_TRACE_POINTS
39 #include "trace-s390.h"
41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
43 struct kvm_stats_debugfs_item debugfs_entries
[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace
) },
45 { "exit_null", VCPU_STAT(exit_null
) },
46 { "exit_validity", VCPU_STAT(exit_validity
) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request
) },
48 { "exit_external_request", VCPU_STAT(exit_external_request
) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt
) },
50 { "exit_instruction", VCPU_STAT(exit_instruction
) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption
) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program
) },
53 { "halt_wakeup", VCPU_STAT(halt_wakeup
) },
54 { "instruction_lctlg", VCPU_STAT(instruction_lctlg
) },
55 { "instruction_lctl", VCPU_STAT(instruction_lctl
) },
56 { "instruction_stctl", VCPU_STAT(instruction_stctl
) },
57 { "instruction_stctg", VCPU_STAT(instruction_stctg
) },
58 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal
) },
59 { "deliver_external_call", VCPU_STAT(deliver_external_call
) },
60 { "deliver_service_signal", VCPU_STAT(deliver_service_signal
) },
61 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt
) },
62 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal
) },
63 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal
) },
64 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal
) },
65 { "deliver_program_interruption", VCPU_STAT(deliver_program_int
) },
66 { "exit_wait_state", VCPU_STAT(exit_wait_state
) },
67 { "instruction_pfmf", VCPU_STAT(instruction_pfmf
) },
68 { "instruction_stidp", VCPU_STAT(instruction_stidp
) },
69 { "instruction_spx", VCPU_STAT(instruction_spx
) },
70 { "instruction_stpx", VCPU_STAT(instruction_stpx
) },
71 { "instruction_stap", VCPU_STAT(instruction_stap
) },
72 { "instruction_storage_key", VCPU_STAT(instruction_storage_key
) },
73 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock
) },
74 { "instruction_stsch", VCPU_STAT(instruction_stsch
) },
75 { "instruction_chsc", VCPU_STAT(instruction_chsc
) },
76 { "instruction_essa", VCPU_STAT(instruction_essa
) },
77 { "instruction_stsi", VCPU_STAT(instruction_stsi
) },
78 { "instruction_stfl", VCPU_STAT(instruction_stfl
) },
79 { "instruction_tprot", VCPU_STAT(instruction_tprot
) },
80 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense
) },
81 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running
) },
82 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call
) },
83 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency
) },
84 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency
) },
85 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start
) },
86 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop
) },
87 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status
) },
88 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status
) },
89 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch
) },
90 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix
) },
91 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart
) },
92 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset
) },
93 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset
) },
94 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown
) },
95 { "diagnose_10", VCPU_STAT(diagnose_10
) },
96 { "diagnose_44", VCPU_STAT(diagnose_44
) },
97 { "diagnose_9c", VCPU_STAT(diagnose_9c
) },
101 unsigned long *vfacilities
;
102 static struct gmap_notifier gmap_notifier
;
104 /* test availability of vfacility */
105 int test_vfacility(unsigned long nr
)
107 return __test_facility(nr
, (void *) vfacilities
);
110 /* Section: not file related */
111 int kvm_arch_hardware_enable(void)
113 /* every s390 is virtualization enabled ;-) */
117 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
);
119 int kvm_arch_hardware_setup(void)
121 gmap_notifier
.notifier_call
= kvm_gmap_notifier
;
122 gmap_register_ipte_notifier(&gmap_notifier
);
126 void kvm_arch_hardware_unsetup(void)
128 gmap_unregister_ipte_notifier(&gmap_notifier
);
131 int kvm_arch_init(void *opaque
)
133 /* Register floating interrupt controller interface. */
134 return kvm_register_device_ops(&kvm_flic_ops
, KVM_DEV_TYPE_FLIC
);
137 /* Section: device related */
138 long kvm_arch_dev_ioctl(struct file
*filp
,
139 unsigned int ioctl
, unsigned long arg
)
141 if (ioctl
== KVM_S390_ENABLE_SIE
)
142 return s390_enable_sie();
146 int kvm_vm_ioctl_check_extension(struct kvm
*kvm
, long ext
)
151 case KVM_CAP_S390_PSW
:
152 case KVM_CAP_S390_GMAP
:
153 case KVM_CAP_SYNC_MMU
:
154 #ifdef CONFIG_KVM_S390_UCONTROL
155 case KVM_CAP_S390_UCONTROL
:
157 case KVM_CAP_ASYNC_PF
:
158 case KVM_CAP_SYNC_REGS
:
159 case KVM_CAP_ONE_REG
:
160 case KVM_CAP_ENABLE_CAP
:
161 case KVM_CAP_S390_CSS_SUPPORT
:
163 case KVM_CAP_IOEVENTFD
:
164 case KVM_CAP_DEVICE_CTRL
:
165 case KVM_CAP_ENABLE_CAP_VM
:
166 case KVM_CAP_S390_IRQCHIP
:
167 case KVM_CAP_VM_ATTRIBUTES
:
168 case KVM_CAP_MP_STATE
:
171 case KVM_CAP_NR_VCPUS
:
172 case KVM_CAP_MAX_VCPUS
:
175 case KVM_CAP_NR_MEMSLOTS
:
176 r
= KVM_USER_MEM_SLOTS
;
178 case KVM_CAP_S390_COW
:
179 r
= MACHINE_HAS_ESOP
;
187 static void kvm_s390_sync_dirty_log(struct kvm
*kvm
,
188 struct kvm_memory_slot
*memslot
)
190 gfn_t cur_gfn
, last_gfn
;
191 unsigned long address
;
192 struct gmap
*gmap
= kvm
->arch
.gmap
;
194 down_read(&gmap
->mm
->mmap_sem
);
195 /* Loop over all guest pages */
196 last_gfn
= memslot
->base_gfn
+ memslot
->npages
;
197 for (cur_gfn
= memslot
->base_gfn
; cur_gfn
<= last_gfn
; cur_gfn
++) {
198 address
= gfn_to_hva_memslot(memslot
, cur_gfn
);
200 if (gmap_test_and_clear_dirty(address
, gmap
))
201 mark_page_dirty(kvm
, cur_gfn
);
203 up_read(&gmap
->mm
->mmap_sem
);
206 /* Section: vm related */
208 * Get (and clear) the dirty memory log for a memory slot.
210 int kvm_vm_ioctl_get_dirty_log(struct kvm
*kvm
,
211 struct kvm_dirty_log
*log
)
215 struct kvm_memory_slot
*memslot
;
218 mutex_lock(&kvm
->slots_lock
);
221 if (log
->slot
>= KVM_USER_MEM_SLOTS
)
224 memslot
= id_to_memslot(kvm
->memslots
, log
->slot
);
226 if (!memslot
->dirty_bitmap
)
229 kvm_s390_sync_dirty_log(kvm
, memslot
);
230 r
= kvm_get_dirty_log(kvm
, log
, &is_dirty
);
234 /* Clear the dirty log */
236 n
= kvm_dirty_bitmap_bytes(memslot
);
237 memset(memslot
->dirty_bitmap
, 0, n
);
241 mutex_unlock(&kvm
->slots_lock
);
245 static int kvm_vm_ioctl_enable_cap(struct kvm
*kvm
, struct kvm_enable_cap
*cap
)
253 case KVM_CAP_S390_IRQCHIP
:
254 kvm
->arch
.use_irqchip
= 1;
264 static int kvm_s390_mem_control(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
268 switch (attr
->attr
) {
269 case KVM_S390_VM_MEM_ENABLE_CMMA
:
271 mutex_lock(&kvm
->lock
);
272 if (atomic_read(&kvm
->online_vcpus
) == 0) {
273 kvm
->arch
.use_cmma
= 1;
276 mutex_unlock(&kvm
->lock
);
278 case KVM_S390_VM_MEM_CLR_CMMA
:
279 mutex_lock(&kvm
->lock
);
280 idx
= srcu_read_lock(&kvm
->srcu
);
281 s390_reset_cmma(kvm
->arch
.gmap
->mm
);
282 srcu_read_unlock(&kvm
->srcu
, idx
);
283 mutex_unlock(&kvm
->lock
);
293 static int kvm_s390_vm_set_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
297 switch (attr
->group
) {
298 case KVM_S390_VM_MEM_CTRL
:
299 ret
= kvm_s390_mem_control(kvm
, attr
);
309 static int kvm_s390_vm_get_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
314 static int kvm_s390_vm_has_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
318 switch (attr
->group
) {
319 case KVM_S390_VM_MEM_CTRL
:
320 switch (attr
->attr
) {
321 case KVM_S390_VM_MEM_ENABLE_CMMA
:
322 case KVM_S390_VM_MEM_CLR_CMMA
:
338 long kvm_arch_vm_ioctl(struct file
*filp
,
339 unsigned int ioctl
, unsigned long arg
)
341 struct kvm
*kvm
= filp
->private_data
;
342 void __user
*argp
= (void __user
*)arg
;
343 struct kvm_device_attr attr
;
347 case KVM_S390_INTERRUPT
: {
348 struct kvm_s390_interrupt s390int
;
351 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
353 r
= kvm_s390_inject_vm(kvm
, &s390int
);
356 case KVM_ENABLE_CAP
: {
357 struct kvm_enable_cap cap
;
359 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
361 r
= kvm_vm_ioctl_enable_cap(kvm
, &cap
);
364 case KVM_CREATE_IRQCHIP
: {
365 struct kvm_irq_routing_entry routing
;
368 if (kvm
->arch
.use_irqchip
) {
369 /* Set up dummy routing. */
370 memset(&routing
, 0, sizeof(routing
));
371 kvm_set_irq_routing(kvm
, &routing
, 0, 0);
376 case KVM_SET_DEVICE_ATTR
: {
378 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
380 r
= kvm_s390_vm_set_attr(kvm
, &attr
);
383 case KVM_GET_DEVICE_ATTR
: {
385 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
387 r
= kvm_s390_vm_get_attr(kvm
, &attr
);
390 case KVM_HAS_DEVICE_ATTR
: {
392 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
394 r
= kvm_s390_vm_has_attr(kvm
, &attr
);
404 static int kvm_s390_crypto_init(struct kvm
*kvm
)
406 if (!test_vfacility(76))
409 kvm
->arch
.crypto
.crycb
= kzalloc(sizeof(*kvm
->arch
.crypto
.crycb
),
410 GFP_KERNEL
| GFP_DMA
);
411 if (!kvm
->arch
.crypto
.crycb
)
414 kvm
->arch
.crypto
.crycbd
= (__u32
) (unsigned long) kvm
->arch
.crypto
.crycb
|
420 int kvm_arch_init_vm(struct kvm
*kvm
, unsigned long type
)
424 static unsigned long sca_offset
;
427 #ifdef CONFIG_KVM_S390_UCONTROL
428 if (type
& ~KVM_VM_S390_UCONTROL
)
430 if ((type
& KVM_VM_S390_UCONTROL
) && (!capable(CAP_SYS_ADMIN
)))
437 rc
= s390_enable_sie();
443 kvm
->arch
.sca
= (struct sca_block
*) get_zeroed_page(GFP_KERNEL
);
446 spin_lock(&kvm_lock
);
447 sca_offset
= (sca_offset
+ 16) & 0x7f0;
448 kvm
->arch
.sca
= (struct sca_block
*) ((char *) kvm
->arch
.sca
+ sca_offset
);
449 spin_unlock(&kvm_lock
);
451 sprintf(debug_name
, "kvm-%u", current
->pid
);
453 kvm
->arch
.dbf
= debug_register(debug_name
, 8, 2, 8 * sizeof(long));
457 if (kvm_s390_crypto_init(kvm
) < 0)
460 spin_lock_init(&kvm
->arch
.float_int
.lock
);
461 INIT_LIST_HEAD(&kvm
->arch
.float_int
.list
);
462 init_waitqueue_head(&kvm
->arch
.ipte_wq
);
463 mutex_init(&kvm
->arch
.ipte_mutex
);
465 debug_register_view(kvm
->arch
.dbf
, &debug_sprintf_view
);
466 VM_EVENT(kvm
, 3, "%s", "vm created");
468 if (type
& KVM_VM_S390_UCONTROL
) {
469 kvm
->arch
.gmap
= NULL
;
471 kvm
->arch
.gmap
= gmap_alloc(current
->mm
, (1UL << 44) - 1);
474 kvm
->arch
.gmap
->private = kvm
;
475 kvm
->arch
.gmap
->pfault_enabled
= 0;
478 kvm
->arch
.css_support
= 0;
479 kvm
->arch
.use_irqchip
= 0;
481 spin_lock_init(&kvm
->arch
.start_stop_lock
);
485 kfree(kvm
->arch
.crypto
.crycb
);
487 debug_unregister(kvm
->arch
.dbf
);
489 free_page((unsigned long)(kvm
->arch
.sca
));
494 void kvm_arch_vcpu_destroy(struct kvm_vcpu
*vcpu
)
496 VCPU_EVENT(vcpu
, 3, "%s", "free cpu");
497 trace_kvm_s390_destroy_vcpu(vcpu
->vcpu_id
);
498 kvm_s390_clear_local_irqs(vcpu
);
499 kvm_clear_async_pf_completion_queue(vcpu
);
500 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
501 clear_bit(63 - vcpu
->vcpu_id
,
502 (unsigned long *) &vcpu
->kvm
->arch
.sca
->mcn
);
503 if (vcpu
->kvm
->arch
.sca
->cpu
[vcpu
->vcpu_id
].sda
==
504 (__u64
) vcpu
->arch
.sie_block
)
505 vcpu
->kvm
->arch
.sca
->cpu
[vcpu
->vcpu_id
].sda
= 0;
509 if (kvm_is_ucontrol(vcpu
->kvm
))
510 gmap_free(vcpu
->arch
.gmap
);
512 if (kvm_s390_cmma_enabled(vcpu
->kvm
))
513 kvm_s390_vcpu_unsetup_cmma(vcpu
);
514 free_page((unsigned long)(vcpu
->arch
.sie_block
));
516 kvm_vcpu_uninit(vcpu
);
517 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
520 static void kvm_free_vcpus(struct kvm
*kvm
)
523 struct kvm_vcpu
*vcpu
;
525 kvm_for_each_vcpu(i
, vcpu
, kvm
)
526 kvm_arch_vcpu_destroy(vcpu
);
528 mutex_lock(&kvm
->lock
);
529 for (i
= 0; i
< atomic_read(&kvm
->online_vcpus
); i
++)
530 kvm
->vcpus
[i
] = NULL
;
532 atomic_set(&kvm
->online_vcpus
, 0);
533 mutex_unlock(&kvm
->lock
);
536 void kvm_arch_destroy_vm(struct kvm
*kvm
)
539 free_page((unsigned long)(kvm
->arch
.sca
));
540 debug_unregister(kvm
->arch
.dbf
);
541 kfree(kvm
->arch
.crypto
.crycb
);
542 if (!kvm_is_ucontrol(kvm
))
543 gmap_free(kvm
->arch
.gmap
);
544 kvm_s390_destroy_adapters(kvm
);
545 kvm_s390_clear_float_irqs(kvm
);
548 /* Section: vcpu related */
549 int kvm_arch_vcpu_init(struct kvm_vcpu
*vcpu
)
551 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
552 kvm_clear_async_pf_completion_queue(vcpu
);
553 if (kvm_is_ucontrol(vcpu
->kvm
)) {
554 vcpu
->arch
.gmap
= gmap_alloc(current
->mm
, -1UL);
555 if (!vcpu
->arch
.gmap
)
557 vcpu
->arch
.gmap
->private = vcpu
->kvm
;
561 vcpu
->arch
.gmap
= vcpu
->kvm
->arch
.gmap
;
562 vcpu
->run
->kvm_valid_regs
= KVM_SYNC_PREFIX
|
571 void kvm_arch_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
573 save_fp_ctl(&vcpu
->arch
.host_fpregs
.fpc
);
574 save_fp_regs(vcpu
->arch
.host_fpregs
.fprs
);
575 save_access_regs(vcpu
->arch
.host_acrs
);
576 restore_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
577 restore_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
578 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
579 gmap_enable(vcpu
->arch
.gmap
);
580 atomic_set_mask(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
583 void kvm_arch_vcpu_put(struct kvm_vcpu
*vcpu
)
585 atomic_clear_mask(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
586 gmap_disable(vcpu
->arch
.gmap
);
587 save_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
588 save_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
589 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
590 restore_fp_ctl(&vcpu
->arch
.host_fpregs
.fpc
);
591 restore_fp_regs(vcpu
->arch
.host_fpregs
.fprs
);
592 restore_access_regs(vcpu
->arch
.host_acrs
);
595 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu
*vcpu
)
597 /* this equals initial cpu reset in pop, but we don't switch to ESA */
598 vcpu
->arch
.sie_block
->gpsw
.mask
= 0UL;
599 vcpu
->arch
.sie_block
->gpsw
.addr
= 0UL;
600 kvm_s390_set_prefix(vcpu
, 0);
601 vcpu
->arch
.sie_block
->cputm
= 0UL;
602 vcpu
->arch
.sie_block
->ckc
= 0UL;
603 vcpu
->arch
.sie_block
->todpr
= 0;
604 memset(vcpu
->arch
.sie_block
->gcr
, 0, 16 * sizeof(__u64
));
605 vcpu
->arch
.sie_block
->gcr
[0] = 0xE0UL
;
606 vcpu
->arch
.sie_block
->gcr
[14] = 0xC2000000UL
;
607 vcpu
->arch
.guest_fpregs
.fpc
= 0;
608 asm volatile("lfpc %0" : : "Q" (vcpu
->arch
.guest_fpregs
.fpc
));
609 vcpu
->arch
.sie_block
->gbea
= 1;
610 vcpu
->arch
.sie_block
->pp
= 0;
611 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
612 kvm_clear_async_pf_completion_queue(vcpu
);
613 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
))
614 kvm_s390_vcpu_stop(vcpu
);
615 kvm_s390_clear_local_irqs(vcpu
);
618 void kvm_arch_vcpu_postcreate(struct kvm_vcpu
*vcpu
)
622 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu
*vcpu
)
624 if (!test_vfacility(76))
627 vcpu
->arch
.sie_block
->crycbd
= vcpu
->kvm
->arch
.crypto
.crycbd
;
630 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu
*vcpu
)
632 free_page(vcpu
->arch
.sie_block
->cbrlo
);
633 vcpu
->arch
.sie_block
->cbrlo
= 0;
636 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu
*vcpu
)
638 vcpu
->arch
.sie_block
->cbrlo
= get_zeroed_page(GFP_KERNEL
);
639 if (!vcpu
->arch
.sie_block
->cbrlo
)
642 vcpu
->arch
.sie_block
->ecb2
|= 0x80;
643 vcpu
->arch
.sie_block
->ecb2
&= ~0x08;
647 int kvm_arch_vcpu_setup(struct kvm_vcpu
*vcpu
)
651 atomic_set(&vcpu
->arch
.sie_block
->cpuflags
, CPUSTAT_ZARCH
|
655 vcpu
->arch
.sie_block
->ecb
= 6;
656 if (test_vfacility(50) && test_vfacility(73))
657 vcpu
->arch
.sie_block
->ecb
|= 0x10;
659 vcpu
->arch
.sie_block
->ecb2
= 8;
660 vcpu
->arch
.sie_block
->eca
= 0xD1002000U
;
662 vcpu
->arch
.sie_block
->eca
|= 1;
663 vcpu
->arch
.sie_block
->fac
= (int) (long) vfacilities
;
664 vcpu
->arch
.sie_block
->ictl
|= ICTL_ISKE
| ICTL_SSKE
| ICTL_RRBE
|
667 if (kvm_s390_cmma_enabled(vcpu
->kvm
)) {
668 rc
= kvm_s390_vcpu_setup_cmma(vcpu
);
672 hrtimer_init(&vcpu
->arch
.ckc_timer
, CLOCK_REALTIME
, HRTIMER_MODE_ABS
);
673 vcpu
->arch
.ckc_timer
.function
= kvm_s390_idle_wakeup
;
674 get_cpu_id(&vcpu
->arch
.cpu_id
);
675 vcpu
->arch
.cpu_id
.version
= 0xff;
677 kvm_s390_vcpu_crypto_setup(vcpu
);
682 struct kvm_vcpu
*kvm_arch_vcpu_create(struct kvm
*kvm
,
685 struct kvm_vcpu
*vcpu
;
686 struct sie_page
*sie_page
;
689 if (id
>= KVM_MAX_VCPUS
)
694 vcpu
= kmem_cache_zalloc(kvm_vcpu_cache
, GFP_KERNEL
);
698 sie_page
= (struct sie_page
*) get_zeroed_page(GFP_KERNEL
);
702 vcpu
->arch
.sie_block
= &sie_page
->sie_block
;
703 vcpu
->arch
.sie_block
->itdba
= (unsigned long) &sie_page
->itdb
;
705 vcpu
->arch
.sie_block
->icpua
= id
;
706 if (!kvm_is_ucontrol(kvm
)) {
707 if (!kvm
->arch
.sca
) {
711 if (!kvm
->arch
.sca
->cpu
[id
].sda
)
712 kvm
->arch
.sca
->cpu
[id
].sda
=
713 (__u64
) vcpu
->arch
.sie_block
;
714 vcpu
->arch
.sie_block
->scaoh
=
715 (__u32
)(((__u64
)kvm
->arch
.sca
) >> 32);
716 vcpu
->arch
.sie_block
->scaol
= (__u32
)(__u64
)kvm
->arch
.sca
;
717 set_bit(63 - id
, (unsigned long *) &kvm
->arch
.sca
->mcn
);
720 spin_lock_init(&vcpu
->arch
.local_int
.lock
);
721 vcpu
->arch
.local_int
.float_int
= &kvm
->arch
.float_int
;
722 vcpu
->arch
.local_int
.wq
= &vcpu
->wq
;
723 vcpu
->arch
.local_int
.cpuflags
= &vcpu
->arch
.sie_block
->cpuflags
;
725 rc
= kvm_vcpu_init(vcpu
, kvm
, id
);
727 goto out_free_sie_block
;
728 VM_EVENT(kvm
, 3, "create cpu %d at %p, sie block at %p", id
, vcpu
,
729 vcpu
->arch
.sie_block
);
730 trace_kvm_s390_create_vcpu(id
, vcpu
, vcpu
->arch
.sie_block
);
734 free_page((unsigned long)(vcpu
->arch
.sie_block
));
736 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
741 int kvm_arch_vcpu_runnable(struct kvm_vcpu
*vcpu
)
743 return kvm_cpu_has_interrupt(vcpu
);
746 void s390_vcpu_block(struct kvm_vcpu
*vcpu
)
748 atomic_set_mask(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
751 void s390_vcpu_unblock(struct kvm_vcpu
*vcpu
)
753 atomic_clear_mask(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
757 * Kick a guest cpu out of SIE and wait until SIE is not running.
758 * If the CPU is not running (e.g. waiting as idle) the function will
759 * return immediately. */
760 void exit_sie(struct kvm_vcpu
*vcpu
)
762 atomic_set_mask(CPUSTAT_STOP_INT
, &vcpu
->arch
.sie_block
->cpuflags
);
763 while (vcpu
->arch
.sie_block
->prog0c
& PROG_IN_SIE
)
767 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
768 void exit_sie_sync(struct kvm_vcpu
*vcpu
)
770 s390_vcpu_block(vcpu
);
774 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
)
777 struct kvm
*kvm
= gmap
->private;
778 struct kvm_vcpu
*vcpu
;
780 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
781 /* match against both prefix pages */
782 if (kvm_s390_get_prefix(vcpu
) == (address
& ~0x1000UL
)) {
783 VCPU_EVENT(vcpu
, 2, "gmap notifier for %lx", address
);
784 kvm_make_request(KVM_REQ_MMU_RELOAD
, vcpu
);
790 int kvm_arch_vcpu_should_kick(struct kvm_vcpu
*vcpu
)
792 /* kvm common code refers to this, but never calls it */
797 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu
*vcpu
,
798 struct kvm_one_reg
*reg
)
803 case KVM_REG_S390_TODPR
:
804 r
= put_user(vcpu
->arch
.sie_block
->todpr
,
805 (u32 __user
*)reg
->addr
);
807 case KVM_REG_S390_EPOCHDIFF
:
808 r
= put_user(vcpu
->arch
.sie_block
->epoch
,
809 (u64 __user
*)reg
->addr
);
811 case KVM_REG_S390_CPU_TIMER
:
812 r
= put_user(vcpu
->arch
.sie_block
->cputm
,
813 (u64 __user
*)reg
->addr
);
815 case KVM_REG_S390_CLOCK_COMP
:
816 r
= put_user(vcpu
->arch
.sie_block
->ckc
,
817 (u64 __user
*)reg
->addr
);
819 case KVM_REG_S390_PFTOKEN
:
820 r
= put_user(vcpu
->arch
.pfault_token
,
821 (u64 __user
*)reg
->addr
);
823 case KVM_REG_S390_PFCOMPARE
:
824 r
= put_user(vcpu
->arch
.pfault_compare
,
825 (u64 __user
*)reg
->addr
);
827 case KVM_REG_S390_PFSELECT
:
828 r
= put_user(vcpu
->arch
.pfault_select
,
829 (u64 __user
*)reg
->addr
);
831 case KVM_REG_S390_PP
:
832 r
= put_user(vcpu
->arch
.sie_block
->pp
,
833 (u64 __user
*)reg
->addr
);
835 case KVM_REG_S390_GBEA
:
836 r
= put_user(vcpu
->arch
.sie_block
->gbea
,
837 (u64 __user
*)reg
->addr
);
846 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu
*vcpu
,
847 struct kvm_one_reg
*reg
)
852 case KVM_REG_S390_TODPR
:
853 r
= get_user(vcpu
->arch
.sie_block
->todpr
,
854 (u32 __user
*)reg
->addr
);
856 case KVM_REG_S390_EPOCHDIFF
:
857 r
= get_user(vcpu
->arch
.sie_block
->epoch
,
858 (u64 __user
*)reg
->addr
);
860 case KVM_REG_S390_CPU_TIMER
:
861 r
= get_user(vcpu
->arch
.sie_block
->cputm
,
862 (u64 __user
*)reg
->addr
);
864 case KVM_REG_S390_CLOCK_COMP
:
865 r
= get_user(vcpu
->arch
.sie_block
->ckc
,
866 (u64 __user
*)reg
->addr
);
868 case KVM_REG_S390_PFTOKEN
:
869 r
= get_user(vcpu
->arch
.pfault_token
,
870 (u64 __user
*)reg
->addr
);
872 case KVM_REG_S390_PFCOMPARE
:
873 r
= get_user(vcpu
->arch
.pfault_compare
,
874 (u64 __user
*)reg
->addr
);
876 case KVM_REG_S390_PFSELECT
:
877 r
= get_user(vcpu
->arch
.pfault_select
,
878 (u64 __user
*)reg
->addr
);
880 case KVM_REG_S390_PP
:
881 r
= get_user(vcpu
->arch
.sie_block
->pp
,
882 (u64 __user
*)reg
->addr
);
884 case KVM_REG_S390_GBEA
:
885 r
= get_user(vcpu
->arch
.sie_block
->gbea
,
886 (u64 __user
*)reg
->addr
);
895 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu
*vcpu
)
897 kvm_s390_vcpu_initial_reset(vcpu
);
901 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
903 memcpy(&vcpu
->run
->s
.regs
.gprs
, ®s
->gprs
, sizeof(regs
->gprs
));
907 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
909 memcpy(®s
->gprs
, &vcpu
->run
->s
.regs
.gprs
, sizeof(regs
->gprs
));
913 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu
*vcpu
,
914 struct kvm_sregs
*sregs
)
916 memcpy(&vcpu
->run
->s
.regs
.acrs
, &sregs
->acrs
, sizeof(sregs
->acrs
));
917 memcpy(&vcpu
->arch
.sie_block
->gcr
, &sregs
->crs
, sizeof(sregs
->crs
));
918 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
922 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu
*vcpu
,
923 struct kvm_sregs
*sregs
)
925 memcpy(&sregs
->acrs
, &vcpu
->run
->s
.regs
.acrs
, sizeof(sregs
->acrs
));
926 memcpy(&sregs
->crs
, &vcpu
->arch
.sie_block
->gcr
, sizeof(sregs
->crs
));
930 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
932 if (test_fp_ctl(fpu
->fpc
))
934 memcpy(&vcpu
->arch
.guest_fpregs
.fprs
, &fpu
->fprs
, sizeof(fpu
->fprs
));
935 vcpu
->arch
.guest_fpregs
.fpc
= fpu
->fpc
;
936 restore_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
937 restore_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
941 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
943 memcpy(&fpu
->fprs
, &vcpu
->arch
.guest_fpregs
.fprs
, sizeof(fpu
->fprs
));
944 fpu
->fpc
= vcpu
->arch
.guest_fpregs
.fpc
;
948 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu
*vcpu
, psw_t psw
)
952 if (!is_vcpu_stopped(vcpu
))
955 vcpu
->run
->psw_mask
= psw
.mask
;
956 vcpu
->run
->psw_addr
= psw
.addr
;
961 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
962 struct kvm_translation
*tr
)
964 return -EINVAL
; /* not implemented yet */
967 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
968 KVM_GUESTDBG_USE_HW_BP | \
971 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu
*vcpu
,
972 struct kvm_guest_debug
*dbg
)
976 vcpu
->guest_debug
= 0;
977 kvm_s390_clear_bp_data(vcpu
);
979 if (dbg
->control
& ~VALID_GUESTDBG_FLAGS
)
982 if (dbg
->control
& KVM_GUESTDBG_ENABLE
) {
983 vcpu
->guest_debug
= dbg
->control
;
984 /* enforce guest PER */
985 atomic_set_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
987 if (dbg
->control
& KVM_GUESTDBG_USE_HW_BP
)
988 rc
= kvm_s390_import_bp_data(vcpu
, dbg
);
990 atomic_clear_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
991 vcpu
->arch
.guestdbg
.last_bp
= 0;
995 vcpu
->guest_debug
= 0;
996 kvm_s390_clear_bp_data(vcpu
);
997 atomic_clear_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
1003 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu
*vcpu
,
1004 struct kvm_mp_state
*mp_state
)
1006 /* CHECK_STOP and LOAD are not supported yet */
1007 return is_vcpu_stopped(vcpu
) ? KVM_MP_STATE_STOPPED
:
1008 KVM_MP_STATE_OPERATING
;
1011 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu
*vcpu
,
1012 struct kvm_mp_state
*mp_state
)
1016 /* user space knows about this interface - let it control the state */
1017 vcpu
->kvm
->arch
.user_cpu_state_ctrl
= 1;
1019 switch (mp_state
->mp_state
) {
1020 case KVM_MP_STATE_STOPPED
:
1021 kvm_s390_vcpu_stop(vcpu
);
1023 case KVM_MP_STATE_OPERATING
:
1024 kvm_s390_vcpu_start(vcpu
);
1026 case KVM_MP_STATE_LOAD
:
1027 case KVM_MP_STATE_CHECK_STOP
:
1028 /* fall through - CHECK_STOP and LOAD are not supported yet */
1036 bool kvm_s390_cmma_enabled(struct kvm
*kvm
)
1038 if (!MACHINE_IS_LPAR
)
1040 /* only enable for z10 and later */
1041 if (!MACHINE_HAS_EDAT1
)
1043 if (!kvm
->arch
.use_cmma
)
1048 static bool ibs_enabled(struct kvm_vcpu
*vcpu
)
1050 return atomic_read(&vcpu
->arch
.sie_block
->cpuflags
) & CPUSTAT_IBS
;
1053 static int kvm_s390_handle_requests(struct kvm_vcpu
*vcpu
)
1056 s390_vcpu_unblock(vcpu
);
1058 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1059 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1060 * This ensures that the ipte instruction for this request has
1061 * already finished. We might race against a second unmapper that
1062 * wants to set the blocking bit. Lets just retry the request loop.
1064 if (kvm_check_request(KVM_REQ_MMU_RELOAD
, vcpu
)) {
1066 rc
= gmap_ipte_notify(vcpu
->arch
.gmap
,
1067 kvm_s390_get_prefix(vcpu
),
1074 if (kvm_check_request(KVM_REQ_TLB_FLUSH
, vcpu
)) {
1075 vcpu
->arch
.sie_block
->ihcpu
= 0xffff;
1079 if (kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
)) {
1080 if (!ibs_enabled(vcpu
)) {
1081 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 1);
1082 atomic_set_mask(CPUSTAT_IBS
,
1083 &vcpu
->arch
.sie_block
->cpuflags
);
1088 if (kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
)) {
1089 if (ibs_enabled(vcpu
)) {
1090 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 0);
1091 atomic_clear_mask(CPUSTAT_IBS
,
1092 &vcpu
->arch
.sie_block
->cpuflags
);
1097 /* nothing to do, just clear the request */
1098 clear_bit(KVM_REQ_UNHALT
, &vcpu
->requests
);
1104 * kvm_arch_fault_in_page - fault-in guest page if necessary
1105 * @vcpu: The corresponding virtual cpu
1106 * @gpa: Guest physical address
1107 * @writable: Whether the page should be writable or not
1109 * Make sure that a guest page has been faulted-in on the host.
1111 * Return: Zero on success, negative error code otherwise.
1113 long kvm_arch_fault_in_page(struct kvm_vcpu
*vcpu
, gpa_t gpa
, int writable
)
1115 return gmap_fault(vcpu
->arch
.gmap
, gpa
,
1116 writable
? FAULT_FLAG_WRITE
: 0);
1119 static void __kvm_inject_pfault_token(struct kvm_vcpu
*vcpu
, bool start_token
,
1120 unsigned long token
)
1122 struct kvm_s390_interrupt inti
;
1123 struct kvm_s390_irq irq
;
1126 irq
.u
.ext
.ext_params2
= token
;
1127 irq
.type
= KVM_S390_INT_PFAULT_INIT
;
1128 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu
, &irq
));
1130 inti
.type
= KVM_S390_INT_PFAULT_DONE
;
1131 inti
.parm64
= token
;
1132 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu
->kvm
, &inti
));
1136 void kvm_arch_async_page_not_present(struct kvm_vcpu
*vcpu
,
1137 struct kvm_async_pf
*work
)
1139 trace_kvm_s390_pfault_init(vcpu
, work
->arch
.pfault_token
);
1140 __kvm_inject_pfault_token(vcpu
, true, work
->arch
.pfault_token
);
1143 void kvm_arch_async_page_present(struct kvm_vcpu
*vcpu
,
1144 struct kvm_async_pf
*work
)
1146 trace_kvm_s390_pfault_done(vcpu
, work
->arch
.pfault_token
);
1147 __kvm_inject_pfault_token(vcpu
, false, work
->arch
.pfault_token
);
1150 void kvm_arch_async_page_ready(struct kvm_vcpu
*vcpu
,
1151 struct kvm_async_pf
*work
)
1153 /* s390 will always inject the page directly */
1156 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu
*vcpu
)
1159 * s390 will always inject the page directly,
1160 * but we still want check_async_completion to cleanup
1165 static int kvm_arch_setup_async_pf(struct kvm_vcpu
*vcpu
)
1168 struct kvm_arch_async_pf arch
;
1171 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
1173 if ((vcpu
->arch
.sie_block
->gpsw
.mask
& vcpu
->arch
.pfault_select
) !=
1174 vcpu
->arch
.pfault_compare
)
1176 if (psw_extint_disabled(vcpu
))
1178 if (kvm_cpu_has_interrupt(vcpu
))
1180 if (!(vcpu
->arch
.sie_block
->gcr
[0] & 0x200ul
))
1182 if (!vcpu
->arch
.gmap
->pfault_enabled
)
1185 hva
= gfn_to_hva(vcpu
->kvm
, gpa_to_gfn(current
->thread
.gmap_addr
));
1186 hva
+= current
->thread
.gmap_addr
& ~PAGE_MASK
;
1187 if (read_guest_real(vcpu
, vcpu
->arch
.pfault_token
, &arch
.pfault_token
, 8))
1190 rc
= kvm_setup_async_pf(vcpu
, current
->thread
.gmap_addr
, hva
, &arch
);
1194 static int vcpu_pre_run(struct kvm_vcpu
*vcpu
)
1199 * On s390 notifications for arriving pages will be delivered directly
1200 * to the guest but the house keeping for completed pfaults is
1201 * handled outside the worker.
1203 kvm_check_async_pf_completion(vcpu
);
1205 memcpy(&vcpu
->arch
.sie_block
->gg14
, &vcpu
->run
->s
.regs
.gprs
[14], 16);
1210 if (test_cpu_flag(CIF_MCCK_PENDING
))
1213 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1214 rc
= kvm_s390_deliver_pending_interrupts(vcpu
);
1219 rc
= kvm_s390_handle_requests(vcpu
);
1223 if (guestdbg_enabled(vcpu
)) {
1224 kvm_s390_backup_guest_per_regs(vcpu
);
1225 kvm_s390_patch_guest_per_regs(vcpu
);
1228 vcpu
->arch
.sie_block
->icptcode
= 0;
1229 cpuflags
= atomic_read(&vcpu
->arch
.sie_block
->cpuflags
);
1230 VCPU_EVENT(vcpu
, 6, "entering sie flags %x", cpuflags
);
1231 trace_kvm_s390_sie_enter(vcpu
, cpuflags
);
1236 static int vcpu_post_run(struct kvm_vcpu
*vcpu
, int exit_reason
)
1240 VCPU_EVENT(vcpu
, 6, "exit sie icptcode %d",
1241 vcpu
->arch
.sie_block
->icptcode
);
1242 trace_kvm_s390_sie_exit(vcpu
, vcpu
->arch
.sie_block
->icptcode
);
1244 if (guestdbg_enabled(vcpu
))
1245 kvm_s390_restore_guest_per_regs(vcpu
);
1247 if (exit_reason
>= 0) {
1249 } else if (kvm_is_ucontrol(vcpu
->kvm
)) {
1250 vcpu
->run
->exit_reason
= KVM_EXIT_S390_UCONTROL
;
1251 vcpu
->run
->s390_ucontrol
.trans_exc_code
=
1252 current
->thread
.gmap_addr
;
1253 vcpu
->run
->s390_ucontrol
.pgm_code
= 0x10;
1256 } else if (current
->thread
.gmap_pfault
) {
1257 trace_kvm_s390_major_guest_pfault(vcpu
);
1258 current
->thread
.gmap_pfault
= 0;
1259 if (kvm_arch_setup_async_pf(vcpu
)) {
1262 gpa_t gpa
= current
->thread
.gmap_addr
;
1263 rc
= kvm_arch_fault_in_page(vcpu
, gpa
, 1);
1268 VCPU_EVENT(vcpu
, 3, "%s", "fault in sie instruction");
1269 trace_kvm_s390_sie_fault(vcpu
);
1270 rc
= kvm_s390_inject_program_int(vcpu
, PGM_ADDRESSING
);
1273 memcpy(&vcpu
->run
->s
.regs
.gprs
[14], &vcpu
->arch
.sie_block
->gg14
, 16);
1276 if (kvm_is_ucontrol(vcpu
->kvm
))
1277 /* Don't exit for host interrupts. */
1278 rc
= vcpu
->arch
.sie_block
->icptcode
? -EOPNOTSUPP
: 0;
1280 rc
= kvm_handle_sie_intercept(vcpu
);
1286 static int __vcpu_run(struct kvm_vcpu
*vcpu
)
1288 int rc
, exit_reason
;
1291 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1292 * ning the guest), so that memslots (and other stuff) are protected
1294 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1297 rc
= vcpu_pre_run(vcpu
);
1301 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
1303 * As PF_VCPU will be used in fault handler, between
1304 * guest_enter and guest_exit should be no uaccess.
1309 exit_reason
= sie64a(vcpu
->arch
.sie_block
,
1310 vcpu
->run
->s
.regs
.gprs
);
1312 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1314 rc
= vcpu_post_run(vcpu
, exit_reason
);
1315 } while (!signal_pending(current
) && !guestdbg_exit_pending(vcpu
) && !rc
);
1317 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
1321 static void sync_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1323 vcpu
->arch
.sie_block
->gpsw
.mask
= kvm_run
->psw_mask
;
1324 vcpu
->arch
.sie_block
->gpsw
.addr
= kvm_run
->psw_addr
;
1325 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PREFIX
)
1326 kvm_s390_set_prefix(vcpu
, kvm_run
->s
.regs
.prefix
);
1327 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_CRS
) {
1328 memcpy(&vcpu
->arch
.sie_block
->gcr
, &kvm_run
->s
.regs
.crs
, 128);
1329 /* some control register changes require a tlb flush */
1330 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
1332 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_ARCH0
) {
1333 vcpu
->arch
.sie_block
->cputm
= kvm_run
->s
.regs
.cputm
;
1334 vcpu
->arch
.sie_block
->ckc
= kvm_run
->s
.regs
.ckc
;
1335 vcpu
->arch
.sie_block
->todpr
= kvm_run
->s
.regs
.todpr
;
1336 vcpu
->arch
.sie_block
->pp
= kvm_run
->s
.regs
.pp
;
1337 vcpu
->arch
.sie_block
->gbea
= kvm_run
->s
.regs
.gbea
;
1339 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PFAULT
) {
1340 vcpu
->arch
.pfault_token
= kvm_run
->s
.regs
.pft
;
1341 vcpu
->arch
.pfault_select
= kvm_run
->s
.regs
.pfs
;
1342 vcpu
->arch
.pfault_compare
= kvm_run
->s
.regs
.pfc
;
1344 kvm_run
->kvm_dirty_regs
= 0;
1347 static void store_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1349 kvm_run
->psw_mask
= vcpu
->arch
.sie_block
->gpsw
.mask
;
1350 kvm_run
->psw_addr
= vcpu
->arch
.sie_block
->gpsw
.addr
;
1351 kvm_run
->s
.regs
.prefix
= kvm_s390_get_prefix(vcpu
);
1352 memcpy(&kvm_run
->s
.regs
.crs
, &vcpu
->arch
.sie_block
->gcr
, 128);
1353 kvm_run
->s
.regs
.cputm
= vcpu
->arch
.sie_block
->cputm
;
1354 kvm_run
->s
.regs
.ckc
= vcpu
->arch
.sie_block
->ckc
;
1355 kvm_run
->s
.regs
.todpr
= vcpu
->arch
.sie_block
->todpr
;
1356 kvm_run
->s
.regs
.pp
= vcpu
->arch
.sie_block
->pp
;
1357 kvm_run
->s
.regs
.gbea
= vcpu
->arch
.sie_block
->gbea
;
1358 kvm_run
->s
.regs
.pft
= vcpu
->arch
.pfault_token
;
1359 kvm_run
->s
.regs
.pfs
= vcpu
->arch
.pfault_select
;
1360 kvm_run
->s
.regs
.pfc
= vcpu
->arch
.pfault_compare
;
1363 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1368 if (guestdbg_exit_pending(vcpu
)) {
1369 kvm_s390_prepare_debug_exit(vcpu
);
1373 if (vcpu
->sigset_active
)
1374 sigprocmask(SIG_SETMASK
, &vcpu
->sigset
, &sigsaved
);
1376 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
)) {
1377 kvm_s390_vcpu_start(vcpu
);
1378 } else if (is_vcpu_stopped(vcpu
)) {
1379 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1384 sync_regs(vcpu
, kvm_run
);
1387 rc
= __vcpu_run(vcpu
);
1389 if (signal_pending(current
) && !rc
) {
1390 kvm_run
->exit_reason
= KVM_EXIT_INTR
;
1394 if (guestdbg_exit_pending(vcpu
) && !rc
) {
1395 kvm_s390_prepare_debug_exit(vcpu
);
1399 if (rc
== -EOPNOTSUPP
) {
1400 /* intercept cannot be handled in-kernel, prepare kvm-run */
1401 kvm_run
->exit_reason
= KVM_EXIT_S390_SIEIC
;
1402 kvm_run
->s390_sieic
.icptcode
= vcpu
->arch
.sie_block
->icptcode
;
1403 kvm_run
->s390_sieic
.ipa
= vcpu
->arch
.sie_block
->ipa
;
1404 kvm_run
->s390_sieic
.ipb
= vcpu
->arch
.sie_block
->ipb
;
1408 if (rc
== -EREMOTE
) {
1409 /* intercept was handled, but userspace support is needed
1410 * kvm_run has been prepared by the handler */
1414 store_regs(vcpu
, kvm_run
);
1416 if (vcpu
->sigset_active
)
1417 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
1419 vcpu
->stat
.exit_userspace
++;
1424 * store status at address
1425 * we use have two special cases:
1426 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1427 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1429 int kvm_s390_store_status_unloaded(struct kvm_vcpu
*vcpu
, unsigned long gpa
)
1431 unsigned char archmode
= 1;
1436 if (gpa
== KVM_S390_STORE_STATUS_NOADDR
) {
1437 if (write_guest_abs(vcpu
, 163, &archmode
, 1))
1439 gpa
= SAVE_AREA_BASE
;
1440 } else if (gpa
== KVM_S390_STORE_STATUS_PREFIXED
) {
1441 if (write_guest_real(vcpu
, 163, &archmode
, 1))
1443 gpa
= kvm_s390_real_to_abs(vcpu
, SAVE_AREA_BASE
);
1445 rc
= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, fp_regs
),
1446 vcpu
->arch
.guest_fpregs
.fprs
, 128);
1447 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, gp_regs
),
1448 vcpu
->run
->s
.regs
.gprs
, 128);
1449 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, psw
),
1450 &vcpu
->arch
.sie_block
->gpsw
, 16);
1451 px
= kvm_s390_get_prefix(vcpu
);
1452 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, pref_reg
),
1454 rc
|= write_guest_abs(vcpu
,
1455 gpa
+ offsetof(struct save_area
, fp_ctrl_reg
),
1456 &vcpu
->arch
.guest_fpregs
.fpc
, 4);
1457 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, tod_reg
),
1458 &vcpu
->arch
.sie_block
->todpr
, 4);
1459 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, timer
),
1460 &vcpu
->arch
.sie_block
->cputm
, 8);
1461 clkcomp
= vcpu
->arch
.sie_block
->ckc
>> 8;
1462 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, clk_cmp
),
1464 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, acc_regs
),
1465 &vcpu
->run
->s
.regs
.acrs
, 64);
1466 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, ctrl_regs
),
1467 &vcpu
->arch
.sie_block
->gcr
, 128);
1468 return rc
? -EFAULT
: 0;
1471 int kvm_s390_vcpu_store_status(struct kvm_vcpu
*vcpu
, unsigned long addr
)
1474 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1475 * copying in vcpu load/put. Lets update our copies before we save
1476 * it into the save area
1478 save_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
1479 save_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
1480 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
1482 return kvm_s390_store_status_unloaded(vcpu
, addr
);
1485 static void __disable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
1487 kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
);
1488 kvm_make_request(KVM_REQ_DISABLE_IBS
, vcpu
);
1489 exit_sie_sync(vcpu
);
1492 static void __disable_ibs_on_all_vcpus(struct kvm
*kvm
)
1495 struct kvm_vcpu
*vcpu
;
1497 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
1498 __disable_ibs_on_vcpu(vcpu
);
1502 static void __enable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
1504 kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
);
1505 kvm_make_request(KVM_REQ_ENABLE_IBS
, vcpu
);
1506 exit_sie_sync(vcpu
);
1509 void kvm_s390_vcpu_start(struct kvm_vcpu
*vcpu
)
1511 int i
, online_vcpus
, started_vcpus
= 0;
1513 if (!is_vcpu_stopped(vcpu
))
1516 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 1);
1517 /* Only one cpu at a time may enter/leave the STOPPED state. */
1518 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
1519 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
1521 for (i
= 0; i
< online_vcpus
; i
++) {
1522 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
]))
1526 if (started_vcpus
== 0) {
1527 /* we're the only active VCPU -> speed it up */
1528 __enable_ibs_on_vcpu(vcpu
);
1529 } else if (started_vcpus
== 1) {
1531 * As we are starting a second VCPU, we have to disable
1532 * the IBS facility on all VCPUs to remove potentially
1533 * oustanding ENABLE requests.
1535 __disable_ibs_on_all_vcpus(vcpu
->kvm
);
1538 atomic_clear_mask(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
1540 * Another VCPU might have used IBS while we were offline.
1541 * Let's play safe and flush the VCPU at startup.
1543 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
1544 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
1548 void kvm_s390_vcpu_stop(struct kvm_vcpu
*vcpu
)
1550 int i
, online_vcpus
, started_vcpus
= 0;
1551 struct kvm_vcpu
*started_vcpu
= NULL
;
1553 if (is_vcpu_stopped(vcpu
))
1556 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 0);
1557 /* Only one cpu at a time may enter/leave the STOPPED state. */
1558 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
1559 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
1561 /* Need to lock access to action_bits to avoid a SIGP race condition */
1562 spin_lock(&vcpu
->arch
.local_int
.lock
);
1563 atomic_set_mask(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
1565 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1566 vcpu
->arch
.local_int
.action_bits
&=
1567 ~(ACTION_STOP_ON_STOP
| ACTION_STORE_ON_STOP
);
1568 spin_unlock(&vcpu
->arch
.local_int
.lock
);
1570 __disable_ibs_on_vcpu(vcpu
);
1572 for (i
= 0; i
< online_vcpus
; i
++) {
1573 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
])) {
1575 started_vcpu
= vcpu
->kvm
->vcpus
[i
];
1579 if (started_vcpus
== 1) {
1581 * As we only have one VCPU left, we want to enable the
1582 * IBS facility for that VCPU to speed it up.
1584 __enable_ibs_on_vcpu(started_vcpu
);
1587 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
1591 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu
*vcpu
,
1592 struct kvm_enable_cap
*cap
)
1600 case KVM_CAP_S390_CSS_SUPPORT
:
1601 if (!vcpu
->kvm
->arch
.css_support
) {
1602 vcpu
->kvm
->arch
.css_support
= 1;
1603 trace_kvm_s390_enable_css(vcpu
->kvm
);
1614 long kvm_arch_vcpu_ioctl(struct file
*filp
,
1615 unsigned int ioctl
, unsigned long arg
)
1617 struct kvm_vcpu
*vcpu
= filp
->private_data
;
1618 void __user
*argp
= (void __user
*)arg
;
1623 case KVM_S390_INTERRUPT
: {
1624 struct kvm_s390_interrupt s390int
;
1625 struct kvm_s390_irq s390irq
;
1628 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
1630 if (s390int_to_s390irq(&s390int
, &s390irq
))
1632 r
= kvm_s390_inject_vcpu(vcpu
, &s390irq
);
1635 case KVM_S390_STORE_STATUS
:
1636 idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1637 r
= kvm_s390_vcpu_store_status(vcpu
, arg
);
1638 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
1640 case KVM_S390_SET_INITIAL_PSW
: {
1644 if (copy_from_user(&psw
, argp
, sizeof(psw
)))
1646 r
= kvm_arch_vcpu_ioctl_set_initial_psw(vcpu
, psw
);
1649 case KVM_S390_INITIAL_RESET
:
1650 r
= kvm_arch_vcpu_ioctl_initial_reset(vcpu
);
1652 case KVM_SET_ONE_REG
:
1653 case KVM_GET_ONE_REG
: {
1654 struct kvm_one_reg reg
;
1656 if (copy_from_user(®
, argp
, sizeof(reg
)))
1658 if (ioctl
== KVM_SET_ONE_REG
)
1659 r
= kvm_arch_vcpu_ioctl_set_one_reg(vcpu
, ®
);
1661 r
= kvm_arch_vcpu_ioctl_get_one_reg(vcpu
, ®
);
1664 #ifdef CONFIG_KVM_S390_UCONTROL
1665 case KVM_S390_UCAS_MAP
: {
1666 struct kvm_s390_ucas_mapping ucasmap
;
1668 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
1673 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1678 r
= gmap_map_segment(vcpu
->arch
.gmap
, ucasmap
.user_addr
,
1679 ucasmap
.vcpu_addr
, ucasmap
.length
);
1682 case KVM_S390_UCAS_UNMAP
: {
1683 struct kvm_s390_ucas_mapping ucasmap
;
1685 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
1690 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1695 r
= gmap_unmap_segment(vcpu
->arch
.gmap
, ucasmap
.vcpu_addr
,
1700 case KVM_S390_VCPU_FAULT
: {
1701 r
= gmap_fault(vcpu
->arch
.gmap
, arg
, 0);
1704 case KVM_ENABLE_CAP
:
1706 struct kvm_enable_cap cap
;
1708 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
1710 r
= kvm_vcpu_ioctl_enable_cap(vcpu
, &cap
);
1719 int kvm_arch_vcpu_fault(struct kvm_vcpu
*vcpu
, struct vm_fault
*vmf
)
1721 #ifdef CONFIG_KVM_S390_UCONTROL
1722 if ((vmf
->pgoff
== KVM_S390_SIE_PAGE_OFFSET
)
1723 && (kvm_is_ucontrol(vcpu
->kvm
))) {
1724 vmf
->page
= virt_to_page(vcpu
->arch
.sie_block
);
1725 get_page(vmf
->page
);
1729 return VM_FAULT_SIGBUS
;
1732 int kvm_arch_create_memslot(struct kvm
*kvm
, struct kvm_memory_slot
*slot
,
1733 unsigned long npages
)
1738 /* Section: memory related */
1739 int kvm_arch_prepare_memory_region(struct kvm
*kvm
,
1740 struct kvm_memory_slot
*memslot
,
1741 struct kvm_userspace_memory_region
*mem
,
1742 enum kvm_mr_change change
)
1744 /* A few sanity checks. We can have memory slots which have to be
1745 located/ended at a segment boundary (1MB). The memory in userland is
1746 ok to be fragmented into various different vmas. It is okay to mmap()
1747 and munmap() stuff in this slot after doing this call at any time */
1749 if (mem
->userspace_addr
& 0xffffful
)
1752 if (mem
->memory_size
& 0xffffful
)
1758 void kvm_arch_commit_memory_region(struct kvm
*kvm
,
1759 struct kvm_userspace_memory_region
*mem
,
1760 const struct kvm_memory_slot
*old
,
1761 enum kvm_mr_change change
)
1765 /* If the basics of the memslot do not change, we do not want
1766 * to update the gmap. Every update causes several unnecessary
1767 * segment translation exceptions. This is usually handled just
1768 * fine by the normal fault handler + gmap, but it will also
1769 * cause faults on the prefix page of running guest CPUs.
1771 if (old
->userspace_addr
== mem
->userspace_addr
&&
1772 old
->base_gfn
* PAGE_SIZE
== mem
->guest_phys_addr
&&
1773 old
->npages
* PAGE_SIZE
== mem
->memory_size
)
1776 rc
= gmap_map_segment(kvm
->arch
.gmap
, mem
->userspace_addr
,
1777 mem
->guest_phys_addr
, mem
->memory_size
);
1779 printk(KERN_WARNING
"kvm-s390: failed to commit memory region\n");
1783 static int __init
kvm_s390_init(void)
1786 ret
= kvm_init(NULL
, sizeof(struct kvm_vcpu
), 0, THIS_MODULE
);
1791 * guests can ask for up to 255+1 double words, we need a full page
1792 * to hold the maximum amount of facilities. On the other hand, we
1793 * only set facilities that are known to work in KVM.
1795 vfacilities
= (unsigned long *) get_zeroed_page(GFP_KERNEL
|GFP_DMA
);
1800 memcpy(vfacilities
, S390_lowcore
.stfle_fac_list
, 16);
1801 vfacilities
[0] &= 0xff82fffbf47c2000UL
;
1802 vfacilities
[1] &= 0x005c000000000000UL
;
1806 static void __exit
kvm_s390_exit(void)
1808 free_page((unsigned long) vfacilities
);
1812 module_init(kvm_s390_init
);
1813 module_exit(kvm_s390_exit
);
1816 * Enable autoloading of the kvm module.
1817 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1818 * since x86 takes a different approach.
1820 #include <linux/miscdevice.h>
1821 MODULE_ALIAS_MISCDEV(KVM_MINOR
);
1822 MODULE_ALIAS("devname:kvm");