2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
32 #include <asm/pgtable.h>
35 #include <asm/switch_to.h>
41 #define KMSG_COMPONENT "kvm-s390"
43 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
45 #define CREATE_TRACE_POINTS
47 #include "trace-s390.h"
49 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
51 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
52 (KVM_MAX_VCPUS + LOCAL_IRQS))
54 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
56 struct kvm_stats_debugfs_item debugfs_entries
[] = {
57 { "userspace_handled", VCPU_STAT(exit_userspace
) },
58 { "exit_null", VCPU_STAT(exit_null
) },
59 { "exit_validity", VCPU_STAT(exit_validity
) },
60 { "exit_stop_request", VCPU_STAT(exit_stop_request
) },
61 { "exit_external_request", VCPU_STAT(exit_external_request
) },
62 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt
) },
63 { "exit_instruction", VCPU_STAT(exit_instruction
) },
64 { "exit_program_interruption", VCPU_STAT(exit_program_interruption
) },
65 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program
) },
66 { "halt_successful_poll", VCPU_STAT(halt_successful_poll
) },
67 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll
) },
68 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid
) },
69 { "halt_wakeup", VCPU_STAT(halt_wakeup
) },
70 { "instruction_lctlg", VCPU_STAT(instruction_lctlg
) },
71 { "instruction_lctl", VCPU_STAT(instruction_lctl
) },
72 { "instruction_stctl", VCPU_STAT(instruction_stctl
) },
73 { "instruction_stctg", VCPU_STAT(instruction_stctg
) },
74 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal
) },
75 { "deliver_external_call", VCPU_STAT(deliver_external_call
) },
76 { "deliver_service_signal", VCPU_STAT(deliver_service_signal
) },
77 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt
) },
78 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal
) },
79 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal
) },
80 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal
) },
81 { "deliver_program_interruption", VCPU_STAT(deliver_program_int
) },
82 { "exit_wait_state", VCPU_STAT(exit_wait_state
) },
83 { "instruction_pfmf", VCPU_STAT(instruction_pfmf
) },
84 { "instruction_stidp", VCPU_STAT(instruction_stidp
) },
85 { "instruction_spx", VCPU_STAT(instruction_spx
) },
86 { "instruction_stpx", VCPU_STAT(instruction_stpx
) },
87 { "instruction_stap", VCPU_STAT(instruction_stap
) },
88 { "instruction_storage_key", VCPU_STAT(instruction_storage_key
) },
89 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock
) },
90 { "instruction_stsch", VCPU_STAT(instruction_stsch
) },
91 { "instruction_chsc", VCPU_STAT(instruction_chsc
) },
92 { "instruction_essa", VCPU_STAT(instruction_essa
) },
93 { "instruction_stsi", VCPU_STAT(instruction_stsi
) },
94 { "instruction_stfl", VCPU_STAT(instruction_stfl
) },
95 { "instruction_tprot", VCPU_STAT(instruction_tprot
) },
96 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense
) },
97 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running
) },
98 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call
) },
99 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency
) },
100 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency
) },
101 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start
) },
102 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop
) },
103 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status
) },
104 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status
) },
105 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status
) },
106 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch
) },
107 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix
) },
108 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart
) },
109 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset
) },
110 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset
) },
111 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown
) },
112 { "diagnose_10", VCPU_STAT(diagnose_10
) },
113 { "diagnose_44", VCPU_STAT(diagnose_44
) },
114 { "diagnose_9c", VCPU_STAT(diagnose_9c
) },
115 { "diagnose_258", VCPU_STAT(diagnose_258
) },
116 { "diagnose_308", VCPU_STAT(diagnose_308
) },
117 { "diagnose_500", VCPU_STAT(diagnose_500
) },
121 /* upper facilities limit for kvm */
122 unsigned long kvm_s390_fac_list_mask
[16] = {
123 0xffe6000000000000UL
,
124 0x005e000000000000UL
,
127 unsigned long kvm_s390_fac_list_mask_size(void)
129 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask
) > S390_ARCH_FAC_MASK_SIZE_U64
);
130 return ARRAY_SIZE(kvm_s390_fac_list_mask
);
133 static struct gmap_notifier gmap_notifier
;
134 debug_info_t
*kvm_s390_dbf
;
136 /* Section: not file related */
137 int kvm_arch_hardware_enable(void)
139 /* every s390 is virtualization enabled ;-) */
143 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
);
146 * This callback is executed during stop_machine(). All CPUs are therefore
147 * temporarily stopped. In order not to change guest behavior, we have to
148 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
149 * so a CPU won't be stopped while calculating with the epoch.
151 static int kvm_clock_sync(struct notifier_block
*notifier
, unsigned long val
,
155 struct kvm_vcpu
*vcpu
;
157 unsigned long long *delta
= v
;
159 list_for_each_entry(kvm
, &vm_list
, vm_list
) {
160 kvm
->arch
.epoch
-= *delta
;
161 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
162 vcpu
->arch
.sie_block
->epoch
-= *delta
;
163 if (vcpu
->arch
.cputm_enabled
)
164 vcpu
->arch
.cputm_start
+= *delta
;
170 static struct notifier_block kvm_clock_notifier
= {
171 .notifier_call
= kvm_clock_sync
,
174 int kvm_arch_hardware_setup(void)
176 gmap_notifier
.notifier_call
= kvm_gmap_notifier
;
177 gmap_register_ipte_notifier(&gmap_notifier
);
178 atomic_notifier_chain_register(&s390_epoch_delta_notifier
,
179 &kvm_clock_notifier
);
183 void kvm_arch_hardware_unsetup(void)
185 gmap_unregister_ipte_notifier(&gmap_notifier
);
186 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier
,
187 &kvm_clock_notifier
);
190 int kvm_arch_init(void *opaque
)
192 kvm_s390_dbf
= debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
196 if (debug_register_view(kvm_s390_dbf
, &debug_sprintf_view
)) {
197 debug_unregister(kvm_s390_dbf
);
201 /* Register floating interrupt controller interface. */
202 return kvm_register_device_ops(&kvm_flic_ops
, KVM_DEV_TYPE_FLIC
);
205 void kvm_arch_exit(void)
207 debug_unregister(kvm_s390_dbf
);
210 /* Section: device related */
211 long kvm_arch_dev_ioctl(struct file
*filp
,
212 unsigned int ioctl
, unsigned long arg
)
214 if (ioctl
== KVM_S390_ENABLE_SIE
)
215 return s390_enable_sie();
219 int kvm_vm_ioctl_check_extension(struct kvm
*kvm
, long ext
)
224 case KVM_CAP_S390_PSW
:
225 case KVM_CAP_S390_GMAP
:
226 case KVM_CAP_SYNC_MMU
:
227 #ifdef CONFIG_KVM_S390_UCONTROL
228 case KVM_CAP_S390_UCONTROL
:
230 case KVM_CAP_ASYNC_PF
:
231 case KVM_CAP_SYNC_REGS
:
232 case KVM_CAP_ONE_REG
:
233 case KVM_CAP_ENABLE_CAP
:
234 case KVM_CAP_S390_CSS_SUPPORT
:
235 case KVM_CAP_IOEVENTFD
:
236 case KVM_CAP_DEVICE_CTRL
:
237 case KVM_CAP_ENABLE_CAP_VM
:
238 case KVM_CAP_S390_IRQCHIP
:
239 case KVM_CAP_VM_ATTRIBUTES
:
240 case KVM_CAP_MP_STATE
:
241 case KVM_CAP_S390_INJECT_IRQ
:
242 case KVM_CAP_S390_USER_SIGP
:
243 case KVM_CAP_S390_USER_STSI
:
244 case KVM_CAP_S390_SKEYS
:
245 case KVM_CAP_S390_IRQ_STATE
:
248 case KVM_CAP_S390_MEM_OP
:
251 case KVM_CAP_NR_VCPUS
:
252 case KVM_CAP_MAX_VCPUS
:
253 r
= sclp
.has_esca
? KVM_S390_ESCA_CPU_SLOTS
254 : KVM_S390_BSCA_CPU_SLOTS
;
256 case KVM_CAP_NR_MEMSLOTS
:
257 r
= KVM_USER_MEM_SLOTS
;
259 case KVM_CAP_S390_COW
:
260 r
= MACHINE_HAS_ESOP
;
262 case KVM_CAP_S390_VECTOR_REGISTERS
:
265 case KVM_CAP_S390_RI
:
266 r
= test_facility(64);
274 static void kvm_s390_sync_dirty_log(struct kvm
*kvm
,
275 struct kvm_memory_slot
*memslot
)
277 gfn_t cur_gfn
, last_gfn
;
278 unsigned long address
;
279 struct gmap
*gmap
= kvm
->arch
.gmap
;
281 /* Loop over all guest pages */
282 last_gfn
= memslot
->base_gfn
+ memslot
->npages
;
283 for (cur_gfn
= memslot
->base_gfn
; cur_gfn
<= last_gfn
; cur_gfn
++) {
284 address
= gfn_to_hva_memslot(memslot
, cur_gfn
);
286 if (test_and_clear_guest_dirty(gmap
->mm
, address
))
287 mark_page_dirty(kvm
, cur_gfn
);
288 if (fatal_signal_pending(current
))
294 /* Section: vm related */
295 static void sca_del_vcpu(struct kvm_vcpu
*vcpu
);
298 * Get (and clear) the dirty memory log for a memory slot.
300 int kvm_vm_ioctl_get_dirty_log(struct kvm
*kvm
,
301 struct kvm_dirty_log
*log
)
305 struct kvm_memslots
*slots
;
306 struct kvm_memory_slot
*memslot
;
309 mutex_lock(&kvm
->slots_lock
);
312 if (log
->slot
>= KVM_USER_MEM_SLOTS
)
315 slots
= kvm_memslots(kvm
);
316 memslot
= id_to_memslot(slots
, log
->slot
);
318 if (!memslot
->dirty_bitmap
)
321 kvm_s390_sync_dirty_log(kvm
, memslot
);
322 r
= kvm_get_dirty_log(kvm
, log
, &is_dirty
);
326 /* Clear the dirty log */
328 n
= kvm_dirty_bitmap_bytes(memslot
);
329 memset(memslot
->dirty_bitmap
, 0, n
);
333 mutex_unlock(&kvm
->slots_lock
);
337 static int kvm_vm_ioctl_enable_cap(struct kvm
*kvm
, struct kvm_enable_cap
*cap
)
345 case KVM_CAP_S390_IRQCHIP
:
346 VM_EVENT(kvm
, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
347 kvm
->arch
.use_irqchip
= 1;
350 case KVM_CAP_S390_USER_SIGP
:
351 VM_EVENT(kvm
, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
352 kvm
->arch
.user_sigp
= 1;
355 case KVM_CAP_S390_VECTOR_REGISTERS
:
356 mutex_lock(&kvm
->lock
);
357 if (atomic_read(&kvm
->online_vcpus
)) {
359 } else if (MACHINE_HAS_VX
) {
360 set_kvm_facility(kvm
->arch
.model
.fac_mask
, 129);
361 set_kvm_facility(kvm
->arch
.model
.fac_list
, 129);
365 mutex_unlock(&kvm
->lock
);
366 VM_EVENT(kvm
, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
367 r
? "(not available)" : "(success)");
369 case KVM_CAP_S390_RI
:
371 mutex_lock(&kvm
->lock
);
372 if (atomic_read(&kvm
->online_vcpus
)) {
374 } else if (test_facility(64)) {
375 set_kvm_facility(kvm
->arch
.model
.fac_mask
, 64);
376 set_kvm_facility(kvm
->arch
.model
.fac_list
, 64);
379 mutex_unlock(&kvm
->lock
);
380 VM_EVENT(kvm
, 3, "ENABLE: CAP_S390_RI %s",
381 r
? "(not available)" : "(success)");
383 case KVM_CAP_S390_USER_STSI
:
384 VM_EVENT(kvm
, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
385 kvm
->arch
.user_stsi
= 1;
395 static int kvm_s390_get_mem_control(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
399 switch (attr
->attr
) {
400 case KVM_S390_VM_MEM_LIMIT_SIZE
:
402 VM_EVENT(kvm
, 3, "QUERY: max guest memory: %lu bytes",
403 kvm
->arch
.mem_limit
);
404 if (put_user(kvm
->arch
.mem_limit
, (u64 __user
*)attr
->addr
))
414 static int kvm_s390_set_mem_control(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
418 switch (attr
->attr
) {
419 case KVM_S390_VM_MEM_ENABLE_CMMA
:
420 /* enable CMMA only for z10 and later (EDAT_1) */
422 if (!MACHINE_IS_LPAR
|| !MACHINE_HAS_EDAT1
)
426 VM_EVENT(kvm
, 3, "%s", "ENABLE: CMMA support");
427 mutex_lock(&kvm
->lock
);
428 if (atomic_read(&kvm
->online_vcpus
) == 0) {
429 kvm
->arch
.use_cmma
= 1;
432 mutex_unlock(&kvm
->lock
);
434 case KVM_S390_VM_MEM_CLR_CMMA
:
436 if (!kvm
->arch
.use_cmma
)
439 VM_EVENT(kvm
, 3, "%s", "RESET: CMMA states");
440 mutex_lock(&kvm
->lock
);
441 idx
= srcu_read_lock(&kvm
->srcu
);
442 s390_reset_cmma(kvm
->arch
.gmap
->mm
);
443 srcu_read_unlock(&kvm
->srcu
, idx
);
444 mutex_unlock(&kvm
->lock
);
447 case KVM_S390_VM_MEM_LIMIT_SIZE
: {
448 unsigned long new_limit
;
450 if (kvm_is_ucontrol(kvm
))
453 if (get_user(new_limit
, (u64 __user
*)attr
->addr
))
456 if (kvm
->arch
.mem_limit
!= KVM_S390_NO_MEM_LIMIT
&&
457 new_limit
> kvm
->arch
.mem_limit
)
463 /* gmap_alloc takes last usable address */
464 if (new_limit
!= KVM_S390_NO_MEM_LIMIT
)
468 mutex_lock(&kvm
->lock
);
469 if (atomic_read(&kvm
->online_vcpus
) == 0) {
470 /* gmap_alloc will round the limit up */
471 struct gmap
*new = gmap_alloc(current
->mm
, new_limit
);
476 gmap_free(kvm
->arch
.gmap
);
478 kvm
->arch
.gmap
= new;
482 mutex_unlock(&kvm
->lock
);
483 VM_EVENT(kvm
, 3, "SET: max guest address: %lu", new_limit
);
484 VM_EVENT(kvm
, 3, "New guest asce: 0x%pK",
485 (void *) kvm
->arch
.gmap
->asce
);
495 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu
*vcpu
);
497 static int kvm_s390_vm_set_crypto(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
499 struct kvm_vcpu
*vcpu
;
502 if (!test_kvm_facility(kvm
, 76))
505 mutex_lock(&kvm
->lock
);
506 switch (attr
->attr
) {
507 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW
:
509 kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
,
510 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
511 kvm
->arch
.crypto
.aes_kw
= 1;
512 VM_EVENT(kvm
, 3, "%s", "ENABLE: AES keywrapping support");
514 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW
:
516 kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
,
517 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
518 kvm
->arch
.crypto
.dea_kw
= 1;
519 VM_EVENT(kvm
, 3, "%s", "ENABLE: DEA keywrapping support");
521 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW
:
522 kvm
->arch
.crypto
.aes_kw
= 0;
523 memset(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
, 0,
524 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
525 VM_EVENT(kvm
, 3, "%s", "DISABLE: AES keywrapping support");
527 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW
:
528 kvm
->arch
.crypto
.dea_kw
= 0;
529 memset(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
, 0,
530 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
531 VM_EVENT(kvm
, 3, "%s", "DISABLE: DEA keywrapping support");
534 mutex_unlock(&kvm
->lock
);
538 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
539 kvm_s390_vcpu_crypto_setup(vcpu
);
542 mutex_unlock(&kvm
->lock
);
546 static int kvm_s390_set_tod_high(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
550 if (copy_from_user(>od_high
, (void __user
*)attr
->addr
,
556 VM_EVENT(kvm
, 3, "SET: TOD extension: 0x%x", gtod_high
);
561 static int kvm_s390_set_tod_low(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
565 if (copy_from_user(>od
, (void __user
*)attr
->addr
, sizeof(gtod
)))
568 kvm_s390_set_tod_clock(kvm
, gtod
);
569 VM_EVENT(kvm
, 3, "SET: TOD base: 0x%llx", gtod
);
573 static int kvm_s390_set_tod(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
580 switch (attr
->attr
) {
581 case KVM_S390_VM_TOD_HIGH
:
582 ret
= kvm_s390_set_tod_high(kvm
, attr
);
584 case KVM_S390_VM_TOD_LOW
:
585 ret
= kvm_s390_set_tod_low(kvm
, attr
);
594 static int kvm_s390_get_tod_high(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
598 if (copy_to_user((void __user
*)attr
->addr
, >od_high
,
601 VM_EVENT(kvm
, 3, "QUERY: TOD extension: 0x%x", gtod_high
);
606 static int kvm_s390_get_tod_low(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
610 gtod
= kvm_s390_get_tod_clock_fast(kvm
);
611 if (copy_to_user((void __user
*)attr
->addr
, >od
, sizeof(gtod
)))
613 VM_EVENT(kvm
, 3, "QUERY: TOD base: 0x%llx", gtod
);
618 static int kvm_s390_get_tod(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
625 switch (attr
->attr
) {
626 case KVM_S390_VM_TOD_HIGH
:
627 ret
= kvm_s390_get_tod_high(kvm
, attr
);
629 case KVM_S390_VM_TOD_LOW
:
630 ret
= kvm_s390_get_tod_low(kvm
, attr
);
639 static int kvm_s390_set_processor(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
641 struct kvm_s390_vm_cpu_processor
*proc
;
642 u16 lowest_ibc
, unblocked_ibc
;
645 mutex_lock(&kvm
->lock
);
646 if (atomic_read(&kvm
->online_vcpus
)) {
650 proc
= kzalloc(sizeof(*proc
), GFP_KERNEL
);
655 if (!copy_from_user(proc
, (void __user
*)attr
->addr
,
657 kvm
->arch
.model
.cpuid
= proc
->cpuid
;
658 lowest_ibc
= sclp
.ibc
>> 16 & 0xfff;
659 unblocked_ibc
= sclp
.ibc
& 0xfff;
661 if (proc
->ibc
> unblocked_ibc
)
662 kvm
->arch
.model
.ibc
= unblocked_ibc
;
663 else if (proc
->ibc
< lowest_ibc
)
664 kvm
->arch
.model
.ibc
= lowest_ibc
;
666 kvm
->arch
.model
.ibc
= proc
->ibc
;
668 memcpy(kvm
->arch
.model
.fac_list
, proc
->fac_list
,
669 S390_ARCH_FAC_LIST_SIZE_BYTE
);
674 mutex_unlock(&kvm
->lock
);
678 static int kvm_s390_set_cpu_model(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
682 switch (attr
->attr
) {
683 case KVM_S390_VM_CPU_PROCESSOR
:
684 ret
= kvm_s390_set_processor(kvm
, attr
);
690 static int kvm_s390_get_processor(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
692 struct kvm_s390_vm_cpu_processor
*proc
;
695 proc
= kzalloc(sizeof(*proc
), GFP_KERNEL
);
700 proc
->cpuid
= kvm
->arch
.model
.cpuid
;
701 proc
->ibc
= kvm
->arch
.model
.ibc
;
702 memcpy(&proc
->fac_list
, kvm
->arch
.model
.fac_list
,
703 S390_ARCH_FAC_LIST_SIZE_BYTE
);
704 if (copy_to_user((void __user
*)attr
->addr
, proc
, sizeof(*proc
)))
711 static int kvm_s390_get_machine(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
713 struct kvm_s390_vm_cpu_machine
*mach
;
716 mach
= kzalloc(sizeof(*mach
), GFP_KERNEL
);
721 get_cpu_id((struct cpuid
*) &mach
->cpuid
);
722 mach
->ibc
= sclp
.ibc
;
723 memcpy(&mach
->fac_mask
, kvm
->arch
.model
.fac_mask
,
724 S390_ARCH_FAC_LIST_SIZE_BYTE
);
725 memcpy((unsigned long *)&mach
->fac_list
, S390_lowcore
.stfle_fac_list
,
726 S390_ARCH_FAC_LIST_SIZE_BYTE
);
727 if (copy_to_user((void __user
*)attr
->addr
, mach
, sizeof(*mach
)))
734 static int kvm_s390_get_cpu_model(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
738 switch (attr
->attr
) {
739 case KVM_S390_VM_CPU_PROCESSOR
:
740 ret
= kvm_s390_get_processor(kvm
, attr
);
742 case KVM_S390_VM_CPU_MACHINE
:
743 ret
= kvm_s390_get_machine(kvm
, attr
);
749 static int kvm_s390_vm_set_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
753 switch (attr
->group
) {
754 case KVM_S390_VM_MEM_CTRL
:
755 ret
= kvm_s390_set_mem_control(kvm
, attr
);
757 case KVM_S390_VM_TOD
:
758 ret
= kvm_s390_set_tod(kvm
, attr
);
760 case KVM_S390_VM_CPU_MODEL
:
761 ret
= kvm_s390_set_cpu_model(kvm
, attr
);
763 case KVM_S390_VM_CRYPTO
:
764 ret
= kvm_s390_vm_set_crypto(kvm
, attr
);
774 static int kvm_s390_vm_get_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
778 switch (attr
->group
) {
779 case KVM_S390_VM_MEM_CTRL
:
780 ret
= kvm_s390_get_mem_control(kvm
, attr
);
782 case KVM_S390_VM_TOD
:
783 ret
= kvm_s390_get_tod(kvm
, attr
);
785 case KVM_S390_VM_CPU_MODEL
:
786 ret
= kvm_s390_get_cpu_model(kvm
, attr
);
796 static int kvm_s390_vm_has_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
800 switch (attr
->group
) {
801 case KVM_S390_VM_MEM_CTRL
:
802 switch (attr
->attr
) {
803 case KVM_S390_VM_MEM_ENABLE_CMMA
:
804 case KVM_S390_VM_MEM_CLR_CMMA
:
805 case KVM_S390_VM_MEM_LIMIT_SIZE
:
813 case KVM_S390_VM_TOD
:
814 switch (attr
->attr
) {
815 case KVM_S390_VM_TOD_LOW
:
816 case KVM_S390_VM_TOD_HIGH
:
824 case KVM_S390_VM_CPU_MODEL
:
825 switch (attr
->attr
) {
826 case KVM_S390_VM_CPU_PROCESSOR
:
827 case KVM_S390_VM_CPU_MACHINE
:
835 case KVM_S390_VM_CRYPTO
:
836 switch (attr
->attr
) {
837 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW
:
838 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW
:
839 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW
:
840 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW
:
856 static long kvm_s390_get_skeys(struct kvm
*kvm
, struct kvm_s390_skeys
*args
)
860 unsigned long curkey
;
863 if (args
->flags
!= 0)
866 /* Is this guest using storage keys? */
867 if (!mm_use_skey(current
->mm
))
868 return KVM_S390_GET_SKEYS_NONE
;
870 /* Enforce sane limit on memory allocation */
871 if (args
->count
< 1 || args
->count
> KVM_S390_SKEYS_MAX
)
874 keys
= kmalloc_array(args
->count
, sizeof(uint8_t),
875 GFP_KERNEL
| __GFP_NOWARN
);
877 keys
= vmalloc(sizeof(uint8_t) * args
->count
);
881 for (i
= 0; i
< args
->count
; i
++) {
882 hva
= gfn_to_hva(kvm
, args
->start_gfn
+ i
);
883 if (kvm_is_error_hva(hva
)) {
888 curkey
= get_guest_storage_key(current
->mm
, hva
);
889 if (IS_ERR_VALUE(curkey
)) {
896 r
= copy_to_user((uint8_t __user
*)args
->skeydata_addr
, keys
,
897 sizeof(uint8_t) * args
->count
);
905 static long kvm_s390_set_skeys(struct kvm
*kvm
, struct kvm_s390_skeys
*args
)
911 if (args
->flags
!= 0)
914 /* Enforce sane limit on memory allocation */
915 if (args
->count
< 1 || args
->count
> KVM_S390_SKEYS_MAX
)
918 keys
= kmalloc_array(args
->count
, sizeof(uint8_t),
919 GFP_KERNEL
| __GFP_NOWARN
);
921 keys
= vmalloc(sizeof(uint8_t) * args
->count
);
925 r
= copy_from_user(keys
, (uint8_t __user
*)args
->skeydata_addr
,
926 sizeof(uint8_t) * args
->count
);
932 /* Enable storage key handling for the guest */
933 r
= s390_enable_skey();
937 for (i
= 0; i
< args
->count
; i
++) {
938 hva
= gfn_to_hva(kvm
, args
->start_gfn
+ i
);
939 if (kvm_is_error_hva(hva
)) {
944 /* Lowest order bit is reserved */
945 if (keys
[i
] & 0x01) {
950 r
= set_guest_storage_key(current
->mm
, hva
,
951 (unsigned long)keys
[i
], 0);
960 long kvm_arch_vm_ioctl(struct file
*filp
,
961 unsigned int ioctl
, unsigned long arg
)
963 struct kvm
*kvm
= filp
->private_data
;
964 void __user
*argp
= (void __user
*)arg
;
965 struct kvm_device_attr attr
;
969 case KVM_S390_INTERRUPT
: {
970 struct kvm_s390_interrupt s390int
;
973 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
975 r
= kvm_s390_inject_vm(kvm
, &s390int
);
978 case KVM_ENABLE_CAP
: {
979 struct kvm_enable_cap cap
;
981 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
983 r
= kvm_vm_ioctl_enable_cap(kvm
, &cap
);
986 case KVM_CREATE_IRQCHIP
: {
987 struct kvm_irq_routing_entry routing
;
990 if (kvm
->arch
.use_irqchip
) {
991 /* Set up dummy routing. */
992 memset(&routing
, 0, sizeof(routing
));
993 r
= kvm_set_irq_routing(kvm
, &routing
, 0, 0);
997 case KVM_SET_DEVICE_ATTR
: {
999 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
1001 r
= kvm_s390_vm_set_attr(kvm
, &attr
);
1004 case KVM_GET_DEVICE_ATTR
: {
1006 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
1008 r
= kvm_s390_vm_get_attr(kvm
, &attr
);
1011 case KVM_HAS_DEVICE_ATTR
: {
1013 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
1015 r
= kvm_s390_vm_has_attr(kvm
, &attr
);
1018 case KVM_S390_GET_SKEYS
: {
1019 struct kvm_s390_skeys args
;
1022 if (copy_from_user(&args
, argp
,
1023 sizeof(struct kvm_s390_skeys
)))
1025 r
= kvm_s390_get_skeys(kvm
, &args
);
1028 case KVM_S390_SET_SKEYS
: {
1029 struct kvm_s390_skeys args
;
1032 if (copy_from_user(&args
, argp
,
1033 sizeof(struct kvm_s390_skeys
)))
1035 r
= kvm_s390_set_skeys(kvm
, &args
);
1045 static int kvm_s390_query_ap_config(u8
*config
)
1047 u32 fcn_code
= 0x04000000UL
;
1050 memset(config
, 0, 128);
1054 ".long 0xb2af0000\n" /* PQAP(QCI) */
1060 : "r" (fcn_code
), "r" (config
)
1061 : "cc", "0", "2", "memory"
1067 static int kvm_s390_apxa_installed(void)
1072 if (test_facility(12)) {
1073 cc
= kvm_s390_query_ap_config(config
);
1076 pr_err("PQAP(QCI) failed with cc=%d", cc
);
1078 return config
[0] & 0x40;
1084 static void kvm_s390_set_crycb_format(struct kvm
*kvm
)
1086 kvm
->arch
.crypto
.crycbd
= (__u32
)(unsigned long) kvm
->arch
.crypto
.crycb
;
1088 if (kvm_s390_apxa_installed())
1089 kvm
->arch
.crypto
.crycbd
|= CRYCB_FORMAT2
;
1091 kvm
->arch
.crypto
.crycbd
|= CRYCB_FORMAT1
;
1094 static u64
kvm_s390_get_initial_cpuid(void)
1099 cpuid
.version
= 0xff;
1100 return *((u64
*) &cpuid
);
1103 static void kvm_s390_crypto_init(struct kvm
*kvm
)
1105 if (!test_kvm_facility(kvm
, 76))
1108 kvm
->arch
.crypto
.crycb
= &kvm
->arch
.sie_page2
->crycb
;
1109 kvm_s390_set_crycb_format(kvm
);
1111 /* Enable AES/DEA protected key functions by default */
1112 kvm
->arch
.crypto
.aes_kw
= 1;
1113 kvm
->arch
.crypto
.dea_kw
= 1;
1114 get_random_bytes(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
,
1115 sizeof(kvm
->arch
.crypto
.crycb
->aes_wrapping_key_mask
));
1116 get_random_bytes(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
,
1117 sizeof(kvm
->arch
.crypto
.crycb
->dea_wrapping_key_mask
));
1120 static void sca_dispose(struct kvm
*kvm
)
1122 if (kvm
->arch
.use_esca
)
1123 free_pages_exact(kvm
->arch
.sca
, sizeof(struct esca_block
));
1125 free_page((unsigned long)(kvm
->arch
.sca
));
1126 kvm
->arch
.sca
= NULL
;
1129 int kvm_arch_init_vm(struct kvm
*kvm
, unsigned long type
)
1132 char debug_name
[16];
1133 static unsigned long sca_offset
;
1136 #ifdef CONFIG_KVM_S390_UCONTROL
1137 if (type
& ~KVM_VM_S390_UCONTROL
)
1139 if ((type
& KVM_VM_S390_UCONTROL
) && (!capable(CAP_SYS_ADMIN
)))
1146 rc
= s390_enable_sie();
1152 kvm
->arch
.use_esca
= 0; /* start with basic SCA */
1153 rwlock_init(&kvm
->arch
.sca_lock
);
1154 kvm
->arch
.sca
= (struct bsca_block
*) get_zeroed_page(GFP_KERNEL
);
1157 spin_lock(&kvm_lock
);
1159 if (sca_offset
+ sizeof(struct bsca_block
) > PAGE_SIZE
)
1161 kvm
->arch
.sca
= (struct bsca_block
*)
1162 ((char *) kvm
->arch
.sca
+ sca_offset
);
1163 spin_unlock(&kvm_lock
);
1165 sprintf(debug_name
, "kvm-%u", current
->pid
);
1167 kvm
->arch
.dbf
= debug_register(debug_name
, 32, 1, 7 * sizeof(long));
1171 kvm
->arch
.sie_page2
=
1172 (struct sie_page2
*) get_zeroed_page(GFP_KERNEL
| GFP_DMA
);
1173 if (!kvm
->arch
.sie_page2
)
1176 /* Populate the facility mask initially. */
1177 memcpy(kvm
->arch
.model
.fac_mask
, S390_lowcore
.stfle_fac_list
,
1178 S390_ARCH_FAC_LIST_SIZE_BYTE
);
1179 for (i
= 0; i
< S390_ARCH_FAC_LIST_SIZE_U64
; i
++) {
1180 if (i
< kvm_s390_fac_list_mask_size())
1181 kvm
->arch
.model
.fac_mask
[i
] &= kvm_s390_fac_list_mask
[i
];
1183 kvm
->arch
.model
.fac_mask
[i
] = 0UL;
1186 /* Populate the facility list initially. */
1187 kvm
->arch
.model
.fac_list
= kvm
->arch
.sie_page2
->fac_list
;
1188 memcpy(kvm
->arch
.model
.fac_list
, kvm
->arch
.model
.fac_mask
,
1189 S390_ARCH_FAC_LIST_SIZE_BYTE
);
1191 kvm
->arch
.model
.cpuid
= kvm_s390_get_initial_cpuid();
1192 kvm
->arch
.model
.ibc
= sclp
.ibc
& 0x0fff;
1194 kvm_s390_crypto_init(kvm
);
1196 spin_lock_init(&kvm
->arch
.float_int
.lock
);
1197 for (i
= 0; i
< FIRQ_LIST_COUNT
; i
++)
1198 INIT_LIST_HEAD(&kvm
->arch
.float_int
.lists
[i
]);
1199 init_waitqueue_head(&kvm
->arch
.ipte_wq
);
1200 mutex_init(&kvm
->arch
.ipte_mutex
);
1202 debug_register_view(kvm
->arch
.dbf
, &debug_sprintf_view
);
1203 VM_EVENT(kvm
, 3, "vm created with type %lu", type
);
1205 if (type
& KVM_VM_S390_UCONTROL
) {
1206 kvm
->arch
.gmap
= NULL
;
1207 kvm
->arch
.mem_limit
= KVM_S390_NO_MEM_LIMIT
;
1209 if (sclp
.hamax
== U64_MAX
)
1210 kvm
->arch
.mem_limit
= TASK_MAX_SIZE
;
1212 kvm
->arch
.mem_limit
= min_t(unsigned long, TASK_MAX_SIZE
,
1214 kvm
->arch
.gmap
= gmap_alloc(current
->mm
, kvm
->arch
.mem_limit
- 1);
1215 if (!kvm
->arch
.gmap
)
1217 kvm
->arch
.gmap
->private = kvm
;
1218 kvm
->arch
.gmap
->pfault_enabled
= 0;
1221 kvm
->arch
.css_support
= 0;
1222 kvm
->arch
.use_irqchip
= 0;
1223 kvm
->arch
.epoch
= 0;
1225 spin_lock_init(&kvm
->arch
.start_stop_lock
);
1226 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm
, current
->pid
);
1230 free_page((unsigned long)kvm
->arch
.sie_page2
);
1231 debug_unregister(kvm
->arch
.dbf
);
1233 KVM_EVENT(3, "creation of vm failed: %d", rc
);
1237 void kvm_arch_vcpu_destroy(struct kvm_vcpu
*vcpu
)
1239 VCPU_EVENT(vcpu
, 3, "%s", "free cpu");
1240 trace_kvm_s390_destroy_vcpu(vcpu
->vcpu_id
);
1241 kvm_s390_clear_local_irqs(vcpu
);
1242 kvm_clear_async_pf_completion_queue(vcpu
);
1243 if (!kvm_is_ucontrol(vcpu
->kvm
))
1246 if (kvm_is_ucontrol(vcpu
->kvm
))
1247 gmap_free(vcpu
->arch
.gmap
);
1249 if (vcpu
->kvm
->arch
.use_cmma
)
1250 kvm_s390_vcpu_unsetup_cmma(vcpu
);
1251 free_page((unsigned long)(vcpu
->arch
.sie_block
));
1253 kvm_vcpu_uninit(vcpu
);
1254 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1257 static void kvm_free_vcpus(struct kvm
*kvm
)
1260 struct kvm_vcpu
*vcpu
;
1262 kvm_for_each_vcpu(i
, vcpu
, kvm
)
1263 kvm_arch_vcpu_destroy(vcpu
);
1265 mutex_lock(&kvm
->lock
);
1266 for (i
= 0; i
< atomic_read(&kvm
->online_vcpus
); i
++)
1267 kvm
->vcpus
[i
] = NULL
;
1269 atomic_set(&kvm
->online_vcpus
, 0);
1270 mutex_unlock(&kvm
->lock
);
1273 void kvm_arch_destroy_vm(struct kvm
*kvm
)
1275 kvm_free_vcpus(kvm
);
1277 debug_unregister(kvm
->arch
.dbf
);
1278 free_page((unsigned long)kvm
->arch
.sie_page2
);
1279 if (!kvm_is_ucontrol(kvm
))
1280 gmap_free(kvm
->arch
.gmap
);
1281 kvm_s390_destroy_adapters(kvm
);
1282 kvm_s390_clear_float_irqs(kvm
);
1283 KVM_EVENT(3, "vm 0x%pK destroyed", kvm
);
1286 /* Section: vcpu related */
1287 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu
*vcpu
)
1289 vcpu
->arch
.gmap
= gmap_alloc(current
->mm
, -1UL);
1290 if (!vcpu
->arch
.gmap
)
1292 vcpu
->arch
.gmap
->private = vcpu
->kvm
;
1297 static void sca_del_vcpu(struct kvm_vcpu
*vcpu
)
1299 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
1300 if (vcpu
->kvm
->arch
.use_esca
) {
1301 struct esca_block
*sca
= vcpu
->kvm
->arch
.sca
;
1303 clear_bit_inv(vcpu
->vcpu_id
, (unsigned long *) sca
->mcn
);
1304 sca
->cpu
[vcpu
->vcpu_id
].sda
= 0;
1306 struct bsca_block
*sca
= vcpu
->kvm
->arch
.sca
;
1308 clear_bit_inv(vcpu
->vcpu_id
, (unsigned long *) &sca
->mcn
);
1309 sca
->cpu
[vcpu
->vcpu_id
].sda
= 0;
1311 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
1314 static void sca_add_vcpu(struct kvm_vcpu
*vcpu
)
1316 read_lock(&vcpu
->kvm
->arch
.sca_lock
);
1317 if (vcpu
->kvm
->arch
.use_esca
) {
1318 struct esca_block
*sca
= vcpu
->kvm
->arch
.sca
;
1320 sca
->cpu
[vcpu
->vcpu_id
].sda
= (__u64
) vcpu
->arch
.sie_block
;
1321 vcpu
->arch
.sie_block
->scaoh
= (__u32
)(((__u64
)sca
) >> 32);
1322 vcpu
->arch
.sie_block
->scaol
= (__u32
)(__u64
)sca
& ~0x3fU
;
1323 vcpu
->arch
.sie_block
->ecb2
|= 0x04U
;
1324 set_bit_inv(vcpu
->vcpu_id
, (unsigned long *) sca
->mcn
);
1326 struct bsca_block
*sca
= vcpu
->kvm
->arch
.sca
;
1328 sca
->cpu
[vcpu
->vcpu_id
].sda
= (__u64
) vcpu
->arch
.sie_block
;
1329 vcpu
->arch
.sie_block
->scaoh
= (__u32
)(((__u64
)sca
) >> 32);
1330 vcpu
->arch
.sie_block
->scaol
= (__u32
)(__u64
)sca
;
1331 set_bit_inv(vcpu
->vcpu_id
, (unsigned long *) &sca
->mcn
);
1333 read_unlock(&vcpu
->kvm
->arch
.sca_lock
);
1336 /* Basic SCA to Extended SCA data copy routines */
1337 static inline void sca_copy_entry(struct esca_entry
*d
, struct bsca_entry
*s
)
1340 d
->sigp_ctrl
.c
= s
->sigp_ctrl
.c
;
1341 d
->sigp_ctrl
.scn
= s
->sigp_ctrl
.scn
;
1344 static void sca_copy_b_to_e(struct esca_block
*d
, struct bsca_block
*s
)
1348 d
->ipte_control
= s
->ipte_control
;
1350 for (i
= 0; i
< KVM_S390_BSCA_CPU_SLOTS
; i
++)
1351 sca_copy_entry(&d
->cpu
[i
], &s
->cpu
[i
]);
1354 static int sca_switch_to_extended(struct kvm
*kvm
)
1356 struct bsca_block
*old_sca
= kvm
->arch
.sca
;
1357 struct esca_block
*new_sca
;
1358 struct kvm_vcpu
*vcpu
;
1359 unsigned int vcpu_idx
;
1362 new_sca
= alloc_pages_exact(sizeof(*new_sca
), GFP_KERNEL
|__GFP_ZERO
);
1366 scaoh
= (u32
)((u64
)(new_sca
) >> 32);
1367 scaol
= (u32
)(u64
)(new_sca
) & ~0x3fU
;
1369 kvm_s390_vcpu_block_all(kvm
);
1370 write_lock(&kvm
->arch
.sca_lock
);
1372 sca_copy_b_to_e(new_sca
, old_sca
);
1374 kvm_for_each_vcpu(vcpu_idx
, vcpu
, kvm
) {
1375 vcpu
->arch
.sie_block
->scaoh
= scaoh
;
1376 vcpu
->arch
.sie_block
->scaol
= scaol
;
1377 vcpu
->arch
.sie_block
->ecb2
|= 0x04U
;
1379 kvm
->arch
.sca
= new_sca
;
1380 kvm
->arch
.use_esca
= 1;
1382 write_unlock(&kvm
->arch
.sca_lock
);
1383 kvm_s390_vcpu_unblock_all(kvm
);
1385 free_page((unsigned long)old_sca
);
1387 VM_EVENT(kvm
, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1388 old_sca
, kvm
->arch
.sca
);
1392 static int sca_can_add_vcpu(struct kvm
*kvm
, unsigned int id
)
1396 if (id
< KVM_S390_BSCA_CPU_SLOTS
)
1401 mutex_lock(&kvm
->lock
);
1402 rc
= kvm
->arch
.use_esca
? 0 : sca_switch_to_extended(kvm
);
1403 mutex_unlock(&kvm
->lock
);
1405 return rc
== 0 && id
< KVM_S390_ESCA_CPU_SLOTS
;
1408 int kvm_arch_vcpu_init(struct kvm_vcpu
*vcpu
)
1410 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
1411 kvm_clear_async_pf_completion_queue(vcpu
);
1412 vcpu
->run
->kvm_valid_regs
= KVM_SYNC_PREFIX
|
1418 if (test_kvm_facility(vcpu
->kvm
, 64))
1419 vcpu
->run
->kvm_valid_regs
|= KVM_SYNC_RICCB
;
1420 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1421 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1424 vcpu
->run
->kvm_valid_regs
|= KVM_SYNC_VRS
;
1426 vcpu
->run
->kvm_valid_regs
|= KVM_SYNC_FPRS
;
1428 if (kvm_is_ucontrol(vcpu
->kvm
))
1429 return __kvm_ucontrol_vcpu_init(vcpu
);
1434 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1435 static void __start_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1437 WARN_ON_ONCE(vcpu
->arch
.cputm_start
!= 0);
1438 raw_write_seqcount_begin(&vcpu
->arch
.cputm_seqcount
);
1439 vcpu
->arch
.cputm_start
= get_tod_clock_fast();
1440 raw_write_seqcount_end(&vcpu
->arch
.cputm_seqcount
);
1443 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1444 static void __stop_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1446 WARN_ON_ONCE(vcpu
->arch
.cputm_start
== 0);
1447 raw_write_seqcount_begin(&vcpu
->arch
.cputm_seqcount
);
1448 vcpu
->arch
.sie_block
->cputm
-= get_tod_clock_fast() - vcpu
->arch
.cputm_start
;
1449 vcpu
->arch
.cputm_start
= 0;
1450 raw_write_seqcount_end(&vcpu
->arch
.cputm_seqcount
);
1453 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1454 static void __enable_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1456 WARN_ON_ONCE(vcpu
->arch
.cputm_enabled
);
1457 vcpu
->arch
.cputm_enabled
= true;
1458 __start_cpu_timer_accounting(vcpu
);
1461 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1462 static void __disable_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1464 WARN_ON_ONCE(!vcpu
->arch
.cputm_enabled
);
1465 __stop_cpu_timer_accounting(vcpu
);
1466 vcpu
->arch
.cputm_enabled
= false;
1469 static void enable_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1471 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1472 __enable_cpu_timer_accounting(vcpu
);
1476 static void disable_cpu_timer_accounting(struct kvm_vcpu
*vcpu
)
1478 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1479 __disable_cpu_timer_accounting(vcpu
);
1483 /* set the cpu timer - may only be called from the VCPU thread itself */
1484 void kvm_s390_set_cpu_timer(struct kvm_vcpu
*vcpu
, __u64 cputm
)
1486 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1487 raw_write_seqcount_begin(&vcpu
->arch
.cputm_seqcount
);
1488 if (vcpu
->arch
.cputm_enabled
)
1489 vcpu
->arch
.cputm_start
= get_tod_clock_fast();
1490 vcpu
->arch
.sie_block
->cputm
= cputm
;
1491 raw_write_seqcount_end(&vcpu
->arch
.cputm_seqcount
);
1495 /* update and get the cpu timer - can also be called from other VCPU threads */
1496 __u64
kvm_s390_get_cpu_timer(struct kvm_vcpu
*vcpu
)
1501 if (unlikely(!vcpu
->arch
.cputm_enabled
))
1502 return vcpu
->arch
.sie_block
->cputm
;
1504 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1506 seq
= raw_read_seqcount(&vcpu
->arch
.cputm_seqcount
);
1508 * If the writer would ever execute a read in the critical
1509 * section, e.g. in irq context, we have a deadlock.
1511 WARN_ON_ONCE((seq
& 1) && smp_processor_id() == vcpu
->cpu
);
1512 value
= vcpu
->arch
.sie_block
->cputm
;
1513 /* if cputm_start is 0, accounting is being started/stopped */
1514 if (likely(vcpu
->arch
.cputm_start
))
1515 value
-= get_tod_clock_fast() - vcpu
->arch
.cputm_start
;
1516 } while (read_seqcount_retry(&vcpu
->arch
.cputm_seqcount
, seq
& ~1));
1521 void kvm_arch_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
1523 /* Save host register state */
1525 vcpu
->arch
.host_fpregs
.fpc
= current
->thread
.fpu
.fpc
;
1526 vcpu
->arch
.host_fpregs
.regs
= current
->thread
.fpu
.regs
;
1529 current
->thread
.fpu
.regs
= vcpu
->run
->s
.regs
.vrs
;
1531 current
->thread
.fpu
.regs
= vcpu
->run
->s
.regs
.fprs
;
1532 current
->thread
.fpu
.fpc
= vcpu
->run
->s
.regs
.fpc
;
1533 if (test_fp_ctl(current
->thread
.fpu
.fpc
))
1534 /* User space provided an invalid FPC, let's clear it */
1535 current
->thread
.fpu
.fpc
= 0;
1537 save_access_regs(vcpu
->arch
.host_acrs
);
1538 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
1539 gmap_enable(vcpu
->arch
.gmap
);
1540 atomic_or(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
1541 if (vcpu
->arch
.cputm_enabled
&& !is_vcpu_idle(vcpu
))
1542 __start_cpu_timer_accounting(vcpu
);
1546 void kvm_arch_vcpu_put(struct kvm_vcpu
*vcpu
)
1549 if (vcpu
->arch
.cputm_enabled
&& !is_vcpu_idle(vcpu
))
1550 __stop_cpu_timer_accounting(vcpu
);
1551 atomic_andnot(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
1552 gmap_disable(vcpu
->arch
.gmap
);
1554 /* Save guest register state */
1556 vcpu
->run
->s
.regs
.fpc
= current
->thread
.fpu
.fpc
;
1558 /* Restore host register state */
1559 current
->thread
.fpu
.fpc
= vcpu
->arch
.host_fpregs
.fpc
;
1560 current
->thread
.fpu
.regs
= vcpu
->arch
.host_fpregs
.regs
;
1562 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
1563 restore_access_regs(vcpu
->arch
.host_acrs
);
1566 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu
*vcpu
)
1568 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1569 vcpu
->arch
.sie_block
->gpsw
.mask
= 0UL;
1570 vcpu
->arch
.sie_block
->gpsw
.addr
= 0UL;
1571 kvm_s390_set_prefix(vcpu
, 0);
1572 kvm_s390_set_cpu_timer(vcpu
, 0);
1573 vcpu
->arch
.sie_block
->ckc
= 0UL;
1574 vcpu
->arch
.sie_block
->todpr
= 0;
1575 memset(vcpu
->arch
.sie_block
->gcr
, 0, 16 * sizeof(__u64
));
1576 vcpu
->arch
.sie_block
->gcr
[0] = 0xE0UL
;
1577 vcpu
->arch
.sie_block
->gcr
[14] = 0xC2000000UL
;
1578 /* make sure the new fpc will be lazily loaded */
1580 current
->thread
.fpu
.fpc
= 0;
1581 vcpu
->arch
.sie_block
->gbea
= 1;
1582 vcpu
->arch
.sie_block
->pp
= 0;
1583 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
1584 kvm_clear_async_pf_completion_queue(vcpu
);
1585 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
))
1586 kvm_s390_vcpu_stop(vcpu
);
1587 kvm_s390_clear_local_irqs(vcpu
);
1590 void kvm_arch_vcpu_postcreate(struct kvm_vcpu
*vcpu
)
1592 mutex_lock(&vcpu
->kvm
->lock
);
1594 vcpu
->arch
.sie_block
->epoch
= vcpu
->kvm
->arch
.epoch
;
1596 mutex_unlock(&vcpu
->kvm
->lock
);
1597 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1598 vcpu
->arch
.gmap
= vcpu
->kvm
->arch
.gmap
;
1604 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu
*vcpu
)
1606 if (!test_kvm_facility(vcpu
->kvm
, 76))
1609 vcpu
->arch
.sie_block
->ecb3
&= ~(ECB3_AES
| ECB3_DEA
);
1611 if (vcpu
->kvm
->arch
.crypto
.aes_kw
)
1612 vcpu
->arch
.sie_block
->ecb3
|= ECB3_AES
;
1613 if (vcpu
->kvm
->arch
.crypto
.dea_kw
)
1614 vcpu
->arch
.sie_block
->ecb3
|= ECB3_DEA
;
1616 vcpu
->arch
.sie_block
->crycbd
= vcpu
->kvm
->arch
.crypto
.crycbd
;
1619 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu
*vcpu
)
1621 free_page(vcpu
->arch
.sie_block
->cbrlo
);
1622 vcpu
->arch
.sie_block
->cbrlo
= 0;
1625 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu
*vcpu
)
1627 vcpu
->arch
.sie_block
->cbrlo
= get_zeroed_page(GFP_KERNEL
);
1628 if (!vcpu
->arch
.sie_block
->cbrlo
)
1631 vcpu
->arch
.sie_block
->ecb2
|= 0x80;
1632 vcpu
->arch
.sie_block
->ecb2
&= ~0x08;
1636 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu
*vcpu
)
1638 struct kvm_s390_cpu_model
*model
= &vcpu
->kvm
->arch
.model
;
1640 vcpu
->arch
.sie_block
->ibc
= model
->ibc
;
1641 if (test_kvm_facility(vcpu
->kvm
, 7))
1642 vcpu
->arch
.sie_block
->fac
= (u32
)(u64
) model
->fac_list
;
1645 int kvm_arch_vcpu_setup(struct kvm_vcpu
*vcpu
)
1649 atomic_set(&vcpu
->arch
.sie_block
->cpuflags
, CPUSTAT_ZARCH
|
1653 if (test_kvm_facility(vcpu
->kvm
, 78))
1654 atomic_or(CPUSTAT_GED2
, &vcpu
->arch
.sie_block
->cpuflags
);
1655 else if (test_kvm_facility(vcpu
->kvm
, 8))
1656 atomic_or(CPUSTAT_GED
, &vcpu
->arch
.sie_block
->cpuflags
);
1658 kvm_s390_vcpu_setup_model(vcpu
);
1660 vcpu
->arch
.sie_block
->ecb
= 0x02;
1661 if (test_kvm_facility(vcpu
->kvm
, 9))
1662 vcpu
->arch
.sie_block
->ecb
|= 0x04;
1663 if (test_kvm_facility(vcpu
->kvm
, 50) && test_kvm_facility(vcpu
->kvm
, 73))
1664 vcpu
->arch
.sie_block
->ecb
|= 0x10;
1666 if (test_kvm_facility(vcpu
->kvm
, 8))
1667 vcpu
->arch
.sie_block
->ecb2
|= 0x08;
1668 vcpu
->arch
.sie_block
->eca
= 0xC1002000U
;
1670 vcpu
->arch
.sie_block
->eca
|= 1;
1671 if (sclp
.has_sigpif
)
1672 vcpu
->arch
.sie_block
->eca
|= 0x10000000U
;
1673 if (test_kvm_facility(vcpu
->kvm
, 64))
1674 vcpu
->arch
.sie_block
->ecb3
|= 0x01;
1675 if (test_kvm_facility(vcpu
->kvm
, 129)) {
1676 vcpu
->arch
.sie_block
->eca
|= 0x00020000;
1677 vcpu
->arch
.sie_block
->ecd
|= 0x20000000;
1679 vcpu
->arch
.sie_block
->riccbd
= (unsigned long) &vcpu
->run
->s
.regs
.riccb
;
1680 vcpu
->arch
.sie_block
->ictl
|= ICTL_ISKE
| ICTL_SSKE
| ICTL_RRBE
;
1682 if (vcpu
->kvm
->arch
.use_cmma
) {
1683 rc
= kvm_s390_vcpu_setup_cmma(vcpu
);
1687 hrtimer_init(&vcpu
->arch
.ckc_timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
1688 vcpu
->arch
.ckc_timer
.function
= kvm_s390_idle_wakeup
;
1690 kvm_s390_vcpu_crypto_setup(vcpu
);
1695 struct kvm_vcpu
*kvm_arch_vcpu_create(struct kvm
*kvm
,
1698 struct kvm_vcpu
*vcpu
;
1699 struct sie_page
*sie_page
;
1702 if (!kvm_is_ucontrol(kvm
) && !sca_can_add_vcpu(kvm
, id
))
1707 vcpu
= kmem_cache_zalloc(kvm_vcpu_cache
, GFP_KERNEL
);
1711 sie_page
= (struct sie_page
*) get_zeroed_page(GFP_KERNEL
);
1715 vcpu
->arch
.sie_block
= &sie_page
->sie_block
;
1716 vcpu
->arch
.sie_block
->itdba
= (unsigned long) &sie_page
->itdb
;
1718 vcpu
->arch
.sie_block
->icpua
= id
;
1719 spin_lock_init(&vcpu
->arch
.local_int
.lock
);
1720 vcpu
->arch
.local_int
.float_int
= &kvm
->arch
.float_int
;
1721 vcpu
->arch
.local_int
.wq
= &vcpu
->wq
;
1722 vcpu
->arch
.local_int
.cpuflags
= &vcpu
->arch
.sie_block
->cpuflags
;
1723 seqcount_init(&vcpu
->arch
.cputm_seqcount
);
1725 rc
= kvm_vcpu_init(vcpu
, kvm
, id
);
1727 goto out_free_sie_block
;
1728 VM_EVENT(kvm
, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id
, vcpu
,
1729 vcpu
->arch
.sie_block
);
1730 trace_kvm_s390_create_vcpu(id
, vcpu
, vcpu
->arch
.sie_block
);
1734 free_page((unsigned long)(vcpu
->arch
.sie_block
));
1736 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
1741 int kvm_arch_vcpu_runnable(struct kvm_vcpu
*vcpu
)
1743 return kvm_s390_vcpu_has_irq(vcpu
, 0);
1746 void kvm_s390_vcpu_block(struct kvm_vcpu
*vcpu
)
1748 atomic_or(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
1752 void kvm_s390_vcpu_unblock(struct kvm_vcpu
*vcpu
)
1754 atomic_andnot(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
1757 static void kvm_s390_vcpu_request(struct kvm_vcpu
*vcpu
)
1759 atomic_or(PROG_REQUEST
, &vcpu
->arch
.sie_block
->prog20
);
1763 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu
*vcpu
)
1765 atomic_andnot(PROG_REQUEST
, &vcpu
->arch
.sie_block
->prog20
);
1769 * Kick a guest cpu out of SIE and wait until SIE is not running.
1770 * If the CPU is not running (e.g. waiting as idle) the function will
1771 * return immediately. */
1772 void exit_sie(struct kvm_vcpu
*vcpu
)
1774 atomic_or(CPUSTAT_STOP_INT
, &vcpu
->arch
.sie_block
->cpuflags
);
1775 while (vcpu
->arch
.sie_block
->prog0c
& PROG_IN_SIE
)
1779 /* Kick a guest cpu out of SIE to process a request synchronously */
1780 void kvm_s390_sync_request(int req
, struct kvm_vcpu
*vcpu
)
1782 kvm_make_request(req
, vcpu
);
1783 kvm_s390_vcpu_request(vcpu
);
1786 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
)
1789 struct kvm
*kvm
= gmap
->private;
1790 struct kvm_vcpu
*vcpu
;
1792 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
1793 /* match against both prefix pages */
1794 if (kvm_s390_get_prefix(vcpu
) == (address
& ~0x1000UL
)) {
1795 VCPU_EVENT(vcpu
, 2, "gmap notifier for %lx", address
);
1796 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD
, vcpu
);
1801 int kvm_arch_vcpu_should_kick(struct kvm_vcpu
*vcpu
)
1803 /* kvm common code refers to this, but never calls it */
1808 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu
*vcpu
,
1809 struct kvm_one_reg
*reg
)
1814 case KVM_REG_S390_TODPR
:
1815 r
= put_user(vcpu
->arch
.sie_block
->todpr
,
1816 (u32 __user
*)reg
->addr
);
1818 case KVM_REG_S390_EPOCHDIFF
:
1819 r
= put_user(vcpu
->arch
.sie_block
->epoch
,
1820 (u64 __user
*)reg
->addr
);
1822 case KVM_REG_S390_CPU_TIMER
:
1823 r
= put_user(kvm_s390_get_cpu_timer(vcpu
),
1824 (u64 __user
*)reg
->addr
);
1826 case KVM_REG_S390_CLOCK_COMP
:
1827 r
= put_user(vcpu
->arch
.sie_block
->ckc
,
1828 (u64 __user
*)reg
->addr
);
1830 case KVM_REG_S390_PFTOKEN
:
1831 r
= put_user(vcpu
->arch
.pfault_token
,
1832 (u64 __user
*)reg
->addr
);
1834 case KVM_REG_S390_PFCOMPARE
:
1835 r
= put_user(vcpu
->arch
.pfault_compare
,
1836 (u64 __user
*)reg
->addr
);
1838 case KVM_REG_S390_PFSELECT
:
1839 r
= put_user(vcpu
->arch
.pfault_select
,
1840 (u64 __user
*)reg
->addr
);
1842 case KVM_REG_S390_PP
:
1843 r
= put_user(vcpu
->arch
.sie_block
->pp
,
1844 (u64 __user
*)reg
->addr
);
1846 case KVM_REG_S390_GBEA
:
1847 r
= put_user(vcpu
->arch
.sie_block
->gbea
,
1848 (u64 __user
*)reg
->addr
);
1857 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu
*vcpu
,
1858 struct kvm_one_reg
*reg
)
1864 case KVM_REG_S390_TODPR
:
1865 r
= get_user(vcpu
->arch
.sie_block
->todpr
,
1866 (u32 __user
*)reg
->addr
);
1868 case KVM_REG_S390_EPOCHDIFF
:
1869 r
= get_user(vcpu
->arch
.sie_block
->epoch
,
1870 (u64 __user
*)reg
->addr
);
1872 case KVM_REG_S390_CPU_TIMER
:
1873 r
= get_user(val
, (u64 __user
*)reg
->addr
);
1875 kvm_s390_set_cpu_timer(vcpu
, val
);
1877 case KVM_REG_S390_CLOCK_COMP
:
1878 r
= get_user(vcpu
->arch
.sie_block
->ckc
,
1879 (u64 __user
*)reg
->addr
);
1881 case KVM_REG_S390_PFTOKEN
:
1882 r
= get_user(vcpu
->arch
.pfault_token
,
1883 (u64 __user
*)reg
->addr
);
1884 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
1885 kvm_clear_async_pf_completion_queue(vcpu
);
1887 case KVM_REG_S390_PFCOMPARE
:
1888 r
= get_user(vcpu
->arch
.pfault_compare
,
1889 (u64 __user
*)reg
->addr
);
1891 case KVM_REG_S390_PFSELECT
:
1892 r
= get_user(vcpu
->arch
.pfault_select
,
1893 (u64 __user
*)reg
->addr
);
1895 case KVM_REG_S390_PP
:
1896 r
= get_user(vcpu
->arch
.sie_block
->pp
,
1897 (u64 __user
*)reg
->addr
);
1899 case KVM_REG_S390_GBEA
:
1900 r
= get_user(vcpu
->arch
.sie_block
->gbea
,
1901 (u64 __user
*)reg
->addr
);
1910 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu
*vcpu
)
1912 kvm_s390_vcpu_initial_reset(vcpu
);
1916 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
1918 memcpy(&vcpu
->run
->s
.regs
.gprs
, ®s
->gprs
, sizeof(regs
->gprs
));
1922 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
1924 memcpy(®s
->gprs
, &vcpu
->run
->s
.regs
.gprs
, sizeof(regs
->gprs
));
1928 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu
*vcpu
,
1929 struct kvm_sregs
*sregs
)
1931 memcpy(&vcpu
->run
->s
.regs
.acrs
, &sregs
->acrs
, sizeof(sregs
->acrs
));
1932 memcpy(&vcpu
->arch
.sie_block
->gcr
, &sregs
->crs
, sizeof(sregs
->crs
));
1933 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
1937 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu
*vcpu
,
1938 struct kvm_sregs
*sregs
)
1940 memcpy(&sregs
->acrs
, &vcpu
->run
->s
.regs
.acrs
, sizeof(sregs
->acrs
));
1941 memcpy(&sregs
->crs
, &vcpu
->arch
.sie_block
->gcr
, sizeof(sregs
->crs
));
1945 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
1947 /* make sure the new values will be lazily loaded */
1949 if (test_fp_ctl(fpu
->fpc
))
1951 current
->thread
.fpu
.fpc
= fpu
->fpc
;
1953 convert_fp_to_vx(current
->thread
.fpu
.vxrs
, (freg_t
*)fpu
->fprs
);
1955 memcpy(current
->thread
.fpu
.fprs
, &fpu
->fprs
, sizeof(fpu
->fprs
));
1959 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
1961 /* make sure we have the latest values */
1964 convert_vx_to_fp((freg_t
*)fpu
->fprs
, current
->thread
.fpu
.vxrs
);
1966 memcpy(fpu
->fprs
, current
->thread
.fpu
.fprs
, sizeof(fpu
->fprs
));
1967 fpu
->fpc
= current
->thread
.fpu
.fpc
;
1971 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu
*vcpu
, psw_t psw
)
1975 if (!is_vcpu_stopped(vcpu
))
1978 vcpu
->run
->psw_mask
= psw
.mask
;
1979 vcpu
->run
->psw_addr
= psw
.addr
;
1984 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
1985 struct kvm_translation
*tr
)
1987 return -EINVAL
; /* not implemented yet */
1990 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1991 KVM_GUESTDBG_USE_HW_BP | \
1992 KVM_GUESTDBG_ENABLE)
1994 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu
*vcpu
,
1995 struct kvm_guest_debug
*dbg
)
1999 vcpu
->guest_debug
= 0;
2000 kvm_s390_clear_bp_data(vcpu
);
2002 if (dbg
->control
& ~VALID_GUESTDBG_FLAGS
)
2005 if (dbg
->control
& KVM_GUESTDBG_ENABLE
) {
2006 vcpu
->guest_debug
= dbg
->control
;
2007 /* enforce guest PER */
2008 atomic_or(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
2010 if (dbg
->control
& KVM_GUESTDBG_USE_HW_BP
)
2011 rc
= kvm_s390_import_bp_data(vcpu
, dbg
);
2013 atomic_andnot(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
2014 vcpu
->arch
.guestdbg
.last_bp
= 0;
2018 vcpu
->guest_debug
= 0;
2019 kvm_s390_clear_bp_data(vcpu
);
2020 atomic_andnot(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
2026 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu
*vcpu
,
2027 struct kvm_mp_state
*mp_state
)
2029 /* CHECK_STOP and LOAD are not supported yet */
2030 return is_vcpu_stopped(vcpu
) ? KVM_MP_STATE_STOPPED
:
2031 KVM_MP_STATE_OPERATING
;
2034 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu
*vcpu
,
2035 struct kvm_mp_state
*mp_state
)
2039 /* user space knows about this interface - let it control the state */
2040 vcpu
->kvm
->arch
.user_cpu_state_ctrl
= 1;
2042 switch (mp_state
->mp_state
) {
2043 case KVM_MP_STATE_STOPPED
:
2044 kvm_s390_vcpu_stop(vcpu
);
2046 case KVM_MP_STATE_OPERATING
:
2047 kvm_s390_vcpu_start(vcpu
);
2049 case KVM_MP_STATE_LOAD
:
2050 case KVM_MP_STATE_CHECK_STOP
:
2051 /* fall through - CHECK_STOP and LOAD are not supported yet */
2059 static bool ibs_enabled(struct kvm_vcpu
*vcpu
)
2061 return atomic_read(&vcpu
->arch
.sie_block
->cpuflags
) & CPUSTAT_IBS
;
2064 static int kvm_s390_handle_requests(struct kvm_vcpu
*vcpu
)
2067 kvm_s390_vcpu_request_handled(vcpu
);
2068 if (!vcpu
->requests
)
2071 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2072 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2073 * This ensures that the ipte instruction for this request has
2074 * already finished. We might race against a second unmapper that
2075 * wants to set the blocking bit. Lets just retry the request loop.
2077 if (kvm_check_request(KVM_REQ_MMU_RELOAD
, vcpu
)) {
2079 rc
= gmap_ipte_notify(vcpu
->arch
.gmap
,
2080 kvm_s390_get_prefix(vcpu
),
2087 if (kvm_check_request(KVM_REQ_TLB_FLUSH
, vcpu
)) {
2088 vcpu
->arch
.sie_block
->ihcpu
= 0xffff;
2092 if (kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
)) {
2093 if (!ibs_enabled(vcpu
)) {
2094 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 1);
2095 atomic_or(CPUSTAT_IBS
,
2096 &vcpu
->arch
.sie_block
->cpuflags
);
2101 if (kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
)) {
2102 if (ibs_enabled(vcpu
)) {
2103 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 0);
2104 atomic_andnot(CPUSTAT_IBS
,
2105 &vcpu
->arch
.sie_block
->cpuflags
);
2110 /* nothing to do, just clear the request */
2111 clear_bit(KVM_REQ_UNHALT
, &vcpu
->requests
);
2116 void kvm_s390_set_tod_clock(struct kvm
*kvm
, u64 tod
)
2118 struct kvm_vcpu
*vcpu
;
2121 mutex_lock(&kvm
->lock
);
2123 kvm
->arch
.epoch
= tod
- get_tod_clock();
2124 kvm_s390_vcpu_block_all(kvm
);
2125 kvm_for_each_vcpu(i
, vcpu
, kvm
)
2126 vcpu
->arch
.sie_block
->epoch
= kvm
->arch
.epoch
;
2127 kvm_s390_vcpu_unblock_all(kvm
);
2129 mutex_unlock(&kvm
->lock
);
2133 * kvm_arch_fault_in_page - fault-in guest page if necessary
2134 * @vcpu: The corresponding virtual cpu
2135 * @gpa: Guest physical address
2136 * @writable: Whether the page should be writable or not
2138 * Make sure that a guest page has been faulted-in on the host.
2140 * Return: Zero on success, negative error code otherwise.
2142 long kvm_arch_fault_in_page(struct kvm_vcpu
*vcpu
, gpa_t gpa
, int writable
)
2144 return gmap_fault(vcpu
->arch
.gmap
, gpa
,
2145 writable
? FAULT_FLAG_WRITE
: 0);
2148 static void __kvm_inject_pfault_token(struct kvm_vcpu
*vcpu
, bool start_token
,
2149 unsigned long token
)
2151 struct kvm_s390_interrupt inti
;
2152 struct kvm_s390_irq irq
;
2155 irq
.u
.ext
.ext_params2
= token
;
2156 irq
.type
= KVM_S390_INT_PFAULT_INIT
;
2157 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu
, &irq
));
2159 inti
.type
= KVM_S390_INT_PFAULT_DONE
;
2160 inti
.parm64
= token
;
2161 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu
->kvm
, &inti
));
2165 void kvm_arch_async_page_not_present(struct kvm_vcpu
*vcpu
,
2166 struct kvm_async_pf
*work
)
2168 trace_kvm_s390_pfault_init(vcpu
, work
->arch
.pfault_token
);
2169 __kvm_inject_pfault_token(vcpu
, true, work
->arch
.pfault_token
);
2172 void kvm_arch_async_page_present(struct kvm_vcpu
*vcpu
,
2173 struct kvm_async_pf
*work
)
2175 trace_kvm_s390_pfault_done(vcpu
, work
->arch
.pfault_token
);
2176 __kvm_inject_pfault_token(vcpu
, false, work
->arch
.pfault_token
);
2179 void kvm_arch_async_page_ready(struct kvm_vcpu
*vcpu
,
2180 struct kvm_async_pf
*work
)
2182 /* s390 will always inject the page directly */
2185 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu
*vcpu
)
2188 * s390 will always inject the page directly,
2189 * but we still want check_async_completion to cleanup
2194 static int kvm_arch_setup_async_pf(struct kvm_vcpu
*vcpu
)
2197 struct kvm_arch_async_pf arch
;
2200 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
2202 if ((vcpu
->arch
.sie_block
->gpsw
.mask
& vcpu
->arch
.pfault_select
) !=
2203 vcpu
->arch
.pfault_compare
)
2205 if (psw_extint_disabled(vcpu
))
2207 if (kvm_s390_vcpu_has_irq(vcpu
, 0))
2209 if (!(vcpu
->arch
.sie_block
->gcr
[0] & 0x200ul
))
2211 if (!vcpu
->arch
.gmap
->pfault_enabled
)
2214 hva
= gfn_to_hva(vcpu
->kvm
, gpa_to_gfn(current
->thread
.gmap_addr
));
2215 hva
+= current
->thread
.gmap_addr
& ~PAGE_MASK
;
2216 if (read_guest_real(vcpu
, vcpu
->arch
.pfault_token
, &arch
.pfault_token
, 8))
2219 rc
= kvm_setup_async_pf(vcpu
, current
->thread
.gmap_addr
, hva
, &arch
);
2223 static int vcpu_pre_run(struct kvm_vcpu
*vcpu
)
2228 * On s390 notifications for arriving pages will be delivered directly
2229 * to the guest but the house keeping for completed pfaults is
2230 * handled outside the worker.
2232 kvm_check_async_pf_completion(vcpu
);
2234 vcpu
->arch
.sie_block
->gg14
= vcpu
->run
->s
.regs
.gprs
[14];
2235 vcpu
->arch
.sie_block
->gg15
= vcpu
->run
->s
.regs
.gprs
[15];
2240 if (test_cpu_flag(CIF_MCCK_PENDING
))
2243 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
2244 rc
= kvm_s390_deliver_pending_interrupts(vcpu
);
2249 rc
= kvm_s390_handle_requests(vcpu
);
2253 if (guestdbg_enabled(vcpu
)) {
2254 kvm_s390_backup_guest_per_regs(vcpu
);
2255 kvm_s390_patch_guest_per_regs(vcpu
);
2258 vcpu
->arch
.sie_block
->icptcode
= 0;
2259 cpuflags
= atomic_read(&vcpu
->arch
.sie_block
->cpuflags
);
2260 VCPU_EVENT(vcpu
, 6, "entering sie flags %x", cpuflags
);
2261 trace_kvm_s390_sie_enter(vcpu
, cpuflags
);
2266 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu
*vcpu
)
2268 struct kvm_s390_pgm_info pgm_info
= {
2269 .code
= PGM_ADDRESSING
,
2274 VCPU_EVENT(vcpu
, 3, "%s", "fault in sie instruction");
2275 trace_kvm_s390_sie_fault(vcpu
);
2278 * We want to inject an addressing exception, which is defined as a
2279 * suppressing or terminating exception. However, since we came here
2280 * by a DAT access exception, the PSW still points to the faulting
2281 * instruction since DAT exceptions are nullifying. So we've got
2282 * to look up the current opcode to get the length of the instruction
2283 * to be able to forward the PSW.
2285 rc
= read_guest_instr(vcpu
, &opcode
, 1);
2286 ilen
= insn_length(opcode
);
2290 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2291 * Forward by arbitrary ilc, injection will take care of
2292 * nullification if necessary.
2294 pgm_info
= vcpu
->arch
.pgm
;
2297 pgm_info
.flags
= ilen
| KVM_S390_PGM_FLAGS_ILC_VALID
;
2298 kvm_s390_forward_psw(vcpu
, ilen
);
2299 return kvm_s390_inject_prog_irq(vcpu
, &pgm_info
);
2302 static int vcpu_post_run(struct kvm_vcpu
*vcpu
, int exit_reason
)
2304 VCPU_EVENT(vcpu
, 6, "exit sie icptcode %d",
2305 vcpu
->arch
.sie_block
->icptcode
);
2306 trace_kvm_s390_sie_exit(vcpu
, vcpu
->arch
.sie_block
->icptcode
);
2308 if (guestdbg_enabled(vcpu
))
2309 kvm_s390_restore_guest_per_regs(vcpu
);
2311 vcpu
->run
->s
.regs
.gprs
[14] = vcpu
->arch
.sie_block
->gg14
;
2312 vcpu
->run
->s
.regs
.gprs
[15] = vcpu
->arch
.sie_block
->gg15
;
2314 if (vcpu
->arch
.sie_block
->icptcode
> 0) {
2315 int rc
= kvm_handle_sie_intercept(vcpu
);
2317 if (rc
!= -EOPNOTSUPP
)
2319 vcpu
->run
->exit_reason
= KVM_EXIT_S390_SIEIC
;
2320 vcpu
->run
->s390_sieic
.icptcode
= vcpu
->arch
.sie_block
->icptcode
;
2321 vcpu
->run
->s390_sieic
.ipa
= vcpu
->arch
.sie_block
->ipa
;
2322 vcpu
->run
->s390_sieic
.ipb
= vcpu
->arch
.sie_block
->ipb
;
2324 } else if (exit_reason
!= -EFAULT
) {
2325 vcpu
->stat
.exit_null
++;
2327 } else if (kvm_is_ucontrol(vcpu
->kvm
)) {
2328 vcpu
->run
->exit_reason
= KVM_EXIT_S390_UCONTROL
;
2329 vcpu
->run
->s390_ucontrol
.trans_exc_code
=
2330 current
->thread
.gmap_addr
;
2331 vcpu
->run
->s390_ucontrol
.pgm_code
= 0x10;
2333 } else if (current
->thread
.gmap_pfault
) {
2334 trace_kvm_s390_major_guest_pfault(vcpu
);
2335 current
->thread
.gmap_pfault
= 0;
2336 if (kvm_arch_setup_async_pf(vcpu
))
2338 return kvm_arch_fault_in_page(vcpu
, current
->thread
.gmap_addr
, 1);
2340 return vcpu_post_run_fault_in_sie(vcpu
);
2343 static int __vcpu_run(struct kvm_vcpu
*vcpu
)
2345 int rc
, exit_reason
;
2348 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2349 * ning the guest), so that memslots (and other stuff) are protected
2351 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2354 rc
= vcpu_pre_run(vcpu
);
2358 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
2360 * As PF_VCPU will be used in fault handler, between
2361 * guest_enter and guest_exit should be no uaccess.
2363 local_irq_disable();
2364 __kvm_guest_enter();
2365 __disable_cpu_timer_accounting(vcpu
);
2367 exit_reason
= sie64a(vcpu
->arch
.sie_block
,
2368 vcpu
->run
->s
.regs
.gprs
);
2369 local_irq_disable();
2370 __enable_cpu_timer_accounting(vcpu
);
2373 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2375 rc
= vcpu_post_run(vcpu
, exit_reason
);
2376 } while (!signal_pending(current
) && !guestdbg_exit_pending(vcpu
) && !rc
);
2378 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
2382 static void sync_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
2384 vcpu
->arch
.sie_block
->gpsw
.mask
= kvm_run
->psw_mask
;
2385 vcpu
->arch
.sie_block
->gpsw
.addr
= kvm_run
->psw_addr
;
2386 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PREFIX
)
2387 kvm_s390_set_prefix(vcpu
, kvm_run
->s
.regs
.prefix
);
2388 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_CRS
) {
2389 memcpy(&vcpu
->arch
.sie_block
->gcr
, &kvm_run
->s
.regs
.crs
, 128);
2390 /* some control register changes require a tlb flush */
2391 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
2393 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_ARCH0
) {
2394 kvm_s390_set_cpu_timer(vcpu
, kvm_run
->s
.regs
.cputm
);
2395 vcpu
->arch
.sie_block
->ckc
= kvm_run
->s
.regs
.ckc
;
2396 vcpu
->arch
.sie_block
->todpr
= kvm_run
->s
.regs
.todpr
;
2397 vcpu
->arch
.sie_block
->pp
= kvm_run
->s
.regs
.pp
;
2398 vcpu
->arch
.sie_block
->gbea
= kvm_run
->s
.regs
.gbea
;
2400 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PFAULT
) {
2401 vcpu
->arch
.pfault_token
= kvm_run
->s
.regs
.pft
;
2402 vcpu
->arch
.pfault_select
= kvm_run
->s
.regs
.pfs
;
2403 vcpu
->arch
.pfault_compare
= kvm_run
->s
.regs
.pfc
;
2404 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
2405 kvm_clear_async_pf_completion_queue(vcpu
);
2407 kvm_run
->kvm_dirty_regs
= 0;
2410 static void store_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
2412 kvm_run
->psw_mask
= vcpu
->arch
.sie_block
->gpsw
.mask
;
2413 kvm_run
->psw_addr
= vcpu
->arch
.sie_block
->gpsw
.addr
;
2414 kvm_run
->s
.regs
.prefix
= kvm_s390_get_prefix(vcpu
);
2415 memcpy(&kvm_run
->s
.regs
.crs
, &vcpu
->arch
.sie_block
->gcr
, 128);
2416 kvm_run
->s
.regs
.cputm
= kvm_s390_get_cpu_timer(vcpu
);
2417 kvm_run
->s
.regs
.ckc
= vcpu
->arch
.sie_block
->ckc
;
2418 kvm_run
->s
.regs
.todpr
= vcpu
->arch
.sie_block
->todpr
;
2419 kvm_run
->s
.regs
.pp
= vcpu
->arch
.sie_block
->pp
;
2420 kvm_run
->s
.regs
.gbea
= vcpu
->arch
.sie_block
->gbea
;
2421 kvm_run
->s
.regs
.pft
= vcpu
->arch
.pfault_token
;
2422 kvm_run
->s
.regs
.pfs
= vcpu
->arch
.pfault_select
;
2423 kvm_run
->s
.regs
.pfc
= vcpu
->arch
.pfault_compare
;
2426 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
2431 if (guestdbg_exit_pending(vcpu
)) {
2432 kvm_s390_prepare_debug_exit(vcpu
);
2436 if (vcpu
->sigset_active
)
2437 sigprocmask(SIG_SETMASK
, &vcpu
->sigset
, &sigsaved
);
2439 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
)) {
2440 kvm_s390_vcpu_start(vcpu
);
2441 } else if (is_vcpu_stopped(vcpu
)) {
2442 pr_err_ratelimited("can't run stopped vcpu %d\n",
2447 sync_regs(vcpu
, kvm_run
);
2448 enable_cpu_timer_accounting(vcpu
);
2451 rc
= __vcpu_run(vcpu
);
2453 if (signal_pending(current
) && !rc
) {
2454 kvm_run
->exit_reason
= KVM_EXIT_INTR
;
2458 if (guestdbg_exit_pending(vcpu
) && !rc
) {
2459 kvm_s390_prepare_debug_exit(vcpu
);
2463 if (rc
== -EREMOTE
) {
2464 /* userspace support is needed, kvm_run has been prepared */
2468 disable_cpu_timer_accounting(vcpu
);
2469 store_regs(vcpu
, kvm_run
);
2471 if (vcpu
->sigset_active
)
2472 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
2474 vcpu
->stat
.exit_userspace
++;
2479 * store status at address
2480 * we use have two special cases:
2481 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2482 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2484 int kvm_s390_store_status_unloaded(struct kvm_vcpu
*vcpu
, unsigned long gpa
)
2486 unsigned char archmode
= 1;
2487 freg_t fprs
[NUM_FPRS
];
2492 px
= kvm_s390_get_prefix(vcpu
);
2493 if (gpa
== KVM_S390_STORE_STATUS_NOADDR
) {
2494 if (write_guest_abs(vcpu
, 163, &archmode
, 1))
2497 } else if (gpa
== KVM_S390_STORE_STATUS_PREFIXED
) {
2498 if (write_guest_real(vcpu
, 163, &archmode
, 1))
2502 gpa
-= __LC_FPREGS_SAVE_AREA
;
2504 /* manually convert vector registers if necessary */
2505 if (MACHINE_HAS_VX
) {
2506 convert_vx_to_fp(fprs
, (__vector128
*) vcpu
->run
->s
.regs
.vrs
);
2507 rc
= write_guest_abs(vcpu
, gpa
+ __LC_FPREGS_SAVE_AREA
,
2510 rc
= write_guest_abs(vcpu
, gpa
+ __LC_FPREGS_SAVE_AREA
,
2511 vcpu
->run
->s
.regs
.fprs
, 128);
2513 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_GPREGS_SAVE_AREA
,
2514 vcpu
->run
->s
.regs
.gprs
, 128);
2515 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_PSW_SAVE_AREA
,
2516 &vcpu
->arch
.sie_block
->gpsw
, 16);
2517 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_PREFIX_SAVE_AREA
,
2519 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_FP_CREG_SAVE_AREA
,
2520 &vcpu
->run
->s
.regs
.fpc
, 4);
2521 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_TOD_PROGREG_SAVE_AREA
,
2522 &vcpu
->arch
.sie_block
->todpr
, 4);
2523 cputm
= kvm_s390_get_cpu_timer(vcpu
);
2524 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_CPU_TIMER_SAVE_AREA
,
2526 clkcomp
= vcpu
->arch
.sie_block
->ckc
>> 8;
2527 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_CLOCK_COMP_SAVE_AREA
,
2529 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_AREGS_SAVE_AREA
,
2530 &vcpu
->run
->s
.regs
.acrs
, 64);
2531 rc
|= write_guest_abs(vcpu
, gpa
+ __LC_CREGS_SAVE_AREA
,
2532 &vcpu
->arch
.sie_block
->gcr
, 128);
2533 return rc
? -EFAULT
: 0;
2536 int kvm_s390_vcpu_store_status(struct kvm_vcpu
*vcpu
, unsigned long addr
)
2539 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2540 * copying in vcpu load/put. Lets update our copies before we save
2541 * it into the save area
2544 vcpu
->run
->s
.regs
.fpc
= current
->thread
.fpu
.fpc
;
2545 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
2547 return kvm_s390_store_status_unloaded(vcpu
, addr
);
2551 * store additional status at address
2553 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu
*vcpu
,
2556 /* Only bits 0-53 are used for address formation */
2557 if (!(gpa
& ~0x3ff))
2560 return write_guest_abs(vcpu
, gpa
& ~0x3ff,
2561 (void *)&vcpu
->run
->s
.regs
.vrs
, 512);
2564 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu
*vcpu
, unsigned long addr
)
2566 if (!test_kvm_facility(vcpu
->kvm
, 129))
2570 * The guest VXRS are in the host VXRs due to the lazy
2571 * copying in vcpu load/put. We can simply call save_fpu_regs()
2572 * to save the current register state because we are in the
2573 * middle of a load/put cycle.
2575 * Let's update our copies before we save it into the save area.
2579 return kvm_s390_store_adtl_status_unloaded(vcpu
, addr
);
2582 static void __disable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
2584 kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
);
2585 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS
, vcpu
);
2588 static void __disable_ibs_on_all_vcpus(struct kvm
*kvm
)
2591 struct kvm_vcpu
*vcpu
;
2593 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
2594 __disable_ibs_on_vcpu(vcpu
);
2598 static void __enable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
2600 kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
);
2601 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS
, vcpu
);
2604 void kvm_s390_vcpu_start(struct kvm_vcpu
*vcpu
)
2606 int i
, online_vcpus
, started_vcpus
= 0;
2608 if (!is_vcpu_stopped(vcpu
))
2611 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 1);
2612 /* Only one cpu at a time may enter/leave the STOPPED state. */
2613 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
2614 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
2616 for (i
= 0; i
< online_vcpus
; i
++) {
2617 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
]))
2621 if (started_vcpus
== 0) {
2622 /* we're the only active VCPU -> speed it up */
2623 __enable_ibs_on_vcpu(vcpu
);
2624 } else if (started_vcpus
== 1) {
2626 * As we are starting a second VCPU, we have to disable
2627 * the IBS facility on all VCPUs to remove potentially
2628 * oustanding ENABLE requests.
2630 __disable_ibs_on_all_vcpus(vcpu
->kvm
);
2633 atomic_andnot(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
2635 * Another VCPU might have used IBS while we were offline.
2636 * Let's play safe and flush the VCPU at startup.
2638 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
2639 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
2643 void kvm_s390_vcpu_stop(struct kvm_vcpu
*vcpu
)
2645 int i
, online_vcpus
, started_vcpus
= 0;
2646 struct kvm_vcpu
*started_vcpu
= NULL
;
2648 if (is_vcpu_stopped(vcpu
))
2651 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 0);
2652 /* Only one cpu at a time may enter/leave the STOPPED state. */
2653 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
2654 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
2656 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2657 kvm_s390_clear_stop_irq(vcpu
);
2659 atomic_or(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
2660 __disable_ibs_on_vcpu(vcpu
);
2662 for (i
= 0; i
< online_vcpus
; i
++) {
2663 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
])) {
2665 started_vcpu
= vcpu
->kvm
->vcpus
[i
];
2669 if (started_vcpus
== 1) {
2671 * As we only have one VCPU left, we want to enable the
2672 * IBS facility for that VCPU to speed it up.
2674 __enable_ibs_on_vcpu(started_vcpu
);
2677 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
2681 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu
*vcpu
,
2682 struct kvm_enable_cap
*cap
)
2690 case KVM_CAP_S390_CSS_SUPPORT
:
2691 if (!vcpu
->kvm
->arch
.css_support
) {
2692 vcpu
->kvm
->arch
.css_support
= 1;
2693 VM_EVENT(vcpu
->kvm
, 3, "%s", "ENABLE: CSS support");
2694 trace_kvm_s390_enable_css(vcpu
->kvm
);
2705 static long kvm_s390_guest_mem_op(struct kvm_vcpu
*vcpu
,
2706 struct kvm_s390_mem_op
*mop
)
2708 void __user
*uaddr
= (void __user
*)mop
->buf
;
2709 void *tmpbuf
= NULL
;
2711 const u64 supported_flags
= KVM_S390_MEMOP_F_INJECT_EXCEPTION
2712 | KVM_S390_MEMOP_F_CHECK_ONLY
;
2714 if (mop
->flags
& ~supported_flags
)
2717 if (mop
->size
> MEM_OP_MAX_SIZE
)
2720 if (!(mop
->flags
& KVM_S390_MEMOP_F_CHECK_ONLY
)) {
2721 tmpbuf
= vmalloc(mop
->size
);
2726 srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2729 case KVM_S390_MEMOP_LOGICAL_READ
:
2730 if (mop
->flags
& KVM_S390_MEMOP_F_CHECK_ONLY
) {
2731 r
= check_gva_range(vcpu
, mop
->gaddr
, mop
->ar
,
2732 mop
->size
, GACC_FETCH
);
2735 r
= read_guest(vcpu
, mop
->gaddr
, mop
->ar
, tmpbuf
, mop
->size
);
2737 if (copy_to_user(uaddr
, tmpbuf
, mop
->size
))
2741 case KVM_S390_MEMOP_LOGICAL_WRITE
:
2742 if (mop
->flags
& KVM_S390_MEMOP_F_CHECK_ONLY
) {
2743 r
= check_gva_range(vcpu
, mop
->gaddr
, mop
->ar
,
2744 mop
->size
, GACC_STORE
);
2747 if (copy_from_user(tmpbuf
, uaddr
, mop
->size
)) {
2751 r
= write_guest(vcpu
, mop
->gaddr
, mop
->ar
, tmpbuf
, mop
->size
);
2757 srcu_read_unlock(&vcpu
->kvm
->srcu
, srcu_idx
);
2759 if (r
> 0 && (mop
->flags
& KVM_S390_MEMOP_F_INJECT_EXCEPTION
) != 0)
2760 kvm_s390_inject_prog_irq(vcpu
, &vcpu
->arch
.pgm
);
2766 long kvm_arch_vcpu_ioctl(struct file
*filp
,
2767 unsigned int ioctl
, unsigned long arg
)
2769 struct kvm_vcpu
*vcpu
= filp
->private_data
;
2770 void __user
*argp
= (void __user
*)arg
;
2775 case KVM_S390_IRQ
: {
2776 struct kvm_s390_irq s390irq
;
2779 if (copy_from_user(&s390irq
, argp
, sizeof(s390irq
)))
2781 r
= kvm_s390_inject_vcpu(vcpu
, &s390irq
);
2784 case KVM_S390_INTERRUPT
: {
2785 struct kvm_s390_interrupt s390int
;
2786 struct kvm_s390_irq s390irq
;
2789 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
2791 if (s390int_to_s390irq(&s390int
, &s390irq
))
2793 r
= kvm_s390_inject_vcpu(vcpu
, &s390irq
);
2796 case KVM_S390_STORE_STATUS
:
2797 idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
2798 r
= kvm_s390_vcpu_store_status(vcpu
, arg
);
2799 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
2801 case KVM_S390_SET_INITIAL_PSW
: {
2805 if (copy_from_user(&psw
, argp
, sizeof(psw
)))
2807 r
= kvm_arch_vcpu_ioctl_set_initial_psw(vcpu
, psw
);
2810 case KVM_S390_INITIAL_RESET
:
2811 r
= kvm_arch_vcpu_ioctl_initial_reset(vcpu
);
2813 case KVM_SET_ONE_REG
:
2814 case KVM_GET_ONE_REG
: {
2815 struct kvm_one_reg reg
;
2817 if (copy_from_user(®
, argp
, sizeof(reg
)))
2819 if (ioctl
== KVM_SET_ONE_REG
)
2820 r
= kvm_arch_vcpu_ioctl_set_one_reg(vcpu
, ®
);
2822 r
= kvm_arch_vcpu_ioctl_get_one_reg(vcpu
, ®
);
2825 #ifdef CONFIG_KVM_S390_UCONTROL
2826 case KVM_S390_UCAS_MAP
: {
2827 struct kvm_s390_ucas_mapping ucasmap
;
2829 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
2834 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
2839 r
= gmap_map_segment(vcpu
->arch
.gmap
, ucasmap
.user_addr
,
2840 ucasmap
.vcpu_addr
, ucasmap
.length
);
2843 case KVM_S390_UCAS_UNMAP
: {
2844 struct kvm_s390_ucas_mapping ucasmap
;
2846 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
2851 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
2856 r
= gmap_unmap_segment(vcpu
->arch
.gmap
, ucasmap
.vcpu_addr
,
2861 case KVM_S390_VCPU_FAULT
: {
2862 r
= gmap_fault(vcpu
->arch
.gmap
, arg
, 0);
2865 case KVM_ENABLE_CAP
:
2867 struct kvm_enable_cap cap
;
2869 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
2871 r
= kvm_vcpu_ioctl_enable_cap(vcpu
, &cap
);
2874 case KVM_S390_MEM_OP
: {
2875 struct kvm_s390_mem_op mem_op
;
2877 if (copy_from_user(&mem_op
, argp
, sizeof(mem_op
)) == 0)
2878 r
= kvm_s390_guest_mem_op(vcpu
, &mem_op
);
2883 case KVM_S390_SET_IRQ_STATE
: {
2884 struct kvm_s390_irq_state irq_state
;
2887 if (copy_from_user(&irq_state
, argp
, sizeof(irq_state
)))
2889 if (irq_state
.len
> VCPU_IRQS_MAX_BUF
||
2890 irq_state
.len
== 0 ||
2891 irq_state
.len
% sizeof(struct kvm_s390_irq
) > 0) {
2895 r
= kvm_s390_set_irq_state(vcpu
,
2896 (void __user
*) irq_state
.buf
,
2900 case KVM_S390_GET_IRQ_STATE
: {
2901 struct kvm_s390_irq_state irq_state
;
2904 if (copy_from_user(&irq_state
, argp
, sizeof(irq_state
)))
2906 if (irq_state
.len
== 0) {
2910 r
= kvm_s390_get_irq_state(vcpu
,
2911 (__u8 __user
*) irq_state
.buf
,
2921 int kvm_arch_vcpu_fault(struct kvm_vcpu
*vcpu
, struct vm_fault
*vmf
)
2923 #ifdef CONFIG_KVM_S390_UCONTROL
2924 if ((vmf
->pgoff
== KVM_S390_SIE_PAGE_OFFSET
)
2925 && (kvm_is_ucontrol(vcpu
->kvm
))) {
2926 vmf
->page
= virt_to_page(vcpu
->arch
.sie_block
);
2927 get_page(vmf
->page
);
2931 return VM_FAULT_SIGBUS
;
2934 int kvm_arch_create_memslot(struct kvm
*kvm
, struct kvm_memory_slot
*slot
,
2935 unsigned long npages
)
2940 /* Section: memory related */
2941 int kvm_arch_prepare_memory_region(struct kvm
*kvm
,
2942 struct kvm_memory_slot
*memslot
,
2943 const struct kvm_userspace_memory_region
*mem
,
2944 enum kvm_mr_change change
)
2946 /* A few sanity checks. We can have memory slots which have to be
2947 located/ended at a segment boundary (1MB). The memory in userland is
2948 ok to be fragmented into various different vmas. It is okay to mmap()
2949 and munmap() stuff in this slot after doing this call at any time */
2951 if (mem
->userspace_addr
& 0xffffful
)
2954 if (mem
->memory_size
& 0xffffful
)
2957 if (mem
->guest_phys_addr
+ mem
->memory_size
> kvm
->arch
.mem_limit
)
2963 void kvm_arch_commit_memory_region(struct kvm
*kvm
,
2964 const struct kvm_userspace_memory_region
*mem
,
2965 const struct kvm_memory_slot
*old
,
2966 const struct kvm_memory_slot
*new,
2967 enum kvm_mr_change change
)
2971 /* If the basics of the memslot do not change, we do not want
2972 * to update the gmap. Every update causes several unnecessary
2973 * segment translation exceptions. This is usually handled just
2974 * fine by the normal fault handler + gmap, but it will also
2975 * cause faults on the prefix page of running guest CPUs.
2977 if (old
->userspace_addr
== mem
->userspace_addr
&&
2978 old
->base_gfn
* PAGE_SIZE
== mem
->guest_phys_addr
&&
2979 old
->npages
* PAGE_SIZE
== mem
->memory_size
)
2982 rc
= gmap_map_segment(kvm
->arch
.gmap
, mem
->userspace_addr
,
2983 mem
->guest_phys_addr
, mem
->memory_size
);
2985 pr_warn("failed to commit memory region\n");
2989 static inline unsigned long nonhyp_mask(int i
)
2991 unsigned int nonhyp_fai
= (sclp
.hmfai
<< i
* 2) >> 30;
2993 return 0x0000ffffffffffffUL
>> (nonhyp_fai
<< 4);
2996 void kvm_arch_vcpu_block_finish(struct kvm_vcpu
*vcpu
)
2998 vcpu
->valid_wakeup
= false;
3001 static int __init
kvm_s390_init(void)
3005 if (!sclp
.has_sief2
) {
3006 pr_info("SIE not available\n");
3010 for (i
= 0; i
< 16; i
++)
3011 kvm_s390_fac_list_mask
[i
] |=
3012 S390_lowcore
.stfle_fac_list
[i
] & nonhyp_mask(i
);
3014 return kvm_init(NULL
, sizeof(struct kvm_vcpu
), 0, THIS_MODULE
);
3017 static void __exit
kvm_s390_exit(void)
3022 module_init(kvm_s390_init
);
3023 module_exit(kvm_s390_exit
);
3026 * Enable autoloading of the kvm module.
3027 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3028 * since x86 takes a different approach.
3030 #include <linux/miscdevice.h>
3031 MODULE_ALIAS_MISCDEV(KVM_MINOR
);
3032 MODULE_ALIAS("devname:kvm");