2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/slab.h>
26 #include <linux/timer.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/lowcore.h>
29 #include <asm/pgtable.h>
31 #include <asm/switch_to.h>
32 #include <asm/facility.h>
37 #define CREATE_TRACE_POINTS
39 #include "trace-s390.h"
41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
43 struct kvm_stats_debugfs_item debugfs_entries
[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace
) },
45 { "exit_null", VCPU_STAT(exit_null
) },
46 { "exit_validity", VCPU_STAT(exit_validity
) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request
) },
48 { "exit_external_request", VCPU_STAT(exit_external_request
) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt
) },
50 { "exit_instruction", VCPU_STAT(exit_instruction
) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption
) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program
) },
53 { "halt_wakeup", VCPU_STAT(halt_wakeup
) },
54 { "instruction_lctlg", VCPU_STAT(instruction_lctlg
) },
55 { "instruction_lctl", VCPU_STAT(instruction_lctl
) },
56 { "instruction_stctl", VCPU_STAT(instruction_stctl
) },
57 { "instruction_stctg", VCPU_STAT(instruction_stctg
) },
58 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal
) },
59 { "deliver_external_call", VCPU_STAT(deliver_external_call
) },
60 { "deliver_service_signal", VCPU_STAT(deliver_service_signal
) },
61 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt
) },
62 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal
) },
63 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal
) },
64 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal
) },
65 { "deliver_program_interruption", VCPU_STAT(deliver_program_int
) },
66 { "exit_wait_state", VCPU_STAT(exit_wait_state
) },
67 { "instruction_pfmf", VCPU_STAT(instruction_pfmf
) },
68 { "instruction_stidp", VCPU_STAT(instruction_stidp
) },
69 { "instruction_spx", VCPU_STAT(instruction_spx
) },
70 { "instruction_stpx", VCPU_STAT(instruction_stpx
) },
71 { "instruction_stap", VCPU_STAT(instruction_stap
) },
72 { "instruction_storage_key", VCPU_STAT(instruction_storage_key
) },
73 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock
) },
74 { "instruction_stsch", VCPU_STAT(instruction_stsch
) },
75 { "instruction_chsc", VCPU_STAT(instruction_chsc
) },
76 { "instruction_essa", VCPU_STAT(instruction_essa
) },
77 { "instruction_stsi", VCPU_STAT(instruction_stsi
) },
78 { "instruction_stfl", VCPU_STAT(instruction_stfl
) },
79 { "instruction_tprot", VCPU_STAT(instruction_tprot
) },
80 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense
) },
81 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running
) },
82 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call
) },
83 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency
) },
84 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop
) },
85 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch
) },
86 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix
) },
87 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart
) },
88 { "diagnose_10", VCPU_STAT(diagnose_10
) },
89 { "diagnose_44", VCPU_STAT(diagnose_44
) },
90 { "diagnose_9c", VCPU_STAT(diagnose_9c
) },
94 unsigned long *vfacilities
;
95 static struct gmap_notifier gmap_notifier
;
97 /* test availability of vfacility */
98 int test_vfacility(unsigned long nr
)
100 return __test_facility(nr
, (void *) vfacilities
);
103 /* Section: not file related */
104 int kvm_arch_hardware_enable(void)
106 /* every s390 is virtualization enabled ;-) */
110 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
);
112 int kvm_arch_hardware_setup(void)
114 gmap_notifier
.notifier_call
= kvm_gmap_notifier
;
115 gmap_register_ipte_notifier(&gmap_notifier
);
119 void kvm_arch_hardware_unsetup(void)
121 gmap_unregister_ipte_notifier(&gmap_notifier
);
124 int kvm_arch_init(void *opaque
)
126 /* Register floating interrupt controller interface. */
127 return kvm_register_device_ops(&kvm_flic_ops
, KVM_DEV_TYPE_FLIC
);
130 /* Section: device related */
131 long kvm_arch_dev_ioctl(struct file
*filp
,
132 unsigned int ioctl
, unsigned long arg
)
134 if (ioctl
== KVM_S390_ENABLE_SIE
)
135 return s390_enable_sie();
139 int kvm_vm_ioctl_check_extension(struct kvm
*kvm
, long ext
)
144 case KVM_CAP_S390_PSW
:
145 case KVM_CAP_S390_GMAP
:
146 case KVM_CAP_SYNC_MMU
:
147 #ifdef CONFIG_KVM_S390_UCONTROL
148 case KVM_CAP_S390_UCONTROL
:
150 case KVM_CAP_ASYNC_PF
:
151 case KVM_CAP_SYNC_REGS
:
152 case KVM_CAP_ONE_REG
:
153 case KVM_CAP_ENABLE_CAP
:
154 case KVM_CAP_S390_CSS_SUPPORT
:
156 case KVM_CAP_IOEVENTFD
:
157 case KVM_CAP_DEVICE_CTRL
:
158 case KVM_CAP_ENABLE_CAP_VM
:
159 case KVM_CAP_S390_IRQCHIP
:
160 case KVM_CAP_VM_ATTRIBUTES
:
161 case KVM_CAP_MP_STATE
:
164 case KVM_CAP_NR_VCPUS
:
165 case KVM_CAP_MAX_VCPUS
:
168 case KVM_CAP_NR_MEMSLOTS
:
169 r
= KVM_USER_MEM_SLOTS
;
171 case KVM_CAP_S390_COW
:
172 r
= MACHINE_HAS_ESOP
;
180 static void kvm_s390_sync_dirty_log(struct kvm
*kvm
,
181 struct kvm_memory_slot
*memslot
)
183 gfn_t cur_gfn
, last_gfn
;
184 unsigned long address
;
185 struct gmap
*gmap
= kvm
->arch
.gmap
;
187 down_read(&gmap
->mm
->mmap_sem
);
188 /* Loop over all guest pages */
189 last_gfn
= memslot
->base_gfn
+ memslot
->npages
;
190 for (cur_gfn
= memslot
->base_gfn
; cur_gfn
<= last_gfn
; cur_gfn
++) {
191 address
= gfn_to_hva_memslot(memslot
, cur_gfn
);
193 if (gmap_test_and_clear_dirty(address
, gmap
))
194 mark_page_dirty(kvm
, cur_gfn
);
196 up_read(&gmap
->mm
->mmap_sem
);
199 /* Section: vm related */
201 * Get (and clear) the dirty memory log for a memory slot.
203 int kvm_vm_ioctl_get_dirty_log(struct kvm
*kvm
,
204 struct kvm_dirty_log
*log
)
208 struct kvm_memory_slot
*memslot
;
211 mutex_lock(&kvm
->slots_lock
);
214 if (log
->slot
>= KVM_USER_MEM_SLOTS
)
217 memslot
= id_to_memslot(kvm
->memslots
, log
->slot
);
219 if (!memslot
->dirty_bitmap
)
222 kvm_s390_sync_dirty_log(kvm
, memslot
);
223 r
= kvm_get_dirty_log(kvm
, log
, &is_dirty
);
227 /* Clear the dirty log */
229 n
= kvm_dirty_bitmap_bytes(memslot
);
230 memset(memslot
->dirty_bitmap
, 0, n
);
234 mutex_unlock(&kvm
->slots_lock
);
238 static int kvm_vm_ioctl_enable_cap(struct kvm
*kvm
, struct kvm_enable_cap
*cap
)
246 case KVM_CAP_S390_IRQCHIP
:
247 kvm
->arch
.use_irqchip
= 1;
257 static int kvm_s390_mem_control(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
261 switch (attr
->attr
) {
262 case KVM_S390_VM_MEM_ENABLE_CMMA
:
264 mutex_lock(&kvm
->lock
);
265 if (atomic_read(&kvm
->online_vcpus
) == 0) {
266 kvm
->arch
.use_cmma
= 1;
269 mutex_unlock(&kvm
->lock
);
271 case KVM_S390_VM_MEM_CLR_CMMA
:
272 mutex_lock(&kvm
->lock
);
273 idx
= srcu_read_lock(&kvm
->srcu
);
274 s390_reset_cmma(kvm
->arch
.gmap
->mm
);
275 srcu_read_unlock(&kvm
->srcu
, idx
);
276 mutex_unlock(&kvm
->lock
);
286 static int kvm_s390_vm_set_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
290 switch (attr
->group
) {
291 case KVM_S390_VM_MEM_CTRL
:
292 ret
= kvm_s390_mem_control(kvm
, attr
);
302 static int kvm_s390_vm_get_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
307 static int kvm_s390_vm_has_attr(struct kvm
*kvm
, struct kvm_device_attr
*attr
)
311 switch (attr
->group
) {
312 case KVM_S390_VM_MEM_CTRL
:
313 switch (attr
->attr
) {
314 case KVM_S390_VM_MEM_ENABLE_CMMA
:
315 case KVM_S390_VM_MEM_CLR_CMMA
:
331 long kvm_arch_vm_ioctl(struct file
*filp
,
332 unsigned int ioctl
, unsigned long arg
)
334 struct kvm
*kvm
= filp
->private_data
;
335 void __user
*argp
= (void __user
*)arg
;
336 struct kvm_device_attr attr
;
340 case KVM_S390_INTERRUPT
: {
341 struct kvm_s390_interrupt s390int
;
344 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
346 r
= kvm_s390_inject_vm(kvm
, &s390int
);
349 case KVM_ENABLE_CAP
: {
350 struct kvm_enable_cap cap
;
352 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
354 r
= kvm_vm_ioctl_enable_cap(kvm
, &cap
);
357 case KVM_CREATE_IRQCHIP
: {
358 struct kvm_irq_routing_entry routing
;
361 if (kvm
->arch
.use_irqchip
) {
362 /* Set up dummy routing. */
363 memset(&routing
, 0, sizeof(routing
));
364 kvm_set_irq_routing(kvm
, &routing
, 0, 0);
369 case KVM_SET_DEVICE_ATTR
: {
371 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
373 r
= kvm_s390_vm_set_attr(kvm
, &attr
);
376 case KVM_GET_DEVICE_ATTR
: {
378 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
380 r
= kvm_s390_vm_get_attr(kvm
, &attr
);
383 case KVM_HAS_DEVICE_ATTR
: {
385 if (copy_from_user(&attr
, (void __user
*)arg
, sizeof(attr
)))
387 r
= kvm_s390_vm_has_attr(kvm
, &attr
);
397 static int kvm_s390_crypto_init(struct kvm
*kvm
)
399 if (!test_vfacility(76))
402 kvm
->arch
.crypto
.crycb
= kzalloc(sizeof(*kvm
->arch
.crypto
.crycb
),
403 GFP_KERNEL
| GFP_DMA
);
404 if (!kvm
->arch
.crypto
.crycb
)
407 kvm
->arch
.crypto
.crycbd
= (__u32
) (unsigned long) kvm
->arch
.crypto
.crycb
|
413 int kvm_arch_init_vm(struct kvm
*kvm
, unsigned long type
)
417 static unsigned long sca_offset
;
420 #ifdef CONFIG_KVM_S390_UCONTROL
421 if (type
& ~KVM_VM_S390_UCONTROL
)
423 if ((type
& KVM_VM_S390_UCONTROL
) && (!capable(CAP_SYS_ADMIN
)))
430 rc
= s390_enable_sie();
436 kvm
->arch
.sca
= (struct sca_block
*) get_zeroed_page(GFP_KERNEL
);
439 spin_lock(&kvm_lock
);
440 sca_offset
= (sca_offset
+ 16) & 0x7f0;
441 kvm
->arch
.sca
= (struct sca_block
*) ((char *) kvm
->arch
.sca
+ sca_offset
);
442 spin_unlock(&kvm_lock
);
444 sprintf(debug_name
, "kvm-%u", current
->pid
);
446 kvm
->arch
.dbf
= debug_register(debug_name
, 8, 2, 8 * sizeof(long));
450 if (kvm_s390_crypto_init(kvm
) < 0)
453 spin_lock_init(&kvm
->arch
.float_int
.lock
);
454 INIT_LIST_HEAD(&kvm
->arch
.float_int
.list
);
455 init_waitqueue_head(&kvm
->arch
.ipte_wq
);
457 debug_register_view(kvm
->arch
.dbf
, &debug_sprintf_view
);
458 VM_EVENT(kvm
, 3, "%s", "vm created");
460 if (type
& KVM_VM_S390_UCONTROL
) {
461 kvm
->arch
.gmap
= NULL
;
463 kvm
->arch
.gmap
= gmap_alloc(current
->mm
, (1UL << 44) - 1);
466 kvm
->arch
.gmap
->private = kvm
;
467 kvm
->arch
.gmap
->pfault_enabled
= 0;
470 kvm
->arch
.css_support
= 0;
471 kvm
->arch
.use_irqchip
= 0;
473 spin_lock_init(&kvm
->arch
.start_stop_lock
);
477 kfree(kvm
->arch
.crypto
.crycb
);
479 debug_unregister(kvm
->arch
.dbf
);
481 free_page((unsigned long)(kvm
->arch
.sca
));
486 void kvm_arch_vcpu_destroy(struct kvm_vcpu
*vcpu
)
488 VCPU_EVENT(vcpu
, 3, "%s", "free cpu");
489 trace_kvm_s390_destroy_vcpu(vcpu
->vcpu_id
);
490 kvm_s390_clear_local_irqs(vcpu
);
491 kvm_clear_async_pf_completion_queue(vcpu
);
492 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
493 clear_bit(63 - vcpu
->vcpu_id
,
494 (unsigned long *) &vcpu
->kvm
->arch
.sca
->mcn
);
495 if (vcpu
->kvm
->arch
.sca
->cpu
[vcpu
->vcpu_id
].sda
==
496 (__u64
) vcpu
->arch
.sie_block
)
497 vcpu
->kvm
->arch
.sca
->cpu
[vcpu
->vcpu_id
].sda
= 0;
501 if (kvm_is_ucontrol(vcpu
->kvm
))
502 gmap_free(vcpu
->arch
.gmap
);
504 if (kvm_s390_cmma_enabled(vcpu
->kvm
))
505 kvm_s390_vcpu_unsetup_cmma(vcpu
);
506 free_page((unsigned long)(vcpu
->arch
.sie_block
));
508 kvm_vcpu_uninit(vcpu
);
509 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
512 static void kvm_free_vcpus(struct kvm
*kvm
)
515 struct kvm_vcpu
*vcpu
;
517 kvm_for_each_vcpu(i
, vcpu
, kvm
)
518 kvm_arch_vcpu_destroy(vcpu
);
520 mutex_lock(&kvm
->lock
);
521 for (i
= 0; i
< atomic_read(&kvm
->online_vcpus
); i
++)
522 kvm
->vcpus
[i
] = NULL
;
524 atomic_set(&kvm
->online_vcpus
, 0);
525 mutex_unlock(&kvm
->lock
);
528 void kvm_arch_destroy_vm(struct kvm
*kvm
)
531 free_page((unsigned long)(kvm
->arch
.sca
));
532 debug_unregister(kvm
->arch
.dbf
);
533 kfree(kvm
->arch
.crypto
.crycb
);
534 if (!kvm_is_ucontrol(kvm
))
535 gmap_free(kvm
->arch
.gmap
);
536 kvm_s390_destroy_adapters(kvm
);
537 kvm_s390_clear_float_irqs(kvm
);
540 /* Section: vcpu related */
541 int kvm_arch_vcpu_init(struct kvm_vcpu
*vcpu
)
543 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
544 kvm_clear_async_pf_completion_queue(vcpu
);
545 if (kvm_is_ucontrol(vcpu
->kvm
)) {
546 vcpu
->arch
.gmap
= gmap_alloc(current
->mm
, -1UL);
547 if (!vcpu
->arch
.gmap
)
549 vcpu
->arch
.gmap
->private = vcpu
->kvm
;
553 vcpu
->arch
.gmap
= vcpu
->kvm
->arch
.gmap
;
554 vcpu
->run
->kvm_valid_regs
= KVM_SYNC_PREFIX
|
563 void kvm_arch_vcpu_load(struct kvm_vcpu
*vcpu
, int cpu
)
565 save_fp_ctl(&vcpu
->arch
.host_fpregs
.fpc
);
566 save_fp_regs(vcpu
->arch
.host_fpregs
.fprs
);
567 save_access_regs(vcpu
->arch
.host_acrs
);
568 restore_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
569 restore_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
570 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
571 gmap_enable(vcpu
->arch
.gmap
);
572 atomic_set_mask(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
575 void kvm_arch_vcpu_put(struct kvm_vcpu
*vcpu
)
577 atomic_clear_mask(CPUSTAT_RUNNING
, &vcpu
->arch
.sie_block
->cpuflags
);
578 gmap_disable(vcpu
->arch
.gmap
);
579 save_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
580 save_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
581 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
582 restore_fp_ctl(&vcpu
->arch
.host_fpregs
.fpc
);
583 restore_fp_regs(vcpu
->arch
.host_fpregs
.fprs
);
584 restore_access_regs(vcpu
->arch
.host_acrs
);
587 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu
*vcpu
)
589 /* this equals initial cpu reset in pop, but we don't switch to ESA */
590 vcpu
->arch
.sie_block
->gpsw
.mask
= 0UL;
591 vcpu
->arch
.sie_block
->gpsw
.addr
= 0UL;
592 kvm_s390_set_prefix(vcpu
, 0);
593 vcpu
->arch
.sie_block
->cputm
= 0UL;
594 vcpu
->arch
.sie_block
->ckc
= 0UL;
595 vcpu
->arch
.sie_block
->todpr
= 0;
596 memset(vcpu
->arch
.sie_block
->gcr
, 0, 16 * sizeof(__u64
));
597 vcpu
->arch
.sie_block
->gcr
[0] = 0xE0UL
;
598 vcpu
->arch
.sie_block
->gcr
[14] = 0xC2000000UL
;
599 vcpu
->arch
.guest_fpregs
.fpc
= 0;
600 asm volatile("lfpc %0" : : "Q" (vcpu
->arch
.guest_fpregs
.fpc
));
601 vcpu
->arch
.sie_block
->gbea
= 1;
602 vcpu
->arch
.sie_block
->pp
= 0;
603 vcpu
->arch
.pfault_token
= KVM_S390_PFAULT_TOKEN_INVALID
;
604 kvm_clear_async_pf_completion_queue(vcpu
);
605 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
))
606 kvm_s390_vcpu_stop(vcpu
);
607 kvm_s390_clear_local_irqs(vcpu
);
610 int kvm_arch_vcpu_postcreate(struct kvm_vcpu
*vcpu
)
615 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu
*vcpu
)
617 if (!test_vfacility(76))
620 vcpu
->arch
.sie_block
->crycbd
= vcpu
->kvm
->arch
.crypto
.crycbd
;
623 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu
*vcpu
)
625 free_page(vcpu
->arch
.sie_block
->cbrlo
);
626 vcpu
->arch
.sie_block
->cbrlo
= 0;
629 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu
*vcpu
)
631 vcpu
->arch
.sie_block
->cbrlo
= get_zeroed_page(GFP_KERNEL
);
632 if (!vcpu
->arch
.sie_block
->cbrlo
)
635 vcpu
->arch
.sie_block
->ecb2
|= 0x80;
636 vcpu
->arch
.sie_block
->ecb2
&= ~0x08;
640 int kvm_arch_vcpu_setup(struct kvm_vcpu
*vcpu
)
644 atomic_set(&vcpu
->arch
.sie_block
->cpuflags
, CPUSTAT_ZARCH
|
648 vcpu
->arch
.sie_block
->ecb
= 6;
649 if (test_vfacility(50) && test_vfacility(73))
650 vcpu
->arch
.sie_block
->ecb
|= 0x10;
652 vcpu
->arch
.sie_block
->ecb2
= 8;
653 vcpu
->arch
.sie_block
->eca
= 0xD1002000U
;
655 vcpu
->arch
.sie_block
->eca
|= 1;
656 vcpu
->arch
.sie_block
->fac
= (int) (long) vfacilities
;
657 vcpu
->arch
.sie_block
->ictl
|= ICTL_ISKE
| ICTL_SSKE
| ICTL_RRBE
|
660 if (kvm_s390_cmma_enabled(vcpu
->kvm
)) {
661 rc
= kvm_s390_vcpu_setup_cmma(vcpu
);
665 hrtimer_init(&vcpu
->arch
.ckc_timer
, CLOCK_REALTIME
, HRTIMER_MODE_ABS
);
666 vcpu
->arch
.ckc_timer
.function
= kvm_s390_idle_wakeup
;
667 get_cpu_id(&vcpu
->arch
.cpu_id
);
668 vcpu
->arch
.cpu_id
.version
= 0xff;
670 kvm_s390_vcpu_crypto_setup(vcpu
);
675 struct kvm_vcpu
*kvm_arch_vcpu_create(struct kvm
*kvm
,
678 struct kvm_vcpu
*vcpu
;
679 struct sie_page
*sie_page
;
682 if (id
>= KVM_MAX_VCPUS
)
687 vcpu
= kmem_cache_zalloc(kvm_vcpu_cache
, GFP_KERNEL
);
691 sie_page
= (struct sie_page
*) get_zeroed_page(GFP_KERNEL
);
695 vcpu
->arch
.sie_block
= &sie_page
->sie_block
;
696 vcpu
->arch
.sie_block
->itdba
= (unsigned long) &sie_page
->itdb
;
698 vcpu
->arch
.sie_block
->icpua
= id
;
699 if (!kvm_is_ucontrol(kvm
)) {
700 if (!kvm
->arch
.sca
) {
704 if (!kvm
->arch
.sca
->cpu
[id
].sda
)
705 kvm
->arch
.sca
->cpu
[id
].sda
=
706 (__u64
) vcpu
->arch
.sie_block
;
707 vcpu
->arch
.sie_block
->scaoh
=
708 (__u32
)(((__u64
)kvm
->arch
.sca
) >> 32);
709 vcpu
->arch
.sie_block
->scaol
= (__u32
)(__u64
)kvm
->arch
.sca
;
710 set_bit(63 - id
, (unsigned long *) &kvm
->arch
.sca
->mcn
);
713 spin_lock_init(&vcpu
->arch
.local_int
.lock
);
714 INIT_LIST_HEAD(&vcpu
->arch
.local_int
.list
);
715 vcpu
->arch
.local_int
.float_int
= &kvm
->arch
.float_int
;
716 vcpu
->arch
.local_int
.wq
= &vcpu
->wq
;
717 vcpu
->arch
.local_int
.cpuflags
= &vcpu
->arch
.sie_block
->cpuflags
;
719 rc
= kvm_vcpu_init(vcpu
, kvm
, id
);
721 goto out_free_sie_block
;
722 VM_EVENT(kvm
, 3, "create cpu %d at %p, sie block at %p", id
, vcpu
,
723 vcpu
->arch
.sie_block
);
724 trace_kvm_s390_create_vcpu(id
, vcpu
, vcpu
->arch
.sie_block
);
728 free_page((unsigned long)(vcpu
->arch
.sie_block
));
730 kmem_cache_free(kvm_vcpu_cache
, vcpu
);
735 int kvm_arch_vcpu_runnable(struct kvm_vcpu
*vcpu
)
737 return kvm_cpu_has_interrupt(vcpu
);
740 void s390_vcpu_block(struct kvm_vcpu
*vcpu
)
742 atomic_set_mask(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
745 void s390_vcpu_unblock(struct kvm_vcpu
*vcpu
)
747 atomic_clear_mask(PROG_BLOCK_SIE
, &vcpu
->arch
.sie_block
->prog20
);
751 * Kick a guest cpu out of SIE and wait until SIE is not running.
752 * If the CPU is not running (e.g. waiting as idle) the function will
753 * return immediately. */
754 void exit_sie(struct kvm_vcpu
*vcpu
)
756 atomic_set_mask(CPUSTAT_STOP_INT
, &vcpu
->arch
.sie_block
->cpuflags
);
757 while (vcpu
->arch
.sie_block
->prog0c
& PROG_IN_SIE
)
761 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
762 void exit_sie_sync(struct kvm_vcpu
*vcpu
)
764 s390_vcpu_block(vcpu
);
768 static void kvm_gmap_notifier(struct gmap
*gmap
, unsigned long address
)
771 struct kvm
*kvm
= gmap
->private;
772 struct kvm_vcpu
*vcpu
;
774 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
775 /* match against both prefix pages */
776 if (kvm_s390_get_prefix(vcpu
) == (address
& ~0x1000UL
)) {
777 VCPU_EVENT(vcpu
, 2, "gmap notifier for %lx", address
);
778 kvm_make_request(KVM_REQ_MMU_RELOAD
, vcpu
);
784 int kvm_arch_vcpu_should_kick(struct kvm_vcpu
*vcpu
)
786 /* kvm common code refers to this, but never calls it */
791 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu
*vcpu
,
792 struct kvm_one_reg
*reg
)
797 case KVM_REG_S390_TODPR
:
798 r
= put_user(vcpu
->arch
.sie_block
->todpr
,
799 (u32 __user
*)reg
->addr
);
801 case KVM_REG_S390_EPOCHDIFF
:
802 r
= put_user(vcpu
->arch
.sie_block
->epoch
,
803 (u64 __user
*)reg
->addr
);
805 case KVM_REG_S390_CPU_TIMER
:
806 r
= put_user(vcpu
->arch
.sie_block
->cputm
,
807 (u64 __user
*)reg
->addr
);
809 case KVM_REG_S390_CLOCK_COMP
:
810 r
= put_user(vcpu
->arch
.sie_block
->ckc
,
811 (u64 __user
*)reg
->addr
);
813 case KVM_REG_S390_PFTOKEN
:
814 r
= put_user(vcpu
->arch
.pfault_token
,
815 (u64 __user
*)reg
->addr
);
817 case KVM_REG_S390_PFCOMPARE
:
818 r
= put_user(vcpu
->arch
.pfault_compare
,
819 (u64 __user
*)reg
->addr
);
821 case KVM_REG_S390_PFSELECT
:
822 r
= put_user(vcpu
->arch
.pfault_select
,
823 (u64 __user
*)reg
->addr
);
825 case KVM_REG_S390_PP
:
826 r
= put_user(vcpu
->arch
.sie_block
->pp
,
827 (u64 __user
*)reg
->addr
);
829 case KVM_REG_S390_GBEA
:
830 r
= put_user(vcpu
->arch
.sie_block
->gbea
,
831 (u64 __user
*)reg
->addr
);
840 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu
*vcpu
,
841 struct kvm_one_reg
*reg
)
846 case KVM_REG_S390_TODPR
:
847 r
= get_user(vcpu
->arch
.sie_block
->todpr
,
848 (u32 __user
*)reg
->addr
);
850 case KVM_REG_S390_EPOCHDIFF
:
851 r
= get_user(vcpu
->arch
.sie_block
->epoch
,
852 (u64 __user
*)reg
->addr
);
854 case KVM_REG_S390_CPU_TIMER
:
855 r
= get_user(vcpu
->arch
.sie_block
->cputm
,
856 (u64 __user
*)reg
->addr
);
858 case KVM_REG_S390_CLOCK_COMP
:
859 r
= get_user(vcpu
->arch
.sie_block
->ckc
,
860 (u64 __user
*)reg
->addr
);
862 case KVM_REG_S390_PFTOKEN
:
863 r
= get_user(vcpu
->arch
.pfault_token
,
864 (u64 __user
*)reg
->addr
);
866 case KVM_REG_S390_PFCOMPARE
:
867 r
= get_user(vcpu
->arch
.pfault_compare
,
868 (u64 __user
*)reg
->addr
);
870 case KVM_REG_S390_PFSELECT
:
871 r
= get_user(vcpu
->arch
.pfault_select
,
872 (u64 __user
*)reg
->addr
);
874 case KVM_REG_S390_PP
:
875 r
= get_user(vcpu
->arch
.sie_block
->pp
,
876 (u64 __user
*)reg
->addr
);
878 case KVM_REG_S390_GBEA
:
879 r
= get_user(vcpu
->arch
.sie_block
->gbea
,
880 (u64 __user
*)reg
->addr
);
889 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu
*vcpu
)
891 kvm_s390_vcpu_initial_reset(vcpu
);
895 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
897 memcpy(&vcpu
->run
->s
.regs
.gprs
, ®s
->gprs
, sizeof(regs
->gprs
));
901 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
903 memcpy(®s
->gprs
, &vcpu
->run
->s
.regs
.gprs
, sizeof(regs
->gprs
));
907 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu
*vcpu
,
908 struct kvm_sregs
*sregs
)
910 memcpy(&vcpu
->run
->s
.regs
.acrs
, &sregs
->acrs
, sizeof(sregs
->acrs
));
911 memcpy(&vcpu
->arch
.sie_block
->gcr
, &sregs
->crs
, sizeof(sregs
->crs
));
912 restore_access_regs(vcpu
->run
->s
.regs
.acrs
);
916 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu
*vcpu
,
917 struct kvm_sregs
*sregs
)
919 memcpy(&sregs
->acrs
, &vcpu
->run
->s
.regs
.acrs
, sizeof(sregs
->acrs
));
920 memcpy(&sregs
->crs
, &vcpu
->arch
.sie_block
->gcr
, sizeof(sregs
->crs
));
924 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
926 if (test_fp_ctl(fpu
->fpc
))
928 memcpy(&vcpu
->arch
.guest_fpregs
.fprs
, &fpu
->fprs
, sizeof(fpu
->fprs
));
929 vcpu
->arch
.guest_fpregs
.fpc
= fpu
->fpc
;
930 restore_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
931 restore_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
935 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
937 memcpy(&fpu
->fprs
, &vcpu
->arch
.guest_fpregs
.fprs
, sizeof(fpu
->fprs
));
938 fpu
->fpc
= vcpu
->arch
.guest_fpregs
.fpc
;
942 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu
*vcpu
, psw_t psw
)
946 if (!is_vcpu_stopped(vcpu
))
949 vcpu
->run
->psw_mask
= psw
.mask
;
950 vcpu
->run
->psw_addr
= psw
.addr
;
955 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
956 struct kvm_translation
*tr
)
958 return -EINVAL
; /* not implemented yet */
961 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
962 KVM_GUESTDBG_USE_HW_BP | \
965 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu
*vcpu
,
966 struct kvm_guest_debug
*dbg
)
970 vcpu
->guest_debug
= 0;
971 kvm_s390_clear_bp_data(vcpu
);
973 if (dbg
->control
& ~VALID_GUESTDBG_FLAGS
)
976 if (dbg
->control
& KVM_GUESTDBG_ENABLE
) {
977 vcpu
->guest_debug
= dbg
->control
;
978 /* enforce guest PER */
979 atomic_set_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
981 if (dbg
->control
& KVM_GUESTDBG_USE_HW_BP
)
982 rc
= kvm_s390_import_bp_data(vcpu
, dbg
);
984 atomic_clear_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
985 vcpu
->arch
.guestdbg
.last_bp
= 0;
989 vcpu
->guest_debug
= 0;
990 kvm_s390_clear_bp_data(vcpu
);
991 atomic_clear_mask(CPUSTAT_P
, &vcpu
->arch
.sie_block
->cpuflags
);
997 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu
*vcpu
,
998 struct kvm_mp_state
*mp_state
)
1000 /* CHECK_STOP and LOAD are not supported yet */
1001 return is_vcpu_stopped(vcpu
) ? KVM_MP_STATE_STOPPED
:
1002 KVM_MP_STATE_OPERATING
;
1005 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu
*vcpu
,
1006 struct kvm_mp_state
*mp_state
)
1010 /* user space knows about this interface - let it control the state */
1011 vcpu
->kvm
->arch
.user_cpu_state_ctrl
= 1;
1013 switch (mp_state
->mp_state
) {
1014 case KVM_MP_STATE_STOPPED
:
1015 kvm_s390_vcpu_stop(vcpu
);
1017 case KVM_MP_STATE_OPERATING
:
1018 kvm_s390_vcpu_start(vcpu
);
1020 case KVM_MP_STATE_LOAD
:
1021 case KVM_MP_STATE_CHECK_STOP
:
1022 /* fall through - CHECK_STOP and LOAD are not supported yet */
1030 bool kvm_s390_cmma_enabled(struct kvm
*kvm
)
1032 if (!MACHINE_IS_LPAR
)
1034 /* only enable for z10 and later */
1035 if (!MACHINE_HAS_EDAT1
)
1037 if (!kvm
->arch
.use_cmma
)
1042 static bool ibs_enabled(struct kvm_vcpu
*vcpu
)
1044 return atomic_read(&vcpu
->arch
.sie_block
->cpuflags
) & CPUSTAT_IBS
;
1047 static int kvm_s390_handle_requests(struct kvm_vcpu
*vcpu
)
1050 s390_vcpu_unblock(vcpu
);
1052 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1053 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1054 * This ensures that the ipte instruction for this request has
1055 * already finished. We might race against a second unmapper that
1056 * wants to set the blocking bit. Lets just retry the request loop.
1058 if (kvm_check_request(KVM_REQ_MMU_RELOAD
, vcpu
)) {
1060 rc
= gmap_ipte_notify(vcpu
->arch
.gmap
,
1061 kvm_s390_get_prefix(vcpu
),
1068 if (kvm_check_request(KVM_REQ_TLB_FLUSH
, vcpu
)) {
1069 vcpu
->arch
.sie_block
->ihcpu
= 0xffff;
1073 if (kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
)) {
1074 if (!ibs_enabled(vcpu
)) {
1075 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 1);
1076 atomic_set_mask(CPUSTAT_IBS
,
1077 &vcpu
->arch
.sie_block
->cpuflags
);
1082 if (kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
)) {
1083 if (ibs_enabled(vcpu
)) {
1084 trace_kvm_s390_enable_disable_ibs(vcpu
->vcpu_id
, 0);
1085 atomic_clear_mask(CPUSTAT_IBS
,
1086 &vcpu
->arch
.sie_block
->cpuflags
);
1091 /* nothing to do, just clear the request */
1092 clear_bit(KVM_REQ_UNHALT
, &vcpu
->requests
);
1098 * kvm_arch_fault_in_page - fault-in guest page if necessary
1099 * @vcpu: The corresponding virtual cpu
1100 * @gpa: Guest physical address
1101 * @writable: Whether the page should be writable or not
1103 * Make sure that a guest page has been faulted-in on the host.
1105 * Return: Zero on success, negative error code otherwise.
1107 long kvm_arch_fault_in_page(struct kvm_vcpu
*vcpu
, gpa_t gpa
, int writable
)
1109 return gmap_fault(vcpu
->arch
.gmap
, gpa
,
1110 writable
? FAULT_FLAG_WRITE
: 0);
1113 static void __kvm_inject_pfault_token(struct kvm_vcpu
*vcpu
, bool start_token
,
1114 unsigned long token
)
1116 struct kvm_s390_interrupt inti
;
1117 inti
.parm64
= token
;
1120 inti
.type
= KVM_S390_INT_PFAULT_INIT
;
1121 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu
, &inti
));
1123 inti
.type
= KVM_S390_INT_PFAULT_DONE
;
1124 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu
->kvm
, &inti
));
1128 void kvm_arch_async_page_not_present(struct kvm_vcpu
*vcpu
,
1129 struct kvm_async_pf
*work
)
1131 trace_kvm_s390_pfault_init(vcpu
, work
->arch
.pfault_token
);
1132 __kvm_inject_pfault_token(vcpu
, true, work
->arch
.pfault_token
);
1135 void kvm_arch_async_page_present(struct kvm_vcpu
*vcpu
,
1136 struct kvm_async_pf
*work
)
1138 trace_kvm_s390_pfault_done(vcpu
, work
->arch
.pfault_token
);
1139 __kvm_inject_pfault_token(vcpu
, false, work
->arch
.pfault_token
);
1142 void kvm_arch_async_page_ready(struct kvm_vcpu
*vcpu
,
1143 struct kvm_async_pf
*work
)
1145 /* s390 will always inject the page directly */
1148 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu
*vcpu
)
1151 * s390 will always inject the page directly,
1152 * but we still want check_async_completion to cleanup
1157 static int kvm_arch_setup_async_pf(struct kvm_vcpu
*vcpu
)
1160 struct kvm_arch_async_pf arch
;
1163 if (vcpu
->arch
.pfault_token
== KVM_S390_PFAULT_TOKEN_INVALID
)
1165 if ((vcpu
->arch
.sie_block
->gpsw
.mask
& vcpu
->arch
.pfault_select
) !=
1166 vcpu
->arch
.pfault_compare
)
1168 if (psw_extint_disabled(vcpu
))
1170 if (kvm_cpu_has_interrupt(vcpu
))
1172 if (!(vcpu
->arch
.sie_block
->gcr
[0] & 0x200ul
))
1174 if (!vcpu
->arch
.gmap
->pfault_enabled
)
1177 hva
= gfn_to_hva(vcpu
->kvm
, gpa_to_gfn(current
->thread
.gmap_addr
));
1178 hva
+= current
->thread
.gmap_addr
& ~PAGE_MASK
;
1179 if (read_guest_real(vcpu
, vcpu
->arch
.pfault_token
, &arch
.pfault_token
, 8))
1182 rc
= kvm_setup_async_pf(vcpu
, current
->thread
.gmap_addr
, hva
, &arch
);
1186 static int vcpu_pre_run(struct kvm_vcpu
*vcpu
)
1191 * On s390 notifications for arriving pages will be delivered directly
1192 * to the guest but the house keeping for completed pfaults is
1193 * handled outside the worker.
1195 kvm_check_async_pf_completion(vcpu
);
1197 memcpy(&vcpu
->arch
.sie_block
->gg14
, &vcpu
->run
->s
.regs
.gprs
[14], 16);
1202 if (test_cpu_flag(CIF_MCCK_PENDING
))
1205 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1206 rc
= kvm_s390_deliver_pending_interrupts(vcpu
);
1211 rc
= kvm_s390_handle_requests(vcpu
);
1215 if (guestdbg_enabled(vcpu
)) {
1216 kvm_s390_backup_guest_per_regs(vcpu
);
1217 kvm_s390_patch_guest_per_regs(vcpu
);
1220 vcpu
->arch
.sie_block
->icptcode
= 0;
1221 cpuflags
= atomic_read(&vcpu
->arch
.sie_block
->cpuflags
);
1222 VCPU_EVENT(vcpu
, 6, "entering sie flags %x", cpuflags
);
1223 trace_kvm_s390_sie_enter(vcpu
, cpuflags
);
1228 static int vcpu_post_run(struct kvm_vcpu
*vcpu
, int exit_reason
)
1232 VCPU_EVENT(vcpu
, 6, "exit sie icptcode %d",
1233 vcpu
->arch
.sie_block
->icptcode
);
1234 trace_kvm_s390_sie_exit(vcpu
, vcpu
->arch
.sie_block
->icptcode
);
1236 if (guestdbg_enabled(vcpu
))
1237 kvm_s390_restore_guest_per_regs(vcpu
);
1239 if (exit_reason
>= 0) {
1241 } else if (kvm_is_ucontrol(vcpu
->kvm
)) {
1242 vcpu
->run
->exit_reason
= KVM_EXIT_S390_UCONTROL
;
1243 vcpu
->run
->s390_ucontrol
.trans_exc_code
=
1244 current
->thread
.gmap_addr
;
1245 vcpu
->run
->s390_ucontrol
.pgm_code
= 0x10;
1248 } else if (current
->thread
.gmap_pfault
) {
1249 trace_kvm_s390_major_guest_pfault(vcpu
);
1250 current
->thread
.gmap_pfault
= 0;
1251 if (kvm_arch_setup_async_pf(vcpu
)) {
1254 gpa_t gpa
= current
->thread
.gmap_addr
;
1255 rc
= kvm_arch_fault_in_page(vcpu
, gpa
, 1);
1260 VCPU_EVENT(vcpu
, 3, "%s", "fault in sie instruction");
1261 trace_kvm_s390_sie_fault(vcpu
);
1262 rc
= kvm_s390_inject_program_int(vcpu
, PGM_ADDRESSING
);
1265 memcpy(&vcpu
->run
->s
.regs
.gprs
[14], &vcpu
->arch
.sie_block
->gg14
, 16);
1268 if (kvm_is_ucontrol(vcpu
->kvm
))
1269 /* Don't exit for host interrupts. */
1270 rc
= vcpu
->arch
.sie_block
->icptcode
? -EOPNOTSUPP
: 0;
1272 rc
= kvm_handle_sie_intercept(vcpu
);
1278 static int __vcpu_run(struct kvm_vcpu
*vcpu
)
1280 int rc
, exit_reason
;
1283 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1284 * ning the guest), so that memslots (and other stuff) are protected
1286 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1289 rc
= vcpu_pre_run(vcpu
);
1293 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
1295 * As PF_VCPU will be used in fault handler, between
1296 * guest_enter and guest_exit should be no uaccess.
1301 exit_reason
= sie64a(vcpu
->arch
.sie_block
,
1302 vcpu
->run
->s
.regs
.gprs
);
1304 vcpu
->srcu_idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1306 rc
= vcpu_post_run(vcpu
, exit_reason
);
1307 } while (!signal_pending(current
) && !guestdbg_exit_pending(vcpu
) && !rc
);
1309 srcu_read_unlock(&vcpu
->kvm
->srcu
, vcpu
->srcu_idx
);
1313 static void sync_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1315 vcpu
->arch
.sie_block
->gpsw
.mask
= kvm_run
->psw_mask
;
1316 vcpu
->arch
.sie_block
->gpsw
.addr
= kvm_run
->psw_addr
;
1317 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PREFIX
)
1318 kvm_s390_set_prefix(vcpu
, kvm_run
->s
.regs
.prefix
);
1319 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_CRS
) {
1320 memcpy(&vcpu
->arch
.sie_block
->gcr
, &kvm_run
->s
.regs
.crs
, 128);
1321 /* some control register changes require a tlb flush */
1322 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
1324 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_ARCH0
) {
1325 vcpu
->arch
.sie_block
->cputm
= kvm_run
->s
.regs
.cputm
;
1326 vcpu
->arch
.sie_block
->ckc
= kvm_run
->s
.regs
.ckc
;
1327 vcpu
->arch
.sie_block
->todpr
= kvm_run
->s
.regs
.todpr
;
1328 vcpu
->arch
.sie_block
->pp
= kvm_run
->s
.regs
.pp
;
1329 vcpu
->arch
.sie_block
->gbea
= kvm_run
->s
.regs
.gbea
;
1331 if (kvm_run
->kvm_dirty_regs
& KVM_SYNC_PFAULT
) {
1332 vcpu
->arch
.pfault_token
= kvm_run
->s
.regs
.pft
;
1333 vcpu
->arch
.pfault_select
= kvm_run
->s
.regs
.pfs
;
1334 vcpu
->arch
.pfault_compare
= kvm_run
->s
.regs
.pfc
;
1336 kvm_run
->kvm_dirty_regs
= 0;
1339 static void store_regs(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1341 kvm_run
->psw_mask
= vcpu
->arch
.sie_block
->gpsw
.mask
;
1342 kvm_run
->psw_addr
= vcpu
->arch
.sie_block
->gpsw
.addr
;
1343 kvm_run
->s
.regs
.prefix
= kvm_s390_get_prefix(vcpu
);
1344 memcpy(&kvm_run
->s
.regs
.crs
, &vcpu
->arch
.sie_block
->gcr
, 128);
1345 kvm_run
->s
.regs
.cputm
= vcpu
->arch
.sie_block
->cputm
;
1346 kvm_run
->s
.regs
.ckc
= vcpu
->arch
.sie_block
->ckc
;
1347 kvm_run
->s
.regs
.todpr
= vcpu
->arch
.sie_block
->todpr
;
1348 kvm_run
->s
.regs
.pp
= vcpu
->arch
.sie_block
->pp
;
1349 kvm_run
->s
.regs
.gbea
= vcpu
->arch
.sie_block
->gbea
;
1350 kvm_run
->s
.regs
.pft
= vcpu
->arch
.pfault_token
;
1351 kvm_run
->s
.regs
.pfs
= vcpu
->arch
.pfault_select
;
1352 kvm_run
->s
.regs
.pfc
= vcpu
->arch
.pfault_compare
;
1355 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
1360 if (guestdbg_exit_pending(vcpu
)) {
1361 kvm_s390_prepare_debug_exit(vcpu
);
1365 if (vcpu
->sigset_active
)
1366 sigprocmask(SIG_SETMASK
, &vcpu
->sigset
, &sigsaved
);
1368 if (!kvm_s390_user_cpu_state_ctrl(vcpu
->kvm
)) {
1369 kvm_s390_vcpu_start(vcpu
);
1370 } else if (is_vcpu_stopped(vcpu
)) {
1371 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1376 sync_regs(vcpu
, kvm_run
);
1379 rc
= __vcpu_run(vcpu
);
1381 if (signal_pending(current
) && !rc
) {
1382 kvm_run
->exit_reason
= KVM_EXIT_INTR
;
1386 if (guestdbg_exit_pending(vcpu
) && !rc
) {
1387 kvm_s390_prepare_debug_exit(vcpu
);
1391 if (rc
== -EOPNOTSUPP
) {
1392 /* intercept cannot be handled in-kernel, prepare kvm-run */
1393 kvm_run
->exit_reason
= KVM_EXIT_S390_SIEIC
;
1394 kvm_run
->s390_sieic
.icptcode
= vcpu
->arch
.sie_block
->icptcode
;
1395 kvm_run
->s390_sieic
.ipa
= vcpu
->arch
.sie_block
->ipa
;
1396 kvm_run
->s390_sieic
.ipb
= vcpu
->arch
.sie_block
->ipb
;
1400 if (rc
== -EREMOTE
) {
1401 /* intercept was handled, but userspace support is needed
1402 * kvm_run has been prepared by the handler */
1406 store_regs(vcpu
, kvm_run
);
1408 if (vcpu
->sigset_active
)
1409 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
1411 vcpu
->stat
.exit_userspace
++;
1416 * store status at address
1417 * we use have two special cases:
1418 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1419 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1421 int kvm_s390_store_status_unloaded(struct kvm_vcpu
*vcpu
, unsigned long gpa
)
1423 unsigned char archmode
= 1;
1428 if (gpa
== KVM_S390_STORE_STATUS_NOADDR
) {
1429 if (write_guest_abs(vcpu
, 163, &archmode
, 1))
1431 gpa
= SAVE_AREA_BASE
;
1432 } else if (gpa
== KVM_S390_STORE_STATUS_PREFIXED
) {
1433 if (write_guest_real(vcpu
, 163, &archmode
, 1))
1435 gpa
= kvm_s390_real_to_abs(vcpu
, SAVE_AREA_BASE
);
1437 rc
= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, fp_regs
),
1438 vcpu
->arch
.guest_fpregs
.fprs
, 128);
1439 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, gp_regs
),
1440 vcpu
->run
->s
.regs
.gprs
, 128);
1441 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, psw
),
1442 &vcpu
->arch
.sie_block
->gpsw
, 16);
1443 px
= kvm_s390_get_prefix(vcpu
);
1444 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, pref_reg
),
1446 rc
|= write_guest_abs(vcpu
,
1447 gpa
+ offsetof(struct save_area
, fp_ctrl_reg
),
1448 &vcpu
->arch
.guest_fpregs
.fpc
, 4);
1449 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, tod_reg
),
1450 &vcpu
->arch
.sie_block
->todpr
, 4);
1451 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, timer
),
1452 &vcpu
->arch
.sie_block
->cputm
, 8);
1453 clkcomp
= vcpu
->arch
.sie_block
->ckc
>> 8;
1454 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, clk_cmp
),
1456 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, acc_regs
),
1457 &vcpu
->run
->s
.regs
.acrs
, 64);
1458 rc
|= write_guest_abs(vcpu
, gpa
+ offsetof(struct save_area
, ctrl_regs
),
1459 &vcpu
->arch
.sie_block
->gcr
, 128);
1460 return rc
? -EFAULT
: 0;
1463 int kvm_s390_vcpu_store_status(struct kvm_vcpu
*vcpu
, unsigned long addr
)
1466 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1467 * copying in vcpu load/put. Lets update our copies before we save
1468 * it into the save area
1470 save_fp_ctl(&vcpu
->arch
.guest_fpregs
.fpc
);
1471 save_fp_regs(vcpu
->arch
.guest_fpregs
.fprs
);
1472 save_access_regs(vcpu
->run
->s
.regs
.acrs
);
1474 return kvm_s390_store_status_unloaded(vcpu
, addr
);
1477 static void __disable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
1479 kvm_check_request(KVM_REQ_ENABLE_IBS
, vcpu
);
1480 kvm_make_request(KVM_REQ_DISABLE_IBS
, vcpu
);
1481 exit_sie_sync(vcpu
);
1484 static void __disable_ibs_on_all_vcpus(struct kvm
*kvm
)
1487 struct kvm_vcpu
*vcpu
;
1489 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
1490 __disable_ibs_on_vcpu(vcpu
);
1494 static void __enable_ibs_on_vcpu(struct kvm_vcpu
*vcpu
)
1496 kvm_check_request(KVM_REQ_DISABLE_IBS
, vcpu
);
1497 kvm_make_request(KVM_REQ_ENABLE_IBS
, vcpu
);
1498 exit_sie_sync(vcpu
);
1501 void kvm_s390_vcpu_start(struct kvm_vcpu
*vcpu
)
1503 int i
, online_vcpus
, started_vcpus
= 0;
1505 if (!is_vcpu_stopped(vcpu
))
1508 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 1);
1509 /* Only one cpu at a time may enter/leave the STOPPED state. */
1510 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
1511 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
1513 for (i
= 0; i
< online_vcpus
; i
++) {
1514 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
]))
1518 if (started_vcpus
== 0) {
1519 /* we're the only active VCPU -> speed it up */
1520 __enable_ibs_on_vcpu(vcpu
);
1521 } else if (started_vcpus
== 1) {
1523 * As we are starting a second VCPU, we have to disable
1524 * the IBS facility on all VCPUs to remove potentially
1525 * oustanding ENABLE requests.
1527 __disable_ibs_on_all_vcpus(vcpu
->kvm
);
1530 atomic_clear_mask(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
1532 * Another VCPU might have used IBS while we were offline.
1533 * Let's play safe and flush the VCPU at startup.
1535 kvm_make_request(KVM_REQ_TLB_FLUSH
, vcpu
);
1536 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
1540 void kvm_s390_vcpu_stop(struct kvm_vcpu
*vcpu
)
1542 int i
, online_vcpus
, started_vcpus
= 0;
1543 struct kvm_vcpu
*started_vcpu
= NULL
;
1545 if (is_vcpu_stopped(vcpu
))
1548 trace_kvm_s390_vcpu_start_stop(vcpu
->vcpu_id
, 0);
1549 /* Only one cpu at a time may enter/leave the STOPPED state. */
1550 spin_lock(&vcpu
->kvm
->arch
.start_stop_lock
);
1551 online_vcpus
= atomic_read(&vcpu
->kvm
->online_vcpus
);
1553 /* Need to lock access to action_bits to avoid a SIGP race condition */
1554 spin_lock(&vcpu
->arch
.local_int
.lock
);
1555 atomic_set_mask(CPUSTAT_STOPPED
, &vcpu
->arch
.sie_block
->cpuflags
);
1557 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1558 vcpu
->arch
.local_int
.action_bits
&=
1559 ~(ACTION_STOP_ON_STOP
| ACTION_STORE_ON_STOP
);
1560 spin_unlock(&vcpu
->arch
.local_int
.lock
);
1562 __disable_ibs_on_vcpu(vcpu
);
1564 for (i
= 0; i
< online_vcpus
; i
++) {
1565 if (!is_vcpu_stopped(vcpu
->kvm
->vcpus
[i
])) {
1567 started_vcpu
= vcpu
->kvm
->vcpus
[i
];
1571 if (started_vcpus
== 1) {
1573 * As we only have one VCPU left, we want to enable the
1574 * IBS facility for that VCPU to speed it up.
1576 __enable_ibs_on_vcpu(started_vcpu
);
1579 spin_unlock(&vcpu
->kvm
->arch
.start_stop_lock
);
1583 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu
*vcpu
,
1584 struct kvm_enable_cap
*cap
)
1592 case KVM_CAP_S390_CSS_SUPPORT
:
1593 if (!vcpu
->kvm
->arch
.css_support
) {
1594 vcpu
->kvm
->arch
.css_support
= 1;
1595 trace_kvm_s390_enable_css(vcpu
->kvm
);
1606 long kvm_arch_vcpu_ioctl(struct file
*filp
,
1607 unsigned int ioctl
, unsigned long arg
)
1609 struct kvm_vcpu
*vcpu
= filp
->private_data
;
1610 void __user
*argp
= (void __user
*)arg
;
1615 case KVM_S390_INTERRUPT
: {
1616 struct kvm_s390_interrupt s390int
;
1619 if (copy_from_user(&s390int
, argp
, sizeof(s390int
)))
1621 r
= kvm_s390_inject_vcpu(vcpu
, &s390int
);
1624 case KVM_S390_STORE_STATUS
:
1625 idx
= srcu_read_lock(&vcpu
->kvm
->srcu
);
1626 r
= kvm_s390_vcpu_store_status(vcpu
, arg
);
1627 srcu_read_unlock(&vcpu
->kvm
->srcu
, idx
);
1629 case KVM_S390_SET_INITIAL_PSW
: {
1633 if (copy_from_user(&psw
, argp
, sizeof(psw
)))
1635 r
= kvm_arch_vcpu_ioctl_set_initial_psw(vcpu
, psw
);
1638 case KVM_S390_INITIAL_RESET
:
1639 r
= kvm_arch_vcpu_ioctl_initial_reset(vcpu
);
1641 case KVM_SET_ONE_REG
:
1642 case KVM_GET_ONE_REG
: {
1643 struct kvm_one_reg reg
;
1645 if (copy_from_user(®
, argp
, sizeof(reg
)))
1647 if (ioctl
== KVM_SET_ONE_REG
)
1648 r
= kvm_arch_vcpu_ioctl_set_one_reg(vcpu
, ®
);
1650 r
= kvm_arch_vcpu_ioctl_get_one_reg(vcpu
, ®
);
1653 #ifdef CONFIG_KVM_S390_UCONTROL
1654 case KVM_S390_UCAS_MAP
: {
1655 struct kvm_s390_ucas_mapping ucasmap
;
1657 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
1662 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1667 r
= gmap_map_segment(vcpu
->arch
.gmap
, ucasmap
.user_addr
,
1668 ucasmap
.vcpu_addr
, ucasmap
.length
);
1671 case KVM_S390_UCAS_UNMAP
: {
1672 struct kvm_s390_ucas_mapping ucasmap
;
1674 if (copy_from_user(&ucasmap
, argp
, sizeof(ucasmap
))) {
1679 if (!kvm_is_ucontrol(vcpu
->kvm
)) {
1684 r
= gmap_unmap_segment(vcpu
->arch
.gmap
, ucasmap
.vcpu_addr
,
1689 case KVM_S390_VCPU_FAULT
: {
1690 r
= gmap_fault(vcpu
->arch
.gmap
, arg
, 0);
1693 case KVM_ENABLE_CAP
:
1695 struct kvm_enable_cap cap
;
1697 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
1699 r
= kvm_vcpu_ioctl_enable_cap(vcpu
, &cap
);
1708 int kvm_arch_vcpu_fault(struct kvm_vcpu
*vcpu
, struct vm_fault
*vmf
)
1710 #ifdef CONFIG_KVM_S390_UCONTROL
1711 if ((vmf
->pgoff
== KVM_S390_SIE_PAGE_OFFSET
)
1712 && (kvm_is_ucontrol(vcpu
->kvm
))) {
1713 vmf
->page
= virt_to_page(vcpu
->arch
.sie_block
);
1714 get_page(vmf
->page
);
1718 return VM_FAULT_SIGBUS
;
1721 int kvm_arch_create_memslot(struct kvm
*kvm
, struct kvm_memory_slot
*slot
,
1722 unsigned long npages
)
1727 /* Section: memory related */
1728 int kvm_arch_prepare_memory_region(struct kvm
*kvm
,
1729 struct kvm_memory_slot
*memslot
,
1730 struct kvm_userspace_memory_region
*mem
,
1731 enum kvm_mr_change change
)
1733 /* A few sanity checks. We can have memory slots which have to be
1734 located/ended at a segment boundary (1MB). The memory in userland is
1735 ok to be fragmented into various different vmas. It is okay to mmap()
1736 and munmap() stuff in this slot after doing this call at any time */
1738 if (mem
->userspace_addr
& 0xffffful
)
1741 if (mem
->memory_size
& 0xffffful
)
1747 void kvm_arch_commit_memory_region(struct kvm
*kvm
,
1748 struct kvm_userspace_memory_region
*mem
,
1749 const struct kvm_memory_slot
*old
,
1750 enum kvm_mr_change change
)
1754 /* If the basics of the memslot do not change, we do not want
1755 * to update the gmap. Every update causes several unnecessary
1756 * segment translation exceptions. This is usually handled just
1757 * fine by the normal fault handler + gmap, but it will also
1758 * cause faults on the prefix page of running guest CPUs.
1760 if (old
->userspace_addr
== mem
->userspace_addr
&&
1761 old
->base_gfn
* PAGE_SIZE
== mem
->guest_phys_addr
&&
1762 old
->npages
* PAGE_SIZE
== mem
->memory_size
)
1765 rc
= gmap_map_segment(kvm
->arch
.gmap
, mem
->userspace_addr
,
1766 mem
->guest_phys_addr
, mem
->memory_size
);
1768 printk(KERN_WARNING
"kvm-s390: failed to commit memory region\n");
1772 static int __init
kvm_s390_init(void)
1775 ret
= kvm_init(NULL
, sizeof(struct kvm_vcpu
), 0, THIS_MODULE
);
1780 * guests can ask for up to 255+1 double words, we need a full page
1781 * to hold the maximum amount of facilities. On the other hand, we
1782 * only set facilities that are known to work in KVM.
1784 vfacilities
= (unsigned long *) get_zeroed_page(GFP_KERNEL
|GFP_DMA
);
1789 memcpy(vfacilities
, S390_lowcore
.stfle_fac_list
, 16);
1790 vfacilities
[0] &= 0xff82fffbf47c2000UL
;
1791 vfacilities
[1] &= 0x005c000000000000UL
;
1795 static void __exit
kvm_s390_exit(void)
1797 free_page((unsigned long) vfacilities
);
1801 module_init(kvm_s390_init
);
1802 module_exit(kvm_s390_exit
);
1805 * Enable autoloading of the kvm module.
1806 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1807 * since x86 takes a different approach.
1809 #include <linux/miscdevice.h>
1810 MODULE_ALIAS_MISCDEV(KVM_MINOR
);
1811 MODULE_ALIAS("devname:kvm");