projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/sage/ceph...
[deliverable/linux.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
cbb870c8 29#include <asm/asm-offsets.h>
b0c632db 30#include <asm/lowcore.h>
fdf03650 31#include <asm/etr.h>
b0c632db 32#include <asm/pgtable.h>
1e133ab2 33#include <asm/gmap.h>
f5daba1d 34#include <asm/nmi.h>
a0616cde 35#include <asm/switch_to.h>
6d3da241 36#include <asm/isc.h>
1526bf9c 37#include <asm/sclp.h>
8f2abe6a 38#include "kvm-s390.h"
b0c632db
HC		 39#include "gaccess.h"
40
ea2cdd27
DH		 41#define KMSG_COMPONENT "kvm-s390"
42#undef pr_fmt
43#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
44
5786fffa
CH		 45#define CREATE_TRACE_POINTS
46#include "trace.h"
ade38c31 47#include "trace-s390.h"
5786fffa 48
41408c28 49#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 50#define LOCAL_IRQS 32
51#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
52 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 53
b0c632db
HC		 54#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
55
56struct kvm_stats_debugfs_item debugfs_entries[] = {
57 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 58 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 59 { "exit_validity", VCPU_STAT(exit_validity) },
60 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
61 { "exit_external_request", VCPU_STAT(exit_external_request) },
62 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 63 { "exit_instruction", VCPU_STAT(exit_instruction) },
64 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
65 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f7819512 66 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 67 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 68 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 69 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 70 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 71 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 72 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
73 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 74 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 75 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 76 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
77 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
78 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
79 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
80 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
81 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
82 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 83 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 84 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
85 { "instruction_spx", VCPU_STAT(instruction_spx) },
86 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
87 { "instruction_stap", VCPU_STAT(instruction_stap) },
88 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 89 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 90 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
91 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 92 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 93 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
94 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 95 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 96 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 97 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 98 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 99 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 100 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
101 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 102 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 103 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
104 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 105 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 106 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
107 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
108 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 109 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
110 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
111 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 112 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 113 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 114 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 115 { "diagnose_258", VCPU_STAT(diagnose_258) },
116 { "diagnose_308", VCPU_STAT(diagnose_308) },
117 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 118 { NULL }
119};
120
9d8d5786 121/* upper facilities limit for kvm */
60a37709
AY		 122unsigned long kvm_s390_fac_list_mask[16] = {
123 0xffe6000000000000UL,
124 0x005e000000000000UL,
9d8d5786 125};
b0c632db 126
9d8d5786 127unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 128{
9d8d5786
MM		 129 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
130 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 131}
132
9d8d5786 133static struct gmap_notifier gmap_notifier;
78f26131 134debug_info_t *kvm_s390_dbf;
9d8d5786 135
b0c632db 136/* Section: not file related */
13a34e06 137int kvm_arch_hardware_enable(void)
b0c632db
HC		 138{
139 /* every s390 is virtualization enabled ;-) */
10474ae8 140 return 0;
b0c632db
HC		 141}
142
2c70fe44
CB		 143static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
144
fdf03650
FZ		 145/*
146 * This callback is executed during stop_machine(). All CPUs are therefore
147 * temporarily stopped. In order not to change guest behavior, we have to
148 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
149 * so a CPU won't be stopped while calculating with the epoch.
150 */
151static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
152 void *v)
153{
154 struct kvm *kvm;
155 struct kvm_vcpu *vcpu;
156 int i;
157 unsigned long long *delta = v;
158
159 list_for_each_entry(kvm, &vm_list, vm_list) {
160 kvm->arch.epoch -= *delta;
161 kvm_for_each_vcpu(i, vcpu, kvm) {
162 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 163 if (vcpu->arch.cputm_enabled)
164 vcpu->arch.cputm_start += *delta;
fdf03650
FZ		 165 }
166 }
167 return NOTIFY_OK;
168}
169
170static struct notifier_block kvm_clock_notifier = {
171 .notifier_call = kvm_clock_sync,
172};
173
b0c632db
HC		 174int kvm_arch_hardware_setup(void)
175{
2c70fe44
CB		 176 gmap_notifier.notifier_call = kvm_gmap_notifier;
177 gmap_register_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 178 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
179 &kvm_clock_notifier);
b0c632db
HC		 180 return 0;
181}
182
183void kvm_arch_hardware_unsetup(void)
184{
2c70fe44 185 gmap_unregister_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 186 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
187 &kvm_clock_notifier);
b0c632db
HC		 188}
189
b0c632db
HC		 190int kvm_arch_init(void *opaque)
191{
78f26131
CB		 192 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
193 if (!kvm_s390_dbf)
194 return -ENOMEM;
195
196 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
197 debug_unregister(kvm_s390_dbf);
198 return -ENOMEM;
199 }
200
84877d93
CH		 201 /* Register floating interrupt controller interface. */
202 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 203}
204
78f26131
CB		 205void kvm_arch_exit(void)
206{
207 debug_unregister(kvm_s390_dbf);
208}
209
b0c632db
HC		 210/* Section: device related */
211long kvm_arch_dev_ioctl(struct file *filp,
212 unsigned int ioctl, unsigned long arg)
213{
214 if (ioctl == KVM_S390_ENABLE_SIE)
215 return s390_enable_sie();
216 return -EINVAL;
217}
218
784aa3d7 219int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 220{
d7b0b5eb
CO		 221 int r;
222
2bd0ac4e 223 switch (ext) {
d7b0b5eb 224 case KVM_CAP_S390_PSW:
b6cf8788 225 case KVM_CAP_S390_GMAP:
52e16b18 226 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 227#ifdef CONFIG_KVM_S390_UCONTROL
228 case KVM_CAP_S390_UCONTROL:
229#endif
3c038e6b 230 case KVM_CAP_ASYNC_PF:
60b413c9 231 case KVM_CAP_SYNC_REGS:
14eebd91 232 case KVM_CAP_ONE_REG:
d6712df9 233 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 234 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 235 case KVM_CAP_IOEVENTFD:
c05c4186 236 case KVM_CAP_DEVICE_CTRL:
d938dc55 237 case KVM_CAP_ENABLE_CAP_VM:
78599d90 238 case KVM_CAP_S390_IRQCHIP:
f2061656 239 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 240 case KVM_CAP_MP_STATE:
47b43c52 241 case KVM_CAP_S390_INJECT_IRQ:
2444b352 242 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 243 case KVM_CAP_S390_USER_STSI:
30ee2a98 244 case KVM_CAP_S390_SKEYS:
816c7667 245 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 246 r = 1;
247 break;
41408c28
TH		 248 case KVM_CAP_S390_MEM_OP:
249 r = MEM_OP_MAX_SIZE;
250 break;
e726b1bd
CB		 251 case KVM_CAP_NR_VCPUS:
252 case KVM_CAP_MAX_VCPUS:
fe0edcb7
ED		 253 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
254 : KVM_S390_BSCA_CPU_SLOTS;
e726b1bd 255 break;
e1e2e605
NW		 256 case KVM_CAP_NR_MEMSLOTS:
257 r = KVM_USER_MEM_SLOTS;
258 break;
1526bf9c 259 case KVM_CAP_S390_COW:
abf09bed 260 r = MACHINE_HAS_ESOP;
1526bf9c 261 break;
68c55750
EF		 262 case KVM_CAP_S390_VECTOR_REGISTERS:
263 r = MACHINE_HAS_VX;
264 break;
c6e5f166
FZ		 265 case KVM_CAP_S390_RI:
266 r = test_facility(64);
267 break;
2bd0ac4e 268 default:
d7b0b5eb 269 r = 0;
2bd0ac4e 270 }
d7b0b5eb 271 return r;
b0c632db
HC		 272}
273
15f36ebd
JH		 274static void kvm_s390_sync_dirty_log(struct kvm *kvm,
275 struct kvm_memory_slot *memslot)
276{
277 gfn_t cur_gfn, last_gfn;
278 unsigned long address;
279 struct gmap *gmap = kvm->arch.gmap;
280
15f36ebd
JH		 281 /* Loop over all guest pages */
282 last_gfn = memslot->base_gfn + memslot->npages;
283 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
284 address = gfn_to_hva_memslot(memslot, cur_gfn);
285
1e133ab2 286 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 287 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 288 if (fatal_signal_pending(current))
289 return;
70c88a00 290 cond_resched();
15f36ebd 291 }
15f36ebd
JH		 292}
293
b0c632db 294/* Section: vm related */
a6e2f683
ED		 295static void sca_del_vcpu(struct kvm_vcpu *vcpu);
296
b0c632db
HC		 297/*
298 * Get (and clear) the dirty memory log for a memory slot.
299 */
300int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
301 struct kvm_dirty_log *log)
302{
15f36ebd
JH		 303 int r;
304 unsigned long n;
9f6b8029 305 struct kvm_memslots *slots;
15f36ebd
JH		 306 struct kvm_memory_slot *memslot;
307 int is_dirty = 0;
308
309 mutex_lock(&kvm->slots_lock);
310
311 r = -EINVAL;
312 if (log->slot >= KVM_USER_MEM_SLOTS)
313 goto out;
314
9f6b8029
PB		 315 slots = kvm_memslots(kvm);
316 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 317 r = -ENOENT;
318 if (!memslot->dirty_bitmap)
319 goto out;
320
321 kvm_s390_sync_dirty_log(kvm, memslot);
322 r = kvm_get_dirty_log(kvm, log, &is_dirty);
323 if (r)
324 goto out;
325
326 /* Clear the dirty log */
327 if (is_dirty) {
328 n = kvm_dirty_bitmap_bytes(memslot);
329 memset(memslot->dirty_bitmap, 0, n);
330 }
331 r = 0;
332out:
333 mutex_unlock(&kvm->slots_lock);
334 return r;
b0c632db
HC		 335}
336
d938dc55
CH		 337static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
338{
339 int r;
340
341 if (cap->flags)
342 return -EINVAL;
343
344 switch (cap->cap) {
84223598 345 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 346 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 347 kvm->arch.use_irqchip = 1;
348 r = 0;
349 break;
2444b352 350 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 351 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 352 kvm->arch.user_sigp = 1;
353 r = 0;
354 break;
68c55750 355 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b
DH		 356 mutex_lock(&kvm->lock);
357 if (atomic_read(&kvm->online_vcpus)) {
358 r = -EBUSY;
359 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 360 set_kvm_facility(kvm->arch.model.fac_mask, 129);
361 set_kvm_facility(kvm->arch.model.fac_list, 129);
18280d8b
MM		 362 r = 0;
363 } else
364 r = -EINVAL;
5967c17b 365 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 366 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
367 r ? "(not available)" : "(success)");
68c55750 368 break;
c6e5f166
FZ		 369 case KVM_CAP_S390_RI:
370 r = -EINVAL;
371 mutex_lock(&kvm->lock);
372 if (atomic_read(&kvm->online_vcpus)) {
373 r = -EBUSY;
374 } else if (test_facility(64)) {
c54f0d6a
DH		 375 set_kvm_facility(kvm->arch.model.fac_mask, 64);
376 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 377 r = 0;
378 }
379 mutex_unlock(&kvm->lock);
380 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
381 r ? "(not available)" : "(success)");
382 break;
e44fc8c9 383 case KVM_CAP_S390_USER_STSI:
c92ea7b9 384 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 385 kvm->arch.user_stsi = 1;
386 r = 0;
387 break;
d938dc55
CH		 388 default:
389 r = -EINVAL;
390 break;
391 }
392 return r;
393}
394
8c0a7ce6
DD		 395static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
396{
397 int ret;
398
399 switch (attr->attr) {
400 case KVM_S390_VM_MEM_LIMIT_SIZE:
401 ret = 0;
c92ea7b9 402 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 403 kvm->arch.mem_limit);
404 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 405 ret = -EFAULT;
406 break;
407 default:
408 ret = -ENXIO;
409 break;
410 }
411 return ret;
412}
413
414static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 415{
416 int ret;
417 unsigned int idx;
418 switch (attr->attr) {
419 case KVM_S390_VM_MEM_ENABLE_CMMA:
e6db1d61
DD		 420 /* enable CMMA only for z10 and later (EDAT_1) */
421 ret = -EINVAL;
422 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
423 break;
424
4f718eab 425 ret = -EBUSY;
c92ea7b9 426 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab
DD		 427 mutex_lock(&kvm->lock);
428 if (atomic_read(&kvm->online_vcpus) == 0) {
429 kvm->arch.use_cmma = 1;
430 ret = 0;
431 }
432 mutex_unlock(&kvm->lock);
433 break;
434 case KVM_S390_VM_MEM_CLR_CMMA:
c3489155
DD		 435 ret = -EINVAL;
436 if (!kvm->arch.use_cmma)
437 break;
438
c92ea7b9 439 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 440 mutex_lock(&kvm->lock);
441 idx = srcu_read_lock(&kvm->srcu);
a13cff31 442 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 443 srcu_read_unlock(&kvm->srcu, idx);
444 mutex_unlock(&kvm->lock);
445 ret = 0;
446 break;
8c0a7ce6
DD		 447 case KVM_S390_VM_MEM_LIMIT_SIZE: {
448 unsigned long new_limit;
449
450 if (kvm_is_ucontrol(kvm))
451 return -EINVAL;
452
453 if (get_user(new_limit, (u64 __user *)attr->addr))
454 return -EFAULT;
455
a3a92c31
DD		 456 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
457 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 458 return -E2BIG;
459
a3a92c31
DD		 460 if (!new_limit)
461 return -EINVAL;
462
463 /* gmap_alloc takes last usable address */
464 if (new_limit != KVM_S390_NO_MEM_LIMIT)
465 new_limit -= 1;
466
8c0a7ce6
DD		 467 ret = -EBUSY;
468 mutex_lock(&kvm->lock);
469 if (atomic_read(&kvm->online_vcpus) == 0) {
470 /* gmap_alloc will round the limit up */
471 struct gmap *new = gmap_alloc(current->mm, new_limit);
472
473 if (!new) {
474 ret = -ENOMEM;
475 } else {
476 gmap_free(kvm->arch.gmap);
477 new->private = kvm;
478 kvm->arch.gmap = new;
479 ret = 0;
480 }
481 }
482 mutex_unlock(&kvm->lock);
a3a92c31
DD		 483 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
484 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
485 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 486 break;
487 }
4f718eab
DD		 488 default:
489 ret = -ENXIO;
490 break;
491 }
492 return ret;
493}
494
a374e892
TK		 495static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
496
497static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
498{
499 struct kvm_vcpu *vcpu;
500 int i;
501
9d8d5786 502 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 503 return -EINVAL;
504
505 mutex_lock(&kvm->lock);
506 switch (attr->attr) {
507 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
508 get_random_bytes(
509 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
510 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
511 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 512 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 513 break;
514 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
515 get_random_bytes(
516 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
517 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
518 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 519 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 520 break;
521 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
522 kvm->arch.crypto.aes_kw = 0;
523 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
524 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 525 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 526 break;
527 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
528 kvm->arch.crypto.dea_kw = 0;
529 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
530 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 531 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 532 break;
533 default:
534 mutex_unlock(&kvm->lock);
535 return -ENXIO;
536 }
537
538 kvm_for_each_vcpu(i, vcpu, kvm) {
539 kvm_s390_vcpu_crypto_setup(vcpu);
540 exit_sie(vcpu);
541 }
542 mutex_unlock(&kvm->lock);
543 return 0;
544}
545
72f25020
JH		 546static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
547{
548 u8 gtod_high;
549
550 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
551 sizeof(gtod_high)))
552 return -EFAULT;
553
554 if (gtod_high != 0)
555 return -EINVAL;
58c383c6 556 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 557
558 return 0;
559}
560
561static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
562{
5a3d883a 563 u64 gtod;
72f25020
JH		 564
565 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
566 return -EFAULT;
567
25ed1675 568 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 569 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 570 return 0;
571}
572
573static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
574{
575 int ret;
576
577 if (attr->flags)
578 return -EINVAL;
579
580 switch (attr->attr) {
581 case KVM_S390_VM_TOD_HIGH:
582 ret = kvm_s390_set_tod_high(kvm, attr);
583 break;
584 case KVM_S390_VM_TOD_LOW:
585 ret = kvm_s390_set_tod_low(kvm, attr);
586 break;
587 default:
588 ret = -ENXIO;
589 break;
590 }
591 return ret;
592}
593
594static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
595{
596 u8 gtod_high = 0;
597
598 if (copy_to_user((void __user *)attr->addr, &gtod_high,
599 sizeof(gtod_high)))
600 return -EFAULT;
58c383c6 601 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 602
603 return 0;
604}
605
606static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
607{
5a3d883a 608 u64 gtod;
72f25020 609
60417fcc 610 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 611 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
612 return -EFAULT;
58c383c6 613 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 614
615 return 0;
616}
617
618static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
619{
620 int ret;
621
622 if (attr->flags)
623 return -EINVAL;
624
625 switch (attr->attr) {
626 case KVM_S390_VM_TOD_HIGH:
627 ret = kvm_s390_get_tod_high(kvm, attr);
628 break;
629 case KVM_S390_VM_TOD_LOW:
630 ret = kvm_s390_get_tod_low(kvm, attr);
631 break;
632 default:
633 ret = -ENXIO;
634 break;
635 }
636 return ret;
637}
638
658b6eda
MM		 639static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
640{
641 struct kvm_s390_vm_cpu_processor *proc;
053dd230 642 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 643 int ret = 0;
644
645 mutex_lock(&kvm->lock);
646 if (atomic_read(&kvm->online_vcpus)) {
647 ret = -EBUSY;
648 goto out;
649 }
650 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
651 if (!proc) {
652 ret = -ENOMEM;
653 goto out;
654 }
655 if (!copy_from_user(proc, (void __user *)attr->addr,
656 sizeof(*proc))) {
9bb0ec09 657 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 658 lowest_ibc = sclp.ibc >> 16 & 0xfff;
659 unblocked_ibc = sclp.ibc & 0xfff;
660 if (lowest_ibc) {
661 if (proc->ibc > unblocked_ibc)
662 kvm->arch.model.ibc = unblocked_ibc;
663 else if (proc->ibc < lowest_ibc)
664 kvm->arch.model.ibc = lowest_ibc;
665 else
666 kvm->arch.model.ibc = proc->ibc;
667 }
c54f0d6a 668 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 669 S390_ARCH_FAC_LIST_SIZE_BYTE);
670 } else
671 ret = -EFAULT;
672 kfree(proc);
673out:
674 mutex_unlock(&kvm->lock);
675 return ret;
676}
677
678static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
679{
680 int ret = -ENXIO;
681
682 switch (attr->attr) {
683 case KVM_S390_VM_CPU_PROCESSOR:
684 ret = kvm_s390_set_processor(kvm, attr);
685 break;
686 }
687 return ret;
688}
689
690static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
691{
692 struct kvm_s390_vm_cpu_processor *proc;
693 int ret = 0;
694
695 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
696 if (!proc) {
697 ret = -ENOMEM;
698 goto out;
699 }
9bb0ec09 700 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 701 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 702 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
703 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 704 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
705 ret = -EFAULT;
706 kfree(proc);
707out:
708 return ret;
709}
710
711static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
712{
713 struct kvm_s390_vm_cpu_machine *mach;
714 int ret = 0;
715
716 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
717 if (!mach) {
718 ret = -ENOMEM;
719 goto out;
720 }
721 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 722 mach->ibc = sclp.ibc;
c54f0d6a 723 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 724 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 725 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 726 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 727 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
728 ret = -EFAULT;
729 kfree(mach);
730out:
731 return ret;
732}
733
734static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
735{
736 int ret = -ENXIO;
737
738 switch (attr->attr) {
739 case KVM_S390_VM_CPU_PROCESSOR:
740 ret = kvm_s390_get_processor(kvm, attr);
741 break;
742 case KVM_S390_VM_CPU_MACHINE:
743 ret = kvm_s390_get_machine(kvm, attr);
744 break;
745 }
746 return ret;
747}
748
f2061656
DD		 749static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
750{
751 int ret;
752
753 switch (attr->group) {
4f718eab 754 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 755 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 756 break;
72f25020
JH		 757 case KVM_S390_VM_TOD:
758 ret = kvm_s390_set_tod(kvm, attr);
759 break;
658b6eda
MM		 760 case KVM_S390_VM_CPU_MODEL:
761 ret = kvm_s390_set_cpu_model(kvm, attr);
762 break;
a374e892
TK		 763 case KVM_S390_VM_CRYPTO:
764 ret = kvm_s390_vm_set_crypto(kvm, attr);
765 break;
f2061656
DD		 766 default:
767 ret = -ENXIO;
768 break;
769 }
770
771 return ret;
772}
773
774static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
775{
8c0a7ce6
DD		 776 int ret;
777
778 switch (attr->group) {
779 case KVM_S390_VM_MEM_CTRL:
780 ret = kvm_s390_get_mem_control(kvm, attr);
781 break;
72f25020
JH		 782 case KVM_S390_VM_TOD:
783 ret = kvm_s390_get_tod(kvm, attr);
784 break;
658b6eda
MM		 785 case KVM_S390_VM_CPU_MODEL:
786 ret = kvm_s390_get_cpu_model(kvm, attr);
787 break;
8c0a7ce6
DD		 788 default:
789 ret = -ENXIO;
790 break;
791 }
792
793 return ret;
f2061656
DD		 794}
795
796static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
797{
798 int ret;
799
800 switch (attr->group) {
4f718eab
DD		 801 case KVM_S390_VM_MEM_CTRL:
802 switch (attr->attr) {
803 case KVM_S390_VM_MEM_ENABLE_CMMA:
804 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 805 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 806 ret = 0;
807 break;
808 default:
809 ret = -ENXIO;
810 break;
811 }
812 break;
72f25020
JH		 813 case KVM_S390_VM_TOD:
814 switch (attr->attr) {
815 case KVM_S390_VM_TOD_LOW:
816 case KVM_S390_VM_TOD_HIGH:
817 ret = 0;
818 break;
819 default:
820 ret = -ENXIO;
821 break;
822 }
823 break;
658b6eda
MM		 824 case KVM_S390_VM_CPU_MODEL:
825 switch (attr->attr) {
826 case KVM_S390_VM_CPU_PROCESSOR:
827 case KVM_S390_VM_CPU_MACHINE:
828 ret = 0;
829 break;
830 default:
831 ret = -ENXIO;
832 break;
833 }
834 break;
a374e892
TK		 835 case KVM_S390_VM_CRYPTO:
836 switch (attr->attr) {
837 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
838 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
839 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
840 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
841 ret = 0;
842 break;
843 default:
844 ret = -ENXIO;
845 break;
846 }
847 break;
f2061656
DD		 848 default:
849 ret = -ENXIO;
850 break;
851 }
852
853 return ret;
854}
855
30ee2a98
JH		 856static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
857{
858 uint8_t *keys;
859 uint64_t hva;
860 unsigned long curkey;
861 int i, r = 0;
862
863 if (args->flags != 0)
864 return -EINVAL;
865
866 /* Is this guest using storage keys? */
867 if (!mm_use_skey(current->mm))
868 return KVM_S390_GET_SKEYS_NONE;
869
870 /* Enforce sane limit on memory allocation */
871 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
872 return -EINVAL;
873
874 keys = kmalloc_array(args->count, sizeof(uint8_t),
875 GFP_KERNEL | __GFP_NOWARN);
876 if (!keys)
877 keys = vmalloc(sizeof(uint8_t) * args->count);
878 if (!keys)
879 return -ENOMEM;
880
881 for (i = 0; i < args->count; i++) {
882 hva = gfn_to_hva(kvm, args->start_gfn + i);
883 if (kvm_is_error_hva(hva)) {
884 r = -EFAULT;
885 goto out;
886 }
887
888 curkey = get_guest_storage_key(current->mm, hva);
889 if (IS_ERR_VALUE(curkey)) {
890 r = curkey;
891 goto out;
892 }
893 keys[i] = curkey;
894 }
895
896 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
897 sizeof(uint8_t) * args->count);
898 if (r)
899 r = -EFAULT;
900out:
901 kvfree(keys);
902 return r;
903}
904
905static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
906{
907 uint8_t *keys;
908 uint64_t hva;
909 int i, r = 0;
910
911 if (args->flags != 0)
912 return -EINVAL;
913
914 /* Enforce sane limit on memory allocation */
915 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
916 return -EINVAL;
917
918 keys = kmalloc_array(args->count, sizeof(uint8_t),
919 GFP_KERNEL | __GFP_NOWARN);
920 if (!keys)
921 keys = vmalloc(sizeof(uint8_t) * args->count);
922 if (!keys)
923 return -ENOMEM;
924
925 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
926 sizeof(uint8_t) * args->count);
927 if (r) {
928 r = -EFAULT;
929 goto out;
930 }
931
932 /* Enable storage key handling for the guest */
14d4a425
DD		 933 r = s390_enable_skey();
934 if (r)
935 goto out;
30ee2a98
JH		 936
937 for (i = 0; i < args->count; i++) {
938 hva = gfn_to_hva(kvm, args->start_gfn + i);
939 if (kvm_is_error_hva(hva)) {
940 r = -EFAULT;
941 goto out;
942 }
943
944 /* Lowest order bit is reserved */
945 if (keys[i] & 0x01) {
946 r = -EINVAL;
947 goto out;
948 }
949
950 r = set_guest_storage_key(current->mm, hva,
951 (unsigned long)keys[i], 0);
952 if (r)
953 goto out;
954 }
955out:
956 kvfree(keys);
957 return r;
958}
959
b0c632db
HC		 960long kvm_arch_vm_ioctl(struct file *filp,
961 unsigned int ioctl, unsigned long arg)
962{
963 struct kvm *kvm = filp->private_data;
964 void __user *argp = (void __user *)arg;
f2061656 965 struct kvm_device_attr attr;
b0c632db
HC		 966 int r;
967
968 switch (ioctl) {
ba5c1e9b
CO		 969 case KVM_S390_INTERRUPT: {
970 struct kvm_s390_interrupt s390int;
971
972 r = -EFAULT;
973 if (copy_from_user(&s390int, argp, sizeof(s390int)))
974 break;
975 r = kvm_s390_inject_vm(kvm, &s390int);
976 break;
977 }
d938dc55
CH		 978 case KVM_ENABLE_CAP: {
979 struct kvm_enable_cap cap;
980 r = -EFAULT;
981 if (copy_from_user(&cap, argp, sizeof(cap)))
982 break;
983 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
984 break;
985 }
84223598
CH		 986 case KVM_CREATE_IRQCHIP: {
987 struct kvm_irq_routing_entry routing;
988
989 r = -EINVAL;
990 if (kvm->arch.use_irqchip) {
991 /* Set up dummy routing. */
992 memset(&routing, 0, sizeof(routing));
152b2839 993 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 994 }
995 break;
996 }
f2061656
DD		 997 case KVM_SET_DEVICE_ATTR: {
998 r = -EFAULT;
999 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1000 break;
1001 r = kvm_s390_vm_set_attr(kvm, &attr);
1002 break;
1003 }
1004 case KVM_GET_DEVICE_ATTR: {
1005 r = -EFAULT;
1006 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1007 break;
1008 r = kvm_s390_vm_get_attr(kvm, &attr);
1009 break;
1010 }
1011 case KVM_HAS_DEVICE_ATTR: {
1012 r = -EFAULT;
1013 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1014 break;
1015 r = kvm_s390_vm_has_attr(kvm, &attr);
1016 break;
1017 }
30ee2a98
JH		 1018 case KVM_S390_GET_SKEYS: {
1019 struct kvm_s390_skeys args;
1020
1021 r = -EFAULT;
1022 if (copy_from_user(&args, argp,
1023 sizeof(struct kvm_s390_skeys)))
1024 break;
1025 r = kvm_s390_get_skeys(kvm, &args);
1026 break;
1027 }
1028 case KVM_S390_SET_SKEYS: {
1029 struct kvm_s390_skeys args;
1030
1031 r = -EFAULT;
1032 if (copy_from_user(&args, argp,
1033 sizeof(struct kvm_s390_skeys)))
1034 break;
1035 r = kvm_s390_set_skeys(kvm, &args);
1036 break;
1037 }
b0c632db 1038 default:
367e1319 1039 r = -ENOTTY;
b0c632db
HC		 1040 }
1041
1042 return r;
1043}
1044
45c9b47c
TK		 1045static int kvm_s390_query_ap_config(u8 *config)
1046{
1047 u32 fcn_code = 0x04000000UL;
86044c8c 1048 u32 cc = 0;
45c9b47c 1049
86044c8c 1050 memset(config, 0, 128);
45c9b47c
TK		 1051 asm volatile(
1052 "lgr 0,%1\n"
1053 "lgr 2,%2\n"
1054 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1055 "0: ipm %0\n"
45c9b47c 1056 "srl %0,28\n"
86044c8c
CB		 1057 "1:\n"
1058 EX_TABLE(0b, 1b)
1059 : "+r" (cc)
45c9b47c
TK		 1060 : "r" (fcn_code), "r" (config)
1061 : "cc", "0", "2", "memory"
1062 );
1063
1064 return cc;
1065}
1066
1067static int kvm_s390_apxa_installed(void)
1068{
1069 u8 config[128];
1070 int cc;
1071
a6aacc3f 1072 if (test_facility(12)) {
45c9b47c
TK		 1073 cc = kvm_s390_query_ap_config(config);
1074
1075 if (cc)
1076 pr_err("PQAP(QCI) failed with cc=%d", cc);
1077 else
1078 return config[0] & 0x40;
1079 }
1080
1081 return 0;
1082}
1083
1084static void kvm_s390_set_crycb_format(struct kvm *kvm)
1085{
1086 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1087
1088 if (kvm_s390_apxa_installed())
1089 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1090 else
1091 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1092}
1093
9bb0ec09 1094static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1095{
9bb0ec09
DH		 1096 struct cpuid cpuid;
1097
1098 get_cpu_id(&cpuid);
1099 cpuid.version = 0xff;
1100 return *((u64 *) &cpuid);
9d8d5786
MM		 1101}
1102
c54f0d6a 1103static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1104{
9d8d5786 1105 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1106 return;
5102ee87 1107
c54f0d6a 1108 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1109 kvm_s390_set_crycb_format(kvm);
5102ee87 1110
ed6f76b4
TK		 1111 /* Enable AES/DEA protected key functions by default */
1112 kvm->arch.crypto.aes_kw = 1;
1113 kvm->arch.crypto.dea_kw = 1;
1114 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1115 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1116 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1117 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1118}
1119
7d43bafc
ED		 1120static void sca_dispose(struct kvm *kvm)
1121{
1122 if (kvm->arch.use_esca)
5e044315 1123 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1124 else
1125 free_page((unsigned long)(kvm->arch.sca));
1126 kvm->arch.sca = NULL;
1127}
1128
e08b9637 1129int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1130{
9d8d5786 1131 int i, rc;
b0c632db 1132 char debug_name[16];
f6c137ff 1133 static unsigned long sca_offset;
b0c632db 1134
e08b9637
CO		 1135 rc = -EINVAL;
1136#ifdef CONFIG_KVM_S390_UCONTROL
1137 if (type & ~KVM_VM_S390_UCONTROL)
1138 goto out_err;
1139 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1140 goto out_err;
1141#else
1142 if (type)
1143 goto out_err;
1144#endif
1145
b0c632db
HC		 1146 rc = s390_enable_sie();
1147 if (rc)
d89f5eff 1148 goto out_err;
b0c632db 1149
b290411a
CO		 1150 rc = -ENOMEM;
1151
7d43bafc 1152 kvm->arch.use_esca = 0; /* start with basic SCA */
5e044315 1153 rwlock_init(&kvm->arch.sca_lock);
bc784cce 1154 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
b0c632db 1155 if (!kvm->arch.sca)
d89f5eff 1156 goto out_err;
f6c137ff 1157 spin_lock(&kvm_lock);
c5c2c393 1158 sca_offset += 16;
bc784cce 1159 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1160 sca_offset = 0;
bc784cce
ED		 1161 kvm->arch.sca = (struct bsca_block *)
1162 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1163 spin_unlock(&kvm_lock);
b0c632db
HC		 1164
1165 sprintf(debug_name, "kvm-%u", current->pid);
1166
1cb9cf72 1167 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1168 if (!kvm->arch.dbf)
40f5b735 1169 goto out_err;
b0c632db 1170
c54f0d6a
DH		 1171 kvm->arch.sie_page2 =
1172 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1173 if (!kvm->arch.sie_page2)
40f5b735 1174 goto out_err;
9d8d5786 1175
fb5bf93f 1176 /* Populate the facility mask initially. */
c54f0d6a 1177 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
94422ee8 1178 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1179 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1180 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1181 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1182 else
c54f0d6a 1183 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1184 }
1185
981467c9 1186 /* Populate the facility list initially. */
c54f0d6a
DH		 1187 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1188 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1189 S390_ARCH_FAC_LIST_SIZE_BYTE);
1190
9bb0ec09 1191 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1192 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1193
c54f0d6a 1194 kvm_s390_crypto_init(kvm);
5102ee87 1195
ba5c1e9b 1196 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1197 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1198 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1199 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1200 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1201
b0c632db 1202 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1203 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1204
e08b9637
CO		 1205 if (type & KVM_VM_S390_UCONTROL) {
1206 kvm->arch.gmap = NULL;
a3a92c31 1207 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1208 } else {
32e6b236
GH		 1209 if (sclp.hamax == U64_MAX)
1210 kvm->arch.mem_limit = TASK_MAX_SIZE;
1211 else
1212 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1213 sclp.hamax + 1);
a3a92c31 1214 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1215 if (!kvm->arch.gmap)
40f5b735 1216 goto out_err;
2c70fe44 1217 kvm->arch.gmap->private = kvm;
24eb3a82 1218 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1219 }
fa6b7fe9
CH		 1220
1221 kvm->arch.css_support = 0;
84223598 1222 kvm->arch.use_irqchip = 0;
72f25020 1223 kvm->arch.epoch = 0;
fa6b7fe9 1224
8ad35755 1225 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1226 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1227
d89f5eff 1228 return 0;
40f5b735 1229out_err:
c54f0d6a 1230 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1231 debug_unregister(kvm->arch.dbf);
7d43bafc 1232 sca_dispose(kvm);
78f26131 1233 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1234 return rc;
b0c632db
HC		 1235}
1236
d329c035
CB		 1237void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1238{
1239 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1240 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1241 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1242 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1243 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1244 sca_del_vcpu(vcpu);
27e0393f
CO		 1245
1246 if (kvm_is_ucontrol(vcpu->kvm))
1247 gmap_free(vcpu->arch.gmap);
1248
e6db1d61 1249 if (vcpu->kvm->arch.use_cmma)
b31605c1 1250 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1251 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1252
6692cef3 1253 kvm_vcpu_uninit(vcpu);
b110feaf 1254 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1255}
1256
1257static void kvm_free_vcpus(struct kvm *kvm)
1258{
1259 unsigned int i;
988a2cae 1260 struct kvm_vcpu *vcpu;
d329c035 1261
988a2cae
GN		 1262 kvm_for_each_vcpu(i, vcpu, kvm)
1263 kvm_arch_vcpu_destroy(vcpu);
1264
1265 mutex_lock(&kvm->lock);
1266 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1267 kvm->vcpus[i] = NULL;
1268
1269 atomic_set(&kvm->online_vcpus, 0);
1270 mutex_unlock(&kvm->lock);
d329c035
CB		 1271}
1272
b0c632db
HC		 1273void kvm_arch_destroy_vm(struct kvm *kvm)
1274{
d329c035 1275 kvm_free_vcpus(kvm);
7d43bafc 1276 sca_dispose(kvm);
d329c035 1277 debug_unregister(kvm->arch.dbf);
c54f0d6a 1278 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f
CO		 1279 if (!kvm_is_ucontrol(kvm))
1280 gmap_free(kvm->arch.gmap);
841b91c5 1281 kvm_s390_destroy_adapters(kvm);
67335e63 1282 kvm_s390_clear_float_irqs(kvm);
8335713a 1283 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1284}
1285
1286/* Section: vcpu related */
dafd032a
DD		 1287static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1288{
1289 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1290 if (!vcpu->arch.gmap)
1291 return -ENOMEM;
1292 vcpu->arch.gmap->private = vcpu->kvm;
1293
1294 return 0;
1295}
1296
a6e2f683
ED		 1297static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1298{
5e044315 1299 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1300 if (vcpu->kvm->arch.use_esca) {
1301 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1302
7d43bafc 1303 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1304 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1305 } else {
1306 struct bsca_block *sca = vcpu->kvm->arch.sca;
1307
1308 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1309 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1310 }
5e044315 1311 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1312}
1313
eaa78f34 1314static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1315{
eaa78f34
DH		 1316 read_lock(&vcpu->kvm->arch.sca_lock);
1317 if (vcpu->kvm->arch.use_esca) {
1318 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1319
eaa78f34 1320 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1321 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1322 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1323 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1324 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1325 } else {
eaa78f34 1326 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1327
eaa78f34 1328 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1329 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1330 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1331 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1332 }
eaa78f34 1333 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1334}
1335
1336/* Basic SCA to Extended SCA data copy routines */
1337static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1338{
1339 d->sda = s->sda;
1340 d->sigp_ctrl.c = s->sigp_ctrl.c;
1341 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1342}
1343
1344static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1345{
1346 int i;
1347
1348 d->ipte_control = s->ipte_control;
1349 d->mcn[0] = s->mcn;
1350 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1351 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1352}
1353
1354static int sca_switch_to_extended(struct kvm *kvm)
1355{
1356 struct bsca_block *old_sca = kvm->arch.sca;
1357 struct esca_block *new_sca;
1358 struct kvm_vcpu *vcpu;
1359 unsigned int vcpu_idx;
1360 u32 scaol, scaoh;
1361
1362 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1363 if (!new_sca)
1364 return -ENOMEM;
1365
1366 scaoh = (u32)((u64)(new_sca) >> 32);
1367 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1368
1369 kvm_s390_vcpu_block_all(kvm);
1370 write_lock(&kvm->arch.sca_lock);
1371
1372 sca_copy_b_to_e(new_sca, old_sca);
1373
1374 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1375 vcpu->arch.sie_block->scaoh = scaoh;
1376 vcpu->arch.sie_block->scaol = scaol;
1377 vcpu->arch.sie_block->ecb2 |= 0x04U;
1378 }
1379 kvm->arch.sca = new_sca;
1380 kvm->arch.use_esca = 1;
1381
1382 write_unlock(&kvm->arch.sca_lock);
1383 kvm_s390_vcpu_unblock_all(kvm);
1384
1385 free_page((unsigned long)old_sca);
1386
8335713a
CB		 1387 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1388 old_sca, kvm->arch.sca);
5e044315 1389 return 0;
a6e2f683
ED		 1390}
1391
1392static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1393{
5e044315
ED		 1394 int rc;
1395
1396 if (id < KVM_S390_BSCA_CPU_SLOTS)
1397 return true;
1398 if (!sclp.has_esca)
1399 return false;
1400
1401 mutex_lock(&kvm->lock);
1402 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1403 mutex_unlock(&kvm->lock);
1404
1405 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1406}
1407
b0c632db
HC		 1408int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1409{
3c038e6b
DD		 1410 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1411 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1412 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1413 KVM_SYNC_GPRS |
9eed0735 1414 KVM_SYNC_ACRS |
b028ee3e
DH		 1415 KVM_SYNC_CRS |
1416 KVM_SYNC_ARCH0 |
1417 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1418 if (test_kvm_facility(vcpu->kvm, 64))
1419 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1420 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1421 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1422 */
1423 if (MACHINE_HAS_VX)
68c55750 1424 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1425 else
1426 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1427
1428 if (kvm_is_ucontrol(vcpu->kvm))
1429 return __kvm_ucontrol_vcpu_init(vcpu);
1430
b0c632db
HC		 1431 return 0;
1432}
1433
db0758b2
DH		 1434/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1435static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1436{
1437 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1438 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1439 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1440 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1441}
1442
1443/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1444static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1445{
1446 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1447 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1448 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1449 vcpu->arch.cputm_start = 0;
9c23a131 1450 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1451}
1452
1453/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1454static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1455{
1456 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1457 vcpu->arch.cputm_enabled = true;
1458 __start_cpu_timer_accounting(vcpu);
1459}
1460
1461/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1462static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1463{
1464 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1465 __stop_cpu_timer_accounting(vcpu);
1466 vcpu->arch.cputm_enabled = false;
1467}
1468
1469static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1470{
1471 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1472 __enable_cpu_timer_accounting(vcpu);
1473 preempt_enable();
1474}
1475
1476static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1477{
1478 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1479 __disable_cpu_timer_accounting(vcpu);
1480 preempt_enable();
1481}
1482
4287f247
DH		 1483/* set the cpu timer - may only be called from the VCPU thread itself */
1484void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1485{
db0758b2 1486 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1487 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1488 if (vcpu->arch.cputm_enabled)
1489 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1490 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1491 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1492 preempt_enable();
4287f247
DH		 1493}
1494
db0758b2 1495/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1496__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1497{
9c23a131 1498 unsigned int seq;
db0758b2 1499 __u64 value;
db0758b2
DH		 1500
1501 if (unlikely(!vcpu->arch.cputm_enabled))
1502 return vcpu->arch.sie_block->cputm;
1503
9c23a131
DH		 1504 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1505 do {
1506 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1507 /*
1508 * If the writer would ever execute a read in the critical
1509 * section, e.g. in irq context, we have a deadlock.
1510 */
1511 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1512 value = vcpu->arch.sie_block->cputm;
1513 /* if cputm_start is 0, accounting is being started/stopped */
1514 if (likely(vcpu->arch.cputm_start))
1515 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1516 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1517 preempt_enable();
db0758b2 1518 return value;
4287f247
DH		 1519}
1520
b0c632db
HC		 1521void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1522{
9977e886 1523 /* Save host register state */
d0164ee2 1524 save_fpu_regs();
9abc2a08
DH		 1525 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1526 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1527
6fd8e67d
DH		 1528 if (MACHINE_HAS_VX)
1529 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1530 else
1531 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1532 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1533 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1534 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1535 current->thread.fpu.fpc = 0;
1536
1537 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1538 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1539 gmap_enable(vcpu->arch.gmap);
805de8f4 1540 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1541 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1542 __start_cpu_timer_accounting(vcpu);
01a745ac 1543 vcpu->cpu = cpu;
b0c632db
HC		 1544}
1545
1546void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1547{
01a745ac 1548 vcpu->cpu = -1;
5ebda316 1549 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1550 __stop_cpu_timer_accounting(vcpu);
805de8f4 1551 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1552 gmap_disable(vcpu->arch.gmap);
9977e886 1553
9abc2a08 1554 /* Save guest register state */
d0164ee2 1555 save_fpu_regs();
9abc2a08 1556 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1557
9abc2a08
DH		 1558 /* Restore host register state */
1559 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1560 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1561
1562 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1563 restore_access_regs(vcpu->arch.host_acrs);
1564}
1565
1566static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1567{
1568 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1569 vcpu->arch.sie_block->gpsw.mask = 0UL;
1570 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1571 kvm_s390_set_prefix(vcpu, 0);
4287f247 1572 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1573 vcpu->arch.sie_block->ckc = 0UL;
1574 vcpu->arch.sie_block->todpr = 0;
1575 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1576 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1577 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1578 /* make sure the new fpc will be lazily loaded */
1579 save_fpu_regs();
1580 current->thread.fpu.fpc = 0;
b0c632db 1581 vcpu->arch.sie_block->gbea = 1;
672550fb 1582 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1583 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1584 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1585 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1586 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1587 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1588}
1589
31928aa5 1590void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1591{
72f25020 1592 mutex_lock(&vcpu->kvm->lock);
fdf03650 1593 preempt_disable();
72f25020 1594 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1595 preempt_enable();
72f25020 1596 mutex_unlock(&vcpu->kvm->lock);
25508824 1597 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1598 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1599 sca_add_vcpu(vcpu);
25508824
DH		 1600 }
1601
42897d86
MT		 1602}
1603
5102ee87
TK		 1604static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1605{
9d8d5786 1606 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1607 return;
1608
a374e892
TK		 1609 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1610
1611 if (vcpu->kvm->arch.crypto.aes_kw)
1612 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1613 if (vcpu->kvm->arch.crypto.dea_kw)
1614 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1615
5102ee87
TK		 1616 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1617}
1618
b31605c1
DD		 1619void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1620{
1621 free_page(vcpu->arch.sie_block->cbrlo);
1622 vcpu->arch.sie_block->cbrlo = 0;
1623}
1624
1625int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1626{
1627 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1628 if (!vcpu->arch.sie_block->cbrlo)
1629 return -ENOMEM;
1630
1631 vcpu->arch.sie_block->ecb2 |= 0x80;
1632 vcpu->arch.sie_block->ecb2 &= ~0x08;
1633 return 0;
1634}
1635
91520f1a
MM		 1636static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1637{
1638 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1639
91520f1a 1640 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1641 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1642 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1643}
1644
b0c632db
HC		 1645int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1646{
b31605c1 1647 int rc = 0;
b31288fa 1648
9e6dabef
CH		 1649 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1650 CPUSTAT_SM |
a4a4f191
GH		 1651 CPUSTAT_STOPPED);
1652
53df84f8 1653 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1654 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1655 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1656 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1657
91520f1a
MM		 1658 kvm_s390_vcpu_setup_model(vcpu);
1659
bd50e8ec
DH		 1660 vcpu->arch.sie_block->ecb = 0x02;
1661 if (test_kvm_facility(vcpu->kvm, 9))
1662 vcpu->arch.sie_block->ecb |= 0x04;
9d8d5786 1663 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1664 vcpu->arch.sie_block->ecb |= 0x10;
1665
d6af0b49
DH		 1666 if (test_kvm_facility(vcpu->kvm, 8))
1667 vcpu->arch.sie_block->ecb2 |= 0x08;
ea5f4969 1668 vcpu->arch.sie_block->eca = 0xC1002000U;
37c5f6c8 1669 if (sclp.has_siif)
217a4406 1670 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1671 if (sclp.has_sigpif)
ea5f4969 1672 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1673 if (test_kvm_facility(vcpu->kvm, 64))
1674 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1675 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1676 vcpu->arch.sie_block->eca |= 0x00020000;
1677 vcpu->arch.sie_block->ecd |= 0x20000000;
1678 }
c6e5f166 1679 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1680 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1681
e6db1d61 1682 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1683 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1684 if (rc)
1685 return rc;
b31288fa 1686 }
0ac96caf 1687 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1688 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1689
5102ee87
TK		 1690 kvm_s390_vcpu_crypto_setup(vcpu);
1691
b31605c1 1692 return rc;
b0c632db
HC		 1693}
1694
1695struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1696 unsigned int id)
1697{
4d47555a 1698 struct kvm_vcpu *vcpu;
7feb6bb8 1699 struct sie_page *sie_page;
4d47555a
CO		 1700 int rc = -EINVAL;
1701
4215825e 1702 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1703 goto out;
1704
1705 rc = -ENOMEM;
b0c632db 1706
b110feaf 1707 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1708 if (!vcpu)
4d47555a 1709 goto out;
b0c632db 1710
7feb6bb8
MM		 1711 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1712 if (!sie_page)
b0c632db
HC		 1713 goto out_free_cpu;
1714
7feb6bb8
MM		 1715 vcpu->arch.sie_block = &sie_page->sie_block;
1716 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1717
b0c632db 1718 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1719 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1720 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1721 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1722 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 1723 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 1724
b0c632db
HC		 1725 rc = kvm_vcpu_init(vcpu, kvm, id);
1726 if (rc)
9abc2a08 1727 goto out_free_sie_block;
8335713a 1728 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1729 vcpu->arch.sie_block);
ade38c31 1730 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1731
b0c632db 1732 return vcpu;
7b06bf2f
WY		 1733out_free_sie_block:
1734 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1735out_free_cpu:
b110feaf 1736 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1737out:
b0c632db
HC		 1738 return ERR_PTR(rc);
1739}
1740
b0c632db
HC		 1741int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1742{
9a022067 1743 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1744}
1745
27406cd5 1746void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1747{
805de8f4 1748 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1749 exit_sie(vcpu);
49b99e1e
CB		 1750}
1751
27406cd5 1752void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1753{
805de8f4 1754 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1755}
1756
8e236546
CB		 1757static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1758{
805de8f4 1759 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1760 exit_sie(vcpu);
8e236546
CB		 1761}
1762
1763static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1764{
9bf9fde2 1765 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1766}
1767
49b99e1e
CB		 1768/*
1769 * Kick a guest cpu out of SIE and wait until SIE is not running.
1770 * If the CPU is not running (e.g. waiting as idle) the function will
1771 * return immediately. */
1772void exit_sie(struct kvm_vcpu *vcpu)
1773{
805de8f4 1774 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1775 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1776 cpu_relax();
1777}
1778
8e236546
CB		 1779/* Kick a guest cpu out of SIE to process a request synchronously */
1780void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1781{
8e236546
CB		 1782 kvm_make_request(req, vcpu);
1783 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1784}
1785
2c70fe44
CB		 1786static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1787{
1788 int i;
1789 struct kvm *kvm = gmap->private;
1790 struct kvm_vcpu *vcpu;
1791
1792 kvm_for_each_vcpu(i, vcpu, kvm) {
1793 /* match against both prefix pages */
fda902cb 1794 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44 1795 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
8e236546 1796 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1797 }
1798 }
1799}
1800
b6d33834
CD		 1801int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1802{
1803 /* kvm common code refers to this, but never calls it */
1804 BUG();
1805 return 0;
1806}
1807
14eebd91
CO		 1808static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1809 struct kvm_one_reg *reg)
1810{
1811 int r = -EINVAL;
1812
1813 switch (reg->id) {
29b7c71b
CO		 1814 case KVM_REG_S390_TODPR:
1815 r = put_user(vcpu->arch.sie_block->todpr,
1816 (u32 __user *)reg->addr);
1817 break;
1818 case KVM_REG_S390_EPOCHDIFF:
1819 r = put_user(vcpu->arch.sie_block->epoch,
1820 (u64 __user *)reg->addr);
1821 break;
46a6dd1c 1822 case KVM_REG_S390_CPU_TIMER:
4287f247 1823 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 1824 (u64 __user *)reg->addr);
1825 break;
1826 case KVM_REG_S390_CLOCK_COMP:
1827 r = put_user(vcpu->arch.sie_block->ckc,
1828 (u64 __user *)reg->addr);
1829 break;
536336c2
DD		 1830 case KVM_REG_S390_PFTOKEN:
1831 r = put_user(vcpu->arch.pfault_token,
1832 (u64 __user *)reg->addr);
1833 break;
1834 case KVM_REG_S390_PFCOMPARE:
1835 r = put_user(vcpu->arch.pfault_compare,
1836 (u64 __user *)reg->addr);
1837 break;
1838 case KVM_REG_S390_PFSELECT:
1839 r = put_user(vcpu->arch.pfault_select,
1840 (u64 __user *)reg->addr);
1841 break;
672550fb
CB		 1842 case KVM_REG_S390_PP:
1843 r = put_user(vcpu->arch.sie_block->pp,
1844 (u64 __user *)reg->addr);
1845 break;
afa45ff5
CB		 1846 case KVM_REG_S390_GBEA:
1847 r = put_user(vcpu->arch.sie_block->gbea,
1848 (u64 __user *)reg->addr);
1849 break;
14eebd91
CO		 1850 default:
1851 break;
1852 }
1853
1854 return r;
1855}
1856
1857static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1858 struct kvm_one_reg *reg)
1859{
1860 int r = -EINVAL;
4287f247 1861 __u64 val;
14eebd91
CO		 1862
1863 switch (reg->id) {
29b7c71b
CO		 1864 case KVM_REG_S390_TODPR:
1865 r = get_user(vcpu->arch.sie_block->todpr,
1866 (u32 __user *)reg->addr);
1867 break;
1868 case KVM_REG_S390_EPOCHDIFF:
1869 r = get_user(vcpu->arch.sie_block->epoch,
1870 (u64 __user *)reg->addr);
1871 break;
46a6dd1c 1872 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 1873 r = get_user(val, (u64 __user *)reg->addr);
1874 if (!r)
1875 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 1876 break;
1877 case KVM_REG_S390_CLOCK_COMP:
1878 r = get_user(vcpu->arch.sie_block->ckc,
1879 (u64 __user *)reg->addr);
1880 break;
536336c2
DD		 1881 case KVM_REG_S390_PFTOKEN:
1882 r = get_user(vcpu->arch.pfault_token,
1883 (u64 __user *)reg->addr);
9fbd8082
DH		 1884 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1885 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 1886 break;
1887 case KVM_REG_S390_PFCOMPARE:
1888 r = get_user(vcpu->arch.pfault_compare,
1889 (u64 __user *)reg->addr);
1890 break;
1891 case KVM_REG_S390_PFSELECT:
1892 r = get_user(vcpu->arch.pfault_select,
1893 (u64 __user *)reg->addr);
1894 break;
672550fb
CB		 1895 case KVM_REG_S390_PP:
1896 r = get_user(vcpu->arch.sie_block->pp,
1897 (u64 __user *)reg->addr);
1898 break;
afa45ff5
CB		 1899 case KVM_REG_S390_GBEA:
1900 r = get_user(vcpu->arch.sie_block->gbea,
1901 (u64 __user *)reg->addr);
1902 break;
14eebd91
CO		 1903 default:
1904 break;
1905 }
1906
1907 return r;
1908}
b6d33834 1909
b0c632db
HC		 1910static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1911{
b0c632db 1912 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 1913 return 0;
1914}
1915
1916int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1917{
5a32c1af 1918 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 1919 return 0;
1920}
1921
1922int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1923{
5a32c1af 1924 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 1925 return 0;
1926}
1927
1928int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1929 struct kvm_sregs *sregs)
1930{
59674c1a 1931 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 1932 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 1933 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1934 return 0;
1935}
1936
1937int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1938 struct kvm_sregs *sregs)
1939{
59674c1a 1940 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1941 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1942 return 0;
1943}
1944
1945int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1946{
9abc2a08
DH		 1947 /* make sure the new values will be lazily loaded */
1948 save_fpu_regs();
4725c860
MS		 1949 if (test_fp_ctl(fpu->fpc))
1950 return -EINVAL;
9abc2a08
DH		 1951 current->thread.fpu.fpc = fpu->fpc;
1952 if (MACHINE_HAS_VX)
1953 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
1954 else
1955 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 1956 return 0;
1957}
1958
1959int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1960{
9abc2a08
DH		 1961 /* make sure we have the latest values */
1962 save_fpu_regs();
1963 if (MACHINE_HAS_VX)
1964 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
1965 else
1966 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
1967 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 1968 return 0;
1969}
1970
1971static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1972{
1973 int rc = 0;
1974
7a42fdc2 1975 if (!is_vcpu_stopped(vcpu))
b0c632db 1976 rc = -EBUSY;
d7b0b5eb
CO		 1977 else {
1978 vcpu->run->psw_mask = psw.mask;
1979 vcpu->run->psw_addr = psw.addr;
1980 }
b0c632db
HC		 1981 return rc;
1982}
1983
1984int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1985 struct kvm_translation *tr)
1986{
1987 return -EINVAL; /* not implemented yet */
1988}
1989
27291e21
DH		 1990#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1991 KVM_GUESTDBG_USE_HW_BP | \
1992 KVM_GUESTDBG_ENABLE)
1993
d0bfb940
JK		 1994int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1995 struct kvm_guest_debug *dbg)
b0c632db 1996{
27291e21
DH		 1997 int rc = 0;
1998
1999 vcpu->guest_debug = 0;
2000 kvm_s390_clear_bp_data(vcpu);
2001
2de3bfc2 2002 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 2003 return -EINVAL;
2004
2005 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2006 vcpu->guest_debug = dbg->control;
2007 /* enforce guest PER */
805de8f4 2008 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2009
2010 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2011 rc = kvm_s390_import_bp_data(vcpu, dbg);
2012 } else {
805de8f4 2013 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2014 vcpu->arch.guestdbg.last_bp = 0;
2015 }
2016
2017 if (rc) {
2018 vcpu->guest_debug = 0;
2019 kvm_s390_clear_bp_data(vcpu);
805de8f4 2020 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2021 }
2022
2023 return rc;
b0c632db
HC		 2024}
2025
62d9f0db
MT		 2026int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2027 struct kvm_mp_state *mp_state)
2028{
6352e4d2
DH		 2029 /* CHECK_STOP and LOAD are not supported yet */
2030 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2031 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2032}
2033
2034int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2035 struct kvm_mp_state *mp_state)
2036{
6352e4d2
DH		 2037 int rc = 0;
2038
2039 /* user space knows about this interface - let it control the state */
2040 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2041
2042 switch (mp_state->mp_state) {
2043 case KVM_MP_STATE_STOPPED:
2044 kvm_s390_vcpu_stop(vcpu);
2045 break;
2046 case KVM_MP_STATE_OPERATING:
2047 kvm_s390_vcpu_start(vcpu);
2048 break;
2049 case KVM_MP_STATE_LOAD:
2050 case KVM_MP_STATE_CHECK_STOP:
2051 /* fall through - CHECK_STOP and LOAD are not supported yet */
2052 default:
2053 rc = -ENXIO;
2054 }
2055
2056 return rc;
62d9f0db
MT		 2057}
2058
8ad35755
DH		 2059static bool ibs_enabled(struct kvm_vcpu *vcpu)
2060{
2061 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2062}
2063
2c70fe44
CB		 2064static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2065{
8ad35755 2066retry:
8e236546 2067 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2068 if (!vcpu->requests)
2069 return 0;
2c70fe44
CB		 2070 /*
2071 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2072 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2073 * This ensures that the ipte instruction for this request has
2074 * already finished. We might race against a second unmapper that
2075 * wants to set the blocking bit. Lets just retry the request loop.
2076 */
8ad35755 2077 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 2078 int rc;
2079 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 2080 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 2081 PAGE_SIZE * 2);
2082 if (rc)
2083 return rc;
8ad35755 2084 goto retry;
2c70fe44 2085 }
8ad35755 2086
d3d692c8
DH		 2087 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2088 vcpu->arch.sie_block->ihcpu = 0xffff;
2089 goto retry;
2090 }
2091
8ad35755
DH		 2092 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2093 if (!ibs_enabled(vcpu)) {
2094 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2095 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2096 &vcpu->arch.sie_block->cpuflags);
2097 }
2098 goto retry;
2c70fe44 2099 }
8ad35755
DH		 2100
2101 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2102 if (ibs_enabled(vcpu)) {
2103 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2104 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2105 &vcpu->arch.sie_block->cpuflags);
2106 }
2107 goto retry;
2108 }
2109
0759d068
DH		 2110 /* nothing to do, just clear the request */
2111 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2112
2c70fe44
CB		 2113 return 0;
2114}
2115
25ed1675
DH		 2116void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2117{
2118 struct kvm_vcpu *vcpu;
2119 int i;
2120
2121 mutex_lock(&kvm->lock);
2122 preempt_disable();
2123 kvm->arch.epoch = tod - get_tod_clock();
2124 kvm_s390_vcpu_block_all(kvm);
2125 kvm_for_each_vcpu(i, vcpu, kvm)
2126 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2127 kvm_s390_vcpu_unblock_all(kvm);
2128 preempt_enable();
2129 mutex_unlock(&kvm->lock);
2130}
2131
fa576c58
TH		 2132/**
2133 * kvm_arch_fault_in_page - fault-in guest page if necessary
2134 * @vcpu: The corresponding virtual cpu
2135 * @gpa: Guest physical address
2136 * @writable: Whether the page should be writable or not
2137 *
2138 * Make sure that a guest page has been faulted-in on the host.
2139 *
2140 * Return: Zero on success, negative error code otherwise.
2141 */
2142long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2143{
527e30b4
MS		 2144 return gmap_fault(vcpu->arch.gmap, gpa,
2145 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2146}
2147
3c038e6b
DD		 2148static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2149 unsigned long token)
2150{
2151 struct kvm_s390_interrupt inti;
383d0b05 2152 struct kvm_s390_irq irq;
3c038e6b
DD		 2153
2154 if (start_token) {
383d0b05
JF		 2155 irq.u.ext.ext_params2 = token;
2156 irq.type = KVM_S390_INT_PFAULT_INIT;
2157 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2158 } else {
2159 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2160 inti.parm64 = token;
3c038e6b
DD		 2161 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2162 }
2163}
2164
2165void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2166 struct kvm_async_pf *work)
2167{
2168 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2169 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2170}
2171
2172void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2173 struct kvm_async_pf *work)
2174{
2175 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2176 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2177}
2178
2179void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2180 struct kvm_async_pf *work)
2181{
2182 /* s390 will always inject the page directly */
2183}
2184
2185bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2186{
2187 /*
2188 * s390 will always inject the page directly,
2189 * but we still want check_async_completion to cleanup
2190 */
2191 return true;
2192}
2193
2194static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2195{
2196 hva_t hva;
2197 struct kvm_arch_async_pf arch;
2198 int rc;
2199
2200 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2201 return 0;
2202 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2203 vcpu->arch.pfault_compare)
2204 return 0;
2205 if (psw_extint_disabled(vcpu))
2206 return 0;
9a022067 2207 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2208 return 0;
2209 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2210 return 0;
2211 if (!vcpu->arch.gmap->pfault_enabled)
2212 return 0;
2213
81480cc1
HC		 2214 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2215 hva += current->thread.gmap_addr & ~PAGE_MASK;
2216 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2217 return 0;
2218
2219 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2220 return rc;
2221}
2222
3fb4c40f 2223static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2224{
3fb4c40f 2225 int rc, cpuflags;
e168bf8d 2226
3c038e6b
DD		 2227 /*
2228 * On s390 notifications for arriving pages will be delivered directly
2229 * to the guest but the house keeping for completed pfaults is
2230 * handled outside the worker.
2231 */
2232 kvm_check_async_pf_completion(vcpu);
2233
7ec7c8c7
CB		 2234 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2235 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2236
2237 if (need_resched())
2238 schedule();
2239
d3a73acb 2240 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2241 s390_handle_mcck();
2242
79395031
JF		 2243 if (!kvm_is_ucontrol(vcpu->kvm)) {
2244 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2245 if (rc)
2246 return rc;
2247 }
0ff31867 2248
2c70fe44
CB		 2249 rc = kvm_s390_handle_requests(vcpu);
2250 if (rc)
2251 return rc;
2252
27291e21
DH		 2253 if (guestdbg_enabled(vcpu)) {
2254 kvm_s390_backup_guest_per_regs(vcpu);
2255 kvm_s390_patch_guest_per_regs(vcpu);
2256 }
2257
b0c632db 2258 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2259 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2260 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2261 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2262
3fb4c40f
TH		 2263 return 0;
2264}
2265
492d8642
TH		 2266static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2267{
56317920
DH		 2268 struct kvm_s390_pgm_info pgm_info = {
2269 .code = PGM_ADDRESSING,
2270 };
2271 u8 opcode, ilen;
492d8642
TH		 2272 int rc;
2273
2274 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2275 trace_kvm_s390_sie_fault(vcpu);
2276
2277 /*
2278 * We want to inject an addressing exception, which is defined as a
2279 * suppressing or terminating exception. However, since we came here
2280 * by a DAT access exception, the PSW still points to the faulting
2281 * instruction since DAT exceptions are nullifying. So we've got
2282 * to look up the current opcode to get the length of the instruction
2283 * to be able to forward the PSW.
2284 */
65977322 2285 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2286 ilen = insn_length(opcode);
9b0d721a
DH		 2287 if (rc < 0) {
2288 return rc;
2289 } else if (rc) {
2290 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2291 * Forward by arbitrary ilc, injection will take care of
2292 * nullification if necessary.
2293 */
2294 pgm_info = vcpu->arch.pgm;
2295 ilen = 4;
2296 }
56317920
DH		 2297 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2298 kvm_s390_forward_psw(vcpu, ilen);
2299 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2300}
2301
3fb4c40f
TH		 2302static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2303{
2b29a9fd
DD		 2304 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2305 vcpu->arch.sie_block->icptcode);
2306 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2307
27291e21
DH		 2308 if (guestdbg_enabled(vcpu))
2309 kvm_s390_restore_guest_per_regs(vcpu);
2310
7ec7c8c7
CB		 2311 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2312 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2313
2314 if (vcpu->arch.sie_block->icptcode > 0) {
2315 int rc = kvm_handle_sie_intercept(vcpu);
2316
2317 if (rc != -EOPNOTSUPP)
2318 return rc;
2319 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2320 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2321 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2322 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2323 return -EREMOTE;
2324 } else if (exit_reason != -EFAULT) {
2325 vcpu->stat.exit_null++;
2326 return 0;
210b1607
TH		 2327 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2328 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2329 vcpu->run->s390_ucontrol.trans_exc_code =
2330 current->thread.gmap_addr;
2331 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2332 return -EREMOTE;
24eb3a82 2333 } else if (current->thread.gmap_pfault) {
3c038e6b 2334 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2335 current->thread.gmap_pfault = 0;
71f116bf
DH		 2336 if (kvm_arch_setup_async_pf(vcpu))
2337 return 0;
2338 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2339 }
71f116bf 2340 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2341}
2342
2343static int __vcpu_run(struct kvm_vcpu *vcpu)
2344{
2345 int rc, exit_reason;
2346
800c1065
TH		 2347 /*
2348 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2349 * ning the guest), so that memslots (and other stuff) are protected
2350 */
2351 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2352
a76ccff6
TH		 2353 do {
2354 rc = vcpu_pre_run(vcpu);
2355 if (rc)
2356 break;
3fb4c40f 2357
800c1065 2358 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2359 /*
2360 * As PF_VCPU will be used in fault handler, between
2361 * guest_enter and guest_exit should be no uaccess.
2362 */
0097d12e
CB		 2363 local_irq_disable();
2364 __kvm_guest_enter();
db0758b2 2365 __disable_cpu_timer_accounting(vcpu);
0097d12e 2366 local_irq_enable();
a76ccff6
TH		 2367 exit_reason = sie64a(vcpu->arch.sie_block,
2368 vcpu->run->s.regs.gprs);
0097d12e 2369 local_irq_disable();
db0758b2 2370 __enable_cpu_timer_accounting(vcpu);
0097d12e
CB		 2371 __kvm_guest_exit();
2372 local_irq_enable();
800c1065 2373 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2374
2375 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2376 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2377
800c1065 2378 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2379 return rc;
b0c632db
HC		 2380}
2381
b028ee3e
DH		 2382static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2383{
2384 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2385 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2386 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2387 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2388 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2389 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2390 /* some control register changes require a tlb flush */
2391 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2392 }
2393 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2394 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2395 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2396 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2397 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2398 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2399 }
2400 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2401 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2402 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2403 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2404 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2405 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2406 }
2407 kvm_run->kvm_dirty_regs = 0;
2408}
2409
2410static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2411{
2412 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2413 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2414 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2415 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2416 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2417 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2418 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2419 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2420 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2421 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2422 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2423 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2424}
2425
b0c632db
HC		 2426int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2427{
8f2abe6a 2428 int rc;
b0c632db
HC		 2429 sigset_t sigsaved;
2430
27291e21
DH		 2431 if (guestdbg_exit_pending(vcpu)) {
2432 kvm_s390_prepare_debug_exit(vcpu);
2433 return 0;
2434 }
2435
b0c632db
HC		 2436 if (vcpu->sigset_active)
2437 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2438
6352e4d2
DH		 2439 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2440 kvm_s390_vcpu_start(vcpu);
2441 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2442 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2443 vcpu->vcpu_id);
2444 return -EINVAL;
2445 }
b0c632db 2446
b028ee3e 2447 sync_regs(vcpu, kvm_run);
db0758b2 2448 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2449
dab4079d 2450 might_fault();
a76ccff6 2451 rc = __vcpu_run(vcpu);
9ace903d 2452
b1d16c49
CE		 2453 if (signal_pending(current) && !rc) {
2454 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2455 rc = -EINTR;
b1d16c49 2456 }
8f2abe6a 2457
27291e21
DH		 2458 if (guestdbg_exit_pending(vcpu) && !rc) {
2459 kvm_s390_prepare_debug_exit(vcpu);
2460 rc = 0;
2461 }
2462
8f2abe6a 2463 if (rc == -EREMOTE) {
71f116bf 2464 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2465 rc = 0;
2466 }
b0c632db 2467
db0758b2 2468 disable_cpu_timer_accounting(vcpu);
b028ee3e 2469 store_regs(vcpu, kvm_run);
d7b0b5eb 2470
b0c632db
HC		 2471 if (vcpu->sigset_active)
2472 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2473
b0c632db 2474 vcpu->stat.exit_userspace++;
7e8e6ab4 2475 return rc;
b0c632db
HC		 2476}
2477
b0c632db
HC		 2478/*
2479 * store status at address
2480 * we use have two special cases:
2481 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2482 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2483 */
d0bce605 2484int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2485{
092670cd 2486 unsigned char archmode = 1;
9abc2a08 2487 freg_t fprs[NUM_FPRS];
fda902cb 2488 unsigned int px;
4287f247 2489 u64 clkcomp, cputm;
d0bce605 2490 int rc;
b0c632db 2491
d9a3a09a 2492 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2493 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2494 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2495 return -EFAULT;
d9a3a09a 2496 gpa = 0;
d0bce605
HC		 2497 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2498 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2499 return -EFAULT;
d9a3a09a
MS		 2500 gpa = px;
2501 } else
2502 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2503
2504 /* manually convert vector registers if necessary */
2505 if (MACHINE_HAS_VX) {
9522b37f 2506 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2507 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2508 fprs, 128);
2509 } else {
2510 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2511 vcpu->run->s.regs.fprs, 128);
9abc2a08 2512 }
d9a3a09a 2513 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2514 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2515 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2516 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2517 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2518 &px, 4);
d9a3a09a 2519 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2520 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2521 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2522 &vcpu->arch.sie_block->todpr, 4);
4287f247 2523 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2524 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2525 &cputm, 8);
178bd789 2526 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2527 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2528 &clkcomp, 8);
d9a3a09a 2529 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2530 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2531 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2532 &vcpu->arch.sie_block->gcr, 128);
2533 return rc ? -EFAULT : 0;
b0c632db
HC		 2534}
2535
e879892c
TH		 2536int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2537{
2538 /*
2539 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2540 * copying in vcpu load/put. Lets update our copies before we save
2541 * it into the save area
2542 */
d0164ee2 2543 save_fpu_regs();
9abc2a08 2544 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2545 save_access_regs(vcpu->run->s.regs.acrs);
2546
2547 return kvm_s390_store_status_unloaded(vcpu, addr);
2548}
2549
bc17de7c
EF		 2550/*
2551 * store additional status at address
2552 */
2553int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2554 unsigned long gpa)
2555{
2556 /* Only bits 0-53 are used for address formation */
2557 if (!(gpa & ~0x3ff))
2558 return 0;
2559
2560 return write_guest_abs(vcpu, gpa & ~0x3ff,
2561 (void *)&vcpu->run->s.regs.vrs, 512);
2562}
2563
2564int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2565{
2566 if (!test_kvm_facility(vcpu->kvm, 129))
2567 return 0;
2568
2569 /*
2570 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2571 * copying in vcpu load/put. We can simply call save_fpu_regs()
2572 * to save the current register state because we are in the
2573 * middle of a load/put cycle.
2574 *
2575 * Let's update our copies before we save it into the save area.
bc17de7c 2576 */
d0164ee2 2577 save_fpu_regs();
bc17de7c
EF		 2578
2579 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2580}
2581
8ad35755
DH		 2582static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2583{
2584 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2585 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2586}
2587
2588static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2589{
2590 unsigned int i;
2591 struct kvm_vcpu *vcpu;
2592
2593 kvm_for_each_vcpu(i, vcpu, kvm) {
2594 __disable_ibs_on_vcpu(vcpu);
2595 }
2596}
2597
2598static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2599{
2600 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2601 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2602}
2603
6852d7b6
DH		 2604void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2605{
8ad35755
DH		 2606 int i, online_vcpus, started_vcpus = 0;
2607
2608 if (!is_vcpu_stopped(vcpu))
2609 return;
2610
6852d7b6 2611 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2612 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2613 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2614 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2615
2616 for (i = 0; i < online_vcpus; i++) {
2617 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2618 started_vcpus++;
2619 }
2620
2621 if (started_vcpus == 0) {
2622 /* we're the only active VCPU -> speed it up */
2623 __enable_ibs_on_vcpu(vcpu);
2624 } else if (started_vcpus == 1) {
2625 /*
2626 * As we are starting a second VCPU, we have to disable
2627 * the IBS facility on all VCPUs to remove potentially
2628 * oustanding ENABLE requests.
2629 */
2630 __disable_ibs_on_all_vcpus(vcpu->kvm);
2631 }
2632
805de8f4 2633 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2634 /*
2635 * Another VCPU might have used IBS while we were offline.
2636 * Let's play safe and flush the VCPU at startup.
2637 */
d3d692c8 2638 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2639 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2640 return;
6852d7b6
DH		 2641}
2642
2643void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2644{
8ad35755
DH		 2645 int i, online_vcpus, started_vcpus = 0;
2646 struct kvm_vcpu *started_vcpu = NULL;
2647
2648 if (is_vcpu_stopped(vcpu))
2649 return;
2650
6852d7b6 2651 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2652 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2653 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2654 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2655
32f5ff63 2656 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2657 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2658
805de8f4 2659 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2660 __disable_ibs_on_vcpu(vcpu);
2661
2662 for (i = 0; i < online_vcpus; i++) {
2663 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2664 started_vcpus++;
2665 started_vcpu = vcpu->kvm->vcpus[i];
2666 }
2667 }
2668
2669 if (started_vcpus == 1) {
2670 /*
2671 * As we only have one VCPU left, we want to enable the
2672 * IBS facility for that VCPU to speed it up.
2673 */
2674 __enable_ibs_on_vcpu(started_vcpu);
2675 }
2676
433b9ee4 2677 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2678 return;
6852d7b6
DH		 2679}
2680
d6712df9
CH		 2681static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2682 struct kvm_enable_cap *cap)
2683{
2684 int r;
2685
2686 if (cap->flags)
2687 return -EINVAL;
2688
2689 switch (cap->cap) {
fa6b7fe9
CH		 2690 case KVM_CAP_S390_CSS_SUPPORT:
2691 if (!vcpu->kvm->arch.css_support) {
2692 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2693 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2694 trace_kvm_s390_enable_css(vcpu->kvm);
2695 }
2696 r = 0;
2697 break;
d6712df9
CH		 2698 default:
2699 r = -EINVAL;
2700 break;
2701 }
2702 return r;
2703}
2704
41408c28
TH		 2705static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2706 struct kvm_s390_mem_op *mop)
2707{
2708 void __user *uaddr = (void __user *)mop->buf;
2709 void *tmpbuf = NULL;
2710 int r, srcu_idx;
2711 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2712 | KVM_S390_MEMOP_F_CHECK_ONLY;
2713
2714 if (mop->flags & ~supported_flags)
2715 return -EINVAL;
2716
2717 if (mop->size > MEM_OP_MAX_SIZE)
2718 return -E2BIG;
2719
2720 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2721 tmpbuf = vmalloc(mop->size);
2722 if (!tmpbuf)
2723 return -ENOMEM;
2724 }
2725
2726 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2727
2728 switch (mop->op) {
2729 case KVM_S390_MEMOP_LOGICAL_READ:
2730 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2731 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2732 mop->size, GACC_FETCH);
41408c28
TH		 2733 break;
2734 }
2735 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2736 if (r == 0) {
2737 if (copy_to_user(uaddr, tmpbuf, mop->size))
2738 r = -EFAULT;
2739 }
2740 break;
2741 case KVM_S390_MEMOP_LOGICAL_WRITE:
2742 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2743 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2744 mop->size, GACC_STORE);
41408c28
TH		 2745 break;
2746 }
2747 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2748 r = -EFAULT;
2749 break;
2750 }
2751 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2752 break;
2753 default:
2754 r = -EINVAL;
2755 }
2756
2757 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2758
2759 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2760 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2761
2762 vfree(tmpbuf);
2763 return r;
2764}
2765
b0c632db
HC		 2766long kvm_arch_vcpu_ioctl(struct file *filp,
2767 unsigned int ioctl, unsigned long arg)
2768{
2769 struct kvm_vcpu *vcpu = filp->private_data;
2770 void __user *argp = (void __user *)arg;
800c1065 2771 int idx;
bc923cc9 2772 long r;
b0c632db 2773
93736624 2774 switch (ioctl) {
47b43c52
JF		 2775 case KVM_S390_IRQ: {
2776 struct kvm_s390_irq s390irq;
2777
2778 r = -EFAULT;
2779 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2780 break;
2781 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2782 break;
2783 }
93736624 2784 case KVM_S390_INTERRUPT: {
ba5c1e9b 2785 struct kvm_s390_interrupt s390int;
383d0b05 2786 struct kvm_s390_irq s390irq;
ba5c1e9b 2787
93736624 2788 r = -EFAULT;
ba5c1e9b 2789 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2790 break;
383d0b05
JF		 2791 if (s390int_to_s390irq(&s390int, &s390irq))
2792 return -EINVAL;
2793 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2794 break;
ba5c1e9b 2795 }
b0c632db 2796 case KVM_S390_STORE_STATUS:
800c1065 2797 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 2798 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 2799 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 2800 break;
b0c632db
HC		 2801 case KVM_S390_SET_INITIAL_PSW: {
2802 psw_t psw;
2803
bc923cc9 2804 r = -EFAULT;
b0c632db 2805 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 2806 break;
2807 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2808 break;
b0c632db
HC		 2809 }
2810 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 2811 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2812 break;
14eebd91
CO		 2813 case KVM_SET_ONE_REG:
2814 case KVM_GET_ONE_REG: {
2815 struct kvm_one_reg reg;
2816 r = -EFAULT;
2817 if (copy_from_user(&reg, argp, sizeof(reg)))
2818 break;
2819 if (ioctl == KVM_SET_ONE_REG)
2820 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2821 else
2822 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2823 break;
2824 }
27e0393f
CO		 2825#ifdef CONFIG_KVM_S390_UCONTROL
2826 case KVM_S390_UCAS_MAP: {
2827 struct kvm_s390_ucas_mapping ucasmap;
2828
2829 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2830 r = -EFAULT;
2831 break;
2832 }
2833
2834 if (!kvm_is_ucontrol(vcpu->kvm)) {
2835 r = -EINVAL;
2836 break;
2837 }
2838
2839 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2840 ucasmap.vcpu_addr, ucasmap.length);
2841 break;
2842 }
2843 case KVM_S390_UCAS_UNMAP: {
2844 struct kvm_s390_ucas_mapping ucasmap;
2845
2846 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2847 r = -EFAULT;
2848 break;
2849 }
2850
2851 if (!kvm_is_ucontrol(vcpu->kvm)) {
2852 r = -EINVAL;
2853 break;
2854 }
2855
2856 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2857 ucasmap.length);
2858 break;
2859 }
2860#endif
ccc7910f 2861 case KVM_S390_VCPU_FAULT: {
527e30b4 2862 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 2863 break;
2864 }
d6712df9
CH		 2865 case KVM_ENABLE_CAP:
2866 {
2867 struct kvm_enable_cap cap;
2868 r = -EFAULT;
2869 if (copy_from_user(&cap, argp, sizeof(cap)))
2870 break;
2871 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2872 break;
2873 }
41408c28
TH		 2874 case KVM_S390_MEM_OP: {
2875 struct kvm_s390_mem_op mem_op;
2876
2877 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2878 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2879 else
2880 r = -EFAULT;
2881 break;
2882 }
816c7667
JF		 2883 case KVM_S390_SET_IRQ_STATE: {
2884 struct kvm_s390_irq_state irq_state;
2885
2886 r = -EFAULT;
2887 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2888 break;
2889 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2890 irq_state.len == 0 ||
2891 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2892 r = -EINVAL;
2893 break;
2894 }
2895 r = kvm_s390_set_irq_state(vcpu,
2896 (void __user *) irq_state.buf,
2897 irq_state.len);
2898 break;
2899 }
2900 case KVM_S390_GET_IRQ_STATE: {
2901 struct kvm_s390_irq_state irq_state;
2902
2903 r = -EFAULT;
2904 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2905 break;
2906 if (irq_state.len == 0) {
2907 r = -EINVAL;
2908 break;
2909 }
2910 r = kvm_s390_get_irq_state(vcpu,
2911 (__u8 __user *) irq_state.buf,
2912 irq_state.len);
2913 break;
2914 }
b0c632db 2915 default:
3e6afcf1 2916 r = -ENOTTY;
b0c632db 2917 }
bc923cc9 2918 return r;
b0c632db
HC		 2919}
2920
5b1c1493
CO		 2921int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2922{
2923#ifdef CONFIG_KVM_S390_UCONTROL
2924 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2925 && (kvm_is_ucontrol(vcpu->kvm))) {
2926 vmf->page = virt_to_page(vcpu->arch.sie_block);
2927 get_page(vmf->page);
2928 return 0;
2929 }
2930#endif
2931 return VM_FAULT_SIGBUS;
2932}
2933
5587027c
AK		 2934int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2935 unsigned long npages)
db3fe4eb
TY		 2936{
2937 return 0;
2938}
2939
b0c632db 2940/* Section: memory related */
f7784b8e
MT		 2941int kvm_arch_prepare_memory_region(struct kvm *kvm,
2942 struct kvm_memory_slot *memslot,
09170a49 2943 const struct kvm_userspace_memory_region *mem,
7b6195a9 2944 enum kvm_mr_change change)
b0c632db 2945{
dd2887e7
NW		 2946 /* A few sanity checks. We can have memory slots which have to be
2947 located/ended at a segment boundary (1MB). The memory in userland is
2948 ok to be fragmented into various different vmas. It is okay to mmap()
2949 and munmap() stuff in this slot after doing this call at any time */
b0c632db 2950
598841ca 2951 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 2952 return -EINVAL;
2953
598841ca 2954 if (mem->memory_size & 0xffffful)
b0c632db
HC		 2955 return -EINVAL;
2956
a3a92c31
DD		 2957 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
2958 return -EINVAL;
2959
f7784b8e
MT		 2960 return 0;
2961}
2962
2963void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 2964 const struct kvm_userspace_memory_region *mem,
8482644a 2965 const struct kvm_memory_slot *old,
f36f3f28 2966 const struct kvm_memory_slot *new,
8482644a 2967 enum kvm_mr_change change)
f7784b8e 2968{
f7850c92 2969 int rc;
f7784b8e 2970
2cef4deb
CB		 2971 /* If the basics of the memslot do not change, we do not want
2972 * to update the gmap. Every update causes several unnecessary
2973 * segment translation exceptions. This is usually handled just
2974 * fine by the normal fault handler + gmap, but it will also
2975 * cause faults on the prefix page of running guest CPUs.
2976 */
2977 if (old->userspace_addr == mem->userspace_addr &&
2978 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2979 old->npages * PAGE_SIZE == mem->memory_size)
2980 return;
598841ca
CO		 2981
2982 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2983 mem->guest_phys_addr, mem->memory_size);
2984 if (rc)
ea2cdd27 2985 pr_warn("failed to commit memory region\n");
598841ca 2986 return;
b0c632db
HC		 2987}
2988
60a37709
AY		 2989static inline unsigned long nonhyp_mask(int i)
2990{
2991 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
2992
2993 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
2994}
2995
3491caf2
CB		 2996void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
2997{
2998 vcpu->valid_wakeup = false;
2999}
3000
b0c632db
HC		 3001static int __init kvm_s390_init(void)
3002{
60a37709
AY		 3003 int i;
3004
07197fd0
DH		 3005 if (!sclp.has_sief2) {
3006 pr_info("SIE not available\n");
3007 return -ENODEV;
3008 }
3009
60a37709
AY		 3010 for (i = 0; i < 16; i++)
3011 kvm_s390_fac_list_mask[i] |=
3012 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3013
9d8d5786 3014 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3015}
3016
3017static void __exit kvm_s390_exit(void)
3018{
3019 kvm_exit();
3020}
3021
3022module_init(kvm_s390_init);
3023module_exit(kvm_s390_exit);
566af940
CH		 3024
3025/*
3026 * Enable autoloading of the kvm module.
3027 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3028 * since x86 takes a different approach.
3029 */
3030#include <linux/miscdevice.h>
3031MODULE_ALIAS_MISCDEV(KVM_MINOR);
3032MODULE_ALIAS("devname:kvm");
Linux kernel - Deliverables
This page took 0.638509 seconds and 5 git commands to generate.