2 * Copyright (C) 2015, 2016 ARM Ltd.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 #include <linux/uaccess.h>
18 #include <linux/interrupt.h>
19 #include <linux/cpu.h>
20 #include <linux/kvm_host.h>
21 #include <kvm/arm_vgic.h>
22 #include <asm/kvm_mmu.h>
26 * Initialization rules: there are multiple stages to the vgic
27 * initialization, both for the distributor and the CPU interfaces.
31 * - kvm_vgic_early_init(): initialization of static data that doesn't
32 * depend on any sizing information or emulation type. No allocation
35 * - vgic_init(): allocation and initialization of the generic data
36 * structures that depend on sizing information (number of CPUs,
37 * number of interrupts). Also initializes the vcpu specific data
38 * structures. Can be executed lazily for GICv2.
42 * - kvm_vgic_cpu_early_init(): initialization of static data that
43 * doesn't depend on any sizing information or emulation type. No
44 * allocation is allowed there.
50 * Those 2 functions should not be needed anymore but they
51 * still are called from arm.c
53 void kvm_vgic_early_init(struct kvm
*kvm
)
57 void kvm_vgic_vcpu_early_init(struct kvm_vcpu
*vcpu
)
64 * kvm_vgic_create: triggered by the instantiation of the VGIC device by
65 * user space, either through the legacy KVM_CREATE_IRQCHIP ioctl (v2 only)
66 * or through the generic KVM_CREATE_DEVICE API ioctl.
67 * irqchip_in_kernel() tells you if this function succeeded or not.
68 * @kvm: kvm struct pointer
69 * @type: KVM_DEV_TYPE_ARM_VGIC_V[23]
71 int kvm_vgic_create(struct kvm
*kvm
, u32 type
)
73 int i
, vcpu_lock_idx
= -1, ret
;
74 struct kvm_vcpu
*vcpu
;
76 mutex_lock(&kvm
->lock
);
78 if (irqchip_in_kernel(kvm
)) {
84 * This function is also called by the KVM_CREATE_IRQCHIP handler,
85 * which had no chance yet to check the availability of the GICv2
86 * emulation. So check this here again. KVM_CREATE_DEVICE does
87 * the proper checks already.
89 if (type
== KVM_DEV_TYPE_ARM_VGIC_V2
&&
90 !kvm_vgic_global_state
.can_emulate_gicv2
) {
96 * Any time a vcpu is run, vcpu_load is called which tries to grab the
97 * vcpu->mutex. By grabbing the vcpu->mutex of all VCPUs we ensure
98 * that no other VCPUs are run while we create the vgic.
101 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
102 if (!mutex_trylock(&vcpu
->mutex
))
107 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
108 if (vcpu
->arch
.has_run_once
)
113 if (type
== KVM_DEV_TYPE_ARM_VGIC_V2
)
114 kvm
->arch
.max_vcpus
= VGIC_V2_MAX_CPUS
;
116 kvm
->arch
.max_vcpus
= VGIC_V3_MAX_CPUS
;
118 if (atomic_read(&kvm
->online_vcpus
) > kvm
->arch
.max_vcpus
) {
123 kvm
->arch
.vgic
.in_kernel
= true;
124 kvm
->arch
.vgic
.vgic_model
= type
;
127 * kvm_vgic_global_state.vctrl_base is set on vgic probe (kvm_arch_init)
128 * it is stored in distributor struct for asm save/restore purpose
130 kvm
->arch
.vgic
.vctrl_base
= kvm_vgic_global_state
.vctrl_base
;
132 kvm
->arch
.vgic
.vgic_dist_base
= VGIC_ADDR_UNDEF
;
133 kvm
->arch
.vgic
.vgic_cpu_base
= VGIC_ADDR_UNDEF
;
134 kvm
->arch
.vgic
.vgic_redist_base
= VGIC_ADDR_UNDEF
;
137 for (; vcpu_lock_idx
>= 0; vcpu_lock_idx
--) {
138 vcpu
= kvm_get_vcpu(kvm
, vcpu_lock_idx
);
139 mutex_unlock(&vcpu
->mutex
);
143 mutex_unlock(&kvm
->lock
);
150 * kvm_vgic_dist_init: initialize the dist data structures
151 * @kvm: kvm struct pointer
152 * @nr_spis: number of spis, frozen by caller
154 static int kvm_vgic_dist_init(struct kvm
*kvm
, unsigned int nr_spis
)
156 struct vgic_dist
*dist
= &kvm
->arch
.vgic
;
157 struct kvm_vcpu
*vcpu0
= kvm_get_vcpu(kvm
, 0);
160 INIT_LIST_HEAD(&dist
->lpi_list_head
);
161 spin_lock_init(&dist
->lpi_list_lock
);
163 dist
->spis
= kcalloc(nr_spis
, sizeof(struct vgic_irq
), GFP_KERNEL
);
168 * In the following code we do not take the irq struct lock since
169 * no other action on irq structs can happen while the VGIC is
170 * not initialized yet:
171 * If someone wants to inject an interrupt or does a MMIO access, we
172 * require prior initialization in case of a virtual GICv3 or trigger
173 * initialization when using a virtual GICv2.
175 for (i
= 0; i
< nr_spis
; i
++) {
176 struct vgic_irq
*irq
= &dist
->spis
[i
];
178 irq
->intid
= i
+ VGIC_NR_PRIVATE_IRQS
;
179 INIT_LIST_HEAD(&irq
->ap_list
);
180 spin_lock_init(&irq
->irq_lock
);
182 irq
->target_vcpu
= vcpu0
;
183 kref_init(&irq
->refcount
);
184 if (dist
->vgic_model
== KVM_DEV_TYPE_ARM_VGIC_V2
)
193 * kvm_vgic_vcpu_init: initialize the vcpu data structures and
194 * enable the VCPU interface
195 * @vcpu: the VCPU which's VGIC should be initialized
197 static void kvm_vgic_vcpu_init(struct kvm_vcpu
*vcpu
)
199 struct vgic_cpu
*vgic_cpu
= &vcpu
->arch
.vgic_cpu
;
202 INIT_LIST_HEAD(&vgic_cpu
->ap_list_head
);
203 spin_lock_init(&vgic_cpu
->ap_list_lock
);
206 * Enable and configure all SGIs to be edge-triggered and
207 * configure all PPIs as level-triggered.
209 for (i
= 0; i
< VGIC_NR_PRIVATE_IRQS
; i
++) {
210 struct vgic_irq
*irq
= &vgic_cpu
->private_irqs
[i
];
212 INIT_LIST_HEAD(&irq
->ap_list
);
213 spin_lock_init(&irq
->irq_lock
);
216 irq
->target_vcpu
= vcpu
;
217 irq
->targets
= 1U << vcpu
->vcpu_id
;
218 kref_init(&irq
->refcount
);
219 if (vgic_irq_is_sgi(i
)) {
222 irq
->config
= VGIC_CONFIG_EDGE
;
225 irq
->config
= VGIC_CONFIG_LEVEL
;
228 if (kvm_vgic_global_state
.type
== VGIC_V2
)
229 vgic_v2_enable(vcpu
);
231 vgic_v3_enable(vcpu
);
235 * vgic_init: allocates and initializes dist and vcpu data structures
236 * depending on two dimensioning parameters:
237 * - the number of spis
238 * - the number of vcpus
239 * The function is generally called when nr_spis has been explicitly set
240 * by the guest through the KVM DEVICE API. If not nr_spis is set to 256.
241 * vgic_initialized() returns true when this function has succeeded.
242 * Must be called with kvm->lock held!
244 int vgic_init(struct kvm
*kvm
)
246 struct vgic_dist
*dist
= &kvm
->arch
.vgic
;
247 struct kvm_vcpu
*vcpu
;
250 if (vgic_initialized(kvm
))
253 /* freeze the number of spis */
255 dist
->nr_spis
= VGIC_NR_IRQS_LEGACY
- VGIC_NR_PRIVATE_IRQS
;
257 ret
= kvm_vgic_dist_init(kvm
, dist
->nr_spis
);
261 if (vgic_has_its(kvm
))
262 dist
->msis_require_devid
= true;
264 kvm_for_each_vcpu(i
, vcpu
, kvm
)
265 kvm_vgic_vcpu_init(vcpu
);
267 ret
= kvm_vgic_setup_default_irq_routing(kvm
);
271 dist
->initialized
= true;
276 static void kvm_vgic_dist_destroy(struct kvm
*kvm
)
278 struct vgic_dist
*dist
= &kvm
->arch
.vgic
;
280 mutex_lock(&kvm
->lock
);
283 dist
->initialized
= false;
288 mutex_unlock(&kvm
->lock
);
291 void kvm_vgic_vcpu_destroy(struct kvm_vcpu
*vcpu
)
293 struct vgic_cpu
*vgic_cpu
= &vcpu
->arch
.vgic_cpu
;
295 INIT_LIST_HEAD(&vgic_cpu
->ap_list_head
);
298 void kvm_vgic_destroy(struct kvm
*kvm
)
300 struct kvm_vcpu
*vcpu
;
303 kvm_vgic_dist_destroy(kvm
);
305 kvm_for_each_vcpu(i
, vcpu
, kvm
)
306 kvm_vgic_vcpu_destroy(vcpu
);
310 * vgic_lazy_init: Lazy init is only allowed if the GIC exposed to the guest
311 * is a GICv2. A GICv3 must be explicitly initialized by the guest using the
312 * KVM_DEV_ARM_VGIC_GRP_CTRL KVM_DEVICE group.
313 * @kvm: kvm struct pointer
315 int vgic_lazy_init(struct kvm
*kvm
)
319 if (unlikely(!vgic_initialized(kvm
))) {
321 * We only provide the automatic initialization of the VGIC
322 * for the legacy case of a GICv2. Any other type must
323 * be explicitly initialized once setup with the respective
326 if (kvm
->arch
.vgic
.vgic_model
!= KVM_DEV_TYPE_ARM_VGIC_V2
)
329 mutex_lock(&kvm
->lock
);
330 ret
= vgic_init(kvm
);
331 mutex_unlock(&kvm
->lock
);
337 /* RESOURCE MAPPING */
340 * Map the MMIO regions depending on the VGIC model exposed to the guest
341 * called on the first VCPU run.
342 * Also map the virtual CPU interface into the VM.
343 * v2/v3 derivatives call vgic_init if not already done.
344 * vgic_ready() returns true if this function has succeeded.
345 * @kvm: kvm struct pointer
347 int kvm_vgic_map_resources(struct kvm
*kvm
)
349 struct vgic_dist
*dist
= &kvm
->arch
.vgic
;
352 mutex_lock(&kvm
->lock
);
353 if (!irqchip_in_kernel(kvm
))
356 if (dist
->vgic_model
== KVM_DEV_TYPE_ARM_VGIC_V2
)
357 ret
= vgic_v2_map_resources(kvm
);
359 ret
= vgic_v3_map_resources(kvm
);
361 mutex_unlock(&kvm
->lock
);
367 static int vgic_init_cpu_starting(unsigned int cpu
)
369 enable_percpu_irq(kvm_vgic_global_state
.maint_irq
, 0);
374 static int vgic_init_cpu_dying(unsigned int cpu
)
376 disable_percpu_irq(kvm_vgic_global_state
.maint_irq
);
380 static irqreturn_t
vgic_maintenance_handler(int irq
, void *data
)
383 * We cannot rely on the vgic maintenance interrupt to be
384 * delivered synchronously. This means we can only use it to
385 * exit the VM, and we perform the handling of EOIed
386 * interrupts on the exit path (see vgic_process_maintenance).
392 * kvm_vgic_hyp_init: populates the kvm_vgic_global_state variable
393 * according to the host GIC model. Accordingly calls either
394 * vgic_v2/v3_probe which registers the KVM_DEVICE that can be
395 * instantiated by a guest later on .
397 int kvm_vgic_hyp_init(void)
399 const struct gic_kvm_info
*gic_kvm_info
;
402 gic_kvm_info
= gic_get_kvm_info();
406 if (!gic_kvm_info
->maint_irq
) {
407 kvm_err("No vgic maintenance irq\n");
411 switch (gic_kvm_info
->type
) {
413 ret
= vgic_v2_probe(gic_kvm_info
);
416 ret
= vgic_v3_probe(gic_kvm_info
);
425 kvm_vgic_global_state
.maint_irq
= gic_kvm_info
->maint_irq
;
426 ret
= request_percpu_irq(kvm_vgic_global_state
.maint_irq
,
427 vgic_maintenance_handler
,
428 "vgic", kvm_get_running_vcpus());
430 kvm_err("Cannot register interrupt %d\n",
431 kvm_vgic_global_state
.maint_irq
);
435 ret
= cpuhp_setup_state(CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING
,
436 "AP_KVM_ARM_VGIC_INIT_STARTING",
437 vgic_init_cpu_starting
, vgic_init_cpu_dying
);
439 kvm_err("Cannot register vgic CPU notifier\n");
443 kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state
.maint_irq
);
447 free_percpu_irq(kvm_vgic_global_state
.maint_irq
,
448 kvm_get_running_vcpus());