4 * Copyright (C) 2015,2016 ARM Ltd.
5 * Author: Andre Przywara <andre.przywara@arm.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/cpu.h>
21 #include <linux/kvm.h>
22 #include <linux/kvm_host.h>
23 #include <linux/interrupt.h>
24 #include <linux/list.h>
25 #include <linux/uaccess.h>
27 #include <linux/irqchip/arm-gic-v3.h>
29 #include <asm/kvm_emulate.h>
30 #include <asm/kvm_arm.h>
31 #include <asm/kvm_mmu.h>
34 #include "vgic-mmio.h"
37 struct list_head dev_list
;
39 /* the head for the list of ITTEs */
40 struct list_head itt_head
;
44 #define COLLECTION_NOT_MAPPED ((u32)~0)
46 struct its_collection
{
47 struct list_head coll_list
;
53 #define its_is_collection_mapped(coll) ((coll) && \
54 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
57 struct list_head itte_list
;
59 struct its_collection
*collection
;
65 * We only implement 48 bits of PA at the moment, although the ITS
66 * supports more. Let's be restrictive here.
68 #define CBASER_ADDRESS(x) ((x) & GENMASK_ULL(47, 12))
70 static unsigned long vgic_mmio_read_its_ctlr(struct kvm
*vcpu
,
72 gpa_t addr
, unsigned int len
)
76 mutex_lock(&its
->cmd_lock
);
77 if (its
->creadr
== its
->cwriter
)
78 reg
|= GITS_CTLR_QUIESCENT
;
80 reg
|= GITS_CTLR_ENABLE
;
81 mutex_unlock(&its
->cmd_lock
);
86 static void vgic_mmio_write_its_ctlr(struct kvm
*kvm
, struct vgic_its
*its
,
87 gpa_t addr
, unsigned int len
,
90 its
->enabled
= !!(val
& GITS_CTLR_ENABLE
);
93 static unsigned long vgic_mmio_read_its_typer(struct kvm
*kvm
,
95 gpa_t addr
, unsigned int len
)
97 u64 reg
= GITS_TYPER_PLPIS
;
100 * We use linear CPU numbers for redistributor addressing,
101 * so GITS_TYPER.PTA is 0.
102 * Also we force all PROPBASER registers to be the same, so
103 * CommonLPIAff is 0 as well.
104 * To avoid memory waste in the guest, we keep the number of IDBits and
105 * DevBits low - as least for the time being.
107 reg
|= 0x0f << GITS_TYPER_DEVBITS_SHIFT
;
108 reg
|= 0x0f << GITS_TYPER_IDBITS_SHIFT
;
110 return extract_bytes(reg
, addr
& 7, len
);
113 static unsigned long vgic_mmio_read_its_iidr(struct kvm
*kvm
,
114 struct vgic_its
*its
,
115 gpa_t addr
, unsigned int len
)
117 return (PRODUCT_ID_KVM
<< 24) | (IMPLEMENTER_ARM
<< 0);
120 static unsigned long vgic_mmio_read_its_idregs(struct kvm
*kvm
,
121 struct vgic_its
*its
,
122 gpa_t addr
, unsigned int len
)
124 switch (addr
& 0xffff) {
126 return 0x92; /* part number, bits[7:0] */
128 return 0xb4; /* part number, bits[11:8] */
130 return GIC_PIDR2_ARCH_GICv3
| 0x0b;
132 return 0x40; /* This is a 64K software visible page */
133 /* The following are the ID registers for (any) GIC. */
147 /* Requires the its_lock to be held. */
148 static void its_free_itte(struct kvm
*kvm
, struct its_itte
*itte
)
150 list_del(&itte
->itte_list
);
154 static int vgic_its_handle_command(struct kvm
*kvm
, struct vgic_its
*its
,
160 static u64
vgic_sanitise_its_baser(u64 reg
)
162 reg
= vgic_sanitise_field(reg
, GITS_BASER_SHAREABILITY_MASK
,
163 GITS_BASER_SHAREABILITY_SHIFT
,
164 vgic_sanitise_shareability
);
165 reg
= vgic_sanitise_field(reg
, GITS_BASER_INNER_CACHEABILITY_MASK
,
166 GITS_BASER_INNER_CACHEABILITY_SHIFT
,
167 vgic_sanitise_inner_cacheability
);
168 reg
= vgic_sanitise_field(reg
, GITS_BASER_OUTER_CACHEABILITY_MASK
,
169 GITS_BASER_OUTER_CACHEABILITY_SHIFT
,
170 vgic_sanitise_outer_cacheability
);
172 /* Bits 15:12 contain bits 51:48 of the PA, which we don't support. */
173 reg
&= ~GENMASK_ULL(15, 12);
175 /* We support only one (ITS) page size: 64K */
176 reg
= (reg
& ~GITS_BASER_PAGE_SIZE_MASK
) | GITS_BASER_PAGE_SIZE_64K
;
181 static u64
vgic_sanitise_its_cbaser(u64 reg
)
183 reg
= vgic_sanitise_field(reg
, GITS_CBASER_SHAREABILITY_MASK
,
184 GITS_CBASER_SHAREABILITY_SHIFT
,
185 vgic_sanitise_shareability
);
186 reg
= vgic_sanitise_field(reg
, GITS_CBASER_INNER_CACHEABILITY_MASK
,
187 GITS_CBASER_INNER_CACHEABILITY_SHIFT
,
188 vgic_sanitise_inner_cacheability
);
189 reg
= vgic_sanitise_field(reg
, GITS_CBASER_OUTER_CACHEABILITY_MASK
,
190 GITS_CBASER_OUTER_CACHEABILITY_SHIFT
,
191 vgic_sanitise_outer_cacheability
);
194 * Sanitise the physical address to be 64k aligned.
195 * Also limit the physical addresses to 48 bits.
197 reg
&= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
202 static unsigned long vgic_mmio_read_its_cbaser(struct kvm
*kvm
,
203 struct vgic_its
*its
,
204 gpa_t addr
, unsigned int len
)
206 return extract_bytes(its
->cbaser
, addr
& 7, len
);
209 static void vgic_mmio_write_its_cbaser(struct kvm
*kvm
, struct vgic_its
*its
,
210 gpa_t addr
, unsigned int len
,
213 /* When GITS_CTLR.Enable is 1, this register is RO. */
217 mutex_lock(&its
->cmd_lock
);
218 its
->cbaser
= update_64bit_reg(its
->cbaser
, addr
& 7, len
, val
);
219 its
->cbaser
= vgic_sanitise_its_cbaser(its
->cbaser
);
222 * CWRITER is architecturally UNKNOWN on reset, but we need to reset
223 * it to CREADR to make sure we start with an empty command buffer.
225 its
->cwriter
= its
->creadr
;
226 mutex_unlock(&its
->cmd_lock
);
229 #define ITS_CMD_BUFFER_SIZE(baser) ((((baser) & 0xff) + 1) << 12)
230 #define ITS_CMD_SIZE 32
231 #define ITS_CMD_OFFSET(reg) ((reg) & GENMASK(19, 5))
234 * By writing to CWRITER the guest announces new commands to be processed.
235 * To avoid any races in the first place, we take the its_cmd lock, which
236 * protects our ring buffer variables, so that there is only one user
237 * per ITS handling commands at a given time.
239 static void vgic_mmio_write_its_cwriter(struct kvm
*kvm
, struct vgic_its
*its
,
240 gpa_t addr
, unsigned int len
,
250 mutex_lock(&its
->cmd_lock
);
252 reg
= update_64bit_reg(its
->cwriter
, addr
& 7, len
, val
);
253 reg
= ITS_CMD_OFFSET(reg
);
254 if (reg
>= ITS_CMD_BUFFER_SIZE(its
->cbaser
)) {
255 mutex_unlock(&its
->cmd_lock
);
260 cbaser
= CBASER_ADDRESS(its
->cbaser
);
262 while (its
->cwriter
!= its
->creadr
) {
263 int ret
= kvm_read_guest(kvm
, cbaser
+ its
->creadr
,
264 cmd_buf
, ITS_CMD_SIZE
);
266 * If kvm_read_guest() fails, this could be due to the guest
267 * programming a bogus value in CBASER or something else going
268 * wrong from which we cannot easily recover.
269 * According to section 6.3.2 in the GICv3 spec we can just
270 * ignore that command then.
273 vgic_its_handle_command(kvm
, its
, cmd_buf
);
275 its
->creadr
+= ITS_CMD_SIZE
;
276 if (its
->creadr
== ITS_CMD_BUFFER_SIZE(its
->cbaser
))
280 mutex_unlock(&its
->cmd_lock
);
283 static unsigned long vgic_mmio_read_its_cwriter(struct kvm
*kvm
,
284 struct vgic_its
*its
,
285 gpa_t addr
, unsigned int len
)
287 return extract_bytes(its
->cwriter
, addr
& 0x7, len
);
290 static unsigned long vgic_mmio_read_its_creadr(struct kvm
*kvm
,
291 struct vgic_its
*its
,
292 gpa_t addr
, unsigned int len
)
294 return extract_bytes(its
->creadr
, addr
& 0x7, len
);
297 #define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
298 static unsigned long vgic_mmio_read_its_baser(struct kvm
*kvm
,
299 struct vgic_its
*its
,
300 gpa_t addr
, unsigned int len
)
304 switch (BASER_INDEX(addr
)) {
306 reg
= its
->baser_device_table
;
309 reg
= its
->baser_coll_table
;
316 return extract_bytes(reg
, addr
& 7, len
);
319 #define GITS_BASER_RO_MASK (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
320 static void vgic_mmio_write_its_baser(struct kvm
*kvm
,
321 struct vgic_its
*its
,
322 gpa_t addr
, unsigned int len
,
325 u64 entry_size
, device_type
;
326 u64 reg
, *regptr
, clearbits
= 0;
328 /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
332 switch (BASER_INDEX(addr
)) {
334 regptr
= &its
->baser_device_table
;
336 device_type
= GITS_BASER_TYPE_DEVICE
;
339 regptr
= &its
->baser_coll_table
;
341 device_type
= GITS_BASER_TYPE_COLLECTION
;
342 clearbits
= GITS_BASER_INDIRECT
;
348 reg
= update_64bit_reg(*regptr
, addr
& 7, len
, val
);
349 reg
&= ~GITS_BASER_RO_MASK
;
352 reg
|= (entry_size
- 1) << GITS_BASER_ENTRY_SIZE_SHIFT
;
353 reg
|= device_type
<< GITS_BASER_TYPE_SHIFT
;
354 reg
= vgic_sanitise_its_baser(reg
);
359 #define REGISTER_ITS_DESC(off, rd, wr, length, acc) \
363 .access_flags = acc, \
368 static void its_mmio_write_wi(struct kvm
*kvm
, struct vgic_its
*its
,
369 gpa_t addr
, unsigned int len
, unsigned long val
)
374 static struct vgic_register_region its_registers
[] = {
375 REGISTER_ITS_DESC(GITS_CTLR
,
376 vgic_mmio_read_its_ctlr
, vgic_mmio_write_its_ctlr
, 4,
378 REGISTER_ITS_DESC(GITS_IIDR
,
379 vgic_mmio_read_its_iidr
, its_mmio_write_wi
, 4,
381 REGISTER_ITS_DESC(GITS_TYPER
,
382 vgic_mmio_read_its_typer
, its_mmio_write_wi
, 8,
383 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
384 REGISTER_ITS_DESC(GITS_CBASER
,
385 vgic_mmio_read_its_cbaser
, vgic_mmio_write_its_cbaser
, 8,
386 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
387 REGISTER_ITS_DESC(GITS_CWRITER
,
388 vgic_mmio_read_its_cwriter
, vgic_mmio_write_its_cwriter
, 8,
389 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
390 REGISTER_ITS_DESC(GITS_CREADR
,
391 vgic_mmio_read_its_creadr
, its_mmio_write_wi
, 8,
392 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
393 REGISTER_ITS_DESC(GITS_BASER
,
394 vgic_mmio_read_its_baser
, vgic_mmio_write_its_baser
, 0x40,
395 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
396 REGISTER_ITS_DESC(GITS_IDREGS_BASE
,
397 vgic_mmio_read_its_idregs
, its_mmio_write_wi
, 0x30,
401 static int vgic_its_init_its(struct kvm
*kvm
, struct vgic_its
*its
)
403 struct vgic_io_device
*iodev
= &its
->iodev
;
406 if (its
->initialized
)
409 if (IS_VGIC_ADDR_UNDEF(its
->vgic_its_base
))
412 iodev
->regions
= its_registers
;
413 iodev
->nr_regions
= ARRAY_SIZE(its_registers
);
414 kvm_iodevice_init(&iodev
->dev
, &kvm_io_gic_ops
);
416 iodev
->base_addr
= its
->vgic_its_base
;
417 iodev
->iodev_type
= IODEV_ITS
;
419 mutex_lock(&kvm
->slots_lock
);
420 ret
= kvm_io_bus_register_dev(kvm
, KVM_MMIO_BUS
, iodev
->base_addr
,
421 KVM_VGIC_V3_ITS_SIZE
, &iodev
->dev
);
422 mutex_unlock(&kvm
->slots_lock
);
425 its
->initialized
= true;
430 #define INITIAL_BASER_VALUE \
431 (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) | \
432 GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) | \
433 GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) | \
434 ((8ULL - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | \
435 GITS_BASER_PAGE_SIZE_64K)
437 #define INITIAL_PROPBASER_VALUE \
438 (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb) | \
439 GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner) | \
440 GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
442 static int vgic_its_create(struct kvm_device
*dev
, u32 type
)
444 struct vgic_its
*its
;
446 if (type
!= KVM_DEV_TYPE_ARM_VGIC_ITS
)
449 its
= kzalloc(sizeof(struct vgic_its
), GFP_KERNEL
);
453 mutex_init(&its
->its_lock
);
454 mutex_init(&its
->cmd_lock
);
456 its
->vgic_its_base
= VGIC_ADDR_UNDEF
;
458 INIT_LIST_HEAD(&its
->device_list
);
459 INIT_LIST_HEAD(&its
->collection_list
);
461 dev
->kvm
->arch
.vgic
.has_its
= true;
462 its
->initialized
= false;
463 its
->enabled
= false;
465 its
->baser_device_table
= INITIAL_BASER_VALUE
|
466 ((u64
)GITS_BASER_TYPE_DEVICE
<< GITS_BASER_TYPE_SHIFT
);
467 its
->baser_coll_table
= INITIAL_BASER_VALUE
|
468 ((u64
)GITS_BASER_TYPE_COLLECTION
<< GITS_BASER_TYPE_SHIFT
);
469 dev
->kvm
->arch
.vgic
.propbaser
= INITIAL_PROPBASER_VALUE
;
476 static void vgic_its_destroy(struct kvm_device
*kvm_dev
)
478 struct kvm
*kvm
= kvm_dev
->kvm
;
479 struct vgic_its
*its
= kvm_dev
->private;
480 struct its_device
*dev
;
481 struct its_itte
*itte
;
482 struct list_head
*dev_cur
, *dev_temp
;
483 struct list_head
*cur
, *temp
;
486 * We may end up here without the lists ever having been initialized.
487 * Check this and bail out early to avoid dereferencing a NULL pointer.
489 if (!its
->device_list
.next
)
492 mutex_lock(&its
->its_lock
);
493 list_for_each_safe(dev_cur
, dev_temp
, &its
->device_list
) {
494 dev
= container_of(dev_cur
, struct its_device
, dev_list
);
495 list_for_each_safe(cur
, temp
, &dev
->itt_head
) {
496 itte
= (container_of(cur
, struct its_itte
, itte_list
));
497 its_free_itte(kvm
, itte
);
503 list_for_each_safe(cur
, temp
, &its
->collection_list
) {
505 kfree(container_of(cur
, struct its_collection
, coll_list
));
507 mutex_unlock(&its
->its_lock
);
512 static int vgic_its_has_attr(struct kvm_device
*dev
,
513 struct kvm_device_attr
*attr
)
515 switch (attr
->group
) {
516 case KVM_DEV_ARM_VGIC_GRP_ADDR
:
517 switch (attr
->attr
) {
518 case KVM_VGIC_ITS_ADDR_TYPE
:
522 case KVM_DEV_ARM_VGIC_GRP_CTRL
:
523 switch (attr
->attr
) {
524 case KVM_DEV_ARM_VGIC_CTRL_INIT
:
532 static int vgic_its_set_attr(struct kvm_device
*dev
,
533 struct kvm_device_attr
*attr
)
535 struct vgic_its
*its
= dev
->private;
538 switch (attr
->group
) {
539 case KVM_DEV_ARM_VGIC_GRP_ADDR
: {
540 u64 __user
*uaddr
= (u64 __user
*)(long)attr
->addr
;
541 unsigned long type
= (unsigned long)attr
->attr
;
544 if (type
!= KVM_VGIC_ITS_ADDR_TYPE
)
547 if (its
->initialized
)
550 if (copy_from_user(&addr
, uaddr
, sizeof(addr
)))
553 ret
= vgic_check_ioaddr(dev
->kvm
, &its
->vgic_its_base
,
558 its
->vgic_its_base
= addr
;
562 case KVM_DEV_ARM_VGIC_GRP_CTRL
:
563 switch (attr
->attr
) {
564 case KVM_DEV_ARM_VGIC_CTRL_INIT
:
565 return vgic_its_init_its(dev
->kvm
, its
);
572 static int vgic_its_get_attr(struct kvm_device
*dev
,
573 struct kvm_device_attr
*attr
)
575 switch (attr
->group
) {
576 case KVM_DEV_ARM_VGIC_GRP_ADDR
: {
577 struct vgic_its
*its
= dev
->private;
578 u64 addr
= its
->vgic_its_base
;
579 u64 __user
*uaddr
= (u64 __user
*)(long)attr
->addr
;
580 unsigned long type
= (unsigned long)attr
->attr
;
582 if (type
!= KVM_VGIC_ITS_ADDR_TYPE
)
585 if (copy_to_user(uaddr
, &addr
, sizeof(addr
)))
596 static struct kvm_device_ops kvm_arm_vgic_its_ops
= {
597 .name
= "kvm-arm-vgic-its",
598 .create
= vgic_its_create
,
599 .destroy
= vgic_its_destroy
,
600 .set_attr
= vgic_its_set_attr
,
601 .get_attr
= vgic_its_get_attr
,
602 .has_attr
= vgic_its_has_attr
,
605 int kvm_vgic_register_its_device(void)
607 return kvm_register_device_ops(&kvm_arm_vgic_its_ops
,
608 KVM_DEV_TYPE_ARM_VGIC_ITS
);