4 * Copyright (C) 2015,2016 ARM Ltd.
5 * Author: Andre Przywara <andre.przywara@arm.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/cpu.h>
21 #include <linux/kvm.h>
22 #include <linux/kvm_host.h>
23 #include <linux/interrupt.h>
24 #include <linux/list.h>
25 #include <linux/uaccess.h>
27 #include <linux/irqchip/arm-gic-v3.h>
29 #include <asm/kvm_emulate.h>
30 #include <asm/kvm_arm.h>
31 #include <asm/kvm_mmu.h>
34 #include "vgic-mmio.h"
37 struct list_head dev_list
;
39 /* the head for the list of ITTEs */
40 struct list_head itt_head
;
44 #define COLLECTION_NOT_MAPPED ((u32)~0)
46 struct its_collection
{
47 struct list_head coll_list
;
53 #define its_is_collection_mapped(coll) ((coll) && \
54 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
57 struct list_head itte_list
;
60 struct its_collection
*collection
;
66 * We only implement 48 bits of PA at the moment, although the ITS
67 * supports more. Let's be restrictive here.
69 #define CBASER_ADDRESS(x) ((x) & GENMASK_ULL(47, 12))
71 static unsigned long vgic_mmio_read_its_ctlr(struct kvm
*vcpu
,
73 gpa_t addr
, unsigned int len
)
77 mutex_lock(&its
->cmd_lock
);
78 if (its
->creadr
== its
->cwriter
)
79 reg
|= GITS_CTLR_QUIESCENT
;
81 reg
|= GITS_CTLR_ENABLE
;
82 mutex_unlock(&its
->cmd_lock
);
87 static void vgic_mmio_write_its_ctlr(struct kvm
*kvm
, struct vgic_its
*its
,
88 gpa_t addr
, unsigned int len
,
91 its
->enabled
= !!(val
& GITS_CTLR_ENABLE
);
94 static unsigned long vgic_mmio_read_its_typer(struct kvm
*kvm
,
96 gpa_t addr
, unsigned int len
)
98 u64 reg
= GITS_TYPER_PLPIS
;
101 * We use linear CPU numbers for redistributor addressing,
102 * so GITS_TYPER.PTA is 0.
103 * Also we force all PROPBASER registers to be the same, so
104 * CommonLPIAff is 0 as well.
105 * To avoid memory waste in the guest, we keep the number of IDBits and
106 * DevBits low - as least for the time being.
108 reg
|= 0x0f << GITS_TYPER_DEVBITS_SHIFT
;
109 reg
|= 0x0f << GITS_TYPER_IDBITS_SHIFT
;
111 return extract_bytes(reg
, addr
& 7, len
);
114 static unsigned long vgic_mmio_read_its_iidr(struct kvm
*kvm
,
115 struct vgic_its
*its
,
116 gpa_t addr
, unsigned int len
)
118 return (PRODUCT_ID_KVM
<< 24) | (IMPLEMENTER_ARM
<< 0);
121 static unsigned long vgic_mmio_read_its_idregs(struct kvm
*kvm
,
122 struct vgic_its
*its
,
123 gpa_t addr
, unsigned int len
)
125 switch (addr
& 0xffff) {
127 return 0x92; /* part number, bits[7:0] */
129 return 0xb4; /* part number, bits[11:8] */
131 return GIC_PIDR2_ARCH_GICv3
| 0x0b;
133 return 0x40; /* This is a 64K software visible page */
134 /* The following are the ID registers for (any) GIC. */
148 /* Requires the its_lock to be held. */
149 static void its_free_itte(struct kvm
*kvm
, struct its_itte
*itte
)
151 list_del(&itte
->itte_list
);
153 /* This put matches the get in vgic_add_lpi. */
154 vgic_put_irq(kvm
, itte
->irq
);
159 static int vgic_its_handle_command(struct kvm
*kvm
, struct vgic_its
*its
,
165 static u64
vgic_sanitise_its_baser(u64 reg
)
167 reg
= vgic_sanitise_field(reg
, GITS_BASER_SHAREABILITY_MASK
,
168 GITS_BASER_SHAREABILITY_SHIFT
,
169 vgic_sanitise_shareability
);
170 reg
= vgic_sanitise_field(reg
, GITS_BASER_INNER_CACHEABILITY_MASK
,
171 GITS_BASER_INNER_CACHEABILITY_SHIFT
,
172 vgic_sanitise_inner_cacheability
);
173 reg
= vgic_sanitise_field(reg
, GITS_BASER_OUTER_CACHEABILITY_MASK
,
174 GITS_BASER_OUTER_CACHEABILITY_SHIFT
,
175 vgic_sanitise_outer_cacheability
);
177 /* Bits 15:12 contain bits 51:48 of the PA, which we don't support. */
178 reg
&= ~GENMASK_ULL(15, 12);
180 /* We support only one (ITS) page size: 64K */
181 reg
= (reg
& ~GITS_BASER_PAGE_SIZE_MASK
) | GITS_BASER_PAGE_SIZE_64K
;
186 static u64
vgic_sanitise_its_cbaser(u64 reg
)
188 reg
= vgic_sanitise_field(reg
, GITS_CBASER_SHAREABILITY_MASK
,
189 GITS_CBASER_SHAREABILITY_SHIFT
,
190 vgic_sanitise_shareability
);
191 reg
= vgic_sanitise_field(reg
, GITS_CBASER_INNER_CACHEABILITY_MASK
,
192 GITS_CBASER_INNER_CACHEABILITY_SHIFT
,
193 vgic_sanitise_inner_cacheability
);
194 reg
= vgic_sanitise_field(reg
, GITS_CBASER_OUTER_CACHEABILITY_MASK
,
195 GITS_CBASER_OUTER_CACHEABILITY_SHIFT
,
196 vgic_sanitise_outer_cacheability
);
199 * Sanitise the physical address to be 64k aligned.
200 * Also limit the physical addresses to 48 bits.
202 reg
&= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
207 static unsigned long vgic_mmio_read_its_cbaser(struct kvm
*kvm
,
208 struct vgic_its
*its
,
209 gpa_t addr
, unsigned int len
)
211 return extract_bytes(its
->cbaser
, addr
& 7, len
);
214 static void vgic_mmio_write_its_cbaser(struct kvm
*kvm
, struct vgic_its
*its
,
215 gpa_t addr
, unsigned int len
,
218 /* When GITS_CTLR.Enable is 1, this register is RO. */
222 mutex_lock(&its
->cmd_lock
);
223 its
->cbaser
= update_64bit_reg(its
->cbaser
, addr
& 7, len
, val
);
224 its
->cbaser
= vgic_sanitise_its_cbaser(its
->cbaser
);
227 * CWRITER is architecturally UNKNOWN on reset, but we need to reset
228 * it to CREADR to make sure we start with an empty command buffer.
230 its
->cwriter
= its
->creadr
;
231 mutex_unlock(&its
->cmd_lock
);
234 #define ITS_CMD_BUFFER_SIZE(baser) ((((baser) & 0xff) + 1) << 12)
235 #define ITS_CMD_SIZE 32
236 #define ITS_CMD_OFFSET(reg) ((reg) & GENMASK(19, 5))
239 * By writing to CWRITER the guest announces new commands to be processed.
240 * To avoid any races in the first place, we take the its_cmd lock, which
241 * protects our ring buffer variables, so that there is only one user
242 * per ITS handling commands at a given time.
244 static void vgic_mmio_write_its_cwriter(struct kvm
*kvm
, struct vgic_its
*its
,
245 gpa_t addr
, unsigned int len
,
255 mutex_lock(&its
->cmd_lock
);
257 reg
= update_64bit_reg(its
->cwriter
, addr
& 7, len
, val
);
258 reg
= ITS_CMD_OFFSET(reg
);
259 if (reg
>= ITS_CMD_BUFFER_SIZE(its
->cbaser
)) {
260 mutex_unlock(&its
->cmd_lock
);
265 cbaser
= CBASER_ADDRESS(its
->cbaser
);
267 while (its
->cwriter
!= its
->creadr
) {
268 int ret
= kvm_read_guest(kvm
, cbaser
+ its
->creadr
,
269 cmd_buf
, ITS_CMD_SIZE
);
271 * If kvm_read_guest() fails, this could be due to the guest
272 * programming a bogus value in CBASER or something else going
273 * wrong from which we cannot easily recover.
274 * According to section 6.3.2 in the GICv3 spec we can just
275 * ignore that command then.
278 vgic_its_handle_command(kvm
, its
, cmd_buf
);
280 its
->creadr
+= ITS_CMD_SIZE
;
281 if (its
->creadr
== ITS_CMD_BUFFER_SIZE(its
->cbaser
))
285 mutex_unlock(&its
->cmd_lock
);
288 static unsigned long vgic_mmio_read_its_cwriter(struct kvm
*kvm
,
289 struct vgic_its
*its
,
290 gpa_t addr
, unsigned int len
)
292 return extract_bytes(its
->cwriter
, addr
& 0x7, len
);
295 static unsigned long vgic_mmio_read_its_creadr(struct kvm
*kvm
,
296 struct vgic_its
*its
,
297 gpa_t addr
, unsigned int len
)
299 return extract_bytes(its
->creadr
, addr
& 0x7, len
);
302 #define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
303 static unsigned long vgic_mmio_read_its_baser(struct kvm
*kvm
,
304 struct vgic_its
*its
,
305 gpa_t addr
, unsigned int len
)
309 switch (BASER_INDEX(addr
)) {
311 reg
= its
->baser_device_table
;
314 reg
= its
->baser_coll_table
;
321 return extract_bytes(reg
, addr
& 7, len
);
324 #define GITS_BASER_RO_MASK (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
325 static void vgic_mmio_write_its_baser(struct kvm
*kvm
,
326 struct vgic_its
*its
,
327 gpa_t addr
, unsigned int len
,
330 u64 entry_size
, device_type
;
331 u64 reg
, *regptr
, clearbits
= 0;
333 /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
337 switch (BASER_INDEX(addr
)) {
339 regptr
= &its
->baser_device_table
;
341 device_type
= GITS_BASER_TYPE_DEVICE
;
344 regptr
= &its
->baser_coll_table
;
346 device_type
= GITS_BASER_TYPE_COLLECTION
;
347 clearbits
= GITS_BASER_INDIRECT
;
353 reg
= update_64bit_reg(*regptr
, addr
& 7, len
, val
);
354 reg
&= ~GITS_BASER_RO_MASK
;
357 reg
|= (entry_size
- 1) << GITS_BASER_ENTRY_SIZE_SHIFT
;
358 reg
|= device_type
<< GITS_BASER_TYPE_SHIFT
;
359 reg
= vgic_sanitise_its_baser(reg
);
364 #define REGISTER_ITS_DESC(off, rd, wr, length, acc) \
368 .access_flags = acc, \
373 static void its_mmio_write_wi(struct kvm
*kvm
, struct vgic_its
*its
,
374 gpa_t addr
, unsigned int len
, unsigned long val
)
379 static struct vgic_register_region its_registers
[] = {
380 REGISTER_ITS_DESC(GITS_CTLR
,
381 vgic_mmio_read_its_ctlr
, vgic_mmio_write_its_ctlr
, 4,
383 REGISTER_ITS_DESC(GITS_IIDR
,
384 vgic_mmio_read_its_iidr
, its_mmio_write_wi
, 4,
386 REGISTER_ITS_DESC(GITS_TYPER
,
387 vgic_mmio_read_its_typer
, its_mmio_write_wi
, 8,
388 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
389 REGISTER_ITS_DESC(GITS_CBASER
,
390 vgic_mmio_read_its_cbaser
, vgic_mmio_write_its_cbaser
, 8,
391 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
392 REGISTER_ITS_DESC(GITS_CWRITER
,
393 vgic_mmio_read_its_cwriter
, vgic_mmio_write_its_cwriter
, 8,
394 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
395 REGISTER_ITS_DESC(GITS_CREADR
,
396 vgic_mmio_read_its_creadr
, its_mmio_write_wi
, 8,
397 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
398 REGISTER_ITS_DESC(GITS_BASER
,
399 vgic_mmio_read_its_baser
, vgic_mmio_write_its_baser
, 0x40,
400 VGIC_ACCESS_64bit
| VGIC_ACCESS_32bit
),
401 REGISTER_ITS_DESC(GITS_IDREGS_BASE
,
402 vgic_mmio_read_its_idregs
, its_mmio_write_wi
, 0x30,
406 static int vgic_its_init_its(struct kvm
*kvm
, struct vgic_its
*its
)
408 struct vgic_io_device
*iodev
= &its
->iodev
;
411 if (its
->initialized
)
414 if (IS_VGIC_ADDR_UNDEF(its
->vgic_its_base
))
417 iodev
->regions
= its_registers
;
418 iodev
->nr_regions
= ARRAY_SIZE(its_registers
);
419 kvm_iodevice_init(&iodev
->dev
, &kvm_io_gic_ops
);
421 iodev
->base_addr
= its
->vgic_its_base
;
422 iodev
->iodev_type
= IODEV_ITS
;
424 mutex_lock(&kvm
->slots_lock
);
425 ret
= kvm_io_bus_register_dev(kvm
, KVM_MMIO_BUS
, iodev
->base_addr
,
426 KVM_VGIC_V3_ITS_SIZE
, &iodev
->dev
);
427 mutex_unlock(&kvm
->slots_lock
);
430 its
->initialized
= true;
435 #define INITIAL_BASER_VALUE \
436 (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) | \
437 GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) | \
438 GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) | \
439 ((8ULL - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | \
440 GITS_BASER_PAGE_SIZE_64K)
442 #define INITIAL_PROPBASER_VALUE \
443 (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb) | \
444 GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner) | \
445 GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
447 static int vgic_its_create(struct kvm_device
*dev
, u32 type
)
449 struct vgic_its
*its
;
451 if (type
!= KVM_DEV_TYPE_ARM_VGIC_ITS
)
454 its
= kzalloc(sizeof(struct vgic_its
), GFP_KERNEL
);
458 mutex_init(&its
->its_lock
);
459 mutex_init(&its
->cmd_lock
);
461 its
->vgic_its_base
= VGIC_ADDR_UNDEF
;
463 INIT_LIST_HEAD(&its
->device_list
);
464 INIT_LIST_HEAD(&its
->collection_list
);
466 dev
->kvm
->arch
.vgic
.has_its
= true;
467 its
->initialized
= false;
468 its
->enabled
= false;
470 its
->baser_device_table
= INITIAL_BASER_VALUE
|
471 ((u64
)GITS_BASER_TYPE_DEVICE
<< GITS_BASER_TYPE_SHIFT
);
472 its
->baser_coll_table
= INITIAL_BASER_VALUE
|
473 ((u64
)GITS_BASER_TYPE_COLLECTION
<< GITS_BASER_TYPE_SHIFT
);
474 dev
->kvm
->arch
.vgic
.propbaser
= INITIAL_PROPBASER_VALUE
;
481 static void vgic_its_destroy(struct kvm_device
*kvm_dev
)
483 struct kvm
*kvm
= kvm_dev
->kvm
;
484 struct vgic_its
*its
= kvm_dev
->private;
485 struct its_device
*dev
;
486 struct its_itte
*itte
;
487 struct list_head
*dev_cur
, *dev_temp
;
488 struct list_head
*cur
, *temp
;
491 * We may end up here without the lists ever having been initialized.
492 * Check this and bail out early to avoid dereferencing a NULL pointer.
494 if (!its
->device_list
.next
)
497 mutex_lock(&its
->its_lock
);
498 list_for_each_safe(dev_cur
, dev_temp
, &its
->device_list
) {
499 dev
= container_of(dev_cur
, struct its_device
, dev_list
);
500 list_for_each_safe(cur
, temp
, &dev
->itt_head
) {
501 itte
= (container_of(cur
, struct its_itte
, itte_list
));
502 its_free_itte(kvm
, itte
);
508 list_for_each_safe(cur
, temp
, &its
->collection_list
) {
510 kfree(container_of(cur
, struct its_collection
, coll_list
));
512 mutex_unlock(&its
->its_lock
);
517 static int vgic_its_has_attr(struct kvm_device
*dev
,
518 struct kvm_device_attr
*attr
)
520 switch (attr
->group
) {
521 case KVM_DEV_ARM_VGIC_GRP_ADDR
:
522 switch (attr
->attr
) {
523 case KVM_VGIC_ITS_ADDR_TYPE
:
527 case KVM_DEV_ARM_VGIC_GRP_CTRL
:
528 switch (attr
->attr
) {
529 case KVM_DEV_ARM_VGIC_CTRL_INIT
:
537 static int vgic_its_set_attr(struct kvm_device
*dev
,
538 struct kvm_device_attr
*attr
)
540 struct vgic_its
*its
= dev
->private;
543 switch (attr
->group
) {
544 case KVM_DEV_ARM_VGIC_GRP_ADDR
: {
545 u64 __user
*uaddr
= (u64 __user
*)(long)attr
->addr
;
546 unsigned long type
= (unsigned long)attr
->attr
;
549 if (type
!= KVM_VGIC_ITS_ADDR_TYPE
)
552 if (its
->initialized
)
555 if (copy_from_user(&addr
, uaddr
, sizeof(addr
)))
558 ret
= vgic_check_ioaddr(dev
->kvm
, &its
->vgic_its_base
,
563 its
->vgic_its_base
= addr
;
567 case KVM_DEV_ARM_VGIC_GRP_CTRL
:
568 switch (attr
->attr
) {
569 case KVM_DEV_ARM_VGIC_CTRL_INIT
:
570 return vgic_its_init_its(dev
->kvm
, its
);
577 static int vgic_its_get_attr(struct kvm_device
*dev
,
578 struct kvm_device_attr
*attr
)
580 switch (attr
->group
) {
581 case KVM_DEV_ARM_VGIC_GRP_ADDR
: {
582 struct vgic_its
*its
= dev
->private;
583 u64 addr
= its
->vgic_its_base
;
584 u64 __user
*uaddr
= (u64 __user
*)(long)attr
->addr
;
585 unsigned long type
= (unsigned long)attr
->attr
;
587 if (type
!= KVM_VGIC_ITS_ADDR_TYPE
)
590 if (copy_to_user(uaddr
, &addr
, sizeof(addr
)))
601 static struct kvm_device_ops kvm_arm_vgic_its_ops
= {
602 .name
= "kvm-arm-vgic-its",
603 .create
= vgic_its_create
,
604 .destroy
= vgic_its_destroy
,
605 .set_attr
= vgic_its_set_attr
,
606 .get_attr
= vgic_its_get_attr
,
607 .has_attr
= vgic_its_has_attr
,
610 int kvm_vgic_register_its_device(void)
612 return kvm_register_device_ops(&kvm_arm_vgic_its_ops
,
613 KVM_DEV_TYPE_ARM_VGIC_ITS
);