2 #define pr_fmt(fmt) "DMAR-IR: " fmt
4 #include <linux/interrupt.h>
5 #include <linux/dmar.h>
6 #include <linux/spinlock.h>
7 #include <linux/slab.h>
8 #include <linux/jiffies.h>
9 #include <linux/hpet.h>
10 #include <linux/pci.h>
11 #include <linux/irq.h>
12 #include <linux/intel-iommu.h>
13 #include <linux/acpi.h>
14 #include <linux/irqdomain.h>
15 #include <linux/crash_dump.h>
16 #include <asm/io_apic.h>
19 #include <asm/irq_remapping.h>
20 #include <asm/pci-direct.h>
21 #include <asm/msidef.h>
23 #include "irq_remapping.h"
31 struct intel_iommu
*iommu
;
33 unsigned int bus
; /* PCI bus number */
34 unsigned int devfn
; /* PCI devfn number */
38 struct intel_iommu
*iommu
;
45 struct intel_iommu
*iommu
;
52 struct intel_ir_data
{
53 struct irq_2_iommu irq_2_iommu
;
54 struct irte irte_entry
;
56 struct msi_msg msi_entry
;
60 #define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
61 #define IRTE_DEST(dest) ((eim_mode) ? dest : dest << 8)
63 static int __read_mostly eim_mode
;
64 static struct ioapic_scope ir_ioapic
[MAX_IO_APICS
];
65 static struct hpet_scope ir_hpet
[MAX_HPET_TBS
];
72 * ->iommu->register_lock
74 * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called
75 * in single-threaded environment with interrupt disabled, so no need to tabke
76 * the dmar_global_lock.
78 static DEFINE_RAW_SPINLOCK(irq_2_ir_lock
);
79 static struct irq_domain_ops intel_ir_domain_ops
;
81 static void iommu_disable_irq_remapping(struct intel_iommu
*iommu
);
82 static int __init
parse_ioapics_under_ir(void);
84 static bool ir_pre_enabled(struct intel_iommu
*iommu
)
86 return (iommu
->flags
& VTD_FLAG_IRQ_REMAP_PRE_ENABLED
);
89 static void clear_ir_pre_enabled(struct intel_iommu
*iommu
)
91 iommu
->flags
&= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED
;
94 static void init_ir_status(struct intel_iommu
*iommu
)
98 gsts
= readl(iommu
->reg
+ DMAR_GSTS_REG
);
99 if (gsts
& DMA_GSTS_IRES
)
100 iommu
->flags
|= VTD_FLAG_IRQ_REMAP_PRE_ENABLED
;
103 static int alloc_irte(struct intel_iommu
*iommu
, int irq
,
104 struct irq_2_iommu
*irq_iommu
, u16 count
)
106 struct ir_table
*table
= iommu
->ir_table
;
107 unsigned int mask
= 0;
111 if (!count
|| !irq_iommu
)
115 count
= __roundup_pow_of_two(count
);
119 if (mask
> ecap_max_handle_mask(iommu
->ecap
)) {
120 pr_err("Requested mask %x exceeds the max invalidation handle"
121 " mask value %Lx\n", mask
,
122 ecap_max_handle_mask(iommu
->ecap
));
126 raw_spin_lock_irqsave(&irq_2_ir_lock
, flags
);
127 index
= bitmap_find_free_region(table
->bitmap
,
128 INTR_REMAP_TABLE_ENTRIES
, mask
);
130 pr_warn("IR%d: can't allocate an IRTE\n", iommu
->seq_id
);
132 irq_iommu
->iommu
= iommu
;
133 irq_iommu
->irte_index
= index
;
134 irq_iommu
->sub_handle
= 0;
135 irq_iommu
->irte_mask
= mask
;
136 irq_iommu
->mode
= IRQ_REMAPPING
;
138 raw_spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
143 static int qi_flush_iec(struct intel_iommu
*iommu
, int index
, int mask
)
147 desc
.low
= QI_IEC_IIDEX(index
) | QI_IEC_TYPE
| QI_IEC_IM(mask
)
151 return qi_submit_sync(&desc
, iommu
);
154 static int modify_irte(struct irq_2_iommu
*irq_iommu
,
155 struct irte
*irte_modified
)
157 struct intel_iommu
*iommu
;
165 raw_spin_lock_irqsave(&irq_2_ir_lock
, flags
);
167 iommu
= irq_iommu
->iommu
;
169 index
= irq_iommu
->irte_index
+ irq_iommu
->sub_handle
;
170 irte
= &iommu
->ir_table
->base
[index
];
172 set_64bit(&irte
->low
, irte_modified
->low
);
173 set_64bit(&irte
->high
, irte_modified
->high
);
174 __iommu_flush_cache(iommu
, irte
, sizeof(*irte
));
176 rc
= qi_flush_iec(iommu
, index
, 0);
178 /* Update iommu mode according to the IRTE mode */
179 irq_iommu
->mode
= irte
->pst
? IRQ_POSTING
: IRQ_REMAPPING
;
180 raw_spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
185 static struct intel_iommu
*map_hpet_to_ir(u8 hpet_id
)
189 for (i
= 0; i
< MAX_HPET_TBS
; i
++)
190 if (ir_hpet
[i
].id
== hpet_id
&& ir_hpet
[i
].iommu
)
191 return ir_hpet
[i
].iommu
;
195 static struct intel_iommu
*map_ioapic_to_ir(int apic
)
199 for (i
= 0; i
< MAX_IO_APICS
; i
++)
200 if (ir_ioapic
[i
].id
== apic
&& ir_ioapic
[i
].iommu
)
201 return ir_ioapic
[i
].iommu
;
205 static struct intel_iommu
*map_dev_to_ir(struct pci_dev
*dev
)
207 struct dmar_drhd_unit
*drhd
;
209 drhd
= dmar_find_matched_drhd_unit(dev
);
216 static int clear_entries(struct irq_2_iommu
*irq_iommu
)
218 struct irte
*start
, *entry
, *end
;
219 struct intel_iommu
*iommu
;
222 if (irq_iommu
->sub_handle
)
225 iommu
= irq_iommu
->iommu
;
226 index
= irq_iommu
->irte_index
;
228 start
= iommu
->ir_table
->base
+ index
;
229 end
= start
+ (1 << irq_iommu
->irte_mask
);
231 for (entry
= start
; entry
< end
; entry
++) {
232 set_64bit(&entry
->low
, 0);
233 set_64bit(&entry
->high
, 0);
235 bitmap_release_region(iommu
->ir_table
->bitmap
, index
,
236 irq_iommu
->irte_mask
);
238 return qi_flush_iec(iommu
, index
, irq_iommu
->irte_mask
);
242 * source validation type
244 #define SVT_NO_VERIFY 0x0 /* no verification is required */
245 #define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */
246 #define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
249 * source-id qualifier
251 #define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
252 #define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
253 * the third least significant bit
255 #define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
256 * the second and third least significant bits
258 #define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
259 * the least three significant bits
263 * set SVT, SQ and SID fields of irte to verify
264 * source ids of interrupt requests
266 static void set_irte_sid(struct irte
*irte
, unsigned int svt
,
267 unsigned int sq
, unsigned int sid
)
269 if (disable_sourceid_checking
)
276 static int set_ioapic_sid(struct irte
*irte
, int apic
)
284 down_read(&dmar_global_lock
);
285 for (i
= 0; i
< MAX_IO_APICS
; i
++) {
286 if (ir_ioapic
[i
].iommu
&& ir_ioapic
[i
].id
== apic
) {
287 sid
= (ir_ioapic
[i
].bus
<< 8) | ir_ioapic
[i
].devfn
;
291 up_read(&dmar_global_lock
);
294 pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic
);
298 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_ALL_16
, sid
);
303 static int set_hpet_sid(struct irte
*irte
, u8 id
)
311 down_read(&dmar_global_lock
);
312 for (i
= 0; i
< MAX_HPET_TBS
; i
++) {
313 if (ir_hpet
[i
].iommu
&& ir_hpet
[i
].id
== id
) {
314 sid
= (ir_hpet
[i
].bus
<< 8) | ir_hpet
[i
].devfn
;
318 up_read(&dmar_global_lock
);
321 pr_warn("Failed to set source-id of HPET block (%d)\n", id
);
326 * Should really use SQ_ALL_16. Some platforms are broken.
327 * While we figure out the right quirks for these broken platforms, use
328 * SQ_13_IGNORE_3 for now.
330 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_13_IGNORE_3
, sid
);
335 struct set_msi_sid_data
{
336 struct pci_dev
*pdev
;
340 static int set_msi_sid_cb(struct pci_dev
*pdev
, u16 alias
, void *opaque
)
342 struct set_msi_sid_data
*data
= opaque
;
350 static int set_msi_sid(struct irte
*irte
, struct pci_dev
*dev
)
352 struct set_msi_sid_data data
;
357 pci_for_each_dma_alias(dev
, set_msi_sid_cb
, &data
);
360 * DMA alias provides us with a PCI device and alias. The only case
361 * where the it will return an alias on a different bus than the
362 * device is the case of a PCIe-to-PCI bridge, where the alias is for
363 * the subordinate bus. In this case we can only verify the bus.
365 * If the alias device is on a different bus than our source device
366 * then we have a topology based alias, use it.
368 * Otherwise, the alias is for a device DMA quirk and we cannot
369 * assume that MSI uses the same requester ID. Therefore use the
372 if (PCI_BUS_NUM(data
.alias
) != data
.pdev
->bus
->number
)
373 set_irte_sid(irte
, SVT_VERIFY_BUS
, SQ_ALL_16
,
374 PCI_DEVID(PCI_BUS_NUM(data
.alias
),
376 else if (data
.pdev
->bus
->number
!= dev
->bus
->number
)
377 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_ALL_16
, data
.alias
);
379 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_ALL_16
,
380 PCI_DEVID(dev
->bus
->number
, dev
->devfn
));
385 static int iommu_load_old_irte(struct intel_iommu
*iommu
)
387 struct irte __iomem
*old_ir_table
;
388 phys_addr_t irt_phys
;
393 if (!is_kdump_kernel()) {
394 pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n",
396 clear_ir_pre_enabled(iommu
);
397 iommu_disable_irq_remapping(iommu
);
401 /* Check whether the old ir-table has the same size as ours */
402 irta
= dmar_readq(iommu
->reg
+ DMAR_IRTA_REG
);
403 if ((irta
& INTR_REMAP_TABLE_REG_SIZE_MASK
)
404 != INTR_REMAP_TABLE_REG_SIZE
)
407 irt_phys
= irta
& VTD_PAGE_MASK
;
408 size
= INTR_REMAP_TABLE_ENTRIES
*sizeof(struct irte
);
410 /* Map the old IR table */
411 old_ir_table
= ioremap_cache(irt_phys
, size
);
416 memcpy_fromio(iommu
->ir_table
->base
, old_ir_table
, size
);
418 __iommu_flush_cache(iommu
, iommu
->ir_table
->base
, size
);
421 * Now check the table for used entries and mark those as
422 * allocated in the bitmap
424 for (i
= 0; i
< INTR_REMAP_TABLE_ENTRIES
; i
++) {
425 if (iommu
->ir_table
->base
[i
].present
)
426 bitmap_set(iommu
->ir_table
->bitmap
, i
, 1);
429 iounmap(old_ir_table
);
435 static void iommu_set_irq_remapping(struct intel_iommu
*iommu
, int mode
)
441 addr
= virt_to_phys((void *)iommu
->ir_table
->base
);
443 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
445 dmar_writeq(iommu
->reg
+ DMAR_IRTA_REG
,
446 (addr
) | IR_X2APIC_MODE(mode
) | INTR_REMAP_TABLE_REG_SIZE
);
448 /* Set interrupt-remapping table pointer */
449 writel(iommu
->gcmd
| DMA_GCMD_SIRTP
, iommu
->reg
+ DMAR_GCMD_REG
);
451 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
452 readl
, (sts
& DMA_GSTS_IRTPS
), sts
);
453 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
456 * Global invalidation of interrupt entry cache to make sure the
457 * hardware uses the new irq remapping table.
459 qi_global_iec(iommu
);
462 static void iommu_enable_irq_remapping(struct intel_iommu
*iommu
)
467 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
469 /* Enable interrupt-remapping */
470 iommu
->gcmd
|= DMA_GCMD_IRE
;
471 iommu
->gcmd
&= ~DMA_GCMD_CFI
; /* Block compatibility-format MSIs */
472 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
474 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
475 readl
, (sts
& DMA_GSTS_IRES
), sts
);
478 * With CFI clear in the Global Command register, we should be
479 * protected from dangerous (i.e. compatibility) interrupts
480 * regardless of x2apic status. Check just to be sure.
482 if (sts
& DMA_GSTS_CFIS
)
484 "Compatibility-format IRQs enabled despite intr remapping;\n"
485 "you are vulnerable to IRQ injection.\n");
487 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
490 static int intel_setup_irq_remapping(struct intel_iommu
*iommu
)
492 struct ir_table
*ir_table
;
494 unsigned long *bitmap
;
499 ir_table
= kzalloc(sizeof(struct ir_table
), GFP_KERNEL
);
503 pages
= alloc_pages_node(iommu
->node
, GFP_KERNEL
| __GFP_ZERO
,
504 INTR_REMAP_PAGE_ORDER
);
506 pr_err("IR%d: failed to allocate pages of order %d\n",
507 iommu
->seq_id
, INTR_REMAP_PAGE_ORDER
);
511 bitmap
= kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES
),
512 sizeof(long), GFP_ATOMIC
);
513 if (bitmap
== NULL
) {
514 pr_err("IR%d: failed to allocate bitmap\n", iommu
->seq_id
);
518 iommu
->ir_domain
= irq_domain_add_hierarchy(arch_get_ir_parent_domain(),
519 0, INTR_REMAP_TABLE_ENTRIES
,
520 NULL
, &intel_ir_domain_ops
,
522 if (!iommu
->ir_domain
) {
523 pr_err("IR%d: failed to allocate irqdomain\n", iommu
->seq_id
);
524 goto out_free_bitmap
;
526 iommu
->ir_msi_domain
= arch_create_msi_irq_domain(iommu
->ir_domain
);
528 ir_table
->base
= page_address(pages
);
529 ir_table
->bitmap
= bitmap
;
530 iommu
->ir_table
= ir_table
;
533 * If the queued invalidation is already initialized,
534 * shouldn't disable it.
538 * Clear previous faults.
540 dmar_fault(-1, iommu
);
541 dmar_disable_qi(iommu
);
543 if (dmar_enable_qi(iommu
)) {
544 pr_err("Failed to enable queued invalidation\n");
545 goto out_free_bitmap
;
549 init_ir_status(iommu
);
551 if (ir_pre_enabled(iommu
)) {
552 if (iommu_load_old_irte(iommu
))
553 pr_err("Failed to copy IR table for %s from previous kernel\n",
556 pr_info("Copied IR table for %s from previous kernel\n",
560 iommu_set_irq_remapping(iommu
, eim_mode
);
567 __free_pages(pages
, INTR_REMAP_PAGE_ORDER
);
571 iommu
->ir_table
= NULL
;
576 static void intel_teardown_irq_remapping(struct intel_iommu
*iommu
)
578 if (iommu
&& iommu
->ir_table
) {
579 if (iommu
->ir_msi_domain
) {
580 irq_domain_remove(iommu
->ir_msi_domain
);
581 iommu
->ir_msi_domain
= NULL
;
583 if (iommu
->ir_domain
) {
584 irq_domain_remove(iommu
->ir_domain
);
585 iommu
->ir_domain
= NULL
;
587 free_pages((unsigned long)iommu
->ir_table
->base
,
588 INTR_REMAP_PAGE_ORDER
);
589 kfree(iommu
->ir_table
->bitmap
);
590 kfree(iommu
->ir_table
);
591 iommu
->ir_table
= NULL
;
596 * Disable Interrupt Remapping.
598 static void iommu_disable_irq_remapping(struct intel_iommu
*iommu
)
603 if (!ecap_ir_support(iommu
->ecap
))
607 * global invalidation of interrupt entry cache before disabling
608 * interrupt-remapping.
610 qi_global_iec(iommu
);
612 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
614 sts
= dmar_readq(iommu
->reg
+ DMAR_GSTS_REG
);
615 if (!(sts
& DMA_GSTS_IRES
))
618 iommu
->gcmd
&= ~DMA_GCMD_IRE
;
619 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
621 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
622 readl
, !(sts
& DMA_GSTS_IRES
), sts
);
625 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
628 static int __init
dmar_x2apic_optout(void)
630 struct acpi_table_dmar
*dmar
;
631 dmar
= (struct acpi_table_dmar
*)dmar_tbl
;
632 if (!dmar
|| no_x2apic_optout
)
634 return dmar
->flags
& DMAR_X2APIC_OPT_OUT
;
637 static void __init
intel_cleanup_irq_remapping(void)
639 struct dmar_drhd_unit
*drhd
;
640 struct intel_iommu
*iommu
;
642 for_each_iommu(iommu
, drhd
) {
643 if (ecap_ir_support(iommu
->ecap
)) {
644 iommu_disable_irq_remapping(iommu
);
645 intel_teardown_irq_remapping(iommu
);
649 if (x2apic_supported())
650 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
653 static int __init
intel_prepare_irq_remapping(void)
655 struct dmar_drhd_unit
*drhd
;
656 struct intel_iommu
*iommu
;
659 if (irq_remap_broken
) {
660 pr_warn("This system BIOS has enabled interrupt remapping\n"
661 "on a chipset that contains an erratum making that\n"
662 "feature unstable. To maintain system stability\n"
663 "interrupt remapping is being disabled. Please\n"
664 "contact your BIOS vendor for an update\n");
665 add_taint(TAINT_FIRMWARE_WORKAROUND
, LOCKDEP_STILL_OK
);
669 if (dmar_table_init() < 0)
672 if (!dmar_ir_support())
675 if (parse_ioapics_under_ir() != 1) {
676 pr_info("Not enabling interrupt remapping\n");
680 /* First make sure all IOMMUs support IRQ remapping */
681 for_each_iommu(iommu
, drhd
)
682 if (!ecap_ir_support(iommu
->ecap
))
685 /* Detect remapping mode: lapic or x2apic */
686 if (x2apic_supported()) {
687 eim
= !dmar_x2apic_optout();
689 pr_info("x2apic is disabled because BIOS sets x2apic opt out bit.");
690 pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n");
694 for_each_iommu(iommu
, drhd
) {
695 if (eim
&& !ecap_eim_support(iommu
->ecap
)) {
696 pr_info("%s does not support EIM\n", iommu
->name
);
703 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
705 /* Do the initializations early */
706 for_each_iommu(iommu
, drhd
) {
707 if (intel_setup_irq_remapping(iommu
)) {
708 pr_err("Failed to setup irq remapping for %s\n",
717 intel_cleanup_irq_remapping();
722 * Set Posted-Interrupts capability.
724 static inline void set_irq_posting_cap(void)
726 struct dmar_drhd_unit
*drhd
;
727 struct intel_iommu
*iommu
;
729 if (!disable_irq_post
) {
730 intel_irq_remap_ops
.capability
|= 1 << IRQ_POSTING_CAP
;
732 for_each_iommu(iommu
, drhd
)
733 if (!cap_pi_support(iommu
->cap
)) {
734 intel_irq_remap_ops
.capability
&=
735 ~(1 << IRQ_POSTING_CAP
);
741 static int __init
intel_enable_irq_remapping(void)
743 struct dmar_drhd_unit
*drhd
;
744 struct intel_iommu
*iommu
;
748 * Setup Interrupt-remapping for all the DRHD's now.
750 for_each_iommu(iommu
, drhd
) {
751 if (!ir_pre_enabled(iommu
))
752 iommu_enable_irq_remapping(iommu
);
759 irq_remapping_enabled
= 1;
761 set_irq_posting_cap();
763 pr_info("Enabled IRQ remapping in %s mode\n", eim_mode
? "x2apic" : "xapic");
765 return eim_mode
? IRQ_REMAP_X2APIC_MODE
: IRQ_REMAP_XAPIC_MODE
;
768 intel_cleanup_irq_remapping();
772 static int ir_parse_one_hpet_scope(struct acpi_dmar_device_scope
*scope
,
773 struct intel_iommu
*iommu
,
774 struct acpi_dmar_hardware_unit
*drhd
)
776 struct acpi_dmar_pci_path
*path
;
778 int count
, free
= -1;
781 path
= (struct acpi_dmar_pci_path
*)(scope
+ 1);
782 count
= (scope
->length
- sizeof(struct acpi_dmar_device_scope
))
783 / sizeof(struct acpi_dmar_pci_path
);
785 while (--count
> 0) {
787 * Access PCI directly due to the PCI
788 * subsystem isn't initialized yet.
790 bus
= read_pci_config_byte(bus
, path
->device
, path
->function
,
795 for (count
= 0; count
< MAX_HPET_TBS
; count
++) {
796 if (ir_hpet
[count
].iommu
== iommu
&&
797 ir_hpet
[count
].id
== scope
->enumeration_id
)
799 else if (ir_hpet
[count
].iommu
== NULL
&& free
== -1)
803 pr_warn("Exceeded Max HPET blocks\n");
807 ir_hpet
[free
].iommu
= iommu
;
808 ir_hpet
[free
].id
= scope
->enumeration_id
;
809 ir_hpet
[free
].bus
= bus
;
810 ir_hpet
[free
].devfn
= PCI_DEVFN(path
->device
, path
->function
);
811 pr_info("HPET id %d under DRHD base 0x%Lx\n",
812 scope
->enumeration_id
, drhd
->address
);
817 static int ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope
*scope
,
818 struct intel_iommu
*iommu
,
819 struct acpi_dmar_hardware_unit
*drhd
)
821 struct acpi_dmar_pci_path
*path
;
823 int count
, free
= -1;
826 path
= (struct acpi_dmar_pci_path
*)(scope
+ 1);
827 count
= (scope
->length
- sizeof(struct acpi_dmar_device_scope
))
828 / sizeof(struct acpi_dmar_pci_path
);
830 while (--count
> 0) {
832 * Access PCI directly due to the PCI
833 * subsystem isn't initialized yet.
835 bus
= read_pci_config_byte(bus
, path
->device
, path
->function
,
840 for (count
= 0; count
< MAX_IO_APICS
; count
++) {
841 if (ir_ioapic
[count
].iommu
== iommu
&&
842 ir_ioapic
[count
].id
== scope
->enumeration_id
)
844 else if (ir_ioapic
[count
].iommu
== NULL
&& free
== -1)
848 pr_warn("Exceeded Max IO APICS\n");
852 ir_ioapic
[free
].bus
= bus
;
853 ir_ioapic
[free
].devfn
= PCI_DEVFN(path
->device
, path
->function
);
854 ir_ioapic
[free
].iommu
= iommu
;
855 ir_ioapic
[free
].id
= scope
->enumeration_id
;
856 pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n",
857 scope
->enumeration_id
, drhd
->address
, iommu
->seq_id
);
862 static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header
*header
,
863 struct intel_iommu
*iommu
)
866 struct acpi_dmar_hardware_unit
*drhd
;
867 struct acpi_dmar_device_scope
*scope
;
870 drhd
= (struct acpi_dmar_hardware_unit
*)header
;
871 start
= (void *)(drhd
+ 1);
872 end
= ((void *)drhd
) + header
->length
;
874 while (start
< end
&& ret
== 0) {
876 if (scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_IOAPIC
)
877 ret
= ir_parse_one_ioapic_scope(scope
, iommu
, drhd
);
878 else if (scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_HPET
)
879 ret
= ir_parse_one_hpet_scope(scope
, iommu
, drhd
);
880 start
+= scope
->length
;
886 static void ir_remove_ioapic_hpet_scope(struct intel_iommu
*iommu
)
890 for (i
= 0; i
< MAX_HPET_TBS
; i
++)
891 if (ir_hpet
[i
].iommu
== iommu
)
892 ir_hpet
[i
].iommu
= NULL
;
894 for (i
= 0; i
< MAX_IO_APICS
; i
++)
895 if (ir_ioapic
[i
].iommu
== iommu
)
896 ir_ioapic
[i
].iommu
= NULL
;
900 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
903 static int __init
parse_ioapics_under_ir(void)
905 struct dmar_drhd_unit
*drhd
;
906 struct intel_iommu
*iommu
;
907 bool ir_supported
= false;
910 for_each_iommu(iommu
, drhd
)
911 if (ecap_ir_support(iommu
->ecap
)) {
912 if (ir_parse_ioapic_hpet_scope(drhd
->hdr
, iommu
))
921 for (ioapic_idx
= 0; ioapic_idx
< nr_ioapics
; ioapic_idx
++) {
922 int ioapic_id
= mpc_ioapic_id(ioapic_idx
);
923 if (!map_ioapic_to_ir(ioapic_id
)) {
924 pr_err(FW_BUG
"ioapic %d has no mapping iommu, "
925 "interrupt remapping will be disabled\n",
934 static int __init
ir_dev_scope_init(void)
938 if (!irq_remapping_enabled
)
941 down_write(&dmar_global_lock
);
942 ret
= dmar_dev_scope_init();
943 up_write(&dmar_global_lock
);
947 rootfs_initcall(ir_dev_scope_init
);
949 static void disable_irq_remapping(void)
951 struct dmar_drhd_unit
*drhd
;
952 struct intel_iommu
*iommu
= NULL
;
955 * Disable Interrupt-remapping for all the DRHD's now.
957 for_each_iommu(iommu
, drhd
) {
958 if (!ecap_ir_support(iommu
->ecap
))
961 iommu_disable_irq_remapping(iommu
);
965 * Clear Posted-Interrupts capability.
967 if (!disable_irq_post
)
968 intel_irq_remap_ops
.capability
&= ~(1 << IRQ_POSTING_CAP
);
971 static int reenable_irq_remapping(int eim
)
973 struct dmar_drhd_unit
*drhd
;
975 struct intel_iommu
*iommu
= NULL
;
977 for_each_iommu(iommu
, drhd
)
979 dmar_reenable_qi(iommu
);
982 * Setup Interrupt-remapping for all the DRHD's now.
984 for_each_iommu(iommu
, drhd
) {
985 if (!ecap_ir_support(iommu
->ecap
))
988 /* Set up interrupt remapping for iommu.*/
989 iommu_set_irq_remapping(iommu
, eim
);
990 iommu_enable_irq_remapping(iommu
);
997 set_irq_posting_cap();
1003 * handle error condition gracefully here!
1008 static void prepare_irte(struct irte
*irte
, int vector
, unsigned int dest
)
1010 memset(irte
, 0, sizeof(*irte
));
1013 irte
->dst_mode
= apic
->irq_dest_mode
;
1015 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the
1016 * actual level or edge trigger will be setup in the IO-APIC
1017 * RTE. This will help simplify level triggered irq migration.
1018 * For more details, see the comments (in io_apic.c) explainig IO-APIC
1019 * irq migration in the presence of interrupt-remapping.
1021 irte
->trigger_mode
= 0;
1022 irte
->dlvry_mode
= apic
->irq_delivery_mode
;
1023 irte
->vector
= vector
;
1024 irte
->dest_id
= IRTE_DEST(dest
);
1025 irte
->redir_hint
= 1;
1028 static struct irq_domain
*intel_get_ir_irq_domain(struct irq_alloc_info
*info
)
1030 struct intel_iommu
*iommu
= NULL
;
1035 switch (info
->type
) {
1036 case X86_IRQ_ALLOC_TYPE_IOAPIC
:
1037 iommu
= map_ioapic_to_ir(info
->ioapic_id
);
1039 case X86_IRQ_ALLOC_TYPE_HPET
:
1040 iommu
= map_hpet_to_ir(info
->hpet_id
);
1042 case X86_IRQ_ALLOC_TYPE_MSI
:
1043 case X86_IRQ_ALLOC_TYPE_MSIX
:
1044 iommu
= map_dev_to_ir(info
->msi_dev
);
1051 return iommu
? iommu
->ir_domain
: NULL
;
1054 static struct irq_domain
*intel_get_irq_domain(struct irq_alloc_info
*info
)
1056 struct intel_iommu
*iommu
;
1061 switch (info
->type
) {
1062 case X86_IRQ_ALLOC_TYPE_MSI
:
1063 case X86_IRQ_ALLOC_TYPE_MSIX
:
1064 iommu
= map_dev_to_ir(info
->msi_dev
);
1066 return iommu
->ir_msi_domain
;
1075 struct irq_remap_ops intel_irq_remap_ops
= {
1076 .prepare
= intel_prepare_irq_remapping
,
1077 .enable
= intel_enable_irq_remapping
,
1078 .disable
= disable_irq_remapping
,
1079 .reenable
= reenable_irq_remapping
,
1080 .enable_faulting
= enable_drhd_fault_handling
,
1081 .get_ir_irq_domain
= intel_get_ir_irq_domain
,
1082 .get_irq_domain
= intel_get_irq_domain
,
1086 * Migrate the IO-APIC irq in the presence of intr-remapping.
1088 * For both level and edge triggered, irq migration is a simple atomic
1089 * update(of vector and cpu destination) of IRTE and flush the hardware cache.
1091 * For level triggered, we eliminate the io-apic RTE modification (with the
1092 * updated vector information), by using a virtual vector (io-apic pin number).
1093 * Real vector that is used for interrupting cpu will be coming from
1094 * the interrupt-remapping table entry.
1096 * As the migration is a simple atomic update of IRTE, the same mechanism
1097 * is used to migrate MSI irq's in the presence of interrupt-remapping.
1100 intel_ir_set_affinity(struct irq_data
*data
, const struct cpumask
*mask
,
1103 struct intel_ir_data
*ir_data
= data
->chip_data
;
1104 struct irte
*irte
= &ir_data
->irte_entry
;
1105 struct irq_cfg
*cfg
= irqd_cfg(data
);
1106 struct irq_data
*parent
= data
->parent_data
;
1109 ret
= parent
->chip
->irq_set_affinity(parent
, mask
, force
);
1110 if (ret
< 0 || ret
== IRQ_SET_MASK_OK_DONE
)
1114 * Atomically updates the IRTE with the new destination, vector
1115 * and flushes the interrupt entry cache.
1117 irte
->vector
= cfg
->vector
;
1118 irte
->dest_id
= IRTE_DEST(cfg
->dest_apicid
);
1120 /* Update the hardware only if the interrupt is in remapped mode. */
1121 if (ir_data
->irq_2_iommu
.mode
== IRQ_REMAPPING
)
1122 modify_irte(&ir_data
->irq_2_iommu
, irte
);
1125 * After this point, all the interrupts will start arriving
1126 * at the new destination. So, time to cleanup the previous
1127 * vector allocation.
1129 send_cleanup_vector(cfg
);
1131 return IRQ_SET_MASK_OK_DONE
;
1134 static void intel_ir_compose_msi_msg(struct irq_data
*irq_data
,
1135 struct msi_msg
*msg
)
1137 struct intel_ir_data
*ir_data
= irq_data
->chip_data
;
1139 *msg
= ir_data
->msi_entry
;
1142 static int intel_ir_set_vcpu_affinity(struct irq_data
*data
, void *info
)
1144 struct intel_ir_data
*ir_data
= data
->chip_data
;
1145 struct vcpu_data
*vcpu_pi_info
= info
;
1147 /* stop posting interrupts, back to remapping mode */
1148 if (!vcpu_pi_info
) {
1149 modify_irte(&ir_data
->irq_2_iommu
, &ir_data
->irte_entry
);
1151 struct irte irte_pi
;
1154 * We are not caching the posted interrupt entry. We
1155 * copy the data from the remapped entry and modify
1156 * the fields which are relevant for posted mode. The
1157 * cached remapped entry is used for switching back to
1160 memset(&irte_pi
, 0, sizeof(irte_pi
));
1161 dmar_copy_shared_irte(&irte_pi
, &ir_data
->irte_entry
);
1163 /* Update the posted mode fields */
1165 irte_pi
.p_urgent
= 0;
1166 irte_pi
.p_vector
= vcpu_pi_info
->vector
;
1167 irte_pi
.pda_l
= (vcpu_pi_info
->pi_desc_addr
>>
1168 (32 - PDA_LOW_BIT
)) & ~(-1UL << PDA_LOW_BIT
);
1169 irte_pi
.pda_h
= (vcpu_pi_info
->pi_desc_addr
>> 32) &
1170 ~(-1UL << PDA_HIGH_BIT
);
1172 modify_irte(&ir_data
->irq_2_iommu
, &irte_pi
);
1178 static struct irq_chip intel_ir_chip
= {
1179 .irq_ack
= ir_ack_apic_edge
,
1180 .irq_set_affinity
= intel_ir_set_affinity
,
1181 .irq_compose_msi_msg
= intel_ir_compose_msi_msg
,
1182 .irq_set_vcpu_affinity
= intel_ir_set_vcpu_affinity
,
1185 static void intel_irq_remapping_prepare_irte(struct intel_ir_data
*data
,
1186 struct irq_cfg
*irq_cfg
,
1187 struct irq_alloc_info
*info
,
1188 int index
, int sub_handle
)
1190 struct IR_IO_APIC_route_entry
*entry
;
1191 struct irte
*irte
= &data
->irte_entry
;
1192 struct msi_msg
*msg
= &data
->msi_entry
;
1194 prepare_irte(irte
, irq_cfg
->vector
, irq_cfg
->dest_apicid
);
1195 switch (info
->type
) {
1196 case X86_IRQ_ALLOC_TYPE_IOAPIC
:
1197 /* Set source-id of interrupt request */
1198 set_ioapic_sid(irte
, info
->ioapic_id
);
1199 apic_printk(APIC_VERBOSE
, KERN_DEBUG
"IOAPIC[%d]: Set IRTE entry (P:%d FPD:%d Dst_Mode:%d Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X Avail:%X Vector:%02X Dest:%08X SID:%04X SQ:%X SVT:%X)\n",
1200 info
->ioapic_id
, irte
->present
, irte
->fpd
,
1201 irte
->dst_mode
, irte
->redir_hint
,
1202 irte
->trigger_mode
, irte
->dlvry_mode
,
1203 irte
->avail
, irte
->vector
, irte
->dest_id
,
1204 irte
->sid
, irte
->sq
, irte
->svt
);
1206 entry
= (struct IR_IO_APIC_route_entry
*)info
->ioapic_entry
;
1207 info
->ioapic_entry
= NULL
;
1208 memset(entry
, 0, sizeof(*entry
));
1209 entry
->index2
= (index
>> 15) & 0x1;
1212 entry
->index
= (index
& 0x7fff);
1214 * IO-APIC RTE will be configured with virtual vector.
1215 * irq handler will do the explicit EOI to the io-apic.
1217 entry
->vector
= info
->ioapic_pin
;
1218 entry
->mask
= 0; /* enable IRQ */
1219 entry
->trigger
= info
->ioapic_trigger
;
1220 entry
->polarity
= info
->ioapic_polarity
;
1221 if (info
->ioapic_trigger
)
1222 entry
->mask
= 1; /* Mask level triggered irqs. */
1225 case X86_IRQ_ALLOC_TYPE_HPET
:
1226 case X86_IRQ_ALLOC_TYPE_MSI
:
1227 case X86_IRQ_ALLOC_TYPE_MSIX
:
1228 if (info
->type
== X86_IRQ_ALLOC_TYPE_HPET
)
1229 set_hpet_sid(irte
, info
->hpet_id
);
1231 set_msi_sid(irte
, info
->msi_dev
);
1233 msg
->address_hi
= MSI_ADDR_BASE_HI
;
1234 msg
->data
= sub_handle
;
1235 msg
->address_lo
= MSI_ADDR_BASE_LO
| MSI_ADDR_IR_EXT_INT
|
1237 MSI_ADDR_IR_INDEX1(index
) |
1238 MSI_ADDR_IR_INDEX2(index
);
1247 static void intel_free_irq_resources(struct irq_domain
*domain
,
1248 unsigned int virq
, unsigned int nr_irqs
)
1250 struct irq_data
*irq_data
;
1251 struct intel_ir_data
*data
;
1252 struct irq_2_iommu
*irq_iommu
;
1253 unsigned long flags
;
1255 for (i
= 0; i
< nr_irqs
; i
++) {
1256 irq_data
= irq_domain_get_irq_data(domain
, virq
+ i
);
1257 if (irq_data
&& irq_data
->chip_data
) {
1258 data
= irq_data
->chip_data
;
1259 irq_iommu
= &data
->irq_2_iommu
;
1260 raw_spin_lock_irqsave(&irq_2_ir_lock
, flags
);
1261 clear_entries(irq_iommu
);
1262 raw_spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
1263 irq_domain_reset_irq_data(irq_data
);
1269 static int intel_irq_remapping_alloc(struct irq_domain
*domain
,
1270 unsigned int virq
, unsigned int nr_irqs
,
1273 struct intel_iommu
*iommu
= domain
->host_data
;
1274 struct irq_alloc_info
*info
= arg
;
1275 struct intel_ir_data
*data
, *ird
;
1276 struct irq_data
*irq_data
;
1277 struct irq_cfg
*irq_cfg
;
1280 if (!info
|| !iommu
)
1282 if (nr_irqs
> 1 && info
->type
!= X86_IRQ_ALLOC_TYPE_MSI
&&
1283 info
->type
!= X86_IRQ_ALLOC_TYPE_MSIX
)
1287 * With IRQ remapping enabled, don't need contiguous CPU vectors
1288 * to support multiple MSI interrupts.
1290 if (info
->type
== X86_IRQ_ALLOC_TYPE_MSI
)
1291 info
->flags
&= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS
;
1293 ret
= irq_domain_alloc_irqs_parent(domain
, virq
, nr_irqs
, arg
);
1298 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
1300 goto out_free_parent
;
1302 down_read(&dmar_global_lock
);
1303 index
= alloc_irte(iommu
, virq
, &data
->irq_2_iommu
, nr_irqs
);
1304 up_read(&dmar_global_lock
);
1306 pr_warn("Failed to allocate IRTE\n");
1308 goto out_free_parent
;
1311 for (i
= 0; i
< nr_irqs
; i
++) {
1312 irq_data
= irq_domain_get_irq_data(domain
, virq
+ i
);
1313 irq_cfg
= irqd_cfg(irq_data
);
1314 if (!irq_data
|| !irq_cfg
) {
1320 ird
= kzalloc(sizeof(*ird
), GFP_KERNEL
);
1323 /* Initialize the common data */
1324 ird
->irq_2_iommu
= data
->irq_2_iommu
;
1325 ird
->irq_2_iommu
.sub_handle
= i
;
1330 irq_data
->hwirq
= (index
<< 16) + i
;
1331 irq_data
->chip_data
= ird
;
1332 irq_data
->chip
= &intel_ir_chip
;
1333 intel_irq_remapping_prepare_irte(ird
, irq_cfg
, info
, index
, i
);
1334 irq_set_status_flags(virq
+ i
, IRQ_MOVE_PCNTXT
);
1339 intel_free_irq_resources(domain
, virq
, i
);
1341 irq_domain_free_irqs_common(domain
, virq
, nr_irqs
);
1345 static void intel_irq_remapping_free(struct irq_domain
*domain
,
1346 unsigned int virq
, unsigned int nr_irqs
)
1348 intel_free_irq_resources(domain
, virq
, nr_irqs
);
1349 irq_domain_free_irqs_common(domain
, virq
, nr_irqs
);
1352 static void intel_irq_remapping_activate(struct irq_domain
*domain
,
1353 struct irq_data
*irq_data
)
1355 struct intel_ir_data
*data
= irq_data
->chip_data
;
1357 modify_irte(&data
->irq_2_iommu
, &data
->irte_entry
);
1360 static void intel_irq_remapping_deactivate(struct irq_domain
*domain
,
1361 struct irq_data
*irq_data
)
1363 struct intel_ir_data
*data
= irq_data
->chip_data
;
1366 memset(&entry
, 0, sizeof(entry
));
1367 modify_irte(&data
->irq_2_iommu
, &entry
);
1370 static struct irq_domain_ops intel_ir_domain_ops
= {
1371 .alloc
= intel_irq_remapping_alloc
,
1372 .free
= intel_irq_remapping_free
,
1373 .activate
= intel_irq_remapping_activate
,
1374 .deactivate
= intel_irq_remapping_deactivate
,
1378 * Support of Interrupt Remapping Unit Hotplug
1380 static int dmar_ir_add(struct dmar_drhd_unit
*dmaru
, struct intel_iommu
*iommu
)
1383 int eim
= x2apic_enabled();
1385 if (eim
&& !ecap_eim_support(iommu
->ecap
)) {
1386 pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n",
1387 iommu
->reg_phys
, iommu
->ecap
);
1391 if (ir_parse_ioapic_hpet_scope(dmaru
->hdr
, iommu
)) {
1392 pr_warn("DRHD %Lx: failed to parse managed IOAPIC/HPET\n",
1397 /* TODO: check all IOAPICs are covered by IOMMU */
1399 /* Setup Interrupt-remapping now. */
1400 ret
= intel_setup_irq_remapping(iommu
);
1402 pr_err("Failed to setup irq remapping for %s\n",
1404 intel_teardown_irq_remapping(iommu
);
1405 ir_remove_ioapic_hpet_scope(iommu
);
1407 iommu_enable_irq_remapping(iommu
);
1413 int dmar_ir_hotplug(struct dmar_drhd_unit
*dmaru
, bool insert
)
1416 struct intel_iommu
*iommu
= dmaru
->iommu
;
1418 if (!irq_remapping_enabled
)
1422 if (!ecap_ir_support(iommu
->ecap
))
1424 if (irq_remapping_cap(IRQ_POSTING_CAP
) &&
1425 !cap_pi_support(iommu
->cap
))
1429 if (!iommu
->ir_table
)
1430 ret
= dmar_ir_add(dmaru
, iommu
);
1432 if (iommu
->ir_table
) {
1433 if (!bitmap_empty(iommu
->ir_table
->bitmap
,
1434 INTR_REMAP_TABLE_ENTRIES
)) {
1437 iommu_disable_irq_remapping(iommu
);
1438 intel_teardown_irq_remapping(iommu
);
1439 ir_remove_ioapic_hpet_scope(iommu
);