2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
20 #include <linux/init.h>
21 #include <linux/bitmap.h>
22 #include <linux/debugfs.h>
23 #include <linux/export.h>
24 #include <linux/slab.h>
25 #include <linux/irq.h>
26 #include <linux/interrupt.h>
27 #include <linux/spinlock.h>
28 #include <linux/pci.h>
29 #include <linux/dmar.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/mempool.h>
32 #include <linux/memory.h>
33 #include <linux/timer.h>
34 #include <linux/iova.h>
35 #include <linux/iommu.h>
36 #include <linux/intel-iommu.h>
37 #include <linux/syscore_ops.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/pci-ats.h>
41 #include <linux/memblock.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/cacheflush.h>
44 #include <asm/iommu.h>
46 #include "irq_remapping.h"
49 #define ROOT_SIZE VTD_PAGE_SIZE
50 #define CONTEXT_SIZE VTD_PAGE_SIZE
52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
56 #define IOAPIC_RANGE_START (0xfee00000)
57 #define IOAPIC_RANGE_END (0xfeefffff)
58 #define IOVA_START_ADDR (0x1000)
60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
62 #define MAX_AGAW_WIDTH 64
63 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
65 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
68 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
74 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
75 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
76 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
78 /* page table handling */
79 #define LEVEL_STRIDE (9)
80 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
98 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
100 static inline int agaw_to_level(int agaw
)
105 static inline int agaw_to_width(int agaw
)
107 return min_t(int, 30 + agaw
* LEVEL_STRIDE
, MAX_AGAW_WIDTH
);
110 static inline int width_to_agaw(int width
)
112 return DIV_ROUND_UP(width
- 30, LEVEL_STRIDE
);
115 static inline unsigned int level_to_offset_bits(int level
)
117 return (level
- 1) * LEVEL_STRIDE
;
120 static inline int pfn_level_offset(unsigned long pfn
, int level
)
122 return (pfn
>> level_to_offset_bits(level
)) & LEVEL_MASK
;
125 static inline unsigned long level_mask(int level
)
127 return -1UL << level_to_offset_bits(level
);
130 static inline unsigned long level_size(int level
)
132 return 1UL << level_to_offset_bits(level
);
135 static inline unsigned long align_to_level(unsigned long pfn
, int level
)
137 return (pfn
+ level_size(level
) - 1) & level_mask(level
);
140 static inline unsigned long lvl_to_nr_pages(unsigned int lvl
)
142 return 1 << min_t(int, (lvl
- 1) * LEVEL_STRIDE
, MAX_AGAW_PFN_WIDTH
);
145 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn
)
149 return dma_pfn
>> (PAGE_SHIFT
- VTD_PAGE_SHIFT
);
152 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn
)
154 return mm_pfn
<< (PAGE_SHIFT
- VTD_PAGE_SHIFT
);
156 static inline unsigned long page_to_dma_pfn(struct page
*pg
)
158 return mm_to_dma_pfn(page_to_pfn(pg
));
160 static inline unsigned long virt_to_dma_pfn(void *p
)
162 return page_to_dma_pfn(virt_to_page(p
));
165 /* global iommu list, set NULL for ignored DMAR units */
166 static struct intel_iommu
**g_iommus
;
168 static void __init
check_tylersburg_isoch(void);
169 static int rwbf_quirk
;
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
175 static int force_on
= 0;
180 * 12-63: Context Ptr (12 - (haw-1))
187 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188 static inline bool root_present(struct root_entry
*root
)
190 return (root
->val
& 1);
192 static inline void set_root_present(struct root_entry
*root
)
196 static inline void set_root_value(struct root_entry
*root
, unsigned long value
)
198 root
->val
|= value
& VTD_PAGE_MASK
;
201 static inline struct context_entry
*
202 get_context_addr_from_root(struct root_entry
*root
)
204 return (struct context_entry
*)
205 (root_present(root
)?phys_to_virt(
206 root
->val
& VTD_PAGE_MASK
) :
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
221 struct context_entry
{
226 static inline bool context_present(struct context_entry
*context
)
228 return (context
->lo
& 1);
230 static inline void context_set_present(struct context_entry
*context
)
235 static inline void context_set_fault_enable(struct context_entry
*context
)
237 context
->lo
&= (((u64
)-1) << 2) | 1;
240 static inline void context_set_translation_type(struct context_entry
*context
,
243 context
->lo
&= (((u64
)-1) << 4) | 3;
244 context
->lo
|= (value
& 3) << 2;
247 static inline void context_set_address_root(struct context_entry
*context
,
250 context
->lo
|= value
& VTD_PAGE_MASK
;
253 static inline void context_set_address_width(struct context_entry
*context
,
256 context
->hi
|= value
& 7;
259 static inline void context_set_domain_id(struct context_entry
*context
,
262 context
->hi
|= (value
& ((1 << 16) - 1)) << 8;
265 static inline void context_clear_entry(struct context_entry
*context
)
278 * 12-63: Host physcial address
284 static inline void dma_clear_pte(struct dma_pte
*pte
)
289 static inline u64
dma_pte_addr(struct dma_pte
*pte
)
292 return pte
->val
& VTD_PAGE_MASK
;
294 /* Must have a full atomic 64-bit read */
295 return __cmpxchg64(&pte
->val
, 0ULL, 0ULL) & VTD_PAGE_MASK
;
299 static inline bool dma_pte_present(struct dma_pte
*pte
)
301 return (pte
->val
& 3) != 0;
304 static inline bool dma_pte_superpage(struct dma_pte
*pte
)
306 return (pte
->val
& (1 << 7));
309 static inline int first_pte_in_page(struct dma_pte
*pte
)
311 return !((unsigned long)pte
& ~VTD_PAGE_MASK
);
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
320 static struct dmar_domain
*si_domain
;
321 static int hw_pass_through
= 1;
323 /* devices under the same p2p bridge are owned in one domain */
324 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
326 /* domain represents a virtual machine, more than one devices
327 * across iommus may be owned in one domain, e.g. kvm guest.
329 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
331 /* si_domain contains mulitple devices */
332 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
334 /* define the limit of IOMMUs supported in each domain */
336 # define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
338 # define IOMMU_UNITS_SUPPORTED 64
342 int id
; /* domain id */
343 int nid
; /* node id */
344 DECLARE_BITMAP(iommu_bmp
, IOMMU_UNITS_SUPPORTED
);
345 /* bitmap of iommus this domain uses*/
347 struct list_head devices
; /* all devices' list */
348 struct iova_domain iovad
; /* iova's that belong to this domain */
350 struct dma_pte
*pgd
; /* virtual address */
351 int gaw
; /* max guest address width */
353 /* adjusted guest address width, 0 is level 2 30-bit */
356 int flags
; /* flags to find out type of domain */
358 int iommu_coherency
;/* indicate coherency of iommu access */
359 int iommu_snooping
; /* indicate snooping control feature*/
360 int iommu_count
; /* reference count of iommu */
361 int iommu_superpage
;/* Level of superpages supported:
362 0 == 4KiB (no superpages), 1 == 2MiB,
363 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
364 spinlock_t iommu_lock
; /* protect iommu set in domain */
365 u64 max_addr
; /* maximum mapped address */
368 /* PCI domain-device relationship */
369 struct device_domain_info
{
370 struct list_head link
; /* link to domain siblings */
371 struct list_head global
; /* link to global list */
372 int segment
; /* PCI domain */
373 u8 bus
; /* PCI bus number */
374 u8 devfn
; /* PCI devfn number */
375 struct device
*dev
; /* it's NULL for PCIe-to-PCI bridge */
376 struct intel_iommu
*iommu
; /* IOMMU used by this device */
377 struct dmar_domain
*domain
; /* pointer to domain */
380 struct dmar_rmrr_unit
{
381 struct list_head list
; /* list of rmrr units */
382 struct acpi_dmar_header
*hdr
; /* ACPI header */
383 u64 base_address
; /* reserved base address*/
384 u64 end_address
; /* reserved end address */
385 struct dmar_dev_scope
*devices
; /* target devices */
386 int devices_cnt
; /* target device count */
389 struct dmar_atsr_unit
{
390 struct list_head list
; /* list of ATSR units */
391 struct acpi_dmar_header
*hdr
; /* ACPI header */
392 struct dmar_dev_scope
*devices
; /* target devices */
393 int devices_cnt
; /* target device count */
394 u8 include_all
:1; /* include all ports */
397 static LIST_HEAD(dmar_atsr_units
);
398 static LIST_HEAD(dmar_rmrr_units
);
400 #define for_each_rmrr_units(rmrr) \
401 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
403 static void flush_unmaps_timeout(unsigned long data
);
405 static DEFINE_TIMER(unmap_timer
, flush_unmaps_timeout
, 0, 0);
407 #define HIGH_WATER_MARK 250
408 struct deferred_flush_tables
{
410 struct iova
*iova
[HIGH_WATER_MARK
];
411 struct dmar_domain
*domain
[HIGH_WATER_MARK
];
412 struct page
*freelist
[HIGH_WATER_MARK
];
415 static struct deferred_flush_tables
*deferred_flush
;
417 /* bitmap for indexing intel_iommus */
418 static int g_num_of_iommus
;
420 static DEFINE_SPINLOCK(async_umap_flush_lock
);
421 static LIST_HEAD(unmaps_to_do
);
424 static long list_size
;
426 static void domain_exit(struct dmar_domain
*domain
);
427 static void domain_remove_dev_info(struct dmar_domain
*domain
);
428 static void domain_remove_one_dev_info(struct dmar_domain
*domain
,
429 struct pci_dev
*pdev
);
430 static void iommu_detach_dependent_devices(struct intel_iommu
*iommu
,
433 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
434 int dmar_disabled
= 0;
436 int dmar_disabled
= 1;
437 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
439 int intel_iommu_enabled
= 0;
440 EXPORT_SYMBOL_GPL(intel_iommu_enabled
);
442 static int dmar_map_gfx
= 1;
443 static int dmar_forcedac
;
444 static int intel_iommu_strict
;
445 static int intel_iommu_superpage
= 1;
447 int intel_iommu_gfx_mapped
;
448 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped
);
450 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
451 static DEFINE_SPINLOCK(device_domain_lock
);
452 static LIST_HEAD(device_domain_list
);
454 static struct iommu_ops intel_iommu_ops
;
456 static int __init
intel_iommu_setup(char *str
)
461 if (!strncmp(str
, "on", 2)) {
463 printk(KERN_INFO
"Intel-IOMMU: enabled\n");
464 } else if (!strncmp(str
, "off", 3)) {
466 printk(KERN_INFO
"Intel-IOMMU: disabled\n");
467 } else if (!strncmp(str
, "igfx_off", 8)) {
470 "Intel-IOMMU: disable GFX device mapping\n");
471 } else if (!strncmp(str
, "forcedac", 8)) {
473 "Intel-IOMMU: Forcing DAC for PCI devices\n");
475 } else if (!strncmp(str
, "strict", 6)) {
477 "Intel-IOMMU: disable batched IOTLB flush\n");
478 intel_iommu_strict
= 1;
479 } else if (!strncmp(str
, "sp_off", 6)) {
481 "Intel-IOMMU: disable supported super page\n");
482 intel_iommu_superpage
= 0;
485 str
+= strcspn(str
, ",");
491 __setup("intel_iommu=", intel_iommu_setup
);
493 static struct kmem_cache
*iommu_domain_cache
;
494 static struct kmem_cache
*iommu_devinfo_cache
;
495 static struct kmem_cache
*iommu_iova_cache
;
497 static inline void *alloc_pgtable_page(int node
)
502 page
= alloc_pages_node(node
, GFP_ATOMIC
| __GFP_ZERO
, 0);
504 vaddr
= page_address(page
);
508 static inline void free_pgtable_page(void *vaddr
)
510 free_page((unsigned long)vaddr
);
513 static inline void *alloc_domain_mem(void)
515 return kmem_cache_alloc(iommu_domain_cache
, GFP_ATOMIC
);
518 static void free_domain_mem(void *vaddr
)
520 kmem_cache_free(iommu_domain_cache
, vaddr
);
523 static inline void * alloc_devinfo_mem(void)
525 return kmem_cache_alloc(iommu_devinfo_cache
, GFP_ATOMIC
);
528 static inline void free_devinfo_mem(void *vaddr
)
530 kmem_cache_free(iommu_devinfo_cache
, vaddr
);
533 struct iova
*alloc_iova_mem(void)
535 return kmem_cache_alloc(iommu_iova_cache
, GFP_ATOMIC
);
538 void free_iova_mem(struct iova
*iova
)
540 kmem_cache_free(iommu_iova_cache
, iova
);
544 static int __iommu_calculate_agaw(struct intel_iommu
*iommu
, int max_gaw
)
549 sagaw
= cap_sagaw(iommu
->cap
);
550 for (agaw
= width_to_agaw(max_gaw
);
552 if (test_bit(agaw
, &sagaw
))
560 * Calculate max SAGAW for each iommu.
562 int iommu_calculate_max_sagaw(struct intel_iommu
*iommu
)
564 return __iommu_calculate_agaw(iommu
, MAX_AGAW_WIDTH
);
568 * calculate agaw for each iommu.
569 * "SAGAW" may be different across iommus, use a default agaw, and
570 * get a supported less agaw for iommus that don't support the default agaw.
572 int iommu_calculate_agaw(struct intel_iommu
*iommu
)
574 return __iommu_calculate_agaw(iommu
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
577 /* This functionin only returns single iommu in a domain */
578 static struct intel_iommu
*domain_get_iommu(struct dmar_domain
*domain
)
582 /* si_domain and vm domain should not get here. */
583 BUG_ON(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
);
584 BUG_ON(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
);
586 iommu_id
= find_first_bit(domain
->iommu_bmp
, g_num_of_iommus
);
587 if (iommu_id
< 0 || iommu_id
>= g_num_of_iommus
)
590 return g_iommus
[iommu_id
];
593 static void domain_update_iommu_coherency(struct dmar_domain
*domain
)
595 struct dmar_drhd_unit
*drhd
;
596 struct intel_iommu
*iommu
;
599 domain
->iommu_coherency
= 1;
601 for_each_set_bit(i
, domain
->iommu_bmp
, g_num_of_iommus
) {
603 if (!ecap_coherent(g_iommus
[i
]->ecap
)) {
604 domain
->iommu_coherency
= 0;
611 /* No hardware attached; use lowest common denominator */
613 for_each_active_iommu(iommu
, drhd
) {
614 if (!ecap_coherent(iommu
->ecap
)) {
615 domain
->iommu_coherency
= 0;
622 static void domain_update_iommu_snooping(struct dmar_domain
*domain
)
626 domain
->iommu_snooping
= 1;
628 for_each_set_bit(i
, domain
->iommu_bmp
, g_num_of_iommus
) {
629 if (!ecap_sc_support(g_iommus
[i
]->ecap
)) {
630 domain
->iommu_snooping
= 0;
636 static void domain_update_iommu_superpage(struct dmar_domain
*domain
)
638 struct dmar_drhd_unit
*drhd
;
639 struct intel_iommu
*iommu
= NULL
;
642 if (!intel_iommu_superpage
) {
643 domain
->iommu_superpage
= 0;
647 /* set iommu_superpage to the smallest common denominator */
649 for_each_active_iommu(iommu
, drhd
) {
650 mask
&= cap_super_page_val(iommu
->cap
);
657 domain
->iommu_superpage
= fls(mask
);
660 /* Some capabilities may be different across iommus */
661 static void domain_update_iommu_cap(struct dmar_domain
*domain
)
663 domain_update_iommu_coherency(domain
);
664 domain_update_iommu_snooping(domain
);
665 domain_update_iommu_superpage(domain
);
668 static struct intel_iommu
*device_to_iommu(int segment
, u8 bus
, u8 devfn
)
670 struct dmar_drhd_unit
*drhd
= NULL
;
671 struct intel_iommu
*iommu
;
673 struct pci_dev
*pdev
;
677 for_each_active_iommu(iommu
, drhd
) {
678 if (segment
!= drhd
->segment
)
681 for_each_active_dev_scope(drhd
->devices
,
682 drhd
->devices_cnt
, i
, dev
) {
683 if (!dev_is_pci(dev
))
685 pdev
= to_pci_dev(dev
);
686 if (pdev
->bus
->number
== bus
&& pdev
->devfn
== devfn
)
688 if (pdev
->subordinate
&&
689 pdev
->subordinate
->number
<= bus
&&
690 pdev
->subordinate
->busn_res
.end
>= bus
)
694 if (drhd
->include_all
)
704 static void domain_flush_cache(struct dmar_domain
*domain
,
705 void *addr
, int size
)
707 if (!domain
->iommu_coherency
)
708 clflush_cache_range(addr
, size
);
711 /* Gets context entry for a given bus and devfn */
712 static struct context_entry
* device_to_context_entry(struct intel_iommu
*iommu
,
715 struct root_entry
*root
;
716 struct context_entry
*context
;
717 unsigned long phy_addr
;
720 spin_lock_irqsave(&iommu
->lock
, flags
);
721 root
= &iommu
->root_entry
[bus
];
722 context
= get_context_addr_from_root(root
);
724 context
= (struct context_entry
*)
725 alloc_pgtable_page(iommu
->node
);
727 spin_unlock_irqrestore(&iommu
->lock
, flags
);
730 __iommu_flush_cache(iommu
, (void *)context
, CONTEXT_SIZE
);
731 phy_addr
= virt_to_phys((void *)context
);
732 set_root_value(root
, phy_addr
);
733 set_root_present(root
);
734 __iommu_flush_cache(iommu
, root
, sizeof(*root
));
736 spin_unlock_irqrestore(&iommu
->lock
, flags
);
737 return &context
[devfn
];
740 static int device_context_mapped(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
742 struct root_entry
*root
;
743 struct context_entry
*context
;
747 spin_lock_irqsave(&iommu
->lock
, flags
);
748 root
= &iommu
->root_entry
[bus
];
749 context
= get_context_addr_from_root(root
);
754 ret
= context_present(&context
[devfn
]);
756 spin_unlock_irqrestore(&iommu
->lock
, flags
);
760 static void clear_context_table(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
762 struct root_entry
*root
;
763 struct context_entry
*context
;
766 spin_lock_irqsave(&iommu
->lock
, flags
);
767 root
= &iommu
->root_entry
[bus
];
768 context
= get_context_addr_from_root(root
);
770 context_clear_entry(&context
[devfn
]);
771 __iommu_flush_cache(iommu
, &context
[devfn
], \
774 spin_unlock_irqrestore(&iommu
->lock
, flags
);
777 static void free_context_table(struct intel_iommu
*iommu
)
779 struct root_entry
*root
;
782 struct context_entry
*context
;
784 spin_lock_irqsave(&iommu
->lock
, flags
);
785 if (!iommu
->root_entry
) {
788 for (i
= 0; i
< ROOT_ENTRY_NR
; i
++) {
789 root
= &iommu
->root_entry
[i
];
790 context
= get_context_addr_from_root(root
);
792 free_pgtable_page(context
);
794 free_pgtable_page(iommu
->root_entry
);
795 iommu
->root_entry
= NULL
;
797 spin_unlock_irqrestore(&iommu
->lock
, flags
);
800 static struct dma_pte
*pfn_to_dma_pte(struct dmar_domain
*domain
,
801 unsigned long pfn
, int *target_level
)
803 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
804 struct dma_pte
*parent
, *pte
= NULL
;
805 int level
= agaw_to_level(domain
->agaw
);
808 BUG_ON(!domain
->pgd
);
810 if (addr_width
< BITS_PER_LONG
&& pfn
>> addr_width
)
811 /* Address beyond IOMMU's addressing capabilities. */
814 parent
= domain
->pgd
;
819 offset
= pfn_level_offset(pfn
, level
);
820 pte
= &parent
[offset
];
821 if (!*target_level
&& (dma_pte_superpage(pte
) || !dma_pte_present(pte
)))
823 if (level
== *target_level
)
826 if (!dma_pte_present(pte
)) {
829 tmp_page
= alloc_pgtable_page(domain
->nid
);
834 domain_flush_cache(domain
, tmp_page
, VTD_PAGE_SIZE
);
835 pteval
= ((uint64_t)virt_to_dma_pfn(tmp_page
) << VTD_PAGE_SHIFT
) | DMA_PTE_READ
| DMA_PTE_WRITE
;
836 if (cmpxchg64(&pte
->val
, 0ULL, pteval
)) {
837 /* Someone else set it while we were thinking; use theirs. */
838 free_pgtable_page(tmp_page
);
841 domain_flush_cache(domain
, pte
, sizeof(*pte
));
847 parent
= phys_to_virt(dma_pte_addr(pte
));
852 *target_level
= level
;
858 /* return address's pte at specific level */
859 static struct dma_pte
*dma_pfn_level_pte(struct dmar_domain
*domain
,
861 int level
, int *large_page
)
863 struct dma_pte
*parent
, *pte
= NULL
;
864 int total
= agaw_to_level(domain
->agaw
);
867 parent
= domain
->pgd
;
868 while (level
<= total
) {
869 offset
= pfn_level_offset(pfn
, total
);
870 pte
= &parent
[offset
];
874 if (!dma_pte_present(pte
)) {
879 if (pte
->val
& DMA_PTE_LARGE_PAGE
) {
884 parent
= phys_to_virt(dma_pte_addr(pte
));
890 /* clear last level pte, a tlb flush should be followed */
891 static void dma_pte_clear_range(struct dmar_domain
*domain
,
892 unsigned long start_pfn
,
893 unsigned long last_pfn
)
895 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
896 unsigned int large_page
= 1;
897 struct dma_pte
*first_pte
, *pte
;
899 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
900 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
901 BUG_ON(start_pfn
> last_pfn
);
903 /* we don't need lock here; nobody else touches the iova range */
906 first_pte
= pte
= dma_pfn_level_pte(domain
, start_pfn
, 1, &large_page
);
908 start_pfn
= align_to_level(start_pfn
+ 1, large_page
+ 1);
913 start_pfn
+= lvl_to_nr_pages(large_page
);
915 } while (start_pfn
<= last_pfn
&& !first_pte_in_page(pte
));
917 domain_flush_cache(domain
, first_pte
,
918 (void *)pte
- (void *)first_pte
);
920 } while (start_pfn
&& start_pfn
<= last_pfn
);
923 static void dma_pte_free_level(struct dmar_domain
*domain
, int level
,
924 struct dma_pte
*pte
, unsigned long pfn
,
925 unsigned long start_pfn
, unsigned long last_pfn
)
927 pfn
= max(start_pfn
, pfn
);
928 pte
= &pte
[pfn_level_offset(pfn
, level
)];
931 unsigned long level_pfn
;
932 struct dma_pte
*level_pte
;
934 if (!dma_pte_present(pte
) || dma_pte_superpage(pte
))
937 level_pfn
= pfn
& level_mask(level
- 1);
938 level_pte
= phys_to_virt(dma_pte_addr(pte
));
941 dma_pte_free_level(domain
, level
- 1, level_pte
,
942 level_pfn
, start_pfn
, last_pfn
);
944 /* If range covers entire pagetable, free it */
945 if (!(start_pfn
> level_pfn
||
946 last_pfn
< level_pfn
+ level_size(level
) - 1)) {
948 domain_flush_cache(domain
, pte
, sizeof(*pte
));
949 free_pgtable_page(level_pte
);
952 pfn
+= level_size(level
);
953 } while (!first_pte_in_page(++pte
) && pfn
<= last_pfn
);
956 /* free page table pages. last level pte should already be cleared */
957 static void dma_pte_free_pagetable(struct dmar_domain
*domain
,
958 unsigned long start_pfn
,
959 unsigned long last_pfn
)
961 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
963 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
964 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
965 BUG_ON(start_pfn
> last_pfn
);
967 /* We don't need lock here; nobody else touches the iova range */
968 dma_pte_free_level(domain
, agaw_to_level(domain
->agaw
),
969 domain
->pgd
, 0, start_pfn
, last_pfn
);
972 if (start_pfn
== 0 && last_pfn
== DOMAIN_MAX_PFN(domain
->gaw
)) {
973 free_pgtable_page(domain
->pgd
);
978 /* When a page at a given level is being unlinked from its parent, we don't
979 need to *modify* it at all. All we need to do is make a list of all the
980 pages which can be freed just as soon as we've flushed the IOTLB and we
981 know the hardware page-walk will no longer touch them.
982 The 'pte' argument is the *parent* PTE, pointing to the page that is to
984 static struct page
*dma_pte_list_pagetables(struct dmar_domain
*domain
,
985 int level
, struct dma_pte
*pte
,
986 struct page
*freelist
)
990 pg
= pfn_to_page(dma_pte_addr(pte
) >> PAGE_SHIFT
);
991 pg
->freelist
= freelist
;
997 for (pte
= page_address(pg
); !first_pte_in_page(pte
); pte
++) {
998 if (dma_pte_present(pte
) && !dma_pte_superpage(pte
))
999 freelist
= dma_pte_list_pagetables(domain
, level
- 1,
1006 static struct page
*dma_pte_clear_level(struct dmar_domain
*domain
, int level
,
1007 struct dma_pte
*pte
, unsigned long pfn
,
1008 unsigned long start_pfn
,
1009 unsigned long last_pfn
,
1010 struct page
*freelist
)
1012 struct dma_pte
*first_pte
= NULL
, *last_pte
= NULL
;
1014 pfn
= max(start_pfn
, pfn
);
1015 pte
= &pte
[pfn_level_offset(pfn
, level
)];
1018 unsigned long level_pfn
;
1020 if (!dma_pte_present(pte
))
1023 level_pfn
= pfn
& level_mask(level
);
1025 /* If range covers entire pagetable, free it */
1026 if (start_pfn
<= level_pfn
&&
1027 last_pfn
>= level_pfn
+ level_size(level
) - 1) {
1028 /* These suborbinate page tables are going away entirely. Don't
1029 bother to clear them; we're just going to *free* them. */
1030 if (level
> 1 && !dma_pte_superpage(pte
))
1031 freelist
= dma_pte_list_pagetables(domain
, level
- 1, pte
, freelist
);
1037 } else if (level
> 1) {
1038 /* Recurse down into a level that isn't *entirely* obsolete */
1039 freelist
= dma_pte_clear_level(domain
, level
- 1,
1040 phys_to_virt(dma_pte_addr(pte
)),
1041 level_pfn
, start_pfn
, last_pfn
,
1045 pfn
+= level_size(level
);
1046 } while (!first_pte_in_page(++pte
) && pfn
<= last_pfn
);
1049 domain_flush_cache(domain
, first_pte
,
1050 (void *)++last_pte
- (void *)first_pte
);
1055 /* We can't just free the pages because the IOMMU may still be walking
1056 the page tables, and may have cached the intermediate levels. The
1057 pages can only be freed after the IOTLB flush has been done. */
1058 struct page
*domain_unmap(struct dmar_domain
*domain
,
1059 unsigned long start_pfn
,
1060 unsigned long last_pfn
)
1062 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
1063 struct page
*freelist
= NULL
;
1065 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
1066 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
1067 BUG_ON(start_pfn
> last_pfn
);
1069 /* we don't need lock here; nobody else touches the iova range */
1070 freelist
= dma_pte_clear_level(domain
, agaw_to_level(domain
->agaw
),
1071 domain
->pgd
, 0, start_pfn
, last_pfn
, NULL
);
1074 if (start_pfn
== 0 && last_pfn
== DOMAIN_MAX_PFN(domain
->gaw
)) {
1075 struct page
*pgd_page
= virt_to_page(domain
->pgd
);
1076 pgd_page
->freelist
= freelist
;
1077 freelist
= pgd_page
;
1085 void dma_free_pagelist(struct page
*freelist
)
1089 while ((pg
= freelist
)) {
1090 freelist
= pg
->freelist
;
1091 free_pgtable_page(page_address(pg
));
1095 /* iommu handling */
1096 static int iommu_alloc_root_entry(struct intel_iommu
*iommu
)
1098 struct root_entry
*root
;
1099 unsigned long flags
;
1101 root
= (struct root_entry
*)alloc_pgtable_page(iommu
->node
);
1105 __iommu_flush_cache(iommu
, root
, ROOT_SIZE
);
1107 spin_lock_irqsave(&iommu
->lock
, flags
);
1108 iommu
->root_entry
= root
;
1109 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1114 static void iommu_set_root_entry(struct intel_iommu
*iommu
)
1120 addr
= iommu
->root_entry
;
1122 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1123 dmar_writeq(iommu
->reg
+ DMAR_RTADDR_REG
, virt_to_phys(addr
));
1125 writel(iommu
->gcmd
| DMA_GCMD_SRTP
, iommu
->reg
+ DMAR_GCMD_REG
);
1127 /* Make sure hardware complete it */
1128 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1129 readl
, (sts
& DMA_GSTS_RTPS
), sts
);
1131 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1134 static void iommu_flush_write_buffer(struct intel_iommu
*iommu
)
1139 if (!rwbf_quirk
&& !cap_rwbf(iommu
->cap
))
1142 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1143 writel(iommu
->gcmd
| DMA_GCMD_WBF
, iommu
->reg
+ DMAR_GCMD_REG
);
1145 /* Make sure hardware complete it */
1146 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1147 readl
, (!(val
& DMA_GSTS_WBFS
)), val
);
1149 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1152 /* return value determine if we need a write buffer flush */
1153 static void __iommu_flush_context(struct intel_iommu
*iommu
,
1154 u16 did
, u16 source_id
, u8 function_mask
,
1161 case DMA_CCMD_GLOBAL_INVL
:
1162 val
= DMA_CCMD_GLOBAL_INVL
;
1164 case DMA_CCMD_DOMAIN_INVL
:
1165 val
= DMA_CCMD_DOMAIN_INVL
|DMA_CCMD_DID(did
);
1167 case DMA_CCMD_DEVICE_INVL
:
1168 val
= DMA_CCMD_DEVICE_INVL
|DMA_CCMD_DID(did
)
1169 | DMA_CCMD_SID(source_id
) | DMA_CCMD_FM(function_mask
);
1174 val
|= DMA_CCMD_ICC
;
1176 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1177 dmar_writeq(iommu
->reg
+ DMAR_CCMD_REG
, val
);
1179 /* Make sure hardware complete it */
1180 IOMMU_WAIT_OP(iommu
, DMAR_CCMD_REG
,
1181 dmar_readq
, (!(val
& DMA_CCMD_ICC
)), val
);
1183 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1186 /* return value determine if we need a write buffer flush */
1187 static void __iommu_flush_iotlb(struct intel_iommu
*iommu
, u16 did
,
1188 u64 addr
, unsigned int size_order
, u64 type
)
1190 int tlb_offset
= ecap_iotlb_offset(iommu
->ecap
);
1191 u64 val
= 0, val_iva
= 0;
1195 case DMA_TLB_GLOBAL_FLUSH
:
1196 /* global flush doesn't need set IVA_REG */
1197 val
= DMA_TLB_GLOBAL_FLUSH
|DMA_TLB_IVT
;
1199 case DMA_TLB_DSI_FLUSH
:
1200 val
= DMA_TLB_DSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
1202 case DMA_TLB_PSI_FLUSH
:
1203 val
= DMA_TLB_PSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
1204 /* IH bit is passed in as part of address */
1205 val_iva
= size_order
| addr
;
1210 /* Note: set drain read/write */
1213 * This is probably to be super secure.. Looks like we can
1214 * ignore it without any impact.
1216 if (cap_read_drain(iommu
->cap
))
1217 val
|= DMA_TLB_READ_DRAIN
;
1219 if (cap_write_drain(iommu
->cap
))
1220 val
|= DMA_TLB_WRITE_DRAIN
;
1222 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1223 /* Note: Only uses first TLB reg currently */
1225 dmar_writeq(iommu
->reg
+ tlb_offset
, val_iva
);
1226 dmar_writeq(iommu
->reg
+ tlb_offset
+ 8, val
);
1228 /* Make sure hardware complete it */
1229 IOMMU_WAIT_OP(iommu
, tlb_offset
+ 8,
1230 dmar_readq
, (!(val
& DMA_TLB_IVT
)), val
);
1232 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1234 /* check IOTLB invalidation granularity */
1235 if (DMA_TLB_IAIG(val
) == 0)
1236 printk(KERN_ERR
"IOMMU: flush IOTLB failed\n");
1237 if (DMA_TLB_IAIG(val
) != DMA_TLB_IIRG(type
))
1238 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1239 (unsigned long long)DMA_TLB_IIRG(type
),
1240 (unsigned long long)DMA_TLB_IAIG(val
));
1243 static struct device_domain_info
*
1244 iommu_support_dev_iotlb (struct dmar_domain
*domain
, struct intel_iommu
*iommu
,
1248 unsigned long flags
;
1249 struct device_domain_info
*info
;
1250 struct pci_dev
*pdev
;
1252 if (!ecap_dev_iotlb_support(iommu
->ecap
))
1258 spin_lock_irqsave(&device_domain_lock
, flags
);
1259 list_for_each_entry(info
, &domain
->devices
, link
)
1260 if (info
->bus
== bus
&& info
->devfn
== devfn
) {
1264 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1266 if (!found
|| !info
->dev
|| !dev_is_pci(info
->dev
))
1269 pdev
= to_pci_dev(info
->dev
);
1271 if (!pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_ATS
))
1274 if (!dmar_find_matched_atsr_unit(pdev
))
1280 static void iommu_enable_dev_iotlb(struct device_domain_info
*info
)
1282 if (!info
|| !dev_is_pci(info
->dev
))
1285 pci_enable_ats(to_pci_dev(info
->dev
), VTD_PAGE_SHIFT
);
1288 static void iommu_disable_dev_iotlb(struct device_domain_info
*info
)
1290 if (!info
->dev
|| !dev_is_pci(info
->dev
) ||
1291 !pci_ats_enabled(to_pci_dev(info
->dev
)))
1294 pci_disable_ats(to_pci_dev(info
->dev
));
1297 static void iommu_flush_dev_iotlb(struct dmar_domain
*domain
,
1298 u64 addr
, unsigned mask
)
1301 unsigned long flags
;
1302 struct device_domain_info
*info
;
1304 spin_lock_irqsave(&device_domain_lock
, flags
);
1305 list_for_each_entry(info
, &domain
->devices
, link
) {
1306 struct pci_dev
*pdev
;
1307 if (!info
->dev
|| !dev_is_pci(info
->dev
))
1310 pdev
= to_pci_dev(info
->dev
);
1311 if (!pci_ats_enabled(pdev
))
1314 sid
= info
->bus
<< 8 | info
->devfn
;
1315 qdep
= pci_ats_queue_depth(pdev
);
1316 qi_flush_dev_iotlb(info
->iommu
, sid
, qdep
, addr
, mask
);
1318 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1321 static void iommu_flush_iotlb_psi(struct intel_iommu
*iommu
, u16 did
,
1322 unsigned long pfn
, unsigned int pages
, int ih
, int map
)
1324 unsigned int mask
= ilog2(__roundup_pow_of_two(pages
));
1325 uint64_t addr
= (uint64_t)pfn
<< VTD_PAGE_SHIFT
;
1332 * Fallback to domain selective flush if no PSI support or the size is
1334 * PSI requires page size to be 2 ^ x, and the base address is naturally
1335 * aligned to the size
1337 if (!cap_pgsel_inv(iommu
->cap
) || mask
> cap_max_amask_val(iommu
->cap
))
1338 iommu
->flush
.flush_iotlb(iommu
, did
, 0, 0,
1341 iommu
->flush
.flush_iotlb(iommu
, did
, addr
| ih
, mask
,
1345 * In caching mode, changes of pages from non-present to present require
1346 * flush. However, device IOTLB doesn't need to be flushed in this case.
1348 if (!cap_caching_mode(iommu
->cap
) || !map
)
1349 iommu_flush_dev_iotlb(iommu
->domains
[did
], addr
, mask
);
1352 static void iommu_disable_protect_mem_regions(struct intel_iommu
*iommu
)
1355 unsigned long flags
;
1357 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
1358 pmen
= readl(iommu
->reg
+ DMAR_PMEN_REG
);
1359 pmen
&= ~DMA_PMEN_EPM
;
1360 writel(pmen
, iommu
->reg
+ DMAR_PMEN_REG
);
1362 /* wait for the protected region status bit to clear */
1363 IOMMU_WAIT_OP(iommu
, DMAR_PMEN_REG
,
1364 readl
, !(pmen
& DMA_PMEN_PRS
), pmen
);
1366 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1369 static int iommu_enable_translation(struct intel_iommu
*iommu
)
1372 unsigned long flags
;
1374 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
1375 iommu
->gcmd
|= DMA_GCMD_TE
;
1376 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1378 /* Make sure hardware complete it */
1379 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1380 readl
, (sts
& DMA_GSTS_TES
), sts
);
1382 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1386 static int iommu_disable_translation(struct intel_iommu
*iommu
)
1391 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1392 iommu
->gcmd
&= ~DMA_GCMD_TE
;
1393 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1395 /* Make sure hardware complete it */
1396 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1397 readl
, (!(sts
& DMA_GSTS_TES
)), sts
);
1399 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1404 static int iommu_init_domains(struct intel_iommu
*iommu
)
1406 unsigned long ndomains
;
1407 unsigned long nlongs
;
1409 ndomains
= cap_ndoms(iommu
->cap
);
1410 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1411 iommu
->seq_id
, ndomains
);
1412 nlongs
= BITS_TO_LONGS(ndomains
);
1414 spin_lock_init(&iommu
->lock
);
1416 /* TBD: there might be 64K domains,
1417 * consider other allocation for future chip
1419 iommu
->domain_ids
= kcalloc(nlongs
, sizeof(unsigned long), GFP_KERNEL
);
1420 if (!iommu
->domain_ids
) {
1421 pr_err("IOMMU%d: allocating domain id array failed\n",
1425 iommu
->domains
= kcalloc(ndomains
, sizeof(struct dmar_domain
*),
1427 if (!iommu
->domains
) {
1428 pr_err("IOMMU%d: allocating domain array failed\n",
1430 kfree(iommu
->domain_ids
);
1431 iommu
->domain_ids
= NULL
;
1436 * if Caching mode is set, then invalid translations are tagged
1437 * with domainid 0. Hence we need to pre-allocate it.
1439 if (cap_caching_mode(iommu
->cap
))
1440 set_bit(0, iommu
->domain_ids
);
1444 static void free_dmar_iommu(struct intel_iommu
*iommu
)
1446 struct dmar_domain
*domain
;
1448 unsigned long flags
;
1450 if ((iommu
->domains
) && (iommu
->domain_ids
)) {
1451 for_each_set_bit(i
, iommu
->domain_ids
, cap_ndoms(iommu
->cap
)) {
1453 * Domain id 0 is reserved for invalid translation
1454 * if hardware supports caching mode.
1456 if (cap_caching_mode(iommu
->cap
) && i
== 0)
1459 domain
= iommu
->domains
[i
];
1460 clear_bit(i
, iommu
->domain_ids
);
1462 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1463 count
= --domain
->iommu_count
;
1464 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1466 domain_exit(domain
);
1470 if (iommu
->gcmd
& DMA_GCMD_TE
)
1471 iommu_disable_translation(iommu
);
1473 kfree(iommu
->domains
);
1474 kfree(iommu
->domain_ids
);
1475 iommu
->domains
= NULL
;
1476 iommu
->domain_ids
= NULL
;
1478 g_iommus
[iommu
->seq_id
] = NULL
;
1480 /* free context mapping */
1481 free_context_table(iommu
);
1484 static struct dmar_domain
*alloc_domain(bool vm
)
1486 /* domain id for virtual machine, it won't be set in context */
1487 static atomic_t vm_domid
= ATOMIC_INIT(0);
1488 struct dmar_domain
*domain
;
1490 domain
= alloc_domain_mem();
1495 domain
->iommu_count
= 0;
1496 memset(domain
->iommu_bmp
, 0, sizeof(domain
->iommu_bmp
));
1498 spin_lock_init(&domain
->iommu_lock
);
1499 INIT_LIST_HEAD(&domain
->devices
);
1501 domain
->id
= atomic_inc_return(&vm_domid
);
1502 domain
->flags
= DOMAIN_FLAG_VIRTUAL_MACHINE
;
1508 static int iommu_attach_domain(struct dmar_domain
*domain
,
1509 struct intel_iommu
*iommu
)
1512 unsigned long ndomains
;
1513 unsigned long flags
;
1515 ndomains
= cap_ndoms(iommu
->cap
);
1517 spin_lock_irqsave(&iommu
->lock
, flags
);
1519 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1520 if (num
>= ndomains
) {
1521 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1522 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1527 domain
->iommu_count
++;
1528 set_bit(num
, iommu
->domain_ids
);
1529 set_bit(iommu
->seq_id
, domain
->iommu_bmp
);
1530 iommu
->domains
[num
] = domain
;
1531 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1536 static void iommu_detach_domain(struct dmar_domain
*domain
,
1537 struct intel_iommu
*iommu
)
1539 unsigned long flags
;
1542 spin_lock_irqsave(&iommu
->lock
, flags
);
1543 ndomains
= cap_ndoms(iommu
->cap
);
1544 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
1545 if (iommu
->domains
[num
] == domain
) {
1546 clear_bit(num
, iommu
->domain_ids
);
1547 iommu
->domains
[num
] = NULL
;
1551 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1554 static struct iova_domain reserved_iova_list
;
1555 static struct lock_class_key reserved_rbtree_key
;
1557 static int dmar_init_reserved_ranges(void)
1559 struct pci_dev
*pdev
= NULL
;
1563 init_iova_domain(&reserved_iova_list
, DMA_32BIT_PFN
);
1565 lockdep_set_class(&reserved_iova_list
.iova_rbtree_lock
,
1566 &reserved_rbtree_key
);
1568 /* IOAPIC ranges shouldn't be accessed by DMA */
1569 iova
= reserve_iova(&reserved_iova_list
, IOVA_PFN(IOAPIC_RANGE_START
),
1570 IOVA_PFN(IOAPIC_RANGE_END
));
1572 printk(KERN_ERR
"Reserve IOAPIC range failed\n");
1576 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1577 for_each_pci_dev(pdev
) {
1580 for (i
= 0; i
< PCI_NUM_RESOURCES
; i
++) {
1581 r
= &pdev
->resource
[i
];
1582 if (!r
->flags
|| !(r
->flags
& IORESOURCE_MEM
))
1584 iova
= reserve_iova(&reserved_iova_list
,
1588 printk(KERN_ERR
"Reserve iova failed\n");
1596 static void domain_reserve_special_ranges(struct dmar_domain
*domain
)
1598 copy_reserved_iova(&reserved_iova_list
, &domain
->iovad
);
1601 static inline int guestwidth_to_adjustwidth(int gaw
)
1604 int r
= (gaw
- 12) % 9;
1615 static int domain_init(struct dmar_domain
*domain
, int guest_width
)
1617 struct intel_iommu
*iommu
;
1618 int adjust_width
, agaw
;
1619 unsigned long sagaw
;
1621 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
1622 domain_reserve_special_ranges(domain
);
1624 /* calculate AGAW */
1625 iommu
= domain_get_iommu(domain
);
1626 if (guest_width
> cap_mgaw(iommu
->cap
))
1627 guest_width
= cap_mgaw(iommu
->cap
);
1628 domain
->gaw
= guest_width
;
1629 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
1630 agaw
= width_to_agaw(adjust_width
);
1631 sagaw
= cap_sagaw(iommu
->cap
);
1632 if (!test_bit(agaw
, &sagaw
)) {
1633 /* hardware doesn't support it, choose a bigger one */
1634 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw
);
1635 agaw
= find_next_bit(&sagaw
, 5, agaw
);
1639 domain
->agaw
= agaw
;
1641 if (ecap_coherent(iommu
->ecap
))
1642 domain
->iommu_coherency
= 1;
1644 domain
->iommu_coherency
= 0;
1646 if (ecap_sc_support(iommu
->ecap
))
1647 domain
->iommu_snooping
= 1;
1649 domain
->iommu_snooping
= 0;
1651 if (intel_iommu_superpage
)
1652 domain
->iommu_superpage
= fls(cap_super_page_val(iommu
->cap
));
1654 domain
->iommu_superpage
= 0;
1656 domain
->nid
= iommu
->node
;
1658 /* always allocate the top pgd */
1659 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page(domain
->nid
);
1662 __iommu_flush_cache(iommu
, domain
->pgd
, PAGE_SIZE
);
1666 static void domain_exit(struct dmar_domain
*domain
)
1668 struct dmar_drhd_unit
*drhd
;
1669 struct intel_iommu
*iommu
;
1670 struct page
*freelist
= NULL
;
1672 /* Domain 0 is reserved, so dont process it */
1676 /* Flush any lazy unmaps that may reference this domain */
1677 if (!intel_iommu_strict
)
1678 flush_unmaps_timeout(0);
1680 /* remove associated devices */
1681 domain_remove_dev_info(domain
);
1684 put_iova_domain(&domain
->iovad
);
1686 freelist
= domain_unmap(domain
, 0, DOMAIN_MAX_PFN(domain
->gaw
));
1688 /* clear attached or cached domains */
1690 for_each_active_iommu(iommu
, drhd
)
1691 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
1692 test_bit(iommu
->seq_id
, domain
->iommu_bmp
))
1693 iommu_detach_domain(domain
, iommu
);
1696 dma_free_pagelist(freelist
);
1698 free_domain_mem(domain
);
1701 static int domain_context_mapping_one(struct dmar_domain
*domain
,
1702 struct intel_iommu
*iommu
,
1703 u8 bus
, u8 devfn
, int translation
)
1705 struct context_entry
*context
;
1706 unsigned long flags
;
1707 struct dma_pte
*pgd
;
1709 unsigned long ndomains
;
1712 struct device_domain_info
*info
= NULL
;
1714 pr_debug("Set context mapping for %02x:%02x.%d\n",
1715 bus
, PCI_SLOT(devfn
), PCI_FUNC(devfn
));
1717 BUG_ON(!domain
->pgd
);
1718 BUG_ON(translation
!= CONTEXT_TT_PASS_THROUGH
&&
1719 translation
!= CONTEXT_TT_MULTI_LEVEL
);
1721 context
= device_to_context_entry(iommu
, bus
, devfn
);
1724 spin_lock_irqsave(&iommu
->lock
, flags
);
1725 if (context_present(context
)) {
1726 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1733 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
1734 domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
) {
1737 /* find an available domain id for this device in iommu */
1738 ndomains
= cap_ndoms(iommu
->cap
);
1739 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
1740 if (iommu
->domains
[num
] == domain
) {
1748 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1749 if (num
>= ndomains
) {
1750 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1751 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1755 set_bit(num
, iommu
->domain_ids
);
1756 iommu
->domains
[num
] = domain
;
1760 /* Skip top levels of page tables for
1761 * iommu which has less agaw than default.
1762 * Unnecessary for PT mode.
1764 if (translation
!= CONTEXT_TT_PASS_THROUGH
) {
1765 for (agaw
= domain
->agaw
; agaw
!= iommu
->agaw
; agaw
--) {
1766 pgd
= phys_to_virt(dma_pte_addr(pgd
));
1767 if (!dma_pte_present(pgd
)) {
1768 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1775 context_set_domain_id(context
, id
);
1777 if (translation
!= CONTEXT_TT_PASS_THROUGH
) {
1778 info
= iommu_support_dev_iotlb(domain
, iommu
, bus
, devfn
);
1779 translation
= info
? CONTEXT_TT_DEV_IOTLB
:
1780 CONTEXT_TT_MULTI_LEVEL
;
1783 * In pass through mode, AW must be programmed to indicate the largest
1784 * AGAW value supported by hardware. And ASR is ignored by hardware.
1786 if (unlikely(translation
== CONTEXT_TT_PASS_THROUGH
))
1787 context_set_address_width(context
, iommu
->msagaw
);
1789 context_set_address_root(context
, virt_to_phys(pgd
));
1790 context_set_address_width(context
, iommu
->agaw
);
1793 context_set_translation_type(context
, translation
);
1794 context_set_fault_enable(context
);
1795 context_set_present(context
);
1796 domain_flush_cache(domain
, context
, sizeof(*context
));
1799 * It's a non-present to present mapping. If hardware doesn't cache
1800 * non-present entry we only need to flush the write-buffer. If the
1801 * _does_ cache non-present entries, then it does so in the special
1802 * domain #0, which we have to flush:
1804 if (cap_caching_mode(iommu
->cap
)) {
1805 iommu
->flush
.flush_context(iommu
, 0,
1806 (((u16
)bus
) << 8) | devfn
,
1807 DMA_CCMD_MASK_NOBIT
,
1808 DMA_CCMD_DEVICE_INVL
);
1809 iommu
->flush
.flush_iotlb(iommu
, domain
->id
, 0, 0, DMA_TLB_DSI_FLUSH
);
1811 iommu_flush_write_buffer(iommu
);
1813 iommu_enable_dev_iotlb(info
);
1814 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1816 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1817 if (!test_and_set_bit(iommu
->seq_id
, domain
->iommu_bmp
)) {
1818 domain
->iommu_count
++;
1819 if (domain
->iommu_count
== 1)
1820 domain
->nid
= iommu
->node
;
1821 domain_update_iommu_cap(domain
);
1823 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1828 domain_context_mapping(struct dmar_domain
*domain
, struct pci_dev
*pdev
,
1832 struct pci_dev
*tmp
, *parent
;
1833 struct intel_iommu
*iommu
;
1835 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
1840 ret
= domain_context_mapping_one(domain
, iommu
,
1841 pdev
->bus
->number
, pdev
->devfn
,
1846 /* dependent device mapping */
1847 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1850 /* Secondary interface's bus number and devfn 0 */
1851 parent
= pdev
->bus
->self
;
1852 while (parent
!= tmp
) {
1853 ret
= domain_context_mapping_one(domain
, iommu
,
1854 parent
->bus
->number
,
1855 parent
->devfn
, translation
);
1858 parent
= parent
->bus
->self
;
1860 if (pci_is_pcie(tmp
)) /* this is a PCIe-to-PCI bridge */
1861 return domain_context_mapping_one(domain
, iommu
,
1862 tmp
->subordinate
->number
, 0,
1864 else /* this is a legacy PCI bridge */
1865 return domain_context_mapping_one(domain
, iommu
,
1871 static int domain_context_mapped(struct pci_dev
*pdev
)
1874 struct pci_dev
*tmp
, *parent
;
1875 struct intel_iommu
*iommu
;
1877 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
1882 ret
= device_context_mapped(iommu
, pdev
->bus
->number
, pdev
->devfn
);
1885 /* dependent device mapping */
1886 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1889 /* Secondary interface's bus number and devfn 0 */
1890 parent
= pdev
->bus
->self
;
1891 while (parent
!= tmp
) {
1892 ret
= device_context_mapped(iommu
, parent
->bus
->number
,
1896 parent
= parent
->bus
->self
;
1898 if (pci_is_pcie(tmp
))
1899 return device_context_mapped(iommu
, tmp
->subordinate
->number
,
1902 return device_context_mapped(iommu
, tmp
->bus
->number
,
1906 /* Returns a number of VTD pages, but aligned to MM page size */
1907 static inline unsigned long aligned_nrpages(unsigned long host_addr
,
1910 host_addr
&= ~PAGE_MASK
;
1911 return PAGE_ALIGN(host_addr
+ size
) >> VTD_PAGE_SHIFT
;
1914 /* Return largest possible superpage level for a given mapping */
1915 static inline int hardware_largepage_caps(struct dmar_domain
*domain
,
1916 unsigned long iov_pfn
,
1917 unsigned long phy_pfn
,
1918 unsigned long pages
)
1920 int support
, level
= 1;
1921 unsigned long pfnmerge
;
1923 support
= domain
->iommu_superpage
;
1925 /* To use a large page, the virtual *and* physical addresses
1926 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1927 of them will mean we have to use smaller pages. So just
1928 merge them and check both at once. */
1929 pfnmerge
= iov_pfn
| phy_pfn
;
1931 while (support
&& !(pfnmerge
& ~VTD_STRIDE_MASK
)) {
1932 pages
>>= VTD_STRIDE_SHIFT
;
1935 pfnmerge
>>= VTD_STRIDE_SHIFT
;
1942 static int __domain_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
1943 struct scatterlist
*sg
, unsigned long phys_pfn
,
1944 unsigned long nr_pages
, int prot
)
1946 struct dma_pte
*first_pte
= NULL
, *pte
= NULL
;
1947 phys_addr_t
uninitialized_var(pteval
);
1948 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
1949 unsigned long sg_res
;
1950 unsigned int largepage_lvl
= 0;
1951 unsigned long lvl_pages
= 0;
1953 BUG_ON(addr_width
< BITS_PER_LONG
&& (iov_pfn
+ nr_pages
- 1) >> addr_width
);
1955 if ((prot
& (DMA_PTE_READ
|DMA_PTE_WRITE
)) == 0)
1958 prot
&= DMA_PTE_READ
| DMA_PTE_WRITE
| DMA_PTE_SNP
;
1963 sg_res
= nr_pages
+ 1;
1964 pteval
= ((phys_addr_t
)phys_pfn
<< VTD_PAGE_SHIFT
) | prot
;
1967 while (nr_pages
> 0) {
1971 sg_res
= aligned_nrpages(sg
->offset
, sg
->length
);
1972 sg
->dma_address
= ((dma_addr_t
)iov_pfn
<< VTD_PAGE_SHIFT
) + sg
->offset
;
1973 sg
->dma_length
= sg
->length
;
1974 pteval
= page_to_phys(sg_page(sg
)) | prot
;
1975 phys_pfn
= pteval
>> VTD_PAGE_SHIFT
;
1979 largepage_lvl
= hardware_largepage_caps(domain
, iov_pfn
, phys_pfn
, sg_res
);
1981 first_pte
= pte
= pfn_to_dma_pte(domain
, iov_pfn
, &largepage_lvl
);
1984 /* It is large page*/
1985 if (largepage_lvl
> 1) {
1986 pteval
|= DMA_PTE_LARGE_PAGE
;
1987 /* Ensure that old small page tables are removed to make room
1988 for superpage, if they exist. */
1989 dma_pte_clear_range(domain
, iov_pfn
,
1990 iov_pfn
+ lvl_to_nr_pages(largepage_lvl
) - 1);
1991 dma_pte_free_pagetable(domain
, iov_pfn
,
1992 iov_pfn
+ lvl_to_nr_pages(largepage_lvl
) - 1);
1994 pteval
&= ~(uint64_t)DMA_PTE_LARGE_PAGE
;
1998 /* We don't need lock here, nobody else
1999 * touches the iova range
2001 tmp
= cmpxchg64_local(&pte
->val
, 0ULL, pteval
);
2003 static int dumps
= 5;
2004 printk(KERN_CRIT
"ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2005 iov_pfn
, tmp
, (unsigned long long)pteval
);
2008 debug_dma_dump_mappings(NULL
);
2013 lvl_pages
= lvl_to_nr_pages(largepage_lvl
);
2015 BUG_ON(nr_pages
< lvl_pages
);
2016 BUG_ON(sg_res
< lvl_pages
);
2018 nr_pages
-= lvl_pages
;
2019 iov_pfn
+= lvl_pages
;
2020 phys_pfn
+= lvl_pages
;
2021 pteval
+= lvl_pages
* VTD_PAGE_SIZE
;
2022 sg_res
-= lvl_pages
;
2024 /* If the next PTE would be the first in a new page, then we
2025 need to flush the cache on the entries we've just written.
2026 And then we'll need to recalculate 'pte', so clear it and
2027 let it get set again in the if (!pte) block above.
2029 If we're done (!nr_pages) we need to flush the cache too.
2031 Also if we've been setting superpages, we may need to
2032 recalculate 'pte' and switch back to smaller pages for the
2033 end of the mapping, if the trailing size is not enough to
2034 use another superpage (i.e. sg_res < lvl_pages). */
2036 if (!nr_pages
|| first_pte_in_page(pte
) ||
2037 (largepage_lvl
> 1 && sg_res
< lvl_pages
)) {
2038 domain_flush_cache(domain
, first_pte
,
2039 (void *)pte
- (void *)first_pte
);
2043 if (!sg_res
&& nr_pages
)
2049 static inline int domain_sg_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
2050 struct scatterlist
*sg
, unsigned long nr_pages
,
2053 return __domain_mapping(domain
, iov_pfn
, sg
, 0, nr_pages
, prot
);
2056 static inline int domain_pfn_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
2057 unsigned long phys_pfn
, unsigned long nr_pages
,
2060 return __domain_mapping(domain
, iov_pfn
, NULL
, phys_pfn
, nr_pages
, prot
);
2063 static void iommu_detach_dev(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
2068 clear_context_table(iommu
, bus
, devfn
);
2069 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
2070 DMA_CCMD_GLOBAL_INVL
);
2071 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
2074 static inline void unlink_domain_info(struct device_domain_info
*info
)
2076 assert_spin_locked(&device_domain_lock
);
2077 list_del(&info
->link
);
2078 list_del(&info
->global
);
2080 info
->dev
->archdata
.iommu
= NULL
;
2083 static void domain_remove_dev_info(struct dmar_domain
*domain
)
2085 struct device_domain_info
*info
;
2086 unsigned long flags
, flags2
;
2088 spin_lock_irqsave(&device_domain_lock
, flags
);
2089 while (!list_empty(&domain
->devices
)) {
2090 info
= list_entry(domain
->devices
.next
,
2091 struct device_domain_info
, link
);
2092 unlink_domain_info(info
);
2093 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2095 iommu_disable_dev_iotlb(info
);
2096 iommu_detach_dev(info
->iommu
, info
->bus
, info
->devfn
);
2098 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) {
2099 iommu_detach_dependent_devices(info
->iommu
, info
->dev
);
2100 /* clear this iommu in iommu_bmp, update iommu count
2103 spin_lock_irqsave(&domain
->iommu_lock
, flags2
);
2104 if (test_and_clear_bit(info
->iommu
->seq_id
,
2105 domain
->iommu_bmp
)) {
2106 domain
->iommu_count
--;
2107 domain_update_iommu_cap(domain
);
2109 spin_unlock_irqrestore(&domain
->iommu_lock
, flags2
);
2112 free_devinfo_mem(info
);
2113 spin_lock_irqsave(&device_domain_lock
, flags
);
2115 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2120 * Note: we use struct device->archdata.iommu stores the info
2122 static struct dmar_domain
*find_domain(struct device
*dev
)
2124 struct device_domain_info
*info
;
2126 /* No lock here, assumes no domain exit in normal case */
2127 info
= dev
->archdata
.iommu
;
2129 return info
->domain
;
2133 static inline struct device_domain_info
*
2134 dmar_search_domain_by_dev_info(int segment
, int bus
, int devfn
)
2136 struct device_domain_info
*info
;
2138 list_for_each_entry(info
, &device_domain_list
, global
)
2139 if (info
->segment
== segment
&& info
->bus
== bus
&&
2140 info
->devfn
== devfn
)
2146 static struct dmar_domain
*dmar_insert_dev_info(struct intel_iommu
*iommu
,
2147 int segment
, int bus
, int devfn
,
2149 struct dmar_domain
*domain
)
2151 struct dmar_domain
*found
= NULL
;
2152 struct device_domain_info
*info
;
2153 unsigned long flags
;
2155 info
= alloc_devinfo_mem();
2159 info
->segment
= segment
;
2161 info
->devfn
= devfn
;
2163 info
->domain
= domain
;
2164 info
->iommu
= iommu
;
2166 domain
->flags
|= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES
;
2168 spin_lock_irqsave(&device_domain_lock
, flags
);
2170 found
= find_domain(dev
);
2172 struct device_domain_info
*info2
;
2173 info2
= dmar_search_domain_by_dev_info(segment
, bus
, devfn
);
2175 found
= info2
->domain
;
2178 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2179 free_devinfo_mem(info
);
2180 /* Caller must free the original domain */
2184 list_add(&info
->link
, &domain
->devices
);
2185 list_add(&info
->global
, &device_domain_list
);
2187 dev
->archdata
.iommu
= info
;
2188 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2193 /* domain is initialized */
2194 static struct dmar_domain
*get_domain_for_dev(struct pci_dev
*pdev
, int gaw
)
2196 struct dmar_domain
*domain
, *free
= NULL
;
2197 struct intel_iommu
*iommu
= NULL
;
2198 struct device_domain_info
*info
;
2199 struct dmar_drhd_unit
*drhd
;
2200 struct pci_dev
*dev_tmp
;
2201 unsigned long flags
;
2202 int bus
= 0, devfn
= 0;
2205 domain
= find_domain(&pdev
->dev
);
2209 segment
= pci_domain_nr(pdev
->bus
);
2211 dev_tmp
= pci_find_upstream_pcie_bridge(pdev
);
2213 if (pci_is_pcie(dev_tmp
)) {
2214 bus
= dev_tmp
->subordinate
->number
;
2217 bus
= dev_tmp
->bus
->number
;
2218 devfn
= dev_tmp
->devfn
;
2220 spin_lock_irqsave(&device_domain_lock
, flags
);
2221 info
= dmar_search_domain_by_dev_info(segment
, bus
, devfn
);
2223 iommu
= info
->iommu
;
2224 domain
= info
->domain
;
2226 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2231 drhd
= dmar_find_matched_drhd_unit(pdev
);
2233 printk(KERN_ERR
"IOMMU: can't find DMAR for device %s\n",
2237 iommu
= drhd
->iommu
;
2239 /* Allocate and intialize new domain for the device */
2240 domain
= alloc_domain(false);
2243 if (iommu_attach_domain(domain
, iommu
)) {
2244 free_domain_mem(domain
);
2248 if (domain_init(domain
, gaw
))
2251 /* register pcie-to-pci device */
2253 domain
= dmar_insert_dev_info(iommu
, segment
, bus
, devfn
, NULL
,
2260 domain
= dmar_insert_dev_info(iommu
, segment
, pdev
->bus
->number
,
2261 pdev
->devfn
, &pdev
->dev
, domain
);
2269 static int iommu_identity_mapping
;
2270 #define IDENTMAP_ALL 1
2271 #define IDENTMAP_GFX 2
2272 #define IDENTMAP_AZALIA 4
2274 static int iommu_domain_identity_map(struct dmar_domain
*domain
,
2275 unsigned long long start
,
2276 unsigned long long end
)
2278 unsigned long first_vpfn
= start
>> VTD_PAGE_SHIFT
;
2279 unsigned long last_vpfn
= end
>> VTD_PAGE_SHIFT
;
2281 if (!reserve_iova(&domain
->iovad
, dma_to_mm_pfn(first_vpfn
),
2282 dma_to_mm_pfn(last_vpfn
))) {
2283 printk(KERN_ERR
"IOMMU: reserve iova failed\n");
2287 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2288 start
, end
, domain
->id
);
2290 * RMRR range might have overlap with physical memory range,
2293 dma_pte_clear_range(domain
, first_vpfn
, last_vpfn
);
2295 return domain_pfn_mapping(domain
, first_vpfn
, first_vpfn
,
2296 last_vpfn
- first_vpfn
+ 1,
2297 DMA_PTE_READ
|DMA_PTE_WRITE
);
2300 static int iommu_prepare_identity_map(struct pci_dev
*pdev
,
2301 unsigned long long start
,
2302 unsigned long long end
)
2304 struct dmar_domain
*domain
;
2307 domain
= get_domain_for_dev(pdev
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
2311 /* For _hardware_ passthrough, don't bother. But for software
2312 passthrough, we do it anyway -- it may indicate a memory
2313 range which is reserved in E820, so which didn't get set
2314 up to start with in si_domain */
2315 if (domain
== si_domain
&& hw_pass_through
) {
2316 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2317 pci_name(pdev
), start
, end
);
2322 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2323 pci_name(pdev
), start
, end
);
2326 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2327 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2328 dmi_get_system_info(DMI_BIOS_VENDOR
),
2329 dmi_get_system_info(DMI_BIOS_VERSION
),
2330 dmi_get_system_info(DMI_PRODUCT_VERSION
));
2335 if (end
>> agaw_to_width(domain
->agaw
)) {
2336 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2337 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2338 agaw_to_width(domain
->agaw
),
2339 dmi_get_system_info(DMI_BIOS_VENDOR
),
2340 dmi_get_system_info(DMI_BIOS_VERSION
),
2341 dmi_get_system_info(DMI_PRODUCT_VERSION
));
2346 ret
= iommu_domain_identity_map(domain
, start
, end
);
2350 /* context entry init */
2351 ret
= domain_context_mapping(domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
2358 domain_exit(domain
);
2362 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit
*rmrr
,
2363 struct pci_dev
*pdev
)
2365 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2367 return iommu_prepare_identity_map(pdev
, rmrr
->base_address
,
2371 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2372 static inline void iommu_prepare_isa(void)
2374 struct pci_dev
*pdev
;
2377 pdev
= pci_get_class(PCI_CLASS_BRIDGE_ISA
<< 8, NULL
);
2381 printk(KERN_INFO
"IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2382 ret
= iommu_prepare_identity_map(pdev
, 0, 16*1024*1024 - 1);
2385 printk(KERN_ERR
"IOMMU: Failed to create 0-16MiB identity map; "
2386 "floppy might not work\n");
2390 static inline void iommu_prepare_isa(void)
2394 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2396 static int md_domain_init(struct dmar_domain
*domain
, int guest_width
);
2398 static int __init
si_domain_init(int hw
)
2400 struct dmar_drhd_unit
*drhd
;
2401 struct intel_iommu
*iommu
;
2404 si_domain
= alloc_domain(false);
2408 si_domain
->flags
= DOMAIN_FLAG_STATIC_IDENTITY
;
2410 for_each_active_iommu(iommu
, drhd
) {
2411 ret
= iommu_attach_domain(si_domain
, iommu
);
2413 domain_exit(si_domain
);
2418 if (md_domain_init(si_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
2419 domain_exit(si_domain
);
2423 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2429 for_each_online_node(nid
) {
2430 unsigned long start_pfn
, end_pfn
;
2433 for_each_mem_pfn_range(i
, nid
, &start_pfn
, &end_pfn
, NULL
) {
2434 ret
= iommu_domain_identity_map(si_domain
,
2435 PFN_PHYS(start_pfn
), PFN_PHYS(end_pfn
));
2444 static int identity_mapping(struct pci_dev
*pdev
)
2446 struct device_domain_info
*info
;
2448 if (likely(!iommu_identity_mapping
))
2451 info
= pdev
->dev
.archdata
.iommu
;
2452 if (info
&& info
!= DUMMY_DEVICE_DOMAIN_INFO
)
2453 return (info
->domain
== si_domain
);
2458 static int domain_add_dev_info(struct dmar_domain
*domain
,
2459 struct pci_dev
*pdev
,
2462 struct dmar_domain
*ndomain
;
2463 struct intel_iommu
*iommu
;
2466 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
),
2467 pdev
->bus
->number
, pdev
->devfn
);
2471 ndomain
= dmar_insert_dev_info(iommu
, pci_domain_nr(pdev
->bus
),
2472 pdev
->bus
->number
, pdev
->devfn
,
2473 &pdev
->dev
, domain
);
2474 if (ndomain
!= domain
)
2477 ret
= domain_context_mapping(domain
, pdev
, translation
);
2479 domain_remove_one_dev_info(domain
, pdev
);
2486 static bool device_has_rmrr(struct pci_dev
*dev
)
2488 struct dmar_rmrr_unit
*rmrr
;
2493 for_each_rmrr_units(rmrr
) {
2495 * Return TRUE if this RMRR contains the device that
2498 for_each_active_dev_scope(rmrr
->devices
,
2499 rmrr
->devices_cnt
, i
, tmp
)
2500 if (tmp
== &dev
->dev
) {
2509 static int iommu_should_identity_map(struct pci_dev
*pdev
, int startup
)
2513 * We want to prevent any device associated with an RMRR from
2514 * getting placed into the SI Domain. This is done because
2515 * problems exist when devices are moved in and out of domains
2516 * and their respective RMRR info is lost. We exempt USB devices
2517 * from this process due to their usage of RMRRs that are known
2518 * to not be needed after BIOS hand-off to OS.
2520 if (device_has_rmrr(pdev
) &&
2521 (pdev
->class >> 8) != PCI_CLASS_SERIAL_USB
)
2524 if ((iommu_identity_mapping
& IDENTMAP_AZALIA
) && IS_AZALIA(pdev
))
2527 if ((iommu_identity_mapping
& IDENTMAP_GFX
) && IS_GFX_DEVICE(pdev
))
2530 if (!(iommu_identity_mapping
& IDENTMAP_ALL
))
2534 * We want to start off with all devices in the 1:1 domain, and
2535 * take them out later if we find they can't access all of memory.
2537 * However, we can't do this for PCI devices behind bridges,
2538 * because all PCI devices behind the same bridge will end up
2539 * with the same source-id on their transactions.
2541 * Practically speaking, we can't change things around for these
2542 * devices at run-time, because we can't be sure there'll be no
2543 * DMA transactions in flight for any of their siblings.
2545 * So PCI devices (unless they're on the root bus) as well as
2546 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2547 * the 1:1 domain, just in _case_ one of their siblings turns out
2548 * not to be able to map all of memory.
2550 if (!pci_is_pcie(pdev
)) {
2551 if (!pci_is_root_bus(pdev
->bus
))
2553 if (pdev
->class >> 8 == PCI_CLASS_BRIDGE_PCI
)
2555 } else if (pci_pcie_type(pdev
) == PCI_EXP_TYPE_PCI_BRIDGE
)
2559 * At boot time, we don't yet know if devices will be 64-bit capable.
2560 * Assume that they will -- if they turn out not to be, then we can
2561 * take them out of the 1:1 domain later.
2565 * If the device's dma_mask is less than the system's memory
2566 * size then this is not a candidate for identity mapping.
2568 u64 dma_mask
= pdev
->dma_mask
;
2570 if (pdev
->dev
.coherent_dma_mask
&&
2571 pdev
->dev
.coherent_dma_mask
< dma_mask
)
2572 dma_mask
= pdev
->dev
.coherent_dma_mask
;
2574 return dma_mask
>= dma_get_required_mask(&pdev
->dev
);
2580 static int __init
iommu_prepare_static_identity_mapping(int hw
)
2582 struct pci_dev
*pdev
= NULL
;
2585 ret
= si_domain_init(hw
);
2589 for_each_pci_dev(pdev
) {
2590 if (iommu_should_identity_map(pdev
, 1)) {
2591 ret
= domain_add_dev_info(si_domain
, pdev
,
2592 hw
? CONTEXT_TT_PASS_THROUGH
:
2593 CONTEXT_TT_MULTI_LEVEL
);
2595 /* device not associated with an iommu */
2600 pr_info("IOMMU: %s identity mapping for device %s\n",
2601 hw
? "hardware" : "software", pci_name(pdev
));
2608 static int __init
init_dmars(void)
2610 struct dmar_drhd_unit
*drhd
;
2611 struct dmar_rmrr_unit
*rmrr
;
2613 struct intel_iommu
*iommu
;
2619 * initialize and program root entry to not present
2622 for_each_drhd_unit(drhd
) {
2624 * lock not needed as this is only incremented in the single
2625 * threaded kernel __init code path all other access are read
2628 if (g_num_of_iommus
< IOMMU_UNITS_SUPPORTED
) {
2632 printk_once(KERN_ERR
"intel-iommu: exceeded %d IOMMUs\n",
2633 IOMMU_UNITS_SUPPORTED
);
2636 g_iommus
= kcalloc(g_num_of_iommus
, sizeof(struct intel_iommu
*),
2639 printk(KERN_ERR
"Allocating global iommu array failed\n");
2644 deferred_flush
= kzalloc(g_num_of_iommus
*
2645 sizeof(struct deferred_flush_tables
), GFP_KERNEL
);
2646 if (!deferred_flush
) {
2651 for_each_active_iommu(iommu
, drhd
) {
2652 g_iommus
[iommu
->seq_id
] = iommu
;
2654 ret
= iommu_init_domains(iommu
);
2660 * we could share the same root & context tables
2661 * among all IOMMU's. Need to Split it later.
2663 ret
= iommu_alloc_root_entry(iommu
);
2665 printk(KERN_ERR
"IOMMU: allocate root entry failed\n");
2668 if (!ecap_pass_through(iommu
->ecap
))
2669 hw_pass_through
= 0;
2673 * Start from the sane iommu hardware state.
2675 for_each_active_iommu(iommu
, drhd
) {
2677 * If the queued invalidation is already initialized by us
2678 * (for example, while enabling interrupt-remapping) then
2679 * we got the things already rolling from a sane state.
2685 * Clear any previous faults.
2687 dmar_fault(-1, iommu
);
2689 * Disable queued invalidation if supported and already enabled
2690 * before OS handover.
2692 dmar_disable_qi(iommu
);
2695 for_each_active_iommu(iommu
, drhd
) {
2696 if (dmar_enable_qi(iommu
)) {
2698 * Queued Invalidate not enabled, use Register Based
2701 iommu
->flush
.flush_context
= __iommu_flush_context
;
2702 iommu
->flush
.flush_iotlb
= __iommu_flush_iotlb
;
2703 printk(KERN_INFO
"IOMMU %d 0x%Lx: using Register based "
2706 (unsigned long long)drhd
->reg_base_addr
);
2708 iommu
->flush
.flush_context
= qi_flush_context
;
2709 iommu
->flush
.flush_iotlb
= qi_flush_iotlb
;
2710 printk(KERN_INFO
"IOMMU %d 0x%Lx: using Queued "
2713 (unsigned long long)drhd
->reg_base_addr
);
2717 if (iommu_pass_through
)
2718 iommu_identity_mapping
|= IDENTMAP_ALL
;
2720 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2721 iommu_identity_mapping
|= IDENTMAP_GFX
;
2724 check_tylersburg_isoch();
2727 * If pass through is not set or not enabled, setup context entries for
2728 * identity mappings for rmrr, gfx, and isa and may fall back to static
2729 * identity mapping if iommu_identity_mapping is set.
2731 if (iommu_identity_mapping
) {
2732 ret
= iommu_prepare_static_identity_mapping(hw_pass_through
);
2734 printk(KERN_CRIT
"Failed to setup IOMMU pass-through\n");
2740 * for each dev attached to rmrr
2742 * locate drhd for dev, alloc domain for dev
2743 * allocate free domain
2744 * allocate page table entries for rmrr
2745 * if context not allocated for bus
2746 * allocate and init context
2747 * set present in root table for this bus
2748 * init context with domain, translation etc
2752 printk(KERN_INFO
"IOMMU: Setting RMRR:\n");
2753 for_each_rmrr_units(rmrr
) {
2754 /* some BIOS lists non-exist devices in DMAR table. */
2755 for_each_active_dev_scope(rmrr
->devices
, rmrr
->devices_cnt
,
2757 if (!dev_is_pci(dev
))
2759 ret
= iommu_prepare_rmrr_dev(rmrr
, to_pci_dev(dev
));
2762 "IOMMU: mapping reserved region failed\n");
2766 iommu_prepare_isa();
2771 * global invalidate context cache
2772 * global invalidate iotlb
2773 * enable translation
2775 for_each_iommu(iommu
, drhd
) {
2776 if (drhd
->ignored
) {
2778 * we always have to disable PMRs or DMA may fail on
2782 iommu_disable_protect_mem_regions(iommu
);
2786 iommu_flush_write_buffer(iommu
);
2788 ret
= dmar_set_interrupt(iommu
);
2792 iommu_set_root_entry(iommu
);
2794 iommu
->flush
.flush_context(iommu
, 0, 0, 0, DMA_CCMD_GLOBAL_INVL
);
2795 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
2797 ret
= iommu_enable_translation(iommu
);
2801 iommu_disable_protect_mem_regions(iommu
);
2807 for_each_active_iommu(iommu
, drhd
)
2808 free_dmar_iommu(iommu
);
2809 kfree(deferred_flush
);
2816 /* This takes a number of _MM_ pages, not VTD pages */
2817 static struct iova
*intel_alloc_iova(struct device
*dev
,
2818 struct dmar_domain
*domain
,
2819 unsigned long nrpages
, uint64_t dma_mask
)
2821 struct pci_dev
*pdev
= to_pci_dev(dev
);
2822 struct iova
*iova
= NULL
;
2824 /* Restrict dma_mask to the width that the iommu can handle */
2825 dma_mask
= min_t(uint64_t, DOMAIN_MAX_ADDR(domain
->gaw
), dma_mask
);
2827 if (!dmar_forcedac
&& dma_mask
> DMA_BIT_MASK(32)) {
2829 * First try to allocate an io virtual address in
2830 * DMA_BIT_MASK(32) and if that fails then try allocating
2833 iova
= alloc_iova(&domain
->iovad
, nrpages
,
2834 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2838 iova
= alloc_iova(&domain
->iovad
, nrpages
, IOVA_PFN(dma_mask
), 1);
2839 if (unlikely(!iova
)) {
2840 printk(KERN_ERR
"Allocating %ld-page iova for %s failed",
2841 nrpages
, pci_name(pdev
));
2848 static struct dmar_domain
*__get_valid_domain_for_dev(struct pci_dev
*pdev
)
2850 struct dmar_domain
*domain
;
2853 domain
= get_domain_for_dev(pdev
,
2854 DEFAULT_DOMAIN_ADDRESS_WIDTH
);
2857 "Allocating domain for %s failed", pci_name(pdev
));
2861 /* make sure context mapping is ok */
2862 if (unlikely(!domain_context_mapped(pdev
))) {
2863 ret
= domain_context_mapping(domain
, pdev
,
2864 CONTEXT_TT_MULTI_LEVEL
);
2867 "Domain context map for %s failed",
2876 static inline struct dmar_domain
*get_valid_domain_for_dev(struct pci_dev
*dev
)
2878 struct device_domain_info
*info
;
2880 /* No lock here, assumes no domain exit in normal case */
2881 info
= dev
->dev
.archdata
.iommu
;
2883 return info
->domain
;
2885 return __get_valid_domain_for_dev(dev
);
2888 static int iommu_dummy(struct device
*dev
)
2890 return dev
->archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
;
2893 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2894 static int iommu_no_mapping(struct device
*dev
)
2896 struct pci_dev
*pdev
;
2899 if (unlikely(!dev_is_pci(dev
)))
2902 if (iommu_dummy(dev
))
2905 if (!iommu_identity_mapping
)
2908 pdev
= to_pci_dev(dev
);
2909 found
= identity_mapping(pdev
);
2911 if (iommu_should_identity_map(pdev
, 0))
2915 * 32 bit DMA is removed from si_domain and fall back
2916 * to non-identity mapping.
2918 domain_remove_one_dev_info(si_domain
, pdev
);
2919 printk(KERN_INFO
"32bit %s uses non-identity mapping\n",
2925 * In case of a detached 64 bit DMA device from vm, the device
2926 * is put into si_domain for identity mapping.
2928 if (iommu_should_identity_map(pdev
, 0)) {
2930 ret
= domain_add_dev_info(si_domain
, pdev
,
2932 CONTEXT_TT_PASS_THROUGH
:
2933 CONTEXT_TT_MULTI_LEVEL
);
2935 printk(KERN_INFO
"64bit %s uses identity mapping\n",
2945 static dma_addr_t
__intel_map_single(struct device
*hwdev
, phys_addr_t paddr
,
2946 size_t size
, int dir
, u64 dma_mask
)
2948 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2949 struct dmar_domain
*domain
;
2950 phys_addr_t start_paddr
;
2954 struct intel_iommu
*iommu
;
2955 unsigned long paddr_pfn
= paddr
>> PAGE_SHIFT
;
2957 BUG_ON(dir
== DMA_NONE
);
2959 if (iommu_no_mapping(hwdev
))
2962 domain
= get_valid_domain_for_dev(pdev
);
2966 iommu
= domain_get_iommu(domain
);
2967 size
= aligned_nrpages(paddr
, size
);
2969 iova
= intel_alloc_iova(hwdev
, domain
, dma_to_mm_pfn(size
), dma_mask
);
2974 * Check if DMAR supports zero-length reads on write only
2977 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
2978 !cap_zlr(iommu
->cap
))
2979 prot
|= DMA_PTE_READ
;
2980 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
2981 prot
|= DMA_PTE_WRITE
;
2983 * paddr - (paddr + size) might be partial page, we should map the whole
2984 * page. Note: if two part of one page are separately mapped, we
2985 * might have two guest_addr mapping to the same host paddr, but this
2986 * is not a big problem
2988 ret
= domain_pfn_mapping(domain
, mm_to_dma_pfn(iova
->pfn_lo
),
2989 mm_to_dma_pfn(paddr_pfn
), size
, prot
);
2993 /* it's a non-present to present mapping. Only flush if caching mode */
2994 if (cap_caching_mode(iommu
->cap
))
2995 iommu_flush_iotlb_psi(iommu
, domain
->id
, mm_to_dma_pfn(iova
->pfn_lo
), size
, 0, 1);
2997 iommu_flush_write_buffer(iommu
);
2999 start_paddr
= (phys_addr_t
)iova
->pfn_lo
<< PAGE_SHIFT
;
3000 start_paddr
+= paddr
& ~PAGE_MASK
;
3005 __free_iova(&domain
->iovad
, iova
);
3006 printk(KERN_ERR
"Device %s request: %zx@%llx dir %d --- failed\n",
3007 pci_name(pdev
), size
, (unsigned long long)paddr
, dir
);
3011 static dma_addr_t
intel_map_page(struct device
*dev
, struct page
*page
,
3012 unsigned long offset
, size_t size
,
3013 enum dma_data_direction dir
,
3014 struct dma_attrs
*attrs
)
3016 return __intel_map_single(dev
, page_to_phys(page
) + offset
, size
,
3017 dir
, to_pci_dev(dev
)->dma_mask
);
3020 static void flush_unmaps(void)
3026 /* just flush them all */
3027 for (i
= 0; i
< g_num_of_iommus
; i
++) {
3028 struct intel_iommu
*iommu
= g_iommus
[i
];
3032 if (!deferred_flush
[i
].next
)
3035 /* In caching mode, global flushes turn emulation expensive */
3036 if (!cap_caching_mode(iommu
->cap
))
3037 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3038 DMA_TLB_GLOBAL_FLUSH
);
3039 for (j
= 0; j
< deferred_flush
[i
].next
; j
++) {
3041 struct iova
*iova
= deferred_flush
[i
].iova
[j
];
3042 struct dmar_domain
*domain
= deferred_flush
[i
].domain
[j
];
3044 /* On real hardware multiple invalidations are expensive */
3045 if (cap_caching_mode(iommu
->cap
))
3046 iommu_flush_iotlb_psi(iommu
, domain
->id
,
3047 iova
->pfn_lo
, iova
->pfn_hi
- iova
->pfn_lo
+ 1,
3048 !deferred_flush
[i
].freelist
[j
], 0);
3050 mask
= ilog2(mm_to_dma_pfn(iova
->pfn_hi
- iova
->pfn_lo
+ 1));
3051 iommu_flush_dev_iotlb(deferred_flush
[i
].domain
[j
],
3052 (uint64_t)iova
->pfn_lo
<< PAGE_SHIFT
, mask
);
3054 __free_iova(&deferred_flush
[i
].domain
[j
]->iovad
, iova
);
3055 if (deferred_flush
[i
].freelist
[j
])
3056 dma_free_pagelist(deferred_flush
[i
].freelist
[j
]);
3058 deferred_flush
[i
].next
= 0;
3064 static void flush_unmaps_timeout(unsigned long data
)
3066 unsigned long flags
;
3068 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
3070 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
3073 static void add_unmap(struct dmar_domain
*dom
, struct iova
*iova
, struct page
*freelist
)
3075 unsigned long flags
;
3077 struct intel_iommu
*iommu
;
3079 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
3080 if (list_size
== HIGH_WATER_MARK
)
3083 iommu
= domain_get_iommu(dom
);
3084 iommu_id
= iommu
->seq_id
;
3086 next
= deferred_flush
[iommu_id
].next
;
3087 deferred_flush
[iommu_id
].domain
[next
] = dom
;
3088 deferred_flush
[iommu_id
].iova
[next
] = iova
;
3089 deferred_flush
[iommu_id
].freelist
[next
] = freelist
;
3090 deferred_flush
[iommu_id
].next
++;
3093 mod_timer(&unmap_timer
, jiffies
+ msecs_to_jiffies(10));
3097 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
3100 static void intel_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
3101 size_t size
, enum dma_data_direction dir
,
3102 struct dma_attrs
*attrs
)
3104 struct pci_dev
*pdev
= to_pci_dev(dev
);
3105 struct dmar_domain
*domain
;
3106 unsigned long start_pfn
, last_pfn
;
3108 struct intel_iommu
*iommu
;
3109 struct page
*freelist
;
3111 if (iommu_no_mapping(dev
))
3114 domain
= find_domain(dev
);
3117 iommu
= domain_get_iommu(domain
);
3119 iova
= find_iova(&domain
->iovad
, IOVA_PFN(dev_addr
));
3120 if (WARN_ONCE(!iova
, "Driver unmaps unmatched page at PFN %llx\n",
3121 (unsigned long long)dev_addr
))
3124 start_pfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3125 last_pfn
= mm_to_dma_pfn(iova
->pfn_hi
+ 1) - 1;
3127 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3128 pci_name(pdev
), start_pfn
, last_pfn
);
3130 freelist
= domain_unmap(domain
, start_pfn
, last_pfn
);
3132 if (intel_iommu_strict
) {
3133 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_pfn
,
3134 last_pfn
- start_pfn
+ 1, !freelist
, 0);
3136 __free_iova(&domain
->iovad
, iova
);
3137 dma_free_pagelist(freelist
);
3139 add_unmap(domain
, iova
, freelist
);
3141 * queue up the release of the unmap to save the 1/6th of the
3142 * cpu used up by the iotlb flush operation...
3147 static void *intel_alloc_coherent(struct device
*hwdev
, size_t size
,
3148 dma_addr_t
*dma_handle
, gfp_t flags
,
3149 struct dma_attrs
*attrs
)
3154 size
= PAGE_ALIGN(size
);
3155 order
= get_order(size
);
3157 if (!iommu_no_mapping(hwdev
))
3158 flags
&= ~(GFP_DMA
| GFP_DMA32
);
3159 else if (hwdev
->coherent_dma_mask
< dma_get_required_mask(hwdev
)) {
3160 if (hwdev
->coherent_dma_mask
< DMA_BIT_MASK(32))
3166 vaddr
= (void *)__get_free_pages(flags
, order
);
3169 memset(vaddr
, 0, size
);
3171 *dma_handle
= __intel_map_single(hwdev
, virt_to_bus(vaddr
), size
,
3173 hwdev
->coherent_dma_mask
);
3176 free_pages((unsigned long)vaddr
, order
);
3180 static void intel_free_coherent(struct device
*hwdev
, size_t size
, void *vaddr
,
3181 dma_addr_t dma_handle
, struct dma_attrs
*attrs
)
3185 size
= PAGE_ALIGN(size
);
3186 order
= get_order(size
);
3188 intel_unmap_page(hwdev
, dma_handle
, size
, DMA_BIDIRECTIONAL
, NULL
);
3189 free_pages((unsigned long)vaddr
, order
);
3192 static void intel_unmap_sg(struct device
*hwdev
, struct scatterlist
*sglist
,
3193 int nelems
, enum dma_data_direction dir
,
3194 struct dma_attrs
*attrs
)
3196 struct dmar_domain
*domain
;
3197 unsigned long start_pfn
, last_pfn
;
3199 struct intel_iommu
*iommu
;
3200 struct page
*freelist
;
3202 if (iommu_no_mapping(hwdev
))
3205 domain
= find_domain(hwdev
);
3208 iommu
= domain_get_iommu(domain
);
3210 iova
= find_iova(&domain
->iovad
, IOVA_PFN(sglist
[0].dma_address
));
3211 if (WARN_ONCE(!iova
, "Driver unmaps unmatched sglist at PFN %llx\n",
3212 (unsigned long long)sglist
[0].dma_address
))
3215 start_pfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3216 last_pfn
= mm_to_dma_pfn(iova
->pfn_hi
+ 1) - 1;
3218 freelist
= domain_unmap(domain
, start_pfn
, last_pfn
);
3220 if (intel_iommu_strict
) {
3221 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_pfn
,
3222 last_pfn
- start_pfn
+ 1, !freelist
, 0);
3224 __free_iova(&domain
->iovad
, iova
);
3225 dma_free_pagelist(freelist
);
3227 add_unmap(domain
, iova
, freelist
);
3229 * queue up the release of the unmap to save the 1/6th of the
3230 * cpu used up by the iotlb flush operation...
3235 static int intel_nontranslate_map_sg(struct device
*hddev
,
3236 struct scatterlist
*sglist
, int nelems
, int dir
)
3239 struct scatterlist
*sg
;
3241 for_each_sg(sglist
, sg
, nelems
, i
) {
3242 BUG_ON(!sg_page(sg
));
3243 sg
->dma_address
= page_to_phys(sg_page(sg
)) + sg
->offset
;
3244 sg
->dma_length
= sg
->length
;
3249 static int intel_map_sg(struct device
*hwdev
, struct scatterlist
*sglist
, int nelems
,
3250 enum dma_data_direction dir
, struct dma_attrs
*attrs
)
3253 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
3254 struct dmar_domain
*domain
;
3257 struct iova
*iova
= NULL
;
3259 struct scatterlist
*sg
;
3260 unsigned long start_vpfn
;
3261 struct intel_iommu
*iommu
;
3263 BUG_ON(dir
== DMA_NONE
);
3264 if (iommu_no_mapping(hwdev
))
3265 return intel_nontranslate_map_sg(hwdev
, sglist
, nelems
, dir
);
3267 domain
= get_valid_domain_for_dev(pdev
);
3271 iommu
= domain_get_iommu(domain
);
3273 for_each_sg(sglist
, sg
, nelems
, i
)
3274 size
+= aligned_nrpages(sg
->offset
, sg
->length
);
3276 iova
= intel_alloc_iova(hwdev
, domain
, dma_to_mm_pfn(size
),
3279 sglist
->dma_length
= 0;
3284 * Check if DMAR supports zero-length reads on write only
3287 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
3288 !cap_zlr(iommu
->cap
))
3289 prot
|= DMA_PTE_READ
;
3290 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
3291 prot
|= DMA_PTE_WRITE
;
3293 start_vpfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3295 ret
= domain_sg_mapping(domain
, start_vpfn
, sglist
, size
, prot
);
3296 if (unlikely(ret
)) {
3297 /* clear the page */
3298 dma_pte_clear_range(domain
, start_vpfn
,
3299 start_vpfn
+ size
- 1);
3300 /* free page tables */
3301 dma_pte_free_pagetable(domain
, start_vpfn
,
3302 start_vpfn
+ size
- 1);
3304 __free_iova(&domain
->iovad
, iova
);
3308 /* it's a non-present to present mapping. Only flush if caching mode */
3309 if (cap_caching_mode(iommu
->cap
))
3310 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_vpfn
, size
, 0, 1);
3312 iommu_flush_write_buffer(iommu
);
3317 static int intel_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
3322 struct dma_map_ops intel_dma_ops
= {
3323 .alloc
= intel_alloc_coherent
,
3324 .free
= intel_free_coherent
,
3325 .map_sg
= intel_map_sg
,
3326 .unmap_sg
= intel_unmap_sg
,
3327 .map_page
= intel_map_page
,
3328 .unmap_page
= intel_unmap_page
,
3329 .mapping_error
= intel_mapping_error
,
3332 static inline int iommu_domain_cache_init(void)
3336 iommu_domain_cache
= kmem_cache_create("iommu_domain",
3337 sizeof(struct dmar_domain
),
3342 if (!iommu_domain_cache
) {
3343 printk(KERN_ERR
"Couldn't create iommu_domain cache\n");
3350 static inline int iommu_devinfo_cache_init(void)
3354 iommu_devinfo_cache
= kmem_cache_create("iommu_devinfo",
3355 sizeof(struct device_domain_info
),
3359 if (!iommu_devinfo_cache
) {
3360 printk(KERN_ERR
"Couldn't create devinfo cache\n");
3367 static inline int iommu_iova_cache_init(void)
3371 iommu_iova_cache
= kmem_cache_create("iommu_iova",
3372 sizeof(struct iova
),
3376 if (!iommu_iova_cache
) {
3377 printk(KERN_ERR
"Couldn't create iova cache\n");
3384 static int __init
iommu_init_mempool(void)
3387 ret
= iommu_iova_cache_init();
3391 ret
= iommu_domain_cache_init();
3395 ret
= iommu_devinfo_cache_init();
3399 kmem_cache_destroy(iommu_domain_cache
);
3401 kmem_cache_destroy(iommu_iova_cache
);
3406 static void __init
iommu_exit_mempool(void)
3408 kmem_cache_destroy(iommu_devinfo_cache
);
3409 kmem_cache_destroy(iommu_domain_cache
);
3410 kmem_cache_destroy(iommu_iova_cache
);
3414 static void quirk_ioat_snb_local_iommu(struct pci_dev
*pdev
)
3416 struct dmar_drhd_unit
*drhd
;
3420 /* We know that this device on this chipset has its own IOMMU.
3421 * If we find it under a different IOMMU, then the BIOS is lying
3422 * to us. Hope that the IOMMU for this device is actually
3423 * disabled, and it needs no translation...
3425 rc
= pci_bus_read_config_dword(pdev
->bus
, PCI_DEVFN(0, 0), 0xb0, &vtbar
);
3427 /* "can't" happen */
3428 dev_info(&pdev
->dev
, "failed to run vt-d quirk\n");
3431 vtbar
&= 0xffff0000;
3433 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3434 drhd
= dmar_find_matched_drhd_unit(pdev
);
3435 if (WARN_TAINT_ONCE(!drhd
|| drhd
->reg_base_addr
- vtbar
!= 0xa000,
3436 TAINT_FIRMWARE_WORKAROUND
,
3437 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3438 pdev
->dev
.archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
3440 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL
, PCI_DEVICE_ID_INTEL_IOAT_SNB
, quirk_ioat_snb_local_iommu
);
3442 static void __init
init_no_remapping_devices(void)
3444 struct dmar_drhd_unit
*drhd
;
3448 for_each_drhd_unit(drhd
) {
3449 if (!drhd
->include_all
) {
3450 for_each_active_dev_scope(drhd
->devices
,
3451 drhd
->devices_cnt
, i
, dev
)
3453 /* ignore DMAR unit if no devices exist */
3454 if (i
== drhd
->devices_cnt
)
3459 for_each_active_drhd_unit(drhd
) {
3460 if (drhd
->include_all
)
3463 for_each_active_dev_scope(drhd
->devices
,
3464 drhd
->devices_cnt
, i
, dev
)
3465 if (!dev_is_pci(dev
) || !IS_GFX_DEVICE(to_pci_dev(dev
)))
3467 if (i
< drhd
->devices_cnt
)
3470 /* This IOMMU has *only* gfx devices. Either bypass it or
3471 set the gfx_mapped flag, as appropriate */
3473 intel_iommu_gfx_mapped
= 1;
3476 for_each_active_dev_scope(drhd
->devices
,
3477 drhd
->devices_cnt
, i
, dev
)
3478 dev
->archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
3483 #ifdef CONFIG_SUSPEND
3484 static int init_iommu_hw(void)
3486 struct dmar_drhd_unit
*drhd
;
3487 struct intel_iommu
*iommu
= NULL
;
3489 for_each_active_iommu(iommu
, drhd
)
3491 dmar_reenable_qi(iommu
);
3493 for_each_iommu(iommu
, drhd
) {
3494 if (drhd
->ignored
) {
3496 * we always have to disable PMRs or DMA may fail on
3500 iommu_disable_protect_mem_regions(iommu
);
3504 iommu_flush_write_buffer(iommu
);
3506 iommu_set_root_entry(iommu
);
3508 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
3509 DMA_CCMD_GLOBAL_INVL
);
3510 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3511 DMA_TLB_GLOBAL_FLUSH
);
3512 if (iommu_enable_translation(iommu
))
3514 iommu_disable_protect_mem_regions(iommu
);
3520 static void iommu_flush_all(void)
3522 struct dmar_drhd_unit
*drhd
;
3523 struct intel_iommu
*iommu
;
3525 for_each_active_iommu(iommu
, drhd
) {
3526 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
3527 DMA_CCMD_GLOBAL_INVL
);
3528 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3529 DMA_TLB_GLOBAL_FLUSH
);
3533 static int iommu_suspend(void)
3535 struct dmar_drhd_unit
*drhd
;
3536 struct intel_iommu
*iommu
= NULL
;
3539 for_each_active_iommu(iommu
, drhd
) {
3540 iommu
->iommu_state
= kzalloc(sizeof(u32
) * MAX_SR_DMAR_REGS
,
3542 if (!iommu
->iommu_state
)
3548 for_each_active_iommu(iommu
, drhd
) {
3549 iommu_disable_translation(iommu
);
3551 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
3553 iommu
->iommu_state
[SR_DMAR_FECTL_REG
] =
3554 readl(iommu
->reg
+ DMAR_FECTL_REG
);
3555 iommu
->iommu_state
[SR_DMAR_FEDATA_REG
] =
3556 readl(iommu
->reg
+ DMAR_FEDATA_REG
);
3557 iommu
->iommu_state
[SR_DMAR_FEADDR_REG
] =
3558 readl(iommu
->reg
+ DMAR_FEADDR_REG
);
3559 iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
] =
3560 readl(iommu
->reg
+ DMAR_FEUADDR_REG
);
3562 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
3567 for_each_active_iommu(iommu
, drhd
)
3568 kfree(iommu
->iommu_state
);
3573 static void iommu_resume(void)
3575 struct dmar_drhd_unit
*drhd
;
3576 struct intel_iommu
*iommu
= NULL
;
3579 if (init_iommu_hw()) {
3581 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3583 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3587 for_each_active_iommu(iommu
, drhd
) {
3589 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
3591 writel(iommu
->iommu_state
[SR_DMAR_FECTL_REG
],
3592 iommu
->reg
+ DMAR_FECTL_REG
);
3593 writel(iommu
->iommu_state
[SR_DMAR_FEDATA_REG
],
3594 iommu
->reg
+ DMAR_FEDATA_REG
);
3595 writel(iommu
->iommu_state
[SR_DMAR_FEADDR_REG
],
3596 iommu
->reg
+ DMAR_FEADDR_REG
);
3597 writel(iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
],
3598 iommu
->reg
+ DMAR_FEUADDR_REG
);
3600 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
3603 for_each_active_iommu(iommu
, drhd
)
3604 kfree(iommu
->iommu_state
);
3607 static struct syscore_ops iommu_syscore_ops
= {
3608 .resume
= iommu_resume
,
3609 .suspend
= iommu_suspend
,
3612 static void __init
init_iommu_pm_ops(void)
3614 register_syscore_ops(&iommu_syscore_ops
);
3618 static inline void init_iommu_pm_ops(void) {}
3619 #endif /* CONFIG_PM */
3622 int __init
dmar_parse_one_rmrr(struct acpi_dmar_header
*header
)
3624 struct acpi_dmar_reserved_memory
*rmrr
;
3625 struct dmar_rmrr_unit
*rmrru
;
3627 rmrru
= kzalloc(sizeof(*rmrru
), GFP_KERNEL
);
3631 rmrru
->hdr
= header
;
3632 rmrr
= (struct acpi_dmar_reserved_memory
*)header
;
3633 rmrru
->base_address
= rmrr
->base_address
;
3634 rmrru
->end_address
= rmrr
->end_address
;
3635 rmrru
->devices
= dmar_alloc_dev_scope((void *)(rmrr
+ 1),
3636 ((void *)rmrr
) + rmrr
->header
.length
,
3637 &rmrru
->devices_cnt
);
3638 if (rmrru
->devices_cnt
&& rmrru
->devices
== NULL
) {
3643 list_add(&rmrru
->list
, &dmar_rmrr_units
);
3648 int __init
dmar_parse_one_atsr(struct acpi_dmar_header
*hdr
)
3650 struct acpi_dmar_atsr
*atsr
;
3651 struct dmar_atsr_unit
*atsru
;
3653 atsr
= container_of(hdr
, struct acpi_dmar_atsr
, header
);
3654 atsru
= kzalloc(sizeof(*atsru
), GFP_KERNEL
);
3659 atsru
->include_all
= atsr
->flags
& 0x1;
3660 if (!atsru
->include_all
) {
3661 atsru
->devices
= dmar_alloc_dev_scope((void *)(atsr
+ 1),
3662 (void *)atsr
+ atsr
->header
.length
,
3663 &atsru
->devices_cnt
);
3664 if (atsru
->devices_cnt
&& atsru
->devices
== NULL
) {
3670 list_add_rcu(&atsru
->list
, &dmar_atsr_units
);
3675 static void intel_iommu_free_atsr(struct dmar_atsr_unit
*atsru
)
3677 dmar_free_dev_scope(&atsru
->devices
, &atsru
->devices_cnt
);
3681 static void intel_iommu_free_dmars(void)
3683 struct dmar_rmrr_unit
*rmrru
, *rmrr_n
;
3684 struct dmar_atsr_unit
*atsru
, *atsr_n
;
3686 list_for_each_entry_safe(rmrru
, rmrr_n
, &dmar_rmrr_units
, list
) {
3687 list_del(&rmrru
->list
);
3688 dmar_free_dev_scope(&rmrru
->devices
, &rmrru
->devices_cnt
);
3692 list_for_each_entry_safe(atsru
, atsr_n
, &dmar_atsr_units
, list
) {
3693 list_del(&atsru
->list
);
3694 intel_iommu_free_atsr(atsru
);
3698 int dmar_find_matched_atsr_unit(struct pci_dev
*dev
)
3701 struct pci_bus
*bus
;
3702 struct pci_dev
*bridge
= NULL
;
3704 struct acpi_dmar_atsr
*atsr
;
3705 struct dmar_atsr_unit
*atsru
;
3707 dev
= pci_physfn(dev
);
3708 for (bus
= dev
->bus
; bus
; bus
= bus
->parent
) {
3710 if (!bridge
|| !pci_is_pcie(bridge
) ||
3711 pci_pcie_type(bridge
) == PCI_EXP_TYPE_PCI_BRIDGE
)
3713 if (pci_pcie_type(bridge
) == PCI_EXP_TYPE_ROOT_PORT
)
3720 list_for_each_entry_rcu(atsru
, &dmar_atsr_units
, list
) {
3721 atsr
= container_of(atsru
->hdr
, struct acpi_dmar_atsr
, header
);
3722 if (atsr
->segment
!= pci_domain_nr(dev
->bus
))
3725 for_each_dev_scope(atsru
->devices
, atsru
->devices_cnt
, i
, tmp
)
3726 if (tmp
== &bridge
->dev
)
3729 if (atsru
->include_all
)
3739 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info
*info
)
3742 struct dmar_rmrr_unit
*rmrru
;
3743 struct dmar_atsr_unit
*atsru
;
3744 struct acpi_dmar_atsr
*atsr
;
3745 struct acpi_dmar_reserved_memory
*rmrr
;
3747 if (!intel_iommu_enabled
&& system_state
!= SYSTEM_BOOTING
)
3750 list_for_each_entry(rmrru
, &dmar_rmrr_units
, list
) {
3751 rmrr
= container_of(rmrru
->hdr
,
3752 struct acpi_dmar_reserved_memory
, header
);
3753 if (info
->event
== BUS_NOTIFY_ADD_DEVICE
) {
3754 ret
= dmar_insert_dev_scope(info
, (void *)(rmrr
+ 1),
3755 ((void *)rmrr
) + rmrr
->header
.length
,
3756 rmrr
->segment
, rmrru
->devices
,
3757 rmrru
->devices_cnt
);
3762 } else if (info
->event
== BUS_NOTIFY_DEL_DEVICE
) {
3763 if (dmar_remove_dev_scope(info
, rmrr
->segment
,
3764 rmrru
->devices
, rmrru
->devices_cnt
))
3769 list_for_each_entry(atsru
, &dmar_atsr_units
, list
) {
3770 if (atsru
->include_all
)
3773 atsr
= container_of(atsru
->hdr
, struct acpi_dmar_atsr
, header
);
3774 if (info
->event
== BUS_NOTIFY_ADD_DEVICE
) {
3775 ret
= dmar_insert_dev_scope(info
, (void *)(atsr
+ 1),
3776 (void *)atsr
+ atsr
->header
.length
,
3777 atsr
->segment
, atsru
->devices
,
3778 atsru
->devices_cnt
);
3783 } else if (info
->event
== BUS_NOTIFY_DEL_DEVICE
) {
3784 if (dmar_remove_dev_scope(info
, atsr
->segment
,
3785 atsru
->devices
, atsru
->devices_cnt
))
3794 * Here we only respond to action of unbound device from driver.
3796 * Added device is not attached to its DMAR domain here yet. That will happen
3797 * when mapping the device to iova.
3799 static int device_notifier(struct notifier_block
*nb
,
3800 unsigned long action
, void *data
)
3802 struct device
*dev
= data
;
3803 struct pci_dev
*pdev
= to_pci_dev(dev
);
3804 struct dmar_domain
*domain
;
3806 if (iommu_dummy(dev
))
3809 if (action
!= BUS_NOTIFY_UNBOUND_DRIVER
&&
3810 action
!= BUS_NOTIFY_DEL_DEVICE
)
3813 domain
= find_domain(dev
);
3817 down_read(&dmar_global_lock
);
3818 domain_remove_one_dev_info(domain
, pdev
);
3819 if (!(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) &&
3820 !(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
) &&
3821 list_empty(&domain
->devices
))
3822 domain_exit(domain
);
3823 up_read(&dmar_global_lock
);
3828 static struct notifier_block device_nb
= {
3829 .notifier_call
= device_notifier
,
3832 static int intel_iommu_memory_notifier(struct notifier_block
*nb
,
3833 unsigned long val
, void *v
)
3835 struct memory_notify
*mhp
= v
;
3836 unsigned long long start
, end
;
3837 unsigned long start_vpfn
, last_vpfn
;
3840 case MEM_GOING_ONLINE
:
3841 start
= mhp
->start_pfn
<< PAGE_SHIFT
;
3842 end
= ((mhp
->start_pfn
+ mhp
->nr_pages
) << PAGE_SHIFT
) - 1;
3843 if (iommu_domain_identity_map(si_domain
, start
, end
)) {
3844 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3851 case MEM_CANCEL_ONLINE
:
3852 start_vpfn
= mm_to_dma_pfn(mhp
->start_pfn
);
3853 last_vpfn
= mm_to_dma_pfn(mhp
->start_pfn
+ mhp
->nr_pages
- 1);
3854 while (start_vpfn
<= last_vpfn
) {
3856 struct dmar_drhd_unit
*drhd
;
3857 struct intel_iommu
*iommu
;
3858 struct page
*freelist
;
3860 iova
= find_iova(&si_domain
->iovad
, start_vpfn
);
3862 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3867 iova
= split_and_remove_iova(&si_domain
->iovad
, iova
,
3868 start_vpfn
, last_vpfn
);
3870 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3871 start_vpfn
, last_vpfn
);
3875 freelist
= domain_unmap(si_domain
, iova
->pfn_lo
,
3879 for_each_active_iommu(iommu
, drhd
)
3880 iommu_flush_iotlb_psi(iommu
, si_domain
->id
,
3882 iova
->pfn_hi
- iova
->pfn_lo
+ 1,
3885 dma_free_pagelist(freelist
);
3887 start_vpfn
= iova
->pfn_hi
+ 1;
3888 free_iova_mem(iova
);
3896 static struct notifier_block intel_iommu_memory_nb
= {
3897 .notifier_call
= intel_iommu_memory_notifier
,
3901 int __init
intel_iommu_init(void)
3904 struct dmar_drhd_unit
*drhd
;
3905 struct intel_iommu
*iommu
;
3907 /* VT-d is required for a TXT/tboot launch, so enforce that */
3908 force_on
= tboot_force_iommu();
3910 if (iommu_init_mempool()) {
3912 panic("tboot: Failed to initialize iommu memory\n");
3916 down_write(&dmar_global_lock
);
3917 if (dmar_table_init()) {
3919 panic("tboot: Failed to initialize DMAR table\n");
3924 * Disable translation if already enabled prior to OS handover.
3926 for_each_active_iommu(iommu
, drhd
)
3927 if (iommu
->gcmd
& DMA_GCMD_TE
)
3928 iommu_disable_translation(iommu
);
3930 if (dmar_dev_scope_init() < 0) {
3932 panic("tboot: Failed to initialize DMAR device scope\n");
3936 if (no_iommu
|| dmar_disabled
)
3939 if (list_empty(&dmar_rmrr_units
))
3940 printk(KERN_INFO
"DMAR: No RMRR found\n");
3942 if (list_empty(&dmar_atsr_units
))
3943 printk(KERN_INFO
"DMAR: No ATSR found\n");
3945 if (dmar_init_reserved_ranges()) {
3947 panic("tboot: Failed to reserve iommu ranges\n");
3948 goto out_free_reserved_range
;
3951 init_no_remapping_devices();
3956 panic("tboot: Failed to initialize DMARs\n");
3957 printk(KERN_ERR
"IOMMU: dmar init failed\n");
3958 goto out_free_reserved_range
;
3960 up_write(&dmar_global_lock
);
3962 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3964 init_timer(&unmap_timer
);
3965 #ifdef CONFIG_SWIOTLB
3968 dma_ops
= &intel_dma_ops
;
3970 init_iommu_pm_ops();
3972 bus_set_iommu(&pci_bus_type
, &intel_iommu_ops
);
3973 bus_register_notifier(&pci_bus_type
, &device_nb
);
3974 if (si_domain
&& !hw_pass_through
)
3975 register_memory_notifier(&intel_iommu_memory_nb
);
3977 intel_iommu_enabled
= 1;
3981 out_free_reserved_range
:
3982 put_iova_domain(&reserved_iova_list
);
3984 intel_iommu_free_dmars();
3985 up_write(&dmar_global_lock
);
3986 iommu_exit_mempool();
3990 static void iommu_detach_dependent_devices(struct intel_iommu
*iommu
,
3993 struct pci_dev
*tmp
, *parent
, *pdev
;
3995 if (!iommu
|| !dev
|| !dev_is_pci(dev
))
3998 pdev
= to_pci_dev(dev
);
4000 /* dependent device detach */
4001 tmp
= pci_find_upstream_pcie_bridge(pdev
);
4002 /* Secondary interface's bus number and devfn 0 */
4004 parent
= pdev
->bus
->self
;
4005 while (parent
!= tmp
) {
4006 iommu_detach_dev(iommu
, parent
->bus
->number
,
4008 parent
= parent
->bus
->self
;
4010 if (pci_is_pcie(tmp
)) /* this is a PCIe-to-PCI bridge */
4011 iommu_detach_dev(iommu
,
4012 tmp
->subordinate
->number
, 0);
4013 else /* this is a legacy PCI bridge */
4014 iommu_detach_dev(iommu
, tmp
->bus
->number
,
4019 static void domain_remove_one_dev_info(struct dmar_domain
*domain
,
4020 struct pci_dev
*pdev
)
4022 struct device_domain_info
*info
, *tmp
;
4023 struct intel_iommu
*iommu
;
4024 unsigned long flags
;
4027 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
4032 spin_lock_irqsave(&device_domain_lock
, flags
);
4033 list_for_each_entry_safe(info
, tmp
, &domain
->devices
, link
) {
4034 if (info
->segment
== pci_domain_nr(pdev
->bus
) &&
4035 info
->bus
== pdev
->bus
->number
&&
4036 info
->devfn
== pdev
->devfn
) {
4037 unlink_domain_info(info
);
4038 spin_unlock_irqrestore(&device_domain_lock
, flags
);
4040 iommu_disable_dev_iotlb(info
);
4041 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
4042 iommu_detach_dependent_devices(iommu
, &pdev
->dev
);
4043 free_devinfo_mem(info
);
4045 spin_lock_irqsave(&device_domain_lock
, flags
);
4053 /* if there is no other devices under the same iommu
4054 * owned by this domain, clear this iommu in iommu_bmp
4055 * update iommu count and coherency
4057 if (info
->iommu
== iommu
)
4061 spin_unlock_irqrestore(&device_domain_lock
, flags
);
4064 unsigned long tmp_flags
;
4065 spin_lock_irqsave(&domain
->iommu_lock
, tmp_flags
);
4066 clear_bit(iommu
->seq_id
, domain
->iommu_bmp
);
4067 domain
->iommu_count
--;
4068 domain_update_iommu_cap(domain
);
4069 spin_unlock_irqrestore(&domain
->iommu_lock
, tmp_flags
);
4071 if (!(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) &&
4072 !(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
)) {
4073 spin_lock_irqsave(&iommu
->lock
, tmp_flags
);
4074 clear_bit(domain
->id
, iommu
->domain_ids
);
4075 iommu
->domains
[domain
->id
] = NULL
;
4076 spin_unlock_irqrestore(&iommu
->lock
, tmp_flags
);
4081 static int md_domain_init(struct dmar_domain
*domain
, int guest_width
)
4085 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
4086 domain_reserve_special_ranges(domain
);
4088 /* calculate AGAW */
4089 domain
->gaw
= guest_width
;
4090 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
4091 domain
->agaw
= width_to_agaw(adjust_width
);
4093 domain
->iommu_coherency
= 0;
4094 domain
->iommu_snooping
= 0;
4095 domain
->iommu_superpage
= 0;
4096 domain
->max_addr
= 0;
4099 /* always allocate the top pgd */
4100 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page(domain
->nid
);
4103 domain_flush_cache(domain
, domain
->pgd
, PAGE_SIZE
);
4107 static int intel_iommu_domain_init(struct iommu_domain
*domain
)
4109 struct dmar_domain
*dmar_domain
;
4111 dmar_domain
= alloc_domain(true);
4114 "intel_iommu_domain_init: dmar_domain == NULL\n");
4117 if (md_domain_init(dmar_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
4119 "intel_iommu_domain_init() failed\n");
4120 domain_exit(dmar_domain
);
4123 domain_update_iommu_cap(dmar_domain
);
4124 domain
->priv
= dmar_domain
;
4126 domain
->geometry
.aperture_start
= 0;
4127 domain
->geometry
.aperture_end
= __DOMAIN_MAX_ADDR(dmar_domain
->gaw
);
4128 domain
->geometry
.force_aperture
= true;
4133 static void intel_iommu_domain_destroy(struct iommu_domain
*domain
)
4135 struct dmar_domain
*dmar_domain
= domain
->priv
;
4137 domain
->priv
= NULL
;
4138 domain_exit(dmar_domain
);
4141 static int intel_iommu_attach_device(struct iommu_domain
*domain
,
4144 struct dmar_domain
*dmar_domain
= domain
->priv
;
4145 struct pci_dev
*pdev
= to_pci_dev(dev
);
4146 struct intel_iommu
*iommu
;
4149 /* normally pdev is not mapped */
4150 if (unlikely(domain_context_mapped(pdev
))) {
4151 struct dmar_domain
*old_domain
;
4153 old_domain
= find_domain(dev
);
4155 if (dmar_domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
4156 dmar_domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
)
4157 domain_remove_one_dev_info(old_domain
, pdev
);
4159 domain_remove_dev_info(old_domain
);
4163 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
4168 /* check if this iommu agaw is sufficient for max mapped address */
4169 addr_width
= agaw_to_width(iommu
->agaw
);
4170 if (addr_width
> cap_mgaw(iommu
->cap
))
4171 addr_width
= cap_mgaw(iommu
->cap
);
4173 if (dmar_domain
->max_addr
> (1LL << addr_width
)) {
4174 printk(KERN_ERR
"%s: iommu width (%d) is not "
4175 "sufficient for the mapped address (%llx)\n",
4176 __func__
, addr_width
, dmar_domain
->max_addr
);
4179 dmar_domain
->gaw
= addr_width
;
4182 * Knock out extra levels of page tables if necessary
4184 while (iommu
->agaw
< dmar_domain
->agaw
) {
4185 struct dma_pte
*pte
;
4187 pte
= dmar_domain
->pgd
;
4188 if (dma_pte_present(pte
)) {
4189 dmar_domain
->pgd
= (struct dma_pte
*)
4190 phys_to_virt(dma_pte_addr(pte
));
4191 free_pgtable_page(pte
);
4193 dmar_domain
->agaw
--;
4196 return domain_add_dev_info(dmar_domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
4199 static void intel_iommu_detach_device(struct iommu_domain
*domain
,
4202 struct dmar_domain
*dmar_domain
= domain
->priv
;
4203 struct pci_dev
*pdev
= to_pci_dev(dev
);
4205 domain_remove_one_dev_info(dmar_domain
, pdev
);
4208 static int intel_iommu_map(struct iommu_domain
*domain
,
4209 unsigned long iova
, phys_addr_t hpa
,
4210 size_t size
, int iommu_prot
)
4212 struct dmar_domain
*dmar_domain
= domain
->priv
;
4217 if (iommu_prot
& IOMMU_READ
)
4218 prot
|= DMA_PTE_READ
;
4219 if (iommu_prot
& IOMMU_WRITE
)
4220 prot
|= DMA_PTE_WRITE
;
4221 if ((iommu_prot
& IOMMU_CACHE
) && dmar_domain
->iommu_snooping
)
4222 prot
|= DMA_PTE_SNP
;
4224 max_addr
= iova
+ size
;
4225 if (dmar_domain
->max_addr
< max_addr
) {
4228 /* check if minimum agaw is sufficient for mapped address */
4229 end
= __DOMAIN_MAX_ADDR(dmar_domain
->gaw
) + 1;
4230 if (end
< max_addr
) {
4231 printk(KERN_ERR
"%s: iommu width (%d) is not "
4232 "sufficient for the mapped address (%llx)\n",
4233 __func__
, dmar_domain
->gaw
, max_addr
);
4236 dmar_domain
->max_addr
= max_addr
;
4238 /* Round up size to next multiple of PAGE_SIZE, if it and
4239 the low bits of hpa would take us onto the next page */
4240 size
= aligned_nrpages(hpa
, size
);
4241 ret
= domain_pfn_mapping(dmar_domain
, iova
>> VTD_PAGE_SHIFT
,
4242 hpa
>> VTD_PAGE_SHIFT
, size
, prot
);
4246 static size_t intel_iommu_unmap(struct iommu_domain
*domain
,
4247 unsigned long iova
, size_t size
)
4249 struct dmar_domain
*dmar_domain
= domain
->priv
;
4250 struct page
*freelist
= NULL
;
4251 struct intel_iommu
*iommu
;
4252 unsigned long start_pfn
, last_pfn
;
4253 unsigned int npages
;
4254 int iommu_id
, num
, ndomains
, level
= 0;
4256 /* Cope with horrid API which requires us to unmap more than the
4257 size argument if it happens to be a large-page mapping. */
4258 if (!pfn_to_dma_pte(dmar_domain
, iova
>> VTD_PAGE_SHIFT
, &level
))
4261 if (size
< VTD_PAGE_SIZE
<< level_to_offset_bits(level
))
4262 size
= VTD_PAGE_SIZE
<< level_to_offset_bits(level
);
4264 start_pfn
= iova
>> VTD_PAGE_SHIFT
;
4265 last_pfn
= (iova
+ size
- 1) >> VTD_PAGE_SHIFT
;
4267 freelist
= domain_unmap(dmar_domain
, start_pfn
, last_pfn
);
4269 npages
= last_pfn
- start_pfn
+ 1;
4271 for_each_set_bit(iommu_id
, dmar_domain
->iommu_bmp
, g_num_of_iommus
) {
4272 iommu
= g_iommus
[iommu_id
];
4275 * find bit position of dmar_domain
4277 ndomains
= cap_ndoms(iommu
->cap
);
4278 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
4279 if (iommu
->domains
[num
] == dmar_domain
)
4280 iommu_flush_iotlb_psi(iommu
, num
, start_pfn
,
4281 npages
, !freelist
, 0);
4286 dma_free_pagelist(freelist
);
4288 if (dmar_domain
->max_addr
== iova
+ size
)
4289 dmar_domain
->max_addr
= iova
;
4294 static phys_addr_t
intel_iommu_iova_to_phys(struct iommu_domain
*domain
,
4297 struct dmar_domain
*dmar_domain
= domain
->priv
;
4298 struct dma_pte
*pte
;
4302 pte
= pfn_to_dma_pte(dmar_domain
, iova
>> VTD_PAGE_SHIFT
, &level
);
4304 phys
= dma_pte_addr(pte
);
4309 static int intel_iommu_domain_has_cap(struct iommu_domain
*domain
,
4312 struct dmar_domain
*dmar_domain
= domain
->priv
;
4314 if (cap
== IOMMU_CAP_CACHE_COHERENCY
)
4315 return dmar_domain
->iommu_snooping
;
4316 if (cap
== IOMMU_CAP_INTR_REMAP
)
4317 return irq_remapping_enabled
;
4322 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4324 static int intel_iommu_add_device(struct device
*dev
)
4326 struct pci_dev
*pdev
= to_pci_dev(dev
);
4327 struct pci_dev
*bridge
, *dma_pdev
= NULL
;
4328 struct iommu_group
*group
;
4331 if (!device_to_iommu(pci_domain_nr(pdev
->bus
),
4332 pdev
->bus
->number
, pdev
->devfn
))
4335 bridge
= pci_find_upstream_pcie_bridge(pdev
);
4337 if (pci_is_pcie(bridge
))
4338 dma_pdev
= pci_get_domain_bus_and_slot(
4339 pci_domain_nr(pdev
->bus
),
4340 bridge
->subordinate
->number
, 0);
4342 dma_pdev
= pci_dev_get(bridge
);
4344 dma_pdev
= pci_dev_get(pdev
);
4346 /* Account for quirked devices */
4347 swap_pci_ref(&dma_pdev
, pci_get_dma_source(dma_pdev
));
4350 * If it's a multifunction device that does not support our
4351 * required ACS flags, add to the same group as lowest numbered
4352 * function that also does not suport the required ACS flags.
4354 if (dma_pdev
->multifunction
&&
4355 !pci_acs_enabled(dma_pdev
, REQ_ACS_FLAGS
)) {
4356 u8 i
, slot
= PCI_SLOT(dma_pdev
->devfn
);
4358 for (i
= 0; i
< 8; i
++) {
4359 struct pci_dev
*tmp
;
4361 tmp
= pci_get_slot(dma_pdev
->bus
, PCI_DEVFN(slot
, i
));
4365 if (!pci_acs_enabled(tmp
, REQ_ACS_FLAGS
)) {
4366 swap_pci_ref(&dma_pdev
, tmp
);
4374 * Devices on the root bus go through the iommu. If that's not us,
4375 * find the next upstream device and test ACS up to the root bus.
4376 * Finding the next device may require skipping virtual buses.
4378 while (!pci_is_root_bus(dma_pdev
->bus
)) {
4379 struct pci_bus
*bus
= dma_pdev
->bus
;
4381 while (!bus
->self
) {
4382 if (!pci_is_root_bus(bus
))
4388 if (pci_acs_path_enabled(bus
->self
, NULL
, REQ_ACS_FLAGS
))
4391 swap_pci_ref(&dma_pdev
, pci_dev_get(bus
->self
));
4395 group
= iommu_group_get(&dma_pdev
->dev
);
4396 pci_dev_put(dma_pdev
);
4398 group
= iommu_group_alloc();
4400 return PTR_ERR(group
);
4403 ret
= iommu_group_add_device(group
, dev
);
4405 iommu_group_put(group
);
4409 static void intel_iommu_remove_device(struct device
*dev
)
4411 iommu_group_remove_device(dev
);
4414 static struct iommu_ops intel_iommu_ops
= {
4415 .domain_init
= intel_iommu_domain_init
,
4416 .domain_destroy
= intel_iommu_domain_destroy
,
4417 .attach_dev
= intel_iommu_attach_device
,
4418 .detach_dev
= intel_iommu_detach_device
,
4419 .map
= intel_iommu_map
,
4420 .unmap
= intel_iommu_unmap
,
4421 .iova_to_phys
= intel_iommu_iova_to_phys
,
4422 .domain_has_cap
= intel_iommu_domain_has_cap
,
4423 .add_device
= intel_iommu_add_device
,
4424 .remove_device
= intel_iommu_remove_device
,
4425 .pgsize_bitmap
= INTEL_IOMMU_PGSIZES
,
4428 static void quirk_iommu_g4x_gfx(struct pci_dev
*dev
)
4430 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4431 printk(KERN_INFO
"DMAR: Disabling IOMMU for graphics on this chipset\n");
4435 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2a40, quirk_iommu_g4x_gfx
);
4436 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e00, quirk_iommu_g4x_gfx
);
4437 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e10, quirk_iommu_g4x_gfx
);
4438 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e20, quirk_iommu_g4x_gfx
);
4439 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e30, quirk_iommu_g4x_gfx
);
4440 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e40, quirk_iommu_g4x_gfx
);
4441 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e90, quirk_iommu_g4x_gfx
);
4443 static void quirk_iommu_rwbf(struct pci_dev
*dev
)
4446 * Mobile 4 Series Chipset neglects to set RWBF capability,
4447 * but needs it. Same seems to hold for the desktop versions.
4449 printk(KERN_INFO
"DMAR: Forcing write-buffer flush capability\n");
4453 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2a40, quirk_iommu_rwbf
);
4454 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e00, quirk_iommu_rwbf
);
4455 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e10, quirk_iommu_rwbf
);
4456 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e20, quirk_iommu_rwbf
);
4457 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e30, quirk_iommu_rwbf
);
4458 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e40, quirk_iommu_rwbf
);
4459 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e90, quirk_iommu_rwbf
);
4462 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
4463 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4464 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
4465 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
4466 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4467 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4468 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4469 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4471 static void quirk_calpella_no_shadow_gtt(struct pci_dev
*dev
)
4475 if (pci_read_config_word(dev
, GGC
, &ggc
))
4478 if (!(ggc
& GGC_MEMORY_VT_ENABLED
)) {
4479 printk(KERN_INFO
"DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4481 } else if (dmar_map_gfx
) {
4482 /* we have to ensure the gfx device is idle before we flush */
4483 printk(KERN_INFO
"DMAR: Disabling batched IOTLB flush on Ironlake\n");
4484 intel_iommu_strict
= 1;
4487 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0040, quirk_calpella_no_shadow_gtt
);
4488 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0044, quirk_calpella_no_shadow_gtt
);
4489 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0062, quirk_calpella_no_shadow_gtt
);
4490 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x006a, quirk_calpella_no_shadow_gtt
);
4492 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4493 ISOCH DMAR unit for the Azalia sound device, but not give it any
4494 TLB entries, which causes it to deadlock. Check for that. We do
4495 this in a function called from init_dmars(), instead of in a PCI
4496 quirk, because we don't want to print the obnoxious "BIOS broken"
4497 message if VT-d is actually disabled.
4499 static void __init
check_tylersburg_isoch(void)
4501 struct pci_dev
*pdev
;
4502 uint32_t vtisochctrl
;
4504 /* If there's no Azalia in the system anyway, forget it. */
4505 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x3a3e, NULL
);
4510 /* System Management Registers. Might be hidden, in which case
4511 we can't do the sanity check. But that's OK, because the
4512 known-broken BIOSes _don't_ actually hide it, so far. */
4513 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x342e, NULL
);
4517 if (pci_read_config_dword(pdev
, 0x188, &vtisochctrl
)) {
4524 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4525 if (vtisochctrl
& 1)
4528 /* Drop all bits other than the number of TLB entries */
4529 vtisochctrl
&= 0x1c;
4531 /* If we have the recommended number of TLB entries (16), fine. */
4532 if (vtisochctrl
== 0x10)
4535 /* Zero TLB entries? You get to ride the short bus to school. */
4537 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4538 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4539 dmi_get_system_info(DMI_BIOS_VENDOR
),
4540 dmi_get_system_info(DMI_BIOS_VERSION
),
4541 dmi_get_system_info(DMI_PRODUCT_VERSION
));
4542 iommu_identity_mapping
|= IDENTMAP_AZALIA
;
4546 printk(KERN_WARNING
"DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",