2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
20 #include <linux/init.h>
21 #include <linux/bitmap.h>
22 #include <linux/debugfs.h>
23 #include <linux/export.h>
24 #include <linux/slab.h>
25 #include <linux/irq.h>
26 #include <linux/interrupt.h>
27 #include <linux/spinlock.h>
28 #include <linux/pci.h>
29 #include <linux/dmar.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/mempool.h>
32 #include <linux/memory.h>
33 #include <linux/timer.h>
34 #include <linux/iova.h>
35 #include <linux/iommu.h>
36 #include <linux/intel-iommu.h>
37 #include <linux/syscore_ops.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/pci-ats.h>
41 #include <linux/memblock.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/cacheflush.h>
44 #include <asm/iommu.h>
46 #include "irq_remapping.h"
49 #define ROOT_SIZE VTD_PAGE_SIZE
50 #define CONTEXT_SIZE VTD_PAGE_SIZE
52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
56 #define IOAPIC_RANGE_START (0xfee00000)
57 #define IOAPIC_RANGE_END (0xfeefffff)
58 #define IOVA_START_ADDR (0x1000)
60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
62 #define MAX_AGAW_WIDTH 64
63 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
65 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
68 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
74 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
75 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
76 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
78 /* page table handling */
79 #define LEVEL_STRIDE (9)
80 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
98 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
100 static inline int agaw_to_level(int agaw
)
105 static inline int agaw_to_width(int agaw
)
107 return min_t(int, 30 + agaw
* LEVEL_STRIDE
, MAX_AGAW_WIDTH
);
110 static inline int width_to_agaw(int width
)
112 return DIV_ROUND_UP(width
- 30, LEVEL_STRIDE
);
115 static inline unsigned int level_to_offset_bits(int level
)
117 return (level
- 1) * LEVEL_STRIDE
;
120 static inline int pfn_level_offset(unsigned long pfn
, int level
)
122 return (pfn
>> level_to_offset_bits(level
)) & LEVEL_MASK
;
125 static inline unsigned long level_mask(int level
)
127 return -1UL << level_to_offset_bits(level
);
130 static inline unsigned long level_size(int level
)
132 return 1UL << level_to_offset_bits(level
);
135 static inline unsigned long align_to_level(unsigned long pfn
, int level
)
137 return (pfn
+ level_size(level
) - 1) & level_mask(level
);
140 static inline unsigned long lvl_to_nr_pages(unsigned int lvl
)
142 return 1 << min_t(int, (lvl
- 1) * LEVEL_STRIDE
, MAX_AGAW_PFN_WIDTH
);
145 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn
)
149 return dma_pfn
>> (PAGE_SHIFT
- VTD_PAGE_SHIFT
);
152 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn
)
154 return mm_pfn
<< (PAGE_SHIFT
- VTD_PAGE_SHIFT
);
156 static inline unsigned long page_to_dma_pfn(struct page
*pg
)
158 return mm_to_dma_pfn(page_to_pfn(pg
));
160 static inline unsigned long virt_to_dma_pfn(void *p
)
162 return page_to_dma_pfn(virt_to_page(p
));
165 /* global iommu list, set NULL for ignored DMAR units */
166 static struct intel_iommu
**g_iommus
;
168 static void __init
check_tylersburg_isoch(void);
169 static int rwbf_quirk
;
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
175 static int force_on
= 0;
180 * 12-63: Context Ptr (12 - (haw-1))
187 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188 static inline bool root_present(struct root_entry
*root
)
190 return (root
->val
& 1);
192 static inline void set_root_present(struct root_entry
*root
)
196 static inline void set_root_value(struct root_entry
*root
, unsigned long value
)
198 root
->val
|= value
& VTD_PAGE_MASK
;
201 static inline struct context_entry
*
202 get_context_addr_from_root(struct root_entry
*root
)
204 return (struct context_entry
*)
205 (root_present(root
)?phys_to_virt(
206 root
->val
& VTD_PAGE_MASK
) :
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
221 struct context_entry
{
226 static inline bool context_present(struct context_entry
*context
)
228 return (context
->lo
& 1);
230 static inline void context_set_present(struct context_entry
*context
)
235 static inline void context_set_fault_enable(struct context_entry
*context
)
237 context
->lo
&= (((u64
)-1) << 2) | 1;
240 static inline void context_set_translation_type(struct context_entry
*context
,
243 context
->lo
&= (((u64
)-1) << 4) | 3;
244 context
->lo
|= (value
& 3) << 2;
247 static inline void context_set_address_root(struct context_entry
*context
,
250 context
->lo
|= value
& VTD_PAGE_MASK
;
253 static inline void context_set_address_width(struct context_entry
*context
,
256 context
->hi
|= value
& 7;
259 static inline void context_set_domain_id(struct context_entry
*context
,
262 context
->hi
|= (value
& ((1 << 16) - 1)) << 8;
265 static inline void context_clear_entry(struct context_entry
*context
)
278 * 12-63: Host physcial address
284 static inline void dma_clear_pte(struct dma_pte
*pte
)
289 static inline u64
dma_pte_addr(struct dma_pte
*pte
)
292 return pte
->val
& VTD_PAGE_MASK
;
294 /* Must have a full atomic 64-bit read */
295 return __cmpxchg64(&pte
->val
, 0ULL, 0ULL) & VTD_PAGE_MASK
;
299 static inline bool dma_pte_present(struct dma_pte
*pte
)
301 return (pte
->val
& 3) != 0;
304 static inline bool dma_pte_superpage(struct dma_pte
*pte
)
306 return (pte
->val
& (1 << 7));
309 static inline int first_pte_in_page(struct dma_pte
*pte
)
311 return !((unsigned long)pte
& ~VTD_PAGE_MASK
);
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
320 static struct dmar_domain
*si_domain
;
321 static int hw_pass_through
= 1;
323 /* devices under the same p2p bridge are owned in one domain */
324 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
326 /* domain represents a virtual machine, more than one devices
327 * across iommus may be owned in one domain, e.g. kvm guest.
329 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
331 /* si_domain contains mulitple devices */
332 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
334 /* define the limit of IOMMUs supported in each domain */
336 # define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
338 # define IOMMU_UNITS_SUPPORTED 64
342 int id
; /* domain id */
343 int nid
; /* node id */
344 DECLARE_BITMAP(iommu_bmp
, IOMMU_UNITS_SUPPORTED
);
345 /* bitmap of iommus this domain uses*/
347 struct list_head devices
; /* all devices' list */
348 struct iova_domain iovad
; /* iova's that belong to this domain */
350 struct dma_pte
*pgd
; /* virtual address */
351 int gaw
; /* max guest address width */
353 /* adjusted guest address width, 0 is level 2 30-bit */
356 int flags
; /* flags to find out type of domain */
358 int iommu_coherency
;/* indicate coherency of iommu access */
359 int iommu_snooping
; /* indicate snooping control feature*/
360 int iommu_count
; /* reference count of iommu */
361 int iommu_superpage
;/* Level of superpages supported:
362 0 == 4KiB (no superpages), 1 == 2MiB,
363 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
364 spinlock_t iommu_lock
; /* protect iommu set in domain */
365 u64 max_addr
; /* maximum mapped address */
368 /* PCI domain-device relationship */
369 struct device_domain_info
{
370 struct list_head link
; /* link to domain siblings */
371 struct list_head global
; /* link to global list */
372 int segment
; /* PCI domain */
373 u8 bus
; /* PCI bus number */
374 u8 devfn
; /* PCI devfn number */
375 struct pci_dev
*dev
; /* it's NULL for PCIe-to-PCI bridge */
376 struct intel_iommu
*iommu
; /* IOMMU used by this device */
377 struct dmar_domain
*domain
; /* pointer to domain */
380 struct dmar_rmrr_unit
{
381 struct list_head list
; /* list of rmrr units */
382 struct acpi_dmar_header
*hdr
; /* ACPI header */
383 u64 base_address
; /* reserved base address*/
384 u64 end_address
; /* reserved end address */
385 struct pci_dev __rcu
**devices
; /* target devices */
386 int devices_cnt
; /* target device count */
389 struct dmar_atsr_unit
{
390 struct list_head list
; /* list of ATSR units */
391 struct acpi_dmar_header
*hdr
; /* ACPI header */
392 struct pci_dev __rcu
**devices
; /* target devices */
393 int devices_cnt
; /* target device count */
394 u8 include_all
:1; /* include all ports */
397 static LIST_HEAD(dmar_atsr_units
);
398 static LIST_HEAD(dmar_rmrr_units
);
400 #define for_each_rmrr_units(rmrr) \
401 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
403 static void flush_unmaps_timeout(unsigned long data
);
405 static DEFINE_TIMER(unmap_timer
, flush_unmaps_timeout
, 0, 0);
407 #define HIGH_WATER_MARK 250
408 struct deferred_flush_tables
{
410 struct iova
*iova
[HIGH_WATER_MARK
];
411 struct dmar_domain
*domain
[HIGH_WATER_MARK
];
412 struct page
*freelist
[HIGH_WATER_MARK
];
415 static struct deferred_flush_tables
*deferred_flush
;
417 /* bitmap for indexing intel_iommus */
418 static int g_num_of_iommus
;
420 static DEFINE_SPINLOCK(async_umap_flush_lock
);
421 static LIST_HEAD(unmaps_to_do
);
424 static long list_size
;
426 static void domain_exit(struct dmar_domain
*domain
);
427 static void domain_remove_dev_info(struct dmar_domain
*domain
);
428 static void domain_remove_one_dev_info(struct dmar_domain
*domain
,
429 struct pci_dev
*pdev
);
430 static void iommu_detach_dependent_devices(struct intel_iommu
*iommu
,
431 struct pci_dev
*pdev
);
433 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
434 int dmar_disabled
= 0;
436 int dmar_disabled
= 1;
437 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
439 int intel_iommu_enabled
= 0;
440 EXPORT_SYMBOL_GPL(intel_iommu_enabled
);
442 static int dmar_map_gfx
= 1;
443 static int dmar_forcedac
;
444 static int intel_iommu_strict
;
445 static int intel_iommu_superpage
= 1;
447 int intel_iommu_gfx_mapped
;
448 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped
);
450 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
451 static DEFINE_SPINLOCK(device_domain_lock
);
452 static LIST_HEAD(device_domain_list
);
454 static struct iommu_ops intel_iommu_ops
;
456 static int __init
intel_iommu_setup(char *str
)
461 if (!strncmp(str
, "on", 2)) {
463 printk(KERN_INFO
"Intel-IOMMU: enabled\n");
464 } else if (!strncmp(str
, "off", 3)) {
466 printk(KERN_INFO
"Intel-IOMMU: disabled\n");
467 } else if (!strncmp(str
, "igfx_off", 8)) {
470 "Intel-IOMMU: disable GFX device mapping\n");
471 } else if (!strncmp(str
, "forcedac", 8)) {
473 "Intel-IOMMU: Forcing DAC for PCI devices\n");
475 } else if (!strncmp(str
, "strict", 6)) {
477 "Intel-IOMMU: disable batched IOTLB flush\n");
478 intel_iommu_strict
= 1;
479 } else if (!strncmp(str
, "sp_off", 6)) {
481 "Intel-IOMMU: disable supported super page\n");
482 intel_iommu_superpage
= 0;
485 str
+= strcspn(str
, ",");
491 __setup("intel_iommu=", intel_iommu_setup
);
493 static struct kmem_cache
*iommu_domain_cache
;
494 static struct kmem_cache
*iommu_devinfo_cache
;
495 static struct kmem_cache
*iommu_iova_cache
;
497 static inline void *alloc_pgtable_page(int node
)
502 page
= alloc_pages_node(node
, GFP_ATOMIC
| __GFP_ZERO
, 0);
504 vaddr
= page_address(page
);
508 static inline void free_pgtable_page(void *vaddr
)
510 free_page((unsigned long)vaddr
);
513 static inline void *alloc_domain_mem(void)
515 return kmem_cache_alloc(iommu_domain_cache
, GFP_ATOMIC
);
518 static void free_domain_mem(void *vaddr
)
520 kmem_cache_free(iommu_domain_cache
, vaddr
);
523 static inline void * alloc_devinfo_mem(void)
525 return kmem_cache_alloc(iommu_devinfo_cache
, GFP_ATOMIC
);
528 static inline void free_devinfo_mem(void *vaddr
)
530 kmem_cache_free(iommu_devinfo_cache
, vaddr
);
533 struct iova
*alloc_iova_mem(void)
535 return kmem_cache_alloc(iommu_iova_cache
, GFP_ATOMIC
);
538 void free_iova_mem(struct iova
*iova
)
540 kmem_cache_free(iommu_iova_cache
, iova
);
544 static int __iommu_calculate_agaw(struct intel_iommu
*iommu
, int max_gaw
)
549 sagaw
= cap_sagaw(iommu
->cap
);
550 for (agaw
= width_to_agaw(max_gaw
);
552 if (test_bit(agaw
, &sagaw
))
560 * Calculate max SAGAW for each iommu.
562 int iommu_calculate_max_sagaw(struct intel_iommu
*iommu
)
564 return __iommu_calculate_agaw(iommu
, MAX_AGAW_WIDTH
);
568 * calculate agaw for each iommu.
569 * "SAGAW" may be different across iommus, use a default agaw, and
570 * get a supported less agaw for iommus that don't support the default agaw.
572 int iommu_calculate_agaw(struct intel_iommu
*iommu
)
574 return __iommu_calculate_agaw(iommu
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
577 /* This functionin only returns single iommu in a domain */
578 static struct intel_iommu
*domain_get_iommu(struct dmar_domain
*domain
)
582 /* si_domain and vm domain should not get here. */
583 BUG_ON(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
);
584 BUG_ON(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
);
586 iommu_id
= find_first_bit(domain
->iommu_bmp
, g_num_of_iommus
);
587 if (iommu_id
< 0 || iommu_id
>= g_num_of_iommus
)
590 return g_iommus
[iommu_id
];
593 static void domain_update_iommu_coherency(struct dmar_domain
*domain
)
595 struct dmar_drhd_unit
*drhd
;
596 struct intel_iommu
*iommu
;
599 domain
->iommu_coherency
= 1;
601 for_each_set_bit(i
, domain
->iommu_bmp
, g_num_of_iommus
) {
603 if (!ecap_coherent(g_iommus
[i
]->ecap
)) {
604 domain
->iommu_coherency
= 0;
611 /* No hardware attached; use lowest common denominator */
613 for_each_active_iommu(iommu
, drhd
) {
614 if (!ecap_coherent(iommu
->ecap
)) {
615 domain
->iommu_coherency
= 0;
622 static void domain_update_iommu_snooping(struct dmar_domain
*domain
)
626 domain
->iommu_snooping
= 1;
628 for_each_set_bit(i
, domain
->iommu_bmp
, g_num_of_iommus
) {
629 if (!ecap_sc_support(g_iommus
[i
]->ecap
)) {
630 domain
->iommu_snooping
= 0;
636 static void domain_update_iommu_superpage(struct dmar_domain
*domain
)
638 struct dmar_drhd_unit
*drhd
;
639 struct intel_iommu
*iommu
= NULL
;
642 if (!intel_iommu_superpage
) {
643 domain
->iommu_superpage
= 0;
647 /* set iommu_superpage to the smallest common denominator */
649 for_each_active_iommu(iommu
, drhd
) {
650 mask
&= cap_super_page_val(iommu
->cap
);
657 domain
->iommu_superpage
= fls(mask
);
660 /* Some capabilities may be different across iommus */
661 static void domain_update_iommu_cap(struct dmar_domain
*domain
)
663 domain_update_iommu_coherency(domain
);
664 domain_update_iommu_snooping(domain
);
665 domain_update_iommu_superpage(domain
);
668 static struct intel_iommu
*device_to_iommu(int segment
, u8 bus
, u8 devfn
)
670 struct dmar_drhd_unit
*drhd
= NULL
;
671 struct intel_iommu
*iommu
;
676 for_each_active_iommu(iommu
, drhd
) {
677 if (segment
!= drhd
->segment
)
680 for_each_active_dev_scope(drhd
->devices
,
681 drhd
->devices_cnt
, i
, dev
) {
682 if (dev
->bus
->number
== bus
&& dev
->devfn
== devfn
)
684 if (dev
->subordinate
&&
685 dev
->subordinate
->number
<= bus
&&
686 dev
->subordinate
->busn_res
.end
>= bus
)
690 if (drhd
->include_all
)
700 static void domain_flush_cache(struct dmar_domain
*domain
,
701 void *addr
, int size
)
703 if (!domain
->iommu_coherency
)
704 clflush_cache_range(addr
, size
);
707 /* Gets context entry for a given bus and devfn */
708 static struct context_entry
* device_to_context_entry(struct intel_iommu
*iommu
,
711 struct root_entry
*root
;
712 struct context_entry
*context
;
713 unsigned long phy_addr
;
716 spin_lock_irqsave(&iommu
->lock
, flags
);
717 root
= &iommu
->root_entry
[bus
];
718 context
= get_context_addr_from_root(root
);
720 context
= (struct context_entry
*)
721 alloc_pgtable_page(iommu
->node
);
723 spin_unlock_irqrestore(&iommu
->lock
, flags
);
726 __iommu_flush_cache(iommu
, (void *)context
, CONTEXT_SIZE
);
727 phy_addr
= virt_to_phys((void *)context
);
728 set_root_value(root
, phy_addr
);
729 set_root_present(root
);
730 __iommu_flush_cache(iommu
, root
, sizeof(*root
));
732 spin_unlock_irqrestore(&iommu
->lock
, flags
);
733 return &context
[devfn
];
736 static int device_context_mapped(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
738 struct root_entry
*root
;
739 struct context_entry
*context
;
743 spin_lock_irqsave(&iommu
->lock
, flags
);
744 root
= &iommu
->root_entry
[bus
];
745 context
= get_context_addr_from_root(root
);
750 ret
= context_present(&context
[devfn
]);
752 spin_unlock_irqrestore(&iommu
->lock
, flags
);
756 static void clear_context_table(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
758 struct root_entry
*root
;
759 struct context_entry
*context
;
762 spin_lock_irqsave(&iommu
->lock
, flags
);
763 root
= &iommu
->root_entry
[bus
];
764 context
= get_context_addr_from_root(root
);
766 context_clear_entry(&context
[devfn
]);
767 __iommu_flush_cache(iommu
, &context
[devfn
], \
770 spin_unlock_irqrestore(&iommu
->lock
, flags
);
773 static void free_context_table(struct intel_iommu
*iommu
)
775 struct root_entry
*root
;
778 struct context_entry
*context
;
780 spin_lock_irqsave(&iommu
->lock
, flags
);
781 if (!iommu
->root_entry
) {
784 for (i
= 0; i
< ROOT_ENTRY_NR
; i
++) {
785 root
= &iommu
->root_entry
[i
];
786 context
= get_context_addr_from_root(root
);
788 free_pgtable_page(context
);
790 free_pgtable_page(iommu
->root_entry
);
791 iommu
->root_entry
= NULL
;
793 spin_unlock_irqrestore(&iommu
->lock
, flags
);
796 static struct dma_pte
*pfn_to_dma_pte(struct dmar_domain
*domain
,
797 unsigned long pfn
, int *target_level
)
799 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
800 struct dma_pte
*parent
, *pte
= NULL
;
801 int level
= agaw_to_level(domain
->agaw
);
804 BUG_ON(!domain
->pgd
);
806 if (addr_width
< BITS_PER_LONG
&& pfn
>> addr_width
)
807 /* Address beyond IOMMU's addressing capabilities. */
810 parent
= domain
->pgd
;
815 offset
= pfn_level_offset(pfn
, level
);
816 pte
= &parent
[offset
];
817 if (!*target_level
&& (dma_pte_superpage(pte
) || !dma_pte_present(pte
)))
819 if (level
== *target_level
)
822 if (!dma_pte_present(pte
)) {
825 tmp_page
= alloc_pgtable_page(domain
->nid
);
830 domain_flush_cache(domain
, tmp_page
, VTD_PAGE_SIZE
);
831 pteval
= ((uint64_t)virt_to_dma_pfn(tmp_page
) << VTD_PAGE_SHIFT
) | DMA_PTE_READ
| DMA_PTE_WRITE
;
832 if (cmpxchg64(&pte
->val
, 0ULL, pteval
)) {
833 /* Someone else set it while we were thinking; use theirs. */
834 free_pgtable_page(tmp_page
);
837 domain_flush_cache(domain
, pte
, sizeof(*pte
));
843 parent
= phys_to_virt(dma_pte_addr(pte
));
848 *target_level
= level
;
854 /* return address's pte at specific level */
855 static struct dma_pte
*dma_pfn_level_pte(struct dmar_domain
*domain
,
857 int level
, int *large_page
)
859 struct dma_pte
*parent
, *pte
= NULL
;
860 int total
= agaw_to_level(domain
->agaw
);
863 parent
= domain
->pgd
;
864 while (level
<= total
) {
865 offset
= pfn_level_offset(pfn
, total
);
866 pte
= &parent
[offset
];
870 if (!dma_pte_present(pte
)) {
875 if (pte
->val
& DMA_PTE_LARGE_PAGE
) {
880 parent
= phys_to_virt(dma_pte_addr(pte
));
886 /* clear last level pte, a tlb flush should be followed */
887 static void dma_pte_clear_range(struct dmar_domain
*domain
,
888 unsigned long start_pfn
,
889 unsigned long last_pfn
)
891 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
892 unsigned int large_page
= 1;
893 struct dma_pte
*first_pte
, *pte
;
895 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
896 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
897 BUG_ON(start_pfn
> last_pfn
);
899 /* we don't need lock here; nobody else touches the iova range */
902 first_pte
= pte
= dma_pfn_level_pte(domain
, start_pfn
, 1, &large_page
);
904 start_pfn
= align_to_level(start_pfn
+ 1, large_page
+ 1);
909 start_pfn
+= lvl_to_nr_pages(large_page
);
911 } while (start_pfn
<= last_pfn
&& !first_pte_in_page(pte
));
913 domain_flush_cache(domain
, first_pte
,
914 (void *)pte
- (void *)first_pte
);
916 } while (start_pfn
&& start_pfn
<= last_pfn
);
919 static void dma_pte_free_level(struct dmar_domain
*domain
, int level
,
920 struct dma_pte
*pte
, unsigned long pfn
,
921 unsigned long start_pfn
, unsigned long last_pfn
)
923 pfn
= max(start_pfn
, pfn
);
924 pte
= &pte
[pfn_level_offset(pfn
, level
)];
927 unsigned long level_pfn
;
928 struct dma_pte
*level_pte
;
930 if (!dma_pte_present(pte
) || dma_pte_superpage(pte
))
933 level_pfn
= pfn
& level_mask(level
- 1);
934 level_pte
= phys_to_virt(dma_pte_addr(pte
));
937 dma_pte_free_level(domain
, level
- 1, level_pte
,
938 level_pfn
, start_pfn
, last_pfn
);
940 /* If range covers entire pagetable, free it */
941 if (!(start_pfn
> level_pfn
||
942 last_pfn
< level_pfn
+ level_size(level
) - 1)) {
944 domain_flush_cache(domain
, pte
, sizeof(*pte
));
945 free_pgtable_page(level_pte
);
948 pfn
+= level_size(level
);
949 } while (!first_pte_in_page(++pte
) && pfn
<= last_pfn
);
952 /* free page table pages. last level pte should already be cleared */
953 static void dma_pte_free_pagetable(struct dmar_domain
*domain
,
954 unsigned long start_pfn
,
955 unsigned long last_pfn
)
957 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
959 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
960 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
961 BUG_ON(start_pfn
> last_pfn
);
963 /* We don't need lock here; nobody else touches the iova range */
964 dma_pte_free_level(domain
, agaw_to_level(domain
->agaw
),
965 domain
->pgd
, 0, start_pfn
, last_pfn
);
968 if (start_pfn
== 0 && last_pfn
== DOMAIN_MAX_PFN(domain
->gaw
)) {
969 free_pgtable_page(domain
->pgd
);
974 /* When a page at a given level is being unlinked from its parent, we don't
975 need to *modify* it at all. All we need to do is make a list of all the
976 pages which can be freed just as soon as we've flushed the IOTLB and we
977 know the hardware page-walk will no longer touch them.
978 The 'pte' argument is the *parent* PTE, pointing to the page that is to
980 static struct page
*dma_pte_list_pagetables(struct dmar_domain
*domain
,
981 int level
, struct dma_pte
*pte
,
982 struct page
*freelist
)
986 pg
= pfn_to_page(dma_pte_addr(pte
) >> PAGE_SHIFT
);
987 pg
->freelist
= freelist
;
993 for (pte
= page_address(pg
); !first_pte_in_page(pte
); pte
++) {
994 if (dma_pte_present(pte
) && !dma_pte_superpage(pte
))
995 freelist
= dma_pte_list_pagetables(domain
, level
- 1,
1002 static struct page
*dma_pte_clear_level(struct dmar_domain
*domain
, int level
,
1003 struct dma_pte
*pte
, unsigned long pfn
,
1004 unsigned long start_pfn
,
1005 unsigned long last_pfn
,
1006 struct page
*freelist
)
1008 struct dma_pte
*first_pte
= NULL
, *last_pte
= NULL
;
1010 pfn
= max(start_pfn
, pfn
);
1011 pte
= &pte
[pfn_level_offset(pfn
, level
)];
1014 unsigned long level_pfn
;
1016 if (!dma_pte_present(pte
))
1019 level_pfn
= pfn
& level_mask(level
);
1021 /* If range covers entire pagetable, free it */
1022 if (start_pfn
<= level_pfn
&&
1023 last_pfn
>= level_pfn
+ level_size(level
) - 1) {
1024 /* These suborbinate page tables are going away entirely. Don't
1025 bother to clear them; we're just going to *free* them. */
1026 if (level
> 1 && !dma_pte_superpage(pte
))
1027 freelist
= dma_pte_list_pagetables(domain
, level
- 1, pte
, freelist
);
1033 } else if (level
> 1) {
1034 /* Recurse down into a level that isn't *entirely* obsolete */
1035 freelist
= dma_pte_clear_level(domain
, level
- 1,
1036 phys_to_virt(dma_pte_addr(pte
)),
1037 level_pfn
, start_pfn
, last_pfn
,
1041 pfn
+= level_size(level
);
1042 } while (!first_pte_in_page(++pte
) && pfn
<= last_pfn
);
1045 domain_flush_cache(domain
, first_pte
,
1046 (void *)++last_pte
- (void *)first_pte
);
1051 /* We can't just free the pages because the IOMMU may still be walking
1052 the page tables, and may have cached the intermediate levels. The
1053 pages can only be freed after the IOTLB flush has been done. */
1054 struct page
*domain_unmap(struct dmar_domain
*domain
,
1055 unsigned long start_pfn
,
1056 unsigned long last_pfn
)
1058 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
1059 struct page
*freelist
= NULL
;
1061 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
1062 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
1063 BUG_ON(start_pfn
> last_pfn
);
1065 /* we don't need lock here; nobody else touches the iova range */
1066 freelist
= dma_pte_clear_level(domain
, agaw_to_level(domain
->agaw
),
1067 domain
->pgd
, 0, start_pfn
, last_pfn
, NULL
);
1070 if (start_pfn
== 0 && last_pfn
== DOMAIN_MAX_PFN(domain
->gaw
)) {
1071 struct page
*pgd_page
= virt_to_page(domain
->pgd
);
1072 pgd_page
->freelist
= freelist
;
1073 freelist
= pgd_page
;
1081 void dma_free_pagelist(struct page
*freelist
)
1085 while ((pg
= freelist
)) {
1086 freelist
= pg
->freelist
;
1087 free_pgtable_page(page_address(pg
));
1091 /* iommu handling */
1092 static int iommu_alloc_root_entry(struct intel_iommu
*iommu
)
1094 struct root_entry
*root
;
1095 unsigned long flags
;
1097 root
= (struct root_entry
*)alloc_pgtable_page(iommu
->node
);
1101 __iommu_flush_cache(iommu
, root
, ROOT_SIZE
);
1103 spin_lock_irqsave(&iommu
->lock
, flags
);
1104 iommu
->root_entry
= root
;
1105 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1110 static void iommu_set_root_entry(struct intel_iommu
*iommu
)
1116 addr
= iommu
->root_entry
;
1118 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1119 dmar_writeq(iommu
->reg
+ DMAR_RTADDR_REG
, virt_to_phys(addr
));
1121 writel(iommu
->gcmd
| DMA_GCMD_SRTP
, iommu
->reg
+ DMAR_GCMD_REG
);
1123 /* Make sure hardware complete it */
1124 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1125 readl
, (sts
& DMA_GSTS_RTPS
), sts
);
1127 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1130 static void iommu_flush_write_buffer(struct intel_iommu
*iommu
)
1135 if (!rwbf_quirk
&& !cap_rwbf(iommu
->cap
))
1138 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1139 writel(iommu
->gcmd
| DMA_GCMD_WBF
, iommu
->reg
+ DMAR_GCMD_REG
);
1141 /* Make sure hardware complete it */
1142 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1143 readl
, (!(val
& DMA_GSTS_WBFS
)), val
);
1145 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1148 /* return value determine if we need a write buffer flush */
1149 static void __iommu_flush_context(struct intel_iommu
*iommu
,
1150 u16 did
, u16 source_id
, u8 function_mask
,
1157 case DMA_CCMD_GLOBAL_INVL
:
1158 val
= DMA_CCMD_GLOBAL_INVL
;
1160 case DMA_CCMD_DOMAIN_INVL
:
1161 val
= DMA_CCMD_DOMAIN_INVL
|DMA_CCMD_DID(did
);
1163 case DMA_CCMD_DEVICE_INVL
:
1164 val
= DMA_CCMD_DEVICE_INVL
|DMA_CCMD_DID(did
)
1165 | DMA_CCMD_SID(source_id
) | DMA_CCMD_FM(function_mask
);
1170 val
|= DMA_CCMD_ICC
;
1172 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1173 dmar_writeq(iommu
->reg
+ DMAR_CCMD_REG
, val
);
1175 /* Make sure hardware complete it */
1176 IOMMU_WAIT_OP(iommu
, DMAR_CCMD_REG
,
1177 dmar_readq
, (!(val
& DMA_CCMD_ICC
)), val
);
1179 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1182 /* return value determine if we need a write buffer flush */
1183 static void __iommu_flush_iotlb(struct intel_iommu
*iommu
, u16 did
,
1184 u64 addr
, unsigned int size_order
, u64 type
)
1186 int tlb_offset
= ecap_iotlb_offset(iommu
->ecap
);
1187 u64 val
= 0, val_iva
= 0;
1191 case DMA_TLB_GLOBAL_FLUSH
:
1192 /* global flush doesn't need set IVA_REG */
1193 val
= DMA_TLB_GLOBAL_FLUSH
|DMA_TLB_IVT
;
1195 case DMA_TLB_DSI_FLUSH
:
1196 val
= DMA_TLB_DSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
1198 case DMA_TLB_PSI_FLUSH
:
1199 val
= DMA_TLB_PSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
1200 /* IH bit is passed in as part of address */
1201 val_iva
= size_order
| addr
;
1206 /* Note: set drain read/write */
1209 * This is probably to be super secure.. Looks like we can
1210 * ignore it without any impact.
1212 if (cap_read_drain(iommu
->cap
))
1213 val
|= DMA_TLB_READ_DRAIN
;
1215 if (cap_write_drain(iommu
->cap
))
1216 val
|= DMA_TLB_WRITE_DRAIN
;
1218 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1219 /* Note: Only uses first TLB reg currently */
1221 dmar_writeq(iommu
->reg
+ tlb_offset
, val_iva
);
1222 dmar_writeq(iommu
->reg
+ tlb_offset
+ 8, val
);
1224 /* Make sure hardware complete it */
1225 IOMMU_WAIT_OP(iommu
, tlb_offset
+ 8,
1226 dmar_readq
, (!(val
& DMA_TLB_IVT
)), val
);
1228 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1230 /* check IOTLB invalidation granularity */
1231 if (DMA_TLB_IAIG(val
) == 0)
1232 printk(KERN_ERR
"IOMMU: flush IOTLB failed\n");
1233 if (DMA_TLB_IAIG(val
) != DMA_TLB_IIRG(type
))
1234 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1235 (unsigned long long)DMA_TLB_IIRG(type
),
1236 (unsigned long long)DMA_TLB_IAIG(val
));
1239 static struct device_domain_info
*iommu_support_dev_iotlb(
1240 struct dmar_domain
*domain
, int segment
, u8 bus
, u8 devfn
)
1243 unsigned long flags
;
1244 struct device_domain_info
*info
;
1245 struct intel_iommu
*iommu
= device_to_iommu(segment
, bus
, devfn
);
1247 if (!ecap_dev_iotlb_support(iommu
->ecap
))
1253 spin_lock_irqsave(&device_domain_lock
, flags
);
1254 list_for_each_entry(info
, &domain
->devices
, link
)
1255 if (info
->bus
== bus
&& info
->devfn
== devfn
) {
1259 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1261 if (!found
|| !info
->dev
)
1264 if (!pci_find_ext_capability(info
->dev
, PCI_EXT_CAP_ID_ATS
))
1267 if (!dmar_find_matched_atsr_unit(info
->dev
))
1270 info
->iommu
= iommu
;
1275 static void iommu_enable_dev_iotlb(struct device_domain_info
*info
)
1280 pci_enable_ats(info
->dev
, VTD_PAGE_SHIFT
);
1283 static void iommu_disable_dev_iotlb(struct device_domain_info
*info
)
1285 if (!info
->dev
|| !pci_ats_enabled(info
->dev
))
1288 pci_disable_ats(info
->dev
);
1291 static void iommu_flush_dev_iotlb(struct dmar_domain
*domain
,
1292 u64 addr
, unsigned mask
)
1295 unsigned long flags
;
1296 struct device_domain_info
*info
;
1298 spin_lock_irqsave(&device_domain_lock
, flags
);
1299 list_for_each_entry(info
, &domain
->devices
, link
) {
1300 if (!info
->dev
|| !pci_ats_enabled(info
->dev
))
1303 sid
= info
->bus
<< 8 | info
->devfn
;
1304 qdep
= pci_ats_queue_depth(info
->dev
);
1305 qi_flush_dev_iotlb(info
->iommu
, sid
, qdep
, addr
, mask
);
1307 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1310 static void iommu_flush_iotlb_psi(struct intel_iommu
*iommu
, u16 did
,
1311 unsigned long pfn
, unsigned int pages
, int ih
, int map
)
1313 unsigned int mask
= ilog2(__roundup_pow_of_two(pages
));
1314 uint64_t addr
= (uint64_t)pfn
<< VTD_PAGE_SHIFT
;
1321 * Fallback to domain selective flush if no PSI support or the size is
1323 * PSI requires page size to be 2 ^ x, and the base address is naturally
1324 * aligned to the size
1326 if (!cap_pgsel_inv(iommu
->cap
) || mask
> cap_max_amask_val(iommu
->cap
))
1327 iommu
->flush
.flush_iotlb(iommu
, did
, 0, 0,
1330 iommu
->flush
.flush_iotlb(iommu
, did
, addr
| ih
, mask
,
1334 * In caching mode, changes of pages from non-present to present require
1335 * flush. However, device IOTLB doesn't need to be flushed in this case.
1337 if (!cap_caching_mode(iommu
->cap
) || !map
)
1338 iommu_flush_dev_iotlb(iommu
->domains
[did
], addr
, mask
);
1341 static void iommu_disable_protect_mem_regions(struct intel_iommu
*iommu
)
1344 unsigned long flags
;
1346 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
1347 pmen
= readl(iommu
->reg
+ DMAR_PMEN_REG
);
1348 pmen
&= ~DMA_PMEN_EPM
;
1349 writel(pmen
, iommu
->reg
+ DMAR_PMEN_REG
);
1351 /* wait for the protected region status bit to clear */
1352 IOMMU_WAIT_OP(iommu
, DMAR_PMEN_REG
,
1353 readl
, !(pmen
& DMA_PMEN_PRS
), pmen
);
1355 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1358 static int iommu_enable_translation(struct intel_iommu
*iommu
)
1361 unsigned long flags
;
1363 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
1364 iommu
->gcmd
|= DMA_GCMD_TE
;
1365 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1367 /* Make sure hardware complete it */
1368 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1369 readl
, (sts
& DMA_GSTS_TES
), sts
);
1371 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1375 static int iommu_disable_translation(struct intel_iommu
*iommu
)
1380 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1381 iommu
->gcmd
&= ~DMA_GCMD_TE
;
1382 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1384 /* Make sure hardware complete it */
1385 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1386 readl
, (!(sts
& DMA_GSTS_TES
)), sts
);
1388 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1393 static int iommu_init_domains(struct intel_iommu
*iommu
)
1395 unsigned long ndomains
;
1396 unsigned long nlongs
;
1398 ndomains
= cap_ndoms(iommu
->cap
);
1399 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1400 iommu
->seq_id
, ndomains
);
1401 nlongs
= BITS_TO_LONGS(ndomains
);
1403 spin_lock_init(&iommu
->lock
);
1405 /* TBD: there might be 64K domains,
1406 * consider other allocation for future chip
1408 iommu
->domain_ids
= kcalloc(nlongs
, sizeof(unsigned long), GFP_KERNEL
);
1409 if (!iommu
->domain_ids
) {
1410 pr_err("IOMMU%d: allocating domain id array failed\n",
1414 iommu
->domains
= kcalloc(ndomains
, sizeof(struct dmar_domain
*),
1416 if (!iommu
->domains
) {
1417 pr_err("IOMMU%d: allocating domain array failed\n",
1419 kfree(iommu
->domain_ids
);
1420 iommu
->domain_ids
= NULL
;
1425 * if Caching mode is set, then invalid translations are tagged
1426 * with domainid 0. Hence we need to pre-allocate it.
1428 if (cap_caching_mode(iommu
->cap
))
1429 set_bit(0, iommu
->domain_ids
);
1433 static void free_dmar_iommu(struct intel_iommu
*iommu
)
1435 struct dmar_domain
*domain
;
1437 unsigned long flags
;
1439 if ((iommu
->domains
) && (iommu
->domain_ids
)) {
1440 for_each_set_bit(i
, iommu
->domain_ids
, cap_ndoms(iommu
->cap
)) {
1442 * Domain id 0 is reserved for invalid translation
1443 * if hardware supports caching mode.
1445 if (cap_caching_mode(iommu
->cap
) && i
== 0)
1448 domain
= iommu
->domains
[i
];
1449 clear_bit(i
, iommu
->domain_ids
);
1451 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1452 count
= --domain
->iommu_count
;
1453 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1455 domain_exit(domain
);
1459 if (iommu
->gcmd
& DMA_GCMD_TE
)
1460 iommu_disable_translation(iommu
);
1462 kfree(iommu
->domains
);
1463 kfree(iommu
->domain_ids
);
1464 iommu
->domains
= NULL
;
1465 iommu
->domain_ids
= NULL
;
1467 g_iommus
[iommu
->seq_id
] = NULL
;
1469 /* free context mapping */
1470 free_context_table(iommu
);
1473 static struct dmar_domain
*alloc_domain(bool vm
)
1475 /* domain id for virtual machine, it won't be set in context */
1476 static atomic_t vm_domid
= ATOMIC_INIT(0);
1477 struct dmar_domain
*domain
;
1479 domain
= alloc_domain_mem();
1484 domain
->iommu_count
= 0;
1485 memset(domain
->iommu_bmp
, 0, sizeof(domain
->iommu_bmp
));
1487 spin_lock_init(&domain
->iommu_lock
);
1488 INIT_LIST_HEAD(&domain
->devices
);
1490 domain
->id
= atomic_inc_return(&vm_domid
);
1491 domain
->flags
= DOMAIN_FLAG_VIRTUAL_MACHINE
;
1497 static int iommu_attach_domain(struct dmar_domain
*domain
,
1498 struct intel_iommu
*iommu
)
1501 unsigned long ndomains
;
1502 unsigned long flags
;
1504 ndomains
= cap_ndoms(iommu
->cap
);
1506 spin_lock_irqsave(&iommu
->lock
, flags
);
1508 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1509 if (num
>= ndomains
) {
1510 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1511 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1516 domain
->iommu_count
++;
1517 set_bit(num
, iommu
->domain_ids
);
1518 set_bit(iommu
->seq_id
, domain
->iommu_bmp
);
1519 iommu
->domains
[num
] = domain
;
1520 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1525 static void iommu_detach_domain(struct dmar_domain
*domain
,
1526 struct intel_iommu
*iommu
)
1528 unsigned long flags
;
1531 spin_lock_irqsave(&iommu
->lock
, flags
);
1532 ndomains
= cap_ndoms(iommu
->cap
);
1533 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
1534 if (iommu
->domains
[num
] == domain
) {
1535 clear_bit(num
, iommu
->domain_ids
);
1536 iommu
->domains
[num
] = NULL
;
1540 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1543 static struct iova_domain reserved_iova_list
;
1544 static struct lock_class_key reserved_rbtree_key
;
1546 static int dmar_init_reserved_ranges(void)
1548 struct pci_dev
*pdev
= NULL
;
1552 init_iova_domain(&reserved_iova_list
, DMA_32BIT_PFN
);
1554 lockdep_set_class(&reserved_iova_list
.iova_rbtree_lock
,
1555 &reserved_rbtree_key
);
1557 /* IOAPIC ranges shouldn't be accessed by DMA */
1558 iova
= reserve_iova(&reserved_iova_list
, IOVA_PFN(IOAPIC_RANGE_START
),
1559 IOVA_PFN(IOAPIC_RANGE_END
));
1561 printk(KERN_ERR
"Reserve IOAPIC range failed\n");
1565 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1566 for_each_pci_dev(pdev
) {
1569 for (i
= 0; i
< PCI_NUM_RESOURCES
; i
++) {
1570 r
= &pdev
->resource
[i
];
1571 if (!r
->flags
|| !(r
->flags
& IORESOURCE_MEM
))
1573 iova
= reserve_iova(&reserved_iova_list
,
1577 printk(KERN_ERR
"Reserve iova failed\n");
1585 static void domain_reserve_special_ranges(struct dmar_domain
*domain
)
1587 copy_reserved_iova(&reserved_iova_list
, &domain
->iovad
);
1590 static inline int guestwidth_to_adjustwidth(int gaw
)
1593 int r
= (gaw
- 12) % 9;
1604 static int domain_init(struct dmar_domain
*domain
, int guest_width
)
1606 struct intel_iommu
*iommu
;
1607 int adjust_width
, agaw
;
1608 unsigned long sagaw
;
1610 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
1611 domain_reserve_special_ranges(domain
);
1613 /* calculate AGAW */
1614 iommu
= domain_get_iommu(domain
);
1615 if (guest_width
> cap_mgaw(iommu
->cap
))
1616 guest_width
= cap_mgaw(iommu
->cap
);
1617 domain
->gaw
= guest_width
;
1618 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
1619 agaw
= width_to_agaw(adjust_width
);
1620 sagaw
= cap_sagaw(iommu
->cap
);
1621 if (!test_bit(agaw
, &sagaw
)) {
1622 /* hardware doesn't support it, choose a bigger one */
1623 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw
);
1624 agaw
= find_next_bit(&sagaw
, 5, agaw
);
1628 domain
->agaw
= agaw
;
1630 if (ecap_coherent(iommu
->ecap
))
1631 domain
->iommu_coherency
= 1;
1633 domain
->iommu_coherency
= 0;
1635 if (ecap_sc_support(iommu
->ecap
))
1636 domain
->iommu_snooping
= 1;
1638 domain
->iommu_snooping
= 0;
1640 if (intel_iommu_superpage
)
1641 domain
->iommu_superpage
= fls(cap_super_page_val(iommu
->cap
));
1643 domain
->iommu_superpage
= 0;
1645 domain
->nid
= iommu
->node
;
1647 /* always allocate the top pgd */
1648 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page(domain
->nid
);
1651 __iommu_flush_cache(iommu
, domain
->pgd
, PAGE_SIZE
);
1655 static void domain_exit(struct dmar_domain
*domain
)
1657 struct dmar_drhd_unit
*drhd
;
1658 struct intel_iommu
*iommu
;
1659 struct page
*freelist
= NULL
;
1661 /* Domain 0 is reserved, so dont process it */
1665 /* Flush any lazy unmaps that may reference this domain */
1666 if (!intel_iommu_strict
)
1667 flush_unmaps_timeout(0);
1669 /* remove associated devices */
1670 domain_remove_dev_info(domain
);
1673 put_iova_domain(&domain
->iovad
);
1675 freelist
= domain_unmap(domain
, 0, DOMAIN_MAX_PFN(domain
->gaw
));
1677 /* clear attached or cached domains */
1679 for_each_active_iommu(iommu
, drhd
)
1680 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
1681 test_bit(iommu
->seq_id
, domain
->iommu_bmp
))
1682 iommu_detach_domain(domain
, iommu
);
1685 dma_free_pagelist(freelist
);
1687 free_domain_mem(domain
);
1690 static int domain_context_mapping_one(struct dmar_domain
*domain
, int segment
,
1691 u8 bus
, u8 devfn
, int translation
)
1693 struct context_entry
*context
;
1694 unsigned long flags
;
1695 struct intel_iommu
*iommu
;
1696 struct dma_pte
*pgd
;
1698 unsigned long ndomains
;
1701 struct device_domain_info
*info
= NULL
;
1703 pr_debug("Set context mapping for %02x:%02x.%d\n",
1704 bus
, PCI_SLOT(devfn
), PCI_FUNC(devfn
));
1706 BUG_ON(!domain
->pgd
);
1707 BUG_ON(translation
!= CONTEXT_TT_PASS_THROUGH
&&
1708 translation
!= CONTEXT_TT_MULTI_LEVEL
);
1710 iommu
= device_to_iommu(segment
, bus
, devfn
);
1714 context
= device_to_context_entry(iommu
, bus
, devfn
);
1717 spin_lock_irqsave(&iommu
->lock
, flags
);
1718 if (context_present(context
)) {
1719 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1726 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
1727 domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
) {
1730 /* find an available domain id for this device in iommu */
1731 ndomains
= cap_ndoms(iommu
->cap
);
1732 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
1733 if (iommu
->domains
[num
] == domain
) {
1741 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1742 if (num
>= ndomains
) {
1743 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1744 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1748 set_bit(num
, iommu
->domain_ids
);
1749 iommu
->domains
[num
] = domain
;
1753 /* Skip top levels of page tables for
1754 * iommu which has less agaw than default.
1755 * Unnecessary for PT mode.
1757 if (translation
!= CONTEXT_TT_PASS_THROUGH
) {
1758 for (agaw
= domain
->agaw
; agaw
!= iommu
->agaw
; agaw
--) {
1759 pgd
= phys_to_virt(dma_pte_addr(pgd
));
1760 if (!dma_pte_present(pgd
)) {
1761 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1768 context_set_domain_id(context
, id
);
1770 if (translation
!= CONTEXT_TT_PASS_THROUGH
) {
1771 info
= iommu_support_dev_iotlb(domain
, segment
, bus
, devfn
);
1772 translation
= info
? CONTEXT_TT_DEV_IOTLB
:
1773 CONTEXT_TT_MULTI_LEVEL
;
1776 * In pass through mode, AW must be programmed to indicate the largest
1777 * AGAW value supported by hardware. And ASR is ignored by hardware.
1779 if (unlikely(translation
== CONTEXT_TT_PASS_THROUGH
))
1780 context_set_address_width(context
, iommu
->msagaw
);
1782 context_set_address_root(context
, virt_to_phys(pgd
));
1783 context_set_address_width(context
, iommu
->agaw
);
1786 context_set_translation_type(context
, translation
);
1787 context_set_fault_enable(context
);
1788 context_set_present(context
);
1789 domain_flush_cache(domain
, context
, sizeof(*context
));
1792 * It's a non-present to present mapping. If hardware doesn't cache
1793 * non-present entry we only need to flush the write-buffer. If the
1794 * _does_ cache non-present entries, then it does so in the special
1795 * domain #0, which we have to flush:
1797 if (cap_caching_mode(iommu
->cap
)) {
1798 iommu
->flush
.flush_context(iommu
, 0,
1799 (((u16
)bus
) << 8) | devfn
,
1800 DMA_CCMD_MASK_NOBIT
,
1801 DMA_CCMD_DEVICE_INVL
);
1802 iommu
->flush
.flush_iotlb(iommu
, domain
->id
, 0, 0, DMA_TLB_DSI_FLUSH
);
1804 iommu_flush_write_buffer(iommu
);
1806 iommu_enable_dev_iotlb(info
);
1807 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1809 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1810 if (!test_and_set_bit(iommu
->seq_id
, domain
->iommu_bmp
)) {
1811 domain
->iommu_count
++;
1812 if (domain
->iommu_count
== 1)
1813 domain
->nid
= iommu
->node
;
1814 domain_update_iommu_cap(domain
);
1816 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1821 domain_context_mapping(struct dmar_domain
*domain
, struct pci_dev
*pdev
,
1825 struct pci_dev
*tmp
, *parent
;
1827 ret
= domain_context_mapping_one(domain
, pci_domain_nr(pdev
->bus
),
1828 pdev
->bus
->number
, pdev
->devfn
,
1833 /* dependent device mapping */
1834 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1837 /* Secondary interface's bus number and devfn 0 */
1838 parent
= pdev
->bus
->self
;
1839 while (parent
!= tmp
) {
1840 ret
= domain_context_mapping_one(domain
,
1841 pci_domain_nr(parent
->bus
),
1842 parent
->bus
->number
,
1843 parent
->devfn
, translation
);
1846 parent
= parent
->bus
->self
;
1848 if (pci_is_pcie(tmp
)) /* this is a PCIe-to-PCI bridge */
1849 return domain_context_mapping_one(domain
,
1850 pci_domain_nr(tmp
->subordinate
),
1851 tmp
->subordinate
->number
, 0,
1853 else /* this is a legacy PCI bridge */
1854 return domain_context_mapping_one(domain
,
1855 pci_domain_nr(tmp
->bus
),
1861 static int domain_context_mapped(struct pci_dev
*pdev
)
1864 struct pci_dev
*tmp
, *parent
;
1865 struct intel_iommu
*iommu
;
1867 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
1872 ret
= device_context_mapped(iommu
, pdev
->bus
->number
, pdev
->devfn
);
1875 /* dependent device mapping */
1876 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1879 /* Secondary interface's bus number and devfn 0 */
1880 parent
= pdev
->bus
->self
;
1881 while (parent
!= tmp
) {
1882 ret
= device_context_mapped(iommu
, parent
->bus
->number
,
1886 parent
= parent
->bus
->self
;
1888 if (pci_is_pcie(tmp
))
1889 return device_context_mapped(iommu
, tmp
->subordinate
->number
,
1892 return device_context_mapped(iommu
, tmp
->bus
->number
,
1896 /* Returns a number of VTD pages, but aligned to MM page size */
1897 static inline unsigned long aligned_nrpages(unsigned long host_addr
,
1900 host_addr
&= ~PAGE_MASK
;
1901 return PAGE_ALIGN(host_addr
+ size
) >> VTD_PAGE_SHIFT
;
1904 /* Return largest possible superpage level for a given mapping */
1905 static inline int hardware_largepage_caps(struct dmar_domain
*domain
,
1906 unsigned long iov_pfn
,
1907 unsigned long phy_pfn
,
1908 unsigned long pages
)
1910 int support
, level
= 1;
1911 unsigned long pfnmerge
;
1913 support
= domain
->iommu_superpage
;
1915 /* To use a large page, the virtual *and* physical addresses
1916 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1917 of them will mean we have to use smaller pages. So just
1918 merge them and check both at once. */
1919 pfnmerge
= iov_pfn
| phy_pfn
;
1921 while (support
&& !(pfnmerge
& ~VTD_STRIDE_MASK
)) {
1922 pages
>>= VTD_STRIDE_SHIFT
;
1925 pfnmerge
>>= VTD_STRIDE_SHIFT
;
1932 static int __domain_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
1933 struct scatterlist
*sg
, unsigned long phys_pfn
,
1934 unsigned long nr_pages
, int prot
)
1936 struct dma_pte
*first_pte
= NULL
, *pte
= NULL
;
1937 phys_addr_t
uninitialized_var(pteval
);
1938 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
1939 unsigned long sg_res
;
1940 unsigned int largepage_lvl
= 0;
1941 unsigned long lvl_pages
= 0;
1943 BUG_ON(addr_width
< BITS_PER_LONG
&& (iov_pfn
+ nr_pages
- 1) >> addr_width
);
1945 if ((prot
& (DMA_PTE_READ
|DMA_PTE_WRITE
)) == 0)
1948 prot
&= DMA_PTE_READ
| DMA_PTE_WRITE
| DMA_PTE_SNP
;
1953 sg_res
= nr_pages
+ 1;
1954 pteval
= ((phys_addr_t
)phys_pfn
<< VTD_PAGE_SHIFT
) | prot
;
1957 while (nr_pages
> 0) {
1961 sg_res
= aligned_nrpages(sg
->offset
, sg
->length
);
1962 sg
->dma_address
= ((dma_addr_t
)iov_pfn
<< VTD_PAGE_SHIFT
) + sg
->offset
;
1963 sg
->dma_length
= sg
->length
;
1964 pteval
= page_to_phys(sg_page(sg
)) | prot
;
1965 phys_pfn
= pteval
>> VTD_PAGE_SHIFT
;
1969 largepage_lvl
= hardware_largepage_caps(domain
, iov_pfn
, phys_pfn
, sg_res
);
1971 first_pte
= pte
= pfn_to_dma_pte(domain
, iov_pfn
, &largepage_lvl
);
1974 /* It is large page*/
1975 if (largepage_lvl
> 1) {
1976 pteval
|= DMA_PTE_LARGE_PAGE
;
1977 /* Ensure that old small page tables are removed to make room
1978 for superpage, if they exist. */
1979 dma_pte_clear_range(domain
, iov_pfn
,
1980 iov_pfn
+ lvl_to_nr_pages(largepage_lvl
) - 1);
1981 dma_pte_free_pagetable(domain
, iov_pfn
,
1982 iov_pfn
+ lvl_to_nr_pages(largepage_lvl
) - 1);
1984 pteval
&= ~(uint64_t)DMA_PTE_LARGE_PAGE
;
1988 /* We don't need lock here, nobody else
1989 * touches the iova range
1991 tmp
= cmpxchg64_local(&pte
->val
, 0ULL, pteval
);
1993 static int dumps
= 5;
1994 printk(KERN_CRIT
"ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1995 iov_pfn
, tmp
, (unsigned long long)pteval
);
1998 debug_dma_dump_mappings(NULL
);
2003 lvl_pages
= lvl_to_nr_pages(largepage_lvl
);
2005 BUG_ON(nr_pages
< lvl_pages
);
2006 BUG_ON(sg_res
< lvl_pages
);
2008 nr_pages
-= lvl_pages
;
2009 iov_pfn
+= lvl_pages
;
2010 phys_pfn
+= lvl_pages
;
2011 pteval
+= lvl_pages
* VTD_PAGE_SIZE
;
2012 sg_res
-= lvl_pages
;
2014 /* If the next PTE would be the first in a new page, then we
2015 need to flush the cache on the entries we've just written.
2016 And then we'll need to recalculate 'pte', so clear it and
2017 let it get set again in the if (!pte) block above.
2019 If we're done (!nr_pages) we need to flush the cache too.
2021 Also if we've been setting superpages, we may need to
2022 recalculate 'pte' and switch back to smaller pages for the
2023 end of the mapping, if the trailing size is not enough to
2024 use another superpage (i.e. sg_res < lvl_pages). */
2026 if (!nr_pages
|| first_pte_in_page(pte
) ||
2027 (largepage_lvl
> 1 && sg_res
< lvl_pages
)) {
2028 domain_flush_cache(domain
, first_pte
,
2029 (void *)pte
- (void *)first_pte
);
2033 if (!sg_res
&& nr_pages
)
2039 static inline int domain_sg_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
2040 struct scatterlist
*sg
, unsigned long nr_pages
,
2043 return __domain_mapping(domain
, iov_pfn
, sg
, 0, nr_pages
, prot
);
2046 static inline int domain_pfn_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
2047 unsigned long phys_pfn
, unsigned long nr_pages
,
2050 return __domain_mapping(domain
, iov_pfn
, NULL
, phys_pfn
, nr_pages
, prot
);
2053 static void iommu_detach_dev(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
2058 clear_context_table(iommu
, bus
, devfn
);
2059 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
2060 DMA_CCMD_GLOBAL_INVL
);
2061 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
2064 static inline void unlink_domain_info(struct device_domain_info
*info
)
2066 assert_spin_locked(&device_domain_lock
);
2067 list_del(&info
->link
);
2068 list_del(&info
->global
);
2070 info
->dev
->dev
.archdata
.iommu
= NULL
;
2073 static void domain_remove_dev_info(struct dmar_domain
*domain
)
2075 struct device_domain_info
*info
;
2076 unsigned long flags
, flags2
;
2077 struct intel_iommu
*iommu
;
2079 spin_lock_irqsave(&device_domain_lock
, flags
);
2080 while (!list_empty(&domain
->devices
)) {
2081 info
= list_entry(domain
->devices
.next
,
2082 struct device_domain_info
, link
);
2083 unlink_domain_info(info
);
2084 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2086 iommu_disable_dev_iotlb(info
);
2087 iommu
= device_to_iommu(info
->segment
, info
->bus
, info
->devfn
);
2088 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
2090 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) {
2091 iommu_detach_dependent_devices(iommu
, info
->dev
);
2092 /* clear this iommu in iommu_bmp, update iommu count
2095 spin_lock_irqsave(&domain
->iommu_lock
, flags2
);
2096 if (test_and_clear_bit(iommu
->seq_id
,
2097 domain
->iommu_bmp
)) {
2098 domain
->iommu_count
--;
2099 domain_update_iommu_cap(domain
);
2101 spin_unlock_irqrestore(&domain
->iommu_lock
, flags2
);
2104 free_devinfo_mem(info
);
2105 spin_lock_irqsave(&device_domain_lock
, flags
);
2107 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2112 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
2114 static struct dmar_domain
*
2115 find_domain(struct pci_dev
*pdev
)
2117 struct device_domain_info
*info
;
2119 /* No lock here, assumes no domain exit in normal case */
2120 info
= pdev
->dev
.archdata
.iommu
;
2122 return info
->domain
;
2126 static inline struct dmar_domain
*
2127 dmar_search_domain_by_dev_info(int segment
, int bus
, int devfn
)
2129 struct device_domain_info
*info
;
2131 list_for_each_entry(info
, &device_domain_list
, global
)
2132 if (info
->segment
== segment
&& info
->bus
== bus
&&
2133 info
->devfn
== devfn
)
2134 return info
->domain
;
2139 static int dmar_insert_dev_info(int segment
, int bus
, int devfn
,
2140 struct pci_dev
*dev
, struct dmar_domain
**domp
)
2142 struct dmar_domain
*found
, *domain
= *domp
;
2143 struct device_domain_info
*info
;
2144 unsigned long flags
;
2146 info
= alloc_devinfo_mem();
2150 info
->segment
= segment
;
2152 info
->devfn
= devfn
;
2154 info
->domain
= domain
;
2156 domain
->flags
|= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES
;
2158 spin_lock_irqsave(&device_domain_lock
, flags
);
2160 found
= find_domain(dev
);
2162 found
= dmar_search_domain_by_dev_info(segment
, bus
, devfn
);
2164 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2165 free_devinfo_mem(info
);
2166 if (found
!= domain
) {
2167 domain_exit(domain
);
2171 list_add(&info
->link
, &domain
->devices
);
2172 list_add(&info
->global
, &device_domain_list
);
2174 dev
->dev
.archdata
.iommu
= info
;
2175 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2181 /* domain is initialized */
2182 static struct dmar_domain
*get_domain_for_dev(struct pci_dev
*pdev
, int gaw
)
2184 struct dmar_domain
*domain
, *free
= NULL
;
2185 struct intel_iommu
*iommu
;
2186 struct dmar_drhd_unit
*drhd
;
2187 struct pci_dev
*dev_tmp
;
2188 unsigned long flags
;
2189 int bus
= 0, devfn
= 0;
2192 domain
= find_domain(pdev
);
2196 segment
= pci_domain_nr(pdev
->bus
);
2198 dev_tmp
= pci_find_upstream_pcie_bridge(pdev
);
2200 if (pci_is_pcie(dev_tmp
)) {
2201 bus
= dev_tmp
->subordinate
->number
;
2204 bus
= dev_tmp
->bus
->number
;
2205 devfn
= dev_tmp
->devfn
;
2207 spin_lock_irqsave(&device_domain_lock
, flags
);
2208 domain
= dmar_search_domain_by_dev_info(segment
, bus
, devfn
);
2209 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2210 /* pcie-pci bridge already has a domain, uses it */
2215 drhd
= dmar_find_matched_drhd_unit(pdev
);
2217 printk(KERN_ERR
"IOMMU: can't find DMAR for device %s\n",
2221 iommu
= drhd
->iommu
;
2223 /* Allocate and intialize new domain for the device */
2224 domain
= alloc_domain(false);
2227 if (iommu_attach_domain(domain
, iommu
)) {
2228 free_domain_mem(domain
);
2232 if (domain_init(domain
, gaw
))
2235 /* register pcie-to-pci device */
2237 if (dmar_insert_dev_info(segment
, bus
, devfn
, NULL
, &domain
))
2244 if (dmar_insert_dev_info(segment
, pdev
->bus
->number
, pdev
->devfn
,
2245 pdev
, &domain
) == 0)
2250 /* recheck it here, maybe others set it */
2251 return find_domain(pdev
);
2254 static int iommu_identity_mapping
;
2255 #define IDENTMAP_ALL 1
2256 #define IDENTMAP_GFX 2
2257 #define IDENTMAP_AZALIA 4
2259 static int iommu_domain_identity_map(struct dmar_domain
*domain
,
2260 unsigned long long start
,
2261 unsigned long long end
)
2263 unsigned long first_vpfn
= start
>> VTD_PAGE_SHIFT
;
2264 unsigned long last_vpfn
= end
>> VTD_PAGE_SHIFT
;
2266 if (!reserve_iova(&domain
->iovad
, dma_to_mm_pfn(first_vpfn
),
2267 dma_to_mm_pfn(last_vpfn
))) {
2268 printk(KERN_ERR
"IOMMU: reserve iova failed\n");
2272 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2273 start
, end
, domain
->id
);
2275 * RMRR range might have overlap with physical memory range,
2278 dma_pte_clear_range(domain
, first_vpfn
, last_vpfn
);
2280 return domain_pfn_mapping(domain
, first_vpfn
, first_vpfn
,
2281 last_vpfn
- first_vpfn
+ 1,
2282 DMA_PTE_READ
|DMA_PTE_WRITE
);
2285 static int iommu_prepare_identity_map(struct pci_dev
*pdev
,
2286 unsigned long long start
,
2287 unsigned long long end
)
2289 struct dmar_domain
*domain
;
2292 domain
= get_domain_for_dev(pdev
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
2296 /* For _hardware_ passthrough, don't bother. But for software
2297 passthrough, we do it anyway -- it may indicate a memory
2298 range which is reserved in E820, so which didn't get set
2299 up to start with in si_domain */
2300 if (domain
== si_domain
&& hw_pass_through
) {
2301 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2302 pci_name(pdev
), start
, end
);
2307 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2308 pci_name(pdev
), start
, end
);
2311 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2312 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2313 dmi_get_system_info(DMI_BIOS_VENDOR
),
2314 dmi_get_system_info(DMI_BIOS_VERSION
),
2315 dmi_get_system_info(DMI_PRODUCT_VERSION
));
2320 if (end
>> agaw_to_width(domain
->agaw
)) {
2321 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2322 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2323 agaw_to_width(domain
->agaw
),
2324 dmi_get_system_info(DMI_BIOS_VENDOR
),
2325 dmi_get_system_info(DMI_BIOS_VERSION
),
2326 dmi_get_system_info(DMI_PRODUCT_VERSION
));
2331 ret
= iommu_domain_identity_map(domain
, start
, end
);
2335 /* context entry init */
2336 ret
= domain_context_mapping(domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
2343 domain_exit(domain
);
2347 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit
*rmrr
,
2348 struct pci_dev
*pdev
)
2350 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2352 return iommu_prepare_identity_map(pdev
, rmrr
->base_address
,
2356 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2357 static inline void iommu_prepare_isa(void)
2359 struct pci_dev
*pdev
;
2362 pdev
= pci_get_class(PCI_CLASS_BRIDGE_ISA
<< 8, NULL
);
2366 printk(KERN_INFO
"IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2367 ret
= iommu_prepare_identity_map(pdev
, 0, 16*1024*1024 - 1);
2370 printk(KERN_ERR
"IOMMU: Failed to create 0-16MiB identity map; "
2371 "floppy might not work\n");
2375 static inline void iommu_prepare_isa(void)
2379 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2381 static int md_domain_init(struct dmar_domain
*domain
, int guest_width
);
2383 static int __init
si_domain_init(int hw
)
2385 struct dmar_drhd_unit
*drhd
;
2386 struct intel_iommu
*iommu
;
2389 si_domain
= alloc_domain(false);
2393 si_domain
->flags
= DOMAIN_FLAG_STATIC_IDENTITY
;
2395 for_each_active_iommu(iommu
, drhd
) {
2396 ret
= iommu_attach_domain(si_domain
, iommu
);
2398 domain_exit(si_domain
);
2403 if (md_domain_init(si_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
2404 domain_exit(si_domain
);
2408 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2414 for_each_online_node(nid
) {
2415 unsigned long start_pfn
, end_pfn
;
2418 for_each_mem_pfn_range(i
, nid
, &start_pfn
, &end_pfn
, NULL
) {
2419 ret
= iommu_domain_identity_map(si_domain
,
2420 PFN_PHYS(start_pfn
), PFN_PHYS(end_pfn
));
2429 static int identity_mapping(struct pci_dev
*pdev
)
2431 struct device_domain_info
*info
;
2433 if (likely(!iommu_identity_mapping
))
2436 info
= pdev
->dev
.archdata
.iommu
;
2437 if (info
&& info
!= DUMMY_DEVICE_DOMAIN_INFO
)
2438 return (info
->domain
== si_domain
);
2443 static int domain_add_dev_info(struct dmar_domain
*domain
,
2444 struct pci_dev
*pdev
,
2447 struct device_domain_info
*info
;
2448 unsigned long flags
;
2451 info
= alloc_devinfo_mem();
2455 info
->segment
= pci_domain_nr(pdev
->bus
);
2456 info
->bus
= pdev
->bus
->number
;
2457 info
->devfn
= pdev
->devfn
;
2459 info
->domain
= domain
;
2461 spin_lock_irqsave(&device_domain_lock
, flags
);
2462 list_add(&info
->link
, &domain
->devices
);
2463 list_add(&info
->global
, &device_domain_list
);
2464 pdev
->dev
.archdata
.iommu
= info
;
2465 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2467 ret
= domain_context_mapping(domain
, pdev
, translation
);
2469 spin_lock_irqsave(&device_domain_lock
, flags
);
2470 unlink_domain_info(info
);
2471 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2472 free_devinfo_mem(info
);
2479 static bool device_has_rmrr(struct pci_dev
*dev
)
2481 struct dmar_rmrr_unit
*rmrr
;
2482 struct pci_dev
*tmp
;
2486 for_each_rmrr_units(rmrr
) {
2488 * Return TRUE if this RMRR contains the device that
2491 for_each_active_dev_scope(rmrr
->devices
,
2492 rmrr
->devices_cnt
, i
, tmp
)
2502 static int iommu_should_identity_map(struct pci_dev
*pdev
, int startup
)
2506 * We want to prevent any device associated with an RMRR from
2507 * getting placed into the SI Domain. This is done because
2508 * problems exist when devices are moved in and out of domains
2509 * and their respective RMRR info is lost. We exempt USB devices
2510 * from this process due to their usage of RMRRs that are known
2511 * to not be needed after BIOS hand-off to OS.
2513 if (device_has_rmrr(pdev
) &&
2514 (pdev
->class >> 8) != PCI_CLASS_SERIAL_USB
)
2517 if ((iommu_identity_mapping
& IDENTMAP_AZALIA
) && IS_AZALIA(pdev
))
2520 if ((iommu_identity_mapping
& IDENTMAP_GFX
) && IS_GFX_DEVICE(pdev
))
2523 if (!(iommu_identity_mapping
& IDENTMAP_ALL
))
2527 * We want to start off with all devices in the 1:1 domain, and
2528 * take them out later if we find they can't access all of memory.
2530 * However, we can't do this for PCI devices behind bridges,
2531 * because all PCI devices behind the same bridge will end up
2532 * with the same source-id on their transactions.
2534 * Practically speaking, we can't change things around for these
2535 * devices at run-time, because we can't be sure there'll be no
2536 * DMA transactions in flight for any of their siblings.
2538 * So PCI devices (unless they're on the root bus) as well as
2539 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2540 * the 1:1 domain, just in _case_ one of their siblings turns out
2541 * not to be able to map all of memory.
2543 if (!pci_is_pcie(pdev
)) {
2544 if (!pci_is_root_bus(pdev
->bus
))
2546 if (pdev
->class >> 8 == PCI_CLASS_BRIDGE_PCI
)
2548 } else if (pci_pcie_type(pdev
) == PCI_EXP_TYPE_PCI_BRIDGE
)
2552 * At boot time, we don't yet know if devices will be 64-bit capable.
2553 * Assume that they will -- if they turn out not to be, then we can
2554 * take them out of the 1:1 domain later.
2558 * If the device's dma_mask is less than the system's memory
2559 * size then this is not a candidate for identity mapping.
2561 u64 dma_mask
= pdev
->dma_mask
;
2563 if (pdev
->dev
.coherent_dma_mask
&&
2564 pdev
->dev
.coherent_dma_mask
< dma_mask
)
2565 dma_mask
= pdev
->dev
.coherent_dma_mask
;
2567 return dma_mask
>= dma_get_required_mask(&pdev
->dev
);
2573 static int __init
iommu_prepare_static_identity_mapping(int hw
)
2575 struct pci_dev
*pdev
= NULL
;
2578 ret
= si_domain_init(hw
);
2582 for_each_pci_dev(pdev
) {
2583 if (iommu_should_identity_map(pdev
, 1)) {
2584 ret
= domain_add_dev_info(si_domain
, pdev
,
2585 hw
? CONTEXT_TT_PASS_THROUGH
:
2586 CONTEXT_TT_MULTI_LEVEL
);
2588 /* device not associated with an iommu */
2593 pr_info("IOMMU: %s identity mapping for device %s\n",
2594 hw
? "hardware" : "software", pci_name(pdev
));
2601 static int __init
init_dmars(void)
2603 struct dmar_drhd_unit
*drhd
;
2604 struct dmar_rmrr_unit
*rmrr
;
2605 struct pci_dev
*pdev
;
2606 struct intel_iommu
*iommu
;
2612 * initialize and program root entry to not present
2615 for_each_drhd_unit(drhd
) {
2617 * lock not needed as this is only incremented in the single
2618 * threaded kernel __init code path all other access are read
2621 if (g_num_of_iommus
< IOMMU_UNITS_SUPPORTED
) {
2625 printk_once(KERN_ERR
"intel-iommu: exceeded %d IOMMUs\n",
2626 IOMMU_UNITS_SUPPORTED
);
2629 g_iommus
= kcalloc(g_num_of_iommus
, sizeof(struct intel_iommu
*),
2632 printk(KERN_ERR
"Allocating global iommu array failed\n");
2637 deferred_flush
= kzalloc(g_num_of_iommus
*
2638 sizeof(struct deferred_flush_tables
), GFP_KERNEL
);
2639 if (!deferred_flush
) {
2644 for_each_active_iommu(iommu
, drhd
) {
2645 g_iommus
[iommu
->seq_id
] = iommu
;
2647 ret
= iommu_init_domains(iommu
);
2653 * we could share the same root & context tables
2654 * among all IOMMU's. Need to Split it later.
2656 ret
= iommu_alloc_root_entry(iommu
);
2658 printk(KERN_ERR
"IOMMU: allocate root entry failed\n");
2661 if (!ecap_pass_through(iommu
->ecap
))
2662 hw_pass_through
= 0;
2666 * Start from the sane iommu hardware state.
2668 for_each_active_iommu(iommu
, drhd
) {
2670 * If the queued invalidation is already initialized by us
2671 * (for example, while enabling interrupt-remapping) then
2672 * we got the things already rolling from a sane state.
2678 * Clear any previous faults.
2680 dmar_fault(-1, iommu
);
2682 * Disable queued invalidation if supported and already enabled
2683 * before OS handover.
2685 dmar_disable_qi(iommu
);
2688 for_each_active_iommu(iommu
, drhd
) {
2689 if (dmar_enable_qi(iommu
)) {
2691 * Queued Invalidate not enabled, use Register Based
2694 iommu
->flush
.flush_context
= __iommu_flush_context
;
2695 iommu
->flush
.flush_iotlb
= __iommu_flush_iotlb
;
2696 printk(KERN_INFO
"IOMMU %d 0x%Lx: using Register based "
2699 (unsigned long long)drhd
->reg_base_addr
);
2701 iommu
->flush
.flush_context
= qi_flush_context
;
2702 iommu
->flush
.flush_iotlb
= qi_flush_iotlb
;
2703 printk(KERN_INFO
"IOMMU %d 0x%Lx: using Queued "
2706 (unsigned long long)drhd
->reg_base_addr
);
2710 if (iommu_pass_through
)
2711 iommu_identity_mapping
|= IDENTMAP_ALL
;
2713 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2714 iommu_identity_mapping
|= IDENTMAP_GFX
;
2717 check_tylersburg_isoch();
2720 * If pass through is not set or not enabled, setup context entries for
2721 * identity mappings for rmrr, gfx, and isa and may fall back to static
2722 * identity mapping if iommu_identity_mapping is set.
2724 if (iommu_identity_mapping
) {
2725 ret
= iommu_prepare_static_identity_mapping(hw_pass_through
);
2727 printk(KERN_CRIT
"Failed to setup IOMMU pass-through\n");
2733 * for each dev attached to rmrr
2735 * locate drhd for dev, alloc domain for dev
2736 * allocate free domain
2737 * allocate page table entries for rmrr
2738 * if context not allocated for bus
2739 * allocate and init context
2740 * set present in root table for this bus
2741 * init context with domain, translation etc
2745 printk(KERN_INFO
"IOMMU: Setting RMRR:\n");
2746 for_each_rmrr_units(rmrr
) {
2747 /* some BIOS lists non-exist devices in DMAR table. */
2748 for_each_active_dev_scope(rmrr
->devices
, rmrr
->devices_cnt
,
2750 ret
= iommu_prepare_rmrr_dev(rmrr
, pdev
);
2753 "IOMMU: mapping reserved region failed\n");
2757 iommu_prepare_isa();
2762 * global invalidate context cache
2763 * global invalidate iotlb
2764 * enable translation
2766 for_each_iommu(iommu
, drhd
) {
2767 if (drhd
->ignored
) {
2769 * we always have to disable PMRs or DMA may fail on
2773 iommu_disable_protect_mem_regions(iommu
);
2777 iommu_flush_write_buffer(iommu
);
2779 ret
= dmar_set_interrupt(iommu
);
2783 iommu_set_root_entry(iommu
);
2785 iommu
->flush
.flush_context(iommu
, 0, 0, 0, DMA_CCMD_GLOBAL_INVL
);
2786 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
2788 ret
= iommu_enable_translation(iommu
);
2792 iommu_disable_protect_mem_regions(iommu
);
2798 for_each_active_iommu(iommu
, drhd
)
2799 free_dmar_iommu(iommu
);
2800 kfree(deferred_flush
);
2807 /* This takes a number of _MM_ pages, not VTD pages */
2808 static struct iova
*intel_alloc_iova(struct device
*dev
,
2809 struct dmar_domain
*domain
,
2810 unsigned long nrpages
, uint64_t dma_mask
)
2812 struct pci_dev
*pdev
= to_pci_dev(dev
);
2813 struct iova
*iova
= NULL
;
2815 /* Restrict dma_mask to the width that the iommu can handle */
2816 dma_mask
= min_t(uint64_t, DOMAIN_MAX_ADDR(domain
->gaw
), dma_mask
);
2818 if (!dmar_forcedac
&& dma_mask
> DMA_BIT_MASK(32)) {
2820 * First try to allocate an io virtual address in
2821 * DMA_BIT_MASK(32) and if that fails then try allocating
2824 iova
= alloc_iova(&domain
->iovad
, nrpages
,
2825 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2829 iova
= alloc_iova(&domain
->iovad
, nrpages
, IOVA_PFN(dma_mask
), 1);
2830 if (unlikely(!iova
)) {
2831 printk(KERN_ERR
"Allocating %ld-page iova for %s failed",
2832 nrpages
, pci_name(pdev
));
2839 static struct dmar_domain
*__get_valid_domain_for_dev(struct pci_dev
*pdev
)
2841 struct dmar_domain
*domain
;
2844 domain
= get_domain_for_dev(pdev
,
2845 DEFAULT_DOMAIN_ADDRESS_WIDTH
);
2848 "Allocating domain for %s failed", pci_name(pdev
));
2852 /* make sure context mapping is ok */
2853 if (unlikely(!domain_context_mapped(pdev
))) {
2854 ret
= domain_context_mapping(domain
, pdev
,
2855 CONTEXT_TT_MULTI_LEVEL
);
2858 "Domain context map for %s failed",
2867 static inline struct dmar_domain
*get_valid_domain_for_dev(struct pci_dev
*dev
)
2869 struct device_domain_info
*info
;
2871 /* No lock here, assumes no domain exit in normal case */
2872 info
= dev
->dev
.archdata
.iommu
;
2874 return info
->domain
;
2876 return __get_valid_domain_for_dev(dev
);
2879 static int iommu_dummy(struct pci_dev
*pdev
)
2881 return pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
;
2884 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2885 static int iommu_no_mapping(struct device
*dev
)
2887 struct pci_dev
*pdev
;
2890 if (unlikely(!dev_is_pci(dev
)))
2893 pdev
= to_pci_dev(dev
);
2894 if (iommu_dummy(pdev
))
2897 if (!iommu_identity_mapping
)
2900 found
= identity_mapping(pdev
);
2902 if (iommu_should_identity_map(pdev
, 0))
2906 * 32 bit DMA is removed from si_domain and fall back
2907 * to non-identity mapping.
2909 domain_remove_one_dev_info(si_domain
, pdev
);
2910 printk(KERN_INFO
"32bit %s uses non-identity mapping\n",
2916 * In case of a detached 64 bit DMA device from vm, the device
2917 * is put into si_domain for identity mapping.
2919 if (iommu_should_identity_map(pdev
, 0)) {
2921 ret
= domain_add_dev_info(si_domain
, pdev
,
2923 CONTEXT_TT_PASS_THROUGH
:
2924 CONTEXT_TT_MULTI_LEVEL
);
2926 printk(KERN_INFO
"64bit %s uses identity mapping\n",
2936 static dma_addr_t
__intel_map_single(struct device
*hwdev
, phys_addr_t paddr
,
2937 size_t size
, int dir
, u64 dma_mask
)
2939 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2940 struct dmar_domain
*domain
;
2941 phys_addr_t start_paddr
;
2945 struct intel_iommu
*iommu
;
2946 unsigned long paddr_pfn
= paddr
>> PAGE_SHIFT
;
2948 BUG_ON(dir
== DMA_NONE
);
2950 if (iommu_no_mapping(hwdev
))
2953 domain
= get_valid_domain_for_dev(pdev
);
2957 iommu
= domain_get_iommu(domain
);
2958 size
= aligned_nrpages(paddr
, size
);
2960 iova
= intel_alloc_iova(hwdev
, domain
, dma_to_mm_pfn(size
), dma_mask
);
2965 * Check if DMAR supports zero-length reads on write only
2968 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
2969 !cap_zlr(iommu
->cap
))
2970 prot
|= DMA_PTE_READ
;
2971 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
2972 prot
|= DMA_PTE_WRITE
;
2974 * paddr - (paddr + size) might be partial page, we should map the whole
2975 * page. Note: if two part of one page are separately mapped, we
2976 * might have two guest_addr mapping to the same host paddr, but this
2977 * is not a big problem
2979 ret
= domain_pfn_mapping(domain
, mm_to_dma_pfn(iova
->pfn_lo
),
2980 mm_to_dma_pfn(paddr_pfn
), size
, prot
);
2984 /* it's a non-present to present mapping. Only flush if caching mode */
2985 if (cap_caching_mode(iommu
->cap
))
2986 iommu_flush_iotlb_psi(iommu
, domain
->id
, mm_to_dma_pfn(iova
->pfn_lo
), size
, 0, 1);
2988 iommu_flush_write_buffer(iommu
);
2990 start_paddr
= (phys_addr_t
)iova
->pfn_lo
<< PAGE_SHIFT
;
2991 start_paddr
+= paddr
& ~PAGE_MASK
;
2996 __free_iova(&domain
->iovad
, iova
);
2997 printk(KERN_ERR
"Device %s request: %zx@%llx dir %d --- failed\n",
2998 pci_name(pdev
), size
, (unsigned long long)paddr
, dir
);
3002 static dma_addr_t
intel_map_page(struct device
*dev
, struct page
*page
,
3003 unsigned long offset
, size_t size
,
3004 enum dma_data_direction dir
,
3005 struct dma_attrs
*attrs
)
3007 return __intel_map_single(dev
, page_to_phys(page
) + offset
, size
,
3008 dir
, to_pci_dev(dev
)->dma_mask
);
3011 static void flush_unmaps(void)
3017 /* just flush them all */
3018 for (i
= 0; i
< g_num_of_iommus
; i
++) {
3019 struct intel_iommu
*iommu
= g_iommus
[i
];
3023 if (!deferred_flush
[i
].next
)
3026 /* In caching mode, global flushes turn emulation expensive */
3027 if (!cap_caching_mode(iommu
->cap
))
3028 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3029 DMA_TLB_GLOBAL_FLUSH
);
3030 for (j
= 0; j
< deferred_flush
[i
].next
; j
++) {
3032 struct iova
*iova
= deferred_flush
[i
].iova
[j
];
3033 struct dmar_domain
*domain
= deferred_flush
[i
].domain
[j
];
3035 /* On real hardware multiple invalidations are expensive */
3036 if (cap_caching_mode(iommu
->cap
))
3037 iommu_flush_iotlb_psi(iommu
, domain
->id
,
3038 iova
->pfn_lo
, iova
->pfn_hi
- iova
->pfn_lo
+ 1,
3039 !deferred_flush
[i
].freelist
[j
], 0);
3041 mask
= ilog2(mm_to_dma_pfn(iova
->pfn_hi
- iova
->pfn_lo
+ 1));
3042 iommu_flush_dev_iotlb(deferred_flush
[i
].domain
[j
],
3043 (uint64_t)iova
->pfn_lo
<< PAGE_SHIFT
, mask
);
3045 __free_iova(&deferred_flush
[i
].domain
[j
]->iovad
, iova
);
3046 if (deferred_flush
[i
].freelist
[j
])
3047 dma_free_pagelist(deferred_flush
[i
].freelist
[j
]);
3049 deferred_flush
[i
].next
= 0;
3055 static void flush_unmaps_timeout(unsigned long data
)
3057 unsigned long flags
;
3059 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
3061 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
3064 static void add_unmap(struct dmar_domain
*dom
, struct iova
*iova
, struct page
*freelist
)
3066 unsigned long flags
;
3068 struct intel_iommu
*iommu
;
3070 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
3071 if (list_size
== HIGH_WATER_MARK
)
3074 iommu
= domain_get_iommu(dom
);
3075 iommu_id
= iommu
->seq_id
;
3077 next
= deferred_flush
[iommu_id
].next
;
3078 deferred_flush
[iommu_id
].domain
[next
] = dom
;
3079 deferred_flush
[iommu_id
].iova
[next
] = iova
;
3080 deferred_flush
[iommu_id
].freelist
[next
] = freelist
;
3081 deferred_flush
[iommu_id
].next
++;
3084 mod_timer(&unmap_timer
, jiffies
+ msecs_to_jiffies(10));
3088 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
3091 static void intel_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
3092 size_t size
, enum dma_data_direction dir
,
3093 struct dma_attrs
*attrs
)
3095 struct pci_dev
*pdev
= to_pci_dev(dev
);
3096 struct dmar_domain
*domain
;
3097 unsigned long start_pfn
, last_pfn
;
3099 struct intel_iommu
*iommu
;
3100 struct page
*freelist
;
3102 if (iommu_no_mapping(dev
))
3105 domain
= find_domain(pdev
);
3108 iommu
= domain_get_iommu(domain
);
3110 iova
= find_iova(&domain
->iovad
, IOVA_PFN(dev_addr
));
3111 if (WARN_ONCE(!iova
, "Driver unmaps unmatched page at PFN %llx\n",
3112 (unsigned long long)dev_addr
))
3115 start_pfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3116 last_pfn
= mm_to_dma_pfn(iova
->pfn_hi
+ 1) - 1;
3118 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3119 pci_name(pdev
), start_pfn
, last_pfn
);
3121 freelist
= domain_unmap(domain
, start_pfn
, last_pfn
);
3123 if (intel_iommu_strict
) {
3124 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_pfn
,
3125 last_pfn
- start_pfn
+ 1, !freelist
, 0);
3127 __free_iova(&domain
->iovad
, iova
);
3128 dma_free_pagelist(freelist
);
3130 add_unmap(domain
, iova
, freelist
);
3132 * queue up the release of the unmap to save the 1/6th of the
3133 * cpu used up by the iotlb flush operation...
3138 static void *intel_alloc_coherent(struct device
*hwdev
, size_t size
,
3139 dma_addr_t
*dma_handle
, gfp_t flags
,
3140 struct dma_attrs
*attrs
)
3145 size
= PAGE_ALIGN(size
);
3146 order
= get_order(size
);
3148 if (!iommu_no_mapping(hwdev
))
3149 flags
&= ~(GFP_DMA
| GFP_DMA32
);
3150 else if (hwdev
->coherent_dma_mask
< dma_get_required_mask(hwdev
)) {
3151 if (hwdev
->coherent_dma_mask
< DMA_BIT_MASK(32))
3157 vaddr
= (void *)__get_free_pages(flags
, order
);
3160 memset(vaddr
, 0, size
);
3162 *dma_handle
= __intel_map_single(hwdev
, virt_to_bus(vaddr
), size
,
3164 hwdev
->coherent_dma_mask
);
3167 free_pages((unsigned long)vaddr
, order
);
3171 static void intel_free_coherent(struct device
*hwdev
, size_t size
, void *vaddr
,
3172 dma_addr_t dma_handle
, struct dma_attrs
*attrs
)
3176 size
= PAGE_ALIGN(size
);
3177 order
= get_order(size
);
3179 intel_unmap_page(hwdev
, dma_handle
, size
, DMA_BIDIRECTIONAL
, NULL
);
3180 free_pages((unsigned long)vaddr
, order
);
3183 static void intel_unmap_sg(struct device
*hwdev
, struct scatterlist
*sglist
,
3184 int nelems
, enum dma_data_direction dir
,
3185 struct dma_attrs
*attrs
)
3187 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
3188 struct dmar_domain
*domain
;
3189 unsigned long start_pfn
, last_pfn
;
3191 struct intel_iommu
*iommu
;
3192 struct page
*freelist
;
3194 if (iommu_no_mapping(hwdev
))
3197 domain
= find_domain(pdev
);
3200 iommu
= domain_get_iommu(domain
);
3202 iova
= find_iova(&domain
->iovad
, IOVA_PFN(sglist
[0].dma_address
));
3203 if (WARN_ONCE(!iova
, "Driver unmaps unmatched sglist at PFN %llx\n",
3204 (unsigned long long)sglist
[0].dma_address
))
3207 start_pfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3208 last_pfn
= mm_to_dma_pfn(iova
->pfn_hi
+ 1) - 1;
3210 freelist
= domain_unmap(domain
, start_pfn
, last_pfn
);
3212 if (intel_iommu_strict
) {
3213 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_pfn
,
3214 last_pfn
- start_pfn
+ 1, !freelist
, 0);
3216 __free_iova(&domain
->iovad
, iova
);
3217 dma_free_pagelist(freelist
);
3219 add_unmap(domain
, iova
, freelist
);
3221 * queue up the release of the unmap to save the 1/6th of the
3222 * cpu used up by the iotlb flush operation...
3227 static int intel_nontranslate_map_sg(struct device
*hddev
,
3228 struct scatterlist
*sglist
, int nelems
, int dir
)
3231 struct scatterlist
*sg
;
3233 for_each_sg(sglist
, sg
, nelems
, i
) {
3234 BUG_ON(!sg_page(sg
));
3235 sg
->dma_address
= page_to_phys(sg_page(sg
)) + sg
->offset
;
3236 sg
->dma_length
= sg
->length
;
3241 static int intel_map_sg(struct device
*hwdev
, struct scatterlist
*sglist
, int nelems
,
3242 enum dma_data_direction dir
, struct dma_attrs
*attrs
)
3245 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
3246 struct dmar_domain
*domain
;
3249 struct iova
*iova
= NULL
;
3251 struct scatterlist
*sg
;
3252 unsigned long start_vpfn
;
3253 struct intel_iommu
*iommu
;
3255 BUG_ON(dir
== DMA_NONE
);
3256 if (iommu_no_mapping(hwdev
))
3257 return intel_nontranslate_map_sg(hwdev
, sglist
, nelems
, dir
);
3259 domain
= get_valid_domain_for_dev(pdev
);
3263 iommu
= domain_get_iommu(domain
);
3265 for_each_sg(sglist
, sg
, nelems
, i
)
3266 size
+= aligned_nrpages(sg
->offset
, sg
->length
);
3268 iova
= intel_alloc_iova(hwdev
, domain
, dma_to_mm_pfn(size
),
3271 sglist
->dma_length
= 0;
3276 * Check if DMAR supports zero-length reads on write only
3279 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
3280 !cap_zlr(iommu
->cap
))
3281 prot
|= DMA_PTE_READ
;
3282 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
3283 prot
|= DMA_PTE_WRITE
;
3285 start_vpfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3287 ret
= domain_sg_mapping(domain
, start_vpfn
, sglist
, size
, prot
);
3288 if (unlikely(ret
)) {
3289 /* clear the page */
3290 dma_pte_clear_range(domain
, start_vpfn
,
3291 start_vpfn
+ size
- 1);
3292 /* free page tables */
3293 dma_pte_free_pagetable(domain
, start_vpfn
,
3294 start_vpfn
+ size
- 1);
3296 __free_iova(&domain
->iovad
, iova
);
3300 /* it's a non-present to present mapping. Only flush if caching mode */
3301 if (cap_caching_mode(iommu
->cap
))
3302 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_vpfn
, size
, 0, 1);
3304 iommu_flush_write_buffer(iommu
);
3309 static int intel_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
3314 struct dma_map_ops intel_dma_ops
= {
3315 .alloc
= intel_alloc_coherent
,
3316 .free
= intel_free_coherent
,
3317 .map_sg
= intel_map_sg
,
3318 .unmap_sg
= intel_unmap_sg
,
3319 .map_page
= intel_map_page
,
3320 .unmap_page
= intel_unmap_page
,
3321 .mapping_error
= intel_mapping_error
,
3324 static inline int iommu_domain_cache_init(void)
3328 iommu_domain_cache
= kmem_cache_create("iommu_domain",
3329 sizeof(struct dmar_domain
),
3334 if (!iommu_domain_cache
) {
3335 printk(KERN_ERR
"Couldn't create iommu_domain cache\n");
3342 static inline int iommu_devinfo_cache_init(void)
3346 iommu_devinfo_cache
= kmem_cache_create("iommu_devinfo",
3347 sizeof(struct device_domain_info
),
3351 if (!iommu_devinfo_cache
) {
3352 printk(KERN_ERR
"Couldn't create devinfo cache\n");
3359 static inline int iommu_iova_cache_init(void)
3363 iommu_iova_cache
= kmem_cache_create("iommu_iova",
3364 sizeof(struct iova
),
3368 if (!iommu_iova_cache
) {
3369 printk(KERN_ERR
"Couldn't create iova cache\n");
3376 static int __init
iommu_init_mempool(void)
3379 ret
= iommu_iova_cache_init();
3383 ret
= iommu_domain_cache_init();
3387 ret
= iommu_devinfo_cache_init();
3391 kmem_cache_destroy(iommu_domain_cache
);
3393 kmem_cache_destroy(iommu_iova_cache
);
3398 static void __init
iommu_exit_mempool(void)
3400 kmem_cache_destroy(iommu_devinfo_cache
);
3401 kmem_cache_destroy(iommu_domain_cache
);
3402 kmem_cache_destroy(iommu_iova_cache
);
3406 static void quirk_ioat_snb_local_iommu(struct pci_dev
*pdev
)
3408 struct dmar_drhd_unit
*drhd
;
3412 /* We know that this device on this chipset has its own IOMMU.
3413 * If we find it under a different IOMMU, then the BIOS is lying
3414 * to us. Hope that the IOMMU for this device is actually
3415 * disabled, and it needs no translation...
3417 rc
= pci_bus_read_config_dword(pdev
->bus
, PCI_DEVFN(0, 0), 0xb0, &vtbar
);
3419 /* "can't" happen */
3420 dev_info(&pdev
->dev
, "failed to run vt-d quirk\n");
3423 vtbar
&= 0xffff0000;
3425 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3426 drhd
= dmar_find_matched_drhd_unit(pdev
);
3427 if (WARN_TAINT_ONCE(!drhd
|| drhd
->reg_base_addr
- vtbar
!= 0xa000,
3428 TAINT_FIRMWARE_WORKAROUND
,
3429 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3430 pdev
->dev
.archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
3432 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL
, PCI_DEVICE_ID_INTEL_IOAT_SNB
, quirk_ioat_snb_local_iommu
);
3434 static void __init
init_no_remapping_devices(void)
3436 struct dmar_drhd_unit
*drhd
;
3437 struct pci_dev
*dev
;
3440 for_each_drhd_unit(drhd
) {
3441 if (!drhd
->include_all
) {
3442 for_each_active_dev_scope(drhd
->devices
,
3443 drhd
->devices_cnt
, i
, dev
)
3445 /* ignore DMAR unit if no pci devices exist */
3446 if (i
== drhd
->devices_cnt
)
3451 for_each_active_drhd_unit(drhd
) {
3452 if (drhd
->include_all
)
3455 for_each_active_dev_scope(drhd
->devices
,
3456 drhd
->devices_cnt
, i
, dev
)
3457 if (!IS_GFX_DEVICE(dev
))
3459 if (i
< drhd
->devices_cnt
)
3462 /* This IOMMU has *only* gfx devices. Either bypass it or
3463 set the gfx_mapped flag, as appropriate */
3465 intel_iommu_gfx_mapped
= 1;
3468 for_each_active_dev_scope(drhd
->devices
,
3469 drhd
->devices_cnt
, i
, dev
)
3470 dev
->dev
.archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
3475 #ifdef CONFIG_SUSPEND
3476 static int init_iommu_hw(void)
3478 struct dmar_drhd_unit
*drhd
;
3479 struct intel_iommu
*iommu
= NULL
;
3481 for_each_active_iommu(iommu
, drhd
)
3483 dmar_reenable_qi(iommu
);
3485 for_each_iommu(iommu
, drhd
) {
3486 if (drhd
->ignored
) {
3488 * we always have to disable PMRs or DMA may fail on
3492 iommu_disable_protect_mem_regions(iommu
);
3496 iommu_flush_write_buffer(iommu
);
3498 iommu_set_root_entry(iommu
);
3500 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
3501 DMA_CCMD_GLOBAL_INVL
);
3502 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3503 DMA_TLB_GLOBAL_FLUSH
);
3504 if (iommu_enable_translation(iommu
))
3506 iommu_disable_protect_mem_regions(iommu
);
3512 static void iommu_flush_all(void)
3514 struct dmar_drhd_unit
*drhd
;
3515 struct intel_iommu
*iommu
;
3517 for_each_active_iommu(iommu
, drhd
) {
3518 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
3519 DMA_CCMD_GLOBAL_INVL
);
3520 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3521 DMA_TLB_GLOBAL_FLUSH
);
3525 static int iommu_suspend(void)
3527 struct dmar_drhd_unit
*drhd
;
3528 struct intel_iommu
*iommu
= NULL
;
3531 for_each_active_iommu(iommu
, drhd
) {
3532 iommu
->iommu_state
= kzalloc(sizeof(u32
) * MAX_SR_DMAR_REGS
,
3534 if (!iommu
->iommu_state
)
3540 for_each_active_iommu(iommu
, drhd
) {
3541 iommu_disable_translation(iommu
);
3543 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
3545 iommu
->iommu_state
[SR_DMAR_FECTL_REG
] =
3546 readl(iommu
->reg
+ DMAR_FECTL_REG
);
3547 iommu
->iommu_state
[SR_DMAR_FEDATA_REG
] =
3548 readl(iommu
->reg
+ DMAR_FEDATA_REG
);
3549 iommu
->iommu_state
[SR_DMAR_FEADDR_REG
] =
3550 readl(iommu
->reg
+ DMAR_FEADDR_REG
);
3551 iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
] =
3552 readl(iommu
->reg
+ DMAR_FEUADDR_REG
);
3554 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
3559 for_each_active_iommu(iommu
, drhd
)
3560 kfree(iommu
->iommu_state
);
3565 static void iommu_resume(void)
3567 struct dmar_drhd_unit
*drhd
;
3568 struct intel_iommu
*iommu
= NULL
;
3571 if (init_iommu_hw()) {
3573 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3575 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3579 for_each_active_iommu(iommu
, drhd
) {
3581 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
3583 writel(iommu
->iommu_state
[SR_DMAR_FECTL_REG
],
3584 iommu
->reg
+ DMAR_FECTL_REG
);
3585 writel(iommu
->iommu_state
[SR_DMAR_FEDATA_REG
],
3586 iommu
->reg
+ DMAR_FEDATA_REG
);
3587 writel(iommu
->iommu_state
[SR_DMAR_FEADDR_REG
],
3588 iommu
->reg
+ DMAR_FEADDR_REG
);
3589 writel(iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
],
3590 iommu
->reg
+ DMAR_FEUADDR_REG
);
3592 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
3595 for_each_active_iommu(iommu
, drhd
)
3596 kfree(iommu
->iommu_state
);
3599 static struct syscore_ops iommu_syscore_ops
= {
3600 .resume
= iommu_resume
,
3601 .suspend
= iommu_suspend
,
3604 static void __init
init_iommu_pm_ops(void)
3606 register_syscore_ops(&iommu_syscore_ops
);
3610 static inline void init_iommu_pm_ops(void) {}
3611 #endif /* CONFIG_PM */
3614 int __init
dmar_parse_one_rmrr(struct acpi_dmar_header
*header
)
3616 struct acpi_dmar_reserved_memory
*rmrr
;
3617 struct dmar_rmrr_unit
*rmrru
;
3619 rmrru
= kzalloc(sizeof(*rmrru
), GFP_KERNEL
);
3623 rmrru
->hdr
= header
;
3624 rmrr
= (struct acpi_dmar_reserved_memory
*)header
;
3625 rmrru
->base_address
= rmrr
->base_address
;
3626 rmrru
->end_address
= rmrr
->end_address
;
3627 rmrru
->devices
= dmar_alloc_dev_scope((void *)(rmrr
+ 1),
3628 ((void *)rmrr
) + rmrr
->header
.length
,
3629 &rmrru
->devices_cnt
);
3630 if (rmrru
->devices_cnt
&& rmrru
->devices
== NULL
) {
3635 list_add(&rmrru
->list
, &dmar_rmrr_units
);
3640 int __init
dmar_parse_one_atsr(struct acpi_dmar_header
*hdr
)
3642 struct acpi_dmar_atsr
*atsr
;
3643 struct dmar_atsr_unit
*atsru
;
3645 atsr
= container_of(hdr
, struct acpi_dmar_atsr
, header
);
3646 atsru
= kzalloc(sizeof(*atsru
), GFP_KERNEL
);
3651 atsru
->include_all
= atsr
->flags
& 0x1;
3652 if (!atsru
->include_all
) {
3653 atsru
->devices
= dmar_alloc_dev_scope((void *)(atsr
+ 1),
3654 (void *)atsr
+ atsr
->header
.length
,
3655 &atsru
->devices_cnt
);
3656 if (atsru
->devices_cnt
&& atsru
->devices
== NULL
) {
3662 list_add_rcu(&atsru
->list
, &dmar_atsr_units
);
3667 static void intel_iommu_free_atsr(struct dmar_atsr_unit
*atsru
)
3669 dmar_free_dev_scope(&atsru
->devices
, &atsru
->devices_cnt
);
3673 static void intel_iommu_free_dmars(void)
3675 struct dmar_rmrr_unit
*rmrru
, *rmrr_n
;
3676 struct dmar_atsr_unit
*atsru
, *atsr_n
;
3678 list_for_each_entry_safe(rmrru
, rmrr_n
, &dmar_rmrr_units
, list
) {
3679 list_del(&rmrru
->list
);
3680 dmar_free_dev_scope(&rmrru
->devices
, &rmrru
->devices_cnt
);
3684 list_for_each_entry_safe(atsru
, atsr_n
, &dmar_atsr_units
, list
) {
3685 list_del(&atsru
->list
);
3686 intel_iommu_free_atsr(atsru
);
3690 int dmar_find_matched_atsr_unit(struct pci_dev
*dev
)
3693 struct pci_bus
*bus
;
3694 struct pci_dev
*bridge
= NULL
, *tmp
;
3695 struct acpi_dmar_atsr
*atsr
;
3696 struct dmar_atsr_unit
*atsru
;
3698 dev
= pci_physfn(dev
);
3699 for (bus
= dev
->bus
; bus
; bus
= bus
->parent
) {
3701 if (!bridge
|| !pci_is_pcie(bridge
) ||
3702 pci_pcie_type(bridge
) == PCI_EXP_TYPE_PCI_BRIDGE
)
3704 if (pci_pcie_type(bridge
) == PCI_EXP_TYPE_ROOT_PORT
)
3711 list_for_each_entry_rcu(atsru
, &dmar_atsr_units
, list
) {
3712 atsr
= container_of(atsru
->hdr
, struct acpi_dmar_atsr
, header
);
3713 if (atsr
->segment
!= pci_domain_nr(dev
->bus
))
3716 for_each_dev_scope(atsru
->devices
, atsru
->devices_cnt
, i
, tmp
)
3720 if (atsru
->include_all
)
3730 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info
*info
)
3733 struct dmar_rmrr_unit
*rmrru
;
3734 struct dmar_atsr_unit
*atsru
;
3735 struct acpi_dmar_atsr
*atsr
;
3736 struct acpi_dmar_reserved_memory
*rmrr
;
3738 if (!intel_iommu_enabled
&& system_state
!= SYSTEM_BOOTING
)
3741 list_for_each_entry(rmrru
, &dmar_rmrr_units
, list
) {
3742 rmrr
= container_of(rmrru
->hdr
,
3743 struct acpi_dmar_reserved_memory
, header
);
3744 if (info
->event
== BUS_NOTIFY_ADD_DEVICE
) {
3745 ret
= dmar_insert_dev_scope(info
, (void *)(rmrr
+ 1),
3746 ((void *)rmrr
) + rmrr
->header
.length
,
3747 rmrr
->segment
, rmrru
->devices
,
3748 rmrru
->devices_cnt
);
3753 } else if (info
->event
== BUS_NOTIFY_DEL_DEVICE
) {
3754 if (dmar_remove_dev_scope(info
, rmrr
->segment
,
3755 rmrru
->devices
, rmrru
->devices_cnt
))
3760 list_for_each_entry(atsru
, &dmar_atsr_units
, list
) {
3761 if (atsru
->include_all
)
3764 atsr
= container_of(atsru
->hdr
, struct acpi_dmar_atsr
, header
);
3765 if (info
->event
== BUS_NOTIFY_ADD_DEVICE
) {
3766 ret
= dmar_insert_dev_scope(info
, (void *)(atsr
+ 1),
3767 (void *)atsr
+ atsr
->header
.length
,
3768 atsr
->segment
, atsru
->devices
,
3769 atsru
->devices_cnt
);
3774 } else if (info
->event
== BUS_NOTIFY_DEL_DEVICE
) {
3775 if (dmar_remove_dev_scope(info
, atsr
->segment
,
3776 atsru
->devices
, atsru
->devices_cnt
))
3785 * Here we only respond to action of unbound device from driver.
3787 * Added device is not attached to its DMAR domain here yet. That will happen
3788 * when mapping the device to iova.
3790 static int device_notifier(struct notifier_block
*nb
,
3791 unsigned long action
, void *data
)
3793 struct device
*dev
= data
;
3794 struct pci_dev
*pdev
= to_pci_dev(dev
);
3795 struct dmar_domain
*domain
;
3797 if (iommu_dummy(pdev
))
3800 if (action
!= BUS_NOTIFY_UNBOUND_DRIVER
&&
3801 action
!= BUS_NOTIFY_DEL_DEVICE
)
3804 domain
= find_domain(pdev
);
3808 down_read(&dmar_global_lock
);
3809 domain_remove_one_dev_info(domain
, pdev
);
3810 if (!(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) &&
3811 !(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
) &&
3812 list_empty(&domain
->devices
))
3813 domain_exit(domain
);
3814 up_read(&dmar_global_lock
);
3819 static struct notifier_block device_nb
= {
3820 .notifier_call
= device_notifier
,
3823 static int intel_iommu_memory_notifier(struct notifier_block
*nb
,
3824 unsigned long val
, void *v
)
3826 struct memory_notify
*mhp
= v
;
3827 unsigned long long start
, end
;
3828 unsigned long start_vpfn
, last_vpfn
;
3831 case MEM_GOING_ONLINE
:
3832 start
= mhp
->start_pfn
<< PAGE_SHIFT
;
3833 end
= ((mhp
->start_pfn
+ mhp
->nr_pages
) << PAGE_SHIFT
) - 1;
3834 if (iommu_domain_identity_map(si_domain
, start
, end
)) {
3835 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3842 case MEM_CANCEL_ONLINE
:
3843 start_vpfn
= mm_to_dma_pfn(mhp
->start_pfn
);
3844 last_vpfn
= mm_to_dma_pfn(mhp
->start_pfn
+ mhp
->nr_pages
- 1);
3845 while (start_vpfn
<= last_vpfn
) {
3847 struct dmar_drhd_unit
*drhd
;
3848 struct intel_iommu
*iommu
;
3849 struct page
*freelist
;
3851 iova
= find_iova(&si_domain
->iovad
, start_vpfn
);
3853 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3858 iova
= split_and_remove_iova(&si_domain
->iovad
, iova
,
3859 start_vpfn
, last_vpfn
);
3861 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3862 start_vpfn
, last_vpfn
);
3866 freelist
= domain_unmap(si_domain
, iova
->pfn_lo
,
3870 for_each_active_iommu(iommu
, drhd
)
3871 iommu_flush_iotlb_psi(iommu
, si_domain
->id
,
3873 iova
->pfn_hi
- iova
->pfn_lo
+ 1,
3876 dma_free_pagelist(freelist
);
3878 start_vpfn
= iova
->pfn_hi
+ 1;
3879 free_iova_mem(iova
);
3887 static struct notifier_block intel_iommu_memory_nb
= {
3888 .notifier_call
= intel_iommu_memory_notifier
,
3892 int __init
intel_iommu_init(void)
3895 struct dmar_drhd_unit
*drhd
;
3896 struct intel_iommu
*iommu
;
3898 /* VT-d is required for a TXT/tboot launch, so enforce that */
3899 force_on
= tboot_force_iommu();
3901 if (iommu_init_mempool()) {
3903 panic("tboot: Failed to initialize iommu memory\n");
3907 down_write(&dmar_global_lock
);
3908 if (dmar_table_init()) {
3910 panic("tboot: Failed to initialize DMAR table\n");
3915 * Disable translation if already enabled prior to OS handover.
3917 for_each_active_iommu(iommu
, drhd
)
3918 if (iommu
->gcmd
& DMA_GCMD_TE
)
3919 iommu_disable_translation(iommu
);
3921 if (dmar_dev_scope_init() < 0) {
3923 panic("tboot: Failed to initialize DMAR device scope\n");
3927 if (no_iommu
|| dmar_disabled
)
3930 if (list_empty(&dmar_rmrr_units
))
3931 printk(KERN_INFO
"DMAR: No RMRR found\n");
3933 if (list_empty(&dmar_atsr_units
))
3934 printk(KERN_INFO
"DMAR: No ATSR found\n");
3936 if (dmar_init_reserved_ranges()) {
3938 panic("tboot: Failed to reserve iommu ranges\n");
3939 goto out_free_reserved_range
;
3942 init_no_remapping_devices();
3947 panic("tboot: Failed to initialize DMARs\n");
3948 printk(KERN_ERR
"IOMMU: dmar init failed\n");
3949 goto out_free_reserved_range
;
3951 up_write(&dmar_global_lock
);
3953 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3955 init_timer(&unmap_timer
);
3956 #ifdef CONFIG_SWIOTLB
3959 dma_ops
= &intel_dma_ops
;
3961 init_iommu_pm_ops();
3963 bus_set_iommu(&pci_bus_type
, &intel_iommu_ops
);
3964 bus_register_notifier(&pci_bus_type
, &device_nb
);
3965 if (si_domain
&& !hw_pass_through
)
3966 register_memory_notifier(&intel_iommu_memory_nb
);
3968 intel_iommu_enabled
= 1;
3972 out_free_reserved_range
:
3973 put_iova_domain(&reserved_iova_list
);
3975 intel_iommu_free_dmars();
3976 up_write(&dmar_global_lock
);
3977 iommu_exit_mempool();
3981 static void iommu_detach_dependent_devices(struct intel_iommu
*iommu
,
3982 struct pci_dev
*pdev
)
3984 struct pci_dev
*tmp
, *parent
;
3986 if (!iommu
|| !pdev
)
3989 /* dependent device detach */
3990 tmp
= pci_find_upstream_pcie_bridge(pdev
);
3991 /* Secondary interface's bus number and devfn 0 */
3993 parent
= pdev
->bus
->self
;
3994 while (parent
!= tmp
) {
3995 iommu_detach_dev(iommu
, parent
->bus
->number
,
3997 parent
= parent
->bus
->self
;
3999 if (pci_is_pcie(tmp
)) /* this is a PCIe-to-PCI bridge */
4000 iommu_detach_dev(iommu
,
4001 tmp
->subordinate
->number
, 0);
4002 else /* this is a legacy PCI bridge */
4003 iommu_detach_dev(iommu
, tmp
->bus
->number
,
4008 static void domain_remove_one_dev_info(struct dmar_domain
*domain
,
4009 struct pci_dev
*pdev
)
4011 struct device_domain_info
*info
, *tmp
;
4012 struct intel_iommu
*iommu
;
4013 unsigned long flags
;
4016 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
4021 spin_lock_irqsave(&device_domain_lock
, flags
);
4022 list_for_each_entry_safe(info
, tmp
, &domain
->devices
, link
) {
4023 if (info
->segment
== pci_domain_nr(pdev
->bus
) &&
4024 info
->bus
== pdev
->bus
->number
&&
4025 info
->devfn
== pdev
->devfn
) {
4026 unlink_domain_info(info
);
4027 spin_unlock_irqrestore(&device_domain_lock
, flags
);
4029 iommu_disable_dev_iotlb(info
);
4030 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
4031 iommu_detach_dependent_devices(iommu
, pdev
);
4032 free_devinfo_mem(info
);
4034 spin_lock_irqsave(&device_domain_lock
, flags
);
4042 /* if there is no other devices under the same iommu
4043 * owned by this domain, clear this iommu in iommu_bmp
4044 * update iommu count and coherency
4046 if (iommu
== device_to_iommu(info
->segment
, info
->bus
,
4051 spin_unlock_irqrestore(&device_domain_lock
, flags
);
4054 unsigned long tmp_flags
;
4055 spin_lock_irqsave(&domain
->iommu_lock
, tmp_flags
);
4056 clear_bit(iommu
->seq_id
, domain
->iommu_bmp
);
4057 domain
->iommu_count
--;
4058 domain_update_iommu_cap(domain
);
4059 spin_unlock_irqrestore(&domain
->iommu_lock
, tmp_flags
);
4061 if (!(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) &&
4062 !(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
)) {
4063 spin_lock_irqsave(&iommu
->lock
, tmp_flags
);
4064 clear_bit(domain
->id
, iommu
->domain_ids
);
4065 iommu
->domains
[domain
->id
] = NULL
;
4066 spin_unlock_irqrestore(&iommu
->lock
, tmp_flags
);
4071 static int md_domain_init(struct dmar_domain
*domain
, int guest_width
)
4075 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
4076 domain_reserve_special_ranges(domain
);
4078 /* calculate AGAW */
4079 domain
->gaw
= guest_width
;
4080 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
4081 domain
->agaw
= width_to_agaw(adjust_width
);
4083 domain
->iommu_coherency
= 0;
4084 domain
->iommu_snooping
= 0;
4085 domain
->iommu_superpage
= 0;
4086 domain
->max_addr
= 0;
4089 /* always allocate the top pgd */
4090 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page(domain
->nid
);
4093 domain_flush_cache(domain
, domain
->pgd
, PAGE_SIZE
);
4097 static int intel_iommu_domain_init(struct iommu_domain
*domain
)
4099 struct dmar_domain
*dmar_domain
;
4101 dmar_domain
= alloc_domain(true);
4104 "intel_iommu_domain_init: dmar_domain == NULL\n");
4107 if (md_domain_init(dmar_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
4109 "intel_iommu_domain_init() failed\n");
4110 domain_exit(dmar_domain
);
4113 domain_update_iommu_cap(dmar_domain
);
4114 domain
->priv
= dmar_domain
;
4116 domain
->geometry
.aperture_start
= 0;
4117 domain
->geometry
.aperture_end
= __DOMAIN_MAX_ADDR(dmar_domain
->gaw
);
4118 domain
->geometry
.force_aperture
= true;
4123 static void intel_iommu_domain_destroy(struct iommu_domain
*domain
)
4125 struct dmar_domain
*dmar_domain
= domain
->priv
;
4127 domain
->priv
= NULL
;
4128 domain_exit(dmar_domain
);
4131 static int intel_iommu_attach_device(struct iommu_domain
*domain
,
4134 struct dmar_domain
*dmar_domain
= domain
->priv
;
4135 struct pci_dev
*pdev
= to_pci_dev(dev
);
4136 struct intel_iommu
*iommu
;
4139 /* normally pdev is not mapped */
4140 if (unlikely(domain_context_mapped(pdev
))) {
4141 struct dmar_domain
*old_domain
;
4143 old_domain
= find_domain(pdev
);
4145 if (dmar_domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
4146 dmar_domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
)
4147 domain_remove_one_dev_info(old_domain
, pdev
);
4149 domain_remove_dev_info(old_domain
);
4153 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
4158 /* check if this iommu agaw is sufficient for max mapped address */
4159 addr_width
= agaw_to_width(iommu
->agaw
);
4160 if (addr_width
> cap_mgaw(iommu
->cap
))
4161 addr_width
= cap_mgaw(iommu
->cap
);
4163 if (dmar_domain
->max_addr
> (1LL << addr_width
)) {
4164 printk(KERN_ERR
"%s: iommu width (%d) is not "
4165 "sufficient for the mapped address (%llx)\n",
4166 __func__
, addr_width
, dmar_domain
->max_addr
);
4169 dmar_domain
->gaw
= addr_width
;
4172 * Knock out extra levels of page tables if necessary
4174 while (iommu
->agaw
< dmar_domain
->agaw
) {
4175 struct dma_pte
*pte
;
4177 pte
= dmar_domain
->pgd
;
4178 if (dma_pte_present(pte
)) {
4179 dmar_domain
->pgd
= (struct dma_pte
*)
4180 phys_to_virt(dma_pte_addr(pte
));
4181 free_pgtable_page(pte
);
4183 dmar_domain
->agaw
--;
4186 return domain_add_dev_info(dmar_domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
4189 static void intel_iommu_detach_device(struct iommu_domain
*domain
,
4192 struct dmar_domain
*dmar_domain
= domain
->priv
;
4193 struct pci_dev
*pdev
= to_pci_dev(dev
);
4195 domain_remove_one_dev_info(dmar_domain
, pdev
);
4198 static int intel_iommu_map(struct iommu_domain
*domain
,
4199 unsigned long iova
, phys_addr_t hpa
,
4200 size_t size
, int iommu_prot
)
4202 struct dmar_domain
*dmar_domain
= domain
->priv
;
4207 if (iommu_prot
& IOMMU_READ
)
4208 prot
|= DMA_PTE_READ
;
4209 if (iommu_prot
& IOMMU_WRITE
)
4210 prot
|= DMA_PTE_WRITE
;
4211 if ((iommu_prot
& IOMMU_CACHE
) && dmar_domain
->iommu_snooping
)
4212 prot
|= DMA_PTE_SNP
;
4214 max_addr
= iova
+ size
;
4215 if (dmar_domain
->max_addr
< max_addr
) {
4218 /* check if minimum agaw is sufficient for mapped address */
4219 end
= __DOMAIN_MAX_ADDR(dmar_domain
->gaw
) + 1;
4220 if (end
< max_addr
) {
4221 printk(KERN_ERR
"%s: iommu width (%d) is not "
4222 "sufficient for the mapped address (%llx)\n",
4223 __func__
, dmar_domain
->gaw
, max_addr
);
4226 dmar_domain
->max_addr
= max_addr
;
4228 /* Round up size to next multiple of PAGE_SIZE, if it and
4229 the low bits of hpa would take us onto the next page */
4230 size
= aligned_nrpages(hpa
, size
);
4231 ret
= domain_pfn_mapping(dmar_domain
, iova
>> VTD_PAGE_SHIFT
,
4232 hpa
>> VTD_PAGE_SHIFT
, size
, prot
);
4236 static size_t intel_iommu_unmap(struct iommu_domain
*domain
,
4237 unsigned long iova
, size_t size
)
4239 struct dmar_domain
*dmar_domain
= domain
->priv
;
4240 struct page
*freelist
= NULL
;
4241 struct intel_iommu
*iommu
;
4242 unsigned long start_pfn
, last_pfn
;
4243 unsigned int npages
;
4244 int iommu_id
, num
, ndomains
, level
= 0;
4246 /* Cope with horrid API which requires us to unmap more than the
4247 size argument if it happens to be a large-page mapping. */
4248 if (!pfn_to_dma_pte(dmar_domain
, iova
>> VTD_PAGE_SHIFT
, &level
))
4251 if (size
< VTD_PAGE_SIZE
<< level_to_offset_bits(level
))
4252 size
= VTD_PAGE_SIZE
<< level_to_offset_bits(level
);
4254 start_pfn
= iova
>> VTD_PAGE_SHIFT
;
4255 last_pfn
= (iova
+ size
- 1) >> VTD_PAGE_SHIFT
;
4257 freelist
= domain_unmap(dmar_domain
, start_pfn
, last_pfn
);
4259 npages
= last_pfn
- start_pfn
+ 1;
4261 for_each_set_bit(iommu_id
, dmar_domain
->iommu_bmp
, g_num_of_iommus
) {
4262 iommu
= g_iommus
[iommu_id
];
4265 * find bit position of dmar_domain
4267 ndomains
= cap_ndoms(iommu
->cap
);
4268 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
4269 if (iommu
->domains
[num
] == dmar_domain
)
4270 iommu_flush_iotlb_psi(iommu
, num
, start_pfn
,
4271 npages
, !freelist
, 0);
4276 dma_free_pagelist(freelist
);
4278 if (dmar_domain
->max_addr
== iova
+ size
)
4279 dmar_domain
->max_addr
= iova
;
4284 static phys_addr_t
intel_iommu_iova_to_phys(struct iommu_domain
*domain
,
4287 struct dmar_domain
*dmar_domain
= domain
->priv
;
4288 struct dma_pte
*pte
;
4292 pte
= pfn_to_dma_pte(dmar_domain
, iova
>> VTD_PAGE_SHIFT
, &level
);
4294 phys
= dma_pte_addr(pte
);
4299 static int intel_iommu_domain_has_cap(struct iommu_domain
*domain
,
4302 struct dmar_domain
*dmar_domain
= domain
->priv
;
4304 if (cap
== IOMMU_CAP_CACHE_COHERENCY
)
4305 return dmar_domain
->iommu_snooping
;
4306 if (cap
== IOMMU_CAP_INTR_REMAP
)
4307 return irq_remapping_enabled
;
4312 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4314 static int intel_iommu_add_device(struct device
*dev
)
4316 struct pci_dev
*pdev
= to_pci_dev(dev
);
4317 struct pci_dev
*bridge
, *dma_pdev
= NULL
;
4318 struct iommu_group
*group
;
4321 if (!device_to_iommu(pci_domain_nr(pdev
->bus
),
4322 pdev
->bus
->number
, pdev
->devfn
))
4325 bridge
= pci_find_upstream_pcie_bridge(pdev
);
4327 if (pci_is_pcie(bridge
))
4328 dma_pdev
= pci_get_domain_bus_and_slot(
4329 pci_domain_nr(pdev
->bus
),
4330 bridge
->subordinate
->number
, 0);
4332 dma_pdev
= pci_dev_get(bridge
);
4334 dma_pdev
= pci_dev_get(pdev
);
4336 /* Account for quirked devices */
4337 swap_pci_ref(&dma_pdev
, pci_get_dma_source(dma_pdev
));
4340 * If it's a multifunction device that does not support our
4341 * required ACS flags, add to the same group as lowest numbered
4342 * function that also does not suport the required ACS flags.
4344 if (dma_pdev
->multifunction
&&
4345 !pci_acs_enabled(dma_pdev
, REQ_ACS_FLAGS
)) {
4346 u8 i
, slot
= PCI_SLOT(dma_pdev
->devfn
);
4348 for (i
= 0; i
< 8; i
++) {
4349 struct pci_dev
*tmp
;
4351 tmp
= pci_get_slot(dma_pdev
->bus
, PCI_DEVFN(slot
, i
));
4355 if (!pci_acs_enabled(tmp
, REQ_ACS_FLAGS
)) {
4356 swap_pci_ref(&dma_pdev
, tmp
);
4364 * Devices on the root bus go through the iommu. If that's not us,
4365 * find the next upstream device and test ACS up to the root bus.
4366 * Finding the next device may require skipping virtual buses.
4368 while (!pci_is_root_bus(dma_pdev
->bus
)) {
4369 struct pci_bus
*bus
= dma_pdev
->bus
;
4371 while (!bus
->self
) {
4372 if (!pci_is_root_bus(bus
))
4378 if (pci_acs_path_enabled(bus
->self
, NULL
, REQ_ACS_FLAGS
))
4381 swap_pci_ref(&dma_pdev
, pci_dev_get(bus
->self
));
4385 group
= iommu_group_get(&dma_pdev
->dev
);
4386 pci_dev_put(dma_pdev
);
4388 group
= iommu_group_alloc();
4390 return PTR_ERR(group
);
4393 ret
= iommu_group_add_device(group
, dev
);
4395 iommu_group_put(group
);
4399 static void intel_iommu_remove_device(struct device
*dev
)
4401 iommu_group_remove_device(dev
);
4404 static struct iommu_ops intel_iommu_ops
= {
4405 .domain_init
= intel_iommu_domain_init
,
4406 .domain_destroy
= intel_iommu_domain_destroy
,
4407 .attach_dev
= intel_iommu_attach_device
,
4408 .detach_dev
= intel_iommu_detach_device
,
4409 .map
= intel_iommu_map
,
4410 .unmap
= intel_iommu_unmap
,
4411 .iova_to_phys
= intel_iommu_iova_to_phys
,
4412 .domain_has_cap
= intel_iommu_domain_has_cap
,
4413 .add_device
= intel_iommu_add_device
,
4414 .remove_device
= intel_iommu_remove_device
,
4415 .pgsize_bitmap
= INTEL_IOMMU_PGSIZES
,
4418 static void quirk_iommu_g4x_gfx(struct pci_dev
*dev
)
4420 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4421 printk(KERN_INFO
"DMAR: Disabling IOMMU for graphics on this chipset\n");
4425 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2a40, quirk_iommu_g4x_gfx
);
4426 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e00, quirk_iommu_g4x_gfx
);
4427 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e10, quirk_iommu_g4x_gfx
);
4428 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e20, quirk_iommu_g4x_gfx
);
4429 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e30, quirk_iommu_g4x_gfx
);
4430 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e40, quirk_iommu_g4x_gfx
);
4431 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e90, quirk_iommu_g4x_gfx
);
4433 static void quirk_iommu_rwbf(struct pci_dev
*dev
)
4436 * Mobile 4 Series Chipset neglects to set RWBF capability,
4437 * but needs it. Same seems to hold for the desktop versions.
4439 printk(KERN_INFO
"DMAR: Forcing write-buffer flush capability\n");
4443 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2a40, quirk_iommu_rwbf
);
4444 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e00, quirk_iommu_rwbf
);
4445 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e10, quirk_iommu_rwbf
);
4446 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e20, quirk_iommu_rwbf
);
4447 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e30, quirk_iommu_rwbf
);
4448 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e40, quirk_iommu_rwbf
);
4449 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e90, quirk_iommu_rwbf
);
4452 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
4453 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4454 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
4455 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
4456 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4457 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4458 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4459 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4461 static void quirk_calpella_no_shadow_gtt(struct pci_dev
*dev
)
4465 if (pci_read_config_word(dev
, GGC
, &ggc
))
4468 if (!(ggc
& GGC_MEMORY_VT_ENABLED
)) {
4469 printk(KERN_INFO
"DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4471 } else if (dmar_map_gfx
) {
4472 /* we have to ensure the gfx device is idle before we flush */
4473 printk(KERN_INFO
"DMAR: Disabling batched IOTLB flush on Ironlake\n");
4474 intel_iommu_strict
= 1;
4477 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0040, quirk_calpella_no_shadow_gtt
);
4478 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0044, quirk_calpella_no_shadow_gtt
);
4479 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0062, quirk_calpella_no_shadow_gtt
);
4480 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x006a, quirk_calpella_no_shadow_gtt
);
4482 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4483 ISOCH DMAR unit for the Azalia sound device, but not give it any
4484 TLB entries, which causes it to deadlock. Check for that. We do
4485 this in a function called from init_dmars(), instead of in a PCI
4486 quirk, because we don't want to print the obnoxious "BIOS broken"
4487 message if VT-d is actually disabled.
4489 static void __init
check_tylersburg_isoch(void)
4491 struct pci_dev
*pdev
;
4492 uint32_t vtisochctrl
;
4494 /* If there's no Azalia in the system anyway, forget it. */
4495 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x3a3e, NULL
);
4500 /* System Management Registers. Might be hidden, in which case
4501 we can't do the sanity check. But that's OK, because the
4502 known-broken BIOSes _don't_ actually hide it, so far. */
4503 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x342e, NULL
);
4507 if (pci_read_config_dword(pdev
, 0x188, &vtisochctrl
)) {
4514 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4515 if (vtisochctrl
& 1)
4518 /* Drop all bits other than the number of TLB entries */
4519 vtisochctrl
&= 0x1c;
4521 /* If we have the recommended number of TLB entries (16), fine. */
4522 if (vtisochctrl
== 0x10)
4525 /* Zero TLB entries? You get to ride the short bus to school. */
4527 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4528 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4529 dmi_get_system_info(DMI_BIOS_VENDOR
),
4530 dmi_get_system_info(DMI_BIOS_VERSION
),
4531 dmi_get_system_info(DMI_PRODUCT_VERSION
));
4532 iommu_identity_mapping
|= IDENTMAP_AZALIA
;
4536 printk(KERN_WARNING
"DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",