2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
20 #include <linux/init.h>
21 #include <linux/bitmap.h>
22 #include <linux/debugfs.h>
23 #include <linux/export.h>
24 #include <linux/slab.h>
25 #include <linux/irq.h>
26 #include <linux/interrupt.h>
27 #include <linux/spinlock.h>
28 #include <linux/pci.h>
29 #include <linux/dmar.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/mempool.h>
32 #include <linux/memory.h>
33 #include <linux/timer.h>
34 #include <linux/iova.h>
35 #include <linux/iommu.h>
36 #include <linux/intel-iommu.h>
37 #include <linux/syscore_ops.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/pci-ats.h>
41 #include <linux/memblock.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/cacheflush.h>
44 #include <asm/iommu.h>
46 #include "irq_remapping.h"
49 #define ROOT_SIZE VTD_PAGE_SIZE
50 #define CONTEXT_SIZE VTD_PAGE_SIZE
52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
56 #define IOAPIC_RANGE_START (0xfee00000)
57 #define IOAPIC_RANGE_END (0xfeefffff)
58 #define IOVA_START_ADDR (0x1000)
60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
62 #define MAX_AGAW_WIDTH 64
63 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
65 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
68 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
74 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
75 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
76 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
78 /* page table handling */
79 #define LEVEL_STRIDE (9)
80 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
98 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
100 static inline int agaw_to_level(int agaw
)
105 static inline int agaw_to_width(int agaw
)
107 return min_t(int, 30 + agaw
* LEVEL_STRIDE
, MAX_AGAW_WIDTH
);
110 static inline int width_to_agaw(int width
)
112 return DIV_ROUND_UP(width
- 30, LEVEL_STRIDE
);
115 static inline unsigned int level_to_offset_bits(int level
)
117 return (level
- 1) * LEVEL_STRIDE
;
120 static inline int pfn_level_offset(unsigned long pfn
, int level
)
122 return (pfn
>> level_to_offset_bits(level
)) & LEVEL_MASK
;
125 static inline unsigned long level_mask(int level
)
127 return -1UL << level_to_offset_bits(level
);
130 static inline unsigned long level_size(int level
)
132 return 1UL << level_to_offset_bits(level
);
135 static inline unsigned long align_to_level(unsigned long pfn
, int level
)
137 return (pfn
+ level_size(level
) - 1) & level_mask(level
);
140 static inline unsigned long lvl_to_nr_pages(unsigned int lvl
)
142 return 1 << min_t(int, (lvl
- 1) * LEVEL_STRIDE
, MAX_AGAW_PFN_WIDTH
);
145 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn
)
149 return dma_pfn
>> (PAGE_SHIFT
- VTD_PAGE_SHIFT
);
152 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn
)
154 return mm_pfn
<< (PAGE_SHIFT
- VTD_PAGE_SHIFT
);
156 static inline unsigned long page_to_dma_pfn(struct page
*pg
)
158 return mm_to_dma_pfn(page_to_pfn(pg
));
160 static inline unsigned long virt_to_dma_pfn(void *p
)
162 return page_to_dma_pfn(virt_to_page(p
));
165 /* global iommu list, set NULL for ignored DMAR units */
166 static struct intel_iommu
**g_iommus
;
168 static void __init
check_tylersburg_isoch(void);
169 static int rwbf_quirk
;
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
175 static int force_on
= 0;
180 * 12-63: Context Ptr (12 - (haw-1))
187 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188 static inline bool root_present(struct root_entry
*root
)
190 return (root
->val
& 1);
192 static inline void set_root_present(struct root_entry
*root
)
196 static inline void set_root_value(struct root_entry
*root
, unsigned long value
)
198 root
->val
|= value
& VTD_PAGE_MASK
;
201 static inline struct context_entry
*
202 get_context_addr_from_root(struct root_entry
*root
)
204 return (struct context_entry
*)
205 (root_present(root
)?phys_to_virt(
206 root
->val
& VTD_PAGE_MASK
) :
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
221 struct context_entry
{
226 static inline bool context_present(struct context_entry
*context
)
228 return (context
->lo
& 1);
230 static inline void context_set_present(struct context_entry
*context
)
235 static inline void context_set_fault_enable(struct context_entry
*context
)
237 context
->lo
&= (((u64
)-1) << 2) | 1;
240 static inline void context_set_translation_type(struct context_entry
*context
,
243 context
->lo
&= (((u64
)-1) << 4) | 3;
244 context
->lo
|= (value
& 3) << 2;
247 static inline void context_set_address_root(struct context_entry
*context
,
250 context
->lo
|= value
& VTD_PAGE_MASK
;
253 static inline void context_set_address_width(struct context_entry
*context
,
256 context
->hi
|= value
& 7;
259 static inline void context_set_domain_id(struct context_entry
*context
,
262 context
->hi
|= (value
& ((1 << 16) - 1)) << 8;
265 static inline void context_clear_entry(struct context_entry
*context
)
278 * 12-63: Host physcial address
284 static inline void dma_clear_pte(struct dma_pte
*pte
)
289 static inline u64
dma_pte_addr(struct dma_pte
*pte
)
292 return pte
->val
& VTD_PAGE_MASK
;
294 /* Must have a full atomic 64-bit read */
295 return __cmpxchg64(&pte
->val
, 0ULL, 0ULL) & VTD_PAGE_MASK
;
299 static inline bool dma_pte_present(struct dma_pte
*pte
)
301 return (pte
->val
& 3) != 0;
304 static inline bool dma_pte_superpage(struct dma_pte
*pte
)
306 return (pte
->val
& (1 << 7));
309 static inline int first_pte_in_page(struct dma_pte
*pte
)
311 return !((unsigned long)pte
& ~VTD_PAGE_MASK
);
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
320 static struct dmar_domain
*si_domain
;
321 static int hw_pass_through
= 1;
323 /* devices under the same p2p bridge are owned in one domain */
324 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
326 /* domain represents a virtual machine, more than one devices
327 * across iommus may be owned in one domain, e.g. kvm guest.
329 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
331 /* si_domain contains mulitple devices */
332 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
334 /* define the limit of IOMMUs supported in each domain */
336 # define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
338 # define IOMMU_UNITS_SUPPORTED 64
342 int id
; /* domain id */
343 int nid
; /* node id */
344 DECLARE_BITMAP(iommu_bmp
, IOMMU_UNITS_SUPPORTED
);
345 /* bitmap of iommus this domain uses*/
347 struct list_head devices
; /* all devices' list */
348 struct iova_domain iovad
; /* iova's that belong to this domain */
350 struct dma_pte
*pgd
; /* virtual address */
351 int gaw
; /* max guest address width */
353 /* adjusted guest address width, 0 is level 2 30-bit */
356 int flags
; /* flags to find out type of domain */
358 int iommu_coherency
;/* indicate coherency of iommu access */
359 int iommu_snooping
; /* indicate snooping control feature*/
360 int iommu_count
; /* reference count of iommu */
361 int iommu_superpage
;/* Level of superpages supported:
362 0 == 4KiB (no superpages), 1 == 2MiB,
363 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
364 spinlock_t iommu_lock
; /* protect iommu set in domain */
365 u64 max_addr
; /* maximum mapped address */
368 /* PCI domain-device relationship */
369 struct device_domain_info
{
370 struct list_head link
; /* link to domain siblings */
371 struct list_head global
; /* link to global list */
372 u8 bus
; /* PCI bus number */
373 u8 devfn
; /* PCI devfn number */
374 struct device
*dev
; /* it's NULL for PCIe-to-PCI bridge */
375 struct intel_iommu
*iommu
; /* IOMMU used by this device */
376 struct dmar_domain
*domain
; /* pointer to domain */
379 struct dmar_rmrr_unit
{
380 struct list_head list
; /* list of rmrr units */
381 struct acpi_dmar_header
*hdr
; /* ACPI header */
382 u64 base_address
; /* reserved base address*/
383 u64 end_address
; /* reserved end address */
384 struct dmar_dev_scope
*devices
; /* target devices */
385 int devices_cnt
; /* target device count */
388 struct dmar_atsr_unit
{
389 struct list_head list
; /* list of ATSR units */
390 struct acpi_dmar_header
*hdr
; /* ACPI header */
391 struct dmar_dev_scope
*devices
; /* target devices */
392 int devices_cnt
; /* target device count */
393 u8 include_all
:1; /* include all ports */
396 static LIST_HEAD(dmar_atsr_units
);
397 static LIST_HEAD(dmar_rmrr_units
);
399 #define for_each_rmrr_units(rmrr) \
400 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
402 static void flush_unmaps_timeout(unsigned long data
);
404 static DEFINE_TIMER(unmap_timer
, flush_unmaps_timeout
, 0, 0);
406 #define HIGH_WATER_MARK 250
407 struct deferred_flush_tables
{
409 struct iova
*iova
[HIGH_WATER_MARK
];
410 struct dmar_domain
*domain
[HIGH_WATER_MARK
];
411 struct page
*freelist
[HIGH_WATER_MARK
];
414 static struct deferred_flush_tables
*deferred_flush
;
416 /* bitmap for indexing intel_iommus */
417 static int g_num_of_iommus
;
419 static DEFINE_SPINLOCK(async_umap_flush_lock
);
420 static LIST_HEAD(unmaps_to_do
);
423 static long list_size
;
425 static void domain_exit(struct dmar_domain
*domain
);
426 static void domain_remove_dev_info(struct dmar_domain
*domain
);
427 static void domain_remove_one_dev_info(struct dmar_domain
*domain
,
429 static void iommu_detach_dependent_devices(struct intel_iommu
*iommu
,
432 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
433 int dmar_disabled
= 0;
435 int dmar_disabled
= 1;
436 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
438 int intel_iommu_enabled
= 0;
439 EXPORT_SYMBOL_GPL(intel_iommu_enabled
);
441 static int dmar_map_gfx
= 1;
442 static int dmar_forcedac
;
443 static int intel_iommu_strict
;
444 static int intel_iommu_superpage
= 1;
446 int intel_iommu_gfx_mapped
;
447 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped
);
449 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
450 static DEFINE_SPINLOCK(device_domain_lock
);
451 static LIST_HEAD(device_domain_list
);
453 static struct iommu_ops intel_iommu_ops
;
455 static int __init
intel_iommu_setup(char *str
)
460 if (!strncmp(str
, "on", 2)) {
462 printk(KERN_INFO
"Intel-IOMMU: enabled\n");
463 } else if (!strncmp(str
, "off", 3)) {
465 printk(KERN_INFO
"Intel-IOMMU: disabled\n");
466 } else if (!strncmp(str
, "igfx_off", 8)) {
469 "Intel-IOMMU: disable GFX device mapping\n");
470 } else if (!strncmp(str
, "forcedac", 8)) {
472 "Intel-IOMMU: Forcing DAC for PCI devices\n");
474 } else if (!strncmp(str
, "strict", 6)) {
476 "Intel-IOMMU: disable batched IOTLB flush\n");
477 intel_iommu_strict
= 1;
478 } else if (!strncmp(str
, "sp_off", 6)) {
480 "Intel-IOMMU: disable supported super page\n");
481 intel_iommu_superpage
= 0;
484 str
+= strcspn(str
, ",");
490 __setup("intel_iommu=", intel_iommu_setup
);
492 static struct kmem_cache
*iommu_domain_cache
;
493 static struct kmem_cache
*iommu_devinfo_cache
;
494 static struct kmem_cache
*iommu_iova_cache
;
496 static inline void *alloc_pgtable_page(int node
)
501 page
= alloc_pages_node(node
, GFP_ATOMIC
| __GFP_ZERO
, 0);
503 vaddr
= page_address(page
);
507 static inline void free_pgtable_page(void *vaddr
)
509 free_page((unsigned long)vaddr
);
512 static inline void *alloc_domain_mem(void)
514 return kmem_cache_alloc(iommu_domain_cache
, GFP_ATOMIC
);
517 static void free_domain_mem(void *vaddr
)
519 kmem_cache_free(iommu_domain_cache
, vaddr
);
522 static inline void * alloc_devinfo_mem(void)
524 return kmem_cache_alloc(iommu_devinfo_cache
, GFP_ATOMIC
);
527 static inline void free_devinfo_mem(void *vaddr
)
529 kmem_cache_free(iommu_devinfo_cache
, vaddr
);
532 struct iova
*alloc_iova_mem(void)
534 return kmem_cache_alloc(iommu_iova_cache
, GFP_ATOMIC
);
537 void free_iova_mem(struct iova
*iova
)
539 kmem_cache_free(iommu_iova_cache
, iova
);
543 static int __iommu_calculate_agaw(struct intel_iommu
*iommu
, int max_gaw
)
548 sagaw
= cap_sagaw(iommu
->cap
);
549 for (agaw
= width_to_agaw(max_gaw
);
551 if (test_bit(agaw
, &sagaw
))
559 * Calculate max SAGAW for each iommu.
561 int iommu_calculate_max_sagaw(struct intel_iommu
*iommu
)
563 return __iommu_calculate_agaw(iommu
, MAX_AGAW_WIDTH
);
567 * calculate agaw for each iommu.
568 * "SAGAW" may be different across iommus, use a default agaw, and
569 * get a supported less agaw for iommus that don't support the default agaw.
571 int iommu_calculate_agaw(struct intel_iommu
*iommu
)
573 return __iommu_calculate_agaw(iommu
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
576 /* This functionin only returns single iommu in a domain */
577 static struct intel_iommu
*domain_get_iommu(struct dmar_domain
*domain
)
581 /* si_domain and vm domain should not get here. */
582 BUG_ON(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
);
583 BUG_ON(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
);
585 iommu_id
= find_first_bit(domain
->iommu_bmp
, g_num_of_iommus
);
586 if (iommu_id
< 0 || iommu_id
>= g_num_of_iommus
)
589 return g_iommus
[iommu_id
];
592 static void domain_update_iommu_coherency(struct dmar_domain
*domain
)
594 struct dmar_drhd_unit
*drhd
;
595 struct intel_iommu
*iommu
;
598 domain
->iommu_coherency
= 1;
600 for_each_set_bit(i
, domain
->iommu_bmp
, g_num_of_iommus
) {
602 if (!ecap_coherent(g_iommus
[i
]->ecap
)) {
603 domain
->iommu_coherency
= 0;
610 /* No hardware attached; use lowest common denominator */
612 for_each_active_iommu(iommu
, drhd
) {
613 if (!ecap_coherent(iommu
->ecap
)) {
614 domain
->iommu_coherency
= 0;
621 static void domain_update_iommu_snooping(struct dmar_domain
*domain
)
625 domain
->iommu_snooping
= 1;
627 for_each_set_bit(i
, domain
->iommu_bmp
, g_num_of_iommus
) {
628 if (!ecap_sc_support(g_iommus
[i
]->ecap
)) {
629 domain
->iommu_snooping
= 0;
635 static void domain_update_iommu_superpage(struct dmar_domain
*domain
)
637 struct dmar_drhd_unit
*drhd
;
638 struct intel_iommu
*iommu
= NULL
;
641 if (!intel_iommu_superpage
) {
642 domain
->iommu_superpage
= 0;
646 /* set iommu_superpage to the smallest common denominator */
648 for_each_active_iommu(iommu
, drhd
) {
649 mask
&= cap_super_page_val(iommu
->cap
);
656 domain
->iommu_superpage
= fls(mask
);
659 /* Some capabilities may be different across iommus */
660 static void domain_update_iommu_cap(struct dmar_domain
*domain
)
662 domain_update_iommu_coherency(domain
);
663 domain_update_iommu_snooping(domain
);
664 domain_update_iommu_superpage(domain
);
667 static struct intel_iommu
*device_to_iommu(struct device
*dev
, u8
*bus
, u8
*devfn
)
669 struct dmar_drhd_unit
*drhd
= NULL
;
670 struct intel_iommu
*iommu
;
672 struct pci_dev
*ptmp
, *pdev
= NULL
;
676 if (dev_is_pci(dev
)) {
677 pdev
= to_pci_dev(dev
);
678 segment
= pci_domain_nr(pdev
->bus
);
679 } else if (ACPI_COMPANION(dev
))
680 dev
= &ACPI_COMPANION(dev
)->dev
;
683 for_each_active_iommu(iommu
, drhd
) {
684 if (pdev
&& segment
!= drhd
->segment
)
687 for_each_active_dev_scope(drhd
->devices
,
688 drhd
->devices_cnt
, i
, tmp
) {
690 *bus
= drhd
->devices
[i
].bus
;
691 *devfn
= drhd
->devices
[i
].devfn
;
695 if (!pdev
|| !dev_is_pci(tmp
))
698 ptmp
= to_pci_dev(tmp
);
699 if (ptmp
->subordinate
&&
700 ptmp
->subordinate
->number
<= pdev
->bus
->number
&&
701 ptmp
->subordinate
->busn_res
.end
>= pdev
->bus
->number
)
705 if (pdev
&& drhd
->include_all
) {
707 *bus
= pdev
->bus
->number
;
708 *devfn
= pdev
->devfn
;
719 static void domain_flush_cache(struct dmar_domain
*domain
,
720 void *addr
, int size
)
722 if (!domain
->iommu_coherency
)
723 clflush_cache_range(addr
, size
);
726 /* Gets context entry for a given bus and devfn */
727 static struct context_entry
* device_to_context_entry(struct intel_iommu
*iommu
,
730 struct root_entry
*root
;
731 struct context_entry
*context
;
732 unsigned long phy_addr
;
735 spin_lock_irqsave(&iommu
->lock
, flags
);
736 root
= &iommu
->root_entry
[bus
];
737 context
= get_context_addr_from_root(root
);
739 context
= (struct context_entry
*)
740 alloc_pgtable_page(iommu
->node
);
742 spin_unlock_irqrestore(&iommu
->lock
, flags
);
745 __iommu_flush_cache(iommu
, (void *)context
, CONTEXT_SIZE
);
746 phy_addr
= virt_to_phys((void *)context
);
747 set_root_value(root
, phy_addr
);
748 set_root_present(root
);
749 __iommu_flush_cache(iommu
, root
, sizeof(*root
));
751 spin_unlock_irqrestore(&iommu
->lock
, flags
);
752 return &context
[devfn
];
755 static int device_context_mapped(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
757 struct root_entry
*root
;
758 struct context_entry
*context
;
762 spin_lock_irqsave(&iommu
->lock
, flags
);
763 root
= &iommu
->root_entry
[bus
];
764 context
= get_context_addr_from_root(root
);
769 ret
= context_present(&context
[devfn
]);
771 spin_unlock_irqrestore(&iommu
->lock
, flags
);
775 static void clear_context_table(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
777 struct root_entry
*root
;
778 struct context_entry
*context
;
781 spin_lock_irqsave(&iommu
->lock
, flags
);
782 root
= &iommu
->root_entry
[bus
];
783 context
= get_context_addr_from_root(root
);
785 context_clear_entry(&context
[devfn
]);
786 __iommu_flush_cache(iommu
, &context
[devfn
], \
789 spin_unlock_irqrestore(&iommu
->lock
, flags
);
792 static void free_context_table(struct intel_iommu
*iommu
)
794 struct root_entry
*root
;
797 struct context_entry
*context
;
799 spin_lock_irqsave(&iommu
->lock
, flags
);
800 if (!iommu
->root_entry
) {
803 for (i
= 0; i
< ROOT_ENTRY_NR
; i
++) {
804 root
= &iommu
->root_entry
[i
];
805 context
= get_context_addr_from_root(root
);
807 free_pgtable_page(context
);
809 free_pgtable_page(iommu
->root_entry
);
810 iommu
->root_entry
= NULL
;
812 spin_unlock_irqrestore(&iommu
->lock
, flags
);
815 static struct dma_pte
*pfn_to_dma_pte(struct dmar_domain
*domain
,
816 unsigned long pfn
, int *target_level
)
818 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
819 struct dma_pte
*parent
, *pte
= NULL
;
820 int level
= agaw_to_level(domain
->agaw
);
823 BUG_ON(!domain
->pgd
);
825 if (addr_width
< BITS_PER_LONG
&& pfn
>> addr_width
)
826 /* Address beyond IOMMU's addressing capabilities. */
829 parent
= domain
->pgd
;
834 offset
= pfn_level_offset(pfn
, level
);
835 pte
= &parent
[offset
];
836 if (!*target_level
&& (dma_pte_superpage(pte
) || !dma_pte_present(pte
)))
838 if (level
== *target_level
)
841 if (!dma_pte_present(pte
)) {
844 tmp_page
= alloc_pgtable_page(domain
->nid
);
849 domain_flush_cache(domain
, tmp_page
, VTD_PAGE_SIZE
);
850 pteval
= ((uint64_t)virt_to_dma_pfn(tmp_page
) << VTD_PAGE_SHIFT
) | DMA_PTE_READ
| DMA_PTE_WRITE
;
851 if (cmpxchg64(&pte
->val
, 0ULL, pteval
)) {
852 /* Someone else set it while we were thinking; use theirs. */
853 free_pgtable_page(tmp_page
);
856 domain_flush_cache(domain
, pte
, sizeof(*pte
));
862 parent
= phys_to_virt(dma_pte_addr(pte
));
867 *target_level
= level
;
873 /* return address's pte at specific level */
874 static struct dma_pte
*dma_pfn_level_pte(struct dmar_domain
*domain
,
876 int level
, int *large_page
)
878 struct dma_pte
*parent
, *pte
= NULL
;
879 int total
= agaw_to_level(domain
->agaw
);
882 parent
= domain
->pgd
;
883 while (level
<= total
) {
884 offset
= pfn_level_offset(pfn
, total
);
885 pte
= &parent
[offset
];
889 if (!dma_pte_present(pte
)) {
894 if (pte
->val
& DMA_PTE_LARGE_PAGE
) {
899 parent
= phys_to_virt(dma_pte_addr(pte
));
905 /* clear last level pte, a tlb flush should be followed */
906 static void dma_pte_clear_range(struct dmar_domain
*domain
,
907 unsigned long start_pfn
,
908 unsigned long last_pfn
)
910 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
911 unsigned int large_page
= 1;
912 struct dma_pte
*first_pte
, *pte
;
914 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
915 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
916 BUG_ON(start_pfn
> last_pfn
);
918 /* we don't need lock here; nobody else touches the iova range */
921 first_pte
= pte
= dma_pfn_level_pte(domain
, start_pfn
, 1, &large_page
);
923 start_pfn
= align_to_level(start_pfn
+ 1, large_page
+ 1);
928 start_pfn
+= lvl_to_nr_pages(large_page
);
930 } while (start_pfn
<= last_pfn
&& !first_pte_in_page(pte
));
932 domain_flush_cache(domain
, first_pte
,
933 (void *)pte
- (void *)first_pte
);
935 } while (start_pfn
&& start_pfn
<= last_pfn
);
938 static void dma_pte_free_level(struct dmar_domain
*domain
, int level
,
939 struct dma_pte
*pte
, unsigned long pfn
,
940 unsigned long start_pfn
, unsigned long last_pfn
)
942 pfn
= max(start_pfn
, pfn
);
943 pte
= &pte
[pfn_level_offset(pfn
, level
)];
946 unsigned long level_pfn
;
947 struct dma_pte
*level_pte
;
949 if (!dma_pte_present(pte
) || dma_pte_superpage(pte
))
952 level_pfn
= pfn
& level_mask(level
- 1);
953 level_pte
= phys_to_virt(dma_pte_addr(pte
));
956 dma_pte_free_level(domain
, level
- 1, level_pte
,
957 level_pfn
, start_pfn
, last_pfn
);
959 /* If range covers entire pagetable, free it */
960 if (!(start_pfn
> level_pfn
||
961 last_pfn
< level_pfn
+ level_size(level
) - 1)) {
963 domain_flush_cache(domain
, pte
, sizeof(*pte
));
964 free_pgtable_page(level_pte
);
967 pfn
+= level_size(level
);
968 } while (!first_pte_in_page(++pte
) && pfn
<= last_pfn
);
971 /* free page table pages. last level pte should already be cleared */
972 static void dma_pte_free_pagetable(struct dmar_domain
*domain
,
973 unsigned long start_pfn
,
974 unsigned long last_pfn
)
976 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
978 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
979 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
980 BUG_ON(start_pfn
> last_pfn
);
982 /* We don't need lock here; nobody else touches the iova range */
983 dma_pte_free_level(domain
, agaw_to_level(domain
->agaw
),
984 domain
->pgd
, 0, start_pfn
, last_pfn
);
987 if (start_pfn
== 0 && last_pfn
== DOMAIN_MAX_PFN(domain
->gaw
)) {
988 free_pgtable_page(domain
->pgd
);
993 /* When a page at a given level is being unlinked from its parent, we don't
994 need to *modify* it at all. All we need to do is make a list of all the
995 pages which can be freed just as soon as we've flushed the IOTLB and we
996 know the hardware page-walk will no longer touch them.
997 The 'pte' argument is the *parent* PTE, pointing to the page that is to
999 static struct page
*dma_pte_list_pagetables(struct dmar_domain
*domain
,
1000 int level
, struct dma_pte
*pte
,
1001 struct page
*freelist
)
1005 pg
= pfn_to_page(dma_pte_addr(pte
) >> PAGE_SHIFT
);
1006 pg
->freelist
= freelist
;
1012 for (pte
= page_address(pg
); !first_pte_in_page(pte
); pte
++) {
1013 if (dma_pte_present(pte
) && !dma_pte_superpage(pte
))
1014 freelist
= dma_pte_list_pagetables(domain
, level
- 1,
1021 static struct page
*dma_pte_clear_level(struct dmar_domain
*domain
, int level
,
1022 struct dma_pte
*pte
, unsigned long pfn
,
1023 unsigned long start_pfn
,
1024 unsigned long last_pfn
,
1025 struct page
*freelist
)
1027 struct dma_pte
*first_pte
= NULL
, *last_pte
= NULL
;
1029 pfn
= max(start_pfn
, pfn
);
1030 pte
= &pte
[pfn_level_offset(pfn
, level
)];
1033 unsigned long level_pfn
;
1035 if (!dma_pte_present(pte
))
1038 level_pfn
= pfn
& level_mask(level
);
1040 /* If range covers entire pagetable, free it */
1041 if (start_pfn
<= level_pfn
&&
1042 last_pfn
>= level_pfn
+ level_size(level
) - 1) {
1043 /* These suborbinate page tables are going away entirely. Don't
1044 bother to clear them; we're just going to *free* them. */
1045 if (level
> 1 && !dma_pte_superpage(pte
))
1046 freelist
= dma_pte_list_pagetables(domain
, level
- 1, pte
, freelist
);
1052 } else if (level
> 1) {
1053 /* Recurse down into a level that isn't *entirely* obsolete */
1054 freelist
= dma_pte_clear_level(domain
, level
- 1,
1055 phys_to_virt(dma_pte_addr(pte
)),
1056 level_pfn
, start_pfn
, last_pfn
,
1060 pfn
+= level_size(level
);
1061 } while (!first_pte_in_page(++pte
) && pfn
<= last_pfn
);
1064 domain_flush_cache(domain
, first_pte
,
1065 (void *)++last_pte
- (void *)first_pte
);
1070 /* We can't just free the pages because the IOMMU may still be walking
1071 the page tables, and may have cached the intermediate levels. The
1072 pages can only be freed after the IOTLB flush has been done. */
1073 struct page
*domain_unmap(struct dmar_domain
*domain
,
1074 unsigned long start_pfn
,
1075 unsigned long last_pfn
)
1077 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
1078 struct page
*freelist
= NULL
;
1080 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
1081 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
1082 BUG_ON(start_pfn
> last_pfn
);
1084 /* we don't need lock here; nobody else touches the iova range */
1085 freelist
= dma_pte_clear_level(domain
, agaw_to_level(domain
->agaw
),
1086 domain
->pgd
, 0, start_pfn
, last_pfn
, NULL
);
1089 if (start_pfn
== 0 && last_pfn
== DOMAIN_MAX_PFN(domain
->gaw
)) {
1090 struct page
*pgd_page
= virt_to_page(domain
->pgd
);
1091 pgd_page
->freelist
= freelist
;
1092 freelist
= pgd_page
;
1100 void dma_free_pagelist(struct page
*freelist
)
1104 while ((pg
= freelist
)) {
1105 freelist
= pg
->freelist
;
1106 free_pgtable_page(page_address(pg
));
1110 /* iommu handling */
1111 static int iommu_alloc_root_entry(struct intel_iommu
*iommu
)
1113 struct root_entry
*root
;
1114 unsigned long flags
;
1116 root
= (struct root_entry
*)alloc_pgtable_page(iommu
->node
);
1120 __iommu_flush_cache(iommu
, root
, ROOT_SIZE
);
1122 spin_lock_irqsave(&iommu
->lock
, flags
);
1123 iommu
->root_entry
= root
;
1124 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1129 static void iommu_set_root_entry(struct intel_iommu
*iommu
)
1135 addr
= iommu
->root_entry
;
1137 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1138 dmar_writeq(iommu
->reg
+ DMAR_RTADDR_REG
, virt_to_phys(addr
));
1140 writel(iommu
->gcmd
| DMA_GCMD_SRTP
, iommu
->reg
+ DMAR_GCMD_REG
);
1142 /* Make sure hardware complete it */
1143 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1144 readl
, (sts
& DMA_GSTS_RTPS
), sts
);
1146 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1149 static void iommu_flush_write_buffer(struct intel_iommu
*iommu
)
1154 if (!rwbf_quirk
&& !cap_rwbf(iommu
->cap
))
1157 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1158 writel(iommu
->gcmd
| DMA_GCMD_WBF
, iommu
->reg
+ DMAR_GCMD_REG
);
1160 /* Make sure hardware complete it */
1161 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1162 readl
, (!(val
& DMA_GSTS_WBFS
)), val
);
1164 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1167 /* return value determine if we need a write buffer flush */
1168 static void __iommu_flush_context(struct intel_iommu
*iommu
,
1169 u16 did
, u16 source_id
, u8 function_mask
,
1176 case DMA_CCMD_GLOBAL_INVL
:
1177 val
= DMA_CCMD_GLOBAL_INVL
;
1179 case DMA_CCMD_DOMAIN_INVL
:
1180 val
= DMA_CCMD_DOMAIN_INVL
|DMA_CCMD_DID(did
);
1182 case DMA_CCMD_DEVICE_INVL
:
1183 val
= DMA_CCMD_DEVICE_INVL
|DMA_CCMD_DID(did
)
1184 | DMA_CCMD_SID(source_id
) | DMA_CCMD_FM(function_mask
);
1189 val
|= DMA_CCMD_ICC
;
1191 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1192 dmar_writeq(iommu
->reg
+ DMAR_CCMD_REG
, val
);
1194 /* Make sure hardware complete it */
1195 IOMMU_WAIT_OP(iommu
, DMAR_CCMD_REG
,
1196 dmar_readq
, (!(val
& DMA_CCMD_ICC
)), val
);
1198 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1201 /* return value determine if we need a write buffer flush */
1202 static void __iommu_flush_iotlb(struct intel_iommu
*iommu
, u16 did
,
1203 u64 addr
, unsigned int size_order
, u64 type
)
1205 int tlb_offset
= ecap_iotlb_offset(iommu
->ecap
);
1206 u64 val
= 0, val_iva
= 0;
1210 case DMA_TLB_GLOBAL_FLUSH
:
1211 /* global flush doesn't need set IVA_REG */
1212 val
= DMA_TLB_GLOBAL_FLUSH
|DMA_TLB_IVT
;
1214 case DMA_TLB_DSI_FLUSH
:
1215 val
= DMA_TLB_DSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
1217 case DMA_TLB_PSI_FLUSH
:
1218 val
= DMA_TLB_PSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
1219 /* IH bit is passed in as part of address */
1220 val_iva
= size_order
| addr
;
1225 /* Note: set drain read/write */
1228 * This is probably to be super secure.. Looks like we can
1229 * ignore it without any impact.
1231 if (cap_read_drain(iommu
->cap
))
1232 val
|= DMA_TLB_READ_DRAIN
;
1234 if (cap_write_drain(iommu
->cap
))
1235 val
|= DMA_TLB_WRITE_DRAIN
;
1237 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1238 /* Note: Only uses first TLB reg currently */
1240 dmar_writeq(iommu
->reg
+ tlb_offset
, val_iva
);
1241 dmar_writeq(iommu
->reg
+ tlb_offset
+ 8, val
);
1243 /* Make sure hardware complete it */
1244 IOMMU_WAIT_OP(iommu
, tlb_offset
+ 8,
1245 dmar_readq
, (!(val
& DMA_TLB_IVT
)), val
);
1247 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1249 /* check IOTLB invalidation granularity */
1250 if (DMA_TLB_IAIG(val
) == 0)
1251 printk(KERN_ERR
"IOMMU: flush IOTLB failed\n");
1252 if (DMA_TLB_IAIG(val
) != DMA_TLB_IIRG(type
))
1253 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1254 (unsigned long long)DMA_TLB_IIRG(type
),
1255 (unsigned long long)DMA_TLB_IAIG(val
));
1258 static struct device_domain_info
*
1259 iommu_support_dev_iotlb (struct dmar_domain
*domain
, struct intel_iommu
*iommu
,
1263 unsigned long flags
;
1264 struct device_domain_info
*info
;
1265 struct pci_dev
*pdev
;
1267 if (!ecap_dev_iotlb_support(iommu
->ecap
))
1273 spin_lock_irqsave(&device_domain_lock
, flags
);
1274 list_for_each_entry(info
, &domain
->devices
, link
)
1275 if (info
->bus
== bus
&& info
->devfn
== devfn
) {
1279 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1281 if (!found
|| !info
->dev
|| !dev_is_pci(info
->dev
))
1284 pdev
= to_pci_dev(info
->dev
);
1286 if (!pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_ATS
))
1289 if (!dmar_find_matched_atsr_unit(pdev
))
1295 static void iommu_enable_dev_iotlb(struct device_domain_info
*info
)
1297 if (!info
|| !dev_is_pci(info
->dev
))
1300 pci_enable_ats(to_pci_dev(info
->dev
), VTD_PAGE_SHIFT
);
1303 static void iommu_disable_dev_iotlb(struct device_domain_info
*info
)
1305 if (!info
->dev
|| !dev_is_pci(info
->dev
) ||
1306 !pci_ats_enabled(to_pci_dev(info
->dev
)))
1309 pci_disable_ats(to_pci_dev(info
->dev
));
1312 static void iommu_flush_dev_iotlb(struct dmar_domain
*domain
,
1313 u64 addr
, unsigned mask
)
1316 unsigned long flags
;
1317 struct device_domain_info
*info
;
1319 spin_lock_irqsave(&device_domain_lock
, flags
);
1320 list_for_each_entry(info
, &domain
->devices
, link
) {
1321 struct pci_dev
*pdev
;
1322 if (!info
->dev
|| !dev_is_pci(info
->dev
))
1325 pdev
= to_pci_dev(info
->dev
);
1326 if (!pci_ats_enabled(pdev
))
1329 sid
= info
->bus
<< 8 | info
->devfn
;
1330 qdep
= pci_ats_queue_depth(pdev
);
1331 qi_flush_dev_iotlb(info
->iommu
, sid
, qdep
, addr
, mask
);
1333 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1336 static void iommu_flush_iotlb_psi(struct intel_iommu
*iommu
, u16 did
,
1337 unsigned long pfn
, unsigned int pages
, int ih
, int map
)
1339 unsigned int mask
= ilog2(__roundup_pow_of_two(pages
));
1340 uint64_t addr
= (uint64_t)pfn
<< VTD_PAGE_SHIFT
;
1347 * Fallback to domain selective flush if no PSI support or the size is
1349 * PSI requires page size to be 2 ^ x, and the base address is naturally
1350 * aligned to the size
1352 if (!cap_pgsel_inv(iommu
->cap
) || mask
> cap_max_amask_val(iommu
->cap
))
1353 iommu
->flush
.flush_iotlb(iommu
, did
, 0, 0,
1356 iommu
->flush
.flush_iotlb(iommu
, did
, addr
| ih
, mask
,
1360 * In caching mode, changes of pages from non-present to present require
1361 * flush. However, device IOTLB doesn't need to be flushed in this case.
1363 if (!cap_caching_mode(iommu
->cap
) || !map
)
1364 iommu_flush_dev_iotlb(iommu
->domains
[did
], addr
, mask
);
1367 static void iommu_disable_protect_mem_regions(struct intel_iommu
*iommu
)
1370 unsigned long flags
;
1372 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
1373 pmen
= readl(iommu
->reg
+ DMAR_PMEN_REG
);
1374 pmen
&= ~DMA_PMEN_EPM
;
1375 writel(pmen
, iommu
->reg
+ DMAR_PMEN_REG
);
1377 /* wait for the protected region status bit to clear */
1378 IOMMU_WAIT_OP(iommu
, DMAR_PMEN_REG
,
1379 readl
, !(pmen
& DMA_PMEN_PRS
), pmen
);
1381 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1384 static int iommu_enable_translation(struct intel_iommu
*iommu
)
1387 unsigned long flags
;
1389 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
1390 iommu
->gcmd
|= DMA_GCMD_TE
;
1391 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1393 /* Make sure hardware complete it */
1394 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1395 readl
, (sts
& DMA_GSTS_TES
), sts
);
1397 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1401 static int iommu_disable_translation(struct intel_iommu
*iommu
)
1406 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
1407 iommu
->gcmd
&= ~DMA_GCMD_TE
;
1408 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1410 /* Make sure hardware complete it */
1411 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1412 readl
, (!(sts
& DMA_GSTS_TES
)), sts
);
1414 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1419 static int iommu_init_domains(struct intel_iommu
*iommu
)
1421 unsigned long ndomains
;
1422 unsigned long nlongs
;
1424 ndomains
= cap_ndoms(iommu
->cap
);
1425 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1426 iommu
->seq_id
, ndomains
);
1427 nlongs
= BITS_TO_LONGS(ndomains
);
1429 spin_lock_init(&iommu
->lock
);
1431 /* TBD: there might be 64K domains,
1432 * consider other allocation for future chip
1434 iommu
->domain_ids
= kcalloc(nlongs
, sizeof(unsigned long), GFP_KERNEL
);
1435 if (!iommu
->domain_ids
) {
1436 pr_err("IOMMU%d: allocating domain id array failed\n",
1440 iommu
->domains
= kcalloc(ndomains
, sizeof(struct dmar_domain
*),
1442 if (!iommu
->domains
) {
1443 pr_err("IOMMU%d: allocating domain array failed\n",
1445 kfree(iommu
->domain_ids
);
1446 iommu
->domain_ids
= NULL
;
1451 * if Caching mode is set, then invalid translations are tagged
1452 * with domainid 0. Hence we need to pre-allocate it.
1454 if (cap_caching_mode(iommu
->cap
))
1455 set_bit(0, iommu
->domain_ids
);
1459 static void free_dmar_iommu(struct intel_iommu
*iommu
)
1461 struct dmar_domain
*domain
;
1463 unsigned long flags
;
1465 if ((iommu
->domains
) && (iommu
->domain_ids
)) {
1466 for_each_set_bit(i
, iommu
->domain_ids
, cap_ndoms(iommu
->cap
)) {
1468 * Domain id 0 is reserved for invalid translation
1469 * if hardware supports caching mode.
1471 if (cap_caching_mode(iommu
->cap
) && i
== 0)
1474 domain
= iommu
->domains
[i
];
1475 clear_bit(i
, iommu
->domain_ids
);
1477 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1478 count
= --domain
->iommu_count
;
1479 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1481 domain_exit(domain
);
1485 if (iommu
->gcmd
& DMA_GCMD_TE
)
1486 iommu_disable_translation(iommu
);
1488 kfree(iommu
->domains
);
1489 kfree(iommu
->domain_ids
);
1490 iommu
->domains
= NULL
;
1491 iommu
->domain_ids
= NULL
;
1493 g_iommus
[iommu
->seq_id
] = NULL
;
1495 /* free context mapping */
1496 free_context_table(iommu
);
1499 static struct dmar_domain
*alloc_domain(bool vm
)
1501 /* domain id for virtual machine, it won't be set in context */
1502 static atomic_t vm_domid
= ATOMIC_INIT(0);
1503 struct dmar_domain
*domain
;
1505 domain
= alloc_domain_mem();
1510 domain
->iommu_count
= 0;
1511 memset(domain
->iommu_bmp
, 0, sizeof(domain
->iommu_bmp
));
1513 spin_lock_init(&domain
->iommu_lock
);
1514 INIT_LIST_HEAD(&domain
->devices
);
1516 domain
->id
= atomic_inc_return(&vm_domid
);
1517 domain
->flags
= DOMAIN_FLAG_VIRTUAL_MACHINE
;
1523 static int iommu_attach_domain(struct dmar_domain
*domain
,
1524 struct intel_iommu
*iommu
)
1527 unsigned long ndomains
;
1528 unsigned long flags
;
1530 ndomains
= cap_ndoms(iommu
->cap
);
1532 spin_lock_irqsave(&iommu
->lock
, flags
);
1534 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1535 if (num
>= ndomains
) {
1536 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1537 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1542 domain
->iommu_count
++;
1543 set_bit(num
, iommu
->domain_ids
);
1544 set_bit(iommu
->seq_id
, domain
->iommu_bmp
);
1545 iommu
->domains
[num
] = domain
;
1546 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1551 static void iommu_detach_domain(struct dmar_domain
*domain
,
1552 struct intel_iommu
*iommu
)
1554 unsigned long flags
;
1557 spin_lock_irqsave(&iommu
->lock
, flags
);
1558 ndomains
= cap_ndoms(iommu
->cap
);
1559 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
1560 if (iommu
->domains
[num
] == domain
) {
1561 clear_bit(num
, iommu
->domain_ids
);
1562 iommu
->domains
[num
] = NULL
;
1566 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1569 static struct iova_domain reserved_iova_list
;
1570 static struct lock_class_key reserved_rbtree_key
;
1572 static int dmar_init_reserved_ranges(void)
1574 struct pci_dev
*pdev
= NULL
;
1578 init_iova_domain(&reserved_iova_list
, DMA_32BIT_PFN
);
1580 lockdep_set_class(&reserved_iova_list
.iova_rbtree_lock
,
1581 &reserved_rbtree_key
);
1583 /* IOAPIC ranges shouldn't be accessed by DMA */
1584 iova
= reserve_iova(&reserved_iova_list
, IOVA_PFN(IOAPIC_RANGE_START
),
1585 IOVA_PFN(IOAPIC_RANGE_END
));
1587 printk(KERN_ERR
"Reserve IOAPIC range failed\n");
1591 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1592 for_each_pci_dev(pdev
) {
1595 for (i
= 0; i
< PCI_NUM_RESOURCES
; i
++) {
1596 r
= &pdev
->resource
[i
];
1597 if (!r
->flags
|| !(r
->flags
& IORESOURCE_MEM
))
1599 iova
= reserve_iova(&reserved_iova_list
,
1603 printk(KERN_ERR
"Reserve iova failed\n");
1611 static void domain_reserve_special_ranges(struct dmar_domain
*domain
)
1613 copy_reserved_iova(&reserved_iova_list
, &domain
->iovad
);
1616 static inline int guestwidth_to_adjustwidth(int gaw
)
1619 int r
= (gaw
- 12) % 9;
1630 static int domain_init(struct dmar_domain
*domain
, int guest_width
)
1632 struct intel_iommu
*iommu
;
1633 int adjust_width
, agaw
;
1634 unsigned long sagaw
;
1636 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
1637 domain_reserve_special_ranges(domain
);
1639 /* calculate AGAW */
1640 iommu
= domain_get_iommu(domain
);
1641 if (guest_width
> cap_mgaw(iommu
->cap
))
1642 guest_width
= cap_mgaw(iommu
->cap
);
1643 domain
->gaw
= guest_width
;
1644 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
1645 agaw
= width_to_agaw(adjust_width
);
1646 sagaw
= cap_sagaw(iommu
->cap
);
1647 if (!test_bit(agaw
, &sagaw
)) {
1648 /* hardware doesn't support it, choose a bigger one */
1649 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw
);
1650 agaw
= find_next_bit(&sagaw
, 5, agaw
);
1654 domain
->agaw
= agaw
;
1656 if (ecap_coherent(iommu
->ecap
))
1657 domain
->iommu_coherency
= 1;
1659 domain
->iommu_coherency
= 0;
1661 if (ecap_sc_support(iommu
->ecap
))
1662 domain
->iommu_snooping
= 1;
1664 domain
->iommu_snooping
= 0;
1666 if (intel_iommu_superpage
)
1667 domain
->iommu_superpage
= fls(cap_super_page_val(iommu
->cap
));
1669 domain
->iommu_superpage
= 0;
1671 domain
->nid
= iommu
->node
;
1673 /* always allocate the top pgd */
1674 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page(domain
->nid
);
1677 __iommu_flush_cache(iommu
, domain
->pgd
, PAGE_SIZE
);
1681 static void domain_exit(struct dmar_domain
*domain
)
1683 struct dmar_drhd_unit
*drhd
;
1684 struct intel_iommu
*iommu
;
1685 struct page
*freelist
= NULL
;
1687 /* Domain 0 is reserved, so dont process it */
1691 /* Flush any lazy unmaps that may reference this domain */
1692 if (!intel_iommu_strict
)
1693 flush_unmaps_timeout(0);
1695 /* remove associated devices */
1696 domain_remove_dev_info(domain
);
1699 put_iova_domain(&domain
->iovad
);
1701 freelist
= domain_unmap(domain
, 0, DOMAIN_MAX_PFN(domain
->gaw
));
1703 /* clear attached or cached domains */
1705 for_each_active_iommu(iommu
, drhd
)
1706 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
1707 test_bit(iommu
->seq_id
, domain
->iommu_bmp
))
1708 iommu_detach_domain(domain
, iommu
);
1711 dma_free_pagelist(freelist
);
1713 free_domain_mem(domain
);
1716 static int domain_context_mapping_one(struct dmar_domain
*domain
,
1717 struct intel_iommu
*iommu
,
1718 u8 bus
, u8 devfn
, int translation
)
1720 struct context_entry
*context
;
1721 unsigned long flags
;
1722 struct dma_pte
*pgd
;
1724 unsigned long ndomains
;
1727 struct device_domain_info
*info
= NULL
;
1729 pr_debug("Set context mapping for %02x:%02x.%d\n",
1730 bus
, PCI_SLOT(devfn
), PCI_FUNC(devfn
));
1732 BUG_ON(!domain
->pgd
);
1733 BUG_ON(translation
!= CONTEXT_TT_PASS_THROUGH
&&
1734 translation
!= CONTEXT_TT_MULTI_LEVEL
);
1736 context
= device_to_context_entry(iommu
, bus
, devfn
);
1739 spin_lock_irqsave(&iommu
->lock
, flags
);
1740 if (context_present(context
)) {
1741 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1748 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
1749 domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
) {
1752 /* find an available domain id for this device in iommu */
1753 ndomains
= cap_ndoms(iommu
->cap
);
1754 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
1755 if (iommu
->domains
[num
] == domain
) {
1763 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1764 if (num
>= ndomains
) {
1765 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1766 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1770 set_bit(num
, iommu
->domain_ids
);
1771 iommu
->domains
[num
] = domain
;
1775 /* Skip top levels of page tables for
1776 * iommu which has less agaw than default.
1777 * Unnecessary for PT mode.
1779 if (translation
!= CONTEXT_TT_PASS_THROUGH
) {
1780 for (agaw
= domain
->agaw
; agaw
!= iommu
->agaw
; agaw
--) {
1781 pgd
= phys_to_virt(dma_pte_addr(pgd
));
1782 if (!dma_pte_present(pgd
)) {
1783 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1790 context_set_domain_id(context
, id
);
1792 if (translation
!= CONTEXT_TT_PASS_THROUGH
) {
1793 info
= iommu_support_dev_iotlb(domain
, iommu
, bus
, devfn
);
1794 translation
= info
? CONTEXT_TT_DEV_IOTLB
:
1795 CONTEXT_TT_MULTI_LEVEL
;
1798 * In pass through mode, AW must be programmed to indicate the largest
1799 * AGAW value supported by hardware. And ASR is ignored by hardware.
1801 if (unlikely(translation
== CONTEXT_TT_PASS_THROUGH
))
1802 context_set_address_width(context
, iommu
->msagaw
);
1804 context_set_address_root(context
, virt_to_phys(pgd
));
1805 context_set_address_width(context
, iommu
->agaw
);
1808 context_set_translation_type(context
, translation
);
1809 context_set_fault_enable(context
);
1810 context_set_present(context
);
1811 domain_flush_cache(domain
, context
, sizeof(*context
));
1814 * It's a non-present to present mapping. If hardware doesn't cache
1815 * non-present entry we only need to flush the write-buffer. If the
1816 * _does_ cache non-present entries, then it does so in the special
1817 * domain #0, which we have to flush:
1819 if (cap_caching_mode(iommu
->cap
)) {
1820 iommu
->flush
.flush_context(iommu
, 0,
1821 (((u16
)bus
) << 8) | devfn
,
1822 DMA_CCMD_MASK_NOBIT
,
1823 DMA_CCMD_DEVICE_INVL
);
1824 iommu
->flush
.flush_iotlb(iommu
, domain
->id
, 0, 0, DMA_TLB_DSI_FLUSH
);
1826 iommu_flush_write_buffer(iommu
);
1828 iommu_enable_dev_iotlb(info
);
1829 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1831 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1832 if (!test_and_set_bit(iommu
->seq_id
, domain
->iommu_bmp
)) {
1833 domain
->iommu_count
++;
1834 if (domain
->iommu_count
== 1)
1835 domain
->nid
= iommu
->node
;
1836 domain_update_iommu_cap(domain
);
1838 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1843 domain_context_mapping(struct dmar_domain
*domain
, struct device
*dev
,
1847 struct pci_dev
*pdev
, *tmp
, *parent
;
1848 struct intel_iommu
*iommu
;
1851 iommu
= device_to_iommu(dev
, &bus
, &devfn
);
1855 ret
= domain_context_mapping_one(domain
, iommu
, bus
, devfn
,
1857 if (ret
|| !dev_is_pci(dev
))
1860 /* dependent device mapping */
1861 pdev
= to_pci_dev(dev
);
1862 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1865 /* Secondary interface's bus number and devfn 0 */
1866 parent
= pdev
->bus
->self
;
1867 while (parent
!= tmp
) {
1868 ret
= domain_context_mapping_one(domain
, iommu
,
1869 parent
->bus
->number
,
1870 parent
->devfn
, translation
);
1873 parent
= parent
->bus
->self
;
1875 if (pci_is_pcie(tmp
)) /* this is a PCIe-to-PCI bridge */
1876 return domain_context_mapping_one(domain
, iommu
,
1877 tmp
->subordinate
->number
, 0,
1879 else /* this is a legacy PCI bridge */
1880 return domain_context_mapping_one(domain
, iommu
,
1886 static int domain_context_mapped(struct device
*dev
)
1889 struct pci_dev
*pdev
, *tmp
, *parent
;
1890 struct intel_iommu
*iommu
;
1893 iommu
= device_to_iommu(dev
, &bus
, &devfn
);
1897 ret
= device_context_mapped(iommu
, bus
, devfn
);
1898 if (!ret
|| !dev_is_pci(dev
))
1901 /* dependent device mapping */
1902 pdev
= to_pci_dev(dev
);
1903 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1906 /* Secondary interface's bus number and devfn 0 */
1907 parent
= pdev
->bus
->self
;
1908 while (parent
!= tmp
) {
1909 ret
= device_context_mapped(iommu
, parent
->bus
->number
,
1913 parent
= parent
->bus
->self
;
1915 if (pci_is_pcie(tmp
))
1916 return device_context_mapped(iommu
, tmp
->subordinate
->number
,
1919 return device_context_mapped(iommu
, tmp
->bus
->number
,
1923 /* Returns a number of VTD pages, but aligned to MM page size */
1924 static inline unsigned long aligned_nrpages(unsigned long host_addr
,
1927 host_addr
&= ~PAGE_MASK
;
1928 return PAGE_ALIGN(host_addr
+ size
) >> VTD_PAGE_SHIFT
;
1931 /* Return largest possible superpage level for a given mapping */
1932 static inline int hardware_largepage_caps(struct dmar_domain
*domain
,
1933 unsigned long iov_pfn
,
1934 unsigned long phy_pfn
,
1935 unsigned long pages
)
1937 int support
, level
= 1;
1938 unsigned long pfnmerge
;
1940 support
= domain
->iommu_superpage
;
1942 /* To use a large page, the virtual *and* physical addresses
1943 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1944 of them will mean we have to use smaller pages. So just
1945 merge them and check both at once. */
1946 pfnmerge
= iov_pfn
| phy_pfn
;
1948 while (support
&& !(pfnmerge
& ~VTD_STRIDE_MASK
)) {
1949 pages
>>= VTD_STRIDE_SHIFT
;
1952 pfnmerge
>>= VTD_STRIDE_SHIFT
;
1959 static int __domain_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
1960 struct scatterlist
*sg
, unsigned long phys_pfn
,
1961 unsigned long nr_pages
, int prot
)
1963 struct dma_pte
*first_pte
= NULL
, *pte
= NULL
;
1964 phys_addr_t
uninitialized_var(pteval
);
1965 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
1966 unsigned long sg_res
;
1967 unsigned int largepage_lvl
= 0;
1968 unsigned long lvl_pages
= 0;
1970 BUG_ON(addr_width
< BITS_PER_LONG
&& (iov_pfn
+ nr_pages
- 1) >> addr_width
);
1972 if ((prot
& (DMA_PTE_READ
|DMA_PTE_WRITE
)) == 0)
1975 prot
&= DMA_PTE_READ
| DMA_PTE_WRITE
| DMA_PTE_SNP
;
1980 sg_res
= nr_pages
+ 1;
1981 pteval
= ((phys_addr_t
)phys_pfn
<< VTD_PAGE_SHIFT
) | prot
;
1984 while (nr_pages
> 0) {
1988 sg_res
= aligned_nrpages(sg
->offset
, sg
->length
);
1989 sg
->dma_address
= ((dma_addr_t
)iov_pfn
<< VTD_PAGE_SHIFT
) + sg
->offset
;
1990 sg
->dma_length
= sg
->length
;
1991 pteval
= page_to_phys(sg_page(sg
)) | prot
;
1992 phys_pfn
= pteval
>> VTD_PAGE_SHIFT
;
1996 largepage_lvl
= hardware_largepage_caps(domain
, iov_pfn
, phys_pfn
, sg_res
);
1998 first_pte
= pte
= pfn_to_dma_pte(domain
, iov_pfn
, &largepage_lvl
);
2001 /* It is large page*/
2002 if (largepage_lvl
> 1) {
2003 pteval
|= DMA_PTE_LARGE_PAGE
;
2004 /* Ensure that old small page tables are removed to make room
2005 for superpage, if they exist. */
2006 dma_pte_clear_range(domain
, iov_pfn
,
2007 iov_pfn
+ lvl_to_nr_pages(largepage_lvl
) - 1);
2008 dma_pte_free_pagetable(domain
, iov_pfn
,
2009 iov_pfn
+ lvl_to_nr_pages(largepage_lvl
) - 1);
2011 pteval
&= ~(uint64_t)DMA_PTE_LARGE_PAGE
;
2015 /* We don't need lock here, nobody else
2016 * touches the iova range
2018 tmp
= cmpxchg64_local(&pte
->val
, 0ULL, pteval
);
2020 static int dumps
= 5;
2021 printk(KERN_CRIT
"ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2022 iov_pfn
, tmp
, (unsigned long long)pteval
);
2025 debug_dma_dump_mappings(NULL
);
2030 lvl_pages
= lvl_to_nr_pages(largepage_lvl
);
2032 BUG_ON(nr_pages
< lvl_pages
);
2033 BUG_ON(sg_res
< lvl_pages
);
2035 nr_pages
-= lvl_pages
;
2036 iov_pfn
+= lvl_pages
;
2037 phys_pfn
+= lvl_pages
;
2038 pteval
+= lvl_pages
* VTD_PAGE_SIZE
;
2039 sg_res
-= lvl_pages
;
2041 /* If the next PTE would be the first in a new page, then we
2042 need to flush the cache on the entries we've just written.
2043 And then we'll need to recalculate 'pte', so clear it and
2044 let it get set again in the if (!pte) block above.
2046 If we're done (!nr_pages) we need to flush the cache too.
2048 Also if we've been setting superpages, we may need to
2049 recalculate 'pte' and switch back to smaller pages for the
2050 end of the mapping, if the trailing size is not enough to
2051 use another superpage (i.e. sg_res < lvl_pages). */
2053 if (!nr_pages
|| first_pte_in_page(pte
) ||
2054 (largepage_lvl
> 1 && sg_res
< lvl_pages
)) {
2055 domain_flush_cache(domain
, first_pte
,
2056 (void *)pte
- (void *)first_pte
);
2060 if (!sg_res
&& nr_pages
)
2066 static inline int domain_sg_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
2067 struct scatterlist
*sg
, unsigned long nr_pages
,
2070 return __domain_mapping(domain
, iov_pfn
, sg
, 0, nr_pages
, prot
);
2073 static inline int domain_pfn_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
2074 unsigned long phys_pfn
, unsigned long nr_pages
,
2077 return __domain_mapping(domain
, iov_pfn
, NULL
, phys_pfn
, nr_pages
, prot
);
2080 static void iommu_detach_dev(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
2085 clear_context_table(iommu
, bus
, devfn
);
2086 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
2087 DMA_CCMD_GLOBAL_INVL
);
2088 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
2091 static inline void unlink_domain_info(struct device_domain_info
*info
)
2093 assert_spin_locked(&device_domain_lock
);
2094 list_del(&info
->link
);
2095 list_del(&info
->global
);
2097 info
->dev
->archdata
.iommu
= NULL
;
2100 static void domain_remove_dev_info(struct dmar_domain
*domain
)
2102 struct device_domain_info
*info
;
2103 unsigned long flags
, flags2
;
2105 spin_lock_irqsave(&device_domain_lock
, flags
);
2106 while (!list_empty(&domain
->devices
)) {
2107 info
= list_entry(domain
->devices
.next
,
2108 struct device_domain_info
, link
);
2109 unlink_domain_info(info
);
2110 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2112 iommu_disable_dev_iotlb(info
);
2113 iommu_detach_dev(info
->iommu
, info
->bus
, info
->devfn
);
2115 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) {
2116 iommu_detach_dependent_devices(info
->iommu
, info
->dev
);
2117 /* clear this iommu in iommu_bmp, update iommu count
2120 spin_lock_irqsave(&domain
->iommu_lock
, flags2
);
2121 if (test_and_clear_bit(info
->iommu
->seq_id
,
2122 domain
->iommu_bmp
)) {
2123 domain
->iommu_count
--;
2124 domain_update_iommu_cap(domain
);
2126 spin_unlock_irqrestore(&domain
->iommu_lock
, flags2
);
2129 free_devinfo_mem(info
);
2130 spin_lock_irqsave(&device_domain_lock
, flags
);
2132 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2137 * Note: we use struct device->archdata.iommu stores the info
2139 static struct dmar_domain
*find_domain(struct device
*dev
)
2141 struct device_domain_info
*info
;
2143 /* No lock here, assumes no domain exit in normal case */
2144 info
= dev
->archdata
.iommu
;
2146 return info
->domain
;
2150 static inline struct device_domain_info
*
2151 dmar_search_domain_by_dev_info(int segment
, int bus
, int devfn
)
2153 struct device_domain_info
*info
;
2155 list_for_each_entry(info
, &device_domain_list
, global
)
2156 if (info
->iommu
->segment
== segment
&& info
->bus
== bus
&&
2157 info
->devfn
== devfn
)
2163 static struct dmar_domain
*dmar_insert_dev_info(struct intel_iommu
*iommu
,
2166 struct dmar_domain
*domain
)
2168 struct dmar_domain
*found
= NULL
;
2169 struct device_domain_info
*info
;
2170 unsigned long flags
;
2172 info
= alloc_devinfo_mem();
2177 info
->devfn
= devfn
;
2179 info
->domain
= domain
;
2180 info
->iommu
= iommu
;
2182 domain
->flags
|= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES
;
2184 spin_lock_irqsave(&device_domain_lock
, flags
);
2186 found
= find_domain(dev
);
2188 struct device_domain_info
*info2
;
2189 info2
= dmar_search_domain_by_dev_info(iommu
->segment
, bus
, devfn
);
2191 found
= info2
->domain
;
2194 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2195 free_devinfo_mem(info
);
2196 /* Caller must free the original domain */
2200 list_add(&info
->link
, &domain
->devices
);
2201 list_add(&info
->global
, &device_domain_list
);
2203 dev
->archdata
.iommu
= info
;
2204 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2209 /* domain is initialized */
2210 static struct dmar_domain
*get_domain_for_dev(struct device
*dev
, int gaw
)
2212 struct dmar_domain
*domain
, *free
= NULL
;
2213 struct intel_iommu
*iommu
= NULL
;
2214 struct device_domain_info
*info
;
2215 struct pci_dev
*dev_tmp
= NULL
;
2216 unsigned long flags
;
2217 u8 bus
, devfn
, bridge_bus
, bridge_devfn
;
2219 domain
= find_domain(dev
);
2223 if (dev_is_pci(dev
)) {
2224 struct pci_dev
*pdev
= to_pci_dev(dev
);
2227 segment
= pci_domain_nr(pdev
->bus
);
2228 dev_tmp
= pci_find_upstream_pcie_bridge(pdev
);
2230 if (pci_is_pcie(dev_tmp
)) {
2231 bridge_bus
= dev_tmp
->subordinate
->number
;
2234 bridge_bus
= dev_tmp
->bus
->number
;
2235 bridge_devfn
= dev_tmp
->devfn
;
2237 spin_lock_irqsave(&device_domain_lock
, flags
);
2238 info
= dmar_search_domain_by_dev_info(segment
, bus
, devfn
);
2240 iommu
= info
->iommu
;
2241 domain
= info
->domain
;
2243 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2244 /* pcie-pci bridge already has a domain, uses it */
2250 iommu
= device_to_iommu(dev
, &bus
, &devfn
);
2254 /* Allocate and initialize new domain for the device */
2255 domain
= alloc_domain(false);
2258 if (iommu_attach_domain(domain
, iommu
)) {
2259 free_domain_mem(domain
);
2263 if (domain_init(domain
, gaw
))
2266 /* register pcie-to-pci device */
2268 domain
= dmar_insert_dev_info(iommu
, bridge_bus
, bridge_devfn
,
2275 domain
= dmar_insert_dev_info(iommu
, bus
, devfn
, dev
, domain
);
2283 static int iommu_identity_mapping
;
2284 #define IDENTMAP_ALL 1
2285 #define IDENTMAP_GFX 2
2286 #define IDENTMAP_AZALIA 4
2288 static int iommu_domain_identity_map(struct dmar_domain
*domain
,
2289 unsigned long long start
,
2290 unsigned long long end
)
2292 unsigned long first_vpfn
= start
>> VTD_PAGE_SHIFT
;
2293 unsigned long last_vpfn
= end
>> VTD_PAGE_SHIFT
;
2295 if (!reserve_iova(&domain
->iovad
, dma_to_mm_pfn(first_vpfn
),
2296 dma_to_mm_pfn(last_vpfn
))) {
2297 printk(KERN_ERR
"IOMMU: reserve iova failed\n");
2301 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2302 start
, end
, domain
->id
);
2304 * RMRR range might have overlap with physical memory range,
2307 dma_pte_clear_range(domain
, first_vpfn
, last_vpfn
);
2309 return domain_pfn_mapping(domain
, first_vpfn
, first_vpfn
,
2310 last_vpfn
- first_vpfn
+ 1,
2311 DMA_PTE_READ
|DMA_PTE_WRITE
);
2314 static int iommu_prepare_identity_map(struct device
*dev
,
2315 unsigned long long start
,
2316 unsigned long long end
)
2318 struct dmar_domain
*domain
;
2321 domain
= get_domain_for_dev(dev
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
2325 /* For _hardware_ passthrough, don't bother. But for software
2326 passthrough, we do it anyway -- it may indicate a memory
2327 range which is reserved in E820, so which didn't get set
2328 up to start with in si_domain */
2329 if (domain
== si_domain
&& hw_pass_through
) {
2330 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2331 dev_name(dev
), start
, end
);
2336 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2337 dev_name(dev
), start
, end
);
2340 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2341 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2342 dmi_get_system_info(DMI_BIOS_VENDOR
),
2343 dmi_get_system_info(DMI_BIOS_VERSION
),
2344 dmi_get_system_info(DMI_PRODUCT_VERSION
));
2349 if (end
>> agaw_to_width(domain
->agaw
)) {
2350 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2351 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2352 agaw_to_width(domain
->agaw
),
2353 dmi_get_system_info(DMI_BIOS_VENDOR
),
2354 dmi_get_system_info(DMI_BIOS_VERSION
),
2355 dmi_get_system_info(DMI_PRODUCT_VERSION
));
2360 ret
= iommu_domain_identity_map(domain
, start
, end
);
2364 /* context entry init */
2365 ret
= domain_context_mapping(domain
, dev
, CONTEXT_TT_MULTI_LEVEL
);
2372 domain_exit(domain
);
2376 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit
*rmrr
,
2379 if (dev
->archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2381 return iommu_prepare_identity_map(dev
, rmrr
->base_address
,
2385 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2386 static inline void iommu_prepare_isa(void)
2388 struct pci_dev
*pdev
;
2391 pdev
= pci_get_class(PCI_CLASS_BRIDGE_ISA
<< 8, NULL
);
2395 printk(KERN_INFO
"IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2396 ret
= iommu_prepare_identity_map(&pdev
->dev
, 0, 16*1024*1024 - 1);
2399 printk(KERN_ERR
"IOMMU: Failed to create 0-16MiB identity map; "
2400 "floppy might not work\n");
2404 static inline void iommu_prepare_isa(void)
2408 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2410 static int md_domain_init(struct dmar_domain
*domain
, int guest_width
);
2412 static int __init
si_domain_init(int hw
)
2414 struct dmar_drhd_unit
*drhd
;
2415 struct intel_iommu
*iommu
;
2418 si_domain
= alloc_domain(false);
2422 si_domain
->flags
= DOMAIN_FLAG_STATIC_IDENTITY
;
2424 for_each_active_iommu(iommu
, drhd
) {
2425 ret
= iommu_attach_domain(si_domain
, iommu
);
2427 domain_exit(si_domain
);
2432 if (md_domain_init(si_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
2433 domain_exit(si_domain
);
2437 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2443 for_each_online_node(nid
) {
2444 unsigned long start_pfn
, end_pfn
;
2447 for_each_mem_pfn_range(i
, nid
, &start_pfn
, &end_pfn
, NULL
) {
2448 ret
= iommu_domain_identity_map(si_domain
,
2449 PFN_PHYS(start_pfn
), PFN_PHYS(end_pfn
));
2458 static int identity_mapping(struct device
*dev
)
2460 struct device_domain_info
*info
;
2462 if (likely(!iommu_identity_mapping
))
2465 info
= dev
->archdata
.iommu
;
2466 if (info
&& info
!= DUMMY_DEVICE_DOMAIN_INFO
)
2467 return (info
->domain
== si_domain
);
2472 static int domain_add_dev_info(struct dmar_domain
*domain
,
2473 struct pci_dev
*pdev
,
2476 struct dmar_domain
*ndomain
;
2477 struct intel_iommu
*iommu
;
2481 iommu
= device_to_iommu(&pdev
->dev
, &bus
, &devfn
);
2485 ndomain
= dmar_insert_dev_info(iommu
, bus
, devfn
, &pdev
->dev
, domain
);
2486 if (ndomain
!= domain
)
2489 ret
= domain_context_mapping(domain
, &pdev
->dev
, translation
);
2491 domain_remove_one_dev_info(domain
, &pdev
->dev
);
2498 static bool device_has_rmrr(struct device
*dev
)
2500 struct dmar_rmrr_unit
*rmrr
;
2505 for_each_rmrr_units(rmrr
) {
2507 * Return TRUE if this RMRR contains the device that
2510 for_each_active_dev_scope(rmrr
->devices
,
2511 rmrr
->devices_cnt
, i
, tmp
)
2521 static int iommu_should_identity_map(struct device
*dev
, int startup
)
2524 if (dev_is_pci(dev
)) {
2525 struct pci_dev
*pdev
= to_pci_dev(dev
);
2528 * We want to prevent any device associated with an RMRR from
2529 * getting placed into the SI Domain. This is done because
2530 * problems exist when devices are moved in and out of domains
2531 * and their respective RMRR info is lost. We exempt USB devices
2532 * from this process due to their usage of RMRRs that are known
2533 * to not be needed after BIOS hand-off to OS.
2535 if (device_has_rmrr(dev
) &&
2536 (pdev
->class >> 8) != PCI_CLASS_SERIAL_USB
)
2539 if ((iommu_identity_mapping
& IDENTMAP_AZALIA
) && IS_AZALIA(pdev
))
2542 if ((iommu_identity_mapping
& IDENTMAP_GFX
) && IS_GFX_DEVICE(pdev
))
2545 if (!(iommu_identity_mapping
& IDENTMAP_ALL
))
2549 * We want to start off with all devices in the 1:1 domain, and
2550 * take them out later if we find they can't access all of memory.
2552 * However, we can't do this for PCI devices behind bridges,
2553 * because all PCI devices behind the same bridge will end up
2554 * with the same source-id on their transactions.
2556 * Practically speaking, we can't change things around for these
2557 * devices at run-time, because we can't be sure there'll be no
2558 * DMA transactions in flight for any of their siblings.
2560 * So PCI devices (unless they're on the root bus) as well as
2561 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2562 * the 1:1 domain, just in _case_ one of their siblings turns out
2563 * not to be able to map all of memory.
2565 if (!pci_is_pcie(pdev
)) {
2566 if (!pci_is_root_bus(pdev
->bus
))
2568 if (pdev
->class >> 8 == PCI_CLASS_BRIDGE_PCI
)
2570 } else if (pci_pcie_type(pdev
) == PCI_EXP_TYPE_PCI_BRIDGE
)
2573 if (device_has_rmrr(dev
))
2578 * At boot time, we don't yet know if devices will be 64-bit capable.
2579 * Assume that they will — if they turn out not to be, then we can
2580 * take them out of the 1:1 domain later.
2584 * If the device's dma_mask is less than the system's memory
2585 * size then this is not a candidate for identity mapping.
2587 u64 dma_mask
= *dev
->dma_mask
;
2589 if (dev
->coherent_dma_mask
&&
2590 dev
->coherent_dma_mask
< dma_mask
)
2591 dma_mask
= dev
->coherent_dma_mask
;
2593 return dma_mask
>= dma_get_required_mask(dev
);
2599 static int __init
iommu_prepare_static_identity_mapping(int hw
)
2601 struct pci_dev
*pdev
= NULL
;
2604 ret
= si_domain_init(hw
);
2608 for_each_pci_dev(pdev
) {
2609 if (iommu_should_identity_map(&pdev
->dev
, 1)) {
2610 ret
= domain_add_dev_info(si_domain
, pdev
,
2611 hw
? CONTEXT_TT_PASS_THROUGH
:
2612 CONTEXT_TT_MULTI_LEVEL
);
2614 /* device not associated with an iommu */
2619 pr_info("IOMMU: %s identity mapping for device %s\n",
2620 hw
? "hardware" : "software", pci_name(pdev
));
2627 static int __init
init_dmars(void)
2629 struct dmar_drhd_unit
*drhd
;
2630 struct dmar_rmrr_unit
*rmrr
;
2632 struct intel_iommu
*iommu
;
2638 * initialize and program root entry to not present
2641 for_each_drhd_unit(drhd
) {
2643 * lock not needed as this is only incremented in the single
2644 * threaded kernel __init code path all other access are read
2647 if (g_num_of_iommus
< IOMMU_UNITS_SUPPORTED
) {
2651 printk_once(KERN_ERR
"intel-iommu: exceeded %d IOMMUs\n",
2652 IOMMU_UNITS_SUPPORTED
);
2655 g_iommus
= kcalloc(g_num_of_iommus
, sizeof(struct intel_iommu
*),
2658 printk(KERN_ERR
"Allocating global iommu array failed\n");
2663 deferred_flush
= kzalloc(g_num_of_iommus
*
2664 sizeof(struct deferred_flush_tables
), GFP_KERNEL
);
2665 if (!deferred_flush
) {
2670 for_each_active_iommu(iommu
, drhd
) {
2671 g_iommus
[iommu
->seq_id
] = iommu
;
2673 ret
= iommu_init_domains(iommu
);
2679 * we could share the same root & context tables
2680 * among all IOMMU's. Need to Split it later.
2682 ret
= iommu_alloc_root_entry(iommu
);
2684 printk(KERN_ERR
"IOMMU: allocate root entry failed\n");
2687 if (!ecap_pass_through(iommu
->ecap
))
2688 hw_pass_through
= 0;
2692 * Start from the sane iommu hardware state.
2694 for_each_active_iommu(iommu
, drhd
) {
2696 * If the queued invalidation is already initialized by us
2697 * (for example, while enabling interrupt-remapping) then
2698 * we got the things already rolling from a sane state.
2704 * Clear any previous faults.
2706 dmar_fault(-1, iommu
);
2708 * Disable queued invalidation if supported and already enabled
2709 * before OS handover.
2711 dmar_disable_qi(iommu
);
2714 for_each_active_iommu(iommu
, drhd
) {
2715 if (dmar_enable_qi(iommu
)) {
2717 * Queued Invalidate not enabled, use Register Based
2720 iommu
->flush
.flush_context
= __iommu_flush_context
;
2721 iommu
->flush
.flush_iotlb
= __iommu_flush_iotlb
;
2722 printk(KERN_INFO
"IOMMU %d 0x%Lx: using Register based "
2725 (unsigned long long)drhd
->reg_base_addr
);
2727 iommu
->flush
.flush_context
= qi_flush_context
;
2728 iommu
->flush
.flush_iotlb
= qi_flush_iotlb
;
2729 printk(KERN_INFO
"IOMMU %d 0x%Lx: using Queued "
2732 (unsigned long long)drhd
->reg_base_addr
);
2736 if (iommu_pass_through
)
2737 iommu_identity_mapping
|= IDENTMAP_ALL
;
2739 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2740 iommu_identity_mapping
|= IDENTMAP_GFX
;
2743 check_tylersburg_isoch();
2746 * If pass through is not set or not enabled, setup context entries for
2747 * identity mappings for rmrr, gfx, and isa and may fall back to static
2748 * identity mapping if iommu_identity_mapping is set.
2750 if (iommu_identity_mapping
) {
2751 ret
= iommu_prepare_static_identity_mapping(hw_pass_through
);
2753 printk(KERN_CRIT
"Failed to setup IOMMU pass-through\n");
2759 * for each dev attached to rmrr
2761 * locate drhd for dev, alloc domain for dev
2762 * allocate free domain
2763 * allocate page table entries for rmrr
2764 * if context not allocated for bus
2765 * allocate and init context
2766 * set present in root table for this bus
2767 * init context with domain, translation etc
2771 printk(KERN_INFO
"IOMMU: Setting RMRR:\n");
2772 for_each_rmrr_units(rmrr
) {
2773 /* some BIOS lists non-exist devices in DMAR table. */
2774 for_each_active_dev_scope(rmrr
->devices
, rmrr
->devices_cnt
,
2776 ret
= iommu_prepare_rmrr_dev(rmrr
, dev
);
2779 "IOMMU: mapping reserved region failed\n");
2783 iommu_prepare_isa();
2788 * global invalidate context cache
2789 * global invalidate iotlb
2790 * enable translation
2792 for_each_iommu(iommu
, drhd
) {
2793 if (drhd
->ignored
) {
2795 * we always have to disable PMRs or DMA may fail on
2799 iommu_disable_protect_mem_regions(iommu
);
2803 iommu_flush_write_buffer(iommu
);
2805 ret
= dmar_set_interrupt(iommu
);
2809 iommu_set_root_entry(iommu
);
2811 iommu
->flush
.flush_context(iommu
, 0, 0, 0, DMA_CCMD_GLOBAL_INVL
);
2812 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
2814 ret
= iommu_enable_translation(iommu
);
2818 iommu_disable_protect_mem_regions(iommu
);
2824 for_each_active_iommu(iommu
, drhd
)
2825 free_dmar_iommu(iommu
);
2826 kfree(deferred_flush
);
2833 /* This takes a number of _MM_ pages, not VTD pages */
2834 static struct iova
*intel_alloc_iova(struct device
*dev
,
2835 struct dmar_domain
*domain
,
2836 unsigned long nrpages
, uint64_t dma_mask
)
2838 struct iova
*iova
= NULL
;
2840 /* Restrict dma_mask to the width that the iommu can handle */
2841 dma_mask
= min_t(uint64_t, DOMAIN_MAX_ADDR(domain
->gaw
), dma_mask
);
2843 if (!dmar_forcedac
&& dma_mask
> DMA_BIT_MASK(32)) {
2845 * First try to allocate an io virtual address in
2846 * DMA_BIT_MASK(32) and if that fails then try allocating
2849 iova
= alloc_iova(&domain
->iovad
, nrpages
,
2850 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2854 iova
= alloc_iova(&domain
->iovad
, nrpages
, IOVA_PFN(dma_mask
), 1);
2855 if (unlikely(!iova
)) {
2856 printk(KERN_ERR
"Allocating %ld-page iova for %s failed",
2857 nrpages
, dev_name(dev
));
2864 static struct dmar_domain
*__get_valid_domain_for_dev(struct device
*dev
)
2866 struct dmar_domain
*domain
;
2869 domain
= get_domain_for_dev(dev
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
2871 printk(KERN_ERR
"Allocating domain for %s failed",
2876 /* make sure context mapping is ok */
2877 if (unlikely(!domain_context_mapped(dev
))) {
2878 ret
= domain_context_mapping(domain
, dev
, CONTEXT_TT_MULTI_LEVEL
);
2880 printk(KERN_ERR
"Domain context map for %s failed",
2889 static inline struct dmar_domain
*get_valid_domain_for_dev(struct device
*dev
)
2891 struct device_domain_info
*info
;
2893 /* No lock here, assumes no domain exit in normal case */
2894 info
= dev
->archdata
.iommu
;
2896 return info
->domain
;
2898 return __get_valid_domain_for_dev(dev
);
2901 static int iommu_dummy(struct device
*dev
)
2903 return dev
->archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
;
2906 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2907 static int iommu_no_mapping(struct device
*dev
)
2909 struct pci_dev
*pdev
;
2912 if (unlikely(!dev_is_pci(dev
)))
2915 if (iommu_dummy(dev
))
2918 if (!iommu_identity_mapping
)
2921 pdev
= to_pci_dev(dev
);
2922 found
= identity_mapping(dev
);
2924 if (iommu_should_identity_map(&pdev
->dev
, 0))
2928 * 32 bit DMA is removed from si_domain and fall back
2929 * to non-identity mapping.
2931 domain_remove_one_dev_info(si_domain
, dev
);
2932 printk(KERN_INFO
"32bit %s uses non-identity mapping\n",
2938 * In case of a detached 64 bit DMA device from vm, the device
2939 * is put into si_domain for identity mapping.
2941 if (iommu_should_identity_map(&pdev
->dev
, 0)) {
2943 ret
= domain_add_dev_info(si_domain
, pdev
,
2945 CONTEXT_TT_PASS_THROUGH
:
2946 CONTEXT_TT_MULTI_LEVEL
);
2948 printk(KERN_INFO
"64bit %s uses identity mapping\n",
2958 static dma_addr_t
__intel_map_single(struct device
*dev
, phys_addr_t paddr
,
2959 size_t size
, int dir
, u64 dma_mask
)
2961 struct dmar_domain
*domain
;
2962 phys_addr_t start_paddr
;
2966 struct intel_iommu
*iommu
;
2967 unsigned long paddr_pfn
= paddr
>> PAGE_SHIFT
;
2969 BUG_ON(dir
== DMA_NONE
);
2971 if (iommu_no_mapping(dev
))
2974 domain
= get_valid_domain_for_dev(dev
);
2978 iommu
= domain_get_iommu(domain
);
2979 size
= aligned_nrpages(paddr
, size
);
2981 iova
= intel_alloc_iova(dev
, domain
, dma_to_mm_pfn(size
), dma_mask
);
2986 * Check if DMAR supports zero-length reads on write only
2989 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
2990 !cap_zlr(iommu
->cap
))
2991 prot
|= DMA_PTE_READ
;
2992 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
2993 prot
|= DMA_PTE_WRITE
;
2995 * paddr - (paddr + size) might be partial page, we should map the whole
2996 * page. Note: if two part of one page are separately mapped, we
2997 * might have two guest_addr mapping to the same host paddr, but this
2998 * is not a big problem
3000 ret
= domain_pfn_mapping(domain
, mm_to_dma_pfn(iova
->pfn_lo
),
3001 mm_to_dma_pfn(paddr_pfn
), size
, prot
);
3005 /* it's a non-present to present mapping. Only flush if caching mode */
3006 if (cap_caching_mode(iommu
->cap
))
3007 iommu_flush_iotlb_psi(iommu
, domain
->id
, mm_to_dma_pfn(iova
->pfn_lo
), size
, 0, 1);
3009 iommu_flush_write_buffer(iommu
);
3011 start_paddr
= (phys_addr_t
)iova
->pfn_lo
<< PAGE_SHIFT
;
3012 start_paddr
+= paddr
& ~PAGE_MASK
;
3017 __free_iova(&domain
->iovad
, iova
);
3018 printk(KERN_ERR
"Device %s request: %zx@%llx dir %d --- failed\n",
3019 dev_name(dev
), size
, (unsigned long long)paddr
, dir
);
3023 static dma_addr_t
intel_map_page(struct device
*dev
, struct page
*page
,
3024 unsigned long offset
, size_t size
,
3025 enum dma_data_direction dir
,
3026 struct dma_attrs
*attrs
)
3028 return __intel_map_single(dev
, page_to_phys(page
) + offset
, size
,
3029 dir
, to_pci_dev(dev
)->dma_mask
);
3032 static void flush_unmaps(void)
3038 /* just flush them all */
3039 for (i
= 0; i
< g_num_of_iommus
; i
++) {
3040 struct intel_iommu
*iommu
= g_iommus
[i
];
3044 if (!deferred_flush
[i
].next
)
3047 /* In caching mode, global flushes turn emulation expensive */
3048 if (!cap_caching_mode(iommu
->cap
))
3049 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3050 DMA_TLB_GLOBAL_FLUSH
);
3051 for (j
= 0; j
< deferred_flush
[i
].next
; j
++) {
3053 struct iova
*iova
= deferred_flush
[i
].iova
[j
];
3054 struct dmar_domain
*domain
= deferred_flush
[i
].domain
[j
];
3056 /* On real hardware multiple invalidations are expensive */
3057 if (cap_caching_mode(iommu
->cap
))
3058 iommu_flush_iotlb_psi(iommu
, domain
->id
,
3059 iova
->pfn_lo
, iova
->pfn_hi
- iova
->pfn_lo
+ 1,
3060 !deferred_flush
[i
].freelist
[j
], 0);
3062 mask
= ilog2(mm_to_dma_pfn(iova
->pfn_hi
- iova
->pfn_lo
+ 1));
3063 iommu_flush_dev_iotlb(deferred_flush
[i
].domain
[j
],
3064 (uint64_t)iova
->pfn_lo
<< PAGE_SHIFT
, mask
);
3066 __free_iova(&deferred_flush
[i
].domain
[j
]->iovad
, iova
);
3067 if (deferred_flush
[i
].freelist
[j
])
3068 dma_free_pagelist(deferred_flush
[i
].freelist
[j
]);
3070 deferred_flush
[i
].next
= 0;
3076 static void flush_unmaps_timeout(unsigned long data
)
3078 unsigned long flags
;
3080 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
3082 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
3085 static void add_unmap(struct dmar_domain
*dom
, struct iova
*iova
, struct page
*freelist
)
3087 unsigned long flags
;
3089 struct intel_iommu
*iommu
;
3091 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
3092 if (list_size
== HIGH_WATER_MARK
)
3095 iommu
= domain_get_iommu(dom
);
3096 iommu_id
= iommu
->seq_id
;
3098 next
= deferred_flush
[iommu_id
].next
;
3099 deferred_flush
[iommu_id
].domain
[next
] = dom
;
3100 deferred_flush
[iommu_id
].iova
[next
] = iova
;
3101 deferred_flush
[iommu_id
].freelist
[next
] = freelist
;
3102 deferred_flush
[iommu_id
].next
++;
3105 mod_timer(&unmap_timer
, jiffies
+ msecs_to_jiffies(10));
3109 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
3112 static void intel_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
3113 size_t size
, enum dma_data_direction dir
,
3114 struct dma_attrs
*attrs
)
3116 struct dmar_domain
*domain
;
3117 unsigned long start_pfn
, last_pfn
;
3119 struct intel_iommu
*iommu
;
3120 struct page
*freelist
;
3122 if (iommu_no_mapping(dev
))
3125 domain
= find_domain(dev
);
3128 iommu
= domain_get_iommu(domain
);
3130 iova
= find_iova(&domain
->iovad
, IOVA_PFN(dev_addr
));
3131 if (WARN_ONCE(!iova
, "Driver unmaps unmatched page at PFN %llx\n",
3132 (unsigned long long)dev_addr
))
3135 start_pfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3136 last_pfn
= mm_to_dma_pfn(iova
->pfn_hi
+ 1) - 1;
3138 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3139 dev_name(dev
), start_pfn
, last_pfn
);
3141 freelist
= domain_unmap(domain
, start_pfn
, last_pfn
);
3143 if (intel_iommu_strict
) {
3144 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_pfn
,
3145 last_pfn
- start_pfn
+ 1, !freelist
, 0);
3147 __free_iova(&domain
->iovad
, iova
);
3148 dma_free_pagelist(freelist
);
3150 add_unmap(domain
, iova
, freelist
);
3152 * queue up the release of the unmap to save the 1/6th of the
3153 * cpu used up by the iotlb flush operation...
3158 static void *intel_alloc_coherent(struct device
*dev
, size_t size
,
3159 dma_addr_t
*dma_handle
, gfp_t flags
,
3160 struct dma_attrs
*attrs
)
3165 size
= PAGE_ALIGN(size
);
3166 order
= get_order(size
);
3168 if (!iommu_no_mapping(dev
))
3169 flags
&= ~(GFP_DMA
| GFP_DMA32
);
3170 else if (dev
->coherent_dma_mask
< dma_get_required_mask(dev
)) {
3171 if (dev
->coherent_dma_mask
< DMA_BIT_MASK(32))
3177 vaddr
= (void *)__get_free_pages(flags
, order
);
3180 memset(vaddr
, 0, size
);
3182 *dma_handle
= __intel_map_single(dev
, virt_to_bus(vaddr
), size
,
3184 dev
->coherent_dma_mask
);
3187 free_pages((unsigned long)vaddr
, order
);
3191 static void intel_free_coherent(struct device
*dev
, size_t size
, void *vaddr
,
3192 dma_addr_t dma_handle
, struct dma_attrs
*attrs
)
3196 size
= PAGE_ALIGN(size
);
3197 order
= get_order(size
);
3199 intel_unmap_page(dev
, dma_handle
, size
, DMA_BIDIRECTIONAL
, NULL
);
3200 free_pages((unsigned long)vaddr
, order
);
3203 static void intel_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
3204 int nelems
, enum dma_data_direction dir
,
3205 struct dma_attrs
*attrs
)
3207 struct dmar_domain
*domain
;
3208 unsigned long start_pfn
, last_pfn
;
3210 struct intel_iommu
*iommu
;
3211 struct page
*freelist
;
3213 if (iommu_no_mapping(dev
))
3216 domain
= find_domain(dev
);
3219 iommu
= domain_get_iommu(domain
);
3221 iova
= find_iova(&domain
->iovad
, IOVA_PFN(sglist
[0].dma_address
));
3222 if (WARN_ONCE(!iova
, "Driver unmaps unmatched sglist at PFN %llx\n",
3223 (unsigned long long)sglist
[0].dma_address
))
3226 start_pfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3227 last_pfn
= mm_to_dma_pfn(iova
->pfn_hi
+ 1) - 1;
3229 freelist
= domain_unmap(domain
, start_pfn
, last_pfn
);
3231 if (intel_iommu_strict
) {
3232 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_pfn
,
3233 last_pfn
- start_pfn
+ 1, !freelist
, 0);
3235 __free_iova(&domain
->iovad
, iova
);
3236 dma_free_pagelist(freelist
);
3238 add_unmap(domain
, iova
, freelist
);
3240 * queue up the release of the unmap to save the 1/6th of the
3241 * cpu used up by the iotlb flush operation...
3246 static int intel_nontranslate_map_sg(struct device
*hddev
,
3247 struct scatterlist
*sglist
, int nelems
, int dir
)
3250 struct scatterlist
*sg
;
3252 for_each_sg(sglist
, sg
, nelems
, i
) {
3253 BUG_ON(!sg_page(sg
));
3254 sg
->dma_address
= page_to_phys(sg_page(sg
)) + sg
->offset
;
3255 sg
->dma_length
= sg
->length
;
3260 static int intel_map_sg(struct device
*dev
, struct scatterlist
*sglist
, int nelems
,
3261 enum dma_data_direction dir
, struct dma_attrs
*attrs
)
3264 struct dmar_domain
*domain
;
3267 struct iova
*iova
= NULL
;
3269 struct scatterlist
*sg
;
3270 unsigned long start_vpfn
;
3271 struct intel_iommu
*iommu
;
3273 BUG_ON(dir
== DMA_NONE
);
3274 if (iommu_no_mapping(dev
))
3275 return intel_nontranslate_map_sg(dev
, sglist
, nelems
, dir
);
3277 domain
= get_valid_domain_for_dev(dev
);
3281 iommu
= domain_get_iommu(domain
);
3283 for_each_sg(sglist
, sg
, nelems
, i
)
3284 size
+= aligned_nrpages(sg
->offset
, sg
->length
);
3286 iova
= intel_alloc_iova(dev
, domain
, dma_to_mm_pfn(size
),
3289 sglist
->dma_length
= 0;
3294 * Check if DMAR supports zero-length reads on write only
3297 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
3298 !cap_zlr(iommu
->cap
))
3299 prot
|= DMA_PTE_READ
;
3300 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
3301 prot
|= DMA_PTE_WRITE
;
3303 start_vpfn
= mm_to_dma_pfn(iova
->pfn_lo
);
3305 ret
= domain_sg_mapping(domain
, start_vpfn
, sglist
, size
, prot
);
3306 if (unlikely(ret
)) {
3307 /* clear the page */
3308 dma_pte_clear_range(domain
, start_vpfn
,
3309 start_vpfn
+ size
- 1);
3310 /* free page tables */
3311 dma_pte_free_pagetable(domain
, start_vpfn
,
3312 start_vpfn
+ size
- 1);
3314 __free_iova(&domain
->iovad
, iova
);
3318 /* it's a non-present to present mapping. Only flush if caching mode */
3319 if (cap_caching_mode(iommu
->cap
))
3320 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_vpfn
, size
, 0, 1);
3322 iommu_flush_write_buffer(iommu
);
3327 static int intel_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
3332 struct dma_map_ops intel_dma_ops
= {
3333 .alloc
= intel_alloc_coherent
,
3334 .free
= intel_free_coherent
,
3335 .map_sg
= intel_map_sg
,
3336 .unmap_sg
= intel_unmap_sg
,
3337 .map_page
= intel_map_page
,
3338 .unmap_page
= intel_unmap_page
,
3339 .mapping_error
= intel_mapping_error
,
3342 static inline int iommu_domain_cache_init(void)
3346 iommu_domain_cache
= kmem_cache_create("iommu_domain",
3347 sizeof(struct dmar_domain
),
3352 if (!iommu_domain_cache
) {
3353 printk(KERN_ERR
"Couldn't create iommu_domain cache\n");
3360 static inline int iommu_devinfo_cache_init(void)
3364 iommu_devinfo_cache
= kmem_cache_create("iommu_devinfo",
3365 sizeof(struct device_domain_info
),
3369 if (!iommu_devinfo_cache
) {
3370 printk(KERN_ERR
"Couldn't create devinfo cache\n");
3377 static inline int iommu_iova_cache_init(void)
3381 iommu_iova_cache
= kmem_cache_create("iommu_iova",
3382 sizeof(struct iova
),
3386 if (!iommu_iova_cache
) {
3387 printk(KERN_ERR
"Couldn't create iova cache\n");
3394 static int __init
iommu_init_mempool(void)
3397 ret
= iommu_iova_cache_init();
3401 ret
= iommu_domain_cache_init();
3405 ret
= iommu_devinfo_cache_init();
3409 kmem_cache_destroy(iommu_domain_cache
);
3411 kmem_cache_destroy(iommu_iova_cache
);
3416 static void __init
iommu_exit_mempool(void)
3418 kmem_cache_destroy(iommu_devinfo_cache
);
3419 kmem_cache_destroy(iommu_domain_cache
);
3420 kmem_cache_destroy(iommu_iova_cache
);
3424 static void quirk_ioat_snb_local_iommu(struct pci_dev
*pdev
)
3426 struct dmar_drhd_unit
*drhd
;
3430 /* We know that this device on this chipset has its own IOMMU.
3431 * If we find it under a different IOMMU, then the BIOS is lying
3432 * to us. Hope that the IOMMU for this device is actually
3433 * disabled, and it needs no translation...
3435 rc
= pci_bus_read_config_dword(pdev
->bus
, PCI_DEVFN(0, 0), 0xb0, &vtbar
);
3437 /* "can't" happen */
3438 dev_info(&pdev
->dev
, "failed to run vt-d quirk\n");
3441 vtbar
&= 0xffff0000;
3443 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3444 drhd
= dmar_find_matched_drhd_unit(pdev
);
3445 if (WARN_TAINT_ONCE(!drhd
|| drhd
->reg_base_addr
- vtbar
!= 0xa000,
3446 TAINT_FIRMWARE_WORKAROUND
,
3447 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3448 pdev
->dev
.archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
3450 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL
, PCI_DEVICE_ID_INTEL_IOAT_SNB
, quirk_ioat_snb_local_iommu
);
3452 static void __init
init_no_remapping_devices(void)
3454 struct dmar_drhd_unit
*drhd
;
3458 for_each_drhd_unit(drhd
) {
3459 if (!drhd
->include_all
) {
3460 for_each_active_dev_scope(drhd
->devices
,
3461 drhd
->devices_cnt
, i
, dev
)
3463 /* ignore DMAR unit if no devices exist */
3464 if (i
== drhd
->devices_cnt
)
3469 for_each_active_drhd_unit(drhd
) {
3470 if (drhd
->include_all
)
3473 for_each_active_dev_scope(drhd
->devices
,
3474 drhd
->devices_cnt
, i
, dev
)
3475 if (!dev_is_pci(dev
) || !IS_GFX_DEVICE(to_pci_dev(dev
)))
3477 if (i
< drhd
->devices_cnt
)
3480 /* This IOMMU has *only* gfx devices. Either bypass it or
3481 set the gfx_mapped flag, as appropriate */
3483 intel_iommu_gfx_mapped
= 1;
3486 for_each_active_dev_scope(drhd
->devices
,
3487 drhd
->devices_cnt
, i
, dev
)
3488 dev
->archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
3493 #ifdef CONFIG_SUSPEND
3494 static int init_iommu_hw(void)
3496 struct dmar_drhd_unit
*drhd
;
3497 struct intel_iommu
*iommu
= NULL
;
3499 for_each_active_iommu(iommu
, drhd
)
3501 dmar_reenable_qi(iommu
);
3503 for_each_iommu(iommu
, drhd
) {
3504 if (drhd
->ignored
) {
3506 * we always have to disable PMRs or DMA may fail on
3510 iommu_disable_protect_mem_regions(iommu
);
3514 iommu_flush_write_buffer(iommu
);
3516 iommu_set_root_entry(iommu
);
3518 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
3519 DMA_CCMD_GLOBAL_INVL
);
3520 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3521 DMA_TLB_GLOBAL_FLUSH
);
3522 if (iommu_enable_translation(iommu
))
3524 iommu_disable_protect_mem_regions(iommu
);
3530 static void iommu_flush_all(void)
3532 struct dmar_drhd_unit
*drhd
;
3533 struct intel_iommu
*iommu
;
3535 for_each_active_iommu(iommu
, drhd
) {
3536 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
3537 DMA_CCMD_GLOBAL_INVL
);
3538 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3539 DMA_TLB_GLOBAL_FLUSH
);
3543 static int iommu_suspend(void)
3545 struct dmar_drhd_unit
*drhd
;
3546 struct intel_iommu
*iommu
= NULL
;
3549 for_each_active_iommu(iommu
, drhd
) {
3550 iommu
->iommu_state
= kzalloc(sizeof(u32
) * MAX_SR_DMAR_REGS
,
3552 if (!iommu
->iommu_state
)
3558 for_each_active_iommu(iommu
, drhd
) {
3559 iommu_disable_translation(iommu
);
3561 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
3563 iommu
->iommu_state
[SR_DMAR_FECTL_REG
] =
3564 readl(iommu
->reg
+ DMAR_FECTL_REG
);
3565 iommu
->iommu_state
[SR_DMAR_FEDATA_REG
] =
3566 readl(iommu
->reg
+ DMAR_FEDATA_REG
);
3567 iommu
->iommu_state
[SR_DMAR_FEADDR_REG
] =
3568 readl(iommu
->reg
+ DMAR_FEADDR_REG
);
3569 iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
] =
3570 readl(iommu
->reg
+ DMAR_FEUADDR_REG
);
3572 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
3577 for_each_active_iommu(iommu
, drhd
)
3578 kfree(iommu
->iommu_state
);
3583 static void iommu_resume(void)
3585 struct dmar_drhd_unit
*drhd
;
3586 struct intel_iommu
*iommu
= NULL
;
3589 if (init_iommu_hw()) {
3591 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3593 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3597 for_each_active_iommu(iommu
, drhd
) {
3599 raw_spin_lock_irqsave(&iommu
->register_lock
, flag
);
3601 writel(iommu
->iommu_state
[SR_DMAR_FECTL_REG
],
3602 iommu
->reg
+ DMAR_FECTL_REG
);
3603 writel(iommu
->iommu_state
[SR_DMAR_FEDATA_REG
],
3604 iommu
->reg
+ DMAR_FEDATA_REG
);
3605 writel(iommu
->iommu_state
[SR_DMAR_FEADDR_REG
],
3606 iommu
->reg
+ DMAR_FEADDR_REG
);
3607 writel(iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
],
3608 iommu
->reg
+ DMAR_FEUADDR_REG
);
3610 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
3613 for_each_active_iommu(iommu
, drhd
)
3614 kfree(iommu
->iommu_state
);
3617 static struct syscore_ops iommu_syscore_ops
= {
3618 .resume
= iommu_resume
,
3619 .suspend
= iommu_suspend
,
3622 static void __init
init_iommu_pm_ops(void)
3624 register_syscore_ops(&iommu_syscore_ops
);
3628 static inline void init_iommu_pm_ops(void) {}
3629 #endif /* CONFIG_PM */
3632 int __init
dmar_parse_one_rmrr(struct acpi_dmar_header
*header
)
3634 struct acpi_dmar_reserved_memory
*rmrr
;
3635 struct dmar_rmrr_unit
*rmrru
;
3637 rmrru
= kzalloc(sizeof(*rmrru
), GFP_KERNEL
);
3641 rmrru
->hdr
= header
;
3642 rmrr
= (struct acpi_dmar_reserved_memory
*)header
;
3643 rmrru
->base_address
= rmrr
->base_address
;
3644 rmrru
->end_address
= rmrr
->end_address
;
3645 rmrru
->devices
= dmar_alloc_dev_scope((void *)(rmrr
+ 1),
3646 ((void *)rmrr
) + rmrr
->header
.length
,
3647 &rmrru
->devices_cnt
);
3648 if (rmrru
->devices_cnt
&& rmrru
->devices
== NULL
) {
3653 list_add(&rmrru
->list
, &dmar_rmrr_units
);
3658 int __init
dmar_parse_one_atsr(struct acpi_dmar_header
*hdr
)
3660 struct acpi_dmar_atsr
*atsr
;
3661 struct dmar_atsr_unit
*atsru
;
3663 atsr
= container_of(hdr
, struct acpi_dmar_atsr
, header
);
3664 atsru
= kzalloc(sizeof(*atsru
), GFP_KERNEL
);
3669 atsru
->include_all
= atsr
->flags
& 0x1;
3670 if (!atsru
->include_all
) {
3671 atsru
->devices
= dmar_alloc_dev_scope((void *)(atsr
+ 1),
3672 (void *)atsr
+ atsr
->header
.length
,
3673 &atsru
->devices_cnt
);
3674 if (atsru
->devices_cnt
&& atsru
->devices
== NULL
) {
3680 list_add_rcu(&atsru
->list
, &dmar_atsr_units
);
3685 static void intel_iommu_free_atsr(struct dmar_atsr_unit
*atsru
)
3687 dmar_free_dev_scope(&atsru
->devices
, &atsru
->devices_cnt
);
3691 static void intel_iommu_free_dmars(void)
3693 struct dmar_rmrr_unit
*rmrru
, *rmrr_n
;
3694 struct dmar_atsr_unit
*atsru
, *atsr_n
;
3696 list_for_each_entry_safe(rmrru
, rmrr_n
, &dmar_rmrr_units
, list
) {
3697 list_del(&rmrru
->list
);
3698 dmar_free_dev_scope(&rmrru
->devices
, &rmrru
->devices_cnt
);
3702 list_for_each_entry_safe(atsru
, atsr_n
, &dmar_atsr_units
, list
) {
3703 list_del(&atsru
->list
);
3704 intel_iommu_free_atsr(atsru
);
3708 int dmar_find_matched_atsr_unit(struct pci_dev
*dev
)
3711 struct pci_bus
*bus
;
3712 struct pci_dev
*bridge
= NULL
;
3714 struct acpi_dmar_atsr
*atsr
;
3715 struct dmar_atsr_unit
*atsru
;
3717 dev
= pci_physfn(dev
);
3718 for (bus
= dev
->bus
; bus
; bus
= bus
->parent
) {
3720 if (!bridge
|| !pci_is_pcie(bridge
) ||
3721 pci_pcie_type(bridge
) == PCI_EXP_TYPE_PCI_BRIDGE
)
3723 if (pci_pcie_type(bridge
) == PCI_EXP_TYPE_ROOT_PORT
)
3730 list_for_each_entry_rcu(atsru
, &dmar_atsr_units
, list
) {
3731 atsr
= container_of(atsru
->hdr
, struct acpi_dmar_atsr
, header
);
3732 if (atsr
->segment
!= pci_domain_nr(dev
->bus
))
3735 for_each_dev_scope(atsru
->devices
, atsru
->devices_cnt
, i
, tmp
)
3736 if (tmp
== &bridge
->dev
)
3739 if (atsru
->include_all
)
3749 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info
*info
)
3752 struct dmar_rmrr_unit
*rmrru
;
3753 struct dmar_atsr_unit
*atsru
;
3754 struct acpi_dmar_atsr
*atsr
;
3755 struct acpi_dmar_reserved_memory
*rmrr
;
3757 if (!intel_iommu_enabled
&& system_state
!= SYSTEM_BOOTING
)
3760 list_for_each_entry(rmrru
, &dmar_rmrr_units
, list
) {
3761 rmrr
= container_of(rmrru
->hdr
,
3762 struct acpi_dmar_reserved_memory
, header
);
3763 if (info
->event
== BUS_NOTIFY_ADD_DEVICE
) {
3764 ret
= dmar_insert_dev_scope(info
, (void *)(rmrr
+ 1),
3765 ((void *)rmrr
) + rmrr
->header
.length
,
3766 rmrr
->segment
, rmrru
->devices
,
3767 rmrru
->devices_cnt
);
3772 } else if (info
->event
== BUS_NOTIFY_DEL_DEVICE
) {
3773 if (dmar_remove_dev_scope(info
, rmrr
->segment
,
3774 rmrru
->devices
, rmrru
->devices_cnt
))
3779 list_for_each_entry(atsru
, &dmar_atsr_units
, list
) {
3780 if (atsru
->include_all
)
3783 atsr
= container_of(atsru
->hdr
, struct acpi_dmar_atsr
, header
);
3784 if (info
->event
== BUS_NOTIFY_ADD_DEVICE
) {
3785 ret
= dmar_insert_dev_scope(info
, (void *)(atsr
+ 1),
3786 (void *)atsr
+ atsr
->header
.length
,
3787 atsr
->segment
, atsru
->devices
,
3788 atsru
->devices_cnt
);
3793 } else if (info
->event
== BUS_NOTIFY_DEL_DEVICE
) {
3794 if (dmar_remove_dev_scope(info
, atsr
->segment
,
3795 atsru
->devices
, atsru
->devices_cnt
))
3804 * Here we only respond to action of unbound device from driver.
3806 * Added device is not attached to its DMAR domain here yet. That will happen
3807 * when mapping the device to iova.
3809 static int device_notifier(struct notifier_block
*nb
,
3810 unsigned long action
, void *data
)
3812 struct device
*dev
= data
;
3813 struct dmar_domain
*domain
;
3815 if (iommu_dummy(dev
))
3818 if (action
!= BUS_NOTIFY_UNBOUND_DRIVER
&&
3819 action
!= BUS_NOTIFY_DEL_DEVICE
)
3822 domain
= find_domain(dev
);
3826 down_read(&dmar_global_lock
);
3827 domain_remove_one_dev_info(domain
, dev
);
3828 if (!(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) &&
3829 !(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
) &&
3830 list_empty(&domain
->devices
))
3831 domain_exit(domain
);
3832 up_read(&dmar_global_lock
);
3837 static struct notifier_block device_nb
= {
3838 .notifier_call
= device_notifier
,
3841 static int intel_iommu_memory_notifier(struct notifier_block
*nb
,
3842 unsigned long val
, void *v
)
3844 struct memory_notify
*mhp
= v
;
3845 unsigned long long start
, end
;
3846 unsigned long start_vpfn
, last_vpfn
;
3849 case MEM_GOING_ONLINE
:
3850 start
= mhp
->start_pfn
<< PAGE_SHIFT
;
3851 end
= ((mhp
->start_pfn
+ mhp
->nr_pages
) << PAGE_SHIFT
) - 1;
3852 if (iommu_domain_identity_map(si_domain
, start
, end
)) {
3853 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3860 case MEM_CANCEL_ONLINE
:
3861 start_vpfn
= mm_to_dma_pfn(mhp
->start_pfn
);
3862 last_vpfn
= mm_to_dma_pfn(mhp
->start_pfn
+ mhp
->nr_pages
- 1);
3863 while (start_vpfn
<= last_vpfn
) {
3865 struct dmar_drhd_unit
*drhd
;
3866 struct intel_iommu
*iommu
;
3867 struct page
*freelist
;
3869 iova
= find_iova(&si_domain
->iovad
, start_vpfn
);
3871 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3876 iova
= split_and_remove_iova(&si_domain
->iovad
, iova
,
3877 start_vpfn
, last_vpfn
);
3879 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3880 start_vpfn
, last_vpfn
);
3884 freelist
= domain_unmap(si_domain
, iova
->pfn_lo
,
3888 for_each_active_iommu(iommu
, drhd
)
3889 iommu_flush_iotlb_psi(iommu
, si_domain
->id
,
3891 iova
->pfn_hi
- iova
->pfn_lo
+ 1,
3894 dma_free_pagelist(freelist
);
3896 start_vpfn
= iova
->pfn_hi
+ 1;
3897 free_iova_mem(iova
);
3905 static struct notifier_block intel_iommu_memory_nb
= {
3906 .notifier_call
= intel_iommu_memory_notifier
,
3910 int __init
intel_iommu_init(void)
3913 struct dmar_drhd_unit
*drhd
;
3914 struct intel_iommu
*iommu
;
3916 /* VT-d is required for a TXT/tboot launch, so enforce that */
3917 force_on
= tboot_force_iommu();
3919 if (iommu_init_mempool()) {
3921 panic("tboot: Failed to initialize iommu memory\n");
3925 down_write(&dmar_global_lock
);
3926 if (dmar_table_init()) {
3928 panic("tboot: Failed to initialize DMAR table\n");
3933 * Disable translation if already enabled prior to OS handover.
3935 for_each_active_iommu(iommu
, drhd
)
3936 if (iommu
->gcmd
& DMA_GCMD_TE
)
3937 iommu_disable_translation(iommu
);
3939 if (dmar_dev_scope_init() < 0) {
3941 panic("tboot: Failed to initialize DMAR device scope\n");
3945 if (no_iommu
|| dmar_disabled
)
3948 if (list_empty(&dmar_rmrr_units
))
3949 printk(KERN_INFO
"DMAR: No RMRR found\n");
3951 if (list_empty(&dmar_atsr_units
))
3952 printk(KERN_INFO
"DMAR: No ATSR found\n");
3954 if (dmar_init_reserved_ranges()) {
3956 panic("tboot: Failed to reserve iommu ranges\n");
3957 goto out_free_reserved_range
;
3960 init_no_remapping_devices();
3965 panic("tboot: Failed to initialize DMARs\n");
3966 printk(KERN_ERR
"IOMMU: dmar init failed\n");
3967 goto out_free_reserved_range
;
3969 up_write(&dmar_global_lock
);
3971 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3973 init_timer(&unmap_timer
);
3974 #ifdef CONFIG_SWIOTLB
3977 dma_ops
= &intel_dma_ops
;
3979 init_iommu_pm_ops();
3981 bus_set_iommu(&pci_bus_type
, &intel_iommu_ops
);
3982 bus_register_notifier(&pci_bus_type
, &device_nb
);
3983 if (si_domain
&& !hw_pass_through
)
3984 register_memory_notifier(&intel_iommu_memory_nb
);
3986 intel_iommu_enabled
= 1;
3990 out_free_reserved_range
:
3991 put_iova_domain(&reserved_iova_list
);
3993 intel_iommu_free_dmars();
3994 up_write(&dmar_global_lock
);
3995 iommu_exit_mempool();
3999 static void iommu_detach_dependent_devices(struct intel_iommu
*iommu
,
4002 struct pci_dev
*tmp
, *parent
, *pdev
;
4004 if (!iommu
|| !dev
|| !dev_is_pci(dev
))
4007 pdev
= to_pci_dev(dev
);
4009 /* dependent device detach */
4010 tmp
= pci_find_upstream_pcie_bridge(pdev
);
4011 /* Secondary interface's bus number and devfn 0 */
4013 parent
= pdev
->bus
->self
;
4014 while (parent
!= tmp
) {
4015 iommu_detach_dev(iommu
, parent
->bus
->number
,
4017 parent
= parent
->bus
->self
;
4019 if (pci_is_pcie(tmp
)) /* this is a PCIe-to-PCI bridge */
4020 iommu_detach_dev(iommu
,
4021 tmp
->subordinate
->number
, 0);
4022 else /* this is a legacy PCI bridge */
4023 iommu_detach_dev(iommu
, tmp
->bus
->number
,
4028 static void domain_remove_one_dev_info(struct dmar_domain
*domain
,
4031 struct device_domain_info
*info
, *tmp
;
4032 struct intel_iommu
*iommu
;
4033 unsigned long flags
;
4037 iommu
= device_to_iommu(dev
, &bus
, &devfn
);
4041 spin_lock_irqsave(&device_domain_lock
, flags
);
4042 list_for_each_entry_safe(info
, tmp
, &domain
->devices
, link
) {
4043 if (info
->iommu
== iommu
&& info
->bus
== bus
&&
4044 info
->devfn
== devfn
) {
4045 unlink_domain_info(info
);
4046 spin_unlock_irqrestore(&device_domain_lock
, flags
);
4048 iommu_disable_dev_iotlb(info
);
4049 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
4050 iommu_detach_dependent_devices(iommu
, dev
);
4051 free_devinfo_mem(info
);
4053 spin_lock_irqsave(&device_domain_lock
, flags
);
4061 /* if there is no other devices under the same iommu
4062 * owned by this domain, clear this iommu in iommu_bmp
4063 * update iommu count and coherency
4065 if (info
->iommu
== iommu
)
4069 spin_unlock_irqrestore(&device_domain_lock
, flags
);
4072 unsigned long tmp_flags
;
4073 spin_lock_irqsave(&domain
->iommu_lock
, tmp_flags
);
4074 clear_bit(iommu
->seq_id
, domain
->iommu_bmp
);
4075 domain
->iommu_count
--;
4076 domain_update_iommu_cap(domain
);
4077 spin_unlock_irqrestore(&domain
->iommu_lock
, tmp_flags
);
4079 if (!(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) &&
4080 !(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
)) {
4081 spin_lock_irqsave(&iommu
->lock
, tmp_flags
);
4082 clear_bit(domain
->id
, iommu
->domain_ids
);
4083 iommu
->domains
[domain
->id
] = NULL
;
4084 spin_unlock_irqrestore(&iommu
->lock
, tmp_flags
);
4089 static int md_domain_init(struct dmar_domain
*domain
, int guest_width
)
4093 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
4094 domain_reserve_special_ranges(domain
);
4096 /* calculate AGAW */
4097 domain
->gaw
= guest_width
;
4098 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
4099 domain
->agaw
= width_to_agaw(adjust_width
);
4101 domain
->iommu_coherency
= 0;
4102 domain
->iommu_snooping
= 0;
4103 domain
->iommu_superpage
= 0;
4104 domain
->max_addr
= 0;
4107 /* always allocate the top pgd */
4108 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page(domain
->nid
);
4111 domain_flush_cache(domain
, domain
->pgd
, PAGE_SIZE
);
4115 static int intel_iommu_domain_init(struct iommu_domain
*domain
)
4117 struct dmar_domain
*dmar_domain
;
4119 dmar_domain
= alloc_domain(true);
4122 "intel_iommu_domain_init: dmar_domain == NULL\n");
4125 if (md_domain_init(dmar_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
4127 "intel_iommu_domain_init() failed\n");
4128 domain_exit(dmar_domain
);
4131 domain_update_iommu_cap(dmar_domain
);
4132 domain
->priv
= dmar_domain
;
4134 domain
->geometry
.aperture_start
= 0;
4135 domain
->geometry
.aperture_end
= __DOMAIN_MAX_ADDR(dmar_domain
->gaw
);
4136 domain
->geometry
.force_aperture
= true;
4141 static void intel_iommu_domain_destroy(struct iommu_domain
*domain
)
4143 struct dmar_domain
*dmar_domain
= domain
->priv
;
4145 domain
->priv
= NULL
;
4146 domain_exit(dmar_domain
);
4149 static int intel_iommu_attach_device(struct iommu_domain
*domain
,
4152 struct dmar_domain
*dmar_domain
= domain
->priv
;
4153 struct pci_dev
*pdev
= to_pci_dev(dev
);
4154 struct intel_iommu
*iommu
;
4158 /* normally pdev is not mapped */
4159 if (unlikely(domain_context_mapped(&pdev
->dev
))) {
4160 struct dmar_domain
*old_domain
;
4162 old_domain
= find_domain(dev
);
4164 if (dmar_domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
4165 dmar_domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
)
4166 domain_remove_one_dev_info(old_domain
, dev
);
4168 domain_remove_dev_info(old_domain
);
4172 iommu
= device_to_iommu(dev
, &bus
, &devfn
);
4176 /* check if this iommu agaw is sufficient for max mapped address */
4177 addr_width
= agaw_to_width(iommu
->agaw
);
4178 if (addr_width
> cap_mgaw(iommu
->cap
))
4179 addr_width
= cap_mgaw(iommu
->cap
);
4181 if (dmar_domain
->max_addr
> (1LL << addr_width
)) {
4182 printk(KERN_ERR
"%s: iommu width (%d) is not "
4183 "sufficient for the mapped address (%llx)\n",
4184 __func__
, addr_width
, dmar_domain
->max_addr
);
4187 dmar_domain
->gaw
= addr_width
;
4190 * Knock out extra levels of page tables if necessary
4192 while (iommu
->agaw
< dmar_domain
->agaw
) {
4193 struct dma_pte
*pte
;
4195 pte
= dmar_domain
->pgd
;
4196 if (dma_pte_present(pte
)) {
4197 dmar_domain
->pgd
= (struct dma_pte
*)
4198 phys_to_virt(dma_pte_addr(pte
));
4199 free_pgtable_page(pte
);
4201 dmar_domain
->agaw
--;
4204 return domain_add_dev_info(dmar_domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
4207 static void intel_iommu_detach_device(struct iommu_domain
*domain
,
4210 struct dmar_domain
*dmar_domain
= domain
->priv
;
4212 domain_remove_one_dev_info(dmar_domain
, dev
);
4215 static int intel_iommu_map(struct iommu_domain
*domain
,
4216 unsigned long iova
, phys_addr_t hpa
,
4217 size_t size
, int iommu_prot
)
4219 struct dmar_domain
*dmar_domain
= domain
->priv
;
4224 if (iommu_prot
& IOMMU_READ
)
4225 prot
|= DMA_PTE_READ
;
4226 if (iommu_prot
& IOMMU_WRITE
)
4227 prot
|= DMA_PTE_WRITE
;
4228 if ((iommu_prot
& IOMMU_CACHE
) && dmar_domain
->iommu_snooping
)
4229 prot
|= DMA_PTE_SNP
;
4231 max_addr
= iova
+ size
;
4232 if (dmar_domain
->max_addr
< max_addr
) {
4235 /* check if minimum agaw is sufficient for mapped address */
4236 end
= __DOMAIN_MAX_ADDR(dmar_domain
->gaw
) + 1;
4237 if (end
< max_addr
) {
4238 printk(KERN_ERR
"%s: iommu width (%d) is not "
4239 "sufficient for the mapped address (%llx)\n",
4240 __func__
, dmar_domain
->gaw
, max_addr
);
4243 dmar_domain
->max_addr
= max_addr
;
4245 /* Round up size to next multiple of PAGE_SIZE, if it and
4246 the low bits of hpa would take us onto the next page */
4247 size
= aligned_nrpages(hpa
, size
);
4248 ret
= domain_pfn_mapping(dmar_domain
, iova
>> VTD_PAGE_SHIFT
,
4249 hpa
>> VTD_PAGE_SHIFT
, size
, prot
);
4253 static size_t intel_iommu_unmap(struct iommu_domain
*domain
,
4254 unsigned long iova
, size_t size
)
4256 struct dmar_domain
*dmar_domain
= domain
->priv
;
4257 struct page
*freelist
= NULL
;
4258 struct intel_iommu
*iommu
;
4259 unsigned long start_pfn
, last_pfn
;
4260 unsigned int npages
;
4261 int iommu_id
, num
, ndomains
, level
= 0;
4263 /* Cope with horrid API which requires us to unmap more than the
4264 size argument if it happens to be a large-page mapping. */
4265 if (!pfn_to_dma_pte(dmar_domain
, iova
>> VTD_PAGE_SHIFT
, &level
))
4268 if (size
< VTD_PAGE_SIZE
<< level_to_offset_bits(level
))
4269 size
= VTD_PAGE_SIZE
<< level_to_offset_bits(level
);
4271 start_pfn
= iova
>> VTD_PAGE_SHIFT
;
4272 last_pfn
= (iova
+ size
- 1) >> VTD_PAGE_SHIFT
;
4274 freelist
= domain_unmap(dmar_domain
, start_pfn
, last_pfn
);
4276 npages
= last_pfn
- start_pfn
+ 1;
4278 for_each_set_bit(iommu_id
, dmar_domain
->iommu_bmp
, g_num_of_iommus
) {
4279 iommu
= g_iommus
[iommu_id
];
4282 * find bit position of dmar_domain
4284 ndomains
= cap_ndoms(iommu
->cap
);
4285 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
4286 if (iommu
->domains
[num
] == dmar_domain
)
4287 iommu_flush_iotlb_psi(iommu
, num
, start_pfn
,
4288 npages
, !freelist
, 0);
4293 dma_free_pagelist(freelist
);
4295 if (dmar_domain
->max_addr
== iova
+ size
)
4296 dmar_domain
->max_addr
= iova
;
4301 static phys_addr_t
intel_iommu_iova_to_phys(struct iommu_domain
*domain
,
4304 struct dmar_domain
*dmar_domain
= domain
->priv
;
4305 struct dma_pte
*pte
;
4309 pte
= pfn_to_dma_pte(dmar_domain
, iova
>> VTD_PAGE_SHIFT
, &level
);
4311 phys
= dma_pte_addr(pte
);
4316 static int intel_iommu_domain_has_cap(struct iommu_domain
*domain
,
4319 struct dmar_domain
*dmar_domain
= domain
->priv
;
4321 if (cap
== IOMMU_CAP_CACHE_COHERENCY
)
4322 return dmar_domain
->iommu_snooping
;
4323 if (cap
== IOMMU_CAP_INTR_REMAP
)
4324 return irq_remapping_enabled
;
4329 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4331 static int intel_iommu_add_device(struct device
*dev
)
4333 struct pci_dev
*pdev
= to_pci_dev(dev
);
4334 struct pci_dev
*bridge
, *dma_pdev
= NULL
;
4335 struct iommu_group
*group
;
4339 if (!device_to_iommu(dev
, &bus
, &devfn
))
4342 bridge
= pci_find_upstream_pcie_bridge(pdev
);
4344 if (pci_is_pcie(bridge
))
4345 dma_pdev
= pci_get_domain_bus_and_slot(
4346 pci_domain_nr(pdev
->bus
),
4347 bridge
->subordinate
->number
, 0);
4349 dma_pdev
= pci_dev_get(bridge
);
4351 dma_pdev
= pci_dev_get(pdev
);
4353 /* Account for quirked devices */
4354 swap_pci_ref(&dma_pdev
, pci_get_dma_source(dma_pdev
));
4357 * If it's a multifunction device that does not support our
4358 * required ACS flags, add to the same group as lowest numbered
4359 * function that also does not suport the required ACS flags.
4361 if (dma_pdev
->multifunction
&&
4362 !pci_acs_enabled(dma_pdev
, REQ_ACS_FLAGS
)) {
4363 u8 i
, slot
= PCI_SLOT(dma_pdev
->devfn
);
4365 for (i
= 0; i
< 8; i
++) {
4366 struct pci_dev
*tmp
;
4368 tmp
= pci_get_slot(dma_pdev
->bus
, PCI_DEVFN(slot
, i
));
4372 if (!pci_acs_enabled(tmp
, REQ_ACS_FLAGS
)) {
4373 swap_pci_ref(&dma_pdev
, tmp
);
4381 * Devices on the root bus go through the iommu. If that's not us,
4382 * find the next upstream device and test ACS up to the root bus.
4383 * Finding the next device may require skipping virtual buses.
4385 while (!pci_is_root_bus(dma_pdev
->bus
)) {
4386 struct pci_bus
*bus
= dma_pdev
->bus
;
4388 while (!bus
->self
) {
4389 if (!pci_is_root_bus(bus
))
4395 if (pci_acs_path_enabled(bus
->self
, NULL
, REQ_ACS_FLAGS
))
4398 swap_pci_ref(&dma_pdev
, pci_dev_get(bus
->self
));
4402 group
= iommu_group_get(&dma_pdev
->dev
);
4403 pci_dev_put(dma_pdev
);
4405 group
= iommu_group_alloc();
4407 return PTR_ERR(group
);
4410 ret
= iommu_group_add_device(group
, dev
);
4412 iommu_group_put(group
);
4416 static void intel_iommu_remove_device(struct device
*dev
)
4418 iommu_group_remove_device(dev
);
4421 static struct iommu_ops intel_iommu_ops
= {
4422 .domain_init
= intel_iommu_domain_init
,
4423 .domain_destroy
= intel_iommu_domain_destroy
,
4424 .attach_dev
= intel_iommu_attach_device
,
4425 .detach_dev
= intel_iommu_detach_device
,
4426 .map
= intel_iommu_map
,
4427 .unmap
= intel_iommu_unmap
,
4428 .iova_to_phys
= intel_iommu_iova_to_phys
,
4429 .domain_has_cap
= intel_iommu_domain_has_cap
,
4430 .add_device
= intel_iommu_add_device
,
4431 .remove_device
= intel_iommu_remove_device
,
4432 .pgsize_bitmap
= INTEL_IOMMU_PGSIZES
,
4435 static void quirk_iommu_g4x_gfx(struct pci_dev
*dev
)
4437 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4438 printk(KERN_INFO
"DMAR: Disabling IOMMU for graphics on this chipset\n");
4442 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2a40, quirk_iommu_g4x_gfx
);
4443 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e00, quirk_iommu_g4x_gfx
);
4444 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e10, quirk_iommu_g4x_gfx
);
4445 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e20, quirk_iommu_g4x_gfx
);
4446 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e30, quirk_iommu_g4x_gfx
);
4447 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e40, quirk_iommu_g4x_gfx
);
4448 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e90, quirk_iommu_g4x_gfx
);
4450 static void quirk_iommu_rwbf(struct pci_dev
*dev
)
4453 * Mobile 4 Series Chipset neglects to set RWBF capability,
4454 * but needs it. Same seems to hold for the desktop versions.
4456 printk(KERN_INFO
"DMAR: Forcing write-buffer flush capability\n");
4460 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2a40, quirk_iommu_rwbf
);
4461 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e00, quirk_iommu_rwbf
);
4462 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e10, quirk_iommu_rwbf
);
4463 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e20, quirk_iommu_rwbf
);
4464 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e30, quirk_iommu_rwbf
);
4465 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e40, quirk_iommu_rwbf
);
4466 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2e90, quirk_iommu_rwbf
);
4469 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
4470 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4471 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
4472 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
4473 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4474 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4475 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4476 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4478 static void quirk_calpella_no_shadow_gtt(struct pci_dev
*dev
)
4482 if (pci_read_config_word(dev
, GGC
, &ggc
))
4485 if (!(ggc
& GGC_MEMORY_VT_ENABLED
)) {
4486 printk(KERN_INFO
"DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4488 } else if (dmar_map_gfx
) {
4489 /* we have to ensure the gfx device is idle before we flush */
4490 printk(KERN_INFO
"DMAR: Disabling batched IOTLB flush on Ironlake\n");
4491 intel_iommu_strict
= 1;
4494 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0040, quirk_calpella_no_shadow_gtt
);
4495 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0044, quirk_calpella_no_shadow_gtt
);
4496 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0062, quirk_calpella_no_shadow_gtt
);
4497 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x006a, quirk_calpella_no_shadow_gtt
);
4499 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4500 ISOCH DMAR unit for the Azalia sound device, but not give it any
4501 TLB entries, which causes it to deadlock. Check for that. We do
4502 this in a function called from init_dmars(), instead of in a PCI
4503 quirk, because we don't want to print the obnoxious "BIOS broken"
4504 message if VT-d is actually disabled.
4506 static void __init
check_tylersburg_isoch(void)
4508 struct pci_dev
*pdev
;
4509 uint32_t vtisochctrl
;
4511 /* If there's no Azalia in the system anyway, forget it. */
4512 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x3a3e, NULL
);
4517 /* System Management Registers. Might be hidden, in which case
4518 we can't do the sanity check. But that's OK, because the
4519 known-broken BIOSes _don't_ actually hide it, so far. */
4520 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x342e, NULL
);
4524 if (pci_read_config_dword(pdev
, 0x188, &vtisochctrl
)) {
4531 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4532 if (vtisochctrl
& 1)
4535 /* Drop all bits other than the number of TLB entries */
4536 vtisochctrl
&= 0x1c;
4538 /* If we have the recommended number of TLB entries (16), fine. */
4539 if (vtisochctrl
== 0x10)
4542 /* Zero TLB entries? You get to ride the short bus to school. */
4544 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4545 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4546 dmi_get_system_info(DMI_BIOS_VENDOR
),
4547 dmi_get_system_info(DMI_BIOS_VERSION
),
4548 dmi_get_system_info(DMI_PRODUCT_VERSION
));
4549 iommu_identity_mapping
|= IDENTMAP_AZALIA
;
4553 printk(KERN_WARNING
"DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",