projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: Clean up and fix page table clear/free behaviour
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 18 */
19
20#include <linux/init.h>
21#include <linux/bitmap.h>
5e0d2a6f 22#include <linux/debugfs.h>
54485c30 23#include <linux/export.h>
ba395927
KA		 24#include <linux/slab.h>
25#include <linux/irq.h>
26#include <linux/interrupt.h>
ba395927
KA		 27#include <linux/spinlock.h>
28#include <linux/pci.h>
29#include <linux/dmar.h>
30#include <linux/dma-mapping.h>
31#include <linux/mempool.h>
75f05569 32#include <linux/memory.h>
5e0d2a6f 33#include <linux/timer.h>
38717946 34#include <linux/iova.h>
5d450806 35#include <linux/iommu.h>
38717946 36#include <linux/intel-iommu.h>
134fac3f 37#include <linux/syscore_ops.h>
69575d38 38#include <linux/tboot.h>
adb2fe02 39#include <linux/dmi.h>
5cdede24 40#include <linux/pci-ats.h>
0ee332c1 41#include <linux/memblock.h>
8a8f422d 42#include <asm/irq_remapping.h>
ba395927 43#include <asm/cacheflush.h>
46a7fa27 44#include <asm/iommu.h>
ba395927 45
078e1ee2 46#include "irq_remapping.h"
61e015ac 47#include "pci.h"
078e1ee2 48
5b6985ce
FY		 49#define ROOT_SIZE VTD_PAGE_SIZE
50#define CONTEXT_SIZE VTD_PAGE_SIZE
51
ba395927
KA		 52#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 54#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 55
56#define IOAPIC_RANGE_START (0xfee00000)
57#define IOAPIC_RANGE_END (0xfeefffff)
58#define IOVA_START_ADDR (0x1000)
59
60#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
4ed0d3e6 62#define MAX_AGAW_WIDTH 64
5c645b35 63#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 64
2ebe3151
DW		 65#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
67
68/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 73
f27be03b 74#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 75#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 76#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 77
df08cdc7
AM		 78/* page table handling */
79#define LEVEL_STRIDE (9)
80#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
81
6d1c56a9
OBC		 82/*
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
86 * that we support.
87 *
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
91 *
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
94 *
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
97 */
98#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
99
df08cdc7
AM		 100static inline int agaw_to_level(int agaw)
101{
102 return agaw + 2;
103}
104
105static inline int agaw_to_width(int agaw)
106{
5c645b35 107 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 108}
109
110static inline int width_to_agaw(int width)
111{
5c645b35 112 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 113}
114
115static inline unsigned int level_to_offset_bits(int level)
116{
117 return (level - 1) * LEVEL_STRIDE;
118}
119
120static inline int pfn_level_offset(unsigned long pfn, int level)
121{
122 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
123}
124
125static inline unsigned long level_mask(int level)
126{
127 return -1UL << level_to_offset_bits(level);
128}
129
130static inline unsigned long level_size(int level)
131{
132 return 1UL << level_to_offset_bits(level);
133}
134
135static inline unsigned long align_to_level(unsigned long pfn, int level)
136{
137 return (pfn + level_size(level) - 1) & level_mask(level);
138}
fd18de50 139
6dd9a7c7
YS		 140static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
141{
5c645b35 142 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 143}
144
dd4e8319
DW		 145/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
148{
149 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
150}
151
152static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
153{
154 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
155}
156static inline unsigned long page_to_dma_pfn(struct page *pg)
157{
158 return mm_to_dma_pfn(page_to_pfn(pg));
159}
160static inline unsigned long virt_to_dma_pfn(void *p)
161{
162 return page_to_dma_pfn(virt_to_page(p));
163}
164
d9630fe9
WH		 165/* global iommu list, set NULL for ignored DMAR units */
166static struct intel_iommu **g_iommus;
167
e0fc7e0b 168static void __init check_tylersburg_isoch(void);
9af88143
DW		 169static int rwbf_quirk;
170
b779260b
JC		 171/*
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
174 */
175static int force_on = 0;
176
46b08e1a
MM		 177/*
178 * 0: Present
179 * 1-11: Reserved
180 * 12-63: Context Ptr (12 - (haw-1))
181 * 64-127: Reserved
182 */
183struct root_entry {
184 u64 val;
185 u64 rsvd1;
186};
187#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188static inline bool root_present(struct root_entry *root)
189{
190 return (root->val & 1);
191}
192static inline void set_root_present(struct root_entry *root)
193{
194 root->val |= 1;
195}
196static inline void set_root_value(struct root_entry *root, unsigned long value)
197{
198 root->val |= value & VTD_PAGE_MASK;
199}
200
201static inline struct context_entry *
202get_context_addr_from_root(struct root_entry *root)
203{
204 return (struct context_entry *)
205 (root_present(root)?phys_to_virt(
206 root->val & VTD_PAGE_MASK) :
207 NULL);
208}
209
7a8fc25e
MM		 210/*
211 * low 64 bits:
212 * 0: present
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
216 * high 64 bits:
217 * 0-2: address width
218 * 3-6: aval
219 * 8-23: domain id
220 */
221struct context_entry {
222 u64 lo;
223 u64 hi;
224};
c07e7d21
MM		 225
226static inline bool context_present(struct context_entry *context)
227{
228 return (context->lo & 1);
229}
230static inline void context_set_present(struct context_entry *context)
231{
232 context->lo |= 1;
233}
234
235static inline void context_set_fault_enable(struct context_entry *context)
236{
237 context->lo &= (((u64)-1) << 2) | 1;
238}
239
c07e7d21
MM		 240static inline void context_set_translation_type(struct context_entry *context,
241 unsigned long value)
242{
243 context->lo &= (((u64)-1) << 4) | 3;
244 context->lo |= (value & 3) << 2;
245}
246
247static inline void context_set_address_root(struct context_entry *context,
248 unsigned long value)
249{
250 context->lo |= value & VTD_PAGE_MASK;
251}
252
253static inline void context_set_address_width(struct context_entry *context,
254 unsigned long value)
255{
256 context->hi |= value & 7;
257}
258
259static inline void context_set_domain_id(struct context_entry *context,
260 unsigned long value)
261{
262 context->hi |= (value & ((1 << 16) - 1)) << 8;
263}
264
265static inline void context_clear_entry(struct context_entry *context)
266{
267 context->lo = 0;
268 context->hi = 0;
269}
7a8fc25e 270
622ba12a
MM		 271/*
272 * 0: readable
273 * 1: writable
274 * 2-6: reserved
275 * 7: super page
9cf06697
SY		 276 * 8-10: available
277 * 11: snoop behavior
622ba12a
MM		 278 * 12-63: Host physcial address
279 */
280struct dma_pte {
281 u64 val;
282};
622ba12a 283
19c239ce
MM		 284static inline void dma_clear_pte(struct dma_pte *pte)
285{
286 pte->val = 0;
287}
288
19c239ce
MM		 289static inline u64 dma_pte_addr(struct dma_pte *pte)
290{
c85994e4
DW		 291#ifdef CONFIG_64BIT
292 return pte->val & VTD_PAGE_MASK;
293#else
294 /* Must have a full atomic 64-bit read */
1a8bd481 295 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 296#endif
19c239ce
MM		 297}
298
19c239ce
MM		 299static inline bool dma_pte_present(struct dma_pte *pte)
300{
301 return (pte->val & 3) != 0;
302}
622ba12a 303
4399c8bf
AK		 304static inline bool dma_pte_superpage(struct dma_pte *pte)
305{
306 return (pte->val & (1 << 7));
307}
308
75e6bf96
DW		 309static inline int first_pte_in_page(struct dma_pte *pte)
310{
311 return !((unsigned long)pte & ~VTD_PAGE_MASK);
312}
313
2c2e2c38
FY		 314/*
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
319 */
19943b0e
DW		 320static struct dmar_domain *si_domain;
321static int hw_pass_through = 1;
2c2e2c38 322
3b5410e7 323/* devices under the same p2p bridge are owned in one domain */
cdc7b837 324#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 325
1ce28feb
WH		 326/* domain represents a virtual machine, more than one devices
327 * across iommus may be owned in one domain, e.g. kvm guest.
328 */
329#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
330
2c2e2c38
FY		 331/* si_domain contains mulitple devices */
332#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
333
1b198bb0
MT		 334/* define the limit of IOMMUs supported in each domain */
335#ifdef CONFIG_X86
336# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
337#else
338# define IOMMU_UNITS_SUPPORTED 64
339#endif
340
99126f7c
MM		 341struct dmar_domain {
342 int id; /* domain id */
4c923d47 343 int nid; /* node id */
1b198bb0
MT		 344 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
345 /* bitmap of iommus this domain uses*/
99126f7c
MM		 346
347 struct list_head devices; /* all devices' list */
348 struct iova_domain iovad; /* iova's that belong to this domain */
349
350 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 351 int gaw; /* max guest address width */
352
353 /* adjusted guest address width, 0 is level 2 30-bit */
354 int agaw;
355
3b5410e7 356 int flags; /* flags to find out type of domain */
8e604097
WH		 357
358 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 359 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 360 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 361 int iommu_superpage;/* Level of superpages supported:
362 0 == 4KiB (no superpages), 1 == 2MiB,
363 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 364 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 365 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 366};
367
a647dacb
MM		 368/* PCI domain-device relationship */
369struct device_domain_info {
370 struct list_head link; /* link to domain siblings */
371 struct list_head global; /* link to global list */
276dbf99
DW		 372 int segment; /* PCI domain */
373 u8 bus; /* PCI bus number */
a647dacb 374 u8 devfn; /* PCI devfn number */
45e829ea 375 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 376 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 377 struct dmar_domain *domain; /* pointer to domain */
378};
379
b94e4117
JL		 380struct dmar_rmrr_unit {
381 struct list_head list; /* list of rmrr units */
382 struct acpi_dmar_header *hdr; /* ACPI header */
383 u64 base_address; /* reserved base address*/
384 u64 end_address; /* reserved end address */
0e242612 385 struct pci_dev __rcu **devices; /* target devices */
b94e4117
JL		 386 int devices_cnt; /* target device count */
387};
388
389struct dmar_atsr_unit {
390 struct list_head list; /* list of ATSR units */
391 struct acpi_dmar_header *hdr; /* ACPI header */
0e242612 392 struct pci_dev __rcu **devices; /* target devices */
b94e4117
JL		 393 int devices_cnt; /* target device count */
394 u8 include_all:1; /* include all ports */
395};
396
397static LIST_HEAD(dmar_atsr_units);
398static LIST_HEAD(dmar_rmrr_units);
399
400#define for_each_rmrr_units(rmrr) \
401 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
402
5e0d2a6f 403static void flush_unmaps_timeout(unsigned long data);
404
b707cb02 405static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 406
80b20dd8 407#define HIGH_WATER_MARK 250
408struct deferred_flush_tables {
409 int next;
410 struct iova *iova[HIGH_WATER_MARK];
411 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 412 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 413};
414
415static struct deferred_flush_tables *deferred_flush;
416
5e0d2a6f 417/* bitmap for indexing intel_iommus */
5e0d2a6f 418static int g_num_of_iommus;
419
420static DEFINE_SPINLOCK(async_umap_flush_lock);
421static LIST_HEAD(unmaps_to_do);
422
423static int timer_on;
424static long list_size;
5e0d2a6f 425
92d03cc8 426static void domain_exit(struct dmar_domain *domain);
ba395927 427static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117
JL		 428static void domain_remove_one_dev_info(struct dmar_domain *domain,
429 struct pci_dev *pdev);
92d03cc8
JL		 430static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
431 struct pci_dev *pdev);
ba395927 432
d3f13810 433#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 434int dmar_disabled = 0;
435#else
436int dmar_disabled = 1;
d3f13810 437#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 438
8bc1f85c
ED		 439int intel_iommu_enabled = 0;
440EXPORT_SYMBOL_GPL(intel_iommu_enabled);
441
2d9e667e 442static int dmar_map_gfx = 1;
7d3b03ce 443static int dmar_forcedac;
5e0d2a6f 444static int intel_iommu_strict;
6dd9a7c7 445static int intel_iommu_superpage = 1;
ba395927 446
c0771df8
DW		 447int intel_iommu_gfx_mapped;
448EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
449
ba395927
KA		 450#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
451static DEFINE_SPINLOCK(device_domain_lock);
452static LIST_HEAD(device_domain_list);
453
a8bcbb0d
JR		 454static struct iommu_ops intel_iommu_ops;
455
ba395927
KA		 456static int __init intel_iommu_setup(char *str)
457{
458 if (!str)
459 return -EINVAL;
460 while (*str) {
0cd5c3c8
KM		 461 if (!strncmp(str, "on", 2)) {
462 dmar_disabled = 0;
463 printk(KERN_INFO "Intel-IOMMU: enabled\n");
464 } else if (!strncmp(str, "off", 3)) {
ba395927 465 dmar_disabled = 1;
0cd5c3c8 466 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 467 } else if (!strncmp(str, "igfx_off", 8)) {
468 dmar_map_gfx = 0;
469 printk(KERN_INFO
470 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 471 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 472 printk(KERN_INFO
7d3b03ce
KA		 473 "Intel-IOMMU: Forcing DAC for PCI devices\n");
474 dmar_forcedac = 1;
5e0d2a6f 475 } else if (!strncmp(str, "strict", 6)) {
476 printk(KERN_INFO
477 "Intel-IOMMU: disable batched IOTLB flush\n");
478 intel_iommu_strict = 1;
6dd9a7c7
YS		 479 } else if (!strncmp(str, "sp_off", 6)) {
480 printk(KERN_INFO
481 "Intel-IOMMU: disable supported super page\n");
482 intel_iommu_superpage = 0;
ba395927
KA		 483 }
484
485 str += strcspn(str, ",");
486 while (*str == ',')
487 str++;
488 }
489 return 0;
490}
491__setup("intel_iommu=", intel_iommu_setup);
492
493static struct kmem_cache *iommu_domain_cache;
494static struct kmem_cache *iommu_devinfo_cache;
495static struct kmem_cache *iommu_iova_cache;
496
4c923d47 497static inline void *alloc_pgtable_page(int node)
eb3fa7cb 498{
4c923d47
SS		 499 struct page *page;
500 void *vaddr = NULL;
eb3fa7cb 501
4c923d47
SS		 502 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
503 if (page)
504 vaddr = page_address(page);
eb3fa7cb 505 return vaddr;
ba395927
KA		 506}
507
508static inline void free_pgtable_page(void *vaddr)
509{
510 free_page((unsigned long)vaddr);
511}
512
513static inline void *alloc_domain_mem(void)
514{
354bb65e 515 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 516}
517
38717946 518static void free_domain_mem(void *vaddr)
ba395927
KA		 519{
520 kmem_cache_free(iommu_domain_cache, vaddr);
521}
522
523static inline void * alloc_devinfo_mem(void)
524{
354bb65e 525 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 526}
527
528static inline void free_devinfo_mem(void *vaddr)
529{
530 kmem_cache_free(iommu_devinfo_cache, vaddr);
531}
532
533struct iova *alloc_iova_mem(void)
534{
354bb65e 535 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 536}
537
538void free_iova_mem(struct iova *iova)
539{
540 kmem_cache_free(iommu_iova_cache, iova);
541}
542
1b573683 543
4ed0d3e6 544static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 545{
546 unsigned long sagaw;
547 int agaw = -1;
548
549 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 550 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 551 agaw >= 0; agaw--) {
552 if (test_bit(agaw, &sagaw))
553 break;
554 }
555
556 return agaw;
557}
558
4ed0d3e6
FY		 559/*
560 * Calculate max SAGAW for each iommu.
561 */
562int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
563{
564 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
565}
566
567/*
568 * calculate agaw for each iommu.
569 * "SAGAW" may be different across iommus, use a default agaw, and
570 * get a supported less agaw for iommus that don't support the default agaw.
571 */
572int iommu_calculate_agaw(struct intel_iommu *iommu)
573{
574 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
575}
576
2c2e2c38 577/* This functionin only returns single iommu in a domain */
8c11e798
WH		 578static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
579{
580 int iommu_id;
581
2c2e2c38 582 /* si_domain and vm domain should not get here. */
1ce28feb 583 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 584 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 585
1b198bb0 586 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 587 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
588 return NULL;
589
590 return g_iommus[iommu_id];
591}
592
8e604097
WH		 593static void domain_update_iommu_coherency(struct dmar_domain *domain)
594{
595 int i;
596
2e12bc29
AW		 597 i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
598
599 domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
8e604097 600
1b198bb0 601 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 602 if (!ecap_coherent(g_iommus[i]->ecap)) {
603 domain->iommu_coherency = 0;
604 break;
605 }
8e604097
WH		 606 }
607}
608
58c610bd
SY		 609static void domain_update_iommu_snooping(struct dmar_domain *domain)
610{
611 int i;
612
613 domain->iommu_snooping = 1;
614
1b198bb0 615 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 616 if (!ecap_sc_support(g_iommus[i]->ecap)) {
617 domain->iommu_snooping = 0;
618 break;
619 }
58c610bd
SY		 620 }
621}
622
6dd9a7c7
YS		 623static void domain_update_iommu_superpage(struct dmar_domain *domain)
624{
8140a95d
AK		 625 struct dmar_drhd_unit *drhd;
626 struct intel_iommu *iommu = NULL;
627 int mask = 0xf;
6dd9a7c7
YS		 628
629 if (!intel_iommu_superpage) {
630 domain->iommu_superpage = 0;
631 return;
632 }
633
8140a95d 634 /* set iommu_superpage to the smallest common denominator */
0e242612 635 rcu_read_lock();
8140a95d
AK		 636 for_each_active_iommu(iommu, drhd) {
637 mask &= cap_super_page_val(iommu->cap);
6dd9a7c7
YS		 638 if (!mask) {
639 break;
640 }
641 }
0e242612
JL		 642 rcu_read_unlock();
643
6dd9a7c7
YS		 644 domain->iommu_superpage = fls(mask);
645}
646
58c610bd
SY		 647/* Some capabilities may be different across iommus */
648static void domain_update_iommu_cap(struct dmar_domain *domain)
649{
650 domain_update_iommu_coherency(domain);
651 domain_update_iommu_snooping(domain);
6dd9a7c7 652 domain_update_iommu_superpage(domain);
58c610bd
SY		 653}
654
276dbf99 655static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 656{
657 struct dmar_drhd_unit *drhd = NULL;
b683b230
JL		 658 struct intel_iommu *iommu;
659 struct pci_dev *dev;
c7151a8d
WH		 660 int i;
661
0e242612 662 rcu_read_lock();
b683b230 663 for_each_active_iommu(iommu, drhd) {
276dbf99
DW		 664 if (segment != drhd->segment)
665 continue;
c7151a8d 666
b683b230
JL		 667 for_each_active_dev_scope(drhd->devices,
668 drhd->devices_cnt, i, dev) {
669 if (dev->bus->number == bus && dev->devfn == devfn)
670 goto out;
671 if (dev->subordinate &&
672 dev->subordinate->number <= bus &&
673 dev->subordinate->busn_res.end >= bus)
674 goto out;
924b6231 675 }
c7151a8d
WH		 676
677 if (drhd->include_all)
b683b230 678 goto out;
c7151a8d 679 }
b683b230
JL		 680 iommu = NULL;
681out:
0e242612 682 rcu_read_unlock();
c7151a8d 683
b683b230 684 return iommu;
c7151a8d
WH		 685}
686
5331fe6f
WH		 687static void domain_flush_cache(struct dmar_domain *domain,
688 void *addr, int size)
689{
690 if (!domain->iommu_coherency)
691 clflush_cache_range(addr, size);
692}
693
ba395927
KA		 694/* Gets context entry for a given bus and devfn */
695static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
696 u8 bus, u8 devfn)
697{
698 struct root_entry *root;
699 struct context_entry *context;
700 unsigned long phy_addr;
701 unsigned long flags;
702
703 spin_lock_irqsave(&iommu->lock, flags);
704 root = &iommu->root_entry[bus];
705 context = get_context_addr_from_root(root);
706 if (!context) {
4c923d47
SS		 707 context = (struct context_entry *)
708 alloc_pgtable_page(iommu->node);
ba395927
KA		 709 if (!context) {
710 spin_unlock_irqrestore(&iommu->lock, flags);
711 return NULL;
712 }
5b6985ce 713 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 714 phy_addr = virt_to_phys((void *)context);
715 set_root_value(root, phy_addr);
716 set_root_present(root);
717 __iommu_flush_cache(iommu, root, sizeof(*root));
718 }
719 spin_unlock_irqrestore(&iommu->lock, flags);
720 return &context[devfn];
721}
722
723static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
724{
725 struct root_entry *root;
726 struct context_entry *context;
727 int ret;
728 unsigned long flags;
729
730 spin_lock_irqsave(&iommu->lock, flags);
731 root = &iommu->root_entry[bus];
732 context = get_context_addr_from_root(root);
733 if (!context) {
734 ret = 0;
735 goto out;
736 }
c07e7d21 737 ret = context_present(&context[devfn]);
ba395927
KA		 738out:
739 spin_unlock_irqrestore(&iommu->lock, flags);
740 return ret;
741}
742
743static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
744{
745 struct root_entry *root;
746 struct context_entry *context;
747 unsigned long flags;
748
749 spin_lock_irqsave(&iommu->lock, flags);
750 root = &iommu->root_entry[bus];
751 context = get_context_addr_from_root(root);
752 if (context) {
c07e7d21 753 context_clear_entry(&context[devfn]);
ba395927
KA		 754 __iommu_flush_cache(iommu, &context[devfn], \
755 sizeof(*context));
756 }
757 spin_unlock_irqrestore(&iommu->lock, flags);
758}
759
760static void free_context_table(struct intel_iommu *iommu)
761{
762 struct root_entry *root;
763 int i;
764 unsigned long flags;
765 struct context_entry *context;
766
767 spin_lock_irqsave(&iommu->lock, flags);
768 if (!iommu->root_entry) {
769 goto out;
770 }
771 for (i = 0; i < ROOT_ENTRY_NR; i++) {
772 root = &iommu->root_entry[i];
773 context = get_context_addr_from_root(root);
774 if (context)
775 free_pgtable_page(context);
776 }
777 free_pgtable_page(iommu->root_entry);
778 iommu->root_entry = NULL;
779out:
780 spin_unlock_irqrestore(&iommu->lock, flags);
781}
782
b026fd28 783static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 784 unsigned long pfn, int *target_level)
ba395927 785{
b026fd28 786 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 787 struct dma_pte *parent, *pte = NULL;
788 int level = agaw_to_level(domain->agaw);
4399c8bf 789 int offset;
ba395927
KA		 790
791 BUG_ON(!domain->pgd);
f9423606
JS		 792
793 if (addr_width < BITS_PER_LONG && pfn >> addr_width)
794 /* Address beyond IOMMU's addressing capabilities. */
795 return NULL;
796
ba395927
KA		 797 parent = domain->pgd;
798
5cf0a76f 799 while (1) {
ba395927
KA		 800 void *tmp_page;
801
b026fd28 802 offset = pfn_level_offset(pfn, level);
ba395927 803 pte = &parent[offset];
5cf0a76f 804 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 805 break;
5cf0a76f 806 if (level == *target_level)
ba395927
KA		 807 break;
808
19c239ce 809 if (!dma_pte_present(pte)) {
c85994e4
DW		 810 uint64_t pteval;
811
4c923d47 812 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 813
206a73c1 814 if (!tmp_page)
ba395927 815 return NULL;
206a73c1 816
c85994e4 817 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 818 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 819 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
820 /* Someone else set it while we were thinking; use theirs. */
821 free_pgtable_page(tmp_page);
822 } else {
823 dma_pte_addr(pte);
824 domain_flush_cache(domain, pte, sizeof(*pte));
825 }
ba395927 826 }
5cf0a76f
DW		 827 if (level == 1)
828 break;
829
19c239ce 830 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 831 level--;
832 }
833
5cf0a76f
DW		 834 if (!*target_level)
835 *target_level = level;
836
ba395927
KA		 837 return pte;
838}
839
6dd9a7c7 840
ba395927 841/* return address's pte at specific level */
90dcfb5e
DW		 842static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
843 unsigned long pfn,
6dd9a7c7 844 int level, int *large_page)
ba395927
KA		 845{
846 struct dma_pte *parent, *pte = NULL;
847 int total = agaw_to_level(domain->agaw);
848 int offset;
849
850 parent = domain->pgd;
851 while (level <= total) {
90dcfb5e 852 offset = pfn_level_offset(pfn, total);
ba395927
KA		 853 pte = &parent[offset];
854 if (level == total)
855 return pte;
856
6dd9a7c7
YS		 857 if (!dma_pte_present(pte)) {
858 *large_page = total;
ba395927 859 break;
6dd9a7c7
YS		 860 }
861
862 if (pte->val & DMA_PTE_LARGE_PAGE) {
863 *large_page = total;
864 return pte;
865 }
866
19c239ce 867 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 868 total--;
869 }
870 return NULL;
871}
872
ba395927 873/* clear last level pte, a tlb flush should be followed */
5cf0a76f 874static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 875 unsigned long start_pfn,
876 unsigned long last_pfn)
ba395927 877{
04b18e65 878 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 879 unsigned int large_page = 1;
310a5ab9 880 struct dma_pte *first_pte, *pte;
66eae846 881
04b18e65 882 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 883 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 884 BUG_ON(start_pfn > last_pfn);
ba395927 885
04b18e65 886 /* we don't need lock here; nobody else touches the iova range */
59c36286 887 do {
6dd9a7c7
YS		 888 large_page = 1;
889 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 890 if (!pte) {
6dd9a7c7 891 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 892 continue;
893 }
6dd9a7c7 894 do {
310a5ab9 895 dma_clear_pte(pte);
6dd9a7c7 896 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 897 pte++;
75e6bf96
DW		 898 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
899
310a5ab9
DW		 900 domain_flush_cache(domain, first_pte,
901 (void *)pte - (void *)first_pte);
59c36286
DW		 902
903 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 904}
905
3269ee0b
AW		 906static void dma_pte_free_level(struct dmar_domain *domain, int level,
907 struct dma_pte *pte, unsigned long pfn,
908 unsigned long start_pfn, unsigned long last_pfn)
909{
910 pfn = max(start_pfn, pfn);
911 pte = &pte[pfn_level_offset(pfn, level)];
912
913 do {
914 unsigned long level_pfn;
915 struct dma_pte *level_pte;
916
917 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
918 goto next;
919
920 level_pfn = pfn & level_mask(level - 1);
921 level_pte = phys_to_virt(dma_pte_addr(pte));
922
923 if (level > 2)
924 dma_pte_free_level(domain, level - 1, level_pte,
925 level_pfn, start_pfn, last_pfn);
926
927 /* If range covers entire pagetable, free it */
928 if (!(start_pfn > level_pfn ||
08336fd2 929 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 930 dma_clear_pte(pte);
931 domain_flush_cache(domain, pte, sizeof(*pte));
932 free_pgtable_page(level_pte);
933 }
934next:
935 pfn += level_size(level);
936 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
937}
938
ba395927
KA		 939/* free page table pages. last level pte should already be cleared */
940static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 941 unsigned long start_pfn,
942 unsigned long last_pfn)
ba395927 943{
6660c63a 944 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927 945
6660c63a
DW		 946 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
947 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 948 BUG_ON(start_pfn > last_pfn);
ba395927 949
f3a0a52f 950 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 951 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
952 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 953
ba395927 954 /* free pgd */
d794dc9b 955 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 956 free_pgtable_page(domain->pgd);
957 domain->pgd = NULL;
958 }
959}
960
ea8ea460
DW		 961/* When a page at a given level is being unlinked from its parent, we don't
962 need to *modify* it at all. All we need to do is make a list of all the
963 pages which can be freed just as soon as we've flushed the IOTLB and we
964 know the hardware page-walk will no longer touch them.
965 The 'pte' argument is the *parent* PTE, pointing to the page that is to
966 be freed. */
967static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
968 int level, struct dma_pte *pte,
969 struct page *freelist)
970{
971 struct page *pg;
972
973 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
974 pg->freelist = freelist;
975 freelist = pg;
976
977 if (level == 1)
978 return freelist;
979
980 for (pte = page_address(pg); !first_pte_in_page(pte); pte++) {
981 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
982 freelist = dma_pte_list_pagetables(domain, level - 1,
983 pte, freelist);
984 }
985
986 return freelist;
987}
988
989static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
990 struct dma_pte *pte, unsigned long pfn,
991 unsigned long start_pfn,
992 unsigned long last_pfn,
993 struct page *freelist)
994{
995 struct dma_pte *first_pte = NULL, *last_pte = NULL;
996
997 pfn = max(start_pfn, pfn);
998 pte = &pte[pfn_level_offset(pfn, level)];
999
1000 do {
1001 unsigned long level_pfn;
1002
1003 if (!dma_pte_present(pte))
1004 goto next;
1005
1006 level_pfn = pfn & level_mask(level);
1007
1008 /* If range covers entire pagetable, free it */
1009 if (start_pfn <= level_pfn &&
1010 last_pfn >= level_pfn + level_size(level) - 1) {
1011 /* These suborbinate page tables are going away entirely. Don't
1012 bother to clear them; we're just going to *free* them. */
1013 if (level > 1 && !dma_pte_superpage(pte))
1014 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1015
1016 dma_clear_pte(pte);
1017 if (!first_pte)
1018 first_pte = pte;
1019 last_pte = pte;
1020 } else if (level > 1) {
1021 /* Recurse down into a level that isn't *entirely* obsolete */
1022 freelist = dma_pte_clear_level(domain, level - 1,
1023 phys_to_virt(dma_pte_addr(pte)),
1024 level_pfn, start_pfn, last_pfn,
1025 freelist);
1026 }
1027next:
1028 pfn += level_size(level);
1029 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1030
1031 if (first_pte)
1032 domain_flush_cache(domain, first_pte,
1033 (void *)++last_pte - (void *)first_pte);
1034
1035 return freelist;
1036}
1037
1038/* We can't just free the pages because the IOMMU may still be walking
1039 the page tables, and may have cached the intermediate levels. The
1040 pages can only be freed after the IOTLB flush has been done. */
1041struct page *domain_unmap(struct dmar_domain *domain,
1042 unsigned long start_pfn,
1043 unsigned long last_pfn)
1044{
1045 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1046 struct page *freelist = NULL;
1047
1048 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
1049 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
1050 BUG_ON(start_pfn > last_pfn);
1051
1052 /* we don't need lock here; nobody else touches the iova range */
1053 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1054 domain->pgd, 0, start_pfn, last_pfn, NULL);
1055
1056 /* free pgd */
1057 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1058 struct page *pgd_page = virt_to_page(domain->pgd);
1059 pgd_page->freelist = freelist;
1060 freelist = pgd_page;
1061
1062 domain->pgd = NULL;
1063 }
1064
1065 return freelist;
1066}
1067
1068void dma_free_pagelist(struct page *freelist)
1069{
1070 struct page *pg;
1071
1072 while ((pg = freelist)) {
1073 freelist = pg->freelist;
1074 free_pgtable_page(page_address(pg));
1075 }
1076}
1077
ba395927
KA		 1078/* iommu handling */
1079static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1080{
1081 struct root_entry *root;
1082 unsigned long flags;
1083
4c923d47 1084 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 1085 if (!root)
1086 return -ENOMEM;
1087
5b6985ce 1088 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1089
1090 spin_lock_irqsave(&iommu->lock, flags);
1091 iommu->root_entry = root;
1092 spin_unlock_irqrestore(&iommu->lock, flags);
1093
1094 return 0;
1095}
1096
ba395927
KA		 1097static void iommu_set_root_entry(struct intel_iommu *iommu)
1098{
1099 void *addr;
c416daa9 1100 u32 sts;
ba395927
KA		 1101 unsigned long flag;
1102
1103 addr = iommu->root_entry;
1104
1f5b3c3f 1105 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1106 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1107
c416daa9 1108 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1109
1110 /* Make sure hardware complete it */
1111 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1112 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1113
1f5b3c3f 1114 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1115}
1116
1117static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1118{
1119 u32 val;
1120 unsigned long flag;
1121
9af88143 1122 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1123 return;
ba395927 1124
1f5b3c3f 1125 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1126 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1127
1128 /* Make sure hardware complete it */
1129 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1130 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1131
1f5b3c3f 1132 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1133}
1134
1135/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1136static void __iommu_flush_context(struct intel_iommu *iommu,
1137 u16 did, u16 source_id, u8 function_mask,
1138 u64 type)
ba395927
KA		 1139{
1140 u64 val = 0;
1141 unsigned long flag;
1142
ba395927
KA		 1143 switch (type) {
1144 case DMA_CCMD_GLOBAL_INVL:
1145 val = DMA_CCMD_GLOBAL_INVL;
1146 break;
1147 case DMA_CCMD_DOMAIN_INVL:
1148 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1149 break;
1150 case DMA_CCMD_DEVICE_INVL:
1151 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1152 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1153 break;
1154 default:
1155 BUG();
1156 }
1157 val |= DMA_CCMD_ICC;
1158
1f5b3c3f 1159 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1160 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1161
1162 /* Make sure hardware complete it */
1163 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1164 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1165
1f5b3c3f 1166 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1167}
1168
ba395927 1169/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1170static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1171 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1172{
1173 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1174 u64 val = 0, val_iva = 0;
1175 unsigned long flag;
1176
ba395927
KA		 1177 switch (type) {
1178 case DMA_TLB_GLOBAL_FLUSH:
1179 /* global flush doesn't need set IVA_REG */
1180 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1181 break;
1182 case DMA_TLB_DSI_FLUSH:
1183 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1184 break;
1185 case DMA_TLB_PSI_FLUSH:
1186 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1187 /* IH bit is passed in as part of address */
ba395927
KA		 1188 val_iva = size_order | addr;
1189 break;
1190 default:
1191 BUG();
1192 }
1193 /* Note: set drain read/write */
1194#if 0
1195 /*
1196 * This is probably to be super secure.. Looks like we can
1197 * ignore it without any impact.
1198 */
1199 if (cap_read_drain(iommu->cap))
1200 val |= DMA_TLB_READ_DRAIN;
1201#endif
1202 if (cap_write_drain(iommu->cap))
1203 val |= DMA_TLB_WRITE_DRAIN;
1204
1f5b3c3f 1205 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1206 /* Note: Only uses first TLB reg currently */
1207 if (val_iva)
1208 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1209 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1210
1211 /* Make sure hardware complete it */
1212 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1213 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1214
1f5b3c3f 1215 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1216
1217 /* check IOTLB invalidation granularity */
1218 if (DMA_TLB_IAIG(val) == 0)
1219 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1220 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1221 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1222 (unsigned long long)DMA_TLB_IIRG(type),
1223 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1224}
1225
93a23a72
YZ		 1226static struct device_domain_info *iommu_support_dev_iotlb(
1227 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1228{
1229 int found = 0;
1230 unsigned long flags;
1231 struct device_domain_info *info;
1232 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1233
1234 if (!ecap_dev_iotlb_support(iommu->ecap))
1235 return NULL;
1236
1237 if (!iommu->qi)
1238 return NULL;
1239
1240 spin_lock_irqsave(&device_domain_lock, flags);
1241 list_for_each_entry(info, &domain->devices, link)
1242 if (info->bus == bus && info->devfn == devfn) {
1243 found = 1;
1244 break;
1245 }
1246 spin_unlock_irqrestore(&device_domain_lock, flags);
1247
1248 if (!found || !info->dev)
1249 return NULL;
1250
1251 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1252 return NULL;
1253
1254 if (!dmar_find_matched_atsr_unit(info->dev))
1255 return NULL;
1256
1257 info->iommu = iommu;
1258
1259 return info;
1260}
1261
1262static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1263{
93a23a72
YZ		 1264 if (!info)
1265 return;
1266
1267 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1268}
1269
1270static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1271{
1272 if (!info->dev || !pci_ats_enabled(info->dev))
1273 return;
1274
1275 pci_disable_ats(info->dev);
1276}
1277
1278static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1279 u64 addr, unsigned mask)
1280{
1281 u16 sid, qdep;
1282 unsigned long flags;
1283 struct device_domain_info *info;
1284
1285 spin_lock_irqsave(&device_domain_lock, flags);
1286 list_for_each_entry(info, &domain->devices, link) {
1287 if (!info->dev || !pci_ats_enabled(info->dev))
1288 continue;
1289
1290 sid = info->bus << 8 | info->devfn;
1291 qdep = pci_ats_queue_depth(info->dev);
1292 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1293 }
1294 spin_unlock_irqrestore(&device_domain_lock, flags);
1295}
1296
1f0ef2aa 1297static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1298 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1299{
9dd2fe89 1300 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1301 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1302
ba395927
KA		 1303 BUG_ON(pages == 0);
1304
ea8ea460
DW		 1305 if (ih)
1306 ih = 1 << 6;
ba395927 1307 /*
9dd2fe89
YZ		 1308 * Fallback to domain selective flush if no PSI support or the size is
1309 * too big.
ba395927
KA		 1310 * PSI requires page size to be 2 ^ x, and the base address is naturally
1311 * aligned to the size
1312 */
9dd2fe89
YZ		 1313 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1314 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1315 DMA_TLB_DSI_FLUSH);
9dd2fe89 1316 else
ea8ea460 1317 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1318 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1319
1320 /*
82653633
NA		 1321 * In caching mode, changes of pages from non-present to present require
1322 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1323 */
82653633 1324 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1325 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1326}
1327
f8bab735 1328static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1329{
1330 u32 pmen;
1331 unsigned long flags;
1332
1f5b3c3f 1333 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1334 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1335 pmen &= ~DMA_PMEN_EPM;
1336 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1337
1338 /* wait for the protected region status bit to clear */
1339 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1340 readl, !(pmen & DMA_PMEN_PRS), pmen);
1341
1f5b3c3f 1342 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1343}
1344
ba395927
KA		 1345static int iommu_enable_translation(struct intel_iommu *iommu)
1346{
1347 u32 sts;
1348 unsigned long flags;
1349
1f5b3c3f 1350 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1351 iommu->gcmd |= DMA_GCMD_TE;
1352 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1353
1354 /* Make sure hardware complete it */
1355 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1356 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1357
1f5b3c3f 1358 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1359 return 0;
1360}
1361
1362static int iommu_disable_translation(struct intel_iommu *iommu)
1363{
1364 u32 sts;
1365 unsigned long flag;
1366
1f5b3c3f 1367 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1368 iommu->gcmd &= ~DMA_GCMD_TE;
1369 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1370
1371 /* Make sure hardware complete it */
1372 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1373 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1374
1f5b3c3f 1375 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1376 return 0;
1377}
1378
3460a6d9 1379
ba395927
KA		 1380static int iommu_init_domains(struct intel_iommu *iommu)
1381{
1382 unsigned long ndomains;
1383 unsigned long nlongs;
1384
1385 ndomains = cap_ndoms(iommu->cap);
852bdb04
JL		 1386 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1387 iommu->seq_id, ndomains);
ba395927
KA		 1388 nlongs = BITS_TO_LONGS(ndomains);
1389
94a91b50
DD		 1390 spin_lock_init(&iommu->lock);
1391
ba395927
KA		 1392 /* TBD: there might be 64K domains,
1393 * consider other allocation for future chip
1394 */
1395 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1396 if (!iommu->domain_ids) {
852bdb04
JL		 1397 pr_err("IOMMU%d: allocating domain id array failed\n",
1398 iommu->seq_id);
ba395927
KA		 1399 return -ENOMEM;
1400 }
1401 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1402 GFP_KERNEL);
1403 if (!iommu->domains) {
852bdb04
JL		 1404 pr_err("IOMMU%d: allocating domain array failed\n",
1405 iommu->seq_id);
1406 kfree(iommu->domain_ids);
1407 iommu->domain_ids = NULL;
ba395927
KA		 1408 return -ENOMEM;
1409 }
1410
1411 /*
1412 * if Caching mode is set, then invalid translations are tagged
1413 * with domainid 0. Hence we need to pre-allocate it.
1414 */
1415 if (cap_caching_mode(iommu->cap))
1416 set_bit(0, iommu->domain_ids);
1417 return 0;
1418}
ba395927 1419
a868e6b7 1420static void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1421{
1422 struct dmar_domain *domain;
5ced12af 1423 int i, count;
c7151a8d 1424 unsigned long flags;
ba395927 1425
94a91b50 1426 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1427 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1428 /*
1429 * Domain id 0 is reserved for invalid translation
1430 * if hardware supports caching mode.
1431 */
1432 if (cap_caching_mode(iommu->cap) && i == 0)
1433 continue;
1434
94a91b50
DD		 1435 domain = iommu->domains[i];
1436 clear_bit(i, iommu->domain_ids);
1437
1438 spin_lock_irqsave(&domain->iommu_lock, flags);
5ced12af
JL		 1439 count = --domain->iommu_count;
1440 spin_unlock_irqrestore(&domain->iommu_lock, flags);
92d03cc8
JL		 1441 if (count == 0)
1442 domain_exit(domain);
5e98c4b1 1443 }
ba395927
KA		 1444 }
1445
1446 if (iommu->gcmd & DMA_GCMD_TE)
1447 iommu_disable_translation(iommu);
1448
ba395927
KA		 1449 kfree(iommu->domains);
1450 kfree(iommu->domain_ids);
a868e6b7
JL		 1451 iommu->domains = NULL;
1452 iommu->domain_ids = NULL;
ba395927 1453
d9630fe9
WH		 1454 g_iommus[iommu->seq_id] = NULL;
1455
ba395927
KA		 1456 /* free context mapping */
1457 free_context_table(iommu);
ba395927
KA		 1458}
1459
92d03cc8 1460static struct dmar_domain *alloc_domain(bool vm)
ba395927 1461{
92d03cc8
JL		 1462 /* domain id for virtual machine, it won't be set in context */
1463 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1464 struct dmar_domain *domain;
ba395927
KA		 1465
1466 domain = alloc_domain_mem();
1467 if (!domain)
1468 return NULL;
1469
4c923d47 1470 domain->nid = -1;
92d03cc8 1471 domain->iommu_count = 0;
1b198bb0 1472 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
2c2e2c38 1473 domain->flags = 0;
92d03cc8
JL		 1474 spin_lock_init(&domain->iommu_lock);
1475 INIT_LIST_HEAD(&domain->devices);
1476 if (vm) {
1477 domain->id = atomic_inc_return(&vm_domid);
1478 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
1479 }
2c2e2c38
FY		 1480
1481 return domain;
1482}
1483
1484static int iommu_attach_domain(struct dmar_domain *domain,
1485 struct intel_iommu *iommu)
1486{
1487 int num;
1488 unsigned long ndomains;
1489 unsigned long flags;
1490
ba395927
KA		 1491 ndomains = cap_ndoms(iommu->cap);
1492
1493 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1494
ba395927
KA		 1495 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1496 if (num >= ndomains) {
1497 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1498 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1499 return -ENOMEM;
ba395927
KA		 1500 }
1501
ba395927 1502 domain->id = num;
9ebd682e 1503 domain->iommu_count++;
2c2e2c38 1504 set_bit(num, iommu->domain_ids);
1b198bb0 1505 set_bit(iommu->seq_id, domain->iommu_bmp);
ba395927
KA		 1506 iommu->domains[num] = domain;
1507 spin_unlock_irqrestore(&iommu->lock, flags);
1508
2c2e2c38 1509 return 0;
ba395927
KA		 1510}
1511
2c2e2c38
FY		 1512static void iommu_detach_domain(struct dmar_domain *domain,
1513 struct intel_iommu *iommu)
ba395927
KA		 1514{
1515 unsigned long flags;
2c2e2c38 1516 int num, ndomains;
ba395927 1517
8c11e798 1518 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1519 ndomains = cap_ndoms(iommu->cap);
a45946ab 1520 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38 1521 if (iommu->domains[num] == domain) {
92d03cc8
JL		 1522 clear_bit(num, iommu->domain_ids);
1523 iommu->domains[num] = NULL;
2c2e2c38
FY		 1524 break;
1525 }
2c2e2c38 1526 }
8c11e798 1527 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1528}
1529
1530static struct iova_domain reserved_iova_list;
8a443df4 1531static struct lock_class_key reserved_rbtree_key;
ba395927 1532
51a63e67 1533static int dmar_init_reserved_ranges(void)
ba395927
KA		 1534{
1535 struct pci_dev *pdev = NULL;
1536 struct iova *iova;
1537 int i;
ba395927 1538
f661197e 1539 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1540
8a443df4
MG		 1541 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1542 &reserved_rbtree_key);
1543
ba395927
KA		 1544 /* IOAPIC ranges shouldn't be accessed by DMA */
1545 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1546 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1547 if (!iova) {
ba395927 1548 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1549 return -ENODEV;
1550 }
ba395927
KA		 1551
1552 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1553 for_each_pci_dev(pdev) {
1554 struct resource *r;
1555
1556 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1557 r = &pdev->resource[i];
1558 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1559 continue;
1a4a4551
DW		 1560 iova = reserve_iova(&reserved_iova_list,
1561 IOVA_PFN(r->start),
1562 IOVA_PFN(r->end));
51a63e67 1563 if (!iova) {
ba395927 1564 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1565 return -ENODEV;
1566 }
ba395927
KA		 1567 }
1568 }
51a63e67 1569 return 0;
ba395927
KA		 1570}
1571
1572static void domain_reserve_special_ranges(struct dmar_domain *domain)
1573{
1574 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1575}
1576
1577static inline int guestwidth_to_adjustwidth(int gaw)
1578{
1579 int agaw;
1580 int r = (gaw - 12) % 9;
1581
1582 if (r == 0)
1583 agaw = gaw;
1584 else
1585 agaw = gaw + 9 - r;
1586 if (agaw > 64)
1587 agaw = 64;
1588 return agaw;
1589}
1590
1591static int domain_init(struct dmar_domain *domain, int guest_width)
1592{
1593 struct intel_iommu *iommu;
1594 int adjust_width, agaw;
1595 unsigned long sagaw;
1596
f661197e 1597 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
ba395927
KA		 1598 domain_reserve_special_ranges(domain);
1599
1600 /* calculate AGAW */
8c11e798 1601 iommu = domain_get_iommu(domain);
ba395927
KA		 1602 if (guest_width > cap_mgaw(iommu->cap))
1603 guest_width = cap_mgaw(iommu->cap);
1604 domain->gaw = guest_width;
1605 adjust_width = guestwidth_to_adjustwidth(guest_width);
1606 agaw = width_to_agaw(adjust_width);
1607 sagaw = cap_sagaw(iommu->cap);
1608 if (!test_bit(agaw, &sagaw)) {
1609 /* hardware doesn't support it, choose a bigger one */
1610 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1611 agaw = find_next_bit(&sagaw, 5, agaw);
1612 if (agaw >= 5)
1613 return -ENODEV;
1614 }
1615 domain->agaw = agaw;
ba395927 1616
8e604097
WH		 1617 if (ecap_coherent(iommu->ecap))
1618 domain->iommu_coherency = 1;
1619 else
1620 domain->iommu_coherency = 0;
1621
58c610bd
SY		 1622 if (ecap_sc_support(iommu->ecap))
1623 domain->iommu_snooping = 1;
1624 else
1625 domain->iommu_snooping = 0;
1626
6dd9a7c7 1627 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
4c923d47 1628 domain->nid = iommu->node;
c7151a8d 1629
ba395927 1630 /* always allocate the top pgd */
4c923d47 1631 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1632 if (!domain->pgd)
1633 return -ENOMEM;
5b6985ce 1634 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1635 return 0;
1636}
1637
1638static void domain_exit(struct dmar_domain *domain)
1639{
2c2e2c38
FY		 1640 struct dmar_drhd_unit *drhd;
1641 struct intel_iommu *iommu;
ea8ea460 1642 struct page *freelist = NULL;
ba395927
KA		 1643
1644 /* Domain 0 is reserved, so dont process it */
1645 if (!domain)
1646 return;
1647
7b668357
AW		 1648 /* Flush any lazy unmaps that may reference this domain */
1649 if (!intel_iommu_strict)
1650 flush_unmaps_timeout(0);
1651
92d03cc8 1652 /* remove associated devices */
ba395927 1653 domain_remove_dev_info(domain);
92d03cc8 1654
ba395927
KA		 1655 /* destroy iovas */
1656 put_iova_domain(&domain->iovad);
ba395927 1657
ea8ea460 1658 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1659
92d03cc8 1660 /* clear attached or cached domains */
0e242612 1661 rcu_read_lock();
2c2e2c38 1662 for_each_active_iommu(iommu, drhd)
92d03cc8
JL		 1663 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1664 test_bit(iommu->seq_id, domain->iommu_bmp))
2c2e2c38 1665 iommu_detach_domain(domain, iommu);
0e242612 1666 rcu_read_unlock();
2c2e2c38 1667
ea8ea460
DW		 1668 dma_free_pagelist(freelist);
1669
ba395927
KA		 1670 free_domain_mem(domain);
1671}
1672
4ed0d3e6
FY		 1673static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1674 u8 bus, u8 devfn, int translation)
ba395927
KA		 1675{
1676 struct context_entry *context;
ba395927 1677 unsigned long flags;
5331fe6f 1678 struct intel_iommu *iommu;
ea6606b0
WH		 1679 struct dma_pte *pgd;
1680 unsigned long num;
1681 unsigned long ndomains;
1682 int id;
1683 int agaw;
93a23a72 1684 struct device_domain_info *info = NULL;
ba395927
KA		 1685
1686 pr_debug("Set context mapping for %02x:%02x.%d\n",
1687 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1688
ba395927 1689 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1690 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1691 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1692
276dbf99 1693 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1694 if (!iommu)
1695 return -ENODEV;
1696
ba395927
KA		 1697 context = device_to_context_entry(iommu, bus, devfn);
1698 if (!context)
1699 return -ENOMEM;
1700 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1701 if (context_present(context)) {
ba395927
KA		 1702 spin_unlock_irqrestore(&iommu->lock, flags);
1703 return 0;
1704 }
1705
ea6606b0
WH		 1706 id = domain->id;
1707 pgd = domain->pgd;
1708
2c2e2c38
FY		 1709 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1710 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1711 int found = 0;
1712
1713 /* find an available domain id for this device in iommu */
1714 ndomains = cap_ndoms(iommu->cap);
a45946ab 1715 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1716 if (iommu->domains[num] == domain) {
1717 id = num;
1718 found = 1;
1719 break;
1720 }
ea6606b0
WH		 1721 }
1722
1723 if (found == 0) {
1724 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1725 if (num >= ndomains) {
1726 spin_unlock_irqrestore(&iommu->lock, flags);
1727 printk(KERN_ERR "IOMMU: no free domain ids\n");
1728 return -EFAULT;
1729 }
1730
1731 set_bit(num, iommu->domain_ids);
1732 iommu->domains[num] = domain;
1733 id = num;
1734 }
1735
1736 /* Skip top levels of page tables for
1737 * iommu which has less agaw than default.
1672af11 1738 * Unnecessary for PT mode.
ea6606b0 1739 */
1672af11
CW		 1740 if (translation != CONTEXT_TT_PASS_THROUGH) {
1741 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1742 pgd = phys_to_virt(dma_pte_addr(pgd));
1743 if (!dma_pte_present(pgd)) {
1744 spin_unlock_irqrestore(&iommu->lock, flags);
1745 return -ENOMEM;
1746 }
ea6606b0
WH		 1747 }
1748 }
1749 }
1750
1751 context_set_domain_id(context, id);
4ed0d3e6 1752
93a23a72
YZ		 1753 if (translation != CONTEXT_TT_PASS_THROUGH) {
1754 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1755 translation = info ? CONTEXT_TT_DEV_IOTLB :
1756 CONTEXT_TT_MULTI_LEVEL;
1757 }
4ed0d3e6
FY		 1758 /*
1759 * In pass through mode, AW must be programmed to indicate the largest
1760 * AGAW value supported by hardware. And ASR is ignored by hardware.
1761 */
93a23a72 1762 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1763 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1764 else {
1765 context_set_address_root(context, virt_to_phys(pgd));
1766 context_set_address_width(context, iommu->agaw);
1767 }
4ed0d3e6
FY		 1768
1769 context_set_translation_type(context, translation);
c07e7d21
MM		 1770 context_set_fault_enable(context);
1771 context_set_present(context);
5331fe6f 1772 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1773
4c25a2c1
DW		 1774 /*
1775 * It's a non-present to present mapping. If hardware doesn't cache
1776 * non-present entry we only need to flush the write-buffer. If the
1777 * _does_ cache non-present entries, then it does so in the special
1778 * domain #0, which we have to flush:
1779 */
1780 if (cap_caching_mode(iommu->cap)) {
1781 iommu->flush.flush_context(iommu, 0,
1782 (((u16)bus) << 8) | devfn,
1783 DMA_CCMD_MASK_NOBIT,
1784 DMA_CCMD_DEVICE_INVL);
82653633 1785 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1786 } else {
ba395927 1787 iommu_flush_write_buffer(iommu);
4c25a2c1 1788 }
93a23a72 1789 iommu_enable_dev_iotlb(info);
ba395927 1790 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1791
1792 spin_lock_irqsave(&domain->iommu_lock, flags);
1b198bb0 1793 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
c7151a8d 1794 domain->iommu_count++;
4c923d47
SS		 1795 if (domain->iommu_count == 1)
1796 domain->nid = iommu->node;
58c610bd 1797 domain_update_iommu_cap(domain);
c7151a8d
WH		 1798 }
1799 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1800 return 0;
1801}
1802
1803static int
4ed0d3e6
FY		 1804domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1805 int translation)
ba395927
KA		 1806{
1807 int ret;
1808 struct pci_dev *tmp, *parent;
1809
276dbf99 1810 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1811 pdev->bus->number, pdev->devfn,
1812 translation);
ba395927
KA		 1813 if (ret)
1814 return ret;
1815
1816 /* dependent device mapping */
1817 tmp = pci_find_upstream_pcie_bridge(pdev);
1818 if (!tmp)
1819 return 0;
1820 /* Secondary interface's bus number and devfn 0 */
1821 parent = pdev->bus->self;
1822 while (parent != tmp) {
276dbf99
DW		 1823 ret = domain_context_mapping_one(domain,
1824 pci_domain_nr(parent->bus),
1825 parent->bus->number,
4ed0d3e6 1826 parent->devfn, translation);
ba395927
KA		 1827 if (ret)
1828 return ret;
1829 parent = parent->bus->self;
1830 }
45e829ea 1831 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1832 return domain_context_mapping_one(domain,
276dbf99 1833 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1834 tmp->subordinate->number, 0,
1835 translation);
ba395927
KA		 1836 else /* this is a legacy PCI bridge */
1837 return domain_context_mapping_one(domain,
276dbf99
DW		 1838 pci_domain_nr(tmp->bus),
1839 tmp->bus->number,
4ed0d3e6
FY		 1840 tmp->devfn,
1841 translation);
ba395927
KA		 1842}
1843
5331fe6f 1844static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1845{
1846 int ret;
1847 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1848 struct intel_iommu *iommu;
1849
276dbf99
DW		 1850 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1851 pdev->devfn);
5331fe6f
WH		 1852 if (!iommu)
1853 return -ENODEV;
ba395927 1854
276dbf99 1855 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1856 if (!ret)
1857 return ret;
1858 /* dependent device mapping */
1859 tmp = pci_find_upstream_pcie_bridge(pdev);
1860 if (!tmp)
1861 return ret;
1862 /* Secondary interface's bus number and devfn 0 */
1863 parent = pdev->bus->self;
1864 while (parent != tmp) {
8c11e798 1865 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1866 parent->devfn);
ba395927
KA		 1867 if (!ret)
1868 return ret;
1869 parent = parent->bus->self;
1870 }
5f4d91a1 1871 if (pci_is_pcie(tmp))
276dbf99
DW		 1872 return device_context_mapped(iommu, tmp->subordinate->number,
1873 0);
ba395927 1874 else
276dbf99
DW		 1875 return device_context_mapped(iommu, tmp->bus->number,
1876 tmp->devfn);
ba395927
KA		 1877}
1878
f532959b
FY		 1879/* Returns a number of VTD pages, but aligned to MM page size */
1880static inline unsigned long aligned_nrpages(unsigned long host_addr,
1881 size_t size)
1882{
1883 host_addr &= ~PAGE_MASK;
1884 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1885}
1886
6dd9a7c7
YS		 1887/* Return largest possible superpage level for a given mapping */
1888static inline int hardware_largepage_caps(struct dmar_domain *domain,
1889 unsigned long iov_pfn,
1890 unsigned long phy_pfn,
1891 unsigned long pages)
1892{
1893 int support, level = 1;
1894 unsigned long pfnmerge;
1895
1896 support = domain->iommu_superpage;
1897
1898 /* To use a large page, the virtual *and* physical addresses
1899 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1900 of them will mean we have to use smaller pages. So just
1901 merge them and check both at once. */
1902 pfnmerge = iov_pfn | phy_pfn;
1903
1904 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1905 pages >>= VTD_STRIDE_SHIFT;
1906 if (!pages)
1907 break;
1908 pfnmerge >>= VTD_STRIDE_SHIFT;
1909 level++;
1910 support--;
1911 }
1912 return level;
1913}
1914
9051aa02
DW		 1915static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1916 struct scatterlist *sg, unsigned long phys_pfn,
1917 unsigned long nr_pages, int prot)
e1605495
DW		 1918{
1919 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1920 phys_addr_t uninitialized_var(pteval);
e1605495 1921 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1922 unsigned long sg_res;
6dd9a7c7
YS		 1923 unsigned int largepage_lvl = 0;
1924 unsigned long lvl_pages = 0;
e1605495
DW		 1925
1926 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1927
1928 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1929 return -EINVAL;
1930
1931 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1932
9051aa02
DW		 1933 if (sg)
1934 sg_res = 0;
1935 else {
1936 sg_res = nr_pages + 1;
1937 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1938 }
1939
6dd9a7c7 1940 while (nr_pages > 0) {
c85994e4
DW		 1941 uint64_t tmp;
1942
e1605495 1943 if (!sg_res) {
f532959b 1944 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1945 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1946 sg->dma_length = sg->length;
1947 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 1948 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 1949 }
6dd9a7c7 1950
e1605495 1951 if (!pte) {
6dd9a7c7
YS		 1952 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1953
5cf0a76f 1954 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 1955 if (!pte)
1956 return -ENOMEM;
6dd9a7c7 1957 /* It is large page*/
6491d4d0 1958 if (largepage_lvl > 1) {
6dd9a7c7 1959 pteval |= DMA_PTE_LARGE_PAGE;
6491d4d0
WD		 1960 /* Ensure that old small page tables are removed to make room
1961 for superpage, if they exist. */
1962 dma_pte_clear_range(domain, iov_pfn,
1963 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1964 dma_pte_free_pagetable(domain, iov_pfn,
1965 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1966 } else {
6dd9a7c7 1967 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 1968 }
6dd9a7c7 1969
e1605495
DW		 1970 }
1971 /* We don't need lock here, nobody else
1972 * touches the iova range
1973 */
7766a3fb 1974 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1975 if (tmp) {
1bf20f0d 1976 static int dumps = 5;
c85994e4
DW		 1977 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1978 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1979 if (dumps) {
1980 dumps--;
1981 debug_dma_dump_mappings(NULL);
1982 }
1983 WARN_ON(1);
1984 }
6dd9a7c7
YS		 1985
1986 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1987
1988 BUG_ON(nr_pages < lvl_pages);
1989 BUG_ON(sg_res < lvl_pages);
1990
1991 nr_pages -= lvl_pages;
1992 iov_pfn += lvl_pages;
1993 phys_pfn += lvl_pages;
1994 pteval += lvl_pages * VTD_PAGE_SIZE;
1995 sg_res -= lvl_pages;
1996
1997 /* If the next PTE would be the first in a new page, then we
1998 need to flush the cache on the entries we've just written.
1999 And then we'll need to recalculate 'pte', so clear it and
2000 let it get set again in the if (!pte) block above.
2001
2002 If we're done (!nr_pages) we need to flush the cache too.
2003
2004 Also if we've been setting superpages, we may need to
2005 recalculate 'pte' and switch back to smaller pages for the
2006 end of the mapping, if the trailing size is not enough to
2007 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2008 pte++;
6dd9a7c7
YS		 2009 if (!nr_pages || first_pte_in_page(pte) ||
2010 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2011 domain_flush_cache(domain, first_pte,
2012 (void *)pte - (void *)first_pte);
2013 pte = NULL;
2014 }
6dd9a7c7
YS		 2015
2016 if (!sg_res && nr_pages)
e1605495
DW		 2017 sg = sg_next(sg);
2018 }
2019 return 0;
2020}
2021
9051aa02
DW		 2022static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2023 struct scatterlist *sg, unsigned long nr_pages,
2024 int prot)
ba395927 2025{
9051aa02
DW		 2026 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2027}
6f6a00e4 2028
9051aa02
DW		 2029static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2030 unsigned long phys_pfn, unsigned long nr_pages,
2031 int prot)
2032{
2033 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2034}
2035
c7151a8d 2036static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2037{
c7151a8d
WH		 2038 if (!iommu)
2039 return;
8c11e798
WH		 2040
2041 clear_context_table(iommu, bus, devfn);
2042 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2043 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2044 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2045}
2046
109b9b04
DW		 2047static inline void unlink_domain_info(struct device_domain_info *info)
2048{
2049 assert_spin_locked(&device_domain_lock);
2050 list_del(&info->link);
2051 list_del(&info->global);
2052 if (info->dev)
2053 info->dev->dev.archdata.iommu = NULL;
2054}
2055
ba395927
KA		 2056static void domain_remove_dev_info(struct dmar_domain *domain)
2057{
2058 struct device_domain_info *info;
92d03cc8 2059 unsigned long flags, flags2;
c7151a8d 2060 struct intel_iommu *iommu;
ba395927
KA		 2061
2062 spin_lock_irqsave(&device_domain_lock, flags);
2063 while (!list_empty(&domain->devices)) {
2064 info = list_entry(domain->devices.next,
2065 struct device_domain_info, link);
109b9b04 2066 unlink_domain_info(info);
ba395927
KA		 2067 spin_unlock_irqrestore(&device_domain_lock, flags);
2068
93a23a72 2069 iommu_disable_dev_iotlb(info);
276dbf99 2070 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 2071 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927 2072
92d03cc8
JL		 2073 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
2074 iommu_detach_dependent_devices(iommu, info->dev);
2075 /* clear this iommu in iommu_bmp, update iommu count
2076 * and capabilities
2077 */
2078 spin_lock_irqsave(&domain->iommu_lock, flags2);
2079 if (test_and_clear_bit(iommu->seq_id,
2080 domain->iommu_bmp)) {
2081 domain->iommu_count--;
2082 domain_update_iommu_cap(domain);
2083 }
2084 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2085 }
2086
2087 free_devinfo_mem(info);
ba395927
KA		 2088 spin_lock_irqsave(&device_domain_lock, flags);
2089 }
2090 spin_unlock_irqrestore(&device_domain_lock, flags);
2091}
2092
2093/*
2094 * find_domain
358dd8ac 2095 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 2096 */
38717946 2097static struct dmar_domain *
ba395927
KA		 2098find_domain(struct pci_dev *pdev)
2099{
2100 struct device_domain_info *info;
2101
2102 /* No lock here, assumes no domain exit in normal case */
358dd8ac 2103 info = pdev->dev.archdata.iommu;
ba395927
KA		 2104 if (info)
2105 return info->domain;
2106 return NULL;
2107}
2108
745f2586
JL		 2109static inline struct dmar_domain *
2110dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2111{
2112 struct device_domain_info *info;
2113
2114 list_for_each_entry(info, &device_domain_list, global)
2115 if (info->segment == segment && info->bus == bus &&
2116 info->devfn == devfn)
2117 return info->domain;
2118
2119 return NULL;
2120}
2121
2122static int dmar_insert_dev_info(int segment, int bus, int devfn,
2123 struct pci_dev *dev, struct dmar_domain **domp)
2124{
2125 struct dmar_domain *found, *domain = *domp;
2126 struct device_domain_info *info;
2127 unsigned long flags;
2128
2129 info = alloc_devinfo_mem();
2130 if (!info)
2131 return -ENOMEM;
2132
2133 info->segment = segment;
2134 info->bus = bus;
2135 info->devfn = devfn;
2136 info->dev = dev;
2137 info->domain = domain;
2138 if (!dev)
2139 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2140
2141 spin_lock_irqsave(&device_domain_lock, flags);
2142 if (dev)
2143 found = find_domain(dev);
2144 else
2145 found = dmar_search_domain_by_dev_info(segment, bus, devfn);
2146 if (found) {
2147 spin_unlock_irqrestore(&device_domain_lock, flags);
2148 free_devinfo_mem(info);
2149 if (found != domain) {
2150 domain_exit(domain);
2151 *domp = found;
2152 }
2153 } else {
2154 list_add(&info->link, &domain->devices);
2155 list_add(&info->global, &device_domain_list);
2156 if (dev)
2157 dev->dev.archdata.iommu = info;
2158 spin_unlock_irqrestore(&device_domain_lock, flags);
2159 }
2160
2161 return 0;
2162}
2163
ba395927
KA		 2164/* domain is initialized */
2165static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
2166{
e85bb5d4 2167 struct dmar_domain *domain, *free = NULL;
ba395927
KA		 2168 struct intel_iommu *iommu;
2169 struct dmar_drhd_unit *drhd;
ba395927
KA		 2170 struct pci_dev *dev_tmp;
2171 unsigned long flags;
2172 int bus = 0, devfn = 0;
276dbf99 2173 int segment;
ba395927
KA		 2174
2175 domain = find_domain(pdev);
2176 if (domain)
2177 return domain;
2178
276dbf99
DW		 2179 segment = pci_domain_nr(pdev->bus);
2180
ba395927
KA		 2181 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
2182 if (dev_tmp) {
5f4d91a1 2183 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 2184 bus = dev_tmp->subordinate->number;
2185 devfn = 0;
2186 } else {
2187 bus = dev_tmp->bus->number;
2188 devfn = dev_tmp->devfn;
2189 }
2190 spin_lock_irqsave(&device_domain_lock, flags);
745f2586 2191 domain = dmar_search_domain_by_dev_info(segment, bus, devfn);
ba395927
KA		 2192 spin_unlock_irqrestore(&device_domain_lock, flags);
2193 /* pcie-pci bridge already has a domain, uses it */
745f2586 2194 if (domain)
ba395927 2195 goto found_domain;
ba395927
KA		 2196 }
2197
ba395927
KA		 2198 drhd = dmar_find_matched_drhd_unit(pdev);
2199 if (!drhd) {
2200 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
2201 pci_name(pdev));
2202 return NULL;
2203 }
2204 iommu = drhd->iommu;
2205
745f2586 2206 /* Allocate and intialize new domain for the device */
92d03cc8 2207 domain = alloc_domain(false);
745f2586
JL		 2208 if (!domain)
2209 goto error;
2210 if (iommu_attach_domain(domain, iommu)) {
2fe9723d 2211 free_domain_mem(domain);
ba395927 2212 goto error;
2c2e2c38 2213 }
e85bb5d4
JL		 2214 free = domain;
2215 if (domain_init(domain, gaw))
ba395927 2216 goto error;
ba395927
KA		 2217
2218 /* register pcie-to-pci device */
2219 if (dev_tmp) {
e85bb5d4 2220 if (dmar_insert_dev_info(segment, bus, devfn, NULL, &domain))
ba395927 2221 goto error;
e85bb5d4
JL		 2222 else
2223 free = NULL;
ba395927
KA		 2224 }
2225
2226found_domain:
745f2586
JL		 2227 if (dmar_insert_dev_info(segment, pdev->bus->number, pdev->devfn,
2228 pdev, &domain) == 0)
ba395927 2229 return domain;
ba395927 2230error:
e85bb5d4
JL		 2231 if (free)
2232 domain_exit(free);
ba395927
KA		 2233 /* recheck it here, maybe others set it */
2234 return find_domain(pdev);
2235}
2236
2c2e2c38 2237static int iommu_identity_mapping;
e0fc7e0b
DW		 2238#define IDENTMAP_ALL 1
2239#define IDENTMAP_GFX 2
2240#define IDENTMAP_AZALIA 4
2c2e2c38 2241
b213203e
DW		 2242static int iommu_domain_identity_map(struct dmar_domain *domain,
2243 unsigned long long start,
2244 unsigned long long end)
ba395927 2245{
c5395d5c
DW		 2246 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2247 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2248
2249 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2250 dma_to_mm_pfn(last_vpfn))) {
ba395927 2251 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2252 return -ENOMEM;
ba395927
KA		 2253 }
2254
c5395d5c
DW		 2255 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2256 start, end, domain->id);
ba395927
KA		 2257 /*
2258 * RMRR range might have overlap with physical memory range,
2259 * clear it first
2260 */
c5395d5c 2261 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2262
c5395d5c
DW		 2263 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2264 last_vpfn - first_vpfn + 1,
61df7443 2265 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2266}
2267
2268static int iommu_prepare_identity_map(struct pci_dev *pdev,
2269 unsigned long long start,
2270 unsigned long long end)
2271{
2272 struct dmar_domain *domain;
2273 int ret;
2274
c7ab48d2 2275 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2276 if (!domain)
2277 return -ENOMEM;
2278
19943b0e
DW		 2279 /* For _hardware_ passthrough, don't bother. But for software
2280 passthrough, we do it anyway -- it may indicate a memory
2281 range which is reserved in E820, so which didn't get set
2282 up to start with in si_domain */
2283 if (domain == si_domain && hw_pass_through) {
2284 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2285 pci_name(pdev), start, end);
2286 return 0;
2287 }
2288
2289 printk(KERN_INFO
2290 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2291 pci_name(pdev), start, end);
2ff729f5 2292
5595b528
DW		 2293 if (end < start) {
2294 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2295 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2296 dmi_get_system_info(DMI_BIOS_VENDOR),
2297 dmi_get_system_info(DMI_BIOS_VERSION),
2298 dmi_get_system_info(DMI_PRODUCT_VERSION));
2299 ret = -EIO;
2300 goto error;
2301 }
2302
2ff729f5
DW		 2303 if (end >> agaw_to_width(domain->agaw)) {
2304 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2305 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2306 agaw_to_width(domain->agaw),
2307 dmi_get_system_info(DMI_BIOS_VENDOR),
2308 dmi_get_system_info(DMI_BIOS_VERSION),
2309 dmi_get_system_info(DMI_PRODUCT_VERSION));
2310 ret = -EIO;
2311 goto error;
2312 }
19943b0e 2313
b213203e 2314 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2315 if (ret)
2316 goto error;
2317
2318 /* context entry init */
4ed0d3e6 2319 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2320 if (ret)
2321 goto error;
2322
2323 return 0;
2324
2325 error:
ba395927
KA		 2326 domain_exit(domain);
2327 return ret;
ba395927
KA		 2328}
2329
2330static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2331 struct pci_dev *pdev)
2332{
358dd8ac 2333 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2334 return 0;
2335 return iommu_prepare_identity_map(pdev, rmrr->base_address,
70e535d1 2336 rmrr->end_address);
ba395927
KA		 2337}
2338
d3f13810 2339#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2340static inline void iommu_prepare_isa(void)
2341{
2342 struct pci_dev *pdev;
2343 int ret;
2344
2345 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2346 if (!pdev)
2347 return;
2348
c7ab48d2 2349 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
70e535d1 2350 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2351
2352 if (ret)
c7ab48d2
DW		 2353 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2354 "floppy might not work\n");
49a0429e
KA		 2355
2356}
2357#else
2358static inline void iommu_prepare_isa(void)
2359{
2360 return;
2361}
d3f13810 2362#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2363
2c2e2c38 2364static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2365
071e1374 2366static int __init si_domain_init(int hw)
2c2e2c38
FY		 2367{
2368 struct dmar_drhd_unit *drhd;
2369 struct intel_iommu *iommu;
c7ab48d2 2370 int nid, ret = 0;
2c2e2c38 2371
92d03cc8 2372 si_domain = alloc_domain(false);
2c2e2c38
FY		 2373 if (!si_domain)
2374 return -EFAULT;
2375
92d03cc8
JL		 2376 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2377
2c2e2c38
FY		 2378 for_each_active_iommu(iommu, drhd) {
2379 ret = iommu_attach_domain(si_domain, iommu);
2380 if (ret) {
2381 domain_exit(si_domain);
2382 return -EFAULT;
2383 }
2384 }
2385
2386 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2387 domain_exit(si_domain);
2388 return -EFAULT;
2389 }
2390
9544c003
JL		 2391 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2392 si_domain->id);
2c2e2c38 2393
19943b0e
DW		 2394 if (hw)
2395 return 0;
2396
c7ab48d2 2397 for_each_online_node(nid) {
5dfe8660
TH		 2398 unsigned long start_pfn, end_pfn;
2399 int i;
2400
2401 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2402 ret = iommu_domain_identity_map(si_domain,
2403 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2404 if (ret)
2405 return ret;
2406 }
c7ab48d2
DW		 2407 }
2408
2c2e2c38
FY		 2409 return 0;
2410}
2411
2c2e2c38
FY		 2412static int identity_mapping(struct pci_dev *pdev)
2413{
2414 struct device_domain_info *info;
2415
2416 if (likely(!iommu_identity_mapping))
2417 return 0;
2418
cb452a40
MT		 2419 info = pdev->dev.archdata.iommu;
2420 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2421 return (info->domain == si_domain);
2c2e2c38 2422
2c2e2c38
FY		 2423 return 0;
2424}
2425
2426static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2427 struct pci_dev *pdev,
2428 int translation)
2c2e2c38
FY		 2429{
2430 struct device_domain_info *info;
2431 unsigned long flags;
5fe60f4e 2432 int ret;
2c2e2c38
FY		 2433
2434 info = alloc_devinfo_mem();
2435 if (!info)
2436 return -ENOMEM;
2437
2438 info->segment = pci_domain_nr(pdev->bus);
2439 info->bus = pdev->bus->number;
2440 info->devfn = pdev->devfn;
2441 info->dev = pdev;
2442 info->domain = domain;
2443
2444 spin_lock_irqsave(&device_domain_lock, flags);
2445 list_add(&info->link, &domain->devices);
2446 list_add(&info->global, &device_domain_list);
2447 pdev->dev.archdata.iommu = info;
2448 spin_unlock_irqrestore(&device_domain_lock, flags);
2449
e2ad23d0
DW		 2450 ret = domain_context_mapping(domain, pdev, translation);
2451 if (ret) {
2452 spin_lock_irqsave(&device_domain_lock, flags);
109b9b04 2453 unlink_domain_info(info);
e2ad23d0
DW		 2454 spin_unlock_irqrestore(&device_domain_lock, flags);
2455 free_devinfo_mem(info);
2456 return ret;
2457 }
2458
2c2e2c38
FY		 2459 return 0;
2460}
2461
ea2447f7
TM		 2462static bool device_has_rmrr(struct pci_dev *dev)
2463{
2464 struct dmar_rmrr_unit *rmrr;
b683b230 2465 struct pci_dev *tmp;
ea2447f7
TM		 2466 int i;
2467
0e242612 2468 rcu_read_lock();
ea2447f7 2469 for_each_rmrr_units(rmrr) {
b683b230
JL		 2470 /*
2471 * Return TRUE if this RMRR contains the device that
2472 * is passed in.
2473 */
2474 for_each_active_dev_scope(rmrr->devices,
2475 rmrr->devices_cnt, i, tmp)
2476 if (tmp == dev) {
0e242612 2477 rcu_read_unlock();
ea2447f7 2478 return true;
b683b230 2479 }
ea2447f7 2480 }
0e242612 2481 rcu_read_unlock();
ea2447f7
TM		 2482 return false;
2483}
2484
6941af28
DW		 2485static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2486{
ea2447f7
TM		 2487
2488 /*
2489 * We want to prevent any device associated with an RMRR from
2490 * getting placed into the SI Domain. This is done because
2491 * problems exist when devices are moved in and out of domains
2492 * and their respective RMRR info is lost. We exempt USB devices
2493 * from this process due to their usage of RMRRs that are known
2494 * to not be needed after BIOS hand-off to OS.
2495 */
2496 if (device_has_rmrr(pdev) &&
2497 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2498 return 0;
2499
e0fc7e0b
DW		 2500 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2501 return 1;
2502
2503 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2504 return 1;
2505
2506 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2507 return 0;
6941af28 2508
3dfc813d
DW		 2509 /*
2510 * We want to start off with all devices in the 1:1 domain, and
2511 * take them out later if we find they can't access all of memory.
2512 *
2513 * However, we can't do this for PCI devices behind bridges,
2514 * because all PCI devices behind the same bridge will end up
2515 * with the same source-id on their transactions.
2516 *
2517 * Practically speaking, we can't change things around for these
2518 * devices at run-time, because we can't be sure there'll be no
2519 * DMA transactions in flight for any of their siblings.
2520 *
2521 * So PCI devices (unless they're on the root bus) as well as
2522 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2523 * the 1:1 domain, just in _case_ one of their siblings turns out
2524 * not to be able to map all of memory.
2525 */
5f4d91a1 2526 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2527 if (!pci_is_root_bus(pdev->bus))
2528 return 0;
2529 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2530 return 0;
62f87c0e 2531 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d
DW		 2532 return 0;
2533
2534 /*
2535 * At boot time, we don't yet know if devices will be 64-bit capable.
2536 * Assume that they will -- if they turn out not to be, then we can
2537 * take them out of the 1:1 domain later.
2538 */
8fcc5372
CW		 2539 if (!startup) {
2540 /*
2541 * If the device's dma_mask is less than the system's memory
2542 * size then this is not a candidate for identity mapping.
2543 */
2544 u64 dma_mask = pdev->dma_mask;
2545
2546 if (pdev->dev.coherent_dma_mask &&
2547 pdev->dev.coherent_dma_mask < dma_mask)
2548 dma_mask = pdev->dev.coherent_dma_mask;
2549
2550 return dma_mask >= dma_get_required_mask(&pdev->dev);
2551 }
6941af28
DW		 2552
2553 return 1;
2554}
2555
071e1374 2556static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2557{
2c2e2c38
FY		 2558 struct pci_dev *pdev = NULL;
2559 int ret;
2560
19943b0e 2561 ret = si_domain_init(hw);
2c2e2c38
FY		 2562 if (ret)
2563 return -EFAULT;
2564
2c2e2c38 2565 for_each_pci_dev(pdev) {
6941af28 2566 if (iommu_should_identity_map(pdev, 1)) {
5fe60f4e 2567 ret = domain_add_dev_info(si_domain, pdev,
eae460b6
MT		 2568 hw ? CONTEXT_TT_PASS_THROUGH :
2569 CONTEXT_TT_MULTI_LEVEL);
2570 if (ret) {
2571 /* device not associated with an iommu */
2572 if (ret == -ENODEV)
2573 continue;
62edf5dc 2574 return ret;
eae460b6
MT		 2575 }
2576 pr_info("IOMMU: %s identity mapping for device %s\n",
2577 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2578 }
2c2e2c38
FY		 2579 }
2580
2581 return 0;
2582}
2583
b779260b 2584static int __init init_dmars(void)
ba395927
KA		 2585{
2586 struct dmar_drhd_unit *drhd;
2587 struct dmar_rmrr_unit *rmrr;
2588 struct pci_dev *pdev;
2589 struct intel_iommu *iommu;
9d783ba0 2590 int i, ret;
2c2e2c38 2591
ba395927
KA		 2592 /*
2593 * for each drhd
2594 * allocate root
2595 * initialize and program root entry to not present
2596 * endfor
2597 */
2598 for_each_drhd_unit(drhd) {
5e0d2a6f 2599 /*
2600 * lock not needed as this is only incremented in the single
2601 * threaded kernel __init code path all other access are read
2602 * only
2603 */
1b198bb0
MT		 2604 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2605 g_num_of_iommus++;
2606 continue;
2607 }
2608 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2609 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2610 }
2611
d9630fe9
WH		 2612 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2613 GFP_KERNEL);
2614 if (!g_iommus) {
2615 printk(KERN_ERR "Allocating global iommu array failed\n");
2616 ret = -ENOMEM;
2617 goto error;
2618 }
2619
80b20dd8 2620 deferred_flush = kzalloc(g_num_of_iommus *
2621 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2622 if (!deferred_flush) {
5e0d2a6f 2623 ret = -ENOMEM;
989d51fc 2624 goto free_g_iommus;
5e0d2a6f 2625 }
2626
7c919779 2627 for_each_active_iommu(iommu, drhd) {
d9630fe9 2628 g_iommus[iommu->seq_id] = iommu;
ba395927 2629
e61d98d8
SS		 2630 ret = iommu_init_domains(iommu);
2631 if (ret)
989d51fc 2632 goto free_iommu;
e61d98d8 2633
ba395927
KA		 2634 /*
2635 * TBD:
2636 * we could share the same root & context tables
25985edc 2637 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2638 */
2639 ret = iommu_alloc_root_entry(iommu);
2640 if (ret) {
2641 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
989d51fc 2642 goto free_iommu;
ba395927 2643 }
4ed0d3e6 2644 if (!ecap_pass_through(iommu->ecap))
19943b0e 2645 hw_pass_through = 0;
ba395927
KA		 2646 }
2647
1531a6a6
SS		 2648 /*
2649 * Start from the sane iommu hardware state.
2650 */
7c919779 2651 for_each_active_iommu(iommu, drhd) {
1531a6a6
SS		 2652 /*
2653 * If the queued invalidation is already initialized by us
2654 * (for example, while enabling interrupt-remapping) then
2655 * we got the things already rolling from a sane state.
2656 */
2657 if (iommu->qi)
2658 continue;
2659
2660 /*
2661 * Clear any previous faults.
2662 */
2663 dmar_fault(-1, iommu);
2664 /*
2665 * Disable queued invalidation if supported and already enabled
2666 * before OS handover.
2667 */
2668 dmar_disable_qi(iommu);
2669 }
2670
7c919779 2671 for_each_active_iommu(iommu, drhd) {
a77b67d4
YS		 2672 if (dmar_enable_qi(iommu)) {
2673 /*
2674 * Queued Invalidate not enabled, use Register Based
2675 * Invalidate
2676 */
2677 iommu->flush.flush_context = __iommu_flush_context;
2678 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2679 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2680 "invalidation\n",
680a7524 2681 iommu->seq_id,
b4e0f9eb 2682 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2683 } else {
2684 iommu->flush.flush_context = qi_flush_context;
2685 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2686 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2687 "invalidation\n",
680a7524 2688 iommu->seq_id,
b4e0f9eb 2689 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2690 }
2691 }
2692
19943b0e 2693 if (iommu_pass_through)
e0fc7e0b
DW		 2694 iommu_identity_mapping |= IDENTMAP_ALL;
2695
d3f13810 2696#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2697 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2698#endif
e0fc7e0b
DW		 2699
2700 check_tylersburg_isoch();
2701
ba395927 2702 /*
19943b0e
DW		 2703 * If pass through is not set or not enabled, setup context entries for
2704 * identity mappings for rmrr, gfx, and isa and may fall back to static
2705 * identity mapping if iommu_identity_mapping is set.
ba395927 2706 */
19943b0e
DW		 2707 if (iommu_identity_mapping) {
2708 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2709 if (ret) {
19943b0e 2710 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
989d51fc 2711 goto free_iommu;
ba395927
KA		 2712 }
2713 }
ba395927 2714 /*
19943b0e
DW		 2715 * For each rmrr
2716 * for each dev attached to rmrr
2717 * do
2718 * locate drhd for dev, alloc domain for dev
2719 * allocate free domain
2720 * allocate page table entries for rmrr
2721 * if context not allocated for bus
2722 * allocate and init context
2723 * set present in root table for this bus
2724 * init context with domain, translation etc
2725 * endfor
2726 * endfor
ba395927 2727 */
19943b0e
DW		 2728 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2729 for_each_rmrr_units(rmrr) {
b683b230
JL		 2730 /* some BIOS lists non-exist devices in DMAR table. */
2731 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2732 i, pdev) {
19943b0e
DW		 2733 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2734 if (ret)
2735 printk(KERN_ERR
2736 "IOMMU: mapping reserved region failed\n");
ba395927 2737 }
4ed0d3e6 2738 }
49a0429e 2739
19943b0e
DW		 2740 iommu_prepare_isa();
2741
ba395927
KA		 2742 /*
2743 * for each drhd
2744 * enable fault log
2745 * global invalidate context cache
2746 * global invalidate iotlb
2747 * enable translation
2748 */
7c919779 2749 for_each_iommu(iommu, drhd) {
51a63e67
JC		 2750 if (drhd->ignored) {
2751 /*
2752 * we always have to disable PMRs or DMA may fail on
2753 * this device
2754 */
2755 if (force_on)
7c919779 2756 iommu_disable_protect_mem_regions(iommu);
ba395927 2757 continue;
51a63e67 2758 }
ba395927
KA		 2759
2760 iommu_flush_write_buffer(iommu);
2761
3460a6d9
KA		 2762 ret = dmar_set_interrupt(iommu);
2763 if (ret)
989d51fc 2764 goto free_iommu;
3460a6d9 2765
ba395927
KA		 2766 iommu_set_root_entry(iommu);
2767
4c25a2c1 2768 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2769 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2770
ba395927
KA		 2771 ret = iommu_enable_translation(iommu);
2772 if (ret)
989d51fc 2773 goto free_iommu;
b94996c9
DW		 2774
2775 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2776 }
2777
2778 return 0;
989d51fc
JL		 2779
2780free_iommu:
7c919779 2781 for_each_active_iommu(iommu, drhd)
a868e6b7 2782 free_dmar_iommu(iommu);
9bdc531e 2783 kfree(deferred_flush);
989d51fc 2784free_g_iommus:
d9630fe9 2785 kfree(g_iommus);
989d51fc 2786error:
ba395927
KA		 2787 return ret;
2788}
2789
5a5e02a6 2790/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2791static struct iova *intel_alloc_iova(struct device *dev,
2792 struct dmar_domain *domain,
2793 unsigned long nrpages, uint64_t dma_mask)
ba395927 2794{
ba395927 2795 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2796 struct iova *iova = NULL;
ba395927 2797
875764de
DW		 2798 /* Restrict dma_mask to the width that the iommu can handle */
2799 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2800
2801 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2802 /*
2803 * First try to allocate an io virtual address in
284901a9 2804 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2805 * from higher range
ba395927 2806 */
875764de
DW		 2807 iova = alloc_iova(&domain->iovad, nrpages,
2808 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2809 if (iova)
2810 return iova;
2811 }
2812 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2813 if (unlikely(!iova)) {
2814 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2815 nrpages, pci_name(pdev));
f76aec76
KA		 2816 return NULL;
2817 }
2818
2819 return iova;
2820}
2821
147202aa 2822static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2823{
2824 struct dmar_domain *domain;
2825 int ret;
2826
2827 domain = get_domain_for_dev(pdev,
2828 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2829 if (!domain) {
2830 printk(KERN_ERR
2831 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2832 return NULL;
ba395927
KA		 2833 }
2834
2835 /* make sure context mapping is ok */
5331fe6f 2836 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2837 ret = domain_context_mapping(domain, pdev,
2838 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2839 if (ret) {
2840 printk(KERN_ERR
2841 "Domain context map for %s failed",
2842 pci_name(pdev));
4fe05bbc 2843 return NULL;
f76aec76 2844 }
ba395927
KA		 2845 }
2846
f76aec76
KA		 2847 return domain;
2848}
2849
147202aa
DW		 2850static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2851{
2852 struct device_domain_info *info;
2853
2854 /* No lock here, assumes no domain exit in normal case */
2855 info = dev->dev.archdata.iommu;
2856 if (likely(info))
2857 return info->domain;
2858
2859 return __get_valid_domain_for_dev(dev);
2860}
2861
2c2e2c38
FY		 2862static int iommu_dummy(struct pci_dev *pdev)
2863{
2864 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2865}
2866
2867/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2868static int iommu_no_mapping(struct device *dev)
2c2e2c38 2869{
73676832 2870 struct pci_dev *pdev;
2c2e2c38
FY		 2871 int found;
2872
dbad0864 2873 if (unlikely(!dev_is_pci(dev)))
73676832
DW		 2874 return 1;
2875
2876 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2877 if (iommu_dummy(pdev))
2878 return 1;
2879
2c2e2c38 2880 if (!iommu_identity_mapping)
1e4c64c4 2881 return 0;
2c2e2c38
FY		 2882
2883 found = identity_mapping(pdev);
2884 if (found) {
6941af28 2885 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2886 return 1;
2887 else {
2888 /*
2889 * 32 bit DMA is removed from si_domain and fall back
2890 * to non-identity mapping.
2891 */
2892 domain_remove_one_dev_info(si_domain, pdev);
2893 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2894 pci_name(pdev));
2895 return 0;
2896 }
2897 } else {
2898 /*
2899 * In case of a detached 64 bit DMA device from vm, the device
2900 * is put into si_domain for identity mapping.
2901 */
6941af28 2902 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2903 int ret;
5fe60f4e
DW		 2904 ret = domain_add_dev_info(si_domain, pdev,
2905 hw_pass_through ?
2906 CONTEXT_TT_PASS_THROUGH :
2907 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2908 if (!ret) {
2909 printk(KERN_INFO "64bit %s uses identity mapping\n",
2910 pci_name(pdev));
2911 return 1;
2912 }
2913 }
2914 }
2915
1e4c64c4 2916 return 0;
2c2e2c38
FY		 2917}
2918
bb9e6d65
FT		 2919static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2920 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2921{
2922 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2923 struct dmar_domain *domain;
5b6985ce 2924 phys_addr_t start_paddr;
f76aec76
KA		 2925 struct iova *iova;
2926 int prot = 0;
6865f0d1 2927 int ret;
8c11e798 2928 struct intel_iommu *iommu;
33041ec0 2929 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2930
2931 BUG_ON(dir == DMA_NONE);
2c2e2c38 2932
73676832 2933 if (iommu_no_mapping(hwdev))
6865f0d1 2934 return paddr;
f76aec76
KA		 2935
2936 domain = get_valid_domain_for_dev(pdev);
2937 if (!domain)
2938 return 0;
2939
8c11e798 2940 iommu = domain_get_iommu(domain);
88cb6a74 2941 size = aligned_nrpages(paddr, size);
f76aec76 2942
c681d0ba 2943 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 2944 if (!iova)
2945 goto error;
2946
ba395927
KA		 2947 /*
2948 * Check if DMAR supports zero-length reads on write only
2949 * mappings..
2950 */
2951 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2952 !cap_zlr(iommu->cap))
ba395927
KA		 2953 prot |= DMA_PTE_READ;
2954 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2955 prot |= DMA_PTE_WRITE;
2956 /*
6865f0d1 2957 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2958 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2959 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2960 * is not a big problem
2961 */
0ab36de2 2962 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2963 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2964 if (ret)
2965 goto error;
2966
1f0ef2aa
DW		 2967 /* it's a non-present to present mapping. Only flush if caching mode */
2968 if (cap_caching_mode(iommu->cap))
ea8ea460 2969 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 2970 else
8c11e798 2971 iommu_flush_write_buffer(iommu);
f76aec76 2972
03d6a246
DW		 2973 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2974 start_paddr += paddr & ~PAGE_MASK;
2975 return start_paddr;
ba395927 2976
ba395927 2977error:
f76aec76
KA		 2978 if (iova)
2979 __free_iova(&domain->iovad, iova);
4cf2e75d 2980 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2981 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2982 return 0;
2983}
2984
ffbbef5c
FT		 2985static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2986 unsigned long offset, size_t size,
2987 enum dma_data_direction dir,
2988 struct dma_attrs *attrs)
bb9e6d65 2989{
ffbbef5c
FT		 2990 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2991 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2992}
2993
5e0d2a6f 2994static void flush_unmaps(void)
2995{
80b20dd8 2996 int i, j;
5e0d2a6f 2997
5e0d2a6f 2998 timer_on = 0;
2999
3000 /* just flush them all */
3001 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3002 struct intel_iommu *iommu = g_iommus[i];
3003 if (!iommu)
3004 continue;
c42d9f32 3005
9dd2fe89
YZ		 3006 if (!deferred_flush[i].next)
3007 continue;
3008
78d5f0f5
NA		 3009 /* In caching mode, global flushes turn emulation expensive */
3010 if (!cap_caching_mode(iommu->cap))
3011 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3012 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3013 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3014 unsigned long mask;
3015 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3016 struct dmar_domain *domain = deferred_flush[i].domain[j];
3017
3018 /* On real hardware multiple invalidations are expensive */
3019 if (cap_caching_mode(iommu->cap))
3020 iommu_flush_iotlb_psi(iommu, domain->id,
ea8ea460
DW		 3021 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1,
3022 !deferred_flush[i].freelist[j], 0);
78d5f0f5
NA		 3023 else {
3024 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
3025 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3026 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3027 }
93a23a72 3028 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3029 if (deferred_flush[i].freelist[j])
3030 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3031 }
9dd2fe89 3032 deferred_flush[i].next = 0;
5e0d2a6f 3033 }
3034
5e0d2a6f 3035 list_size = 0;
5e0d2a6f 3036}
3037
3038static void flush_unmaps_timeout(unsigned long data)
3039{
80b20dd8 3040 unsigned long flags;
3041
3042 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3043 flush_unmaps();
80b20dd8 3044 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3045}
3046
ea8ea460 3047static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3048{
3049 unsigned long flags;
80b20dd8 3050 int next, iommu_id;
8c11e798 3051 struct intel_iommu *iommu;
5e0d2a6f 3052
3053 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3054 if (list_size == HIGH_WATER_MARK)
3055 flush_unmaps();
3056
8c11e798
WH		 3057 iommu = domain_get_iommu(dom);
3058 iommu_id = iommu->seq_id;
c42d9f32 3059
80b20dd8 3060 next = deferred_flush[iommu_id].next;
3061 deferred_flush[iommu_id].domain[next] = dom;
3062 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3063 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3064 deferred_flush[iommu_id].next++;
5e0d2a6f 3065
3066 if (!timer_on) {
3067 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3068 timer_on = 1;
3069 }
3070 list_size++;
3071 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3072}
3073
ffbbef5c
FT		 3074static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3075 size_t size, enum dma_data_direction dir,
3076 struct dma_attrs *attrs)
ba395927 3077{
ba395927 3078 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 3079 struct dmar_domain *domain;
d794dc9b 3080 unsigned long start_pfn, last_pfn;
ba395927 3081 struct iova *iova;
8c11e798 3082 struct intel_iommu *iommu;
ea8ea460 3083 struct page *freelist;
ba395927 3084
73676832 3085 if (iommu_no_mapping(dev))
f76aec76 3086 return;
2c2e2c38 3087
ba395927
KA		 3088 domain = find_domain(pdev);
3089 BUG_ON(!domain);
3090
8c11e798
WH		 3091 iommu = domain_get_iommu(domain);
3092
ba395927 3093 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3094 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3095 (unsigned long long)dev_addr))
ba395927 3096 return;
ba395927 3097
d794dc9b
DW		 3098 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3099 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3100
d794dc9b
DW		 3101 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3102 pci_name(pdev), start_pfn, last_pfn);
ba395927 3103
ea8ea460 3104 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3105
5e0d2a6f 3106 if (intel_iommu_strict) {
03d6a246 3107 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3108 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3109 /* free iova */
3110 __free_iova(&domain->iovad, iova);
ea8ea460 3111 dma_free_pagelist(freelist);
5e0d2a6f 3112 } else {
ea8ea460 3113 add_unmap(domain, iova, freelist);
5e0d2a6f 3114 /*
3115 * queue up the release of the unmap to save the 1/6th of the
3116 * cpu used up by the iotlb flush operation...
3117 */
5e0d2a6f 3118 }
ba395927
KA		 3119}
3120
d7ab5c46 3121static void *intel_alloc_coherent(struct device *hwdev, size_t size,
baa676fc
AP		 3122 dma_addr_t *dma_handle, gfp_t flags,
3123 struct dma_attrs *attrs)
ba395927
KA		 3124{
3125 void *vaddr;
3126 int order;
3127
5b6985ce 3128 size = PAGE_ALIGN(size);
ba395927 3129 order = get_order(size);
e8bb910d
AW		 3130
3131 if (!iommu_no_mapping(hwdev))
3132 flags &= ~(GFP_DMA | GFP_DMA32);
3133 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
3134 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
3135 flags |= GFP_DMA;
3136 else
3137 flags |= GFP_DMA32;
3138 }
ba395927
KA		 3139
3140 vaddr = (void *)__get_free_pages(flags, order);
3141 if (!vaddr)
3142 return NULL;
3143 memset(vaddr, 0, size);
3144
bb9e6d65
FT		 3145 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
3146 DMA_BIDIRECTIONAL,
3147 hwdev->coherent_dma_mask);
ba395927
KA		 3148 if (*dma_handle)
3149 return vaddr;
3150 free_pages((unsigned long)vaddr, order);
3151 return NULL;
3152}
3153
d7ab5c46 3154static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
baa676fc 3155 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3156{
3157 int order;
3158
5b6985ce 3159 size = PAGE_ALIGN(size);
ba395927
KA		 3160 order = get_order(size);
3161
0db9b7ae 3162 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 3163 free_pages((unsigned long)vaddr, order);
3164}
3165
d7ab5c46
FT		 3166static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
3167 int nelems, enum dma_data_direction dir,
3168 struct dma_attrs *attrs)
ba395927 3169{
ba395927
KA		 3170 struct pci_dev *pdev = to_pci_dev(hwdev);
3171 struct dmar_domain *domain;
d794dc9b 3172 unsigned long start_pfn, last_pfn;
f76aec76 3173 struct iova *iova;
8c11e798 3174 struct intel_iommu *iommu;
ea8ea460 3175 struct page *freelist;
ba395927 3176
73676832 3177 if (iommu_no_mapping(hwdev))
ba395927
KA		 3178 return;
3179
3180 domain = find_domain(pdev);
8c11e798
WH		 3181 BUG_ON(!domain);
3182
3183 iommu = domain_get_iommu(domain);
ba395927 3184
c03ab37c 3185 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 3186 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3187 (unsigned long long)sglist[0].dma_address))
f76aec76 3188 return;
f76aec76 3189
d794dc9b
DW		 3190 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3191 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76 3192
ea8ea460 3193 freelist = domain_unmap(domain, start_pfn, last_pfn);
f76aec76 3194
acea0018
DW		 3195 if (intel_iommu_strict) {
3196 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3197 last_pfn - start_pfn + 1, !freelist, 0);
acea0018
DW		 3198 /* free iova */
3199 __free_iova(&domain->iovad, iova);
ea8ea460 3200 dma_free_pagelist(freelist);
acea0018 3201 } else {
ea8ea460 3202 add_unmap(domain, iova, freelist);
acea0018
DW		 3203 /*
3204 * queue up the release of the unmap to save the 1/6th of the
3205 * cpu used up by the iotlb flush operation...
3206 */
3207 }
ba395927
KA		 3208}
3209
ba395927 3210static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3211 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3212{
3213 int i;
c03ab37c 3214 struct scatterlist *sg;
ba395927 3215
c03ab37c 3216 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3217 BUG_ON(!sg_page(sg));
4cf2e75d 3218 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3219 sg->dma_length = sg->length;
ba395927
KA		 3220 }
3221 return nelems;
3222}
3223
d7ab5c46
FT		 3224static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3225 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3226{
ba395927 3227 int i;
ba395927
KA		 3228 struct pci_dev *pdev = to_pci_dev(hwdev);
3229 struct dmar_domain *domain;
f76aec76
KA		 3230 size_t size = 0;
3231 int prot = 0;
f76aec76
KA		 3232 struct iova *iova = NULL;
3233 int ret;
c03ab37c 3234 struct scatterlist *sg;
b536d24d 3235 unsigned long start_vpfn;
8c11e798 3236 struct intel_iommu *iommu;
ba395927
KA		 3237
3238 BUG_ON(dir == DMA_NONE);
73676832 3239 if (iommu_no_mapping(hwdev))
c03ab37c 3240 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 3241
f76aec76
KA		 3242 domain = get_valid_domain_for_dev(pdev);
3243 if (!domain)
3244 return 0;
3245
8c11e798
WH		 3246 iommu = domain_get_iommu(domain);
3247
b536d24d 3248 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3249 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3250
5a5e02a6
DW		 3251 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3252 pdev->dma_mask);
f76aec76 3253 if (!iova) {
c03ab37c 3254 sglist->dma_length = 0;
f76aec76
KA		 3255 return 0;
3256 }
3257
3258 /*
3259 * Check if DMAR supports zero-length reads on write only
3260 * mappings..
3261 */
3262 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3263 !cap_zlr(iommu->cap))
f76aec76
KA		 3264 prot |= DMA_PTE_READ;
3265 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3266 prot |= DMA_PTE_WRITE;
3267
b536d24d 3268 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3269
f532959b 3270 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3271 if (unlikely(ret)) {
3272 /* clear the page */
3273 dma_pte_clear_range(domain, start_vpfn,
3274 start_vpfn + size - 1);
3275 /* free page tables */
3276 dma_pte_free_pagetable(domain, start_vpfn,
3277 start_vpfn + size - 1);
3278 /* free iova */
3279 __free_iova(&domain->iovad, iova);
3280 return 0;
ba395927
KA		 3281 }
3282
1f0ef2aa
DW		 3283 /* it's a non-present to present mapping. Only flush if caching mode */
3284 if (cap_caching_mode(iommu->cap))
ea8ea460 3285 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3286 else
8c11e798 3287 iommu_flush_write_buffer(iommu);
1f0ef2aa 3288
ba395927
KA		 3289 return nelems;
3290}
3291
dfb805e8
FT		 3292static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3293{
3294 return !dma_addr;
3295}
3296
160c1d8e 3297struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3298 .alloc = intel_alloc_coherent,
3299 .free = intel_free_coherent,
ba395927
KA		 3300 .map_sg = intel_map_sg,
3301 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3302 .map_page = intel_map_page,
3303 .unmap_page = intel_unmap_page,
dfb805e8 3304 .mapping_error = intel_mapping_error,
ba395927
KA		 3305};
3306
3307static inline int iommu_domain_cache_init(void)
3308{
3309 int ret = 0;
3310
3311 iommu_domain_cache = kmem_cache_create("iommu_domain",
3312 sizeof(struct dmar_domain),
3313 0,
3314 SLAB_HWCACHE_ALIGN,
3315
3316 NULL);
3317 if (!iommu_domain_cache) {
3318 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3319 ret = -ENOMEM;
3320 }
3321
3322 return ret;
3323}
3324
3325static inline int iommu_devinfo_cache_init(void)
3326{
3327 int ret = 0;
3328
3329 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3330 sizeof(struct device_domain_info),
3331 0,
3332 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3333 NULL);
3334 if (!iommu_devinfo_cache) {
3335 printk(KERN_ERR "Couldn't create devinfo cache\n");
3336 ret = -ENOMEM;
3337 }
3338
3339 return ret;
3340}
3341
3342static inline int iommu_iova_cache_init(void)
3343{
3344 int ret = 0;
3345
3346 iommu_iova_cache = kmem_cache_create("iommu_iova",
3347 sizeof(struct iova),
3348 0,
3349 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3350 NULL);
3351 if (!iommu_iova_cache) {
3352 printk(KERN_ERR "Couldn't create iova cache\n");
3353 ret = -ENOMEM;
3354 }
3355
3356 return ret;
3357}
3358
3359static int __init iommu_init_mempool(void)
3360{
3361 int ret;
3362 ret = iommu_iova_cache_init();
3363 if (ret)
3364 return ret;
3365
3366 ret = iommu_domain_cache_init();
3367 if (ret)
3368 goto domain_error;
3369
3370 ret = iommu_devinfo_cache_init();
3371 if (!ret)
3372 return ret;
3373
3374 kmem_cache_destroy(iommu_domain_cache);
3375domain_error:
3376 kmem_cache_destroy(iommu_iova_cache);
3377
3378 return -ENOMEM;
3379}
3380
3381static void __init iommu_exit_mempool(void)
3382{
3383 kmem_cache_destroy(iommu_devinfo_cache);
3384 kmem_cache_destroy(iommu_domain_cache);
3385 kmem_cache_destroy(iommu_iova_cache);
3386
3387}
3388
556ab45f
DW		 3389static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3390{
3391 struct dmar_drhd_unit *drhd;
3392 u32 vtbar;
3393 int rc;
3394
3395 /* We know that this device on this chipset has its own IOMMU.
3396 * If we find it under a different IOMMU, then the BIOS is lying
3397 * to us. Hope that the IOMMU for this device is actually
3398 * disabled, and it needs no translation...
3399 */
3400 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3401 if (rc) {
3402 /* "can't" happen */
3403 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3404 return;
3405 }
3406 vtbar &= 0xffff0000;
3407
3408 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3409 drhd = dmar_find_matched_drhd_unit(pdev);
3410 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3411 TAINT_FIRMWARE_WORKAROUND,
3412 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3413 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3414}
3415DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3416
ba395927
KA		 3417static void __init init_no_remapping_devices(void)
3418{
3419 struct dmar_drhd_unit *drhd;
b683b230
JL		 3420 struct pci_dev *dev;
3421 int i;
ba395927
KA		 3422
3423 for_each_drhd_unit(drhd) {
3424 if (!drhd->include_all) {
b683b230
JL		 3425 for_each_active_dev_scope(drhd->devices,
3426 drhd->devices_cnt, i, dev)
3427 break;
ba395927
KA		 3428 /* ignore DMAR unit if no pci devices exist */
3429 if (i == drhd->devices_cnt)
3430 drhd->ignored = 1;
3431 }
3432 }
3433
7c919779 3434 for_each_active_drhd_unit(drhd) {
7c919779 3435 if (drhd->include_all)
ba395927
KA		 3436 continue;
3437
b683b230
JL		 3438 for_each_active_dev_scope(drhd->devices,
3439 drhd->devices_cnt, i, dev)
3440 if (!IS_GFX_DEVICE(dev))
ba395927 3441 break;
ba395927
KA		 3442 if (i < drhd->devices_cnt)
3443 continue;
3444
c0771df8
DW		 3445 /* This IOMMU has *only* gfx devices. Either bypass it or
3446 set the gfx_mapped flag, as appropriate */
3447 if (dmar_map_gfx) {
3448 intel_iommu_gfx_mapped = 1;
3449 } else {
3450 drhd->ignored = 1;
b683b230
JL		 3451 for_each_active_dev_scope(drhd->devices,
3452 drhd->devices_cnt, i, dev)
3453 dev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3454 }
3455 }
3456}
3457
f59c7b69
FY		 3458#ifdef CONFIG_SUSPEND
3459static int init_iommu_hw(void)
3460{
3461 struct dmar_drhd_unit *drhd;
3462 struct intel_iommu *iommu = NULL;
3463
3464 for_each_active_iommu(iommu, drhd)
3465 if (iommu->qi)
3466 dmar_reenable_qi(iommu);
3467
b779260b
JC		 3468 for_each_iommu(iommu, drhd) {
3469 if (drhd->ignored) {
3470 /*
3471 * we always have to disable PMRs or DMA may fail on
3472 * this device
3473 */
3474 if (force_on)
3475 iommu_disable_protect_mem_regions(iommu);
3476 continue;
3477 }
3478
f59c7b69
FY		 3479 iommu_flush_write_buffer(iommu);
3480
3481 iommu_set_root_entry(iommu);
3482
3483 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3484 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3485 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3486 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3487 if (iommu_enable_translation(iommu))
3488 return 1;
b94996c9 3489 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3490 }
3491
3492 return 0;
3493}
3494
3495static void iommu_flush_all(void)
3496{
3497 struct dmar_drhd_unit *drhd;
3498 struct intel_iommu *iommu;
3499
3500 for_each_active_iommu(iommu, drhd) {
3501 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3502 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3503 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3504 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3505 }
3506}
3507
134fac3f 3508static int iommu_suspend(void)
f59c7b69
FY		 3509{
3510 struct dmar_drhd_unit *drhd;
3511 struct intel_iommu *iommu = NULL;
3512 unsigned long flag;
3513
3514 for_each_active_iommu(iommu, drhd) {
3515 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3516 GFP_ATOMIC);
3517 if (!iommu->iommu_state)
3518 goto nomem;
3519 }
3520
3521 iommu_flush_all();
3522
3523 for_each_active_iommu(iommu, drhd) {
3524 iommu_disable_translation(iommu);
3525
1f5b3c3f 3526 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3527
3528 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3529 readl(iommu->reg + DMAR_FECTL_REG);
3530 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3531 readl(iommu->reg + DMAR_FEDATA_REG);
3532 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3533 readl(iommu->reg + DMAR_FEADDR_REG);
3534 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3535 readl(iommu->reg + DMAR_FEUADDR_REG);
3536
1f5b3c3f 3537 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3538 }
3539 return 0;
3540
3541nomem:
3542 for_each_active_iommu(iommu, drhd)
3543 kfree(iommu->iommu_state);
3544
3545 return -ENOMEM;
3546}
3547
134fac3f 3548static void iommu_resume(void)
f59c7b69
FY		 3549{
3550 struct dmar_drhd_unit *drhd;
3551 struct intel_iommu *iommu = NULL;
3552 unsigned long flag;
3553
3554 if (init_iommu_hw()) {
b779260b
JC		 3555 if (force_on)
3556 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3557 else
3558 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3559 return;
f59c7b69
FY		 3560 }
3561
3562 for_each_active_iommu(iommu, drhd) {
3563
1f5b3c3f 3564 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3565
3566 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3567 iommu->reg + DMAR_FECTL_REG);
3568 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3569 iommu->reg + DMAR_FEDATA_REG);
3570 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3571 iommu->reg + DMAR_FEADDR_REG);
3572 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3573 iommu->reg + DMAR_FEUADDR_REG);
3574
1f5b3c3f 3575 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3576 }
3577
3578 for_each_active_iommu(iommu, drhd)
3579 kfree(iommu->iommu_state);
f59c7b69
FY		 3580}
3581
134fac3f 3582static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3583 .resume = iommu_resume,
3584 .suspend = iommu_suspend,
3585};
3586
134fac3f 3587static void __init init_iommu_pm_ops(void)
f59c7b69 3588{
134fac3f 3589 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3590}
3591
3592#else
99592ba4 3593static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3594#endif /* CONFIG_PM */
3595
318fe7df
SS		 3596
3597int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3598{
3599 struct acpi_dmar_reserved_memory *rmrr;
3600 struct dmar_rmrr_unit *rmrru;
3601
3602 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3603 if (!rmrru)
3604 return -ENOMEM;
3605
3606 rmrru->hdr = header;
3607 rmrr = (struct acpi_dmar_reserved_memory *)header;
3608 rmrru->base_address = rmrr->base_address;
3609 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3610 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3611 ((void *)rmrr) + rmrr->header.length,
3612 &rmrru->devices_cnt);
3613 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3614 kfree(rmrru);
3615 return -ENOMEM;
3616 }
318fe7df 3617
2e455289 3618 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3619
2e455289 3620 return 0;
318fe7df
SS		 3621}
3622
318fe7df
SS		 3623int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3624{
3625 struct acpi_dmar_atsr *atsr;
3626 struct dmar_atsr_unit *atsru;
3627
3628 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3629 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3630 if (!atsru)
3631 return -ENOMEM;
3632
3633 atsru->hdr = hdr;
3634 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 3635 if (!atsru->include_all) {
3636 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3637 (void *)atsr + atsr->header.length,
3638 &atsru->devices_cnt);
3639 if (atsru->devices_cnt && atsru->devices == NULL) {
3640 kfree(atsru);
3641 return -ENOMEM;
3642 }
3643 }
318fe7df 3644
0e242612 3645 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 3646
3647 return 0;
3648}
3649
9bdc531e
JL		 3650static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3651{
3652 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3653 kfree(atsru);
3654}
3655
3656static void intel_iommu_free_dmars(void)
3657{
3658 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3659 struct dmar_atsr_unit *atsru, *atsr_n;
3660
3661 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3662 list_del(&rmrru->list);
3663 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3664 kfree(rmrru);
318fe7df
SS		 3665 }
3666
9bdc531e
JL		 3667 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3668 list_del(&atsru->list);
3669 intel_iommu_free_atsr(atsru);
3670 }
318fe7df
SS		 3671}
3672
3673int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3674{
b683b230 3675 int i, ret = 1;
318fe7df 3676 struct pci_bus *bus;
b683b230 3677 struct pci_dev *bridge = NULL, *tmp;
318fe7df
SS		 3678 struct acpi_dmar_atsr *atsr;
3679 struct dmar_atsr_unit *atsru;
3680
3681 dev = pci_physfn(dev);
318fe7df 3682 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 3683 bridge = bus->self;
318fe7df 3684 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3685 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 3686 return 0;
b5f82ddf 3687 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 3688 break;
318fe7df 3689 }
b5f82ddf
JL		 3690 if (!bridge)
3691 return 0;
318fe7df 3692
0e242612 3693 rcu_read_lock();
b5f82ddf
JL		 3694 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3695 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3696 if (atsr->segment != pci_domain_nr(dev->bus))
3697 continue;
3698
b683b230
JL		 3699 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3700 if (tmp == bridge)
3701 goto out;
b5f82ddf
JL		 3702
3703 if (atsru->include_all)
b683b230 3704 goto out;
b5f82ddf 3705 }
b683b230
JL		 3706 ret = 0;
3707out:
0e242612 3708 rcu_read_unlock();
318fe7df 3709
b683b230 3710 return ret;
318fe7df
SS		 3711}
3712
59ce0515
JL		 3713int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3714{
3715 int ret = 0;
3716 struct dmar_rmrr_unit *rmrru;
3717 struct dmar_atsr_unit *atsru;
3718 struct acpi_dmar_atsr *atsr;
3719 struct acpi_dmar_reserved_memory *rmrr;
3720
3721 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3722 return 0;
3723
3724 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3725 rmrr = container_of(rmrru->hdr,
3726 struct acpi_dmar_reserved_memory, header);
3727 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3728 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3729 ((void *)rmrr) + rmrr->header.length,
3730 rmrr->segment, rmrru->devices,
3731 rmrru->devices_cnt);
3732 if (ret > 0)
3733 break;
3734 else if(ret < 0)
3735 return ret;
3736 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3737 if (dmar_remove_dev_scope(info, rmrr->segment,
3738 rmrru->devices, rmrru->devices_cnt))
3739 break;
3740 }
3741 }
3742
3743 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3744 if (atsru->include_all)
3745 continue;
3746
3747 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3748 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3749 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3750 (void *)atsr + atsr->header.length,
3751 atsr->segment, atsru->devices,
3752 atsru->devices_cnt);
3753 if (ret > 0)
3754 break;
3755 else if(ret < 0)
3756 return ret;
3757 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3758 if (dmar_remove_dev_scope(info, atsr->segment,
3759 atsru->devices, atsru->devices_cnt))
3760 break;
3761 }
3762 }
3763
3764 return 0;
3765}
3766
99dcaded
FY		 3767/*
3768 * Here we only respond to action of unbound device from driver.
3769 *
3770 * Added device is not attached to its DMAR domain here yet. That will happen
3771 * when mapping the device to iova.
3772 */
3773static int device_notifier(struct notifier_block *nb,
3774 unsigned long action, void *data)
3775{
3776 struct device *dev = data;
3777 struct pci_dev *pdev = to_pci_dev(dev);
3778 struct dmar_domain *domain;
3779
816997d0 3780 if (iommu_dummy(pdev))
44cd613c
DW		 3781 return 0;
3782
7e7dfab7
JL		 3783 if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
3784 action != BUS_NOTIFY_DEL_DEVICE)
3785 return 0;
3786
99dcaded
FY		 3787 domain = find_domain(pdev);
3788 if (!domain)
3789 return 0;
3790
3a5670e8 3791 down_read(&dmar_global_lock);
7e7dfab7
JL		 3792 domain_remove_one_dev_info(domain, pdev);
3793 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3794 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3795 list_empty(&domain->devices))
3796 domain_exit(domain);
3a5670e8 3797 up_read(&dmar_global_lock);
a97590e5 3798
99dcaded
FY		 3799 return 0;
3800}
3801
3802static struct notifier_block device_nb = {
3803 .notifier_call = device_notifier,
3804};
3805
75f05569
JL		 3806static int intel_iommu_memory_notifier(struct notifier_block *nb,
3807 unsigned long val, void *v)
3808{
3809 struct memory_notify *mhp = v;
3810 unsigned long long start, end;
3811 unsigned long start_vpfn, last_vpfn;
3812
3813 switch (val) {
3814 case MEM_GOING_ONLINE:
3815 start = mhp->start_pfn << PAGE_SHIFT;
3816 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
3817 if (iommu_domain_identity_map(si_domain, start, end)) {
3818 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3819 start, end);
3820 return NOTIFY_BAD;
3821 }
3822 break;
3823
3824 case MEM_OFFLINE:
3825 case MEM_CANCEL_ONLINE:
3826 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3827 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
3828 while (start_vpfn <= last_vpfn) {
3829 struct iova *iova;
3830 struct dmar_drhd_unit *drhd;
3831 struct intel_iommu *iommu;
ea8ea460 3832 struct page *freelist;
75f05569
JL		 3833
3834 iova = find_iova(&si_domain->iovad, start_vpfn);
3835 if (iova == NULL) {
3836 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3837 start_vpfn);
3838 break;
3839 }
3840
3841 iova = split_and_remove_iova(&si_domain->iovad, iova,
3842 start_vpfn, last_vpfn);
3843 if (iova == NULL) {
3844 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3845 start_vpfn, last_vpfn);
3846 return NOTIFY_BAD;
3847 }
3848
ea8ea460
DW		 3849 freelist = domain_unmap(si_domain, iova->pfn_lo,
3850 iova->pfn_hi);
3851
75f05569
JL		 3852 rcu_read_lock();
3853 for_each_active_iommu(iommu, drhd)
3854 iommu_flush_iotlb_psi(iommu, si_domain->id,
3855 iova->pfn_lo,
ea8ea460
DW		 3856 iova->pfn_hi - iova->pfn_lo + 1,
3857 !freelist, 0);
75f05569 3858 rcu_read_unlock();
ea8ea460 3859 dma_free_pagelist(freelist);
75f05569
JL		 3860
3861 start_vpfn = iova->pfn_hi + 1;
3862 free_iova_mem(iova);
3863 }
3864 break;
3865 }
3866
3867 return NOTIFY_OK;
3868}
3869
3870static struct notifier_block intel_iommu_memory_nb = {
3871 .notifier_call = intel_iommu_memory_notifier,
3872 .priority = 0
3873};
3874
ba395927
KA		 3875int __init intel_iommu_init(void)
3876{
9bdc531e 3877 int ret = -ENODEV;
3a93c841 3878 struct dmar_drhd_unit *drhd;
7c919779 3879 struct intel_iommu *iommu;
ba395927 3880
a59b50e9
JC		 3881 /* VT-d is required for a TXT/tboot launch, so enforce that */
3882 force_on = tboot_force_iommu();
3883
3a5670e8
JL		 3884 if (iommu_init_mempool()) {
3885 if (force_on)
3886 panic("tboot: Failed to initialize iommu memory\n");
3887 return -ENOMEM;
3888 }
3889
3890 down_write(&dmar_global_lock);
a59b50e9
JC		 3891 if (dmar_table_init()) {
3892 if (force_on)
3893 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 3894 goto out_free_dmar;
a59b50e9 3895 }
ba395927 3896
3a93c841
TI		 3897 /*
3898 * Disable translation if already enabled prior to OS handover.
3899 */
7c919779 3900 for_each_active_iommu(iommu, drhd)
3a93c841
TI		 3901 if (iommu->gcmd & DMA_GCMD_TE)
3902 iommu_disable_translation(iommu);
3a93c841 3903
c2c7286a 3904 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 3905 if (force_on)
3906 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 3907 goto out_free_dmar;
a59b50e9 3908 }
1886e8a9 3909
75f1cdf1 3910 if (no_iommu || dmar_disabled)
9bdc531e 3911 goto out_free_dmar;
2ae21010 3912
318fe7df
SS		 3913 if (list_empty(&dmar_rmrr_units))
3914 printk(KERN_INFO "DMAR: No RMRR found\n");
3915
3916 if (list_empty(&dmar_atsr_units))
3917 printk(KERN_INFO "DMAR: No ATSR found\n");
3918
51a63e67
JC		 3919 if (dmar_init_reserved_ranges()) {
3920 if (force_on)
3921 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 3922 goto out_free_reserved_range;
51a63e67 3923 }
ba395927
KA		 3924
3925 init_no_remapping_devices();
3926
b779260b 3927 ret = init_dmars();
ba395927 3928 if (ret) {
a59b50e9
JC		 3929 if (force_on)
3930 panic("tboot: Failed to initialize DMARs\n");
ba395927 3931 printk(KERN_ERR "IOMMU: dmar init failed\n");
9bdc531e 3932 goto out_free_reserved_range;
ba395927 3933 }
3a5670e8 3934 up_write(&dmar_global_lock);
ba395927
KA		 3935 printk(KERN_INFO
3936 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3937
5e0d2a6f 3938 init_timer(&unmap_timer);
75f1cdf1
FT		 3939#ifdef CONFIG_SWIOTLB
3940 swiotlb = 0;
3941#endif
19943b0e 3942 dma_ops = &intel_dma_ops;
4ed0d3e6 3943
134fac3f 3944 init_iommu_pm_ops();
a8bcbb0d 3945
4236d97d 3946 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 3947 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 3948 if (si_domain && !hw_pass_through)
3949 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 3950
8bc1f85c
ED		 3951 intel_iommu_enabled = 1;
3952
ba395927 3953 return 0;
9bdc531e
JL		 3954
3955out_free_reserved_range:
3956 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 3957out_free_dmar:
3958 intel_iommu_free_dmars();
3a5670e8
JL		 3959 up_write(&dmar_global_lock);
3960 iommu_exit_mempool();
9bdc531e 3961 return ret;
ba395927 3962}
e820482c 3963
3199aa6b
HW		 3964static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3965 struct pci_dev *pdev)
3966{
3967 struct pci_dev *tmp, *parent;
3968
3969 if (!iommu || !pdev)
3970 return;
3971
3972 /* dependent device detach */
3973 tmp = pci_find_upstream_pcie_bridge(pdev);
3974 /* Secondary interface's bus number and devfn 0 */
3975 if (tmp) {
3976 parent = pdev->bus->self;
3977 while (parent != tmp) {
3978 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3979 parent->devfn);
3199aa6b
HW		 3980 parent = parent->bus->self;
3981 }
45e829ea 3982 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3983 iommu_detach_dev(iommu,
3984 tmp->subordinate->number, 0);
3985 else /* this is a legacy PCI bridge */
276dbf99
DW		 3986 iommu_detach_dev(iommu, tmp->bus->number,
3987 tmp->devfn);
3199aa6b
HW		 3988 }
3989}
3990
2c2e2c38 3991static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3992 struct pci_dev *pdev)
3993{
bca2b916 3994 struct device_domain_info *info, *tmp;
c7151a8d
WH		 3995 struct intel_iommu *iommu;
3996 unsigned long flags;
3997 int found = 0;
c7151a8d 3998
276dbf99
DW		 3999 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
4000 pdev->devfn);
c7151a8d
WH		 4001 if (!iommu)
4002 return;
4003
4004 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4005 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
8519dc44
MH		 4006 if (info->segment == pci_domain_nr(pdev->bus) &&
4007 info->bus == pdev->bus->number &&
c7151a8d 4008 info->devfn == pdev->devfn) {
109b9b04 4009 unlink_domain_info(info);
c7151a8d
WH		 4010 spin_unlock_irqrestore(&device_domain_lock, flags);
4011
93a23a72 4012 iommu_disable_dev_iotlb(info);
c7151a8d 4013 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 4014 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 4015 free_devinfo_mem(info);
4016
4017 spin_lock_irqsave(&device_domain_lock, flags);
4018
4019 if (found)
4020 break;
4021 else
4022 continue;
4023 }
4024
4025 /* if there is no other devices under the same iommu
4026 * owned by this domain, clear this iommu in iommu_bmp
4027 * update iommu count and coherency
4028 */
276dbf99
DW		 4029 if (iommu == device_to_iommu(info->segment, info->bus,
4030 info->devfn))
c7151a8d
WH		 4031 found = 1;
4032 }
4033
3e7abe25
RD		 4034 spin_unlock_irqrestore(&device_domain_lock, flags);
4035
c7151a8d
WH		 4036 if (found == 0) {
4037 unsigned long tmp_flags;
4038 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
1b198bb0 4039 clear_bit(iommu->seq_id, domain->iommu_bmp);
c7151a8d 4040 domain->iommu_count--;
58c610bd 4041 domain_update_iommu_cap(domain);
c7151a8d 4042 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5 4043
9b4554b2
AW		 4044 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
4045 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
4046 spin_lock_irqsave(&iommu->lock, tmp_flags);
4047 clear_bit(domain->id, iommu->domain_ids);
4048 iommu->domains[domain->id] = NULL;
4049 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
4050 }
c7151a8d 4051 }
c7151a8d
WH		 4052}
4053
2c2e2c38 4054static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4055{
4056 int adjust_width;
4057
4058 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 4059 domain_reserve_special_ranges(domain);
4060
4061 /* calculate AGAW */
4062 domain->gaw = guest_width;
4063 adjust_width = guestwidth_to_adjustwidth(guest_width);
4064 domain->agaw = width_to_agaw(adjust_width);
4065
5e98c4b1 4066 domain->iommu_coherency = 0;
c5b15255 4067 domain->iommu_snooping = 0;
6dd9a7c7 4068 domain->iommu_superpage = 0;
fe40f1e0 4069 domain->max_addr = 0;
4c923d47 4070 domain->nid = -1;
5e98c4b1
WH		 4071
4072 /* always allocate the top pgd */
4c923d47 4073 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4074 if (!domain->pgd)
4075 return -ENOMEM;
4076 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4077 return 0;
4078}
4079
5d450806 4080static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 4081{
5d450806 4082 struct dmar_domain *dmar_domain;
38717946 4083
92d03cc8 4084 dmar_domain = alloc_domain(true);
5d450806 4085 if (!dmar_domain) {
38717946 4086 printk(KERN_ERR
5d450806
JR		 4087 "intel_iommu_domain_init: dmar_domain == NULL\n");
4088 return -ENOMEM;
38717946 4089 }
2c2e2c38 4090 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 4091 printk(KERN_ERR
5d450806 4092 "intel_iommu_domain_init() failed\n");
92d03cc8 4093 domain_exit(dmar_domain);
5d450806 4094 return -ENOMEM;
38717946 4095 }
8140a95d 4096 domain_update_iommu_cap(dmar_domain);
5d450806 4097 domain->priv = dmar_domain;
faa3d6f5 4098
8a0e715b
JR		 4099 domain->geometry.aperture_start = 0;
4100 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4101 domain->geometry.force_aperture = true;
4102
5d450806 4103 return 0;
38717946 4104}
38717946 4105
5d450806 4106static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 4107{
5d450806
JR		 4108 struct dmar_domain *dmar_domain = domain->priv;
4109
4110 domain->priv = NULL;
92d03cc8 4111 domain_exit(dmar_domain);
38717946 4112}
38717946 4113
4c5478c9
JR		 4114static int intel_iommu_attach_device(struct iommu_domain *domain,
4115 struct device *dev)
38717946 4116{
4c5478c9
JR		 4117 struct dmar_domain *dmar_domain = domain->priv;
4118 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 4119 struct intel_iommu *iommu;
4120 int addr_width;
faa3d6f5
WH		 4121
4122 /* normally pdev is not mapped */
4123 if (unlikely(domain_context_mapped(pdev))) {
4124 struct dmar_domain *old_domain;
4125
4126 old_domain = find_domain(pdev);
4127 if (old_domain) {
2c2e2c38
FY		 4128 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4129 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4130 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 4131 else
4132 domain_remove_dev_info(old_domain);
4133 }
4134 }
4135
276dbf99
DW		 4136 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
4137 pdev->devfn);
fe40f1e0
WH		 4138 if (!iommu)
4139 return -ENODEV;
4140
4141 /* check if this iommu agaw is sufficient for max mapped address */
4142 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4143 if (addr_width > cap_mgaw(iommu->cap))
4144 addr_width = cap_mgaw(iommu->cap);
4145
4146 if (dmar_domain->max_addr > (1LL << addr_width)) {
4147 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4148 "sufficient for the mapped address (%llx)\n",
a99c47a2 4149 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4150 return -EFAULT;
4151 }
a99c47a2
TL		 4152 dmar_domain->gaw = addr_width;
4153
4154 /*
4155 * Knock out extra levels of page tables if necessary
4156 */
4157 while (iommu->agaw < dmar_domain->agaw) {
4158 struct dma_pte *pte;
4159
4160 pte = dmar_domain->pgd;
4161 if (dma_pte_present(pte)) {
25cbff16
SY		 4162 dmar_domain->pgd = (struct dma_pte *)
4163 phys_to_virt(dma_pte_addr(pte));
7a661013 4164 free_pgtable_page(pte);
a99c47a2
TL		 4165 }
4166 dmar_domain->agaw--;
4167 }
fe40f1e0 4168
5fe60f4e 4169 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 4170}
38717946 4171
4c5478c9
JR		 4172static void intel_iommu_detach_device(struct iommu_domain *domain,
4173 struct device *dev)
38717946 4174{
4c5478c9
JR		 4175 struct dmar_domain *dmar_domain = domain->priv;
4176 struct pci_dev *pdev = to_pci_dev(dev);
4177
2c2e2c38 4178 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 4179}
c7151a8d 4180
b146a1c9
JR		 4181static int intel_iommu_map(struct iommu_domain *domain,
4182 unsigned long iova, phys_addr_t hpa,
5009065d 4183 size_t size, int iommu_prot)
faa3d6f5 4184{
dde57a21 4185 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4186 u64 max_addr;
dde57a21 4187 int prot = 0;
faa3d6f5 4188 int ret;
fe40f1e0 4189
dde57a21
JR		 4190 if (iommu_prot & IOMMU_READ)
4191 prot |= DMA_PTE_READ;
4192 if (iommu_prot & IOMMU_WRITE)
4193 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4194 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4195 prot |= DMA_PTE_SNP;
dde57a21 4196
163cc52c 4197 max_addr = iova + size;
dde57a21 4198 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4199 u64 end;
4200
4201 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4202 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4203 if (end < max_addr) {
8954da1f 4204 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4205 "sufficient for the mapped address (%llx)\n",
8954da1f 4206 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4207 return -EFAULT;
4208 }
dde57a21 4209 dmar_domain->max_addr = max_addr;
fe40f1e0 4210 }
ad051221
DW		 4211 /* Round up size to next multiple of PAGE_SIZE, if it and
4212 the low bits of hpa would take us onto the next page */
88cb6a74 4213 size = aligned_nrpages(hpa, size);
ad051221
DW		 4214 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4215 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4216 return ret;
38717946 4217}
38717946 4218
5009065d 4219static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4220 unsigned long iova, size_t size)
38717946 4221{
dde57a21 4222 struct dmar_domain *dmar_domain = domain->priv;
ea8ea460
DW		 4223 struct page *freelist = NULL;
4224 struct intel_iommu *iommu;
4225 unsigned long start_pfn, last_pfn;
4226 unsigned int npages;
4227 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4228
4229 /* Cope with horrid API which requires us to unmap more than the
4230 size argument if it happens to be a large-page mapping. */
4231 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4232 BUG();
4233
4234 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4235 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4236
ea8ea460
DW		 4237 start_pfn = iova >> VTD_PAGE_SHIFT;
4238 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4239
4240 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4241
4242 npages = last_pfn - start_pfn + 1;
4243
4244 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4245 iommu = g_iommus[iommu_id];
4246
4247 /*
4248 * find bit position of dmar_domain
4249 */
4250 ndomains = cap_ndoms(iommu->cap);
4251 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4252 if (iommu->domains[num] == dmar_domain)
4253 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4254 npages, !freelist, 0);
4255 }
4256
4257 }
4258
4259 dma_free_pagelist(freelist);
fe40f1e0 4260
163cc52c
DW		 4261 if (dmar_domain->max_addr == iova + size)
4262 dmar_domain->max_addr = iova;
b146a1c9 4263
5cf0a76f 4264 return size;
38717946 4265}
38717946 4266
d14d6577 4267static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4268 dma_addr_t iova)
38717946 4269{
d14d6577 4270 struct dmar_domain *dmar_domain = domain->priv;
38717946 4271 struct dma_pte *pte;
5cf0a76f 4272 int level = 0;
faa3d6f5 4273 u64 phys = 0;
38717946 4274
5cf0a76f 4275 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4276 if (pte)
faa3d6f5 4277 phys = dma_pte_addr(pte);
38717946 4278
faa3d6f5 4279 return phys;
38717946 4280}
a8bcbb0d 4281
dbb9fd86
SY		 4282static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4283 unsigned long cap)
4284{
4285 struct dmar_domain *dmar_domain = domain->priv;
4286
4287 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4288 return dmar_domain->iommu_snooping;
323f99cb 4289 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4290 return irq_remapping_enabled;
dbb9fd86
SY		 4291
4292 return 0;
4293}
4294
783f157b 4295#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
70ae6f0d 4296
abdfdde2
AW		 4297static int intel_iommu_add_device(struct device *dev)
4298{
4299 struct pci_dev *pdev = to_pci_dev(dev);
3da4af0a 4300 struct pci_dev *bridge, *dma_pdev = NULL;
abdfdde2
AW		 4301 struct iommu_group *group;
4302 int ret;
70ae6f0d 4303
abdfdde2
AW		 4304 if (!device_to_iommu(pci_domain_nr(pdev->bus),
4305 pdev->bus->number, pdev->devfn))
70ae6f0d
AW		 4306 return -ENODEV;
4307
4308 bridge = pci_find_upstream_pcie_bridge(pdev);
4309 if (bridge) {
abdfdde2
AW		 4310 if (pci_is_pcie(bridge))
4311 dma_pdev = pci_get_domain_bus_and_slot(
4312 pci_domain_nr(pdev->bus),
4313 bridge->subordinate->number, 0);
3da4af0a 4314 if (!dma_pdev)
abdfdde2
AW		 4315 dma_pdev = pci_dev_get(bridge);
4316 } else
4317 dma_pdev = pci_dev_get(pdev);
4318
a4ff1fc2 4319 /* Account for quirked devices */
783f157b
AW		 4320 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4321
a4ff1fc2
AW		 4322 /*
4323 * If it's a multifunction device that does not support our
c14d2690
AW		 4324 * required ACS flags, add to the same group as lowest numbered
4325 * function that also does not suport the required ACS flags.
a4ff1fc2 4326 */
783f157b 4327 if (dma_pdev->multifunction &&
c14d2690
AW		 4328 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4329 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4330
4331 for (i = 0; i < 8; i++) {
4332 struct pci_dev *tmp;
4333
4334 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4335 if (!tmp)
4336 continue;
4337
4338 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4339 swap_pci_ref(&dma_pdev, tmp);
4340 break;
4341 }
4342 pci_dev_put(tmp);
4343 }
4344 }
783f157b 4345
a4ff1fc2
AW		 4346 /*
4347 * Devices on the root bus go through the iommu. If that's not us,
4348 * find the next upstream device and test ACS up to the root bus.
4349 * Finding the next device may require skipping virtual buses.
4350 */
783f157b 4351 while (!pci_is_root_bus(dma_pdev->bus)) {
a4ff1fc2
AW		 4352 struct pci_bus *bus = dma_pdev->bus;
4353
4354 while (!bus->self) {
4355 if (!pci_is_root_bus(bus))
4356 bus = bus->parent;
4357 else
4358 goto root_bus;
4359 }
4360
4361 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
783f157b
AW		 4362 break;
4363
a4ff1fc2 4364 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
783f157b
AW		 4365 }
4366
a4ff1fc2 4367root_bus:
abdfdde2
AW		 4368 group = iommu_group_get(&dma_pdev->dev);
4369 pci_dev_put(dma_pdev);
4370 if (!group) {
4371 group = iommu_group_alloc();
4372 if (IS_ERR(group))
4373 return PTR_ERR(group);
70ae6f0d
AW		 4374 }
4375
abdfdde2 4376 ret = iommu_group_add_device(group, dev);
bcb71abe 4377
abdfdde2
AW		 4378 iommu_group_put(group);
4379 return ret;
4380}
70ae6f0d 4381
abdfdde2
AW		 4382static void intel_iommu_remove_device(struct device *dev)
4383{
4384 iommu_group_remove_device(dev);
70ae6f0d
AW		 4385}
4386
a8bcbb0d
JR		 4387static struct iommu_ops intel_iommu_ops = {
4388 .domain_init = intel_iommu_domain_init,
4389 .domain_destroy = intel_iommu_domain_destroy,
4390 .attach_dev = intel_iommu_attach_device,
4391 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4392 .map = intel_iommu_map,
4393 .unmap = intel_iommu_unmap,
a8bcbb0d 4394 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4395 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4396 .add_device = intel_iommu_add_device,
4397 .remove_device = intel_iommu_remove_device,
6d1c56a9 4398 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4399};
9af88143 4400
9452618e
DV		 4401static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4402{
4403 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4404 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4405 dmar_map_gfx = 0;
4406}
4407
4408DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4409DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4410DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4411DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4412DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4413DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4414DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4415
d34d6517 4416static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4417{
4418 /*
4419 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4420 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4421 */
4422 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4423 rwbf_quirk = 1;
4424}
4425
4426DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4427DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4428DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4429DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4430DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4431DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4432DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4433
eecfd57f
AJ		 4434#define GGC 0x52
4435#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4436#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4437#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4438#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4439#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4440#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4441#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4442#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4443
d34d6517 4444static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4445{
4446 unsigned short ggc;
4447
eecfd57f 4448 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4449 return;
4450
eecfd57f 4451 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4452 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4453 dmar_map_gfx = 0;
6fbcfb3e
DW		 4454 } else if (dmar_map_gfx) {
4455 /* we have to ensure the gfx device is idle before we flush */
4456 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4457 intel_iommu_strict = 1;
4458 }
9eecabcb
DW		 4459}
4460DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4461DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4462DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4463DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4464
e0fc7e0b
DW		 4465/* On Tylersburg chipsets, some BIOSes have been known to enable the
4466 ISOCH DMAR unit for the Azalia sound device, but not give it any
4467 TLB entries, which causes it to deadlock. Check for that. We do
4468 this in a function called from init_dmars(), instead of in a PCI
4469 quirk, because we don't want to print the obnoxious "BIOS broken"
4470 message if VT-d is actually disabled.
4471*/
4472static void __init check_tylersburg_isoch(void)
4473{
4474 struct pci_dev *pdev;
4475 uint32_t vtisochctrl;
4476
4477 /* If there's no Azalia in the system anyway, forget it. */
4478 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4479 if (!pdev)
4480 return;
4481 pci_dev_put(pdev);
4482
4483 /* System Management Registers. Might be hidden, in which case
4484 we can't do the sanity check. But that's OK, because the
4485 known-broken BIOSes _don't_ actually hide it, so far. */
4486 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4487 if (!pdev)
4488 return;
4489
4490 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4491 pci_dev_put(pdev);
4492 return;
4493 }
4494
4495 pci_dev_put(pdev);
4496
4497 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4498 if (vtisochctrl & 1)
4499 return;
4500
4501 /* Drop all bits other than the number of TLB entries */
4502 vtisochctrl &= 0x1c;
4503
4504 /* If we have the recommended number of TLB entries (16), fine. */
4505 if (vtisochctrl == 0x10)
4506 return;
4507
4508 /* Zero TLB entries? You get to ride the short bus to school. */
4509 if (!vtisochctrl) {
4510 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4511 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4512 dmi_get_system_info(DMI_BIOS_VENDOR),
4513 dmi_get_system_info(DMI_BIOS_VERSION),
4514 dmi_get_system_info(DMI_PRODUCT_VERSION));
4515 iommu_identity_mapping |= IDENTMAP_AZALIA;
4516 return;
4517 }
4518
4519 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4520 vtisochctrl);
4521}
Linux kernel - Deliverables
This page took 1.08631 seconds and 5 git commands to generate.