projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu: Make IOVA domain low limit flexible
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 18 */
19
20#include <linux/init.h>
21#include <linux/bitmap.h>
5e0d2a6f 22#include <linux/debugfs.h>
54485c30 23#include <linux/export.h>
ba395927
KA		 24#include <linux/slab.h>
25#include <linux/irq.h>
26#include <linux/interrupt.h>
ba395927
KA		 27#include <linux/spinlock.h>
28#include <linux/pci.h>
29#include <linux/dmar.h>
30#include <linux/dma-mapping.h>
31#include <linux/mempool.h>
75f05569 32#include <linux/memory.h>
5e0d2a6f 33#include <linux/timer.h>
38717946 34#include <linux/iova.h>
5d450806 35#include <linux/iommu.h>
38717946 36#include <linux/intel-iommu.h>
134fac3f 37#include <linux/syscore_ops.h>
69575d38 38#include <linux/tboot.h>
adb2fe02 39#include <linux/dmi.h>
5cdede24 40#include <linux/pci-ats.h>
0ee332c1 41#include <linux/memblock.h>
36746436 42#include <linux/dma-contiguous.h>
8a8f422d 43#include <asm/irq_remapping.h>
ba395927 44#include <asm/cacheflush.h>
46a7fa27 45#include <asm/iommu.h>
ba395927 46
078e1ee2
JR		 47#include "irq_remapping.h"
48
5b6985ce
FY		 49#define ROOT_SIZE VTD_PAGE_SIZE
50#define CONTEXT_SIZE VTD_PAGE_SIZE
51
ba395927
KA		 52#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 54#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 55
56#define IOAPIC_RANGE_START (0xfee00000)
57#define IOAPIC_RANGE_END (0xfeefffff)
58#define IOVA_START_ADDR (0x1000)
59
60#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
4ed0d3e6 62#define MAX_AGAW_WIDTH 64
5c645b35 63#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 64
2ebe3151
DW		 65#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
67
68/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 73
1b722500
RM		 74/* IO virtual address start page frame number */
75#define IOVA_START_PFN (1)
76
f27be03b 77#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 78#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 79#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 80
df08cdc7
AM		 81/* page table handling */
82#define LEVEL_STRIDE (9)
83#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
84
6d1c56a9
OBC		 85/*
86 * This bitmap is used to advertise the page sizes our hardware support
87 * to the IOMMU core, which will then use this information to split
88 * physically contiguous memory regions it is mapping into page sizes
89 * that we support.
90 *
91 * Traditionally the IOMMU core just handed us the mappings directly,
92 * after making sure the size is an order of a 4KiB page and that the
93 * mapping has natural alignment.
94 *
95 * To retain this behavior, we currently advertise that we support
96 * all page sizes that are an order of 4KiB.
97 *
98 * If at some point we'd like to utilize the IOMMU core's new behavior,
99 * we could change this to advertise the real page sizes we support.
100 */
101#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
102
df08cdc7
AM		 103static inline int agaw_to_level(int agaw)
104{
105 return agaw + 2;
106}
107
108static inline int agaw_to_width(int agaw)
109{
5c645b35 110 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 111}
112
113static inline int width_to_agaw(int width)
114{
5c645b35 115 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 116}
117
118static inline unsigned int level_to_offset_bits(int level)
119{
120 return (level - 1) * LEVEL_STRIDE;
121}
122
123static inline int pfn_level_offset(unsigned long pfn, int level)
124{
125 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
126}
127
128static inline unsigned long level_mask(int level)
129{
130 return -1UL << level_to_offset_bits(level);
131}
132
133static inline unsigned long level_size(int level)
134{
135 return 1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long align_to_level(unsigned long pfn, int level)
139{
140 return (pfn + level_size(level) - 1) & level_mask(level);
141}
fd18de50 142
6dd9a7c7
YS		 143static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
144{
5c645b35 145 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 146}
147
dd4e8319
DW		 148/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
149 are never going to work. */
150static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
151{
152 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
153}
154
155static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
156{
157 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159static inline unsigned long page_to_dma_pfn(struct page *pg)
160{
161 return mm_to_dma_pfn(page_to_pfn(pg));
162}
163static inline unsigned long virt_to_dma_pfn(void *p)
164{
165 return page_to_dma_pfn(virt_to_page(p));
166}
167
d9630fe9
WH		 168/* global iommu list, set NULL for ignored DMAR units */
169static struct intel_iommu **g_iommus;
170
e0fc7e0b 171static void __init check_tylersburg_isoch(void);
9af88143
DW		 172static int rwbf_quirk;
173
b779260b
JC		 174/*
175 * set to 1 to panic kernel if can't successfully enable VT-d
176 * (used when kernel is launched w/ TXT)
177 */
178static int force_on = 0;
179
46b08e1a
MM		 180/*
181 * 0: Present
182 * 1-11: Reserved
183 * 12-63: Context Ptr (12 - (haw-1))
184 * 64-127: Reserved
185 */
186struct root_entry {
187 u64 val;
188 u64 rsvd1;
189};
190#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
191static inline bool root_present(struct root_entry *root)
192{
193 return (root->val & 1);
194}
195static inline void set_root_present(struct root_entry *root)
196{
197 root->val |= 1;
198}
199static inline void set_root_value(struct root_entry *root, unsigned long value)
200{
1a2262f9 201 root->val &= ~VTD_PAGE_MASK;
46b08e1a
MM		 202 root->val |= value & VTD_PAGE_MASK;
203}
204
205static inline struct context_entry *
206get_context_addr_from_root(struct root_entry *root)
207{
208 return (struct context_entry *)
209 (root_present(root)?phys_to_virt(
210 root->val & VTD_PAGE_MASK) :
211 NULL);
212}
213
7a8fc25e
MM		 214/*
215 * low 64 bits:
216 * 0: present
217 * 1: fault processing disable
218 * 2-3: translation type
219 * 12-63: address space root
220 * high 64 bits:
221 * 0-2: address width
222 * 3-6: aval
223 * 8-23: domain id
224 */
225struct context_entry {
226 u64 lo;
227 u64 hi;
228};
c07e7d21
MM		 229
230static inline bool context_present(struct context_entry *context)
231{
232 return (context->lo & 1);
233}
234static inline void context_set_present(struct context_entry *context)
235{
236 context->lo |= 1;
237}
238
239static inline void context_set_fault_enable(struct context_entry *context)
240{
241 context->lo &= (((u64)-1) << 2) | 1;
242}
243
c07e7d21
MM		 244static inline void context_set_translation_type(struct context_entry *context,
245 unsigned long value)
246{
247 context->lo &= (((u64)-1) << 4) | 3;
248 context->lo |= (value & 3) << 2;
249}
250
251static inline void context_set_address_root(struct context_entry *context,
252 unsigned long value)
253{
1a2262f9 254 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 255 context->lo |= value & VTD_PAGE_MASK;
256}
257
258static inline void context_set_address_width(struct context_entry *context,
259 unsigned long value)
260{
261 context->hi |= value & 7;
262}
263
264static inline void context_set_domain_id(struct context_entry *context,
265 unsigned long value)
266{
267 context->hi |= (value & ((1 << 16) - 1)) << 8;
268}
269
270static inline void context_clear_entry(struct context_entry *context)
271{
272 context->lo = 0;
273 context->hi = 0;
274}
7a8fc25e 275
622ba12a
MM		 276/*
277 * 0: readable
278 * 1: writable
279 * 2-6: reserved
280 * 7: super page
9cf06697
SY		 281 * 8-10: available
282 * 11: snoop behavior
622ba12a
MM		 283 * 12-63: Host physcial address
284 */
285struct dma_pte {
286 u64 val;
287};
622ba12a 288
19c239ce
MM		 289static inline void dma_clear_pte(struct dma_pte *pte)
290{
291 pte->val = 0;
292}
293
19c239ce
MM		 294static inline u64 dma_pte_addr(struct dma_pte *pte)
295{
c85994e4
DW		 296#ifdef CONFIG_64BIT
297 return pte->val & VTD_PAGE_MASK;
298#else
299 /* Must have a full atomic 64-bit read */
1a8bd481 300 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 301#endif
19c239ce
MM		 302}
303
19c239ce
MM		 304static inline bool dma_pte_present(struct dma_pte *pte)
305{
306 return (pte->val & 3) != 0;
307}
622ba12a 308
4399c8bf
AK		 309static inline bool dma_pte_superpage(struct dma_pte *pte)
310{
c3c75eb7 311 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 312}
313
75e6bf96
DW		 314static inline int first_pte_in_page(struct dma_pte *pte)
315{
316 return !((unsigned long)pte & ~VTD_PAGE_MASK);
317}
318
2c2e2c38
FY		 319/*
320 * This domain is a statically identity mapping domain.
321 * 1. This domain creats a static 1:1 mapping to all usable memory.
322 * 2. It maps to each iommu if successful.
323 * 3. Each iommu mapps to this domain if successful.
324 */
19943b0e
DW		 325static struct dmar_domain *si_domain;
326static int hw_pass_through = 1;
2c2e2c38 327
1ce28feb
WH		 328/* domain represents a virtual machine, more than one devices
329 * across iommus may be owned in one domain, e.g. kvm guest.
330 */
ab8dfe25 331#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 332
2c2e2c38 333/* si_domain contains mulitple devices */
ab8dfe25 334#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 335
99126f7c
MM		 336struct dmar_domain {
337 int id; /* domain id */
4c923d47 338 int nid; /* node id */
78d8e704 339 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
1b198bb0 340 /* bitmap of iommus this domain uses*/
99126f7c
MM		 341
342 struct list_head devices; /* all devices' list */
343 struct iova_domain iovad; /* iova's that belong to this domain */
344
345 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 346 int gaw; /* max guest address width */
347
348 /* adjusted guest address width, 0 is level 2 30-bit */
349 int agaw;
350
3b5410e7 351 int flags; /* flags to find out type of domain */
8e604097
WH		 352
353 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 354 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 355 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 356 int iommu_superpage;/* Level of superpages supported:
357 0 == 4KiB (no superpages), 1 == 2MiB,
358 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 359 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 360 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 361};
362
a647dacb
MM		 363/* PCI domain-device relationship */
364struct device_domain_info {
365 struct list_head link; /* link to domain siblings */
366 struct list_head global; /* link to global list */
276dbf99 367 u8 bus; /* PCI bus number */
a647dacb 368 u8 devfn; /* PCI devfn number */
0bcb3e28 369 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 370 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 371 struct dmar_domain *domain; /* pointer to domain */
372};
373
b94e4117
JL		 374struct dmar_rmrr_unit {
375 struct list_head list; /* list of rmrr units */
376 struct acpi_dmar_header *hdr; /* ACPI header */
377 u64 base_address; /* reserved base address*/
378 u64 end_address; /* reserved end address */
832bd858 379 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 380 int devices_cnt; /* target device count */
381};
382
383struct dmar_atsr_unit {
384 struct list_head list; /* list of ATSR units */
385 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 386 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 387 int devices_cnt; /* target device count */
388 u8 include_all:1; /* include all ports */
389};
390
391static LIST_HEAD(dmar_atsr_units);
392static LIST_HEAD(dmar_rmrr_units);
393
394#define for_each_rmrr_units(rmrr) \
395 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
396
5e0d2a6f 397static void flush_unmaps_timeout(unsigned long data);
398
b707cb02 399static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 400
80b20dd8 401#define HIGH_WATER_MARK 250
402struct deferred_flush_tables {
403 int next;
404 struct iova *iova[HIGH_WATER_MARK];
405 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 406 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 407};
408
409static struct deferred_flush_tables *deferred_flush;
410
5e0d2a6f 411/* bitmap for indexing intel_iommus */
5e0d2a6f 412static int g_num_of_iommus;
413
414static DEFINE_SPINLOCK(async_umap_flush_lock);
415static LIST_HEAD(unmaps_to_do);
416
417static int timer_on;
418static long list_size;
5e0d2a6f 419
92d03cc8 420static void domain_exit(struct dmar_domain *domain);
ba395927 421static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 422static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 423 struct device *dev);
92d03cc8 424static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 425 struct device *dev);
2a46ddf7
JL		 426static int domain_detach_iommu(struct dmar_domain *domain,
427 struct intel_iommu *iommu);
ba395927 428
d3f13810 429#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 430int dmar_disabled = 0;
431#else
432int dmar_disabled = 1;
d3f13810 433#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 434
8bc1f85c
ED		 435int intel_iommu_enabled = 0;
436EXPORT_SYMBOL_GPL(intel_iommu_enabled);
437
2d9e667e 438static int dmar_map_gfx = 1;
7d3b03ce 439static int dmar_forcedac;
5e0d2a6f 440static int intel_iommu_strict;
6dd9a7c7 441static int intel_iommu_superpage = 1;
ba395927 442
c0771df8
DW		 443int intel_iommu_gfx_mapped;
444EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
445
ba395927
KA		 446#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
447static DEFINE_SPINLOCK(device_domain_lock);
448static LIST_HEAD(device_domain_list);
449
b22f6434 450static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 451
ba395927
KA		 452static int __init intel_iommu_setup(char *str)
453{
454 if (!str)
455 return -EINVAL;
456 while (*str) {
0cd5c3c8
KM		 457 if (!strncmp(str, "on", 2)) {
458 dmar_disabled = 0;
459 printk(KERN_INFO "Intel-IOMMU: enabled\n");
460 } else if (!strncmp(str, "off", 3)) {
ba395927 461 dmar_disabled = 1;
0cd5c3c8 462 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 463 } else if (!strncmp(str, "igfx_off", 8)) {
464 dmar_map_gfx = 0;
465 printk(KERN_INFO
466 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 467 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 468 printk(KERN_INFO
7d3b03ce
KA		 469 "Intel-IOMMU: Forcing DAC for PCI devices\n");
470 dmar_forcedac = 1;
5e0d2a6f 471 } else if (!strncmp(str, "strict", 6)) {
472 printk(KERN_INFO
473 "Intel-IOMMU: disable batched IOTLB flush\n");
474 intel_iommu_strict = 1;
6dd9a7c7
YS		 475 } else if (!strncmp(str, "sp_off", 6)) {
476 printk(KERN_INFO
477 "Intel-IOMMU: disable supported super page\n");
478 intel_iommu_superpage = 0;
ba395927
KA		 479 }
480
481 str += strcspn(str, ",");
482 while (*str == ',')
483 str++;
484 }
485 return 0;
486}
487__setup("intel_iommu=", intel_iommu_setup);
488
489static struct kmem_cache *iommu_domain_cache;
490static struct kmem_cache *iommu_devinfo_cache;
ba395927 491
4c923d47 492static inline void *alloc_pgtable_page(int node)
eb3fa7cb 493{
4c923d47
SS		 494 struct page *page;
495 void *vaddr = NULL;
eb3fa7cb 496
4c923d47
SS		 497 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
498 if (page)
499 vaddr = page_address(page);
eb3fa7cb 500 return vaddr;
ba395927
KA		 501}
502
503static inline void free_pgtable_page(void *vaddr)
504{
505 free_page((unsigned long)vaddr);
506}
507
508static inline void *alloc_domain_mem(void)
509{
354bb65e 510 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 511}
512
38717946 513static void free_domain_mem(void *vaddr)
ba395927
KA		 514{
515 kmem_cache_free(iommu_domain_cache, vaddr);
516}
517
518static inline void * alloc_devinfo_mem(void)
519{
354bb65e 520 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 521}
522
523static inline void free_devinfo_mem(void *vaddr)
524{
525 kmem_cache_free(iommu_devinfo_cache, vaddr);
526}
527
ab8dfe25
JL		 528static inline int domain_type_is_vm(struct dmar_domain *domain)
529{
530 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
531}
532
533static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
534{
535 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
536 DOMAIN_FLAG_STATIC_IDENTITY);
537}
1b573683 538
162d1b10
JL		 539static inline int domain_pfn_supported(struct dmar_domain *domain,
540 unsigned long pfn)
541{
542 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
543
544 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
545}
546
4ed0d3e6 547static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 548{
549 unsigned long sagaw;
550 int agaw = -1;
551
552 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 553 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 554 agaw >= 0; agaw--) {
555 if (test_bit(agaw, &sagaw))
556 break;
557 }
558
559 return agaw;
560}
561
4ed0d3e6
FY		 562/*
563 * Calculate max SAGAW for each iommu.
564 */
565int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
566{
567 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
568}
569
570/*
571 * calculate agaw for each iommu.
572 * "SAGAW" may be different across iommus, use a default agaw, and
573 * get a supported less agaw for iommus that don't support the default agaw.
574 */
575int iommu_calculate_agaw(struct intel_iommu *iommu)
576{
577 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
578}
579
2c2e2c38 580/* This functionin only returns single iommu in a domain */
8c11e798
WH		 581static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
582{
583 int iommu_id;
584
2c2e2c38 585 /* si_domain and vm domain should not get here. */
ab8dfe25 586 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 587 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 588 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
589 return NULL;
590
591 return g_iommus[iommu_id];
592}
593
8e604097
WH		 594static void domain_update_iommu_coherency(struct dmar_domain *domain)
595{
d0501960
DW		 596 struct dmar_drhd_unit *drhd;
597 struct intel_iommu *iommu;
598 int i, found = 0;
2e12bc29 599
d0501960 600 domain->iommu_coherency = 1;
8e604097 601
1b198bb0 602 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
d0501960 603 found = 1;
8e604097
WH		 604 if (!ecap_coherent(g_iommus[i]->ecap)) {
605 domain->iommu_coherency = 0;
606 break;
607 }
8e604097 608 }
d0501960
DW		 609 if (found)
610 return;
611
612 /* No hardware attached; use lowest common denominator */
613 rcu_read_lock();
614 for_each_active_iommu(iommu, drhd) {
615 if (!ecap_coherent(iommu->ecap)) {
616 domain->iommu_coherency = 0;
617 break;
618 }
619 }
620 rcu_read_unlock();
8e604097
WH		 621}
622
161f6934 623static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 624{
161f6934
JL		 625 struct dmar_drhd_unit *drhd;
626 struct intel_iommu *iommu;
627 int ret = 1;
58c610bd 628
161f6934
JL		 629 rcu_read_lock();
630 for_each_active_iommu(iommu, drhd) {
631 if (iommu != skip) {
632 if (!ecap_sc_support(iommu->ecap)) {
633 ret = 0;
634 break;
635 }
58c610bd 636 }
58c610bd 637 }
161f6934
JL		 638 rcu_read_unlock();
639
640 return ret;
58c610bd
SY		 641}
642
161f6934 643static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 644{
8140a95d 645 struct dmar_drhd_unit *drhd;
161f6934 646 struct intel_iommu *iommu;
8140a95d 647 int mask = 0xf;
6dd9a7c7
YS		 648
649 if (!intel_iommu_superpage) {
161f6934 650 return 0;
6dd9a7c7
YS		 651 }
652
8140a95d 653 /* set iommu_superpage to the smallest common denominator */
0e242612 654 rcu_read_lock();
8140a95d 655 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 656 if (iommu != skip) {
657 mask &= cap_super_page_val(iommu->cap);
658 if (!mask)
659 break;
6dd9a7c7
YS		 660 }
661 }
0e242612
JL		 662 rcu_read_unlock();
663
161f6934 664 return fls(mask);
6dd9a7c7
YS		 665}
666
58c610bd
SY		 667/* Some capabilities may be different across iommus */
668static void domain_update_iommu_cap(struct dmar_domain *domain)
669{
670 domain_update_iommu_coherency(domain);
161f6934
JL		 671 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
672 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 673}
674
156baca8 675static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 676{
677 struct dmar_drhd_unit *drhd = NULL;
b683b230 678 struct intel_iommu *iommu;
156baca8
DW		 679 struct device *tmp;
680 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 681 u16 segment = 0;
c7151a8d
WH		 682 int i;
683
156baca8
DW		 684 if (dev_is_pci(dev)) {
685 pdev = to_pci_dev(dev);
686 segment = pci_domain_nr(pdev->bus);
687 } else if (ACPI_COMPANION(dev))
688 dev = &ACPI_COMPANION(dev)->dev;
689
0e242612 690 rcu_read_lock();
b683b230 691 for_each_active_iommu(iommu, drhd) {
156baca8 692 if (pdev && segment != drhd->segment)
276dbf99 693 continue;
c7151a8d 694
b683b230 695 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 696 drhd->devices_cnt, i, tmp) {
697 if (tmp == dev) {
698 *bus = drhd->devices[i].bus;
699 *devfn = drhd->devices[i].devfn;
b683b230 700 goto out;
156baca8
DW		 701 }
702
703 if (!pdev || !dev_is_pci(tmp))
704 continue;
705
706 ptmp = to_pci_dev(tmp);
707 if (ptmp->subordinate &&
708 ptmp->subordinate->number <= pdev->bus->number &&
709 ptmp->subordinate->busn_res.end >= pdev->bus->number)
710 goto got_pdev;
924b6231 711 }
c7151a8d 712
156baca8
DW		 713 if (pdev && drhd->include_all) {
714 got_pdev:
715 *bus = pdev->bus->number;
716 *devfn = pdev->devfn;
b683b230 717 goto out;
156baca8 718 }
c7151a8d 719 }
b683b230 720 iommu = NULL;
156baca8 721 out:
0e242612 722 rcu_read_unlock();
c7151a8d 723
b683b230 724 return iommu;
c7151a8d
WH		 725}
726
5331fe6f
WH		 727static void domain_flush_cache(struct dmar_domain *domain,
728 void *addr, int size)
729{
730 if (!domain->iommu_coherency)
731 clflush_cache_range(addr, size);
732}
733
ba395927
KA		 734/* Gets context entry for a given bus and devfn */
735static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
736 u8 bus, u8 devfn)
737{
738 struct root_entry *root;
739 struct context_entry *context;
740 unsigned long phy_addr;
741 unsigned long flags;
742
743 spin_lock_irqsave(&iommu->lock, flags);
744 root = &iommu->root_entry[bus];
745 context = get_context_addr_from_root(root);
746 if (!context) {
4c923d47
SS		 747 context = (struct context_entry *)
748 alloc_pgtable_page(iommu->node);
ba395927
KA		 749 if (!context) {
750 spin_unlock_irqrestore(&iommu->lock, flags);
751 return NULL;
752 }
5b6985ce 753 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 754 phy_addr = virt_to_phys((void *)context);
755 set_root_value(root, phy_addr);
756 set_root_present(root);
757 __iommu_flush_cache(iommu, root, sizeof(*root));
758 }
759 spin_unlock_irqrestore(&iommu->lock, flags);
760 return &context[devfn];
761}
762
763static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
764{
765 struct root_entry *root;
766 struct context_entry *context;
767 int ret;
768 unsigned long flags;
769
770 spin_lock_irqsave(&iommu->lock, flags);
771 root = &iommu->root_entry[bus];
772 context = get_context_addr_from_root(root);
773 if (!context) {
774 ret = 0;
775 goto out;
776 }
c07e7d21 777 ret = context_present(&context[devfn]);
ba395927
KA		 778out:
779 spin_unlock_irqrestore(&iommu->lock, flags);
780 return ret;
781}
782
783static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
784{
785 struct root_entry *root;
786 struct context_entry *context;
787 unsigned long flags;
788
789 spin_lock_irqsave(&iommu->lock, flags);
790 root = &iommu->root_entry[bus];
791 context = get_context_addr_from_root(root);
792 if (context) {
c07e7d21 793 context_clear_entry(&context[devfn]);
ba395927
KA		 794 __iommu_flush_cache(iommu, &context[devfn], \
795 sizeof(*context));
796 }
797 spin_unlock_irqrestore(&iommu->lock, flags);
798}
799
800static void free_context_table(struct intel_iommu *iommu)
801{
802 struct root_entry *root;
803 int i;
804 unsigned long flags;
805 struct context_entry *context;
806
807 spin_lock_irqsave(&iommu->lock, flags);
808 if (!iommu->root_entry) {
809 goto out;
810 }
811 for (i = 0; i < ROOT_ENTRY_NR; i++) {
812 root = &iommu->root_entry[i];
813 context = get_context_addr_from_root(root);
814 if (context)
815 free_pgtable_page(context);
816 }
817 free_pgtable_page(iommu->root_entry);
818 iommu->root_entry = NULL;
819out:
820 spin_unlock_irqrestore(&iommu->lock, flags);
821}
822
b026fd28 823static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 824 unsigned long pfn, int *target_level)
ba395927 825{
ba395927
KA		 826 struct dma_pte *parent, *pte = NULL;
827 int level = agaw_to_level(domain->agaw);
4399c8bf 828 int offset;
ba395927
KA		 829
830 BUG_ON(!domain->pgd);
f9423606 831
162d1b10 832 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 833 /* Address beyond IOMMU's addressing capabilities. */
834 return NULL;
835
ba395927
KA		 836 parent = domain->pgd;
837
5cf0a76f 838 while (1) {
ba395927
KA		 839 void *tmp_page;
840
b026fd28 841 offset = pfn_level_offset(pfn, level);
ba395927 842 pte = &parent[offset];
5cf0a76f 843 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 844 break;
5cf0a76f 845 if (level == *target_level)
ba395927
KA		 846 break;
847
19c239ce 848 if (!dma_pte_present(pte)) {
c85994e4
DW		 849 uint64_t pteval;
850
4c923d47 851 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 852
206a73c1 853 if (!tmp_page)
ba395927 854 return NULL;
206a73c1 855
c85994e4 856 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 857 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 858 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 859 /* Someone else set it while we were thinking; use theirs. */
860 free_pgtable_page(tmp_page);
effad4b5 861 else
c85994e4 862 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 863 }
5cf0a76f
DW		 864 if (level == 1)
865 break;
866
19c239ce 867 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 868 level--;
869 }
870
5cf0a76f
DW		 871 if (!*target_level)
872 *target_level = level;
873
ba395927
KA		 874 return pte;
875}
876
6dd9a7c7 877
ba395927 878/* return address's pte at specific level */
90dcfb5e
DW		 879static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
880 unsigned long pfn,
6dd9a7c7 881 int level, int *large_page)
ba395927
KA		 882{
883 struct dma_pte *parent, *pte = NULL;
884 int total = agaw_to_level(domain->agaw);
885 int offset;
886
887 parent = domain->pgd;
888 while (level <= total) {
90dcfb5e 889 offset = pfn_level_offset(pfn, total);
ba395927
KA		 890 pte = &parent[offset];
891 if (level == total)
892 return pte;
893
6dd9a7c7
YS		 894 if (!dma_pte_present(pte)) {
895 *large_page = total;
ba395927 896 break;
6dd9a7c7
YS		 897 }
898
e16922af 899 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 900 *large_page = total;
901 return pte;
902 }
903
19c239ce 904 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 905 total--;
906 }
907 return NULL;
908}
909
ba395927 910/* clear last level pte, a tlb flush should be followed */
5cf0a76f 911static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 912 unsigned long start_pfn,
913 unsigned long last_pfn)
ba395927 914{
6dd9a7c7 915 unsigned int large_page = 1;
310a5ab9 916 struct dma_pte *first_pte, *pte;
66eae846 917
162d1b10
JL		 918 BUG_ON(!domain_pfn_supported(domain, start_pfn));
919 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 920 BUG_ON(start_pfn > last_pfn);
ba395927 921
04b18e65 922 /* we don't need lock here; nobody else touches the iova range */
59c36286 923 do {
6dd9a7c7
YS		 924 large_page = 1;
925 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 926 if (!pte) {
6dd9a7c7 927 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 928 continue;
929 }
6dd9a7c7 930 do {
310a5ab9 931 dma_clear_pte(pte);
6dd9a7c7 932 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 933 pte++;
75e6bf96
DW		 934 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
935
310a5ab9
DW		 936 domain_flush_cache(domain, first_pte,
937 (void *)pte - (void *)first_pte);
59c36286
DW		 938
939 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 940}
941
3269ee0b
AW		 942static void dma_pte_free_level(struct dmar_domain *domain, int level,
943 struct dma_pte *pte, unsigned long pfn,
944 unsigned long start_pfn, unsigned long last_pfn)
945{
946 pfn = max(start_pfn, pfn);
947 pte = &pte[pfn_level_offset(pfn, level)];
948
949 do {
950 unsigned long level_pfn;
951 struct dma_pte *level_pte;
952
953 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
954 goto next;
955
956 level_pfn = pfn & level_mask(level - 1);
957 level_pte = phys_to_virt(dma_pte_addr(pte));
958
959 if (level > 2)
960 dma_pte_free_level(domain, level - 1, level_pte,
961 level_pfn, start_pfn, last_pfn);
962
963 /* If range covers entire pagetable, free it */
964 if (!(start_pfn > level_pfn ||
08336fd2 965 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 966 dma_clear_pte(pte);
967 domain_flush_cache(domain, pte, sizeof(*pte));
968 free_pgtable_page(level_pte);
969 }
970next:
971 pfn += level_size(level);
972 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
973}
974
ba395927
KA		 975/* free page table pages. last level pte should already be cleared */
976static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 977 unsigned long start_pfn,
978 unsigned long last_pfn)
ba395927 979{
162d1b10
JL		 980 BUG_ON(!domain_pfn_supported(domain, start_pfn));
981 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 982 BUG_ON(start_pfn > last_pfn);
ba395927 983
d41a4adb
JL		 984 dma_pte_clear_range(domain, start_pfn, last_pfn);
985
f3a0a52f 986 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 987 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
988 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 989
ba395927 990 /* free pgd */
d794dc9b 991 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 992 free_pgtable_page(domain->pgd);
993 domain->pgd = NULL;
994 }
995}
996
ea8ea460
DW		 997/* When a page at a given level is being unlinked from its parent, we don't
998 need to *modify* it at all. All we need to do is make a list of all the
999 pages which can be freed just as soon as we've flushed the IOTLB and we
1000 know the hardware page-walk will no longer touch them.
1001 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1002 be freed. */
1003static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1004 int level, struct dma_pte *pte,
1005 struct page *freelist)
1006{
1007 struct page *pg;
1008
1009 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1010 pg->freelist = freelist;
1011 freelist = pg;
1012
1013 if (level == 1)
1014 return freelist;
1015
adeb2590
JL		 1016 pte = page_address(pg);
1017 do {
ea8ea460
DW		 1018 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1019 freelist = dma_pte_list_pagetables(domain, level - 1,
1020 pte, freelist);
adeb2590
JL		 1021 pte++;
1022 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1023
1024 return freelist;
1025}
1026
1027static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1028 struct dma_pte *pte, unsigned long pfn,
1029 unsigned long start_pfn,
1030 unsigned long last_pfn,
1031 struct page *freelist)
1032{
1033 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1034
1035 pfn = max(start_pfn, pfn);
1036 pte = &pte[pfn_level_offset(pfn, level)];
1037
1038 do {
1039 unsigned long level_pfn;
1040
1041 if (!dma_pte_present(pte))
1042 goto next;
1043
1044 level_pfn = pfn & level_mask(level);
1045
1046 /* If range covers entire pagetable, free it */
1047 if (start_pfn <= level_pfn &&
1048 last_pfn >= level_pfn + level_size(level) - 1) {
1049 /* These suborbinate page tables are going away entirely. Don't
1050 bother to clear them; we're just going to *free* them. */
1051 if (level > 1 && !dma_pte_superpage(pte))
1052 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1053
1054 dma_clear_pte(pte);
1055 if (!first_pte)
1056 first_pte = pte;
1057 last_pte = pte;
1058 } else if (level > 1) {
1059 /* Recurse down into a level that isn't *entirely* obsolete */
1060 freelist = dma_pte_clear_level(domain, level - 1,
1061 phys_to_virt(dma_pte_addr(pte)),
1062 level_pfn, start_pfn, last_pfn,
1063 freelist);
1064 }
1065next:
1066 pfn += level_size(level);
1067 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1068
1069 if (first_pte)
1070 domain_flush_cache(domain, first_pte,
1071 (void *)++last_pte - (void *)first_pte);
1072
1073 return freelist;
1074}
1075
1076/* We can't just free the pages because the IOMMU may still be walking
1077 the page tables, and may have cached the intermediate levels. The
1078 pages can only be freed after the IOTLB flush has been done. */
1079struct page *domain_unmap(struct dmar_domain *domain,
1080 unsigned long start_pfn,
1081 unsigned long last_pfn)
1082{
ea8ea460
DW		 1083 struct page *freelist = NULL;
1084
162d1b10
JL		 1085 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1086 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1087 BUG_ON(start_pfn > last_pfn);
1088
1089 /* we don't need lock here; nobody else touches the iova range */
1090 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1091 domain->pgd, 0, start_pfn, last_pfn, NULL);
1092
1093 /* free pgd */
1094 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1095 struct page *pgd_page = virt_to_page(domain->pgd);
1096 pgd_page->freelist = freelist;
1097 freelist = pgd_page;
1098
1099 domain->pgd = NULL;
1100 }
1101
1102 return freelist;
1103}
1104
1105void dma_free_pagelist(struct page *freelist)
1106{
1107 struct page *pg;
1108
1109 while ((pg = freelist)) {
1110 freelist = pg->freelist;
1111 free_pgtable_page(page_address(pg));
1112 }
1113}
1114
ba395927
KA		 1115/* iommu handling */
1116static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1117{
1118 struct root_entry *root;
1119 unsigned long flags;
1120
4c923d47 1121 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46
JL		 1122 if (!root) {
1123 pr_err("IOMMU: allocating root entry for %s failed\n",
1124 iommu->name);
ba395927 1125 return -ENOMEM;
ffebeb46 1126 }
ba395927 1127
5b6985ce 1128 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1129
1130 spin_lock_irqsave(&iommu->lock, flags);
1131 iommu->root_entry = root;
1132 spin_unlock_irqrestore(&iommu->lock, flags);
1133
1134 return 0;
1135}
1136
ba395927
KA		 1137static void iommu_set_root_entry(struct intel_iommu *iommu)
1138{
1139 void *addr;
c416daa9 1140 u32 sts;
ba395927
KA		 1141 unsigned long flag;
1142
1143 addr = iommu->root_entry;
1144
1f5b3c3f 1145 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1146 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1147
c416daa9 1148 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1149
1150 /* Make sure hardware complete it */
1151 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1152 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1153
1f5b3c3f 1154 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1155}
1156
1157static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1158{
1159 u32 val;
1160 unsigned long flag;
1161
9af88143 1162 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1163 return;
ba395927 1164
1f5b3c3f 1165 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1166 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1167
1168 /* Make sure hardware complete it */
1169 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1170 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1171
1f5b3c3f 1172 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1173}
1174
1175/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1176static void __iommu_flush_context(struct intel_iommu *iommu,
1177 u16 did, u16 source_id, u8 function_mask,
1178 u64 type)
ba395927
KA		 1179{
1180 u64 val = 0;
1181 unsigned long flag;
1182
ba395927
KA		 1183 switch (type) {
1184 case DMA_CCMD_GLOBAL_INVL:
1185 val = DMA_CCMD_GLOBAL_INVL;
1186 break;
1187 case DMA_CCMD_DOMAIN_INVL:
1188 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1189 break;
1190 case DMA_CCMD_DEVICE_INVL:
1191 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1192 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1193 break;
1194 default:
1195 BUG();
1196 }
1197 val |= DMA_CCMD_ICC;
1198
1f5b3c3f 1199 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1200 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1201
1202 /* Make sure hardware complete it */
1203 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1204 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1205
1f5b3c3f 1206 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1207}
1208
ba395927 1209/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1210static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1211 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1212{
1213 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1214 u64 val = 0, val_iva = 0;
1215 unsigned long flag;
1216
ba395927
KA		 1217 switch (type) {
1218 case DMA_TLB_GLOBAL_FLUSH:
1219 /* global flush doesn't need set IVA_REG */
1220 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1221 break;
1222 case DMA_TLB_DSI_FLUSH:
1223 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1224 break;
1225 case DMA_TLB_PSI_FLUSH:
1226 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1227 /* IH bit is passed in as part of address */
ba395927
KA		 1228 val_iva = size_order | addr;
1229 break;
1230 default:
1231 BUG();
1232 }
1233 /* Note: set drain read/write */
1234#if 0
1235 /*
1236 * This is probably to be super secure.. Looks like we can
1237 * ignore it without any impact.
1238 */
1239 if (cap_read_drain(iommu->cap))
1240 val |= DMA_TLB_READ_DRAIN;
1241#endif
1242 if (cap_write_drain(iommu->cap))
1243 val |= DMA_TLB_WRITE_DRAIN;
1244
1f5b3c3f 1245 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1246 /* Note: Only uses first TLB reg currently */
1247 if (val_iva)
1248 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1249 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1250
1251 /* Make sure hardware complete it */
1252 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1253 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1254
1f5b3c3f 1255 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1256
1257 /* check IOTLB invalidation granularity */
1258 if (DMA_TLB_IAIG(val) == 0)
1259 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1260 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1261 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1262 (unsigned long long)DMA_TLB_IIRG(type),
1263 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1264}
1265
64ae892b
DW		 1266static struct device_domain_info *
1267iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1268 u8 bus, u8 devfn)
93a23a72
YZ		 1269{
1270 int found = 0;
1271 unsigned long flags;
1272 struct device_domain_info *info;
0bcb3e28 1273 struct pci_dev *pdev;
93a23a72
YZ		 1274
1275 if (!ecap_dev_iotlb_support(iommu->ecap))
1276 return NULL;
1277
1278 if (!iommu->qi)
1279 return NULL;
1280
1281 spin_lock_irqsave(&device_domain_lock, flags);
1282 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1283 if (info->iommu == iommu && info->bus == bus &&
1284 info->devfn == devfn) {
93a23a72
YZ		 1285 found = 1;
1286 break;
1287 }
1288 spin_unlock_irqrestore(&device_domain_lock, flags);
1289
0bcb3e28 1290 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1291 return NULL;
1292
0bcb3e28
DW		 1293 pdev = to_pci_dev(info->dev);
1294
1295 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1296 return NULL;
1297
0bcb3e28 1298 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1299 return NULL;
1300
93a23a72
YZ		 1301 return info;
1302}
1303
1304static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1305{
0bcb3e28 1306 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1307 return;
1308
0bcb3e28 1309 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1310}
1311
1312static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1313{
0bcb3e28
DW		 1314 if (!info->dev || !dev_is_pci(info->dev) ||
1315 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1316 return;
1317
0bcb3e28 1318 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1319}
1320
1321static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1322 u64 addr, unsigned mask)
1323{
1324 u16 sid, qdep;
1325 unsigned long flags;
1326 struct device_domain_info *info;
1327
1328 spin_lock_irqsave(&device_domain_lock, flags);
1329 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1330 struct pci_dev *pdev;
1331 if (!info->dev || !dev_is_pci(info->dev))
1332 continue;
1333
1334 pdev = to_pci_dev(info->dev);
1335 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1336 continue;
1337
1338 sid = info->bus << 8 | info->devfn;
0bcb3e28 1339 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1340 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1341 }
1342 spin_unlock_irqrestore(&device_domain_lock, flags);
1343}
1344
1f0ef2aa 1345static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1346 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1347{
9dd2fe89 1348 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1349 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1350
ba395927
KA		 1351 BUG_ON(pages == 0);
1352
ea8ea460
DW		 1353 if (ih)
1354 ih = 1 << 6;
ba395927 1355 /*
9dd2fe89
YZ		 1356 * Fallback to domain selective flush if no PSI support or the size is
1357 * too big.
ba395927
KA		 1358 * PSI requires page size to be 2 ^ x, and the base address is naturally
1359 * aligned to the size
1360 */
9dd2fe89
YZ		 1361 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1362 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1363 DMA_TLB_DSI_FLUSH);
9dd2fe89 1364 else
ea8ea460 1365 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1366 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1367
1368 /*
82653633
NA		 1369 * In caching mode, changes of pages from non-present to present require
1370 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1371 */
82653633 1372 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1373 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1374}
1375
f8bab735 1376static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1377{
1378 u32 pmen;
1379 unsigned long flags;
1380
1f5b3c3f 1381 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1382 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1383 pmen &= ~DMA_PMEN_EPM;
1384 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1385
1386 /* wait for the protected region status bit to clear */
1387 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1388 readl, !(pmen & DMA_PMEN_PRS), pmen);
1389
1f5b3c3f 1390 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1391}
1392
2a41ccee 1393static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1394{
1395 u32 sts;
1396 unsigned long flags;
1397
1f5b3c3f 1398 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1399 iommu->gcmd |= DMA_GCMD_TE;
1400 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1401
1402 /* Make sure hardware complete it */
1403 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1404 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1405
1f5b3c3f 1406 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1407}
1408
2a41ccee 1409static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1410{
1411 u32 sts;
1412 unsigned long flag;
1413
1f5b3c3f 1414 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1415 iommu->gcmd &= ~DMA_GCMD_TE;
1416 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1417
1418 /* Make sure hardware complete it */
1419 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1420 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1421
1f5b3c3f 1422 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1423}
1424
3460a6d9 1425
ba395927
KA		 1426static int iommu_init_domains(struct intel_iommu *iommu)
1427{
1428 unsigned long ndomains;
1429 unsigned long nlongs;
1430
1431 ndomains = cap_ndoms(iommu->cap);
852bdb04
JL		 1432 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1433 iommu->seq_id, ndomains);
ba395927
KA		 1434 nlongs = BITS_TO_LONGS(ndomains);
1435
94a91b50
DD		 1436 spin_lock_init(&iommu->lock);
1437
ba395927
KA		 1438 /* TBD: there might be 64K domains,
1439 * consider other allocation for future chip
1440 */
1441 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1442 if (!iommu->domain_ids) {
852bdb04
JL		 1443 pr_err("IOMMU%d: allocating domain id array failed\n",
1444 iommu->seq_id);
ba395927
KA		 1445 return -ENOMEM;
1446 }
1447 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1448 GFP_KERNEL);
1449 if (!iommu->domains) {
852bdb04
JL		 1450 pr_err("IOMMU%d: allocating domain array failed\n",
1451 iommu->seq_id);
1452 kfree(iommu->domain_ids);
1453 iommu->domain_ids = NULL;
ba395927
KA		 1454 return -ENOMEM;
1455 }
1456
1457 /*
1458 * if Caching mode is set, then invalid translations are tagged
1459 * with domainid 0. Hence we need to pre-allocate it.
1460 */
1461 if (cap_caching_mode(iommu->cap))
1462 set_bit(0, iommu->domain_ids);
1463 return 0;
1464}
ba395927 1465
ffebeb46 1466static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1467{
1468 struct dmar_domain *domain;
2a46ddf7 1469 int i;
ba395927 1470
94a91b50 1471 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1472 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1473 /*
1474 * Domain id 0 is reserved for invalid translation
1475 * if hardware supports caching mode.
1476 */
1477 if (cap_caching_mode(iommu->cap) && i == 0)
1478 continue;
1479
94a91b50
DD		 1480 domain = iommu->domains[i];
1481 clear_bit(i, iommu->domain_ids);
129ad281
JL		 1482 if (domain_detach_iommu(domain, iommu) == 0 &&
1483 !domain_type_is_vm(domain))
92d03cc8 1484 domain_exit(domain);
5e98c4b1 1485 }
ba395927
KA		 1486 }
1487
1488 if (iommu->gcmd & DMA_GCMD_TE)
1489 iommu_disable_translation(iommu);
ffebeb46 1490}
ba395927 1491
ffebeb46
JL		 1492static void free_dmar_iommu(struct intel_iommu *iommu)
1493{
1494 if ((iommu->domains) && (iommu->domain_ids)) {
1495 kfree(iommu->domains);
1496 kfree(iommu->domain_ids);
1497 iommu->domains = NULL;
1498 iommu->domain_ids = NULL;
1499 }
ba395927 1500
d9630fe9
WH		 1501 g_iommus[iommu->seq_id] = NULL;
1502
ba395927
KA		 1503 /* free context mapping */
1504 free_context_table(iommu);
ba395927
KA		 1505}
1506
ab8dfe25 1507static struct dmar_domain *alloc_domain(int flags)
ba395927 1508{
92d03cc8
JL		 1509 /* domain id for virtual machine, it won't be set in context */
1510 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1511 struct dmar_domain *domain;
ba395927
KA		 1512
1513 domain = alloc_domain_mem();
1514 if (!domain)
1515 return NULL;
1516
ab8dfe25 1517 memset(domain, 0, sizeof(*domain));
4c923d47 1518 domain->nid = -1;
ab8dfe25 1519 domain->flags = flags;
92d03cc8
JL		 1520 spin_lock_init(&domain->iommu_lock);
1521 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1522 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1523 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1524
1525 return domain;
1526}
1527
fb170fb4
JL		 1528static int __iommu_attach_domain(struct dmar_domain *domain,
1529 struct intel_iommu *iommu)
2c2e2c38
FY		 1530{
1531 int num;
1532 unsigned long ndomains;
2c2e2c38 1533
ba395927 1534 ndomains = cap_ndoms(iommu->cap);
ba395927 1535 num = find_first_zero_bit(iommu->domain_ids, ndomains);
fb170fb4
JL		 1536 if (num < ndomains) {
1537 set_bit(num, iommu->domain_ids);
1538 iommu->domains[num] = domain;
1539 } else {
1540 num = -ENOSPC;
ba395927
KA		 1541 }
1542
fb170fb4
JL		 1543 return num;
1544}
1545
1546static int iommu_attach_domain(struct dmar_domain *domain,
1547 struct intel_iommu *iommu)
1548{
1549 int num;
1550 unsigned long flags;
1551
1552 spin_lock_irqsave(&iommu->lock, flags);
1553 num = __iommu_attach_domain(domain, iommu);
44bde614 1554 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4
JL		 1555 if (num < 0)
1556 pr_err("IOMMU: no free domain ids\n");
ba395927 1557
fb170fb4 1558 return num;
ba395927
KA		 1559}
1560
44bde614
JL		 1561static int iommu_attach_vm_domain(struct dmar_domain *domain,
1562 struct intel_iommu *iommu)
1563{
1564 int num;
1565 unsigned long ndomains;
1566
1567 ndomains = cap_ndoms(iommu->cap);
1568 for_each_set_bit(num, iommu->domain_ids, ndomains)
1569 if (iommu->domains[num] == domain)
1570 return num;
1571
1572 return __iommu_attach_domain(domain, iommu);
1573}
1574
2c2e2c38
FY		 1575static void iommu_detach_domain(struct dmar_domain *domain,
1576 struct intel_iommu *iommu)
ba395927
KA		 1577{
1578 unsigned long flags;
2c2e2c38 1579 int num, ndomains;
ba395927 1580
8c11e798 1581 spin_lock_irqsave(&iommu->lock, flags);
fb170fb4
JL		 1582 if (domain_type_is_vm_or_si(domain)) {
1583 ndomains = cap_ndoms(iommu->cap);
1584 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1585 if (iommu->domains[num] == domain) {
1586 clear_bit(num, iommu->domain_ids);
1587 iommu->domains[num] = NULL;
1588 break;
1589 }
2c2e2c38 1590 }
fb170fb4
JL		 1591 } else {
1592 clear_bit(domain->id, iommu->domain_ids);
1593 iommu->domains[domain->id] = NULL;
2c2e2c38 1594 }
8c11e798 1595 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1596}
1597
fb170fb4
JL		 1598static void domain_attach_iommu(struct dmar_domain *domain,
1599 struct intel_iommu *iommu)
1600{
1601 unsigned long flags;
1602
1603 spin_lock_irqsave(&domain->iommu_lock, flags);
1604 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1605 domain->iommu_count++;
1606 if (domain->iommu_count == 1)
1607 domain->nid = iommu->node;
1608 domain_update_iommu_cap(domain);
1609 }
1610 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1611}
1612
1613static int domain_detach_iommu(struct dmar_domain *domain,
1614 struct intel_iommu *iommu)
1615{
1616 unsigned long flags;
1617 int count = INT_MAX;
1618
1619 spin_lock_irqsave(&domain->iommu_lock, flags);
1620 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1621 count = --domain->iommu_count;
1622 domain_update_iommu_cap(domain);
1623 }
1624 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1625
1626 return count;
1627}
1628
ba395927 1629static struct iova_domain reserved_iova_list;
8a443df4 1630static struct lock_class_key reserved_rbtree_key;
ba395927 1631
51a63e67 1632static int dmar_init_reserved_ranges(void)
ba395927
KA		 1633{
1634 struct pci_dev *pdev = NULL;
1635 struct iova *iova;
1636 int i;
ba395927 1637
1b722500 1638 init_iova_domain(&reserved_iova_list, IOVA_START_PFN, DMA_32BIT_PFN);
ba395927 1639
8a443df4
MG		 1640 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1641 &reserved_rbtree_key);
1642
ba395927
KA		 1643 /* IOAPIC ranges shouldn't be accessed by DMA */
1644 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1645 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1646 if (!iova) {
ba395927 1647 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1648 return -ENODEV;
1649 }
ba395927
KA		 1650
1651 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1652 for_each_pci_dev(pdev) {
1653 struct resource *r;
1654
1655 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1656 r = &pdev->resource[i];
1657 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1658 continue;
1a4a4551
DW		 1659 iova = reserve_iova(&reserved_iova_list,
1660 IOVA_PFN(r->start),
1661 IOVA_PFN(r->end));
51a63e67 1662 if (!iova) {
ba395927 1663 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1664 return -ENODEV;
1665 }
ba395927
KA		 1666 }
1667 }
51a63e67 1668 return 0;
ba395927
KA		 1669}
1670
1671static void domain_reserve_special_ranges(struct dmar_domain *domain)
1672{
1673 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1674}
1675
1676static inline int guestwidth_to_adjustwidth(int gaw)
1677{
1678 int agaw;
1679 int r = (gaw - 12) % 9;
1680
1681 if (r == 0)
1682 agaw = gaw;
1683 else
1684 agaw = gaw + 9 - r;
1685 if (agaw > 64)
1686 agaw = 64;
1687 return agaw;
1688}
1689
1690static int domain_init(struct dmar_domain *domain, int guest_width)
1691{
1692 struct intel_iommu *iommu;
1693 int adjust_width, agaw;
1694 unsigned long sagaw;
1695
1b722500 1696 init_iova_domain(&domain->iovad, IOVA_START_PFN, DMA_32BIT_PFN);
ba395927
KA		 1697 domain_reserve_special_ranges(domain);
1698
1699 /* calculate AGAW */
8c11e798 1700 iommu = domain_get_iommu(domain);
ba395927
KA		 1701 if (guest_width > cap_mgaw(iommu->cap))
1702 guest_width = cap_mgaw(iommu->cap);
1703 domain->gaw = guest_width;
1704 adjust_width = guestwidth_to_adjustwidth(guest_width);
1705 agaw = width_to_agaw(adjust_width);
1706 sagaw = cap_sagaw(iommu->cap);
1707 if (!test_bit(agaw, &sagaw)) {
1708 /* hardware doesn't support it, choose a bigger one */
1709 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1710 agaw = find_next_bit(&sagaw, 5, agaw);
1711 if (agaw >= 5)
1712 return -ENODEV;
1713 }
1714 domain->agaw = agaw;
ba395927 1715
8e604097
WH		 1716 if (ecap_coherent(iommu->ecap))
1717 domain->iommu_coherency = 1;
1718 else
1719 domain->iommu_coherency = 0;
1720
58c610bd
SY		 1721 if (ecap_sc_support(iommu->ecap))
1722 domain->iommu_snooping = 1;
1723 else
1724 domain->iommu_snooping = 0;
1725
214e39aa
DW		 1726 if (intel_iommu_superpage)
1727 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1728 else
1729 domain->iommu_superpage = 0;
1730
4c923d47 1731 domain->nid = iommu->node;
c7151a8d 1732
ba395927 1733 /* always allocate the top pgd */
4c923d47 1734 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1735 if (!domain->pgd)
1736 return -ENOMEM;
5b6985ce 1737 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1738 return 0;
1739}
1740
1741static void domain_exit(struct dmar_domain *domain)
1742{
2c2e2c38
FY		 1743 struct dmar_drhd_unit *drhd;
1744 struct intel_iommu *iommu;
ea8ea460 1745 struct page *freelist = NULL;
ba395927
KA		 1746
1747 /* Domain 0 is reserved, so dont process it */
1748 if (!domain)
1749 return;
1750
7b668357
AW		 1751 /* Flush any lazy unmaps that may reference this domain */
1752 if (!intel_iommu_strict)
1753 flush_unmaps_timeout(0);
1754
92d03cc8 1755 /* remove associated devices */
ba395927 1756 domain_remove_dev_info(domain);
92d03cc8 1757
ba395927
KA		 1758 /* destroy iovas */
1759 put_iova_domain(&domain->iovad);
ba395927 1760
ea8ea460 1761 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1762
92d03cc8 1763 /* clear attached or cached domains */
0e242612 1764 rcu_read_lock();
2c2e2c38 1765 for_each_active_iommu(iommu, drhd)
fb170fb4 1766 iommu_detach_domain(domain, iommu);
0e242612 1767 rcu_read_unlock();
2c2e2c38 1768
ea8ea460
DW		 1769 dma_free_pagelist(freelist);
1770
ba395927
KA		 1771 free_domain_mem(domain);
1772}
1773
64ae892b
DW		 1774static int domain_context_mapping_one(struct dmar_domain *domain,
1775 struct intel_iommu *iommu,
1776 u8 bus, u8 devfn, int translation)
ba395927
KA		 1777{
1778 struct context_entry *context;
ba395927 1779 unsigned long flags;
ea6606b0 1780 struct dma_pte *pgd;
ea6606b0
WH		 1781 int id;
1782 int agaw;
93a23a72 1783 struct device_domain_info *info = NULL;
ba395927
KA		 1784
1785 pr_debug("Set context mapping for %02x:%02x.%d\n",
1786 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1787
ba395927 1788 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1789 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1790 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1791
ba395927
KA		 1792 context = device_to_context_entry(iommu, bus, devfn);
1793 if (!context)
1794 return -ENOMEM;
1795 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1796 if (context_present(context)) {
ba395927
KA		 1797 spin_unlock_irqrestore(&iommu->lock, flags);
1798 return 0;
1799 }
1800
ea6606b0
WH		 1801 id = domain->id;
1802 pgd = domain->pgd;
1803
ab8dfe25 1804 if (domain_type_is_vm_or_si(domain)) {
44bde614
JL		 1805 if (domain_type_is_vm(domain)) {
1806 id = iommu_attach_vm_domain(domain, iommu);
fb170fb4 1807 if (id < 0) {
ea6606b0 1808 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4 1809 pr_err("IOMMU: no free domain ids\n");
ea6606b0
WH		 1810 return -EFAULT;
1811 }
ea6606b0
WH		 1812 }
1813
1814 /* Skip top levels of page tables for
1815 * iommu which has less agaw than default.
1672af11 1816 * Unnecessary for PT mode.
ea6606b0 1817 */
1672af11
CW		 1818 if (translation != CONTEXT_TT_PASS_THROUGH) {
1819 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1820 pgd = phys_to_virt(dma_pte_addr(pgd));
1821 if (!dma_pte_present(pgd)) {
1822 spin_unlock_irqrestore(&iommu->lock, flags);
1823 return -ENOMEM;
1824 }
ea6606b0
WH		 1825 }
1826 }
1827 }
1828
1829 context_set_domain_id(context, id);
4ed0d3e6 1830
93a23a72 1831 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1832 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1833 translation = info ? CONTEXT_TT_DEV_IOTLB :
1834 CONTEXT_TT_MULTI_LEVEL;
1835 }
4ed0d3e6
FY		 1836 /*
1837 * In pass through mode, AW must be programmed to indicate the largest
1838 * AGAW value supported by hardware. And ASR is ignored by hardware.
1839 */
93a23a72 1840 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1841 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1842 else {
1843 context_set_address_root(context, virt_to_phys(pgd));
1844 context_set_address_width(context, iommu->agaw);
1845 }
4ed0d3e6
FY		 1846
1847 context_set_translation_type(context, translation);
c07e7d21
MM		 1848 context_set_fault_enable(context);
1849 context_set_present(context);
5331fe6f 1850 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1851
4c25a2c1
DW		 1852 /*
1853 * It's a non-present to present mapping. If hardware doesn't cache
1854 * non-present entry we only need to flush the write-buffer. If the
1855 * _does_ cache non-present entries, then it does so in the special
1856 * domain #0, which we have to flush:
1857 */
1858 if (cap_caching_mode(iommu->cap)) {
1859 iommu->flush.flush_context(iommu, 0,
1860 (((u16)bus) << 8) | devfn,
1861 DMA_CCMD_MASK_NOBIT,
1862 DMA_CCMD_DEVICE_INVL);
18fd779a 1863 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1864 } else {
ba395927 1865 iommu_flush_write_buffer(iommu);
4c25a2c1 1866 }
93a23a72 1867 iommu_enable_dev_iotlb(info);
ba395927 1868 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 1869
fb170fb4
JL		 1870 domain_attach_iommu(domain, iommu);
1871
ba395927
KA		 1872 return 0;
1873}
1874
579305f7
AW		 1875struct domain_context_mapping_data {
1876 struct dmar_domain *domain;
1877 struct intel_iommu *iommu;
1878 int translation;
1879};
1880
1881static int domain_context_mapping_cb(struct pci_dev *pdev,
1882 u16 alias, void *opaque)
1883{
1884 struct domain_context_mapping_data *data = opaque;
1885
1886 return domain_context_mapping_one(data->domain, data->iommu,
1887 PCI_BUS_NUM(alias), alias & 0xff,
1888 data->translation);
1889}
1890
ba395927 1891static int
e1f167f3
DW		 1892domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1893 int translation)
ba395927 1894{
64ae892b 1895 struct intel_iommu *iommu;
156baca8 1896 u8 bus, devfn;
579305f7 1897 struct domain_context_mapping_data data;
64ae892b 1898
e1f167f3 1899 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 1900 if (!iommu)
1901 return -ENODEV;
ba395927 1902
579305f7
AW		 1903 if (!dev_is_pci(dev))
1904 return domain_context_mapping_one(domain, iommu, bus, devfn,
4ed0d3e6 1905 translation);
579305f7
AW		 1906
1907 data.domain = domain;
1908 data.iommu = iommu;
1909 data.translation = translation;
1910
1911 return pci_for_each_dma_alias(to_pci_dev(dev),
1912 &domain_context_mapping_cb, &data);
1913}
1914
1915static int domain_context_mapped_cb(struct pci_dev *pdev,
1916 u16 alias, void *opaque)
1917{
1918 struct intel_iommu *iommu = opaque;
1919
1920 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 1921}
1922
e1f167f3 1923static int domain_context_mapped(struct device *dev)
ba395927 1924{
5331fe6f 1925 struct intel_iommu *iommu;
156baca8 1926 u8 bus, devfn;
5331fe6f 1927
e1f167f3 1928 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 1929 if (!iommu)
1930 return -ENODEV;
ba395927 1931
579305f7
AW		 1932 if (!dev_is_pci(dev))
1933 return device_context_mapped(iommu, bus, devfn);
e1f167f3 1934
579305f7
AW		 1935 return !pci_for_each_dma_alias(to_pci_dev(dev),
1936 domain_context_mapped_cb, iommu);
ba395927
KA		 1937}
1938
f532959b
FY		 1939/* Returns a number of VTD pages, but aligned to MM page size */
1940static inline unsigned long aligned_nrpages(unsigned long host_addr,
1941 size_t size)
1942{
1943 host_addr &= ~PAGE_MASK;
1944 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1945}
1946
6dd9a7c7
YS		 1947/* Return largest possible superpage level for a given mapping */
1948static inline int hardware_largepage_caps(struct dmar_domain *domain,
1949 unsigned long iov_pfn,
1950 unsigned long phy_pfn,
1951 unsigned long pages)
1952{
1953 int support, level = 1;
1954 unsigned long pfnmerge;
1955
1956 support = domain->iommu_superpage;
1957
1958 /* To use a large page, the virtual *and* physical addresses
1959 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1960 of them will mean we have to use smaller pages. So just
1961 merge them and check both at once. */
1962 pfnmerge = iov_pfn | phy_pfn;
1963
1964 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1965 pages >>= VTD_STRIDE_SHIFT;
1966 if (!pages)
1967 break;
1968 pfnmerge >>= VTD_STRIDE_SHIFT;
1969 level++;
1970 support--;
1971 }
1972 return level;
1973}
1974
9051aa02
DW		 1975static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1976 struct scatterlist *sg, unsigned long phys_pfn,
1977 unsigned long nr_pages, int prot)
e1605495
DW		 1978{
1979 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1980 phys_addr_t uninitialized_var(pteval);
cc4f14aa 1981 unsigned long sg_res = 0;
6dd9a7c7
YS		 1982 unsigned int largepage_lvl = 0;
1983 unsigned long lvl_pages = 0;
e1605495 1984
162d1b10 1985 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 1986
1987 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1988 return -EINVAL;
1989
1990 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1991
cc4f14aa
JL		 1992 if (!sg) {
1993 sg_res = nr_pages;
9051aa02
DW		 1994 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1995 }
1996
6dd9a7c7 1997 while (nr_pages > 0) {
c85994e4
DW		 1998 uint64_t tmp;
1999
e1605495 2000 if (!sg_res) {
f532959b 2001 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2002 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2003 sg->dma_length = sg->length;
2004 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2005 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2006 }
6dd9a7c7 2007
e1605495 2008 if (!pte) {
6dd9a7c7
YS		 2009 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2010
5cf0a76f 2011 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2012 if (!pte)
2013 return -ENOMEM;
6dd9a7c7 2014 /* It is large page*/
6491d4d0 2015 if (largepage_lvl > 1) {
6dd9a7c7 2016 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2017 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2018 /*
2019 * Ensure that old small page tables are
2020 * removed to make room for superpage,
2021 * if they exist.
2022 */
6491d4d0 2023 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2024 iov_pfn + lvl_pages - 1);
6491d4d0 2025 } else {
6dd9a7c7 2026 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2027 }
6dd9a7c7 2028
e1605495
DW		 2029 }
2030 /* We don't need lock here, nobody else
2031 * touches the iova range
2032 */
7766a3fb 2033 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2034 if (tmp) {
1bf20f0d 2035 static int dumps = 5;
c85994e4
DW		 2036 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2037 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2038 if (dumps) {
2039 dumps--;
2040 debug_dma_dump_mappings(NULL);
2041 }
2042 WARN_ON(1);
2043 }
6dd9a7c7
YS		 2044
2045 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2046
2047 BUG_ON(nr_pages < lvl_pages);
2048 BUG_ON(sg_res < lvl_pages);
2049
2050 nr_pages -= lvl_pages;
2051 iov_pfn += lvl_pages;
2052 phys_pfn += lvl_pages;
2053 pteval += lvl_pages * VTD_PAGE_SIZE;
2054 sg_res -= lvl_pages;
2055
2056 /* If the next PTE would be the first in a new page, then we
2057 need to flush the cache on the entries we've just written.
2058 And then we'll need to recalculate 'pte', so clear it and
2059 let it get set again in the if (!pte) block above.
2060
2061 If we're done (!nr_pages) we need to flush the cache too.
2062
2063 Also if we've been setting superpages, we may need to
2064 recalculate 'pte' and switch back to smaller pages for the
2065 end of the mapping, if the trailing size is not enough to
2066 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2067 pte++;
6dd9a7c7
YS		 2068 if (!nr_pages || first_pte_in_page(pte) ||
2069 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2070 domain_flush_cache(domain, first_pte,
2071 (void *)pte - (void *)first_pte);
2072 pte = NULL;
2073 }
6dd9a7c7
YS		 2074
2075 if (!sg_res && nr_pages)
e1605495
DW		 2076 sg = sg_next(sg);
2077 }
2078 return 0;
2079}
2080
9051aa02
DW		 2081static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2082 struct scatterlist *sg, unsigned long nr_pages,
2083 int prot)
ba395927 2084{
9051aa02
DW		 2085 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2086}
6f6a00e4 2087
9051aa02
DW		 2088static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2089 unsigned long phys_pfn, unsigned long nr_pages,
2090 int prot)
2091{
2092 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2093}
2094
c7151a8d 2095static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2096{
c7151a8d
WH		 2097 if (!iommu)
2098 return;
8c11e798
WH		 2099
2100 clear_context_table(iommu, bus, devfn);
2101 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2102 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2103 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2104}
2105
109b9b04
DW		 2106static inline void unlink_domain_info(struct device_domain_info *info)
2107{
2108 assert_spin_locked(&device_domain_lock);
2109 list_del(&info->link);
2110 list_del(&info->global);
2111 if (info->dev)
0bcb3e28 2112 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2113}
2114
ba395927
KA		 2115static void domain_remove_dev_info(struct dmar_domain *domain)
2116{
3a74ca01 2117 struct device_domain_info *info, *tmp;
fb170fb4 2118 unsigned long flags;
ba395927
KA		 2119
2120 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2121 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2122 unlink_domain_info(info);
ba395927
KA		 2123 spin_unlock_irqrestore(&device_domain_lock, flags);
2124
93a23a72 2125 iommu_disable_dev_iotlb(info);
7c7faa11 2126 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2127
ab8dfe25 2128 if (domain_type_is_vm(domain)) {
7c7faa11 2129 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2130 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2131 }
2132
2133 free_devinfo_mem(info);
ba395927
KA		 2134 spin_lock_irqsave(&device_domain_lock, flags);
2135 }
2136 spin_unlock_irqrestore(&device_domain_lock, flags);
2137}
2138
2139/*
2140 * find_domain
1525a29a 2141 * Note: we use struct device->archdata.iommu stores the info
ba395927 2142 */
1525a29a 2143static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2144{
2145 struct device_domain_info *info;
2146
2147 /* No lock here, assumes no domain exit in normal case */
1525a29a 2148 info = dev->archdata.iommu;
ba395927
KA		 2149 if (info)
2150 return info->domain;
2151 return NULL;
2152}
2153
5a8f40e8 2154static inline struct device_domain_info *
745f2586
JL		 2155dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2156{
2157 struct device_domain_info *info;
2158
2159 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2160 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2161 info->devfn == devfn)
5a8f40e8 2162 return info;
745f2586
JL		 2163
2164 return NULL;
2165}
2166
5a8f40e8 2167static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2168 int bus, int devfn,
b718cd3d
DW		 2169 struct device *dev,
2170 struct dmar_domain *domain)
745f2586 2171{
5a8f40e8 2172 struct dmar_domain *found = NULL;
745f2586
JL		 2173 struct device_domain_info *info;
2174 unsigned long flags;
2175
2176 info = alloc_devinfo_mem();
2177 if (!info)
b718cd3d 2178 return NULL;
745f2586 2179
745f2586
JL		 2180 info->bus = bus;
2181 info->devfn = devfn;
2182 info->dev = dev;
2183 info->domain = domain;
5a8f40e8 2184 info->iommu = iommu;
745f2586
JL		 2185
2186 spin_lock_irqsave(&device_domain_lock, flags);
2187 if (dev)
0bcb3e28 2188 found = find_domain(dev);
5a8f40e8
DW		 2189 else {
2190 struct device_domain_info *info2;
41e80dca 2191 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2192 if (info2)
2193 found = info2->domain;
2194 }
745f2586
JL		 2195 if (found) {
2196 spin_unlock_irqrestore(&device_domain_lock, flags);
2197 free_devinfo_mem(info);
b718cd3d
DW		 2198 /* Caller must free the original domain */
2199 return found;
745f2586
JL		 2200 }
2201
b718cd3d
DW		 2202 list_add(&info->link, &domain->devices);
2203 list_add(&info->global, &device_domain_list);
2204 if (dev)
2205 dev->archdata.iommu = info;
2206 spin_unlock_irqrestore(&device_domain_lock, flags);
2207
2208 return domain;
745f2586
JL		 2209}
2210
579305f7
AW		 2211static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2212{
2213 *(u16 *)opaque = alias;
2214 return 0;
2215}
2216
ba395927 2217/* domain is initialized */
146922ec 2218static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2219{
579305f7
AW		 2220 struct dmar_domain *domain, *tmp;
2221 struct intel_iommu *iommu;
5a8f40e8 2222 struct device_domain_info *info;
579305f7 2223 u16 dma_alias;
ba395927 2224 unsigned long flags;
aa4d066a 2225 u8 bus, devfn;
ba395927 2226
146922ec 2227 domain = find_domain(dev);
ba395927
KA		 2228 if (domain)
2229 return domain;
2230
579305f7
AW		 2231 iommu = device_to_iommu(dev, &bus, &devfn);
2232 if (!iommu)
2233 return NULL;
2234
146922ec
DW		 2235 if (dev_is_pci(dev)) {
2236 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2237
579305f7
AW		 2238 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2239
2240 spin_lock_irqsave(&device_domain_lock, flags);
2241 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2242 PCI_BUS_NUM(dma_alias),
2243 dma_alias & 0xff);
2244 if (info) {
2245 iommu = info->iommu;
2246 domain = info->domain;
5a8f40e8 2247 }
579305f7 2248 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2249
579305f7
AW		 2250 /* DMA alias already has a domain, uses it */
2251 if (info)
2252 goto found_domain;
2253 }
ba395927 2254
146922ec 2255 /* Allocate and initialize new domain for the device */
ab8dfe25 2256 domain = alloc_domain(0);
745f2586 2257 if (!domain)
579305f7 2258 return NULL;
44bde614
JL		 2259 domain->id = iommu_attach_domain(domain, iommu);
2260 if (domain->id < 0) {
2fe9723d 2261 free_domain_mem(domain);
579305f7 2262 return NULL;
2c2e2c38 2263 }
fb170fb4 2264 domain_attach_iommu(domain, iommu);
579305f7
AW		 2265 if (domain_init(domain, gaw)) {
2266 domain_exit(domain);
2267 return NULL;
2c2e2c38 2268 }
ba395927 2269
579305f7
AW		 2270 /* register PCI DMA alias device */
2271 if (dev_is_pci(dev)) {
2272 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2273 dma_alias & 0xff, NULL, domain);
2274
2275 if (!tmp || tmp != domain) {
2276 domain_exit(domain);
2277 domain = tmp;
2278 }
2279
b718cd3d 2280 if (!domain)
579305f7 2281 return NULL;
ba395927
KA		 2282 }
2283
2284found_domain:
579305f7
AW		 2285 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2286
2287 if (!tmp || tmp != domain) {
2288 domain_exit(domain);
2289 domain = tmp;
2290 }
b718cd3d
DW		 2291
2292 return domain;
ba395927
KA		 2293}
2294
2c2e2c38 2295static int iommu_identity_mapping;
e0fc7e0b
DW		 2296#define IDENTMAP_ALL 1
2297#define IDENTMAP_GFX 2
2298#define IDENTMAP_AZALIA 4
2c2e2c38 2299
b213203e
DW		 2300static int iommu_domain_identity_map(struct dmar_domain *domain,
2301 unsigned long long start,
2302 unsigned long long end)
ba395927 2303{
c5395d5c
DW		 2304 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2305 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2306
2307 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2308 dma_to_mm_pfn(last_vpfn))) {
ba395927 2309 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2310 return -ENOMEM;
ba395927
KA		 2311 }
2312
c5395d5c
DW		 2313 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2314 start, end, domain->id);
ba395927
KA		 2315 /*
2316 * RMRR range might have overlap with physical memory range,
2317 * clear it first
2318 */
c5395d5c 2319 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2320
c5395d5c
DW		 2321 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2322 last_vpfn - first_vpfn + 1,
61df7443 2323 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2324}
2325
0b9d9753 2326static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2327 unsigned long long start,
2328 unsigned long long end)
2329{
2330 struct dmar_domain *domain;
2331 int ret;
2332
0b9d9753 2333 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2334 if (!domain)
2335 return -ENOMEM;
2336
19943b0e
DW		 2337 /* For _hardware_ passthrough, don't bother. But for software
2338 passthrough, we do it anyway -- it may indicate a memory
2339 range which is reserved in E820, so which didn't get set
2340 up to start with in si_domain */
2341 if (domain == si_domain && hw_pass_through) {
2342 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
0b9d9753 2343 dev_name(dev), start, end);
19943b0e
DW		 2344 return 0;
2345 }
2346
2347 printk(KERN_INFO
2348 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
0b9d9753 2349 dev_name(dev), start, end);
2ff729f5 2350
5595b528
DW		 2351 if (end < start) {
2352 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2353 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2354 dmi_get_system_info(DMI_BIOS_VENDOR),
2355 dmi_get_system_info(DMI_BIOS_VERSION),
2356 dmi_get_system_info(DMI_PRODUCT_VERSION));
2357 ret = -EIO;
2358 goto error;
2359 }
2360
2ff729f5
DW		 2361 if (end >> agaw_to_width(domain->agaw)) {
2362 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2363 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2364 agaw_to_width(domain->agaw),
2365 dmi_get_system_info(DMI_BIOS_VENDOR),
2366 dmi_get_system_info(DMI_BIOS_VERSION),
2367 dmi_get_system_info(DMI_PRODUCT_VERSION));
2368 ret = -EIO;
2369 goto error;
2370 }
19943b0e 2371
b213203e 2372 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2373 if (ret)
2374 goto error;
2375
2376 /* context entry init */
0b9d9753 2377 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2378 if (ret)
2379 goto error;
2380
2381 return 0;
2382
2383 error:
ba395927
KA		 2384 domain_exit(domain);
2385 return ret;
ba395927
KA		 2386}
2387
2388static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2389 struct device *dev)
ba395927 2390{
0b9d9753 2391 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2392 return 0;
0b9d9753
DW		 2393 return iommu_prepare_identity_map(dev, rmrr->base_address,
2394 rmrr->end_address);
ba395927
KA		 2395}
2396
d3f13810 2397#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2398static inline void iommu_prepare_isa(void)
2399{
2400 struct pci_dev *pdev;
2401 int ret;
2402
2403 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2404 if (!pdev)
2405 return;
2406
c7ab48d2 2407 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2408 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2409
2410 if (ret)
c7ab48d2
DW		 2411 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2412 "floppy might not work\n");
49a0429e 2413
9b27e82d 2414 pci_dev_put(pdev);
49a0429e
KA		 2415}
2416#else
2417static inline void iommu_prepare_isa(void)
2418{
2419 return;
2420}
d3f13810 2421#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2422
2c2e2c38 2423static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2424
071e1374 2425static int __init si_domain_init(int hw)
2c2e2c38
FY		 2426{
2427 struct dmar_drhd_unit *drhd;
2428 struct intel_iommu *iommu;
c7ab48d2 2429 int nid, ret = 0;
44bde614 2430 bool first = true;
2c2e2c38 2431
ab8dfe25 2432 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2433 if (!si_domain)
2434 return -EFAULT;
2435
2c2e2c38
FY		 2436 for_each_active_iommu(iommu, drhd) {
2437 ret = iommu_attach_domain(si_domain, iommu);
fb170fb4 2438 if (ret < 0) {
2c2e2c38
FY		 2439 domain_exit(si_domain);
2440 return -EFAULT;
44bde614
JL		 2441 } else if (first) {
2442 si_domain->id = ret;
2443 first = false;
2444 } else if (si_domain->id != ret) {
2445 domain_exit(si_domain);
2446 return -EFAULT;
2c2e2c38 2447 }
fb170fb4 2448 domain_attach_iommu(si_domain, iommu);
2c2e2c38
FY		 2449 }
2450
2451 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2452 domain_exit(si_domain);
2453 return -EFAULT;
2454 }
2455
9544c003
JL		 2456 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2457 si_domain->id);
2c2e2c38 2458
19943b0e
DW		 2459 if (hw)
2460 return 0;
2461
c7ab48d2 2462 for_each_online_node(nid) {
5dfe8660
TH		 2463 unsigned long start_pfn, end_pfn;
2464 int i;
2465
2466 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2467 ret = iommu_domain_identity_map(si_domain,
2468 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2469 if (ret)
2470 return ret;
2471 }
c7ab48d2
DW		 2472 }
2473
2c2e2c38
FY		 2474 return 0;
2475}
2476
9b226624 2477static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2478{
2479 struct device_domain_info *info;
2480
2481 if (likely(!iommu_identity_mapping))
2482 return 0;
2483
9b226624 2484 info = dev->archdata.iommu;
cb452a40
MT		 2485 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2486 return (info->domain == si_domain);
2c2e2c38 2487
2c2e2c38
FY		 2488 return 0;
2489}
2490
2491static int domain_add_dev_info(struct dmar_domain *domain,
5913c9bf 2492 struct device *dev, int translation)
2c2e2c38 2493{
0ac72664 2494 struct dmar_domain *ndomain;
5a8f40e8 2495 struct intel_iommu *iommu;
156baca8 2496 u8 bus, devfn;
5fe60f4e 2497 int ret;
2c2e2c38 2498
5913c9bf 2499 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2500 if (!iommu)
2501 return -ENODEV;
2502
5913c9bf 2503 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2504 if (ndomain != domain)
2505 return -EBUSY;
2c2e2c38 2506
5913c9bf 2507 ret = domain_context_mapping(domain, dev, translation);
e2ad23d0 2508 if (ret) {
5913c9bf 2509 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2510 return ret;
2511 }
2512
2c2e2c38
FY		 2513 return 0;
2514}
2515
0b9d9753 2516static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2517{
2518 struct dmar_rmrr_unit *rmrr;
832bd858 2519 struct device *tmp;
ea2447f7
TM		 2520 int i;
2521
0e242612 2522 rcu_read_lock();
ea2447f7 2523 for_each_rmrr_units(rmrr) {
b683b230
JL		 2524 /*
2525 * Return TRUE if this RMRR contains the device that
2526 * is passed in.
2527 */
2528 for_each_active_dev_scope(rmrr->devices,
2529 rmrr->devices_cnt, i, tmp)
0b9d9753 2530 if (tmp == dev) {
0e242612 2531 rcu_read_unlock();
ea2447f7 2532 return true;
b683b230 2533 }
ea2447f7 2534 }
0e242612 2535 rcu_read_unlock();
ea2447f7
TM		 2536 return false;
2537}
2538
c875d2c1
AW		 2539/*
2540 * There are a couple cases where we need to restrict the functionality of
2541 * devices associated with RMRRs. The first is when evaluating a device for
2542 * identity mapping because problems exist when devices are moved in and out
2543 * of domains and their respective RMRR information is lost. This means that
2544 * a device with associated RMRRs will never be in a "passthrough" domain.
2545 * The second is use of the device through the IOMMU API. This interface
2546 * expects to have full control of the IOVA space for the device. We cannot
2547 * satisfy both the requirement that RMRR access is maintained and have an
2548 * unencumbered IOVA space. We also have no ability to quiesce the device's
2549 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2550 * We therefore prevent devices associated with an RMRR from participating in
2551 * the IOMMU API, which eliminates them from device assignment.
2552 *
2553 * In both cases we assume that PCI USB devices with RMRRs have them largely
2554 * for historical reasons and that the RMRR space is not actively used post
2555 * boot. This exclusion may change if vendors begin to abuse it.
2556 */
2557static bool device_is_rmrr_locked(struct device *dev)
2558{
2559 if (!device_has_rmrr(dev))
2560 return false;
2561
2562 if (dev_is_pci(dev)) {
2563 struct pci_dev *pdev = to_pci_dev(dev);
2564
2565 if ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
2566 return false;
2567 }
2568
2569 return true;
2570}
2571
3bdb2591 2572static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2573{
ea2447f7 2574
3bdb2591
DW		 2575 if (dev_is_pci(dev)) {
2576 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2577
c875d2c1 2578 if (device_is_rmrr_locked(dev))
3bdb2591 2579 return 0;
e0fc7e0b 2580
3bdb2591
DW		 2581 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2582 return 1;
e0fc7e0b 2583
3bdb2591
DW		 2584 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2585 return 1;
6941af28 2586
3bdb2591 2587 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2588 return 0;
3bdb2591
DW		 2589
2590 /*
2591 * We want to start off with all devices in the 1:1 domain, and
2592 * take them out later if we find they can't access all of memory.
2593 *
2594 * However, we can't do this for PCI devices behind bridges,
2595 * because all PCI devices behind the same bridge will end up
2596 * with the same source-id on their transactions.
2597 *
2598 * Practically speaking, we can't change things around for these
2599 * devices at run-time, because we can't be sure there'll be no
2600 * DMA transactions in flight for any of their siblings.
2601 *
2602 * So PCI devices (unless they're on the root bus) as well as
2603 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2604 * the 1:1 domain, just in _case_ one of their siblings turns out
2605 * not to be able to map all of memory.
2606 */
2607 if (!pci_is_pcie(pdev)) {
2608 if (!pci_is_root_bus(pdev->bus))
2609 return 0;
2610 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2611 return 0;
2612 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2613 return 0;
3bdb2591
DW		 2614 } else {
2615 if (device_has_rmrr(dev))
2616 return 0;
2617 }
3dfc813d 2618
3bdb2591 2619 /*
3dfc813d 2620 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2621 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2622 * take them out of the 1:1 domain later.
2623 */
8fcc5372
CW		 2624 if (!startup) {
2625 /*
2626 * If the device's dma_mask is less than the system's memory
2627 * size then this is not a candidate for identity mapping.
2628 */
3bdb2591 2629 u64 dma_mask = *dev->dma_mask;
8fcc5372 2630
3bdb2591
DW		 2631 if (dev->coherent_dma_mask &&
2632 dev->coherent_dma_mask < dma_mask)
2633 dma_mask = dev->coherent_dma_mask;
8fcc5372 2634
3bdb2591 2635 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2636 }
6941af28
DW		 2637
2638 return 1;
2639}
2640
cf04eee8
DW		 2641static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2642{
2643 int ret;
2644
2645 if (!iommu_should_identity_map(dev, 1))
2646 return 0;
2647
2648 ret = domain_add_dev_info(si_domain, dev,
2649 hw ? CONTEXT_TT_PASS_THROUGH :
2650 CONTEXT_TT_MULTI_LEVEL);
2651 if (!ret)
2652 pr_info("IOMMU: %s identity mapping for device %s\n",
2653 hw ? "hardware" : "software", dev_name(dev));
2654 else if (ret == -ENODEV)
2655 /* device not associated with an iommu */
2656 ret = 0;
2657
2658 return ret;
2659}
2660
2661
071e1374 2662static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2663{
2c2e2c38 2664 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2665 struct dmar_drhd_unit *drhd;
2666 struct intel_iommu *iommu;
2667 struct device *dev;
2668 int i;
2669 int ret = 0;
2c2e2c38 2670
19943b0e 2671 ret = si_domain_init(hw);
2c2e2c38
FY		 2672 if (ret)
2673 return -EFAULT;
2674
2c2e2c38 2675 for_each_pci_dev(pdev) {
cf04eee8
DW		 2676 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2677 if (ret)
2678 return ret;
2679 }
2680
2681 for_each_active_iommu(iommu, drhd)
2682 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2683 struct acpi_device_physical_node *pn;
2684 struct acpi_device *adev;
2685
2686 if (dev->bus != &acpi_bus_type)
2687 continue;
2688
2689 adev= to_acpi_device(dev);
2690 mutex_lock(&adev->physical_node_lock);
2691 list_for_each_entry(pn, &adev->physical_node_list, node) {
2692 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2693 if (ret)
2694 break;
eae460b6 2695 }
cf04eee8
DW		 2696 mutex_unlock(&adev->physical_node_lock);
2697 if (ret)
2698 return ret;
62edf5dc 2699 }
2c2e2c38
FY		 2700
2701 return 0;
2702}
2703
ffebeb46
JL		 2704static void intel_iommu_init_qi(struct intel_iommu *iommu)
2705{
2706 /*
2707 * Start from the sane iommu hardware state.
2708 * If the queued invalidation is already initialized by us
2709 * (for example, while enabling interrupt-remapping) then
2710 * we got the things already rolling from a sane state.
2711 */
2712 if (!iommu->qi) {
2713 /*
2714 * Clear any previous faults.
2715 */
2716 dmar_fault(-1, iommu);
2717 /*
2718 * Disable queued invalidation if supported and already enabled
2719 * before OS handover.
2720 */
2721 dmar_disable_qi(iommu);
2722 }
2723
2724 if (dmar_enable_qi(iommu)) {
2725 /*
2726 * Queued Invalidate not enabled, use Register Based Invalidate
2727 */
2728 iommu->flush.flush_context = __iommu_flush_context;
2729 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2730 pr_info("IOMMU: %s using Register based invalidation\n",
2731 iommu->name);
2732 } else {
2733 iommu->flush.flush_context = qi_flush_context;
2734 iommu->flush.flush_iotlb = qi_flush_iotlb;
2735 pr_info("IOMMU: %s using Queued invalidation\n", iommu->name);
2736 }
2737}
2738
b779260b 2739static int __init init_dmars(void)
ba395927
KA		 2740{
2741 struct dmar_drhd_unit *drhd;
2742 struct dmar_rmrr_unit *rmrr;
832bd858 2743 struct device *dev;
ba395927 2744 struct intel_iommu *iommu;
9d783ba0 2745 int i, ret;
2c2e2c38 2746
ba395927
KA		 2747 /*
2748 * for each drhd
2749 * allocate root
2750 * initialize and program root entry to not present
2751 * endfor
2752 */
2753 for_each_drhd_unit(drhd) {
5e0d2a6f 2754 /*
2755 * lock not needed as this is only incremented in the single
2756 * threaded kernel __init code path all other access are read
2757 * only
2758 */
78d8e704 2759 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 2760 g_num_of_iommus++;
2761 continue;
2762 }
2763 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
78d8e704 2764 DMAR_UNITS_SUPPORTED);
5e0d2a6f 2765 }
2766
ffebeb46
JL		 2767 /* Preallocate enough resources for IOMMU hot-addition */
2768 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
2769 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
2770
d9630fe9
WH		 2771 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2772 GFP_KERNEL);
2773 if (!g_iommus) {
2774 printk(KERN_ERR "Allocating global iommu array failed\n");
2775 ret = -ENOMEM;
2776 goto error;
2777 }
2778
80b20dd8 2779 deferred_flush = kzalloc(g_num_of_iommus *
2780 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2781 if (!deferred_flush) {
5e0d2a6f 2782 ret = -ENOMEM;
989d51fc 2783 goto free_g_iommus;
5e0d2a6f 2784 }
2785
7c919779 2786 for_each_active_iommu(iommu, drhd) {
d9630fe9 2787 g_iommus[iommu->seq_id] = iommu;
ba395927 2788
e61d98d8
SS		 2789 ret = iommu_init_domains(iommu);
2790 if (ret)
989d51fc 2791 goto free_iommu;
e61d98d8 2792
ba395927
KA		 2793 /*
2794 * TBD:
2795 * we could share the same root & context tables
25985edc 2796 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2797 */
2798 ret = iommu_alloc_root_entry(iommu);
ffebeb46 2799 if (ret)
989d51fc 2800 goto free_iommu;
4ed0d3e6 2801 if (!ecap_pass_through(iommu->ecap))
19943b0e 2802 hw_pass_through = 0;
ba395927
KA		 2803 }
2804
ffebeb46
JL		 2805 for_each_active_iommu(iommu, drhd)
2806 intel_iommu_init_qi(iommu);
a77b67d4 2807
19943b0e 2808 if (iommu_pass_through)
e0fc7e0b
DW		 2809 iommu_identity_mapping |= IDENTMAP_ALL;
2810
d3f13810 2811#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2812 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2813#endif
e0fc7e0b
DW		 2814
2815 check_tylersburg_isoch();
2816
ba395927 2817 /*
19943b0e
DW		 2818 * If pass through is not set or not enabled, setup context entries for
2819 * identity mappings for rmrr, gfx, and isa and may fall back to static
2820 * identity mapping if iommu_identity_mapping is set.
ba395927 2821 */
19943b0e
DW		 2822 if (iommu_identity_mapping) {
2823 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2824 if (ret) {
19943b0e 2825 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
989d51fc 2826 goto free_iommu;
ba395927
KA		 2827 }
2828 }
ba395927 2829 /*
19943b0e
DW		 2830 * For each rmrr
2831 * for each dev attached to rmrr
2832 * do
2833 * locate drhd for dev, alloc domain for dev
2834 * allocate free domain
2835 * allocate page table entries for rmrr
2836 * if context not allocated for bus
2837 * allocate and init context
2838 * set present in root table for this bus
2839 * init context with domain, translation etc
2840 * endfor
2841 * endfor
ba395927 2842 */
19943b0e
DW		 2843 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2844 for_each_rmrr_units(rmrr) {
b683b230
JL		 2845 /* some BIOS lists non-exist devices in DMAR table. */
2846 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 2847 i, dev) {
0b9d9753 2848 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e
DW		 2849 if (ret)
2850 printk(KERN_ERR
2851 "IOMMU: mapping reserved region failed\n");
ba395927 2852 }
4ed0d3e6 2853 }
49a0429e 2854
19943b0e
DW		 2855 iommu_prepare_isa();
2856
ba395927
KA		 2857 /*
2858 * for each drhd
2859 * enable fault log
2860 * global invalidate context cache
2861 * global invalidate iotlb
2862 * enable translation
2863 */
7c919779 2864 for_each_iommu(iommu, drhd) {
51a63e67
JC		 2865 if (drhd->ignored) {
2866 /*
2867 * we always have to disable PMRs or DMA may fail on
2868 * this device
2869 */
2870 if (force_on)
7c919779 2871 iommu_disable_protect_mem_regions(iommu);
ba395927 2872 continue;
51a63e67 2873 }
ba395927
KA		 2874
2875 iommu_flush_write_buffer(iommu);
2876
3460a6d9
KA		 2877 ret = dmar_set_interrupt(iommu);
2878 if (ret)
989d51fc 2879 goto free_iommu;
3460a6d9 2880
ba395927
KA		 2881 iommu_set_root_entry(iommu);
2882
4c25a2c1 2883 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2884 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2a41ccee 2885 iommu_enable_translation(iommu);
b94996c9 2886 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2887 }
2888
2889 return 0;
989d51fc
JL		 2890
2891free_iommu:
ffebeb46
JL		 2892 for_each_active_iommu(iommu, drhd) {
2893 disable_dmar_iommu(iommu);
a868e6b7 2894 free_dmar_iommu(iommu);
ffebeb46 2895 }
9bdc531e 2896 kfree(deferred_flush);
989d51fc 2897free_g_iommus:
d9630fe9 2898 kfree(g_iommus);
989d51fc 2899error:
ba395927
KA		 2900 return ret;
2901}
2902
5a5e02a6 2903/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2904static struct iova *intel_alloc_iova(struct device *dev,
2905 struct dmar_domain *domain,
2906 unsigned long nrpages, uint64_t dma_mask)
ba395927 2907{
ba395927 2908 struct iova *iova = NULL;
ba395927 2909
875764de
DW		 2910 /* Restrict dma_mask to the width that the iommu can handle */
2911 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2912
2913 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2914 /*
2915 * First try to allocate an io virtual address in
284901a9 2916 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2917 * from higher range
ba395927 2918 */
875764de
DW		 2919 iova = alloc_iova(&domain->iovad, nrpages,
2920 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2921 if (iova)
2922 return iova;
2923 }
2924 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2925 if (unlikely(!iova)) {
2926 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
207e3592 2927 nrpages, dev_name(dev));
f76aec76
KA		 2928 return NULL;
2929 }
2930
2931 return iova;
2932}
2933
d4b709f4 2934static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 2935{
2936 struct dmar_domain *domain;
2937 int ret;
2938
d4b709f4 2939 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 2940 if (!domain) {
d4b709f4
DW		 2941 printk(KERN_ERR "Allocating domain for %s failed",
2942 dev_name(dev));
4fe05bbc 2943 return NULL;
ba395927
KA		 2944 }
2945
2946 /* make sure context mapping is ok */
d4b709f4
DW		 2947 if (unlikely(!domain_context_mapped(dev))) {
2948 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
f76aec76 2949 if (ret) {
d4b709f4
DW		 2950 printk(KERN_ERR "Domain context map for %s failed",
2951 dev_name(dev));
4fe05bbc 2952 return NULL;
f76aec76 2953 }
ba395927
KA		 2954 }
2955
f76aec76
KA		 2956 return domain;
2957}
2958
d4b709f4 2959static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 2960{
2961 struct device_domain_info *info;
2962
2963 /* No lock here, assumes no domain exit in normal case */
d4b709f4 2964 info = dev->archdata.iommu;
147202aa
DW		 2965 if (likely(info))
2966 return info->domain;
2967
2968 return __get_valid_domain_for_dev(dev);
2969}
2970
3d89194a 2971static int iommu_dummy(struct device *dev)
2c2e2c38 2972{
3d89194a 2973 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2c2e2c38
FY		 2974}
2975
ecb509ec 2976/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 2977static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 2978{
2979 int found;
2980
3d89194a 2981 if (iommu_dummy(dev))
1e4c64c4
DW		 2982 return 1;
2983
2c2e2c38 2984 if (!iommu_identity_mapping)
1e4c64c4 2985 return 0;
2c2e2c38 2986
9b226624 2987 found = identity_mapping(dev);
2c2e2c38 2988 if (found) {
ecb509ec 2989 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 2990 return 1;
2991 else {
2992 /*
2993 * 32 bit DMA is removed from si_domain and fall back
2994 * to non-identity mapping.
2995 */
bf9c9eda 2996 domain_remove_one_dev_info(si_domain, dev);
2c2e2c38 2997 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
ecb509ec 2998 dev_name(dev));
2c2e2c38
FY		 2999 return 0;
3000 }
3001 } else {
3002 /*
3003 * In case of a detached 64 bit DMA device from vm, the device
3004 * is put into si_domain for identity mapping.
3005 */
ecb509ec 3006 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3007 int ret;
5913c9bf 3008 ret = domain_add_dev_info(si_domain, dev,
5fe60f4e
DW		 3009 hw_pass_through ?
3010 CONTEXT_TT_PASS_THROUGH :
3011 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 3012 if (!ret) {
3013 printk(KERN_INFO "64bit %s uses identity mapping\n",
ecb509ec 3014 dev_name(dev));
2c2e2c38
FY		 3015 return 1;
3016 }
3017 }
3018 }
3019
1e4c64c4 3020 return 0;
2c2e2c38
FY		 3021}
3022
5040a918 3023static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3024 size_t size, int dir, u64 dma_mask)
f76aec76 3025{
f76aec76 3026 struct dmar_domain *domain;
5b6985ce 3027 phys_addr_t start_paddr;
f76aec76
KA		 3028 struct iova *iova;
3029 int prot = 0;
6865f0d1 3030 int ret;
8c11e798 3031 struct intel_iommu *iommu;
33041ec0 3032 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3033
3034 BUG_ON(dir == DMA_NONE);
2c2e2c38 3035
5040a918 3036 if (iommu_no_mapping(dev))
6865f0d1 3037 return paddr;
f76aec76 3038
5040a918 3039 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3040 if (!domain)
3041 return 0;
3042
8c11e798 3043 iommu = domain_get_iommu(domain);
88cb6a74 3044 size = aligned_nrpages(paddr, size);
f76aec76 3045
5040a918 3046 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3047 if (!iova)
3048 goto error;
3049
ba395927
KA		 3050 /*
3051 * Check if DMAR supports zero-length reads on write only
3052 * mappings..
3053 */
3054 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3055 !cap_zlr(iommu->cap))
ba395927
KA		 3056 prot |= DMA_PTE_READ;
3057 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3058 prot |= DMA_PTE_WRITE;
3059 /*
6865f0d1 3060 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3061 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3062 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3063 * is not a big problem
3064 */
0ab36de2 3065 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3066 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3067 if (ret)
3068 goto error;
3069
1f0ef2aa
DW		 3070 /* it's a non-present to present mapping. Only flush if caching mode */
3071 if (cap_caching_mode(iommu->cap))
ea8ea460 3072 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3073 else
8c11e798 3074 iommu_flush_write_buffer(iommu);
f76aec76 3075
03d6a246
DW		 3076 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3077 start_paddr += paddr & ~PAGE_MASK;
3078 return start_paddr;
ba395927 3079
ba395927 3080error:
f76aec76
KA		 3081 if (iova)
3082 __free_iova(&domain->iovad, iova);
4cf2e75d 3083 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3084 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3085 return 0;
3086}
3087
ffbbef5c
FT		 3088static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3089 unsigned long offset, size_t size,
3090 enum dma_data_direction dir,
3091 struct dma_attrs *attrs)
bb9e6d65 3092{
ffbbef5c 3093 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3094 dir, *dev->dma_mask);
bb9e6d65
FT		 3095}
3096
5e0d2a6f 3097static void flush_unmaps(void)
3098{
80b20dd8 3099 int i, j;
5e0d2a6f 3100
5e0d2a6f 3101 timer_on = 0;
3102
3103 /* just flush them all */
3104 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3105 struct intel_iommu *iommu = g_iommus[i];
3106 if (!iommu)
3107 continue;
c42d9f32 3108
9dd2fe89
YZ		 3109 if (!deferred_flush[i].next)
3110 continue;
3111
78d5f0f5
NA		 3112 /* In caching mode, global flushes turn emulation expensive */
3113 if (!cap_caching_mode(iommu->cap))
3114 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3115 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3116 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3117 unsigned long mask;
3118 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3119 struct dmar_domain *domain = deferred_flush[i].domain[j];
3120
3121 /* On real hardware multiple invalidations are expensive */
3122 if (cap_caching_mode(iommu->cap))
3123 iommu_flush_iotlb_psi(iommu, domain->id,
a156ef99 3124 iova->pfn_lo, iova_size(iova),
ea8ea460 3125 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3126 else {
a156ef99 3127 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3128 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3129 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3130 }
93a23a72 3131 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3132 if (deferred_flush[i].freelist[j])
3133 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3134 }
9dd2fe89 3135 deferred_flush[i].next = 0;
5e0d2a6f 3136 }
3137
5e0d2a6f 3138 list_size = 0;
5e0d2a6f 3139}
3140
3141static void flush_unmaps_timeout(unsigned long data)
3142{
80b20dd8 3143 unsigned long flags;
3144
3145 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3146 flush_unmaps();
80b20dd8 3147 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3148}
3149
ea8ea460 3150static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3151{
3152 unsigned long flags;
80b20dd8 3153 int next, iommu_id;
8c11e798 3154 struct intel_iommu *iommu;
5e0d2a6f 3155
3156 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3157 if (list_size == HIGH_WATER_MARK)
3158 flush_unmaps();
3159
8c11e798
WH		 3160 iommu = domain_get_iommu(dom);
3161 iommu_id = iommu->seq_id;
c42d9f32 3162
80b20dd8 3163 next = deferred_flush[iommu_id].next;
3164 deferred_flush[iommu_id].domain[next] = dom;
3165 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3166 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3167 deferred_flush[iommu_id].next++;
5e0d2a6f 3168
3169 if (!timer_on) {
3170 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3171 timer_on = 1;
3172 }
3173 list_size++;
3174 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3175}
3176
d41a4adb 3177static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3178{
f76aec76 3179 struct dmar_domain *domain;
d794dc9b 3180 unsigned long start_pfn, last_pfn;
ba395927 3181 struct iova *iova;
8c11e798 3182 struct intel_iommu *iommu;
ea8ea460 3183 struct page *freelist;
ba395927 3184
73676832 3185 if (iommu_no_mapping(dev))
f76aec76 3186 return;
2c2e2c38 3187
1525a29a 3188 domain = find_domain(dev);
ba395927
KA		 3189 BUG_ON(!domain);
3190
8c11e798
WH		 3191 iommu = domain_get_iommu(domain);
3192
ba395927 3193 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3194 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3195 (unsigned long long)dev_addr))
ba395927 3196 return;
ba395927 3197
d794dc9b
DW		 3198 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3199 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3200
d794dc9b 3201 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3202 dev_name(dev), start_pfn, last_pfn);
ba395927 3203
ea8ea460 3204 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3205
5e0d2a6f 3206 if (intel_iommu_strict) {
03d6a246 3207 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3208 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3209 /* free iova */
3210 __free_iova(&domain->iovad, iova);
ea8ea460 3211 dma_free_pagelist(freelist);
5e0d2a6f 3212 } else {
ea8ea460 3213 add_unmap(domain, iova, freelist);
5e0d2a6f 3214 /*
3215 * queue up the release of the unmap to save the 1/6th of the
3216 * cpu used up by the iotlb flush operation...
3217 */
5e0d2a6f 3218 }
ba395927
KA		 3219}
3220
d41a4adb
JL		 3221static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3222 size_t size, enum dma_data_direction dir,
3223 struct dma_attrs *attrs)
3224{
3225 intel_unmap(dev, dev_addr);
3226}
3227
5040a918 3228static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3229 dma_addr_t *dma_handle, gfp_t flags,
3230 struct dma_attrs *attrs)
ba395927 3231{
36746436 3232 struct page *page = NULL;
ba395927
KA		 3233 int order;
3234
5b6985ce 3235 size = PAGE_ALIGN(size);
ba395927 3236 order = get_order(size);
e8bb910d 3237
5040a918 3238 if (!iommu_no_mapping(dev))
e8bb910d 3239 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3240 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3241 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3242 flags |= GFP_DMA;
3243 else
3244 flags |= GFP_DMA32;
3245 }
ba395927 3246
36746436
AM		 3247 if (flags & __GFP_WAIT) {
3248 unsigned int count = size >> PAGE_SHIFT;
3249
3250 page = dma_alloc_from_contiguous(dev, count, order);
3251 if (page && iommu_no_mapping(dev) &&
3252 page_to_phys(page) + size > dev->coherent_dma_mask) {
3253 dma_release_from_contiguous(dev, page, count);
3254 page = NULL;
3255 }
3256 }
3257
3258 if (!page)
3259 page = alloc_pages(flags, order);
3260 if (!page)
ba395927 3261 return NULL;
36746436 3262 memset(page_address(page), 0, size);
ba395927 3263
36746436 3264 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3265 DMA_BIDIRECTIONAL,
5040a918 3266 dev->coherent_dma_mask);
ba395927 3267 if (*dma_handle)
36746436
AM		 3268 return page_address(page);
3269 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3270 __free_pages(page, order);
3271
ba395927
KA		 3272 return NULL;
3273}
3274
5040a918 3275static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3276 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3277{
3278 int order;
36746436 3279 struct page *page = virt_to_page(vaddr);
ba395927 3280
5b6985ce 3281 size = PAGE_ALIGN(size);
ba395927
KA		 3282 order = get_order(size);
3283
d41a4adb 3284 intel_unmap(dev, dma_handle);
36746436
AM		 3285 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3286 __free_pages(page, order);
ba395927
KA		 3287}
3288
5040a918 3289static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3290 int nelems, enum dma_data_direction dir,
3291 struct dma_attrs *attrs)
ba395927 3292{
d41a4adb 3293 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3294}
3295
ba395927 3296static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3297 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3298{
3299 int i;
c03ab37c 3300 struct scatterlist *sg;
ba395927 3301
c03ab37c 3302 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3303 BUG_ON(!sg_page(sg));
4cf2e75d 3304 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3305 sg->dma_length = sg->length;
ba395927
KA		 3306 }
3307 return nelems;
3308}
3309
5040a918 3310static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3311 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3312{
ba395927 3313 int i;
ba395927 3314 struct dmar_domain *domain;
f76aec76
KA		 3315 size_t size = 0;
3316 int prot = 0;
f76aec76
KA		 3317 struct iova *iova = NULL;
3318 int ret;
c03ab37c 3319 struct scatterlist *sg;
b536d24d 3320 unsigned long start_vpfn;
8c11e798 3321 struct intel_iommu *iommu;
ba395927
KA		 3322
3323 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3324 if (iommu_no_mapping(dev))
3325 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3326
5040a918 3327 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3328 if (!domain)
3329 return 0;
3330
8c11e798
WH		 3331 iommu = domain_get_iommu(domain);
3332
b536d24d 3333 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3334 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3335
5040a918
DW		 3336 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3337 *dev->dma_mask);
f76aec76 3338 if (!iova) {
c03ab37c 3339 sglist->dma_length = 0;
f76aec76
KA		 3340 return 0;
3341 }
3342
3343 /*
3344 * Check if DMAR supports zero-length reads on write only
3345 * mappings..
3346 */
3347 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3348 !cap_zlr(iommu->cap))
f76aec76
KA		 3349 prot |= DMA_PTE_READ;
3350 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3351 prot |= DMA_PTE_WRITE;
3352
b536d24d 3353 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3354
f532959b 3355 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3356 if (unlikely(ret)) {
e1605495
DW		 3357 dma_pte_free_pagetable(domain, start_vpfn,
3358 start_vpfn + size - 1);
e1605495
DW		 3359 __free_iova(&domain->iovad, iova);
3360 return 0;
ba395927
KA		 3361 }
3362
1f0ef2aa
DW		 3363 /* it's a non-present to present mapping. Only flush if caching mode */
3364 if (cap_caching_mode(iommu->cap))
ea8ea460 3365 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3366 else
8c11e798 3367 iommu_flush_write_buffer(iommu);
1f0ef2aa 3368
ba395927
KA		 3369 return nelems;
3370}
3371
dfb805e8
FT		 3372static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3373{
3374 return !dma_addr;
3375}
3376
160c1d8e 3377struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3378 .alloc = intel_alloc_coherent,
3379 .free = intel_free_coherent,
ba395927
KA		 3380 .map_sg = intel_map_sg,
3381 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3382 .map_page = intel_map_page,
3383 .unmap_page = intel_unmap_page,
dfb805e8 3384 .mapping_error = intel_mapping_error,
ba395927
KA		 3385};
3386
3387static inline int iommu_domain_cache_init(void)
3388{
3389 int ret = 0;
3390
3391 iommu_domain_cache = kmem_cache_create("iommu_domain",
3392 sizeof(struct dmar_domain),
3393 0,
3394 SLAB_HWCACHE_ALIGN,
3395
3396 NULL);
3397 if (!iommu_domain_cache) {
3398 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3399 ret = -ENOMEM;
3400 }
3401
3402 return ret;
3403}
3404
3405static inline int iommu_devinfo_cache_init(void)
3406{
3407 int ret = 0;
3408
3409 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3410 sizeof(struct device_domain_info),
3411 0,
3412 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3413 NULL);
3414 if (!iommu_devinfo_cache) {
3415 printk(KERN_ERR "Couldn't create devinfo cache\n");
3416 ret = -ENOMEM;
3417 }
3418
3419 return ret;
3420}
3421
ba395927
KA		 3422static int __init iommu_init_mempool(void)
3423{
3424 int ret;
3425 ret = iommu_iova_cache_init();
3426 if (ret)
3427 return ret;
3428
3429 ret = iommu_domain_cache_init();
3430 if (ret)
3431 goto domain_error;
3432
3433 ret = iommu_devinfo_cache_init();
3434 if (!ret)
3435 return ret;
3436
3437 kmem_cache_destroy(iommu_domain_cache);
3438domain_error:
85b45456 3439 iommu_iova_cache_destroy();
ba395927
KA		 3440
3441 return -ENOMEM;
3442}
3443
3444static void __init iommu_exit_mempool(void)
3445{
3446 kmem_cache_destroy(iommu_devinfo_cache);
3447 kmem_cache_destroy(iommu_domain_cache);
85b45456 3448 iommu_iova_cache_destroy();
ba395927
KA		 3449}
3450
556ab45f
DW		 3451static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3452{
3453 struct dmar_drhd_unit *drhd;
3454 u32 vtbar;
3455 int rc;
3456
3457 /* We know that this device on this chipset has its own IOMMU.
3458 * If we find it under a different IOMMU, then the BIOS is lying
3459 * to us. Hope that the IOMMU for this device is actually
3460 * disabled, and it needs no translation...
3461 */
3462 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3463 if (rc) {
3464 /* "can't" happen */
3465 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3466 return;
3467 }
3468 vtbar &= 0xffff0000;
3469
3470 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3471 drhd = dmar_find_matched_drhd_unit(pdev);
3472 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3473 TAINT_FIRMWARE_WORKAROUND,
3474 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3475 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3476}
3477DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3478
ba395927
KA		 3479static void __init init_no_remapping_devices(void)
3480{
3481 struct dmar_drhd_unit *drhd;
832bd858 3482 struct device *dev;
b683b230 3483 int i;
ba395927
KA		 3484
3485 for_each_drhd_unit(drhd) {
3486 if (!drhd->include_all) {
b683b230
JL		 3487 for_each_active_dev_scope(drhd->devices,
3488 drhd->devices_cnt, i, dev)
3489 break;
832bd858 3490 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3491 if (i == drhd->devices_cnt)
3492 drhd->ignored = 1;
3493 }
3494 }
3495
7c919779 3496 for_each_active_drhd_unit(drhd) {
7c919779 3497 if (drhd->include_all)
ba395927
KA		 3498 continue;
3499
b683b230
JL		 3500 for_each_active_dev_scope(drhd->devices,
3501 drhd->devices_cnt, i, dev)
832bd858 3502 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3503 break;
ba395927
KA		 3504 if (i < drhd->devices_cnt)
3505 continue;
3506
c0771df8
DW		 3507 /* This IOMMU has *only* gfx devices. Either bypass it or
3508 set the gfx_mapped flag, as appropriate */
3509 if (dmar_map_gfx) {
3510 intel_iommu_gfx_mapped = 1;
3511 } else {
3512 drhd->ignored = 1;
b683b230
JL		 3513 for_each_active_dev_scope(drhd->devices,
3514 drhd->devices_cnt, i, dev)
832bd858 3515 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3516 }
3517 }
3518}
3519
f59c7b69
FY		 3520#ifdef CONFIG_SUSPEND
3521static int init_iommu_hw(void)
3522{
3523 struct dmar_drhd_unit *drhd;
3524 struct intel_iommu *iommu = NULL;
3525
3526 for_each_active_iommu(iommu, drhd)
3527 if (iommu->qi)
3528 dmar_reenable_qi(iommu);
3529
b779260b
JC		 3530 for_each_iommu(iommu, drhd) {
3531 if (drhd->ignored) {
3532 /*
3533 * we always have to disable PMRs or DMA may fail on
3534 * this device
3535 */
3536 if (force_on)
3537 iommu_disable_protect_mem_regions(iommu);
3538 continue;
3539 }
3540
f59c7b69
FY		 3541 iommu_flush_write_buffer(iommu);
3542
3543 iommu_set_root_entry(iommu);
3544
3545 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3546 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3547 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3548 iommu_enable_translation(iommu);
b94996c9 3549 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3550 }
3551
3552 return 0;
3553}
3554
3555static void iommu_flush_all(void)
3556{
3557 struct dmar_drhd_unit *drhd;
3558 struct intel_iommu *iommu;
3559
3560 for_each_active_iommu(iommu, drhd) {
3561 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3562 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3563 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3564 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3565 }
3566}
3567
134fac3f 3568static int iommu_suspend(void)
f59c7b69
FY		 3569{
3570 struct dmar_drhd_unit *drhd;
3571 struct intel_iommu *iommu = NULL;
3572 unsigned long flag;
3573
3574 for_each_active_iommu(iommu, drhd) {
3575 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3576 GFP_ATOMIC);
3577 if (!iommu->iommu_state)
3578 goto nomem;
3579 }
3580
3581 iommu_flush_all();
3582
3583 for_each_active_iommu(iommu, drhd) {
3584 iommu_disable_translation(iommu);
3585
1f5b3c3f 3586 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3587
3588 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3589 readl(iommu->reg + DMAR_FECTL_REG);
3590 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3591 readl(iommu->reg + DMAR_FEDATA_REG);
3592 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3593 readl(iommu->reg + DMAR_FEADDR_REG);
3594 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3595 readl(iommu->reg + DMAR_FEUADDR_REG);
3596
1f5b3c3f 3597 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3598 }
3599 return 0;
3600
3601nomem:
3602 for_each_active_iommu(iommu, drhd)
3603 kfree(iommu->iommu_state);
3604
3605 return -ENOMEM;
3606}
3607
134fac3f 3608static void iommu_resume(void)
f59c7b69
FY		 3609{
3610 struct dmar_drhd_unit *drhd;
3611 struct intel_iommu *iommu = NULL;
3612 unsigned long flag;
3613
3614 if (init_iommu_hw()) {
b779260b
JC		 3615 if (force_on)
3616 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3617 else
3618 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3619 return;
f59c7b69
FY		 3620 }
3621
3622 for_each_active_iommu(iommu, drhd) {
3623
1f5b3c3f 3624 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3625
3626 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3627 iommu->reg + DMAR_FECTL_REG);
3628 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3629 iommu->reg + DMAR_FEDATA_REG);
3630 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3631 iommu->reg + DMAR_FEADDR_REG);
3632 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3633 iommu->reg + DMAR_FEUADDR_REG);
3634
1f5b3c3f 3635 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3636 }
3637
3638 for_each_active_iommu(iommu, drhd)
3639 kfree(iommu->iommu_state);
f59c7b69
FY		 3640}
3641
134fac3f 3642static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3643 .resume = iommu_resume,
3644 .suspend = iommu_suspend,
3645};
3646
134fac3f 3647static void __init init_iommu_pm_ops(void)
f59c7b69 3648{
134fac3f 3649 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3650}
3651
3652#else
99592ba4 3653static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3654#endif /* CONFIG_PM */
3655
318fe7df 3656
c2a0b538 3657int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3658{
3659 struct acpi_dmar_reserved_memory *rmrr;
3660 struct dmar_rmrr_unit *rmrru;
3661
3662 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3663 if (!rmrru)
3664 return -ENOMEM;
3665
3666 rmrru->hdr = header;
3667 rmrr = (struct acpi_dmar_reserved_memory *)header;
3668 rmrru->base_address = rmrr->base_address;
3669 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3670 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3671 ((void *)rmrr) + rmrr->header.length,
3672 &rmrru->devices_cnt);
3673 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3674 kfree(rmrru);
3675 return -ENOMEM;
3676 }
318fe7df 3677
2e455289 3678 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3679
2e455289 3680 return 0;
318fe7df
SS		 3681}
3682
6b197249
JL		 3683static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3684{
3685 struct dmar_atsr_unit *atsru;
3686 struct acpi_dmar_atsr *tmp;
3687
3688 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3689 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3690 if (atsr->segment != tmp->segment)
3691 continue;
3692 if (atsr->header.length != tmp->header.length)
3693 continue;
3694 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3695 return atsru;
3696 }
3697
3698 return NULL;
3699}
3700
3701int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 3702{
3703 struct acpi_dmar_atsr *atsr;
3704 struct dmar_atsr_unit *atsru;
3705
6b197249
JL		 3706 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
3707 return 0;
3708
318fe7df 3709 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 3710 atsru = dmar_find_atsr(atsr);
3711 if (atsru)
3712 return 0;
3713
3714 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 3715 if (!atsru)
3716 return -ENOMEM;
3717
6b197249
JL		 3718 /*
3719 * If memory is allocated from slab by ACPI _DSM method, we need to
3720 * copy the memory content because the memory buffer will be freed
3721 * on return.
3722 */
3723 atsru->hdr = (void *)(atsru + 1);
3724 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 3725 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 3726 if (!atsru->include_all) {
3727 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3728 (void *)atsr + atsr->header.length,
3729 &atsru->devices_cnt);
3730 if (atsru->devices_cnt && atsru->devices == NULL) {
3731 kfree(atsru);
3732 return -ENOMEM;
3733 }
3734 }
318fe7df 3735
0e242612 3736 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 3737
3738 return 0;
3739}
3740
9bdc531e
JL		 3741static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3742{
3743 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3744 kfree(atsru);
3745}
3746
6b197249
JL		 3747int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3748{
3749 struct acpi_dmar_atsr *atsr;
3750 struct dmar_atsr_unit *atsru;
3751
3752 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3753 atsru = dmar_find_atsr(atsr);
3754 if (atsru) {
3755 list_del_rcu(&atsru->list);
3756 synchronize_rcu();
3757 intel_iommu_free_atsr(atsru);
3758 }
3759
3760 return 0;
3761}
3762
3763int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3764{
3765 int i;
3766 struct device *dev;
3767 struct acpi_dmar_atsr *atsr;
3768 struct dmar_atsr_unit *atsru;
3769
3770 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3771 atsru = dmar_find_atsr(atsr);
3772 if (!atsru)
3773 return 0;
3774
3775 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
3776 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
3777 i, dev)
3778 return -EBUSY;
3779
3780 return 0;
3781}
3782
ffebeb46
JL		 3783static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3784{
3785 int sp, ret = 0;
3786 struct intel_iommu *iommu = dmaru->iommu;
3787
3788 if (g_iommus[iommu->seq_id])
3789 return 0;
3790
3791 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
3792 pr_warn("IOMMU: %s doesn't support hardware pass through.\n",
3793 iommu->name);
3794 return -ENXIO;
3795 }
3796 if (!ecap_sc_support(iommu->ecap) &&
3797 domain_update_iommu_snooping(iommu)) {
3798 pr_warn("IOMMU: %s doesn't support snooping.\n",
3799 iommu->name);
3800 return -ENXIO;
3801 }
3802 sp = domain_update_iommu_superpage(iommu) - 1;
3803 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
3804 pr_warn("IOMMU: %s doesn't support large page.\n",
3805 iommu->name);
3806 return -ENXIO;
3807 }
3808
3809 /*
3810 * Disable translation if already enabled prior to OS handover.
3811 */
3812 if (iommu->gcmd & DMA_GCMD_TE)
3813 iommu_disable_translation(iommu);
3814
3815 g_iommus[iommu->seq_id] = iommu;
3816 ret = iommu_init_domains(iommu);
3817 if (ret == 0)
3818 ret = iommu_alloc_root_entry(iommu);
3819 if (ret)
3820 goto out;
3821
3822 if (dmaru->ignored) {
3823 /*
3824 * we always have to disable PMRs or DMA may fail on this device
3825 */
3826 if (force_on)
3827 iommu_disable_protect_mem_regions(iommu);
3828 return 0;
3829 }
3830
3831 intel_iommu_init_qi(iommu);
3832 iommu_flush_write_buffer(iommu);
3833 ret = dmar_set_interrupt(iommu);
3834 if (ret)
3835 goto disable_iommu;
3836
3837 iommu_set_root_entry(iommu);
3838 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3839 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3840 iommu_enable_translation(iommu);
3841
3842 if (si_domain) {
3843 ret = iommu_attach_domain(si_domain, iommu);
3844 if (ret < 0 || si_domain->id != ret)
3845 goto disable_iommu;
3846 domain_attach_iommu(si_domain, iommu);
3847 }
3848
3849 iommu_disable_protect_mem_regions(iommu);
3850 return 0;
3851
3852disable_iommu:
3853 disable_dmar_iommu(iommu);
3854out:
3855 free_dmar_iommu(iommu);
3856 return ret;
3857}
3858
6b197249
JL		 3859int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
3860{
ffebeb46
JL		 3861 int ret = 0;
3862 struct intel_iommu *iommu = dmaru->iommu;
3863
3864 if (!intel_iommu_enabled)
3865 return 0;
3866 if (iommu == NULL)
3867 return -EINVAL;
3868
3869 if (insert) {
3870 ret = intel_iommu_add(dmaru);
3871 } else {
3872 disable_dmar_iommu(iommu);
3873 free_dmar_iommu(iommu);
3874 }
3875
3876 return ret;
6b197249
JL		 3877}
3878
9bdc531e
JL		 3879static void intel_iommu_free_dmars(void)
3880{
3881 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3882 struct dmar_atsr_unit *atsru, *atsr_n;
3883
3884 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3885 list_del(&rmrru->list);
3886 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3887 kfree(rmrru);
318fe7df
SS		 3888 }
3889
9bdc531e
JL		 3890 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3891 list_del(&atsru->list);
3892 intel_iommu_free_atsr(atsru);
3893 }
318fe7df
SS		 3894}
3895
3896int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3897{
b683b230 3898 int i, ret = 1;
318fe7df 3899 struct pci_bus *bus;
832bd858
DW		 3900 struct pci_dev *bridge = NULL;
3901 struct device *tmp;
318fe7df
SS		 3902 struct acpi_dmar_atsr *atsr;
3903 struct dmar_atsr_unit *atsru;
3904
3905 dev = pci_physfn(dev);
318fe7df 3906 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 3907 bridge = bus->self;
318fe7df 3908 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3909 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 3910 return 0;
b5f82ddf 3911 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 3912 break;
318fe7df 3913 }
b5f82ddf
JL		 3914 if (!bridge)
3915 return 0;
318fe7df 3916
0e242612 3917 rcu_read_lock();
b5f82ddf
JL		 3918 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3919 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3920 if (atsr->segment != pci_domain_nr(dev->bus))
3921 continue;
3922
b683b230 3923 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 3924 if (tmp == &bridge->dev)
b683b230 3925 goto out;
b5f82ddf
JL		 3926
3927 if (atsru->include_all)
b683b230 3928 goto out;
b5f82ddf 3929 }
b683b230
JL		 3930 ret = 0;
3931out:
0e242612 3932 rcu_read_unlock();
318fe7df 3933
b683b230 3934 return ret;
318fe7df
SS		 3935}
3936
59ce0515
JL		 3937int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3938{
3939 int ret = 0;
3940 struct dmar_rmrr_unit *rmrru;
3941 struct dmar_atsr_unit *atsru;
3942 struct acpi_dmar_atsr *atsr;
3943 struct acpi_dmar_reserved_memory *rmrr;
3944
3945 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3946 return 0;
3947
3948 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3949 rmrr = container_of(rmrru->hdr,
3950 struct acpi_dmar_reserved_memory, header);
3951 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3952 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3953 ((void *)rmrr) + rmrr->header.length,
3954 rmrr->segment, rmrru->devices,
3955 rmrru->devices_cnt);
27e24950 3956 if(ret < 0)
59ce0515
JL		 3957 return ret;
3958 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 3959 dmar_remove_dev_scope(info, rmrr->segment,
3960 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 3961 }
3962 }
3963
3964 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3965 if (atsru->include_all)
3966 continue;
3967
3968 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3969 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3970 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3971 (void *)atsr + atsr->header.length,
3972 atsr->segment, atsru->devices,
3973 atsru->devices_cnt);
3974 if (ret > 0)
3975 break;
3976 else if(ret < 0)
3977 return ret;
3978 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3979 if (dmar_remove_dev_scope(info, atsr->segment,
3980 atsru->devices, atsru->devices_cnt))
3981 break;
3982 }
3983 }
3984
3985 return 0;
3986}
3987
99dcaded
FY		 3988/*
3989 * Here we only respond to action of unbound device from driver.
3990 *
3991 * Added device is not attached to its DMAR domain here yet. That will happen
3992 * when mapping the device to iova.
3993 */
3994static int device_notifier(struct notifier_block *nb,
3995 unsigned long action, void *data)
3996{
3997 struct device *dev = data;
99dcaded
FY		 3998 struct dmar_domain *domain;
3999
3d89194a 4000 if (iommu_dummy(dev))
44cd613c
DW		 4001 return 0;
4002
1196c2fb 4003 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4004 return 0;
4005
1525a29a 4006 domain = find_domain(dev);
99dcaded
FY		 4007 if (!domain)
4008 return 0;
4009
3a5670e8 4010 down_read(&dmar_global_lock);
bf9c9eda 4011 domain_remove_one_dev_info(domain, dev);
ab8dfe25 4012 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4013 domain_exit(domain);
3a5670e8 4014 up_read(&dmar_global_lock);
a97590e5 4015
99dcaded
FY		 4016 return 0;
4017}
4018
4019static struct notifier_block device_nb = {
4020 .notifier_call = device_notifier,
4021};
4022
75f05569
JL		 4023static int intel_iommu_memory_notifier(struct notifier_block *nb,
4024 unsigned long val, void *v)
4025{
4026 struct memory_notify *mhp = v;
4027 unsigned long long start, end;
4028 unsigned long start_vpfn, last_vpfn;
4029
4030 switch (val) {
4031 case MEM_GOING_ONLINE:
4032 start = mhp->start_pfn << PAGE_SHIFT;
4033 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4034 if (iommu_domain_identity_map(si_domain, start, end)) {
4035 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
4036 start, end);
4037 return NOTIFY_BAD;
4038 }
4039 break;
4040
4041 case MEM_OFFLINE:
4042 case MEM_CANCEL_ONLINE:
4043 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4044 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4045 while (start_vpfn <= last_vpfn) {
4046 struct iova *iova;
4047 struct dmar_drhd_unit *drhd;
4048 struct intel_iommu *iommu;
ea8ea460 4049 struct page *freelist;
75f05569
JL		 4050
4051 iova = find_iova(&si_domain->iovad, start_vpfn);
4052 if (iova == NULL) {
4053 pr_debug("dmar: failed get IOVA for PFN %lx\n",
4054 start_vpfn);
4055 break;
4056 }
4057
4058 iova = split_and_remove_iova(&si_domain->iovad, iova,
4059 start_vpfn, last_vpfn);
4060 if (iova == NULL) {
4061 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
4062 start_vpfn, last_vpfn);
4063 return NOTIFY_BAD;
4064 }
4065
ea8ea460
DW		 4066 freelist = domain_unmap(si_domain, iova->pfn_lo,
4067 iova->pfn_hi);
4068
75f05569
JL		 4069 rcu_read_lock();
4070 for_each_active_iommu(iommu, drhd)
4071 iommu_flush_iotlb_psi(iommu, si_domain->id,
a156ef99 4072 iova->pfn_lo, iova_size(iova),
ea8ea460 4073 !freelist, 0);
75f05569 4074 rcu_read_unlock();
ea8ea460 4075 dma_free_pagelist(freelist);
75f05569
JL		 4076
4077 start_vpfn = iova->pfn_hi + 1;
4078 free_iova_mem(iova);
4079 }
4080 break;
4081 }
4082
4083 return NOTIFY_OK;
4084}
4085
4086static struct notifier_block intel_iommu_memory_nb = {
4087 .notifier_call = intel_iommu_memory_notifier,
4088 .priority = 0
4089};
4090
a5459cfe
AW		 4091
4092static ssize_t intel_iommu_show_version(struct device *dev,
4093 struct device_attribute *attr,
4094 char *buf)
4095{
4096 struct intel_iommu *iommu = dev_get_drvdata(dev);
4097 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4098 return sprintf(buf, "%d:%d\n",
4099 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4100}
4101static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4102
4103static ssize_t intel_iommu_show_address(struct device *dev,
4104 struct device_attribute *attr,
4105 char *buf)
4106{
4107 struct intel_iommu *iommu = dev_get_drvdata(dev);
4108 return sprintf(buf, "%llx\n", iommu->reg_phys);
4109}
4110static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4111
4112static ssize_t intel_iommu_show_cap(struct device *dev,
4113 struct device_attribute *attr,
4114 char *buf)
4115{
4116 struct intel_iommu *iommu = dev_get_drvdata(dev);
4117 return sprintf(buf, "%llx\n", iommu->cap);
4118}
4119static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4120
4121static ssize_t intel_iommu_show_ecap(struct device *dev,
4122 struct device_attribute *attr,
4123 char *buf)
4124{
4125 struct intel_iommu *iommu = dev_get_drvdata(dev);
4126 return sprintf(buf, "%llx\n", iommu->ecap);
4127}
4128static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4129
4130static struct attribute *intel_iommu_attrs[] = {
4131 &dev_attr_version.attr,
4132 &dev_attr_address.attr,
4133 &dev_attr_cap.attr,
4134 &dev_attr_ecap.attr,
4135 NULL,
4136};
4137
4138static struct attribute_group intel_iommu_group = {
4139 .name = "intel-iommu",
4140 .attrs = intel_iommu_attrs,
4141};
4142
4143const struct attribute_group *intel_iommu_groups[] = {
4144 &intel_iommu_group,
4145 NULL,
4146};
4147
ba395927
KA		 4148int __init intel_iommu_init(void)
4149{
9bdc531e 4150 int ret = -ENODEV;
3a93c841 4151 struct dmar_drhd_unit *drhd;
7c919779 4152 struct intel_iommu *iommu;
ba395927 4153
a59b50e9
JC		 4154 /* VT-d is required for a TXT/tboot launch, so enforce that */
4155 force_on = tboot_force_iommu();
4156
3a5670e8
JL		 4157 if (iommu_init_mempool()) {
4158 if (force_on)
4159 panic("tboot: Failed to initialize iommu memory\n");
4160 return -ENOMEM;
4161 }
4162
4163 down_write(&dmar_global_lock);
a59b50e9
JC		 4164 if (dmar_table_init()) {
4165 if (force_on)
4166 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4167 goto out_free_dmar;
a59b50e9 4168 }
ba395927 4169
3a93c841
TI		 4170 /*
4171 * Disable translation if already enabled prior to OS handover.
4172 */
7c919779 4173 for_each_active_iommu(iommu, drhd)
3a93c841
TI		 4174 if (iommu->gcmd & DMA_GCMD_TE)
4175 iommu_disable_translation(iommu);
3a93c841 4176
c2c7286a 4177 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4178 if (force_on)
4179 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4180 goto out_free_dmar;
a59b50e9 4181 }
1886e8a9 4182
75f1cdf1 4183 if (no_iommu || dmar_disabled)
9bdc531e 4184 goto out_free_dmar;
2ae21010 4185
318fe7df
SS		 4186 if (list_empty(&dmar_rmrr_units))
4187 printk(KERN_INFO "DMAR: No RMRR found\n");
4188
4189 if (list_empty(&dmar_atsr_units))
4190 printk(KERN_INFO "DMAR: No ATSR found\n");
4191
51a63e67
JC		 4192 if (dmar_init_reserved_ranges()) {
4193 if (force_on)
4194 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4195 goto out_free_reserved_range;
51a63e67 4196 }
ba395927
KA		 4197
4198 init_no_remapping_devices();
4199
b779260b 4200 ret = init_dmars();
ba395927 4201 if (ret) {
a59b50e9
JC		 4202 if (force_on)
4203 panic("tboot: Failed to initialize DMARs\n");
ba395927 4204 printk(KERN_ERR "IOMMU: dmar init failed\n");
9bdc531e 4205 goto out_free_reserved_range;
ba395927 4206 }
3a5670e8 4207 up_write(&dmar_global_lock);
ba395927
KA		 4208 printk(KERN_INFO
4209 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4210
5e0d2a6f 4211 init_timer(&unmap_timer);
75f1cdf1
FT		 4212#ifdef CONFIG_SWIOTLB
4213 swiotlb = 0;
4214#endif
19943b0e 4215 dma_ops = &intel_dma_ops;
4ed0d3e6 4216
134fac3f 4217 init_iommu_pm_ops();
a8bcbb0d 4218
a5459cfe
AW		 4219 for_each_active_iommu(iommu, drhd)
4220 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4221 intel_iommu_groups,
4222 iommu->name);
4223
4236d97d 4224 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4225 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4226 if (si_domain && !hw_pass_through)
4227 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4228
8bc1f85c
ED		 4229 intel_iommu_enabled = 1;
4230
ba395927 4231 return 0;
9bdc531e
JL		 4232
4233out_free_reserved_range:
4234 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4235out_free_dmar:
4236 intel_iommu_free_dmars();
3a5670e8
JL		 4237 up_write(&dmar_global_lock);
4238 iommu_exit_mempool();
9bdc531e 4239 return ret;
ba395927 4240}
e820482c 4241
579305f7
AW		 4242static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4243{
4244 struct intel_iommu *iommu = opaque;
4245
4246 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4247 return 0;
4248}
4249
4250/*
4251 * NB - intel-iommu lacks any sort of reference counting for the users of
4252 * dependent devices. If multiple endpoints have intersecting dependent
4253 * devices, unbinding the driver from any one of them will possibly leave
4254 * the others unable to operate.
4255 */
3199aa6b 4256static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4257 struct device *dev)
3199aa6b 4258{
0bcb3e28 4259 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4260 return;
4261
579305f7 4262 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4263}
4264
2c2e2c38 4265static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4266 struct device *dev)
c7151a8d 4267{
bca2b916 4268 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4269 struct intel_iommu *iommu;
4270 unsigned long flags;
4271 int found = 0;
156baca8 4272 u8 bus, devfn;
c7151a8d 4273
bf9c9eda 4274 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4275 if (!iommu)
4276 return;
4277
4278 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4279 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4280 if (info->iommu == iommu && info->bus == bus &&
4281 info->devfn == devfn) {
109b9b04 4282 unlink_domain_info(info);
c7151a8d
WH		 4283 spin_unlock_irqrestore(&device_domain_lock, flags);
4284
93a23a72 4285 iommu_disable_dev_iotlb(info);
c7151a8d 4286 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4287 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4288 free_devinfo_mem(info);
4289
4290 spin_lock_irqsave(&device_domain_lock, flags);
4291
4292 if (found)
4293 break;
4294 else
4295 continue;
4296 }
4297
4298 /* if there is no other devices under the same iommu
4299 * owned by this domain, clear this iommu in iommu_bmp
4300 * update iommu count and coherency
4301 */
8bbc4410 4302 if (info->iommu == iommu)
c7151a8d
WH		 4303 found = 1;
4304 }
4305
3e7abe25
RD		 4306 spin_unlock_irqrestore(&device_domain_lock, flags);
4307
c7151a8d 4308 if (found == 0) {
fb170fb4
JL		 4309 domain_detach_iommu(domain, iommu);
4310 if (!domain_type_is_vm_or_si(domain))
4311 iommu_detach_domain(domain, iommu);
c7151a8d 4312 }
c7151a8d
WH		 4313}
4314
2c2e2c38 4315static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4316{
4317 int adjust_width;
4318
1b722500 4319 init_iova_domain(&domain->iovad, IOVA_START_PFN, DMA_32BIT_PFN);
5e98c4b1
WH		 4320 domain_reserve_special_ranges(domain);
4321
4322 /* calculate AGAW */
4323 domain->gaw = guest_width;
4324 adjust_width = guestwidth_to_adjustwidth(guest_width);
4325 domain->agaw = width_to_agaw(adjust_width);
4326
5e98c4b1 4327 domain->iommu_coherency = 0;
c5b15255 4328 domain->iommu_snooping = 0;
6dd9a7c7 4329 domain->iommu_superpage = 0;
fe40f1e0 4330 domain->max_addr = 0;
5e98c4b1
WH		 4331
4332 /* always allocate the top pgd */
4c923d47 4333 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4334 if (!domain->pgd)
4335 return -ENOMEM;
4336 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4337 return 0;
4338}
4339
5d450806 4340static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 4341{
5d450806 4342 struct dmar_domain *dmar_domain;
38717946 4343
ab8dfe25 4344 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4345 if (!dmar_domain) {
38717946 4346 printk(KERN_ERR
5d450806
JR		 4347 "intel_iommu_domain_init: dmar_domain == NULL\n");
4348 return -ENOMEM;
38717946 4349 }
2c2e2c38 4350 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 4351 printk(KERN_ERR
5d450806 4352 "intel_iommu_domain_init() failed\n");
92d03cc8 4353 domain_exit(dmar_domain);
5d450806 4354 return -ENOMEM;
38717946 4355 }
8140a95d 4356 domain_update_iommu_cap(dmar_domain);
5d450806 4357 domain->priv = dmar_domain;
faa3d6f5 4358
8a0e715b
JR		 4359 domain->geometry.aperture_start = 0;
4360 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4361 domain->geometry.force_aperture = true;
4362
5d450806 4363 return 0;
38717946 4364}
38717946 4365
5d450806 4366static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 4367{
5d450806
JR		 4368 struct dmar_domain *dmar_domain = domain->priv;
4369
4370 domain->priv = NULL;
92d03cc8 4371 domain_exit(dmar_domain);
38717946 4372}
38717946 4373
4c5478c9
JR		 4374static int intel_iommu_attach_device(struct iommu_domain *domain,
4375 struct device *dev)
38717946 4376{
4c5478c9 4377 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0
WH		 4378 struct intel_iommu *iommu;
4379 int addr_width;
156baca8 4380 u8 bus, devfn;
faa3d6f5 4381
c875d2c1
AW		 4382 if (device_is_rmrr_locked(dev)) {
4383 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4384 return -EPERM;
4385 }
4386
7207d8f9
DW		 4387 /* normally dev is not mapped */
4388 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4389 struct dmar_domain *old_domain;
4390
1525a29a 4391 old_domain = find_domain(dev);
faa3d6f5 4392 if (old_domain) {
ab8dfe25 4393 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4394 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4395 else
4396 domain_remove_dev_info(old_domain);
62c22167
JR		 4397
4398 if (!domain_type_is_vm_or_si(old_domain) &&
4399 list_empty(&old_domain->devices))
4400 domain_exit(old_domain);
faa3d6f5
WH		 4401 }
4402 }
4403
156baca8 4404 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4405 if (!iommu)
4406 return -ENODEV;
4407
4408 /* check if this iommu agaw is sufficient for max mapped address */
4409 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4410 if (addr_width > cap_mgaw(iommu->cap))
4411 addr_width = cap_mgaw(iommu->cap);
4412
4413 if (dmar_domain->max_addr > (1LL << addr_width)) {
4414 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4415 "sufficient for the mapped address (%llx)\n",
a99c47a2 4416 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4417 return -EFAULT;
4418 }
a99c47a2
TL		 4419 dmar_domain->gaw = addr_width;
4420
4421 /*
4422 * Knock out extra levels of page tables if necessary
4423 */
4424 while (iommu->agaw < dmar_domain->agaw) {
4425 struct dma_pte *pte;
4426
4427 pte = dmar_domain->pgd;
4428 if (dma_pte_present(pte)) {
25cbff16
SY		 4429 dmar_domain->pgd = (struct dma_pte *)
4430 phys_to_virt(dma_pte_addr(pte));
7a661013 4431 free_pgtable_page(pte);
a99c47a2
TL		 4432 }
4433 dmar_domain->agaw--;
4434 }
fe40f1e0 4435
5913c9bf 4436 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
38717946 4437}
38717946 4438
4c5478c9
JR		 4439static void intel_iommu_detach_device(struct iommu_domain *domain,
4440 struct device *dev)
38717946 4441{
4c5478c9 4442 struct dmar_domain *dmar_domain = domain->priv;
4c5478c9 4443
bf9c9eda 4444 domain_remove_one_dev_info(dmar_domain, dev);
faa3d6f5 4445}
c7151a8d 4446
b146a1c9
JR		 4447static int intel_iommu_map(struct iommu_domain *domain,
4448 unsigned long iova, phys_addr_t hpa,
5009065d 4449 size_t size, int iommu_prot)
faa3d6f5 4450{
dde57a21 4451 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4452 u64 max_addr;
dde57a21 4453 int prot = 0;
faa3d6f5 4454 int ret;
fe40f1e0 4455
dde57a21
JR		 4456 if (iommu_prot & IOMMU_READ)
4457 prot |= DMA_PTE_READ;
4458 if (iommu_prot & IOMMU_WRITE)
4459 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4460 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4461 prot |= DMA_PTE_SNP;
dde57a21 4462
163cc52c 4463 max_addr = iova + size;
dde57a21 4464 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4465 u64 end;
4466
4467 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4468 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4469 if (end < max_addr) {
8954da1f 4470 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4471 "sufficient for the mapped address (%llx)\n",
8954da1f 4472 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4473 return -EFAULT;
4474 }
dde57a21 4475 dmar_domain->max_addr = max_addr;
fe40f1e0 4476 }
ad051221
DW		 4477 /* Round up size to next multiple of PAGE_SIZE, if it and
4478 the low bits of hpa would take us onto the next page */
88cb6a74 4479 size = aligned_nrpages(hpa, size);
ad051221
DW		 4480 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4481 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4482 return ret;
38717946 4483}
38717946 4484
5009065d 4485static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4486 unsigned long iova, size_t size)
38717946 4487{
dde57a21 4488 struct dmar_domain *dmar_domain = domain->priv;
ea8ea460
DW		 4489 struct page *freelist = NULL;
4490 struct intel_iommu *iommu;
4491 unsigned long start_pfn, last_pfn;
4492 unsigned int npages;
4493 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4494
4495 /* Cope with horrid API which requires us to unmap more than the
4496 size argument if it happens to be a large-page mapping. */
4497 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4498 BUG();
4499
4500 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4501 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4502
ea8ea460
DW		 4503 start_pfn = iova >> VTD_PAGE_SHIFT;
4504 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4505
4506 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4507
4508 npages = last_pfn - start_pfn + 1;
4509
4510 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4511 iommu = g_iommus[iommu_id];
4512
4513 /*
4514 * find bit position of dmar_domain
4515 */
4516 ndomains = cap_ndoms(iommu->cap);
4517 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4518 if (iommu->domains[num] == dmar_domain)
4519 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4520 npages, !freelist, 0);
4521 }
4522
4523 }
4524
4525 dma_free_pagelist(freelist);
fe40f1e0 4526
163cc52c
DW		 4527 if (dmar_domain->max_addr == iova + size)
4528 dmar_domain->max_addr = iova;
b146a1c9 4529
5cf0a76f 4530 return size;
38717946 4531}
38717946 4532
d14d6577 4533static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4534 dma_addr_t iova)
38717946 4535{
d14d6577 4536 struct dmar_domain *dmar_domain = domain->priv;
38717946 4537 struct dma_pte *pte;
5cf0a76f 4538 int level = 0;
faa3d6f5 4539 u64 phys = 0;
38717946 4540
5cf0a76f 4541 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4542 if (pte)
faa3d6f5 4543 phys = dma_pte_addr(pte);
38717946 4544
faa3d6f5 4545 return phys;
38717946 4546}
a8bcbb0d 4547
5d587b8d 4548static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4549{
dbb9fd86 4550 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4551 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4552 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4553 return irq_remapping_enabled == 1;
dbb9fd86 4554
5d587b8d 4555 return false;
dbb9fd86
SY		 4556}
4557
abdfdde2
AW		 4558static int intel_iommu_add_device(struct device *dev)
4559{
a5459cfe 4560 struct intel_iommu *iommu;
abdfdde2 4561 struct iommu_group *group;
156baca8 4562 u8 bus, devfn;
70ae6f0d 4563
a5459cfe
AW		 4564 iommu = device_to_iommu(dev, &bus, &devfn);
4565 if (!iommu)
70ae6f0d
AW		 4566 return -ENODEV;
4567
a5459cfe 4568 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4569
e17f9ff4 4570 group = iommu_group_get_for_dev(dev);
783f157b 4571
e17f9ff4
AW		 4572 if (IS_ERR(group))
4573 return PTR_ERR(group);
bcb71abe 4574
abdfdde2 4575 iommu_group_put(group);
e17f9ff4 4576 return 0;
abdfdde2 4577}
70ae6f0d 4578
abdfdde2
AW		 4579static void intel_iommu_remove_device(struct device *dev)
4580{
a5459cfe
AW		 4581 struct intel_iommu *iommu;
4582 u8 bus, devfn;
4583
4584 iommu = device_to_iommu(dev, &bus, &devfn);
4585 if (!iommu)
4586 return;
4587
abdfdde2 4588 iommu_group_remove_device(dev);
a5459cfe
AW		 4589
4590 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4591}
4592
b22f6434 4593static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4594 .capable = intel_iommu_capable,
a8bcbb0d
JR		 4595 .domain_init = intel_iommu_domain_init,
4596 .domain_destroy = intel_iommu_domain_destroy,
4597 .attach_dev = intel_iommu_attach_device,
4598 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4599 .map = intel_iommu_map,
4600 .unmap = intel_iommu_unmap,
315786eb 4601 .map_sg = default_iommu_map_sg,
a8bcbb0d 4602 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4603 .add_device = intel_iommu_add_device,
4604 .remove_device = intel_iommu_remove_device,
6d1c56a9 4605 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4606};
9af88143 4607
9452618e
DV		 4608static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4609{
4610 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4611 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4612 dmar_map_gfx = 0;
4613}
4614
4615DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4616DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4617DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4618DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4619DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4620DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4621DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4622
d34d6517 4623static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4624{
4625 /*
4626 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4627 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4628 */
4629 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4630 rwbf_quirk = 1;
4631}
4632
4633DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4634DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4635DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4636DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4637DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4638DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4639DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4640
eecfd57f
AJ		 4641#define GGC 0x52
4642#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4643#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4644#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4645#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4646#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4647#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4648#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4649#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4650
d34d6517 4651static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4652{
4653 unsigned short ggc;
4654
eecfd57f 4655 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4656 return;
4657
eecfd57f 4658 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4659 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4660 dmar_map_gfx = 0;
6fbcfb3e
DW		 4661 } else if (dmar_map_gfx) {
4662 /* we have to ensure the gfx device is idle before we flush */
4663 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4664 intel_iommu_strict = 1;
4665 }
9eecabcb
DW		 4666}
4667DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4668DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4669DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4670DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4671
e0fc7e0b
DW		 4672/* On Tylersburg chipsets, some BIOSes have been known to enable the
4673 ISOCH DMAR unit for the Azalia sound device, but not give it any
4674 TLB entries, which causes it to deadlock. Check for that. We do
4675 this in a function called from init_dmars(), instead of in a PCI
4676 quirk, because we don't want to print the obnoxious "BIOS broken"
4677 message if VT-d is actually disabled.
4678*/
4679static void __init check_tylersburg_isoch(void)
4680{
4681 struct pci_dev *pdev;
4682 uint32_t vtisochctrl;
4683
4684 /* If there's no Azalia in the system anyway, forget it. */
4685 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4686 if (!pdev)
4687 return;
4688 pci_dev_put(pdev);
4689
4690 /* System Management Registers. Might be hidden, in which case
4691 we can't do the sanity check. But that's OK, because the
4692 known-broken BIOSes _don't_ actually hide it, so far. */
4693 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4694 if (!pdev)
4695 return;
4696
4697 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4698 pci_dev_put(pdev);
4699 return;
4700 }
4701
4702 pci_dev_put(pdev);
4703
4704 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4705 if (vtisochctrl & 1)
4706 return;
4707
4708 /* Drop all bits other than the number of TLB entries */
4709 vtisochctrl &= 0x1c;
4710
4711 /* If we have the recommended number of TLB entries (16), fine. */
4712 if (vtisochctrl == 0x10)
4713 return;
4714
4715 /* Zero TLB entries? You get to ride the short bus to school. */
4716 if (!vtisochctrl) {
4717 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4718 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4719 dmi_get_system_info(DMI_BIOS_VENDOR),
4720 dmi_get_system_info(DMI_BIOS_VERSION),
4721 dmi_get_system_info(DMI_PRODUCT_VERSION));
4722 iommu_identity_mapping |= IDENTMAP_AZALIA;
4723 return;
4724 }
4725
4726 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4727 vtisochctrl);
4728}
Linux kernel - Deliverables
This page took 1.005305 seconds and 5 git commands to generate.