projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
irqchip/gic-v3-its: Add missing cache flushes
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
28ccce0d
JR		 367/*
368 * Domain represents a virtual machine, more than one devices
1ce28feb
WH		 369 * across iommus may be owned in one domain, e.g. kvm guest.
370 */
ab8dfe25 371#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 372
2c2e2c38 373/* si_domain contains mulitple devices */
ab8dfe25 374#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 375
29a27719
JR		 376#define for_each_domain_iommu(idx, domain) \
377 for (idx = 0; idx < g_num_of_iommus; idx++) \
378 if (domain->iommu_refcnt[idx])
379
99126f7c 380struct dmar_domain {
4c923d47 381 int nid; /* node id */
29a27719
JR		 382
383 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
384 /* Refcount of devices per iommu */
385
99126f7c 386
c0e8a6c8
JR		 387 u16 iommu_did[DMAR_UNITS_SUPPORTED];
388 /* Domain ids per IOMMU. Use u16 since
389 * domain ids are 16 bit wide according
390 * to VT-d spec, section 9.3 */
99126f7c 391
00a77deb 392 struct list_head devices; /* all devices' list */
99126f7c
MM		 393 struct iova_domain iovad; /* iova's that belong to this domain */
394
395 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 396 int gaw; /* max guest address width */
397
398 /* adjusted guest address width, 0 is level 2 30-bit */
399 int agaw;
400
3b5410e7 401 int flags; /* flags to find out type of domain */
8e604097
WH		 402
403 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 404 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 405 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 406 int iommu_superpage;/* Level of superpages supported:
407 0 == 4KiB (no superpages), 1 == 2MiB,
408 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
fe40f1e0 409 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 410
411 struct iommu_domain domain; /* generic domain data structure for
412 iommu core */
99126f7c
MM		 413};
414
a647dacb
MM		 415/* PCI domain-device relationship */
416struct device_domain_info {
417 struct list_head link; /* link to domain siblings */
418 struct list_head global; /* link to global list */
276dbf99 419 u8 bus; /* PCI bus number */
a647dacb 420 u8 devfn; /* PCI devfn number */
fb0cc3aa
BH		 421 struct {
422 u8 enabled:1;
423 u8 qdep;
424 } ats; /* ATS state */
0bcb3e28 425 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 426 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 427 struct dmar_domain *domain; /* pointer to domain */
428};
429
b94e4117
JL		 430struct dmar_rmrr_unit {
431 struct list_head list; /* list of rmrr units */
432 struct acpi_dmar_header *hdr; /* ACPI header */
433 u64 base_address; /* reserved base address*/
434 u64 end_address; /* reserved end address */
832bd858 435 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 436 int devices_cnt; /* target device count */
437};
438
439struct dmar_atsr_unit {
440 struct list_head list; /* list of ATSR units */
441 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 442 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 443 int devices_cnt; /* target device count */
444 u8 include_all:1; /* include all ports */
445};
446
447static LIST_HEAD(dmar_atsr_units);
448static LIST_HEAD(dmar_rmrr_units);
449
450#define for_each_rmrr_units(rmrr) \
451 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
452
5e0d2a6f 453static void flush_unmaps_timeout(unsigned long data);
454
b707cb02 455static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 456
80b20dd8 457#define HIGH_WATER_MARK 250
458struct deferred_flush_tables {
459 int next;
460 struct iova *iova[HIGH_WATER_MARK];
461 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 462 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 463};
464
465static struct deferred_flush_tables *deferred_flush;
466
5e0d2a6f 467/* bitmap for indexing intel_iommus */
5e0d2a6f 468static int g_num_of_iommus;
469
470static DEFINE_SPINLOCK(async_umap_flush_lock);
471static LIST_HEAD(unmaps_to_do);
472
473static int timer_on;
474static long list_size;
5e0d2a6f 475
92d03cc8 476static void domain_exit(struct dmar_domain *domain);
ba395927 477static void domain_remove_dev_info(struct dmar_domain *domain);
e6de0f8d
JR		 478static void dmar_remove_one_dev_info(struct dmar_domain *domain,
479 struct device *dev);
127c7615 480static void __dmar_remove_one_dev_info(struct device_domain_info *info);
2452d9db
JR		 481static void domain_context_clear(struct intel_iommu *iommu,
482 struct device *dev);
2a46ddf7
JL		 483static int domain_detach_iommu(struct dmar_domain *domain,
484 struct intel_iommu *iommu);
ba395927 485
d3f13810 486#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 487int dmar_disabled = 0;
488#else
489int dmar_disabled = 1;
d3f13810 490#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 491
8bc1f85c
ED		 492int intel_iommu_enabled = 0;
493EXPORT_SYMBOL_GPL(intel_iommu_enabled);
494
2d9e667e 495static int dmar_map_gfx = 1;
7d3b03ce 496static int dmar_forcedac;
5e0d2a6f 497static int intel_iommu_strict;
6dd9a7c7 498static int intel_iommu_superpage = 1;
c83b2f20
DW		 499static int intel_iommu_ecs = 1;
500
501/* We only actually use ECS when PASID support (on the new bit 40)
502 * is also advertised. Some early implementations — the ones with
503 * PASID support on bit 28 — have issues even when we *only* use
504 * extended root/context tables. */
505#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
506 ecap_pasid(iommu->ecap))
ba395927 507
c0771df8
DW		 508int intel_iommu_gfx_mapped;
509EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
510
ba395927
KA		 511#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
512static DEFINE_SPINLOCK(device_domain_lock);
513static LIST_HEAD(device_domain_list);
514
b22f6434 515static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 516
4158c2ec
JR		 517static bool translation_pre_enabled(struct intel_iommu *iommu)
518{
519 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
520}
521
091d42e4
JR		 522static void clear_translation_pre_enabled(struct intel_iommu *iommu)
523{
524 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
525}
526
4158c2ec
JR		 527static void init_translation_status(struct intel_iommu *iommu)
528{
529 u32 gsts;
530
531 gsts = readl(iommu->reg + DMAR_GSTS_REG);
532 if (gsts & DMA_GSTS_TES)
533 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
534}
535
00a77deb
JR		 536/* Convert generic 'struct iommu_domain to private struct dmar_domain */
537static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
538{
539 return container_of(dom, struct dmar_domain, domain);
540}
541
ba395927
KA		 542static int __init intel_iommu_setup(char *str)
543{
544 if (!str)
545 return -EINVAL;
546 while (*str) {
0cd5c3c8
KM		 547 if (!strncmp(str, "on", 2)) {
548 dmar_disabled = 0;
9f10e5bf 549 pr_info("IOMMU enabled\n");
0cd5c3c8 550 } else if (!strncmp(str, "off", 3)) {
ba395927 551 dmar_disabled = 1;
9f10e5bf 552 pr_info("IOMMU disabled\n");
ba395927
KA		 553 } else if (!strncmp(str, "igfx_off", 8)) {
554 dmar_map_gfx = 0;
9f10e5bf 555 pr_info("Disable GFX device mapping\n");
7d3b03ce 556 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 557 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 558 dmar_forcedac = 1;
5e0d2a6f 559 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 560 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 561 intel_iommu_strict = 1;
6dd9a7c7 562 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 563 pr_info("Disable supported super page\n");
6dd9a7c7 564 intel_iommu_superpage = 0;
c83b2f20
DW		 565 } else if (!strncmp(str, "ecs_off", 7)) {
566 printk(KERN_INFO
567 "Intel-IOMMU: disable extended context table support\n");
568 intel_iommu_ecs = 0;
ba395927
KA		 569 }
570
571 str += strcspn(str, ",");
572 while (*str == ',')
573 str++;
574 }
575 return 0;
576}
577__setup("intel_iommu=", intel_iommu_setup);
578
579static struct kmem_cache *iommu_domain_cache;
580static struct kmem_cache *iommu_devinfo_cache;
ba395927 581
9452d5bf
JR		 582static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
583{
8bf47816
JR		 584 struct dmar_domain **domains;
585 int idx = did >> 8;
586
587 domains = iommu->domains[idx];
588 if (!domains)
589 return NULL;
590
591 return domains[did & 0xff];
9452d5bf
JR		 592}
593
594static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
595 struct dmar_domain *domain)
596{
8bf47816
JR		 597 struct dmar_domain **domains;
598 int idx = did >> 8;
599
600 if (!iommu->domains[idx]) {
601 size_t size = 256 * sizeof(struct dmar_domain *);
602 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
603 }
604
605 domains = iommu->domains[idx];
606 if (WARN_ON(!domains))
607 return;
608 else
609 domains[did & 0xff] = domain;
9452d5bf
JR		 610}
611
4c923d47 612static inline void *alloc_pgtable_page(int node)
eb3fa7cb 613{
4c923d47
SS		 614 struct page *page;
615 void *vaddr = NULL;
eb3fa7cb 616
4c923d47
SS		 617 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
618 if (page)
619 vaddr = page_address(page);
eb3fa7cb 620 return vaddr;
ba395927
KA		 621}
622
623static inline void free_pgtable_page(void *vaddr)
624{
625 free_page((unsigned long)vaddr);
626}
627
628static inline void *alloc_domain_mem(void)
629{
354bb65e 630 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 631}
632
38717946 633static void free_domain_mem(void *vaddr)
ba395927
KA		 634{
635 kmem_cache_free(iommu_domain_cache, vaddr);
636}
637
638static inline void * alloc_devinfo_mem(void)
639{
354bb65e 640 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 641}
642
643static inline void free_devinfo_mem(void *vaddr)
644{
645 kmem_cache_free(iommu_devinfo_cache, vaddr);
646}
647
ab8dfe25
JL		 648static inline int domain_type_is_vm(struct dmar_domain *domain)
649{
650 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
651}
652
28ccce0d
JR		 653static inline int domain_type_is_si(struct dmar_domain *domain)
654{
655 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
656}
657
ab8dfe25
JL		 658static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
659{
660 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
661 DOMAIN_FLAG_STATIC_IDENTITY);
662}
1b573683 663
162d1b10
JL		 664static inline int domain_pfn_supported(struct dmar_domain *domain,
665 unsigned long pfn)
666{
667 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
668
669 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
670}
671
4ed0d3e6 672static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 673{
674 unsigned long sagaw;
675 int agaw = -1;
676
677 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 678 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 679 agaw >= 0; agaw--) {
680 if (test_bit(agaw, &sagaw))
681 break;
682 }
683
684 return agaw;
685}
686
4ed0d3e6
FY		 687/*
688 * Calculate max SAGAW for each iommu.
689 */
690int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
691{
692 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
693}
694
695/*
696 * calculate agaw for each iommu.
697 * "SAGAW" may be different across iommus, use a default agaw, and
698 * get a supported less agaw for iommus that don't support the default agaw.
699 */
700int iommu_calculate_agaw(struct intel_iommu *iommu)
701{
702 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
703}
704
2c2e2c38 705/* This functionin only returns single iommu in a domain */
8c11e798
WH		 706static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
707{
708 int iommu_id;
709
2c2e2c38 710 /* si_domain and vm domain should not get here. */
ab8dfe25 711 BUG_ON(domain_type_is_vm_or_si(domain));
29a27719
JR		 712 for_each_domain_iommu(iommu_id, domain)
713 break;
714
8c11e798
WH		 715 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
716 return NULL;
717
718 return g_iommus[iommu_id];
719}
720
8e604097
WH		 721static void domain_update_iommu_coherency(struct dmar_domain *domain)
722{
d0501960
DW		 723 struct dmar_drhd_unit *drhd;
724 struct intel_iommu *iommu;
2f119c78
QL		 725 bool found = false;
726 int i;
2e12bc29 727
d0501960 728 domain->iommu_coherency = 1;
8e604097 729
29a27719 730 for_each_domain_iommu(i, domain) {
2f119c78 731 found = true;
8e604097
WH		 732 if (!ecap_coherent(g_iommus[i]->ecap)) {
733 domain->iommu_coherency = 0;
734 break;
735 }
8e604097 736 }
d0501960
DW		 737 if (found)
738 return;
739
740 /* No hardware attached; use lowest common denominator */
741 rcu_read_lock();
742 for_each_active_iommu(iommu, drhd) {
743 if (!ecap_coherent(iommu->ecap)) {
744 domain->iommu_coherency = 0;
745 break;
746 }
747 }
748 rcu_read_unlock();
8e604097
WH		 749}
750
161f6934 751static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 752{
161f6934
JL		 753 struct dmar_drhd_unit *drhd;
754 struct intel_iommu *iommu;
755 int ret = 1;
58c610bd 756
161f6934
JL		 757 rcu_read_lock();
758 for_each_active_iommu(iommu, drhd) {
759 if (iommu != skip) {
760 if (!ecap_sc_support(iommu->ecap)) {
761 ret = 0;
762 break;
763 }
58c610bd 764 }
58c610bd 765 }
161f6934
JL		 766 rcu_read_unlock();
767
768 return ret;
58c610bd
SY		 769}
770
161f6934 771static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 772{
8140a95d 773 struct dmar_drhd_unit *drhd;
161f6934 774 struct intel_iommu *iommu;
8140a95d 775 int mask = 0xf;
6dd9a7c7
YS		 776
777 if (!intel_iommu_superpage) {
161f6934 778 return 0;
6dd9a7c7
YS		 779 }
780
8140a95d 781 /* set iommu_superpage to the smallest common denominator */
0e242612 782 rcu_read_lock();
8140a95d 783 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 784 if (iommu != skip) {
785 mask &= cap_super_page_val(iommu->cap);
786 if (!mask)
787 break;
6dd9a7c7
YS		 788 }
789 }
0e242612
JL		 790 rcu_read_unlock();
791
161f6934 792 return fls(mask);
6dd9a7c7
YS		 793}
794
58c610bd
SY		 795/* Some capabilities may be different across iommus */
796static void domain_update_iommu_cap(struct dmar_domain *domain)
797{
798 domain_update_iommu_coherency(domain);
161f6934
JL		 799 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
800 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 801}
802
03ecc32c
DW		 803static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
804 u8 bus, u8 devfn, int alloc)
805{
806 struct root_entry *root = &iommu->root_entry[bus];
807 struct context_entry *context;
808 u64 *entry;
809
4df4eab1 810 entry = &root->lo;
c83b2f20 811 if (ecs_enabled(iommu)) {
03ecc32c
DW		 812 if (devfn >= 0x80) {
813 devfn -= 0x80;
814 entry = &root->hi;
815 }
816 devfn *= 2;
817 }
03ecc32c
DW		 818 if (*entry & 1)
819 context = phys_to_virt(*entry & VTD_PAGE_MASK);
820 else {
821 unsigned long phy_addr;
822 if (!alloc)
823 return NULL;
824
825 context = alloc_pgtable_page(iommu->node);
826 if (!context)
827 return NULL;
828
829 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
830 phy_addr = virt_to_phys((void *)context);
831 *entry = phy_addr | 1;
832 __iommu_flush_cache(iommu, entry, sizeof(*entry));
833 }
834 return &context[devfn];
835}
836
4ed6a540
DW		 837static int iommu_dummy(struct device *dev)
838{
839 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
840}
841
156baca8 842static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 843{
844 struct dmar_drhd_unit *drhd = NULL;
b683b230 845 struct intel_iommu *iommu;
156baca8
DW		 846 struct device *tmp;
847 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 848 u16 segment = 0;
c7151a8d
WH		 849 int i;
850
4ed6a540
DW		 851 if (iommu_dummy(dev))
852 return NULL;
853
156baca8
DW		 854 if (dev_is_pci(dev)) {
855 pdev = to_pci_dev(dev);
856 segment = pci_domain_nr(pdev->bus);
ca5b74d2 857 } else if (has_acpi_companion(dev))
156baca8
DW		 858 dev = &ACPI_COMPANION(dev)->dev;
859
0e242612 860 rcu_read_lock();
b683b230 861 for_each_active_iommu(iommu, drhd) {
156baca8 862 if (pdev && segment != drhd->segment)
276dbf99 863 continue;
c7151a8d 864
b683b230 865 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 866 drhd->devices_cnt, i, tmp) {
867 if (tmp == dev) {
868 *bus = drhd->devices[i].bus;
869 *devfn = drhd->devices[i].devfn;
b683b230 870 goto out;
156baca8
DW		 871 }
872
873 if (!pdev || !dev_is_pci(tmp))
874 continue;
875
876 ptmp = to_pci_dev(tmp);
877 if (ptmp->subordinate &&
878 ptmp->subordinate->number <= pdev->bus->number &&
879 ptmp->subordinate->busn_res.end >= pdev->bus->number)
880 goto got_pdev;
924b6231 881 }
c7151a8d 882
156baca8
DW		 883 if (pdev && drhd->include_all) {
884 got_pdev:
885 *bus = pdev->bus->number;
886 *devfn = pdev->devfn;
b683b230 887 goto out;
156baca8 888 }
c7151a8d 889 }
b683b230 890 iommu = NULL;
156baca8 891 out:
0e242612 892 rcu_read_unlock();
c7151a8d 893
b683b230 894 return iommu;
c7151a8d
WH		 895}
896
5331fe6f
WH		 897static void domain_flush_cache(struct dmar_domain *domain,
898 void *addr, int size)
899{
900 if (!domain->iommu_coherency)
901 clflush_cache_range(addr, size);
902}
903
ba395927
KA		 904static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
905{
ba395927 906 struct context_entry *context;
03ecc32c 907 int ret = 0;
ba395927
KA		 908 unsigned long flags;
909
910 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 911 context = iommu_context_addr(iommu, bus, devfn, 0);
912 if (context)
913 ret = context_present(context);
ba395927
KA		 914 spin_unlock_irqrestore(&iommu->lock, flags);
915 return ret;
916}
917
918static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
919{
ba395927
KA		 920 struct context_entry *context;
921 unsigned long flags;
922
923 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 924 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 925 if (context) {
03ecc32c
DW		 926 context_clear_entry(context);
927 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 928 }
929 spin_unlock_irqrestore(&iommu->lock, flags);
930}
931
932static void free_context_table(struct intel_iommu *iommu)
933{
ba395927
KA		 934 int i;
935 unsigned long flags;
936 struct context_entry *context;
937
938 spin_lock_irqsave(&iommu->lock, flags);
939 if (!iommu->root_entry) {
940 goto out;
941 }
942 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 943 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 944 if (context)
945 free_pgtable_page(context);
03ecc32c 946
c83b2f20 947 if (!ecs_enabled(iommu))
03ecc32c
DW		 948 continue;
949
950 context = iommu_context_addr(iommu, i, 0x80, 0);
951 if (context)
952 free_pgtable_page(context);
953
ba395927
KA		 954 }
955 free_pgtable_page(iommu->root_entry);
956 iommu->root_entry = NULL;
957out:
958 spin_unlock_irqrestore(&iommu->lock, flags);
959}
960
b026fd28 961static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 962 unsigned long pfn, int *target_level)
ba395927 963{
ba395927
KA		 964 struct dma_pte *parent, *pte = NULL;
965 int level = agaw_to_level(domain->agaw);
4399c8bf 966 int offset;
ba395927
KA		 967
968 BUG_ON(!domain->pgd);
f9423606 969
162d1b10 970 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 971 /* Address beyond IOMMU's addressing capabilities. */
972 return NULL;
973
ba395927
KA		 974 parent = domain->pgd;
975
5cf0a76f 976 while (1) {
ba395927
KA		 977 void *tmp_page;
978
b026fd28 979 offset = pfn_level_offset(pfn, level);
ba395927 980 pte = &parent[offset];
5cf0a76f 981 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 982 break;
5cf0a76f 983 if (level == *target_level)
ba395927
KA		 984 break;
985
19c239ce 986 if (!dma_pte_present(pte)) {
c85994e4
DW		 987 uint64_t pteval;
988
4c923d47 989 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 990
206a73c1 991 if (!tmp_page)
ba395927 992 return NULL;
206a73c1 993
c85994e4 994 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 995 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 996 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 997 /* Someone else set it while we were thinking; use theirs. */
998 free_pgtable_page(tmp_page);
effad4b5 999 else
c85994e4 1000 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 1001 }
5cf0a76f
DW		 1002 if (level == 1)
1003 break;
1004
19c239ce 1005 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1006 level--;
1007 }
1008
5cf0a76f
DW		 1009 if (!*target_level)
1010 *target_level = level;
1011
ba395927
KA		 1012 return pte;
1013}
1014
6dd9a7c7 1015
ba395927 1016/* return address's pte at specific level */
90dcfb5e
DW		 1017static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1018 unsigned long pfn,
6dd9a7c7 1019 int level, int *large_page)
ba395927
KA		 1020{
1021 struct dma_pte *parent, *pte = NULL;
1022 int total = agaw_to_level(domain->agaw);
1023 int offset;
1024
1025 parent = domain->pgd;
1026 while (level <= total) {
90dcfb5e 1027 offset = pfn_level_offset(pfn, total);
ba395927
KA		 1028 pte = &parent[offset];
1029 if (level == total)
1030 return pte;
1031
6dd9a7c7
YS		 1032 if (!dma_pte_present(pte)) {
1033 *large_page = total;
ba395927 1034 break;
6dd9a7c7
YS		 1035 }
1036
e16922af 1037 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 1038 *large_page = total;
1039 return pte;
1040 }
1041
19c239ce 1042 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1043 total--;
1044 }
1045 return NULL;
1046}
1047
ba395927 1048/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1049static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1050 unsigned long start_pfn,
1051 unsigned long last_pfn)
ba395927 1052{
6dd9a7c7 1053 unsigned int large_page = 1;
310a5ab9 1054 struct dma_pte *first_pte, *pte;
66eae846 1055
162d1b10
JL		 1056 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1057 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1058 BUG_ON(start_pfn > last_pfn);
ba395927 1059
04b18e65 1060 /* we don't need lock here; nobody else touches the iova range */
59c36286 1061 do {
6dd9a7c7
YS		 1062 large_page = 1;
1063 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1064 if (!pte) {
6dd9a7c7 1065 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1066 continue;
1067 }
6dd9a7c7 1068 do {
310a5ab9 1069 dma_clear_pte(pte);
6dd9a7c7 1070 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1071 pte++;
75e6bf96
DW		 1072 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1073
310a5ab9
DW		 1074 domain_flush_cache(domain, first_pte,
1075 (void *)pte - (void *)first_pte);
59c36286
DW		 1076
1077 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1078}
1079
3269ee0b
AW		 1080static void dma_pte_free_level(struct dmar_domain *domain, int level,
1081 struct dma_pte *pte, unsigned long pfn,
1082 unsigned long start_pfn, unsigned long last_pfn)
1083{
1084 pfn = max(start_pfn, pfn);
1085 pte = &pte[pfn_level_offset(pfn, level)];
1086
1087 do {
1088 unsigned long level_pfn;
1089 struct dma_pte *level_pte;
1090
1091 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1092 goto next;
1093
1094 level_pfn = pfn & level_mask(level - 1);
1095 level_pte = phys_to_virt(dma_pte_addr(pte));
1096
1097 if (level > 2)
1098 dma_pte_free_level(domain, level - 1, level_pte,
1099 level_pfn, start_pfn, last_pfn);
1100
1101 /* If range covers entire pagetable, free it */
1102 if (!(start_pfn > level_pfn ||
08336fd2 1103 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1104 dma_clear_pte(pte);
1105 domain_flush_cache(domain, pte, sizeof(*pte));
1106 free_pgtable_page(level_pte);
1107 }
1108next:
1109 pfn += level_size(level);
1110 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1111}
1112
ba395927
KA		 1113/* free page table pages. last level pte should already be cleared */
1114static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1115 unsigned long start_pfn,
1116 unsigned long last_pfn)
ba395927 1117{
162d1b10
JL		 1118 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1119 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1120 BUG_ON(start_pfn > last_pfn);
ba395927 1121
d41a4adb
JL		 1122 dma_pte_clear_range(domain, start_pfn, last_pfn);
1123
f3a0a52f 1124 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1125 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1126 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1127
ba395927 1128 /* free pgd */
d794dc9b 1129 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1130 free_pgtable_page(domain->pgd);
1131 domain->pgd = NULL;
1132 }
1133}
1134
ea8ea460
DW		 1135/* When a page at a given level is being unlinked from its parent, we don't
1136 need to *modify* it at all. All we need to do is make a list of all the
1137 pages which can be freed just as soon as we've flushed the IOTLB and we
1138 know the hardware page-walk will no longer touch them.
1139 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1140 be freed. */
1141static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1142 int level, struct dma_pte *pte,
1143 struct page *freelist)
1144{
1145 struct page *pg;
1146
1147 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1148 pg->freelist = freelist;
1149 freelist = pg;
1150
1151 if (level == 1)
1152 return freelist;
1153
adeb2590
JL		 1154 pte = page_address(pg);
1155 do {
ea8ea460
DW		 1156 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1157 freelist = dma_pte_list_pagetables(domain, level - 1,
1158 pte, freelist);
adeb2590
JL		 1159 pte++;
1160 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1161
1162 return freelist;
1163}
1164
1165static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1166 struct dma_pte *pte, unsigned long pfn,
1167 unsigned long start_pfn,
1168 unsigned long last_pfn,
1169 struct page *freelist)
1170{
1171 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1172
1173 pfn = max(start_pfn, pfn);
1174 pte = &pte[pfn_level_offset(pfn, level)];
1175
1176 do {
1177 unsigned long level_pfn;
1178
1179 if (!dma_pte_present(pte))
1180 goto next;
1181
1182 level_pfn = pfn & level_mask(level);
1183
1184 /* If range covers entire pagetable, free it */
1185 if (start_pfn <= level_pfn &&
1186 last_pfn >= level_pfn + level_size(level) - 1) {
1187 /* These suborbinate page tables are going away entirely. Don't
1188 bother to clear them; we're just going to *free* them. */
1189 if (level > 1 && !dma_pte_superpage(pte))
1190 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1191
1192 dma_clear_pte(pte);
1193 if (!first_pte)
1194 first_pte = pte;
1195 last_pte = pte;
1196 } else if (level > 1) {
1197 /* Recurse down into a level that isn't *entirely* obsolete */
1198 freelist = dma_pte_clear_level(domain, level - 1,
1199 phys_to_virt(dma_pte_addr(pte)),
1200 level_pfn, start_pfn, last_pfn,
1201 freelist);
1202 }
1203next:
1204 pfn += level_size(level);
1205 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1206
1207 if (first_pte)
1208 domain_flush_cache(domain, first_pte,
1209 (void *)++last_pte - (void *)first_pte);
1210
1211 return freelist;
1212}
1213
1214/* We can't just free the pages because the IOMMU may still be walking
1215 the page tables, and may have cached the intermediate levels. The
1216 pages can only be freed after the IOTLB flush has been done. */
b690420a
JR		 1217static struct page *domain_unmap(struct dmar_domain *domain,
1218 unsigned long start_pfn,
1219 unsigned long last_pfn)
ea8ea460 1220{
ea8ea460
DW		 1221 struct page *freelist = NULL;
1222
162d1b10
JL		 1223 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1224 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1225 BUG_ON(start_pfn > last_pfn);
1226
1227 /* we don't need lock here; nobody else touches the iova range */
1228 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1229 domain->pgd, 0, start_pfn, last_pfn, NULL);
1230
1231 /* free pgd */
1232 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1233 struct page *pgd_page = virt_to_page(domain->pgd);
1234 pgd_page->freelist = freelist;
1235 freelist = pgd_page;
1236
1237 domain->pgd = NULL;
1238 }
1239
1240 return freelist;
1241}
1242
b690420a 1243static void dma_free_pagelist(struct page *freelist)
ea8ea460
DW		 1244{
1245 struct page *pg;
1246
1247 while ((pg = freelist)) {
1248 freelist = pg->freelist;
1249 free_pgtable_page(page_address(pg));
1250 }
1251}
1252
ba395927
KA		 1253/* iommu handling */
1254static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1255{
1256 struct root_entry *root;
1257 unsigned long flags;
1258
4c923d47 1259 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1260 if (!root) {
9f10e5bf 1261 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1262 iommu->name);
ba395927 1263 return -ENOMEM;
ffebeb46 1264 }
ba395927 1265
5b6985ce 1266 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1267
1268 spin_lock_irqsave(&iommu->lock, flags);
1269 iommu->root_entry = root;
1270 spin_unlock_irqrestore(&iommu->lock, flags);
1271
1272 return 0;
1273}
1274
ba395927
KA		 1275static void iommu_set_root_entry(struct intel_iommu *iommu)
1276{
03ecc32c 1277 u64 addr;
c416daa9 1278 u32 sts;
ba395927
KA		 1279 unsigned long flag;
1280
03ecc32c 1281 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1282 if (ecs_enabled(iommu))
03ecc32c 1283 addr |= DMA_RTADDR_RTT;
ba395927 1284
1f5b3c3f 1285 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1286 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1287
c416daa9 1288 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1289
1290 /* Make sure hardware complete it */
1291 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1292 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1293
1f5b3c3f 1294 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1295}
1296
1297static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1298{
1299 u32 val;
1300 unsigned long flag;
1301
9af88143 1302 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1303 return;
ba395927 1304
1f5b3c3f 1305 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1306 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1307
1308 /* Make sure hardware complete it */
1309 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1310 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1311
1f5b3c3f 1312 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1313}
1314
1315/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1316static void __iommu_flush_context(struct intel_iommu *iommu,
1317 u16 did, u16 source_id, u8 function_mask,
1318 u64 type)
ba395927
KA		 1319{
1320 u64 val = 0;
1321 unsigned long flag;
1322
ba395927
KA		 1323 switch (type) {
1324 case DMA_CCMD_GLOBAL_INVL:
1325 val = DMA_CCMD_GLOBAL_INVL;
1326 break;
1327 case DMA_CCMD_DOMAIN_INVL:
1328 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1329 break;
1330 case DMA_CCMD_DEVICE_INVL:
1331 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1332 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1333 break;
1334 default:
1335 BUG();
1336 }
1337 val |= DMA_CCMD_ICC;
1338
1f5b3c3f 1339 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1340 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1341
1342 /* Make sure hardware complete it */
1343 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1344 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1345
1f5b3c3f 1346 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1347}
1348
ba395927 1349/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1350static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1351 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1352{
1353 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1354 u64 val = 0, val_iva = 0;
1355 unsigned long flag;
1356
ba395927
KA		 1357 switch (type) {
1358 case DMA_TLB_GLOBAL_FLUSH:
1359 /* global flush doesn't need set IVA_REG */
1360 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1361 break;
1362 case DMA_TLB_DSI_FLUSH:
1363 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1364 break;
1365 case DMA_TLB_PSI_FLUSH:
1366 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1367 /* IH bit is passed in as part of address */
ba395927
KA		 1368 val_iva = size_order | addr;
1369 break;
1370 default:
1371 BUG();
1372 }
1373 /* Note: set drain read/write */
1374#if 0
1375 /*
1376 * This is probably to be super secure.. Looks like we can
1377 * ignore it without any impact.
1378 */
1379 if (cap_read_drain(iommu->cap))
1380 val |= DMA_TLB_READ_DRAIN;
1381#endif
1382 if (cap_write_drain(iommu->cap))
1383 val |= DMA_TLB_WRITE_DRAIN;
1384
1f5b3c3f 1385 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1386 /* Note: Only uses first TLB reg currently */
1387 if (val_iva)
1388 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1389 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1390
1391 /* Make sure hardware complete it */
1392 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1393 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1394
1f5b3c3f 1395 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1396
1397 /* check IOTLB invalidation granularity */
1398 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1399 pr_err("Flush IOTLB failed\n");
ba395927 1400 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1401 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1402 (unsigned long long)DMA_TLB_IIRG(type),
1403 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1404}
1405
64ae892b
DW		 1406static struct device_domain_info *
1407iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1408 u8 bus, u8 devfn)
93a23a72 1409{
2f119c78 1410 bool found = false;
93a23a72 1411 struct device_domain_info *info;
0bcb3e28 1412 struct pci_dev *pdev;
93a23a72 1413
55d94043
JR		 1414 assert_spin_locked(&device_domain_lock);
1415
93a23a72
YZ		 1416 if (!ecap_dev_iotlb_support(iommu->ecap))
1417 return NULL;
1418
1419 if (!iommu->qi)
1420 return NULL;
1421
93a23a72 1422 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1423 if (info->iommu == iommu && info->bus == bus &&
1424 info->devfn == devfn) {
2f119c78 1425 found = true;
93a23a72
YZ		 1426 break;
1427 }
93a23a72 1428
0bcb3e28 1429 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1430 return NULL;
1431
0bcb3e28
DW		 1432 pdev = to_pci_dev(info->dev);
1433
1434 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1435 return NULL;
1436
0bcb3e28 1437 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1438 return NULL;
1439
93a23a72
YZ		 1440 return info;
1441}
1442
1443static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1444{
fb0cc3aa
BH		 1445 struct pci_dev *pdev;
1446
0bcb3e28 1447 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1448 return;
1449
fb0cc3aa
BH		 1450 pdev = to_pci_dev(info->dev);
1451 if (pci_enable_ats(pdev, VTD_PAGE_SHIFT))
1452 return;
1453
1454 info->ats.enabled = 1;
1455 info->ats.qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1456}
1457
1458static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1459{
fb0cc3aa 1460 if (!info->ats.enabled)
93a23a72
YZ		 1461 return;
1462
0bcb3e28 1463 pci_disable_ats(to_pci_dev(info->dev));
fb0cc3aa 1464 info->ats.enabled = 0;
93a23a72
YZ		 1465}
1466
1467static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1468 u64 addr, unsigned mask)
1469{
1470 u16 sid, qdep;
1471 unsigned long flags;
1472 struct device_domain_info *info;
1473
1474 spin_lock_irqsave(&device_domain_lock, flags);
1475 list_for_each_entry(info, &domain->devices, link) {
fb0cc3aa 1476 if (!info->ats.enabled)
93a23a72
YZ		 1477 continue;
1478
1479 sid = info->bus << 8 | info->devfn;
fb0cc3aa 1480 qdep = info->ats.qdep;
93a23a72
YZ		 1481 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1482 }
1483 spin_unlock_irqrestore(&device_domain_lock, flags);
1484}
1485
a1ddcbe9
JR		 1486static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1487 struct dmar_domain *domain,
1488 unsigned long pfn, unsigned int pages,
1489 int ih, int map)
ba395927 1490{
9dd2fe89 1491 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1492 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
a1ddcbe9 1493 u16 did = domain->iommu_did[iommu->seq_id];
ba395927 1494
ba395927
KA		 1495 BUG_ON(pages == 0);
1496
ea8ea460
DW		 1497 if (ih)
1498 ih = 1 << 6;
ba395927 1499 /*
9dd2fe89
YZ		 1500 * Fallback to domain selective flush if no PSI support or the size is
1501 * too big.
ba395927
KA		 1502 * PSI requires page size to be 2 ^ x, and the base address is naturally
1503 * aligned to the size
1504 */
9dd2fe89
YZ		 1505 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1506 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1507 DMA_TLB_DSI_FLUSH);
9dd2fe89 1508 else
ea8ea460 1509 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1510 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1511
1512 /*
82653633
NA		 1513 * In caching mode, changes of pages from non-present to present require
1514 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1515 */
82653633 1516 if (!cap_caching_mode(iommu->cap) || !map)
9452d5bf
JR		 1517 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1518 addr, mask);
ba395927
KA		 1519}
1520
f8bab735 1521static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1522{
1523 u32 pmen;
1524 unsigned long flags;
1525
1f5b3c3f 1526 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1527 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1528 pmen &= ~DMA_PMEN_EPM;
1529 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1530
1531 /* wait for the protected region status bit to clear */
1532 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1533 readl, !(pmen & DMA_PMEN_PRS), pmen);
1534
1f5b3c3f 1535 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1536}
1537
2a41ccee 1538static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1539{
1540 u32 sts;
1541 unsigned long flags;
1542
1f5b3c3f 1543 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1544 iommu->gcmd |= DMA_GCMD_TE;
1545 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1546
1547 /* Make sure hardware complete it */
1548 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1549 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1550
1f5b3c3f 1551 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1552}
1553
2a41ccee 1554static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1555{
1556 u32 sts;
1557 unsigned long flag;
1558
1f5b3c3f 1559 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1560 iommu->gcmd &= ~DMA_GCMD_TE;
1561 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1562
1563 /* Make sure hardware complete it */
1564 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1565 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1566
1f5b3c3f 1567 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1568}
1569
3460a6d9 1570
ba395927
KA		 1571static int iommu_init_domains(struct intel_iommu *iommu)
1572{
8bf47816
JR		 1573 u32 ndomains, nlongs;
1574 size_t size;
ba395927
KA		 1575
1576 ndomains = cap_ndoms(iommu->cap);
8bf47816 1577 pr_debug("%s: Number of Domains supported <%d>\n",
9f10e5bf 1578 iommu->name, ndomains);
ba395927
KA		 1579 nlongs = BITS_TO_LONGS(ndomains);
1580
94a91b50
DD		 1581 spin_lock_init(&iommu->lock);
1582
ba395927
KA		 1583 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1584 if (!iommu->domain_ids) {
9f10e5bf
JR		 1585 pr_err("%s: Allocating domain id array failed\n",
1586 iommu->name);
ba395927
KA		 1587 return -ENOMEM;
1588 }
8bf47816
JR		 1589
1590 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1591 iommu->domains = kzalloc(size, GFP_KERNEL);
1592
1593 if (iommu->domains) {
1594 size = 256 * sizeof(struct dmar_domain *);
1595 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1596 }
1597
1598 if (!iommu->domains || !iommu->domains[0]) {
9f10e5bf
JR		 1599 pr_err("%s: Allocating domain array failed\n",
1600 iommu->name);
852bdb04 1601 kfree(iommu->domain_ids);
8bf47816 1602 kfree(iommu->domains);
852bdb04 1603 iommu->domain_ids = NULL;
8bf47816 1604 iommu->domains = NULL;
ba395927
KA		 1605 return -ENOMEM;
1606 }
1607
8bf47816
JR		 1608
1609
ba395927 1610 /*
c0e8a6c8
JR		 1611 * If Caching mode is set, then invalid translations are tagged
1612 * with domain-id 0, hence we need to pre-allocate it. We also
1613 * use domain-id 0 as a marker for non-allocated domain-id, so
1614 * make sure it is not used for a real domain.
ba395927 1615 */
c0e8a6c8
JR		 1616 set_bit(0, iommu->domain_ids);
1617
ba395927
KA		 1618 return 0;
1619}
ba395927 1620
ffebeb46 1621static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927 1622{
29a27719 1623 struct device_domain_info *info, *tmp;
55d94043 1624 unsigned long flags;
ba395927 1625
29a27719
JR		 1626 if (!iommu->domains || !iommu->domain_ids)
1627 return;
a4eaa86c 1628
55d94043 1629 spin_lock_irqsave(&device_domain_lock, flags);
29a27719
JR		 1630 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1631 struct dmar_domain *domain;
1632
1633 if (info->iommu != iommu)
1634 continue;
1635
1636 if (!info->dev || !info->domain)
1637 continue;
1638
1639 domain = info->domain;
1640
e6de0f8d 1641 dmar_remove_one_dev_info(domain, info->dev);
29a27719
JR		 1642
1643 if (!domain_type_is_vm_or_si(domain))
1644 domain_exit(domain);
ba395927 1645 }
55d94043 1646 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 1647
1648 if (iommu->gcmd & DMA_GCMD_TE)
1649 iommu_disable_translation(iommu);
ffebeb46 1650}
ba395927 1651
ffebeb46
JL		 1652static void free_dmar_iommu(struct intel_iommu *iommu)
1653{
1654 if ((iommu->domains) && (iommu->domain_ids)) {
8bf47816
JR		 1655 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1656 int i;
1657
1658 for (i = 0; i < elems; i++)
1659 kfree(iommu->domains[i]);
ffebeb46
JL		 1660 kfree(iommu->domains);
1661 kfree(iommu->domain_ids);
1662 iommu->domains = NULL;
1663 iommu->domain_ids = NULL;
1664 }
ba395927 1665
d9630fe9
WH		 1666 g_iommus[iommu->seq_id] = NULL;
1667
ba395927
KA		 1668 /* free context mapping */
1669 free_context_table(iommu);
ba395927
KA		 1670}
1671
ab8dfe25 1672static struct dmar_domain *alloc_domain(int flags)
ba395927 1673{
ba395927 1674 struct dmar_domain *domain;
ba395927
KA		 1675
1676 domain = alloc_domain_mem();
1677 if (!domain)
1678 return NULL;
1679
ab8dfe25 1680 memset(domain, 0, sizeof(*domain));
4c923d47 1681 domain->nid = -1;
ab8dfe25 1682 domain->flags = flags;
92d03cc8 1683 INIT_LIST_HEAD(&domain->devices);
2c2e2c38
FY		 1684
1685 return domain;
1686}
1687
d160aca5
JR		 1688/* Must be called with iommu->lock */
1689static int domain_attach_iommu(struct dmar_domain *domain,
fb170fb4
JL		 1690 struct intel_iommu *iommu)
1691{
44bde614 1692 unsigned long ndomains;
55d94043 1693 int num;
44bde614 1694
55d94043 1695 assert_spin_locked(&device_domain_lock);
d160aca5 1696 assert_spin_locked(&iommu->lock);
ba395927 1697
29a27719
JR		 1698 domain->iommu_refcnt[iommu->seq_id] += 1;
1699 domain->iommu_count += 1;
1700 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
fb170fb4 1701 ndomains = cap_ndoms(iommu->cap);
d160aca5
JR		 1702 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1703
1704 if (num >= ndomains) {
1705 pr_err("%s: No free domain ids\n", iommu->name);
1706 domain->iommu_refcnt[iommu->seq_id] -= 1;
1707 domain->iommu_count -= 1;
55d94043 1708 return -ENOSPC;
2c2e2c38 1709 }
ba395927 1710
d160aca5
JR		 1711 set_bit(num, iommu->domain_ids);
1712 set_iommu_domain(iommu, num, domain);
1713
1714 domain->iommu_did[iommu->seq_id] = num;
1715 domain->nid = iommu->node;
fb170fb4 1716
fb170fb4
JL		 1717 domain_update_iommu_cap(domain);
1718 }
d160aca5 1719
55d94043 1720 return 0;
fb170fb4
JL		 1721}
1722
1723static int domain_detach_iommu(struct dmar_domain *domain,
1724 struct intel_iommu *iommu)
1725{
d160aca5 1726 int num, count = INT_MAX;
d160aca5 1727
55d94043 1728 assert_spin_locked(&device_domain_lock);
d160aca5 1729 assert_spin_locked(&iommu->lock);
fb170fb4 1730
29a27719
JR		 1731 domain->iommu_refcnt[iommu->seq_id] -= 1;
1732 count = --domain->iommu_count;
1733 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
d160aca5
JR		 1734 num = domain->iommu_did[iommu->seq_id];
1735 clear_bit(num, iommu->domain_ids);
1736 set_iommu_domain(iommu, num, NULL);
fb170fb4 1737
fb170fb4 1738 domain_update_iommu_cap(domain);
c0e8a6c8 1739 domain->iommu_did[iommu->seq_id] = 0;
fb170fb4 1740 }
fb170fb4
JL		 1741
1742 return count;
1743}
1744
ba395927 1745static struct iova_domain reserved_iova_list;
8a443df4 1746static struct lock_class_key reserved_rbtree_key;
ba395927 1747
51a63e67 1748static int dmar_init_reserved_ranges(void)
ba395927
KA		 1749{
1750 struct pci_dev *pdev = NULL;
1751 struct iova *iova;
1752 int i;
ba395927 1753
0fb5fe87
RM		 1754 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1755 DMA_32BIT_PFN);
ba395927 1756
8a443df4
MG		 1757 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1758 &reserved_rbtree_key);
1759
ba395927
KA		 1760 /* IOAPIC ranges shouldn't be accessed by DMA */
1761 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1762 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1763 if (!iova) {
9f10e5bf 1764 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1765 return -ENODEV;
1766 }
ba395927
KA		 1767
1768 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1769 for_each_pci_dev(pdev) {
1770 struct resource *r;
1771
1772 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1773 r = &pdev->resource[i];
1774 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1775 continue;
1a4a4551
DW		 1776 iova = reserve_iova(&reserved_iova_list,
1777 IOVA_PFN(r->start),
1778 IOVA_PFN(r->end));
51a63e67 1779 if (!iova) {
9f10e5bf 1780 pr_err("Reserve iova failed\n");
51a63e67
JC		 1781 return -ENODEV;
1782 }
ba395927
KA		 1783 }
1784 }
51a63e67 1785 return 0;
ba395927
KA		 1786}
1787
1788static void domain_reserve_special_ranges(struct dmar_domain *domain)
1789{
1790 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1791}
1792
1793static inline int guestwidth_to_adjustwidth(int gaw)
1794{
1795 int agaw;
1796 int r = (gaw - 12) % 9;
1797
1798 if (r == 0)
1799 agaw = gaw;
1800 else
1801 agaw = gaw + 9 - r;
1802 if (agaw > 64)
1803 agaw = 64;
1804 return agaw;
1805}
1806
dc534b25
JR		 1807static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1808 int guest_width)
ba395927 1809{
ba395927
KA		 1810 int adjust_width, agaw;
1811 unsigned long sagaw;
1812
0fb5fe87
RM		 1813 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1814 DMA_32BIT_PFN);
ba395927
KA		 1815 domain_reserve_special_ranges(domain);
1816
1817 /* calculate AGAW */
ba395927
KA		 1818 if (guest_width > cap_mgaw(iommu->cap))
1819 guest_width = cap_mgaw(iommu->cap);
1820 domain->gaw = guest_width;
1821 adjust_width = guestwidth_to_adjustwidth(guest_width);
1822 agaw = width_to_agaw(adjust_width);
1823 sagaw = cap_sagaw(iommu->cap);
1824 if (!test_bit(agaw, &sagaw)) {
1825 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1826 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1827 agaw = find_next_bit(&sagaw, 5, agaw);
1828 if (agaw >= 5)
1829 return -ENODEV;
1830 }
1831 domain->agaw = agaw;
ba395927 1832
8e604097
WH		 1833 if (ecap_coherent(iommu->ecap))
1834 domain->iommu_coherency = 1;
1835 else
1836 domain->iommu_coherency = 0;
1837
58c610bd
SY		 1838 if (ecap_sc_support(iommu->ecap))
1839 domain->iommu_snooping = 1;
1840 else
1841 domain->iommu_snooping = 0;
1842
214e39aa
DW		 1843 if (intel_iommu_superpage)
1844 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1845 else
1846 domain->iommu_superpage = 0;
1847
4c923d47 1848 domain->nid = iommu->node;
c7151a8d 1849
ba395927 1850 /* always allocate the top pgd */
4c923d47 1851 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1852 if (!domain->pgd)
1853 return -ENOMEM;
5b6985ce 1854 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1855 return 0;
1856}
1857
1858static void domain_exit(struct dmar_domain *domain)
1859{
ea8ea460 1860 struct page *freelist = NULL;
ba395927
KA		 1861
1862 /* Domain 0 is reserved, so dont process it */
1863 if (!domain)
1864 return;
1865
7b668357
AW		 1866 /* Flush any lazy unmaps that may reference this domain */
1867 if (!intel_iommu_strict)
1868 flush_unmaps_timeout(0);
1869
d160aca5
JR		 1870 /* Remove associated devices and clear attached or cached domains */
1871 rcu_read_lock();
ba395927 1872 domain_remove_dev_info(domain);
d160aca5 1873 rcu_read_unlock();
92d03cc8 1874
ba395927
KA		 1875 /* destroy iovas */
1876 put_iova_domain(&domain->iovad);
ba395927 1877
ea8ea460 1878 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1879
ea8ea460
DW		 1880 dma_free_pagelist(freelist);
1881
ba395927
KA		 1882 free_domain_mem(domain);
1883}
1884
64ae892b
DW		 1885static int domain_context_mapping_one(struct dmar_domain *domain,
1886 struct intel_iommu *iommu,
28ccce0d 1887 u8 bus, u8 devfn)
ba395927 1888{
c6c2cebd 1889 u16 did = domain->iommu_did[iommu->seq_id];
28ccce0d
JR		 1890 int translation = CONTEXT_TT_MULTI_LEVEL;
1891 struct device_domain_info *info = NULL;
ba395927 1892 struct context_entry *context;
ba395927 1893 unsigned long flags;
ea6606b0 1894 struct dma_pte *pgd;
55d94043 1895 int ret, agaw;
28ccce0d 1896
c6c2cebd
JR		 1897 WARN_ON(did == 0);
1898
28ccce0d
JR		 1899 if (hw_pass_through && domain_type_is_si(domain))
1900 translation = CONTEXT_TT_PASS_THROUGH;
ba395927
KA		 1901
1902 pr_debug("Set context mapping for %02x:%02x.%d\n",
1903 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1904
ba395927 1905 BUG_ON(!domain->pgd);
5331fe6f 1906
55d94043
JR		 1907 spin_lock_irqsave(&device_domain_lock, flags);
1908 spin_lock(&iommu->lock);
1909
1910 ret = -ENOMEM;
03ecc32c 1911 context = iommu_context_addr(iommu, bus, devfn, 1);
ba395927 1912 if (!context)
55d94043 1913 goto out_unlock;
ba395927 1914
55d94043
JR		 1915 ret = 0;
1916 if (context_present(context))
1917 goto out_unlock;
cf484d0e 1918
ea6606b0
WH		 1919 pgd = domain->pgd;
1920
de24e553 1921 context_clear_entry(context);
c6c2cebd 1922 context_set_domain_id(context, did);
ea6606b0 1923
de24e553
JR		 1924 /*
1925 * Skip top levels of page tables for iommu which has less agaw
1926 * than default. Unnecessary for PT mode.
1927 */
93a23a72 1928 if (translation != CONTEXT_TT_PASS_THROUGH) {
de24e553 1929 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
55d94043 1930 ret = -ENOMEM;
de24e553 1931 pgd = phys_to_virt(dma_pte_addr(pgd));
55d94043
JR		 1932 if (!dma_pte_present(pgd))
1933 goto out_unlock;
ea6606b0 1934 }
4ed0d3e6 1935
64ae892b 1936 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1937 translation = info ? CONTEXT_TT_DEV_IOTLB :
1938 CONTEXT_TT_MULTI_LEVEL;
de24e553 1939
93a23a72
YZ		 1940 context_set_address_root(context, virt_to_phys(pgd));
1941 context_set_address_width(context, iommu->agaw);
de24e553
JR		 1942 } else {
1943 /*
1944 * In pass through mode, AW must be programmed to
1945 * indicate the largest AGAW value supported by
1946 * hardware. And ASR is ignored by hardware.
1947 */
1948 context_set_address_width(context, iommu->msagaw);
93a23a72 1949 }
4ed0d3e6
FY		 1950
1951 context_set_translation_type(context, translation);
c07e7d21
MM		 1952 context_set_fault_enable(context);
1953 context_set_present(context);
5331fe6f 1954 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1955
4c25a2c1
DW		 1956 /*
1957 * It's a non-present to present mapping. If hardware doesn't cache
1958 * non-present entry we only need to flush the write-buffer. If the
1959 * _does_ cache non-present entries, then it does so in the special
1960 * domain #0, which we have to flush:
1961 */
1962 if (cap_caching_mode(iommu->cap)) {
1963 iommu->flush.flush_context(iommu, 0,
1964 (((u16)bus) << 8) | devfn,
1965 DMA_CCMD_MASK_NOBIT,
1966 DMA_CCMD_DEVICE_INVL);
c6c2cebd 1967 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1968 } else {
ba395927 1969 iommu_flush_write_buffer(iommu);
4c25a2c1 1970 }
93a23a72 1971 iommu_enable_dev_iotlb(info);
c7151a8d 1972
55d94043
JR		 1973 ret = 0;
1974
1975out_unlock:
1976 spin_unlock(&iommu->lock);
1977 spin_unlock_irqrestore(&device_domain_lock, flags);
fb170fb4 1978
ba395927
KA		 1979 return 0;
1980}
1981
579305f7
AW		 1982struct domain_context_mapping_data {
1983 struct dmar_domain *domain;
1984 struct intel_iommu *iommu;
579305f7
AW		 1985};
1986
1987static int domain_context_mapping_cb(struct pci_dev *pdev,
1988 u16 alias, void *opaque)
1989{
1990 struct domain_context_mapping_data *data = opaque;
1991
1992 return domain_context_mapping_one(data->domain, data->iommu,
28ccce0d 1993 PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 1994}
1995
ba395927 1996static int
28ccce0d 1997domain_context_mapping(struct dmar_domain *domain, struct device *dev)
ba395927 1998{
64ae892b 1999 struct intel_iommu *iommu;
156baca8 2000 u8 bus, devfn;
579305f7 2001 struct domain_context_mapping_data data;
64ae892b 2002
e1f167f3 2003 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 2004 if (!iommu)
2005 return -ENODEV;
ba395927 2006
579305f7 2007 if (!dev_is_pci(dev))
28ccce0d 2008 return domain_context_mapping_one(domain, iommu, bus, devfn);
579305f7
AW		 2009
2010 data.domain = domain;
2011 data.iommu = iommu;
579305f7
AW		 2012
2013 return pci_for_each_dma_alias(to_pci_dev(dev),
2014 &domain_context_mapping_cb, &data);
2015}
2016
2017static int domain_context_mapped_cb(struct pci_dev *pdev,
2018 u16 alias, void *opaque)
2019{
2020 struct intel_iommu *iommu = opaque;
2021
2022 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2023}
2024
e1f167f3 2025static int domain_context_mapped(struct device *dev)
ba395927 2026{
5331fe6f 2027 struct intel_iommu *iommu;
156baca8 2028 u8 bus, devfn;
5331fe6f 2029
e1f167f3 2030 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2031 if (!iommu)
2032 return -ENODEV;
ba395927 2033
579305f7
AW		 2034 if (!dev_is_pci(dev))
2035 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2036
579305f7
AW		 2037 return !pci_for_each_dma_alias(to_pci_dev(dev),
2038 domain_context_mapped_cb, iommu);
ba395927
KA		 2039}
2040
f532959b
FY		 2041/* Returns a number of VTD pages, but aligned to MM page size */
2042static inline unsigned long aligned_nrpages(unsigned long host_addr,
2043 size_t size)
2044{
2045 host_addr &= ~PAGE_MASK;
2046 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2047}
2048
6dd9a7c7
YS		 2049/* Return largest possible superpage level for a given mapping */
2050static inline int hardware_largepage_caps(struct dmar_domain *domain,
2051 unsigned long iov_pfn,
2052 unsigned long phy_pfn,
2053 unsigned long pages)
2054{
2055 int support, level = 1;
2056 unsigned long pfnmerge;
2057
2058 support = domain->iommu_superpage;
2059
2060 /* To use a large page, the virtual *and* physical addresses
2061 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2062 of them will mean we have to use smaller pages. So just
2063 merge them and check both at once. */
2064 pfnmerge = iov_pfn | phy_pfn;
2065
2066 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2067 pages >>= VTD_STRIDE_SHIFT;
2068 if (!pages)
2069 break;
2070 pfnmerge >>= VTD_STRIDE_SHIFT;
2071 level++;
2072 support--;
2073 }
2074 return level;
2075}
2076
9051aa02
DW		 2077static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2078 struct scatterlist *sg, unsigned long phys_pfn,
2079 unsigned long nr_pages, int prot)
e1605495
DW		 2080{
2081 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2082 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2083 unsigned long sg_res = 0;
6dd9a7c7
YS		 2084 unsigned int largepage_lvl = 0;
2085 unsigned long lvl_pages = 0;
e1605495 2086
162d1b10 2087 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2088
2089 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2090 return -EINVAL;
2091
2092 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2093
cc4f14aa
JL		 2094 if (!sg) {
2095 sg_res = nr_pages;
9051aa02
DW		 2096 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2097 }
2098
6dd9a7c7 2099 while (nr_pages > 0) {
c85994e4
DW		 2100 uint64_t tmp;
2101
e1605495 2102 if (!sg_res) {
f532959b 2103 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2104 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2105 sg->dma_length = sg->length;
db0fa0cb 2106 pteval = (sg_phys(sg) & PAGE_MASK) | prot;
6dd9a7c7 2107 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2108 }
6dd9a7c7 2109
e1605495 2110 if (!pte) {
6dd9a7c7
YS		 2111 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2112
5cf0a76f 2113 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2114 if (!pte)
2115 return -ENOMEM;
6dd9a7c7 2116 /* It is large page*/
6491d4d0 2117 if (largepage_lvl > 1) {
6dd9a7c7 2118 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2119 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2120 /*
2121 * Ensure that old small page tables are
2122 * removed to make room for superpage,
2123 * if they exist.
2124 */
6491d4d0 2125 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2126 iov_pfn + lvl_pages - 1);
6491d4d0 2127 } else {
6dd9a7c7 2128 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2129 }
6dd9a7c7 2130
e1605495
DW		 2131 }
2132 /* We don't need lock here, nobody else
2133 * touches the iova range
2134 */
7766a3fb 2135 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2136 if (tmp) {
1bf20f0d 2137 static int dumps = 5;
9f10e5bf
JR		 2138 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2139 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2140 if (dumps) {
2141 dumps--;
2142 debug_dma_dump_mappings(NULL);
2143 }
2144 WARN_ON(1);
2145 }
6dd9a7c7
YS		 2146
2147 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2148
2149 BUG_ON(nr_pages < lvl_pages);
2150 BUG_ON(sg_res < lvl_pages);
2151
2152 nr_pages -= lvl_pages;
2153 iov_pfn += lvl_pages;
2154 phys_pfn += lvl_pages;
2155 pteval += lvl_pages * VTD_PAGE_SIZE;
2156 sg_res -= lvl_pages;
2157
2158 /* If the next PTE would be the first in a new page, then we
2159 need to flush the cache on the entries we've just written.
2160 And then we'll need to recalculate 'pte', so clear it and
2161 let it get set again in the if (!pte) block above.
2162
2163 If we're done (!nr_pages) we need to flush the cache too.
2164
2165 Also if we've been setting superpages, we may need to
2166 recalculate 'pte' and switch back to smaller pages for the
2167 end of the mapping, if the trailing size is not enough to
2168 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2169 pte++;
6dd9a7c7
YS		 2170 if (!nr_pages || first_pte_in_page(pte) ||
2171 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2172 domain_flush_cache(domain, first_pte,
2173 (void *)pte - (void *)first_pte);
2174 pte = NULL;
2175 }
6dd9a7c7
YS		 2176
2177 if (!sg_res && nr_pages)
e1605495
DW		 2178 sg = sg_next(sg);
2179 }
2180 return 0;
2181}
2182
9051aa02
DW		 2183static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2184 struct scatterlist *sg, unsigned long nr_pages,
2185 int prot)
ba395927 2186{
9051aa02
DW		 2187 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2188}
6f6a00e4 2189
9051aa02
DW		 2190static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2191 unsigned long phys_pfn, unsigned long nr_pages,
2192 int prot)
2193{
2194 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2195}
2196
2452d9db 2197static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2198{
c7151a8d
WH		 2199 if (!iommu)
2200 return;
8c11e798
WH		 2201
2202 clear_context_table(iommu, bus, devfn);
2203 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2204 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2205 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2206}
2207
109b9b04
DW		 2208static inline void unlink_domain_info(struct device_domain_info *info)
2209{
2210 assert_spin_locked(&device_domain_lock);
2211 list_del(&info->link);
2212 list_del(&info->global);
2213 if (info->dev)
0bcb3e28 2214 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2215}
2216
ba395927
KA		 2217static void domain_remove_dev_info(struct dmar_domain *domain)
2218{
3a74ca01 2219 struct device_domain_info *info, *tmp;
fb170fb4 2220 unsigned long flags;
ba395927
KA		 2221
2222 spin_lock_irqsave(&device_domain_lock, flags);
76f45fe3 2223 list_for_each_entry_safe(info, tmp, &domain->devices, link)
127c7615 2224 __dmar_remove_one_dev_info(info);
ba395927
KA		 2225 spin_unlock_irqrestore(&device_domain_lock, flags);
2226}
2227
2228/*
2229 * find_domain
1525a29a 2230 * Note: we use struct device->archdata.iommu stores the info
ba395927 2231 */
1525a29a 2232static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2233{
2234 struct device_domain_info *info;
2235
2236 /* No lock here, assumes no domain exit in normal case */
1525a29a 2237 info = dev->archdata.iommu;
ba395927
KA		 2238 if (info)
2239 return info->domain;
2240 return NULL;
2241}
2242
5a8f40e8 2243static inline struct device_domain_info *
745f2586
JL		 2244dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2245{
2246 struct device_domain_info *info;
2247
2248 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2249 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2250 info->devfn == devfn)
5a8f40e8 2251 return info;
745f2586
JL		 2252
2253 return NULL;
2254}
2255
5db31569
JR		 2256static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2257 int bus, int devfn,
2258 struct device *dev,
2259 struct dmar_domain *domain)
745f2586 2260{
5a8f40e8 2261 struct dmar_domain *found = NULL;
745f2586
JL		 2262 struct device_domain_info *info;
2263 unsigned long flags;
d160aca5 2264 int ret;
745f2586
JL		 2265
2266 info = alloc_devinfo_mem();
2267 if (!info)
b718cd3d 2268 return NULL;
745f2586 2269
745f2586
JL		 2270 info->bus = bus;
2271 info->devfn = devfn;
fb0cc3aa
BH		 2272 info->ats.enabled = 0;
2273 info->ats.qdep = 0;
745f2586
JL		 2274 info->dev = dev;
2275 info->domain = domain;
5a8f40e8 2276 info->iommu = iommu;
745f2586
JL		 2277
2278 spin_lock_irqsave(&device_domain_lock, flags);
2279 if (dev)
0bcb3e28 2280 found = find_domain(dev);
f303e507
JR		 2281
2282 if (!found) {
5a8f40e8 2283 struct device_domain_info *info2;
41e80dca 2284 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
f303e507
JR		 2285 if (info2) {
2286 found = info2->domain;
2287 info2->dev = dev;
2288 }
5a8f40e8 2289 }
f303e507 2290
745f2586
JL		 2291 if (found) {
2292 spin_unlock_irqrestore(&device_domain_lock, flags);
2293 free_devinfo_mem(info);
b718cd3d
DW		 2294 /* Caller must free the original domain */
2295 return found;
745f2586
JL		 2296 }
2297
d160aca5
JR		 2298 spin_lock(&iommu->lock);
2299 ret = domain_attach_iommu(domain, iommu);
2300 spin_unlock(&iommu->lock);
2301
2302 if (ret) {
c6c2cebd
JR		 2303 spin_unlock_irqrestore(&device_domain_lock, flags);
2304 return NULL;
2305 }
c6c2cebd 2306
b718cd3d
DW		 2307 list_add(&info->link, &domain->devices);
2308 list_add(&info->global, &device_domain_list);
2309 if (dev)
2310 dev->archdata.iommu = info;
2311 spin_unlock_irqrestore(&device_domain_lock, flags);
2312
cc4e2575
JR		 2313 if (dev && domain_context_mapping(domain, dev)) {
2314 pr_err("Domain context map for %s failed\n", dev_name(dev));
e6de0f8d 2315 dmar_remove_one_dev_info(domain, dev);
cc4e2575
JR		 2316 return NULL;
2317 }
2318
b718cd3d 2319 return domain;
745f2586
JL		 2320}
2321
579305f7
AW		 2322static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2323{
2324 *(u16 *)opaque = alias;
2325 return 0;
2326}
2327
ba395927 2328/* domain is initialized */
146922ec 2329static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2330{
cc4e2575 2331 struct device_domain_info *info = NULL;
579305f7
AW		 2332 struct dmar_domain *domain, *tmp;
2333 struct intel_iommu *iommu;
08a7f456 2334 u16 req_id, dma_alias;
ba395927 2335 unsigned long flags;
aa4d066a 2336 u8 bus, devfn;
ba395927 2337
146922ec 2338 domain = find_domain(dev);
ba395927
KA		 2339 if (domain)
2340 return domain;
2341
579305f7
AW		 2342 iommu = device_to_iommu(dev, &bus, &devfn);
2343 if (!iommu)
2344 return NULL;
2345
08a7f456
JR		 2346 req_id = ((u16)bus << 8) | devfn;
2347
146922ec
DW		 2348 if (dev_is_pci(dev)) {
2349 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2350
579305f7
AW		 2351 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2352
2353 spin_lock_irqsave(&device_domain_lock, flags);
2354 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2355 PCI_BUS_NUM(dma_alias),
2356 dma_alias & 0xff);
2357 if (info) {
2358 iommu = info->iommu;
2359 domain = info->domain;
5a8f40e8 2360 }
579305f7 2361 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2362
579305f7
AW		 2363 /* DMA alias already has a domain, uses it */
2364 if (info)
2365 goto found_domain;
2366 }
ba395927 2367
146922ec 2368 /* Allocate and initialize new domain for the device */
ab8dfe25 2369 domain = alloc_domain(0);
745f2586 2370 if (!domain)
579305f7 2371 return NULL;
dc534b25 2372 if (domain_init(domain, iommu, gaw)) {
579305f7
AW		 2373 domain_exit(domain);
2374 return NULL;
2c2e2c38 2375 }
ba395927 2376
579305f7 2377 /* register PCI DMA alias device */
08a7f456 2378 if (req_id != dma_alias && dev_is_pci(dev)) {
5db31569
JR		 2379 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2380 dma_alias & 0xff, NULL, domain);
579305f7
AW		 2381
2382 if (!tmp || tmp != domain) {
2383 domain_exit(domain);
2384 domain = tmp;
2385 }
2386
b718cd3d 2387 if (!domain)
579305f7 2388 return NULL;
ba395927
KA		 2389 }
2390
2391found_domain:
5db31569 2392 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
579305f7
AW		 2393
2394 if (!tmp || tmp != domain) {
2395 domain_exit(domain);
2396 domain = tmp;
2397 }
b718cd3d
DW		 2398
2399 return domain;
ba395927
KA		 2400}
2401
2c2e2c38 2402static int iommu_identity_mapping;
e0fc7e0b
DW		 2403#define IDENTMAP_ALL 1
2404#define IDENTMAP_GFX 2
2405#define IDENTMAP_AZALIA 4
2c2e2c38 2406
b213203e
DW		 2407static int iommu_domain_identity_map(struct dmar_domain *domain,
2408 unsigned long long start,
2409 unsigned long long end)
ba395927 2410{
c5395d5c
DW		 2411 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2412 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2413
2414 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2415 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2416 pr_err("Reserving iova failed\n");
b213203e 2417 return -ENOMEM;
ba395927
KA		 2418 }
2419
af1089ce 2420 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
ba395927
KA		 2421 /*
2422 * RMRR range might have overlap with physical memory range,
2423 * clear it first
2424 */
c5395d5c 2425 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2426
c5395d5c
DW		 2427 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2428 last_vpfn - first_vpfn + 1,
61df7443 2429 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2430}
2431
0b9d9753 2432static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2433 unsigned long long start,
2434 unsigned long long end)
2435{
2436 struct dmar_domain *domain;
2437 int ret;
2438
0b9d9753 2439 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2440 if (!domain)
2441 return -ENOMEM;
2442
19943b0e
DW		 2443 /* For _hardware_ passthrough, don't bother. But for software
2444 passthrough, we do it anyway -- it may indicate a memory
2445 range which is reserved in E820, so which didn't get set
2446 up to start with in si_domain */
2447 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2448 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2449 dev_name(dev), start, end);
19943b0e
DW		 2450 return 0;
2451 }
2452
9f10e5bf
JR		 2453 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2454 dev_name(dev), start, end);
2455
5595b528
DW		 2456 if (end < start) {
2457 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2458 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2459 dmi_get_system_info(DMI_BIOS_VENDOR),
2460 dmi_get_system_info(DMI_BIOS_VERSION),
2461 dmi_get_system_info(DMI_PRODUCT_VERSION));
2462 ret = -EIO;
2463 goto error;
2464 }
2465
2ff729f5
DW		 2466 if (end >> agaw_to_width(domain->agaw)) {
2467 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2468 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2469 agaw_to_width(domain->agaw),
2470 dmi_get_system_info(DMI_BIOS_VENDOR),
2471 dmi_get_system_info(DMI_BIOS_VERSION),
2472 dmi_get_system_info(DMI_PRODUCT_VERSION));
2473 ret = -EIO;
2474 goto error;
2475 }
19943b0e 2476
b213203e 2477 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2478 if (ret)
2479 goto error;
2480
b213203e
DW		 2481 return 0;
2482
2483 error:
ba395927
KA		 2484 domain_exit(domain);
2485 return ret;
ba395927
KA		 2486}
2487
2488static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2489 struct device *dev)
ba395927 2490{
0b9d9753 2491 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2492 return 0;
0b9d9753
DW		 2493 return iommu_prepare_identity_map(dev, rmrr->base_address,
2494 rmrr->end_address);
ba395927
KA		 2495}
2496
d3f13810 2497#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2498static inline void iommu_prepare_isa(void)
2499{
2500 struct pci_dev *pdev;
2501 int ret;
2502
2503 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2504 if (!pdev)
2505 return;
2506
9f10e5bf 2507 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2508 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2509
2510 if (ret)
9f10e5bf 2511 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2512
9b27e82d 2513 pci_dev_put(pdev);
49a0429e
KA		 2514}
2515#else
2516static inline void iommu_prepare_isa(void)
2517{
2518 return;
2519}
d3f13810 2520#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2521
2c2e2c38 2522static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2523
071e1374 2524static int __init si_domain_init(int hw)
2c2e2c38 2525{
c7ab48d2 2526 int nid, ret = 0;
2c2e2c38 2527
ab8dfe25 2528 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2529 if (!si_domain)
2530 return -EFAULT;
2531
2c2e2c38
FY		 2532 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2533 domain_exit(si_domain);
2534 return -EFAULT;
2535 }
2536
0dc79715 2537 pr_debug("Identity mapping domain allocated\n");
2c2e2c38 2538
19943b0e
DW		 2539 if (hw)
2540 return 0;
2541
c7ab48d2 2542 for_each_online_node(nid) {
5dfe8660
TH		 2543 unsigned long start_pfn, end_pfn;
2544 int i;
2545
2546 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2547 ret = iommu_domain_identity_map(si_domain,
2548 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2549 if (ret)
2550 return ret;
2551 }
c7ab48d2
DW		 2552 }
2553
2c2e2c38
FY		 2554 return 0;
2555}
2556
9b226624 2557static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2558{
2559 struct device_domain_info *info;
2560
2561 if (likely(!iommu_identity_mapping))
2562 return 0;
2563
9b226624 2564 info = dev->archdata.iommu;
cb452a40
MT		 2565 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2566 return (info->domain == si_domain);
2c2e2c38 2567
2c2e2c38
FY		 2568 return 0;
2569}
2570
28ccce0d 2571static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2c2e2c38 2572{
0ac72664 2573 struct dmar_domain *ndomain;
5a8f40e8 2574 struct intel_iommu *iommu;
156baca8 2575 u8 bus, devfn;
2c2e2c38 2576
5913c9bf 2577 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2578 if (!iommu)
2579 return -ENODEV;
2580
5db31569 2581 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2582 if (ndomain != domain)
2583 return -EBUSY;
2c2e2c38
FY		 2584
2585 return 0;
2586}
2587
0b9d9753 2588static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2589{
2590 struct dmar_rmrr_unit *rmrr;
832bd858 2591 struct device *tmp;
ea2447f7
TM		 2592 int i;
2593
0e242612 2594 rcu_read_lock();
ea2447f7 2595 for_each_rmrr_units(rmrr) {
b683b230
JL		 2596 /*
2597 * Return TRUE if this RMRR contains the device that
2598 * is passed in.
2599 */
2600 for_each_active_dev_scope(rmrr->devices,
2601 rmrr->devices_cnt, i, tmp)
0b9d9753 2602 if (tmp == dev) {
0e242612 2603 rcu_read_unlock();
ea2447f7 2604 return true;
b683b230 2605 }
ea2447f7 2606 }
0e242612 2607 rcu_read_unlock();
ea2447f7
TM		 2608 return false;
2609}
2610
c875d2c1
AW		 2611/*
2612 * There are a couple cases where we need to restrict the functionality of
2613 * devices associated with RMRRs. The first is when evaluating a device for
2614 * identity mapping because problems exist when devices are moved in and out
2615 * of domains and their respective RMRR information is lost. This means that
2616 * a device with associated RMRRs will never be in a "passthrough" domain.
2617 * The second is use of the device through the IOMMU API. This interface
2618 * expects to have full control of the IOVA space for the device. We cannot
2619 * satisfy both the requirement that RMRR access is maintained and have an
2620 * unencumbered IOVA space. We also have no ability to quiesce the device's
2621 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2622 * We therefore prevent devices associated with an RMRR from participating in
2623 * the IOMMU API, which eliminates them from device assignment.
2624 *
2625 * In both cases we assume that PCI USB devices with RMRRs have them largely
2626 * for historical reasons and that the RMRR space is not actively used post
2627 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2628 *
2629 * The same exception is made for graphics devices, with the requirement that
2630 * any use of the RMRR regions will be torn down before assigning the device
2631 * to a guest.
c875d2c1
AW		 2632 */
2633static bool device_is_rmrr_locked(struct device *dev)
2634{
2635 if (!device_has_rmrr(dev))
2636 return false;
2637
2638 if (dev_is_pci(dev)) {
2639 struct pci_dev *pdev = to_pci_dev(dev);
2640
18436afd 2641 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2642 return false;
2643 }
2644
2645 return true;
2646}
2647
3bdb2591 2648static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2649{
ea2447f7 2650
3bdb2591
DW		 2651 if (dev_is_pci(dev)) {
2652 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2653
c875d2c1 2654 if (device_is_rmrr_locked(dev))
3bdb2591 2655 return 0;
e0fc7e0b 2656
3bdb2591
DW		 2657 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2658 return 1;
e0fc7e0b 2659
3bdb2591
DW		 2660 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2661 return 1;
6941af28 2662
3bdb2591 2663 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2664 return 0;
3bdb2591
DW		 2665
2666 /*
2667 * We want to start off with all devices in the 1:1 domain, and
2668 * take them out later if we find they can't access all of memory.
2669 *
2670 * However, we can't do this for PCI devices behind bridges,
2671 * because all PCI devices behind the same bridge will end up
2672 * with the same source-id on their transactions.
2673 *
2674 * Practically speaking, we can't change things around for these
2675 * devices at run-time, because we can't be sure there'll be no
2676 * DMA transactions in flight for any of their siblings.
2677 *
2678 * So PCI devices (unless they're on the root bus) as well as
2679 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2680 * the 1:1 domain, just in _case_ one of their siblings turns out
2681 * not to be able to map all of memory.
2682 */
2683 if (!pci_is_pcie(pdev)) {
2684 if (!pci_is_root_bus(pdev->bus))
2685 return 0;
2686 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2687 return 0;
2688 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2689 return 0;
3bdb2591
DW		 2690 } else {
2691 if (device_has_rmrr(dev))
2692 return 0;
2693 }
3dfc813d 2694
3bdb2591 2695 /*
3dfc813d 2696 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2697 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2698 * take them out of the 1:1 domain later.
2699 */
8fcc5372
CW		 2700 if (!startup) {
2701 /*
2702 * If the device's dma_mask is less than the system's memory
2703 * size then this is not a candidate for identity mapping.
2704 */
3bdb2591 2705 u64 dma_mask = *dev->dma_mask;
8fcc5372 2706
3bdb2591
DW		 2707 if (dev->coherent_dma_mask &&
2708 dev->coherent_dma_mask < dma_mask)
2709 dma_mask = dev->coherent_dma_mask;
8fcc5372 2710
3bdb2591 2711 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2712 }
6941af28
DW		 2713
2714 return 1;
2715}
2716
cf04eee8
DW		 2717static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2718{
2719 int ret;
2720
2721 if (!iommu_should_identity_map(dev, 1))
2722 return 0;
2723
28ccce0d 2724 ret = domain_add_dev_info(si_domain, dev);
cf04eee8 2725 if (!ret)
9f10e5bf
JR		 2726 pr_info("%s identity mapping for device %s\n",
2727 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2728 else if (ret == -ENODEV)
2729 /* device not associated with an iommu */
2730 ret = 0;
2731
2732 return ret;
2733}
2734
2735
071e1374 2736static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2737{
2c2e2c38 2738 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2739 struct dmar_drhd_unit *drhd;
2740 struct intel_iommu *iommu;
2741 struct device *dev;
2742 int i;
2743 int ret = 0;
2c2e2c38 2744
2c2e2c38 2745 for_each_pci_dev(pdev) {
cf04eee8
DW		 2746 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2747 if (ret)
2748 return ret;
2749 }
2750
2751 for_each_active_iommu(iommu, drhd)
2752 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2753 struct acpi_device_physical_node *pn;
2754 struct acpi_device *adev;
2755
2756 if (dev->bus != &acpi_bus_type)
2757 continue;
86080ccc 2758
cf04eee8
DW		 2759 adev= to_acpi_device(dev);
2760 mutex_lock(&adev->physical_node_lock);
2761 list_for_each_entry(pn, &adev->physical_node_list, node) {
2762 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2763 if (ret)
2764 break;
eae460b6 2765 }
cf04eee8
DW		 2766 mutex_unlock(&adev->physical_node_lock);
2767 if (ret)
2768 return ret;
62edf5dc 2769 }
2c2e2c38
FY		 2770
2771 return 0;
2772}
2773
ffebeb46
JL		 2774static void intel_iommu_init_qi(struct intel_iommu *iommu)
2775{
2776 /*
2777 * Start from the sane iommu hardware state.
2778 * If the queued invalidation is already initialized by us
2779 * (for example, while enabling interrupt-remapping) then
2780 * we got the things already rolling from a sane state.
2781 */
2782 if (!iommu->qi) {
2783 /*
2784 * Clear any previous faults.
2785 */
2786 dmar_fault(-1, iommu);
2787 /*
2788 * Disable queued invalidation if supported and already enabled
2789 * before OS handover.
2790 */
2791 dmar_disable_qi(iommu);
2792 }
2793
2794 if (dmar_enable_qi(iommu)) {
2795 /*
2796 * Queued Invalidate not enabled, use Register Based Invalidate
2797 */
2798 iommu->flush.flush_context = __iommu_flush_context;
2799 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2800 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2801 iommu->name);
2802 } else {
2803 iommu->flush.flush_context = qi_flush_context;
2804 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2805 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2806 }
2807}
2808
091d42e4 2809static int copy_context_table(struct intel_iommu *iommu,
543c8dcf 2810 struct root_entry __iomem *old_re,
091d42e4
JR		 2811 struct context_entry **tbl,
2812 int bus, bool ext)
2813{
dbcd861f 2814 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
543c8dcf
JR		 2815 struct context_entry __iomem *old_ce = NULL;
2816 struct context_entry *new_ce = NULL, ce;
2817 struct root_entry re;
091d42e4
JR		 2818 phys_addr_t old_ce_phys;
2819
2820 tbl_idx = ext ? bus * 2 : bus;
543c8dcf 2821 memcpy_fromio(&re, old_re, sizeof(re));
091d42e4
JR		 2822
2823 for (devfn = 0; devfn < 256; devfn++) {
2824 /* First calculate the correct index */
2825 idx = (ext ? devfn * 2 : devfn) % 256;
2826
2827 if (idx == 0) {
2828 /* First save what we may have and clean up */
2829 if (new_ce) {
2830 tbl[tbl_idx] = new_ce;
2831 __iommu_flush_cache(iommu, new_ce,
2832 VTD_PAGE_SIZE);
2833 pos = 1;
2834 }
2835
2836 if (old_ce)
2837 iounmap(old_ce);
2838
2839 ret = 0;
2840 if (devfn < 0x80)
543c8dcf 2841 old_ce_phys = root_entry_lctp(&re);
091d42e4 2842 else
543c8dcf 2843 old_ce_phys = root_entry_uctp(&re);
091d42e4
JR		 2844
2845 if (!old_ce_phys) {
2846 if (ext && devfn == 0) {
2847 /* No LCTP, try UCTP */
2848 devfn = 0x7f;
2849 continue;
2850 } else {
2851 goto out;
2852 }
2853 }
2854
2855 ret = -ENOMEM;
2856 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2857 if (!old_ce)
2858 goto out;
2859
2860 new_ce = alloc_pgtable_page(iommu->node);
2861 if (!new_ce)
2862 goto out_unmap;
2863
2864 ret = 0;
2865 }
2866
2867 /* Now copy the context entry */
543c8dcf 2868 memcpy_fromio(&ce, old_ce + idx, sizeof(ce));
091d42e4 2869
cf484d0e 2870 if (!__context_present(&ce))
091d42e4
JR		 2871 continue;
2872
dbcd861f
JR		 2873 did = context_domain_id(&ce);
2874 if (did >= 0 && did < cap_ndoms(iommu->cap))
2875 set_bit(did, iommu->domain_ids);
2876
cf484d0e
JR		 2877 /*
2878 * We need a marker for copied context entries. This
2879 * marker needs to work for the old format as well as
2880 * for extended context entries.
2881 *
2882 * Bit 67 of the context entry is used. In the old
2883 * format this bit is available to software, in the
2884 * extended format it is the PGE bit, but PGE is ignored
2885 * by HW if PASIDs are disabled (and thus still
2886 * available).
2887 *
2888 * So disable PASIDs first and then mark the entry
2889 * copied. This means that we don't copy PASID
2890 * translations from the old kernel, but this is fine as
2891 * faults there are not fatal.
2892 */
2893 context_clear_pasid_enable(&ce);
2894 context_set_copied(&ce);
2895
091d42e4
JR		 2896 new_ce[idx] = ce;
2897 }
2898
2899 tbl[tbl_idx + pos] = new_ce;
2900
2901 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2902
2903out_unmap:
2904 iounmap(old_ce);
2905
2906out:
2907 return ret;
2908}
2909
2910static int copy_translation_tables(struct intel_iommu *iommu)
2911{
543c8dcf 2912 struct root_entry __iomem *old_rt;
091d42e4 2913 struct context_entry **ctxt_tbls;
091d42e4
JR		 2914 phys_addr_t old_rt_phys;
2915 int ctxt_table_entries;
2916 unsigned long flags;
2917 u64 rtaddr_reg;
2918 int bus, ret;
c3361f2f 2919 bool new_ext, ext;
091d42e4
JR		 2920
2921 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2922 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2923 new_ext = !!ecap_ecs(iommu->ecap);
2924
2925 /*
2926 * The RTT bit can only be changed when translation is disabled,
2927 * but disabling translation means to open a window for data
2928 * corruption. So bail out and don't copy anything if we would
2929 * have to change the bit.
2930 */
2931 if (new_ext != ext)
2932 return -EINVAL;
091d42e4
JR		 2933
2934 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2935 if (!old_rt_phys)
2936 return -EINVAL;
2937
2938 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2939 if (!old_rt)
2940 return -ENOMEM;
2941
2942 /* This is too big for the stack - allocate it from slab */
2943 ctxt_table_entries = ext ? 512 : 256;
2944 ret = -ENOMEM;
2945 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2946 if (!ctxt_tbls)
2947 goto out_unmap;
2948
2949 for (bus = 0; bus < 256; bus++) {
2950 ret = copy_context_table(iommu, &old_rt[bus],
2951 ctxt_tbls, bus, ext);
2952 if (ret) {
2953 pr_err("%s: Failed to copy context table for bus %d\n",
2954 iommu->name, bus);
2955 continue;
2956 }
2957 }
2958
2959 spin_lock_irqsave(&iommu->lock, flags);
2960
2961 /* Context tables are copied, now write them to the root_entry table */
2962 for (bus = 0; bus < 256; bus++) {
2963 int idx = ext ? bus * 2 : bus;
2964 u64 val;
2965
2966 if (ctxt_tbls[idx]) {
2967 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2968 iommu->root_entry[bus].lo = val;
2969 }
2970
2971 if (!ext || !ctxt_tbls[idx + 1])
2972 continue;
2973
2974 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2975 iommu->root_entry[bus].hi = val;
2976 }
2977
2978 spin_unlock_irqrestore(&iommu->lock, flags);
2979
2980 kfree(ctxt_tbls);
2981
2982 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2983
2984 ret = 0;
2985
2986out_unmap:
2987 iounmap(old_rt);
2988
2989 return ret;
2990}
2991
b779260b 2992static int __init init_dmars(void)
ba395927
KA		 2993{
2994 struct dmar_drhd_unit *drhd;
2995 struct dmar_rmrr_unit *rmrr;
a87f4918 2996 bool copied_tables = false;
832bd858 2997 struct device *dev;
ba395927 2998 struct intel_iommu *iommu;
9d783ba0 2999 int i, ret;
2c2e2c38 3000
ba395927
KA		 3001 /*
3002 * for each drhd
3003 * allocate root
3004 * initialize and program root entry to not present
3005 * endfor
3006 */
3007 for_each_drhd_unit(drhd) {
5e0d2a6f 3008 /*
3009 * lock not needed as this is only incremented in the single
3010 * threaded kernel __init code path all other access are read
3011 * only
3012 */
78d8e704 3013 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3014 g_num_of_iommus++;
3015 continue;
3016 }
9f10e5bf 3017 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3018 }
3019
ffebeb46
JL		 3020 /* Preallocate enough resources for IOMMU hot-addition */
3021 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3022 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3023
d9630fe9
WH		 3024 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3025 GFP_KERNEL);
3026 if (!g_iommus) {
9f10e5bf 3027 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3028 ret = -ENOMEM;
3029 goto error;
3030 }
3031
80b20dd8 3032 deferred_flush = kzalloc(g_num_of_iommus *
3033 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3034 if (!deferred_flush) {
5e0d2a6f 3035 ret = -ENOMEM;
989d51fc 3036 goto free_g_iommus;
5e0d2a6f 3037 }
3038
7c919779 3039 for_each_active_iommu(iommu, drhd) {
d9630fe9 3040 g_iommus[iommu->seq_id] = iommu;
ba395927 3041
b63d80d1
JR		 3042 intel_iommu_init_qi(iommu);
3043
e61d98d8
SS		 3044 ret = iommu_init_domains(iommu);
3045 if (ret)
989d51fc 3046 goto free_iommu;
e61d98d8 3047
4158c2ec
JR		 3048 init_translation_status(iommu);
3049
091d42e4
JR		 3050 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3051 iommu_disable_translation(iommu);
3052 clear_translation_pre_enabled(iommu);
3053 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3054 iommu->name);
3055 }
4158c2ec 3056
ba395927
KA		 3057 /*
3058 * TBD:
3059 * we could share the same root & context tables
25985edc 3060 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3061 */
3062 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3063 if (ret)
989d51fc 3064 goto free_iommu;
5f0a7f76 3065
091d42e4
JR		 3066 if (translation_pre_enabled(iommu)) {
3067 pr_info("Translation already enabled - trying to copy translation structures\n");
3068
3069 ret = copy_translation_tables(iommu);
3070 if (ret) {
3071 /*
3072 * We found the IOMMU with translation
3073 * enabled - but failed to copy over the
3074 * old root-entry table. Try to proceed
3075 * by disabling translation now and
3076 * allocating a clean root-entry table.
3077 * This might cause DMAR faults, but
3078 * probably the dump will still succeed.
3079 */
3080 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3081 iommu->name);
3082 iommu_disable_translation(iommu);
3083 clear_translation_pre_enabled(iommu);
3084 } else {
3085 pr_info("Copied translation tables from previous kernel for %s\n",
3086 iommu->name);
a87f4918 3087 copied_tables = true;
091d42e4
JR		 3088 }
3089 }
3090
5f0a7f76
JR		 3091 iommu_flush_write_buffer(iommu);
3092 iommu_set_root_entry(iommu);
3093 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3094 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3095
4ed0d3e6 3096 if (!ecap_pass_through(iommu->ecap))
19943b0e 3097 hw_pass_through = 0;
ba395927
KA		 3098 }
3099
19943b0e 3100 if (iommu_pass_through)
e0fc7e0b
DW		 3101 iommu_identity_mapping |= IDENTMAP_ALL;
3102
d3f13810 3103#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3104 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3105#endif
e0fc7e0b 3106
86080ccc
JR		 3107 if (iommu_identity_mapping) {
3108 ret = si_domain_init(hw_pass_through);
3109 if (ret)
3110 goto free_iommu;
3111 }
3112
e0fc7e0b
DW		 3113 check_tylersburg_isoch();
3114
a87f4918
JR		 3115 /*
3116 * If we copied translations from a previous kernel in the kdump
3117 * case, we can not assign the devices to domains now, as that
3118 * would eliminate the old mappings. So skip this part and defer
3119 * the assignment to device driver initialization time.
3120 */
3121 if (copied_tables)
3122 goto domains_done;
3123
ba395927 3124 /*
19943b0e
DW		 3125 * If pass through is not set or not enabled, setup context entries for
3126 * identity mappings for rmrr, gfx, and isa and may fall back to static
3127 * identity mapping if iommu_identity_mapping is set.
ba395927 3128 */
19943b0e
DW		 3129 if (iommu_identity_mapping) {
3130 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3131 if (ret) {
9f10e5bf 3132 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3133 goto free_iommu;
ba395927
KA		 3134 }
3135 }
ba395927 3136 /*
19943b0e
DW		 3137 * For each rmrr
3138 * for each dev attached to rmrr
3139 * do
3140 * locate drhd for dev, alloc domain for dev
3141 * allocate free domain
3142 * allocate page table entries for rmrr
3143 * if context not allocated for bus
3144 * allocate and init context
3145 * set present in root table for this bus
3146 * init context with domain, translation etc
3147 * endfor
3148 * endfor
ba395927 3149 */
9f10e5bf 3150 pr_info("Setting RMRR:\n");
19943b0e 3151 for_each_rmrr_units(rmrr) {
b683b230
JL		 3152 /* some BIOS lists non-exist devices in DMAR table. */
3153 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3154 i, dev) {
0b9d9753 3155 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3156 if (ret)
9f10e5bf 3157 pr_err("Mapping reserved region failed\n");
ba395927 3158 }
4ed0d3e6 3159 }
49a0429e 3160
19943b0e
DW		 3161 iommu_prepare_isa();
3162
a87f4918
JR		 3163domains_done:
3164
ba395927
KA		 3165 /*
3166 * for each drhd
3167 * enable fault log
3168 * global invalidate context cache
3169 * global invalidate iotlb
3170 * enable translation
3171 */
7c919779 3172 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3173 if (drhd->ignored) {
3174 /*
3175 * we always have to disable PMRs or DMA may fail on
3176 * this device
3177 */
3178 if (force_on)
7c919779 3179 iommu_disable_protect_mem_regions(iommu);
ba395927 3180 continue;
51a63e67 3181 }
ba395927
KA		 3182
3183 iommu_flush_write_buffer(iommu);
3184
3460a6d9
KA		 3185 ret = dmar_set_interrupt(iommu);
3186 if (ret)
989d51fc 3187 goto free_iommu;
3460a6d9 3188
8939ddf6
JR		 3189 if (!translation_pre_enabled(iommu))
3190 iommu_enable_translation(iommu);
3191
b94996c9 3192 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3193 }
3194
3195 return 0;
989d51fc
JL		 3196
3197free_iommu:
ffebeb46
JL		 3198 for_each_active_iommu(iommu, drhd) {
3199 disable_dmar_iommu(iommu);
a868e6b7 3200 free_dmar_iommu(iommu);
ffebeb46 3201 }
9bdc531e 3202 kfree(deferred_flush);
989d51fc 3203free_g_iommus:
d9630fe9 3204 kfree(g_iommus);
989d51fc 3205error:
ba395927
KA		 3206 return ret;
3207}
3208
5a5e02a6 3209/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3210static struct iova *intel_alloc_iova(struct device *dev,
3211 struct dmar_domain *domain,
3212 unsigned long nrpages, uint64_t dma_mask)
ba395927 3213{
ba395927 3214 struct iova *iova = NULL;
ba395927 3215
875764de
DW		 3216 /* Restrict dma_mask to the width that the iommu can handle */
3217 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3218
3219 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3220 /*
3221 * First try to allocate an io virtual address in
284901a9 3222 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3223 * from higher range
ba395927 3224 */
875764de
DW		 3225 iova = alloc_iova(&domain->iovad, nrpages,
3226 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3227 if (iova)
3228 return iova;
3229 }
3230 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3231 if (unlikely(!iova)) {
9f10e5bf 3232 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3233 nrpages, dev_name(dev));
f76aec76
KA		 3234 return NULL;
3235 }
3236
3237 return iova;
3238}
3239
d4b709f4 3240static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3241{
3242 struct dmar_domain *domain;
f76aec76 3243
d4b709f4 3244 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3245 if (!domain) {
9f10e5bf 3246 pr_err("Allocating domain for %s failed\n",
d4b709f4 3247 dev_name(dev));
4fe05bbc 3248 return NULL;
ba395927
KA		 3249 }
3250
f76aec76
KA		 3251 return domain;
3252}
3253
d4b709f4 3254static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3255{
3256 struct device_domain_info *info;
3257
3258 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3259 info = dev->archdata.iommu;
147202aa
DW		 3260 if (likely(info))
3261 return info->domain;
3262
3263 return __get_valid_domain_for_dev(dev);
3264}
3265
ecb509ec 3266/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3267static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3268{
3269 int found;
3270
3d89194a 3271 if (iommu_dummy(dev))
1e4c64c4
DW		 3272 return 1;
3273
2c2e2c38 3274 if (!iommu_identity_mapping)
1e4c64c4 3275 return 0;
2c2e2c38 3276
9b226624 3277 found = identity_mapping(dev);
2c2e2c38 3278 if (found) {
ecb509ec 3279 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3280 return 1;
3281 else {
3282 /*
3283 * 32 bit DMA is removed from si_domain and fall back
3284 * to non-identity mapping.
3285 */
e6de0f8d 3286 dmar_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3287 pr_info("32bit %s uses non-identity mapping\n",
3288 dev_name(dev));
2c2e2c38
FY		 3289 return 0;
3290 }
3291 } else {
3292 /*
3293 * In case of a detached 64 bit DMA device from vm, the device
3294 * is put into si_domain for identity mapping.
3295 */
ecb509ec 3296 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3297 int ret;
28ccce0d 3298 ret = domain_add_dev_info(si_domain, dev);
2c2e2c38 3299 if (!ret) {
9f10e5bf
JR		 3300 pr_info("64bit %s uses identity mapping\n",
3301 dev_name(dev));
2c2e2c38
FY		 3302 return 1;
3303 }
3304 }
3305 }
3306
1e4c64c4 3307 return 0;
2c2e2c38
FY		 3308}
3309
5040a918 3310static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3311 size_t size, int dir, u64 dma_mask)
f76aec76 3312{
f76aec76 3313 struct dmar_domain *domain;
5b6985ce 3314 phys_addr_t start_paddr;
f76aec76
KA		 3315 struct iova *iova;
3316 int prot = 0;
6865f0d1 3317 int ret;
8c11e798 3318 struct intel_iommu *iommu;
33041ec0 3319 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3320
3321 BUG_ON(dir == DMA_NONE);
2c2e2c38 3322
5040a918 3323 if (iommu_no_mapping(dev))
6865f0d1 3324 return paddr;
f76aec76 3325
5040a918 3326 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3327 if (!domain)
3328 return 0;
3329
8c11e798 3330 iommu = domain_get_iommu(domain);
88cb6a74 3331 size = aligned_nrpages(paddr, size);
f76aec76 3332
5040a918 3333 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3334 if (!iova)
3335 goto error;
3336
ba395927
KA		 3337 /*
3338 * Check if DMAR supports zero-length reads on write only
3339 * mappings..
3340 */
3341 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3342 !cap_zlr(iommu->cap))
ba395927
KA		 3343 prot |= DMA_PTE_READ;
3344 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3345 prot |= DMA_PTE_WRITE;
3346 /*
6865f0d1 3347 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3348 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3349 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3350 * is not a big problem
3351 */
0ab36de2 3352 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3353 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3354 if (ret)
3355 goto error;
3356
1f0ef2aa
DW		 3357 /* it's a non-present to present mapping. Only flush if caching mode */
3358 if (cap_caching_mode(iommu->cap))
a1ddcbe9
JR		 3359 iommu_flush_iotlb_psi(iommu, domain,
3360 mm_to_dma_pfn(iova->pfn_lo),
3361 size, 0, 1);
1f0ef2aa 3362 else
8c11e798 3363 iommu_flush_write_buffer(iommu);
f76aec76 3364
03d6a246
DW		 3365 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3366 start_paddr += paddr & ~PAGE_MASK;
3367 return start_paddr;
ba395927 3368
ba395927 3369error:
f76aec76
KA		 3370 if (iova)
3371 __free_iova(&domain->iovad, iova);
9f10e5bf 3372 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3373 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3374 return 0;
3375}
3376
ffbbef5c
FT		 3377static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3378 unsigned long offset, size_t size,
3379 enum dma_data_direction dir,
3380 struct dma_attrs *attrs)
bb9e6d65 3381{
ffbbef5c 3382 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3383 dir, *dev->dma_mask);
bb9e6d65
FT		 3384}
3385
5e0d2a6f 3386static void flush_unmaps(void)
3387{
80b20dd8 3388 int i, j;
5e0d2a6f 3389
5e0d2a6f 3390 timer_on = 0;
3391
3392 /* just flush them all */
3393 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3394 struct intel_iommu *iommu = g_iommus[i];
3395 if (!iommu)
3396 continue;
c42d9f32 3397
9dd2fe89
YZ		 3398 if (!deferred_flush[i].next)
3399 continue;
3400
78d5f0f5
NA		 3401 /* In caching mode, global flushes turn emulation expensive */
3402 if (!cap_caching_mode(iommu->cap))
3403 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3404 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3405 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3406 unsigned long mask;
3407 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3408 struct dmar_domain *domain = deferred_flush[i].domain[j];
3409
3410 /* On real hardware multiple invalidations are expensive */
3411 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3412 iommu_flush_iotlb_psi(iommu, domain,
a156ef99 3413 iova->pfn_lo, iova_size(iova),
ea8ea460 3414 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3415 else {
a156ef99 3416 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3417 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3418 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3419 }
93a23a72 3420 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3421 if (deferred_flush[i].freelist[j])
3422 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3423 }
9dd2fe89 3424 deferred_flush[i].next = 0;
5e0d2a6f 3425 }
3426
5e0d2a6f 3427 list_size = 0;
5e0d2a6f 3428}
3429
3430static void flush_unmaps_timeout(unsigned long data)
3431{
80b20dd8 3432 unsigned long flags;
3433
3434 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3435 flush_unmaps();
80b20dd8 3436 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3437}
3438
ea8ea460 3439static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3440{
3441 unsigned long flags;
80b20dd8 3442 int next, iommu_id;
8c11e798 3443 struct intel_iommu *iommu;
5e0d2a6f 3444
3445 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3446 if (list_size == HIGH_WATER_MARK)
3447 flush_unmaps();
3448
8c11e798
WH		 3449 iommu = domain_get_iommu(dom);
3450 iommu_id = iommu->seq_id;
c42d9f32 3451
80b20dd8 3452 next = deferred_flush[iommu_id].next;
3453 deferred_flush[iommu_id].domain[next] = dom;
3454 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3455 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3456 deferred_flush[iommu_id].next++;
5e0d2a6f 3457
3458 if (!timer_on) {
3459 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3460 timer_on = 1;
3461 }
3462 list_size++;
3463 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3464}
3465
d41a4adb 3466static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3467{
f76aec76 3468 struct dmar_domain *domain;
d794dc9b 3469 unsigned long start_pfn, last_pfn;
ba395927 3470 struct iova *iova;
8c11e798 3471 struct intel_iommu *iommu;
ea8ea460 3472 struct page *freelist;
ba395927 3473
73676832 3474 if (iommu_no_mapping(dev))
f76aec76 3475 return;
2c2e2c38 3476
1525a29a 3477 domain = find_domain(dev);
ba395927
KA		 3478 BUG_ON(!domain);
3479
8c11e798
WH		 3480 iommu = domain_get_iommu(domain);
3481
ba395927 3482 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3483 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3484 (unsigned long long)dev_addr))
ba395927 3485 return;
ba395927 3486
d794dc9b
DW		 3487 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3488 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3489
d794dc9b 3490 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3491 dev_name(dev), start_pfn, last_pfn);
ba395927 3492
ea8ea460 3493 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3494
5e0d2a6f 3495 if (intel_iommu_strict) {
a1ddcbe9 3496 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
ea8ea460 3497 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3498 /* free iova */
3499 __free_iova(&domain->iovad, iova);
ea8ea460 3500 dma_free_pagelist(freelist);
5e0d2a6f 3501 } else {
ea8ea460 3502 add_unmap(domain, iova, freelist);
5e0d2a6f 3503 /*
3504 * queue up the release of the unmap to save the 1/6th of the
3505 * cpu used up by the iotlb flush operation...
3506 */
5e0d2a6f 3507 }
ba395927
KA		 3508}
3509
d41a4adb
JL		 3510static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3511 size_t size, enum dma_data_direction dir,
3512 struct dma_attrs *attrs)
3513{
3514 intel_unmap(dev, dev_addr);
3515}
3516
5040a918 3517static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3518 dma_addr_t *dma_handle, gfp_t flags,
3519 struct dma_attrs *attrs)
ba395927 3520{
36746436 3521 struct page *page = NULL;
ba395927
KA		 3522 int order;
3523
5b6985ce 3524 size = PAGE_ALIGN(size);
ba395927 3525 order = get_order(size);
e8bb910d 3526
5040a918 3527 if (!iommu_no_mapping(dev))
e8bb910d 3528 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3529 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3530 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3531 flags |= GFP_DMA;
3532 else
3533 flags |= GFP_DMA32;
3534 }
ba395927 3535
36746436
AM		 3536 if (flags & __GFP_WAIT) {
3537 unsigned int count = size >> PAGE_SHIFT;
3538
3539 page = dma_alloc_from_contiguous(dev, count, order);
3540 if (page && iommu_no_mapping(dev) &&
3541 page_to_phys(page) + size > dev->coherent_dma_mask) {
3542 dma_release_from_contiguous(dev, page, count);
3543 page = NULL;
3544 }
3545 }
3546
3547 if (!page)
3548 page = alloc_pages(flags, order);
3549 if (!page)
ba395927 3550 return NULL;
36746436 3551 memset(page_address(page), 0, size);
ba395927 3552
36746436 3553 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3554 DMA_BIDIRECTIONAL,
5040a918 3555 dev->coherent_dma_mask);
ba395927 3556 if (*dma_handle)
36746436
AM		 3557 return page_address(page);
3558 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3559 __free_pages(page, order);
3560
ba395927
KA		 3561 return NULL;
3562}
3563
5040a918 3564static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3565 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3566{
3567 int order;
36746436 3568 struct page *page = virt_to_page(vaddr);
ba395927 3569
5b6985ce 3570 size = PAGE_ALIGN(size);
ba395927
KA		 3571 order = get_order(size);
3572
d41a4adb 3573 intel_unmap(dev, dma_handle);
36746436
AM		 3574 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3575 __free_pages(page, order);
ba395927
KA		 3576}
3577
5040a918 3578static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3579 int nelems, enum dma_data_direction dir,
3580 struct dma_attrs *attrs)
ba395927 3581{
d41a4adb 3582 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3583}
3584
ba395927 3585static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3586 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3587{
3588 int i;
c03ab37c 3589 struct scatterlist *sg;
ba395927 3590
c03ab37c 3591 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3592 BUG_ON(!sg_page(sg));
db0fa0cb 3593 sg->dma_address = sg_phys(sg);
c03ab37c 3594 sg->dma_length = sg->length;
ba395927
KA		 3595 }
3596 return nelems;
3597}
3598
5040a918 3599static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3600 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3601{
ba395927 3602 int i;
ba395927 3603 struct dmar_domain *domain;
f76aec76
KA		 3604 size_t size = 0;
3605 int prot = 0;
f76aec76
KA		 3606 struct iova *iova = NULL;
3607 int ret;
c03ab37c 3608 struct scatterlist *sg;
b536d24d 3609 unsigned long start_vpfn;
8c11e798 3610 struct intel_iommu *iommu;
ba395927
KA		 3611
3612 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3613 if (iommu_no_mapping(dev))
3614 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3615
5040a918 3616 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3617 if (!domain)
3618 return 0;
3619
8c11e798
WH		 3620 iommu = domain_get_iommu(domain);
3621
b536d24d 3622 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3623 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3624
5040a918
DW		 3625 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3626 *dev->dma_mask);
f76aec76 3627 if (!iova) {
c03ab37c 3628 sglist->dma_length = 0;
f76aec76
KA		 3629 return 0;
3630 }
3631
3632 /*
3633 * Check if DMAR supports zero-length reads on write only
3634 * mappings..
3635 */
3636 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3637 !cap_zlr(iommu->cap))
f76aec76
KA		 3638 prot |= DMA_PTE_READ;
3639 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3640 prot |= DMA_PTE_WRITE;
3641
b536d24d 3642 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3643
f532959b 3644 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3645 if (unlikely(ret)) {
e1605495
DW		 3646 dma_pte_free_pagetable(domain, start_vpfn,
3647 start_vpfn + size - 1);
e1605495
DW		 3648 __free_iova(&domain->iovad, iova);
3649 return 0;
ba395927
KA		 3650 }
3651
1f0ef2aa
DW		 3652 /* it's a non-present to present mapping. Only flush if caching mode */
3653 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3654 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
1f0ef2aa 3655 else
8c11e798 3656 iommu_flush_write_buffer(iommu);
1f0ef2aa 3657
ba395927
KA		 3658 return nelems;
3659}
3660
dfb805e8
FT		 3661static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3662{
3663 return !dma_addr;
3664}
3665
160c1d8e 3666struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3667 .alloc = intel_alloc_coherent,
3668 .free = intel_free_coherent,
ba395927
KA		 3669 .map_sg = intel_map_sg,
3670 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3671 .map_page = intel_map_page,
3672 .unmap_page = intel_unmap_page,
dfb805e8 3673 .mapping_error = intel_mapping_error,
ba395927
KA		 3674};
3675
3676static inline int iommu_domain_cache_init(void)
3677{
3678 int ret = 0;
3679
3680 iommu_domain_cache = kmem_cache_create("iommu_domain",
3681 sizeof(struct dmar_domain),
3682 0,
3683 SLAB_HWCACHE_ALIGN,
3684
3685 NULL);
3686 if (!iommu_domain_cache) {
9f10e5bf 3687 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3688 ret = -ENOMEM;
3689 }
3690
3691 return ret;
3692}
3693
3694static inline int iommu_devinfo_cache_init(void)
3695{
3696 int ret = 0;
3697
3698 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3699 sizeof(struct device_domain_info),
3700 0,
3701 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3702 NULL);
3703 if (!iommu_devinfo_cache) {
9f10e5bf 3704 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3705 ret = -ENOMEM;
3706 }
3707
3708 return ret;
3709}
3710
ba395927
KA		 3711static int __init iommu_init_mempool(void)
3712{
3713 int ret;
3714 ret = iommu_iova_cache_init();
3715 if (ret)
3716 return ret;
3717
3718 ret = iommu_domain_cache_init();
3719 if (ret)
3720 goto domain_error;
3721
3722 ret = iommu_devinfo_cache_init();
3723 if (!ret)
3724 return ret;
3725
3726 kmem_cache_destroy(iommu_domain_cache);
3727domain_error:
85b45456 3728 iommu_iova_cache_destroy();
ba395927
KA		 3729
3730 return -ENOMEM;
3731}
3732
3733static void __init iommu_exit_mempool(void)
3734{
3735 kmem_cache_destroy(iommu_devinfo_cache);
3736 kmem_cache_destroy(iommu_domain_cache);
85b45456 3737 iommu_iova_cache_destroy();
ba395927
KA		 3738}
3739
556ab45f
DW		 3740static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3741{
3742 struct dmar_drhd_unit *drhd;
3743 u32 vtbar;
3744 int rc;
3745
3746 /* We know that this device on this chipset has its own IOMMU.
3747 * If we find it under a different IOMMU, then the BIOS is lying
3748 * to us. Hope that the IOMMU for this device is actually
3749 * disabled, and it needs no translation...
3750 */
3751 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3752 if (rc) {
3753 /* "can't" happen */
3754 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3755 return;
3756 }
3757 vtbar &= 0xffff0000;
3758
3759 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3760 drhd = dmar_find_matched_drhd_unit(pdev);
3761 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3762 TAINT_FIRMWARE_WORKAROUND,
3763 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3764 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3765}
3766DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3767
ba395927
KA		 3768static void __init init_no_remapping_devices(void)
3769{
3770 struct dmar_drhd_unit *drhd;
832bd858 3771 struct device *dev;
b683b230 3772 int i;
ba395927
KA		 3773
3774 for_each_drhd_unit(drhd) {
3775 if (!drhd->include_all) {
b683b230
JL		 3776 for_each_active_dev_scope(drhd->devices,
3777 drhd->devices_cnt, i, dev)
3778 break;
832bd858 3779 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3780 if (i == drhd->devices_cnt)
3781 drhd->ignored = 1;
3782 }
3783 }
3784
7c919779 3785 for_each_active_drhd_unit(drhd) {
7c919779 3786 if (drhd->include_all)
ba395927
KA		 3787 continue;
3788
b683b230
JL		 3789 for_each_active_dev_scope(drhd->devices,
3790 drhd->devices_cnt, i, dev)
832bd858 3791 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3792 break;
ba395927
KA		 3793 if (i < drhd->devices_cnt)
3794 continue;
3795
c0771df8
DW		 3796 /* This IOMMU has *only* gfx devices. Either bypass it or
3797 set the gfx_mapped flag, as appropriate */
3798 if (dmar_map_gfx) {
3799 intel_iommu_gfx_mapped = 1;
3800 } else {
3801 drhd->ignored = 1;
b683b230
JL		 3802 for_each_active_dev_scope(drhd->devices,
3803 drhd->devices_cnt, i, dev)
832bd858 3804 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3805 }
3806 }
3807}
3808
f59c7b69
FY		 3809#ifdef CONFIG_SUSPEND
3810static int init_iommu_hw(void)
3811{
3812 struct dmar_drhd_unit *drhd;
3813 struct intel_iommu *iommu = NULL;
3814
3815 for_each_active_iommu(iommu, drhd)
3816 if (iommu->qi)
3817 dmar_reenable_qi(iommu);
3818
b779260b
JC		 3819 for_each_iommu(iommu, drhd) {
3820 if (drhd->ignored) {
3821 /*
3822 * we always have to disable PMRs or DMA may fail on
3823 * this device
3824 */
3825 if (force_on)
3826 iommu_disable_protect_mem_regions(iommu);
3827 continue;
3828 }
3829
f59c7b69
FY		 3830 iommu_flush_write_buffer(iommu);
3831
3832 iommu_set_root_entry(iommu);
3833
3834 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3835 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3836 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3837 iommu_enable_translation(iommu);
b94996c9 3838 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3839 }
3840
3841 return 0;
3842}
3843
3844static void iommu_flush_all(void)
3845{
3846 struct dmar_drhd_unit *drhd;
3847 struct intel_iommu *iommu;
3848
3849 for_each_active_iommu(iommu, drhd) {
3850 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3851 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3852 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3853 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3854 }
3855}
3856
134fac3f 3857static int iommu_suspend(void)
f59c7b69
FY		 3858{
3859 struct dmar_drhd_unit *drhd;
3860 struct intel_iommu *iommu = NULL;
3861 unsigned long flag;
3862
3863 for_each_active_iommu(iommu, drhd) {
3864 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3865 GFP_ATOMIC);
3866 if (!iommu->iommu_state)
3867 goto nomem;
3868 }
3869
3870 iommu_flush_all();
3871
3872 for_each_active_iommu(iommu, drhd) {
3873 iommu_disable_translation(iommu);
3874
1f5b3c3f 3875 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3876
3877 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3878 readl(iommu->reg + DMAR_FECTL_REG);
3879 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3880 readl(iommu->reg + DMAR_FEDATA_REG);
3881 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3882 readl(iommu->reg + DMAR_FEADDR_REG);
3883 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3884 readl(iommu->reg + DMAR_FEUADDR_REG);
3885
1f5b3c3f 3886 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3887 }
3888 return 0;
3889
3890nomem:
3891 for_each_active_iommu(iommu, drhd)
3892 kfree(iommu->iommu_state);
3893
3894 return -ENOMEM;
3895}
3896
134fac3f 3897static void iommu_resume(void)
f59c7b69
FY		 3898{
3899 struct dmar_drhd_unit *drhd;
3900 struct intel_iommu *iommu = NULL;
3901 unsigned long flag;
3902
3903 if (init_iommu_hw()) {
b779260b
JC		 3904 if (force_on)
3905 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3906 else
3907 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3908 return;
f59c7b69
FY		 3909 }
3910
3911 for_each_active_iommu(iommu, drhd) {
3912
1f5b3c3f 3913 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3914
3915 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3916 iommu->reg + DMAR_FECTL_REG);
3917 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3918 iommu->reg + DMAR_FEDATA_REG);
3919 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3920 iommu->reg + DMAR_FEADDR_REG);
3921 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3922 iommu->reg + DMAR_FEUADDR_REG);
3923
1f5b3c3f 3924 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3925 }
3926
3927 for_each_active_iommu(iommu, drhd)
3928 kfree(iommu->iommu_state);
f59c7b69
FY		 3929}
3930
134fac3f 3931static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3932 .resume = iommu_resume,
3933 .suspend = iommu_suspend,
3934};
3935
134fac3f 3936static void __init init_iommu_pm_ops(void)
f59c7b69 3937{
134fac3f 3938 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3939}
3940
3941#else
99592ba4 3942static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3943#endif /* CONFIG_PM */
3944
318fe7df 3945
c2a0b538 3946int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3947{
3948 struct acpi_dmar_reserved_memory *rmrr;
3949 struct dmar_rmrr_unit *rmrru;
3950
3951 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3952 if (!rmrru)
3953 return -ENOMEM;
3954
3955 rmrru->hdr = header;
3956 rmrr = (struct acpi_dmar_reserved_memory *)header;
3957 rmrru->base_address = rmrr->base_address;
3958 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3959 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3960 ((void *)rmrr) + rmrr->header.length,
3961 &rmrru->devices_cnt);
3962 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3963 kfree(rmrru);
3964 return -ENOMEM;
3965 }
318fe7df 3966
2e455289 3967 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3968
2e455289 3969 return 0;
318fe7df
SS		 3970}
3971
6b197249
JL		 3972static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3973{
3974 struct dmar_atsr_unit *atsru;
3975 struct acpi_dmar_atsr *tmp;
3976
3977 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3978 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3979 if (atsr->segment != tmp->segment)
3980 continue;
3981 if (atsr->header.length != tmp->header.length)
3982 continue;
3983 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3984 return atsru;
3985 }
3986
3987 return NULL;
3988}
3989
3990int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 3991{
3992 struct acpi_dmar_atsr *atsr;
3993 struct dmar_atsr_unit *atsru;
3994
6b197249
JL		 3995 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
3996 return 0;
3997
318fe7df 3998 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 3999 atsru = dmar_find_atsr(atsr);
4000 if (atsru)
4001 return 0;
4002
4003 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4004 if (!atsru)
4005 return -ENOMEM;
4006
6b197249
JL		 4007 /*
4008 * If memory is allocated from slab by ACPI _DSM method, we need to
4009 * copy the memory content because the memory buffer will be freed
4010 * on return.
4011 */
4012 atsru->hdr = (void *)(atsru + 1);
4013 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4014 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4015 if (!atsru->include_all) {
4016 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4017 (void *)atsr + atsr->header.length,
4018 &atsru->devices_cnt);
4019 if (atsru->devices_cnt && atsru->devices == NULL) {
4020 kfree(atsru);
4021 return -ENOMEM;
4022 }
4023 }
318fe7df 4024
0e242612 4025 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4026
4027 return 0;
4028}
4029
9bdc531e
JL		 4030static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4031{
4032 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4033 kfree(atsru);
4034}
4035
6b197249
JL		 4036int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4037{
4038 struct acpi_dmar_atsr *atsr;
4039 struct dmar_atsr_unit *atsru;
4040
4041 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4042 atsru = dmar_find_atsr(atsr);
4043 if (atsru) {
4044 list_del_rcu(&atsru->list);
4045 synchronize_rcu();
4046 intel_iommu_free_atsr(atsru);
4047 }
4048
4049 return 0;
4050}
4051
4052int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4053{
4054 int i;
4055 struct device *dev;
4056 struct acpi_dmar_atsr *atsr;
4057 struct dmar_atsr_unit *atsru;
4058
4059 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4060 atsru = dmar_find_atsr(atsr);
4061 if (!atsru)
4062 return 0;
4063
4064 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4065 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4066 i, dev)
4067 return -EBUSY;
4068
4069 return 0;
4070}
4071
ffebeb46
JL		 4072static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4073{
4074 int sp, ret = 0;
4075 struct intel_iommu *iommu = dmaru->iommu;
4076
4077 if (g_iommus[iommu->seq_id])
4078 return 0;
4079
4080 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4081 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4082 iommu->name);
4083 return -ENXIO;
4084 }
4085 if (!ecap_sc_support(iommu->ecap) &&
4086 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4087 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4088 iommu->name);
4089 return -ENXIO;
4090 }
4091 sp = domain_update_iommu_superpage(iommu) - 1;
4092 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4093 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4094 iommu->name);
4095 return -ENXIO;
4096 }
4097
4098 /*
4099 * Disable translation if already enabled prior to OS handover.
4100 */
4101 if (iommu->gcmd & DMA_GCMD_TE)
4102 iommu_disable_translation(iommu);
4103
4104 g_iommus[iommu->seq_id] = iommu;
4105 ret = iommu_init_domains(iommu);
4106 if (ret == 0)
4107 ret = iommu_alloc_root_entry(iommu);
4108 if (ret)
4109 goto out;
4110
4111 if (dmaru->ignored) {
4112 /*
4113 * we always have to disable PMRs or DMA may fail on this device
4114 */
4115 if (force_on)
4116 iommu_disable_protect_mem_regions(iommu);
4117 return 0;
4118 }
4119
4120 intel_iommu_init_qi(iommu);
4121 iommu_flush_write_buffer(iommu);
4122 ret = dmar_set_interrupt(iommu);
4123 if (ret)
4124 goto disable_iommu;
4125
4126 iommu_set_root_entry(iommu);
4127 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4128 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4129 iommu_enable_translation(iommu);
4130
ffebeb46
JL		 4131 iommu_disable_protect_mem_regions(iommu);
4132 return 0;
4133
4134disable_iommu:
4135 disable_dmar_iommu(iommu);
4136out:
4137 free_dmar_iommu(iommu);
4138 return ret;
4139}
4140
6b197249
JL		 4141int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4142{
ffebeb46
JL		 4143 int ret = 0;
4144 struct intel_iommu *iommu = dmaru->iommu;
4145
4146 if (!intel_iommu_enabled)
4147 return 0;
4148 if (iommu == NULL)
4149 return -EINVAL;
4150
4151 if (insert) {
4152 ret = intel_iommu_add(dmaru);
4153 } else {
4154 disable_dmar_iommu(iommu);
4155 free_dmar_iommu(iommu);
4156 }
4157
4158 return ret;
6b197249
JL		 4159}
4160
9bdc531e
JL		 4161static void intel_iommu_free_dmars(void)
4162{
4163 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4164 struct dmar_atsr_unit *atsru, *atsr_n;
4165
4166 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4167 list_del(&rmrru->list);
4168 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4169 kfree(rmrru);
318fe7df
SS		 4170 }
4171
9bdc531e
JL		 4172 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4173 list_del(&atsru->list);
4174 intel_iommu_free_atsr(atsru);
4175 }
318fe7df
SS		 4176}
4177
4178int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4179{
b683b230 4180 int i, ret = 1;
318fe7df 4181 struct pci_bus *bus;
832bd858
DW		 4182 struct pci_dev *bridge = NULL;
4183 struct device *tmp;
318fe7df
SS		 4184 struct acpi_dmar_atsr *atsr;
4185 struct dmar_atsr_unit *atsru;
4186
4187 dev = pci_physfn(dev);
318fe7df 4188 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4189 bridge = bus->self;
318fe7df 4190 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 4191 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4192 return 0;
b5f82ddf 4193 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4194 break;
318fe7df 4195 }
b5f82ddf
JL		 4196 if (!bridge)
4197 return 0;
318fe7df 4198
0e242612 4199 rcu_read_lock();
b5f82ddf
JL		 4200 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4201 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4202 if (atsr->segment != pci_domain_nr(dev->bus))
4203 continue;
4204
b683b230 4205 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4206 if (tmp == &bridge->dev)
b683b230 4207 goto out;
b5f82ddf
JL		 4208
4209 if (atsru->include_all)
b683b230 4210 goto out;
b5f82ddf 4211 }
b683b230
JL		 4212 ret = 0;
4213out:
0e242612 4214 rcu_read_unlock();
318fe7df 4215
b683b230 4216 return ret;
318fe7df
SS		 4217}
4218
59ce0515
JL		 4219int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4220{
4221 int ret = 0;
4222 struct dmar_rmrr_unit *rmrru;
4223 struct dmar_atsr_unit *atsru;
4224 struct acpi_dmar_atsr *atsr;
4225 struct acpi_dmar_reserved_memory *rmrr;
4226
4227 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4228 return 0;
4229
4230 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4231 rmrr = container_of(rmrru->hdr,
4232 struct acpi_dmar_reserved_memory, header);
4233 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4234 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4235 ((void *)rmrr) + rmrr->header.length,
4236 rmrr->segment, rmrru->devices,
4237 rmrru->devices_cnt);
27e24950 4238 if(ret < 0)
59ce0515
JL		 4239 return ret;
4240 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4241 dmar_remove_dev_scope(info, rmrr->segment,
4242 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4243 }
4244 }
4245
4246 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4247 if (atsru->include_all)
4248 continue;
4249
4250 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4251 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4252 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4253 (void *)atsr + atsr->header.length,
4254 atsr->segment, atsru->devices,
4255 atsru->devices_cnt);
4256 if (ret > 0)
4257 break;
4258 else if(ret < 0)
4259 return ret;
4260 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4261 if (dmar_remove_dev_scope(info, atsr->segment,
4262 atsru->devices, atsru->devices_cnt))
4263 break;
4264 }
4265 }
4266
4267 return 0;
4268}
4269
99dcaded
FY		 4270/*
4271 * Here we only respond to action of unbound device from driver.
4272 *
4273 * Added device is not attached to its DMAR domain here yet. That will happen
4274 * when mapping the device to iova.
4275 */
4276static int device_notifier(struct notifier_block *nb,
4277 unsigned long action, void *data)
4278{
4279 struct device *dev = data;
99dcaded
FY		 4280 struct dmar_domain *domain;
4281
3d89194a 4282 if (iommu_dummy(dev))
44cd613c
DW		 4283 return 0;
4284
1196c2fb 4285 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4286 return 0;
4287
1525a29a 4288 domain = find_domain(dev);
99dcaded
FY		 4289 if (!domain)
4290 return 0;
4291
e6de0f8d 4292 dmar_remove_one_dev_info(domain, dev);
ab8dfe25 4293 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4294 domain_exit(domain);
a97590e5 4295
99dcaded
FY		 4296 return 0;
4297}
4298
4299static struct notifier_block device_nb = {
4300 .notifier_call = device_notifier,
4301};
4302
75f05569
JL		 4303static int intel_iommu_memory_notifier(struct notifier_block *nb,
4304 unsigned long val, void *v)
4305{
4306 struct memory_notify *mhp = v;
4307 unsigned long long start, end;
4308 unsigned long start_vpfn, last_vpfn;
4309
4310 switch (val) {
4311 case MEM_GOING_ONLINE:
4312 start = mhp->start_pfn << PAGE_SHIFT;
4313 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4314 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4315 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4316 start, end);
4317 return NOTIFY_BAD;
4318 }
4319 break;
4320
4321 case MEM_OFFLINE:
4322 case MEM_CANCEL_ONLINE:
4323 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4324 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4325 while (start_vpfn <= last_vpfn) {
4326 struct iova *iova;
4327 struct dmar_drhd_unit *drhd;
4328 struct intel_iommu *iommu;
ea8ea460 4329 struct page *freelist;
75f05569
JL		 4330
4331 iova = find_iova(&si_domain->iovad, start_vpfn);
4332 if (iova == NULL) {
9f10e5bf 4333 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4334 start_vpfn);
4335 break;
4336 }
4337
4338 iova = split_and_remove_iova(&si_domain->iovad, iova,
4339 start_vpfn, last_vpfn);
4340 if (iova == NULL) {
9f10e5bf 4341 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4342 start_vpfn, last_vpfn);
4343 return NOTIFY_BAD;
4344 }
4345
ea8ea460
DW		 4346 freelist = domain_unmap(si_domain, iova->pfn_lo,
4347 iova->pfn_hi);
4348
75f05569
JL		 4349 rcu_read_lock();
4350 for_each_active_iommu(iommu, drhd)
a1ddcbe9 4351 iommu_flush_iotlb_psi(iommu, si_domain,
a156ef99 4352 iova->pfn_lo, iova_size(iova),
ea8ea460 4353 !freelist, 0);
75f05569 4354 rcu_read_unlock();
ea8ea460 4355 dma_free_pagelist(freelist);
75f05569
JL		 4356
4357 start_vpfn = iova->pfn_hi + 1;
4358 free_iova_mem(iova);
4359 }
4360 break;
4361 }
4362
4363 return NOTIFY_OK;
4364}
4365
4366static struct notifier_block intel_iommu_memory_nb = {
4367 .notifier_call = intel_iommu_memory_notifier,
4368 .priority = 0
4369};
4370
a5459cfe
AW		 4371
4372static ssize_t intel_iommu_show_version(struct device *dev,
4373 struct device_attribute *attr,
4374 char *buf)
4375{
4376 struct intel_iommu *iommu = dev_get_drvdata(dev);
4377 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4378 return sprintf(buf, "%d:%d\n",
4379 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4380}
4381static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4382
4383static ssize_t intel_iommu_show_address(struct device *dev,
4384 struct device_attribute *attr,
4385 char *buf)
4386{
4387 struct intel_iommu *iommu = dev_get_drvdata(dev);
4388 return sprintf(buf, "%llx\n", iommu->reg_phys);
4389}
4390static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4391
4392static ssize_t intel_iommu_show_cap(struct device *dev,
4393 struct device_attribute *attr,
4394 char *buf)
4395{
4396 struct intel_iommu *iommu = dev_get_drvdata(dev);
4397 return sprintf(buf, "%llx\n", iommu->cap);
4398}
4399static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4400
4401static ssize_t intel_iommu_show_ecap(struct device *dev,
4402 struct device_attribute *attr,
4403 char *buf)
4404{
4405 struct intel_iommu *iommu = dev_get_drvdata(dev);
4406 return sprintf(buf, "%llx\n", iommu->ecap);
4407}
4408static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4409
2238c082
AW		 4410static ssize_t intel_iommu_show_ndoms(struct device *dev,
4411 struct device_attribute *attr,
4412 char *buf)
4413{
4414 struct intel_iommu *iommu = dev_get_drvdata(dev);
4415 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4416}
4417static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4418
4419static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4420 struct device_attribute *attr,
4421 char *buf)
4422{
4423 struct intel_iommu *iommu = dev_get_drvdata(dev);
4424 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4425 cap_ndoms(iommu->cap)));
4426}
4427static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4428
a5459cfe
AW		 4429static struct attribute *intel_iommu_attrs[] = {
4430 &dev_attr_version.attr,
4431 &dev_attr_address.attr,
4432 &dev_attr_cap.attr,
4433 &dev_attr_ecap.attr,
2238c082
AW		 4434 &dev_attr_domains_supported.attr,
4435 &dev_attr_domains_used.attr,
a5459cfe
AW		 4436 NULL,
4437};
4438
4439static struct attribute_group intel_iommu_group = {
4440 .name = "intel-iommu",
4441 .attrs = intel_iommu_attrs,
4442};
4443
4444const struct attribute_group *intel_iommu_groups[] = {
4445 &intel_iommu_group,
4446 NULL,
4447};
4448
ba395927
KA		 4449int __init intel_iommu_init(void)
4450{
9bdc531e 4451 int ret = -ENODEV;
3a93c841 4452 struct dmar_drhd_unit *drhd;
7c919779 4453 struct intel_iommu *iommu;
ba395927 4454
a59b50e9
JC		 4455 /* VT-d is required for a TXT/tboot launch, so enforce that */
4456 force_on = tboot_force_iommu();
4457
3a5670e8
JL		 4458 if (iommu_init_mempool()) {
4459 if (force_on)
4460 panic("tboot: Failed to initialize iommu memory\n");
4461 return -ENOMEM;
4462 }
4463
4464 down_write(&dmar_global_lock);
a59b50e9
JC		 4465 if (dmar_table_init()) {
4466 if (force_on)
4467 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4468 goto out_free_dmar;
a59b50e9 4469 }
ba395927 4470
c2c7286a 4471 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4472 if (force_on)
4473 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4474 goto out_free_dmar;
a59b50e9 4475 }
1886e8a9 4476
75f1cdf1 4477 if (no_iommu || dmar_disabled)
9bdc531e 4478 goto out_free_dmar;
2ae21010 4479
318fe7df 4480 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4481 pr_info("No RMRR found\n");
318fe7df
SS		 4482
4483 if (list_empty(&dmar_atsr_units))
9f10e5bf 4484 pr_info("No ATSR found\n");
318fe7df 4485
51a63e67
JC		 4486 if (dmar_init_reserved_ranges()) {
4487 if (force_on)
4488 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4489 goto out_free_reserved_range;
51a63e67 4490 }
ba395927
KA		 4491
4492 init_no_remapping_devices();
4493
b779260b 4494 ret = init_dmars();
ba395927 4495 if (ret) {
a59b50e9
JC		 4496 if (force_on)
4497 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4498 pr_err("Initialization failed\n");
9bdc531e 4499 goto out_free_reserved_range;
ba395927 4500 }
3a5670e8 4501 up_write(&dmar_global_lock);
9f10e5bf 4502 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4503
5e0d2a6f 4504 init_timer(&unmap_timer);
75f1cdf1
FT		 4505#ifdef CONFIG_SWIOTLB
4506 swiotlb = 0;
4507#endif
19943b0e 4508 dma_ops = &intel_dma_ops;
4ed0d3e6 4509
134fac3f 4510 init_iommu_pm_ops();
a8bcbb0d 4511
a5459cfe
AW		 4512 for_each_active_iommu(iommu, drhd)
4513 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4514 intel_iommu_groups,
2439d4aa 4515 "%s", iommu->name);
a5459cfe 4516
4236d97d 4517 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4518 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4519 if (si_domain && !hw_pass_through)
4520 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4521
8bc1f85c
ED		 4522 intel_iommu_enabled = 1;
4523
ba395927 4524 return 0;
9bdc531e
JL		 4525
4526out_free_reserved_range:
4527 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4528out_free_dmar:
4529 intel_iommu_free_dmars();
3a5670e8
JL		 4530 up_write(&dmar_global_lock);
4531 iommu_exit_mempool();
9bdc531e 4532 return ret;
ba395927 4533}
e820482c 4534
2452d9db 4535static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
579305f7
AW		 4536{
4537 struct intel_iommu *iommu = opaque;
4538
2452d9db 4539 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 4540 return 0;
4541}
4542
4543/*
4544 * NB - intel-iommu lacks any sort of reference counting for the users of
4545 * dependent devices. If multiple endpoints have intersecting dependent
4546 * devices, unbinding the driver from any one of them will possibly leave
4547 * the others unable to operate.
4548 */
2452d9db 4549static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
3199aa6b 4550{
0bcb3e28 4551 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4552 return;
4553
2452d9db 4554 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
3199aa6b
HW		 4555}
4556
127c7615 4557static void __dmar_remove_one_dev_info(struct device_domain_info *info)
c7151a8d 4558{
c7151a8d
WH		 4559 struct intel_iommu *iommu;
4560 unsigned long flags;
c7151a8d 4561
55d94043
JR		 4562 assert_spin_locked(&device_domain_lock);
4563
127c7615 4564 if (WARN_ON(!info))
c7151a8d
WH		 4565 return;
4566
127c7615 4567 iommu = info->iommu;
c7151a8d 4568
127c7615
JR		 4569 if (info->dev) {
4570 iommu_disable_dev_iotlb(info);
4571 domain_context_clear(iommu, info->dev);
4572 }
c7151a8d 4573
b608ac3b 4574 unlink_domain_info(info);
c7151a8d 4575
d160aca5 4576 spin_lock_irqsave(&iommu->lock, flags);
127c7615 4577 domain_detach_iommu(info->domain, iommu);
d160aca5 4578 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 4579
127c7615 4580 free_devinfo_mem(info);
c7151a8d 4581}
c7151a8d 4582
55d94043
JR		 4583static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4584 struct device *dev)
4585{
127c7615 4586 struct device_domain_info *info;
55d94043 4587 unsigned long flags;
3e7abe25 4588
55d94043 4589 spin_lock_irqsave(&device_domain_lock, flags);
127c7615
JR		 4590 info = dev->archdata.iommu;
4591 __dmar_remove_one_dev_info(info);
55d94043 4592 spin_unlock_irqrestore(&device_domain_lock, flags);
c7151a8d
WH		 4593}
4594
2c2e2c38 4595static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4596{
4597 int adjust_width;
4598
0fb5fe87
RM		 4599 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4600 DMA_32BIT_PFN);
5e98c4b1
WH		 4601 domain_reserve_special_ranges(domain);
4602
4603 /* calculate AGAW */
4604 domain->gaw = guest_width;
4605 adjust_width = guestwidth_to_adjustwidth(guest_width);
4606 domain->agaw = width_to_agaw(adjust_width);
4607
5e98c4b1 4608 domain->iommu_coherency = 0;
c5b15255 4609 domain->iommu_snooping = 0;
6dd9a7c7 4610 domain->iommu_superpage = 0;
fe40f1e0 4611 domain->max_addr = 0;
5e98c4b1
WH		 4612
4613 /* always allocate the top pgd */
4c923d47 4614 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4615 if (!domain->pgd)
4616 return -ENOMEM;
4617 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4618 return 0;
4619}
4620
00a77deb 4621static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4622{
5d450806 4623 struct dmar_domain *dmar_domain;
00a77deb
JR		 4624 struct iommu_domain *domain;
4625
4626 if (type != IOMMU_DOMAIN_UNMANAGED)
4627 return NULL;
38717946 4628
ab8dfe25 4629 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4630 if (!dmar_domain) {
9f10e5bf 4631 pr_err("Can't allocate dmar_domain\n");
00a77deb 4632 return NULL;
38717946 4633 }
2c2e2c38 4634 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4635 pr_err("Domain initialization failed\n");
92d03cc8 4636 domain_exit(dmar_domain);
00a77deb 4637 return NULL;
38717946 4638 }
8140a95d 4639 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4640
00a77deb 4641 domain = &dmar_domain->domain;
8a0e715b
JR		 4642 domain->geometry.aperture_start = 0;
4643 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4644 domain->geometry.force_aperture = true;
4645
00a77deb 4646 return domain;
38717946 4647}
38717946 4648
00a77deb 4649static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4650{
00a77deb 4651 domain_exit(to_dmar_domain(domain));
38717946 4652}
38717946 4653
4c5478c9
JR		 4654static int intel_iommu_attach_device(struct iommu_domain *domain,
4655 struct device *dev)
38717946 4656{
00a77deb 4657 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4658 struct intel_iommu *iommu;
4659 int addr_width;
156baca8 4660 u8 bus, devfn;
faa3d6f5 4661
c875d2c1
AW		 4662 if (device_is_rmrr_locked(dev)) {
4663 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4664 return -EPERM;
4665 }
4666
7207d8f9
DW		 4667 /* normally dev is not mapped */
4668 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4669 struct dmar_domain *old_domain;
4670
1525a29a 4671 old_domain = find_domain(dev);
faa3d6f5 4672 if (old_domain) {
d160aca5 4673 rcu_read_lock();
de7e8886 4674 dmar_remove_one_dev_info(old_domain, dev);
d160aca5 4675 rcu_read_unlock();
62c22167
JR		 4676
4677 if (!domain_type_is_vm_or_si(old_domain) &&
4678 list_empty(&old_domain->devices))
4679 domain_exit(old_domain);
faa3d6f5
WH		 4680 }
4681 }
4682
156baca8 4683 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4684 if (!iommu)
4685 return -ENODEV;
4686
4687 /* check if this iommu agaw is sufficient for max mapped address */
4688 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4689 if (addr_width > cap_mgaw(iommu->cap))
4690 addr_width = cap_mgaw(iommu->cap);
4691
4692 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4693 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4694 "sufficient for the mapped address (%llx)\n",
a99c47a2 4695 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4696 return -EFAULT;
4697 }
a99c47a2
TL		 4698 dmar_domain->gaw = addr_width;
4699
4700 /*
4701 * Knock out extra levels of page tables if necessary
4702 */
4703 while (iommu->agaw < dmar_domain->agaw) {
4704 struct dma_pte *pte;
4705
4706 pte = dmar_domain->pgd;
4707 if (dma_pte_present(pte)) {
25cbff16
SY		 4708 dmar_domain->pgd = (struct dma_pte *)
4709 phys_to_virt(dma_pte_addr(pte));
7a661013 4710 free_pgtable_page(pte);
a99c47a2
TL		 4711 }
4712 dmar_domain->agaw--;
4713 }
fe40f1e0 4714
28ccce0d 4715 return domain_add_dev_info(dmar_domain, dev);
38717946 4716}
38717946 4717
4c5478c9
JR		 4718static void intel_iommu_detach_device(struct iommu_domain *domain,
4719 struct device *dev)
38717946 4720{
e6de0f8d 4721 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4722}
c7151a8d 4723
b146a1c9
JR		 4724static int intel_iommu_map(struct iommu_domain *domain,
4725 unsigned long iova, phys_addr_t hpa,
5009065d 4726 size_t size, int iommu_prot)
faa3d6f5 4727{
00a77deb 4728 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4729 u64 max_addr;
dde57a21 4730 int prot = 0;
faa3d6f5 4731 int ret;
fe40f1e0 4732
dde57a21
JR		 4733 if (iommu_prot & IOMMU_READ)
4734 prot |= DMA_PTE_READ;
4735 if (iommu_prot & IOMMU_WRITE)
4736 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4737 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4738 prot |= DMA_PTE_SNP;
dde57a21 4739
163cc52c 4740 max_addr = iova + size;
dde57a21 4741 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4742 u64 end;
4743
4744 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4745 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4746 if (end < max_addr) {
9f10e5bf 4747 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4748 "sufficient for the mapped address (%llx)\n",
8954da1f 4749 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4750 return -EFAULT;
4751 }
dde57a21 4752 dmar_domain->max_addr = max_addr;
fe40f1e0 4753 }
ad051221
DW		 4754 /* Round up size to next multiple of PAGE_SIZE, if it and
4755 the low bits of hpa would take us onto the next page */
88cb6a74 4756 size = aligned_nrpages(hpa, size);
ad051221
DW		 4757 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4758 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4759 return ret;
38717946 4760}
38717946 4761
5009065d 4762static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4763 unsigned long iova, size_t size)
38717946 4764{
00a77deb 4765 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4766 struct page *freelist = NULL;
4767 struct intel_iommu *iommu;
4768 unsigned long start_pfn, last_pfn;
4769 unsigned int npages;
42e8c186 4770 int iommu_id, level = 0;
5cf0a76f
DW		 4771
4772 /* Cope with horrid API which requires us to unmap more than the
4773 size argument if it happens to be a large-page mapping. */
dc02e46e 4774 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5cf0a76f
DW		 4775
4776 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4777 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4778
ea8ea460
DW		 4779 start_pfn = iova >> VTD_PAGE_SHIFT;
4780 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4781
4782 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4783
4784 npages = last_pfn - start_pfn + 1;
4785
29a27719 4786 for_each_domain_iommu(iommu_id, dmar_domain) {
a1ddcbe9 4787 iommu = g_iommus[iommu_id];
ea8ea460 4788
42e8c186
JR		 4789 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
4790 start_pfn, npages, !freelist, 0);
ea8ea460
DW		 4791 }
4792
4793 dma_free_pagelist(freelist);
fe40f1e0 4794
163cc52c
DW		 4795 if (dmar_domain->max_addr == iova + size)
4796 dmar_domain->max_addr = iova;
b146a1c9 4797
5cf0a76f 4798 return size;
38717946 4799}
38717946 4800
d14d6577 4801static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4802 dma_addr_t iova)
38717946 4803{
00a77deb 4804 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4805 struct dma_pte *pte;
5cf0a76f 4806 int level = 0;
faa3d6f5 4807 u64 phys = 0;
38717946 4808
5cf0a76f 4809 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4810 if (pte)
faa3d6f5 4811 phys = dma_pte_addr(pte);
38717946 4812
faa3d6f5 4813 return phys;
38717946 4814}
a8bcbb0d 4815
5d587b8d 4816static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4817{
dbb9fd86 4818 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4819 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4820 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4821 return irq_remapping_enabled == 1;
dbb9fd86 4822
5d587b8d 4823 return false;
dbb9fd86
SY		 4824}
4825
abdfdde2
AW		 4826static int intel_iommu_add_device(struct device *dev)
4827{
a5459cfe 4828 struct intel_iommu *iommu;
abdfdde2 4829 struct iommu_group *group;
156baca8 4830 u8 bus, devfn;
70ae6f0d 4831
a5459cfe
AW		 4832 iommu = device_to_iommu(dev, &bus, &devfn);
4833 if (!iommu)
70ae6f0d
AW		 4834 return -ENODEV;
4835
a5459cfe 4836 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4837
e17f9ff4 4838 group = iommu_group_get_for_dev(dev);
783f157b 4839
e17f9ff4
AW		 4840 if (IS_ERR(group))
4841 return PTR_ERR(group);
bcb71abe 4842
abdfdde2 4843 iommu_group_put(group);
e17f9ff4 4844 return 0;
abdfdde2 4845}
70ae6f0d 4846
abdfdde2
AW		 4847static void intel_iommu_remove_device(struct device *dev)
4848{
a5459cfe
AW		 4849 struct intel_iommu *iommu;
4850 u8 bus, devfn;
4851
4852 iommu = device_to_iommu(dev, &bus, &devfn);
4853 if (!iommu)
4854 return;
4855
abdfdde2 4856 iommu_group_remove_device(dev);
a5459cfe
AW		 4857
4858 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4859}
4860
b22f6434 4861static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4862 .capable = intel_iommu_capable,
00a77deb
JR		 4863 .domain_alloc = intel_iommu_domain_alloc,
4864 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4865 .attach_dev = intel_iommu_attach_device,
4866 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4867 .map = intel_iommu_map,
4868 .unmap = intel_iommu_unmap,
315786eb 4869 .map_sg = default_iommu_map_sg,
a8bcbb0d 4870 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4871 .add_device = intel_iommu_add_device,
4872 .remove_device = intel_iommu_remove_device,
6d1c56a9 4873 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4874};
9af88143 4875
9452618e
DV		 4876static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4877{
4878 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4879 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4880 dmar_map_gfx = 0;
4881}
4882
4883DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4884DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4885DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4886DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4887DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4888DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4889DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4890
d34d6517 4891static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4892{
4893 /*
4894 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4895 * but needs it. Same seems to hold for the desktop versions.
9af88143 4896 */
9f10e5bf 4897 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4898 rwbf_quirk = 1;
4899}
4900
4901DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4902DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4903DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4904DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4905DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4906DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4907DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4908
eecfd57f
AJ		 4909#define GGC 0x52
4910#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4911#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4912#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4913#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4914#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4915#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4916#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4917#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4918
d34d6517 4919static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4920{
4921 unsigned short ggc;
4922
eecfd57f 4923 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4924 return;
4925
eecfd57f 4926 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 4927 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 4928 dmar_map_gfx = 0;
6fbcfb3e
DW		 4929 } else if (dmar_map_gfx) {
4930 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 4931 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 4932 intel_iommu_strict = 1;
4933 }
9eecabcb
DW		 4934}
4935DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4936DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4937DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4938DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4939
e0fc7e0b
DW		 4940/* On Tylersburg chipsets, some BIOSes have been known to enable the
4941 ISOCH DMAR unit for the Azalia sound device, but not give it any
4942 TLB entries, which causes it to deadlock. Check for that. We do
4943 this in a function called from init_dmars(), instead of in a PCI
4944 quirk, because we don't want to print the obnoxious "BIOS broken"
4945 message if VT-d is actually disabled.
4946*/
4947static void __init check_tylersburg_isoch(void)
4948{
4949 struct pci_dev *pdev;
4950 uint32_t vtisochctrl;
4951
4952 /* If there's no Azalia in the system anyway, forget it. */
4953 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4954 if (!pdev)
4955 return;
4956 pci_dev_put(pdev);
4957
4958 /* System Management Registers. Might be hidden, in which case
4959 we can't do the sanity check. But that's OK, because the
4960 known-broken BIOSes _don't_ actually hide it, so far. */
4961 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4962 if (!pdev)
4963 return;
4964
4965 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4966 pci_dev_put(pdev);
4967 return;
4968 }
4969
4970 pci_dev_put(pdev);
4971
4972 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4973 if (vtisochctrl & 1)
4974 return;
4975
4976 /* Drop all bits other than the number of TLB entries */
4977 vtisochctrl &= 0x1c;
4978
4979 /* If we have the recommended number of TLB entries (16), fine. */
4980 if (vtisochctrl == 0x10)
4981 return;
4982
4983 /* Zero TLB entries? You get to ride the short bus to school. */
4984 if (!vtisochctrl) {
4985 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4986 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4987 dmi_get_system_info(DMI_BIOS_VENDOR),
4988 dmi_get_system_info(DMI_BIOS_VERSION),
4989 dmi_get_system_info(DMI_PRODUCT_VERSION));
4990 iommu_identity_mapping |= IDENTMAP_AZALIA;
4991 return;
4992 }
9f10e5bf
JR		 4993
4994 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 4995 vtisochctrl);
4996}
Linux kernel - Deliverables
This page took 1.532624 seconds and 5 git commands to generate.