projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge git://git.infradead.org/~dwmw2/iommu-2.6.31
[deliverable/linux.git] / drivers / pci / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
5e0d2a6f 35#include <linux/timer.h>
38717946 36#include <linux/iova.h>
5d450806 37#include <linux/iommu.h>
38717946 38#include <linux/intel-iommu.h>
f59c7b69 39#include <linux/sysdev.h>
ba395927 40#include <asm/cacheflush.h>
46a7fa27 41#include <asm/iommu.h>
ba395927
KA		 42#include "pci.h"
43
5b6985ce
FY		 44#define ROOT_SIZE VTD_PAGE_SIZE
45#define CONTEXT_SIZE VTD_PAGE_SIZE
46
ba395927
KA		 47#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
48#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
49
50#define IOAPIC_RANGE_START (0xfee00000)
51#define IOAPIC_RANGE_END (0xfeefffff)
52#define IOVA_START_ADDR (0x1000)
53
54#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
55
4ed0d3e6
FY		 56#define MAX_AGAW_WIDTH 64
57
ba395927
KA		 58#define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
59
f27be03b 60#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 61#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 62#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 63
fd18de50
DW		 64#ifndef PHYSICAL_PAGE_MASK
65#define PHYSICAL_PAGE_MASK PAGE_MASK
66#endif
67
d9630fe9
WH		 68/* global iommu list, set NULL for ignored DMAR units */
69static struct intel_iommu **g_iommus;
70
9af88143
DW		 71static int rwbf_quirk;
72
46b08e1a
MM		 73/*
74 * 0: Present
75 * 1-11: Reserved
76 * 12-63: Context Ptr (12 - (haw-1))
77 * 64-127: Reserved
78 */
79struct root_entry {
80 u64 val;
81 u64 rsvd1;
82};
83#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
84static inline bool root_present(struct root_entry *root)
85{
86 return (root->val & 1);
87}
88static inline void set_root_present(struct root_entry *root)
89{
90 root->val |= 1;
91}
92static inline void set_root_value(struct root_entry *root, unsigned long value)
93{
94 root->val |= value & VTD_PAGE_MASK;
95}
96
97static inline struct context_entry *
98get_context_addr_from_root(struct root_entry *root)
99{
100 return (struct context_entry *)
101 (root_present(root)?phys_to_virt(
102 root->val & VTD_PAGE_MASK) :
103 NULL);
104}
105
7a8fc25e
MM		 106/*
107 * low 64 bits:
108 * 0: present
109 * 1: fault processing disable
110 * 2-3: translation type
111 * 12-63: address space root
112 * high 64 bits:
113 * 0-2: address width
114 * 3-6: aval
115 * 8-23: domain id
116 */
117struct context_entry {
118 u64 lo;
119 u64 hi;
120};
c07e7d21
MM		 121
122static inline bool context_present(struct context_entry *context)
123{
124 return (context->lo & 1);
125}
126static inline void context_set_present(struct context_entry *context)
127{
128 context->lo |= 1;
129}
130
131static inline void context_set_fault_enable(struct context_entry *context)
132{
133 context->lo &= (((u64)-1) << 2) | 1;
134}
135
c07e7d21
MM		 136static inline void context_set_translation_type(struct context_entry *context,
137 unsigned long value)
138{
139 context->lo &= (((u64)-1) << 4) | 3;
140 context->lo |= (value & 3) << 2;
141}
142
143static inline void context_set_address_root(struct context_entry *context,
144 unsigned long value)
145{
146 context->lo |= value & VTD_PAGE_MASK;
147}
148
149static inline void context_set_address_width(struct context_entry *context,
150 unsigned long value)
151{
152 context->hi |= value & 7;
153}
154
155static inline void context_set_domain_id(struct context_entry *context,
156 unsigned long value)
157{
158 context->hi |= (value & ((1 << 16) - 1)) << 8;
159}
160
161static inline void context_clear_entry(struct context_entry *context)
162{
163 context->lo = 0;
164 context->hi = 0;
165}
7a8fc25e 166
622ba12a
MM		 167/*
168 * 0: readable
169 * 1: writable
170 * 2-6: reserved
171 * 7: super page
9cf06697
SY		 172 * 8-10: available
173 * 11: snoop behavior
622ba12a
MM		 174 * 12-63: Host physcial address
175 */
176struct dma_pte {
177 u64 val;
178};
622ba12a 179
19c239ce
MM		 180static inline void dma_clear_pte(struct dma_pte *pte)
181{
182 pte->val = 0;
183}
184
185static inline void dma_set_pte_readable(struct dma_pte *pte)
186{
187 pte->val |= DMA_PTE_READ;
188}
189
190static inline void dma_set_pte_writable(struct dma_pte *pte)
191{
192 pte->val |= DMA_PTE_WRITE;
193}
194
9cf06697
SY		 195static inline void dma_set_pte_snp(struct dma_pte *pte)
196{
197 pte->val |= DMA_PTE_SNP;
198}
199
19c239ce
MM		 200static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
201{
202 pte->val = (pte->val & ~3) | (prot & 3);
203}
204
205static inline u64 dma_pte_addr(struct dma_pte *pte)
206{
207 return (pte->val & VTD_PAGE_MASK);
208}
209
210static inline void dma_set_pte_addr(struct dma_pte *pte, u64 addr)
211{
212 pte->val |= (addr & VTD_PAGE_MASK);
213}
214
215static inline bool dma_pte_present(struct dma_pte *pte)
216{
217 return (pte->val & 3) != 0;
218}
622ba12a 219
3b5410e7 220/* devices under the same p2p bridge are owned in one domain */
cdc7b837 221#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 222
1ce28feb
WH		 223/* domain represents a virtual machine, more than one devices
224 * across iommus may be owned in one domain, e.g. kvm guest.
225 */
226#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
227
99126f7c
MM		 228struct dmar_domain {
229 int id; /* domain id */
8c11e798 230 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
99126f7c
MM		 231
232 struct list_head devices; /* all devices' list */
233 struct iova_domain iovad; /* iova's that belong to this domain */
234
235 struct dma_pte *pgd; /* virtual address */
236 spinlock_t mapping_lock; /* page table lock */
237 int gaw; /* max guest address width */
238
239 /* adjusted guest address width, 0 is level 2 30-bit */
240 int agaw;
241
3b5410e7 242 int flags; /* flags to find out type of domain */
8e604097
WH		 243
244 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 245 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d
WH		 246 int iommu_count; /* reference count of iommu */
247 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 248 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 249};
250
a647dacb
MM		 251/* PCI domain-device relationship */
252struct device_domain_info {
253 struct list_head link; /* link to domain siblings */
254 struct list_head global; /* link to global list */
276dbf99
DW		 255 int segment; /* PCI domain */
256 u8 bus; /* PCI bus number */
a647dacb
MM		 257 u8 devfn; /* PCI devfn number */
258 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
93a23a72 259 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 260 struct dmar_domain *domain; /* pointer to domain */
261};
262
5e0d2a6f 263static void flush_unmaps_timeout(unsigned long data);
264
265DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
266
80b20dd8 267#define HIGH_WATER_MARK 250
268struct deferred_flush_tables {
269 int next;
270 struct iova *iova[HIGH_WATER_MARK];
271 struct dmar_domain *domain[HIGH_WATER_MARK];
272};
273
274static struct deferred_flush_tables *deferred_flush;
275
5e0d2a6f 276/* bitmap for indexing intel_iommus */
5e0d2a6f 277static int g_num_of_iommus;
278
279static DEFINE_SPINLOCK(async_umap_flush_lock);
280static LIST_HEAD(unmaps_to_do);
281
282static int timer_on;
283static long list_size;
5e0d2a6f 284
ba395927
KA		 285static void domain_remove_dev_info(struct dmar_domain *domain);
286
0cd5c3c8
KM		 287#ifdef CONFIG_DMAR_DEFAULT_ON
288int dmar_disabled = 0;
289#else
290int dmar_disabled = 1;
291#endif /*CONFIG_DMAR_DEFAULT_ON*/
292
ba395927 293static int __initdata dmar_map_gfx = 1;
7d3b03ce 294static int dmar_forcedac;
5e0d2a6f 295static int intel_iommu_strict;
ba395927
KA		 296
297#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
298static DEFINE_SPINLOCK(device_domain_lock);
299static LIST_HEAD(device_domain_list);
300
a8bcbb0d
JR		 301static struct iommu_ops intel_iommu_ops;
302
ba395927
KA		 303static int __init intel_iommu_setup(char *str)
304{
305 if (!str)
306 return -EINVAL;
307 while (*str) {
0cd5c3c8
KM		 308 if (!strncmp(str, "on", 2)) {
309 dmar_disabled = 0;
310 printk(KERN_INFO "Intel-IOMMU: enabled\n");
311 } else if (!strncmp(str, "off", 3)) {
ba395927 312 dmar_disabled = 1;
0cd5c3c8 313 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 314 } else if (!strncmp(str, "igfx_off", 8)) {
315 dmar_map_gfx = 0;
316 printk(KERN_INFO
317 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 318 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 319 printk(KERN_INFO
7d3b03ce
KA		 320 "Intel-IOMMU: Forcing DAC for PCI devices\n");
321 dmar_forcedac = 1;
5e0d2a6f 322 } else if (!strncmp(str, "strict", 6)) {
323 printk(KERN_INFO
324 "Intel-IOMMU: disable batched IOTLB flush\n");
325 intel_iommu_strict = 1;
ba395927
KA		 326 }
327
328 str += strcspn(str, ",");
329 while (*str == ',')
330 str++;
331 }
332 return 0;
333}
334__setup("intel_iommu=", intel_iommu_setup);
335
336static struct kmem_cache *iommu_domain_cache;
337static struct kmem_cache *iommu_devinfo_cache;
338static struct kmem_cache *iommu_iova_cache;
339
eb3fa7cb
KA		 340static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
341{
342 unsigned int flags;
343 void *vaddr;
344
345 /* trying to avoid low memory issues */
346 flags = current->flags & PF_MEMALLOC;
347 current->flags |= PF_MEMALLOC;
348 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
349 current->flags &= (~PF_MEMALLOC | flags);
350 return vaddr;
351}
352
353
ba395927
KA		 354static inline void *alloc_pgtable_page(void)
355{
eb3fa7cb
KA		 356 unsigned int flags;
357 void *vaddr;
358
359 /* trying to avoid low memory issues */
360 flags = current->flags & PF_MEMALLOC;
361 current->flags |= PF_MEMALLOC;
362 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
363 current->flags &= (~PF_MEMALLOC | flags);
364 return vaddr;
ba395927
KA		 365}
366
367static inline void free_pgtable_page(void *vaddr)
368{
369 free_page((unsigned long)vaddr);
370}
371
372static inline void *alloc_domain_mem(void)
373{
eb3fa7cb 374 return iommu_kmem_cache_alloc(iommu_domain_cache);
ba395927
KA		 375}
376
38717946 377static void free_domain_mem(void *vaddr)
ba395927
KA		 378{
379 kmem_cache_free(iommu_domain_cache, vaddr);
380}
381
382static inline void * alloc_devinfo_mem(void)
383{
eb3fa7cb 384 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
ba395927
KA		 385}
386
387static inline void free_devinfo_mem(void *vaddr)
388{
389 kmem_cache_free(iommu_devinfo_cache, vaddr);
390}
391
392struct iova *alloc_iova_mem(void)
393{
eb3fa7cb 394 return iommu_kmem_cache_alloc(iommu_iova_cache);
ba395927
KA		 395}
396
397void free_iova_mem(struct iova *iova)
398{
399 kmem_cache_free(iommu_iova_cache, iova);
400}
401
1b573683
WH		 402
403static inline int width_to_agaw(int width);
404
4ed0d3e6 405static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 406{
407 unsigned long sagaw;
408 int agaw = -1;
409
410 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 411 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 412 agaw >= 0; agaw--) {
413 if (test_bit(agaw, &sagaw))
414 break;
415 }
416
417 return agaw;
418}
419
4ed0d3e6
FY		 420/*
421 * Calculate max SAGAW for each iommu.
422 */
423int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
424{
425 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
426}
427
428/*
429 * calculate agaw for each iommu.
430 * "SAGAW" may be different across iommus, use a default agaw, and
431 * get a supported less agaw for iommus that don't support the default agaw.
432 */
433int iommu_calculate_agaw(struct intel_iommu *iommu)
434{
435 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
436}
437
8c11e798
WH		 438/* in native case, each domain is related to only one iommu */
439static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
440{
441 int iommu_id;
442
1ce28feb
WH		 443 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
444
8c11e798
WH		 445 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
446 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
447 return NULL;
448
449 return g_iommus[iommu_id];
450}
451
8e604097
WH		 452static void domain_update_iommu_coherency(struct dmar_domain *domain)
453{
454 int i;
455
456 domain->iommu_coherency = 1;
457
458 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
459 for (; i < g_num_of_iommus; ) {
460 if (!ecap_coherent(g_iommus[i]->ecap)) {
461 domain->iommu_coherency = 0;
462 break;
463 }
464 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
465 }
466}
467
58c610bd
SY		 468static void domain_update_iommu_snooping(struct dmar_domain *domain)
469{
470 int i;
471
472 domain->iommu_snooping = 1;
473
474 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
475 for (; i < g_num_of_iommus; ) {
476 if (!ecap_sc_support(g_iommus[i]->ecap)) {
477 domain->iommu_snooping = 0;
478 break;
479 }
480 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
481 }
482}
483
484/* Some capabilities may be different across iommus */
485static void domain_update_iommu_cap(struct dmar_domain *domain)
486{
487 domain_update_iommu_coherency(domain);
488 domain_update_iommu_snooping(domain);
489}
490
276dbf99 491static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 492{
493 struct dmar_drhd_unit *drhd = NULL;
494 int i;
495
496 for_each_drhd_unit(drhd) {
497 if (drhd->ignored)
498 continue;
276dbf99
DW		 499 if (segment != drhd->segment)
500 continue;
c7151a8d 501
924b6231 502 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 503 if (drhd->devices[i] &&
504 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 505 drhd->devices[i]->devfn == devfn)
506 return drhd->iommu;
4958c5dc
DW		 507 if (drhd->devices[i] &&
508 drhd->devices[i]->subordinate &&
924b6231
DW		 509 drhd->devices[i]->subordinate->number <= bus &&
510 drhd->devices[i]->subordinate->subordinate >= bus)
511 return drhd->iommu;
512 }
c7151a8d
WH		 513
514 if (drhd->include_all)
515 return drhd->iommu;
516 }
517
518 return NULL;
519}
520
5331fe6f
WH		 521static void domain_flush_cache(struct dmar_domain *domain,
522 void *addr, int size)
523{
524 if (!domain->iommu_coherency)
525 clflush_cache_range(addr, size);
526}
527
ba395927
KA		 528/* Gets context entry for a given bus and devfn */
529static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
530 u8 bus, u8 devfn)
531{
532 struct root_entry *root;
533 struct context_entry *context;
534 unsigned long phy_addr;
535 unsigned long flags;
536
537 spin_lock_irqsave(&iommu->lock, flags);
538 root = &iommu->root_entry[bus];
539 context = get_context_addr_from_root(root);
540 if (!context) {
541 context = (struct context_entry *)alloc_pgtable_page();
542 if (!context) {
543 spin_unlock_irqrestore(&iommu->lock, flags);
544 return NULL;
545 }
5b6985ce 546 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 547 phy_addr = virt_to_phys((void *)context);
548 set_root_value(root, phy_addr);
549 set_root_present(root);
550 __iommu_flush_cache(iommu, root, sizeof(*root));
551 }
552 spin_unlock_irqrestore(&iommu->lock, flags);
553 return &context[devfn];
554}
555
556static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
557{
558 struct root_entry *root;
559 struct context_entry *context;
560 int ret;
561 unsigned long flags;
562
563 spin_lock_irqsave(&iommu->lock, flags);
564 root = &iommu->root_entry[bus];
565 context = get_context_addr_from_root(root);
566 if (!context) {
567 ret = 0;
568 goto out;
569 }
c07e7d21 570 ret = context_present(&context[devfn]);
ba395927
KA		 571out:
572 spin_unlock_irqrestore(&iommu->lock, flags);
573 return ret;
574}
575
576static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
577{
578 struct root_entry *root;
579 struct context_entry *context;
580 unsigned long flags;
581
582 spin_lock_irqsave(&iommu->lock, flags);
583 root = &iommu->root_entry[bus];
584 context = get_context_addr_from_root(root);
585 if (context) {
c07e7d21 586 context_clear_entry(&context[devfn]);
ba395927
KA		 587 __iommu_flush_cache(iommu, &context[devfn], \
588 sizeof(*context));
589 }
590 spin_unlock_irqrestore(&iommu->lock, flags);
591}
592
593static void free_context_table(struct intel_iommu *iommu)
594{
595 struct root_entry *root;
596 int i;
597 unsigned long flags;
598 struct context_entry *context;
599
600 spin_lock_irqsave(&iommu->lock, flags);
601 if (!iommu->root_entry) {
602 goto out;
603 }
604 for (i = 0; i < ROOT_ENTRY_NR; i++) {
605 root = &iommu->root_entry[i];
606 context = get_context_addr_from_root(root);
607 if (context)
608 free_pgtable_page(context);
609 }
610 free_pgtable_page(iommu->root_entry);
611 iommu->root_entry = NULL;
612out:
613 spin_unlock_irqrestore(&iommu->lock, flags);
614}
615
616/* page table handling */
617#define LEVEL_STRIDE (9)
618#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
619
620static inline int agaw_to_level(int agaw)
621{
622 return agaw + 2;
623}
624
625static inline int agaw_to_width(int agaw)
626{
627 return 30 + agaw * LEVEL_STRIDE;
628
629}
630
631static inline int width_to_agaw(int width)
632{
633 return (width - 30) / LEVEL_STRIDE;
634}
635
636static inline unsigned int level_to_offset_bits(int level)
637{
638 return (12 + (level - 1) * LEVEL_STRIDE);
639}
640
641static inline int address_level_offset(u64 addr, int level)
642{
643 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
644}
645
646static inline u64 level_mask(int level)
647{
648 return ((u64)-1 << level_to_offset_bits(level));
649}
650
651static inline u64 level_size(int level)
652{
653 return ((u64)1 << level_to_offset_bits(level));
654}
655
656static inline u64 align_to_level(u64 addr, int level)
657{
658 return ((addr + level_size(level) - 1) & level_mask(level));
659}
660
661static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
662{
663 int addr_width = agaw_to_width(domain->agaw);
664 struct dma_pte *parent, *pte = NULL;
665 int level = agaw_to_level(domain->agaw);
666 int offset;
667 unsigned long flags;
668
669 BUG_ON(!domain->pgd);
670
671 addr &= (((u64)1) << addr_width) - 1;
672 parent = domain->pgd;
673
674 spin_lock_irqsave(&domain->mapping_lock, flags);
675 while (level > 0) {
676 void *tmp_page;
677
678 offset = address_level_offset(addr, level);
679 pte = &parent[offset];
680 if (level == 1)
681 break;
682
19c239ce 683 if (!dma_pte_present(pte)) {
ba395927
KA		 684 tmp_page = alloc_pgtable_page();
685
686 if (!tmp_page) {
687 spin_unlock_irqrestore(&domain->mapping_lock,
688 flags);
689 return NULL;
690 }
5331fe6f 691 domain_flush_cache(domain, tmp_page, PAGE_SIZE);
19c239ce 692 dma_set_pte_addr(pte, virt_to_phys(tmp_page));
ba395927
KA		 693 /*
694 * high level table always sets r/w, last level page
695 * table control read/write
696 */
19c239ce
MM		 697 dma_set_pte_readable(pte);
698 dma_set_pte_writable(pte);
5331fe6f 699 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 700 }
19c239ce 701 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 702 level--;
703 }
704
705 spin_unlock_irqrestore(&domain->mapping_lock, flags);
706 return pte;
707}
708
709/* return address's pte at specific level */
710static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
711 int level)
712{
713 struct dma_pte *parent, *pte = NULL;
714 int total = agaw_to_level(domain->agaw);
715 int offset;
716
717 parent = domain->pgd;
718 while (level <= total) {
719 offset = address_level_offset(addr, total);
720 pte = &parent[offset];
721 if (level == total)
722 return pte;
723
19c239ce 724 if (!dma_pte_present(pte))
ba395927 725 break;
19c239ce 726 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 727 total--;
728 }
729 return NULL;
730}
731
732/* clear one page's page table */
733static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
734{
735 struct dma_pte *pte = NULL;
736
737 /* get last level pte */
738 pte = dma_addr_level_pte(domain, addr, 1);
739
740 if (pte) {
19c239ce 741 dma_clear_pte(pte);
5331fe6f 742 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927
KA		 743 }
744}
745
746/* clear last level pte, a tlb flush should be followed */
747static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
748{
749 int addr_width = agaw_to_width(domain->agaw);
afeeb7ce 750 int npages;
ba395927
KA		 751
752 start &= (((u64)1) << addr_width) - 1;
753 end &= (((u64)1) << addr_width) - 1;
754 /* in case it's partial page */
31d3568d
FY		 755 start &= PAGE_MASK;
756 end = PAGE_ALIGN(end);
afeeb7ce 757 npages = (end - start) / VTD_PAGE_SIZE;
ba395927
KA		 758
759 /* we don't need lock here, nobody else touches the iova range */
afeeb7ce 760 while (npages--) {
ba395927 761 dma_pte_clear_one(domain, start);
5b6985ce 762 start += VTD_PAGE_SIZE;
ba395927
KA		 763 }
764}
765
766/* free page table pages. last level pte should already be cleared */
767static void dma_pte_free_pagetable(struct dmar_domain *domain,
768 u64 start, u64 end)
769{
770 int addr_width = agaw_to_width(domain->agaw);
771 struct dma_pte *pte;
772 int total = agaw_to_level(domain->agaw);
773 int level;
774 u64 tmp;
775
776 start &= (((u64)1) << addr_width) - 1;
777 end &= (((u64)1) << addr_width) - 1;
778
779 /* we don't need lock here, nobody else touches the iova range */
780 level = 2;
781 while (level <= total) {
782 tmp = align_to_level(start, level);
783 if (tmp >= end || (tmp + level_size(level) > end))
784 return;
785
786 while (tmp < end) {
787 pte = dma_addr_level_pte(domain, tmp, level);
788 if (pte) {
789 free_pgtable_page(
19c239ce
MM		 790 phys_to_virt(dma_pte_addr(pte)));
791 dma_clear_pte(pte);
5331fe6f 792 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927
KA		 793 }
794 tmp += level_size(level);
795 }
796 level++;
797 }
798 /* free pgd */
799 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
800 free_pgtable_page(domain->pgd);
801 domain->pgd = NULL;
802 }
803}
804
805/* iommu handling */
806static int iommu_alloc_root_entry(struct intel_iommu *iommu)
807{
808 struct root_entry *root;
809 unsigned long flags;
810
811 root = (struct root_entry *)alloc_pgtable_page();
812 if (!root)
813 return -ENOMEM;
814
5b6985ce 815 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 816
817 spin_lock_irqsave(&iommu->lock, flags);
818 iommu->root_entry = root;
819 spin_unlock_irqrestore(&iommu->lock, flags);
820
821 return 0;
822}
823
ba395927
KA		 824static void iommu_set_root_entry(struct intel_iommu *iommu)
825{
826 void *addr;
c416daa9 827 u32 sts;
ba395927
KA		 828 unsigned long flag;
829
830 addr = iommu->root_entry;
831
832 spin_lock_irqsave(&iommu->register_lock, flag);
833 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
834
c416daa9 835 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 836
837 /* Make sure hardware complete it */
838 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 839 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927
KA		 840
841 spin_unlock_irqrestore(&iommu->register_lock, flag);
842}
843
844static void iommu_flush_write_buffer(struct intel_iommu *iommu)
845{
846 u32 val;
847 unsigned long flag;
848
9af88143 849 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 850 return;
ba395927
KA		 851
852 spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 853 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 854
855 /* Make sure hardware complete it */
856 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 857 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927
KA		 858
859 spin_unlock_irqrestore(&iommu->register_lock, flag);
860}
861
862/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 863static void __iommu_flush_context(struct intel_iommu *iommu,
864 u16 did, u16 source_id, u8 function_mask,
865 u64 type)
ba395927
KA		 866{
867 u64 val = 0;
868 unsigned long flag;
869
ba395927
KA		 870 switch (type) {
871 case DMA_CCMD_GLOBAL_INVL:
872 val = DMA_CCMD_GLOBAL_INVL;
873 break;
874 case DMA_CCMD_DOMAIN_INVL:
875 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
876 break;
877 case DMA_CCMD_DEVICE_INVL:
878 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
879 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
880 break;
881 default:
882 BUG();
883 }
884 val |= DMA_CCMD_ICC;
885
886 spin_lock_irqsave(&iommu->register_lock, flag);
887 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
888
889 /* Make sure hardware complete it */
890 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
891 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
892
893 spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 894}
895
ba395927 896/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 897static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
898 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 899{
900 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
901 u64 val = 0, val_iva = 0;
902 unsigned long flag;
903
ba395927
KA		 904 switch (type) {
905 case DMA_TLB_GLOBAL_FLUSH:
906 /* global flush doesn't need set IVA_REG */
907 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
908 break;
909 case DMA_TLB_DSI_FLUSH:
910 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
911 break;
912 case DMA_TLB_PSI_FLUSH:
913 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
914 /* Note: always flush non-leaf currently */
915 val_iva = size_order | addr;
916 break;
917 default:
918 BUG();
919 }
920 /* Note: set drain read/write */
921#if 0
922 /*
923 * This is probably to be super secure.. Looks like we can
924 * ignore it without any impact.
925 */
926 if (cap_read_drain(iommu->cap))
927 val |= DMA_TLB_READ_DRAIN;
928#endif
929 if (cap_write_drain(iommu->cap))
930 val |= DMA_TLB_WRITE_DRAIN;
931
932 spin_lock_irqsave(&iommu->register_lock, flag);
933 /* Note: Only uses first TLB reg currently */
934 if (val_iva)
935 dmar_writeq(iommu->reg + tlb_offset, val_iva);
936 dmar_writeq(iommu->reg + tlb_offset + 8, val);
937
938 /* Make sure hardware complete it */
939 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
940 dmar_readq, (!(val & DMA_TLB_IVT)), val);
941
942 spin_unlock_irqrestore(&iommu->register_lock, flag);
943
944 /* check IOTLB invalidation granularity */
945 if (DMA_TLB_IAIG(val) == 0)
946 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
947 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
948 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 949 (unsigned long long)DMA_TLB_IIRG(type),
950 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 951}
952
93a23a72
YZ		 953static struct device_domain_info *iommu_support_dev_iotlb(
954 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
955{
956 int found = 0;
957 unsigned long flags;
958 struct device_domain_info *info;
959 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
960
961 if (!ecap_dev_iotlb_support(iommu->ecap))
962 return NULL;
963
964 if (!iommu->qi)
965 return NULL;
966
967 spin_lock_irqsave(&device_domain_lock, flags);
968 list_for_each_entry(info, &domain->devices, link)
969 if (info->bus == bus && info->devfn == devfn) {
970 found = 1;
971 break;
972 }
973 spin_unlock_irqrestore(&device_domain_lock, flags);
974
975 if (!found || !info->dev)
976 return NULL;
977
978 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
979 return NULL;
980
981 if (!dmar_find_matched_atsr_unit(info->dev))
982 return NULL;
983
984 info->iommu = iommu;
985
986 return info;
987}
988
989static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 990{
93a23a72
YZ		 991 if (!info)
992 return;
993
994 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
995}
996
997static void iommu_disable_dev_iotlb(struct device_domain_info *info)
998{
999 if (!info->dev || !pci_ats_enabled(info->dev))
1000 return;
1001
1002 pci_disable_ats(info->dev);
1003}
1004
1005static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1006 u64 addr, unsigned mask)
1007{
1008 u16 sid, qdep;
1009 unsigned long flags;
1010 struct device_domain_info *info;
1011
1012 spin_lock_irqsave(&device_domain_lock, flags);
1013 list_for_each_entry(info, &domain->devices, link) {
1014 if (!info->dev || !pci_ats_enabled(info->dev))
1015 continue;
1016
1017 sid = info->bus << 8 | info->devfn;
1018 qdep = pci_ats_queue_depth(info->dev);
1019 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1020 }
1021 spin_unlock_irqrestore(&device_domain_lock, flags);
1022}
1023
1f0ef2aa
DW		 1024static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1025 u64 addr, unsigned int pages)
ba395927 1026{
9dd2fe89 1027 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
ba395927 1028
5b6985ce 1029 BUG_ON(addr & (~VTD_PAGE_MASK));
ba395927
KA		 1030 BUG_ON(pages == 0);
1031
ba395927 1032 /*
9dd2fe89
YZ		 1033 * Fallback to domain selective flush if no PSI support or the size is
1034 * too big.
ba395927
KA		 1035 * PSI requires page size to be 2 ^ x, and the base address is naturally
1036 * aligned to the size
1037 */
9dd2fe89
YZ		 1038 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1039 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1040 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1041 else
1042 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1043 DMA_TLB_PSI_FLUSH);
93a23a72
YZ		 1044 if (did)
1045 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1046}
1047
f8bab735 1048static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1049{
1050 u32 pmen;
1051 unsigned long flags;
1052
1053 spin_lock_irqsave(&iommu->register_lock, flags);
1054 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1055 pmen &= ~DMA_PMEN_EPM;
1056 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1057
1058 /* wait for the protected region status bit to clear */
1059 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1060 readl, !(pmen & DMA_PMEN_PRS), pmen);
1061
1062 spin_unlock_irqrestore(&iommu->register_lock, flags);
1063}
1064
ba395927
KA		 1065static int iommu_enable_translation(struct intel_iommu *iommu)
1066{
1067 u32 sts;
1068 unsigned long flags;
1069
1070 spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1071 iommu->gcmd |= DMA_GCMD_TE;
1072 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1073
1074 /* Make sure hardware complete it */
1075 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1076 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1077
ba395927
KA		 1078 spin_unlock_irqrestore(&iommu->register_lock, flags);
1079 return 0;
1080}
1081
1082static int iommu_disable_translation(struct intel_iommu *iommu)
1083{
1084 u32 sts;
1085 unsigned long flag;
1086
1087 spin_lock_irqsave(&iommu->register_lock, flag);
1088 iommu->gcmd &= ~DMA_GCMD_TE;
1089 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1090
1091 /* Make sure hardware complete it */
1092 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1093 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927
KA		 1094
1095 spin_unlock_irqrestore(&iommu->register_lock, flag);
1096 return 0;
1097}
1098
3460a6d9 1099
ba395927
KA		 1100static int iommu_init_domains(struct intel_iommu *iommu)
1101{
1102 unsigned long ndomains;
1103 unsigned long nlongs;
1104
1105 ndomains = cap_ndoms(iommu->cap);
1106 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1107 nlongs = BITS_TO_LONGS(ndomains);
1108
1109 /* TBD: there might be 64K domains,
1110 * consider other allocation for future chip
1111 */
1112 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1113 if (!iommu->domain_ids) {
1114 printk(KERN_ERR "Allocating domain id array failed\n");
1115 return -ENOMEM;
1116 }
1117 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1118 GFP_KERNEL);
1119 if (!iommu->domains) {
1120 printk(KERN_ERR "Allocating domain array failed\n");
1121 kfree(iommu->domain_ids);
1122 return -ENOMEM;
1123 }
1124
e61d98d8
SS		 1125 spin_lock_init(&iommu->lock);
1126
ba395927
KA		 1127 /*
1128 * if Caching mode is set, then invalid translations are tagged
1129 * with domainid 0. Hence we need to pre-allocate it.
1130 */
1131 if (cap_caching_mode(iommu->cap))
1132 set_bit(0, iommu->domain_ids);
1133 return 0;
1134}
ba395927 1135
ba395927
KA		 1136
1137static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1138static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1139
1140void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1141{
1142 struct dmar_domain *domain;
1143 int i;
c7151a8d 1144 unsigned long flags;
ba395927 1145
ba395927
KA		 1146 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1147 for (; i < cap_ndoms(iommu->cap); ) {
1148 domain = iommu->domains[i];
1149 clear_bit(i, iommu->domain_ids);
c7151a8d
WH		 1150
1151 spin_lock_irqsave(&domain->iommu_lock, flags);
5e98c4b1
WH		 1152 if (--domain->iommu_count == 0) {
1153 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1154 vm_domain_exit(domain);
1155 else
1156 domain_exit(domain);
1157 }
c7151a8d
WH		 1158 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1159
ba395927
KA		 1160 i = find_next_bit(iommu->domain_ids,
1161 cap_ndoms(iommu->cap), i+1);
1162 }
1163
1164 if (iommu->gcmd & DMA_GCMD_TE)
1165 iommu_disable_translation(iommu);
1166
1167 if (iommu->irq) {
1168 set_irq_data(iommu->irq, NULL);
1169 /* This will mask the irq */
1170 free_irq(iommu->irq, iommu);
1171 destroy_irq(iommu->irq);
1172 }
1173
1174 kfree(iommu->domains);
1175 kfree(iommu->domain_ids);
1176
d9630fe9
WH		 1177 g_iommus[iommu->seq_id] = NULL;
1178
1179 /* if all iommus are freed, free g_iommus */
1180 for (i = 0; i < g_num_of_iommus; i++) {
1181 if (g_iommus[i])
1182 break;
1183 }
1184
1185 if (i == g_num_of_iommus)
1186 kfree(g_iommus);
1187
ba395927
KA		 1188 /* free context mapping */
1189 free_context_table(iommu);
ba395927
KA		 1190}
1191
1192static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1193{
1194 unsigned long num;
1195 unsigned long ndomains;
1196 struct dmar_domain *domain;
1197 unsigned long flags;
1198
1199 domain = alloc_domain_mem();
1200 if (!domain)
1201 return NULL;
1202
1203 ndomains = cap_ndoms(iommu->cap);
1204
1205 spin_lock_irqsave(&iommu->lock, flags);
1206 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1207 if (num >= ndomains) {
1208 spin_unlock_irqrestore(&iommu->lock, flags);
1209 free_domain_mem(domain);
1210 printk(KERN_ERR "IOMMU: no free domain ids\n");
1211 return NULL;
1212 }
1213
1214 set_bit(num, iommu->domain_ids);
1215 domain->id = num;
8c11e798
WH		 1216 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1217 set_bit(iommu->seq_id, &domain->iommu_bmp);
d71a2f33 1218 domain->flags = 0;
ba395927
KA		 1219 iommu->domains[num] = domain;
1220 spin_unlock_irqrestore(&iommu->lock, flags);
1221
1222 return domain;
1223}
1224
1225static void iommu_free_domain(struct dmar_domain *domain)
1226{
1227 unsigned long flags;
8c11e798
WH		 1228 struct intel_iommu *iommu;
1229
1230 iommu = domain_get_iommu(domain);
ba395927 1231
8c11e798
WH		 1232 spin_lock_irqsave(&iommu->lock, flags);
1233 clear_bit(domain->id, iommu->domain_ids);
1234 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1235}
1236
1237static struct iova_domain reserved_iova_list;
8a443df4
MG		 1238static struct lock_class_key reserved_alloc_key;
1239static struct lock_class_key reserved_rbtree_key;
ba395927
KA		 1240
1241static void dmar_init_reserved_ranges(void)
1242{
1243 struct pci_dev *pdev = NULL;
1244 struct iova *iova;
1245 int i;
1246 u64 addr, size;
1247
f661197e 1248 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1249
8a443df4
MG		 1250 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1251 &reserved_alloc_key);
1252 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1253 &reserved_rbtree_key);
1254
ba395927
KA		 1255 /* IOAPIC ranges shouldn't be accessed by DMA */
1256 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1257 IOVA_PFN(IOAPIC_RANGE_END));
1258 if (!iova)
1259 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1260
1261 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1262 for_each_pci_dev(pdev) {
1263 struct resource *r;
1264
1265 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1266 r = &pdev->resource[i];
1267 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1268 continue;
1269 addr = r->start;
fd18de50 1270 addr &= PHYSICAL_PAGE_MASK;
ba395927 1271 size = r->end - addr;
5b6985ce 1272 size = PAGE_ALIGN(size);
ba395927
KA		 1273 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1274 IOVA_PFN(size + addr) - 1);
1275 if (!iova)
1276 printk(KERN_ERR "Reserve iova failed\n");
1277 }
1278 }
1279
1280}
1281
1282static void domain_reserve_special_ranges(struct dmar_domain *domain)
1283{
1284 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1285}
1286
1287static inline int guestwidth_to_adjustwidth(int gaw)
1288{
1289 int agaw;
1290 int r = (gaw - 12) % 9;
1291
1292 if (r == 0)
1293 agaw = gaw;
1294 else
1295 agaw = gaw + 9 - r;
1296 if (agaw > 64)
1297 agaw = 64;
1298 return agaw;
1299}
1300
1301static int domain_init(struct dmar_domain *domain, int guest_width)
1302{
1303 struct intel_iommu *iommu;
1304 int adjust_width, agaw;
1305 unsigned long sagaw;
1306
f661197e 1307 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
ba395927 1308 spin_lock_init(&domain->mapping_lock);
c7151a8d 1309 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1310
1311 domain_reserve_special_ranges(domain);
1312
1313 /* calculate AGAW */
8c11e798 1314 iommu = domain_get_iommu(domain);
ba395927
KA		 1315 if (guest_width > cap_mgaw(iommu->cap))
1316 guest_width = cap_mgaw(iommu->cap);
1317 domain->gaw = guest_width;
1318 adjust_width = guestwidth_to_adjustwidth(guest_width);
1319 agaw = width_to_agaw(adjust_width);
1320 sagaw = cap_sagaw(iommu->cap);
1321 if (!test_bit(agaw, &sagaw)) {
1322 /* hardware doesn't support it, choose a bigger one */
1323 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1324 agaw = find_next_bit(&sagaw, 5, agaw);
1325 if (agaw >= 5)
1326 return -ENODEV;
1327 }
1328 domain->agaw = agaw;
1329 INIT_LIST_HEAD(&domain->devices);
1330
8e604097
WH		 1331 if (ecap_coherent(iommu->ecap))
1332 domain->iommu_coherency = 1;
1333 else
1334 domain->iommu_coherency = 0;
1335
58c610bd
SY		 1336 if (ecap_sc_support(iommu->ecap))
1337 domain->iommu_snooping = 1;
1338 else
1339 domain->iommu_snooping = 0;
1340
c7151a8d
WH		 1341 domain->iommu_count = 1;
1342
ba395927
KA		 1343 /* always allocate the top pgd */
1344 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1345 if (!domain->pgd)
1346 return -ENOMEM;
5b6985ce 1347 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1348 return 0;
1349}
1350
1351static void domain_exit(struct dmar_domain *domain)
1352{
1353 u64 end;
1354
1355 /* Domain 0 is reserved, so dont process it */
1356 if (!domain)
1357 return;
1358
1359 domain_remove_dev_info(domain);
1360 /* destroy iovas */
1361 put_iova_domain(&domain->iovad);
1362 end = DOMAIN_MAX_ADDR(domain->gaw);
5b6985ce 1363 end = end & (~PAGE_MASK);
ba395927
KA		 1364
1365 /* clear ptes */
1366 dma_pte_clear_range(domain, 0, end);
1367
1368 /* free page tables */
1369 dma_pte_free_pagetable(domain, 0, end);
1370
1371 iommu_free_domain(domain);
1372 free_domain_mem(domain);
1373}
1374
4ed0d3e6
FY		 1375static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1376 u8 bus, u8 devfn, int translation)
ba395927
KA		 1377{
1378 struct context_entry *context;
ba395927 1379 unsigned long flags;
5331fe6f 1380 struct intel_iommu *iommu;
ea6606b0
WH		 1381 struct dma_pte *pgd;
1382 unsigned long num;
1383 unsigned long ndomains;
1384 int id;
1385 int agaw;
93a23a72 1386 struct device_domain_info *info = NULL;
ba395927
KA		 1387
1388 pr_debug("Set context mapping for %02x:%02x.%d\n",
1389 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1390
ba395927 1391 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1392 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1393 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1394
276dbf99 1395 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1396 if (!iommu)
1397 return -ENODEV;
1398
ba395927
KA		 1399 context = device_to_context_entry(iommu, bus, devfn);
1400 if (!context)
1401 return -ENOMEM;
1402 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1403 if (context_present(context)) {
ba395927
KA		 1404 spin_unlock_irqrestore(&iommu->lock, flags);
1405 return 0;
1406 }
1407
ea6606b0
WH		 1408 id = domain->id;
1409 pgd = domain->pgd;
1410
1411 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
1412 int found = 0;
1413
1414 /* find an available domain id for this device in iommu */
1415 ndomains = cap_ndoms(iommu->cap);
1416 num = find_first_bit(iommu->domain_ids, ndomains);
1417 for (; num < ndomains; ) {
1418 if (iommu->domains[num] == domain) {
1419 id = num;
1420 found = 1;
1421 break;
1422 }
1423 num = find_next_bit(iommu->domain_ids,
1424 cap_ndoms(iommu->cap), num+1);
1425 }
1426
1427 if (found == 0) {
1428 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1429 if (num >= ndomains) {
1430 spin_unlock_irqrestore(&iommu->lock, flags);
1431 printk(KERN_ERR "IOMMU: no free domain ids\n");
1432 return -EFAULT;
1433 }
1434
1435 set_bit(num, iommu->domain_ids);
1436 iommu->domains[num] = domain;
1437 id = num;
1438 }
1439
1440 /* Skip top levels of page tables for
1441 * iommu which has less agaw than default.
1442 */
1443 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1444 pgd = phys_to_virt(dma_pte_addr(pgd));
1445 if (!dma_pte_present(pgd)) {
1446 spin_unlock_irqrestore(&iommu->lock, flags);
1447 return -ENOMEM;
1448 }
1449 }
1450 }
1451
1452 context_set_domain_id(context, id);
4ed0d3e6 1453
93a23a72
YZ		 1454 if (translation != CONTEXT_TT_PASS_THROUGH) {
1455 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1456 translation = info ? CONTEXT_TT_DEV_IOTLB :
1457 CONTEXT_TT_MULTI_LEVEL;
1458 }
4ed0d3e6
FY		 1459 /*
1460 * In pass through mode, AW must be programmed to indicate the largest
1461 * AGAW value supported by hardware. And ASR is ignored by hardware.
1462 */
93a23a72 1463 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1464 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1465 else {
1466 context_set_address_root(context, virt_to_phys(pgd));
1467 context_set_address_width(context, iommu->agaw);
1468 }
4ed0d3e6
FY		 1469
1470 context_set_translation_type(context, translation);
c07e7d21
MM		 1471 context_set_fault_enable(context);
1472 context_set_present(context);
5331fe6f 1473 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1474
4c25a2c1
DW		 1475 /*
1476 * It's a non-present to present mapping. If hardware doesn't cache
1477 * non-present entry we only need to flush the write-buffer. If the
1478 * _does_ cache non-present entries, then it does so in the special
1479 * domain #0, which we have to flush:
1480 */
1481 if (cap_caching_mode(iommu->cap)) {
1482 iommu->flush.flush_context(iommu, 0,
1483 (((u16)bus) << 8) | devfn,
1484 DMA_CCMD_MASK_NOBIT,
1485 DMA_CCMD_DEVICE_INVL);
1f0ef2aa 1486 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1487 } else {
ba395927 1488 iommu_flush_write_buffer(iommu);
4c25a2c1 1489 }
93a23a72 1490 iommu_enable_dev_iotlb(info);
ba395927 1491 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1492
1493 spin_lock_irqsave(&domain->iommu_lock, flags);
1494 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1495 domain->iommu_count++;
58c610bd 1496 domain_update_iommu_cap(domain);
c7151a8d
WH		 1497 }
1498 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1499 return 0;
1500}
1501
1502static int
4ed0d3e6
FY		 1503domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1504 int translation)
ba395927
KA		 1505{
1506 int ret;
1507 struct pci_dev *tmp, *parent;
1508
276dbf99 1509 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1510 pdev->bus->number, pdev->devfn,
1511 translation);
ba395927
KA		 1512 if (ret)
1513 return ret;
1514
1515 /* dependent device mapping */
1516 tmp = pci_find_upstream_pcie_bridge(pdev);
1517 if (!tmp)
1518 return 0;
1519 /* Secondary interface's bus number and devfn 0 */
1520 parent = pdev->bus->self;
1521 while (parent != tmp) {
276dbf99
DW		 1522 ret = domain_context_mapping_one(domain,
1523 pci_domain_nr(parent->bus),
1524 parent->bus->number,
4ed0d3e6 1525 parent->devfn, translation);
ba395927
KA		 1526 if (ret)
1527 return ret;
1528 parent = parent->bus->self;
1529 }
1530 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1531 return domain_context_mapping_one(domain,
276dbf99 1532 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1533 tmp->subordinate->number, 0,
1534 translation);
ba395927
KA		 1535 else /* this is a legacy PCI bridge */
1536 return domain_context_mapping_one(domain,
276dbf99
DW		 1537 pci_domain_nr(tmp->bus),
1538 tmp->bus->number,
4ed0d3e6
FY		 1539 tmp->devfn,
1540 translation);
ba395927
KA		 1541}
1542
5331fe6f 1543static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1544{
1545 int ret;
1546 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1547 struct intel_iommu *iommu;
1548
276dbf99
DW		 1549 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1550 pdev->devfn);
5331fe6f
WH		 1551 if (!iommu)
1552 return -ENODEV;
ba395927 1553
276dbf99 1554 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1555 if (!ret)
1556 return ret;
1557 /* dependent device mapping */
1558 tmp = pci_find_upstream_pcie_bridge(pdev);
1559 if (!tmp)
1560 return ret;
1561 /* Secondary interface's bus number and devfn 0 */
1562 parent = pdev->bus->self;
1563 while (parent != tmp) {
8c11e798 1564 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1565 parent->devfn);
ba395927
KA		 1566 if (!ret)
1567 return ret;
1568 parent = parent->bus->self;
1569 }
1570 if (tmp->is_pcie)
276dbf99
DW		 1571 return device_context_mapped(iommu, tmp->subordinate->number,
1572 0);
ba395927 1573 else
276dbf99
DW		 1574 return device_context_mapped(iommu, tmp->bus->number,
1575 tmp->devfn);
ba395927
KA		 1576}
1577
1578static int
1579domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1580 u64 hpa, size_t size, int prot)
1581{
1582 u64 start_pfn, end_pfn;
1583 struct dma_pte *pte;
1584 int index;
5b6985ce
FY		 1585 int addr_width = agaw_to_width(domain->agaw);
1586
1587 hpa &= (((u64)1) << addr_width) - 1;
ba395927
KA		 1588
1589 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1590 return -EINVAL;
5b6985ce
FY		 1591 iova &= PAGE_MASK;
1592 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1593 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
ba395927
KA		 1594 index = 0;
1595 while (start_pfn < end_pfn) {
5b6985ce 1596 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
ba395927
KA		 1597 if (!pte)
1598 return -ENOMEM;
1599 /* We don't need lock here, nobody else
1600 * touches the iova range
1601 */
19c239ce
MM		 1602 BUG_ON(dma_pte_addr(pte));
1603 dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT);
1604 dma_set_pte_prot(pte, prot);
9cf06697
SY		 1605 if (prot & DMA_PTE_SNP)
1606 dma_set_pte_snp(pte);
5331fe6f 1607 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927
KA		 1608 start_pfn++;
1609 index++;
1610 }
1611 return 0;
1612}
1613
c7151a8d 1614static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1615{
c7151a8d
WH		 1616 if (!iommu)
1617 return;
8c11e798
WH		 1618
1619 clear_context_table(iommu, bus, devfn);
1620 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1621 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1622 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1623}
1624
1625static void domain_remove_dev_info(struct dmar_domain *domain)
1626{
1627 struct device_domain_info *info;
1628 unsigned long flags;
c7151a8d 1629 struct intel_iommu *iommu;
ba395927
KA		 1630
1631 spin_lock_irqsave(&device_domain_lock, flags);
1632 while (!list_empty(&domain->devices)) {
1633 info = list_entry(domain->devices.next,
1634 struct device_domain_info, link);
1635 list_del(&info->link);
1636 list_del(&info->global);
1637 if (info->dev)
358dd8ac 1638 info->dev->dev.archdata.iommu = NULL;
ba395927
KA		 1639 spin_unlock_irqrestore(&device_domain_lock, flags);
1640
93a23a72 1641 iommu_disable_dev_iotlb(info);
276dbf99 1642 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1643 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1644 free_devinfo_mem(info);
1645
1646 spin_lock_irqsave(&device_domain_lock, flags);
1647 }
1648 spin_unlock_irqrestore(&device_domain_lock, flags);
1649}
1650
1651/*
1652 * find_domain
358dd8ac 1653 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1654 */
38717946 1655static struct dmar_domain *
ba395927
KA		 1656find_domain(struct pci_dev *pdev)
1657{
1658 struct device_domain_info *info;
1659
1660 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1661 info = pdev->dev.archdata.iommu;
ba395927
KA		 1662 if (info)
1663 return info->domain;
1664 return NULL;
1665}
1666
ba395927
KA		 1667/* domain is initialized */
1668static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1669{
1670 struct dmar_domain *domain, *found = NULL;
1671 struct intel_iommu *iommu;
1672 struct dmar_drhd_unit *drhd;
1673 struct device_domain_info *info, *tmp;
1674 struct pci_dev *dev_tmp;
1675 unsigned long flags;
1676 int bus = 0, devfn = 0;
276dbf99 1677 int segment;
ba395927
KA		 1678
1679 domain = find_domain(pdev);
1680 if (domain)
1681 return domain;
1682
276dbf99
DW		 1683 segment = pci_domain_nr(pdev->bus);
1684
ba395927
KA		 1685 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1686 if (dev_tmp) {
1687 if (dev_tmp->is_pcie) {
1688 bus = dev_tmp->subordinate->number;
1689 devfn = 0;
1690 } else {
1691 bus = dev_tmp->bus->number;
1692 devfn = dev_tmp->devfn;
1693 }
1694 spin_lock_irqsave(&device_domain_lock, flags);
1695 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 1696 if (info->segment == segment &&
1697 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 1698 found = info->domain;
1699 break;
1700 }
1701 }
1702 spin_unlock_irqrestore(&device_domain_lock, flags);
1703 /* pcie-pci bridge already has a domain, uses it */
1704 if (found) {
1705 domain = found;
1706 goto found_domain;
1707 }
1708 }
1709
1710 /* Allocate new domain for the device */
1711 drhd = dmar_find_matched_drhd_unit(pdev);
1712 if (!drhd) {
1713 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1714 pci_name(pdev));
1715 return NULL;
1716 }
1717 iommu = drhd->iommu;
1718
1719 domain = iommu_alloc_domain(iommu);
1720 if (!domain)
1721 goto error;
1722
1723 if (domain_init(domain, gaw)) {
1724 domain_exit(domain);
1725 goto error;
1726 }
1727
1728 /* register pcie-to-pci device */
1729 if (dev_tmp) {
1730 info = alloc_devinfo_mem();
1731 if (!info) {
1732 domain_exit(domain);
1733 goto error;
1734 }
276dbf99 1735 info->segment = segment;
ba395927
KA		 1736 info->bus = bus;
1737 info->devfn = devfn;
1738 info->dev = NULL;
1739 info->domain = domain;
1740 /* This domain is shared by devices under p2p bridge */
3b5410e7 1741 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 1742
1743 /* pcie-to-pci bridge already has a domain, uses it */
1744 found = NULL;
1745 spin_lock_irqsave(&device_domain_lock, flags);
1746 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 1747 if (tmp->segment == segment &&
1748 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 1749 found = tmp->domain;
1750 break;
1751 }
1752 }
1753 if (found) {
1754 free_devinfo_mem(info);
1755 domain_exit(domain);
1756 domain = found;
1757 } else {
1758 list_add(&info->link, &domain->devices);
1759 list_add(&info->global, &device_domain_list);
1760 }
1761 spin_unlock_irqrestore(&device_domain_lock, flags);
1762 }
1763
1764found_domain:
1765 info = alloc_devinfo_mem();
1766 if (!info)
1767 goto error;
276dbf99 1768 info->segment = segment;
ba395927
KA		 1769 info->bus = pdev->bus->number;
1770 info->devfn = pdev->devfn;
1771 info->dev = pdev;
1772 info->domain = domain;
1773 spin_lock_irqsave(&device_domain_lock, flags);
1774 /* somebody is fast */
1775 found = find_domain(pdev);
1776 if (found != NULL) {
1777 spin_unlock_irqrestore(&device_domain_lock, flags);
1778 if (found != domain) {
1779 domain_exit(domain);
1780 domain = found;
1781 }
1782 free_devinfo_mem(info);
1783 return domain;
1784 }
1785 list_add(&info->link, &domain->devices);
1786 list_add(&info->global, &device_domain_list);
358dd8ac 1787 pdev->dev.archdata.iommu = info;
ba395927
KA		 1788 spin_unlock_irqrestore(&device_domain_lock, flags);
1789 return domain;
1790error:
1791 /* recheck it here, maybe others set it */
1792 return find_domain(pdev);
1793}
1794
5b6985ce
FY		 1795static int iommu_prepare_identity_map(struct pci_dev *pdev,
1796 unsigned long long start,
1797 unsigned long long end)
ba395927
KA		 1798{
1799 struct dmar_domain *domain;
1800 unsigned long size;
5b6985ce 1801 unsigned long long base;
ba395927
KA		 1802 int ret;
1803
1804 printk(KERN_INFO
1805 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1806 pci_name(pdev), start, end);
1807 /* page table init */
1808 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1809 if (!domain)
1810 return -ENOMEM;
1811
1812 /* The address might not be aligned */
5b6985ce 1813 base = start & PAGE_MASK;
ba395927 1814 size = end - base;
5b6985ce 1815 size = PAGE_ALIGN(size);
ba395927
KA		 1816 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1817 IOVA_PFN(base + size) - 1)) {
1818 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1819 ret = -ENOMEM;
1820 goto error;
1821 }
1822
1823 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1824 size, base, pci_name(pdev));
1825 /*
1826 * RMRR range might have overlap with physical memory range,
1827 * clear it first
1828 */
1829 dma_pte_clear_range(domain, base, base + size);
1830
1831 ret = domain_page_mapping(domain, base, base, size,
1832 DMA_PTE_READ|DMA_PTE_WRITE);
1833 if (ret)
1834 goto error;
1835
1836 /* context entry init */
4ed0d3e6 1837 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
ba395927
KA		 1838 if (!ret)
1839 return 0;
1840error:
1841 domain_exit(domain);
1842 return ret;
1843
1844}
1845
1846static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1847 struct pci_dev *pdev)
1848{
358dd8ac 1849 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 1850 return 0;
1851 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1852 rmrr->end_address + 1);
1853}
1854
e820482c 1855#ifdef CONFIG_DMAR_GFX_WA
d52d53b8
YL		 1856struct iommu_prepare_data {
1857 struct pci_dev *pdev;
1858 int ret;
1859};
1860
1861static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1862 unsigned long end_pfn, void *datax)
1863{
1864 struct iommu_prepare_data *data;
1865
1866 data = (struct iommu_prepare_data *)datax;
1867
1868 data->ret = iommu_prepare_identity_map(data->pdev,
1869 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1870 return data->ret;
1871
1872}
1873
1874static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1875{
1876 int nid;
1877 struct iommu_prepare_data data;
1878
1879 data.pdev = pdev;
1880 data.ret = 0;
1881
1882 for_each_online_node(nid) {
1883 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1884 if (data.ret)
1885 return data.ret;
1886 }
1887 return data.ret;
1888}
1889
e820482c
KA		 1890static void __init iommu_prepare_gfx_mapping(void)
1891{
1892 struct pci_dev *pdev = NULL;
e820482c
KA		 1893 int ret;
1894
1895 for_each_pci_dev(pdev) {
358dd8ac 1896 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
e820482c
KA		 1897 !IS_GFX_DEVICE(pdev))
1898 continue;
1899 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1900 pci_name(pdev));
d52d53b8
YL		 1901 ret = iommu_prepare_with_active_regions(pdev);
1902 if (ret)
1903 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
e820482c
KA		 1904 }
1905}
2abd7e16
MM		 1906#else /* !CONFIG_DMAR_GFX_WA */
1907static inline void iommu_prepare_gfx_mapping(void)
1908{
1909 return;
1910}
e820482c
KA		 1911#endif
1912
49a0429e
KA		 1913#ifdef CONFIG_DMAR_FLOPPY_WA
1914static inline void iommu_prepare_isa(void)
1915{
1916 struct pci_dev *pdev;
1917 int ret;
1918
1919 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1920 if (!pdev)
1921 return;
1922
1923 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1924 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1925
1926 if (ret)
1c35b8e5 1927 printk(KERN_ERR "IOMMU: Failed to create 0-64M identity map, "
49a0429e
KA		 1928 "floppy might not work\n");
1929
1930}
1931#else
1932static inline void iommu_prepare_isa(void)
1933{
1934 return;
1935}
1936#endif /* !CONFIG_DMAR_FLPY_WA */
1937
4ed0d3e6
FY		 1938/* Initialize each context entry as pass through.*/
1939static int __init init_context_pass_through(void)
1940{
1941 struct pci_dev *pdev = NULL;
1942 struct dmar_domain *domain;
1943 int ret;
1944
1945 for_each_pci_dev(pdev) {
1946 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1947 ret = domain_context_mapping(domain, pdev,
1948 CONTEXT_TT_PASS_THROUGH);
1949 if (ret)
1950 return ret;
1951 }
1952 return 0;
1953}
1954
519a0549 1955static int __init init_dmars(void)
ba395927
KA		 1956{
1957 struct dmar_drhd_unit *drhd;
1958 struct dmar_rmrr_unit *rmrr;
1959 struct pci_dev *pdev;
1960 struct intel_iommu *iommu;
9d783ba0 1961 int i, ret;
4ed0d3e6 1962 int pass_through = 1;
ba395927
KA		 1963
1964 /*
1965 * for each drhd
1966 * allocate root
1967 * initialize and program root entry to not present
1968 * endfor
1969 */
1970 for_each_drhd_unit(drhd) {
5e0d2a6f 1971 g_num_of_iommus++;
1972 /*
1973 * lock not needed as this is only incremented in the single
1974 * threaded kernel __init code path all other access are read
1975 * only
1976 */
1977 }
1978
d9630fe9
WH		 1979 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
1980 GFP_KERNEL);
1981 if (!g_iommus) {
1982 printk(KERN_ERR "Allocating global iommu array failed\n");
1983 ret = -ENOMEM;
1984 goto error;
1985 }
1986
80b20dd8 1987 deferred_flush = kzalloc(g_num_of_iommus *
1988 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1989 if (!deferred_flush) {
d9630fe9 1990 kfree(g_iommus);
5e0d2a6f 1991 ret = -ENOMEM;
1992 goto error;
1993 }
1994
5e0d2a6f 1995 for_each_drhd_unit(drhd) {
1996 if (drhd->ignored)
1997 continue;
1886e8a9
SS		 1998
1999 iommu = drhd->iommu;
d9630fe9 2000 g_iommus[iommu->seq_id] = iommu;
ba395927 2001
e61d98d8
SS		 2002 ret = iommu_init_domains(iommu);
2003 if (ret)
2004 goto error;
2005
ba395927
KA		 2006 /*
2007 * TBD:
2008 * we could share the same root & context tables
2009 * amoung all IOMMU's. Need to Split it later.
2010 */
2011 ret = iommu_alloc_root_entry(iommu);
2012 if (ret) {
2013 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2014 goto error;
2015 }
4ed0d3e6
FY		 2016 if (!ecap_pass_through(iommu->ecap))
2017 pass_through = 0;
ba395927 2018 }
4ed0d3e6
FY		 2019 if (iommu_pass_through)
2020 if (!pass_through) {
2021 printk(KERN_INFO
2022 "Pass Through is not supported by hardware.\n");
2023 iommu_pass_through = 0;
2024 }
ba395927 2025
1531a6a6
SS		 2026 /*
2027 * Start from the sane iommu hardware state.
2028 */
a77b67d4
YS		 2029 for_each_drhd_unit(drhd) {
2030 if (drhd->ignored)
2031 continue;
2032
2033 iommu = drhd->iommu;
1531a6a6
SS		 2034
2035 /*
2036 * If the queued invalidation is already initialized by us
2037 * (for example, while enabling interrupt-remapping) then
2038 * we got the things already rolling from a sane state.
2039 */
2040 if (iommu->qi)
2041 continue;
2042
2043 /*
2044 * Clear any previous faults.
2045 */
2046 dmar_fault(-1, iommu);
2047 /*
2048 * Disable queued invalidation if supported and already enabled
2049 * before OS handover.
2050 */
2051 dmar_disable_qi(iommu);
2052 }
2053
2054 for_each_drhd_unit(drhd) {
2055 if (drhd->ignored)
2056 continue;
2057
2058 iommu = drhd->iommu;
2059
a77b67d4
YS		 2060 if (dmar_enable_qi(iommu)) {
2061 /*
2062 * Queued Invalidate not enabled, use Register Based
2063 * Invalidate
2064 */
2065 iommu->flush.flush_context = __iommu_flush_context;
2066 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2067 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
b4e0f9eb
FT		 2068 "invalidation\n",
2069 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2070 } else {
2071 iommu->flush.flush_context = qi_flush_context;
2072 iommu->flush.flush_iotlb = qi_flush_iotlb;
2073 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
b4e0f9eb
FT		 2074 "invalidation\n",
2075 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2076 }
2077 }
2078
ba395927 2079 /*
4ed0d3e6
FY		 2080 * If pass through is set and enabled, context entries of all pci
2081 * devices are intialized by pass through translation type.
ba395927 2082 */
4ed0d3e6
FY		 2083 if (iommu_pass_through) {
2084 ret = init_context_pass_through();
2085 if (ret) {
2086 printk(KERN_ERR "IOMMU: Pass through init failed.\n");
2087 iommu_pass_through = 0;
ba395927
KA		 2088 }
2089 }
2090
ba395927 2091 /*
4ed0d3e6
FY		 2092 * If pass through is not set or not enabled, setup context entries for
2093 * identity mappings for rmrr, gfx, and isa.
ba395927 2094 */
4ed0d3e6
FY		 2095 if (!iommu_pass_through) {
2096 /*
2097 * For each rmrr
2098 * for each dev attached to rmrr
2099 * do
2100 * locate drhd for dev, alloc domain for dev
2101 * allocate free domain
2102 * allocate page table entries for rmrr
2103 * if context not allocated for bus
2104 * allocate and init context
2105 * set present in root table for this bus
2106 * init context with domain, translation etc
2107 * endfor
2108 * endfor
2109 */
2110 for_each_rmrr_units(rmrr) {
2111 for (i = 0; i < rmrr->devices_cnt; i++) {
2112 pdev = rmrr->devices[i];
2113 /*
2114 * some BIOS lists non-exist devices in DMAR
2115 * table.
2116 */
2117 if (!pdev)
2118 continue;
2119 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2120 if (ret)
2121 printk(KERN_ERR
ba395927 2122 "IOMMU: mapping reserved region failed\n");
4ed0d3e6 2123 }
ba395927 2124 }
ba395927 2125
4ed0d3e6 2126 iommu_prepare_gfx_mapping();
e820482c 2127
4ed0d3e6
FY		 2128 iommu_prepare_isa();
2129 }
49a0429e 2130
ba395927
KA		 2131 /*
2132 * for each drhd
2133 * enable fault log
2134 * global invalidate context cache
2135 * global invalidate iotlb
2136 * enable translation
2137 */
2138 for_each_drhd_unit(drhd) {
2139 if (drhd->ignored)
2140 continue;
2141 iommu = drhd->iommu;
ba395927
KA		 2142
2143 iommu_flush_write_buffer(iommu);
2144
3460a6d9
KA		 2145 ret = dmar_set_interrupt(iommu);
2146 if (ret)
2147 goto error;
2148
ba395927
KA		 2149 iommu_set_root_entry(iommu);
2150
4c25a2c1 2151 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2152 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2153 iommu_disable_protect_mem_regions(iommu);
2154
ba395927
KA		 2155 ret = iommu_enable_translation(iommu);
2156 if (ret)
2157 goto error;
2158 }
2159
2160 return 0;
2161error:
2162 for_each_drhd_unit(drhd) {
2163 if (drhd->ignored)
2164 continue;
2165 iommu = drhd->iommu;
2166 free_iommu(iommu);
2167 }
d9630fe9 2168 kfree(g_iommus);
ba395927
KA		 2169 return ret;
2170}
2171
2172static inline u64 aligned_size(u64 host_addr, size_t size)
2173{
2174 u64 addr;
5b6985ce
FY		 2175 addr = (host_addr & (~PAGE_MASK)) + size;
2176 return PAGE_ALIGN(addr);
ba395927
KA		 2177}
2178
2179struct iova *
f76aec76 2180iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
ba395927 2181{
ba395927
KA		 2182 struct iova *piova;
2183
2184 /* Make sure it's in range */
ba395927 2185 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
f76aec76 2186 if (!size || (IOVA_START_ADDR + size > end))
ba395927
KA		 2187 return NULL;
2188
2189 piova = alloc_iova(&domain->iovad,
5b6985ce 2190 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
ba395927
KA		 2191 return piova;
2192}
2193
f76aec76
KA		 2194static struct iova *
2195__intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
bb9e6d65 2196 size_t size, u64 dma_mask)
ba395927 2197{
ba395927 2198 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2199 struct iova *iova = NULL;
ba395927 2200
284901a9 2201 if (dma_mask <= DMA_BIT_MASK(32) || dmar_forcedac)
bb9e6d65
FT		 2202 iova = iommu_alloc_iova(domain, size, dma_mask);
2203 else {
ba395927
KA		 2204 /*
2205 * First try to allocate an io virtual address in
284901a9 2206 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2207 * from higher range
ba395927 2208 */
284901a9 2209 iova = iommu_alloc_iova(domain, size, DMA_BIT_MASK(32));
ba395927 2210 if (!iova)
bb9e6d65 2211 iova = iommu_alloc_iova(domain, size, dma_mask);
ba395927
KA		 2212 }
2213
2214 if (!iova) {
2215 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
f76aec76
KA		 2216 return NULL;
2217 }
2218
2219 return iova;
2220}
2221
2222static struct dmar_domain *
2223get_valid_domain_for_dev(struct pci_dev *pdev)
2224{
2225 struct dmar_domain *domain;
2226 int ret;
2227
2228 domain = get_domain_for_dev(pdev,
2229 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2230 if (!domain) {
2231 printk(KERN_ERR
2232 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2233 return NULL;
ba395927
KA		 2234 }
2235
2236 /* make sure context mapping is ok */
5331fe6f 2237 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2238 ret = domain_context_mapping(domain, pdev,
2239 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2240 if (ret) {
2241 printk(KERN_ERR
2242 "Domain context map for %s failed",
2243 pci_name(pdev));
4fe05bbc 2244 return NULL;
f76aec76 2245 }
ba395927
KA		 2246 }
2247
f76aec76
KA		 2248 return domain;
2249}
2250
bb9e6d65
FT		 2251static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2252 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2253{
2254 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2255 struct dmar_domain *domain;
5b6985ce 2256 phys_addr_t start_paddr;
f76aec76
KA		 2257 struct iova *iova;
2258 int prot = 0;
6865f0d1 2259 int ret;
8c11e798 2260 struct intel_iommu *iommu;
f76aec76
KA		 2261
2262 BUG_ON(dir == DMA_NONE);
358dd8ac 2263 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
6865f0d1 2264 return paddr;
f76aec76
KA		 2265
2266 domain = get_valid_domain_for_dev(pdev);
2267 if (!domain)
2268 return 0;
2269
8c11e798 2270 iommu = domain_get_iommu(domain);
6865f0d1 2271 size = aligned_size((u64)paddr, size);
f76aec76 2272
bb9e6d65 2273 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
f76aec76
KA		 2274 if (!iova)
2275 goto error;
2276
5b6985ce 2277 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
f76aec76 2278
ba395927
KA		 2279 /*
2280 * Check if DMAR supports zero-length reads on write only
2281 * mappings..
2282 */
2283 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2284 !cap_zlr(iommu->cap))
ba395927
KA		 2285 prot |= DMA_PTE_READ;
2286 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2287 prot |= DMA_PTE_WRITE;
2288 /*
6865f0d1 2289 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2290 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2291 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2292 * is not a big problem
2293 */
6865f0d1 2294 ret = domain_page_mapping(domain, start_paddr,
fd18de50
DW		 2295 ((u64)paddr) & PHYSICAL_PAGE_MASK,
2296 size, prot);
ba395927
KA		 2297 if (ret)
2298 goto error;
2299
1f0ef2aa
DW		 2300 /* it's a non-present to present mapping. Only flush if caching mode */
2301 if (cap_caching_mode(iommu->cap))
2302 iommu_flush_iotlb_psi(iommu, 0, start_paddr,
2303 size >> VTD_PAGE_SHIFT);
2304 else
8c11e798 2305 iommu_flush_write_buffer(iommu);
f76aec76 2306
5b6985ce 2307 return start_paddr + ((u64)paddr & (~PAGE_MASK));
ba395927 2308
ba395927 2309error:
f76aec76
KA		 2310 if (iova)
2311 __free_iova(&domain->iovad, iova);
4cf2e75d 2312 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2313 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2314 return 0;
2315}
2316
ffbbef5c
FT		 2317static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2318 unsigned long offset, size_t size,
2319 enum dma_data_direction dir,
2320 struct dma_attrs *attrs)
bb9e6d65 2321{
ffbbef5c
FT		 2322 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2323 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2324}
2325
5e0d2a6f 2326static void flush_unmaps(void)
2327{
80b20dd8 2328 int i, j;
5e0d2a6f 2329
5e0d2a6f 2330 timer_on = 0;
2331
2332 /* just flush them all */
2333 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2334 struct intel_iommu *iommu = g_iommus[i];
2335 if (!iommu)
2336 continue;
c42d9f32 2337
9dd2fe89
YZ		 2338 if (!deferred_flush[i].next)
2339 continue;
2340
2341 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2342 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2343 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2344 unsigned long mask;
2345 struct iova *iova = deferred_flush[i].iova[j];
2346
2347 mask = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT;
2348 mask = ilog2(mask >> VTD_PAGE_SHIFT);
2349 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2350 iova->pfn_lo << PAGE_SHIFT, mask);
2351 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2352 }
9dd2fe89 2353 deferred_flush[i].next = 0;
5e0d2a6f 2354 }
2355
5e0d2a6f 2356 list_size = 0;
5e0d2a6f 2357}
2358
2359static void flush_unmaps_timeout(unsigned long data)
2360{
80b20dd8 2361 unsigned long flags;
2362
2363 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2364 flush_unmaps();
80b20dd8 2365 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2366}
2367
2368static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2369{
2370 unsigned long flags;
80b20dd8 2371 int next, iommu_id;
8c11e798 2372 struct intel_iommu *iommu;
5e0d2a6f 2373
2374 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2375 if (list_size == HIGH_WATER_MARK)
2376 flush_unmaps();
2377
8c11e798
WH		 2378 iommu = domain_get_iommu(dom);
2379 iommu_id = iommu->seq_id;
c42d9f32 2380
80b20dd8 2381 next = deferred_flush[iommu_id].next;
2382 deferred_flush[iommu_id].domain[next] = dom;
2383 deferred_flush[iommu_id].iova[next] = iova;
2384 deferred_flush[iommu_id].next++;
5e0d2a6f 2385
2386 if (!timer_on) {
2387 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2388 timer_on = 1;
2389 }
2390 list_size++;
2391 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2392}
2393
ffbbef5c
FT		 2394static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2395 size_t size, enum dma_data_direction dir,
2396 struct dma_attrs *attrs)
ba395927 2397{
ba395927 2398 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76
KA		 2399 struct dmar_domain *domain;
2400 unsigned long start_addr;
ba395927 2401 struct iova *iova;
8c11e798 2402 struct intel_iommu *iommu;
ba395927 2403
358dd8ac 2404 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
f76aec76 2405 return;
ba395927
KA		 2406 domain = find_domain(pdev);
2407 BUG_ON(!domain);
2408
8c11e798
WH		 2409 iommu = domain_get_iommu(domain);
2410
ba395927 2411 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
f76aec76 2412 if (!iova)
ba395927 2413 return;
ba395927 2414
5b6985ce 2415 start_addr = iova->pfn_lo << PAGE_SHIFT;
f76aec76 2416 size = aligned_size((u64)dev_addr, size);
ba395927 2417
4cf2e75d 2418 pr_debug("Device %s unmapping: %zx@%llx\n",
5b6985ce 2419 pci_name(pdev), size, (unsigned long long)start_addr);
ba395927 2420
f76aec76
KA		 2421 /* clear the whole page */
2422 dma_pte_clear_range(domain, start_addr, start_addr + size);
2423 /* free page tables */
2424 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
5e0d2a6f 2425 if (intel_iommu_strict) {
1f0ef2aa
DW		 2426 iommu_flush_iotlb_psi(iommu, domain->id, start_addr,
2427 size >> VTD_PAGE_SHIFT);
5e0d2a6f 2428 /* free iova */
2429 __free_iova(&domain->iovad, iova);
2430 } else {
2431 add_unmap(domain, iova);
2432 /*
2433 * queue up the release of the unmap to save the 1/6th of the
2434 * cpu used up by the iotlb flush operation...
2435 */
5e0d2a6f 2436 }
ba395927
KA		 2437}
2438
d7ab5c46
FT		 2439static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2440 int dir)
ffbbef5c
FT		 2441{
2442 intel_unmap_page(dev, dev_addr, size, dir, NULL);
2443}
2444
d7ab5c46
FT		 2445static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2446 dma_addr_t *dma_handle, gfp_t flags)
ba395927
KA		 2447{
2448 void *vaddr;
2449 int order;
2450
5b6985ce 2451 size = PAGE_ALIGN(size);
ba395927
KA		 2452 order = get_order(size);
2453 flags &= ~(GFP_DMA | GFP_DMA32);
2454
2455 vaddr = (void *)__get_free_pages(flags, order);
2456 if (!vaddr)
2457 return NULL;
2458 memset(vaddr, 0, size);
2459
bb9e6d65
FT		 2460 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2461 DMA_BIDIRECTIONAL,
2462 hwdev->coherent_dma_mask);
ba395927
KA		 2463 if (*dma_handle)
2464 return vaddr;
2465 free_pages((unsigned long)vaddr, order);
2466 return NULL;
2467}
2468
d7ab5c46
FT		 2469static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2470 dma_addr_t dma_handle)
ba395927
KA		 2471{
2472 int order;
2473
5b6985ce 2474 size = PAGE_ALIGN(size);
ba395927
KA		 2475 order = get_order(size);
2476
2477 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2478 free_pages((unsigned long)vaddr, order);
2479}
2480
d7ab5c46
FT		 2481static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2482 int nelems, enum dma_data_direction dir,
2483 struct dma_attrs *attrs)
ba395927
KA		 2484{
2485 int i;
2486 struct pci_dev *pdev = to_pci_dev(hwdev);
2487 struct dmar_domain *domain;
f76aec76
KA		 2488 unsigned long start_addr;
2489 struct iova *iova;
2490 size_t size = 0;
4cf2e75d 2491 phys_addr_t addr;
c03ab37c 2492 struct scatterlist *sg;
8c11e798 2493 struct intel_iommu *iommu;
ba395927 2494
358dd8ac 2495 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2496 return;
2497
2498 domain = find_domain(pdev);
8c11e798
WH		 2499 BUG_ON(!domain);
2500
2501 iommu = domain_get_iommu(domain);
ba395927 2502
c03ab37c 2503 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
f76aec76
KA		 2504 if (!iova)
2505 return;
c03ab37c 2506 for_each_sg(sglist, sg, nelems, i) {
4cf2e75d 2507 addr = page_to_phys(sg_page(sg)) + sg->offset;
f76aec76
KA		 2508 size += aligned_size((u64)addr, sg->length);
2509 }
2510
5b6985ce 2511 start_addr = iova->pfn_lo << PAGE_SHIFT;
f76aec76
KA		 2512
2513 /* clear the whole page */
2514 dma_pte_clear_range(domain, start_addr, start_addr + size);
2515 /* free page tables */
2516 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2517
1f0ef2aa
DW		 2518 iommu_flush_iotlb_psi(iommu, domain->id, start_addr,
2519 size >> VTD_PAGE_SHIFT);
f76aec76
KA		 2520
2521 /* free iova */
2522 __free_iova(&domain->iovad, iova);
ba395927
KA		 2523}
2524
ba395927 2525static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 2526 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 2527{
2528 int i;
c03ab37c 2529 struct scatterlist *sg;
ba395927 2530
c03ab37c 2531 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 2532 BUG_ON(!sg_page(sg));
4cf2e75d 2533 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 2534 sg->dma_length = sg->length;
ba395927
KA		 2535 }
2536 return nelems;
2537}
2538
d7ab5c46
FT		 2539static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2540 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 2541{
4cf2e75d 2542 phys_addr_t addr;
ba395927 2543 int i;
ba395927
KA		 2544 struct pci_dev *pdev = to_pci_dev(hwdev);
2545 struct dmar_domain *domain;
f76aec76
KA		 2546 size_t size = 0;
2547 int prot = 0;
2548 size_t offset = 0;
2549 struct iova *iova = NULL;
2550 int ret;
c03ab37c 2551 struct scatterlist *sg;
f76aec76 2552 unsigned long start_addr;
8c11e798 2553 struct intel_iommu *iommu;
ba395927
KA		 2554
2555 BUG_ON(dir == DMA_NONE);
358dd8ac 2556 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
c03ab37c 2557 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 2558
f76aec76
KA		 2559 domain = get_valid_domain_for_dev(pdev);
2560 if (!domain)
2561 return 0;
2562
8c11e798
WH		 2563 iommu = domain_get_iommu(domain);
2564
c03ab37c 2565 for_each_sg(sglist, sg, nelems, i) {
4cf2e75d 2566 addr = page_to_phys(sg_page(sg)) + sg->offset;
f76aec76
KA		 2567 size += aligned_size((u64)addr, sg->length);
2568 }
2569
bb9e6d65 2570 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
f76aec76 2571 if (!iova) {
c03ab37c 2572 sglist->dma_length = 0;
f76aec76
KA		 2573 return 0;
2574 }
2575
2576 /*
2577 * Check if DMAR supports zero-length reads on write only
2578 * mappings..
2579 */
2580 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2581 !cap_zlr(iommu->cap))
f76aec76
KA		 2582 prot |= DMA_PTE_READ;
2583 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2584 prot |= DMA_PTE_WRITE;
2585
5b6985ce 2586 start_addr = iova->pfn_lo << PAGE_SHIFT;
f76aec76 2587 offset = 0;
c03ab37c 2588 for_each_sg(sglist, sg, nelems, i) {
4cf2e75d 2589 addr = page_to_phys(sg_page(sg)) + sg->offset;
f76aec76
KA		 2590 size = aligned_size((u64)addr, sg->length);
2591 ret = domain_page_mapping(domain, start_addr + offset,
fd18de50
DW		 2592 ((u64)addr) & PHYSICAL_PAGE_MASK,
2593 size, prot);
f76aec76
KA		 2594 if (ret) {
2595 /* clear the page */
2596 dma_pte_clear_range(domain, start_addr,
2597 start_addr + offset);
2598 /* free page tables */
2599 dma_pte_free_pagetable(domain, start_addr,
2600 start_addr + offset);
2601 /* free iova */
2602 __free_iova(&domain->iovad, iova);
ba395927
KA		 2603 return 0;
2604 }
f76aec76 2605 sg->dma_address = start_addr + offset +
5b6985ce 2606 ((u64)addr & (~PAGE_MASK));
ba395927 2607 sg->dma_length = sg->length;
f76aec76 2608 offset += size;
ba395927
KA		 2609 }
2610
1f0ef2aa
DW		 2611 /* it's a non-present to present mapping. Only flush if caching mode */
2612 if (cap_caching_mode(iommu->cap))
2613 iommu_flush_iotlb_psi(iommu, 0, start_addr,
2614 offset >> VTD_PAGE_SHIFT);
2615 else
8c11e798 2616 iommu_flush_write_buffer(iommu);
1f0ef2aa 2617
ba395927
KA		 2618 return nelems;
2619}
2620
dfb805e8
FT		 2621static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2622{
2623 return !dma_addr;
2624}
2625
160c1d8e 2626struct dma_map_ops intel_dma_ops = {
ba395927
KA		 2627 .alloc_coherent = intel_alloc_coherent,
2628 .free_coherent = intel_free_coherent,
ba395927
KA		 2629 .map_sg = intel_map_sg,
2630 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 2631 .map_page = intel_map_page,
2632 .unmap_page = intel_unmap_page,
dfb805e8 2633 .mapping_error = intel_mapping_error,
ba395927
KA		 2634};
2635
2636static inline int iommu_domain_cache_init(void)
2637{
2638 int ret = 0;
2639
2640 iommu_domain_cache = kmem_cache_create("iommu_domain",
2641 sizeof(struct dmar_domain),
2642 0,
2643 SLAB_HWCACHE_ALIGN,
2644
2645 NULL);
2646 if (!iommu_domain_cache) {
2647 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2648 ret = -ENOMEM;
2649 }
2650
2651 return ret;
2652}
2653
2654static inline int iommu_devinfo_cache_init(void)
2655{
2656 int ret = 0;
2657
2658 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2659 sizeof(struct device_domain_info),
2660 0,
2661 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2662 NULL);
2663 if (!iommu_devinfo_cache) {
2664 printk(KERN_ERR "Couldn't create devinfo cache\n");
2665 ret = -ENOMEM;
2666 }
2667
2668 return ret;
2669}
2670
2671static inline int iommu_iova_cache_init(void)
2672{
2673 int ret = 0;
2674
2675 iommu_iova_cache = kmem_cache_create("iommu_iova",
2676 sizeof(struct iova),
2677 0,
2678 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2679 NULL);
2680 if (!iommu_iova_cache) {
2681 printk(KERN_ERR "Couldn't create iova cache\n");
2682 ret = -ENOMEM;
2683 }
2684
2685 return ret;
2686}
2687
2688static int __init iommu_init_mempool(void)
2689{
2690 int ret;
2691 ret = iommu_iova_cache_init();
2692 if (ret)
2693 return ret;
2694
2695 ret = iommu_domain_cache_init();
2696 if (ret)
2697 goto domain_error;
2698
2699 ret = iommu_devinfo_cache_init();
2700 if (!ret)
2701 return ret;
2702
2703 kmem_cache_destroy(iommu_domain_cache);
2704domain_error:
2705 kmem_cache_destroy(iommu_iova_cache);
2706
2707 return -ENOMEM;
2708}
2709
2710static void __init iommu_exit_mempool(void)
2711{
2712 kmem_cache_destroy(iommu_devinfo_cache);
2713 kmem_cache_destroy(iommu_domain_cache);
2714 kmem_cache_destroy(iommu_iova_cache);
2715
2716}
2717
ba395927
KA		 2718static void __init init_no_remapping_devices(void)
2719{
2720 struct dmar_drhd_unit *drhd;
2721
2722 for_each_drhd_unit(drhd) {
2723 if (!drhd->include_all) {
2724 int i;
2725 for (i = 0; i < drhd->devices_cnt; i++)
2726 if (drhd->devices[i] != NULL)
2727 break;
2728 /* ignore DMAR unit if no pci devices exist */
2729 if (i == drhd->devices_cnt)
2730 drhd->ignored = 1;
2731 }
2732 }
2733
2734 if (dmar_map_gfx)
2735 return;
2736
2737 for_each_drhd_unit(drhd) {
2738 int i;
2739 if (drhd->ignored || drhd->include_all)
2740 continue;
2741
2742 for (i = 0; i < drhd->devices_cnt; i++)
2743 if (drhd->devices[i] &&
2744 !IS_GFX_DEVICE(drhd->devices[i]))
2745 break;
2746
2747 if (i < drhd->devices_cnt)
2748 continue;
2749
2750 /* bypass IOMMU if it is just for gfx devices */
2751 drhd->ignored = 1;
2752 for (i = 0; i < drhd->devices_cnt; i++) {
2753 if (!drhd->devices[i])
2754 continue;
358dd8ac 2755 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 2756 }
2757 }
2758}
2759
f59c7b69
FY		 2760#ifdef CONFIG_SUSPEND
2761static int init_iommu_hw(void)
2762{
2763 struct dmar_drhd_unit *drhd;
2764 struct intel_iommu *iommu = NULL;
2765
2766 for_each_active_iommu(iommu, drhd)
2767 if (iommu->qi)
2768 dmar_reenable_qi(iommu);
2769
2770 for_each_active_iommu(iommu, drhd) {
2771 iommu_flush_write_buffer(iommu);
2772
2773 iommu_set_root_entry(iommu);
2774
2775 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 2776 DMA_CCMD_GLOBAL_INVL);
f59c7b69 2777 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 2778 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 2779 iommu_disable_protect_mem_regions(iommu);
2780 iommu_enable_translation(iommu);
2781 }
2782
2783 return 0;
2784}
2785
2786static void iommu_flush_all(void)
2787{
2788 struct dmar_drhd_unit *drhd;
2789 struct intel_iommu *iommu;
2790
2791 for_each_active_iommu(iommu, drhd) {
2792 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 2793 DMA_CCMD_GLOBAL_INVL);
f59c7b69 2794 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 2795 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 2796 }
2797}
2798
2799static int iommu_suspend(struct sys_device *dev, pm_message_t state)
2800{
2801 struct dmar_drhd_unit *drhd;
2802 struct intel_iommu *iommu = NULL;
2803 unsigned long flag;
2804
2805 for_each_active_iommu(iommu, drhd) {
2806 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
2807 GFP_ATOMIC);
2808 if (!iommu->iommu_state)
2809 goto nomem;
2810 }
2811
2812 iommu_flush_all();
2813
2814 for_each_active_iommu(iommu, drhd) {
2815 iommu_disable_translation(iommu);
2816
2817 spin_lock_irqsave(&iommu->register_lock, flag);
2818
2819 iommu->iommu_state[SR_DMAR_FECTL_REG] =
2820 readl(iommu->reg + DMAR_FECTL_REG);
2821 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
2822 readl(iommu->reg + DMAR_FEDATA_REG);
2823 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
2824 readl(iommu->reg + DMAR_FEADDR_REG);
2825 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
2826 readl(iommu->reg + DMAR_FEUADDR_REG);
2827
2828 spin_unlock_irqrestore(&iommu->register_lock, flag);
2829 }
2830 return 0;
2831
2832nomem:
2833 for_each_active_iommu(iommu, drhd)
2834 kfree(iommu->iommu_state);
2835
2836 return -ENOMEM;
2837}
2838
2839static int iommu_resume(struct sys_device *dev)
2840{
2841 struct dmar_drhd_unit *drhd;
2842 struct intel_iommu *iommu = NULL;
2843 unsigned long flag;
2844
2845 if (init_iommu_hw()) {
2846 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
2847 return -EIO;
2848 }
2849
2850 for_each_active_iommu(iommu, drhd) {
2851
2852 spin_lock_irqsave(&iommu->register_lock, flag);
2853
2854 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
2855 iommu->reg + DMAR_FECTL_REG);
2856 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
2857 iommu->reg + DMAR_FEDATA_REG);
2858 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
2859 iommu->reg + DMAR_FEADDR_REG);
2860 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
2861 iommu->reg + DMAR_FEUADDR_REG);
2862
2863 spin_unlock_irqrestore(&iommu->register_lock, flag);
2864 }
2865
2866 for_each_active_iommu(iommu, drhd)
2867 kfree(iommu->iommu_state);
2868
2869 return 0;
2870}
2871
2872static struct sysdev_class iommu_sysclass = {
2873 .name = "iommu",
2874 .resume = iommu_resume,
2875 .suspend = iommu_suspend,
2876};
2877
2878static struct sys_device device_iommu = {
2879 .cls = &iommu_sysclass,
2880};
2881
2882static int __init init_iommu_sysfs(void)
2883{
2884 int error;
2885
2886 error = sysdev_class_register(&iommu_sysclass);
2887 if (error)
2888 return error;
2889
2890 error = sysdev_register(&device_iommu);
2891 if (error)
2892 sysdev_class_unregister(&iommu_sysclass);
2893
2894 return error;
2895}
2896
2897#else
2898static int __init init_iommu_sysfs(void)
2899{
2900 return 0;
2901}
2902#endif /* CONFIG_PM */
2903
ba395927
KA		 2904int __init intel_iommu_init(void)
2905{
2906 int ret = 0;
2907
ba395927
KA		 2908 if (dmar_table_init())
2909 return -ENODEV;
2910
1886e8a9
SS		 2911 if (dmar_dev_scope_init())
2912 return -ENODEV;
2913
2ae21010
SS		 2914 /*
2915 * Check the need for DMA-remapping initialization now.
2916 * Above initialization will also be used by Interrupt-remapping.
2917 */
4ed0d3e6 2918 if (no_iommu || (swiotlb && !iommu_pass_through) || dmar_disabled)
2ae21010
SS		 2919 return -ENODEV;
2920
ba395927
KA		 2921 iommu_init_mempool();
2922 dmar_init_reserved_ranges();
2923
2924 init_no_remapping_devices();
2925
2926 ret = init_dmars();
2927 if (ret) {
2928 printk(KERN_ERR "IOMMU: dmar init failed\n");
2929 put_iova_domain(&reserved_iova_list);
2930 iommu_exit_mempool();
2931 return ret;
2932 }
2933 printk(KERN_INFO
2934 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2935
5e0d2a6f 2936 init_timer(&unmap_timer);
ba395927 2937 force_iommu = 1;
4ed0d3e6
FY		 2938
2939 if (!iommu_pass_through) {
2940 printk(KERN_INFO
2941 "Multi-level page-table translation for DMAR.\n");
2942 dma_ops = &intel_dma_ops;
2943 } else
2944 printk(KERN_INFO
2945 "DMAR: Pass through translation for DMAR.\n");
2946
f59c7b69 2947 init_iommu_sysfs();
a8bcbb0d
JR		 2948
2949 register_iommu(&intel_iommu_ops);
2950
ba395927
KA		 2951 return 0;
2952}
e820482c 2953
c7151a8d
WH		 2954static int vm_domain_add_dev_info(struct dmar_domain *domain,
2955 struct pci_dev *pdev)
2956{
2957 struct device_domain_info *info;
2958 unsigned long flags;
2959
2960 info = alloc_devinfo_mem();
2961 if (!info)
2962 return -ENOMEM;
2963
276dbf99 2964 info->segment = pci_domain_nr(pdev->bus);
c7151a8d
WH		 2965 info->bus = pdev->bus->number;
2966 info->devfn = pdev->devfn;
2967 info->dev = pdev;
2968 info->domain = domain;
2969
2970 spin_lock_irqsave(&device_domain_lock, flags);
2971 list_add(&info->link, &domain->devices);
2972 list_add(&info->global, &device_domain_list);
2973 pdev->dev.archdata.iommu = info;
2974 spin_unlock_irqrestore(&device_domain_lock, flags);
2975
2976 return 0;
2977}
2978
3199aa6b
HW		 2979static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
2980 struct pci_dev *pdev)
2981{
2982 struct pci_dev *tmp, *parent;
2983
2984 if (!iommu || !pdev)
2985 return;
2986
2987 /* dependent device detach */
2988 tmp = pci_find_upstream_pcie_bridge(pdev);
2989 /* Secondary interface's bus number and devfn 0 */
2990 if (tmp) {
2991 parent = pdev->bus->self;
2992 while (parent != tmp) {
2993 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 2994 parent->devfn);
3199aa6b
HW		 2995 parent = parent->bus->self;
2996 }
2997 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
2998 iommu_detach_dev(iommu,
2999 tmp->subordinate->number, 0);
3000 else /* this is a legacy PCI bridge */
276dbf99
DW		 3001 iommu_detach_dev(iommu, tmp->bus->number,
3002 tmp->devfn);
3199aa6b
HW		 3003 }
3004}
3005
c7151a8d
WH		 3006static void vm_domain_remove_one_dev_info(struct dmar_domain *domain,
3007 struct pci_dev *pdev)
3008{
3009 struct device_domain_info *info;
3010 struct intel_iommu *iommu;
3011 unsigned long flags;
3012 int found = 0;
3013 struct list_head *entry, *tmp;
3014
276dbf99
DW		 3015 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3016 pdev->devfn);
c7151a8d
WH		 3017 if (!iommu)
3018 return;
3019
3020 spin_lock_irqsave(&device_domain_lock, flags);
3021 list_for_each_safe(entry, tmp, &domain->devices) {
3022 info = list_entry(entry, struct device_domain_info, link);
276dbf99 3023 /* No need to compare PCI domain; it has to be the same */
c7151a8d
WH		 3024 if (info->bus == pdev->bus->number &&
3025 info->devfn == pdev->devfn) {
3026 list_del(&info->link);
3027 list_del(&info->global);
3028 if (info->dev)
3029 info->dev->dev.archdata.iommu = NULL;
3030 spin_unlock_irqrestore(&device_domain_lock, flags);
3031
93a23a72 3032 iommu_disable_dev_iotlb(info);
c7151a8d 3033 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3034 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3035 free_devinfo_mem(info);
3036
3037 spin_lock_irqsave(&device_domain_lock, flags);
3038
3039 if (found)
3040 break;
3041 else
3042 continue;
3043 }
3044
3045 /* if there is no other devices under the same iommu
3046 * owned by this domain, clear this iommu in iommu_bmp
3047 * update iommu count and coherency
3048 */
276dbf99
DW		 3049 if (iommu == device_to_iommu(info->segment, info->bus,
3050 info->devfn))
c7151a8d
WH		 3051 found = 1;
3052 }
3053
3054 if (found == 0) {
3055 unsigned long tmp_flags;
3056 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3057 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3058 domain->iommu_count--;
58c610bd 3059 domain_update_iommu_cap(domain);
c7151a8d
WH		 3060 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3061 }
3062
3063 spin_unlock_irqrestore(&device_domain_lock, flags);
3064}
3065
3066static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3067{
3068 struct device_domain_info *info;
3069 struct intel_iommu *iommu;
3070 unsigned long flags1, flags2;
3071
3072 spin_lock_irqsave(&device_domain_lock, flags1);
3073 while (!list_empty(&domain->devices)) {
3074 info = list_entry(domain->devices.next,
3075 struct device_domain_info, link);
3076 list_del(&info->link);
3077 list_del(&info->global);
3078 if (info->dev)
3079 info->dev->dev.archdata.iommu = NULL;
3080
3081 spin_unlock_irqrestore(&device_domain_lock, flags1);
3082
93a23a72 3083 iommu_disable_dev_iotlb(info);
276dbf99 3084 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3085 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3086 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3087
3088 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3089 * and capabilities
c7151a8d
WH		 3090 */
3091 spin_lock_irqsave(&domain->iommu_lock, flags2);
3092 if (test_and_clear_bit(iommu->seq_id,
3093 &domain->iommu_bmp)) {
3094 domain->iommu_count--;
58c610bd 3095 domain_update_iommu_cap(domain);
c7151a8d
WH		 3096 }
3097 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3098
3099 free_devinfo_mem(info);
3100 spin_lock_irqsave(&device_domain_lock, flags1);
3101 }
3102 spin_unlock_irqrestore(&device_domain_lock, flags1);
3103}
3104
5e98c4b1
WH		 3105/* domain id for virtual machine, it won't be set in context */
3106static unsigned long vm_domid;
3107
fe40f1e0
WH		 3108static int vm_domain_min_agaw(struct dmar_domain *domain)
3109{
3110 int i;
3111 int min_agaw = domain->agaw;
3112
3113 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3114 for (; i < g_num_of_iommus; ) {
3115 if (min_agaw > g_iommus[i]->agaw)
3116 min_agaw = g_iommus[i]->agaw;
3117
3118 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3119 }
3120
3121 return min_agaw;
3122}
3123
5e98c4b1
WH		 3124static struct dmar_domain *iommu_alloc_vm_domain(void)
3125{
3126 struct dmar_domain *domain;
3127
3128 domain = alloc_domain_mem();
3129 if (!domain)
3130 return NULL;
3131
3132 domain->id = vm_domid++;
3133 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3134 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3135
3136 return domain;
3137}
3138
3139static int vm_domain_init(struct dmar_domain *domain, int guest_width)
3140{
3141 int adjust_width;
3142
3143 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3144 spin_lock_init(&domain->mapping_lock);
3145 spin_lock_init(&domain->iommu_lock);
3146
3147 domain_reserve_special_ranges(domain);
3148
3149 /* calculate AGAW */
3150 domain->gaw = guest_width;
3151 adjust_width = guestwidth_to_adjustwidth(guest_width);
3152 domain->agaw = width_to_agaw(adjust_width);
3153
3154 INIT_LIST_HEAD(&domain->devices);
3155
3156 domain->iommu_count = 0;
3157 domain->iommu_coherency = 0;
fe40f1e0 3158 domain->max_addr = 0;
5e98c4b1
WH		 3159
3160 /* always allocate the top pgd */
3161 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
3162 if (!domain->pgd)
3163 return -ENOMEM;
3164 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3165 return 0;
3166}
3167
3168static void iommu_free_vm_domain(struct dmar_domain *domain)
3169{
3170 unsigned long flags;
3171 struct dmar_drhd_unit *drhd;
3172 struct intel_iommu *iommu;
3173 unsigned long i;
3174 unsigned long ndomains;
3175
3176 for_each_drhd_unit(drhd) {
3177 if (drhd->ignored)
3178 continue;
3179 iommu = drhd->iommu;
3180
3181 ndomains = cap_ndoms(iommu->cap);
3182 i = find_first_bit(iommu->domain_ids, ndomains);
3183 for (; i < ndomains; ) {
3184 if (iommu->domains[i] == domain) {
3185 spin_lock_irqsave(&iommu->lock, flags);
3186 clear_bit(i, iommu->domain_ids);
3187 iommu->domains[i] = NULL;
3188 spin_unlock_irqrestore(&iommu->lock, flags);
3189 break;
3190 }
3191 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3192 }
3193 }
3194}
3195
3196static void vm_domain_exit(struct dmar_domain *domain)
3197{
3198 u64 end;
3199
3200 /* Domain 0 is reserved, so dont process it */
3201 if (!domain)
3202 return;
3203
3204 vm_domain_remove_all_dev_info(domain);
3205 /* destroy iovas */
3206 put_iova_domain(&domain->iovad);
3207 end = DOMAIN_MAX_ADDR(domain->gaw);
3208 end = end & (~VTD_PAGE_MASK);
3209
3210 /* clear ptes */
3211 dma_pte_clear_range(domain, 0, end);
3212
3213 /* free page tables */
3214 dma_pte_free_pagetable(domain, 0, end);
3215
3216 iommu_free_vm_domain(domain);
3217 free_domain_mem(domain);
3218}
3219
5d450806 3220static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3221{
5d450806 3222 struct dmar_domain *dmar_domain;
38717946 3223
5d450806
JR		 3224 dmar_domain = iommu_alloc_vm_domain();
3225 if (!dmar_domain) {
38717946 3226 printk(KERN_ERR
5d450806
JR		 3227 "intel_iommu_domain_init: dmar_domain == NULL\n");
3228 return -ENOMEM;
38717946 3229 }
5d450806 3230 if (vm_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3231 printk(KERN_ERR
5d450806
JR		 3232 "intel_iommu_domain_init() failed\n");
3233 vm_domain_exit(dmar_domain);
3234 return -ENOMEM;
38717946 3235 }
5d450806 3236 domain->priv = dmar_domain;
faa3d6f5 3237
5d450806 3238 return 0;
38717946 3239}
38717946 3240
5d450806 3241static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 3242{
5d450806
JR		 3243 struct dmar_domain *dmar_domain = domain->priv;
3244
3245 domain->priv = NULL;
3246 vm_domain_exit(dmar_domain);
38717946 3247}
38717946 3248
4c5478c9
JR		 3249static int intel_iommu_attach_device(struct iommu_domain *domain,
3250 struct device *dev)
38717946 3251{
4c5478c9
JR		 3252 struct dmar_domain *dmar_domain = domain->priv;
3253 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 3254 struct intel_iommu *iommu;
3255 int addr_width;
3256 u64 end;
faa3d6f5
WH		 3257 int ret;
3258
3259 /* normally pdev is not mapped */
3260 if (unlikely(domain_context_mapped(pdev))) {
3261 struct dmar_domain *old_domain;
3262
3263 old_domain = find_domain(pdev);
3264 if (old_domain) {
4c5478c9 3265 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
faa3d6f5
WH		 3266 vm_domain_remove_one_dev_info(old_domain, pdev);
3267 else
3268 domain_remove_dev_info(old_domain);
3269 }
3270 }
3271
276dbf99
DW		 3272 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3273 pdev->devfn);
fe40f1e0
WH		 3274 if (!iommu)
3275 return -ENODEV;
3276
3277 /* check if this iommu agaw is sufficient for max mapped address */
3278 addr_width = agaw_to_width(iommu->agaw);
3279 end = DOMAIN_MAX_ADDR(addr_width);
3280 end = end & VTD_PAGE_MASK;
4c5478c9 3281 if (end < dmar_domain->max_addr) {
fe40f1e0
WH		 3282 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3283 "sufficient for the mapped address (%llx)\n",
4c5478c9 3284 __func__, iommu->agaw, dmar_domain->max_addr);
fe40f1e0
WH		 3285 return -EFAULT;
3286 }
3287
93a23a72 3288 ret = vm_domain_add_dev_info(dmar_domain, pdev);
faa3d6f5
WH		 3289 if (ret)
3290 return ret;
3291
93a23a72 3292 ret = domain_context_mapping(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
faa3d6f5 3293 return ret;
38717946 3294}
38717946 3295
4c5478c9
JR		 3296static void intel_iommu_detach_device(struct iommu_domain *domain,
3297 struct device *dev)
38717946 3298{
4c5478c9
JR		 3299 struct dmar_domain *dmar_domain = domain->priv;
3300 struct pci_dev *pdev = to_pci_dev(dev);
3301
3302 vm_domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 3303}
c7151a8d 3304
dde57a21
JR		 3305static int intel_iommu_map_range(struct iommu_domain *domain,
3306 unsigned long iova, phys_addr_t hpa,
3307 size_t size, int iommu_prot)
faa3d6f5 3308{
dde57a21 3309 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0
WH		 3310 u64 max_addr;
3311 int addr_width;
dde57a21 3312 int prot = 0;
faa3d6f5 3313 int ret;
fe40f1e0 3314
dde57a21
JR		 3315 if (iommu_prot & IOMMU_READ)
3316 prot |= DMA_PTE_READ;
3317 if (iommu_prot & IOMMU_WRITE)
3318 prot |= DMA_PTE_WRITE;
9cf06697
SY		 3319 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3320 prot |= DMA_PTE_SNP;
dde57a21 3321
fe40f1e0 3322 max_addr = (iova & VTD_PAGE_MASK) + VTD_PAGE_ALIGN(size);
dde57a21 3323 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 3324 int min_agaw;
3325 u64 end;
3326
3327 /* check if minimum agaw is sufficient for mapped address */
dde57a21 3328 min_agaw = vm_domain_min_agaw(dmar_domain);
fe40f1e0
WH		 3329 addr_width = agaw_to_width(min_agaw);
3330 end = DOMAIN_MAX_ADDR(addr_width);
3331 end = end & VTD_PAGE_MASK;
3332 if (end < max_addr) {
3333 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3334 "sufficient for the mapped address (%llx)\n",
3335 __func__, min_agaw, max_addr);
3336 return -EFAULT;
3337 }
dde57a21 3338 dmar_domain->max_addr = max_addr;
fe40f1e0
WH		 3339 }
3340
dde57a21 3341 ret = domain_page_mapping(dmar_domain, iova, hpa, size, prot);
faa3d6f5 3342 return ret;
38717946 3343}
38717946 3344
dde57a21
JR		 3345static void intel_iommu_unmap_range(struct iommu_domain *domain,
3346 unsigned long iova, size_t size)
38717946 3347{
dde57a21 3348 struct dmar_domain *dmar_domain = domain->priv;
faa3d6f5
WH		 3349 dma_addr_t base;
3350
3351 /* The address might not be aligned */
3352 base = iova & VTD_PAGE_MASK;
3353 size = VTD_PAGE_ALIGN(size);
dde57a21 3354 dma_pte_clear_range(dmar_domain, base, base + size);
fe40f1e0 3355
dde57a21
JR		 3356 if (dmar_domain->max_addr == base + size)
3357 dmar_domain->max_addr = base;
38717946 3358}
38717946 3359
d14d6577
JR		 3360static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3361 unsigned long iova)
38717946 3362{
d14d6577 3363 struct dmar_domain *dmar_domain = domain->priv;
38717946 3364 struct dma_pte *pte;
faa3d6f5 3365 u64 phys = 0;
38717946 3366
d14d6577 3367 pte = addr_to_dma_pte(dmar_domain, iova);
38717946 3368 if (pte)
faa3d6f5 3369 phys = dma_pte_addr(pte);
38717946 3370
faa3d6f5 3371 return phys;
38717946 3372}
a8bcbb0d 3373
dbb9fd86
SY		 3374static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3375 unsigned long cap)
3376{
3377 struct dmar_domain *dmar_domain = domain->priv;
3378
3379 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3380 return dmar_domain->iommu_snooping;
3381
3382 return 0;
3383}
3384
a8bcbb0d
JR		 3385static struct iommu_ops intel_iommu_ops = {
3386 .domain_init = intel_iommu_domain_init,
3387 .domain_destroy = intel_iommu_domain_destroy,
3388 .attach_dev = intel_iommu_attach_device,
3389 .detach_dev = intel_iommu_detach_device,
3390 .map = intel_iommu_map_range,
3391 .unmap = intel_iommu_unmap_range,
3392 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 3393 .domain_has_cap = intel_iommu_domain_has_cap,
a8bcbb0d 3394};
9af88143
DW		 3395
3396static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3397{
3398 /*
3399 * Mobile 4 Series Chipset neglects to set RWBF capability,
3400 * but needs it:
3401 */
3402 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3403 rwbf_quirk = 1;
3404}
3405
3406DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
Linux kernel - Deliverables
This page took 0.411056 seconds and 5 git commands to generate.