projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'for-linus' of git://ftp.arm.linux.org.uk/~rmk/linux-arm
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 */
19
20 #include <linux/init.h>
21 #include <linux/bitmap.h>
22 #include <linux/debugfs.h>
23 #include <linux/export.h>
24 #include <linux/slab.h>
25 #include <linux/irq.h>
26 #include <linux/interrupt.h>
27 #include <linux/spinlock.h>
28 #include <linux/pci.h>
29 #include <linux/dmar.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/mempool.h>
32 #include <linux/memory.h>
33 #include <linux/timer.h>
34 #include <linux/iova.h>
35 #include <linux/iommu.h>
36 #include <linux/intel-iommu.h>
37 #include <linux/syscore_ops.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/pci-ats.h>
41 #include <linux/memblock.h>
42 #include <linux/dma-contiguous.h>
43 #include <asm/irq_remapping.h>
44 #include <asm/cacheflush.h>
45 #include <asm/iommu.h>
46
47 #include "irq_remapping.h"
48
49 #define ROOT_SIZE VTD_PAGE_SIZE
50 #define CONTEXT_SIZE VTD_PAGE_SIZE
51
52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
55
56 #define IOAPIC_RANGE_START (0xfee00000)
57 #define IOAPIC_RANGE_END (0xfeefffff)
58 #define IOVA_START_ADDR (0x1000)
59
60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
62 #define MAX_AGAW_WIDTH 64
63 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
64
65 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
67
68 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
73
74 /* IO virtual address start page frame number */
75 #define IOVA_START_PFN (1)
76
77 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
78 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
79 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
80
81 /* page table handling */
82 #define LEVEL_STRIDE (9)
83 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
84
85 /*
86 * This bitmap is used to advertise the page sizes our hardware support
87 * to the IOMMU core, which will then use this information to split
88 * physically contiguous memory regions it is mapping into page sizes
89 * that we support.
90 *
91 * Traditionally the IOMMU core just handed us the mappings directly,
92 * after making sure the size is an order of a 4KiB page and that the
93 * mapping has natural alignment.
94 *
95 * To retain this behavior, we currently advertise that we support
96 * all page sizes that are an order of 4KiB.
97 *
98 * If at some point we'd like to utilize the IOMMU core's new behavior,
99 * we could change this to advertise the real page sizes we support.
100 */
101 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
102
103 static inline int agaw_to_level(int agaw)
104 {
105 return agaw + 2;
106 }
107
108 static inline int agaw_to_width(int agaw)
109 {
110 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
111 }
112
113 static inline int width_to_agaw(int width)
114 {
115 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
116 }
117
118 static inline unsigned int level_to_offset_bits(int level)
119 {
120 return (level - 1) * LEVEL_STRIDE;
121 }
122
123 static inline int pfn_level_offset(unsigned long pfn, int level)
124 {
125 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
126 }
127
128 static inline unsigned long level_mask(int level)
129 {
130 return -1UL << level_to_offset_bits(level);
131 }
132
133 static inline unsigned long level_size(int level)
134 {
135 return 1UL << level_to_offset_bits(level);
136 }
137
138 static inline unsigned long align_to_level(unsigned long pfn, int level)
139 {
140 return (pfn + level_size(level) - 1) & level_mask(level);
141 }
142
143 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
144 {
145 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
146 }
147
148 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
149 are never going to work. */
150 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
151 {
152 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
153 }
154
155 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
156 {
157 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
158 }
159 static inline unsigned long page_to_dma_pfn(struct page *pg)
160 {
161 return mm_to_dma_pfn(page_to_pfn(pg));
162 }
163 static inline unsigned long virt_to_dma_pfn(void *p)
164 {
165 return page_to_dma_pfn(virt_to_page(p));
166 }
167
168 /* global iommu list, set NULL for ignored DMAR units */
169 static struct intel_iommu **g_iommus;
170
171 static void __init check_tylersburg_isoch(void);
172 static int rwbf_quirk;
173
174 /*
175 * set to 1 to panic kernel if can't successfully enable VT-d
176 * (used when kernel is launched w/ TXT)
177 */
178 static int force_on = 0;
179
180 /*
181 * 0: Present
182 * 1-11: Reserved
183 * 12-63: Context Ptr (12 - (haw-1))
184 * 64-127: Reserved
185 */
186 struct root_entry {
187 u64 val;
188 u64 rsvd1;
189 };
190 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
191 static inline bool root_present(struct root_entry *root)
192 {
193 return (root->val & 1);
194 }
195 static inline void set_root_present(struct root_entry *root)
196 {
197 root->val |= 1;
198 }
199 static inline void set_root_value(struct root_entry *root, unsigned long value)
200 {
201 root->val &= ~VTD_PAGE_MASK;
202 root->val |= value & VTD_PAGE_MASK;
203 }
204
205 static inline struct context_entry *
206 get_context_addr_from_root(struct root_entry *root)
207 {
208 return (struct context_entry *)
209 (root_present(root)?phys_to_virt(
210 root->val & VTD_PAGE_MASK) :
211 NULL);
212 }
213
214 /*
215 * low 64 bits:
216 * 0: present
217 * 1: fault processing disable
218 * 2-3: translation type
219 * 12-63: address space root
220 * high 64 bits:
221 * 0-2: address width
222 * 3-6: aval
223 * 8-23: domain id
224 */
225 struct context_entry {
226 u64 lo;
227 u64 hi;
228 };
229
230 static inline bool context_present(struct context_entry *context)
231 {
232 return (context->lo & 1);
233 }
234 static inline void context_set_present(struct context_entry *context)
235 {
236 context->lo |= 1;
237 }
238
239 static inline void context_set_fault_enable(struct context_entry *context)
240 {
241 context->lo &= (((u64)-1) << 2) | 1;
242 }
243
244 static inline void context_set_translation_type(struct context_entry *context,
245 unsigned long value)
246 {
247 context->lo &= (((u64)-1) << 4) | 3;
248 context->lo |= (value & 3) << 2;
249 }
250
251 static inline void context_set_address_root(struct context_entry *context,
252 unsigned long value)
253 {
254 context->lo &= ~VTD_PAGE_MASK;
255 context->lo |= value & VTD_PAGE_MASK;
256 }
257
258 static inline void context_set_address_width(struct context_entry *context,
259 unsigned long value)
260 {
261 context->hi |= value & 7;
262 }
263
264 static inline void context_set_domain_id(struct context_entry *context,
265 unsigned long value)
266 {
267 context->hi |= (value & ((1 << 16) - 1)) << 8;
268 }
269
270 static inline void context_clear_entry(struct context_entry *context)
271 {
272 context->lo = 0;
273 context->hi = 0;
274 }
275
276 /*
277 * 0: readable
278 * 1: writable
279 * 2-6: reserved
280 * 7: super page
281 * 8-10: available
282 * 11: snoop behavior
283 * 12-63: Host physcial address
284 */
285 struct dma_pte {
286 u64 val;
287 };
288
289 static inline void dma_clear_pte(struct dma_pte *pte)
290 {
291 pte->val = 0;
292 }
293
294 static inline u64 dma_pte_addr(struct dma_pte *pte)
295 {
296 #ifdef CONFIG_64BIT
297 return pte->val & VTD_PAGE_MASK;
298 #else
299 /* Must have a full atomic 64-bit read */
300 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
301 #endif
302 }
303
304 static inline bool dma_pte_present(struct dma_pte *pte)
305 {
306 return (pte->val & 3) != 0;
307 }
308
309 static inline bool dma_pte_superpage(struct dma_pte *pte)
310 {
311 return (pte->val & DMA_PTE_LARGE_PAGE);
312 }
313
314 static inline int first_pte_in_page(struct dma_pte *pte)
315 {
316 return !((unsigned long)pte & ~VTD_PAGE_MASK);
317 }
318
319 /*
320 * This domain is a statically identity mapping domain.
321 * 1. This domain creats a static 1:1 mapping to all usable memory.
322 * 2. It maps to each iommu if successful.
323 * 3. Each iommu mapps to this domain if successful.
324 */
325 static struct dmar_domain *si_domain;
326 static int hw_pass_through = 1;
327
328 /* domain represents a virtual machine, more than one devices
329 * across iommus may be owned in one domain, e.g. kvm guest.
330 */
331 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
332
333 /* si_domain contains mulitple devices */
334 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
335
336 struct dmar_domain {
337 int id; /* domain id */
338 int nid; /* node id */
339 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
340 /* bitmap of iommus this domain uses*/
341
342 struct list_head devices; /* all devices' list */
343 struct iova_domain iovad; /* iova's that belong to this domain */
344
345 struct dma_pte *pgd; /* virtual address */
346 int gaw; /* max guest address width */
347
348 /* adjusted guest address width, 0 is level 2 30-bit */
349 int agaw;
350
351 int flags; /* flags to find out type of domain */
352
353 int iommu_coherency;/* indicate coherency of iommu access */
354 int iommu_snooping; /* indicate snooping control feature*/
355 int iommu_count; /* reference count of iommu */
356 int iommu_superpage;/* Level of superpages supported:
357 0 == 4KiB (no superpages), 1 == 2MiB,
358 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
359 spinlock_t iommu_lock; /* protect iommu set in domain */
360 u64 max_addr; /* maximum mapped address */
361
362 struct iommu_domain domain; /* generic domain data structure for
363 iommu core */
364 };
365
366 /* PCI domain-device relationship */
367 struct device_domain_info {
368 struct list_head link; /* link to domain siblings */
369 struct list_head global; /* link to global list */
370 u8 bus; /* PCI bus number */
371 u8 devfn; /* PCI devfn number */
372 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
373 struct intel_iommu *iommu; /* IOMMU used by this device */
374 struct dmar_domain *domain; /* pointer to domain */
375 };
376
377 struct dmar_rmrr_unit {
378 struct list_head list; /* list of rmrr units */
379 struct acpi_dmar_header *hdr; /* ACPI header */
380 u64 base_address; /* reserved base address*/
381 u64 end_address; /* reserved end address */
382 struct dmar_dev_scope *devices; /* target devices */
383 int devices_cnt; /* target device count */
384 };
385
386 struct dmar_atsr_unit {
387 struct list_head list; /* list of ATSR units */
388 struct acpi_dmar_header *hdr; /* ACPI header */
389 struct dmar_dev_scope *devices; /* target devices */
390 int devices_cnt; /* target device count */
391 u8 include_all:1; /* include all ports */
392 };
393
394 static LIST_HEAD(dmar_atsr_units);
395 static LIST_HEAD(dmar_rmrr_units);
396
397 #define for_each_rmrr_units(rmrr) \
398 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
399
400 static void flush_unmaps_timeout(unsigned long data);
401
402 static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
403
404 #define HIGH_WATER_MARK 250
405 struct deferred_flush_tables {
406 int next;
407 struct iova *iova[HIGH_WATER_MARK];
408 struct dmar_domain *domain[HIGH_WATER_MARK];
409 struct page *freelist[HIGH_WATER_MARK];
410 };
411
412 static struct deferred_flush_tables *deferred_flush;
413
414 /* bitmap for indexing intel_iommus */
415 static int g_num_of_iommus;
416
417 static DEFINE_SPINLOCK(async_umap_flush_lock);
418 static LIST_HEAD(unmaps_to_do);
419
420 static int timer_on;
421 static long list_size;
422
423 static void domain_exit(struct dmar_domain *domain);
424 static void domain_remove_dev_info(struct dmar_domain *domain);
425 static void domain_remove_one_dev_info(struct dmar_domain *domain,
426 struct device *dev);
427 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
428 struct device *dev);
429 static int domain_detach_iommu(struct dmar_domain *domain,
430 struct intel_iommu *iommu);
431
432 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
433 int dmar_disabled = 0;
434 #else
435 int dmar_disabled = 1;
436 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
437
438 int intel_iommu_enabled = 0;
439 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
440
441 static int dmar_map_gfx = 1;
442 static int dmar_forcedac;
443 static int intel_iommu_strict;
444 static int intel_iommu_superpage = 1;
445
446 int intel_iommu_gfx_mapped;
447 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
448
449 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
450 static DEFINE_SPINLOCK(device_domain_lock);
451 static LIST_HEAD(device_domain_list);
452
453 static const struct iommu_ops intel_iommu_ops;
454
455 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
456 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
457 {
458 return container_of(dom, struct dmar_domain, domain);
459 }
460
461 static int __init intel_iommu_setup(char *str)
462 {
463 if (!str)
464 return -EINVAL;
465 while (*str) {
466 if (!strncmp(str, "on", 2)) {
467 dmar_disabled = 0;
468 printk(KERN_INFO "Intel-IOMMU: enabled\n");
469 } else if (!strncmp(str, "off", 3)) {
470 dmar_disabled = 1;
471 printk(KERN_INFO "Intel-IOMMU: disabled\n");
472 } else if (!strncmp(str, "igfx_off", 8)) {
473 dmar_map_gfx = 0;
474 printk(KERN_INFO
475 "Intel-IOMMU: disable GFX device mapping\n");
476 } else if (!strncmp(str, "forcedac", 8)) {
477 printk(KERN_INFO
478 "Intel-IOMMU: Forcing DAC for PCI devices\n");
479 dmar_forcedac = 1;
480 } else if (!strncmp(str, "strict", 6)) {
481 printk(KERN_INFO
482 "Intel-IOMMU: disable batched IOTLB flush\n");
483 intel_iommu_strict = 1;
484 } else if (!strncmp(str, "sp_off", 6)) {
485 printk(KERN_INFO
486 "Intel-IOMMU: disable supported super page\n");
487 intel_iommu_superpage = 0;
488 }
489
490 str += strcspn(str, ",");
491 while (*str == ',')
492 str++;
493 }
494 return 0;
495 }
496 __setup("intel_iommu=", intel_iommu_setup);
497
498 static struct kmem_cache *iommu_domain_cache;
499 static struct kmem_cache *iommu_devinfo_cache;
500
501 static inline void *alloc_pgtable_page(int node)
502 {
503 struct page *page;
504 void *vaddr = NULL;
505
506 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
507 if (page)
508 vaddr = page_address(page);
509 return vaddr;
510 }
511
512 static inline void free_pgtable_page(void *vaddr)
513 {
514 free_page((unsigned long)vaddr);
515 }
516
517 static inline void *alloc_domain_mem(void)
518 {
519 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
520 }
521
522 static void free_domain_mem(void *vaddr)
523 {
524 kmem_cache_free(iommu_domain_cache, vaddr);
525 }
526
527 static inline void * alloc_devinfo_mem(void)
528 {
529 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
530 }
531
532 static inline void free_devinfo_mem(void *vaddr)
533 {
534 kmem_cache_free(iommu_devinfo_cache, vaddr);
535 }
536
537 static inline int domain_type_is_vm(struct dmar_domain *domain)
538 {
539 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
540 }
541
542 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
543 {
544 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
545 DOMAIN_FLAG_STATIC_IDENTITY);
546 }
547
548 static inline int domain_pfn_supported(struct dmar_domain *domain,
549 unsigned long pfn)
550 {
551 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
552
553 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
554 }
555
556 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
557 {
558 unsigned long sagaw;
559 int agaw = -1;
560
561 sagaw = cap_sagaw(iommu->cap);
562 for (agaw = width_to_agaw(max_gaw);
563 agaw >= 0; agaw--) {
564 if (test_bit(agaw, &sagaw))
565 break;
566 }
567
568 return agaw;
569 }
570
571 /*
572 * Calculate max SAGAW for each iommu.
573 */
574 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
575 {
576 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
577 }
578
579 /*
580 * calculate agaw for each iommu.
581 * "SAGAW" may be different across iommus, use a default agaw, and
582 * get a supported less agaw for iommus that don't support the default agaw.
583 */
584 int iommu_calculate_agaw(struct intel_iommu *iommu)
585 {
586 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
587 }
588
589 /* This functionin only returns single iommu in a domain */
590 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
591 {
592 int iommu_id;
593
594 /* si_domain and vm domain should not get here. */
595 BUG_ON(domain_type_is_vm_or_si(domain));
596 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
597 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
598 return NULL;
599
600 return g_iommus[iommu_id];
601 }
602
603 static void domain_update_iommu_coherency(struct dmar_domain *domain)
604 {
605 struct dmar_drhd_unit *drhd;
606 struct intel_iommu *iommu;
607 bool found = false;
608 int i;
609
610 domain->iommu_coherency = 1;
611
612 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
613 found = true;
614 if (!ecap_coherent(g_iommus[i]->ecap)) {
615 domain->iommu_coherency = 0;
616 break;
617 }
618 }
619 if (found)
620 return;
621
622 /* No hardware attached; use lowest common denominator */
623 rcu_read_lock();
624 for_each_active_iommu(iommu, drhd) {
625 if (!ecap_coherent(iommu->ecap)) {
626 domain->iommu_coherency = 0;
627 break;
628 }
629 }
630 rcu_read_unlock();
631 }
632
633 static int domain_update_iommu_snooping(struct intel_iommu *skip)
634 {
635 struct dmar_drhd_unit *drhd;
636 struct intel_iommu *iommu;
637 int ret = 1;
638
639 rcu_read_lock();
640 for_each_active_iommu(iommu, drhd) {
641 if (iommu != skip) {
642 if (!ecap_sc_support(iommu->ecap)) {
643 ret = 0;
644 break;
645 }
646 }
647 }
648 rcu_read_unlock();
649
650 return ret;
651 }
652
653 static int domain_update_iommu_superpage(struct intel_iommu *skip)
654 {
655 struct dmar_drhd_unit *drhd;
656 struct intel_iommu *iommu;
657 int mask = 0xf;
658
659 if (!intel_iommu_superpage) {
660 return 0;
661 }
662
663 /* set iommu_superpage to the smallest common denominator */
664 rcu_read_lock();
665 for_each_active_iommu(iommu, drhd) {
666 if (iommu != skip) {
667 mask &= cap_super_page_val(iommu->cap);
668 if (!mask)
669 break;
670 }
671 }
672 rcu_read_unlock();
673
674 return fls(mask);
675 }
676
677 /* Some capabilities may be different across iommus */
678 static void domain_update_iommu_cap(struct dmar_domain *domain)
679 {
680 domain_update_iommu_coherency(domain);
681 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
682 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
683 }
684
685 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
686 {
687 struct dmar_drhd_unit *drhd = NULL;
688 struct intel_iommu *iommu;
689 struct device *tmp;
690 struct pci_dev *ptmp, *pdev = NULL;
691 u16 segment = 0;
692 int i;
693
694 if (dev_is_pci(dev)) {
695 pdev = to_pci_dev(dev);
696 segment = pci_domain_nr(pdev->bus);
697 } else if (has_acpi_companion(dev))
698 dev = &ACPI_COMPANION(dev)->dev;
699
700 rcu_read_lock();
701 for_each_active_iommu(iommu, drhd) {
702 if (pdev && segment != drhd->segment)
703 continue;
704
705 for_each_active_dev_scope(drhd->devices,
706 drhd->devices_cnt, i, tmp) {
707 if (tmp == dev) {
708 *bus = drhd->devices[i].bus;
709 *devfn = drhd->devices[i].devfn;
710 goto out;
711 }
712
713 if (!pdev || !dev_is_pci(tmp))
714 continue;
715
716 ptmp = to_pci_dev(tmp);
717 if (ptmp->subordinate &&
718 ptmp->subordinate->number <= pdev->bus->number &&
719 ptmp->subordinate->busn_res.end >= pdev->bus->number)
720 goto got_pdev;
721 }
722
723 if (pdev && drhd->include_all) {
724 got_pdev:
725 *bus = pdev->bus->number;
726 *devfn = pdev->devfn;
727 goto out;
728 }
729 }
730 iommu = NULL;
731 out:
732 rcu_read_unlock();
733
734 return iommu;
735 }
736
737 static void domain_flush_cache(struct dmar_domain *domain,
738 void *addr, int size)
739 {
740 if (!domain->iommu_coherency)
741 clflush_cache_range(addr, size);
742 }
743
744 /* Gets context entry for a given bus and devfn */
745 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
746 u8 bus, u8 devfn)
747 {
748 struct root_entry *root;
749 struct context_entry *context;
750 unsigned long phy_addr;
751 unsigned long flags;
752
753 spin_lock_irqsave(&iommu->lock, flags);
754 root = &iommu->root_entry[bus];
755 context = get_context_addr_from_root(root);
756 if (!context) {
757 context = (struct context_entry *)
758 alloc_pgtable_page(iommu->node);
759 if (!context) {
760 spin_unlock_irqrestore(&iommu->lock, flags);
761 return NULL;
762 }
763 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
764 phy_addr = virt_to_phys((void *)context);
765 set_root_value(root, phy_addr);
766 set_root_present(root);
767 __iommu_flush_cache(iommu, root, sizeof(*root));
768 }
769 spin_unlock_irqrestore(&iommu->lock, flags);
770 return &context[devfn];
771 }
772
773 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
774 {
775 struct root_entry *root;
776 struct context_entry *context;
777 int ret;
778 unsigned long flags;
779
780 spin_lock_irqsave(&iommu->lock, flags);
781 root = &iommu->root_entry[bus];
782 context = get_context_addr_from_root(root);
783 if (!context) {
784 ret = 0;
785 goto out;
786 }
787 ret = context_present(&context[devfn]);
788 out:
789 spin_unlock_irqrestore(&iommu->lock, flags);
790 return ret;
791 }
792
793 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
794 {
795 struct root_entry *root;
796 struct context_entry *context;
797 unsigned long flags;
798
799 spin_lock_irqsave(&iommu->lock, flags);
800 root = &iommu->root_entry[bus];
801 context = get_context_addr_from_root(root);
802 if (context) {
803 context_clear_entry(&context[devfn]);
804 __iommu_flush_cache(iommu, &context[devfn], \
805 sizeof(*context));
806 }
807 spin_unlock_irqrestore(&iommu->lock, flags);
808 }
809
810 static void free_context_table(struct intel_iommu *iommu)
811 {
812 struct root_entry *root;
813 int i;
814 unsigned long flags;
815 struct context_entry *context;
816
817 spin_lock_irqsave(&iommu->lock, flags);
818 if (!iommu->root_entry) {
819 goto out;
820 }
821 for (i = 0; i < ROOT_ENTRY_NR; i++) {
822 root = &iommu->root_entry[i];
823 context = get_context_addr_from_root(root);
824 if (context)
825 free_pgtable_page(context);
826 }
827 free_pgtable_page(iommu->root_entry);
828 iommu->root_entry = NULL;
829 out:
830 spin_unlock_irqrestore(&iommu->lock, flags);
831 }
832
833 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
834 unsigned long pfn, int *target_level)
835 {
836 struct dma_pte *parent, *pte = NULL;
837 int level = agaw_to_level(domain->agaw);
838 int offset;
839
840 BUG_ON(!domain->pgd);
841
842 if (!domain_pfn_supported(domain, pfn))
843 /* Address beyond IOMMU's addressing capabilities. */
844 return NULL;
845
846 parent = domain->pgd;
847
848 while (1) {
849 void *tmp_page;
850
851 offset = pfn_level_offset(pfn, level);
852 pte = &parent[offset];
853 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
854 break;
855 if (level == *target_level)
856 break;
857
858 if (!dma_pte_present(pte)) {
859 uint64_t pteval;
860
861 tmp_page = alloc_pgtable_page(domain->nid);
862
863 if (!tmp_page)
864 return NULL;
865
866 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
867 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
868 if (cmpxchg64(&pte->val, 0ULL, pteval))
869 /* Someone else set it while we were thinking; use theirs. */
870 free_pgtable_page(tmp_page);
871 else
872 domain_flush_cache(domain, pte, sizeof(*pte));
873 }
874 if (level == 1)
875 break;
876
877 parent = phys_to_virt(dma_pte_addr(pte));
878 level--;
879 }
880
881 if (!*target_level)
882 *target_level = level;
883
884 return pte;
885 }
886
887
888 /* return address's pte at specific level */
889 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
890 unsigned long pfn,
891 int level, int *large_page)
892 {
893 struct dma_pte *parent, *pte = NULL;
894 int total = agaw_to_level(domain->agaw);
895 int offset;
896
897 parent = domain->pgd;
898 while (level <= total) {
899 offset = pfn_level_offset(pfn, total);
900 pte = &parent[offset];
901 if (level == total)
902 return pte;
903
904 if (!dma_pte_present(pte)) {
905 *large_page = total;
906 break;
907 }
908
909 if (dma_pte_superpage(pte)) {
910 *large_page = total;
911 return pte;
912 }
913
914 parent = phys_to_virt(dma_pte_addr(pte));
915 total--;
916 }
917 return NULL;
918 }
919
920 /* clear last level pte, a tlb flush should be followed */
921 static void dma_pte_clear_range(struct dmar_domain *domain,
922 unsigned long start_pfn,
923 unsigned long last_pfn)
924 {
925 unsigned int large_page = 1;
926 struct dma_pte *first_pte, *pte;
927
928 BUG_ON(!domain_pfn_supported(domain, start_pfn));
929 BUG_ON(!domain_pfn_supported(domain, last_pfn));
930 BUG_ON(start_pfn > last_pfn);
931
932 /* we don't need lock here; nobody else touches the iova range */
933 do {
934 large_page = 1;
935 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
936 if (!pte) {
937 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
938 continue;
939 }
940 do {
941 dma_clear_pte(pte);
942 start_pfn += lvl_to_nr_pages(large_page);
943 pte++;
944 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
945
946 domain_flush_cache(domain, first_pte,
947 (void *)pte - (void *)first_pte);
948
949 } while (start_pfn && start_pfn <= last_pfn);
950 }
951
952 static void dma_pte_free_level(struct dmar_domain *domain, int level,
953 struct dma_pte *pte, unsigned long pfn,
954 unsigned long start_pfn, unsigned long last_pfn)
955 {
956 pfn = max(start_pfn, pfn);
957 pte = &pte[pfn_level_offset(pfn, level)];
958
959 do {
960 unsigned long level_pfn;
961 struct dma_pte *level_pte;
962
963 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
964 goto next;
965
966 level_pfn = pfn & level_mask(level - 1);
967 level_pte = phys_to_virt(dma_pte_addr(pte));
968
969 if (level > 2)
970 dma_pte_free_level(domain, level - 1, level_pte,
971 level_pfn, start_pfn, last_pfn);
972
973 /* If range covers entire pagetable, free it */
974 if (!(start_pfn > level_pfn ||
975 last_pfn < level_pfn + level_size(level) - 1)) {
976 dma_clear_pte(pte);
977 domain_flush_cache(domain, pte, sizeof(*pte));
978 free_pgtable_page(level_pte);
979 }
980 next:
981 pfn += level_size(level);
982 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
983 }
984
985 /* free page table pages. last level pte should already be cleared */
986 static void dma_pte_free_pagetable(struct dmar_domain *domain,
987 unsigned long start_pfn,
988 unsigned long last_pfn)
989 {
990 BUG_ON(!domain_pfn_supported(domain, start_pfn));
991 BUG_ON(!domain_pfn_supported(domain, last_pfn));
992 BUG_ON(start_pfn > last_pfn);
993
994 dma_pte_clear_range(domain, start_pfn, last_pfn);
995
996 /* We don't need lock here; nobody else touches the iova range */
997 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
998 domain->pgd, 0, start_pfn, last_pfn);
999
1000 /* free pgd */
1001 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1002 free_pgtable_page(domain->pgd);
1003 domain->pgd = NULL;
1004 }
1005 }
1006
1007 /* When a page at a given level is being unlinked from its parent, we don't
1008 need to *modify* it at all. All we need to do is make a list of all the
1009 pages which can be freed just as soon as we've flushed the IOTLB and we
1010 know the hardware page-walk will no longer touch them.
1011 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1012 be freed. */
1013 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1014 int level, struct dma_pte *pte,
1015 struct page *freelist)
1016 {
1017 struct page *pg;
1018
1019 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1020 pg->freelist = freelist;
1021 freelist = pg;
1022
1023 if (level == 1)
1024 return freelist;
1025
1026 pte = page_address(pg);
1027 do {
1028 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1029 freelist = dma_pte_list_pagetables(domain, level - 1,
1030 pte, freelist);
1031 pte++;
1032 } while (!first_pte_in_page(pte));
1033
1034 return freelist;
1035 }
1036
1037 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1038 struct dma_pte *pte, unsigned long pfn,
1039 unsigned long start_pfn,
1040 unsigned long last_pfn,
1041 struct page *freelist)
1042 {
1043 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1044
1045 pfn = max(start_pfn, pfn);
1046 pte = &pte[pfn_level_offset(pfn, level)];
1047
1048 do {
1049 unsigned long level_pfn;
1050
1051 if (!dma_pte_present(pte))
1052 goto next;
1053
1054 level_pfn = pfn & level_mask(level);
1055
1056 /* If range covers entire pagetable, free it */
1057 if (start_pfn <= level_pfn &&
1058 last_pfn >= level_pfn + level_size(level) - 1) {
1059 /* These suborbinate page tables are going away entirely. Don't
1060 bother to clear them; we're just going to *free* them. */
1061 if (level > 1 && !dma_pte_superpage(pte))
1062 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1063
1064 dma_clear_pte(pte);
1065 if (!first_pte)
1066 first_pte = pte;
1067 last_pte = pte;
1068 } else if (level > 1) {
1069 /* Recurse down into a level that isn't *entirely* obsolete */
1070 freelist = dma_pte_clear_level(domain, level - 1,
1071 phys_to_virt(dma_pte_addr(pte)),
1072 level_pfn, start_pfn, last_pfn,
1073 freelist);
1074 }
1075 next:
1076 pfn += level_size(level);
1077 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1078
1079 if (first_pte)
1080 domain_flush_cache(domain, first_pte,
1081 (void *)++last_pte - (void *)first_pte);
1082
1083 return freelist;
1084 }
1085
1086 /* We can't just free the pages because the IOMMU may still be walking
1087 the page tables, and may have cached the intermediate levels. The
1088 pages can only be freed after the IOTLB flush has been done. */
1089 struct page *domain_unmap(struct dmar_domain *domain,
1090 unsigned long start_pfn,
1091 unsigned long last_pfn)
1092 {
1093 struct page *freelist = NULL;
1094
1095 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1096 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1097 BUG_ON(start_pfn > last_pfn);
1098
1099 /* we don't need lock here; nobody else touches the iova range */
1100 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1101 domain->pgd, 0, start_pfn, last_pfn, NULL);
1102
1103 /* free pgd */
1104 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1105 struct page *pgd_page = virt_to_page(domain->pgd);
1106 pgd_page->freelist = freelist;
1107 freelist = pgd_page;
1108
1109 domain->pgd = NULL;
1110 }
1111
1112 return freelist;
1113 }
1114
1115 void dma_free_pagelist(struct page *freelist)
1116 {
1117 struct page *pg;
1118
1119 while ((pg = freelist)) {
1120 freelist = pg->freelist;
1121 free_pgtable_page(page_address(pg));
1122 }
1123 }
1124
1125 /* iommu handling */
1126 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1127 {
1128 struct root_entry *root;
1129 unsigned long flags;
1130
1131 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1132 if (!root) {
1133 pr_err("IOMMU: allocating root entry for %s failed\n",
1134 iommu->name);
1135 return -ENOMEM;
1136 }
1137
1138 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1139
1140 spin_lock_irqsave(&iommu->lock, flags);
1141 iommu->root_entry = root;
1142 spin_unlock_irqrestore(&iommu->lock, flags);
1143
1144 return 0;
1145 }
1146
1147 static void iommu_set_root_entry(struct intel_iommu *iommu)
1148 {
1149 void *addr;
1150 u32 sts;
1151 unsigned long flag;
1152
1153 addr = iommu->root_entry;
1154
1155 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1156 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1157
1158 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1159
1160 /* Make sure hardware complete it */
1161 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1162 readl, (sts & DMA_GSTS_RTPS), sts);
1163
1164 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1165 }
1166
1167 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1168 {
1169 u32 val;
1170 unsigned long flag;
1171
1172 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1173 return;
1174
1175 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1176 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1177
1178 /* Make sure hardware complete it */
1179 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1180 readl, (!(val & DMA_GSTS_WBFS)), val);
1181
1182 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1183 }
1184
1185 /* return value determine if we need a write buffer flush */
1186 static void __iommu_flush_context(struct intel_iommu *iommu,
1187 u16 did, u16 source_id, u8 function_mask,
1188 u64 type)
1189 {
1190 u64 val = 0;
1191 unsigned long flag;
1192
1193 switch (type) {
1194 case DMA_CCMD_GLOBAL_INVL:
1195 val = DMA_CCMD_GLOBAL_INVL;
1196 break;
1197 case DMA_CCMD_DOMAIN_INVL:
1198 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1199 break;
1200 case DMA_CCMD_DEVICE_INVL:
1201 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1202 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1203 break;
1204 default:
1205 BUG();
1206 }
1207 val |= DMA_CCMD_ICC;
1208
1209 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1210 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1211
1212 /* Make sure hardware complete it */
1213 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1214 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1215
1216 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1217 }
1218
1219 /* return value determine if we need a write buffer flush */
1220 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1221 u64 addr, unsigned int size_order, u64 type)
1222 {
1223 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1224 u64 val = 0, val_iva = 0;
1225 unsigned long flag;
1226
1227 switch (type) {
1228 case DMA_TLB_GLOBAL_FLUSH:
1229 /* global flush doesn't need set IVA_REG */
1230 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1231 break;
1232 case DMA_TLB_DSI_FLUSH:
1233 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1234 break;
1235 case DMA_TLB_PSI_FLUSH:
1236 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1237 /* IH bit is passed in as part of address */
1238 val_iva = size_order | addr;
1239 break;
1240 default:
1241 BUG();
1242 }
1243 /* Note: set drain read/write */
1244 #if 0
1245 /*
1246 * This is probably to be super secure.. Looks like we can
1247 * ignore it without any impact.
1248 */
1249 if (cap_read_drain(iommu->cap))
1250 val |= DMA_TLB_READ_DRAIN;
1251 #endif
1252 if (cap_write_drain(iommu->cap))
1253 val |= DMA_TLB_WRITE_DRAIN;
1254
1255 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1256 /* Note: Only uses first TLB reg currently */
1257 if (val_iva)
1258 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1259 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1260
1261 /* Make sure hardware complete it */
1262 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1263 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1264
1265 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1266
1267 /* check IOTLB invalidation granularity */
1268 if (DMA_TLB_IAIG(val) == 0)
1269 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1270 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1271 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1272 (unsigned long long)DMA_TLB_IIRG(type),
1273 (unsigned long long)DMA_TLB_IAIG(val));
1274 }
1275
1276 static struct device_domain_info *
1277 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1278 u8 bus, u8 devfn)
1279 {
1280 bool found = false;
1281 unsigned long flags;
1282 struct device_domain_info *info;
1283 struct pci_dev *pdev;
1284
1285 if (!ecap_dev_iotlb_support(iommu->ecap))
1286 return NULL;
1287
1288 if (!iommu->qi)
1289 return NULL;
1290
1291 spin_lock_irqsave(&device_domain_lock, flags);
1292 list_for_each_entry(info, &domain->devices, link)
1293 if (info->iommu == iommu && info->bus == bus &&
1294 info->devfn == devfn) {
1295 found = true;
1296 break;
1297 }
1298 spin_unlock_irqrestore(&device_domain_lock, flags);
1299
1300 if (!found || !info->dev || !dev_is_pci(info->dev))
1301 return NULL;
1302
1303 pdev = to_pci_dev(info->dev);
1304
1305 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1306 return NULL;
1307
1308 if (!dmar_find_matched_atsr_unit(pdev))
1309 return NULL;
1310
1311 return info;
1312 }
1313
1314 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1315 {
1316 if (!info || !dev_is_pci(info->dev))
1317 return;
1318
1319 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1320 }
1321
1322 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1323 {
1324 if (!info->dev || !dev_is_pci(info->dev) ||
1325 !pci_ats_enabled(to_pci_dev(info->dev)))
1326 return;
1327
1328 pci_disable_ats(to_pci_dev(info->dev));
1329 }
1330
1331 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1332 u64 addr, unsigned mask)
1333 {
1334 u16 sid, qdep;
1335 unsigned long flags;
1336 struct device_domain_info *info;
1337
1338 spin_lock_irqsave(&device_domain_lock, flags);
1339 list_for_each_entry(info, &domain->devices, link) {
1340 struct pci_dev *pdev;
1341 if (!info->dev || !dev_is_pci(info->dev))
1342 continue;
1343
1344 pdev = to_pci_dev(info->dev);
1345 if (!pci_ats_enabled(pdev))
1346 continue;
1347
1348 sid = info->bus << 8 | info->devfn;
1349 qdep = pci_ats_queue_depth(pdev);
1350 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1351 }
1352 spin_unlock_irqrestore(&device_domain_lock, flags);
1353 }
1354
1355 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1356 unsigned long pfn, unsigned int pages, int ih, int map)
1357 {
1358 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1359 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1360
1361 BUG_ON(pages == 0);
1362
1363 if (ih)
1364 ih = 1 << 6;
1365 /*
1366 * Fallback to domain selective flush if no PSI support or the size is
1367 * too big.
1368 * PSI requires page size to be 2 ^ x, and the base address is naturally
1369 * aligned to the size
1370 */
1371 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1372 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1373 DMA_TLB_DSI_FLUSH);
1374 else
1375 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1376 DMA_TLB_PSI_FLUSH);
1377
1378 /*
1379 * In caching mode, changes of pages from non-present to present require
1380 * flush. However, device IOTLB doesn't need to be flushed in this case.
1381 */
1382 if (!cap_caching_mode(iommu->cap) || !map)
1383 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1384 }
1385
1386 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1387 {
1388 u32 pmen;
1389 unsigned long flags;
1390
1391 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1392 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1393 pmen &= ~DMA_PMEN_EPM;
1394 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1395
1396 /* wait for the protected region status bit to clear */
1397 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1398 readl, !(pmen & DMA_PMEN_PRS), pmen);
1399
1400 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1401 }
1402
1403 static void iommu_enable_translation(struct intel_iommu *iommu)
1404 {
1405 u32 sts;
1406 unsigned long flags;
1407
1408 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1409 iommu->gcmd |= DMA_GCMD_TE;
1410 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1411
1412 /* Make sure hardware complete it */
1413 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1414 readl, (sts & DMA_GSTS_TES), sts);
1415
1416 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1417 }
1418
1419 static void iommu_disable_translation(struct intel_iommu *iommu)
1420 {
1421 u32 sts;
1422 unsigned long flag;
1423
1424 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1425 iommu->gcmd &= ~DMA_GCMD_TE;
1426 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1427
1428 /* Make sure hardware complete it */
1429 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1430 readl, (!(sts & DMA_GSTS_TES)), sts);
1431
1432 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1433 }
1434
1435
1436 static int iommu_init_domains(struct intel_iommu *iommu)
1437 {
1438 unsigned long ndomains;
1439 unsigned long nlongs;
1440
1441 ndomains = cap_ndoms(iommu->cap);
1442 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1443 iommu->seq_id, ndomains);
1444 nlongs = BITS_TO_LONGS(ndomains);
1445
1446 spin_lock_init(&iommu->lock);
1447
1448 /* TBD: there might be 64K domains,
1449 * consider other allocation for future chip
1450 */
1451 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1452 if (!iommu->domain_ids) {
1453 pr_err("IOMMU%d: allocating domain id array failed\n",
1454 iommu->seq_id);
1455 return -ENOMEM;
1456 }
1457 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1458 GFP_KERNEL);
1459 if (!iommu->domains) {
1460 pr_err("IOMMU%d: allocating domain array failed\n",
1461 iommu->seq_id);
1462 kfree(iommu->domain_ids);
1463 iommu->domain_ids = NULL;
1464 return -ENOMEM;
1465 }
1466
1467 /*
1468 * if Caching mode is set, then invalid translations are tagged
1469 * with domainid 0. Hence we need to pre-allocate it.
1470 */
1471 if (cap_caching_mode(iommu->cap))
1472 set_bit(0, iommu->domain_ids);
1473 return 0;
1474 }
1475
1476 static void disable_dmar_iommu(struct intel_iommu *iommu)
1477 {
1478 struct dmar_domain *domain;
1479 int i;
1480
1481 if ((iommu->domains) && (iommu->domain_ids)) {
1482 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1483 /*
1484 * Domain id 0 is reserved for invalid translation
1485 * if hardware supports caching mode.
1486 */
1487 if (cap_caching_mode(iommu->cap) && i == 0)
1488 continue;
1489
1490 domain = iommu->domains[i];
1491 clear_bit(i, iommu->domain_ids);
1492 if (domain_detach_iommu(domain, iommu) == 0 &&
1493 !domain_type_is_vm(domain))
1494 domain_exit(domain);
1495 }
1496 }
1497
1498 if (iommu->gcmd & DMA_GCMD_TE)
1499 iommu_disable_translation(iommu);
1500 }
1501
1502 static void free_dmar_iommu(struct intel_iommu *iommu)
1503 {
1504 if ((iommu->domains) && (iommu->domain_ids)) {
1505 kfree(iommu->domains);
1506 kfree(iommu->domain_ids);
1507 iommu->domains = NULL;
1508 iommu->domain_ids = NULL;
1509 }
1510
1511 g_iommus[iommu->seq_id] = NULL;
1512
1513 /* free context mapping */
1514 free_context_table(iommu);
1515 }
1516
1517 static struct dmar_domain *alloc_domain(int flags)
1518 {
1519 /* domain id for virtual machine, it won't be set in context */
1520 static atomic_t vm_domid = ATOMIC_INIT(0);
1521 struct dmar_domain *domain;
1522
1523 domain = alloc_domain_mem();
1524 if (!domain)
1525 return NULL;
1526
1527 memset(domain, 0, sizeof(*domain));
1528 domain->nid = -1;
1529 domain->flags = flags;
1530 spin_lock_init(&domain->iommu_lock);
1531 INIT_LIST_HEAD(&domain->devices);
1532 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1533 domain->id = atomic_inc_return(&vm_domid);
1534
1535 return domain;
1536 }
1537
1538 static int __iommu_attach_domain(struct dmar_domain *domain,
1539 struct intel_iommu *iommu)
1540 {
1541 int num;
1542 unsigned long ndomains;
1543
1544 ndomains = cap_ndoms(iommu->cap);
1545 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1546 if (num < ndomains) {
1547 set_bit(num, iommu->domain_ids);
1548 iommu->domains[num] = domain;
1549 } else {
1550 num = -ENOSPC;
1551 }
1552
1553 return num;
1554 }
1555
1556 static int iommu_attach_domain(struct dmar_domain *domain,
1557 struct intel_iommu *iommu)
1558 {
1559 int num;
1560 unsigned long flags;
1561
1562 spin_lock_irqsave(&iommu->lock, flags);
1563 num = __iommu_attach_domain(domain, iommu);
1564 spin_unlock_irqrestore(&iommu->lock, flags);
1565 if (num < 0)
1566 pr_err("IOMMU: no free domain ids\n");
1567
1568 return num;
1569 }
1570
1571 static int iommu_attach_vm_domain(struct dmar_domain *domain,
1572 struct intel_iommu *iommu)
1573 {
1574 int num;
1575 unsigned long ndomains;
1576
1577 ndomains = cap_ndoms(iommu->cap);
1578 for_each_set_bit(num, iommu->domain_ids, ndomains)
1579 if (iommu->domains[num] == domain)
1580 return num;
1581
1582 return __iommu_attach_domain(domain, iommu);
1583 }
1584
1585 static void iommu_detach_domain(struct dmar_domain *domain,
1586 struct intel_iommu *iommu)
1587 {
1588 unsigned long flags;
1589 int num, ndomains;
1590
1591 spin_lock_irqsave(&iommu->lock, flags);
1592 if (domain_type_is_vm_or_si(domain)) {
1593 ndomains = cap_ndoms(iommu->cap);
1594 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1595 if (iommu->domains[num] == domain) {
1596 clear_bit(num, iommu->domain_ids);
1597 iommu->domains[num] = NULL;
1598 break;
1599 }
1600 }
1601 } else {
1602 clear_bit(domain->id, iommu->domain_ids);
1603 iommu->domains[domain->id] = NULL;
1604 }
1605 spin_unlock_irqrestore(&iommu->lock, flags);
1606 }
1607
1608 static void domain_attach_iommu(struct dmar_domain *domain,
1609 struct intel_iommu *iommu)
1610 {
1611 unsigned long flags;
1612
1613 spin_lock_irqsave(&domain->iommu_lock, flags);
1614 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1615 domain->iommu_count++;
1616 if (domain->iommu_count == 1)
1617 domain->nid = iommu->node;
1618 domain_update_iommu_cap(domain);
1619 }
1620 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1621 }
1622
1623 static int domain_detach_iommu(struct dmar_domain *domain,
1624 struct intel_iommu *iommu)
1625 {
1626 unsigned long flags;
1627 int count = INT_MAX;
1628
1629 spin_lock_irqsave(&domain->iommu_lock, flags);
1630 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1631 count = --domain->iommu_count;
1632 domain_update_iommu_cap(domain);
1633 }
1634 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1635
1636 return count;
1637 }
1638
1639 static struct iova_domain reserved_iova_list;
1640 static struct lock_class_key reserved_rbtree_key;
1641
1642 static int dmar_init_reserved_ranges(void)
1643 {
1644 struct pci_dev *pdev = NULL;
1645 struct iova *iova;
1646 int i;
1647
1648 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1649 DMA_32BIT_PFN);
1650
1651 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1652 &reserved_rbtree_key);
1653
1654 /* IOAPIC ranges shouldn't be accessed by DMA */
1655 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1656 IOVA_PFN(IOAPIC_RANGE_END));
1657 if (!iova) {
1658 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1659 return -ENODEV;
1660 }
1661
1662 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1663 for_each_pci_dev(pdev) {
1664 struct resource *r;
1665
1666 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1667 r = &pdev->resource[i];
1668 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1669 continue;
1670 iova = reserve_iova(&reserved_iova_list,
1671 IOVA_PFN(r->start),
1672 IOVA_PFN(r->end));
1673 if (!iova) {
1674 printk(KERN_ERR "Reserve iova failed\n");
1675 return -ENODEV;
1676 }
1677 }
1678 }
1679 return 0;
1680 }
1681
1682 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1683 {
1684 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1685 }
1686
1687 static inline int guestwidth_to_adjustwidth(int gaw)
1688 {
1689 int agaw;
1690 int r = (gaw - 12) % 9;
1691
1692 if (r == 0)
1693 agaw = gaw;
1694 else
1695 agaw = gaw + 9 - r;
1696 if (agaw > 64)
1697 agaw = 64;
1698 return agaw;
1699 }
1700
1701 static int domain_init(struct dmar_domain *domain, int guest_width)
1702 {
1703 struct intel_iommu *iommu;
1704 int adjust_width, agaw;
1705 unsigned long sagaw;
1706
1707 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1708 DMA_32BIT_PFN);
1709 domain_reserve_special_ranges(domain);
1710
1711 /* calculate AGAW */
1712 iommu = domain_get_iommu(domain);
1713 if (guest_width > cap_mgaw(iommu->cap))
1714 guest_width = cap_mgaw(iommu->cap);
1715 domain->gaw = guest_width;
1716 adjust_width = guestwidth_to_adjustwidth(guest_width);
1717 agaw = width_to_agaw(adjust_width);
1718 sagaw = cap_sagaw(iommu->cap);
1719 if (!test_bit(agaw, &sagaw)) {
1720 /* hardware doesn't support it, choose a bigger one */
1721 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1722 agaw = find_next_bit(&sagaw, 5, agaw);
1723 if (agaw >= 5)
1724 return -ENODEV;
1725 }
1726 domain->agaw = agaw;
1727
1728 if (ecap_coherent(iommu->ecap))
1729 domain->iommu_coherency = 1;
1730 else
1731 domain->iommu_coherency = 0;
1732
1733 if (ecap_sc_support(iommu->ecap))
1734 domain->iommu_snooping = 1;
1735 else
1736 domain->iommu_snooping = 0;
1737
1738 if (intel_iommu_superpage)
1739 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1740 else
1741 domain->iommu_superpage = 0;
1742
1743 domain->nid = iommu->node;
1744
1745 /* always allocate the top pgd */
1746 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1747 if (!domain->pgd)
1748 return -ENOMEM;
1749 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1750 return 0;
1751 }
1752
1753 static void domain_exit(struct dmar_domain *domain)
1754 {
1755 struct page *freelist = NULL;
1756 int i;
1757
1758 /* Domain 0 is reserved, so dont process it */
1759 if (!domain)
1760 return;
1761
1762 /* Flush any lazy unmaps that may reference this domain */
1763 if (!intel_iommu_strict)
1764 flush_unmaps_timeout(0);
1765
1766 /* remove associated devices */
1767 domain_remove_dev_info(domain);
1768
1769 /* destroy iovas */
1770 put_iova_domain(&domain->iovad);
1771
1772 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1773
1774 /* clear attached or cached domains */
1775 rcu_read_lock();
1776 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus)
1777 iommu_detach_domain(domain, g_iommus[i]);
1778 rcu_read_unlock();
1779
1780 dma_free_pagelist(freelist);
1781
1782 free_domain_mem(domain);
1783 }
1784
1785 static int domain_context_mapping_one(struct dmar_domain *domain,
1786 struct intel_iommu *iommu,
1787 u8 bus, u8 devfn, int translation)
1788 {
1789 struct context_entry *context;
1790 unsigned long flags;
1791 struct dma_pte *pgd;
1792 int id;
1793 int agaw;
1794 struct device_domain_info *info = NULL;
1795
1796 pr_debug("Set context mapping for %02x:%02x.%d\n",
1797 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1798
1799 BUG_ON(!domain->pgd);
1800 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1801 translation != CONTEXT_TT_MULTI_LEVEL);
1802
1803 context = device_to_context_entry(iommu, bus, devfn);
1804 if (!context)
1805 return -ENOMEM;
1806 spin_lock_irqsave(&iommu->lock, flags);
1807 if (context_present(context)) {
1808 spin_unlock_irqrestore(&iommu->lock, flags);
1809 return 0;
1810 }
1811
1812 id = domain->id;
1813 pgd = domain->pgd;
1814
1815 if (domain_type_is_vm_or_si(domain)) {
1816 if (domain_type_is_vm(domain)) {
1817 id = iommu_attach_vm_domain(domain, iommu);
1818 if (id < 0) {
1819 spin_unlock_irqrestore(&iommu->lock, flags);
1820 pr_err("IOMMU: no free domain ids\n");
1821 return -EFAULT;
1822 }
1823 }
1824
1825 /* Skip top levels of page tables for
1826 * iommu which has less agaw than default.
1827 * Unnecessary for PT mode.
1828 */
1829 if (translation != CONTEXT_TT_PASS_THROUGH) {
1830 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1831 pgd = phys_to_virt(dma_pte_addr(pgd));
1832 if (!dma_pte_present(pgd)) {
1833 spin_unlock_irqrestore(&iommu->lock, flags);
1834 return -ENOMEM;
1835 }
1836 }
1837 }
1838 }
1839
1840 context_set_domain_id(context, id);
1841
1842 if (translation != CONTEXT_TT_PASS_THROUGH) {
1843 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1844 translation = info ? CONTEXT_TT_DEV_IOTLB :
1845 CONTEXT_TT_MULTI_LEVEL;
1846 }
1847 /*
1848 * In pass through mode, AW must be programmed to indicate the largest
1849 * AGAW value supported by hardware. And ASR is ignored by hardware.
1850 */
1851 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1852 context_set_address_width(context, iommu->msagaw);
1853 else {
1854 context_set_address_root(context, virt_to_phys(pgd));
1855 context_set_address_width(context, iommu->agaw);
1856 }
1857
1858 context_set_translation_type(context, translation);
1859 context_set_fault_enable(context);
1860 context_set_present(context);
1861 domain_flush_cache(domain, context, sizeof(*context));
1862
1863 /*
1864 * It's a non-present to present mapping. If hardware doesn't cache
1865 * non-present entry we only need to flush the write-buffer. If the
1866 * _does_ cache non-present entries, then it does so in the special
1867 * domain #0, which we have to flush:
1868 */
1869 if (cap_caching_mode(iommu->cap)) {
1870 iommu->flush.flush_context(iommu, 0,
1871 (((u16)bus) << 8) | devfn,
1872 DMA_CCMD_MASK_NOBIT,
1873 DMA_CCMD_DEVICE_INVL);
1874 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
1875 } else {
1876 iommu_flush_write_buffer(iommu);
1877 }
1878 iommu_enable_dev_iotlb(info);
1879 spin_unlock_irqrestore(&iommu->lock, flags);
1880
1881 domain_attach_iommu(domain, iommu);
1882
1883 return 0;
1884 }
1885
1886 struct domain_context_mapping_data {
1887 struct dmar_domain *domain;
1888 struct intel_iommu *iommu;
1889 int translation;
1890 };
1891
1892 static int domain_context_mapping_cb(struct pci_dev *pdev,
1893 u16 alias, void *opaque)
1894 {
1895 struct domain_context_mapping_data *data = opaque;
1896
1897 return domain_context_mapping_one(data->domain, data->iommu,
1898 PCI_BUS_NUM(alias), alias & 0xff,
1899 data->translation);
1900 }
1901
1902 static int
1903 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1904 int translation)
1905 {
1906 struct intel_iommu *iommu;
1907 u8 bus, devfn;
1908 struct domain_context_mapping_data data;
1909
1910 iommu = device_to_iommu(dev, &bus, &devfn);
1911 if (!iommu)
1912 return -ENODEV;
1913
1914 if (!dev_is_pci(dev))
1915 return domain_context_mapping_one(domain, iommu, bus, devfn,
1916 translation);
1917
1918 data.domain = domain;
1919 data.iommu = iommu;
1920 data.translation = translation;
1921
1922 return pci_for_each_dma_alias(to_pci_dev(dev),
1923 &domain_context_mapping_cb, &data);
1924 }
1925
1926 static int domain_context_mapped_cb(struct pci_dev *pdev,
1927 u16 alias, void *opaque)
1928 {
1929 struct intel_iommu *iommu = opaque;
1930
1931 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
1932 }
1933
1934 static int domain_context_mapped(struct device *dev)
1935 {
1936 struct intel_iommu *iommu;
1937 u8 bus, devfn;
1938
1939 iommu = device_to_iommu(dev, &bus, &devfn);
1940 if (!iommu)
1941 return -ENODEV;
1942
1943 if (!dev_is_pci(dev))
1944 return device_context_mapped(iommu, bus, devfn);
1945
1946 return !pci_for_each_dma_alias(to_pci_dev(dev),
1947 domain_context_mapped_cb, iommu);
1948 }
1949
1950 /* Returns a number of VTD pages, but aligned to MM page size */
1951 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1952 size_t size)
1953 {
1954 host_addr &= ~PAGE_MASK;
1955 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1956 }
1957
1958 /* Return largest possible superpage level for a given mapping */
1959 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1960 unsigned long iov_pfn,
1961 unsigned long phy_pfn,
1962 unsigned long pages)
1963 {
1964 int support, level = 1;
1965 unsigned long pfnmerge;
1966
1967 support = domain->iommu_superpage;
1968
1969 /* To use a large page, the virtual *and* physical addresses
1970 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1971 of them will mean we have to use smaller pages. So just
1972 merge them and check both at once. */
1973 pfnmerge = iov_pfn | phy_pfn;
1974
1975 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1976 pages >>= VTD_STRIDE_SHIFT;
1977 if (!pages)
1978 break;
1979 pfnmerge >>= VTD_STRIDE_SHIFT;
1980 level++;
1981 support--;
1982 }
1983 return level;
1984 }
1985
1986 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1987 struct scatterlist *sg, unsigned long phys_pfn,
1988 unsigned long nr_pages, int prot)
1989 {
1990 struct dma_pte *first_pte = NULL, *pte = NULL;
1991 phys_addr_t uninitialized_var(pteval);
1992 unsigned long sg_res = 0;
1993 unsigned int largepage_lvl = 0;
1994 unsigned long lvl_pages = 0;
1995
1996 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
1997
1998 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1999 return -EINVAL;
2000
2001 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2002
2003 if (!sg) {
2004 sg_res = nr_pages;
2005 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2006 }
2007
2008 while (nr_pages > 0) {
2009 uint64_t tmp;
2010
2011 if (!sg_res) {
2012 sg_res = aligned_nrpages(sg->offset, sg->length);
2013 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2014 sg->dma_length = sg->length;
2015 pteval = page_to_phys(sg_page(sg)) | prot;
2016 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2017 }
2018
2019 if (!pte) {
2020 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2021
2022 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2023 if (!pte)
2024 return -ENOMEM;
2025 /* It is large page*/
2026 if (largepage_lvl > 1) {
2027 pteval |= DMA_PTE_LARGE_PAGE;
2028 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2029 /*
2030 * Ensure that old small page tables are
2031 * removed to make room for superpage,
2032 * if they exist.
2033 */
2034 dma_pte_free_pagetable(domain, iov_pfn,
2035 iov_pfn + lvl_pages - 1);
2036 } else {
2037 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2038 }
2039
2040 }
2041 /* We don't need lock here, nobody else
2042 * touches the iova range
2043 */
2044 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2045 if (tmp) {
2046 static int dumps = 5;
2047 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2048 iov_pfn, tmp, (unsigned long long)pteval);
2049 if (dumps) {
2050 dumps--;
2051 debug_dma_dump_mappings(NULL);
2052 }
2053 WARN_ON(1);
2054 }
2055
2056 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2057
2058 BUG_ON(nr_pages < lvl_pages);
2059 BUG_ON(sg_res < lvl_pages);
2060
2061 nr_pages -= lvl_pages;
2062 iov_pfn += lvl_pages;
2063 phys_pfn += lvl_pages;
2064 pteval += lvl_pages * VTD_PAGE_SIZE;
2065 sg_res -= lvl_pages;
2066
2067 /* If the next PTE would be the first in a new page, then we
2068 need to flush the cache on the entries we've just written.
2069 And then we'll need to recalculate 'pte', so clear it and
2070 let it get set again in the if (!pte) block above.
2071
2072 If we're done (!nr_pages) we need to flush the cache too.
2073
2074 Also if we've been setting superpages, we may need to
2075 recalculate 'pte' and switch back to smaller pages for the
2076 end of the mapping, if the trailing size is not enough to
2077 use another superpage (i.e. sg_res < lvl_pages). */
2078 pte++;
2079 if (!nr_pages || first_pte_in_page(pte) ||
2080 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2081 domain_flush_cache(domain, first_pte,
2082 (void *)pte - (void *)first_pte);
2083 pte = NULL;
2084 }
2085
2086 if (!sg_res && nr_pages)
2087 sg = sg_next(sg);
2088 }
2089 return 0;
2090 }
2091
2092 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2093 struct scatterlist *sg, unsigned long nr_pages,
2094 int prot)
2095 {
2096 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2097 }
2098
2099 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2100 unsigned long phys_pfn, unsigned long nr_pages,
2101 int prot)
2102 {
2103 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2104 }
2105
2106 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2107 {
2108 if (!iommu)
2109 return;
2110
2111 clear_context_table(iommu, bus, devfn);
2112 iommu->flush.flush_context(iommu, 0, 0, 0,
2113 DMA_CCMD_GLOBAL_INVL);
2114 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2115 }
2116
2117 static inline void unlink_domain_info(struct device_domain_info *info)
2118 {
2119 assert_spin_locked(&device_domain_lock);
2120 list_del(&info->link);
2121 list_del(&info->global);
2122 if (info->dev)
2123 info->dev->archdata.iommu = NULL;
2124 }
2125
2126 static void domain_remove_dev_info(struct dmar_domain *domain)
2127 {
2128 struct device_domain_info *info, *tmp;
2129 unsigned long flags;
2130
2131 spin_lock_irqsave(&device_domain_lock, flags);
2132 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
2133 unlink_domain_info(info);
2134 spin_unlock_irqrestore(&device_domain_lock, flags);
2135
2136 iommu_disable_dev_iotlb(info);
2137 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2138
2139 if (domain_type_is_vm(domain)) {
2140 iommu_detach_dependent_devices(info->iommu, info->dev);
2141 domain_detach_iommu(domain, info->iommu);
2142 }
2143
2144 free_devinfo_mem(info);
2145 spin_lock_irqsave(&device_domain_lock, flags);
2146 }
2147 spin_unlock_irqrestore(&device_domain_lock, flags);
2148 }
2149
2150 /*
2151 * find_domain
2152 * Note: we use struct device->archdata.iommu stores the info
2153 */
2154 static struct dmar_domain *find_domain(struct device *dev)
2155 {
2156 struct device_domain_info *info;
2157
2158 /* No lock here, assumes no domain exit in normal case */
2159 info = dev->archdata.iommu;
2160 if (info)
2161 return info->domain;
2162 return NULL;
2163 }
2164
2165 static inline struct device_domain_info *
2166 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2167 {
2168 struct device_domain_info *info;
2169
2170 list_for_each_entry(info, &device_domain_list, global)
2171 if (info->iommu->segment == segment && info->bus == bus &&
2172 info->devfn == devfn)
2173 return info;
2174
2175 return NULL;
2176 }
2177
2178 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2179 int bus, int devfn,
2180 struct device *dev,
2181 struct dmar_domain *domain)
2182 {
2183 struct dmar_domain *found = NULL;
2184 struct device_domain_info *info;
2185 unsigned long flags;
2186
2187 info = alloc_devinfo_mem();
2188 if (!info)
2189 return NULL;
2190
2191 info->bus = bus;
2192 info->devfn = devfn;
2193 info->dev = dev;
2194 info->domain = domain;
2195 info->iommu = iommu;
2196
2197 spin_lock_irqsave(&device_domain_lock, flags);
2198 if (dev)
2199 found = find_domain(dev);
2200 else {
2201 struct device_domain_info *info2;
2202 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2203 if (info2)
2204 found = info2->domain;
2205 }
2206 if (found) {
2207 spin_unlock_irqrestore(&device_domain_lock, flags);
2208 free_devinfo_mem(info);
2209 /* Caller must free the original domain */
2210 return found;
2211 }
2212
2213 list_add(&info->link, &domain->devices);
2214 list_add(&info->global, &device_domain_list);
2215 if (dev)
2216 dev->archdata.iommu = info;
2217 spin_unlock_irqrestore(&device_domain_lock, flags);
2218
2219 return domain;
2220 }
2221
2222 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2223 {
2224 *(u16 *)opaque = alias;
2225 return 0;
2226 }
2227
2228 /* domain is initialized */
2229 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2230 {
2231 struct dmar_domain *domain, *tmp;
2232 struct intel_iommu *iommu;
2233 struct device_domain_info *info;
2234 u16 dma_alias;
2235 unsigned long flags;
2236 u8 bus, devfn;
2237
2238 domain = find_domain(dev);
2239 if (domain)
2240 return domain;
2241
2242 iommu = device_to_iommu(dev, &bus, &devfn);
2243 if (!iommu)
2244 return NULL;
2245
2246 if (dev_is_pci(dev)) {
2247 struct pci_dev *pdev = to_pci_dev(dev);
2248
2249 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2250
2251 spin_lock_irqsave(&device_domain_lock, flags);
2252 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2253 PCI_BUS_NUM(dma_alias),
2254 dma_alias & 0xff);
2255 if (info) {
2256 iommu = info->iommu;
2257 domain = info->domain;
2258 }
2259 spin_unlock_irqrestore(&device_domain_lock, flags);
2260
2261 /* DMA alias already has a domain, uses it */
2262 if (info)
2263 goto found_domain;
2264 }
2265
2266 /* Allocate and initialize new domain for the device */
2267 domain = alloc_domain(0);
2268 if (!domain)
2269 return NULL;
2270 domain->id = iommu_attach_domain(domain, iommu);
2271 if (domain->id < 0) {
2272 free_domain_mem(domain);
2273 return NULL;
2274 }
2275 domain_attach_iommu(domain, iommu);
2276 if (domain_init(domain, gaw)) {
2277 domain_exit(domain);
2278 return NULL;
2279 }
2280
2281 /* register PCI DMA alias device */
2282 if (dev_is_pci(dev)) {
2283 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2284 dma_alias & 0xff, NULL, domain);
2285
2286 if (!tmp || tmp != domain) {
2287 domain_exit(domain);
2288 domain = tmp;
2289 }
2290
2291 if (!domain)
2292 return NULL;
2293 }
2294
2295 found_domain:
2296 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2297
2298 if (!tmp || tmp != domain) {
2299 domain_exit(domain);
2300 domain = tmp;
2301 }
2302
2303 return domain;
2304 }
2305
2306 static int iommu_identity_mapping;
2307 #define IDENTMAP_ALL 1
2308 #define IDENTMAP_GFX 2
2309 #define IDENTMAP_AZALIA 4
2310
2311 static int iommu_domain_identity_map(struct dmar_domain *domain,
2312 unsigned long long start,
2313 unsigned long long end)
2314 {
2315 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2316 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2317
2318 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2319 dma_to_mm_pfn(last_vpfn))) {
2320 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2321 return -ENOMEM;
2322 }
2323
2324 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2325 start, end, domain->id);
2326 /*
2327 * RMRR range might have overlap with physical memory range,
2328 * clear it first
2329 */
2330 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2331
2332 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2333 last_vpfn - first_vpfn + 1,
2334 DMA_PTE_READ|DMA_PTE_WRITE);
2335 }
2336
2337 static int iommu_prepare_identity_map(struct device *dev,
2338 unsigned long long start,
2339 unsigned long long end)
2340 {
2341 struct dmar_domain *domain;
2342 int ret;
2343
2344 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2345 if (!domain)
2346 return -ENOMEM;
2347
2348 /* For _hardware_ passthrough, don't bother. But for software
2349 passthrough, we do it anyway -- it may indicate a memory
2350 range which is reserved in E820, so which didn't get set
2351 up to start with in si_domain */
2352 if (domain == si_domain && hw_pass_through) {
2353 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2354 dev_name(dev), start, end);
2355 return 0;
2356 }
2357
2358 printk(KERN_INFO
2359 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2360 dev_name(dev), start, end);
2361
2362 if (end < start) {
2363 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2364 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2365 dmi_get_system_info(DMI_BIOS_VENDOR),
2366 dmi_get_system_info(DMI_BIOS_VERSION),
2367 dmi_get_system_info(DMI_PRODUCT_VERSION));
2368 ret = -EIO;
2369 goto error;
2370 }
2371
2372 if (end >> agaw_to_width(domain->agaw)) {
2373 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2374 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2375 agaw_to_width(domain->agaw),
2376 dmi_get_system_info(DMI_BIOS_VENDOR),
2377 dmi_get_system_info(DMI_BIOS_VERSION),
2378 dmi_get_system_info(DMI_PRODUCT_VERSION));
2379 ret = -EIO;
2380 goto error;
2381 }
2382
2383 ret = iommu_domain_identity_map(domain, start, end);
2384 if (ret)
2385 goto error;
2386
2387 /* context entry init */
2388 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2389 if (ret)
2390 goto error;
2391
2392 return 0;
2393
2394 error:
2395 domain_exit(domain);
2396 return ret;
2397 }
2398
2399 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2400 struct device *dev)
2401 {
2402 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2403 return 0;
2404 return iommu_prepare_identity_map(dev, rmrr->base_address,
2405 rmrr->end_address);
2406 }
2407
2408 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2409 static inline void iommu_prepare_isa(void)
2410 {
2411 struct pci_dev *pdev;
2412 int ret;
2413
2414 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2415 if (!pdev)
2416 return;
2417
2418 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2419 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2420
2421 if (ret)
2422 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2423 "floppy might not work\n");
2424
2425 pci_dev_put(pdev);
2426 }
2427 #else
2428 static inline void iommu_prepare_isa(void)
2429 {
2430 return;
2431 }
2432 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2433
2434 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2435
2436 static int __init si_domain_init(int hw)
2437 {
2438 struct dmar_drhd_unit *drhd;
2439 struct intel_iommu *iommu;
2440 int nid, ret = 0;
2441 bool first = true;
2442
2443 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2444 if (!si_domain)
2445 return -EFAULT;
2446
2447 for_each_active_iommu(iommu, drhd) {
2448 ret = iommu_attach_domain(si_domain, iommu);
2449 if (ret < 0) {
2450 domain_exit(si_domain);
2451 return -EFAULT;
2452 } else if (first) {
2453 si_domain->id = ret;
2454 first = false;
2455 } else if (si_domain->id != ret) {
2456 domain_exit(si_domain);
2457 return -EFAULT;
2458 }
2459 domain_attach_iommu(si_domain, iommu);
2460 }
2461
2462 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2463 domain_exit(si_domain);
2464 return -EFAULT;
2465 }
2466
2467 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2468 si_domain->id);
2469
2470 if (hw)
2471 return 0;
2472
2473 for_each_online_node(nid) {
2474 unsigned long start_pfn, end_pfn;
2475 int i;
2476
2477 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2478 ret = iommu_domain_identity_map(si_domain,
2479 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2480 if (ret)
2481 return ret;
2482 }
2483 }
2484
2485 return 0;
2486 }
2487
2488 static int identity_mapping(struct device *dev)
2489 {
2490 struct device_domain_info *info;
2491
2492 if (likely(!iommu_identity_mapping))
2493 return 0;
2494
2495 info = dev->archdata.iommu;
2496 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2497 return (info->domain == si_domain);
2498
2499 return 0;
2500 }
2501
2502 static int domain_add_dev_info(struct dmar_domain *domain,
2503 struct device *dev, int translation)
2504 {
2505 struct dmar_domain *ndomain;
2506 struct intel_iommu *iommu;
2507 u8 bus, devfn;
2508 int ret;
2509
2510 iommu = device_to_iommu(dev, &bus, &devfn);
2511 if (!iommu)
2512 return -ENODEV;
2513
2514 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2515 if (ndomain != domain)
2516 return -EBUSY;
2517
2518 ret = domain_context_mapping(domain, dev, translation);
2519 if (ret) {
2520 domain_remove_one_dev_info(domain, dev);
2521 return ret;
2522 }
2523
2524 return 0;
2525 }
2526
2527 static bool device_has_rmrr(struct device *dev)
2528 {
2529 struct dmar_rmrr_unit *rmrr;
2530 struct device *tmp;
2531 int i;
2532
2533 rcu_read_lock();
2534 for_each_rmrr_units(rmrr) {
2535 /*
2536 * Return TRUE if this RMRR contains the device that
2537 * is passed in.
2538 */
2539 for_each_active_dev_scope(rmrr->devices,
2540 rmrr->devices_cnt, i, tmp)
2541 if (tmp == dev) {
2542 rcu_read_unlock();
2543 return true;
2544 }
2545 }
2546 rcu_read_unlock();
2547 return false;
2548 }
2549
2550 /*
2551 * There are a couple cases where we need to restrict the functionality of
2552 * devices associated with RMRRs. The first is when evaluating a device for
2553 * identity mapping because problems exist when devices are moved in and out
2554 * of domains and their respective RMRR information is lost. This means that
2555 * a device with associated RMRRs will never be in a "passthrough" domain.
2556 * The second is use of the device through the IOMMU API. This interface
2557 * expects to have full control of the IOVA space for the device. We cannot
2558 * satisfy both the requirement that RMRR access is maintained and have an
2559 * unencumbered IOVA space. We also have no ability to quiesce the device's
2560 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2561 * We therefore prevent devices associated with an RMRR from participating in
2562 * the IOMMU API, which eliminates them from device assignment.
2563 *
2564 * In both cases we assume that PCI USB devices with RMRRs have them largely
2565 * for historical reasons and that the RMRR space is not actively used post
2566 * boot. This exclusion may change if vendors begin to abuse it.
2567 */
2568 static bool device_is_rmrr_locked(struct device *dev)
2569 {
2570 if (!device_has_rmrr(dev))
2571 return false;
2572
2573 if (dev_is_pci(dev)) {
2574 struct pci_dev *pdev = to_pci_dev(dev);
2575
2576 if ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
2577 return false;
2578 }
2579
2580 return true;
2581 }
2582
2583 static int iommu_should_identity_map(struct device *dev, int startup)
2584 {
2585
2586 if (dev_is_pci(dev)) {
2587 struct pci_dev *pdev = to_pci_dev(dev);
2588
2589 if (device_is_rmrr_locked(dev))
2590 return 0;
2591
2592 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2593 return 1;
2594
2595 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2596 return 1;
2597
2598 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2599 return 0;
2600
2601 /*
2602 * We want to start off with all devices in the 1:1 domain, and
2603 * take them out later if we find they can't access all of memory.
2604 *
2605 * However, we can't do this for PCI devices behind bridges,
2606 * because all PCI devices behind the same bridge will end up
2607 * with the same source-id on their transactions.
2608 *
2609 * Practically speaking, we can't change things around for these
2610 * devices at run-time, because we can't be sure there'll be no
2611 * DMA transactions in flight for any of their siblings.
2612 *
2613 * So PCI devices (unless they're on the root bus) as well as
2614 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2615 * the 1:1 domain, just in _case_ one of their siblings turns out
2616 * not to be able to map all of memory.
2617 */
2618 if (!pci_is_pcie(pdev)) {
2619 if (!pci_is_root_bus(pdev->bus))
2620 return 0;
2621 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2622 return 0;
2623 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2624 return 0;
2625 } else {
2626 if (device_has_rmrr(dev))
2627 return 0;
2628 }
2629
2630 /*
2631 * At boot time, we don't yet know if devices will be 64-bit capable.
2632 * Assume that they will — if they turn out not to be, then we can
2633 * take them out of the 1:1 domain later.
2634 */
2635 if (!startup) {
2636 /*
2637 * If the device's dma_mask is less than the system's memory
2638 * size then this is not a candidate for identity mapping.
2639 */
2640 u64 dma_mask = *dev->dma_mask;
2641
2642 if (dev->coherent_dma_mask &&
2643 dev->coherent_dma_mask < dma_mask)
2644 dma_mask = dev->coherent_dma_mask;
2645
2646 return dma_mask >= dma_get_required_mask(dev);
2647 }
2648
2649 return 1;
2650 }
2651
2652 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2653 {
2654 int ret;
2655
2656 if (!iommu_should_identity_map(dev, 1))
2657 return 0;
2658
2659 ret = domain_add_dev_info(si_domain, dev,
2660 hw ? CONTEXT_TT_PASS_THROUGH :
2661 CONTEXT_TT_MULTI_LEVEL);
2662 if (!ret)
2663 pr_info("IOMMU: %s identity mapping for device %s\n",
2664 hw ? "hardware" : "software", dev_name(dev));
2665 else if (ret == -ENODEV)
2666 /* device not associated with an iommu */
2667 ret = 0;
2668
2669 return ret;
2670 }
2671
2672
2673 static int __init iommu_prepare_static_identity_mapping(int hw)
2674 {
2675 struct pci_dev *pdev = NULL;
2676 struct dmar_drhd_unit *drhd;
2677 struct intel_iommu *iommu;
2678 struct device *dev;
2679 int i;
2680 int ret = 0;
2681
2682 ret = si_domain_init(hw);
2683 if (ret)
2684 return -EFAULT;
2685
2686 for_each_pci_dev(pdev) {
2687 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2688 if (ret)
2689 return ret;
2690 }
2691
2692 for_each_active_iommu(iommu, drhd)
2693 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2694 struct acpi_device_physical_node *pn;
2695 struct acpi_device *adev;
2696
2697 if (dev->bus != &acpi_bus_type)
2698 continue;
2699
2700 adev= to_acpi_device(dev);
2701 mutex_lock(&adev->physical_node_lock);
2702 list_for_each_entry(pn, &adev->physical_node_list, node) {
2703 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2704 if (ret)
2705 break;
2706 }
2707 mutex_unlock(&adev->physical_node_lock);
2708 if (ret)
2709 return ret;
2710 }
2711
2712 return 0;
2713 }
2714
2715 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2716 {
2717 /*
2718 * Start from the sane iommu hardware state.
2719 * If the queued invalidation is already initialized by us
2720 * (for example, while enabling interrupt-remapping) then
2721 * we got the things already rolling from a sane state.
2722 */
2723 if (!iommu->qi) {
2724 /*
2725 * Clear any previous faults.
2726 */
2727 dmar_fault(-1, iommu);
2728 /*
2729 * Disable queued invalidation if supported and already enabled
2730 * before OS handover.
2731 */
2732 dmar_disable_qi(iommu);
2733 }
2734
2735 if (dmar_enable_qi(iommu)) {
2736 /*
2737 * Queued Invalidate not enabled, use Register Based Invalidate
2738 */
2739 iommu->flush.flush_context = __iommu_flush_context;
2740 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2741 pr_info("IOMMU: %s using Register based invalidation\n",
2742 iommu->name);
2743 } else {
2744 iommu->flush.flush_context = qi_flush_context;
2745 iommu->flush.flush_iotlb = qi_flush_iotlb;
2746 pr_info("IOMMU: %s using Queued invalidation\n", iommu->name);
2747 }
2748 }
2749
2750 static int __init init_dmars(void)
2751 {
2752 struct dmar_drhd_unit *drhd;
2753 struct dmar_rmrr_unit *rmrr;
2754 struct device *dev;
2755 struct intel_iommu *iommu;
2756 int i, ret;
2757
2758 /*
2759 * for each drhd
2760 * allocate root
2761 * initialize and program root entry to not present
2762 * endfor
2763 */
2764 for_each_drhd_unit(drhd) {
2765 /*
2766 * lock not needed as this is only incremented in the single
2767 * threaded kernel __init code path all other access are read
2768 * only
2769 */
2770 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
2771 g_num_of_iommus++;
2772 continue;
2773 }
2774 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2775 DMAR_UNITS_SUPPORTED);
2776 }
2777
2778 /* Preallocate enough resources for IOMMU hot-addition */
2779 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
2780 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
2781
2782 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2783 GFP_KERNEL);
2784 if (!g_iommus) {
2785 printk(KERN_ERR "Allocating global iommu array failed\n");
2786 ret = -ENOMEM;
2787 goto error;
2788 }
2789
2790 deferred_flush = kzalloc(g_num_of_iommus *
2791 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2792 if (!deferred_flush) {
2793 ret = -ENOMEM;
2794 goto free_g_iommus;
2795 }
2796
2797 for_each_active_iommu(iommu, drhd) {
2798 g_iommus[iommu->seq_id] = iommu;
2799
2800 ret = iommu_init_domains(iommu);
2801 if (ret)
2802 goto free_iommu;
2803
2804 /*
2805 * TBD:
2806 * we could share the same root & context tables
2807 * among all IOMMU's. Need to Split it later.
2808 */
2809 ret = iommu_alloc_root_entry(iommu);
2810 if (ret)
2811 goto free_iommu;
2812 if (!ecap_pass_through(iommu->ecap))
2813 hw_pass_through = 0;
2814 }
2815
2816 for_each_active_iommu(iommu, drhd)
2817 intel_iommu_init_qi(iommu);
2818
2819 if (iommu_pass_through)
2820 iommu_identity_mapping |= IDENTMAP_ALL;
2821
2822 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2823 iommu_identity_mapping |= IDENTMAP_GFX;
2824 #endif
2825
2826 check_tylersburg_isoch();
2827
2828 /*
2829 * If pass through is not set or not enabled, setup context entries for
2830 * identity mappings for rmrr, gfx, and isa and may fall back to static
2831 * identity mapping if iommu_identity_mapping is set.
2832 */
2833 if (iommu_identity_mapping) {
2834 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2835 if (ret) {
2836 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2837 goto free_iommu;
2838 }
2839 }
2840 /*
2841 * For each rmrr
2842 * for each dev attached to rmrr
2843 * do
2844 * locate drhd for dev, alloc domain for dev
2845 * allocate free domain
2846 * allocate page table entries for rmrr
2847 * if context not allocated for bus
2848 * allocate and init context
2849 * set present in root table for this bus
2850 * init context with domain, translation etc
2851 * endfor
2852 * endfor
2853 */
2854 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2855 for_each_rmrr_units(rmrr) {
2856 /* some BIOS lists non-exist devices in DMAR table. */
2857 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2858 i, dev) {
2859 ret = iommu_prepare_rmrr_dev(rmrr, dev);
2860 if (ret)
2861 printk(KERN_ERR
2862 "IOMMU: mapping reserved region failed\n");
2863 }
2864 }
2865
2866 iommu_prepare_isa();
2867
2868 /*
2869 * for each drhd
2870 * enable fault log
2871 * global invalidate context cache
2872 * global invalidate iotlb
2873 * enable translation
2874 */
2875 for_each_iommu(iommu, drhd) {
2876 if (drhd->ignored) {
2877 /*
2878 * we always have to disable PMRs or DMA may fail on
2879 * this device
2880 */
2881 if (force_on)
2882 iommu_disable_protect_mem_regions(iommu);
2883 continue;
2884 }
2885
2886 iommu_flush_write_buffer(iommu);
2887
2888 ret = dmar_set_interrupt(iommu);
2889 if (ret)
2890 goto free_iommu;
2891
2892 iommu_set_root_entry(iommu);
2893
2894 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2895 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2896 iommu_enable_translation(iommu);
2897 iommu_disable_protect_mem_regions(iommu);
2898 }
2899
2900 return 0;
2901
2902 free_iommu:
2903 for_each_active_iommu(iommu, drhd) {
2904 disable_dmar_iommu(iommu);
2905 free_dmar_iommu(iommu);
2906 }
2907 kfree(deferred_flush);
2908 free_g_iommus:
2909 kfree(g_iommus);
2910 error:
2911 return ret;
2912 }
2913
2914 /* This takes a number of _MM_ pages, not VTD pages */
2915 static struct iova *intel_alloc_iova(struct device *dev,
2916 struct dmar_domain *domain,
2917 unsigned long nrpages, uint64_t dma_mask)
2918 {
2919 struct iova *iova = NULL;
2920
2921 /* Restrict dma_mask to the width that the iommu can handle */
2922 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2923
2924 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2925 /*
2926 * First try to allocate an io virtual address in
2927 * DMA_BIT_MASK(32) and if that fails then try allocating
2928 * from higher range
2929 */
2930 iova = alloc_iova(&domain->iovad, nrpages,
2931 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2932 if (iova)
2933 return iova;
2934 }
2935 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2936 if (unlikely(!iova)) {
2937 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2938 nrpages, dev_name(dev));
2939 return NULL;
2940 }
2941
2942 return iova;
2943 }
2944
2945 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
2946 {
2947 struct dmar_domain *domain;
2948 int ret;
2949
2950 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2951 if (!domain) {
2952 printk(KERN_ERR "Allocating domain for %s failed",
2953 dev_name(dev));
2954 return NULL;
2955 }
2956
2957 /* make sure context mapping is ok */
2958 if (unlikely(!domain_context_mapped(dev))) {
2959 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2960 if (ret) {
2961 printk(KERN_ERR "Domain context map for %s failed",
2962 dev_name(dev));
2963 return NULL;
2964 }
2965 }
2966
2967 return domain;
2968 }
2969
2970 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
2971 {
2972 struct device_domain_info *info;
2973
2974 /* No lock here, assumes no domain exit in normal case */
2975 info = dev->archdata.iommu;
2976 if (likely(info))
2977 return info->domain;
2978
2979 return __get_valid_domain_for_dev(dev);
2980 }
2981
2982 static int iommu_dummy(struct device *dev)
2983 {
2984 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2985 }
2986
2987 /* Check if the dev needs to go through non-identity map and unmap process.*/
2988 static int iommu_no_mapping(struct device *dev)
2989 {
2990 int found;
2991
2992 if (iommu_dummy(dev))
2993 return 1;
2994
2995 if (!iommu_identity_mapping)
2996 return 0;
2997
2998 found = identity_mapping(dev);
2999 if (found) {
3000 if (iommu_should_identity_map(dev, 0))
3001 return 1;
3002 else {
3003 /*
3004 * 32 bit DMA is removed from si_domain and fall back
3005 * to non-identity mapping.
3006 */
3007 domain_remove_one_dev_info(si_domain, dev);
3008 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
3009 dev_name(dev));
3010 return 0;
3011 }
3012 } else {
3013 /*
3014 * In case of a detached 64 bit DMA device from vm, the device
3015 * is put into si_domain for identity mapping.
3016 */
3017 if (iommu_should_identity_map(dev, 0)) {
3018 int ret;
3019 ret = domain_add_dev_info(si_domain, dev,
3020 hw_pass_through ?
3021 CONTEXT_TT_PASS_THROUGH :
3022 CONTEXT_TT_MULTI_LEVEL);
3023 if (!ret) {
3024 printk(KERN_INFO "64bit %s uses identity mapping\n",
3025 dev_name(dev));
3026 return 1;
3027 }
3028 }
3029 }
3030
3031 return 0;
3032 }
3033
3034 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3035 size_t size, int dir, u64 dma_mask)
3036 {
3037 struct dmar_domain *domain;
3038 phys_addr_t start_paddr;
3039 struct iova *iova;
3040 int prot = 0;
3041 int ret;
3042 struct intel_iommu *iommu;
3043 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3044
3045 BUG_ON(dir == DMA_NONE);
3046
3047 if (iommu_no_mapping(dev))
3048 return paddr;
3049
3050 domain = get_valid_domain_for_dev(dev);
3051 if (!domain)
3052 return 0;
3053
3054 iommu = domain_get_iommu(domain);
3055 size = aligned_nrpages(paddr, size);
3056
3057 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3058 if (!iova)
3059 goto error;
3060
3061 /*
3062 * Check if DMAR supports zero-length reads on write only
3063 * mappings..
3064 */
3065 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3066 !cap_zlr(iommu->cap))
3067 prot |= DMA_PTE_READ;
3068 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3069 prot |= DMA_PTE_WRITE;
3070 /*
3071 * paddr - (paddr + size) might be partial page, we should map the whole
3072 * page. Note: if two part of one page are separately mapped, we
3073 * might have two guest_addr mapping to the same host paddr, but this
3074 * is not a big problem
3075 */
3076 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3077 mm_to_dma_pfn(paddr_pfn), size, prot);
3078 if (ret)
3079 goto error;
3080
3081 /* it's a non-present to present mapping. Only flush if caching mode */
3082 if (cap_caching_mode(iommu->cap))
3083 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
3084 else
3085 iommu_flush_write_buffer(iommu);
3086
3087 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3088 start_paddr += paddr & ~PAGE_MASK;
3089 return start_paddr;
3090
3091 error:
3092 if (iova)
3093 __free_iova(&domain->iovad, iova);
3094 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
3095 dev_name(dev), size, (unsigned long long)paddr, dir);
3096 return 0;
3097 }
3098
3099 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3100 unsigned long offset, size_t size,
3101 enum dma_data_direction dir,
3102 struct dma_attrs *attrs)
3103 {
3104 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3105 dir, *dev->dma_mask);
3106 }
3107
3108 static void flush_unmaps(void)
3109 {
3110 int i, j;
3111
3112 timer_on = 0;
3113
3114 /* just flush them all */
3115 for (i = 0; i < g_num_of_iommus; i++) {
3116 struct intel_iommu *iommu = g_iommus[i];
3117 if (!iommu)
3118 continue;
3119
3120 if (!deferred_flush[i].next)
3121 continue;
3122
3123 /* In caching mode, global flushes turn emulation expensive */
3124 if (!cap_caching_mode(iommu->cap))
3125 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3126 DMA_TLB_GLOBAL_FLUSH);
3127 for (j = 0; j < deferred_flush[i].next; j++) {
3128 unsigned long mask;
3129 struct iova *iova = deferred_flush[i].iova[j];
3130 struct dmar_domain *domain = deferred_flush[i].domain[j];
3131
3132 /* On real hardware multiple invalidations are expensive */
3133 if (cap_caching_mode(iommu->cap))
3134 iommu_flush_iotlb_psi(iommu, domain->id,
3135 iova->pfn_lo, iova_size(iova),
3136 !deferred_flush[i].freelist[j], 0);
3137 else {
3138 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3139 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3140 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3141 }
3142 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3143 if (deferred_flush[i].freelist[j])
3144 dma_free_pagelist(deferred_flush[i].freelist[j]);
3145 }
3146 deferred_flush[i].next = 0;
3147 }
3148
3149 list_size = 0;
3150 }
3151
3152 static void flush_unmaps_timeout(unsigned long data)
3153 {
3154 unsigned long flags;
3155
3156 spin_lock_irqsave(&async_umap_flush_lock, flags);
3157 flush_unmaps();
3158 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3159 }
3160
3161 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3162 {
3163 unsigned long flags;
3164 int next, iommu_id;
3165 struct intel_iommu *iommu;
3166
3167 spin_lock_irqsave(&async_umap_flush_lock, flags);
3168 if (list_size == HIGH_WATER_MARK)
3169 flush_unmaps();
3170
3171 iommu = domain_get_iommu(dom);
3172 iommu_id = iommu->seq_id;
3173
3174 next = deferred_flush[iommu_id].next;
3175 deferred_flush[iommu_id].domain[next] = dom;
3176 deferred_flush[iommu_id].iova[next] = iova;
3177 deferred_flush[iommu_id].freelist[next] = freelist;
3178 deferred_flush[iommu_id].next++;
3179
3180 if (!timer_on) {
3181 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3182 timer_on = 1;
3183 }
3184 list_size++;
3185 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3186 }
3187
3188 static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3189 {
3190 struct dmar_domain *domain;
3191 unsigned long start_pfn, last_pfn;
3192 struct iova *iova;
3193 struct intel_iommu *iommu;
3194 struct page *freelist;
3195
3196 if (iommu_no_mapping(dev))
3197 return;
3198
3199 domain = find_domain(dev);
3200 BUG_ON(!domain);
3201
3202 iommu = domain_get_iommu(domain);
3203
3204 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3205 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3206 (unsigned long long)dev_addr))
3207 return;
3208
3209 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3210 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3211
3212 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3213 dev_name(dev), start_pfn, last_pfn);
3214
3215 freelist = domain_unmap(domain, start_pfn, last_pfn);
3216
3217 if (intel_iommu_strict) {
3218 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3219 last_pfn - start_pfn + 1, !freelist, 0);
3220 /* free iova */
3221 __free_iova(&domain->iovad, iova);
3222 dma_free_pagelist(freelist);
3223 } else {
3224 add_unmap(domain, iova, freelist);
3225 /*
3226 * queue up the release of the unmap to save the 1/6th of the
3227 * cpu used up by the iotlb flush operation...
3228 */
3229 }
3230 }
3231
3232 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3233 size_t size, enum dma_data_direction dir,
3234 struct dma_attrs *attrs)
3235 {
3236 intel_unmap(dev, dev_addr);
3237 }
3238
3239 static void *intel_alloc_coherent(struct device *dev, size_t size,
3240 dma_addr_t *dma_handle, gfp_t flags,
3241 struct dma_attrs *attrs)
3242 {
3243 struct page *page = NULL;
3244 int order;
3245
3246 size = PAGE_ALIGN(size);
3247 order = get_order(size);
3248
3249 if (!iommu_no_mapping(dev))
3250 flags &= ~(GFP_DMA | GFP_DMA32);
3251 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3252 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3253 flags |= GFP_DMA;
3254 else
3255 flags |= GFP_DMA32;
3256 }
3257
3258 if (flags & __GFP_WAIT) {
3259 unsigned int count = size >> PAGE_SHIFT;
3260
3261 page = dma_alloc_from_contiguous(dev, count, order);
3262 if (page && iommu_no_mapping(dev) &&
3263 page_to_phys(page) + size > dev->coherent_dma_mask) {
3264 dma_release_from_contiguous(dev, page, count);
3265 page = NULL;
3266 }
3267 }
3268
3269 if (!page)
3270 page = alloc_pages(flags, order);
3271 if (!page)
3272 return NULL;
3273 memset(page_address(page), 0, size);
3274
3275 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3276 DMA_BIDIRECTIONAL,
3277 dev->coherent_dma_mask);
3278 if (*dma_handle)
3279 return page_address(page);
3280 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3281 __free_pages(page, order);
3282
3283 return NULL;
3284 }
3285
3286 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3287 dma_addr_t dma_handle, struct dma_attrs *attrs)
3288 {
3289 int order;
3290 struct page *page = virt_to_page(vaddr);
3291
3292 size = PAGE_ALIGN(size);
3293 order = get_order(size);
3294
3295 intel_unmap(dev, dma_handle);
3296 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3297 __free_pages(page, order);
3298 }
3299
3300 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3301 int nelems, enum dma_data_direction dir,
3302 struct dma_attrs *attrs)
3303 {
3304 intel_unmap(dev, sglist[0].dma_address);
3305 }
3306
3307 static int intel_nontranslate_map_sg(struct device *hddev,
3308 struct scatterlist *sglist, int nelems, int dir)
3309 {
3310 int i;
3311 struct scatterlist *sg;
3312
3313 for_each_sg(sglist, sg, nelems, i) {
3314 BUG_ON(!sg_page(sg));
3315 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3316 sg->dma_length = sg->length;
3317 }
3318 return nelems;
3319 }
3320
3321 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3322 enum dma_data_direction dir, struct dma_attrs *attrs)
3323 {
3324 int i;
3325 struct dmar_domain *domain;
3326 size_t size = 0;
3327 int prot = 0;
3328 struct iova *iova = NULL;
3329 int ret;
3330 struct scatterlist *sg;
3331 unsigned long start_vpfn;
3332 struct intel_iommu *iommu;
3333
3334 BUG_ON(dir == DMA_NONE);
3335 if (iommu_no_mapping(dev))
3336 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3337
3338 domain = get_valid_domain_for_dev(dev);
3339 if (!domain)
3340 return 0;
3341
3342 iommu = domain_get_iommu(domain);
3343
3344 for_each_sg(sglist, sg, nelems, i)
3345 size += aligned_nrpages(sg->offset, sg->length);
3346
3347 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3348 *dev->dma_mask);
3349 if (!iova) {
3350 sglist->dma_length = 0;
3351 return 0;
3352 }
3353
3354 /*
3355 * Check if DMAR supports zero-length reads on write only
3356 * mappings..
3357 */
3358 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3359 !cap_zlr(iommu->cap))
3360 prot |= DMA_PTE_READ;
3361 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3362 prot |= DMA_PTE_WRITE;
3363
3364 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3365
3366 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3367 if (unlikely(ret)) {
3368 dma_pte_free_pagetable(domain, start_vpfn,
3369 start_vpfn + size - 1);
3370 __free_iova(&domain->iovad, iova);
3371 return 0;
3372 }
3373
3374 /* it's a non-present to present mapping. Only flush if caching mode */
3375 if (cap_caching_mode(iommu->cap))
3376 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
3377 else
3378 iommu_flush_write_buffer(iommu);
3379
3380 return nelems;
3381 }
3382
3383 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3384 {
3385 return !dma_addr;
3386 }
3387
3388 struct dma_map_ops intel_dma_ops = {
3389 .alloc = intel_alloc_coherent,
3390 .free = intel_free_coherent,
3391 .map_sg = intel_map_sg,
3392 .unmap_sg = intel_unmap_sg,
3393 .map_page = intel_map_page,
3394 .unmap_page = intel_unmap_page,
3395 .mapping_error = intel_mapping_error,
3396 };
3397
3398 static inline int iommu_domain_cache_init(void)
3399 {
3400 int ret = 0;
3401
3402 iommu_domain_cache = kmem_cache_create("iommu_domain",
3403 sizeof(struct dmar_domain),
3404 0,
3405 SLAB_HWCACHE_ALIGN,
3406
3407 NULL);
3408 if (!iommu_domain_cache) {
3409 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3410 ret = -ENOMEM;
3411 }
3412
3413 return ret;
3414 }
3415
3416 static inline int iommu_devinfo_cache_init(void)
3417 {
3418 int ret = 0;
3419
3420 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3421 sizeof(struct device_domain_info),
3422 0,
3423 SLAB_HWCACHE_ALIGN,
3424 NULL);
3425 if (!iommu_devinfo_cache) {
3426 printk(KERN_ERR "Couldn't create devinfo cache\n");
3427 ret = -ENOMEM;
3428 }
3429
3430 return ret;
3431 }
3432
3433 static int __init iommu_init_mempool(void)
3434 {
3435 int ret;
3436 ret = iommu_iova_cache_init();
3437 if (ret)
3438 return ret;
3439
3440 ret = iommu_domain_cache_init();
3441 if (ret)
3442 goto domain_error;
3443
3444 ret = iommu_devinfo_cache_init();
3445 if (!ret)
3446 return ret;
3447
3448 kmem_cache_destroy(iommu_domain_cache);
3449 domain_error:
3450 iommu_iova_cache_destroy();
3451
3452 return -ENOMEM;
3453 }
3454
3455 static void __init iommu_exit_mempool(void)
3456 {
3457 kmem_cache_destroy(iommu_devinfo_cache);
3458 kmem_cache_destroy(iommu_domain_cache);
3459 iommu_iova_cache_destroy();
3460 }
3461
3462 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3463 {
3464 struct dmar_drhd_unit *drhd;
3465 u32 vtbar;
3466 int rc;
3467
3468 /* We know that this device on this chipset has its own IOMMU.
3469 * If we find it under a different IOMMU, then the BIOS is lying
3470 * to us. Hope that the IOMMU for this device is actually
3471 * disabled, and it needs no translation...
3472 */
3473 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3474 if (rc) {
3475 /* "can't" happen */
3476 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3477 return;
3478 }
3479 vtbar &= 0xffff0000;
3480
3481 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3482 drhd = dmar_find_matched_drhd_unit(pdev);
3483 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3484 TAINT_FIRMWARE_WORKAROUND,
3485 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3486 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3487 }
3488 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3489
3490 static void __init init_no_remapping_devices(void)
3491 {
3492 struct dmar_drhd_unit *drhd;
3493 struct device *dev;
3494 int i;
3495
3496 for_each_drhd_unit(drhd) {
3497 if (!drhd->include_all) {
3498 for_each_active_dev_scope(drhd->devices,
3499 drhd->devices_cnt, i, dev)
3500 break;
3501 /* ignore DMAR unit if no devices exist */
3502 if (i == drhd->devices_cnt)
3503 drhd->ignored = 1;
3504 }
3505 }
3506
3507 for_each_active_drhd_unit(drhd) {
3508 if (drhd->include_all)
3509 continue;
3510
3511 for_each_active_dev_scope(drhd->devices,
3512 drhd->devices_cnt, i, dev)
3513 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3514 break;
3515 if (i < drhd->devices_cnt)
3516 continue;
3517
3518 /* This IOMMU has *only* gfx devices. Either bypass it or
3519 set the gfx_mapped flag, as appropriate */
3520 if (dmar_map_gfx) {
3521 intel_iommu_gfx_mapped = 1;
3522 } else {
3523 drhd->ignored = 1;
3524 for_each_active_dev_scope(drhd->devices,
3525 drhd->devices_cnt, i, dev)
3526 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3527 }
3528 }
3529 }
3530
3531 #ifdef CONFIG_SUSPEND
3532 static int init_iommu_hw(void)
3533 {
3534 struct dmar_drhd_unit *drhd;
3535 struct intel_iommu *iommu = NULL;
3536
3537 for_each_active_iommu(iommu, drhd)
3538 if (iommu->qi)
3539 dmar_reenable_qi(iommu);
3540
3541 for_each_iommu(iommu, drhd) {
3542 if (drhd->ignored) {
3543 /*
3544 * we always have to disable PMRs or DMA may fail on
3545 * this device
3546 */
3547 if (force_on)
3548 iommu_disable_protect_mem_regions(iommu);
3549 continue;
3550 }
3551
3552 iommu_flush_write_buffer(iommu);
3553
3554 iommu_set_root_entry(iommu);
3555
3556 iommu->flush.flush_context(iommu, 0, 0, 0,
3557 DMA_CCMD_GLOBAL_INVL);
3558 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3559 iommu_enable_translation(iommu);
3560 iommu_disable_protect_mem_regions(iommu);
3561 }
3562
3563 return 0;
3564 }
3565
3566 static void iommu_flush_all(void)
3567 {
3568 struct dmar_drhd_unit *drhd;
3569 struct intel_iommu *iommu;
3570
3571 for_each_active_iommu(iommu, drhd) {
3572 iommu->flush.flush_context(iommu, 0, 0, 0,
3573 DMA_CCMD_GLOBAL_INVL);
3574 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3575 DMA_TLB_GLOBAL_FLUSH);
3576 }
3577 }
3578
3579 static int iommu_suspend(void)
3580 {
3581 struct dmar_drhd_unit *drhd;
3582 struct intel_iommu *iommu = NULL;
3583 unsigned long flag;
3584
3585 for_each_active_iommu(iommu, drhd) {
3586 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3587 GFP_ATOMIC);
3588 if (!iommu->iommu_state)
3589 goto nomem;
3590 }
3591
3592 iommu_flush_all();
3593
3594 for_each_active_iommu(iommu, drhd) {
3595 iommu_disable_translation(iommu);
3596
3597 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3598
3599 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3600 readl(iommu->reg + DMAR_FECTL_REG);
3601 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3602 readl(iommu->reg + DMAR_FEDATA_REG);
3603 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3604 readl(iommu->reg + DMAR_FEADDR_REG);
3605 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3606 readl(iommu->reg + DMAR_FEUADDR_REG);
3607
3608 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3609 }
3610 return 0;
3611
3612 nomem:
3613 for_each_active_iommu(iommu, drhd)
3614 kfree(iommu->iommu_state);
3615
3616 return -ENOMEM;
3617 }
3618
3619 static void iommu_resume(void)
3620 {
3621 struct dmar_drhd_unit *drhd;
3622 struct intel_iommu *iommu = NULL;
3623 unsigned long flag;
3624
3625 if (init_iommu_hw()) {
3626 if (force_on)
3627 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3628 else
3629 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3630 return;
3631 }
3632
3633 for_each_active_iommu(iommu, drhd) {
3634
3635 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3636
3637 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3638 iommu->reg + DMAR_FECTL_REG);
3639 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3640 iommu->reg + DMAR_FEDATA_REG);
3641 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3642 iommu->reg + DMAR_FEADDR_REG);
3643 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3644 iommu->reg + DMAR_FEUADDR_REG);
3645
3646 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3647 }
3648
3649 for_each_active_iommu(iommu, drhd)
3650 kfree(iommu->iommu_state);
3651 }
3652
3653 static struct syscore_ops iommu_syscore_ops = {
3654 .resume = iommu_resume,
3655 .suspend = iommu_suspend,
3656 };
3657
3658 static void __init init_iommu_pm_ops(void)
3659 {
3660 register_syscore_ops(&iommu_syscore_ops);
3661 }
3662
3663 #else
3664 static inline void init_iommu_pm_ops(void) {}
3665 #endif /* CONFIG_PM */
3666
3667
3668 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
3669 {
3670 struct acpi_dmar_reserved_memory *rmrr;
3671 struct dmar_rmrr_unit *rmrru;
3672
3673 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3674 if (!rmrru)
3675 return -ENOMEM;
3676
3677 rmrru->hdr = header;
3678 rmrr = (struct acpi_dmar_reserved_memory *)header;
3679 rmrru->base_address = rmrr->base_address;
3680 rmrru->end_address = rmrr->end_address;
3681 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3682 ((void *)rmrr) + rmrr->header.length,
3683 &rmrru->devices_cnt);
3684 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3685 kfree(rmrru);
3686 return -ENOMEM;
3687 }
3688
3689 list_add(&rmrru->list, &dmar_rmrr_units);
3690
3691 return 0;
3692 }
3693
3694 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3695 {
3696 struct dmar_atsr_unit *atsru;
3697 struct acpi_dmar_atsr *tmp;
3698
3699 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3700 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3701 if (atsr->segment != tmp->segment)
3702 continue;
3703 if (atsr->header.length != tmp->header.length)
3704 continue;
3705 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3706 return atsru;
3707 }
3708
3709 return NULL;
3710 }
3711
3712 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3713 {
3714 struct acpi_dmar_atsr *atsr;
3715 struct dmar_atsr_unit *atsru;
3716
3717 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
3718 return 0;
3719
3720 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3721 atsru = dmar_find_atsr(atsr);
3722 if (atsru)
3723 return 0;
3724
3725 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
3726 if (!atsru)
3727 return -ENOMEM;
3728
3729 /*
3730 * If memory is allocated from slab by ACPI _DSM method, we need to
3731 * copy the memory content because the memory buffer will be freed
3732 * on return.
3733 */
3734 atsru->hdr = (void *)(atsru + 1);
3735 memcpy(atsru->hdr, hdr, hdr->length);
3736 atsru->include_all = atsr->flags & 0x1;
3737 if (!atsru->include_all) {
3738 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3739 (void *)atsr + atsr->header.length,
3740 &atsru->devices_cnt);
3741 if (atsru->devices_cnt && atsru->devices == NULL) {
3742 kfree(atsru);
3743 return -ENOMEM;
3744 }
3745 }
3746
3747 list_add_rcu(&atsru->list, &dmar_atsr_units);
3748
3749 return 0;
3750 }
3751
3752 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3753 {
3754 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3755 kfree(atsru);
3756 }
3757
3758 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3759 {
3760 struct acpi_dmar_atsr *atsr;
3761 struct dmar_atsr_unit *atsru;
3762
3763 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3764 atsru = dmar_find_atsr(atsr);
3765 if (atsru) {
3766 list_del_rcu(&atsru->list);
3767 synchronize_rcu();
3768 intel_iommu_free_atsr(atsru);
3769 }
3770
3771 return 0;
3772 }
3773
3774 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3775 {
3776 int i;
3777 struct device *dev;
3778 struct acpi_dmar_atsr *atsr;
3779 struct dmar_atsr_unit *atsru;
3780
3781 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3782 atsru = dmar_find_atsr(atsr);
3783 if (!atsru)
3784 return 0;
3785
3786 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
3787 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
3788 i, dev)
3789 return -EBUSY;
3790
3791 return 0;
3792 }
3793
3794 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3795 {
3796 int sp, ret = 0;
3797 struct intel_iommu *iommu = dmaru->iommu;
3798
3799 if (g_iommus[iommu->seq_id])
3800 return 0;
3801
3802 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
3803 pr_warn("IOMMU: %s doesn't support hardware pass through.\n",
3804 iommu->name);
3805 return -ENXIO;
3806 }
3807 if (!ecap_sc_support(iommu->ecap) &&
3808 domain_update_iommu_snooping(iommu)) {
3809 pr_warn("IOMMU: %s doesn't support snooping.\n",
3810 iommu->name);
3811 return -ENXIO;
3812 }
3813 sp = domain_update_iommu_superpage(iommu) - 1;
3814 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
3815 pr_warn("IOMMU: %s doesn't support large page.\n",
3816 iommu->name);
3817 return -ENXIO;
3818 }
3819
3820 /*
3821 * Disable translation if already enabled prior to OS handover.
3822 */
3823 if (iommu->gcmd & DMA_GCMD_TE)
3824 iommu_disable_translation(iommu);
3825
3826 g_iommus[iommu->seq_id] = iommu;
3827 ret = iommu_init_domains(iommu);
3828 if (ret == 0)
3829 ret = iommu_alloc_root_entry(iommu);
3830 if (ret)
3831 goto out;
3832
3833 if (dmaru->ignored) {
3834 /*
3835 * we always have to disable PMRs or DMA may fail on this device
3836 */
3837 if (force_on)
3838 iommu_disable_protect_mem_regions(iommu);
3839 return 0;
3840 }
3841
3842 intel_iommu_init_qi(iommu);
3843 iommu_flush_write_buffer(iommu);
3844 ret = dmar_set_interrupt(iommu);
3845 if (ret)
3846 goto disable_iommu;
3847
3848 iommu_set_root_entry(iommu);
3849 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3850 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3851 iommu_enable_translation(iommu);
3852
3853 if (si_domain) {
3854 ret = iommu_attach_domain(si_domain, iommu);
3855 if (ret < 0 || si_domain->id != ret)
3856 goto disable_iommu;
3857 domain_attach_iommu(si_domain, iommu);
3858 }
3859
3860 iommu_disable_protect_mem_regions(iommu);
3861 return 0;
3862
3863 disable_iommu:
3864 disable_dmar_iommu(iommu);
3865 out:
3866 free_dmar_iommu(iommu);
3867 return ret;
3868 }
3869
3870 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
3871 {
3872 int ret = 0;
3873 struct intel_iommu *iommu = dmaru->iommu;
3874
3875 if (!intel_iommu_enabled)
3876 return 0;
3877 if (iommu == NULL)
3878 return -EINVAL;
3879
3880 if (insert) {
3881 ret = intel_iommu_add(dmaru);
3882 } else {
3883 disable_dmar_iommu(iommu);
3884 free_dmar_iommu(iommu);
3885 }
3886
3887 return ret;
3888 }
3889
3890 static void intel_iommu_free_dmars(void)
3891 {
3892 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3893 struct dmar_atsr_unit *atsru, *atsr_n;
3894
3895 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3896 list_del(&rmrru->list);
3897 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3898 kfree(rmrru);
3899 }
3900
3901 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3902 list_del(&atsru->list);
3903 intel_iommu_free_atsr(atsru);
3904 }
3905 }
3906
3907 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3908 {
3909 int i, ret = 1;
3910 struct pci_bus *bus;
3911 struct pci_dev *bridge = NULL;
3912 struct device *tmp;
3913 struct acpi_dmar_atsr *atsr;
3914 struct dmar_atsr_unit *atsru;
3915
3916 dev = pci_physfn(dev);
3917 for (bus = dev->bus; bus; bus = bus->parent) {
3918 bridge = bus->self;
3919 if (!bridge || !pci_is_pcie(bridge) ||
3920 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3921 return 0;
3922 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3923 break;
3924 }
3925 if (!bridge)
3926 return 0;
3927
3928 rcu_read_lock();
3929 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3930 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3931 if (atsr->segment != pci_domain_nr(dev->bus))
3932 continue;
3933
3934 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3935 if (tmp == &bridge->dev)
3936 goto out;
3937
3938 if (atsru->include_all)
3939 goto out;
3940 }
3941 ret = 0;
3942 out:
3943 rcu_read_unlock();
3944
3945 return ret;
3946 }
3947
3948 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3949 {
3950 int ret = 0;
3951 struct dmar_rmrr_unit *rmrru;
3952 struct dmar_atsr_unit *atsru;
3953 struct acpi_dmar_atsr *atsr;
3954 struct acpi_dmar_reserved_memory *rmrr;
3955
3956 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3957 return 0;
3958
3959 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3960 rmrr = container_of(rmrru->hdr,
3961 struct acpi_dmar_reserved_memory, header);
3962 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3963 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3964 ((void *)rmrr) + rmrr->header.length,
3965 rmrr->segment, rmrru->devices,
3966 rmrru->devices_cnt);
3967 if(ret < 0)
3968 return ret;
3969 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3970 dmar_remove_dev_scope(info, rmrr->segment,
3971 rmrru->devices, rmrru->devices_cnt);
3972 }
3973 }
3974
3975 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3976 if (atsru->include_all)
3977 continue;
3978
3979 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3980 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3981 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3982 (void *)atsr + atsr->header.length,
3983 atsr->segment, atsru->devices,
3984 atsru->devices_cnt);
3985 if (ret > 0)
3986 break;
3987 else if(ret < 0)
3988 return ret;
3989 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3990 if (dmar_remove_dev_scope(info, atsr->segment,
3991 atsru->devices, atsru->devices_cnt))
3992 break;
3993 }
3994 }
3995
3996 return 0;
3997 }
3998
3999 /*
4000 * Here we only respond to action of unbound device from driver.
4001 *
4002 * Added device is not attached to its DMAR domain here yet. That will happen
4003 * when mapping the device to iova.
4004 */
4005 static int device_notifier(struct notifier_block *nb,
4006 unsigned long action, void *data)
4007 {
4008 struct device *dev = data;
4009 struct dmar_domain *domain;
4010
4011 if (iommu_dummy(dev))
4012 return 0;
4013
4014 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4015 return 0;
4016
4017 domain = find_domain(dev);
4018 if (!domain)
4019 return 0;
4020
4021 down_read(&dmar_global_lock);
4022 domain_remove_one_dev_info(domain, dev);
4023 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4024 domain_exit(domain);
4025 up_read(&dmar_global_lock);
4026
4027 return 0;
4028 }
4029
4030 static struct notifier_block device_nb = {
4031 .notifier_call = device_notifier,
4032 };
4033
4034 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4035 unsigned long val, void *v)
4036 {
4037 struct memory_notify *mhp = v;
4038 unsigned long long start, end;
4039 unsigned long start_vpfn, last_vpfn;
4040
4041 switch (val) {
4042 case MEM_GOING_ONLINE:
4043 start = mhp->start_pfn << PAGE_SHIFT;
4044 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4045 if (iommu_domain_identity_map(si_domain, start, end)) {
4046 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
4047 start, end);
4048 return NOTIFY_BAD;
4049 }
4050 break;
4051
4052 case MEM_OFFLINE:
4053 case MEM_CANCEL_ONLINE:
4054 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4055 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4056 while (start_vpfn <= last_vpfn) {
4057 struct iova *iova;
4058 struct dmar_drhd_unit *drhd;
4059 struct intel_iommu *iommu;
4060 struct page *freelist;
4061
4062 iova = find_iova(&si_domain->iovad, start_vpfn);
4063 if (iova == NULL) {
4064 pr_debug("dmar: failed get IOVA for PFN %lx\n",
4065 start_vpfn);
4066 break;
4067 }
4068
4069 iova = split_and_remove_iova(&si_domain->iovad, iova,
4070 start_vpfn, last_vpfn);
4071 if (iova == NULL) {
4072 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
4073 start_vpfn, last_vpfn);
4074 return NOTIFY_BAD;
4075 }
4076
4077 freelist = domain_unmap(si_domain, iova->pfn_lo,
4078 iova->pfn_hi);
4079
4080 rcu_read_lock();
4081 for_each_active_iommu(iommu, drhd)
4082 iommu_flush_iotlb_psi(iommu, si_domain->id,
4083 iova->pfn_lo, iova_size(iova),
4084 !freelist, 0);
4085 rcu_read_unlock();
4086 dma_free_pagelist(freelist);
4087
4088 start_vpfn = iova->pfn_hi + 1;
4089 free_iova_mem(iova);
4090 }
4091 break;
4092 }
4093
4094 return NOTIFY_OK;
4095 }
4096
4097 static struct notifier_block intel_iommu_memory_nb = {
4098 .notifier_call = intel_iommu_memory_notifier,
4099 .priority = 0
4100 };
4101
4102
4103 static ssize_t intel_iommu_show_version(struct device *dev,
4104 struct device_attribute *attr,
4105 char *buf)
4106 {
4107 struct intel_iommu *iommu = dev_get_drvdata(dev);
4108 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4109 return sprintf(buf, "%d:%d\n",
4110 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4111 }
4112 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4113
4114 static ssize_t intel_iommu_show_address(struct device *dev,
4115 struct device_attribute *attr,
4116 char *buf)
4117 {
4118 struct intel_iommu *iommu = dev_get_drvdata(dev);
4119 return sprintf(buf, "%llx\n", iommu->reg_phys);
4120 }
4121 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4122
4123 static ssize_t intel_iommu_show_cap(struct device *dev,
4124 struct device_attribute *attr,
4125 char *buf)
4126 {
4127 struct intel_iommu *iommu = dev_get_drvdata(dev);
4128 return sprintf(buf, "%llx\n", iommu->cap);
4129 }
4130 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4131
4132 static ssize_t intel_iommu_show_ecap(struct device *dev,
4133 struct device_attribute *attr,
4134 char *buf)
4135 {
4136 struct intel_iommu *iommu = dev_get_drvdata(dev);
4137 return sprintf(buf, "%llx\n", iommu->ecap);
4138 }
4139 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4140
4141 static struct attribute *intel_iommu_attrs[] = {
4142 &dev_attr_version.attr,
4143 &dev_attr_address.attr,
4144 &dev_attr_cap.attr,
4145 &dev_attr_ecap.attr,
4146 NULL,
4147 };
4148
4149 static struct attribute_group intel_iommu_group = {
4150 .name = "intel-iommu",
4151 .attrs = intel_iommu_attrs,
4152 };
4153
4154 const struct attribute_group *intel_iommu_groups[] = {
4155 &intel_iommu_group,
4156 NULL,
4157 };
4158
4159 int __init intel_iommu_init(void)
4160 {
4161 int ret = -ENODEV;
4162 struct dmar_drhd_unit *drhd;
4163 struct intel_iommu *iommu;
4164
4165 /* VT-d is required for a TXT/tboot launch, so enforce that */
4166 force_on = tboot_force_iommu();
4167
4168 if (iommu_init_mempool()) {
4169 if (force_on)
4170 panic("tboot: Failed to initialize iommu memory\n");
4171 return -ENOMEM;
4172 }
4173
4174 down_write(&dmar_global_lock);
4175 if (dmar_table_init()) {
4176 if (force_on)
4177 panic("tboot: Failed to initialize DMAR table\n");
4178 goto out_free_dmar;
4179 }
4180
4181 /*
4182 * Disable translation if already enabled prior to OS handover.
4183 */
4184 for_each_active_iommu(iommu, drhd)
4185 if (iommu->gcmd & DMA_GCMD_TE)
4186 iommu_disable_translation(iommu);
4187
4188 if (dmar_dev_scope_init() < 0) {
4189 if (force_on)
4190 panic("tboot: Failed to initialize DMAR device scope\n");
4191 goto out_free_dmar;
4192 }
4193
4194 if (no_iommu || dmar_disabled)
4195 goto out_free_dmar;
4196
4197 if (list_empty(&dmar_rmrr_units))
4198 printk(KERN_INFO "DMAR: No RMRR found\n");
4199
4200 if (list_empty(&dmar_atsr_units))
4201 printk(KERN_INFO "DMAR: No ATSR found\n");
4202
4203 if (dmar_init_reserved_ranges()) {
4204 if (force_on)
4205 panic("tboot: Failed to reserve iommu ranges\n");
4206 goto out_free_reserved_range;
4207 }
4208
4209 init_no_remapping_devices();
4210
4211 ret = init_dmars();
4212 if (ret) {
4213 if (force_on)
4214 panic("tboot: Failed to initialize DMARs\n");
4215 printk(KERN_ERR "IOMMU: dmar init failed\n");
4216 goto out_free_reserved_range;
4217 }
4218 up_write(&dmar_global_lock);
4219 printk(KERN_INFO
4220 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4221
4222 init_timer(&unmap_timer);
4223 #ifdef CONFIG_SWIOTLB
4224 swiotlb = 0;
4225 #endif
4226 dma_ops = &intel_dma_ops;
4227
4228 init_iommu_pm_ops();
4229
4230 for_each_active_iommu(iommu, drhd)
4231 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4232 intel_iommu_groups,
4233 iommu->name);
4234
4235 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4236 bus_register_notifier(&pci_bus_type, &device_nb);
4237 if (si_domain && !hw_pass_through)
4238 register_memory_notifier(&intel_iommu_memory_nb);
4239
4240 intel_iommu_enabled = 1;
4241
4242 return 0;
4243
4244 out_free_reserved_range:
4245 put_iova_domain(&reserved_iova_list);
4246 out_free_dmar:
4247 intel_iommu_free_dmars();
4248 up_write(&dmar_global_lock);
4249 iommu_exit_mempool();
4250 return ret;
4251 }
4252
4253 static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4254 {
4255 struct intel_iommu *iommu = opaque;
4256
4257 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4258 return 0;
4259 }
4260
4261 /*
4262 * NB - intel-iommu lacks any sort of reference counting for the users of
4263 * dependent devices. If multiple endpoints have intersecting dependent
4264 * devices, unbinding the driver from any one of them will possibly leave
4265 * the others unable to operate.
4266 */
4267 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4268 struct device *dev)
4269 {
4270 if (!iommu || !dev || !dev_is_pci(dev))
4271 return;
4272
4273 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
4274 }
4275
4276 static void domain_remove_one_dev_info(struct dmar_domain *domain,
4277 struct device *dev)
4278 {
4279 struct device_domain_info *info, *tmp;
4280 struct intel_iommu *iommu;
4281 unsigned long flags;
4282 bool found = false;
4283 u8 bus, devfn;
4284
4285 iommu = device_to_iommu(dev, &bus, &devfn);
4286 if (!iommu)
4287 return;
4288
4289 spin_lock_irqsave(&device_domain_lock, flags);
4290 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4291 if (info->iommu == iommu && info->bus == bus &&
4292 info->devfn == devfn) {
4293 unlink_domain_info(info);
4294 spin_unlock_irqrestore(&device_domain_lock, flags);
4295
4296 iommu_disable_dev_iotlb(info);
4297 iommu_detach_dev(iommu, info->bus, info->devfn);
4298 iommu_detach_dependent_devices(iommu, dev);
4299 free_devinfo_mem(info);
4300
4301 spin_lock_irqsave(&device_domain_lock, flags);
4302
4303 if (found)
4304 break;
4305 else
4306 continue;
4307 }
4308
4309 /* if there is no other devices under the same iommu
4310 * owned by this domain, clear this iommu in iommu_bmp
4311 * update iommu count and coherency
4312 */
4313 if (info->iommu == iommu)
4314 found = true;
4315 }
4316
4317 spin_unlock_irqrestore(&device_domain_lock, flags);
4318
4319 if (found == 0) {
4320 domain_detach_iommu(domain, iommu);
4321 if (!domain_type_is_vm_or_si(domain))
4322 iommu_detach_domain(domain, iommu);
4323 }
4324 }
4325
4326 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4327 {
4328 int adjust_width;
4329
4330 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4331 DMA_32BIT_PFN);
4332 domain_reserve_special_ranges(domain);
4333
4334 /* calculate AGAW */
4335 domain->gaw = guest_width;
4336 adjust_width = guestwidth_to_adjustwidth(guest_width);
4337 domain->agaw = width_to_agaw(adjust_width);
4338
4339 domain->iommu_coherency = 0;
4340 domain->iommu_snooping = 0;
4341 domain->iommu_superpage = 0;
4342 domain->max_addr = 0;
4343
4344 /* always allocate the top pgd */
4345 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4346 if (!domain->pgd)
4347 return -ENOMEM;
4348 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4349 return 0;
4350 }
4351
4352 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4353 {
4354 struct dmar_domain *dmar_domain;
4355 struct iommu_domain *domain;
4356
4357 if (type != IOMMU_DOMAIN_UNMANAGED)
4358 return NULL;
4359
4360 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4361 if (!dmar_domain) {
4362 printk(KERN_ERR
4363 "intel_iommu_domain_init: dmar_domain == NULL\n");
4364 return NULL;
4365 }
4366 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4367 printk(KERN_ERR
4368 "intel_iommu_domain_init() failed\n");
4369 domain_exit(dmar_domain);
4370 return NULL;
4371 }
4372 domain_update_iommu_cap(dmar_domain);
4373
4374 domain = &dmar_domain->domain;
4375 domain->geometry.aperture_start = 0;
4376 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4377 domain->geometry.force_aperture = true;
4378
4379 return domain;
4380 }
4381
4382 static void intel_iommu_domain_free(struct iommu_domain *domain)
4383 {
4384 domain_exit(to_dmar_domain(domain));
4385 }
4386
4387 static int intel_iommu_attach_device(struct iommu_domain *domain,
4388 struct device *dev)
4389 {
4390 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4391 struct intel_iommu *iommu;
4392 int addr_width;
4393 u8 bus, devfn;
4394
4395 if (device_is_rmrr_locked(dev)) {
4396 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4397 return -EPERM;
4398 }
4399
4400 /* normally dev is not mapped */
4401 if (unlikely(domain_context_mapped(dev))) {
4402 struct dmar_domain *old_domain;
4403
4404 old_domain = find_domain(dev);
4405 if (old_domain) {
4406 if (domain_type_is_vm_or_si(dmar_domain))
4407 domain_remove_one_dev_info(old_domain, dev);
4408 else
4409 domain_remove_dev_info(old_domain);
4410
4411 if (!domain_type_is_vm_or_si(old_domain) &&
4412 list_empty(&old_domain->devices))
4413 domain_exit(old_domain);
4414 }
4415 }
4416
4417 iommu = device_to_iommu(dev, &bus, &devfn);
4418 if (!iommu)
4419 return -ENODEV;
4420
4421 /* check if this iommu agaw is sufficient for max mapped address */
4422 addr_width = agaw_to_width(iommu->agaw);
4423 if (addr_width > cap_mgaw(iommu->cap))
4424 addr_width = cap_mgaw(iommu->cap);
4425
4426 if (dmar_domain->max_addr > (1LL << addr_width)) {
4427 printk(KERN_ERR "%s: iommu width (%d) is not "
4428 "sufficient for the mapped address (%llx)\n",
4429 __func__, addr_width, dmar_domain->max_addr);
4430 return -EFAULT;
4431 }
4432 dmar_domain->gaw = addr_width;
4433
4434 /*
4435 * Knock out extra levels of page tables if necessary
4436 */
4437 while (iommu->agaw < dmar_domain->agaw) {
4438 struct dma_pte *pte;
4439
4440 pte = dmar_domain->pgd;
4441 if (dma_pte_present(pte)) {
4442 dmar_domain->pgd = (struct dma_pte *)
4443 phys_to_virt(dma_pte_addr(pte));
4444 free_pgtable_page(pte);
4445 }
4446 dmar_domain->agaw--;
4447 }
4448
4449 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
4450 }
4451
4452 static void intel_iommu_detach_device(struct iommu_domain *domain,
4453 struct device *dev)
4454 {
4455 domain_remove_one_dev_info(to_dmar_domain(domain), dev);
4456 }
4457
4458 static int intel_iommu_map(struct iommu_domain *domain,
4459 unsigned long iova, phys_addr_t hpa,
4460 size_t size, int iommu_prot)
4461 {
4462 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4463 u64 max_addr;
4464 int prot = 0;
4465 int ret;
4466
4467 if (iommu_prot & IOMMU_READ)
4468 prot |= DMA_PTE_READ;
4469 if (iommu_prot & IOMMU_WRITE)
4470 prot |= DMA_PTE_WRITE;
4471 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4472 prot |= DMA_PTE_SNP;
4473
4474 max_addr = iova + size;
4475 if (dmar_domain->max_addr < max_addr) {
4476 u64 end;
4477
4478 /* check if minimum agaw is sufficient for mapped address */
4479 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4480 if (end < max_addr) {
4481 printk(KERN_ERR "%s: iommu width (%d) is not "
4482 "sufficient for the mapped address (%llx)\n",
4483 __func__, dmar_domain->gaw, max_addr);
4484 return -EFAULT;
4485 }
4486 dmar_domain->max_addr = max_addr;
4487 }
4488 /* Round up size to next multiple of PAGE_SIZE, if it and
4489 the low bits of hpa would take us onto the next page */
4490 size = aligned_nrpages(hpa, size);
4491 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4492 hpa >> VTD_PAGE_SHIFT, size, prot);
4493 return ret;
4494 }
4495
4496 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4497 unsigned long iova, size_t size)
4498 {
4499 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4500 struct page *freelist = NULL;
4501 struct intel_iommu *iommu;
4502 unsigned long start_pfn, last_pfn;
4503 unsigned int npages;
4504 int iommu_id, num, ndomains, level = 0;
4505
4506 /* Cope with horrid API which requires us to unmap more than the
4507 size argument if it happens to be a large-page mapping. */
4508 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4509 BUG();
4510
4511 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4512 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4513
4514 start_pfn = iova >> VTD_PAGE_SHIFT;
4515 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4516
4517 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4518
4519 npages = last_pfn - start_pfn + 1;
4520
4521 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4522 iommu = g_iommus[iommu_id];
4523
4524 /*
4525 * find bit position of dmar_domain
4526 */
4527 ndomains = cap_ndoms(iommu->cap);
4528 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4529 if (iommu->domains[num] == dmar_domain)
4530 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4531 npages, !freelist, 0);
4532 }
4533
4534 }
4535
4536 dma_free_pagelist(freelist);
4537
4538 if (dmar_domain->max_addr == iova + size)
4539 dmar_domain->max_addr = iova;
4540
4541 return size;
4542 }
4543
4544 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4545 dma_addr_t iova)
4546 {
4547 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4548 struct dma_pte *pte;
4549 int level = 0;
4550 u64 phys = 0;
4551
4552 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4553 if (pte)
4554 phys = dma_pte_addr(pte);
4555
4556 return phys;
4557 }
4558
4559 static bool intel_iommu_capable(enum iommu_cap cap)
4560 {
4561 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4562 return domain_update_iommu_snooping(NULL) == 1;
4563 if (cap == IOMMU_CAP_INTR_REMAP)
4564 return irq_remapping_enabled == 1;
4565
4566 return false;
4567 }
4568
4569 static int intel_iommu_add_device(struct device *dev)
4570 {
4571 struct intel_iommu *iommu;
4572 struct iommu_group *group;
4573 u8 bus, devfn;
4574
4575 iommu = device_to_iommu(dev, &bus, &devfn);
4576 if (!iommu)
4577 return -ENODEV;
4578
4579 iommu_device_link(iommu->iommu_dev, dev);
4580
4581 group = iommu_group_get_for_dev(dev);
4582
4583 if (IS_ERR(group))
4584 return PTR_ERR(group);
4585
4586 iommu_group_put(group);
4587 return 0;
4588 }
4589
4590 static void intel_iommu_remove_device(struct device *dev)
4591 {
4592 struct intel_iommu *iommu;
4593 u8 bus, devfn;
4594
4595 iommu = device_to_iommu(dev, &bus, &devfn);
4596 if (!iommu)
4597 return;
4598
4599 iommu_group_remove_device(dev);
4600
4601 iommu_device_unlink(iommu->iommu_dev, dev);
4602 }
4603
4604 static const struct iommu_ops intel_iommu_ops = {
4605 .capable = intel_iommu_capable,
4606 .domain_alloc = intel_iommu_domain_alloc,
4607 .domain_free = intel_iommu_domain_free,
4608 .attach_dev = intel_iommu_attach_device,
4609 .detach_dev = intel_iommu_detach_device,
4610 .map = intel_iommu_map,
4611 .unmap = intel_iommu_unmap,
4612 .map_sg = default_iommu_map_sg,
4613 .iova_to_phys = intel_iommu_iova_to_phys,
4614 .add_device = intel_iommu_add_device,
4615 .remove_device = intel_iommu_remove_device,
4616 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
4617 };
4618
4619 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4620 {
4621 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4622 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4623 dmar_map_gfx = 0;
4624 }
4625
4626 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4627 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4628 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4629 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4630 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4631 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4632 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4633
4634 static void quirk_iommu_rwbf(struct pci_dev *dev)
4635 {
4636 /*
4637 * Mobile 4 Series Chipset neglects to set RWBF capability,
4638 * but needs it. Same seems to hold for the desktop versions.
4639 */
4640 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4641 rwbf_quirk = 1;
4642 }
4643
4644 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4645 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4646 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4647 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4648 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4649 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4650 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4651
4652 #define GGC 0x52
4653 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
4654 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4655 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
4656 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
4657 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4658 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4659 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4660 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4661
4662 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4663 {
4664 unsigned short ggc;
4665
4666 if (pci_read_config_word(dev, GGC, &ggc))
4667 return;
4668
4669 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4670 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4671 dmar_map_gfx = 0;
4672 } else if (dmar_map_gfx) {
4673 /* we have to ensure the gfx device is idle before we flush */
4674 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4675 intel_iommu_strict = 1;
4676 }
4677 }
4678 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4679 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4680 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4681 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4682
4683 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4684 ISOCH DMAR unit for the Azalia sound device, but not give it any
4685 TLB entries, which causes it to deadlock. Check for that. We do
4686 this in a function called from init_dmars(), instead of in a PCI
4687 quirk, because we don't want to print the obnoxious "BIOS broken"
4688 message if VT-d is actually disabled.
4689 */
4690 static void __init check_tylersburg_isoch(void)
4691 {
4692 struct pci_dev *pdev;
4693 uint32_t vtisochctrl;
4694
4695 /* If there's no Azalia in the system anyway, forget it. */
4696 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4697 if (!pdev)
4698 return;
4699 pci_dev_put(pdev);
4700
4701 /* System Management Registers. Might be hidden, in which case
4702 we can't do the sanity check. But that's OK, because the
4703 known-broken BIOSes _don't_ actually hide it, so far. */
4704 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4705 if (!pdev)
4706 return;
4707
4708 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4709 pci_dev_put(pdev);
4710 return;
4711 }
4712
4713 pci_dev_put(pdev);
4714
4715 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4716 if (vtisochctrl & 1)
4717 return;
4718
4719 /* Drop all bits other than the number of TLB entries */
4720 vtisochctrl &= 0x1c;
4721
4722 /* If we have the recommended number of TLB entries (16), fine. */
4723 if (vtisochctrl == 0x10)
4724 return;
4725
4726 /* Zero TLB entries? You get to ride the short bus to school. */
4727 if (!vtisochctrl) {
4728 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4729 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4730 dmi_get_system_info(DMI_BIOS_VENDOR),
4731 dmi_get_system_info(DMI_BIOS_VERSION),
4732 dmi_get_system_info(DMI_PRODUCT_VERSION));
4733 iommu_identity_mapping |= IDENTMAP_AZALIA;
4734 return;
4735 }
4736
4737 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4738 vtisochctrl);
4739 }
Linux kernel - Deliverables
This page took 0.169263 seconds and 5 git commands to generate.