projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
iommu/vt-d: Work around broken RMRR firmware entries
[deliverable/linux.git] / drivers / iommu / dmar.c
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
30
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <linux/iommu.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/iommu_table.h>
44
45 #include "irq_remapping.h"
46
47 /*
48 * Assumptions:
49 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
50 * before IO devices managed by that unit.
51 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
52 * after IO devices managed by that unit.
53 * 3) Hotplug events are rare.
54 *
55 * Locking rules for DMA and interrupt remapping related global data structures:
56 * 1) Use dmar_global_lock in process context
57 * 2) Use RCU in interrupt context
58 */
59 DECLARE_RWSEM(dmar_global_lock);
60 LIST_HEAD(dmar_drhd_units);
61
62 struct acpi_table_header * __initdata dmar_tbl;
63 static acpi_size dmar_tbl_size;
64 static int dmar_dev_scope_status = 1;
65
66 static int alloc_iommu(struct dmar_drhd_unit *drhd);
67 static void free_iommu(struct intel_iommu *iommu);
68
69 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
70 {
71 /*
72 * add INCLUDE_ALL at the tail, so scan the list will find it at
73 * the very end.
74 */
75 if (drhd->include_all)
76 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
77 else
78 list_add_rcu(&drhd->list, &dmar_drhd_units);
79 }
80
81 void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
82 {
83 struct acpi_dmar_device_scope *scope;
84
85 *cnt = 0;
86 while (start < end) {
87 scope = start;
88 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
89 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
90 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
91 (*cnt)++;
92 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
93 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
94 pr_warn("Unsupported device scope\n");
95 }
96 start += scope->length;
97 }
98 if (*cnt == 0)
99 return NULL;
100
101 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
102 }
103
104 void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
105 {
106 int i;
107 struct device *tmp_dev;
108
109 if (*devices && *cnt) {
110 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
111 put_device(tmp_dev);
112 kfree(*devices);
113 }
114
115 *devices = NULL;
116 *cnt = 0;
117 }
118
119 /* Optimize out kzalloc()/kfree() for normal cases */
120 static char dmar_pci_notify_info_buf[64];
121
122 static struct dmar_pci_notify_info *
123 dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
124 {
125 int level = 0;
126 size_t size;
127 struct pci_dev *tmp;
128 struct dmar_pci_notify_info *info;
129
130 BUG_ON(dev->is_virtfn);
131
132 /* Only generate path[] for device addition event */
133 if (event == BUS_NOTIFY_ADD_DEVICE)
134 for (tmp = dev; tmp; tmp = tmp->bus->self)
135 level++;
136
137 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
138 if (size <= sizeof(dmar_pci_notify_info_buf)) {
139 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
140 } else {
141 info = kzalloc(size, GFP_KERNEL);
142 if (!info) {
143 pr_warn("Out of memory when allocating notify_info "
144 "for %s.\n", pci_name(dev));
145 if (dmar_dev_scope_status == 0)
146 dmar_dev_scope_status = -ENOMEM;
147 return NULL;
148 }
149 }
150
151 info->event = event;
152 info->dev = dev;
153 info->seg = pci_domain_nr(dev->bus);
154 info->level = level;
155 if (event == BUS_NOTIFY_ADD_DEVICE) {
156 for (tmp = dev; tmp; tmp = tmp->bus->self) {
157 level--;
158 info->path[level].bus = tmp->bus->number;
159 info->path[level].device = PCI_SLOT(tmp->devfn);
160 info->path[level].function = PCI_FUNC(tmp->devfn);
161 if (pci_is_root_bus(tmp->bus))
162 info->bus = tmp->bus->number;
163 }
164 }
165
166 return info;
167 }
168
169 static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
170 {
171 if ((void *)info != dmar_pci_notify_info_buf)
172 kfree(info);
173 }
174
175 static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
176 struct acpi_dmar_pci_path *path, int count)
177 {
178 int i;
179
180 if (info->bus != bus)
181 goto fallback;
182 if (info->level != count)
183 goto fallback;
184
185 for (i = 0; i < count; i++) {
186 if (path[i].device != info->path[i].device ||
187 path[i].function != info->path[i].function)
188 goto fallback;
189 }
190
191 return true;
192
193 fallback:
194
195 if (count != 1)
196 return false;
197
198 i = info->level - 1;
199 if (bus == info->path[i].bus &&
200 path[0].device == info->path[i].device &&
201 path[0].function == info->path[i].function) {
202 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
203 bus, path[0].device, path[0].function);
204 return true;
205 }
206
207 return false;
208 }
209
210 /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
211 int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
212 void *start, void*end, u16 segment,
213 struct dmar_dev_scope *devices,
214 int devices_cnt)
215 {
216 int i, level;
217 struct device *tmp, *dev = &info->dev->dev;
218 struct acpi_dmar_device_scope *scope;
219 struct acpi_dmar_pci_path *path;
220
221 if (segment != info->seg)
222 return 0;
223
224 for (; start < end; start += scope->length) {
225 scope = start;
226 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
227 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
228 continue;
229
230 path = (struct acpi_dmar_pci_path *)(scope + 1);
231 level = (scope->length - sizeof(*scope)) / sizeof(*path);
232 if (!dmar_match_pci_path(info, scope->bus, path, level))
233 continue;
234
235 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
236 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
237 pr_warn("Device scope type does not match for %s\n",
238 pci_name(info->dev));
239 return -EINVAL;
240 }
241
242 for_each_dev_scope(devices, devices_cnt, i, tmp)
243 if (tmp == NULL) {
244 devices[i].bus = info->dev->bus->number;
245 devices[i].devfn = info->dev->devfn;
246 rcu_assign_pointer(devices[i].dev,
247 get_device(dev));
248 return 1;
249 }
250 BUG_ON(i >= devices_cnt);
251 }
252
253 return 0;
254 }
255
256 int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
257 struct dmar_dev_scope *devices, int count)
258 {
259 int index;
260 struct device *tmp;
261
262 if (info->seg != segment)
263 return 0;
264
265 for_each_active_dev_scope(devices, count, index, tmp)
266 if (tmp == &info->dev->dev) {
267 RCU_INIT_POINTER(devices[index].dev, NULL);
268 synchronize_rcu();
269 put_device(tmp);
270 return 1;
271 }
272
273 return 0;
274 }
275
276 static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
277 {
278 int ret = 0;
279 struct dmar_drhd_unit *dmaru;
280 struct acpi_dmar_hardware_unit *drhd;
281
282 for_each_drhd_unit(dmaru) {
283 if (dmaru->include_all)
284 continue;
285
286 drhd = container_of(dmaru->hdr,
287 struct acpi_dmar_hardware_unit, header);
288 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
289 ((void *)drhd) + drhd->header.length,
290 dmaru->segment,
291 dmaru->devices, dmaru->devices_cnt);
292 if (ret != 0)
293 break;
294 }
295 if (ret >= 0)
296 ret = dmar_iommu_notify_scope_dev(info);
297 if (ret < 0 && dmar_dev_scope_status == 0)
298 dmar_dev_scope_status = ret;
299
300 return ret;
301 }
302
303 static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
304 {
305 struct dmar_drhd_unit *dmaru;
306
307 for_each_drhd_unit(dmaru)
308 if (dmar_remove_dev_scope(info, dmaru->segment,
309 dmaru->devices, dmaru->devices_cnt))
310 break;
311 dmar_iommu_notify_scope_dev(info);
312 }
313
314 static int dmar_pci_bus_notifier(struct notifier_block *nb,
315 unsigned long action, void *data)
316 {
317 struct pci_dev *pdev = to_pci_dev(data);
318 struct dmar_pci_notify_info *info;
319
320 /* Only care about add/remove events for physical functions */
321 if (pdev->is_virtfn)
322 return NOTIFY_DONE;
323 if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE)
324 return NOTIFY_DONE;
325
326 info = dmar_alloc_pci_notify_info(pdev, action);
327 if (!info)
328 return NOTIFY_DONE;
329
330 down_write(&dmar_global_lock);
331 if (action == BUS_NOTIFY_ADD_DEVICE)
332 dmar_pci_bus_add_dev(info);
333 else if (action == BUS_NOTIFY_DEL_DEVICE)
334 dmar_pci_bus_del_dev(info);
335 up_write(&dmar_global_lock);
336
337 dmar_free_pci_notify_info(info);
338
339 return NOTIFY_OK;
340 }
341
342 static struct notifier_block dmar_pci_bus_nb = {
343 .notifier_call = dmar_pci_bus_notifier,
344 .priority = INT_MIN,
345 };
346
347 /**
348 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
349 * structure which uniquely represent one DMA remapping hardware unit
350 * present in the platform
351 */
352 static int __init
353 dmar_parse_one_drhd(struct acpi_dmar_header *header)
354 {
355 struct acpi_dmar_hardware_unit *drhd;
356 struct dmar_drhd_unit *dmaru;
357 int ret = 0;
358
359 drhd = (struct acpi_dmar_hardware_unit *)header;
360 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
361 if (!dmaru)
362 return -ENOMEM;
363
364 dmaru->hdr = header;
365 dmaru->reg_base_addr = drhd->address;
366 dmaru->segment = drhd->segment;
367 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
368 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
369 ((void *)drhd) + drhd->header.length,
370 &dmaru->devices_cnt);
371 if (dmaru->devices_cnt && dmaru->devices == NULL) {
372 kfree(dmaru);
373 return -ENOMEM;
374 }
375
376 ret = alloc_iommu(dmaru);
377 if (ret) {
378 dmar_free_dev_scope(&dmaru->devices,
379 &dmaru->devices_cnt);
380 kfree(dmaru);
381 return ret;
382 }
383 dmar_register_drhd_unit(dmaru);
384 return 0;
385 }
386
387 static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
388 {
389 if (dmaru->devices && dmaru->devices_cnt)
390 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
391 if (dmaru->iommu)
392 free_iommu(dmaru->iommu);
393 kfree(dmaru);
394 }
395
396 static int __init dmar_parse_one_andd(struct acpi_dmar_header *header)
397 {
398 struct acpi_dmar_andd *andd = (void *)header;
399
400 /* Check for NUL termination within the designated length */
401 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
402 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
403 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
404 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
405 dmi_get_system_info(DMI_BIOS_VENDOR),
406 dmi_get_system_info(DMI_BIOS_VERSION),
407 dmi_get_system_info(DMI_PRODUCT_VERSION));
408 return -EINVAL;
409 }
410 pr_info("ANDD device: %x name: %s\n", andd->device_number,
411 andd->device_name);
412
413 return 0;
414 }
415
416 #ifdef CONFIG_ACPI_NUMA
417 static int __init
418 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
419 {
420 struct acpi_dmar_rhsa *rhsa;
421 struct dmar_drhd_unit *drhd;
422
423 rhsa = (struct acpi_dmar_rhsa *)header;
424 for_each_drhd_unit(drhd) {
425 if (drhd->reg_base_addr == rhsa->base_address) {
426 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
427
428 if (!node_online(node))
429 node = -1;
430 drhd->iommu->node = node;
431 return 0;
432 }
433 }
434 WARN_TAINT(
435 1, TAINT_FIRMWARE_WORKAROUND,
436 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
437 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
438 drhd->reg_base_addr,
439 dmi_get_system_info(DMI_BIOS_VENDOR),
440 dmi_get_system_info(DMI_BIOS_VERSION),
441 dmi_get_system_info(DMI_PRODUCT_VERSION));
442
443 return 0;
444 }
445 #endif
446
447 static void __init
448 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
449 {
450 struct acpi_dmar_hardware_unit *drhd;
451 struct acpi_dmar_reserved_memory *rmrr;
452 struct acpi_dmar_atsr *atsr;
453 struct acpi_dmar_rhsa *rhsa;
454
455 switch (header->type) {
456 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
457 drhd = container_of(header, struct acpi_dmar_hardware_unit,
458 header);
459 pr_info("DRHD base: %#016Lx flags: %#x\n",
460 (unsigned long long)drhd->address, drhd->flags);
461 break;
462 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
463 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
464 header);
465 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
466 (unsigned long long)rmrr->base_address,
467 (unsigned long long)rmrr->end_address);
468 break;
469 case ACPI_DMAR_TYPE_ROOT_ATS:
470 atsr = container_of(header, struct acpi_dmar_atsr, header);
471 pr_info("ATSR flags: %#x\n", atsr->flags);
472 break;
473 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
474 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
475 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
476 (unsigned long long)rhsa->base_address,
477 rhsa->proximity_domain);
478 break;
479 case ACPI_DMAR_TYPE_NAMESPACE:
480 /* We don't print this here because we need to sanity-check
481 it first. So print it in dmar_parse_one_andd() instead. */
482 break;
483 }
484 }
485
486 /**
487 * dmar_table_detect - checks to see if the platform supports DMAR devices
488 */
489 static int __init dmar_table_detect(void)
490 {
491 acpi_status status = AE_OK;
492
493 /* if we could find DMAR table, then there are DMAR devices */
494 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
495 (struct acpi_table_header **)&dmar_tbl,
496 &dmar_tbl_size);
497
498 if (ACPI_SUCCESS(status) && !dmar_tbl) {
499 pr_warn("Unable to map DMAR\n");
500 status = AE_NOT_FOUND;
501 }
502
503 return (ACPI_SUCCESS(status) ? 1 : 0);
504 }
505
506 /**
507 * parse_dmar_table - parses the DMA reporting table
508 */
509 static int __init
510 parse_dmar_table(void)
511 {
512 struct acpi_table_dmar *dmar;
513 struct acpi_dmar_header *entry_header;
514 int ret = 0;
515 int drhd_count = 0;
516
517 /*
518 * Do it again, earlier dmar_tbl mapping could be mapped with
519 * fixed map.
520 */
521 dmar_table_detect();
522
523 /*
524 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
525 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
526 */
527 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
528
529 dmar = (struct acpi_table_dmar *)dmar_tbl;
530 if (!dmar)
531 return -ENODEV;
532
533 if (dmar->width < PAGE_SHIFT - 1) {
534 pr_warn("Invalid DMAR haw\n");
535 return -EINVAL;
536 }
537
538 pr_info("Host address width %d\n", dmar->width + 1);
539
540 entry_header = (struct acpi_dmar_header *)(dmar + 1);
541 while (((unsigned long)entry_header) <
542 (((unsigned long)dmar) + dmar_tbl->length)) {
543 /* Avoid looping forever on bad ACPI tables */
544 if (entry_header->length == 0) {
545 pr_warn("Invalid 0-length structure\n");
546 ret = -EINVAL;
547 break;
548 }
549
550 dmar_table_print_dmar_entry(entry_header);
551
552 switch (entry_header->type) {
553 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
554 drhd_count++;
555 ret = dmar_parse_one_drhd(entry_header);
556 break;
557 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
558 ret = dmar_parse_one_rmrr(entry_header);
559 break;
560 case ACPI_DMAR_TYPE_ROOT_ATS:
561 ret = dmar_parse_one_atsr(entry_header);
562 break;
563 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
564 #ifdef CONFIG_ACPI_NUMA
565 ret = dmar_parse_one_rhsa(entry_header);
566 #endif
567 break;
568 case ACPI_DMAR_TYPE_NAMESPACE:
569 ret = dmar_parse_one_andd(entry_header);
570 break;
571 default:
572 pr_warn("Unknown DMAR structure type %d\n",
573 entry_header->type);
574 ret = 0; /* for forward compatibility */
575 break;
576 }
577 if (ret)
578 break;
579
580 entry_header = ((void *)entry_header + entry_header->length);
581 }
582 if (drhd_count == 0)
583 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
584 return ret;
585 }
586
587 static int dmar_pci_device_match(struct dmar_dev_scope devices[],
588 int cnt, struct pci_dev *dev)
589 {
590 int index;
591 struct device *tmp;
592
593 while (dev) {
594 for_each_active_dev_scope(devices, cnt, index, tmp)
595 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
596 return 1;
597
598 /* Check our parent */
599 dev = dev->bus->self;
600 }
601
602 return 0;
603 }
604
605 struct dmar_drhd_unit *
606 dmar_find_matched_drhd_unit(struct pci_dev *dev)
607 {
608 struct dmar_drhd_unit *dmaru;
609 struct acpi_dmar_hardware_unit *drhd;
610
611 dev = pci_physfn(dev);
612
613 rcu_read_lock();
614 for_each_drhd_unit(dmaru) {
615 drhd = container_of(dmaru->hdr,
616 struct acpi_dmar_hardware_unit,
617 header);
618
619 if (dmaru->include_all &&
620 drhd->segment == pci_domain_nr(dev->bus))
621 goto out;
622
623 if (dmar_pci_device_match(dmaru->devices,
624 dmaru->devices_cnt, dev))
625 goto out;
626 }
627 dmaru = NULL;
628 out:
629 rcu_read_unlock();
630
631 return dmaru;
632 }
633
634 static void __init dmar_acpi_insert_dev_scope(u8 device_number,
635 struct acpi_device *adev)
636 {
637 struct dmar_drhd_unit *dmaru;
638 struct acpi_dmar_hardware_unit *drhd;
639 struct acpi_dmar_device_scope *scope;
640 struct device *tmp;
641 int i;
642 struct acpi_dmar_pci_path *path;
643
644 for_each_drhd_unit(dmaru) {
645 drhd = container_of(dmaru->hdr,
646 struct acpi_dmar_hardware_unit,
647 header);
648
649 for (scope = (void *)(drhd + 1);
650 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
651 scope = ((void *)scope) + scope->length) {
652 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
653 continue;
654 if (scope->enumeration_id != device_number)
655 continue;
656
657 path = (void *)(scope + 1);
658 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
659 dev_name(&adev->dev), dmaru->reg_base_addr,
660 scope->bus, path->device, path->function);
661 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
662 if (tmp == NULL) {
663 dmaru->devices[i].bus = scope->bus;
664 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
665 path->function);
666 rcu_assign_pointer(dmaru->devices[i].dev,
667 get_device(&adev->dev));
668 return;
669 }
670 BUG_ON(i >= dmaru->devices_cnt);
671 }
672 }
673 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
674 device_number, dev_name(&adev->dev));
675 }
676
677 static int __init dmar_acpi_dev_scope_init(void)
678 {
679 struct acpi_dmar_andd *andd;
680
681 if (dmar_tbl == NULL)
682 return -ENODEV;
683
684 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
685 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
686 andd = ((void *)andd) + andd->header.length) {
687 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
688 acpi_handle h;
689 struct acpi_device *adev;
690
691 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
692 andd->device_name,
693 &h))) {
694 pr_err("Failed to find handle for ACPI object %s\n",
695 andd->device_name);
696 continue;
697 }
698 acpi_bus_get_device(h, &adev);
699 if (!adev) {
700 pr_err("Failed to get device for ACPI object %s\n",
701 andd->device_name);
702 continue;
703 }
704 dmar_acpi_insert_dev_scope(andd->device_number, adev);
705 }
706 }
707 return 0;
708 }
709
710 int __init dmar_dev_scope_init(void)
711 {
712 struct pci_dev *dev = NULL;
713 struct dmar_pci_notify_info *info;
714
715 if (dmar_dev_scope_status != 1)
716 return dmar_dev_scope_status;
717
718 if (list_empty(&dmar_drhd_units)) {
719 dmar_dev_scope_status = -ENODEV;
720 } else {
721 dmar_dev_scope_status = 0;
722
723 dmar_acpi_dev_scope_init();
724
725 for_each_pci_dev(dev) {
726 if (dev->is_virtfn)
727 continue;
728
729 info = dmar_alloc_pci_notify_info(dev,
730 BUS_NOTIFY_ADD_DEVICE);
731 if (!info) {
732 return dmar_dev_scope_status;
733 } else {
734 dmar_pci_bus_add_dev(info);
735 dmar_free_pci_notify_info(info);
736 }
737 }
738
739 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
740 }
741
742 return dmar_dev_scope_status;
743 }
744
745
746 int __init dmar_table_init(void)
747 {
748 static int dmar_table_initialized;
749 int ret;
750
751 if (dmar_table_initialized == 0) {
752 ret = parse_dmar_table();
753 if (ret < 0) {
754 if (ret != -ENODEV)
755 pr_info("parse DMAR table failure.\n");
756 } else if (list_empty(&dmar_drhd_units)) {
757 pr_info("No DMAR devices found\n");
758 ret = -ENODEV;
759 }
760
761 if (ret < 0)
762 dmar_table_initialized = ret;
763 else
764 dmar_table_initialized = 1;
765 }
766
767 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
768 }
769
770 static void warn_invalid_dmar(u64 addr, const char *message)
771 {
772 WARN_TAINT_ONCE(
773 1, TAINT_FIRMWARE_WORKAROUND,
774 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
775 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
776 addr, message,
777 dmi_get_system_info(DMI_BIOS_VENDOR),
778 dmi_get_system_info(DMI_BIOS_VERSION),
779 dmi_get_system_info(DMI_PRODUCT_VERSION));
780 }
781
782 static int __init check_zero_address(void)
783 {
784 struct acpi_table_dmar *dmar;
785 struct acpi_dmar_header *entry_header;
786 struct acpi_dmar_hardware_unit *drhd;
787
788 dmar = (struct acpi_table_dmar *)dmar_tbl;
789 entry_header = (struct acpi_dmar_header *)(dmar + 1);
790
791 while (((unsigned long)entry_header) <
792 (((unsigned long)dmar) + dmar_tbl->length)) {
793 /* Avoid looping forever on bad ACPI tables */
794 if (entry_header->length == 0) {
795 pr_warn("Invalid 0-length structure\n");
796 return 0;
797 }
798
799 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
800 void __iomem *addr;
801 u64 cap, ecap;
802
803 drhd = (void *)entry_header;
804 if (!drhd->address) {
805 warn_invalid_dmar(0, "");
806 goto failed;
807 }
808
809 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
810 if (!addr ) {
811 printk("IOMMU: can't validate: %llx\n", drhd->address);
812 goto failed;
813 }
814 cap = dmar_readq(addr + DMAR_CAP_REG);
815 ecap = dmar_readq(addr + DMAR_ECAP_REG);
816 early_iounmap(addr, VTD_PAGE_SIZE);
817 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
818 warn_invalid_dmar(drhd->address,
819 " returns all ones");
820 goto failed;
821 }
822 }
823
824 entry_header = ((void *)entry_header + entry_header->length);
825 }
826 return 1;
827
828 failed:
829 return 0;
830 }
831
832 int __init detect_intel_iommu(void)
833 {
834 int ret;
835
836 down_write(&dmar_global_lock);
837 ret = dmar_table_detect();
838 if (ret)
839 ret = check_zero_address();
840 {
841 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
842 iommu_detected = 1;
843 /* Make sure ACS will be enabled */
844 pci_request_acs();
845 }
846
847 #ifdef CONFIG_X86
848 if (ret)
849 x86_init.iommu.iommu_init = intel_iommu_init;
850 #endif
851 }
852 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
853 dmar_tbl = NULL;
854 up_write(&dmar_global_lock);
855
856 return ret ? 1 : -ENODEV;
857 }
858
859
860 static void unmap_iommu(struct intel_iommu *iommu)
861 {
862 iounmap(iommu->reg);
863 release_mem_region(iommu->reg_phys, iommu->reg_size);
864 }
865
866 /**
867 * map_iommu: map the iommu's registers
868 * @iommu: the iommu to map
869 * @phys_addr: the physical address of the base resgister
870 *
871 * Memory map the iommu's registers. Start w/ a single page, and
872 * possibly expand if that turns out to be insufficent.
873 */
874 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
875 {
876 int map_size, err=0;
877
878 iommu->reg_phys = phys_addr;
879 iommu->reg_size = VTD_PAGE_SIZE;
880
881 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
882 pr_err("IOMMU: can't reserve memory\n");
883 err = -EBUSY;
884 goto out;
885 }
886
887 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
888 if (!iommu->reg) {
889 pr_err("IOMMU: can't map the region\n");
890 err = -ENOMEM;
891 goto release;
892 }
893
894 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
895 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
896
897 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
898 err = -EINVAL;
899 warn_invalid_dmar(phys_addr, " returns all ones");
900 goto unmap;
901 }
902
903 /* the registers might be more than one page */
904 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
905 cap_max_fault_reg_offset(iommu->cap));
906 map_size = VTD_PAGE_ALIGN(map_size);
907 if (map_size > iommu->reg_size) {
908 iounmap(iommu->reg);
909 release_mem_region(iommu->reg_phys, iommu->reg_size);
910 iommu->reg_size = map_size;
911 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
912 iommu->name)) {
913 pr_err("IOMMU: can't reserve memory\n");
914 err = -EBUSY;
915 goto out;
916 }
917 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
918 if (!iommu->reg) {
919 pr_err("IOMMU: can't map the region\n");
920 err = -ENOMEM;
921 goto release;
922 }
923 }
924 err = 0;
925 goto out;
926
927 unmap:
928 iounmap(iommu->reg);
929 release:
930 release_mem_region(iommu->reg_phys, iommu->reg_size);
931 out:
932 return err;
933 }
934
935 static int alloc_iommu(struct dmar_drhd_unit *drhd)
936 {
937 struct intel_iommu *iommu;
938 u32 ver, sts;
939 static int iommu_allocated = 0;
940 int agaw = 0;
941 int msagaw = 0;
942 int err;
943
944 if (!drhd->reg_base_addr) {
945 warn_invalid_dmar(0, "");
946 return -EINVAL;
947 }
948
949 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
950 if (!iommu)
951 return -ENOMEM;
952
953 iommu->seq_id = iommu_allocated++;
954 sprintf (iommu->name, "dmar%d", iommu->seq_id);
955
956 err = map_iommu(iommu, drhd->reg_base_addr);
957 if (err) {
958 pr_err("IOMMU: failed to map %s\n", iommu->name);
959 goto error;
960 }
961
962 err = -EINVAL;
963 agaw = iommu_calculate_agaw(iommu);
964 if (agaw < 0) {
965 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
966 iommu->seq_id);
967 goto err_unmap;
968 }
969 msagaw = iommu_calculate_max_sagaw(iommu);
970 if (msagaw < 0) {
971 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
972 iommu->seq_id);
973 goto err_unmap;
974 }
975 iommu->agaw = agaw;
976 iommu->msagaw = msagaw;
977 iommu->segment = drhd->segment;
978
979 iommu->node = -1;
980
981 ver = readl(iommu->reg + DMAR_VER_REG);
982 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
983 iommu->seq_id,
984 (unsigned long long)drhd->reg_base_addr,
985 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
986 (unsigned long long)iommu->cap,
987 (unsigned long long)iommu->ecap);
988
989 /* Reflect status in gcmd */
990 sts = readl(iommu->reg + DMAR_GSTS_REG);
991 if (sts & DMA_GSTS_IRES)
992 iommu->gcmd |= DMA_GCMD_IRE;
993 if (sts & DMA_GSTS_TES)
994 iommu->gcmd |= DMA_GCMD_TE;
995 if (sts & DMA_GSTS_QIES)
996 iommu->gcmd |= DMA_GCMD_QIE;
997
998 raw_spin_lock_init(&iommu->register_lock);
999
1000 drhd->iommu = iommu;
1001
1002 if (intel_iommu_enabled)
1003 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1004 intel_iommu_groups,
1005 iommu->name);
1006
1007 return 0;
1008
1009 err_unmap:
1010 unmap_iommu(iommu);
1011 error:
1012 kfree(iommu);
1013 return err;
1014 }
1015
1016 static void free_iommu(struct intel_iommu *iommu)
1017 {
1018 iommu_device_destroy(iommu->iommu_dev);
1019
1020 if (iommu->irq) {
1021 free_irq(iommu->irq, iommu);
1022 irq_set_handler_data(iommu->irq, NULL);
1023 dmar_free_hwirq(iommu->irq);
1024 }
1025
1026 if (iommu->qi) {
1027 free_page((unsigned long)iommu->qi->desc);
1028 kfree(iommu->qi->desc_status);
1029 kfree(iommu->qi);
1030 }
1031
1032 if (iommu->reg)
1033 unmap_iommu(iommu);
1034
1035 kfree(iommu);
1036 }
1037
1038 /*
1039 * Reclaim all the submitted descriptors which have completed its work.
1040 */
1041 static inline void reclaim_free_desc(struct q_inval *qi)
1042 {
1043 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1044 qi->desc_status[qi->free_tail] == QI_ABORT) {
1045 qi->desc_status[qi->free_tail] = QI_FREE;
1046 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1047 qi->free_cnt++;
1048 }
1049 }
1050
1051 static int qi_check_fault(struct intel_iommu *iommu, int index)
1052 {
1053 u32 fault;
1054 int head, tail;
1055 struct q_inval *qi = iommu->qi;
1056 int wait_index = (index + 1) % QI_LENGTH;
1057
1058 if (qi->desc_status[wait_index] == QI_ABORT)
1059 return -EAGAIN;
1060
1061 fault = readl(iommu->reg + DMAR_FSTS_REG);
1062
1063 /*
1064 * If IQE happens, the head points to the descriptor associated
1065 * with the error. No new descriptors are fetched until the IQE
1066 * is cleared.
1067 */
1068 if (fault & DMA_FSTS_IQE) {
1069 head = readl(iommu->reg + DMAR_IQH_REG);
1070 if ((head >> DMAR_IQ_SHIFT) == index) {
1071 pr_err("VT-d detected invalid descriptor: "
1072 "low=%llx, high=%llx\n",
1073 (unsigned long long)qi->desc[index].low,
1074 (unsigned long long)qi->desc[index].high);
1075 memcpy(&qi->desc[index], &qi->desc[wait_index],
1076 sizeof(struct qi_desc));
1077 __iommu_flush_cache(iommu, &qi->desc[index],
1078 sizeof(struct qi_desc));
1079 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1080 return -EINVAL;
1081 }
1082 }
1083
1084 /*
1085 * If ITE happens, all pending wait_desc commands are aborted.
1086 * No new descriptors are fetched until the ITE is cleared.
1087 */
1088 if (fault & DMA_FSTS_ITE) {
1089 head = readl(iommu->reg + DMAR_IQH_REG);
1090 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1091 head |= 1;
1092 tail = readl(iommu->reg + DMAR_IQT_REG);
1093 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1094
1095 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1096
1097 do {
1098 if (qi->desc_status[head] == QI_IN_USE)
1099 qi->desc_status[head] = QI_ABORT;
1100 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1101 } while (head != tail);
1102
1103 if (qi->desc_status[wait_index] == QI_ABORT)
1104 return -EAGAIN;
1105 }
1106
1107 if (fault & DMA_FSTS_ICE)
1108 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1109
1110 return 0;
1111 }
1112
1113 /*
1114 * Submit the queued invalidation descriptor to the remapping
1115 * hardware unit and wait for its completion.
1116 */
1117 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1118 {
1119 int rc;
1120 struct q_inval *qi = iommu->qi;
1121 struct qi_desc *hw, wait_desc;
1122 int wait_index, index;
1123 unsigned long flags;
1124
1125 if (!qi)
1126 return 0;
1127
1128 hw = qi->desc;
1129
1130 restart:
1131 rc = 0;
1132
1133 raw_spin_lock_irqsave(&qi->q_lock, flags);
1134 while (qi->free_cnt < 3) {
1135 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1136 cpu_relax();
1137 raw_spin_lock_irqsave(&qi->q_lock, flags);
1138 }
1139
1140 index = qi->free_head;
1141 wait_index = (index + 1) % QI_LENGTH;
1142
1143 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1144
1145 hw[index] = *desc;
1146
1147 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1148 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1149 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1150
1151 hw[wait_index] = wait_desc;
1152
1153 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1154 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1155
1156 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1157 qi->free_cnt -= 2;
1158
1159 /*
1160 * update the HW tail register indicating the presence of
1161 * new descriptors.
1162 */
1163 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1164
1165 while (qi->desc_status[wait_index] != QI_DONE) {
1166 /*
1167 * We will leave the interrupts disabled, to prevent interrupt
1168 * context to queue another cmd while a cmd is already submitted
1169 * and waiting for completion on this cpu. This is to avoid
1170 * a deadlock where the interrupt context can wait indefinitely
1171 * for free slots in the queue.
1172 */
1173 rc = qi_check_fault(iommu, index);
1174 if (rc)
1175 break;
1176
1177 raw_spin_unlock(&qi->q_lock);
1178 cpu_relax();
1179 raw_spin_lock(&qi->q_lock);
1180 }
1181
1182 qi->desc_status[index] = QI_DONE;
1183
1184 reclaim_free_desc(qi);
1185 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1186
1187 if (rc == -EAGAIN)
1188 goto restart;
1189
1190 return rc;
1191 }
1192
1193 /*
1194 * Flush the global interrupt entry cache.
1195 */
1196 void qi_global_iec(struct intel_iommu *iommu)
1197 {
1198 struct qi_desc desc;
1199
1200 desc.low = QI_IEC_TYPE;
1201 desc.high = 0;
1202
1203 /* should never fail */
1204 qi_submit_sync(&desc, iommu);
1205 }
1206
1207 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1208 u64 type)
1209 {
1210 struct qi_desc desc;
1211
1212 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1213 | QI_CC_GRAN(type) | QI_CC_TYPE;
1214 desc.high = 0;
1215
1216 qi_submit_sync(&desc, iommu);
1217 }
1218
1219 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1220 unsigned int size_order, u64 type)
1221 {
1222 u8 dw = 0, dr = 0;
1223
1224 struct qi_desc desc;
1225 int ih = 0;
1226
1227 if (cap_write_drain(iommu->cap))
1228 dw = 1;
1229
1230 if (cap_read_drain(iommu->cap))
1231 dr = 1;
1232
1233 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1234 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1235 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1236 | QI_IOTLB_AM(size_order);
1237
1238 qi_submit_sync(&desc, iommu);
1239 }
1240
1241 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1242 u64 addr, unsigned mask)
1243 {
1244 struct qi_desc desc;
1245
1246 if (mask) {
1247 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1248 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1249 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1250 } else
1251 desc.high = QI_DEV_IOTLB_ADDR(addr);
1252
1253 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1254 qdep = 0;
1255
1256 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1257 QI_DIOTLB_TYPE;
1258
1259 qi_submit_sync(&desc, iommu);
1260 }
1261
1262 /*
1263 * Disable Queued Invalidation interface.
1264 */
1265 void dmar_disable_qi(struct intel_iommu *iommu)
1266 {
1267 unsigned long flags;
1268 u32 sts;
1269 cycles_t start_time = get_cycles();
1270
1271 if (!ecap_qis(iommu->ecap))
1272 return;
1273
1274 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1275
1276 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1277 if (!(sts & DMA_GSTS_QIES))
1278 goto end;
1279
1280 /*
1281 * Give a chance to HW to complete the pending invalidation requests.
1282 */
1283 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1284 readl(iommu->reg + DMAR_IQH_REG)) &&
1285 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1286 cpu_relax();
1287
1288 iommu->gcmd &= ~DMA_GCMD_QIE;
1289 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1290
1291 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1292 !(sts & DMA_GSTS_QIES), sts);
1293 end:
1294 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1295 }
1296
1297 /*
1298 * Enable queued invalidation.
1299 */
1300 static void __dmar_enable_qi(struct intel_iommu *iommu)
1301 {
1302 u32 sts;
1303 unsigned long flags;
1304 struct q_inval *qi = iommu->qi;
1305
1306 qi->free_head = qi->free_tail = 0;
1307 qi->free_cnt = QI_LENGTH;
1308
1309 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1310
1311 /* write zero to the tail reg */
1312 writel(0, iommu->reg + DMAR_IQT_REG);
1313
1314 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1315
1316 iommu->gcmd |= DMA_GCMD_QIE;
1317 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1318
1319 /* Make sure hardware complete it */
1320 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1321
1322 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1323 }
1324
1325 /*
1326 * Enable Queued Invalidation interface. This is a must to support
1327 * interrupt-remapping. Also used by DMA-remapping, which replaces
1328 * register based IOTLB invalidation.
1329 */
1330 int dmar_enable_qi(struct intel_iommu *iommu)
1331 {
1332 struct q_inval *qi;
1333 struct page *desc_page;
1334
1335 if (!ecap_qis(iommu->ecap))
1336 return -ENOENT;
1337
1338 /*
1339 * queued invalidation is already setup and enabled.
1340 */
1341 if (iommu->qi)
1342 return 0;
1343
1344 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1345 if (!iommu->qi)
1346 return -ENOMEM;
1347
1348 qi = iommu->qi;
1349
1350
1351 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1352 if (!desc_page) {
1353 kfree(qi);
1354 iommu->qi = NULL;
1355 return -ENOMEM;
1356 }
1357
1358 qi->desc = page_address(desc_page);
1359
1360 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1361 if (!qi->desc_status) {
1362 free_page((unsigned long) qi->desc);
1363 kfree(qi);
1364 iommu->qi = NULL;
1365 return -ENOMEM;
1366 }
1367
1368 raw_spin_lock_init(&qi->q_lock);
1369
1370 __dmar_enable_qi(iommu);
1371
1372 return 0;
1373 }
1374
1375 /* iommu interrupt handling. Most stuff are MSI-like. */
1376
1377 enum faulttype {
1378 DMA_REMAP,
1379 INTR_REMAP,
1380 UNKNOWN,
1381 };
1382
1383 static const char *dma_remap_fault_reasons[] =
1384 {
1385 "Software",
1386 "Present bit in root entry is clear",
1387 "Present bit in context entry is clear",
1388 "Invalid context entry",
1389 "Access beyond MGAW",
1390 "PTE Write access is not set",
1391 "PTE Read access is not set",
1392 "Next page table ptr is invalid",
1393 "Root table address invalid",
1394 "Context table ptr is invalid",
1395 "non-zero reserved fields in RTP",
1396 "non-zero reserved fields in CTP",
1397 "non-zero reserved fields in PTE",
1398 "PCE for translation request specifies blocking",
1399 };
1400
1401 static const char *irq_remap_fault_reasons[] =
1402 {
1403 "Detected reserved fields in the decoded interrupt-remapped request",
1404 "Interrupt index exceeded the interrupt-remapping table size",
1405 "Present field in the IRTE entry is clear",
1406 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1407 "Detected reserved fields in the IRTE entry",
1408 "Blocked a compatibility format interrupt request",
1409 "Blocked an interrupt request due to source-id verification failure",
1410 };
1411
1412 static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1413 {
1414 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1415 ARRAY_SIZE(irq_remap_fault_reasons))) {
1416 *fault_type = INTR_REMAP;
1417 return irq_remap_fault_reasons[fault_reason - 0x20];
1418 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1419 *fault_type = DMA_REMAP;
1420 return dma_remap_fault_reasons[fault_reason];
1421 } else {
1422 *fault_type = UNKNOWN;
1423 return "Unknown";
1424 }
1425 }
1426
1427 void dmar_msi_unmask(struct irq_data *data)
1428 {
1429 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1430 unsigned long flag;
1431
1432 /* unmask it */
1433 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1434 writel(0, iommu->reg + DMAR_FECTL_REG);
1435 /* Read a reg to force flush the post write */
1436 readl(iommu->reg + DMAR_FECTL_REG);
1437 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1438 }
1439
1440 void dmar_msi_mask(struct irq_data *data)
1441 {
1442 unsigned long flag;
1443 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1444
1445 /* mask it */
1446 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1447 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1448 /* Read a reg to force flush the post write */
1449 readl(iommu->reg + DMAR_FECTL_REG);
1450 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1451 }
1452
1453 void dmar_msi_write(int irq, struct msi_msg *msg)
1454 {
1455 struct intel_iommu *iommu = irq_get_handler_data(irq);
1456 unsigned long flag;
1457
1458 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1459 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1460 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1461 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1462 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1463 }
1464
1465 void dmar_msi_read(int irq, struct msi_msg *msg)
1466 {
1467 struct intel_iommu *iommu = irq_get_handler_data(irq);
1468 unsigned long flag;
1469
1470 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1471 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1472 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1473 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1474 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1475 }
1476
1477 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1478 u8 fault_reason, u16 source_id, unsigned long long addr)
1479 {
1480 const char *reason;
1481 int fault_type;
1482
1483 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1484
1485 if (fault_type == INTR_REMAP)
1486 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1487 "fault index %llx\n"
1488 "INTR-REMAP:[fault reason %02d] %s\n",
1489 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1490 PCI_FUNC(source_id & 0xFF), addr >> 48,
1491 fault_reason, reason);
1492 else
1493 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1494 "fault addr %llx \n"
1495 "DMAR:[fault reason %02d] %s\n",
1496 (type ? "DMA Read" : "DMA Write"),
1497 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1498 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1499 return 0;
1500 }
1501
1502 #define PRIMARY_FAULT_REG_LEN (16)
1503 irqreturn_t dmar_fault(int irq, void *dev_id)
1504 {
1505 struct intel_iommu *iommu = dev_id;
1506 int reg, fault_index;
1507 u32 fault_status;
1508 unsigned long flag;
1509
1510 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1511 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1512 if (fault_status)
1513 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1514
1515 /* TBD: ignore advanced fault log currently */
1516 if (!(fault_status & DMA_FSTS_PPF))
1517 goto unlock_exit;
1518
1519 fault_index = dma_fsts_fault_record_index(fault_status);
1520 reg = cap_fault_reg_offset(iommu->cap);
1521 while (1) {
1522 u8 fault_reason;
1523 u16 source_id;
1524 u64 guest_addr;
1525 int type;
1526 u32 data;
1527
1528 /* highest 32 bits */
1529 data = readl(iommu->reg + reg +
1530 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1531 if (!(data & DMA_FRCD_F))
1532 break;
1533
1534 fault_reason = dma_frcd_fault_reason(data);
1535 type = dma_frcd_type(data);
1536
1537 data = readl(iommu->reg + reg +
1538 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1539 source_id = dma_frcd_source_id(data);
1540
1541 guest_addr = dmar_readq(iommu->reg + reg +
1542 fault_index * PRIMARY_FAULT_REG_LEN);
1543 guest_addr = dma_frcd_page_addr(guest_addr);
1544 /* clear the fault */
1545 writel(DMA_FRCD_F, iommu->reg + reg +
1546 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1547
1548 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1549
1550 dmar_fault_do_one(iommu, type, fault_reason,
1551 source_id, guest_addr);
1552
1553 fault_index++;
1554 if (fault_index >= cap_num_fault_regs(iommu->cap))
1555 fault_index = 0;
1556 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1557 }
1558
1559 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1560
1561 unlock_exit:
1562 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1563 return IRQ_HANDLED;
1564 }
1565
1566 int dmar_set_interrupt(struct intel_iommu *iommu)
1567 {
1568 int irq, ret;
1569
1570 /*
1571 * Check if the fault interrupt is already initialized.
1572 */
1573 if (iommu->irq)
1574 return 0;
1575
1576 irq = dmar_alloc_hwirq();
1577 if (irq <= 0) {
1578 pr_err("IOMMU: no free vectors\n");
1579 return -EINVAL;
1580 }
1581
1582 irq_set_handler_data(irq, iommu);
1583 iommu->irq = irq;
1584
1585 ret = arch_setup_dmar_msi(irq);
1586 if (ret) {
1587 irq_set_handler_data(irq, NULL);
1588 iommu->irq = 0;
1589 dmar_free_hwirq(irq);
1590 return ret;
1591 }
1592
1593 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1594 if (ret)
1595 pr_err("IOMMU: can't request irq\n");
1596 return ret;
1597 }
1598
1599 int __init enable_drhd_fault_handling(void)
1600 {
1601 struct dmar_drhd_unit *drhd;
1602 struct intel_iommu *iommu;
1603
1604 /*
1605 * Enable fault control interrupt.
1606 */
1607 for_each_iommu(iommu, drhd) {
1608 u32 fault_status;
1609 int ret = dmar_set_interrupt(iommu);
1610
1611 if (ret) {
1612 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1613 (unsigned long long)drhd->reg_base_addr, ret);
1614 return -1;
1615 }
1616
1617 /*
1618 * Clear any previous faults.
1619 */
1620 dmar_fault(iommu->irq, iommu);
1621 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1622 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1623 }
1624
1625 return 0;
1626 }
1627
1628 /*
1629 * Re-enable Queued Invalidation interface.
1630 */
1631 int dmar_reenable_qi(struct intel_iommu *iommu)
1632 {
1633 if (!ecap_qis(iommu->ecap))
1634 return -ENOENT;
1635
1636 if (!iommu->qi)
1637 return -ENOENT;
1638
1639 /*
1640 * First disable queued invalidation.
1641 */
1642 dmar_disable_qi(iommu);
1643 /*
1644 * Then enable queued invalidation again. Since there is no pending
1645 * invalidation requests now, it's safe to re-enable queued
1646 * invalidation.
1647 */
1648 __dmar_enable_qi(iommu);
1649
1650 return 0;
1651 }
1652
1653 /*
1654 * Check interrupt remapping support in DMAR table description.
1655 */
1656 int __init dmar_ir_support(void)
1657 {
1658 struct acpi_table_dmar *dmar;
1659 dmar = (struct acpi_table_dmar *)dmar_tbl;
1660 if (!dmar)
1661 return 0;
1662 return dmar->flags & 0x1;
1663 }
1664
1665 static int __init dmar_free_unused_resources(void)
1666 {
1667 struct dmar_drhd_unit *dmaru, *dmaru_n;
1668
1669 /* DMAR units are in use */
1670 if (irq_remapping_enabled || intel_iommu_enabled)
1671 return 0;
1672
1673 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1674 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1675
1676 down_write(&dmar_global_lock);
1677 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1678 list_del(&dmaru->list);
1679 dmar_free_drhd(dmaru);
1680 }
1681 up_write(&dmar_global_lock);
1682
1683 return 0;
1684 }
1685
1686 late_initcall(dmar_free_unused_resources);
1687 IOMMU_INIT_POST(detect_intel_iommu);
Linux kernel - Deliverables
This page took 0.150772 seconds and 5 git commands to generate.