2 * Copyright (C) 2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <joerg.roedel@amd.com>
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/scatterlist.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/dma-debug.h>
23 #include <linux/spinlock.h>
24 #include <linux/debugfs.h>
25 #include <linux/device.h>
26 #include <linux/types.h>
27 #include <linux/sched.h>
28 #include <linux/list.h>
29 #include <linux/slab.h>
31 #define HASH_SIZE 1024ULL
32 #define HASH_FN_SHIFT 13
33 #define HASH_FN_MASK (HASH_SIZE - 1)
42 struct dma_debug_entry
{
43 struct list_head list
;
55 struct list_head list
;
57 } ____cacheline_aligned_in_smp
;
59 /* Hash list to save the allocated dma addresses */
60 static struct hash_bucket dma_entry_hash
[HASH_SIZE
];
61 /* List of pre-allocated dma_debug_entry's */
62 static LIST_HEAD(free_entries
);
63 /* Lock for the list above */
64 static DEFINE_SPINLOCK(free_entries_lock
);
66 /* Global disable flag - will be set in case of an error */
67 static bool global_disable __read_mostly
;
69 /* Global error count */
70 static u32 error_count
;
72 /* Global error show enable*/
73 static u32 show_all_errors __read_mostly
;
74 /* Number of errors to show */
75 static u32 show_num_errors
= 1;
77 static u32 num_free_entries
;
78 static u32 min_free_entries
;
80 /* number of preallocated entries requested by kernel cmdline */
81 static u32 req_entries
;
83 /* debugfs dentry's for the stuff above */
84 static struct dentry
*dma_debug_dent __read_mostly
;
85 static struct dentry
*global_disable_dent __read_mostly
;
86 static struct dentry
*error_count_dent __read_mostly
;
87 static struct dentry
*show_all_errors_dent __read_mostly
;
88 static struct dentry
*show_num_errors_dent __read_mostly
;
89 static struct dentry
*num_free_entries_dent __read_mostly
;
90 static struct dentry
*min_free_entries_dent __read_mostly
;
92 static const char *type2name
[4] = { "single", "page",
93 "scather-gather", "coherent" };
95 static const char *dir2name
[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
96 "DMA_FROM_DEVICE", "DMA_NONE" };
99 * The access to some variables in this macro is racy. We can't use atomic_t
100 * here because all these variables are exported to debugfs. Some of them even
101 * writeable. This is also the reason why a lock won't help much. But anyway,
102 * the races are no big deal. Here is why:
104 * error_count: the addition is racy, but the worst thing that can happen is
105 * that we don't count some errors
106 * show_num_errors: the subtraction is racy. Also no big deal because in
107 * worst case this will result in one warning more in the
108 * system log than the user configured. This variable is
109 * writeable via debugfs.
111 #define err_printk(dev, format, arg...) do { \
113 if (show_all_errors || show_num_errors > 0) { \
114 WARN(1, "%s %s: " format, \
115 dev_driver_string(dev), \
116 dev_name(dev) , ## arg); \
118 if (!show_all_errors && show_num_errors > 0) \
119 show_num_errors -= 1; \
123 * Hash related functions
125 * Every DMA-API request is saved into a struct dma_debug_entry. To
126 * have quick access to these structs they are stored into a hash.
128 static int hash_fn(struct dma_debug_entry
*entry
)
131 * Hash function is based on the dma address.
132 * We use bits 20-27 here as the index into the hash
134 return (entry
->dev_addr
>> HASH_FN_SHIFT
) & HASH_FN_MASK
;
138 * Request exclusive access to a hash bucket for a given dma_debug_entry.
140 static struct hash_bucket
*get_hash_bucket(struct dma_debug_entry
*entry
,
141 unsigned long *flags
)
143 int idx
= hash_fn(entry
);
144 unsigned long __flags
;
146 spin_lock_irqsave(&dma_entry_hash
[idx
].lock
, __flags
);
148 return &dma_entry_hash
[idx
];
152 * Give up exclusive access to the hash bucket
154 static void put_hash_bucket(struct hash_bucket
*bucket
,
155 unsigned long *flags
)
157 unsigned long __flags
= *flags
;
159 spin_unlock_irqrestore(&bucket
->lock
, __flags
);
163 * Search a given entry in the hash bucket list
165 static struct dma_debug_entry
*hash_bucket_find(struct hash_bucket
*bucket
,
166 struct dma_debug_entry
*ref
)
168 struct dma_debug_entry
*entry
;
170 list_for_each_entry(entry
, &bucket
->list
, list
) {
171 if ((entry
->dev_addr
== ref
->dev_addr
) &&
172 (entry
->dev
== ref
->dev
))
180 * Add an entry to a hash bucket
182 static void hash_bucket_add(struct hash_bucket
*bucket
,
183 struct dma_debug_entry
*entry
)
185 list_add_tail(&entry
->list
, &bucket
->list
);
189 * Remove entry from a hash bucket list
191 static void hash_bucket_del(struct dma_debug_entry
*entry
)
193 list_del(&entry
->list
);
197 * Wrapper function for adding an entry to the hash.
198 * This function takes care of locking itself.
200 static void add_dma_entry(struct dma_debug_entry
*entry
)
202 struct hash_bucket
*bucket
;
205 bucket
= get_hash_bucket(entry
, &flags
);
206 hash_bucket_add(bucket
, entry
);
207 put_hash_bucket(bucket
, &flags
);
210 /* struct dma_entry allocator
212 * The next two functions implement the allocator for
213 * struct dma_debug_entries.
215 static struct dma_debug_entry
*dma_entry_alloc(void)
217 struct dma_debug_entry
*entry
= NULL
;
220 spin_lock_irqsave(&free_entries_lock
, flags
);
222 if (list_empty(&free_entries
)) {
223 printk(KERN_ERR
"DMA-API: debugging out of memory "
225 global_disable
= true;
229 entry
= list_entry(free_entries
.next
, struct dma_debug_entry
, list
);
230 list_del(&entry
->list
);
231 memset(entry
, 0, sizeof(*entry
));
233 num_free_entries
-= 1;
234 if (num_free_entries
< min_free_entries
)
235 min_free_entries
= num_free_entries
;
238 spin_unlock_irqrestore(&free_entries_lock
, flags
);
243 static void dma_entry_free(struct dma_debug_entry
*entry
)
248 * add to beginning of the list - this way the entries are
249 * more likely cache hot when they are reallocated.
251 spin_lock_irqsave(&free_entries_lock
, flags
);
252 list_add(&entry
->list
, &free_entries
);
253 num_free_entries
+= 1;
254 spin_unlock_irqrestore(&free_entries_lock
, flags
);
258 * DMA-API debugging init code
260 * The init code does two things:
261 * 1. Initialize core data structures
262 * 2. Preallocate a given number of dma_debug_entry structs
265 static int prealloc_memory(u32 num_entries
)
267 struct dma_debug_entry
*entry
, *next_entry
;
270 for (i
= 0; i
< num_entries
; ++i
) {
271 entry
= kzalloc(sizeof(*entry
), GFP_KERNEL
);
275 list_add_tail(&entry
->list
, &free_entries
);
278 num_free_entries
= num_entries
;
279 min_free_entries
= num_entries
;
281 printk(KERN_INFO
"DMA-API: preallocated %d debug entries\n",
288 list_for_each_entry_safe(entry
, next_entry
, &free_entries
, list
) {
289 list_del(&entry
->list
);
296 static int dma_debug_fs_init(void)
298 dma_debug_dent
= debugfs_create_dir("dma-api", NULL
);
299 if (!dma_debug_dent
) {
300 printk(KERN_ERR
"DMA-API: can not create debugfs directory\n");
304 global_disable_dent
= debugfs_create_bool("disabled", 0444,
306 (u32
*)&global_disable
);
307 if (!global_disable_dent
)
310 error_count_dent
= debugfs_create_u32("error_count", 0444,
311 dma_debug_dent
, &error_count
);
312 if (!error_count_dent
)
315 show_all_errors_dent
= debugfs_create_u32("all_errors", 0644,
318 if (!show_all_errors_dent
)
321 show_num_errors_dent
= debugfs_create_u32("num_errors", 0644,
324 if (!show_num_errors_dent
)
327 num_free_entries_dent
= debugfs_create_u32("num_free_entries", 0444,
330 if (!num_free_entries_dent
)
333 min_free_entries_dent
= debugfs_create_u32("min_free_entries", 0444,
336 if (!min_free_entries_dent
)
342 debugfs_remove_recursive(dma_debug_dent
);
349 * Let the architectures decide how many entries should be preallocated.
351 void dma_debug_init(u32 num_entries
)
358 for (i
= 0; i
< HASH_SIZE
; ++i
) {
359 INIT_LIST_HEAD(&dma_entry_hash
[i
].list
);
360 dma_entry_hash
[i
].lock
= SPIN_LOCK_UNLOCKED
;
363 if (dma_debug_fs_init() != 0) {
364 printk(KERN_ERR
"DMA-API: error creating debugfs entries "
366 global_disable
= true;
372 num_entries
= req_entries
;
374 if (prealloc_memory(num_entries
) != 0) {
375 printk(KERN_ERR
"DMA-API: debugging out of memory error "
377 global_disable
= true;
382 printk(KERN_INFO
"DMA-API: debugging enabled by kernel config\n");
385 static __init
int dma_debug_cmdline(char *str
)
390 if (strncmp(str
, "off", 3) == 0) {
391 printk(KERN_INFO
"DMA-API: debugging disabled on kernel "
393 global_disable
= true;
399 static __init
int dma_debug_entries_cmdline(char *str
)
406 res
= get_option(&str
, &req_entries
);
414 __setup("dma_debug=", dma_debug_cmdline
);
415 __setup("dma_debug_entries=", dma_debug_entries_cmdline
);
417 static void check_unmap(struct dma_debug_entry
*ref
)
419 struct dma_debug_entry
*entry
;
420 struct hash_bucket
*bucket
;
423 if (dma_mapping_error(ref
->dev
, ref
->dev_addr
))
426 bucket
= get_hash_bucket(ref
, &flags
);
427 entry
= hash_bucket_find(bucket
, ref
);
430 err_printk(ref
->dev
, "DMA-API: device driver tries "
431 "to free DMA memory it has not allocated "
432 "[device address=0x%016llx] [size=%llu bytes]\n",
433 ref
->dev_addr
, ref
->size
);
437 if (ref
->size
!= entry
->size
) {
438 err_printk(ref
->dev
, "DMA-API: device driver frees "
439 "DMA memory with different size "
440 "[device address=0x%016llx] [map size=%llu bytes] "
441 "[unmap size=%llu bytes]\n",
442 ref
->dev_addr
, entry
->size
, ref
->size
);
445 if (ref
->type
!= entry
->type
) {
446 err_printk(ref
->dev
, "DMA-API: device driver frees "
447 "DMA memory with wrong function "
448 "[device address=0x%016llx] [size=%llu bytes] "
449 "[mapped as %s] [unmapped as %s]\n",
450 ref
->dev_addr
, ref
->size
,
451 type2name
[entry
->type
], type2name
[ref
->type
]);
452 } else if ((entry
->type
== dma_debug_coherent
) &&
453 (ref
->paddr
!= entry
->paddr
)) {
454 err_printk(ref
->dev
, "DMA-API: device driver frees "
455 "DMA memory with different CPU address "
456 "[device address=0x%016llx] [size=%llu bytes] "
457 "[cpu alloc address=%p] [cpu free address=%p]",
458 ref
->dev_addr
, ref
->size
,
459 (void *)entry
->paddr
, (void *)ref
->paddr
);
462 if (ref
->sg_call_ents
&& ref
->type
== dma_debug_sg
&&
463 ref
->sg_call_ents
!= entry
->sg_call_ents
) {
464 err_printk(ref
->dev
, "DMA-API: device driver frees "
465 "DMA sg list with different entry count "
466 "[map count=%d] [unmap count=%d]\n",
467 entry
->sg_call_ents
, ref
->sg_call_ents
);
471 * This may be no bug in reality - but most implementations of the
472 * DMA API don't handle this properly, so check for it here
474 if (ref
->direction
!= entry
->direction
) {
475 err_printk(ref
->dev
, "DMA-API: device driver frees "
476 "DMA memory with different direction "
477 "[device address=0x%016llx] [size=%llu bytes] "
478 "[mapped with %s] [unmapped with %s]\n",
479 ref
->dev_addr
, ref
->size
,
480 dir2name
[entry
->direction
],
481 dir2name
[ref
->direction
]);
484 hash_bucket_del(entry
);
485 dma_entry_free(entry
);
488 put_hash_bucket(bucket
, &flags
);
491 static void check_for_stack(struct device
*dev
, void *addr
)
493 if (object_is_on_stack(addr
))
494 err_printk(dev
, "DMA-API: device driver maps memory from stack"
495 " [addr=%p]\n", addr
);
498 static void check_sync(struct device
*dev
, dma_addr_t addr
,
499 u64 size
, u64 offset
, int direction
, bool to_cpu
)
501 struct dma_debug_entry ref
= {
505 .direction
= direction
,
507 struct dma_debug_entry
*entry
;
508 struct hash_bucket
*bucket
;
511 bucket
= get_hash_bucket(&ref
, &flags
);
513 entry
= hash_bucket_find(bucket
, &ref
);
516 err_printk(dev
, "DMA-API: device driver tries "
517 "to sync DMA memory it has not allocated "
518 "[device address=0x%016llx] [size=%llu bytes]\n",
523 if ((offset
+ size
) > entry
->size
) {
524 err_printk(dev
, "DMA-API: device driver syncs"
525 " DMA memory outside allocated range "
526 "[device address=0x%016llx] "
527 "[allocation size=%llu bytes] [sync offset=%llu] "
528 "[sync size=%llu]\n", entry
->dev_addr
, entry
->size
,
532 if (direction
!= entry
->direction
) {
533 err_printk(dev
, "DMA-API: device driver syncs "
534 "DMA memory with different direction "
535 "[device address=0x%016llx] [size=%llu bytes] "
536 "[mapped with %s] [synced with %s]\n",
538 dir2name
[entry
->direction
],
539 dir2name
[direction
]);
542 if (entry
->direction
== DMA_BIDIRECTIONAL
)
545 if (to_cpu
&& !(entry
->direction
== DMA_FROM_DEVICE
) &&
546 !(direction
== DMA_TO_DEVICE
))
547 err_printk(dev
, "DMA-API: device driver syncs "
548 "device read-only DMA memory for cpu "
549 "[device address=0x%016llx] [size=%llu bytes] "
550 "[mapped with %s] [synced with %s]\n",
552 dir2name
[entry
->direction
],
553 dir2name
[direction
]);
555 if (!to_cpu
&& !(entry
->direction
== DMA_TO_DEVICE
) &&
556 !(direction
== DMA_FROM_DEVICE
))
557 err_printk(dev
, "DMA-API: device driver syncs "
558 "device write-only DMA memory to device "
559 "[device address=0x%016llx] [size=%llu bytes] "
560 "[mapped with %s] [synced with %s]\n",
562 dir2name
[entry
->direction
],
563 dir2name
[direction
]);
566 put_hash_bucket(bucket
, &flags
);
570 void debug_dma_map_page(struct device
*dev
, struct page
*page
, size_t offset
,
571 size_t size
, int direction
, dma_addr_t dma_addr
,
574 struct dma_debug_entry
*entry
;
576 if (unlikely(global_disable
))
579 if (unlikely(dma_mapping_error(dev
, dma_addr
)))
582 entry
= dma_entry_alloc();
587 entry
->type
= dma_debug_page
;
588 entry
->paddr
= page_to_phys(page
) + offset
;
589 entry
->dev_addr
= dma_addr
;
591 entry
->direction
= direction
;
594 entry
->type
= dma_debug_single
;
595 check_for_stack(dev
, page_address(page
) + offset
);
598 add_dma_entry(entry
);
600 EXPORT_SYMBOL(debug_dma_map_page
);
602 void debug_dma_unmap_page(struct device
*dev
, dma_addr_t addr
,
603 size_t size
, int direction
, bool map_single
)
605 struct dma_debug_entry ref
= {
606 .type
= dma_debug_page
,
610 .direction
= direction
,
613 if (unlikely(global_disable
))
617 ref
.type
= dma_debug_single
;
621 EXPORT_SYMBOL(debug_dma_unmap_page
);
623 void debug_dma_map_sg(struct device
*dev
, struct scatterlist
*sg
,
624 int nents
, int mapped_ents
, int direction
)
626 struct dma_debug_entry
*entry
;
627 struct scatterlist
*s
;
630 if (unlikely(global_disable
))
633 for_each_sg(sg
, s
, mapped_ents
, i
) {
634 entry
= dma_entry_alloc();
638 entry
->type
= dma_debug_sg
;
640 entry
->paddr
= sg_phys(s
);
641 entry
->size
= s
->length
;
642 entry
->dev_addr
= s
->dma_address
;
643 entry
->direction
= direction
;
644 entry
->sg_call_ents
= nents
;
645 entry
->sg_mapped_ents
= mapped_ents
;
647 check_for_stack(dev
, sg_virt(s
));
649 add_dma_entry(entry
);
652 EXPORT_SYMBOL(debug_dma_map_sg
);
654 void debug_dma_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
657 struct dma_debug_entry
*entry
;
658 struct scatterlist
*s
;
659 int mapped_ents
= 0, i
;
662 if (unlikely(global_disable
))
665 for_each_sg(sglist
, s
, nelems
, i
) {
667 struct dma_debug_entry ref
= {
668 .type
= dma_debug_sg
,
671 .dev_addr
= s
->dma_address
,
677 if (mapped_ents
&& i
>= mapped_ents
)
680 if (mapped_ents
== 0) {
681 struct hash_bucket
*bucket
;
682 ref
.sg_call_ents
= nelems
;
683 bucket
= get_hash_bucket(&ref
, &flags
);
684 entry
= hash_bucket_find(bucket
, &ref
);
686 mapped_ents
= entry
->sg_mapped_ents
;
687 put_hash_bucket(bucket
, &flags
);
693 EXPORT_SYMBOL(debug_dma_unmap_sg
);
695 void debug_dma_alloc_coherent(struct device
*dev
, size_t size
,
696 dma_addr_t dma_addr
, void *virt
)
698 struct dma_debug_entry
*entry
;
700 if (unlikely(global_disable
))
703 if (unlikely(virt
== NULL
))
706 entry
= dma_entry_alloc();
710 entry
->type
= dma_debug_coherent
;
712 entry
->paddr
= virt_to_phys(virt
);
714 entry
->dev_addr
= dma_addr
;
715 entry
->direction
= DMA_BIDIRECTIONAL
;
717 add_dma_entry(entry
);
719 EXPORT_SYMBOL(debug_dma_alloc_coherent
);
721 void debug_dma_free_coherent(struct device
*dev
, size_t size
,
722 void *virt
, dma_addr_t addr
)
724 struct dma_debug_entry ref
= {
725 .type
= dma_debug_coherent
,
727 .paddr
= virt_to_phys(virt
),
730 .direction
= DMA_BIDIRECTIONAL
,
733 if (unlikely(global_disable
))
738 EXPORT_SYMBOL(debug_dma_free_coherent
);
740 void debug_dma_sync_single_for_cpu(struct device
*dev
, dma_addr_t dma_handle
,
741 size_t size
, int direction
)
743 if (unlikely(global_disable
))
746 check_sync(dev
, dma_handle
, size
, 0, direction
, true);
748 EXPORT_SYMBOL(debug_dma_sync_single_for_cpu
);
750 void debug_dma_sync_single_for_device(struct device
*dev
,
751 dma_addr_t dma_handle
, size_t size
,
754 if (unlikely(global_disable
))
757 check_sync(dev
, dma_handle
, size
, 0, direction
, false);
759 EXPORT_SYMBOL(debug_dma_sync_single_for_device
);