2 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
4 * Rewrite, cleanup, new allocation schemes, virtual merging:
5 * Copyright (C) 2004 Olof Johansson, IBM Corporation
6 * and Ben. Herrenschmidt, IBM Corporation
8 * Dynamic DMA mapping support, bus-independent parts.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 #include <linux/init.h>
27 #include <linux/types.h>
28 #include <linux/slab.h>
30 #include <linux/spinlock.h>
31 #include <linux/string.h>
32 #include <linux/dma-mapping.h>
33 #include <linux/bitops.h>
34 #include <linux/iommu-helper.h>
37 #include <asm/iommu.h>
38 #include <asm/pci-bridge.h>
39 #include <asm/machdep.h>
40 #include <asm/kdump.h>
44 #ifdef CONFIG_IOMMU_VMERGE
45 static int novmerge
= 0;
47 static int novmerge
= 1;
50 static int protect4gb
= 1;
52 static inline unsigned long iommu_num_pages(unsigned long vaddr
,
57 npages
= IOMMU_PAGE_ALIGN(vaddr
+ slen
) - (vaddr
& IOMMU_PAGE_MASK
);
58 npages
>>= IOMMU_PAGE_SHIFT
;
63 static int __init
setup_protect4gb(char *str
)
65 if (strcmp(str
, "on") == 0)
67 else if (strcmp(str
, "off") == 0)
73 static int __init
setup_iommu(char *str
)
75 if (!strcmp(str
, "novmerge"))
77 else if (!strcmp(str
, "vmerge"))
82 __setup("protect4gb=", setup_protect4gb
);
83 __setup("iommu=", setup_iommu
);
85 static unsigned long iommu_range_alloc(struct device
*dev
,
86 struct iommu_table
*tbl
,
88 unsigned long *handle
,
90 unsigned int align_order
)
92 unsigned long n
, end
, start
;
94 int largealloc
= npages
> 15;
96 unsigned long align_mask
;
97 unsigned long boundary_size
;
99 align_mask
= 0xffffffffffffffffl
>> (64 - align_order
);
101 /* This allocator was derived from x86_64's bit string search */
104 if (unlikely(npages
== 0)) {
105 if (printk_ratelimit())
107 return DMA_ERROR_CODE
;
110 if (handle
&& *handle
)
113 start
= largealloc
? tbl
->it_largehint
: tbl
->it_hint
;
115 /* Use only half of the table for small allocs (15 pages or less) */
116 limit
= largealloc
? tbl
->it_size
: tbl
->it_halfpoint
;
118 if (largealloc
&& start
< tbl
->it_halfpoint
)
119 start
= tbl
->it_halfpoint
;
121 /* The case below can happen if we have a small segment appended
122 * to a large, or when the previous alloc was at the very end of
123 * the available space. If so, go back to the initial start.
126 start
= largealloc
? tbl
->it_largehint
: tbl
->it_hint
;
130 if (limit
+ tbl
->it_offset
> mask
) {
131 limit
= mask
- tbl
->it_offset
+ 1;
132 /* If we're constrained on address range, first try
133 * at the masked hint to avoid O(n) search complexity,
134 * but on second pass, start at 0.
136 if ((start
& mask
) >= limit
|| pass
> 0)
143 boundary_size
= ALIGN(dma_get_seg_boundary(dev
) + 1,
144 1 << IOMMU_PAGE_SHIFT
);
146 boundary_size
= ALIGN(1UL << 32, 1 << IOMMU_PAGE_SHIFT
);
147 /* 4GB boundary for iseries_hv_alloc and iseries_hv_map */
149 n
= iommu_area_alloc(tbl
->it_map
, limit
, start
, npages
,
150 tbl
->it_offset
, boundary_size
>> IOMMU_PAGE_SHIFT
,
153 if (likely(pass
< 2)) {
154 /* First failure, just rescan the half of the table.
155 * Second failure, rescan the other half of the table.
157 start
= (largealloc
^ pass
) ? tbl
->it_halfpoint
: 0;
158 limit
= pass
? tbl
->it_size
: limit
;
162 /* Third failure, give up */
163 return DMA_ERROR_CODE
;
169 /* Bump the hint to a new block for small allocs. */
171 /* Don't bump to new block to avoid fragmentation */
172 tbl
->it_largehint
= end
;
174 /* Overflow will be taken care of at the next allocation */
175 tbl
->it_hint
= (end
+ tbl
->it_blocksize
- 1) &
176 ~(tbl
->it_blocksize
- 1);
179 /* Update handle for SG allocations */
186 static dma_addr_t
iommu_alloc(struct device
*dev
, struct iommu_table
*tbl
,
187 void *page
, unsigned int npages
,
188 enum dma_data_direction direction
,
189 unsigned long mask
, unsigned int align_order
,
190 struct dma_attrs
*attrs
)
192 unsigned long entry
, flags
;
193 dma_addr_t ret
= DMA_ERROR_CODE
;
195 spin_lock_irqsave(&(tbl
->it_lock
), flags
);
197 entry
= iommu_range_alloc(dev
, tbl
, npages
, NULL
, mask
, align_order
);
199 if (unlikely(entry
== DMA_ERROR_CODE
)) {
200 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
201 return DMA_ERROR_CODE
;
204 entry
+= tbl
->it_offset
; /* Offset into real TCE table */
205 ret
= entry
<< IOMMU_PAGE_SHIFT
; /* Set the return dma address */
207 /* Put the TCEs in the HW table */
208 ppc_md
.tce_build(tbl
, entry
, npages
, (unsigned long)page
& IOMMU_PAGE_MASK
,
212 /* Flush/invalidate TLB caches if necessary */
213 if (ppc_md
.tce_flush
)
214 ppc_md
.tce_flush(tbl
);
216 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
218 /* Make sure updates are seen by hardware */
224 static void __iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
227 unsigned long entry
, free_entry
;
229 entry
= dma_addr
>> IOMMU_PAGE_SHIFT
;
230 free_entry
= entry
- tbl
->it_offset
;
232 if (((free_entry
+ npages
) > tbl
->it_size
) ||
233 (entry
< tbl
->it_offset
)) {
234 if (printk_ratelimit()) {
235 printk(KERN_INFO
"iommu_free: invalid entry\n");
236 printk(KERN_INFO
"\tentry = 0x%lx\n", entry
);
237 printk(KERN_INFO
"\tdma_addr = 0x%lx\n", (u64
)dma_addr
);
238 printk(KERN_INFO
"\tTable = 0x%lx\n", (u64
)tbl
);
239 printk(KERN_INFO
"\tbus# = 0x%lx\n", (u64
)tbl
->it_busno
);
240 printk(KERN_INFO
"\tsize = 0x%lx\n", (u64
)tbl
->it_size
);
241 printk(KERN_INFO
"\tstartOff = 0x%lx\n", (u64
)tbl
->it_offset
);
242 printk(KERN_INFO
"\tindex = 0x%lx\n", (u64
)tbl
->it_index
);
248 ppc_md
.tce_free(tbl
, entry
, npages
);
249 iommu_area_free(tbl
->it_map
, free_entry
, npages
);
252 static void iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
257 spin_lock_irqsave(&(tbl
->it_lock
), flags
);
259 __iommu_free(tbl
, dma_addr
, npages
);
261 /* Make sure TLB cache is flushed if the HW needs it. We do
262 * not do an mb() here on purpose, it is not needed on any of
263 * the current platforms.
265 if (ppc_md
.tce_flush
)
266 ppc_md
.tce_flush(tbl
);
268 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
271 int iommu_map_sg(struct device
*dev
, struct iommu_table
*tbl
,
272 struct scatterlist
*sglist
, int nelems
,
273 unsigned long mask
, enum dma_data_direction direction
,
274 struct dma_attrs
*attrs
)
276 dma_addr_t dma_next
= 0, dma_addr
;
278 struct scatterlist
*s
, *outs
, *segstart
;
279 int outcount
, incount
, i
;
281 unsigned long handle
;
282 unsigned int max_seg_size
;
284 BUG_ON(direction
== DMA_NONE
);
286 if ((nelems
== 0) || !tbl
)
289 outs
= s
= segstart
= &sglist
[0];
294 /* Init first segment length for backout at failure */
295 outs
->dma_length
= 0;
297 DBG("sg mapping %d elements:\n", nelems
);
299 spin_lock_irqsave(&(tbl
->it_lock
), flags
);
301 max_seg_size
= dma_get_max_seg_size(dev
);
302 for_each_sg(sglist
, s
, nelems
, i
) {
303 unsigned long vaddr
, npages
, entry
, slen
;
311 /* Allocate iommu entries for that segment */
312 vaddr
= (unsigned long) sg_virt(s
);
313 npages
= iommu_num_pages(vaddr
, slen
);
315 if (IOMMU_PAGE_SHIFT
< PAGE_SHIFT
&& slen
>= PAGE_SIZE
&&
316 (vaddr
& ~PAGE_MASK
) == 0)
317 align
= PAGE_SHIFT
- IOMMU_PAGE_SHIFT
;
318 entry
= iommu_range_alloc(dev
, tbl
, npages
, &handle
,
319 mask
>> IOMMU_PAGE_SHIFT
, align
);
321 DBG(" - vaddr: %lx, size: %lx\n", vaddr
, slen
);
324 if (unlikely(entry
== DMA_ERROR_CODE
)) {
325 if (printk_ratelimit())
326 printk(KERN_INFO
"iommu_alloc failed, tbl %p vaddr %lx"
327 " npages %lx\n", tbl
, vaddr
, npages
);
331 /* Convert entry to a dma_addr_t */
332 entry
+= tbl
->it_offset
;
333 dma_addr
= entry
<< IOMMU_PAGE_SHIFT
;
334 dma_addr
|= (s
->offset
& ~IOMMU_PAGE_MASK
);
336 DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n",
337 npages
, entry
, dma_addr
);
339 /* Insert into HW table */
340 ppc_md
.tce_build(tbl
, entry
, npages
, vaddr
& IOMMU_PAGE_MASK
,
343 /* If we are in an open segment, try merging */
345 DBG(" - trying merge...\n");
346 /* We cannot merge if:
347 * - allocated dma_addr isn't contiguous to previous allocation
349 if (novmerge
|| (dma_addr
!= dma_next
) ||
350 (outs
->dma_length
+ s
->length
> max_seg_size
)) {
351 /* Can't merge: create a new segment */
354 outs
= sg_next(outs
);
355 DBG(" can't merge, new segment.\n");
357 outs
->dma_length
+= s
->length
;
358 DBG(" merged, new len: %ux\n", outs
->dma_length
);
363 /* This is a new segment, fill entries */
364 DBG(" - filling new segment.\n");
365 outs
->dma_address
= dma_addr
;
366 outs
->dma_length
= slen
;
369 /* Calculate next page pointer for contiguous check */
370 dma_next
= dma_addr
+ slen
;
372 DBG(" - dma next is: %lx\n", dma_next
);
375 /* Flush/invalidate TLB caches if necessary */
376 if (ppc_md
.tce_flush
)
377 ppc_md
.tce_flush(tbl
);
379 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
381 DBG("mapped %d elements:\n", outcount
);
383 /* For the sake of iommu_unmap_sg, we clear out the length in the
384 * next entry of the sglist if we didn't fill the list completely
386 if (outcount
< incount
) {
387 outs
= sg_next(outs
);
388 outs
->dma_address
= DMA_ERROR_CODE
;
389 outs
->dma_length
= 0;
392 /* Make sure updates are seen by hardware */
398 for_each_sg(sglist
, s
, nelems
, i
) {
399 if (s
->dma_length
!= 0) {
400 unsigned long vaddr
, npages
;
402 vaddr
= s
->dma_address
& IOMMU_PAGE_MASK
;
403 npages
= iommu_num_pages(s
->dma_address
, s
->dma_length
);
404 __iommu_free(tbl
, vaddr
, npages
);
405 s
->dma_address
= DMA_ERROR_CODE
;
411 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
416 void iommu_unmap_sg(struct iommu_table
*tbl
, struct scatterlist
*sglist
,
417 int nelems
, enum dma_data_direction direction
,
418 struct dma_attrs
*attrs
)
420 struct scatterlist
*sg
;
423 BUG_ON(direction
== DMA_NONE
);
428 spin_lock_irqsave(&(tbl
->it_lock
), flags
);
433 dma_addr_t dma_handle
= sg
->dma_address
;
435 if (sg
->dma_length
== 0)
437 npages
= iommu_num_pages(dma_handle
, sg
->dma_length
);
438 __iommu_free(tbl
, dma_handle
, npages
);
442 /* Flush/invalidate TLBs if necessary. As for iommu_free(), we
443 * do not do an mb() here, the affected platforms do not need it
446 if (ppc_md
.tce_flush
)
447 ppc_md
.tce_flush(tbl
);
449 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
453 * Build a iommu_table structure. This contains a bit map which
454 * is used to manage allocation of the tce space.
456 struct iommu_table
*iommu_init_table(struct iommu_table
*tbl
, int nid
)
459 static int welcomed
= 0;
462 /* Set aside 1/4 of the table for large allocations. */
463 tbl
->it_halfpoint
= tbl
->it_size
* 3 / 4;
465 /* number of bytes needed for the bitmap */
466 sz
= (tbl
->it_size
+ 7) >> 3;
468 page
= alloc_pages_node(nid
, GFP_ATOMIC
, get_order(sz
));
470 panic("iommu_init_table: Can't allocate %ld bytes\n", sz
);
471 tbl
->it_map
= page_address(page
);
472 memset(tbl
->it_map
, 0, sz
);
475 tbl
->it_largehint
= tbl
->it_halfpoint
;
476 spin_lock_init(&tbl
->it_lock
);
478 #ifdef CONFIG_CRASH_DUMP
479 if (ppc_md
.tce_get
) {
481 unsigned long tceval
;
482 unsigned long tcecount
= 0;
485 * Reserve the existing mappings left by the first kernel.
487 for (index
= 0; index
< tbl
->it_size
; index
++) {
488 tceval
= ppc_md
.tce_get(tbl
, index
+ tbl
->it_offset
);
490 * Freed TCE entry contains 0x7fffffffffffffff on JS20
492 if (tceval
&& (tceval
!= 0x7fffffffffffffffUL
)) {
493 __set_bit(index
, tbl
->it_map
);
497 if ((tbl
->it_size
- tcecount
) < KDUMP_MIN_TCE_ENTRIES
) {
498 printk(KERN_WARNING
"TCE table is full; ");
499 printk(KERN_WARNING
"freeing %d entries for the kdump boot\n",
500 KDUMP_MIN_TCE_ENTRIES
);
501 for (index
= tbl
->it_size
- KDUMP_MIN_TCE_ENTRIES
;
502 index
< tbl
->it_size
; index
++)
503 __clear_bit(index
, tbl
->it_map
);
507 /* Clear the hardware table in case firmware left allocations in it */
508 ppc_md
.tce_free(tbl
, tbl
->it_offset
, tbl
->it_size
);
512 printk(KERN_INFO
"IOMMU table initialized, virtual merging %s\n",
513 novmerge
? "disabled" : "enabled");
520 void iommu_free_table(struct iommu_table
*tbl
, const char *node_name
)
522 unsigned long bitmap_sz
, i
;
525 if (!tbl
|| !tbl
->it_map
) {
526 printk(KERN_ERR
"%s: expected TCE map for %s\n", __func__
,
531 /* verify that table contains no entries */
532 /* it_size is in entries, and we're examining 64 at a time */
533 for (i
= 0; i
< (tbl
->it_size
/64); i
++) {
534 if (tbl
->it_map
[i
] != 0) {
535 printk(KERN_WARNING
"%s: Unexpected TCEs for %s\n",
536 __func__
, node_name
);
541 /* calculate bitmap size in bytes */
542 bitmap_sz
= (tbl
->it_size
+ 7) / 8;
545 order
= get_order(bitmap_sz
);
546 free_pages((unsigned long) tbl
->it_map
, order
);
552 /* Creates TCEs for a user provided buffer. The user buffer must be
553 * contiguous real kernel storage (not vmalloc). The address of the buffer
554 * passed here is the kernel (virtual) address of the buffer. The buffer
555 * need not be page aligned, the dma_addr_t returned will point to the same
556 * byte within the page as vaddr.
558 dma_addr_t
iommu_map_single(struct device
*dev
, struct iommu_table
*tbl
,
559 void *vaddr
, size_t size
, unsigned long mask
,
560 enum dma_data_direction direction
, struct dma_attrs
*attrs
)
562 dma_addr_t dma_handle
= DMA_ERROR_CODE
;
564 unsigned int npages
, align
;
566 BUG_ON(direction
== DMA_NONE
);
568 uaddr
= (unsigned long)vaddr
;
569 npages
= iommu_num_pages(uaddr
, size
);
573 if (IOMMU_PAGE_SHIFT
< PAGE_SHIFT
&& size
>= PAGE_SIZE
&&
574 ((unsigned long)vaddr
& ~PAGE_MASK
) == 0)
575 align
= PAGE_SHIFT
- IOMMU_PAGE_SHIFT
;
577 dma_handle
= iommu_alloc(dev
, tbl
, vaddr
, npages
, direction
,
578 mask
>> IOMMU_PAGE_SHIFT
, align
,
580 if (dma_handle
== DMA_ERROR_CODE
) {
581 if (printk_ratelimit()) {
582 printk(KERN_INFO
"iommu_alloc failed, "
583 "tbl %p vaddr %p npages %d\n",
587 dma_handle
|= (uaddr
& ~IOMMU_PAGE_MASK
);
593 void iommu_unmap_single(struct iommu_table
*tbl
, dma_addr_t dma_handle
,
594 size_t size
, enum dma_data_direction direction
,
595 struct dma_attrs
*attrs
)
599 BUG_ON(direction
== DMA_NONE
);
602 npages
= iommu_num_pages(dma_handle
, size
);
603 iommu_free(tbl
, dma_handle
, npages
);
607 /* Allocates a contiguous real buffer and creates mappings over it.
608 * Returns the virtual address of the buffer and sets dma_handle
609 * to the dma address (mapping) of the first page.
611 void *iommu_alloc_coherent(struct device
*dev
, struct iommu_table
*tbl
,
612 size_t size
, dma_addr_t
*dma_handle
,
613 unsigned long mask
, gfp_t flag
, int node
)
618 unsigned int nio_pages
, io_order
;
621 size
= PAGE_ALIGN(size
);
622 order
= get_order(size
);
625 * Client asked for way too much space. This is checked later
626 * anyway. It is easier to debug here for the drivers than in
629 if (order
>= IOMAP_MAX_ORDER
) {
630 printk("iommu_alloc_consistent size too large: 0x%lx\n", size
);
637 /* Alloc enough pages (and possibly more) */
638 page
= alloc_pages_node(node
, flag
, order
);
641 ret
= page_address(page
);
642 memset(ret
, 0, size
);
644 /* Set up tces to cover the allocated range */
645 nio_pages
= size
>> IOMMU_PAGE_SHIFT
;
646 io_order
= get_iommu_order(size
);
647 mapping
= iommu_alloc(dev
, tbl
, ret
, nio_pages
, DMA_BIDIRECTIONAL
,
648 mask
>> IOMMU_PAGE_SHIFT
, io_order
, NULL
);
649 if (mapping
== DMA_ERROR_CODE
) {
650 free_pages((unsigned long)ret
, order
);
653 *dma_handle
= mapping
;
657 void iommu_free_coherent(struct iommu_table
*tbl
, size_t size
,
658 void *vaddr
, dma_addr_t dma_handle
)
661 unsigned int nio_pages
;
663 size
= PAGE_ALIGN(size
);
664 nio_pages
= size
>> IOMMU_PAGE_SHIFT
;
665 iommu_free(tbl
, dma_handle
, nio_pages
);
666 size
= PAGE_ALIGN(size
);
667 free_pages((unsigned long)vaddr
, get_order(size
));