Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
1beb6a7d | 2 | * arch/powerpc/sysdev/dart_iommu.c |
1da177e4 | 3 | * |
91f14480 | 4 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
1beb6a7d BH |
5 | * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>, |
6 | * IBM Corporation | |
1da177e4 LT |
7 | * |
8 | * Based on pSeries_iommu.c: | |
9 | * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation | |
91f14480 | 10 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
1da177e4 | 11 | * |
1beb6a7d BH |
12 | * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu. |
13 | * | |
1da177e4 | 14 | * |
1da177e4 LT |
15 | * This program is free software; you can redistribute it and/or modify |
16 | * it under the terms of the GNU General Public License as published by | |
17 | * the Free Software Foundation; either version 2 of the License, or | |
18 | * (at your option) any later version. | |
1beb6a7d | 19 | * |
1da177e4 LT |
20 | * This program is distributed in the hope that it will be useful, |
21 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
22 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
23 | * GNU General Public License for more details. | |
1beb6a7d | 24 | * |
1da177e4 LT |
25 | * You should have received a copy of the GNU General Public License |
26 | * along with this program; if not, write to the Free Software | |
27 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
28 | */ | |
29 | ||
1da177e4 LT |
30 | #include <linux/init.h> |
31 | #include <linux/types.h> | |
32 | #include <linux/slab.h> | |
33 | #include <linux/mm.h> | |
34 | #include <linux/spinlock.h> | |
35 | #include <linux/string.h> | |
36 | #include <linux/pci.h> | |
37 | #include <linux/dma-mapping.h> | |
38 | #include <linux/vmalloc.h> | |
39 | #include <asm/io.h> | |
40 | #include <asm/prom.h> | |
1da177e4 LT |
41 | #include <asm/iommu.h> |
42 | #include <asm/pci-bridge.h> | |
43 | #include <asm/machdep.h> | |
44 | #include <asm/abs_addr.h> | |
45 | #include <asm/cacheflush.h> | |
46 | #include <asm/lmb.h> | |
d387899f | 47 | #include <asm/ppc-pci.h> |
1da177e4 | 48 | |
9933f299 DG |
49 | #include "dart.h" |
50 | ||
28897731 | 51 | extern int iommu_is_off; |
1da177e4 LT |
52 | extern int iommu_force_on; |
53 | ||
1da177e4 LT |
54 | /* Physical base address and size of the DART table */ |
55 | unsigned long dart_tablebase; /* exported to htab_initialize */ | |
56 | static unsigned long dart_tablesize; | |
57 | ||
58 | /* Virtual base address of the DART table */ | |
59 | static u32 *dart_vbase; | |
60 | ||
61 | /* Mapped base address for the dart */ | |
6fa2ffe9 | 62 | static unsigned int __iomem *dart; |
1da177e4 LT |
63 | |
64 | /* Dummy val that entries are set to when unused */ | |
65 | static unsigned int dart_emptyval; | |
66 | ||
1beb6a7d BH |
67 | static struct iommu_table iommu_table_dart; |
68 | static int iommu_table_dart_inited; | |
1da177e4 | 69 | static int dart_dirty; |
1beb6a7d | 70 | static int dart_is_u4; |
1da177e4 LT |
71 | |
72 | #define DBG(...) | |
73 | ||
74 | static inline void dart_tlb_invalidate_all(void) | |
75 | { | |
76 | unsigned long l = 0; | |
1beb6a7d | 77 | unsigned int reg, inv_bit; |
1da177e4 LT |
78 | unsigned long limit; |
79 | ||
80 | DBG("dart: flush\n"); | |
81 | ||
82 | /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the | |
83 | * control register and wait for it to clear. | |
84 | * | |
85 | * Gotcha: Sometimes, the DART won't detect that the bit gets | |
86 | * set. If so, clear it and set it again. | |
1beb6a7d | 87 | */ |
1da177e4 LT |
88 | |
89 | limit = 0; | |
90 | ||
1beb6a7d | 91 | inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB; |
1da177e4 | 92 | retry: |
1da177e4 | 93 | l = 0; |
1beb6a7d BH |
94 | reg = DART_IN(DART_CNTL); |
95 | reg |= inv_bit; | |
96 | DART_OUT(DART_CNTL, reg); | |
97 | ||
98 | while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit)) | |
1da177e4 | 99 | l++; |
1beb6a7d | 100 | if (l == (1L << limit)) { |
1da177e4 LT |
101 | if (limit < 4) { |
102 | limit++; | |
feb76c7b OJ |
103 | reg = DART_IN(DART_CNTL); |
104 | reg &= ~inv_bit; | |
1beb6a7d | 105 | DART_OUT(DART_CNTL, reg); |
1da177e4 LT |
106 | goto retry; |
107 | } else | |
1beb6a7d | 108 | panic("DART: TLB did not flush after waiting a long " |
1da177e4 LT |
109 | "time. Buggy U3 ?"); |
110 | } | |
111 | } | |
112 | ||
feb76c7b OJ |
113 | static inline void dart_tlb_invalidate_one(unsigned long bus_rpn) |
114 | { | |
115 | unsigned int reg; | |
116 | unsigned int l, limit; | |
117 | ||
118 | reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE | | |
119 | (bus_rpn & DART_CNTL_U4_IONE_MASK); | |
120 | DART_OUT(DART_CNTL, reg); | |
121 | ||
122 | limit = 0; | |
123 | wait_more: | |
124 | l = 0; | |
125 | while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) { | |
126 | rmb(); | |
127 | l++; | |
128 | } | |
129 | ||
130 | if (l == (1L << limit)) { | |
131 | if (limit < 4) { | |
132 | limit++; | |
133 | goto wait_more; | |
134 | } else | |
135 | panic("DART: TLB did not flush after waiting a long " | |
136 | "time. Buggy U4 ?"); | |
137 | } | |
138 | } | |
139 | ||
1da177e4 LT |
140 | static void dart_flush(struct iommu_table *tbl) |
141 | { | |
feb76c7b | 142 | if (dart_dirty) { |
1da177e4 | 143 | dart_tlb_invalidate_all(); |
feb76c7b OJ |
144 | dart_dirty = 0; |
145 | } | |
1da177e4 LT |
146 | } |
147 | ||
1beb6a7d | 148 | static void dart_build(struct iommu_table *tbl, long index, |
1da177e4 LT |
149 | long npages, unsigned long uaddr, |
150 | enum dma_data_direction direction) | |
151 | { | |
152 | unsigned int *dp; | |
153 | unsigned int rpn; | |
feb76c7b | 154 | long l; |
1da177e4 LT |
155 | |
156 | DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr); | |
157 | ||
d0035c62 OJ |
158 | index <<= DART_PAGE_FACTOR; |
159 | npages <<= DART_PAGE_FACTOR; | |
160 | ||
1da177e4 | 161 | dp = ((unsigned int*)tbl->it_base) + index; |
1beb6a7d | 162 | |
1da177e4 LT |
163 | /* On U3, all memory is contigous, so we can move this |
164 | * out of the loop. | |
165 | */ | |
feb76c7b OJ |
166 | l = npages; |
167 | while (l--) { | |
d0035c62 | 168 | rpn = virt_to_abs(uaddr) >> DART_PAGE_SHIFT; |
1da177e4 LT |
169 | |
170 | *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK); | |
171 | ||
d0035c62 | 172 | uaddr += DART_PAGE_SIZE; |
1da177e4 LT |
173 | } |
174 | ||
feb76c7b OJ |
175 | if (dart_is_u4) { |
176 | rpn = index; | |
177 | mb(); /* make sure all updates have reached memory */ | |
178 | while (npages--) | |
179 | dart_tlb_invalidate_one(rpn++); | |
180 | } else { | |
181 | dart_dirty = 1; | |
182 | } | |
1da177e4 LT |
183 | } |
184 | ||
185 | ||
186 | static void dart_free(struct iommu_table *tbl, long index, long npages) | |
187 | { | |
188 | unsigned int *dp; | |
1beb6a7d | 189 | |
1da177e4 LT |
190 | /* We don't worry about flushing the TLB cache. The only drawback of |
191 | * not doing it is that we won't catch buggy device drivers doing | |
192 | * bad DMAs, but then no 32-bit architecture ever does either. | |
193 | */ | |
194 | ||
195 | DBG("dart: free at: %lx, %lx\n", index, npages); | |
196 | ||
d0035c62 OJ |
197 | index <<= DART_PAGE_FACTOR; |
198 | npages <<= DART_PAGE_FACTOR; | |
199 | ||
1da177e4 | 200 | dp = ((unsigned int *)tbl->it_base) + index; |
1beb6a7d | 201 | |
1da177e4 LT |
202 | while (npages--) |
203 | *(dp++) = dart_emptyval; | |
204 | } | |
205 | ||
206 | ||
207 | static int dart_init(struct device_node *dart_node) | |
208 | { | |
1da177e4 | 209 | unsigned int i; |
1beb6a7d BH |
210 | unsigned long tmp, base, size; |
211 | struct resource r; | |
1da177e4 LT |
212 | |
213 | if (dart_tablebase == 0 || dart_tablesize == 0) { | |
1beb6a7d BH |
214 | printk(KERN_INFO "DART: table not allocated, using " |
215 | "direct DMA\n"); | |
1da177e4 LT |
216 | return -ENODEV; |
217 | } | |
218 | ||
1beb6a7d BH |
219 | if (of_address_to_resource(dart_node, 0, &r)) |
220 | panic("DART: can't get register base ! "); | |
221 | ||
1da177e4 LT |
222 | /* Make sure nothing from the DART range remains in the CPU cache |
223 | * from a previous mapping that existed before the kernel took | |
224 | * over | |
225 | */ | |
1beb6a7d BH |
226 | flush_dcache_phys_range(dart_tablebase, |
227 | dart_tablebase + dart_tablesize); | |
1da177e4 LT |
228 | |
229 | /* Allocate a spare page to map all invalid DART pages. We need to do | |
230 | * that to work around what looks like a problem with the HT bridge | |
231 | * prefetching into invalid pages and corrupting data | |
232 | */ | |
d0035c62 | 233 | tmp = lmb_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE); |
1beb6a7d BH |
234 | dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) & |
235 | DARTMAP_RPNMASK); | |
1da177e4 | 236 | |
1beb6a7d BH |
237 | /* Map in DART registers */ |
238 | dart = ioremap(r.start, r.end - r.start + 1); | |
1da177e4 | 239 | if (dart == NULL) |
1beb6a7d | 240 | panic("DART: Cannot map registers!"); |
1da177e4 | 241 | |
1beb6a7d | 242 | /* Map in DART table */ |
1da177e4 LT |
243 | dart_vbase = ioremap(virt_to_abs(dart_tablebase), dart_tablesize); |
244 | ||
245 | /* Fill initial table */ | |
246 | for (i = 0; i < dart_tablesize/4; i++) | |
247 | dart_vbase[i] = dart_emptyval; | |
248 | ||
249 | /* Initialize DART with table base and enable it. */ | |
1beb6a7d BH |
250 | base = dart_tablebase >> DART_PAGE_SHIFT; |
251 | size = dart_tablesize >> DART_PAGE_SHIFT; | |
252 | if (dart_is_u4) { | |
56c8eaee | 253 | size &= DART_SIZE_U4_SIZE_MASK; |
1beb6a7d BH |
254 | DART_OUT(DART_BASE_U4, base); |
255 | DART_OUT(DART_SIZE_U4, size); | |
256 | DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE); | |
257 | } else { | |
56c8eaee | 258 | size &= DART_CNTL_U3_SIZE_MASK; |
1beb6a7d BH |
259 | DART_OUT(DART_CNTL, |
260 | DART_CNTL_U3_ENABLE | | |
261 | (base << DART_CNTL_U3_BASE_SHIFT) | | |
262 | (size << DART_CNTL_U3_SIZE_SHIFT)); | |
263 | } | |
1da177e4 LT |
264 | |
265 | /* Invalidate DART to get rid of possible stale TLBs */ | |
266 | dart_tlb_invalidate_all(); | |
267 | ||
1beb6a7d BH |
268 | printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n", |
269 | dart_is_u4 ? "U4" : "U3"); | |
1da177e4 LT |
270 | |
271 | return 0; | |
272 | } | |
273 | ||
1beb6a7d | 274 | static void iommu_table_dart_setup(void) |
1da177e4 | 275 | { |
1beb6a7d BH |
276 | iommu_table_dart.it_busno = 0; |
277 | iommu_table_dart.it_offset = 0; | |
1da177e4 | 278 | /* it_size is in number of entries */ |
1beb6a7d | 279 | iommu_table_dart.it_size = (dart_tablesize / sizeof(u32)) >> DART_PAGE_FACTOR; |
1da177e4 LT |
280 | |
281 | /* Initialize the common IOMMU code */ | |
1beb6a7d BH |
282 | iommu_table_dart.it_base = (unsigned long)dart_vbase; |
283 | iommu_table_dart.it_index = 0; | |
284 | iommu_table_dart.it_blocksize = 1; | |
ca1588e7 | 285 | iommu_init_table(&iommu_table_dart, -1); |
1da177e4 LT |
286 | |
287 | /* Reserve the last page of the DART to avoid possible prefetch | |
288 | * past the DART mapped area | |
289 | */ | |
1beb6a7d | 290 | set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map); |
1da177e4 LT |
291 | } |
292 | ||
1beb6a7d | 293 | static void iommu_dev_setup_dart(struct pci_dev *dev) |
1da177e4 LT |
294 | { |
295 | struct device_node *dn; | |
296 | ||
297 | /* We only have one iommu table on the mac for now, which makes | |
298 | * things simple. Setup all PCI devices to point to this table | |
299 | * | |
300 | * We must use pci_device_to_OF_node() to make sure that | |
301 | * we get the real "final" pointer to the device in the | |
302 | * pci_dev sysdata and not the temporary PHB one | |
303 | */ | |
304 | dn = pci_device_to_OF_node(dev); | |
305 | ||
306 | if (dn) | |
1beb6a7d | 307 | PCI_DN(dn)->iommu_table = &iommu_table_dart; |
1da177e4 LT |
308 | } |
309 | ||
1beb6a7d | 310 | static void iommu_bus_setup_dart(struct pci_bus *bus) |
1da177e4 LT |
311 | { |
312 | struct device_node *dn; | |
313 | ||
1beb6a7d BH |
314 | if (!iommu_table_dart_inited) { |
315 | iommu_table_dart_inited = 1; | |
316 | iommu_table_dart_setup(); | |
1da177e4 LT |
317 | } |
318 | ||
319 | dn = pci_bus_to_OF_node(bus); | |
320 | ||
321 | if (dn) | |
1beb6a7d | 322 | PCI_DN(dn)->iommu_table = &iommu_table_dart; |
1da177e4 LT |
323 | } |
324 | ||
325 | static void iommu_dev_setup_null(struct pci_dev *dev) { } | |
326 | static void iommu_bus_setup_null(struct pci_bus *bus) { } | |
327 | ||
1beb6a7d | 328 | void iommu_init_early_dart(void) |
1da177e4 LT |
329 | { |
330 | struct device_node *dn; | |
331 | ||
332 | /* Find the DART in the device-tree */ | |
333 | dn = of_find_compatible_node(NULL, "dart", "u3-dart"); | |
1beb6a7d BH |
334 | if (dn == NULL) { |
335 | dn = of_find_compatible_node(NULL, "dart", "u4-dart"); | |
336 | if (dn == NULL) | |
337 | goto bail; | |
338 | dart_is_u4 = 1; | |
339 | } | |
1da177e4 LT |
340 | |
341 | /* Setup low level TCE operations for the core IOMMU code */ | |
342 | ppc_md.tce_build = dart_build; | |
343 | ppc_md.tce_free = dart_free; | |
344 | ppc_md.tce_flush = dart_flush; | |
345 | ||
346 | /* Initialize the DART HW */ | |
1beb6a7d BH |
347 | if (dart_init(dn) == 0) { |
348 | ppc_md.iommu_dev_setup = iommu_dev_setup_dart; | |
349 | ppc_md.iommu_bus_setup = iommu_bus_setup_dart; | |
1da177e4 LT |
350 | |
351 | /* Setup pci_dma ops */ | |
352 | pci_iommu_init(); | |
1beb6a7d BH |
353 | |
354 | return; | |
1da177e4 | 355 | } |
1beb6a7d BH |
356 | |
357 | bail: | |
358 | /* If init failed, use direct iommu and null setup functions */ | |
359 | ppc_md.iommu_dev_setup = iommu_dev_setup_null; | |
360 | ppc_md.iommu_bus_setup = iommu_bus_setup_null; | |
361 | ||
362 | /* Setup pci_dma ops */ | |
363 | pci_direct_iommu_init(); | |
1da177e4 LT |
364 | } |
365 | ||
366 | ||
1beb6a7d | 367 | void __init alloc_dart_table(void) |
1da177e4 | 368 | { |
28897731 | 369 | /* Only reserve DART space if machine has more than 1GB of RAM |
1da177e4 | 370 | * or if requested with iommu=on on cmdline. |
28897731 OJ |
371 | * |
372 | * 1GB of RAM is picked as limit because some default devices | |
373 | * (i.e. Airport Extreme) have 30 bit address range limits. | |
1da177e4 | 374 | */ |
28897731 OJ |
375 | |
376 | if (iommu_is_off) | |
377 | return; | |
378 | ||
379 | if (!iommu_force_on && lmb_end_of_DRAM() <= 0x40000000ull) | |
1da177e4 LT |
380 | return; |
381 | ||
382 | /* 512 pages (2MB) is max DART tablesize. */ | |
383 | dart_tablesize = 1UL << 21; | |
384 | /* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we | |
385 | * will blow up an entire large page anyway in the kernel mapping | |
386 | */ | |
387 | dart_tablebase = (unsigned long) | |
388 | abs_to_virt(lmb_alloc_base(1UL<<24, 1UL<<24, 0x80000000L)); | |
389 | ||
1beb6a7d | 390 | printk(KERN_INFO "DART table allocated at: %lx\n", dart_tablebase); |
1da177e4 | 391 | } |