Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* $Id: sbus.c,v 1.19 2002/01/23 11:27:32 davem Exp $ |
2 | * sbus.c: UltraSparc SBUS controller support. | |
3 | * | |
4 | * Copyright (C) 1999 David S. Miller (davem@redhat.com) | |
5 | */ | |
6 | ||
7 | #include <linux/kernel.h> | |
8 | #include <linux/types.h> | |
9 | #include <linux/mm.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/init.h> | |
13 | #include <linux/interrupt.h> | |
14 | ||
15 | #include <asm/page.h> | |
16 | #include <asm/sbus.h> | |
17 | #include <asm/io.h> | |
18 | #include <asm/upa.h> | |
19 | #include <asm/cache.h> | |
20 | #include <asm/dma.h> | |
21 | #include <asm/irq.h> | |
22 | #include <asm/starfire.h> | |
23 | ||
24 | #include "iommu_common.h" | |
25 | ||
26 | /* These should be allocated on an SMP_CACHE_BYTES | |
27 | * aligned boundary for optimal performance. | |
28 | * | |
29 | * On SYSIO, using an 8K page size we have 1GB of SBUS | |
30 | * DMA space mapped. We divide this space into equally | |
31 | * sized clusters. We allocate a DMA mapping from the | |
32 | * cluster that matches the order of the allocation, or | |
33 | * if the order is greater than the number of clusters, | |
34 | * we try to allocate from the last cluster. | |
35 | */ | |
36 | ||
37 | #define NCLUSTERS 8UL | |
38 | #define ONE_GIG (1UL * 1024UL * 1024UL * 1024UL) | |
39 | #define CLUSTER_SIZE (ONE_GIG / NCLUSTERS) | |
40 | #define CLUSTER_MASK (CLUSTER_SIZE - 1) | |
41 | #define CLUSTER_NPAGES (CLUSTER_SIZE >> IO_PAGE_SHIFT) | |
42 | #define MAP_BASE ((u32)0xc0000000) | |
43 | ||
44 | struct sbus_iommu { | |
45 | /*0x00*/spinlock_t lock; | |
46 | ||
47 | /*0x08*/iopte_t *page_table; | |
48 | /*0x10*/unsigned long strbuf_regs; | |
49 | /*0x18*/unsigned long iommu_regs; | |
50 | /*0x20*/unsigned long sbus_control_reg; | |
51 | ||
52 | /*0x28*/volatile unsigned long strbuf_flushflag; | |
53 | ||
54 | /* If NCLUSTERS is ever decresed to 4 or lower, | |
55 | * you must increase the size of the type of | |
56 | * these counters. You have been duly warned. -DaveM | |
57 | */ | |
58 | /*0x30*/struct { | |
59 | u16 next; | |
60 | u16 flush; | |
61 | } alloc_info[NCLUSTERS]; | |
62 | ||
63 | /* The lowest used consistent mapping entry. Since | |
64 | * we allocate consistent maps out of cluster 0 this | |
65 | * is relative to the beginning of closter 0. | |
66 | */ | |
67 | /*0x50*/u32 lowest_consistent_map; | |
68 | }; | |
69 | ||
70 | /* Offsets from iommu_regs */ | |
71 | #define SYSIO_IOMMUREG_BASE 0x2400UL | |
72 | #define IOMMU_CONTROL (0x2400UL - 0x2400UL) /* IOMMU control register */ | |
73 | #define IOMMU_TSBBASE (0x2408UL - 0x2400UL) /* TSB base address register */ | |
74 | #define IOMMU_FLUSH (0x2410UL - 0x2400UL) /* IOMMU flush register */ | |
75 | #define IOMMU_VADIAG (0x4400UL - 0x2400UL) /* SBUS virtual address diagnostic */ | |
76 | #define IOMMU_TAGCMP (0x4408UL - 0x2400UL) /* TLB tag compare diagnostics */ | |
77 | #define IOMMU_LRUDIAG (0x4500UL - 0x2400UL) /* IOMMU LRU queue diagnostics */ | |
78 | #define IOMMU_TAGDIAG (0x4580UL - 0x2400UL) /* TLB tag diagnostics */ | |
79 | #define IOMMU_DRAMDIAG (0x4600UL - 0x2400UL) /* TLB data RAM diagnostics */ | |
80 | ||
81 | #define IOMMU_DRAM_VALID (1UL << 30UL) | |
82 | ||
83 | static void __iommu_flushall(struct sbus_iommu *iommu) | |
84 | { | |
85 | unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG; | |
86 | int entry; | |
87 | ||
88 | for (entry = 0; entry < 16; entry++) { | |
89 | upa_writeq(0, tag); | |
90 | tag += 8UL; | |
91 | } | |
92 | upa_readq(iommu->sbus_control_reg); | |
93 | ||
94 | for (entry = 0; entry < NCLUSTERS; entry++) { | |
95 | iommu->alloc_info[entry].flush = | |
96 | iommu->alloc_info[entry].next; | |
97 | } | |
98 | } | |
99 | ||
100 | static void iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages) | |
101 | { | |
102 | while (npages--) | |
103 | upa_writeq(base + (npages << IO_PAGE_SHIFT), | |
104 | iommu->iommu_regs + IOMMU_FLUSH); | |
105 | upa_readq(iommu->sbus_control_reg); | |
106 | } | |
107 | ||
108 | /* Offsets from strbuf_regs */ | |
109 | #define SYSIO_STRBUFREG_BASE 0x2800UL | |
110 | #define STRBUF_CONTROL (0x2800UL - 0x2800UL) /* Control */ | |
111 | #define STRBUF_PFLUSH (0x2808UL - 0x2800UL) /* Page flush/invalidate */ | |
112 | #define STRBUF_FSYNC (0x2810UL - 0x2800UL) /* Flush synchronization */ | |
113 | #define STRBUF_DRAMDIAG (0x5000UL - 0x2800UL) /* data RAM diagnostic */ | |
114 | #define STRBUF_ERRDIAG (0x5400UL - 0x2800UL) /* error status diagnostics */ | |
115 | #define STRBUF_PTAGDIAG (0x5800UL - 0x2800UL) /* Page tag diagnostics */ | |
116 | #define STRBUF_LTAGDIAG (0x5900UL - 0x2800UL) /* Line tag diagnostics */ | |
117 | ||
118 | #define STRBUF_TAG_VALID 0x02UL | |
119 | ||
7c963ad1 | 120 | static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages, int direction) |
1da177e4 | 121 | { |
4dbc30fb DM |
122 | unsigned long n; |
123 | int limit; | |
124 | ||
4dbc30fb DM |
125 | n = npages; |
126 | while (n--) | |
127 | upa_writeq(base + (n << IO_PAGE_SHIFT), | |
1da177e4 LT |
128 | iommu->strbuf_regs + STRBUF_PFLUSH); |
129 | ||
7c963ad1 DM |
130 | /* If the device could not have possibly put dirty data into |
131 | * the streaming cache, no flush-flag synchronization needs | |
132 | * to be performed. | |
133 | */ | |
134 | if (direction == SBUS_DMA_TODEVICE) | |
135 | return; | |
136 | ||
137 | iommu->strbuf_flushflag = 0UL; | |
138 | ||
1da177e4 LT |
139 | /* Whoopee cushion! */ |
140 | upa_writeq(__pa(&iommu->strbuf_flushflag), | |
141 | iommu->strbuf_regs + STRBUF_FSYNC); | |
142 | upa_readq(iommu->sbus_control_reg); | |
4dbc30fb | 143 | |
a228dfd5 | 144 | limit = 100000; |
4dbc30fb DM |
145 | while (iommu->strbuf_flushflag == 0UL) { |
146 | limit--; | |
147 | if (!limit) | |
148 | break; | |
a228dfd5 | 149 | udelay(1); |
4f07118f | 150 | rmb(); |
4dbc30fb DM |
151 | } |
152 | if (!limit) | |
153 | printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout " | |
154 | "vaddr[%08x] npages[%ld]\n", | |
155 | base, npages); | |
1da177e4 LT |
156 | } |
157 | ||
158 | static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages) | |
159 | { | |
160 | iopte_t *iopte, *limit, *first, *cluster; | |
161 | unsigned long cnum, ent, nent, flush_point, found; | |
162 | ||
163 | cnum = 0; | |
164 | nent = 1; | |
165 | while ((1UL << cnum) < npages) | |
166 | cnum++; | |
167 | if(cnum >= NCLUSTERS) { | |
168 | nent = 1UL << (cnum - NCLUSTERS); | |
169 | cnum = NCLUSTERS - 1; | |
170 | } | |
171 | iopte = iommu->page_table + (cnum * CLUSTER_NPAGES); | |
172 | ||
173 | if (cnum == 0) | |
174 | limit = (iommu->page_table + | |
175 | iommu->lowest_consistent_map); | |
176 | else | |
177 | limit = (iopte + CLUSTER_NPAGES); | |
178 | ||
179 | iopte += ((ent = iommu->alloc_info[cnum].next) << cnum); | |
180 | flush_point = iommu->alloc_info[cnum].flush; | |
181 | ||
182 | first = iopte; | |
183 | cluster = NULL; | |
184 | found = 0; | |
185 | for (;;) { | |
186 | if (iopte_val(*iopte) == 0UL) { | |
187 | found++; | |
188 | if (!cluster) | |
189 | cluster = iopte; | |
190 | } else { | |
191 | /* Used cluster in the way */ | |
192 | cluster = NULL; | |
193 | found = 0; | |
194 | } | |
195 | ||
196 | if (found == nent) | |
197 | break; | |
198 | ||
199 | iopte += (1 << cnum); | |
200 | ent++; | |
201 | if (iopte >= limit) { | |
202 | iopte = (iommu->page_table + (cnum * CLUSTER_NPAGES)); | |
203 | ent = 0; | |
204 | ||
205 | /* Multiple cluster allocations must not wrap */ | |
206 | cluster = NULL; | |
207 | found = 0; | |
208 | } | |
209 | if (ent == flush_point) | |
210 | __iommu_flushall(iommu); | |
211 | if (iopte == first) | |
212 | goto bad; | |
213 | } | |
214 | ||
215 | /* ent/iopte points to the last cluster entry we're going to use, | |
216 | * so save our place for the next allocation. | |
217 | */ | |
218 | if ((iopte + (1 << cnum)) >= limit) | |
219 | ent = 0; | |
220 | else | |
221 | ent = ent + 1; | |
222 | iommu->alloc_info[cnum].next = ent; | |
223 | if (ent == flush_point) | |
224 | __iommu_flushall(iommu); | |
225 | ||
226 | /* I've got your streaming cluster right here buddy boy... */ | |
227 | return cluster; | |
228 | ||
229 | bad: | |
230 | printk(KERN_EMERG "sbus: alloc_streaming_cluster of npages(%ld) failed!\n", | |
231 | npages); | |
232 | return NULL; | |
233 | } | |
234 | ||
235 | static void free_streaming_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages) | |
236 | { | |
237 | unsigned long cnum, ent, nent; | |
238 | iopte_t *iopte; | |
239 | ||
240 | cnum = 0; | |
241 | nent = 1; | |
242 | while ((1UL << cnum) < npages) | |
243 | cnum++; | |
244 | if(cnum >= NCLUSTERS) { | |
245 | nent = 1UL << (cnum - NCLUSTERS); | |
246 | cnum = NCLUSTERS - 1; | |
247 | } | |
248 | ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum); | |
249 | iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT); | |
250 | do { | |
251 | iopte_val(*iopte) = 0UL; | |
252 | iopte += 1 << cnum; | |
253 | } while(--nent); | |
254 | ||
255 | /* If the global flush might not have caught this entry, | |
256 | * adjust the flush point such that we will flush before | |
257 | * ever trying to reuse it. | |
258 | */ | |
259 | #define between(X,Y,Z) (((Z) - (Y)) >= ((X) - (Y))) | |
260 | if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush)) | |
261 | iommu->alloc_info[cnum].flush = ent; | |
262 | #undef between | |
263 | } | |
264 | ||
265 | /* We allocate consistent mappings from the end of cluster zero. */ | |
266 | static iopte_t *alloc_consistent_cluster(struct sbus_iommu *iommu, unsigned long npages) | |
267 | { | |
268 | iopte_t *iopte; | |
269 | ||
270 | iopte = iommu->page_table + (1 * CLUSTER_NPAGES); | |
271 | while (iopte > iommu->page_table) { | |
272 | iopte--; | |
273 | if (!(iopte_val(*iopte) & IOPTE_VALID)) { | |
274 | unsigned long tmp = npages; | |
275 | ||
276 | while (--tmp) { | |
277 | iopte--; | |
278 | if (iopte_val(*iopte) & IOPTE_VALID) | |
279 | break; | |
280 | } | |
281 | if (tmp == 0) { | |
282 | u32 entry = (iopte - iommu->page_table); | |
283 | ||
284 | if (entry < iommu->lowest_consistent_map) | |
285 | iommu->lowest_consistent_map = entry; | |
286 | return iopte; | |
287 | } | |
288 | } | |
289 | } | |
290 | return NULL; | |
291 | } | |
292 | ||
293 | static void free_consistent_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages) | |
294 | { | |
295 | iopte_t *iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT); | |
296 | ||
297 | if ((iopte - iommu->page_table) == iommu->lowest_consistent_map) { | |
298 | iopte_t *walk = iopte + npages; | |
299 | iopte_t *limit; | |
300 | ||
301 | limit = iommu->page_table + CLUSTER_NPAGES; | |
302 | while (walk < limit) { | |
303 | if (iopte_val(*walk) != 0UL) | |
304 | break; | |
305 | walk++; | |
306 | } | |
307 | iommu->lowest_consistent_map = | |
308 | (walk - iommu->page_table); | |
309 | } | |
310 | ||
311 | while (npages--) | |
312 | *iopte++ = __iopte(0UL); | |
313 | } | |
314 | ||
315 | void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma_addr) | |
316 | { | |
317 | unsigned long order, first_page, flags; | |
318 | struct sbus_iommu *iommu; | |
319 | iopte_t *iopte; | |
320 | void *ret; | |
321 | int npages; | |
322 | ||
323 | if (size <= 0 || sdev == NULL || dvma_addr == NULL) | |
324 | return NULL; | |
325 | ||
326 | size = IO_PAGE_ALIGN(size); | |
327 | order = get_order(size); | |
328 | if (order >= 10) | |
329 | return NULL; | |
330 | first_page = __get_free_pages(GFP_KERNEL, order); | |
331 | if (first_page == 0UL) | |
332 | return NULL; | |
333 | memset((char *)first_page, 0, PAGE_SIZE << order); | |
334 | ||
335 | iommu = sdev->bus->iommu; | |
336 | ||
337 | spin_lock_irqsave(&iommu->lock, flags); | |
338 | iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT); | |
339 | if (iopte == NULL) { | |
340 | spin_unlock_irqrestore(&iommu->lock, flags); | |
341 | free_pages(first_page, order); | |
342 | return NULL; | |
343 | } | |
344 | ||
345 | /* Ok, we're committed at this point. */ | |
346 | *dvma_addr = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); | |
347 | ret = (void *) first_page; | |
348 | npages = size >> IO_PAGE_SHIFT; | |
349 | while (npages--) { | |
350 | *iopte++ = __iopte(IOPTE_VALID | IOPTE_CACHE | IOPTE_WRITE | | |
351 | (__pa(first_page) & IOPTE_PAGE)); | |
352 | first_page += IO_PAGE_SIZE; | |
353 | } | |
354 | iommu_flush(iommu, *dvma_addr, size >> IO_PAGE_SHIFT); | |
355 | spin_unlock_irqrestore(&iommu->lock, flags); | |
356 | ||
357 | return ret; | |
358 | } | |
359 | ||
360 | void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_addr_t dvma) | |
361 | { | |
362 | unsigned long order, npages; | |
363 | struct sbus_iommu *iommu; | |
364 | ||
365 | if (size <= 0 || sdev == NULL || cpu == NULL) | |
366 | return; | |
367 | ||
368 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | |
369 | iommu = sdev->bus->iommu; | |
370 | ||
371 | spin_lock_irq(&iommu->lock); | |
372 | free_consistent_cluster(iommu, dvma, npages); | |
373 | iommu_flush(iommu, dvma, npages); | |
374 | spin_unlock_irq(&iommu->lock); | |
375 | ||
376 | order = get_order(size); | |
377 | if (order < 10) | |
378 | free_pages((unsigned long)cpu, order); | |
379 | } | |
380 | ||
381 | dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t size, int dir) | |
382 | { | |
383 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
384 | unsigned long npages, pbase, flags; | |
385 | iopte_t *iopte; | |
386 | u32 dma_base, offset; | |
387 | unsigned long iopte_bits; | |
388 | ||
389 | if (dir == SBUS_DMA_NONE) | |
390 | BUG(); | |
391 | ||
392 | pbase = (unsigned long) ptr; | |
393 | offset = (u32) (pbase & ~IO_PAGE_MASK); | |
394 | size = (IO_PAGE_ALIGN(pbase + size) - (pbase & IO_PAGE_MASK)); | |
395 | pbase = (unsigned long) __pa(pbase & IO_PAGE_MASK); | |
396 | ||
397 | spin_lock_irqsave(&iommu->lock, flags); | |
398 | npages = size >> IO_PAGE_SHIFT; | |
399 | iopte = alloc_streaming_cluster(iommu, npages); | |
400 | if (iopte == NULL) | |
401 | goto bad; | |
402 | dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); | |
403 | npages = size >> IO_PAGE_SHIFT; | |
404 | iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; | |
405 | if (dir != SBUS_DMA_TODEVICE) | |
406 | iopte_bits |= IOPTE_WRITE; | |
407 | while (npages--) { | |
408 | *iopte++ = __iopte(iopte_bits | (pbase & IOPTE_PAGE)); | |
409 | pbase += IO_PAGE_SIZE; | |
410 | } | |
411 | npages = size >> IO_PAGE_SHIFT; | |
412 | spin_unlock_irqrestore(&iommu->lock, flags); | |
413 | ||
414 | return (dma_base | offset); | |
415 | ||
416 | bad: | |
417 | spin_unlock_irqrestore(&iommu->lock, flags); | |
418 | BUG(); | |
419 | return 0; | |
420 | } | |
421 | ||
422 | void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t dma_addr, size_t size, int direction) | |
423 | { | |
424 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
425 | u32 dma_base = dma_addr & IO_PAGE_MASK; | |
426 | unsigned long flags; | |
427 | ||
428 | size = (IO_PAGE_ALIGN(dma_addr + size) - dma_base); | |
429 | ||
430 | spin_lock_irqsave(&iommu->lock, flags); | |
431 | free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT); | |
7c963ad1 | 432 | sbus_strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT, direction); |
1da177e4 LT |
433 | spin_unlock_irqrestore(&iommu->lock, flags); |
434 | } | |
435 | ||
436 | #define SG_ENT_PHYS_ADDRESS(SG) \ | |
437 | (__pa(page_address((SG)->page)) + (SG)->offset) | |
438 | ||
439 | static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, int nelems, unsigned long iopte_bits) | |
440 | { | |
441 | struct scatterlist *dma_sg = sg; | |
442 | struct scatterlist *sg_end = sg + nelems; | |
443 | int i; | |
444 | ||
445 | for (i = 0; i < nused; i++) { | |
446 | unsigned long pteval = ~0UL; | |
447 | u32 dma_npages; | |
448 | ||
449 | dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) + | |
450 | dma_sg->dma_length + | |
451 | ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT; | |
452 | do { | |
453 | unsigned long offset; | |
454 | signed int len; | |
455 | ||
456 | /* If we are here, we know we have at least one | |
457 | * more page to map. So walk forward until we | |
458 | * hit a page crossing, and begin creating new | |
459 | * mappings from that spot. | |
460 | */ | |
461 | for (;;) { | |
462 | unsigned long tmp; | |
463 | ||
464 | tmp = (unsigned long) SG_ENT_PHYS_ADDRESS(sg); | |
465 | len = sg->length; | |
466 | if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) { | |
467 | pteval = tmp & IO_PAGE_MASK; | |
468 | offset = tmp & (IO_PAGE_SIZE - 1UL); | |
469 | break; | |
470 | } | |
471 | if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) { | |
472 | pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK; | |
473 | offset = 0UL; | |
474 | len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL))); | |
475 | break; | |
476 | } | |
477 | sg++; | |
478 | } | |
479 | ||
480 | pteval = ((pteval & IOPTE_PAGE) | iopte_bits); | |
481 | while (len > 0) { | |
482 | *iopte++ = __iopte(pteval); | |
483 | pteval += IO_PAGE_SIZE; | |
484 | len -= (IO_PAGE_SIZE - offset); | |
485 | offset = 0; | |
486 | dma_npages--; | |
487 | } | |
488 | ||
489 | pteval = (pteval & IOPTE_PAGE) + len; | |
490 | sg++; | |
491 | ||
492 | /* Skip over any tail mappings we've fully mapped, | |
493 | * adjusting pteval along the way. Stop when we | |
494 | * detect a page crossing event. | |
495 | */ | |
496 | while (sg < sg_end && | |
497 | (pteval << (64 - IO_PAGE_SHIFT)) != 0UL && | |
498 | (pteval == SG_ENT_PHYS_ADDRESS(sg)) && | |
499 | ((pteval ^ | |
500 | (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) { | |
501 | pteval += sg->length; | |
502 | sg++; | |
503 | } | |
504 | if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL) | |
505 | pteval = ~0UL; | |
506 | } while (dma_npages != 0); | |
507 | dma_sg++; | |
508 | } | |
509 | } | |
510 | ||
511 | int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int dir) | |
512 | { | |
513 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
514 | unsigned long flags, npages; | |
515 | iopte_t *iopte; | |
516 | u32 dma_base; | |
517 | struct scatterlist *sgtmp; | |
518 | int used; | |
519 | unsigned long iopte_bits; | |
520 | ||
521 | if (dir == SBUS_DMA_NONE) | |
522 | BUG(); | |
523 | ||
524 | /* Fast path single entry scatterlists. */ | |
525 | if (nents == 1) { | |
526 | sg->dma_address = | |
527 | sbus_map_single(sdev, | |
528 | (page_address(sg->page) + sg->offset), | |
529 | sg->length, dir); | |
530 | sg->dma_length = sg->length; | |
531 | return 1; | |
532 | } | |
533 | ||
534 | npages = prepare_sg(sg, nents); | |
535 | ||
536 | spin_lock_irqsave(&iommu->lock, flags); | |
537 | iopte = alloc_streaming_cluster(iommu, npages); | |
538 | if (iopte == NULL) | |
539 | goto bad; | |
540 | dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); | |
541 | ||
542 | /* Normalize DVMA addresses. */ | |
543 | sgtmp = sg; | |
544 | used = nents; | |
545 | ||
546 | while (used && sgtmp->dma_length) { | |
547 | sgtmp->dma_address += dma_base; | |
548 | sgtmp++; | |
549 | used--; | |
550 | } | |
551 | used = nents - used; | |
552 | ||
553 | iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; | |
554 | if (dir != SBUS_DMA_TODEVICE) | |
555 | iopte_bits |= IOPTE_WRITE; | |
556 | ||
557 | fill_sg(iopte, sg, used, nents, iopte_bits); | |
558 | #ifdef VERIFY_SG | |
559 | verify_sglist(sg, nents, iopte, npages); | |
560 | #endif | |
561 | spin_unlock_irqrestore(&iommu->lock, flags); | |
562 | ||
563 | return used; | |
564 | ||
565 | bad: | |
566 | spin_unlock_irqrestore(&iommu->lock, flags); | |
567 | BUG(); | |
568 | return 0; | |
569 | } | |
570 | ||
571 | void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) | |
572 | { | |
573 | unsigned long size, flags; | |
574 | struct sbus_iommu *iommu; | |
575 | u32 dvma_base; | |
576 | int i; | |
577 | ||
578 | /* Fast path single entry scatterlists. */ | |
579 | if (nents == 1) { | |
580 | sbus_unmap_single(sdev, sg->dma_address, sg->dma_length, direction); | |
581 | return; | |
582 | } | |
583 | ||
584 | dvma_base = sg[0].dma_address & IO_PAGE_MASK; | |
585 | for (i = 0; i < nents; i++) { | |
586 | if (sg[i].dma_length == 0) | |
587 | break; | |
588 | } | |
589 | i--; | |
590 | size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - dvma_base; | |
591 | ||
592 | iommu = sdev->bus->iommu; | |
593 | spin_lock_irqsave(&iommu->lock, flags); | |
594 | free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT); | |
7c963ad1 | 595 | sbus_strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT, direction); |
1da177e4 LT |
596 | spin_unlock_irqrestore(&iommu->lock, flags); |
597 | } | |
598 | ||
599 | void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction) | |
600 | { | |
601 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
602 | unsigned long flags; | |
603 | ||
604 | size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK)); | |
605 | ||
606 | spin_lock_irqsave(&iommu->lock, flags); | |
7c963ad1 | 607 | sbus_strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT, direction); |
1da177e4 LT |
608 | spin_unlock_irqrestore(&iommu->lock, flags); |
609 | } | |
610 | ||
611 | void sbus_dma_sync_single_for_device(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction) | |
612 | { | |
613 | } | |
614 | ||
615 | void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) | |
616 | { | |
617 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
618 | unsigned long flags, size; | |
619 | u32 base; | |
620 | int i; | |
621 | ||
622 | base = sg[0].dma_address & IO_PAGE_MASK; | |
623 | for (i = 0; i < nents; i++) { | |
624 | if (sg[i].dma_length == 0) | |
625 | break; | |
626 | } | |
627 | i--; | |
628 | size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base; | |
629 | ||
630 | spin_lock_irqsave(&iommu->lock, flags); | |
7c963ad1 | 631 | sbus_strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT, direction); |
1da177e4 LT |
632 | spin_unlock_irqrestore(&iommu->lock, flags); |
633 | } | |
634 | ||
635 | void sbus_dma_sync_sg_for_device(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) | |
636 | { | |
637 | } | |
638 | ||
639 | /* Enable 64-bit DVMA mode for the given device. */ | |
640 | void sbus_set_sbus64(struct sbus_dev *sdev, int bursts) | |
641 | { | |
642 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
643 | int slot = sdev->slot; | |
644 | unsigned long cfg_reg; | |
645 | u64 val; | |
646 | ||
647 | cfg_reg = iommu->sbus_control_reg; | |
648 | switch (slot) { | |
649 | case 0: | |
650 | cfg_reg += 0x20UL; | |
651 | break; | |
652 | case 1: | |
653 | cfg_reg += 0x28UL; | |
654 | break; | |
655 | case 2: | |
656 | cfg_reg += 0x30UL; | |
657 | break; | |
658 | case 3: | |
659 | cfg_reg += 0x38UL; | |
660 | break; | |
661 | case 13: | |
662 | cfg_reg += 0x40UL; | |
663 | break; | |
664 | case 14: | |
665 | cfg_reg += 0x48UL; | |
666 | break; | |
667 | case 15: | |
668 | cfg_reg += 0x50UL; | |
669 | break; | |
670 | ||
671 | default: | |
672 | return; | |
673 | }; | |
674 | ||
675 | val = upa_readq(cfg_reg); | |
676 | if (val & (1UL << 14UL)) { | |
677 | /* Extended transfer mode already enabled. */ | |
678 | return; | |
679 | } | |
680 | ||
681 | val |= (1UL << 14UL); | |
682 | ||
683 | if (bursts & DMA_BURST8) | |
684 | val |= (1UL << 1UL); | |
685 | if (bursts & DMA_BURST16) | |
686 | val |= (1UL << 2UL); | |
687 | if (bursts & DMA_BURST32) | |
688 | val |= (1UL << 3UL); | |
689 | if (bursts & DMA_BURST64) | |
690 | val |= (1UL << 4UL); | |
691 | upa_writeq(val, cfg_reg); | |
692 | } | |
693 | ||
694 | /* SBUS SYSIO INO number to Sparc PIL level. */ | |
695 | static unsigned char sysio_ino_to_pil[] = { | |
696 | 0, 4, 4, 7, 5, 7, 8, 9, /* SBUS slot 0 */ | |
697 | 0, 4, 4, 7, 5, 7, 8, 9, /* SBUS slot 1 */ | |
698 | 0, 4, 4, 7, 5, 7, 8, 9, /* SBUS slot 2 */ | |
699 | 0, 4, 4, 7, 5, 7, 8, 9, /* SBUS slot 3 */ | |
700 | 4, /* Onboard SCSI */ | |
701 | 5, /* Onboard Ethernet */ | |
702 | /*XXX*/ 8, /* Onboard BPP */ | |
703 | 0, /* Bogon */ | |
704 | 13, /* Audio */ | |
705 | /*XXX*/15, /* PowerFail */ | |
706 | 0, /* Bogon */ | |
707 | 0, /* Bogon */ | |
708 | 12, /* Zilog Serial Channels (incl. Keyboard/Mouse lines) */ | |
709 | 11, /* Floppy */ | |
710 | 0, /* Spare Hardware (bogon for now) */ | |
711 | 0, /* Keyboard (bogon for now) */ | |
712 | 0, /* Mouse (bogon for now) */ | |
713 | 0, /* Serial (bogon for now) */ | |
714 | 0, 0, /* Bogon, Bogon */ | |
715 | 10, /* Timer 0 */ | |
716 | 11, /* Timer 1 */ | |
717 | 0, 0, /* Bogon, Bogon */ | |
718 | 15, /* Uncorrectable SBUS Error */ | |
719 | 15, /* Correctable SBUS Error */ | |
720 | 15, /* SBUS Error */ | |
721 | /*XXX*/ 0, /* Power Management (bogon for now) */ | |
722 | }; | |
723 | ||
724 | /* INO number to IMAP register offset for SYSIO external IRQ's. | |
725 | * This should conform to both Sunfire/Wildfire server and Fusion | |
726 | * desktop designs. | |
727 | */ | |
728 | #define SYSIO_IMAP_SLOT0 0x2c04UL | |
729 | #define SYSIO_IMAP_SLOT1 0x2c0cUL | |
730 | #define SYSIO_IMAP_SLOT2 0x2c14UL | |
731 | #define SYSIO_IMAP_SLOT3 0x2c1cUL | |
732 | #define SYSIO_IMAP_SCSI 0x3004UL | |
733 | #define SYSIO_IMAP_ETH 0x300cUL | |
734 | #define SYSIO_IMAP_BPP 0x3014UL | |
735 | #define SYSIO_IMAP_AUDIO 0x301cUL | |
736 | #define SYSIO_IMAP_PFAIL 0x3024UL | |
737 | #define SYSIO_IMAP_KMS 0x302cUL | |
738 | #define SYSIO_IMAP_FLPY 0x3034UL | |
739 | #define SYSIO_IMAP_SHW 0x303cUL | |
740 | #define SYSIO_IMAP_KBD 0x3044UL | |
741 | #define SYSIO_IMAP_MS 0x304cUL | |
742 | #define SYSIO_IMAP_SER 0x3054UL | |
743 | #define SYSIO_IMAP_TIM0 0x3064UL | |
744 | #define SYSIO_IMAP_TIM1 0x306cUL | |
745 | #define SYSIO_IMAP_UE 0x3074UL | |
746 | #define SYSIO_IMAP_CE 0x307cUL | |
747 | #define SYSIO_IMAP_SBERR 0x3084UL | |
748 | #define SYSIO_IMAP_PMGMT 0x308cUL | |
749 | #define SYSIO_IMAP_GFX 0x3094UL | |
750 | #define SYSIO_IMAP_EUPA 0x309cUL | |
751 | ||
752 | #define bogon ((unsigned long) -1) | |
753 | static unsigned long sysio_irq_offsets[] = { | |
754 | /* SBUS Slot 0 --> 3, level 1 --> 7 */ | |
755 | SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, | |
756 | SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, | |
757 | SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, | |
758 | SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, | |
759 | SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, | |
760 | SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, | |
761 | SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, | |
762 | SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, | |
763 | ||
764 | /* Onboard devices (not relevant/used on SunFire). */ | |
765 | SYSIO_IMAP_SCSI, | |
766 | SYSIO_IMAP_ETH, | |
767 | SYSIO_IMAP_BPP, | |
768 | bogon, | |
769 | SYSIO_IMAP_AUDIO, | |
770 | SYSIO_IMAP_PFAIL, | |
771 | bogon, | |
772 | bogon, | |
773 | SYSIO_IMAP_KMS, | |
774 | SYSIO_IMAP_FLPY, | |
775 | SYSIO_IMAP_SHW, | |
776 | SYSIO_IMAP_KBD, | |
777 | SYSIO_IMAP_MS, | |
778 | SYSIO_IMAP_SER, | |
779 | bogon, | |
780 | bogon, | |
781 | SYSIO_IMAP_TIM0, | |
782 | SYSIO_IMAP_TIM1, | |
783 | bogon, | |
784 | bogon, | |
785 | SYSIO_IMAP_UE, | |
786 | SYSIO_IMAP_CE, | |
787 | SYSIO_IMAP_SBERR, | |
788 | SYSIO_IMAP_PMGMT, | |
789 | }; | |
790 | ||
791 | #undef bogon | |
792 | ||
793 | #define NUM_SYSIO_OFFSETS (sizeof(sysio_irq_offsets) / sizeof(sysio_irq_offsets[0])) | |
794 | ||
795 | /* Convert Interrupt Mapping register pointer to associated | |
796 | * Interrupt Clear register pointer, SYSIO specific version. | |
797 | */ | |
798 | #define SYSIO_ICLR_UNUSED0 0x3400UL | |
799 | #define SYSIO_ICLR_SLOT0 0x340cUL | |
800 | #define SYSIO_ICLR_SLOT1 0x344cUL | |
801 | #define SYSIO_ICLR_SLOT2 0x348cUL | |
802 | #define SYSIO_ICLR_SLOT3 0x34ccUL | |
803 | static unsigned long sysio_imap_to_iclr(unsigned long imap) | |
804 | { | |
805 | unsigned long diff = SYSIO_ICLR_UNUSED0 - SYSIO_IMAP_SLOT0; | |
806 | return imap + diff; | |
807 | } | |
808 | ||
809 | unsigned int sbus_build_irq(void *buscookie, unsigned int ino) | |
810 | { | |
811 | struct sbus_bus *sbus = (struct sbus_bus *)buscookie; | |
812 | struct sbus_iommu *iommu = sbus->iommu; | |
813 | unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL; | |
814 | unsigned long imap, iclr; | |
815 | int pil, sbus_level = 0; | |
816 | ||
817 | pil = sysio_ino_to_pil[ino]; | |
818 | if (!pil) { | |
819 | printk("sbus_irq_build: Bad SYSIO INO[%x]\n", ino); | |
820 | panic("Bad SYSIO IRQ translations..."); | |
821 | } | |
822 | ||
823 | if (PIL_RESERVED(pil)) | |
824 | BUG(); | |
825 | ||
826 | imap = sysio_irq_offsets[ino]; | |
827 | if (imap == ((unsigned long)-1)) { | |
828 | prom_printf("get_irq_translations: Bad SYSIO INO[%x] cpu[%d]\n", | |
829 | ino, pil); | |
830 | prom_halt(); | |
831 | } | |
832 | imap += reg_base; | |
833 | ||
834 | /* SYSIO inconsistency. For external SLOTS, we have to select | |
835 | * the right ICLR register based upon the lower SBUS irq level | |
836 | * bits. | |
837 | */ | |
838 | if (ino >= 0x20) { | |
839 | iclr = sysio_imap_to_iclr(imap); | |
840 | } else { | |
841 | int sbus_slot = (ino & 0x18)>>3; | |
842 | ||
843 | sbus_level = ino & 0x7; | |
844 | ||
845 | switch(sbus_slot) { | |
846 | case 0: | |
847 | iclr = reg_base + SYSIO_ICLR_SLOT0; | |
848 | break; | |
849 | case 1: | |
850 | iclr = reg_base + SYSIO_ICLR_SLOT1; | |
851 | break; | |
852 | case 2: | |
853 | iclr = reg_base + SYSIO_ICLR_SLOT2; | |
854 | break; | |
855 | default: | |
856 | case 3: | |
857 | iclr = reg_base + SYSIO_ICLR_SLOT3; | |
858 | break; | |
859 | }; | |
860 | ||
861 | iclr += ((unsigned long)sbus_level - 1UL) * 8UL; | |
862 | } | |
863 | return build_irq(pil, sbus_level, iclr, imap); | |
864 | } | |
865 | ||
866 | /* Error interrupt handling. */ | |
867 | #define SYSIO_UE_AFSR 0x0030UL | |
868 | #define SYSIO_UE_AFAR 0x0038UL | |
869 | #define SYSIO_UEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */ | |
870 | #define SYSIO_UEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */ | |
871 | #define SYSIO_UEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */ | |
872 | #define SYSIO_UEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO is cause */ | |
873 | #define SYSIO_UEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */ | |
874 | #define SYSIO_UEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/ | |
875 | #define SYSIO_UEAFSR_RESV1 0x03ff000000000000UL /* Reserved */ | |
876 | #define SYSIO_UEAFSR_DOFF 0x0000e00000000000UL /* Doubleword Offset */ | |
877 | #define SYSIO_UEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */ | |
878 | #define SYSIO_UEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */ | |
879 | #define SYSIO_UEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */ | |
880 | static irqreturn_t sysio_ue_handler(int irq, void *dev_id, struct pt_regs *regs) | |
881 | { | |
882 | struct sbus_bus *sbus = dev_id; | |
883 | struct sbus_iommu *iommu = sbus->iommu; | |
884 | unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL; | |
885 | unsigned long afsr_reg, afar_reg; | |
886 | unsigned long afsr, afar, error_bits; | |
887 | int reported; | |
888 | ||
889 | afsr_reg = reg_base + SYSIO_UE_AFSR; | |
890 | afar_reg = reg_base + SYSIO_UE_AFAR; | |
891 | ||
892 | /* Latch error status. */ | |
893 | afsr = upa_readq(afsr_reg); | |
894 | afar = upa_readq(afar_reg); | |
895 | ||
896 | /* Clear primary/secondary error status bits. */ | |
897 | error_bits = afsr & | |
898 | (SYSIO_UEAFSR_PPIO | SYSIO_UEAFSR_PDRD | SYSIO_UEAFSR_PDWR | | |
899 | SYSIO_UEAFSR_SPIO | SYSIO_UEAFSR_SDRD | SYSIO_UEAFSR_SDWR); | |
900 | upa_writeq(error_bits, afsr_reg); | |
901 | ||
902 | /* Log the error. */ | |
903 | printk("SYSIO[%x]: Uncorrectable ECC Error, primary error type[%s]\n", | |
904 | sbus->portid, | |
905 | (((error_bits & SYSIO_UEAFSR_PPIO) ? | |
906 | "PIO" : | |
907 | ((error_bits & SYSIO_UEAFSR_PDRD) ? | |
908 | "DVMA Read" : | |
909 | ((error_bits & SYSIO_UEAFSR_PDWR) ? | |
910 | "DVMA Write" : "???"))))); | |
911 | printk("SYSIO[%x]: DOFF[%lx] SIZE[%lx] MID[%lx]\n", | |
912 | sbus->portid, | |
913 | (afsr & SYSIO_UEAFSR_DOFF) >> 45UL, | |
914 | (afsr & SYSIO_UEAFSR_SIZE) >> 42UL, | |
915 | (afsr & SYSIO_UEAFSR_MID) >> 37UL); | |
916 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
917 | printk("SYSIO[%x]: Secondary UE errors [", sbus->portid); | |
918 | reported = 0; | |
919 | if (afsr & SYSIO_UEAFSR_SPIO) { | |
920 | reported++; | |
921 | printk("(PIO)"); | |
922 | } | |
923 | if (afsr & SYSIO_UEAFSR_SDRD) { | |
924 | reported++; | |
925 | printk("(DVMA Read)"); | |
926 | } | |
927 | if (afsr & SYSIO_UEAFSR_SDWR) { | |
928 | reported++; | |
929 | printk("(DVMA Write)"); | |
930 | } | |
931 | if (!reported) | |
932 | printk("(none)"); | |
933 | printk("]\n"); | |
934 | ||
935 | return IRQ_HANDLED; | |
936 | } | |
937 | ||
938 | #define SYSIO_CE_AFSR 0x0040UL | |
939 | #define SYSIO_CE_AFAR 0x0048UL | |
940 | #define SYSIO_CEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */ | |
941 | #define SYSIO_CEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */ | |
942 | #define SYSIO_CEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */ | |
943 | #define SYSIO_CEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO cause */ | |
944 | #define SYSIO_CEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */ | |
945 | #define SYSIO_CEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/ | |
946 | #define SYSIO_CEAFSR_RESV1 0x0300000000000000UL /* Reserved */ | |
947 | #define SYSIO_CEAFSR_ESYND 0x00ff000000000000UL /* Syndrome Bits */ | |
948 | #define SYSIO_CEAFSR_DOFF 0x0000e00000000000UL /* Double Offset */ | |
949 | #define SYSIO_CEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */ | |
950 | #define SYSIO_CEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */ | |
951 | #define SYSIO_CEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */ | |
952 | static irqreturn_t sysio_ce_handler(int irq, void *dev_id, struct pt_regs *regs) | |
953 | { | |
954 | struct sbus_bus *sbus = dev_id; | |
955 | struct sbus_iommu *iommu = sbus->iommu; | |
956 | unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL; | |
957 | unsigned long afsr_reg, afar_reg; | |
958 | unsigned long afsr, afar, error_bits; | |
959 | int reported; | |
960 | ||
961 | afsr_reg = reg_base + SYSIO_CE_AFSR; | |
962 | afar_reg = reg_base + SYSIO_CE_AFAR; | |
963 | ||
964 | /* Latch error status. */ | |
965 | afsr = upa_readq(afsr_reg); | |
966 | afar = upa_readq(afar_reg); | |
967 | ||
968 | /* Clear primary/secondary error status bits. */ | |
969 | error_bits = afsr & | |
970 | (SYSIO_CEAFSR_PPIO | SYSIO_CEAFSR_PDRD | SYSIO_CEAFSR_PDWR | | |
971 | SYSIO_CEAFSR_SPIO | SYSIO_CEAFSR_SDRD | SYSIO_CEAFSR_SDWR); | |
972 | upa_writeq(error_bits, afsr_reg); | |
973 | ||
974 | printk("SYSIO[%x]: Correctable ECC Error, primary error type[%s]\n", | |
975 | sbus->portid, | |
976 | (((error_bits & SYSIO_CEAFSR_PPIO) ? | |
977 | "PIO" : | |
978 | ((error_bits & SYSIO_CEAFSR_PDRD) ? | |
979 | "DVMA Read" : | |
980 | ((error_bits & SYSIO_CEAFSR_PDWR) ? | |
981 | "DVMA Write" : "???"))))); | |
982 | ||
983 | /* XXX Use syndrome and afar to print out module string just like | |
984 | * XXX UDB CE trap handler does... -DaveM | |
985 | */ | |
986 | printk("SYSIO[%x]: DOFF[%lx] ECC Syndrome[%lx] Size[%lx] MID[%lx]\n", | |
987 | sbus->portid, | |
988 | (afsr & SYSIO_CEAFSR_DOFF) >> 45UL, | |
989 | (afsr & SYSIO_CEAFSR_ESYND) >> 48UL, | |
990 | (afsr & SYSIO_CEAFSR_SIZE) >> 42UL, | |
991 | (afsr & SYSIO_CEAFSR_MID) >> 37UL); | |
992 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
993 | ||
994 | printk("SYSIO[%x]: Secondary CE errors [", sbus->portid); | |
995 | reported = 0; | |
996 | if (afsr & SYSIO_CEAFSR_SPIO) { | |
997 | reported++; | |
998 | printk("(PIO)"); | |
999 | } | |
1000 | if (afsr & SYSIO_CEAFSR_SDRD) { | |
1001 | reported++; | |
1002 | printk("(DVMA Read)"); | |
1003 | } | |
1004 | if (afsr & SYSIO_CEAFSR_SDWR) { | |
1005 | reported++; | |
1006 | printk("(DVMA Write)"); | |
1007 | } | |
1008 | if (!reported) | |
1009 | printk("(none)"); | |
1010 | printk("]\n"); | |
1011 | ||
1012 | return IRQ_HANDLED; | |
1013 | } | |
1014 | ||
1015 | #define SYSIO_SBUS_AFSR 0x2010UL | |
1016 | #define SYSIO_SBUS_AFAR 0x2018UL | |
1017 | #define SYSIO_SBAFSR_PLE 0x8000000000000000UL /* Primary Late PIO Error */ | |
1018 | #define SYSIO_SBAFSR_PTO 0x4000000000000000UL /* Primary SBUS Timeout */ | |
1019 | #define SYSIO_SBAFSR_PBERR 0x2000000000000000UL /* Primary SBUS Error ACK */ | |
1020 | #define SYSIO_SBAFSR_SLE 0x1000000000000000UL /* Secondary Late PIO Error */ | |
1021 | #define SYSIO_SBAFSR_STO 0x0800000000000000UL /* Secondary SBUS Timeout */ | |
1022 | #define SYSIO_SBAFSR_SBERR 0x0400000000000000UL /* Secondary SBUS Error ACK */ | |
1023 | #define SYSIO_SBAFSR_RESV1 0x03ff000000000000UL /* Reserved */ | |
1024 | #define SYSIO_SBAFSR_RD 0x0000800000000000UL /* Primary was late PIO read */ | |
1025 | #define SYSIO_SBAFSR_RESV2 0x0000600000000000UL /* Reserved */ | |
1026 | #define SYSIO_SBAFSR_SIZE 0x00001c0000000000UL /* Size of transfer */ | |
1027 | #define SYSIO_SBAFSR_MID 0x000003e000000000UL /* MID causing the error */ | |
1028 | #define SYSIO_SBAFSR_RESV3 0x0000001fffffffffUL /* Reserved */ | |
1029 | static irqreturn_t sysio_sbus_error_handler(int irq, void *dev_id, struct pt_regs *regs) | |
1030 | { | |
1031 | struct sbus_bus *sbus = dev_id; | |
1032 | struct sbus_iommu *iommu = sbus->iommu; | |
1033 | unsigned long afsr_reg, afar_reg, reg_base; | |
1034 | unsigned long afsr, afar, error_bits; | |
1035 | int reported; | |
1036 | ||
1037 | reg_base = iommu->sbus_control_reg - 0x2000UL; | |
1038 | afsr_reg = reg_base + SYSIO_SBUS_AFSR; | |
1039 | afar_reg = reg_base + SYSIO_SBUS_AFAR; | |
1040 | ||
1041 | afsr = upa_readq(afsr_reg); | |
1042 | afar = upa_readq(afar_reg); | |
1043 | ||
1044 | /* Clear primary/secondary error status bits. */ | |
1045 | error_bits = afsr & | |
1046 | (SYSIO_SBAFSR_PLE | SYSIO_SBAFSR_PTO | SYSIO_SBAFSR_PBERR | | |
1047 | SYSIO_SBAFSR_SLE | SYSIO_SBAFSR_STO | SYSIO_SBAFSR_SBERR); | |
1048 | upa_writeq(error_bits, afsr_reg); | |
1049 | ||
1050 | /* Log the error. */ | |
1051 | printk("SYSIO[%x]: SBUS Error, primary error type[%s] read(%d)\n", | |
1052 | sbus->portid, | |
1053 | (((error_bits & SYSIO_SBAFSR_PLE) ? | |
1054 | "Late PIO Error" : | |
1055 | ((error_bits & SYSIO_SBAFSR_PTO) ? | |
1056 | "Time Out" : | |
1057 | ((error_bits & SYSIO_SBAFSR_PBERR) ? | |
1058 | "Error Ack" : "???")))), | |
1059 | (afsr & SYSIO_SBAFSR_RD) ? 1 : 0); | |
1060 | printk("SYSIO[%x]: size[%lx] MID[%lx]\n", | |
1061 | sbus->portid, | |
1062 | (afsr & SYSIO_SBAFSR_SIZE) >> 42UL, | |
1063 | (afsr & SYSIO_SBAFSR_MID) >> 37UL); | |
1064 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
1065 | printk("SYSIO[%x]: Secondary SBUS errors [", sbus->portid); | |
1066 | reported = 0; | |
1067 | if (afsr & SYSIO_SBAFSR_SLE) { | |
1068 | reported++; | |
1069 | printk("(Late PIO Error)"); | |
1070 | } | |
1071 | if (afsr & SYSIO_SBAFSR_STO) { | |
1072 | reported++; | |
1073 | printk("(Time Out)"); | |
1074 | } | |
1075 | if (afsr & SYSIO_SBAFSR_SBERR) { | |
1076 | reported++; | |
1077 | printk("(Error Ack)"); | |
1078 | } | |
1079 | if (!reported) | |
1080 | printk("(none)"); | |
1081 | printk("]\n"); | |
1082 | ||
1083 | /* XXX check iommu/strbuf for further error status XXX */ | |
1084 | ||
1085 | return IRQ_HANDLED; | |
1086 | } | |
1087 | ||
1088 | #define ECC_CONTROL 0x0020UL | |
1089 | #define SYSIO_ECNTRL_ECCEN 0x8000000000000000UL /* Enable ECC Checking */ | |
1090 | #define SYSIO_ECNTRL_UEEN 0x4000000000000000UL /* Enable UE Interrupts */ | |
1091 | #define SYSIO_ECNTRL_CEEN 0x2000000000000000UL /* Enable CE Interrupts */ | |
1092 | ||
1093 | #define SYSIO_UE_INO 0x34 | |
1094 | #define SYSIO_CE_INO 0x35 | |
1095 | #define SYSIO_SBUSERR_INO 0x36 | |
1096 | ||
1097 | static void __init sysio_register_error_handlers(struct sbus_bus *sbus) | |
1098 | { | |
1099 | struct sbus_iommu *iommu = sbus->iommu; | |
1100 | unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL; | |
1101 | unsigned int irq; | |
1102 | u64 control; | |
1103 | ||
1104 | irq = sbus_build_irq(sbus, SYSIO_UE_INO); | |
1105 | if (request_irq(irq, sysio_ue_handler, | |
1106 | SA_SHIRQ, "SYSIO UE", sbus) < 0) { | |
1107 | prom_printf("SYSIO[%x]: Cannot register UE interrupt.\n", | |
1108 | sbus->portid); | |
1109 | prom_halt(); | |
1110 | } | |
1111 | ||
1112 | irq = sbus_build_irq(sbus, SYSIO_CE_INO); | |
1113 | if (request_irq(irq, sysio_ce_handler, | |
1114 | SA_SHIRQ, "SYSIO CE", sbus) < 0) { | |
1115 | prom_printf("SYSIO[%x]: Cannot register CE interrupt.\n", | |
1116 | sbus->portid); | |
1117 | prom_halt(); | |
1118 | } | |
1119 | ||
1120 | irq = sbus_build_irq(sbus, SYSIO_SBUSERR_INO); | |
1121 | if (request_irq(irq, sysio_sbus_error_handler, | |
1122 | SA_SHIRQ, "SYSIO SBUS Error", sbus) < 0) { | |
1123 | prom_printf("SYSIO[%x]: Cannot register SBUS Error interrupt.\n", | |
1124 | sbus->portid); | |
1125 | prom_halt(); | |
1126 | } | |
1127 | ||
1128 | /* Now turn the error interrupts on and also enable ECC checking. */ | |
1129 | upa_writeq((SYSIO_ECNTRL_ECCEN | | |
1130 | SYSIO_ECNTRL_UEEN | | |
1131 | SYSIO_ECNTRL_CEEN), | |
1132 | reg_base + ECC_CONTROL); | |
1133 | ||
1134 | control = upa_readq(iommu->sbus_control_reg); | |
1135 | control |= 0x100UL; /* SBUS Error Interrupt Enable */ | |
1136 | upa_writeq(control, iommu->sbus_control_reg); | |
1137 | } | |
1138 | ||
1139 | /* Boot time initialization. */ | |
1140 | void __init sbus_iommu_init(int prom_node, struct sbus_bus *sbus) | |
1141 | { | |
1142 | struct linux_prom64_registers rprop; | |
1143 | struct sbus_iommu *iommu; | |
1144 | unsigned long regs, tsb_base; | |
1145 | u64 control; | |
1146 | int err, i; | |
1147 | ||
1148 | sbus->portid = prom_getintdefault(sbus->prom_node, | |
1149 | "upa-portid", -1); | |
1150 | ||
1151 | err = prom_getproperty(prom_node, "reg", | |
1152 | (char *)&rprop, sizeof(rprop)); | |
1153 | if (err < 0) { | |
1154 | prom_printf("sbus_iommu_init: Cannot map SYSIO control registers.\n"); | |
1155 | prom_halt(); | |
1156 | } | |
1157 | regs = rprop.phys_addr; | |
1158 | ||
1159 | iommu = kmalloc(sizeof(*iommu) + SMP_CACHE_BYTES, GFP_ATOMIC); | |
1160 | if (iommu == NULL) { | |
1161 | prom_printf("sbus_iommu_init: Fatal error, kmalloc(iommu) failed\n"); | |
1162 | prom_halt(); | |
1163 | } | |
1164 | ||
1165 | /* Align on E$ line boundary. */ | |
1166 | iommu = (struct sbus_iommu *) | |
1167 | (((unsigned long)iommu + (SMP_CACHE_BYTES - 1UL)) & | |
1168 | ~(SMP_CACHE_BYTES - 1UL)); | |
1169 | ||
1170 | memset(iommu, 0, sizeof(*iommu)); | |
1171 | ||
1172 | /* We start with no consistent mappings. */ | |
1173 | iommu->lowest_consistent_map = CLUSTER_NPAGES; | |
1174 | ||
1175 | for (i = 0; i < NCLUSTERS; i++) { | |
1176 | iommu->alloc_info[i].flush = 0; | |
1177 | iommu->alloc_info[i].next = 0; | |
1178 | } | |
1179 | ||
1180 | /* Setup spinlock. */ | |
1181 | spin_lock_init(&iommu->lock); | |
1182 | ||
1183 | /* Init register offsets. */ | |
1184 | iommu->iommu_regs = regs + SYSIO_IOMMUREG_BASE; | |
1185 | iommu->strbuf_regs = regs + SYSIO_STRBUFREG_BASE; | |
1186 | ||
1187 | /* The SYSIO SBUS control register is used for dummy reads | |
1188 | * in order to ensure write completion. | |
1189 | */ | |
1190 | iommu->sbus_control_reg = regs + 0x2000UL; | |
1191 | ||
1192 | /* Link into SYSIO software state. */ | |
1193 | sbus->iommu = iommu; | |
1194 | ||
1195 | printk("SYSIO: UPA portID %x, at %016lx\n", | |
1196 | sbus->portid, regs); | |
1197 | ||
1198 | /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */ | |
1199 | control = upa_readq(iommu->iommu_regs + IOMMU_CONTROL); | |
1200 | control = ((7UL << 16UL) | | |
1201 | (0UL << 2UL) | | |
1202 | (1UL << 1UL) | | |
1203 | (1UL << 0UL)); | |
1204 | ||
1205 | /* Using the above configuration we need 1MB iommu page | |
1206 | * table (128K ioptes * 8 bytes per iopte). This is | |
1207 | * page order 7 on UltraSparc. | |
1208 | */ | |
1209 | tsb_base = __get_free_pages(GFP_ATOMIC, get_order(IO_TSB_SIZE)); | |
1210 | if (tsb_base == 0UL) { | |
1211 | prom_printf("sbus_iommu_init: Fatal error, cannot alloc TSB table.\n"); | |
1212 | prom_halt(); | |
1213 | } | |
1214 | ||
1215 | iommu->page_table = (iopte_t *) tsb_base; | |
1216 | memset(iommu->page_table, 0, IO_TSB_SIZE); | |
1217 | ||
1218 | upa_writeq(control, iommu->iommu_regs + IOMMU_CONTROL); | |
1219 | ||
1220 | /* Clean out any cruft in the IOMMU using | |
1221 | * diagnostic accesses. | |
1222 | */ | |
1223 | for (i = 0; i < 16; i++) { | |
1224 | unsigned long dram = iommu->iommu_regs + IOMMU_DRAMDIAG; | |
1225 | unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG; | |
1226 | ||
1227 | dram += (unsigned long)i * 8UL; | |
1228 | tag += (unsigned long)i * 8UL; | |
1229 | upa_writeq(0, dram); | |
1230 | upa_writeq(0, tag); | |
1231 | } | |
1232 | upa_readq(iommu->sbus_control_reg); | |
1233 | ||
1234 | /* Give the TSB to SYSIO. */ | |
1235 | upa_writeq(__pa(tsb_base), iommu->iommu_regs + IOMMU_TSBBASE); | |
1236 | ||
1237 | /* Setup streaming buffer, DE=1 SB_EN=1 */ | |
1238 | control = (1UL << 1UL) | (1UL << 0UL); | |
1239 | upa_writeq(control, iommu->strbuf_regs + STRBUF_CONTROL); | |
1240 | ||
1241 | /* Clear out the tags using diagnostics. */ | |
1242 | for (i = 0; i < 16; i++) { | |
1243 | unsigned long ptag, ltag; | |
1244 | ||
1245 | ptag = iommu->strbuf_regs + STRBUF_PTAGDIAG; | |
1246 | ltag = iommu->strbuf_regs + STRBUF_LTAGDIAG; | |
1247 | ptag += (unsigned long)i * 8UL; | |
1248 | ltag += (unsigned long)i * 8UL; | |
1249 | ||
1250 | upa_writeq(0UL, ptag); | |
1251 | upa_writeq(0UL, ltag); | |
1252 | } | |
1253 | ||
1254 | /* Enable DVMA arbitration for all devices/slots. */ | |
1255 | control = upa_readq(iommu->sbus_control_reg); | |
1256 | control |= 0x3fUL; | |
1257 | upa_writeq(control, iommu->sbus_control_reg); | |
1258 | ||
1259 | /* Now some Xfire specific grot... */ | |
1260 | if (this_is_starfire) | |
1261 | sbus->starfire_cookie = starfire_hookup(sbus->portid); | |
1262 | else | |
1263 | sbus->starfire_cookie = NULL; | |
1264 | ||
1265 | sysio_register_error_handlers(sbus); | |
1266 | } |