Commit | Line | Data |
---|---|---|
8f6a93a1 DM |
1 | /* pci_sun4v.c: SUN4V specific PCI controller support. |
2 | * | |
3 | * Copyright (C) 2006 David S. Miller (davem@davemloft.net) | |
4 | */ | |
5 | ||
6 | #include <linux/kernel.h> | |
7 | #include <linux/types.h> | |
8 | #include <linux/pci.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/slab.h> | |
11 | #include <linux/interrupt.h> | |
18397944 | 12 | #include <linux/percpu.h> |
8f6a93a1 DM |
13 | |
14 | #include <asm/pbm.h> | |
15 | #include <asm/iommu.h> | |
16 | #include <asm/irq.h> | |
17 | #include <asm/upa.h> | |
18 | #include <asm/pstate.h> | |
19 | #include <asm/oplib.h> | |
20 | #include <asm/hypervisor.h> | |
21 | ||
22 | #include "pci_impl.h" | |
23 | #include "iommu_common.h" | |
24 | ||
bade5622 DM |
25 | #include "pci_sun4v.h" |
26 | ||
18397944 DM |
27 | #define PGLIST_NENTS 2048 |
28 | ||
29 | struct sun4v_pglist { | |
30 | u64 pglist[PGLIST_NENTS]; | |
31 | }; | |
32 | ||
33 | static DEFINE_PER_CPU(struct sun4v_pglist, iommu_pglists); | |
34 | ||
35 | static long pci_arena_alloc(struct pci_iommu_arena *arena, unsigned long npages) | |
36 | { | |
37 | unsigned long n, i, start, end, limit; | |
38 | int pass; | |
39 | ||
40 | limit = arena->limit; | |
41 | start = arena->hint; | |
42 | pass = 0; | |
43 | ||
44 | again: | |
45 | n = find_next_zero_bit(arena->map, limit, start); | |
46 | end = n + npages; | |
47 | if (unlikely(end >= limit)) { | |
48 | if (likely(pass < 1)) { | |
49 | limit = start; | |
50 | start = 0; | |
51 | pass++; | |
52 | goto again; | |
53 | } else { | |
54 | /* Scanned the whole thing, give up. */ | |
55 | return -1; | |
56 | } | |
57 | } | |
58 | ||
59 | for (i = n; i < end; i++) { | |
60 | if (test_bit(i, arena->map)) { | |
61 | start = i + 1; | |
62 | goto again; | |
63 | } | |
64 | } | |
65 | ||
66 | for (i = n; i < end; i++) | |
67 | __set_bit(i, arena->map); | |
68 | ||
69 | arena->hint = end; | |
70 | ||
71 | return n; | |
72 | } | |
73 | ||
74 | static void pci_arena_free(struct pci_iommu_arena *arena, unsigned long base, unsigned long npages) | |
75 | { | |
76 | unsigned long i; | |
77 | ||
78 | for (i = base; i < (base + npages); i++) | |
79 | __clear_bit(i, arena->map); | |
80 | } | |
81 | ||
8f6a93a1 DM |
82 | static void *pci_4v_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_addrp) |
83 | { | |
18397944 DM |
84 | struct pcidev_cookie *pcp; |
85 | struct pci_iommu *iommu; | |
86 | unsigned long devhandle, flags, order, first_page, npages, n; | |
87 | void *ret; | |
88 | long entry; | |
89 | u64 *pglist; | |
90 | int cpu; | |
91 | ||
92 | size = IO_PAGE_ALIGN(size); | |
93 | order = get_order(size); | |
94 | if (order >= MAX_ORDER) | |
95 | return NULL; | |
96 | ||
97 | npages = size >> IO_PAGE_SHIFT; | |
98 | if (npages > PGLIST_NENTS) | |
99 | return NULL; | |
100 | ||
101 | first_page = __get_free_pages(GFP_ATOMIC, order); | |
102 | if (first_page == 0UL) | |
103 | return NULL; | |
104 | memset((char *)first_page, 0, PAGE_SIZE << order); | |
105 | ||
106 | pcp = pdev->sysdata; | |
107 | devhandle = pcp->pbm->devhandle; | |
108 | iommu = pcp->pbm->iommu; | |
109 | ||
110 | spin_lock_irqsave(&iommu->lock, flags); | |
111 | entry = pci_arena_alloc(&iommu->arena, npages); | |
112 | spin_unlock_irqrestore(&iommu->lock, flags); | |
113 | ||
114 | if (unlikely(entry < 0L)) { | |
115 | free_pages(first_page, order); | |
116 | return NULL; | |
117 | } | |
118 | ||
119 | *dma_addrp = (iommu->page_table_map_base + | |
120 | (entry << IO_PAGE_SHIFT)); | |
121 | ret = (void *) first_page; | |
122 | first_page = __pa(first_page); | |
123 | ||
124 | cpu = get_cpu(); | |
125 | ||
126 | pglist = &__get_cpu_var(iommu_pglists).pglist[0]; | |
127 | for (n = 0; n < npages; n++) | |
128 | pglist[n] = first_page + (n * PAGE_SIZE); | |
129 | ||
130 | do { | |
131 | unsigned long num; | |
132 | ||
133 | num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), | |
134 | npages, | |
135 | (HV_PCI_MAP_ATTR_READ | | |
136 | HV_PCI_MAP_ATTR_WRITE), | |
137 | __pa(pglist)); | |
138 | entry += num; | |
139 | npages -= num; | |
140 | pglist += num; | |
141 | } while (npages != 0); | |
142 | ||
143 | put_cpu(); | |
144 | ||
145 | return ret; | |
8f6a93a1 DM |
146 | } |
147 | ||
148 | static void pci_4v_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_t dvma) | |
149 | { | |
18397944 DM |
150 | struct pcidev_cookie *pcp; |
151 | struct pci_iommu *iommu; | |
152 | unsigned long flags, order, npages, entry, devhandle; | |
153 | ||
154 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | |
155 | pcp = pdev->sysdata; | |
156 | iommu = pcp->pbm->iommu; | |
157 | devhandle = pcp->pbm->devhandle; | |
158 | entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | |
159 | ||
160 | spin_lock_irqsave(&iommu->lock, flags); | |
161 | ||
162 | pci_arena_free(&iommu->arena, entry, npages); | |
163 | ||
164 | do { | |
165 | unsigned long num; | |
166 | ||
167 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | |
168 | npages); | |
169 | entry += num; | |
170 | npages -= num; | |
171 | } while (npages != 0); | |
172 | ||
173 | spin_unlock_irqrestore(&iommu->lock, flags); | |
174 | ||
175 | order = get_order(size); | |
176 | if (order < 10) | |
177 | free_pages((unsigned long)cpu, order); | |
8f6a93a1 DM |
178 | } |
179 | ||
180 | static dma_addr_t pci_4v_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direction) | |
181 | { | |
18397944 DM |
182 | struct pcidev_cookie *pcp; |
183 | struct pci_iommu *iommu; | |
184 | unsigned long flags, npages, oaddr; | |
185 | unsigned long i, base_paddr, devhandle; | |
186 | u32 bus_addr, ret; | |
187 | unsigned long prot; | |
188 | long entry; | |
189 | u64 *pglist; | |
190 | int cpu; | |
191 | ||
192 | pcp = pdev->sysdata; | |
193 | iommu = pcp->pbm->iommu; | |
194 | devhandle = pcp->pbm->devhandle; | |
195 | ||
196 | if (unlikely(direction == PCI_DMA_NONE)) | |
197 | goto bad; | |
198 | ||
199 | oaddr = (unsigned long)ptr; | |
200 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); | |
201 | npages >>= IO_PAGE_SHIFT; | |
202 | if (unlikely(npages > PGLIST_NENTS)) | |
203 | goto bad; | |
204 | ||
205 | spin_lock_irqsave(&iommu->lock, flags); | |
206 | entry = pci_arena_alloc(&iommu->arena, npages); | |
207 | spin_unlock_irqrestore(&iommu->lock, flags); | |
208 | ||
209 | if (unlikely(entry < 0L)) | |
210 | goto bad; | |
211 | ||
212 | bus_addr = (iommu->page_table_map_base + | |
213 | (entry << IO_PAGE_SHIFT)); | |
214 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); | |
215 | base_paddr = __pa(oaddr & IO_PAGE_MASK); | |
216 | prot = HV_PCI_MAP_ATTR_READ; | |
217 | if (direction != PCI_DMA_TODEVICE) | |
218 | prot |= HV_PCI_MAP_ATTR_WRITE; | |
219 | ||
220 | cpu = get_cpu(); | |
221 | ||
222 | pglist = &__get_cpu_var(iommu_pglists).pglist[0]; | |
223 | for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) | |
224 | pglist[i] = base_paddr; | |
225 | ||
226 | do { | |
227 | unsigned long num; | |
228 | ||
229 | num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), | |
230 | npages, prot, | |
231 | __pa(pglist)); | |
232 | entry += num; | |
233 | npages -= num; | |
234 | pglist += num; | |
235 | } while (npages != 0); | |
236 | ||
237 | put_cpu(); | |
238 | ||
239 | return ret; | |
240 | ||
241 | bad: | |
242 | if (printk_ratelimit()) | |
243 | WARN_ON(1); | |
244 | return PCI_DMA_ERROR_CODE; | |
8f6a93a1 DM |
245 | } |
246 | ||
247 | static void pci_4v_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction) | |
248 | { | |
18397944 DM |
249 | struct pcidev_cookie *pcp; |
250 | struct pci_iommu *iommu; | |
251 | unsigned long flags, npages, devhandle; | |
252 | long entry; | |
253 | ||
254 | if (unlikely(direction == PCI_DMA_NONE)) { | |
255 | if (printk_ratelimit()) | |
256 | WARN_ON(1); | |
257 | return; | |
258 | } | |
259 | ||
260 | pcp = pdev->sysdata; | |
261 | iommu = pcp->pbm->iommu; | |
262 | devhandle = pcp->pbm->devhandle; | |
263 | ||
264 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); | |
265 | npages >>= IO_PAGE_SHIFT; | |
266 | bus_addr &= IO_PAGE_MASK; | |
267 | ||
268 | spin_lock_irqsave(&iommu->lock, flags); | |
269 | ||
270 | entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; | |
271 | pci_arena_free(&iommu->arena, entry, npages); | |
272 | ||
273 | do { | |
274 | unsigned long num; | |
275 | ||
276 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | |
277 | npages); | |
278 | entry += num; | |
279 | npages -= num; | |
280 | } while (npages != 0); | |
281 | ||
282 | spin_unlock_irqrestore(&iommu->lock, flags); | |
283 | } | |
284 | ||
285 | #define SG_ENT_PHYS_ADDRESS(SG) \ | |
286 | (__pa(page_address((SG)->page)) + (SG)->offset) | |
287 | ||
288 | static inline void fill_sg(long entry, unsigned long devhandle, | |
289 | struct scatterlist *sg, | |
290 | int nused, int nelems, unsigned long prot) | |
291 | { | |
292 | struct scatterlist *dma_sg = sg; | |
293 | struct scatterlist *sg_end = sg + nelems; | |
294 | int i, cpu, pglist_ent; | |
295 | u64 *pglist; | |
296 | ||
297 | cpu = get_cpu(); | |
298 | pglist = &__get_cpu_var(iommu_pglists).pglist[0]; | |
299 | pglist_ent = 0; | |
300 | for (i = 0; i < nused; i++) { | |
301 | unsigned long pteval = ~0UL; | |
302 | u32 dma_npages; | |
303 | ||
304 | dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) + | |
305 | dma_sg->dma_length + | |
306 | ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT; | |
307 | do { | |
308 | unsigned long offset; | |
309 | signed int len; | |
310 | ||
311 | /* If we are here, we know we have at least one | |
312 | * more page to map. So walk forward until we | |
313 | * hit a page crossing, and begin creating new | |
314 | * mappings from that spot. | |
315 | */ | |
316 | for (;;) { | |
317 | unsigned long tmp; | |
318 | ||
319 | tmp = SG_ENT_PHYS_ADDRESS(sg); | |
320 | len = sg->length; | |
321 | if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) { | |
322 | pteval = tmp & IO_PAGE_MASK; | |
323 | offset = tmp & (IO_PAGE_SIZE - 1UL); | |
324 | break; | |
325 | } | |
326 | if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) { | |
327 | pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK; | |
328 | offset = 0UL; | |
329 | len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL))); | |
330 | break; | |
331 | } | |
332 | sg++; | |
333 | } | |
334 | ||
335 | pteval = (pteval & IOPTE_PAGE); | |
336 | while (len > 0) { | |
337 | pglist[pglist_ent++] = pteval; | |
338 | pteval += IO_PAGE_SIZE; | |
339 | len -= (IO_PAGE_SIZE - offset); | |
340 | offset = 0; | |
341 | dma_npages--; | |
342 | } | |
343 | ||
344 | pteval = (pteval & IOPTE_PAGE) + len; | |
345 | sg++; | |
346 | ||
347 | /* Skip over any tail mappings we've fully mapped, | |
348 | * adjusting pteval along the way. Stop when we | |
349 | * detect a page crossing event. | |
350 | */ | |
351 | while (sg < sg_end && | |
352 | (pteval << (64 - IO_PAGE_SHIFT)) != 0UL && | |
353 | (pteval == SG_ENT_PHYS_ADDRESS(sg)) && | |
354 | ((pteval ^ | |
355 | (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) { | |
356 | pteval += sg->length; | |
357 | sg++; | |
358 | } | |
359 | if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL) | |
360 | pteval = ~0UL; | |
361 | } while (dma_npages != 0); | |
362 | dma_sg++; | |
363 | } | |
364 | ||
365 | BUG_ON(pglist_ent == 0); | |
366 | ||
367 | do { | |
368 | unsigned long num; | |
369 | ||
370 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | |
371 | pglist_ent); | |
372 | entry += num; | |
373 | pglist_ent -= num; | |
374 | } while (pglist_ent != 0); | |
375 | ||
376 | put_cpu(); | |
8f6a93a1 DM |
377 | } |
378 | ||
379 | static int pci_4v_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) | |
380 | { | |
18397944 DM |
381 | struct pcidev_cookie *pcp; |
382 | struct pci_iommu *iommu; | |
383 | unsigned long flags, npages, prot, devhandle; | |
384 | u32 dma_base; | |
385 | struct scatterlist *sgtmp; | |
386 | long entry; | |
387 | int used; | |
388 | ||
389 | /* Fast path single entry scatterlists. */ | |
390 | if (nelems == 1) { | |
391 | sglist->dma_address = | |
392 | pci_4v_map_single(pdev, | |
393 | (page_address(sglist->page) + sglist->offset), | |
394 | sglist->length, direction); | |
395 | if (unlikely(sglist->dma_address == PCI_DMA_ERROR_CODE)) | |
396 | return 0; | |
397 | sglist->dma_length = sglist->length; | |
398 | return 1; | |
399 | } | |
400 | ||
401 | pcp = pdev->sysdata; | |
402 | iommu = pcp->pbm->iommu; | |
403 | devhandle = pcp->pbm->devhandle; | |
404 | ||
405 | if (unlikely(direction == PCI_DMA_NONE)) | |
406 | goto bad; | |
407 | ||
408 | /* Step 1: Prepare scatter list. */ | |
409 | npages = prepare_sg(sglist, nelems); | |
410 | if (unlikely(npages > PGLIST_NENTS)) | |
411 | goto bad; | |
412 | ||
413 | /* Step 2: Allocate a cluster and context, if necessary. */ | |
414 | spin_lock_irqsave(&iommu->lock, flags); | |
415 | entry = pci_arena_alloc(&iommu->arena, npages); | |
416 | spin_unlock_irqrestore(&iommu->lock, flags); | |
417 | ||
418 | if (unlikely(entry < 0L)) | |
419 | goto bad; | |
420 | ||
421 | dma_base = iommu->page_table_map_base + | |
422 | (entry << IO_PAGE_SHIFT); | |
423 | ||
424 | /* Step 3: Normalize DMA addresses. */ | |
425 | used = nelems; | |
426 | ||
427 | sgtmp = sglist; | |
428 | while (used && sgtmp->dma_length) { | |
429 | sgtmp->dma_address += dma_base; | |
430 | sgtmp++; | |
431 | used--; | |
432 | } | |
433 | used = nelems - used; | |
434 | ||
435 | /* Step 4: Create the mappings. */ | |
436 | prot = HV_PCI_MAP_ATTR_READ; | |
437 | if (direction != PCI_DMA_TODEVICE) | |
438 | prot |= HV_PCI_MAP_ATTR_WRITE; | |
439 | ||
440 | fill_sg(entry, devhandle, sglist, used, nelems, prot); | |
441 | ||
442 | return used; | |
443 | ||
444 | bad: | |
445 | if (printk_ratelimit()) | |
446 | WARN_ON(1); | |
447 | return 0; | |
8f6a93a1 DM |
448 | } |
449 | ||
450 | static void pci_4v_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) | |
451 | { | |
18397944 DM |
452 | struct pcidev_cookie *pcp; |
453 | struct pci_iommu *iommu; | |
454 | unsigned long flags, i, npages, devhandle; | |
455 | long entry; | |
456 | u32 bus_addr; | |
457 | ||
458 | if (unlikely(direction == PCI_DMA_NONE)) { | |
459 | if (printk_ratelimit()) | |
460 | WARN_ON(1); | |
461 | } | |
462 | ||
463 | pcp = pdev->sysdata; | |
464 | iommu = pcp->pbm->iommu; | |
465 | devhandle = pcp->pbm->devhandle; | |
466 | ||
467 | bus_addr = sglist->dma_address & IO_PAGE_MASK; | |
468 | ||
469 | for (i = 1; i < nelems; i++) | |
470 | if (sglist[i].dma_length == 0) | |
471 | break; | |
472 | i--; | |
473 | npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - | |
474 | bus_addr) >> IO_PAGE_SHIFT; | |
475 | ||
476 | entry = ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | |
477 | ||
478 | spin_lock_irqsave(&iommu->lock, flags); | |
479 | ||
480 | pci_arena_free(&iommu->arena, entry, npages); | |
481 | ||
482 | do { | |
483 | unsigned long num; | |
484 | ||
485 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | |
486 | npages); | |
487 | entry += num; | |
488 | npages -= num; | |
489 | } while (npages != 0); | |
490 | ||
491 | spin_unlock_irqrestore(&iommu->lock, flags); | |
8f6a93a1 DM |
492 | } |
493 | ||
494 | static void pci_4v_dma_sync_single_for_cpu(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction) | |
495 | { | |
18397944 | 496 | /* Nothing to do... */ |
8f6a93a1 DM |
497 | } |
498 | ||
499 | static void pci_4v_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int direction) | |
500 | { | |
18397944 | 501 | /* Nothing to do... */ |
8f6a93a1 DM |
502 | } |
503 | ||
504 | struct pci_iommu_ops pci_sun4v_iommu_ops = { | |
505 | .alloc_consistent = pci_4v_alloc_consistent, | |
506 | .free_consistent = pci_4v_free_consistent, | |
507 | .map_single = pci_4v_map_single, | |
508 | .unmap_single = pci_4v_unmap_single, | |
509 | .map_sg = pci_4v_map_sg, | |
510 | .unmap_sg = pci_4v_unmap_sg, | |
511 | .dma_sync_single_for_cpu = pci_4v_dma_sync_single_for_cpu, | |
512 | .dma_sync_sg_for_cpu = pci_4v_dma_sync_sg_for_cpu, | |
513 | }; | |
514 | ||
bade5622 DM |
515 | /* SUN4V PCI configuration space accessors. */ |
516 | ||
059833eb DM |
517 | static inline int pci_sun4v_out_of_range(struct pci_pbm_info *pbm, unsigned int bus) |
518 | { | |
519 | if (bus < pbm->pci_first_busno || | |
520 | bus > pbm->pci_last_busno) | |
521 | return 1; | |
522 | return 0; | |
523 | } | |
524 | ||
bade5622 DM |
525 | static int pci_sun4v_read_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, |
526 | int where, int size, u32 *value) | |
527 | { | |
7eae642f | 528 | struct pci_pbm_info *pbm = bus_dev->sysdata; |
059833eb | 529 | u32 devhandle = pbm->devhandle; |
7eae642f DM |
530 | unsigned int bus = bus_dev->number; |
531 | unsigned int device = PCI_SLOT(devfn); | |
532 | unsigned int func = PCI_FUNC(devfn); | |
533 | unsigned long ret; | |
534 | ||
059833eb DM |
535 | if (pci_sun4v_out_of_range(pbm, bus)) { |
536 | ret = ~0UL; | |
537 | } else { | |
538 | ret = pci_sun4v_config_get(devhandle, | |
539 | HV_PCI_DEVICE_BUILD(bus, device, func), | |
540 | where, size); | |
10804828 DM |
541 | #if 0 |
542 | printk("read_pci_cfg: devh[%x] device[%08x] where[%x] sz[%d] " | |
543 | "== [%016lx]\n", | |
544 | devhandle, HV_PCI_DEVICE_BUILD(bus, device, func), | |
545 | where, size, ret); | |
546 | #endif | |
059833eb | 547 | } |
7eae642f DM |
548 | switch (size) { |
549 | case 1: | |
550 | *value = ret & 0xff; | |
551 | break; | |
552 | case 2: | |
553 | *value = ret & 0xffff; | |
554 | break; | |
555 | case 4: | |
556 | *value = ret & 0xffffffff; | |
557 | break; | |
558 | }; | |
559 | ||
560 | ||
561 | return PCIBIOS_SUCCESSFUL; | |
bade5622 DM |
562 | } |
563 | ||
564 | static int pci_sun4v_write_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, | |
565 | int where, int size, u32 value) | |
566 | { | |
7eae642f | 567 | struct pci_pbm_info *pbm = bus_dev->sysdata; |
059833eb | 568 | u32 devhandle = pbm->devhandle; |
7eae642f DM |
569 | unsigned int bus = bus_dev->number; |
570 | unsigned int device = PCI_SLOT(devfn); | |
571 | unsigned int func = PCI_FUNC(devfn); | |
572 | unsigned long ret; | |
573 | ||
059833eb DM |
574 | if (pci_sun4v_out_of_range(pbm, bus)) { |
575 | /* Do nothing. */ | |
576 | } else { | |
577 | ret = pci_sun4v_config_put(devhandle, | |
578 | HV_PCI_DEVICE_BUILD(bus, device, func), | |
579 | where, size, value); | |
10804828 DM |
580 | #if 0 |
581 | printk("write_pci_cfg: devh[%x] device[%08x] where[%x] sz[%d] " | |
582 | "val[%08x] == [%016lx]\n", | |
583 | devhandle, HV_PCI_DEVICE_BUILD(bus, device, func), | |
584 | where, size, value, ret); | |
585 | #endif | |
059833eb | 586 | } |
7eae642f | 587 | return PCIBIOS_SUCCESSFUL; |
bade5622 DM |
588 | } |
589 | ||
590 | static struct pci_ops pci_sun4v_ops = { | |
591 | .read = pci_sun4v_read_pci_cfg, | |
592 | .write = pci_sun4v_write_pci_cfg, | |
593 | }; | |
594 | ||
595 | ||
c2609267 DM |
596 | static void pbm_scan_bus(struct pci_controller_info *p, |
597 | struct pci_pbm_info *pbm) | |
598 | { | |
599 | struct pcidev_cookie *cookie = kmalloc(sizeof(*cookie), GFP_KERNEL); | |
600 | ||
601 | if (!cookie) { | |
602 | prom_printf("%s: Critical allocation failure.\n", pbm->name); | |
603 | prom_halt(); | |
604 | } | |
605 | ||
606 | /* All we care about is the PBM. */ | |
607 | memset(cookie, 0, sizeof(*cookie)); | |
608 | cookie->pbm = pbm; | |
609 | ||
610 | pbm->pci_bus = pci_scan_bus(pbm->pci_first_busno, | |
611 | p->pci_ops, | |
612 | pbm); | |
10804828 | 613 | #if 0 |
c2609267 DM |
614 | pci_fixup_host_bridge_self(pbm->pci_bus); |
615 | pbm->pci_bus->self->sysdata = cookie; | |
10804828 | 616 | #endif |
c2609267 | 617 | |
10804828 DM |
618 | pci_fill_in_pbm_cookies(pbm->pci_bus, pbm, |
619 | prom_getchild(pbm->prom_node)); | |
c2609267 DM |
620 | pci_record_assignments(pbm, pbm->pci_bus); |
621 | pci_assign_unassigned(pbm, pbm->pci_bus); | |
622 | pci_fixup_irq(pbm, pbm->pci_bus); | |
623 | pci_determine_66mhz_disposition(pbm, pbm->pci_bus); | |
624 | pci_setup_busmastering(pbm, pbm->pci_bus); | |
625 | } | |
626 | ||
bade5622 DM |
627 | static void pci_sun4v_scan_bus(struct pci_controller_info *p) |
628 | { | |
c2609267 DM |
629 | if (p->pbm_A.prom_node) { |
630 | p->pbm_A.is_66mhz_capable = | |
631 | prom_getbool(p->pbm_A.prom_node, "66mhz-capable"); | |
632 | ||
633 | pbm_scan_bus(p, &p->pbm_A); | |
634 | } | |
635 | if (p->pbm_B.prom_node) { | |
636 | p->pbm_B.is_66mhz_capable = | |
637 | prom_getbool(p->pbm_B.prom_node, "66mhz-capable"); | |
638 | ||
639 | pbm_scan_bus(p, &p->pbm_B); | |
640 | } | |
641 | ||
642 | /* XXX register error interrupt handlers XXX */ | |
bade5622 DM |
643 | } |
644 | ||
645 | static unsigned int pci_sun4v_irq_build(struct pci_pbm_info *pbm, | |
646 | struct pci_dev *pdev, | |
e3999574 | 647 | unsigned int devino) |
bade5622 | 648 | { |
10804828 DM |
649 | u32 devhandle = pbm->devhandle; |
650 | int pil; | |
651 | ||
10804828 DM |
652 | pil = 4; |
653 | if (pdev) { | |
654 | switch ((pdev->class >> 16) & 0xff) { | |
655 | case PCI_BASE_CLASS_STORAGE: | |
656 | pil = 4; | |
657 | break; | |
658 | ||
659 | case PCI_BASE_CLASS_NETWORK: | |
660 | pil = 6; | |
661 | break; | |
662 | ||
663 | case PCI_BASE_CLASS_DISPLAY: | |
664 | pil = 9; | |
665 | break; | |
666 | ||
667 | case PCI_BASE_CLASS_MULTIMEDIA: | |
668 | case PCI_BASE_CLASS_MEMORY: | |
669 | case PCI_BASE_CLASS_BRIDGE: | |
670 | case PCI_BASE_CLASS_SERIAL: | |
671 | pil = 10; | |
672 | break; | |
673 | ||
674 | default: | |
675 | pil = 4; | |
676 | break; | |
677 | }; | |
678 | } | |
679 | BUG_ON(PIL_RESERVED(pil)); | |
680 | ||
e3999574 | 681 | return sun4v_build_irq(devhandle, devino, pil, IBF_PCI); |
bade5622 DM |
682 | } |
683 | ||
bade5622 DM |
684 | static void pci_sun4v_base_address_update(struct pci_dev *pdev, int resource) |
685 | { | |
686 | struct pcidev_cookie *pcp = pdev->sysdata; | |
687 | struct pci_pbm_info *pbm = pcp->pbm; | |
688 | struct resource *res, *root; | |
689 | u32 reg; | |
690 | int where, size, is_64bit; | |
691 | ||
692 | res = &pdev->resource[resource]; | |
693 | if (resource < 6) { | |
694 | where = PCI_BASE_ADDRESS_0 + (resource * 4); | |
695 | } else if (resource == PCI_ROM_RESOURCE) { | |
696 | where = pdev->rom_base_reg; | |
697 | } else { | |
698 | /* Somebody might have asked allocation of a non-standard resource */ | |
699 | return; | |
700 | } | |
701 | ||
c2609267 | 702 | /* XXX 64-bit MEM handling is not %100 correct... XXX */ |
bade5622 DM |
703 | is_64bit = 0; |
704 | if (res->flags & IORESOURCE_IO) | |
705 | root = &pbm->io_space; | |
706 | else { | |
707 | root = &pbm->mem_space; | |
708 | if ((res->flags & PCI_BASE_ADDRESS_MEM_TYPE_MASK) | |
709 | == PCI_BASE_ADDRESS_MEM_TYPE_64) | |
710 | is_64bit = 1; | |
711 | } | |
712 | ||
713 | size = res->end - res->start; | |
714 | pci_read_config_dword(pdev, where, ®); | |
715 | reg = ((reg & size) | | |
716 | (((u32)(res->start - root->start)) & ~size)); | |
717 | if (resource == PCI_ROM_RESOURCE) { | |
718 | reg |= PCI_ROM_ADDRESS_ENABLE; | |
719 | res->flags |= IORESOURCE_ROM_ENABLE; | |
720 | } | |
721 | pci_write_config_dword(pdev, where, reg); | |
722 | ||
723 | /* This knows that the upper 32-bits of the address | |
724 | * must be zero. Our PCI common layer enforces this. | |
725 | */ | |
726 | if (is_64bit) | |
727 | pci_write_config_dword(pdev, where + 4, 0); | |
728 | } | |
729 | ||
bade5622 DM |
730 | static void pci_sun4v_resource_adjust(struct pci_dev *pdev, |
731 | struct resource *res, | |
732 | struct resource *root) | |
733 | { | |
734 | res->start += root->start; | |
735 | res->end += root->start; | |
736 | } | |
737 | ||
738 | /* Use ranges property to determine where PCI MEM, I/O, and Config | |
739 | * space are for this PCI bus module. | |
740 | */ | |
741 | static void pci_sun4v_determine_mem_io_space(struct pci_pbm_info *pbm) | |
742 | { | |
221b2fb8 | 743 | int i, saw_mem, saw_io; |
bade5622 | 744 | |
221b2fb8 | 745 | saw_mem = saw_io = 0; |
bade5622 DM |
746 | for (i = 0; i < pbm->num_pbm_ranges; i++) { |
747 | struct linux_prom_pci_ranges *pr = &pbm->pbm_ranges[i]; | |
748 | unsigned long a; | |
749 | int type; | |
750 | ||
751 | type = (pr->child_phys_hi >> 24) & 0x3; | |
752 | a = (((unsigned long)pr->parent_phys_hi << 32UL) | | |
753 | ((unsigned long)pr->parent_phys_lo << 0UL)); | |
754 | ||
755 | switch (type) { | |
bade5622 DM |
756 | case 1: |
757 | /* 16-bit IO space, 16MB */ | |
758 | pbm->io_space.start = a; | |
759 | pbm->io_space.end = a + ((16UL*1024UL*1024UL) - 1UL); | |
760 | pbm->io_space.flags = IORESOURCE_IO; | |
761 | saw_io = 1; | |
762 | break; | |
763 | ||
764 | case 2: | |
765 | /* 32-bit MEM space, 2GB */ | |
766 | pbm->mem_space.start = a; | |
767 | pbm->mem_space.end = a + (0x80000000UL - 1UL); | |
768 | pbm->mem_space.flags = IORESOURCE_MEM; | |
769 | saw_mem = 1; | |
770 | break; | |
771 | ||
c2609267 DM |
772 | case 3: |
773 | /* XXX 64-bit MEM handling XXX */ | |
774 | ||
bade5622 DM |
775 | default: |
776 | break; | |
777 | }; | |
778 | } | |
779 | ||
221b2fb8 | 780 | if (!saw_io || !saw_mem) { |
bade5622 DM |
781 | prom_printf("%s: Fatal error, missing %s PBM range.\n", |
782 | pbm->name, | |
221b2fb8 | 783 | (!saw_io ? "IO" : "MEM")); |
bade5622 DM |
784 | prom_halt(); |
785 | } | |
786 | ||
221b2fb8 | 787 | printk("%s: PCI IO[%lx] MEM[%lx]\n", |
bade5622 | 788 | pbm->name, |
bade5622 DM |
789 | pbm->io_space.start, |
790 | pbm->mem_space.start); | |
791 | } | |
792 | ||
793 | static void pbm_register_toplevel_resources(struct pci_controller_info *p, | |
794 | struct pci_pbm_info *pbm) | |
795 | { | |
796 | pbm->io_space.name = pbm->mem_space.name = pbm->name; | |
797 | ||
798 | request_resource(&ioport_resource, &pbm->io_space); | |
799 | request_resource(&iomem_resource, &pbm->mem_space); | |
800 | pci_register_legacy_regions(&pbm->io_space, | |
801 | &pbm->mem_space); | |
802 | } | |
803 | ||
18397944 DM |
804 | static void probe_existing_entries(struct pci_pbm_info *pbm, |
805 | struct pci_iommu *iommu) | |
806 | { | |
807 | struct pci_iommu_arena *arena = &iommu->arena; | |
808 | unsigned long i, devhandle; | |
809 | ||
810 | devhandle = pbm->devhandle; | |
811 | for (i = 0; i < arena->limit; i++) { | |
812 | unsigned long ret, io_attrs, ra; | |
813 | ||
814 | ret = pci_sun4v_iommu_getmap(devhandle, | |
815 | HV_PCI_TSBID(0, i), | |
816 | &io_attrs, &ra); | |
817 | if (ret == HV_EOK) | |
818 | __set_bit(i, arena->map); | |
819 | } | |
820 | } | |
821 | ||
bade5622 DM |
822 | static void pci_sun4v_iommu_init(struct pci_pbm_info *pbm) |
823 | { | |
18397944 DM |
824 | struct pci_iommu *iommu = pbm->iommu; |
825 | unsigned long num_tsb_entries, sz; | |
826 | u32 vdma[2], dma_mask, dma_offset; | |
827 | int err, tsbsize; | |
828 | ||
829 | err = prom_getproperty(pbm->prom_node, "virtual-dma", | |
830 | (char *)&vdma[0], sizeof(vdma)); | |
831 | if (err == 0 || err == -1) { | |
832 | /* No property, use default values. */ | |
833 | vdma[0] = 0x80000000; | |
834 | vdma[1] = 0x80000000; | |
835 | } | |
836 | ||
837 | dma_mask = vdma[0]; | |
838 | switch (vdma[1]) { | |
839 | case 0x20000000: | |
840 | dma_mask |= 0x1fffffff; | |
841 | tsbsize = 64; | |
842 | break; | |
843 | ||
844 | case 0x40000000: | |
845 | dma_mask |= 0x3fffffff; | |
846 | tsbsize = 128; | |
847 | break; | |
848 | ||
849 | case 0x80000000: | |
850 | dma_mask |= 0x7fffffff; | |
851 | tsbsize = 128; | |
852 | break; | |
853 | ||
854 | default: | |
855 | prom_printf("PCI-SUN4V: strange virtual-dma size.\n"); | |
856 | prom_halt(); | |
857 | }; | |
858 | ||
859 | num_tsb_entries = tsbsize / sizeof(iopte_t); | |
860 | ||
861 | dma_offset = vdma[0]; | |
862 | ||
863 | /* Setup initial software IOMMU state. */ | |
864 | spin_lock_init(&iommu->lock); | |
865 | iommu->ctx_lowest_free = 1; | |
866 | iommu->page_table_map_base = dma_offset; | |
867 | iommu->dma_addr_mask = dma_mask; | |
868 | ||
869 | /* Allocate and initialize the free area map. */ | |
870 | sz = num_tsb_entries / 8; | |
871 | sz = (sz + 7UL) & ~7UL; | |
872 | iommu->arena.map = kmalloc(sz, GFP_KERNEL); | |
873 | if (!iommu->arena.map) { | |
874 | prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n"); | |
875 | prom_halt(); | |
876 | } | |
877 | memset(iommu->arena.map, 0, sz); | |
878 | iommu->arena.limit = num_tsb_entries; | |
879 | ||
880 | probe_existing_entries(pbm, iommu); | |
bade5622 DM |
881 | } |
882 | ||
10804828 DM |
883 | /* Don't get this from the root nexus, get it from the "pci@0" node below. */ |
884 | static void pci_sun4v_get_bus_range(struct pci_pbm_info *pbm) | |
885 | { | |
886 | unsigned int busrange[2]; | |
887 | int prom_node = pbm->prom_node; | |
888 | int err; | |
889 | ||
890 | prom_node = prom_getchild(prom_node); | |
891 | if (prom_node == 0) { | |
892 | prom_printf("%s: Fatal error, no child OBP node.\n", pbm->name); | |
893 | prom_halt(); | |
894 | } | |
895 | ||
896 | err = prom_getproperty(prom_node, "bus-range", | |
897 | (char *)&busrange[0], | |
898 | sizeof(busrange)); | |
899 | if (err == 0 || err == -1) { | |
900 | prom_printf("%s: Fatal error, no bus-range.\n", pbm->name); | |
901 | prom_halt(); | |
902 | } | |
903 | ||
904 | pbm->pci_first_busno = busrange[0]; | |
905 | pbm->pci_last_busno = busrange[1]; | |
906 | ||
907 | } | |
908 | ||
3833789b | 909 | static void pci_sun4v_pbm_init(struct pci_controller_info *p, int prom_node, unsigned int devhandle) |
bade5622 DM |
910 | { |
911 | struct pci_pbm_info *pbm; | |
3833789b | 912 | int err, i; |
bade5622 | 913 | |
3833789b DM |
914 | if (devhandle & 0x40) |
915 | pbm = &p->pbm_B; | |
916 | else | |
917 | pbm = &p->pbm_A; | |
bade5622 DM |
918 | |
919 | pbm->parent = p; | |
920 | pbm->prom_node = prom_node; | |
921 | pbm->pci_first_slot = 1; | |
922 | ||
3833789b | 923 | pbm->devhandle = devhandle; |
bade5622 DM |
924 | |
925 | sprintf(pbm->name, "SUN4V-PCI%d PBM%c", | |
926 | p->index, (pbm == &p->pbm_A ? 'A' : 'B')); | |
927 | ||
928 | printk("%s: devhandle[%x]\n", pbm->name, pbm->devhandle); | |
929 | ||
930 | prom_getstring(prom_node, "name", | |
931 | pbm->prom_name, sizeof(pbm->prom_name)); | |
932 | ||
933 | err = prom_getproperty(prom_node, "ranges", | |
934 | (char *) pbm->pbm_ranges, | |
935 | sizeof(pbm->pbm_ranges)); | |
936 | if (err == 0 || err == -1) { | |
937 | prom_printf("%s: Fatal error, no ranges property.\n", | |
938 | pbm->name); | |
939 | prom_halt(); | |
940 | } | |
941 | ||
942 | pbm->num_pbm_ranges = | |
943 | (err / sizeof(struct linux_prom_pci_ranges)); | |
944 | ||
3833789b DM |
945 | /* Mask out the top 8 bits of the ranges, leaving the real |
946 | * physical address. | |
947 | */ | |
948 | for (i = 0; i < pbm->num_pbm_ranges; i++) | |
949 | pbm->pbm_ranges[i].parent_phys_hi &= 0x0fffffff; | |
950 | ||
bade5622 DM |
951 | pci_sun4v_determine_mem_io_space(pbm); |
952 | pbm_register_toplevel_resources(p, pbm); | |
953 | ||
954 | err = prom_getproperty(prom_node, "interrupt-map", | |
955 | (char *)pbm->pbm_intmap, | |
956 | sizeof(pbm->pbm_intmap)); | |
957 | if (err != -1) { | |
958 | pbm->num_pbm_intmap = (err / sizeof(struct linux_prom_pci_intmap)); | |
959 | err = prom_getproperty(prom_node, "interrupt-map-mask", | |
960 | (char *)&pbm->pbm_intmask, | |
961 | sizeof(pbm->pbm_intmask)); | |
962 | if (err == -1) { | |
963 | prom_printf("%s: Fatal error, no " | |
964 | "interrupt-map-mask.\n", pbm->name); | |
965 | prom_halt(); | |
966 | } | |
967 | } else { | |
968 | pbm->num_pbm_intmap = 0; | |
969 | memset(&pbm->pbm_intmask, 0, sizeof(pbm->pbm_intmask)); | |
970 | } | |
971 | ||
10804828 | 972 | pci_sun4v_get_bus_range(pbm); |
bade5622 DM |
973 | pci_sun4v_iommu_init(pbm); |
974 | } | |
975 | ||
8f6a93a1 DM |
976 | void sun4v_pci_init(int node, char *model_name) |
977 | { | |
bade5622 DM |
978 | struct pci_controller_info *p; |
979 | struct pci_iommu *iommu; | |
3833789b DM |
980 | struct linux_prom64_registers regs; |
981 | unsigned int devhandle; | |
982 | ||
983 | prom_getproperty(node, "reg", (char *)®s, sizeof(regs)); | |
984 | devhandle = (regs.phys_addr >> 32UL) & 0x0fffffff;; | |
985 | ||
986 | for (p = pci_controller_root; p; p = p->next) { | |
987 | struct pci_pbm_info *pbm; | |
988 | ||
989 | if (p->pbm_A.prom_node && p->pbm_B.prom_node) | |
990 | continue; | |
991 | ||
992 | pbm = (p->pbm_A.prom_node ? | |
993 | &p->pbm_A : | |
994 | &p->pbm_B); | |
995 | ||
0b522497 | 996 | if (pbm->devhandle == (devhandle ^ 0x40)) { |
3833789b | 997 | pci_sun4v_pbm_init(p, node, devhandle); |
0b522497 DM |
998 | return; |
999 | } | |
3833789b | 1000 | } |
bade5622 DM |
1001 | |
1002 | p = kmalloc(sizeof(struct pci_controller_info), GFP_ATOMIC); | |
1003 | if (!p) { | |
1004 | prom_printf("SUN4V_PCI: Fatal memory allocation error.\n"); | |
1005 | prom_halt(); | |
1006 | } | |
1007 | memset(p, 0, sizeof(*p)); | |
1008 | ||
1009 | iommu = kmalloc(sizeof(struct pci_iommu), GFP_ATOMIC); | |
1010 | if (!iommu) { | |
1011 | prom_printf("SCHIZO: Fatal memory allocation error.\n"); | |
1012 | prom_halt(); | |
1013 | } | |
1014 | memset(iommu, 0, sizeof(*iommu)); | |
1015 | p->pbm_A.iommu = iommu; | |
1016 | ||
1017 | iommu = kmalloc(sizeof(struct pci_iommu), GFP_ATOMIC); | |
1018 | if (!iommu) { | |
1019 | prom_printf("SCHIZO: Fatal memory allocation error.\n"); | |
1020 | prom_halt(); | |
1021 | } | |
1022 | memset(iommu, 0, sizeof(*iommu)); | |
1023 | p->pbm_B.iommu = iommu; | |
1024 | ||
1025 | p->next = pci_controller_root; | |
1026 | pci_controller_root = p; | |
1027 | ||
1028 | p->index = pci_num_controllers++; | |
1029 | p->pbms_same_domain = 0; | |
1030 | ||
1031 | p->scan_bus = pci_sun4v_scan_bus; | |
1032 | p->irq_build = pci_sun4v_irq_build; | |
1033 | p->base_address_update = pci_sun4v_base_address_update; | |
1034 | p->resource_adjust = pci_sun4v_resource_adjust; | |
1035 | p->pci_ops = &pci_sun4v_ops; | |
1036 | ||
1037 | /* Like PSYCHO and SCHIZO we have a 2GB aligned area | |
1038 | * for memory space. | |
1039 | */ | |
1040 | pci_memspace_mask = 0x7fffffffUL; | |
1041 | ||
3833789b | 1042 | pci_sun4v_pbm_init(p, node, devhandle); |
8f6a93a1 | 1043 | } |