Commit | Line | Data |
---|---|---|
8f6a93a1 DM |
1 | /* pci_sun4v.c: SUN4V specific PCI controller support. |
2 | * | |
d284142c | 3 | * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net) |
8f6a93a1 DM |
4 | */ |
5 | ||
6 | #include <linux/kernel.h> | |
7 | #include <linux/types.h> | |
8 | #include <linux/pci.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/slab.h> | |
11 | #include <linux/interrupt.h> | |
18397944 | 12 | #include <linux/percpu.h> |
35a17eb6 DM |
13 | #include <linux/irq.h> |
14 | #include <linux/msi.h> | |
7b64db60 | 15 | #include <linux/export.h> |
59db8102 | 16 | #include <linux/log2.h> |
3822b509 | 17 | #include <linux/of_device.h> |
8f6a93a1 | 18 | |
8f6a93a1 DM |
19 | #include <asm/iommu.h> |
20 | #include <asm/irq.h> | |
8f6a93a1 | 21 | #include <asm/hypervisor.h> |
e87dc350 | 22 | #include <asm/prom.h> |
8f6a93a1 DM |
23 | |
24 | #include "pci_impl.h" | |
25 | #include "iommu_common.h" | |
26 | ||
bade5622 DM |
27 | #include "pci_sun4v.h" |
28 | ||
3822b509 DM |
29 | #define DRIVER_NAME "pci_sun4v" |
30 | #define PFX DRIVER_NAME ": " | |
31 | ||
e01c0d6d DM |
32 | static unsigned long vpci_major = 1; |
33 | static unsigned long vpci_minor = 1; | |
34 | ||
7c8f486a | 35 | #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) |
18397944 | 36 | |
16ce82d8 | 37 | struct iommu_batch { |
ad7ad57c | 38 | struct device *dev; /* Device mapping is for. */ |
6a32fd4d DM |
39 | unsigned long prot; /* IOMMU page protections */ |
40 | unsigned long entry; /* Index into IOTSB. */ | |
41 | u64 *pglist; /* List of physical pages */ | |
42 | unsigned long npages; /* Number of pages in list. */ | |
18397944 DM |
43 | }; |
44 | ||
ad7ad57c | 45 | static DEFINE_PER_CPU(struct iommu_batch, iommu_batch); |
d3ae4b5b | 46 | static int iommu_batch_initialized; |
6a32fd4d DM |
47 | |
48 | /* Interrupts must be disabled. */ | |
ad7ad57c | 49 | static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry) |
6a32fd4d | 50 | { |
ad7ad57c | 51 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); |
6a32fd4d | 52 | |
ad7ad57c | 53 | p->dev = dev; |
6a32fd4d DM |
54 | p->prot = prot; |
55 | p->entry = entry; | |
56 | p->npages = 0; | |
57 | } | |
58 | ||
59 | /* Interrupts must be disabled. */ | |
ad7ad57c | 60 | static long iommu_batch_flush(struct iommu_batch *p) |
6a32fd4d | 61 | { |
ad7ad57c | 62 | struct pci_pbm_info *pbm = p->dev->archdata.host_controller; |
a2fb23af | 63 | unsigned long devhandle = pbm->devhandle; |
6a32fd4d DM |
64 | unsigned long prot = p->prot; |
65 | unsigned long entry = p->entry; | |
66 | u64 *pglist = p->pglist; | |
67 | unsigned long npages = p->npages; | |
68 | ||
d82965c1 | 69 | while (npages != 0) { |
6a32fd4d DM |
70 | long num; |
71 | ||
72 | num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), | |
73 | npages, prot, __pa(pglist)); | |
74 | if (unlikely(num < 0)) { | |
75 | if (printk_ratelimit()) | |
ad7ad57c | 76 | printk("iommu_batch_flush: IOMMU map of " |
90181136 | 77 | "[%08lx:%08llx:%lx:%lx:%lx] failed with " |
6a32fd4d DM |
78 | "status %ld\n", |
79 | devhandle, HV_PCI_TSBID(0, entry), | |
80 | npages, prot, __pa(pglist), num); | |
81 | return -1; | |
82 | } | |
83 | ||
84 | entry += num; | |
85 | npages -= num; | |
86 | pglist += num; | |
d82965c1 | 87 | } |
6a32fd4d DM |
88 | |
89 | p->entry = entry; | |
90 | p->npages = 0; | |
91 | ||
92 | return 0; | |
93 | } | |
94 | ||
13fa14e1 DM |
95 | static inline void iommu_batch_new_entry(unsigned long entry) |
96 | { | |
97 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); | |
98 | ||
99 | if (p->entry + p->npages == entry) | |
100 | return; | |
101 | if (p->entry != ~0UL) | |
102 | iommu_batch_flush(p); | |
103 | p->entry = entry; | |
104 | } | |
105 | ||
6a32fd4d | 106 | /* Interrupts must be disabled. */ |
ad7ad57c | 107 | static inline long iommu_batch_add(u64 phys_page) |
6a32fd4d | 108 | { |
ad7ad57c | 109 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); |
6a32fd4d DM |
110 | |
111 | BUG_ON(p->npages >= PGLIST_NENTS); | |
112 | ||
113 | p->pglist[p->npages++] = phys_page; | |
114 | if (p->npages == PGLIST_NENTS) | |
ad7ad57c | 115 | return iommu_batch_flush(p); |
6a32fd4d DM |
116 | |
117 | return 0; | |
118 | } | |
119 | ||
120 | /* Interrupts must be disabled. */ | |
ad7ad57c | 121 | static inline long iommu_batch_end(void) |
6a32fd4d | 122 | { |
ad7ad57c | 123 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); |
6a32fd4d DM |
124 | |
125 | BUG_ON(p->npages >= PGLIST_NENTS); | |
126 | ||
ad7ad57c | 127 | return iommu_batch_flush(p); |
6a32fd4d | 128 | } |
18397944 | 129 | |
ad7ad57c | 130 | static void *dma_4v_alloc_coherent(struct device *dev, size_t size, |
c416258a AP |
131 | dma_addr_t *dma_addrp, gfp_t gfp, |
132 | struct dma_attrs *attrs) | |
8f6a93a1 | 133 | { |
7c8f486a | 134 | unsigned long flags, order, first_page, npages, n; |
c1b1a5f1 DM |
135 | struct iommu *iommu; |
136 | struct page *page; | |
18397944 DM |
137 | void *ret; |
138 | long entry; | |
c1b1a5f1 | 139 | int nid; |
18397944 DM |
140 | |
141 | size = IO_PAGE_ALIGN(size); | |
142 | order = get_order(size); | |
6a32fd4d | 143 | if (unlikely(order >= MAX_ORDER)) |
18397944 DM |
144 | return NULL; |
145 | ||
146 | npages = size >> IO_PAGE_SHIFT; | |
18397944 | 147 | |
c1b1a5f1 DM |
148 | nid = dev->archdata.numa_node; |
149 | page = alloc_pages_node(nid, gfp, order); | |
150 | if (unlikely(!page)) | |
18397944 | 151 | return NULL; |
e7a0453e | 152 | |
c1b1a5f1 | 153 | first_page = (unsigned long) page_address(page); |
18397944 DM |
154 | memset((char *)first_page, 0, PAGE_SIZE << order); |
155 | ||
ad7ad57c | 156 | iommu = dev->archdata.iommu; |
18397944 DM |
157 | |
158 | spin_lock_irqsave(&iommu->lock, flags); | |
d284142c | 159 | entry = iommu_range_alloc(dev, iommu, npages, NULL); |
18397944 DM |
160 | spin_unlock_irqrestore(&iommu->lock, flags); |
161 | ||
d284142c DM |
162 | if (unlikely(entry == DMA_ERROR_CODE)) |
163 | goto range_alloc_fail; | |
18397944 DM |
164 | |
165 | *dma_addrp = (iommu->page_table_map_base + | |
166 | (entry << IO_PAGE_SHIFT)); | |
167 | ret = (void *) first_page; | |
168 | first_page = __pa(first_page); | |
169 | ||
6a32fd4d | 170 | local_irq_save(flags); |
18397944 | 171 | |
ad7ad57c DM |
172 | iommu_batch_start(dev, |
173 | (HV_PCI_MAP_ATTR_READ | | |
174 | HV_PCI_MAP_ATTR_WRITE), | |
175 | entry); | |
18397944 | 176 | |
6a32fd4d | 177 | for (n = 0; n < npages; n++) { |
ad7ad57c | 178 | long err = iommu_batch_add(first_page + (n * PAGE_SIZE)); |
6a32fd4d DM |
179 | if (unlikely(err < 0L)) |
180 | goto iommu_map_fail; | |
181 | } | |
18397944 | 182 | |
ad7ad57c | 183 | if (unlikely(iommu_batch_end() < 0L)) |
6a32fd4d | 184 | goto iommu_map_fail; |
18397944 | 185 | |
6a32fd4d | 186 | local_irq_restore(flags); |
18397944 DM |
187 | |
188 | return ret; | |
6a32fd4d DM |
189 | |
190 | iommu_map_fail: | |
191 | /* Interrupts are disabled. */ | |
192 | spin_lock(&iommu->lock); | |
d284142c | 193 | iommu_range_free(iommu, *dma_addrp, npages); |
6a32fd4d DM |
194 | spin_unlock_irqrestore(&iommu->lock, flags); |
195 | ||
d284142c | 196 | range_alloc_fail: |
6a32fd4d DM |
197 | free_pages(first_page, order); |
198 | return NULL; | |
8f6a93a1 DM |
199 | } |
200 | ||
ad7ad57c | 201 | static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, |
c416258a | 202 | dma_addr_t dvma, struct dma_attrs *attrs) |
8f6a93a1 | 203 | { |
a2fb23af | 204 | struct pci_pbm_info *pbm; |
16ce82d8 | 205 | struct iommu *iommu; |
7c8f486a DM |
206 | unsigned long flags, order, npages, entry; |
207 | u32 devhandle; | |
18397944 DM |
208 | |
209 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | |
ad7ad57c DM |
210 | iommu = dev->archdata.iommu; |
211 | pbm = dev->archdata.host_controller; | |
a2fb23af | 212 | devhandle = pbm->devhandle; |
18397944 DM |
213 | entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); |
214 | ||
215 | spin_lock_irqsave(&iommu->lock, flags); | |
216 | ||
d284142c | 217 | iommu_range_free(iommu, dvma, npages); |
18397944 DM |
218 | |
219 | do { | |
220 | unsigned long num; | |
221 | ||
222 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | |
223 | npages); | |
224 | entry += num; | |
225 | npages -= num; | |
226 | } while (npages != 0); | |
227 | ||
228 | spin_unlock_irqrestore(&iommu->lock, flags); | |
229 | ||
230 | order = get_order(size); | |
231 | if (order < 10) | |
232 | free_pages((unsigned long)cpu, order); | |
8f6a93a1 DM |
233 | } |
234 | ||
797a7568 FT |
235 | static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, |
236 | unsigned long offset, size_t sz, | |
bc0a14f1 FT |
237 | enum dma_data_direction direction, |
238 | struct dma_attrs *attrs) | |
8f6a93a1 | 239 | { |
16ce82d8 | 240 | struct iommu *iommu; |
18397944 | 241 | unsigned long flags, npages, oaddr; |
7c8f486a | 242 | unsigned long i, base_paddr; |
6a32fd4d | 243 | u32 bus_addr, ret; |
18397944 DM |
244 | unsigned long prot; |
245 | long entry; | |
18397944 | 246 | |
ad7ad57c | 247 | iommu = dev->archdata.iommu; |
18397944 | 248 | |
ad7ad57c | 249 | if (unlikely(direction == DMA_NONE)) |
18397944 DM |
250 | goto bad; |
251 | ||
797a7568 | 252 | oaddr = (unsigned long)(page_address(page) + offset); |
18397944 DM |
253 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
254 | npages >>= IO_PAGE_SHIFT; | |
18397944 DM |
255 | |
256 | spin_lock_irqsave(&iommu->lock, flags); | |
d284142c | 257 | entry = iommu_range_alloc(dev, iommu, npages, NULL); |
18397944 DM |
258 | spin_unlock_irqrestore(&iommu->lock, flags); |
259 | ||
d284142c | 260 | if (unlikely(entry == DMA_ERROR_CODE)) |
18397944 DM |
261 | goto bad; |
262 | ||
263 | bus_addr = (iommu->page_table_map_base + | |
264 | (entry << IO_PAGE_SHIFT)); | |
265 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); | |
266 | base_paddr = __pa(oaddr & IO_PAGE_MASK); | |
267 | prot = HV_PCI_MAP_ATTR_READ; | |
ad7ad57c | 268 | if (direction != DMA_TO_DEVICE) |
18397944 DM |
269 | prot |= HV_PCI_MAP_ATTR_WRITE; |
270 | ||
6a32fd4d | 271 | local_irq_save(flags); |
18397944 | 272 | |
ad7ad57c | 273 | iommu_batch_start(dev, prot, entry); |
18397944 | 274 | |
6a32fd4d | 275 | for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { |
ad7ad57c | 276 | long err = iommu_batch_add(base_paddr); |
6a32fd4d DM |
277 | if (unlikely(err < 0L)) |
278 | goto iommu_map_fail; | |
279 | } | |
ad7ad57c | 280 | if (unlikely(iommu_batch_end() < 0L)) |
6a32fd4d | 281 | goto iommu_map_fail; |
18397944 | 282 | |
6a32fd4d | 283 | local_irq_restore(flags); |
18397944 DM |
284 | |
285 | return ret; | |
286 | ||
287 | bad: | |
288 | if (printk_ratelimit()) | |
289 | WARN_ON(1); | |
ad7ad57c | 290 | return DMA_ERROR_CODE; |
6a32fd4d DM |
291 | |
292 | iommu_map_fail: | |
293 | /* Interrupts are disabled. */ | |
294 | spin_lock(&iommu->lock); | |
d284142c | 295 | iommu_range_free(iommu, bus_addr, npages); |
6a32fd4d DM |
296 | spin_unlock_irqrestore(&iommu->lock, flags); |
297 | ||
ad7ad57c | 298 | return DMA_ERROR_CODE; |
8f6a93a1 DM |
299 | } |
300 | ||
797a7568 | 301 | static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, |
bc0a14f1 FT |
302 | size_t sz, enum dma_data_direction direction, |
303 | struct dma_attrs *attrs) | |
8f6a93a1 | 304 | { |
a2fb23af | 305 | struct pci_pbm_info *pbm; |
16ce82d8 | 306 | struct iommu *iommu; |
7c8f486a | 307 | unsigned long flags, npages; |
18397944 | 308 | long entry; |
7c8f486a | 309 | u32 devhandle; |
18397944 | 310 | |
ad7ad57c | 311 | if (unlikely(direction == DMA_NONE)) { |
18397944 DM |
312 | if (printk_ratelimit()) |
313 | WARN_ON(1); | |
314 | return; | |
315 | } | |
316 | ||
ad7ad57c DM |
317 | iommu = dev->archdata.iommu; |
318 | pbm = dev->archdata.host_controller; | |
a2fb23af | 319 | devhandle = pbm->devhandle; |
18397944 DM |
320 | |
321 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); | |
322 | npages >>= IO_PAGE_SHIFT; | |
323 | bus_addr &= IO_PAGE_MASK; | |
324 | ||
325 | spin_lock_irqsave(&iommu->lock, flags); | |
326 | ||
d284142c | 327 | iommu_range_free(iommu, bus_addr, npages); |
18397944 | 328 | |
d284142c | 329 | entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; |
18397944 DM |
330 | do { |
331 | unsigned long num; | |
332 | ||
333 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | |
334 | npages); | |
335 | entry += num; | |
336 | npages -= num; | |
337 | } while (npages != 0); | |
338 | ||
339 | spin_unlock_irqrestore(&iommu->lock, flags); | |
340 | } | |
341 | ||
ad7ad57c | 342 | static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, |
bc0a14f1 FT |
343 | int nelems, enum dma_data_direction direction, |
344 | struct dma_attrs *attrs) | |
8f6a93a1 | 345 | { |
13fa14e1 DM |
346 | struct scatterlist *s, *outs, *segstart; |
347 | unsigned long flags, handle, prot; | |
348 | dma_addr_t dma_next = 0, dma_addr; | |
349 | unsigned int max_seg_size; | |
f0880257 | 350 | unsigned long seg_boundary_size; |
13fa14e1 | 351 | int outcount, incount, i; |
16ce82d8 | 352 | struct iommu *iommu; |
f0880257 | 353 | unsigned long base_shift; |
13fa14e1 DM |
354 | long err; |
355 | ||
356 | BUG_ON(direction == DMA_NONE); | |
18397944 | 357 | |
ad7ad57c | 358 | iommu = dev->archdata.iommu; |
13fa14e1 DM |
359 | if (nelems == 0 || !iommu) |
360 | return 0; | |
18397944 | 361 | |
13fa14e1 DM |
362 | prot = HV_PCI_MAP_ATTR_READ; |
363 | if (direction != DMA_TO_DEVICE) | |
364 | prot |= HV_PCI_MAP_ATTR_WRITE; | |
18397944 | 365 | |
13fa14e1 DM |
366 | outs = s = segstart = &sglist[0]; |
367 | outcount = 1; | |
368 | incount = nelems; | |
369 | handle = 0; | |
18397944 | 370 | |
13fa14e1 DM |
371 | /* Init first segment length for backout at failure */ |
372 | outs->dma_length = 0; | |
18397944 | 373 | |
13fa14e1 | 374 | spin_lock_irqsave(&iommu->lock, flags); |
18397944 | 375 | |
13fa14e1 | 376 | iommu_batch_start(dev, prot, ~0UL); |
18397944 | 377 | |
13fa14e1 | 378 | max_seg_size = dma_get_max_seg_size(dev); |
f0880257 FT |
379 | seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, |
380 | IO_PAGE_SIZE) >> IO_PAGE_SHIFT; | |
381 | base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; | |
13fa14e1 | 382 | for_each_sg(sglist, s, nelems, i) { |
f0880257 | 383 | unsigned long paddr, npages, entry, out_entry = 0, slen; |
38192d52 | 384 | |
13fa14e1 DM |
385 | slen = s->length; |
386 | /* Sanity check */ | |
387 | if (slen == 0) { | |
388 | dma_next = 0; | |
389 | continue; | |
390 | } | |
391 | /* Allocate iommu entries for that segment */ | |
392 | paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); | |
0fcff28f | 393 | npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); |
13fa14e1 | 394 | entry = iommu_range_alloc(dev, iommu, npages, &handle); |
38192d52 | 395 | |
13fa14e1 DM |
396 | /* Handle failure */ |
397 | if (unlikely(entry == DMA_ERROR_CODE)) { | |
398 | if (printk_ratelimit()) | |
399 | printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" | |
400 | " npages %lx\n", iommu, paddr, npages); | |
401 | goto iommu_map_failed; | |
402 | } | |
38192d52 | 403 | |
13fa14e1 | 404 | iommu_batch_new_entry(entry); |
38192d52 | 405 | |
13fa14e1 DM |
406 | /* Convert entry to a dma_addr_t */ |
407 | dma_addr = iommu->page_table_map_base + | |
408 | (entry << IO_PAGE_SHIFT); | |
409 | dma_addr |= (s->offset & ~IO_PAGE_MASK); | |
38192d52 | 410 | |
13fa14e1 | 411 | /* Insert into HW table */ |
38192d52 | 412 | paddr &= IO_PAGE_MASK; |
13fa14e1 | 413 | while (npages--) { |
38192d52 | 414 | err = iommu_batch_add(paddr); |
13fa14e1 | 415 | if (unlikely(err < 0L)) |
38192d52 | 416 | goto iommu_map_failed; |
13fa14e1 DM |
417 | paddr += IO_PAGE_SIZE; |
418 | } | |
419 | ||
420 | /* If we are in an open segment, try merging */ | |
421 | if (segstart != s) { | |
422 | /* We cannot merge if: | |
423 | * - allocated dma_addr isn't contiguous to previous allocation | |
424 | */ | |
425 | if ((dma_addr != dma_next) || | |
f0880257 FT |
426 | (outs->dma_length + s->length > max_seg_size) || |
427 | (is_span_boundary(out_entry, base_shift, | |
428 | seg_boundary_size, outs, s))) { | |
13fa14e1 DM |
429 | /* Can't merge: create a new segment */ |
430 | segstart = s; | |
431 | outcount++; | |
432 | outs = sg_next(outs); | |
433 | } else { | |
434 | outs->dma_length += s->length; | |
38192d52 | 435 | } |
13fa14e1 | 436 | } |
38192d52 | 437 | |
13fa14e1 DM |
438 | if (segstart == s) { |
439 | /* This is a new segment, fill entries */ | |
440 | outs->dma_address = dma_addr; | |
441 | outs->dma_length = slen; | |
f0880257 | 442 | out_entry = entry; |
38192d52 | 443 | } |
13fa14e1 DM |
444 | |
445 | /* Calculate next page pointer for contiguous check */ | |
446 | dma_next = dma_addr + slen; | |
38192d52 DM |
447 | } |
448 | ||
449 | err = iommu_batch_end(); | |
450 | ||
6a32fd4d DM |
451 | if (unlikely(err < 0L)) |
452 | goto iommu_map_failed; | |
18397944 | 453 | |
13fa14e1 | 454 | spin_unlock_irqrestore(&iommu->lock, flags); |
18397944 | 455 | |
13fa14e1 DM |
456 | if (outcount < incount) { |
457 | outs = sg_next(outs); | |
458 | outs->dma_address = DMA_ERROR_CODE; | |
459 | outs->dma_length = 0; | |
460 | } | |
461 | ||
462 | return outcount; | |
6a32fd4d DM |
463 | |
464 | iommu_map_failed: | |
13fa14e1 DM |
465 | for_each_sg(sglist, s, nelems, i) { |
466 | if (s->dma_length != 0) { | |
467 | unsigned long vaddr, npages; | |
468 | ||
469 | vaddr = s->dma_address & IO_PAGE_MASK; | |
0fcff28f JR |
470 | npages = iommu_num_pages(s->dma_address, s->dma_length, |
471 | IO_PAGE_SIZE); | |
13fa14e1 DM |
472 | iommu_range_free(iommu, vaddr, npages); |
473 | /* XXX demap? XXX */ | |
474 | s->dma_address = DMA_ERROR_CODE; | |
475 | s->dma_length = 0; | |
476 | } | |
477 | if (s == outs) | |
478 | break; | |
479 | } | |
6a32fd4d DM |
480 | spin_unlock_irqrestore(&iommu->lock, flags); |
481 | ||
482 | return 0; | |
8f6a93a1 DM |
483 | } |
484 | ||
ad7ad57c | 485 | static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, |
bc0a14f1 FT |
486 | int nelems, enum dma_data_direction direction, |
487 | struct dma_attrs *attrs) | |
8f6a93a1 | 488 | { |
a2fb23af | 489 | struct pci_pbm_info *pbm; |
13fa14e1 | 490 | struct scatterlist *sg; |
16ce82d8 | 491 | struct iommu *iommu; |
13fa14e1 DM |
492 | unsigned long flags; |
493 | u32 devhandle; | |
18397944 | 494 | |
13fa14e1 | 495 | BUG_ON(direction == DMA_NONE); |
18397944 | 496 | |
ad7ad57c DM |
497 | iommu = dev->archdata.iommu; |
498 | pbm = dev->archdata.host_controller; | |
a2fb23af | 499 | devhandle = pbm->devhandle; |
18397944 | 500 | |
18397944 DM |
501 | spin_lock_irqsave(&iommu->lock, flags); |
502 | ||
13fa14e1 DM |
503 | sg = sglist; |
504 | while (nelems--) { | |
505 | dma_addr_t dma_handle = sg->dma_address; | |
506 | unsigned int len = sg->dma_length; | |
507 | unsigned long npages, entry; | |
508 | ||
509 | if (!len) | |
510 | break; | |
0fcff28f | 511 | npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); |
13fa14e1 DM |
512 | iommu_range_free(iommu, dma_handle, npages); |
513 | ||
514 | entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | |
515 | while (npages) { | |
516 | unsigned long num; | |
517 | ||
518 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | |
519 | npages); | |
520 | entry += num; | |
521 | npages -= num; | |
522 | } | |
18397944 | 523 | |
13fa14e1 DM |
524 | sg = sg_next(sg); |
525 | } | |
18397944 DM |
526 | |
527 | spin_unlock_irqrestore(&iommu->lock, flags); | |
8f6a93a1 DM |
528 | } |
529 | ||
02f7a189 | 530 | static struct dma_map_ops sun4v_dma_ops = { |
c416258a AP |
531 | .alloc = dma_4v_alloc_coherent, |
532 | .free = dma_4v_free_coherent, | |
797a7568 FT |
533 | .map_page = dma_4v_map_page, |
534 | .unmap_page = dma_4v_unmap_page, | |
ad7ad57c DM |
535 | .map_sg = dma_4v_map_sg, |
536 | .unmap_sg = dma_4v_unmap_sg, | |
8f6a93a1 DM |
537 | }; |
538 | ||
9a2ed5cc DM |
539 | static void __devinit pci_sun4v_scan_bus(struct pci_pbm_info *pbm, |
540 | struct device *parent) | |
bade5622 | 541 | { |
e87dc350 DM |
542 | struct property *prop; |
543 | struct device_node *dp; | |
544 | ||
61c7a080 | 545 | dp = pbm->op->dev.of_node; |
34768bc8 DM |
546 | prop = of_find_property(dp, "66mhz-capable", NULL); |
547 | pbm->is_66mhz_capable = (prop != NULL); | |
e822358a | 548 | pbm->pci_bus = pci_scan_one_pbm(pbm, parent); |
c2609267 DM |
549 | |
550 | /* XXX register error interrupt handlers XXX */ | |
bade5622 DM |
551 | } |
552 | ||
9a2ed5cc DM |
553 | static unsigned long __devinit probe_existing_entries(struct pci_pbm_info *pbm, |
554 | struct iommu *iommu) | |
18397944 | 555 | { |
9b3627f3 | 556 | struct iommu_arena *arena = &iommu->arena; |
e7a0453e | 557 | unsigned long i, cnt = 0; |
7c8f486a | 558 | u32 devhandle; |
18397944 DM |
559 | |
560 | devhandle = pbm->devhandle; | |
561 | for (i = 0; i < arena->limit; i++) { | |
562 | unsigned long ret, io_attrs, ra; | |
563 | ||
564 | ret = pci_sun4v_iommu_getmap(devhandle, | |
565 | HV_PCI_TSBID(0, i), | |
566 | &io_attrs, &ra); | |
e7a0453e | 567 | if (ret == HV_EOK) { |
c2a5a46b DM |
568 | if (page_in_phys_avail(ra)) { |
569 | pci_sun4v_iommu_demap(devhandle, | |
570 | HV_PCI_TSBID(0, i), 1); | |
571 | } else { | |
572 | cnt++; | |
573 | __set_bit(i, arena->map); | |
574 | } | |
e7a0453e | 575 | } |
18397944 | 576 | } |
e7a0453e DM |
577 | |
578 | return cnt; | |
18397944 DM |
579 | } |
580 | ||
9a2ed5cc | 581 | static int __devinit pci_sun4v_iommu_init(struct pci_pbm_info *pbm) |
bade5622 | 582 | { |
8aef7278 | 583 | static const u32 vdma_default[] = { 0x80000000, 0x80000000 }; |
16ce82d8 | 584 | struct iommu *iommu = pbm->iommu; |
c6fee081 | 585 | unsigned long num_tsb_entries, sz; |
8aef7278 DM |
586 | u32 dma_mask, dma_offset; |
587 | const u32 *vdma; | |
588 | ||
61c7a080 | 589 | vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", NULL); |
8aef7278 DM |
590 | if (!vdma) |
591 | vdma = vdma_default; | |
18397944 | 592 | |
59db8102 | 593 | if ((vdma[0] | vdma[1]) & ~IO_PAGE_MASK) { |
3822b509 DM |
594 | printk(KERN_ERR PFX "Strange virtual-dma[%08x:%08x].\n", |
595 | vdma[0], vdma[1]); | |
596 | return -EINVAL; | |
18397944 DM |
597 | }; |
598 | ||
59db8102 DM |
599 | dma_mask = (roundup_pow_of_two(vdma[1]) - 1UL); |
600 | num_tsb_entries = vdma[1] / IO_PAGE_SIZE; | |
18397944 DM |
601 | |
602 | dma_offset = vdma[0]; | |
603 | ||
604 | /* Setup initial software IOMMU state. */ | |
605 | spin_lock_init(&iommu->lock); | |
606 | iommu->ctx_lowest_free = 1; | |
607 | iommu->page_table_map_base = dma_offset; | |
608 | iommu->dma_addr_mask = dma_mask; | |
609 | ||
610 | /* Allocate and initialize the free area map. */ | |
59db8102 | 611 | sz = (num_tsb_entries + 7) / 8; |
18397944 | 612 | sz = (sz + 7UL) & ~7UL; |
982c2064 | 613 | iommu->arena.map = kzalloc(sz, GFP_KERNEL); |
18397944 | 614 | if (!iommu->arena.map) { |
3822b509 DM |
615 | printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); |
616 | return -ENOMEM; | |
18397944 | 617 | } |
18397944 DM |
618 | iommu->arena.limit = num_tsb_entries; |
619 | ||
e7a0453e | 620 | sz = probe_existing_entries(pbm, iommu); |
c2a5a46b DM |
621 | if (sz) |
622 | printk("%s: Imported %lu TSB entries from OBP\n", | |
623 | pbm->name, sz); | |
3822b509 DM |
624 | |
625 | return 0; | |
bade5622 DM |
626 | } |
627 | ||
35a17eb6 DM |
628 | #ifdef CONFIG_PCI_MSI |
629 | struct pci_sun4v_msiq_entry { | |
630 | u64 version_type; | |
631 | #define MSIQ_VERSION_MASK 0xffffffff00000000UL | |
632 | #define MSIQ_VERSION_SHIFT 32 | |
633 | #define MSIQ_TYPE_MASK 0x00000000000000ffUL | |
634 | #define MSIQ_TYPE_SHIFT 0 | |
635 | #define MSIQ_TYPE_NONE 0x00 | |
636 | #define MSIQ_TYPE_MSG 0x01 | |
637 | #define MSIQ_TYPE_MSI32 0x02 | |
638 | #define MSIQ_TYPE_MSI64 0x03 | |
639 | #define MSIQ_TYPE_INTX 0x08 | |
640 | #define MSIQ_TYPE_NONE2 0xff | |
641 | ||
642 | u64 intx_sysino; | |
643 | u64 reserved1; | |
644 | u64 stick; | |
645 | u64 req_id; /* bus/device/func */ | |
646 | #define MSIQ_REQID_BUS_MASK 0xff00UL | |
647 | #define MSIQ_REQID_BUS_SHIFT 8 | |
648 | #define MSIQ_REQID_DEVICE_MASK 0x00f8UL | |
649 | #define MSIQ_REQID_DEVICE_SHIFT 3 | |
650 | #define MSIQ_REQID_FUNC_MASK 0x0007UL | |
651 | #define MSIQ_REQID_FUNC_SHIFT 0 | |
652 | ||
653 | u64 msi_address; | |
654 | ||
e5dd42e4 | 655 | /* The format of this value is message type dependent. |
35a17eb6 DM |
656 | * For MSI bits 15:0 are the data from the MSI packet. |
657 | * For MSI-X bits 31:0 are the data from the MSI packet. | |
658 | * For MSG, the message code and message routing code where: | |
659 | * bits 39:32 is the bus/device/fn of the msg target-id | |
660 | * bits 18:16 is the message routing code | |
661 | * bits 7:0 is the message code | |
662 | * For INTx the low order 2-bits are: | |
663 | * 00 - INTA | |
664 | * 01 - INTB | |
665 | * 10 - INTC | |
666 | * 11 - INTD | |
667 | */ | |
668 | u64 msi_data; | |
669 | ||
670 | u64 reserved2; | |
671 | }; | |
672 | ||
759f89e0 DM |
673 | static int pci_sun4v_get_head(struct pci_pbm_info *pbm, unsigned long msiqid, |
674 | unsigned long *head) | |
35a17eb6 | 675 | { |
759f89e0 | 676 | unsigned long err, limit; |
35a17eb6 | 677 | |
759f89e0 | 678 | err = pci_sun4v_msiq_gethead(pbm->devhandle, msiqid, head); |
35a17eb6 | 679 | if (unlikely(err)) |
759f89e0 | 680 | return -ENXIO; |
35a17eb6 | 681 | |
759f89e0 DM |
682 | limit = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); |
683 | if (unlikely(*head >= limit)) | |
684 | return -EFBIG; | |
685 | ||
686 | return 0; | |
687 | } | |
688 | ||
689 | static int pci_sun4v_dequeue_msi(struct pci_pbm_info *pbm, | |
690 | unsigned long msiqid, unsigned long *head, | |
691 | unsigned long *msi) | |
692 | { | |
693 | struct pci_sun4v_msiq_entry *ep; | |
694 | unsigned long err, type; | |
695 | ||
696 | /* Note: void pointer arithmetic, 'head' is a byte offset */ | |
697 | ep = (pbm->msi_queues + ((msiqid - pbm->msiq_first) * | |
698 | (pbm->msiq_ent_count * | |
699 | sizeof(struct pci_sun4v_msiq_entry))) + | |
700 | *head); | |
701 | ||
702 | if ((ep->version_type & MSIQ_TYPE_MASK) == 0) | |
703 | return 0; | |
35a17eb6 | 704 | |
759f89e0 DM |
705 | type = (ep->version_type & MSIQ_TYPE_MASK) >> MSIQ_TYPE_SHIFT; |
706 | if (unlikely(type != MSIQ_TYPE_MSI32 && | |
707 | type != MSIQ_TYPE_MSI64)) | |
708 | return -EINVAL; | |
35a17eb6 | 709 | |
759f89e0 DM |
710 | *msi = ep->msi_data; |
711 | ||
712 | err = pci_sun4v_msi_setstate(pbm->devhandle, | |
713 | ep->msi_data /* msi_num */, | |
714 | HV_MSISTATE_IDLE); | |
715 | if (unlikely(err)) | |
716 | return -ENXIO; | |
35a17eb6 | 717 | |
759f89e0 DM |
718 | /* Clear the entry. */ |
719 | ep->version_type &= ~MSIQ_TYPE_MASK; | |
35a17eb6 | 720 | |
759f89e0 DM |
721 | (*head) += sizeof(struct pci_sun4v_msiq_entry); |
722 | if (*head >= | |
723 | (pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry))) | |
724 | *head = 0; | |
35a17eb6 | 725 | |
759f89e0 | 726 | return 1; |
35a17eb6 DM |
727 | } |
728 | ||
759f89e0 DM |
729 | static int pci_sun4v_set_head(struct pci_pbm_info *pbm, unsigned long msiqid, |
730 | unsigned long head) | |
35a17eb6 | 731 | { |
759f89e0 | 732 | unsigned long err; |
35a17eb6 | 733 | |
759f89e0 DM |
734 | err = pci_sun4v_msiq_sethead(pbm->devhandle, msiqid, head); |
735 | if (unlikely(err)) | |
736 | return -EINVAL; | |
35a17eb6 | 737 | |
759f89e0 DM |
738 | return 0; |
739 | } | |
35a17eb6 | 740 | |
759f89e0 DM |
741 | static int pci_sun4v_msi_setup(struct pci_pbm_info *pbm, unsigned long msiqid, |
742 | unsigned long msi, int is_msi64) | |
743 | { | |
744 | if (pci_sun4v_msi_setmsiq(pbm->devhandle, msi, msiqid, | |
745 | (is_msi64 ? | |
746 | HV_MSITYPE_MSI64 : HV_MSITYPE_MSI32))) | |
747 | return -ENXIO; | |
748 | if (pci_sun4v_msi_setstate(pbm->devhandle, msi, HV_MSISTATE_IDLE)) | |
749 | return -ENXIO; | |
750 | if (pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_VALID)) | |
751 | return -ENXIO; | |
35a17eb6 DM |
752 | return 0; |
753 | } | |
754 | ||
759f89e0 | 755 | static int pci_sun4v_msi_teardown(struct pci_pbm_info *pbm, unsigned long msi) |
35a17eb6 | 756 | { |
759f89e0 DM |
757 | unsigned long err, msiqid; |
758 | ||
759 | err = pci_sun4v_msi_getmsiq(pbm->devhandle, msi, &msiqid); | |
760 | if (err) | |
761 | return -ENXIO; | |
762 | ||
763 | pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_INVALID); | |
764 | ||
765 | return 0; | |
35a17eb6 DM |
766 | } |
767 | ||
759f89e0 | 768 | static int pci_sun4v_msiq_alloc(struct pci_pbm_info *pbm) |
35a17eb6 DM |
769 | { |
770 | unsigned long q_size, alloc_size, pages, order; | |
771 | int i; | |
772 | ||
773 | q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); | |
774 | alloc_size = (pbm->msiq_num * q_size); | |
775 | order = get_order(alloc_size); | |
776 | pages = __get_free_pages(GFP_KERNEL | __GFP_COMP, order); | |
777 | if (pages == 0UL) { | |
778 | printk(KERN_ERR "MSI: Cannot allocate MSI queues (o=%lu).\n", | |
779 | order); | |
780 | return -ENOMEM; | |
781 | } | |
782 | memset((char *)pages, 0, PAGE_SIZE << order); | |
783 | pbm->msi_queues = (void *) pages; | |
784 | ||
785 | for (i = 0; i < pbm->msiq_num; i++) { | |
786 | unsigned long err, base = __pa(pages + (i * q_size)); | |
787 | unsigned long ret1, ret2; | |
788 | ||
789 | err = pci_sun4v_msiq_conf(pbm->devhandle, | |
790 | pbm->msiq_first + i, | |
791 | base, pbm->msiq_ent_count); | |
792 | if (err) { | |
793 | printk(KERN_ERR "MSI: msiq register fails (err=%lu)\n", | |
794 | err); | |
795 | goto h_error; | |
796 | } | |
797 | ||
798 | err = pci_sun4v_msiq_info(pbm->devhandle, | |
799 | pbm->msiq_first + i, | |
800 | &ret1, &ret2); | |
801 | if (err) { | |
802 | printk(KERN_ERR "MSI: Cannot read msiq (err=%lu)\n", | |
803 | err); | |
804 | goto h_error; | |
805 | } | |
806 | if (ret1 != base || ret2 != pbm->msiq_ent_count) { | |
807 | printk(KERN_ERR "MSI: Bogus qconf " | |
808 | "expected[%lx:%x] got[%lx:%lx]\n", | |
809 | base, pbm->msiq_ent_count, | |
810 | ret1, ret2); | |
811 | goto h_error; | |
812 | } | |
813 | } | |
814 | ||
815 | return 0; | |
816 | ||
817 | h_error: | |
818 | free_pages(pages, order); | |
819 | return -EINVAL; | |
820 | } | |
821 | ||
759f89e0 | 822 | static void pci_sun4v_msiq_free(struct pci_pbm_info *pbm) |
35a17eb6 | 823 | { |
759f89e0 | 824 | unsigned long q_size, alloc_size, pages, order; |
35a17eb6 DM |
825 | int i; |
826 | ||
759f89e0 DM |
827 | for (i = 0; i < pbm->msiq_num; i++) { |
828 | unsigned long msiqid = pbm->msiq_first + i; | |
35a17eb6 | 829 | |
759f89e0 | 830 | (void) pci_sun4v_msiq_conf(pbm->devhandle, msiqid, 0UL, 0); |
35a17eb6 | 831 | } |
7fe3730d | 832 | |
759f89e0 DM |
833 | q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); |
834 | alloc_size = (pbm->msiq_num * q_size); | |
835 | order = get_order(alloc_size); | |
35a17eb6 | 836 | |
759f89e0 | 837 | pages = (unsigned long) pbm->msi_queues; |
35a17eb6 | 838 | |
759f89e0 | 839 | free_pages(pages, order); |
35a17eb6 | 840 | |
759f89e0 | 841 | pbm->msi_queues = NULL; |
35a17eb6 DM |
842 | } |
843 | ||
759f89e0 DM |
844 | static int pci_sun4v_msiq_build_irq(struct pci_pbm_info *pbm, |
845 | unsigned long msiqid, | |
846 | unsigned long devino) | |
35a17eb6 | 847 | { |
44ed3c0c | 848 | unsigned int irq = sun4v_build_irq(pbm->devhandle, devino); |
35a17eb6 | 849 | |
44ed3c0c | 850 | if (!irq) |
759f89e0 | 851 | return -ENOMEM; |
35a17eb6 | 852 | |
759f89e0 DM |
853 | if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID)) |
854 | return -EINVAL; | |
7cc85833 DM |
855 | if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE)) |
856 | return -EINVAL; | |
35a17eb6 | 857 | |
44ed3c0c | 858 | return irq; |
35a17eb6 | 859 | } |
e9870c4c | 860 | |
759f89e0 DM |
861 | static const struct sparc64_msiq_ops pci_sun4v_msiq_ops = { |
862 | .get_head = pci_sun4v_get_head, | |
863 | .dequeue_msi = pci_sun4v_dequeue_msi, | |
864 | .set_head = pci_sun4v_set_head, | |
865 | .msi_setup = pci_sun4v_msi_setup, | |
866 | .msi_teardown = pci_sun4v_msi_teardown, | |
867 | .msiq_alloc = pci_sun4v_msiq_alloc, | |
868 | .msiq_free = pci_sun4v_msiq_free, | |
869 | .msiq_build_irq = pci_sun4v_msiq_build_irq, | |
870 | }; | |
871 | ||
e9870c4c DM |
872 | static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) |
873 | { | |
759f89e0 | 874 | sparc64_pbm_msi_init(pbm, &pci_sun4v_msiq_ops); |
e9870c4c | 875 | } |
35a17eb6 DM |
876 | #else /* CONFIG_PCI_MSI */ |
877 | static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) | |
878 | { | |
879 | } | |
880 | #endif /* !(CONFIG_PCI_MSI) */ | |
881 | ||
9a2ed5cc | 882 | static int __devinit pci_sun4v_pbm_init(struct pci_pbm_info *pbm, |
cd4cd730 | 883 | struct platform_device *op, u32 devhandle) |
bade5622 | 884 | { |
61c7a080 | 885 | struct device_node *dp = op->dev.of_node; |
3822b509 | 886 | int err; |
bade5622 | 887 | |
c1b1a5f1 DM |
888 | pbm->numa_node = of_node_to_nid(dp); |
889 | ||
ca3dd88e DM |
890 | pbm->pci_ops = &sun4v_pci_ops; |
891 | pbm->config_space_reg_bits = 12; | |
34768bc8 | 892 | |
6c108f12 DM |
893 | pbm->index = pci_num_pbms++; |
894 | ||
22fecbae | 895 | pbm->op = op; |
bade5622 | 896 | |
3833789b | 897 | pbm->devhandle = devhandle; |
bade5622 | 898 | |
e87dc350 | 899 | pbm->name = dp->full_name; |
bade5622 | 900 | |
e87dc350 | 901 | printk("%s: SUN4V PCI Bus Module\n", pbm->name); |
c1b1a5f1 | 902 | printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node); |
bade5622 | 903 | |
9fd8b647 | 904 | pci_determine_mem_io_space(pbm); |
bade5622 | 905 | |
cfa0652c | 906 | pci_get_pbm_props(pbm); |
3822b509 DM |
907 | |
908 | err = pci_sun4v_iommu_init(pbm); | |
909 | if (err) | |
910 | return err; | |
911 | ||
35a17eb6 | 912 | pci_sun4v_msi_init(pbm); |
3822b509 | 913 | |
e822358a | 914 | pci_sun4v_scan_bus(pbm, &op->dev); |
3822b509 | 915 | |
d3ae4b5b DM |
916 | pbm->next = pci_pbm_root; |
917 | pci_pbm_root = pbm; | |
918 | ||
3822b509 | 919 | return 0; |
bade5622 DM |
920 | } |
921 | ||
4ebb24f7 | 922 | static int __devinit pci_sun4v_probe(struct platform_device *op) |
8f6a93a1 | 923 | { |
3822b509 | 924 | const struct linux_prom64_registers *regs; |
e01c0d6d | 925 | static int hvapi_negotiated = 0; |
34768bc8 | 926 | struct pci_pbm_info *pbm; |
3822b509 | 927 | struct device_node *dp; |
16ce82d8 | 928 | struct iommu *iommu; |
7c8f486a | 929 | u32 devhandle; |
d7472c38 | 930 | int i, err; |
3833789b | 931 | |
61c7a080 | 932 | dp = op->dev.of_node; |
3822b509 | 933 | |
e01c0d6d | 934 | if (!hvapi_negotiated++) { |
8d2aec51 DM |
935 | err = sun4v_hvapi_register(HV_GRP_PCI, |
936 | vpci_major, | |
937 | &vpci_minor); | |
e01c0d6d DM |
938 | |
939 | if (err) { | |
3822b509 DM |
940 | printk(KERN_ERR PFX "Could not register hvapi, " |
941 | "err=%d\n", err); | |
942 | return err; | |
e01c0d6d | 943 | } |
3822b509 | 944 | printk(KERN_INFO PFX "Registered hvapi major[%lu] minor[%lu]\n", |
e01c0d6d | 945 | vpci_major, vpci_minor); |
ad7ad57c DM |
946 | |
947 | dma_ops = &sun4v_dma_ops; | |
e01c0d6d DM |
948 | } |
949 | ||
3822b509 | 950 | regs = of_get_property(dp, "reg", NULL); |
d7472c38 | 951 | err = -ENODEV; |
3822b509 DM |
952 | if (!regs) { |
953 | printk(KERN_ERR PFX "Could not find config registers\n"); | |
d7472c38 | 954 | goto out_err; |
75c6d141 | 955 | } |
e87dc350 | 956 | devhandle = (regs->phys_addr >> 32UL) & 0x0fffffff; |
3833789b | 957 | |
d7472c38 | 958 | err = -ENOMEM; |
d3ae4b5b DM |
959 | if (!iommu_batch_initialized) { |
960 | for_each_possible_cpu(i) { | |
961 | unsigned long page = get_zeroed_page(GFP_KERNEL); | |
7c8f486a | 962 | |
d3ae4b5b DM |
963 | if (!page) |
964 | goto out_err; | |
7c8f486a | 965 | |
d3ae4b5b DM |
966 | per_cpu(iommu_batch, i).pglist = (u64 *) page; |
967 | } | |
968 | iommu_batch_initialized = 1; | |
bade5622 | 969 | } |
7c8f486a | 970 | |
d3ae4b5b DM |
971 | pbm = kzalloc(sizeof(*pbm), GFP_KERNEL); |
972 | if (!pbm) { | |
973 | printk(KERN_ERR PFX "Could not allocate pci_pbm_info\n"); | |
d7472c38 | 974 | goto out_err; |
3822b509 | 975 | } |
7c8f486a | 976 | |
d3ae4b5b | 977 | iommu = kzalloc(sizeof(struct iommu), GFP_KERNEL); |
3822b509 | 978 | if (!iommu) { |
d3ae4b5b | 979 | printk(KERN_ERR PFX "Could not allocate pbm iommu\n"); |
d7472c38 | 980 | goto out_free_controller; |
3822b509 | 981 | } |
7c8f486a | 982 | |
d3ae4b5b | 983 | pbm->iommu = iommu; |
bade5622 | 984 | |
d3ae4b5b DM |
985 | err = pci_sun4v_pbm_init(pbm, op, devhandle); |
986 | if (err) | |
987 | goto out_free_iommu; | |
7c8f486a | 988 | |
d3ae4b5b | 989 | dev_set_drvdata(&op->dev, pbm); |
bade5622 | 990 | |
d3ae4b5b | 991 | return 0; |
7c8f486a | 992 | |
d3ae4b5b DM |
993 | out_free_iommu: |
994 | kfree(pbm->iommu); | |
d7472c38 DM |
995 | |
996 | out_free_controller: | |
d3ae4b5b | 997 | kfree(pbm); |
d7472c38 DM |
998 | |
999 | out_err: | |
1000 | return err; | |
8f6a93a1 | 1001 | } |
3822b509 | 1002 | |
3628aa06 | 1003 | static const struct of_device_id pci_sun4v_match[] = { |
3822b509 DM |
1004 | { |
1005 | .name = "pci", | |
1006 | .compatible = "SUNW,sun4v-pci", | |
1007 | }, | |
1008 | {}, | |
1009 | }; | |
1010 | ||
4ebb24f7 | 1011 | static struct platform_driver pci_sun4v_driver = { |
4018294b GL |
1012 | .driver = { |
1013 | .name = DRIVER_NAME, | |
1014 | .owner = THIS_MODULE, | |
1015 | .of_match_table = pci_sun4v_match, | |
1016 | }, | |
3822b509 DM |
1017 | .probe = pci_sun4v_probe, |
1018 | }; | |
1019 | ||
1020 | static int __init pci_sun4v_init(void) | |
1021 | { | |
4ebb24f7 | 1022 | return platform_driver_register(&pci_sun4v_driver); |
3822b509 DM |
1023 | } |
1024 | ||
1025 | subsys_initcall(pci_sun4v_init); |