1 /* pci_sun4v.c: SUN4V specific PCI controller support.
3 * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
6 #include <linux/kernel.h>
7 #include <linux/types.h>
9 #include <linux/init.h>
10 #include <linux/slab.h>
11 #include <linux/interrupt.h>
12 #include <linux/percpu.h>
13 #include <linux/irq.h>
14 #include <linux/msi.h>
15 #include <linux/log2.h>
17 #include <asm/iommu.h>
20 #include <asm/pstate.h>
21 #include <asm/oplib.h>
22 #include <asm/hypervisor.h>
26 #include "iommu_common.h"
28 #include "pci_sun4v.h"
30 static unsigned long vpci_major
= 1;
31 static unsigned long vpci_minor
= 1;
33 #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64))
36 struct device
*dev
; /* Device mapping is for. */
37 unsigned long prot
; /* IOMMU page protections */
38 unsigned long entry
; /* Index into IOTSB. */
39 u64
*pglist
; /* List of physical pages */
40 unsigned long npages
; /* Number of pages in list. */
43 static DEFINE_PER_CPU(struct iommu_batch
, iommu_batch
);
45 /* Interrupts must be disabled. */
46 static inline void iommu_batch_start(struct device
*dev
, unsigned long prot
, unsigned long entry
)
48 struct iommu_batch
*p
= &__get_cpu_var(iommu_batch
);
56 /* Interrupts must be disabled. */
57 static long iommu_batch_flush(struct iommu_batch
*p
)
59 struct pci_pbm_info
*pbm
= p
->dev
->archdata
.host_controller
;
60 unsigned long devhandle
= pbm
->devhandle
;
61 unsigned long prot
= p
->prot
;
62 unsigned long entry
= p
->entry
;
63 u64
*pglist
= p
->pglist
;
64 unsigned long npages
= p
->npages
;
69 num
= pci_sun4v_iommu_map(devhandle
, HV_PCI_TSBID(0, entry
),
70 npages
, prot
, __pa(pglist
));
71 if (unlikely(num
< 0)) {
72 if (printk_ratelimit())
73 printk("iommu_batch_flush: IOMMU map of "
74 "[%08lx:%08lx:%lx:%lx:%lx] failed with "
76 devhandle
, HV_PCI_TSBID(0, entry
),
77 npages
, prot
, __pa(pglist
), num
);
92 /* Interrupts must be disabled. */
93 static inline long iommu_batch_add(u64 phys_page
)
95 struct iommu_batch
*p
= &__get_cpu_var(iommu_batch
);
97 BUG_ON(p
->npages
>= PGLIST_NENTS
);
99 p
->pglist
[p
->npages
++] = phys_page
;
100 if (p
->npages
== PGLIST_NENTS
)
101 return iommu_batch_flush(p
);
106 /* Interrupts must be disabled. */
107 static inline long iommu_batch_end(void)
109 struct iommu_batch
*p
= &__get_cpu_var(iommu_batch
);
111 BUG_ON(p
->npages
>= PGLIST_NENTS
);
113 return iommu_batch_flush(p
);
116 static long arena_alloc(struct iommu_arena
*arena
, unsigned long npages
)
118 unsigned long n
, i
, start
, end
, limit
;
121 limit
= arena
->limit
;
126 n
= find_next_zero_bit(arena
->map
, limit
, start
);
128 if (unlikely(end
>= limit
)) {
129 if (likely(pass
< 1)) {
135 /* Scanned the whole thing, give up. */
140 for (i
= n
; i
< end
; i
++) {
141 if (test_bit(i
, arena
->map
)) {
147 for (i
= n
; i
< end
; i
++)
148 __set_bit(i
, arena
->map
);
155 static void arena_free(struct iommu_arena
*arena
, unsigned long base
,
156 unsigned long npages
)
160 for (i
= base
; i
< (base
+ npages
); i
++)
161 __clear_bit(i
, arena
->map
);
164 static void *dma_4v_alloc_coherent(struct device
*dev
, size_t size
,
165 dma_addr_t
*dma_addrp
, gfp_t gfp
)
168 unsigned long flags
, order
, first_page
, npages
, n
;
172 size
= IO_PAGE_ALIGN(size
);
173 order
= get_order(size
);
174 if (unlikely(order
>= MAX_ORDER
))
177 npages
= size
>> IO_PAGE_SHIFT
;
179 first_page
= __get_free_pages(gfp
, order
);
180 if (unlikely(first_page
== 0UL))
183 memset((char *)first_page
, 0, PAGE_SIZE
<< order
);
185 iommu
= dev
->archdata
.iommu
;
187 spin_lock_irqsave(&iommu
->lock
, flags
);
188 entry
= arena_alloc(&iommu
->arena
, npages
);
189 spin_unlock_irqrestore(&iommu
->lock
, flags
);
191 if (unlikely(entry
< 0L))
192 goto arena_alloc_fail
;
194 *dma_addrp
= (iommu
->page_table_map_base
+
195 (entry
<< IO_PAGE_SHIFT
));
196 ret
= (void *) first_page
;
197 first_page
= __pa(first_page
);
199 local_irq_save(flags
);
201 iommu_batch_start(dev
,
202 (HV_PCI_MAP_ATTR_READ
|
203 HV_PCI_MAP_ATTR_WRITE
),
206 for (n
= 0; n
< npages
; n
++) {
207 long err
= iommu_batch_add(first_page
+ (n
* PAGE_SIZE
));
208 if (unlikely(err
< 0L))
212 if (unlikely(iommu_batch_end() < 0L))
215 local_irq_restore(flags
);
220 /* Interrupts are disabled. */
221 spin_lock(&iommu
->lock
);
222 arena_free(&iommu
->arena
, entry
, npages
);
223 spin_unlock_irqrestore(&iommu
->lock
, flags
);
226 free_pages(first_page
, order
);
230 static void dma_4v_free_coherent(struct device
*dev
, size_t size
, void *cpu
,
233 struct pci_pbm_info
*pbm
;
235 unsigned long flags
, order
, npages
, entry
;
238 npages
= IO_PAGE_ALIGN(size
) >> IO_PAGE_SHIFT
;
239 iommu
= dev
->archdata
.iommu
;
240 pbm
= dev
->archdata
.host_controller
;
241 devhandle
= pbm
->devhandle
;
242 entry
= ((dvma
- iommu
->page_table_map_base
) >> IO_PAGE_SHIFT
);
244 spin_lock_irqsave(&iommu
->lock
, flags
);
246 arena_free(&iommu
->arena
, entry
, npages
);
251 num
= pci_sun4v_iommu_demap(devhandle
, HV_PCI_TSBID(0, entry
),
255 } while (npages
!= 0);
257 spin_unlock_irqrestore(&iommu
->lock
, flags
);
259 order
= get_order(size
);
261 free_pages((unsigned long)cpu
, order
);
264 static dma_addr_t
dma_4v_map_single(struct device
*dev
, void *ptr
, size_t sz
,
265 enum dma_data_direction direction
)
268 unsigned long flags
, npages
, oaddr
;
269 unsigned long i
, base_paddr
;
274 iommu
= dev
->archdata
.iommu
;
276 if (unlikely(direction
== DMA_NONE
))
279 oaddr
= (unsigned long)ptr
;
280 npages
= IO_PAGE_ALIGN(oaddr
+ sz
) - (oaddr
& IO_PAGE_MASK
);
281 npages
>>= IO_PAGE_SHIFT
;
283 spin_lock_irqsave(&iommu
->lock
, flags
);
284 entry
= arena_alloc(&iommu
->arena
, npages
);
285 spin_unlock_irqrestore(&iommu
->lock
, flags
);
287 if (unlikely(entry
< 0L))
290 bus_addr
= (iommu
->page_table_map_base
+
291 (entry
<< IO_PAGE_SHIFT
));
292 ret
= bus_addr
| (oaddr
& ~IO_PAGE_MASK
);
293 base_paddr
= __pa(oaddr
& IO_PAGE_MASK
);
294 prot
= HV_PCI_MAP_ATTR_READ
;
295 if (direction
!= DMA_TO_DEVICE
)
296 prot
|= HV_PCI_MAP_ATTR_WRITE
;
298 local_irq_save(flags
);
300 iommu_batch_start(dev
, prot
, entry
);
302 for (i
= 0; i
< npages
; i
++, base_paddr
+= IO_PAGE_SIZE
) {
303 long err
= iommu_batch_add(base_paddr
);
304 if (unlikely(err
< 0L))
307 if (unlikely(iommu_batch_end() < 0L))
310 local_irq_restore(flags
);
315 if (printk_ratelimit())
317 return DMA_ERROR_CODE
;
320 /* Interrupts are disabled. */
321 spin_lock(&iommu
->lock
);
322 arena_free(&iommu
->arena
, entry
, npages
);
323 spin_unlock_irqrestore(&iommu
->lock
, flags
);
325 return DMA_ERROR_CODE
;
328 static void dma_4v_unmap_single(struct device
*dev
, dma_addr_t bus_addr
,
329 size_t sz
, enum dma_data_direction direction
)
331 struct pci_pbm_info
*pbm
;
333 unsigned long flags
, npages
;
337 if (unlikely(direction
== DMA_NONE
)) {
338 if (printk_ratelimit())
343 iommu
= dev
->archdata
.iommu
;
344 pbm
= dev
->archdata
.host_controller
;
345 devhandle
= pbm
->devhandle
;
347 npages
= IO_PAGE_ALIGN(bus_addr
+ sz
) - (bus_addr
& IO_PAGE_MASK
);
348 npages
>>= IO_PAGE_SHIFT
;
349 bus_addr
&= IO_PAGE_MASK
;
351 spin_lock_irqsave(&iommu
->lock
, flags
);
353 entry
= (bus_addr
- iommu
->page_table_map_base
) >> IO_PAGE_SHIFT
;
354 arena_free(&iommu
->arena
, entry
, npages
);
359 num
= pci_sun4v_iommu_demap(devhandle
, HV_PCI_TSBID(0, entry
),
363 } while (npages
!= 0);
365 spin_unlock_irqrestore(&iommu
->lock
, flags
);
368 #define SG_ENT_PHYS_ADDRESS(SG) \
369 (__pa(page_address((SG)->page)) + (SG)->offset)
371 static inline long fill_sg(long entry
, struct device
*dev
,
372 struct scatterlist
*sg
,
373 int nused
, int nelems
, unsigned long prot
)
375 struct scatterlist
*dma_sg
= sg
;
376 struct scatterlist
*sg_end
= sg
+ nelems
;
380 local_irq_save(flags
);
382 iommu_batch_start(dev
, prot
, entry
);
384 for (i
= 0; i
< nused
; i
++) {
385 unsigned long pteval
= ~0UL;
388 dma_npages
= ((dma_sg
->dma_address
& (IO_PAGE_SIZE
- 1UL)) +
390 ((IO_PAGE_SIZE
- 1UL))) >> IO_PAGE_SHIFT
;
392 unsigned long offset
;
395 /* If we are here, we know we have at least one
396 * more page to map. So walk forward until we
397 * hit a page crossing, and begin creating new
398 * mappings from that spot.
403 tmp
= SG_ENT_PHYS_ADDRESS(sg
);
405 if (((tmp
^ pteval
) >> IO_PAGE_SHIFT
) != 0UL) {
406 pteval
= tmp
& IO_PAGE_MASK
;
407 offset
= tmp
& (IO_PAGE_SIZE
- 1UL);
410 if (((tmp
^ (tmp
+ len
- 1UL)) >> IO_PAGE_SHIFT
) != 0UL) {
411 pteval
= (tmp
+ IO_PAGE_SIZE
) & IO_PAGE_MASK
;
413 len
-= (IO_PAGE_SIZE
- (tmp
& (IO_PAGE_SIZE
- 1UL)));
419 pteval
= (pteval
& IOPTE_PAGE
);
423 err
= iommu_batch_add(pteval
);
424 if (unlikely(err
< 0L))
425 goto iommu_map_failed
;
427 pteval
+= IO_PAGE_SIZE
;
428 len
-= (IO_PAGE_SIZE
- offset
);
433 pteval
= (pteval
& IOPTE_PAGE
) + len
;
436 /* Skip over any tail mappings we've fully mapped,
437 * adjusting pteval along the way. Stop when we
438 * detect a page crossing event.
440 while (sg
< sg_end
&&
441 (pteval
<< (64 - IO_PAGE_SHIFT
)) != 0UL &&
442 (pteval
== SG_ENT_PHYS_ADDRESS(sg
)) &&
444 (SG_ENT_PHYS_ADDRESS(sg
) + sg
->length
- 1UL)) >> IO_PAGE_SHIFT
) == 0UL) {
445 pteval
+= sg
->length
;
448 if ((pteval
<< (64 - IO_PAGE_SHIFT
)) == 0UL)
450 } while (dma_npages
!= 0);
454 if (unlikely(iommu_batch_end() < 0L))
455 goto iommu_map_failed
;
457 local_irq_restore(flags
);
461 local_irq_restore(flags
);
465 static int dma_4v_map_sg(struct device
*dev
, struct scatterlist
*sglist
,
466 int nelems
, enum dma_data_direction direction
)
469 unsigned long flags
, npages
, prot
;
471 struct scatterlist
*sgtmp
;
475 /* Fast path single entry scatterlists. */
477 sglist
->dma_address
=
478 dma_4v_map_single(dev
,
479 (page_address(sglist
->page
) +
481 sglist
->length
, direction
);
482 if (unlikely(sglist
->dma_address
== DMA_ERROR_CODE
))
484 sglist
->dma_length
= sglist
->length
;
488 iommu
= dev
->archdata
.iommu
;
490 if (unlikely(direction
== DMA_NONE
))
493 /* Step 1: Prepare scatter list. */
494 npages
= prepare_sg(sglist
, nelems
);
496 /* Step 2: Allocate a cluster and context, if necessary. */
497 spin_lock_irqsave(&iommu
->lock
, flags
);
498 entry
= arena_alloc(&iommu
->arena
, npages
);
499 spin_unlock_irqrestore(&iommu
->lock
, flags
);
501 if (unlikely(entry
< 0L))
504 dma_base
= iommu
->page_table_map_base
+
505 (entry
<< IO_PAGE_SHIFT
);
507 /* Step 3: Normalize DMA addresses. */
511 while (used
&& sgtmp
->dma_length
) {
512 sgtmp
->dma_address
+= dma_base
;
516 used
= nelems
- used
;
518 /* Step 4: Create the mappings. */
519 prot
= HV_PCI_MAP_ATTR_READ
;
520 if (direction
!= DMA_TO_DEVICE
)
521 prot
|= HV_PCI_MAP_ATTR_WRITE
;
523 err
= fill_sg(entry
, dev
, sglist
, used
, nelems
, prot
);
524 if (unlikely(err
< 0L))
525 goto iommu_map_failed
;
530 if (printk_ratelimit())
535 spin_lock_irqsave(&iommu
->lock
, flags
);
536 arena_free(&iommu
->arena
, entry
, npages
);
537 spin_unlock_irqrestore(&iommu
->lock
, flags
);
542 static void dma_4v_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
543 int nelems
, enum dma_data_direction direction
)
545 struct pci_pbm_info
*pbm
;
547 unsigned long flags
, i
, npages
;
549 u32 devhandle
, bus_addr
;
551 if (unlikely(direction
== DMA_NONE
)) {
552 if (printk_ratelimit())
556 iommu
= dev
->archdata
.iommu
;
557 pbm
= dev
->archdata
.host_controller
;
558 devhandle
= pbm
->devhandle
;
560 bus_addr
= sglist
->dma_address
& IO_PAGE_MASK
;
562 for (i
= 1; i
< nelems
; i
++)
563 if (sglist
[i
].dma_length
== 0)
566 npages
= (IO_PAGE_ALIGN(sglist
[i
].dma_address
+ sglist
[i
].dma_length
) -
567 bus_addr
) >> IO_PAGE_SHIFT
;
569 entry
= ((bus_addr
- iommu
->page_table_map_base
) >> IO_PAGE_SHIFT
);
571 spin_lock_irqsave(&iommu
->lock
, flags
);
573 arena_free(&iommu
->arena
, entry
, npages
);
578 num
= pci_sun4v_iommu_demap(devhandle
, HV_PCI_TSBID(0, entry
),
582 } while (npages
!= 0);
584 spin_unlock_irqrestore(&iommu
->lock
, flags
);
587 static void dma_4v_sync_single_for_cpu(struct device
*dev
,
588 dma_addr_t bus_addr
, size_t sz
,
589 enum dma_data_direction direction
)
591 /* Nothing to do... */
594 static void dma_4v_sync_sg_for_cpu(struct device
*dev
,
595 struct scatterlist
*sglist
, int nelems
,
596 enum dma_data_direction direction
)
598 /* Nothing to do... */
601 const struct dma_ops sun4v_dma_ops
= {
602 .alloc_coherent
= dma_4v_alloc_coherent
,
603 .free_coherent
= dma_4v_free_coherent
,
604 .map_single
= dma_4v_map_single
,
605 .unmap_single
= dma_4v_unmap_single
,
606 .map_sg
= dma_4v_map_sg
,
607 .unmap_sg
= dma_4v_unmap_sg
,
608 .sync_single_for_cpu
= dma_4v_sync_single_for_cpu
,
609 .sync_sg_for_cpu
= dma_4v_sync_sg_for_cpu
,
612 static void pci_sun4v_scan_bus(struct pci_pbm_info
*pbm
)
614 struct property
*prop
;
615 struct device_node
*dp
;
618 prop
= of_find_property(dp
, "66mhz-capable", NULL
);
619 pbm
->is_66mhz_capable
= (prop
!= NULL
);
620 pbm
->pci_bus
= pci_scan_one_pbm(pbm
);
622 /* XXX register error interrupt handlers XXX */
625 static unsigned long probe_existing_entries(struct pci_pbm_info
*pbm
,
628 struct iommu_arena
*arena
= &iommu
->arena
;
629 unsigned long i
, cnt
= 0;
632 devhandle
= pbm
->devhandle
;
633 for (i
= 0; i
< arena
->limit
; i
++) {
634 unsigned long ret
, io_attrs
, ra
;
636 ret
= pci_sun4v_iommu_getmap(devhandle
,
640 if (page_in_phys_avail(ra
)) {
641 pci_sun4v_iommu_demap(devhandle
,
642 HV_PCI_TSBID(0, i
), 1);
645 __set_bit(i
, arena
->map
);
653 static void pci_sun4v_iommu_init(struct pci_pbm_info
*pbm
)
655 struct iommu
*iommu
= pbm
->iommu
;
656 struct property
*prop
;
657 unsigned long num_tsb_entries
, sz
, tsbsize
;
658 u32 vdma
[2], dma_mask
, dma_offset
;
660 prop
= of_find_property(pbm
->prom_node
, "virtual-dma", NULL
);
662 u32
*val
= prop
->value
;
667 /* No property, use default values. */
668 vdma
[0] = 0x80000000;
669 vdma
[1] = 0x80000000;
672 if ((vdma
[0] | vdma
[1]) & ~IO_PAGE_MASK
) {
673 prom_printf("PCI-SUN4V: strange virtual-dma[%08x:%08x].\n",
678 dma_mask
= (roundup_pow_of_two(vdma
[1]) - 1UL);
679 num_tsb_entries
= vdma
[1] / IO_PAGE_SIZE
;
680 tsbsize
= num_tsb_entries
* sizeof(iopte_t
);
682 dma_offset
= vdma
[0];
684 /* Setup initial software IOMMU state. */
685 spin_lock_init(&iommu
->lock
);
686 iommu
->ctx_lowest_free
= 1;
687 iommu
->page_table_map_base
= dma_offset
;
688 iommu
->dma_addr_mask
= dma_mask
;
690 /* Allocate and initialize the free area map. */
691 sz
= (num_tsb_entries
+ 7) / 8;
692 sz
= (sz
+ 7UL) & ~7UL;
693 iommu
->arena
.map
= kzalloc(sz
, GFP_KERNEL
);
694 if (!iommu
->arena
.map
) {
695 prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n");
698 iommu
->arena
.limit
= num_tsb_entries
;
700 sz
= probe_existing_entries(pbm
, iommu
);
702 printk("%s: Imported %lu TSB entries from OBP\n",
706 #ifdef CONFIG_PCI_MSI
707 struct pci_sun4v_msiq_entry
{
709 #define MSIQ_VERSION_MASK 0xffffffff00000000UL
710 #define MSIQ_VERSION_SHIFT 32
711 #define MSIQ_TYPE_MASK 0x00000000000000ffUL
712 #define MSIQ_TYPE_SHIFT 0
713 #define MSIQ_TYPE_NONE 0x00
714 #define MSIQ_TYPE_MSG 0x01
715 #define MSIQ_TYPE_MSI32 0x02
716 #define MSIQ_TYPE_MSI64 0x03
717 #define MSIQ_TYPE_INTX 0x08
718 #define MSIQ_TYPE_NONE2 0xff
723 u64 req_id
; /* bus/device/func */
724 #define MSIQ_REQID_BUS_MASK 0xff00UL
725 #define MSIQ_REQID_BUS_SHIFT 8
726 #define MSIQ_REQID_DEVICE_MASK 0x00f8UL
727 #define MSIQ_REQID_DEVICE_SHIFT 3
728 #define MSIQ_REQID_FUNC_MASK 0x0007UL
729 #define MSIQ_REQID_FUNC_SHIFT 0
733 /* The format of this value is message type dependent.
734 * For MSI bits 15:0 are the data from the MSI packet.
735 * For MSI-X bits 31:0 are the data from the MSI packet.
736 * For MSG, the message code and message routing code where:
737 * bits 39:32 is the bus/device/fn of the msg target-id
738 * bits 18:16 is the message routing code
739 * bits 7:0 is the message code
740 * For INTx the low order 2-bits are:
751 /* For now this just runs as a pre-handler for the real interrupt handler.
752 * So we just walk through the queue and ACK all the entries, update the
753 * head pointer, and return.
755 * In the longer term it would be nice to do something more integrated
756 * wherein we can pass in some of this MSI info to the drivers. This
757 * would be most useful for PCIe fabric error messages, although we could
758 * invoke those directly from the loop here in order to pass the info around.
760 static void pci_sun4v_msi_prehandler(unsigned int ino
, void *data1
, void *data2
)
762 struct pci_pbm_info
*pbm
= data1
;
763 struct pci_sun4v_msiq_entry
*base
, *ep
;
764 unsigned long msiqid
, orig_head
, head
, type
, err
;
766 msiqid
= (unsigned long) data2
;
769 err
= pci_sun4v_msiq_gethead(pbm
->devhandle
, msiqid
, &head
);
773 if (unlikely(head
>= (pbm
->msiq_ent_count
* sizeof(struct pci_sun4v_msiq_entry
))))
776 head
/= sizeof(struct pci_sun4v_msiq_entry
);
778 base
= (pbm
->msi_queues
+ ((msiqid
- pbm
->msiq_first
) *
779 (pbm
->msiq_ent_count
*
780 sizeof(struct pci_sun4v_msiq_entry
))));
782 while ((ep
->version_type
& MSIQ_TYPE_MASK
) != 0) {
783 type
= (ep
->version_type
& MSIQ_TYPE_MASK
) >> MSIQ_TYPE_SHIFT
;
784 if (unlikely(type
!= MSIQ_TYPE_MSI32
&&
785 type
!= MSIQ_TYPE_MSI64
))
788 pci_sun4v_msi_setstate(pbm
->devhandle
,
789 ep
->msi_data
/* msi_num */,
792 /* Clear the entry. */
793 ep
->version_type
&= ~MSIQ_TYPE_MASK
;
795 /* Go to next entry in ring. */
797 if (head
>= pbm
->msiq_ent_count
)
802 if (likely(head
!= orig_head
)) {
803 /* ACK entries by updating head pointer. */
804 head
*= sizeof(struct pci_sun4v_msiq_entry
);
805 err
= pci_sun4v_msiq_sethead(pbm
->devhandle
, msiqid
, head
);
812 printk(KERN_EMERG
"MSI: Hypervisor set head gives error %lu\n", err
);
816 printk(KERN_EMERG
"MSI: Hypervisor get head gives error %lu\n", err
);
819 printk(KERN_EMERG
"MSI: devhandle[%x] msiqid[%lx] head[%lu]\n",
820 pbm
->devhandle
, msiqid
, head
);
824 printk(KERN_EMERG
"MSI: Hypervisor gives bad offset %lx max(%lx)\n",
825 head
, pbm
->msiq_ent_count
* sizeof(struct pci_sun4v_msiq_entry
));
829 printk(KERN_EMERG
"MSI: Entry has bad type %lx\n", type
);
833 static int msi_bitmap_alloc(struct pci_pbm_info
*pbm
)
835 unsigned long size
, bits_per_ulong
;
837 bits_per_ulong
= sizeof(unsigned long) * 8;
838 size
= (pbm
->msi_num
+ (bits_per_ulong
- 1)) & ~(bits_per_ulong
- 1);
840 BUG_ON(size
% sizeof(unsigned long));
842 pbm
->msi_bitmap
= kzalloc(size
, GFP_KERNEL
);
843 if (!pbm
->msi_bitmap
)
849 static void msi_bitmap_free(struct pci_pbm_info
*pbm
)
851 kfree(pbm
->msi_bitmap
);
852 pbm
->msi_bitmap
= NULL
;
855 static int msi_queue_alloc(struct pci_pbm_info
*pbm
)
857 unsigned long q_size
, alloc_size
, pages
, order
;
860 q_size
= pbm
->msiq_ent_count
* sizeof(struct pci_sun4v_msiq_entry
);
861 alloc_size
= (pbm
->msiq_num
* q_size
);
862 order
= get_order(alloc_size
);
863 pages
= __get_free_pages(GFP_KERNEL
| __GFP_COMP
, order
);
865 printk(KERN_ERR
"MSI: Cannot allocate MSI queues (o=%lu).\n",
869 memset((char *)pages
, 0, PAGE_SIZE
<< order
);
870 pbm
->msi_queues
= (void *) pages
;
872 for (i
= 0; i
< pbm
->msiq_num
; i
++) {
873 unsigned long err
, base
= __pa(pages
+ (i
* q_size
));
874 unsigned long ret1
, ret2
;
876 err
= pci_sun4v_msiq_conf(pbm
->devhandle
,
878 base
, pbm
->msiq_ent_count
);
880 printk(KERN_ERR
"MSI: msiq register fails (err=%lu)\n",
885 err
= pci_sun4v_msiq_info(pbm
->devhandle
,
889 printk(KERN_ERR
"MSI: Cannot read msiq (err=%lu)\n",
893 if (ret1
!= base
|| ret2
!= pbm
->msiq_ent_count
) {
894 printk(KERN_ERR
"MSI: Bogus qconf "
895 "expected[%lx:%x] got[%lx:%lx]\n",
896 base
, pbm
->msiq_ent_count
,
905 free_pages(pages
, order
);
910 static int alloc_msi(struct pci_pbm_info
*pbm
)
914 for (i
= 0; i
< pbm
->msi_num
; i
++) {
915 if (!test_and_set_bit(i
, pbm
->msi_bitmap
))
916 return i
+ pbm
->msi_first
;
922 static void free_msi(struct pci_pbm_info
*pbm
, int msi_num
)
924 msi_num
-= pbm
->msi_first
;
925 clear_bit(msi_num
, pbm
->msi_bitmap
);
928 static int pci_sun4v_setup_msi_irq(unsigned int *virt_irq_p
,
929 struct pci_dev
*pdev
,
930 struct msi_desc
*entry
)
932 struct pci_pbm_info
*pbm
= pdev
->dev
.archdata
.host_controller
;
933 unsigned long devino
, msiqid
;
939 msi_num
= alloc_msi(pbm
);
943 devino
= sun4v_build_msi(pbm
->devhandle
, virt_irq_p
,
944 pbm
->msiq_first_devino
,
945 (pbm
->msiq_first_devino
+
951 msiqid
= ((devino
- pbm
->msiq_first_devino
) +
955 if (pci_sun4v_msiq_setstate(pbm
->devhandle
, msiqid
, HV_MSIQSTATE_IDLE
))
959 if (pci_sun4v_msiq_setvalid(pbm
->devhandle
, msiqid
, HV_MSIQ_VALID
))
962 if (pci_sun4v_msi_setmsiq(pbm
->devhandle
,
964 (entry
->msi_attrib
.is_64
?
965 HV_MSITYPE_MSI64
: HV_MSITYPE_MSI32
)))
968 if (pci_sun4v_msi_setstate(pbm
->devhandle
, msi_num
, HV_MSISTATE_IDLE
))
971 if (pci_sun4v_msi_setvalid(pbm
->devhandle
, msi_num
, HV_MSIVALID_VALID
))
974 pdev
->dev
.archdata
.msi_num
= msi_num
;
976 if (entry
->msi_attrib
.is_64
) {
977 msg
.address_hi
= pbm
->msi64_start
>> 32;
978 msg
.address_lo
= pbm
->msi64_start
& 0xffffffff;
981 msg
.address_lo
= pbm
->msi32_start
;
985 set_irq_msi(*virt_irq_p
, entry
);
986 write_msi_msg(*virt_irq_p
, &msg
);
988 irq_install_pre_handler(*virt_irq_p
,
989 pci_sun4v_msi_prehandler
,
990 pbm
, (void *) msiqid
);
995 free_msi(pbm
, msi_num
);
996 sun4v_destroy_msi(*virt_irq_p
);
1002 static void pci_sun4v_teardown_msi_irq(unsigned int virt_irq
,
1003 struct pci_dev
*pdev
)
1005 struct pci_pbm_info
*pbm
= pdev
->dev
.archdata
.host_controller
;
1006 unsigned long msiqid
, err
;
1007 unsigned int msi_num
;
1009 msi_num
= pdev
->dev
.archdata
.msi_num
;
1010 err
= pci_sun4v_msi_getmsiq(pbm
->devhandle
, msi_num
, &msiqid
);
1012 printk(KERN_ERR
"%s: getmsiq gives error %lu\n",
1017 pci_sun4v_msi_setvalid(pbm
->devhandle
, msi_num
, HV_MSIVALID_INVALID
);
1018 pci_sun4v_msiq_setvalid(pbm
->devhandle
, msiqid
, HV_MSIQ_INVALID
);
1020 free_msi(pbm
, msi_num
);
1022 /* The sun4v_destroy_msi() will liberate the devino and thus the MSIQ
1025 sun4v_destroy_msi(virt_irq
);
1028 static void pci_sun4v_msi_init(struct pci_pbm_info
*pbm
)
1033 val
= of_get_property(pbm
->prom_node
, "#msi-eqs", &len
);
1034 if (!val
|| len
!= 4)
1036 pbm
->msiq_num
= *val
;
1037 if (pbm
->msiq_num
) {
1038 const struct msiq_prop
{
1043 const struct msi_range_prop
{
1047 const struct addr_range_prop
{
1056 val
= of_get_property(pbm
->prom_node
, "msi-eq-size", &len
);
1057 if (!val
|| len
!= 4)
1060 pbm
->msiq_ent_count
= *val
;
1062 mqp
= of_get_property(pbm
->prom_node
,
1063 "msi-eq-to-devino", &len
);
1064 if (!mqp
|| len
!= sizeof(struct msiq_prop
))
1067 pbm
->msiq_first
= mqp
->first_msiq
;
1068 pbm
->msiq_first_devino
= mqp
->first_devino
;
1070 val
= of_get_property(pbm
->prom_node
, "#msi", &len
);
1071 if (!val
|| len
!= 4)
1073 pbm
->msi_num
= *val
;
1075 mrng
= of_get_property(pbm
->prom_node
, "msi-ranges", &len
);
1076 if (!mrng
|| len
!= sizeof(struct msi_range_prop
))
1078 pbm
->msi_first
= mrng
->first_msi
;
1080 val
= of_get_property(pbm
->prom_node
, "msi-data-mask", &len
);
1081 if (!val
|| len
!= 4)
1083 pbm
->msi_data_mask
= *val
;
1085 val
= of_get_property(pbm
->prom_node
, "msix-data-width", &len
);
1086 if (!val
|| len
!= 4)
1088 pbm
->msix_data_width
= *val
;
1090 arng
= of_get_property(pbm
->prom_node
, "msi-address-ranges",
1092 if (!arng
|| len
!= sizeof(struct addr_range_prop
))
1094 pbm
->msi32_start
= ((u64
)arng
->msi32_high
<< 32) |
1095 (u64
) arng
->msi32_low
;
1096 pbm
->msi64_start
= ((u64
)arng
->msi64_high
<< 32) |
1097 (u64
) arng
->msi64_low
;
1098 pbm
->msi32_len
= arng
->msi32_len
;
1099 pbm
->msi64_len
= arng
->msi64_len
;
1101 if (msi_bitmap_alloc(pbm
))
1104 if (msi_queue_alloc(pbm
)) {
1105 msi_bitmap_free(pbm
);
1109 printk(KERN_INFO
"%s: MSI Queue first[%u] num[%u] count[%u] "
1112 pbm
->msiq_first
, pbm
->msiq_num
,
1113 pbm
->msiq_ent_count
,
1114 pbm
->msiq_first_devino
);
1115 printk(KERN_INFO
"%s: MSI first[%u] num[%u] mask[0x%x] "
1118 pbm
->msi_first
, pbm
->msi_num
, pbm
->msi_data_mask
,
1119 pbm
->msix_data_width
);
1120 printk(KERN_INFO
"%s: MSI addr32[0x%lx:0x%x] "
1121 "addr64[0x%lx:0x%x]\n",
1123 pbm
->msi32_start
, pbm
->msi32_len
,
1124 pbm
->msi64_start
, pbm
->msi64_len
);
1125 printk(KERN_INFO
"%s: MSI queues at RA [%p]\n",
1129 pbm
->setup_msi_irq
= pci_sun4v_setup_msi_irq
;
1130 pbm
->teardown_msi_irq
= pci_sun4v_teardown_msi_irq
;
1136 printk(KERN_INFO
"%s: No MSI support.\n", pbm
->name
);
1138 #else /* CONFIG_PCI_MSI */
1139 static void pci_sun4v_msi_init(struct pci_pbm_info
*pbm
)
1142 #endif /* !(CONFIG_PCI_MSI) */
1144 static void __init
pci_sun4v_pbm_init(struct pci_controller_info
*p
, struct device_node
*dp
, u32 devhandle
)
1146 struct pci_pbm_info
*pbm
;
1148 if (devhandle
& 0x40)
1153 pbm
->next
= pci_pbm_root
;
1156 pbm
->scan_bus
= pci_sun4v_scan_bus
;
1157 pbm
->pci_ops
= &sun4v_pci_ops
;
1158 pbm
->config_space_reg_bits
= 12;
1160 pbm
->index
= pci_num_pbms
++;
1163 pbm
->prom_node
= dp
;
1165 pbm
->devhandle
= devhandle
;
1167 pbm
->name
= dp
->full_name
;
1169 printk("%s: SUN4V PCI Bus Module\n", pbm
->name
);
1171 pci_determine_mem_io_space(pbm
);
1173 pci_get_pbm_props(pbm
);
1174 pci_sun4v_iommu_init(pbm
);
1175 pci_sun4v_msi_init(pbm
);
1178 void __init
sun4v_pci_init(struct device_node
*dp
, char *model_name
)
1180 static int hvapi_negotiated
= 0;
1181 struct pci_controller_info
*p
;
1182 struct pci_pbm_info
*pbm
;
1183 struct iommu
*iommu
;
1184 struct property
*prop
;
1185 struct linux_prom64_registers
*regs
;
1189 if (!hvapi_negotiated
++) {
1190 int err
= sun4v_hvapi_register(HV_GRP_PCI
,
1195 prom_printf("SUN4V_PCI: Could not register hvapi, "
1199 printk("SUN4V_PCI: Registered hvapi major[%lu] minor[%lu]\n",
1200 vpci_major
, vpci_minor
);
1202 dma_ops
= &sun4v_dma_ops
;
1205 prop
= of_find_property(dp
, "reg", NULL
);
1208 devhandle
= (regs
->phys_addr
>> 32UL) & 0x0fffffff;
1210 for (pbm
= pci_pbm_root
; pbm
; pbm
= pbm
->next
) {
1211 if (pbm
->devhandle
== (devhandle
^ 0x40)) {
1212 pci_sun4v_pbm_init(pbm
->parent
, dp
, devhandle
);
1217 for_each_possible_cpu(i
) {
1218 unsigned long page
= get_zeroed_page(GFP_ATOMIC
);
1221 goto fatal_memory_error
;
1223 per_cpu(iommu_batch
, i
).pglist
= (u64
*) page
;
1226 p
= kzalloc(sizeof(struct pci_controller_info
), GFP_ATOMIC
);
1228 goto fatal_memory_error
;
1230 iommu
= kzalloc(sizeof(struct iommu
), GFP_ATOMIC
);
1232 goto fatal_memory_error
;
1234 p
->pbm_A
.iommu
= iommu
;
1236 iommu
= kzalloc(sizeof(struct iommu
), GFP_ATOMIC
);
1238 goto fatal_memory_error
;
1240 p
->pbm_B
.iommu
= iommu
;
1242 /* Like PSYCHO and SCHIZO we have a 2GB aligned area
1245 pci_memspace_mask
= 0x7fffffffUL
;
1247 pci_sun4v_pbm_init(p
, dp
, devhandle
);
1251 prom_printf("SUN4V_PCI: Fatal memory allocation error.\n");