3 * Purpose: PCI Message Signaled Interrupt (MSI)
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
11 #include <linux/irq.h>
12 #include <linux/interrupt.h>
13 #include <linux/export.h>
14 #include <linux/ioport.h>
15 #include <linux/pci.h>
16 #include <linux/proc_fs.h>
17 #include <linux/msi.h>
18 #include <linux/smp.h>
19 #include <linux/errno.h>
21 #include <linux/slab.h>
22 #include <linux/irqdomain.h>
26 static int pci_msi_enable
= 1;
27 int pci_msi_ignore_mask
;
29 #define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1)
31 #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
32 static struct irq_domain
*pci_msi_default_domain
;
33 static DEFINE_MUTEX(pci_msi_domain_lock
);
35 struct irq_domain
* __weak
arch_get_pci_msi_domain(struct pci_dev
*dev
)
37 return pci_msi_default_domain
;
40 static struct irq_domain
*pci_msi_get_domain(struct pci_dev
*dev
)
42 struct irq_domain
*domain
;
44 domain
= dev_get_msi_domain(&dev
->dev
);
48 return arch_get_pci_msi_domain(dev
);
51 static int pci_msi_setup_msi_irqs(struct pci_dev
*dev
, int nvec
, int type
)
53 struct irq_domain
*domain
;
55 domain
= pci_msi_get_domain(dev
);
57 return pci_msi_domain_alloc_irqs(domain
, dev
, nvec
, type
);
59 return arch_setup_msi_irqs(dev
, nvec
, type
);
62 static void pci_msi_teardown_msi_irqs(struct pci_dev
*dev
)
64 struct irq_domain
*domain
;
66 domain
= pci_msi_get_domain(dev
);
68 pci_msi_domain_free_irqs(domain
, dev
);
70 arch_teardown_msi_irqs(dev
);
73 #define pci_msi_setup_msi_irqs arch_setup_msi_irqs
74 #define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs
79 int __weak
arch_setup_msi_irq(struct pci_dev
*dev
, struct msi_desc
*desc
)
81 struct msi_controller
*chip
= dev
->bus
->msi
;
84 if (!chip
|| !chip
->setup_irq
)
87 err
= chip
->setup_irq(chip
, dev
, desc
);
91 irq_set_chip_data(desc
->irq
, chip
);
96 void __weak
arch_teardown_msi_irq(unsigned int irq
)
98 struct msi_controller
*chip
= irq_get_chip_data(irq
);
100 if (!chip
|| !chip
->teardown_irq
)
103 chip
->teardown_irq(chip
, irq
);
106 int __weak
arch_setup_msi_irqs(struct pci_dev
*dev
, int nvec
, int type
)
108 struct msi_desc
*entry
;
112 * If an architecture wants to support multiple MSI, it needs to
113 * override arch_setup_msi_irqs()
115 if (type
== PCI_CAP_ID_MSI
&& nvec
> 1)
118 for_each_pci_msi_entry(entry
, dev
) {
119 ret
= arch_setup_msi_irq(dev
, entry
);
130 * We have a default implementation available as a separate non-weak
131 * function, as it is used by the Xen x86 PCI code
133 void default_teardown_msi_irqs(struct pci_dev
*dev
)
136 struct msi_desc
*entry
;
138 for_each_pci_msi_entry(entry
, dev
)
140 for (i
= 0; i
< entry
->nvec_used
; i
++)
141 arch_teardown_msi_irq(entry
->irq
+ i
);
144 void __weak
arch_teardown_msi_irqs(struct pci_dev
*dev
)
146 return default_teardown_msi_irqs(dev
);
149 static void default_restore_msi_irq(struct pci_dev
*dev
, int irq
)
151 struct msi_desc
*entry
;
154 if (dev
->msix_enabled
) {
155 for_each_pci_msi_entry(entry
, dev
) {
156 if (irq
== entry
->irq
)
159 } else if (dev
->msi_enabled
) {
160 entry
= irq_get_msi_desc(irq
);
164 __pci_write_msi_msg(entry
, &entry
->msg
);
167 void __weak
arch_restore_msi_irqs(struct pci_dev
*dev
)
169 return default_restore_msi_irqs(dev
);
172 static inline __attribute_const__ u32
msi_mask(unsigned x
)
174 /* Don't shift by >= width of type */
177 return (1 << (1 << x
)) - 1;
181 * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to
182 * mask all MSI interrupts by clearing the MSI enable bit does not work
183 * reliably as devices without an INTx disable bit will then generate a
184 * level IRQ which will never be cleared.
186 u32
__pci_msi_desc_mask_irq(struct msi_desc
*desc
, u32 mask
, u32 flag
)
188 u32 mask_bits
= desc
->masked
;
190 if (pci_msi_ignore_mask
|| !desc
->msi_attrib
.maskbit
)
195 pci_write_config_dword(msi_desc_to_pci_dev(desc
), desc
->mask_pos
,
201 static void msi_mask_irq(struct msi_desc
*desc
, u32 mask
, u32 flag
)
203 desc
->masked
= __pci_msi_desc_mask_irq(desc
, mask
, flag
);
207 * This internal function does not flush PCI writes to the device.
208 * All users must ensure that they read from the device before either
209 * assuming that the device state is up to date, or returning out of this
210 * file. This saves a few milliseconds when initialising devices with lots
211 * of MSI-X interrupts.
213 u32
__pci_msix_desc_mask_irq(struct msi_desc
*desc
, u32 flag
)
215 u32 mask_bits
= desc
->masked
;
216 unsigned offset
= desc
->msi_attrib
.entry_nr
* PCI_MSIX_ENTRY_SIZE
+
217 PCI_MSIX_ENTRY_VECTOR_CTRL
;
219 if (pci_msi_ignore_mask
)
222 mask_bits
&= ~PCI_MSIX_ENTRY_CTRL_MASKBIT
;
224 mask_bits
|= PCI_MSIX_ENTRY_CTRL_MASKBIT
;
225 writel(mask_bits
, desc
->mask_base
+ offset
);
230 static void msix_mask_irq(struct msi_desc
*desc
, u32 flag
)
232 desc
->masked
= __pci_msix_desc_mask_irq(desc
, flag
);
235 static void msi_set_mask_bit(struct irq_data
*data
, u32 flag
)
237 struct msi_desc
*desc
= irq_data_get_msi_desc(data
);
239 if (desc
->msi_attrib
.is_msix
) {
240 msix_mask_irq(desc
, flag
);
241 readl(desc
->mask_base
); /* Flush write to device */
243 unsigned offset
= data
->irq
- desc
->irq
;
244 msi_mask_irq(desc
, 1 << offset
, flag
<< offset
);
249 * pci_msi_mask_irq - Generic irq chip callback to mask PCI/MSI interrupts
250 * @data: pointer to irqdata associated to that interrupt
252 void pci_msi_mask_irq(struct irq_data
*data
)
254 msi_set_mask_bit(data
, 1);
258 * pci_msi_unmask_irq - Generic irq chip callback to unmask PCI/MSI interrupts
259 * @data: pointer to irqdata associated to that interrupt
261 void pci_msi_unmask_irq(struct irq_data
*data
)
263 msi_set_mask_bit(data
, 0);
266 void default_restore_msi_irqs(struct pci_dev
*dev
)
268 struct msi_desc
*entry
;
270 for_each_pci_msi_entry(entry
, dev
)
271 default_restore_msi_irq(dev
, entry
->irq
);
274 void __pci_read_msi_msg(struct msi_desc
*entry
, struct msi_msg
*msg
)
276 struct pci_dev
*dev
= msi_desc_to_pci_dev(entry
);
278 BUG_ON(dev
->current_state
!= PCI_D0
);
280 if (entry
->msi_attrib
.is_msix
) {
281 void __iomem
*base
= entry
->mask_base
+
282 entry
->msi_attrib
.entry_nr
* PCI_MSIX_ENTRY_SIZE
;
284 msg
->address_lo
= readl(base
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
285 msg
->address_hi
= readl(base
+ PCI_MSIX_ENTRY_UPPER_ADDR
);
286 msg
->data
= readl(base
+ PCI_MSIX_ENTRY_DATA
);
288 int pos
= dev
->msi_cap
;
291 pci_read_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_LO
,
293 if (entry
->msi_attrib
.is_64
) {
294 pci_read_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_HI
,
296 pci_read_config_word(dev
, pos
+ PCI_MSI_DATA_64
, &data
);
299 pci_read_config_word(dev
, pos
+ PCI_MSI_DATA_32
, &data
);
305 void __pci_write_msi_msg(struct msi_desc
*entry
, struct msi_msg
*msg
)
307 struct pci_dev
*dev
= msi_desc_to_pci_dev(entry
);
309 if (dev
->current_state
!= PCI_D0
) {
310 /* Don't touch the hardware now */
311 } else if (entry
->msi_attrib
.is_msix
) {
313 base
= entry
->mask_base
+
314 entry
->msi_attrib
.entry_nr
* PCI_MSIX_ENTRY_SIZE
;
316 writel(msg
->address_lo
, base
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
317 writel(msg
->address_hi
, base
+ PCI_MSIX_ENTRY_UPPER_ADDR
);
318 writel(msg
->data
, base
+ PCI_MSIX_ENTRY_DATA
);
320 int pos
= dev
->msi_cap
;
323 pci_read_config_word(dev
, pos
+ PCI_MSI_FLAGS
, &msgctl
);
324 msgctl
&= ~PCI_MSI_FLAGS_QSIZE
;
325 msgctl
|= entry
->msi_attrib
.multiple
<< 4;
326 pci_write_config_word(dev
, pos
+ PCI_MSI_FLAGS
, msgctl
);
328 pci_write_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_LO
,
330 if (entry
->msi_attrib
.is_64
) {
331 pci_write_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_HI
,
333 pci_write_config_word(dev
, pos
+ PCI_MSI_DATA_64
,
336 pci_write_config_word(dev
, pos
+ PCI_MSI_DATA_32
,
343 void pci_write_msi_msg(unsigned int irq
, struct msi_msg
*msg
)
345 struct msi_desc
*entry
= irq_get_msi_desc(irq
);
347 __pci_write_msi_msg(entry
, msg
);
349 EXPORT_SYMBOL_GPL(pci_write_msi_msg
);
351 static void free_msi_irqs(struct pci_dev
*dev
)
353 struct list_head
*msi_list
= dev_to_msi_list(&dev
->dev
);
354 struct msi_desc
*entry
, *tmp
;
355 struct attribute
**msi_attrs
;
356 struct device_attribute
*dev_attr
;
359 for_each_pci_msi_entry(entry
, dev
)
361 for (i
= 0; i
< entry
->nvec_used
; i
++)
362 BUG_ON(irq_has_action(entry
->irq
+ i
));
364 pci_msi_teardown_msi_irqs(dev
);
366 list_for_each_entry_safe(entry
, tmp
, msi_list
, list
) {
367 if (entry
->msi_attrib
.is_msix
) {
368 if (list_is_last(&entry
->list
, msi_list
))
369 iounmap(entry
->mask_base
);
372 list_del(&entry
->list
);
376 if (dev
->msi_irq_groups
) {
377 sysfs_remove_groups(&dev
->dev
.kobj
, dev
->msi_irq_groups
);
378 msi_attrs
= dev
->msi_irq_groups
[0]->attrs
;
379 while (msi_attrs
[count
]) {
380 dev_attr
= container_of(msi_attrs
[count
],
381 struct device_attribute
, attr
);
382 kfree(dev_attr
->attr
.name
);
387 kfree(dev
->msi_irq_groups
[0]);
388 kfree(dev
->msi_irq_groups
);
389 dev
->msi_irq_groups
= NULL
;
393 static void pci_intx_for_msi(struct pci_dev
*dev
, int enable
)
395 if (!(dev
->dev_flags
& PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG
))
396 pci_intx(dev
, enable
);
399 static void __pci_restore_msi_state(struct pci_dev
*dev
)
402 struct msi_desc
*entry
;
404 if (!dev
->msi_enabled
)
407 entry
= irq_get_msi_desc(dev
->irq
);
409 pci_intx_for_msi(dev
, 0);
410 pci_msi_set_enable(dev
, 0);
411 arch_restore_msi_irqs(dev
);
413 pci_read_config_word(dev
, dev
->msi_cap
+ PCI_MSI_FLAGS
, &control
);
414 msi_mask_irq(entry
, msi_mask(entry
->msi_attrib
.multi_cap
),
416 control
&= ~PCI_MSI_FLAGS_QSIZE
;
417 control
|= (entry
->msi_attrib
.multiple
<< 4) | PCI_MSI_FLAGS_ENABLE
;
418 pci_write_config_word(dev
, dev
->msi_cap
+ PCI_MSI_FLAGS
, control
);
421 static void __pci_restore_msix_state(struct pci_dev
*dev
)
423 struct msi_desc
*entry
;
425 if (!dev
->msix_enabled
)
427 BUG_ON(list_empty(dev_to_msi_list(&dev
->dev
)));
429 /* route the table */
430 pci_intx_for_msi(dev
, 0);
431 pci_msix_clear_and_set_ctrl(dev
, 0,
432 PCI_MSIX_FLAGS_ENABLE
| PCI_MSIX_FLAGS_MASKALL
);
434 arch_restore_msi_irqs(dev
);
435 for_each_pci_msi_entry(entry
, dev
)
436 msix_mask_irq(entry
, entry
->masked
);
438 pci_msix_clear_and_set_ctrl(dev
, PCI_MSIX_FLAGS_MASKALL
, 0);
441 void pci_restore_msi_state(struct pci_dev
*dev
)
443 __pci_restore_msi_state(dev
);
444 __pci_restore_msix_state(dev
);
446 EXPORT_SYMBOL_GPL(pci_restore_msi_state
);
448 static ssize_t
msi_mode_show(struct device
*dev
, struct device_attribute
*attr
,
451 struct msi_desc
*entry
;
455 retval
= kstrtoul(attr
->attr
.name
, 10, &irq
);
459 entry
= irq_get_msi_desc(irq
);
461 return sprintf(buf
, "%s\n",
462 entry
->msi_attrib
.is_msix
? "msix" : "msi");
467 static int populate_msi_sysfs(struct pci_dev
*pdev
)
469 struct attribute
**msi_attrs
;
470 struct attribute
*msi_attr
;
471 struct device_attribute
*msi_dev_attr
;
472 struct attribute_group
*msi_irq_group
;
473 const struct attribute_group
**msi_irq_groups
;
474 struct msi_desc
*entry
;
479 /* Determine how many msi entries we have */
480 for_each_pci_msi_entry(entry
, pdev
)
485 /* Dynamically create the MSI attributes for the PCI device */
486 msi_attrs
= kzalloc(sizeof(void *) * (num_msi
+ 1), GFP_KERNEL
);
489 for_each_pci_msi_entry(entry
, pdev
) {
490 msi_dev_attr
= kzalloc(sizeof(*msi_dev_attr
), GFP_KERNEL
);
493 msi_attrs
[count
] = &msi_dev_attr
->attr
;
495 sysfs_attr_init(&msi_dev_attr
->attr
);
496 msi_dev_attr
->attr
.name
= kasprintf(GFP_KERNEL
, "%d",
498 if (!msi_dev_attr
->attr
.name
)
500 msi_dev_attr
->attr
.mode
= S_IRUGO
;
501 msi_dev_attr
->show
= msi_mode_show
;
505 msi_irq_group
= kzalloc(sizeof(*msi_irq_group
), GFP_KERNEL
);
508 msi_irq_group
->name
= "msi_irqs";
509 msi_irq_group
->attrs
= msi_attrs
;
511 msi_irq_groups
= kzalloc(sizeof(void *) * 2, GFP_KERNEL
);
513 goto error_irq_group
;
514 msi_irq_groups
[0] = msi_irq_group
;
516 ret
= sysfs_create_groups(&pdev
->dev
.kobj
, msi_irq_groups
);
518 goto error_irq_groups
;
519 pdev
->msi_irq_groups
= msi_irq_groups
;
524 kfree(msi_irq_groups
);
526 kfree(msi_irq_group
);
529 msi_attr
= msi_attrs
[count
];
531 msi_dev_attr
= container_of(msi_attr
, struct device_attribute
, attr
);
532 kfree(msi_attr
->name
);
535 msi_attr
= msi_attrs
[count
];
541 static struct msi_desc
*msi_setup_entry(struct pci_dev
*dev
, int nvec
)
544 struct msi_desc
*entry
;
546 /* MSI Entry Initialization */
547 entry
= alloc_msi_entry(&dev
->dev
);
551 pci_read_config_word(dev
, dev
->msi_cap
+ PCI_MSI_FLAGS
, &control
);
553 entry
->msi_attrib
.is_msix
= 0;
554 entry
->msi_attrib
.is_64
= !!(control
& PCI_MSI_FLAGS_64BIT
);
555 entry
->msi_attrib
.entry_nr
= 0;
556 entry
->msi_attrib
.maskbit
= !!(control
& PCI_MSI_FLAGS_MASKBIT
);
557 entry
->msi_attrib
.default_irq
= dev
->irq
; /* Save IOAPIC IRQ */
558 entry
->msi_attrib
.multi_cap
= (control
& PCI_MSI_FLAGS_QMASK
) >> 1;
559 entry
->msi_attrib
.multiple
= ilog2(__roundup_pow_of_two(nvec
));
560 entry
->nvec_used
= nvec
;
562 if (control
& PCI_MSI_FLAGS_64BIT
)
563 entry
->mask_pos
= dev
->msi_cap
+ PCI_MSI_MASK_64
;
565 entry
->mask_pos
= dev
->msi_cap
+ PCI_MSI_MASK_32
;
567 /* Save the initial mask status */
568 if (entry
->msi_attrib
.maskbit
)
569 pci_read_config_dword(dev
, entry
->mask_pos
, &entry
->masked
);
574 static int msi_verify_entries(struct pci_dev
*dev
)
576 struct msi_desc
*entry
;
578 for_each_pci_msi_entry(entry
, dev
) {
579 if (!dev
->no_64bit_msi
|| !entry
->msg
.address_hi
)
581 dev_err(&dev
->dev
, "Device has broken 64-bit MSI but arch"
582 " tried to assign one above 4G\n");
589 * msi_capability_init - configure device's MSI capability structure
590 * @dev: pointer to the pci_dev data structure of MSI device function
591 * @nvec: number of interrupts to allocate
593 * Setup the MSI capability structure of the device with the requested
594 * number of interrupts. A return value of zero indicates the successful
595 * setup of an entry with the new MSI irq. A negative return value indicates
596 * an error, and a positive return value indicates the number of interrupts
597 * which could have been allocated.
599 static int msi_capability_init(struct pci_dev
*dev
, int nvec
)
601 struct msi_desc
*entry
;
605 pci_msi_set_enable(dev
, 0); /* Disable MSI during set up */
607 entry
= msi_setup_entry(dev
, nvec
);
611 /* All MSIs are unmasked by default, Mask them all */
612 mask
= msi_mask(entry
->msi_attrib
.multi_cap
);
613 msi_mask_irq(entry
, mask
, mask
);
615 list_add_tail(&entry
->list
, dev_to_msi_list(&dev
->dev
));
617 /* Configure MSI capability structure */
618 ret
= pci_msi_setup_msi_irqs(dev
, nvec
, PCI_CAP_ID_MSI
);
620 msi_mask_irq(entry
, mask
, ~mask
);
625 ret
= msi_verify_entries(dev
);
627 msi_mask_irq(entry
, mask
, ~mask
);
632 ret
= populate_msi_sysfs(dev
);
634 msi_mask_irq(entry
, mask
, ~mask
);
639 /* Set MSI enabled bits */
640 pci_intx_for_msi(dev
, 0);
641 pci_msi_set_enable(dev
, 1);
642 dev
->msi_enabled
= 1;
644 pcibios_free_irq(dev
);
645 dev
->irq
= entry
->irq
;
649 static void __iomem
*msix_map_region(struct pci_dev
*dev
, unsigned nr_entries
)
651 resource_size_t phys_addr
;
656 pci_read_config_dword(dev
, dev
->msix_cap
+ PCI_MSIX_TABLE
,
658 bir
= (u8
)(table_offset
& PCI_MSIX_TABLE_BIR
);
659 flags
= pci_resource_flags(dev
, bir
);
660 if (!flags
|| (flags
& IORESOURCE_UNSET
))
663 table_offset
&= PCI_MSIX_TABLE_OFFSET
;
664 phys_addr
= pci_resource_start(dev
, bir
) + table_offset
;
666 return ioremap_nocache(phys_addr
, nr_entries
* PCI_MSIX_ENTRY_SIZE
);
669 static int msix_setup_entries(struct pci_dev
*dev
, void __iomem
*base
,
670 struct msix_entry
*entries
, int nvec
)
672 struct msi_desc
*entry
;
675 for (i
= 0; i
< nvec
; i
++) {
676 entry
= alloc_msi_entry(&dev
->dev
);
682 /* No enough memory. Don't try again */
686 entry
->msi_attrib
.is_msix
= 1;
687 entry
->msi_attrib
.is_64
= 1;
688 entry
->msi_attrib
.entry_nr
= entries
[i
].entry
;
689 entry
->msi_attrib
.default_irq
= dev
->irq
;
690 entry
->mask_base
= base
;
691 entry
->nvec_used
= 1;
693 list_add_tail(&entry
->list
, dev_to_msi_list(&dev
->dev
));
699 static void msix_program_entries(struct pci_dev
*dev
,
700 struct msix_entry
*entries
)
702 struct msi_desc
*entry
;
705 for_each_pci_msi_entry(entry
, dev
) {
706 int offset
= entries
[i
].entry
* PCI_MSIX_ENTRY_SIZE
+
707 PCI_MSIX_ENTRY_VECTOR_CTRL
;
709 entries
[i
].vector
= entry
->irq
;
710 entry
->masked
= readl(entry
->mask_base
+ offset
);
711 msix_mask_irq(entry
, 1);
717 * msix_capability_init - configure device's MSI-X capability
718 * @dev: pointer to the pci_dev data structure of MSI-X device function
719 * @entries: pointer to an array of struct msix_entry entries
720 * @nvec: number of @entries
722 * Setup the MSI-X capability structure of device function with a
723 * single MSI-X irq. A return of zero indicates the successful setup of
724 * requested MSI-X entries with allocated irqs or non-zero for otherwise.
726 static int msix_capability_init(struct pci_dev
*dev
,
727 struct msix_entry
*entries
, int nvec
)
733 /* Ensure MSI-X is disabled while it is set up */
734 pci_msix_clear_and_set_ctrl(dev
, PCI_MSIX_FLAGS_ENABLE
, 0);
736 pci_read_config_word(dev
, dev
->msix_cap
+ PCI_MSIX_FLAGS
, &control
);
737 /* Request & Map MSI-X table region */
738 base
= msix_map_region(dev
, msix_table_size(control
));
742 ret
= msix_setup_entries(dev
, base
, entries
, nvec
);
746 ret
= pci_msi_setup_msi_irqs(dev
, nvec
, PCI_CAP_ID_MSIX
);
750 /* Check if all MSI entries honor device restrictions */
751 ret
= msi_verify_entries(dev
);
756 * Some devices require MSI-X to be enabled before we can touch the
757 * MSI-X registers. We need to mask all the vectors to prevent
758 * interrupts coming in before they're fully set up.
760 pci_msix_clear_and_set_ctrl(dev
, 0,
761 PCI_MSIX_FLAGS_MASKALL
| PCI_MSIX_FLAGS_ENABLE
);
763 msix_program_entries(dev
, entries
);
765 ret
= populate_msi_sysfs(dev
);
769 /* Set MSI-X enabled bits and unmask the function */
770 pci_intx_for_msi(dev
, 0);
771 dev
->msix_enabled
= 1;
772 pci_msix_clear_and_set_ctrl(dev
, PCI_MSIX_FLAGS_MASKALL
, 0);
774 pcibios_free_irq(dev
);
780 * If we had some success, report the number of irqs
781 * we succeeded in setting up.
783 struct msi_desc
*entry
;
786 for_each_pci_msi_entry(entry
, dev
) {
801 * pci_msi_supported - check whether MSI may be enabled on a device
802 * @dev: pointer to the pci_dev data structure of MSI device function
803 * @nvec: how many MSIs have been requested ?
805 * Look at global flags, the device itself, and its parent buses
806 * to determine if MSI/-X are supported for the device. If MSI/-X is
807 * supported return 1, else return 0.
809 static int pci_msi_supported(struct pci_dev
*dev
, int nvec
)
813 /* MSI must be globally enabled and supported by the device */
817 if (!dev
|| dev
->no_msi
|| dev
->current_state
!= PCI_D0
)
821 * You can't ask to have 0 or less MSIs configured.
823 * b) the list manipulation code assumes nvec >= 1.
829 * Any bridge which does NOT route MSI transactions from its
830 * secondary bus to its primary bus must set NO_MSI flag on
831 * the secondary pci_bus.
832 * We expect only arch-specific PCI host bus controller driver
833 * or quirks for specific PCI bridges to be setting NO_MSI.
835 for (bus
= dev
->bus
; bus
; bus
= bus
->parent
)
836 if (bus
->bus_flags
& PCI_BUS_FLAGS_NO_MSI
)
843 * pci_msi_vec_count - Return the number of MSI vectors a device can send
844 * @dev: device to report about
846 * This function returns the number of MSI vectors a device requested via
847 * Multiple Message Capable register. It returns a negative errno if the
848 * device is not capable sending MSI interrupts. Otherwise, the call succeeds
849 * and returns a power of two, up to a maximum of 2^5 (32), according to the
852 int pci_msi_vec_count(struct pci_dev
*dev
)
860 pci_read_config_word(dev
, dev
->msi_cap
+ PCI_MSI_FLAGS
, &msgctl
);
861 ret
= 1 << ((msgctl
& PCI_MSI_FLAGS_QMASK
) >> 1);
865 EXPORT_SYMBOL(pci_msi_vec_count
);
867 void pci_msi_shutdown(struct pci_dev
*dev
)
869 struct msi_desc
*desc
;
872 if (!pci_msi_enable
|| !dev
|| !dev
->msi_enabled
)
875 BUG_ON(list_empty(dev_to_msi_list(&dev
->dev
)));
876 desc
= first_pci_msi_entry(dev
);
878 pci_msi_set_enable(dev
, 0);
879 pci_intx_for_msi(dev
, 1);
880 dev
->msi_enabled
= 0;
882 /* Return the device with MSI unmasked as initial states */
883 mask
= msi_mask(desc
->msi_attrib
.multi_cap
);
884 /* Keep cached state to be restored */
885 __pci_msi_desc_mask_irq(desc
, mask
, ~mask
);
887 /* Restore dev->irq to its default pin-assertion irq */
888 dev
->irq
= desc
->msi_attrib
.default_irq
;
889 pcibios_alloc_irq(dev
);
892 void pci_disable_msi(struct pci_dev
*dev
)
894 if (!pci_msi_enable
|| !dev
|| !dev
->msi_enabled
)
897 pci_msi_shutdown(dev
);
900 EXPORT_SYMBOL(pci_disable_msi
);
903 * pci_msix_vec_count - return the number of device's MSI-X table entries
904 * @dev: pointer to the pci_dev data structure of MSI-X device function
905 * This function returns the number of device's MSI-X table entries and
906 * therefore the number of MSI-X vectors device is capable of sending.
907 * It returns a negative errno if the device is not capable of sending MSI-X
910 int pci_msix_vec_count(struct pci_dev
*dev
)
917 pci_read_config_word(dev
, dev
->msix_cap
+ PCI_MSIX_FLAGS
, &control
);
918 return msix_table_size(control
);
920 EXPORT_SYMBOL(pci_msix_vec_count
);
923 * pci_enable_msix - configure device's MSI-X capability structure
924 * @dev: pointer to the pci_dev data structure of MSI-X device function
925 * @entries: pointer to an array of MSI-X entries
926 * @nvec: number of MSI-X irqs requested for allocation by device driver
928 * Setup the MSI-X capability structure of device function with the number
929 * of requested irqs upon its software driver call to request for
930 * MSI-X mode enabled on its hardware device function. A return of zero
931 * indicates the successful configuration of MSI-X capability structure
932 * with new allocated MSI-X irqs. A return of < 0 indicates a failure.
933 * Or a return of > 0 indicates that driver request is exceeding the number
934 * of irqs or MSI-X vectors available. Driver should use the returned value to
935 * re-send its request.
937 int pci_enable_msix(struct pci_dev
*dev
, struct msix_entry
*entries
, int nvec
)
942 if (!pci_msi_supported(dev
, nvec
))
948 nr_entries
= pci_msix_vec_count(dev
);
951 if (nvec
> nr_entries
)
954 /* Check for any invalid entries */
955 for (i
= 0; i
< nvec
; i
++) {
956 if (entries
[i
].entry
>= nr_entries
)
957 return -EINVAL
; /* invalid entry */
958 for (j
= i
+ 1; j
< nvec
; j
++) {
959 if (entries
[i
].entry
== entries
[j
].entry
)
960 return -EINVAL
; /* duplicate entry */
963 WARN_ON(!!dev
->msix_enabled
);
965 /* Check whether driver already requested for MSI irq */
966 if (dev
->msi_enabled
) {
967 dev_info(&dev
->dev
, "can't enable MSI-X (MSI IRQ already assigned)\n");
970 return msix_capability_init(dev
, entries
, nvec
);
972 EXPORT_SYMBOL(pci_enable_msix
);
974 void pci_msix_shutdown(struct pci_dev
*dev
)
976 struct msi_desc
*entry
;
978 if (!pci_msi_enable
|| !dev
|| !dev
->msix_enabled
)
981 /* Return the device with MSI-X masked as initial states */
982 for_each_pci_msi_entry(entry
, dev
) {
983 /* Keep cached states to be restored */
984 __pci_msix_desc_mask_irq(entry
, 1);
987 pci_msix_clear_and_set_ctrl(dev
, PCI_MSIX_FLAGS_ENABLE
, 0);
988 pci_intx_for_msi(dev
, 1);
989 dev
->msix_enabled
= 0;
990 pcibios_alloc_irq(dev
);
993 void pci_disable_msix(struct pci_dev
*dev
)
995 if (!pci_msi_enable
|| !dev
|| !dev
->msix_enabled
)
998 pci_msix_shutdown(dev
);
1001 EXPORT_SYMBOL(pci_disable_msix
);
1003 void pci_no_msi(void)
1009 * pci_msi_enabled - is MSI enabled?
1011 * Returns true if MSI has not been disabled by the command-line option
1014 int pci_msi_enabled(void)
1016 return pci_msi_enable
;
1018 EXPORT_SYMBOL(pci_msi_enabled
);
1020 void pci_msi_init_pci_dev(struct pci_dev
*dev
)
1025 * pci_enable_msi_range - configure device's MSI capability structure
1026 * @dev: device to configure
1027 * @minvec: minimal number of interrupts to configure
1028 * @maxvec: maximum number of interrupts to configure
1030 * This function tries to allocate a maximum possible number of interrupts in a
1031 * range between @minvec and @maxvec. It returns a negative errno if an error
1032 * occurs. If it succeeds, it returns the actual number of interrupts allocated
1033 * and updates the @dev's irq member to the lowest new interrupt number;
1034 * the other interrupt numbers allocated to this device are consecutive.
1036 int pci_enable_msi_range(struct pci_dev
*dev
, int minvec
, int maxvec
)
1041 if (!pci_msi_supported(dev
, minvec
))
1044 WARN_ON(!!dev
->msi_enabled
);
1046 /* Check whether driver already requested MSI-X irqs */
1047 if (dev
->msix_enabled
) {
1049 "can't enable MSI (MSI-X already enabled)\n");
1053 if (maxvec
< minvec
)
1056 nvec
= pci_msi_vec_count(dev
);
1059 else if (nvec
< minvec
)
1061 else if (nvec
> maxvec
)
1065 rc
= msi_capability_init(dev
, nvec
);
1068 } else if (rc
> 0) {
1077 EXPORT_SYMBOL(pci_enable_msi_range
);
1080 * pci_enable_msix_range - configure device's MSI-X capability structure
1081 * @dev: pointer to the pci_dev data structure of MSI-X device function
1082 * @entries: pointer to an array of MSI-X entries
1083 * @minvec: minimum number of MSI-X irqs requested
1084 * @maxvec: maximum number of MSI-X irqs requested
1086 * Setup the MSI-X capability structure of device function with a maximum
1087 * possible number of interrupts in the range between @minvec and @maxvec
1088 * upon its software driver call to request for MSI-X mode enabled on its
1089 * hardware device function. It returns a negative errno if an error occurs.
1090 * If it succeeds, it returns the actual number of interrupts allocated and
1091 * indicates the successful configuration of MSI-X capability structure
1092 * with new allocated MSI-X interrupts.
1094 int pci_enable_msix_range(struct pci_dev
*dev
, struct msix_entry
*entries
,
1095 int minvec
, int maxvec
)
1100 if (maxvec
< minvec
)
1104 rc
= pci_enable_msix(dev
, entries
, nvec
);
1107 } else if (rc
> 0) {
1116 EXPORT_SYMBOL(pci_enable_msix_range
);
1118 struct pci_dev
*msi_desc_to_pci_dev(struct msi_desc
*desc
)
1120 return to_pci_dev(desc
->dev
);
1123 void *msi_desc_to_pci_sysdata(struct msi_desc
*desc
)
1125 struct pci_dev
*dev
= msi_desc_to_pci_dev(desc
);
1127 return dev
->bus
->sysdata
;
1129 EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata
);
1131 #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
1133 * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space
1134 * @irq_data: Pointer to interrupt data of the MSI interrupt
1135 * @msg: Pointer to the message
1137 void pci_msi_domain_write_msg(struct irq_data
*irq_data
, struct msi_msg
*msg
)
1139 struct msi_desc
*desc
= irq_data_get_msi_desc(irq_data
);
1142 * For MSI-X desc->irq is always equal to irq_data->irq. For
1143 * MSI only the first interrupt of MULTI MSI passes the test.
1145 if (desc
->irq
== irq_data
->irq
)
1146 __pci_write_msi_msg(desc
, msg
);
1150 * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
1151 * @dev: Pointer to the PCI device
1152 * @desc: Pointer to the msi descriptor
1154 * The ID number is only used within the irqdomain.
1156 irq_hw_number_t
pci_msi_domain_calc_hwirq(struct pci_dev
*dev
,
1157 struct msi_desc
*desc
)
1159 return (irq_hw_number_t
)desc
->msi_attrib
.entry_nr
|
1160 PCI_DEVID(dev
->bus
->number
, dev
->devfn
) << 11 |
1161 (pci_domain_nr(dev
->bus
) & 0xFFFFFFFF) << 27;
1164 static inline bool pci_msi_desc_is_multi_msi(struct msi_desc
*desc
)
1166 return !desc
->msi_attrib
.is_msix
&& desc
->nvec_used
> 1;
1170 * pci_msi_domain_check_cap - Verify that @domain supports the capabilities for @dev
1171 * @domain: The interrupt domain to check
1172 * @info: The domain info for verification
1173 * @dev: The device to check
1176 * 0 if the functionality is supported
1177 * 1 if Multi MSI is requested, but the domain does not support it
1178 * -ENOTSUPP otherwise
1180 int pci_msi_domain_check_cap(struct irq_domain
*domain
,
1181 struct msi_domain_info
*info
, struct device
*dev
)
1183 struct msi_desc
*desc
= first_pci_msi_entry(to_pci_dev(dev
));
1185 /* Special handling to support pci_enable_msi_range() */
1186 if (pci_msi_desc_is_multi_msi(desc
) &&
1187 !(info
->flags
& MSI_FLAG_MULTI_PCI_MSI
))
1189 else if (desc
->msi_attrib
.is_msix
&& !(info
->flags
& MSI_FLAG_PCI_MSIX
))
1195 static int pci_msi_domain_handle_error(struct irq_domain
*domain
,
1196 struct msi_desc
*desc
, int error
)
1198 /* Special handling to support pci_enable_msi_range() */
1199 if (pci_msi_desc_is_multi_msi(desc
) && error
== -ENOSPC
)
1205 #ifdef GENERIC_MSI_DOMAIN_OPS
1206 static void pci_msi_domain_set_desc(msi_alloc_info_t
*arg
,
1207 struct msi_desc
*desc
)
1210 arg
->hwirq
= pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc
),
1214 #define pci_msi_domain_set_desc NULL
1217 static struct msi_domain_ops pci_msi_domain_ops_default
= {
1218 .set_desc
= pci_msi_domain_set_desc
,
1219 .msi_check
= pci_msi_domain_check_cap
,
1220 .handle_error
= pci_msi_domain_handle_error
,
1223 static void pci_msi_domain_update_dom_ops(struct msi_domain_info
*info
)
1225 struct msi_domain_ops
*ops
= info
->ops
;
1228 info
->ops
= &pci_msi_domain_ops_default
;
1230 if (ops
->set_desc
== NULL
)
1231 ops
->set_desc
= pci_msi_domain_set_desc
;
1232 if (ops
->msi_check
== NULL
)
1233 ops
->msi_check
= pci_msi_domain_check_cap
;
1234 if (ops
->handle_error
== NULL
)
1235 ops
->handle_error
= pci_msi_domain_handle_error
;
1239 static void pci_msi_domain_update_chip_ops(struct msi_domain_info
*info
)
1241 struct irq_chip
*chip
= info
->chip
;
1244 if (!chip
->irq_write_msi_msg
)
1245 chip
->irq_write_msi_msg
= pci_msi_domain_write_msg
;
1249 * pci_msi_create_irq_domain - Creat a MSI interrupt domain
1250 * @node: Optional device-tree node of the interrupt controller
1251 * @info: MSI domain info
1252 * @parent: Parent irq domain
1254 * Updates the domain and chip ops and creates a MSI interrupt domain.
1257 * A domain pointer or NULL in case of failure.
1259 struct irq_domain
*pci_msi_create_irq_domain(struct device_node
*node
,
1260 struct msi_domain_info
*info
,
1261 struct irq_domain
*parent
)
1263 struct irq_domain
*domain
;
1265 if (info
->flags
& MSI_FLAG_USE_DEF_DOM_OPS
)
1266 pci_msi_domain_update_dom_ops(info
);
1267 if (info
->flags
& MSI_FLAG_USE_DEF_CHIP_OPS
)
1268 pci_msi_domain_update_chip_ops(info
);
1270 domain
= msi_create_irq_domain(node
, info
, parent
);
1274 domain
->bus_token
= DOMAIN_BUS_PCI_MSI
;
1279 * pci_msi_domain_alloc_irqs - Allocate interrupts for @dev in @domain
1280 * @domain: The interrupt domain to allocate from
1281 * @dev: The device for which to allocate
1282 * @nvec: The number of interrupts to allocate
1283 * @type: Unused to allow simpler migration from the arch_XXX interfaces
1286 * A virtual interrupt number or an error code in case of failure
1288 int pci_msi_domain_alloc_irqs(struct irq_domain
*domain
, struct pci_dev
*dev
,
1291 return msi_domain_alloc_irqs(domain
, &dev
->dev
, nvec
);
1295 * pci_msi_domain_free_irqs - Free interrupts for @dev in @domain
1296 * @domain: The interrupt domain
1297 * @dev: The device for which to free interrupts
1299 void pci_msi_domain_free_irqs(struct irq_domain
*domain
, struct pci_dev
*dev
)
1301 msi_domain_free_irqs(domain
, &dev
->dev
);
1305 * pci_msi_create_default_irq_domain - Create a default MSI interrupt domain
1306 * @node: Optional device-tree node of the interrupt controller
1307 * @info: MSI domain info
1308 * @parent: Parent irq domain
1310 * Returns: A domain pointer or NULL in case of failure. If successful
1311 * the default PCI/MSI irqdomain pointer is updated.
1313 struct irq_domain
*pci_msi_create_default_irq_domain(struct device_node
*node
,
1314 struct msi_domain_info
*info
, struct irq_domain
*parent
)
1316 struct irq_domain
*domain
;
1318 mutex_lock(&pci_msi_domain_lock
);
1319 if (pci_msi_default_domain
) {
1320 pr_err("PCI: default irq domain for PCI MSI has already been created.\n");
1323 domain
= pci_msi_create_irq_domain(node
, info
, parent
);
1324 pci_msi_default_domain
= domain
;
1326 mutex_unlock(&pci_msi_domain_lock
);
1330 #endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */