1 #include <linux/seq_file.h>
2 #include <linux/cpumask.h>
3 #include <linux/kernel.h>
4 #include <linux/string.h>
5 #include <linux/cpumask.h>
6 #include <linux/errno.h>
11 #include <asm/hw_irq.h>
12 #include <asm/irq_remapping.h>
13 #include <asm/processor.h>
14 #include <asm/x86_init.h>
17 #include "irq_remapping.h"
19 int irq_remapping_enabled
;
21 int disable_irq_remap
;
22 int disable_sourceid_checking
;
25 static struct irq_remap_ops
*remap_ops
;
27 static int msi_alloc_remapped_irq(struct pci_dev
*pdev
, int irq
, int nvec
);
28 static int msi_setup_remapped_irq(struct pci_dev
*pdev
, unsigned int irq
,
29 int index
, int sub_handle
);
30 static int set_remapped_irq_affinity(struct irq_data
*data
,
31 const struct cpumask
*mask
,
34 static bool irq_remapped(struct irq_cfg
*cfg
)
36 return (cfg
->remapped
== 1);
39 static void irq_remapping_disable_io_apic(void)
42 * With interrupt-remapping, for now we will use virtual wire A
43 * mode, as virtual wire B is little complex (need to configure
44 * both IOAPIC RTE as well as interrupt-remapping table entry).
45 * As this gets called during crash dump, keep this simple for
48 if (cpu_has_apic
|| apic_from_smp_config())
49 disconnect_bsp_APIC(0);
52 static int do_setup_msi_irqs(struct pci_dev
*dev
, int nvec
)
54 int node
, ret
, sub_handle
, index
= 0;
56 struct msi_desc
*msidesc
;
58 nvec
= __roundup_pow_of_two(nvec
);
60 WARN_ON(!list_is_singular(&dev
->msi_list
));
61 msidesc
= list_entry(dev
->msi_list
.next
, struct msi_desc
, list
);
62 WARN_ON(msidesc
->irq
);
63 WARN_ON(msidesc
->msi_attrib
.multiple
);
65 node
= dev_to_node(&dev
->dev
);
66 irq
= __create_irqs(get_nr_irqs_gsi(), nvec
, node
);
70 msidesc
->msi_attrib
.multiple
= ilog2(nvec
);
71 for (sub_handle
= 0; sub_handle
< nvec
; sub_handle
++) {
73 index
= msi_alloc_remapped_irq(dev
, irq
, nvec
);
79 ret
= msi_setup_remapped_irq(dev
, irq
+ sub_handle
,
84 ret
= setup_msi_irq(dev
, msidesc
, irq
, sub_handle
);
91 destroy_irqs(irq
, nvec
);
94 * Restore altered MSI descriptor fields and prevent just destroyed
95 * IRQs from tearing down again in default_teardown_msi_irqs()
98 msidesc
->msi_attrib
.multiple
= 0;
103 static int do_setup_msix_irqs(struct pci_dev
*dev
, int nvec
)
105 int node
, ret
, sub_handle
, index
= 0;
106 struct msi_desc
*msidesc
;
109 node
= dev_to_node(&dev
->dev
);
110 irq
= get_nr_irqs_gsi();
113 list_for_each_entry(msidesc
, &dev
->msi_list
, list
) {
115 irq
= create_irq_nr(irq
, node
);
120 ret
= index
= msi_alloc_remapped_irq(dev
, irq
, nvec
);
122 ret
= msi_setup_remapped_irq(dev
, irq
, index
, sub_handle
);
127 ret
= setup_msi_irq(dev
, msidesc
, irq
, 0);
142 static int irq_remapping_setup_msi_irqs(struct pci_dev
*dev
,
145 if (type
== PCI_CAP_ID_MSI
)
146 return do_setup_msi_irqs(dev
, nvec
);
148 return do_setup_msix_irqs(dev
, nvec
);
151 void eoi_ioapic_pin_remapped(int apic
, int pin
, int vector
)
154 * Intr-remapping uses pin number as the virtual vector
155 * in the RTE. Actual vector is programmed in
156 * intr-remapping table entry. Hence for the io-apic
157 * EOI we use the pin number.
159 io_apic_eoi(apic
, pin
);
162 static void __init
irq_remapping_modify_x86_ops(void)
164 x86_io_apic_ops
.disable
= irq_remapping_disable_io_apic
;
165 x86_io_apic_ops
.set_affinity
= set_remapped_irq_affinity
;
166 x86_io_apic_ops
.setup_entry
= setup_ioapic_remapped_entry
;
167 x86_io_apic_ops
.eoi_ioapic_pin
= eoi_ioapic_pin_remapped
;
168 x86_msi
.setup_msi_irqs
= irq_remapping_setup_msi_irqs
;
169 x86_msi
.setup_hpet_msi
= setup_hpet_msi_remapped
;
170 x86_msi
.compose_msi_msg
= compose_remapped_msi_msg
;
173 static __init
int setup_nointremap(char *str
)
175 disable_irq_remap
= 1;
178 early_param("nointremap", setup_nointremap
);
180 static __init
int setup_irqremap(char *str
)
186 if (!strncmp(str
, "on", 2))
187 disable_irq_remap
= 0;
188 else if (!strncmp(str
, "off", 3))
189 disable_irq_remap
= 1;
190 else if (!strncmp(str
, "nosid", 5))
191 disable_sourceid_checking
= 1;
192 else if (!strncmp(str
, "no_x2apic_optout", 16))
193 no_x2apic_optout
= 1;
195 str
+= strcspn(str
, ",");
202 early_param("intremap", setup_irqremap
);
204 void __init
setup_irq_remapping_ops(void)
206 remap_ops
= &intel_irq_remap_ops
;
208 #ifdef CONFIG_AMD_IOMMU
209 if (amd_iommu_irq_ops
.prepare() == 0)
210 remap_ops
= &amd_iommu_irq_ops
;
214 int irq_remapping_supported(void)
216 if (disable_irq_remap
)
219 if (!remap_ops
|| !remap_ops
->supported
)
222 return remap_ops
->supported();
225 int __init
irq_remapping_prepare(void)
227 if (!remap_ops
|| !remap_ops
->prepare
)
230 return remap_ops
->prepare();
233 int __init
irq_remapping_enable(void)
237 if (!remap_ops
|| !remap_ops
->enable
)
240 ret
= remap_ops
->enable();
242 if (irq_remapping_enabled
)
243 irq_remapping_modify_x86_ops();
248 void irq_remapping_disable(void)
250 if (!irq_remapping_enabled
||
255 remap_ops
->disable();
258 int irq_remapping_reenable(int mode
)
260 if (!irq_remapping_enabled
||
262 !remap_ops
->reenable
)
265 return remap_ops
->reenable(mode
);
268 int __init
irq_remap_enable_fault_handling(void)
270 if (!irq_remapping_enabled
)
273 if (!remap_ops
|| !remap_ops
->enable_faulting
)
276 return remap_ops
->enable_faulting();
279 int setup_ioapic_remapped_entry(int irq
,
280 struct IO_APIC_route_entry
*entry
,
281 unsigned int destination
, int vector
,
282 struct io_apic_irq_attr
*attr
)
284 if (!remap_ops
|| !remap_ops
->setup_ioapic_entry
)
287 return remap_ops
->setup_ioapic_entry(irq
, entry
, destination
,
291 int set_remapped_irq_affinity(struct irq_data
*data
, const struct cpumask
*mask
,
294 if (!config_enabled(CONFIG_SMP
) || !remap_ops
||
295 !remap_ops
->set_affinity
)
298 return remap_ops
->set_affinity(data
, mask
, force
);
301 void free_remapped_irq(int irq
)
303 struct irq_cfg
*cfg
= irq_get_chip_data(irq
);
305 if (!remap_ops
|| !remap_ops
->free_irq
)
308 if (irq_remapped(cfg
))
309 remap_ops
->free_irq(irq
);
312 void compose_remapped_msi_msg(struct pci_dev
*pdev
,
313 unsigned int irq
, unsigned int dest
,
314 struct msi_msg
*msg
, u8 hpet_id
)
316 struct irq_cfg
*cfg
= irq_get_chip_data(irq
);
318 if (!irq_remapped(cfg
))
319 native_compose_msi_msg(pdev
, irq
, dest
, msg
, hpet_id
);
320 else if (remap_ops
&& remap_ops
->compose_msi_msg
)
321 remap_ops
->compose_msi_msg(pdev
, irq
, dest
, msg
, hpet_id
);
324 static int msi_alloc_remapped_irq(struct pci_dev
*pdev
, int irq
, int nvec
)
326 if (!remap_ops
|| !remap_ops
->msi_alloc_irq
)
329 return remap_ops
->msi_alloc_irq(pdev
, irq
, nvec
);
332 static int msi_setup_remapped_irq(struct pci_dev
*pdev
, unsigned int irq
,
333 int index
, int sub_handle
)
335 if (!remap_ops
|| !remap_ops
->msi_setup_irq
)
338 return remap_ops
->msi_setup_irq(pdev
, irq
, index
, sub_handle
);
341 int setup_hpet_msi_remapped(unsigned int irq
, unsigned int id
)
343 if (!remap_ops
|| !remap_ops
->setup_hpet_msi
)
346 return remap_ops
->setup_hpet_msi(irq
, id
);
349 void panic_if_irq_remap(const char *msg
)
351 if (irq_remapping_enabled
)
355 static void ir_ack_apic_edge(struct irq_data
*data
)
360 static void ir_ack_apic_level(struct irq_data
*data
)
363 eoi_ioapic_irq(data
->irq
, data
->chip_data
);
366 static void ir_print_prefix(struct irq_data
*data
, struct seq_file
*p
)
368 seq_printf(p
, " IR-%s", data
->chip
->name
);
371 void irq_remap_modify_chip_defaults(struct irq_chip
*chip
)
373 chip
->irq_print_chip
= ir_print_prefix
;
374 chip
->irq_ack
= ir_ack_apic_edge
;
375 chip
->irq_eoi
= ir_ack_apic_level
;
376 chip
->irq_set_affinity
= x86_io_apic_ops
.set_affinity
;
379 bool setup_remapped_irq(int irq
, struct irq_cfg
*cfg
, struct irq_chip
*chip
)
381 if (!irq_remapped(cfg
))
383 irq_set_status_flags(irq
, IRQ_MOVE_PCNTXT
);
384 irq_remap_modify_chip_defaults(chip
);