2 * Local APIC related interfaces to support IOAPIC, MSI, HT_IRQ etc.
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 * Moved from arch/x86/kernel/apic/io_apic.c.
6 * Jiang Liu <jiang.liu@linux.intel.com>
7 * Enable support of hierarchical irqdomains
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
13 #include <linux/interrupt.h>
14 #include <linux/init.h>
15 #include <linux/compiler.h>
16 #include <linux/irqdomain.h>
17 #include <linux/slab.h>
18 #include <asm/hw_irq.h>
20 #include <asm/i8259.h>
22 #include <asm/irq_remapping.h>
24 struct irq_domain
*x86_vector_domain
;
25 static DEFINE_RAW_SPINLOCK(vector_lock
);
26 static struct irq_chip lapic_controller
;
28 void lock_vector_lock(void)
30 /* Used to the online set of cpus does not change
31 * during assign_irq_vector.
33 raw_spin_lock(&vector_lock
);
36 void unlock_vector_lock(void)
38 raw_spin_unlock(&vector_lock
);
41 struct irq_cfg
*irq_cfg(unsigned int irq
)
43 return irqd_cfg(irq_get_irq_data(irq
));
46 struct irq_cfg
*irqd_cfg(struct irq_data
*irq_data
)
51 while (irq_data
->parent_data
)
52 irq_data
= irq_data
->parent_data
;
54 return irq_data
->chip_data
;
57 static struct irq_cfg
*alloc_irq_cfg(int node
)
61 cfg
= kzalloc_node(sizeof(*cfg
), GFP_KERNEL
, node
);
64 if (!zalloc_cpumask_var_node(&cfg
->domain
, GFP_KERNEL
, node
))
66 if (!zalloc_cpumask_var_node(&cfg
->old_domain
, GFP_KERNEL
, node
))
68 #ifdef CONFIG_X86_IO_APIC
69 INIT_LIST_HEAD(&cfg
->irq_2_pin
);
73 free_cpumask_var(cfg
->domain
);
79 struct irq_cfg
*alloc_irq_and_cfg_at(unsigned int at
, int node
)
81 int res
= irq_alloc_desc_at(at
, node
);
92 cfg
= alloc_irq_cfg(node
);
94 irq_set_chip_data(at
, cfg
);
100 static void free_irq_cfg(struct irq_cfg
*cfg
)
103 free_cpumask_var(cfg
->domain
);
104 free_cpumask_var(cfg
->old_domain
);
110 __assign_irq_vector(int irq
, struct irq_cfg
*cfg
, const struct cpumask
*mask
)
113 * NOTE! The local APIC isn't very good at handling
114 * multiple interrupts at the same interrupt level.
115 * As the interrupt level is determined by taking the
116 * vector number and shifting that right by 4, we
117 * want to spread these out a bit so that they don't
118 * all fall in the same interrupt level.
120 * Also, we've got to be careful not to trash gate
121 * 0x80, because int 0x80 is hm, kind of importantish. ;)
123 static int current_vector
= FIRST_EXTERNAL_VECTOR
+ VECTOR_OFFSET_START
;
124 static int current_offset
= VECTOR_OFFSET_START
% 16;
126 cpumask_var_t tmp_mask
;
128 if (cfg
->move_in_progress
)
131 if (!alloc_cpumask_var(&tmp_mask
, GFP_ATOMIC
))
134 /* Only try and allocate irqs on cpus that are present */
136 cpumask_clear(cfg
->old_domain
);
137 cpu
= cpumask_first_and(mask
, cpu_online_mask
);
138 while (cpu
< nr_cpu_ids
) {
139 int new_cpu
, vector
, offset
;
141 apic
->vector_allocation_domain(cpu
, tmp_mask
, mask
);
143 if (cpumask_subset(tmp_mask
, cfg
->domain
)) {
145 if (cpumask_equal(tmp_mask
, cfg
->domain
))
148 * New cpumask using the vector is a proper subset of
149 * the current in use mask. So cleanup the vector
150 * allocation for the members that are not used anymore.
152 cpumask_andnot(cfg
->old_domain
, cfg
->domain
, tmp_mask
);
153 cfg
->move_in_progress
=
154 cpumask_intersects(cfg
->old_domain
, cpu_online_mask
);
155 cpumask_and(cfg
->domain
, cfg
->domain
, tmp_mask
);
159 vector
= current_vector
;
160 offset
= current_offset
;
163 if (vector
>= first_system_vector
) {
164 offset
= (offset
+ 1) % 16;
165 vector
= FIRST_EXTERNAL_VECTOR
+ offset
;
168 if (unlikely(current_vector
== vector
)) {
169 cpumask_or(cfg
->old_domain
, cfg
->old_domain
, tmp_mask
);
170 cpumask_andnot(tmp_mask
, mask
, cfg
->old_domain
);
171 cpu
= cpumask_first_and(tmp_mask
, cpu_online_mask
);
175 if (test_bit(vector
, used_vectors
))
178 for_each_cpu_and(new_cpu
, tmp_mask
, cpu_online_mask
) {
179 if (per_cpu(vector_irq
, new_cpu
)[vector
] >
184 current_vector
= vector
;
185 current_offset
= offset
;
187 cpumask_copy(cfg
->old_domain
, cfg
->domain
);
188 cfg
->move_in_progress
=
189 cpumask_intersects(cfg
->old_domain
, cpu_online_mask
);
191 for_each_cpu_and(new_cpu
, tmp_mask
, cpu_online_mask
)
192 per_cpu(vector_irq
, new_cpu
)[vector
] = irq
;
193 cfg
->vector
= vector
;
194 cpumask_copy(cfg
->domain
, tmp_mask
);
198 free_cpumask_var(tmp_mask
);
201 /* cache destination APIC IDs into cfg->dest_apicid */
202 err
= apic
->cpu_mask_to_apicid_and(mask
, cfg
->domain
,
209 int assign_irq_vector(int irq
, struct irq_cfg
*cfg
, const struct cpumask
*mask
)
214 raw_spin_lock_irqsave(&vector_lock
, flags
);
215 err
= __assign_irq_vector(irq
, cfg
, mask
);
216 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
220 void clear_irq_vector(int irq
, struct irq_cfg
*cfg
)
225 raw_spin_lock_irqsave(&vector_lock
, flags
);
226 BUG_ON(!cfg
->vector
);
228 vector
= cfg
->vector
;
229 for_each_cpu_and(cpu
, cfg
->domain
, cpu_online_mask
)
230 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
233 cpumask_clear(cfg
->domain
);
235 if (likely(!cfg
->move_in_progress
)) {
236 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
240 for_each_cpu_and(cpu
, cfg
->old_domain
, cpu_online_mask
) {
241 for (vector
= FIRST_EXTERNAL_VECTOR
; vector
< NR_VECTORS
;
243 if (per_cpu(vector_irq
, cpu
)[vector
] != irq
)
245 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
249 cfg
->move_in_progress
= 0;
250 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
253 void init_irq_alloc_info(struct irq_alloc_info
*info
,
254 const struct cpumask
*mask
)
256 memset(info
, 0, sizeof(*info
));
260 void copy_irq_alloc_info(struct irq_alloc_info
*dst
, struct irq_alloc_info
*src
)
265 memset(dst
, 0, sizeof(*dst
));
268 static inline const struct cpumask
*
269 irq_alloc_info_get_mask(struct irq_alloc_info
*info
)
271 return (!info
|| !info
->mask
) ? apic
->target_cpus() : info
->mask
;
274 static void x86_vector_free_irqs(struct irq_domain
*domain
,
275 unsigned int virq
, unsigned int nr_irqs
)
277 struct irq_data
*irq_data
;
280 for (i
= 0; i
< nr_irqs
; i
++) {
281 irq_data
= irq_domain_get_irq_data(x86_vector_domain
, virq
+ i
);
282 if (irq_data
&& irq_data
->chip_data
) {
283 free_remapped_irq(virq
);
284 clear_irq_vector(virq
+ i
, irq_data
->chip_data
);
285 free_irq_cfg(irq_data
->chip_data
);
286 irq_domain_reset_irq_data(irq_data
);
291 static int x86_vector_alloc_irqs(struct irq_domain
*domain
, unsigned int virq
,
292 unsigned int nr_irqs
, void *arg
)
294 struct irq_alloc_info
*info
= arg
;
295 const struct cpumask
*mask
;
296 struct irq_data
*irq_data
;
303 /* Currently vector allocator can't guarantee contiguous allocations */
304 if ((info
->flags
& X86_IRQ_ALLOC_CONTIGUOUS_VECTORS
) && nr_irqs
> 1)
307 mask
= irq_alloc_info_get_mask(info
);
308 for (i
= 0; i
< nr_irqs
; i
++) {
309 irq_data
= irq_domain_get_irq_data(domain
, virq
+ i
);
311 cfg
= alloc_irq_cfg(irq_data
->node
);
317 irq_data
->chip
= &lapic_controller
;
318 irq_data
->chip_data
= cfg
;
319 irq_data
->hwirq
= virq
+ i
;
320 err
= assign_irq_vector(virq
, cfg
, mask
);
328 x86_vector_free_irqs(domain
, virq
, i
+ 1);
332 static struct irq_domain_ops x86_vector_domain_ops
= {
333 .alloc
= x86_vector_alloc_irqs
,
334 .free
= x86_vector_free_irqs
,
337 int __init
arch_probe_nr_irqs(void)
341 if (nr_irqs
> (NR_VECTORS
* nr_cpu_ids
))
342 nr_irqs
= NR_VECTORS
* nr_cpu_ids
;
344 nr
= (gsi_top
+ nr_legacy_irqs()) + 8 * nr_cpu_ids
;
345 #if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
347 * for MSI and HT dyn irq
349 if (gsi_top
<= NR_IRQS_LEGACY
)
350 nr
+= 8 * nr_cpu_ids
;
357 return nr_legacy_irqs();
360 int __init
arch_early_irq_init(void)
362 x86_vector_domain
= irq_domain_add_tree(NULL
, &x86_vector_domain_ops
,
364 BUG_ON(x86_vector_domain
== NULL
);
365 irq_set_default_host(x86_vector_domain
);
367 arch_init_msi_domain(x86_vector_domain
);
368 arch_init_htirq_domain(x86_vector_domain
);
370 return arch_early_ioapic_init();
373 static void __setup_vector_irq(int cpu
)
375 /* Initialize vector_irq on a new cpu */
380 * vector_lock will make sure that we don't run into irq vector
381 * assignments that might be happening on another cpu in parallel,
382 * while we setup our initial vector to irq mappings.
384 raw_spin_lock(&vector_lock
);
385 /* Mark the inuse vectors */
386 for_each_active_irq(irq
) {
391 if (!cpumask_test_cpu(cpu
, cfg
->domain
))
393 vector
= cfg
->vector
;
394 per_cpu(vector_irq
, cpu
)[vector
] = irq
;
396 /* Mark the free vectors */
397 for (vector
= 0; vector
< NR_VECTORS
; ++vector
) {
398 irq
= per_cpu(vector_irq
, cpu
)[vector
];
399 if (irq
<= VECTOR_UNDEFINED
)
403 if (!cpumask_test_cpu(cpu
, cfg
->domain
))
404 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
406 raw_spin_unlock(&vector_lock
);
410 * Setup the vector to irq mappings.
412 void setup_vector_irq(int cpu
)
417 * On most of the platforms, legacy PIC delivers the interrupts on the
418 * boot cpu. But there are certain platforms where PIC interrupts are
419 * delivered to multiple cpu's. If the legacy IRQ is handled by the
420 * legacy PIC, for the new cpu that is coming online, setup the static
421 * legacy vector to irq mapping:
423 for (irq
= 0; irq
< nr_legacy_irqs(); irq
++)
424 per_cpu(vector_irq
, cpu
)[IRQ0_VECTOR
+ irq
] = irq
;
426 __setup_vector_irq(cpu
);
429 int apic_retrigger_irq(struct irq_data
*data
)
431 struct irq_cfg
*cfg
= irqd_cfg(data
);
435 raw_spin_lock_irqsave(&vector_lock
, flags
);
436 cpu
= cpumask_first_and(cfg
->domain
, cpu_online_mask
);
437 apic
->send_IPI_mask(cpumask_of(cpu
), cfg
->vector
);
438 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
443 void apic_ack_edge(struct irq_data
*data
)
445 irq_complete_move(irqd_cfg(data
));
451 * Either sets data->affinity to a valid value, and returns
452 * ->cpu_mask_to_apicid of that in dest_id, or returns -1 and
453 * leaves data->affinity untouched.
455 int apic_set_affinity(struct irq_data
*data
, const struct cpumask
*mask
,
456 unsigned int *dest_id
)
458 struct irq_cfg
*cfg
= irqd_cfg(data
);
459 unsigned int irq
= data
->irq
;
462 if (!config_enabled(CONFIG_SMP
))
465 if (!cpumask_intersects(mask
, cpu_online_mask
))
468 err
= assign_irq_vector(irq
, cfg
, mask
);
472 err
= apic
->cpu_mask_to_apicid_and(mask
, cfg
->domain
, dest_id
);
474 if (assign_irq_vector(irq
, cfg
, data
->affinity
))
475 pr_err("Failed to recover vector for irq %d\n", irq
);
479 cpumask_copy(data
->affinity
, mask
);
484 static int vector_set_affinity(struct irq_data
*irq_data
,
485 const struct cpumask
*dest
, bool force
)
487 struct irq_cfg
*cfg
= irq_data
->chip_data
;
488 int err
, irq
= irq_data
->irq
;
490 if (!config_enabled(CONFIG_SMP
))
493 if (!cpumask_intersects(dest
, cpu_online_mask
))
496 err
= assign_irq_vector(irq
, cfg
, dest
);
498 struct irq_data
*top
= irq_get_irq_data(irq
);
500 if (assign_irq_vector(irq
, cfg
, top
->affinity
))
501 pr_err("Failed to recover vector for irq %d\n", irq
);
505 return IRQ_SET_MASK_OK
;
508 static struct irq_chip lapic_controller
= {
509 .irq_ack
= apic_ack_edge
,
510 .irq_set_affinity
= vector_set_affinity
,
511 .irq_retrigger
= apic_retrigger_irq
,
515 void send_cleanup_vector(struct irq_cfg
*cfg
)
517 cpumask_var_t cleanup_mask
;
519 if (unlikely(!alloc_cpumask_var(&cleanup_mask
, GFP_ATOMIC
))) {
522 for_each_cpu_and(i
, cfg
->old_domain
, cpu_online_mask
)
523 apic
->send_IPI_mask(cpumask_of(i
),
524 IRQ_MOVE_CLEANUP_VECTOR
);
526 cpumask_and(cleanup_mask
, cfg
->old_domain
, cpu_online_mask
);
527 apic
->send_IPI_mask(cleanup_mask
, IRQ_MOVE_CLEANUP_VECTOR
);
528 free_cpumask_var(cleanup_mask
);
530 cfg
->move_in_progress
= 0;
533 asmlinkage __visible
void smp_irq_move_cleanup_interrupt(void)
541 me
= smp_processor_id();
542 for (vector
= FIRST_EXTERNAL_VECTOR
; vector
< NR_VECTORS
; vector
++) {
545 struct irq_desc
*desc
;
548 irq
= __this_cpu_read(vector_irq
[vector
]);
550 if (irq
<= VECTOR_UNDEFINED
)
553 desc
= irq_to_desc(irq
);
561 raw_spin_lock(&desc
->lock
);
564 * Check if the irq migration is in progress. If so, we
565 * haven't received the cleanup request yet for this irq.
567 if (cfg
->move_in_progress
)
570 if (vector
== cfg
->vector
&& cpumask_test_cpu(me
, cfg
->domain
))
573 irr
= apic_read(APIC_IRR
+ (vector
/ 32 * 0x10));
575 * Check if the vector that needs to be cleanedup is
576 * registered at the cpu's IRR. If so, then this is not
577 * the best time to clean it up. Lets clean it up in the
578 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
581 if (irr
& (1 << (vector
% 32))) {
582 apic
->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR
);
585 __this_cpu_write(vector_irq
[vector
], VECTOR_UNDEFINED
);
587 raw_spin_unlock(&desc
->lock
);
593 static void __irq_complete_move(struct irq_cfg
*cfg
, unsigned vector
)
597 if (likely(!cfg
->move_in_progress
))
600 me
= smp_processor_id();
602 if (vector
== cfg
->vector
&& cpumask_test_cpu(me
, cfg
->domain
))
603 send_cleanup_vector(cfg
);
606 void irq_complete_move(struct irq_cfg
*cfg
)
608 __irq_complete_move(cfg
, ~get_irq_regs()->orig_ax
);
611 void irq_force_complete_move(int irq
)
613 struct irq_cfg
*cfg
= irq_cfg(irq
);
618 __irq_complete_move(cfg
, cfg
->vector
);
623 * Dynamic irq allocate and deallocation. Should be replaced by irq domains!
625 int arch_setup_hwirq(unsigned int irq
, int node
)
631 cfg
= alloc_irq_cfg(node
);
635 raw_spin_lock_irqsave(&vector_lock
, flags
);
636 ret
= __assign_irq_vector(irq
, cfg
, apic
->target_cpus());
637 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
640 irq_set_chip_data(irq
, cfg
);
646 void arch_teardown_hwirq(unsigned int irq
)
648 struct irq_cfg
*cfg
= irq_cfg(irq
);
650 free_remapped_irq(irq
);
651 clear_irq_vector(irq
, cfg
);
652 irq_set_chip_data(irq
, NULL
);
656 static void __init
print_APIC_field(int base
)
662 for (i
= 0; i
< 8; i
++)
663 pr_cont("%08x", apic_read(base
+ i
*0x10));
668 static void __init
print_local_APIC(void *dummy
)
670 unsigned int i
, v
, ver
, maxlvt
;
673 pr_debug("printing local APIC contents on CPU#%d/%d:\n",
674 smp_processor_id(), hard_smp_processor_id());
675 v
= apic_read(APIC_ID
);
676 pr_info("... APIC ID: %08x (%01x)\n", v
, read_apic_id());
677 v
= apic_read(APIC_LVR
);
678 pr_info("... APIC VERSION: %08x\n", v
);
679 ver
= GET_APIC_VERSION(v
);
680 maxlvt
= lapic_get_maxlvt();
682 v
= apic_read(APIC_TASKPRI
);
683 pr_debug("... APIC TASKPRI: %08x (%02x)\n", v
, v
& APIC_TPRI_MASK
);
686 if (APIC_INTEGRATED(ver
)) {
687 if (!APIC_XAPIC(ver
)) {
688 v
= apic_read(APIC_ARBPRI
);
689 pr_debug("... APIC ARBPRI: %08x (%02x)\n",
690 v
, v
& APIC_ARBPRI_MASK
);
692 v
= apic_read(APIC_PROCPRI
);
693 pr_debug("... APIC PROCPRI: %08x\n", v
);
697 * Remote read supported only in the 82489DX and local APIC for
698 * Pentium processors.
700 if (!APIC_INTEGRATED(ver
) || maxlvt
== 3) {
701 v
= apic_read(APIC_RRR
);
702 pr_debug("... APIC RRR: %08x\n", v
);
705 v
= apic_read(APIC_LDR
);
706 pr_debug("... APIC LDR: %08x\n", v
);
707 if (!x2apic_enabled()) {
708 v
= apic_read(APIC_DFR
);
709 pr_debug("... APIC DFR: %08x\n", v
);
711 v
= apic_read(APIC_SPIV
);
712 pr_debug("... APIC SPIV: %08x\n", v
);
714 pr_debug("... APIC ISR field:\n");
715 print_APIC_field(APIC_ISR
);
716 pr_debug("... APIC TMR field:\n");
717 print_APIC_field(APIC_TMR
);
718 pr_debug("... APIC IRR field:\n");
719 print_APIC_field(APIC_IRR
);
722 if (APIC_INTEGRATED(ver
)) {
723 /* Due to the Pentium erratum 3AP. */
725 apic_write(APIC_ESR
, 0);
727 v
= apic_read(APIC_ESR
);
728 pr_debug("... APIC ESR: %08x\n", v
);
731 icr
= apic_icr_read();
732 pr_debug("... APIC ICR: %08x\n", (u32
)icr
);
733 pr_debug("... APIC ICR2: %08x\n", (u32
)(icr
>> 32));
735 v
= apic_read(APIC_LVTT
);
736 pr_debug("... APIC LVTT: %08x\n", v
);
740 v
= apic_read(APIC_LVTPC
);
741 pr_debug("... APIC LVTPC: %08x\n", v
);
743 v
= apic_read(APIC_LVT0
);
744 pr_debug("... APIC LVT0: %08x\n", v
);
745 v
= apic_read(APIC_LVT1
);
746 pr_debug("... APIC LVT1: %08x\n", v
);
750 v
= apic_read(APIC_LVTERR
);
751 pr_debug("... APIC LVTERR: %08x\n", v
);
754 v
= apic_read(APIC_TMICT
);
755 pr_debug("... APIC TMICT: %08x\n", v
);
756 v
= apic_read(APIC_TMCCT
);
757 pr_debug("... APIC TMCCT: %08x\n", v
);
758 v
= apic_read(APIC_TDCR
);
759 pr_debug("... APIC TDCR: %08x\n", v
);
761 if (boot_cpu_has(X86_FEATURE_EXTAPIC
)) {
762 v
= apic_read(APIC_EFEAT
);
763 maxlvt
= (v
>> 16) & 0xff;
764 pr_debug("... APIC EFEAT: %08x\n", v
);
765 v
= apic_read(APIC_ECTRL
);
766 pr_debug("... APIC ECTRL: %08x\n", v
);
767 for (i
= 0; i
< maxlvt
; i
++) {
768 v
= apic_read(APIC_EILVTn(i
));
769 pr_debug("... APIC EILVT%d: %08x\n", i
, v
);
775 static void __init
print_local_APICs(int maxcpu
)
783 for_each_online_cpu(cpu
) {
786 smp_call_function_single(cpu
, print_local_APIC
, NULL
, 1);
791 static void __init
print_PIC(void)
796 if (!nr_legacy_irqs())
799 pr_debug("\nprinting PIC contents\n");
801 raw_spin_lock_irqsave(&i8259A_lock
, flags
);
803 v
= inb(0xa1) << 8 | inb(0x21);
804 pr_debug("... PIC IMR: %04x\n", v
);
806 v
= inb(0xa0) << 8 | inb(0x20);
807 pr_debug("... PIC IRR: %04x\n", v
);
811 v
= inb(0xa0) << 8 | inb(0x20);
815 raw_spin_unlock_irqrestore(&i8259A_lock
, flags
);
817 pr_debug("... PIC ISR: %04x\n", v
);
819 v
= inb(0x4d1) << 8 | inb(0x4d0);
820 pr_debug("... PIC ELCR: %04x\n", v
);
823 static int show_lapic __initdata
= 1;
824 static __init
int setup_show_lapic(char *arg
)
828 if (strcmp(arg
, "all") == 0) {
829 show_lapic
= CONFIG_NR_CPUS
;
831 get_option(&arg
, &num
);
838 __setup("show_lapic=", setup_show_lapic
);
840 static int __init
print_ICs(void)
842 if (apic_verbosity
== APIC_QUIET
)
847 /* don't print out if apic is not there */
848 if (!cpu_has_apic
&& !apic_from_smp_config())
851 print_local_APICs(show_lapic
);
857 late_initcall(print_ICs
);