2 * Local APIC related interfaces to support IOAPIC, MSI, HT_IRQ etc.
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 * Moved from arch/x86/kernel/apic/io_apic.c.
6 * Jiang Liu <jiang.liu@linux.intel.com>
7 * Enable support of hierarchical irqdomains
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
13 #include <linux/interrupt.h>
14 #include <linux/init.h>
15 #include <linux/compiler.h>
16 #include <linux/irqdomain.h>
17 #include <linux/slab.h>
18 #include <asm/hw_irq.h>
20 #include <asm/i8259.h>
22 #include <asm/irq_remapping.h>
24 struct irq_domain
*x86_vector_domain
;
25 static DEFINE_RAW_SPINLOCK(vector_lock
);
26 static struct irq_chip lapic_controller
;
28 void lock_vector_lock(void)
30 /* Used to the online set of cpus does not change
31 * during assign_irq_vector.
33 raw_spin_lock(&vector_lock
);
36 void unlock_vector_lock(void)
38 raw_spin_unlock(&vector_lock
);
41 struct irq_cfg
*irq_cfg(unsigned int irq
)
43 return irqd_cfg(irq_get_irq_data(irq
));
46 struct irq_cfg
*irqd_cfg(struct irq_data
*irq_data
)
51 while (irq_data
->parent_data
)
52 irq_data
= irq_data
->parent_data
;
54 return irq_data
->chip_data
;
57 static struct irq_cfg
*alloc_irq_cfg(int node
)
61 cfg
= kzalloc_node(sizeof(*cfg
), GFP_KERNEL
, node
);
64 if (!zalloc_cpumask_var_node(&cfg
->domain
, GFP_KERNEL
, node
))
66 if (!zalloc_cpumask_var_node(&cfg
->old_domain
, GFP_KERNEL
, node
))
68 #ifdef CONFIG_X86_IO_APIC
69 INIT_LIST_HEAD(&cfg
->irq_2_pin
);
73 free_cpumask_var(cfg
->domain
);
79 struct irq_cfg
*alloc_irq_and_cfg_at(unsigned int at
, int node
)
81 int res
= irq_alloc_desc_at(at
, node
);
92 cfg
= alloc_irq_cfg(node
);
94 irq_set_chip_data(at
, cfg
);
100 static void free_irq_cfg(struct irq_cfg
*cfg
)
103 free_cpumask_var(cfg
->domain
);
104 free_cpumask_var(cfg
->old_domain
);
110 __assign_irq_vector(int irq
, struct irq_cfg
*cfg
, const struct cpumask
*mask
)
113 * NOTE! The local APIC isn't very good at handling
114 * multiple interrupts at the same interrupt level.
115 * As the interrupt level is determined by taking the
116 * vector number and shifting that right by 4, we
117 * want to spread these out a bit so that they don't
118 * all fall in the same interrupt level.
120 * Also, we've got to be careful not to trash gate
121 * 0x80, because int 0x80 is hm, kind of importantish. ;)
123 static int current_vector
= FIRST_EXTERNAL_VECTOR
+ VECTOR_OFFSET_START
;
124 static int current_offset
= VECTOR_OFFSET_START
% 16;
126 cpumask_var_t tmp_mask
;
128 if (cfg
->move_in_progress
)
131 if (!alloc_cpumask_var(&tmp_mask
, GFP_ATOMIC
))
134 /* Only try and allocate irqs on cpus that are present */
136 cpumask_clear(cfg
->old_domain
);
137 cpu
= cpumask_first_and(mask
, cpu_online_mask
);
138 while (cpu
< nr_cpu_ids
) {
139 int new_cpu
, vector
, offset
;
141 apic
->vector_allocation_domain(cpu
, tmp_mask
, mask
);
143 if (cpumask_subset(tmp_mask
, cfg
->domain
)) {
145 if (cpumask_equal(tmp_mask
, cfg
->domain
))
148 * New cpumask using the vector is a proper subset of
149 * the current in use mask. So cleanup the vector
150 * allocation for the members that are not used anymore.
152 cpumask_andnot(cfg
->old_domain
, cfg
->domain
, tmp_mask
);
153 cfg
->move_in_progress
=
154 cpumask_intersects(cfg
->old_domain
, cpu_online_mask
);
155 cpumask_and(cfg
->domain
, cfg
->domain
, tmp_mask
);
159 vector
= current_vector
;
160 offset
= current_offset
;
163 if (vector
>= first_system_vector
) {
164 offset
= (offset
+ 1) % 16;
165 vector
= FIRST_EXTERNAL_VECTOR
+ offset
;
168 if (unlikely(current_vector
== vector
)) {
169 cpumask_or(cfg
->old_domain
, cfg
->old_domain
, tmp_mask
);
170 cpumask_andnot(tmp_mask
, mask
, cfg
->old_domain
);
171 cpu
= cpumask_first_and(tmp_mask
, cpu_online_mask
);
175 if (test_bit(vector
, used_vectors
))
178 for_each_cpu_and(new_cpu
, tmp_mask
, cpu_online_mask
) {
179 if (per_cpu(vector_irq
, new_cpu
)[vector
] >
184 current_vector
= vector
;
185 current_offset
= offset
;
187 cpumask_copy(cfg
->old_domain
, cfg
->domain
);
188 cfg
->move_in_progress
=
189 cpumask_intersects(cfg
->old_domain
, cpu_online_mask
);
191 for_each_cpu_and(new_cpu
, tmp_mask
, cpu_online_mask
)
192 per_cpu(vector_irq
, new_cpu
)[vector
] = irq
;
193 cfg
->vector
= vector
;
194 cpumask_copy(cfg
->domain
, tmp_mask
);
198 free_cpumask_var(tmp_mask
);
201 /* cache destination APIC IDs into cfg->dest_apicid */
202 err
= apic
->cpu_mask_to_apicid_and(mask
, cfg
->domain
,
209 int assign_irq_vector(int irq
, struct irq_cfg
*cfg
, const struct cpumask
*mask
)
214 raw_spin_lock_irqsave(&vector_lock
, flags
);
215 err
= __assign_irq_vector(irq
, cfg
, mask
);
216 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
220 void clear_irq_vector(int irq
, struct irq_cfg
*cfg
)
225 raw_spin_lock_irqsave(&vector_lock
, flags
);
226 BUG_ON(!cfg
->vector
);
228 vector
= cfg
->vector
;
229 for_each_cpu_and(cpu
, cfg
->domain
, cpu_online_mask
)
230 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
233 cpumask_clear(cfg
->domain
);
235 if (likely(!cfg
->move_in_progress
)) {
236 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
240 for_each_cpu_and(cpu
, cfg
->old_domain
, cpu_online_mask
) {
241 for (vector
= FIRST_EXTERNAL_VECTOR
; vector
< NR_VECTORS
;
243 if (per_cpu(vector_irq
, cpu
)[vector
] != irq
)
245 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
249 cfg
->move_in_progress
= 0;
250 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
253 void init_irq_alloc_info(struct irq_alloc_info
*info
,
254 const struct cpumask
*mask
)
256 memset(info
, 0, sizeof(*info
));
260 void copy_irq_alloc_info(struct irq_alloc_info
*dst
, struct irq_alloc_info
*src
)
265 memset(dst
, 0, sizeof(*dst
));
268 static inline const struct cpumask
*
269 irq_alloc_info_get_mask(struct irq_alloc_info
*info
)
271 return (!info
|| !info
->mask
) ? apic
->target_cpus() : info
->mask
;
274 static void x86_vector_free_irqs(struct irq_domain
*domain
,
275 unsigned int virq
, unsigned int nr_irqs
)
277 struct irq_data
*irq_data
;
280 for (i
= 0; i
< nr_irqs
; i
++) {
281 irq_data
= irq_domain_get_irq_data(x86_vector_domain
, virq
+ i
);
282 if (irq_data
&& irq_data
->chip_data
) {
283 free_remapped_irq(virq
);
284 clear_irq_vector(virq
+ i
, irq_data
->chip_data
);
285 free_irq_cfg(irq_data
->chip_data
);
286 irq_domain_reset_irq_data(irq_data
);
291 static int x86_vector_alloc_irqs(struct irq_domain
*domain
, unsigned int virq
,
292 unsigned int nr_irqs
, void *arg
)
294 struct irq_alloc_info
*info
= arg
;
295 const struct cpumask
*mask
;
296 struct irq_data
*irq_data
;
303 /* Currently vector allocator can't guarantee contiguous allocations */
304 if ((info
->flags
& X86_IRQ_ALLOC_CONTIGUOUS_VECTORS
) && nr_irqs
> 1)
307 mask
= irq_alloc_info_get_mask(info
);
308 for (i
= 0; i
< nr_irqs
; i
++) {
309 irq_data
= irq_domain_get_irq_data(domain
, virq
+ i
);
311 cfg
= alloc_irq_cfg(irq_data
->node
);
317 irq_data
->chip
= &lapic_controller
;
318 irq_data
->chip_data
= cfg
;
319 irq_data
->hwirq
= virq
+ i
;
320 err
= assign_irq_vector(virq
, cfg
, mask
);
328 x86_vector_free_irqs(domain
, virq
, i
+ 1);
332 static struct irq_domain_ops x86_vector_domain_ops
= {
333 .alloc
= x86_vector_alloc_irqs
,
334 .free
= x86_vector_free_irqs
,
337 int __init
arch_probe_nr_irqs(void)
341 if (nr_irqs
> (NR_VECTORS
* nr_cpu_ids
))
342 nr_irqs
= NR_VECTORS
* nr_cpu_ids
;
344 nr
= (gsi_top
+ nr_legacy_irqs()) + 8 * nr_cpu_ids
;
345 #if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
347 * for MSI and HT dyn irq
349 if (gsi_top
<= NR_IRQS_LEGACY
)
350 nr
+= 8 * nr_cpu_ids
;
357 return nr_legacy_irqs();
360 int __init
arch_early_irq_init(void)
362 x86_vector_domain
= irq_domain_add_tree(NULL
, &x86_vector_domain_ops
,
364 BUG_ON(x86_vector_domain
== NULL
);
365 irq_set_default_host(x86_vector_domain
);
367 return arch_early_ioapic_init();
370 static void __setup_vector_irq(int cpu
)
372 /* Initialize vector_irq on a new cpu */
377 * vector_lock will make sure that we don't run into irq vector
378 * assignments that might be happening on another cpu in parallel,
379 * while we setup our initial vector to irq mappings.
381 raw_spin_lock(&vector_lock
);
382 /* Mark the inuse vectors */
383 for_each_active_irq(irq
) {
388 if (!cpumask_test_cpu(cpu
, cfg
->domain
))
390 vector
= cfg
->vector
;
391 per_cpu(vector_irq
, cpu
)[vector
] = irq
;
393 /* Mark the free vectors */
394 for (vector
= 0; vector
< NR_VECTORS
; ++vector
) {
395 irq
= per_cpu(vector_irq
, cpu
)[vector
];
396 if (irq
<= VECTOR_UNDEFINED
)
400 if (!cpumask_test_cpu(cpu
, cfg
->domain
))
401 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
403 raw_spin_unlock(&vector_lock
);
407 * Setup the vector to irq mappings.
409 void setup_vector_irq(int cpu
)
414 * On most of the platforms, legacy PIC delivers the interrupts on the
415 * boot cpu. But there are certain platforms where PIC interrupts are
416 * delivered to multiple cpu's. If the legacy IRQ is handled by the
417 * legacy PIC, for the new cpu that is coming online, setup the static
418 * legacy vector to irq mapping:
420 for (irq
= 0; irq
< nr_legacy_irqs(); irq
++)
421 per_cpu(vector_irq
, cpu
)[IRQ0_VECTOR
+ irq
] = irq
;
423 __setup_vector_irq(cpu
);
426 int apic_retrigger_irq(struct irq_data
*data
)
428 struct irq_cfg
*cfg
= irqd_cfg(data
);
432 raw_spin_lock_irqsave(&vector_lock
, flags
);
433 cpu
= cpumask_first_and(cfg
->domain
, cpu_online_mask
);
434 apic
->send_IPI_mask(cpumask_of(cpu
), cfg
->vector
);
435 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
440 void apic_ack_edge(struct irq_data
*data
)
442 irq_complete_move(irqd_cfg(data
));
448 * Either sets data->affinity to a valid value, and returns
449 * ->cpu_mask_to_apicid of that in dest_id, or returns -1 and
450 * leaves data->affinity untouched.
452 int apic_set_affinity(struct irq_data
*data
, const struct cpumask
*mask
,
453 unsigned int *dest_id
)
455 struct irq_cfg
*cfg
= irqd_cfg(data
);
456 unsigned int irq
= data
->irq
;
459 if (!config_enabled(CONFIG_SMP
))
462 if (!cpumask_intersects(mask
, cpu_online_mask
))
465 err
= assign_irq_vector(irq
, cfg
, mask
);
469 err
= apic
->cpu_mask_to_apicid_and(mask
, cfg
->domain
, dest_id
);
471 if (assign_irq_vector(irq
, cfg
, data
->affinity
))
472 pr_err("Failed to recover vector for irq %d\n", irq
);
476 cpumask_copy(data
->affinity
, mask
);
481 static int vector_set_affinity(struct irq_data
*irq_data
,
482 const struct cpumask
*dest
, bool force
)
484 struct irq_cfg
*cfg
= irq_data
->chip_data
;
485 int err
, irq
= irq_data
->irq
;
487 if (!config_enabled(CONFIG_SMP
))
490 if (!cpumask_intersects(dest
, cpu_online_mask
))
493 err
= assign_irq_vector(irq
, cfg
, dest
);
495 struct irq_data
*top
= irq_get_irq_data(irq
);
497 if (assign_irq_vector(irq
, cfg
, top
->affinity
))
498 pr_err("Failed to recover vector for irq %d\n", irq
);
502 return IRQ_SET_MASK_OK
;
505 static struct irq_chip lapic_controller
= {
506 .irq_ack
= apic_ack_edge
,
507 .irq_set_affinity
= vector_set_affinity
,
508 .irq_retrigger
= apic_retrigger_irq
,
512 void send_cleanup_vector(struct irq_cfg
*cfg
)
514 cpumask_var_t cleanup_mask
;
516 if (unlikely(!alloc_cpumask_var(&cleanup_mask
, GFP_ATOMIC
))) {
519 for_each_cpu_and(i
, cfg
->old_domain
, cpu_online_mask
)
520 apic
->send_IPI_mask(cpumask_of(i
),
521 IRQ_MOVE_CLEANUP_VECTOR
);
523 cpumask_and(cleanup_mask
, cfg
->old_domain
, cpu_online_mask
);
524 apic
->send_IPI_mask(cleanup_mask
, IRQ_MOVE_CLEANUP_VECTOR
);
525 free_cpumask_var(cleanup_mask
);
527 cfg
->move_in_progress
= 0;
530 asmlinkage __visible
void smp_irq_move_cleanup_interrupt(void)
538 me
= smp_processor_id();
539 for (vector
= FIRST_EXTERNAL_VECTOR
; vector
< NR_VECTORS
; vector
++) {
542 struct irq_desc
*desc
;
545 irq
= __this_cpu_read(vector_irq
[vector
]);
547 if (irq
<= VECTOR_UNDEFINED
)
550 desc
= irq_to_desc(irq
);
558 raw_spin_lock(&desc
->lock
);
561 * Check if the irq migration is in progress. If so, we
562 * haven't received the cleanup request yet for this irq.
564 if (cfg
->move_in_progress
)
567 if (vector
== cfg
->vector
&& cpumask_test_cpu(me
, cfg
->domain
))
570 irr
= apic_read(APIC_IRR
+ (vector
/ 32 * 0x10));
572 * Check if the vector that needs to be cleanedup is
573 * registered at the cpu's IRR. If so, then this is not
574 * the best time to clean it up. Lets clean it up in the
575 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
578 if (irr
& (1 << (vector
% 32))) {
579 apic
->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR
);
582 __this_cpu_write(vector_irq
[vector
], VECTOR_UNDEFINED
);
584 raw_spin_unlock(&desc
->lock
);
590 static void __irq_complete_move(struct irq_cfg
*cfg
, unsigned vector
)
594 if (likely(!cfg
->move_in_progress
))
597 me
= smp_processor_id();
599 if (vector
== cfg
->vector
&& cpumask_test_cpu(me
, cfg
->domain
))
600 send_cleanup_vector(cfg
);
603 void irq_complete_move(struct irq_cfg
*cfg
)
605 __irq_complete_move(cfg
, ~get_irq_regs()->orig_ax
);
608 void irq_force_complete_move(int irq
)
610 struct irq_cfg
*cfg
= irq_cfg(irq
);
615 __irq_complete_move(cfg
, cfg
->vector
);
620 * Dynamic irq allocate and deallocation. Should be replaced by irq domains!
622 int arch_setup_hwirq(unsigned int irq
, int node
)
628 cfg
= alloc_irq_cfg(node
);
632 raw_spin_lock_irqsave(&vector_lock
, flags
);
633 ret
= __assign_irq_vector(irq
, cfg
, apic
->target_cpus());
634 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
637 irq_set_chip_data(irq
, cfg
);
643 void arch_teardown_hwirq(unsigned int irq
)
645 struct irq_cfg
*cfg
= irq_cfg(irq
);
647 free_remapped_irq(irq
);
648 clear_irq_vector(irq
, cfg
);
649 irq_set_chip_data(irq
, NULL
);
653 static void __init
print_APIC_field(int base
)
659 for (i
= 0; i
< 8; i
++)
660 pr_cont("%08x", apic_read(base
+ i
*0x10));
665 static void __init
print_local_APIC(void *dummy
)
667 unsigned int i
, v
, ver
, maxlvt
;
670 pr_debug("printing local APIC contents on CPU#%d/%d:\n",
671 smp_processor_id(), hard_smp_processor_id());
672 v
= apic_read(APIC_ID
);
673 pr_info("... APIC ID: %08x (%01x)\n", v
, read_apic_id());
674 v
= apic_read(APIC_LVR
);
675 pr_info("... APIC VERSION: %08x\n", v
);
676 ver
= GET_APIC_VERSION(v
);
677 maxlvt
= lapic_get_maxlvt();
679 v
= apic_read(APIC_TASKPRI
);
680 pr_debug("... APIC TASKPRI: %08x (%02x)\n", v
, v
& APIC_TPRI_MASK
);
683 if (APIC_INTEGRATED(ver
)) {
684 if (!APIC_XAPIC(ver
)) {
685 v
= apic_read(APIC_ARBPRI
);
686 pr_debug("... APIC ARBPRI: %08x (%02x)\n",
687 v
, v
& APIC_ARBPRI_MASK
);
689 v
= apic_read(APIC_PROCPRI
);
690 pr_debug("... APIC PROCPRI: %08x\n", v
);
694 * Remote read supported only in the 82489DX and local APIC for
695 * Pentium processors.
697 if (!APIC_INTEGRATED(ver
) || maxlvt
== 3) {
698 v
= apic_read(APIC_RRR
);
699 pr_debug("... APIC RRR: %08x\n", v
);
702 v
= apic_read(APIC_LDR
);
703 pr_debug("... APIC LDR: %08x\n", v
);
704 if (!x2apic_enabled()) {
705 v
= apic_read(APIC_DFR
);
706 pr_debug("... APIC DFR: %08x\n", v
);
708 v
= apic_read(APIC_SPIV
);
709 pr_debug("... APIC SPIV: %08x\n", v
);
711 pr_debug("... APIC ISR field:\n");
712 print_APIC_field(APIC_ISR
);
713 pr_debug("... APIC TMR field:\n");
714 print_APIC_field(APIC_TMR
);
715 pr_debug("... APIC IRR field:\n");
716 print_APIC_field(APIC_IRR
);
719 if (APIC_INTEGRATED(ver
)) {
720 /* Due to the Pentium erratum 3AP. */
722 apic_write(APIC_ESR
, 0);
724 v
= apic_read(APIC_ESR
);
725 pr_debug("... APIC ESR: %08x\n", v
);
728 icr
= apic_icr_read();
729 pr_debug("... APIC ICR: %08x\n", (u32
)icr
);
730 pr_debug("... APIC ICR2: %08x\n", (u32
)(icr
>> 32));
732 v
= apic_read(APIC_LVTT
);
733 pr_debug("... APIC LVTT: %08x\n", v
);
737 v
= apic_read(APIC_LVTPC
);
738 pr_debug("... APIC LVTPC: %08x\n", v
);
740 v
= apic_read(APIC_LVT0
);
741 pr_debug("... APIC LVT0: %08x\n", v
);
742 v
= apic_read(APIC_LVT1
);
743 pr_debug("... APIC LVT1: %08x\n", v
);
747 v
= apic_read(APIC_LVTERR
);
748 pr_debug("... APIC LVTERR: %08x\n", v
);
751 v
= apic_read(APIC_TMICT
);
752 pr_debug("... APIC TMICT: %08x\n", v
);
753 v
= apic_read(APIC_TMCCT
);
754 pr_debug("... APIC TMCCT: %08x\n", v
);
755 v
= apic_read(APIC_TDCR
);
756 pr_debug("... APIC TDCR: %08x\n", v
);
758 if (boot_cpu_has(X86_FEATURE_EXTAPIC
)) {
759 v
= apic_read(APIC_EFEAT
);
760 maxlvt
= (v
>> 16) & 0xff;
761 pr_debug("... APIC EFEAT: %08x\n", v
);
762 v
= apic_read(APIC_ECTRL
);
763 pr_debug("... APIC ECTRL: %08x\n", v
);
764 for (i
= 0; i
< maxlvt
; i
++) {
765 v
= apic_read(APIC_EILVTn(i
));
766 pr_debug("... APIC EILVT%d: %08x\n", i
, v
);
772 static void __init
print_local_APICs(int maxcpu
)
780 for_each_online_cpu(cpu
) {
783 smp_call_function_single(cpu
, print_local_APIC
, NULL
, 1);
788 static void __init
print_PIC(void)
793 if (!nr_legacy_irqs())
796 pr_debug("\nprinting PIC contents\n");
798 raw_spin_lock_irqsave(&i8259A_lock
, flags
);
800 v
= inb(0xa1) << 8 | inb(0x21);
801 pr_debug("... PIC IMR: %04x\n", v
);
803 v
= inb(0xa0) << 8 | inb(0x20);
804 pr_debug("... PIC IRR: %04x\n", v
);
808 v
= inb(0xa0) << 8 | inb(0x20);
812 raw_spin_unlock_irqrestore(&i8259A_lock
, flags
);
814 pr_debug("... PIC ISR: %04x\n", v
);
816 v
= inb(0x4d1) << 8 | inb(0x4d0);
817 pr_debug("... PIC ELCR: %04x\n", v
);
820 static int show_lapic __initdata
= 1;
821 static __init
int setup_show_lapic(char *arg
)
825 if (strcmp(arg
, "all") == 0) {
826 show_lapic
= CONFIG_NR_CPUS
;
828 get_option(&arg
, &num
);
835 __setup("show_lapic=", setup_show_lapic
);
837 static int __init
print_ICs(void)
839 if (apic_verbosity
== APIC_QUIET
)
844 /* don't print out if apic is not there */
845 if (!cpu_has_apic
&& !apic_from_smp_config())
848 print_local_APICs(show_lapic
);
854 late_initcall(print_ICs
);