2 * Local APIC related interfaces to support IOAPIC, MSI, HT_IRQ etc.
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 * Moved from arch/x86/kernel/apic/io_apic.c.
6 * Jiang Liu <jiang.liu@linux.intel.com>
7 * Enable support of hierarchical irqdomains
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
13 #include <linux/interrupt.h>
14 #include <linux/init.h>
15 #include <linux/compiler.h>
16 #include <linux/irqdomain.h>
17 #include <linux/slab.h>
18 #include <asm/hw_irq.h>
20 #include <asm/i8259.h>
22 #include <asm/irq_remapping.h>
24 struct irq_domain
*x86_vector_domain
;
25 static DEFINE_RAW_SPINLOCK(vector_lock
);
26 static struct irq_chip lapic_controller
;
28 void lock_vector_lock(void)
30 /* Used to the online set of cpus does not change
31 * during assign_irq_vector.
33 raw_spin_lock(&vector_lock
);
36 void unlock_vector_lock(void)
38 raw_spin_unlock(&vector_lock
);
41 struct irq_cfg
*irq_cfg(unsigned int irq
)
43 return irqd_cfg(irq_get_irq_data(irq
));
46 struct irq_cfg
*irqd_cfg(struct irq_data
*irq_data
)
51 while (irq_data
->parent_data
)
52 irq_data
= irq_data
->parent_data
;
54 return irq_data
->chip_data
;
57 static struct irq_cfg
*alloc_irq_cfg(int node
)
61 cfg
= kzalloc_node(sizeof(*cfg
), GFP_KERNEL
, node
);
64 if (!zalloc_cpumask_var_node(&cfg
->domain
, GFP_KERNEL
, node
))
66 if (!zalloc_cpumask_var_node(&cfg
->old_domain
, GFP_KERNEL
, node
))
68 #ifdef CONFIG_X86_IO_APIC
69 INIT_LIST_HEAD(&cfg
->irq_2_pin
);
73 free_cpumask_var(cfg
->domain
);
79 struct irq_cfg
*alloc_irq_and_cfg_at(unsigned int at
, int node
)
81 int res
= irq_alloc_desc_at(at
, node
);
92 cfg
= alloc_irq_cfg(node
);
94 irq_set_chip_data(at
, cfg
);
100 static void free_irq_cfg(struct irq_cfg
*cfg
)
103 free_cpumask_var(cfg
->domain
);
104 free_cpumask_var(cfg
->old_domain
);
110 __assign_irq_vector(int irq
, struct irq_cfg
*cfg
, const struct cpumask
*mask
)
113 * NOTE! The local APIC isn't very good at handling
114 * multiple interrupts at the same interrupt level.
115 * As the interrupt level is determined by taking the
116 * vector number and shifting that right by 4, we
117 * want to spread these out a bit so that they don't
118 * all fall in the same interrupt level.
120 * Also, we've got to be careful not to trash gate
121 * 0x80, because int 0x80 is hm, kind of importantish. ;)
123 static int current_vector
= FIRST_EXTERNAL_VECTOR
+ VECTOR_OFFSET_START
;
124 static int current_offset
= VECTOR_OFFSET_START
% 16;
126 cpumask_var_t tmp_mask
;
128 if (cfg
->move_in_progress
)
131 if (!alloc_cpumask_var(&tmp_mask
, GFP_ATOMIC
))
134 /* Only try and allocate irqs on cpus that are present */
136 cpumask_clear(cfg
->old_domain
);
137 cpu
= cpumask_first_and(mask
, cpu_online_mask
);
138 while (cpu
< nr_cpu_ids
) {
139 int new_cpu
, vector
, offset
;
141 apic
->vector_allocation_domain(cpu
, tmp_mask
, mask
);
143 if (cpumask_subset(tmp_mask
, cfg
->domain
)) {
145 if (cpumask_equal(tmp_mask
, cfg
->domain
))
148 * New cpumask using the vector is a proper subset of
149 * the current in use mask. So cleanup the vector
150 * allocation for the members that are not used anymore.
152 cpumask_andnot(cfg
->old_domain
, cfg
->domain
, tmp_mask
);
153 cfg
->move_in_progress
=
154 cpumask_intersects(cfg
->old_domain
, cpu_online_mask
);
155 cpumask_and(cfg
->domain
, cfg
->domain
, tmp_mask
);
159 vector
= current_vector
;
160 offset
= current_offset
;
163 if (vector
>= first_system_vector
) {
164 offset
= (offset
+ 1) % 16;
165 vector
= FIRST_EXTERNAL_VECTOR
+ offset
;
168 if (unlikely(current_vector
== vector
)) {
169 cpumask_or(cfg
->old_domain
, cfg
->old_domain
, tmp_mask
);
170 cpumask_andnot(tmp_mask
, mask
, cfg
->old_domain
);
171 cpu
= cpumask_first_and(tmp_mask
, cpu_online_mask
);
175 if (test_bit(vector
, used_vectors
))
178 for_each_cpu_and(new_cpu
, tmp_mask
, cpu_online_mask
) {
179 if (per_cpu(vector_irq
, new_cpu
)[vector
] >
184 current_vector
= vector
;
185 current_offset
= offset
;
187 cpumask_copy(cfg
->old_domain
, cfg
->domain
);
188 cfg
->move_in_progress
=
189 cpumask_intersects(cfg
->old_domain
, cpu_online_mask
);
191 for_each_cpu_and(new_cpu
, tmp_mask
, cpu_online_mask
)
192 per_cpu(vector_irq
, new_cpu
)[vector
] = irq
;
193 cfg
->vector
= vector
;
194 cpumask_copy(cfg
->domain
, tmp_mask
);
198 free_cpumask_var(tmp_mask
);
201 /* cache destination APIC IDs into cfg->dest_apicid */
202 err
= apic
->cpu_mask_to_apicid_and(mask
, cfg
->domain
,
209 int assign_irq_vector(int irq
, struct irq_cfg
*cfg
, const struct cpumask
*mask
)
214 raw_spin_lock_irqsave(&vector_lock
, flags
);
215 err
= __assign_irq_vector(irq
, cfg
, mask
);
216 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
220 void clear_irq_vector(int irq
, struct irq_cfg
*cfg
)
225 raw_spin_lock_irqsave(&vector_lock
, flags
);
226 BUG_ON(!cfg
->vector
);
228 vector
= cfg
->vector
;
229 for_each_cpu_and(cpu
, cfg
->domain
, cpu_online_mask
)
230 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
233 cpumask_clear(cfg
->domain
);
235 if (likely(!cfg
->move_in_progress
)) {
236 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
240 for_each_cpu_and(cpu
, cfg
->old_domain
, cpu_online_mask
) {
241 for (vector
= FIRST_EXTERNAL_VECTOR
; vector
< NR_VECTORS
;
243 if (per_cpu(vector_irq
, cpu
)[vector
] != irq
)
245 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
249 cfg
->move_in_progress
= 0;
250 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
253 void init_irq_alloc_info(struct irq_alloc_info
*info
,
254 const struct cpumask
*mask
)
256 memset(info
, 0, sizeof(*info
));
260 void copy_irq_alloc_info(struct irq_alloc_info
*dst
, struct irq_alloc_info
*src
)
265 memset(dst
, 0, sizeof(*dst
));
268 static inline const struct cpumask
*
269 irq_alloc_info_get_mask(struct irq_alloc_info
*info
)
271 return (!info
|| !info
->mask
) ? apic
->target_cpus() : info
->mask
;
274 static void x86_vector_free_irqs(struct irq_domain
*domain
,
275 unsigned int virq
, unsigned int nr_irqs
)
277 struct irq_data
*irq_data
;
280 for (i
= 0; i
< nr_irqs
; i
++) {
281 irq_data
= irq_domain_get_irq_data(x86_vector_domain
, virq
+ i
);
282 if (irq_data
&& irq_data
->chip_data
) {
283 free_remapped_irq(virq
);
284 clear_irq_vector(virq
+ i
, irq_data
->chip_data
);
285 free_irq_cfg(irq_data
->chip_data
);
286 irq_domain_reset_irq_data(irq_data
);
291 static int x86_vector_alloc_irqs(struct irq_domain
*domain
, unsigned int virq
,
292 unsigned int nr_irqs
, void *arg
)
294 struct irq_alloc_info
*info
= arg
;
295 const struct cpumask
*mask
;
296 struct irq_data
*irq_data
;
303 /* Currently vector allocator can't guarantee contiguous allocations */
304 if ((info
->flags
& X86_IRQ_ALLOC_CONTIGUOUS_VECTORS
) && nr_irqs
> 1)
307 mask
= irq_alloc_info_get_mask(info
);
308 for (i
= 0; i
< nr_irqs
; i
++) {
309 irq_data
= irq_domain_get_irq_data(domain
, virq
+ i
);
311 cfg
= alloc_irq_cfg(irq_data
->node
);
317 irq_data
->chip
= &lapic_controller
;
318 irq_data
->chip_data
= cfg
;
319 irq_data
->hwirq
= virq
+ i
;
320 err
= assign_irq_vector(virq
, cfg
, mask
);
328 x86_vector_free_irqs(domain
, virq
, i
+ 1);
332 static struct irq_domain_ops x86_vector_domain_ops
= {
333 .alloc
= x86_vector_alloc_irqs
,
334 .free
= x86_vector_free_irqs
,
337 int __init
arch_probe_nr_irqs(void)
341 if (nr_irqs
> (NR_VECTORS
* nr_cpu_ids
))
342 nr_irqs
= NR_VECTORS
* nr_cpu_ids
;
344 nr
= (gsi_top
+ nr_legacy_irqs()) + 8 * nr_cpu_ids
;
345 #if defined(CONFIG_PCI_MSI) || defined(CONFIG_HT_IRQ)
347 * for MSI and HT dyn irq
349 if (gsi_top
<= NR_IRQS_LEGACY
)
350 nr
+= 8 * nr_cpu_ids
;
357 return nr_legacy_irqs();
360 int __init
arch_early_irq_init(void)
362 x86_vector_domain
= irq_domain_add_tree(NULL
, &x86_vector_domain_ops
,
364 BUG_ON(x86_vector_domain
== NULL
);
365 irq_set_default_host(x86_vector_domain
);
367 arch_init_msi_domain(x86_vector_domain
);
369 return arch_early_ioapic_init();
372 static void __setup_vector_irq(int cpu
)
374 /* Initialize vector_irq on a new cpu */
379 * vector_lock will make sure that we don't run into irq vector
380 * assignments that might be happening on another cpu in parallel,
381 * while we setup our initial vector to irq mappings.
383 raw_spin_lock(&vector_lock
);
384 /* Mark the inuse vectors */
385 for_each_active_irq(irq
) {
390 if (!cpumask_test_cpu(cpu
, cfg
->domain
))
392 vector
= cfg
->vector
;
393 per_cpu(vector_irq
, cpu
)[vector
] = irq
;
395 /* Mark the free vectors */
396 for (vector
= 0; vector
< NR_VECTORS
; ++vector
) {
397 irq
= per_cpu(vector_irq
, cpu
)[vector
];
398 if (irq
<= VECTOR_UNDEFINED
)
402 if (!cpumask_test_cpu(cpu
, cfg
->domain
))
403 per_cpu(vector_irq
, cpu
)[vector
] = VECTOR_UNDEFINED
;
405 raw_spin_unlock(&vector_lock
);
409 * Setup the vector to irq mappings.
411 void setup_vector_irq(int cpu
)
416 * On most of the platforms, legacy PIC delivers the interrupts on the
417 * boot cpu. But there are certain platforms where PIC interrupts are
418 * delivered to multiple cpu's. If the legacy IRQ is handled by the
419 * legacy PIC, for the new cpu that is coming online, setup the static
420 * legacy vector to irq mapping:
422 for (irq
= 0; irq
< nr_legacy_irqs(); irq
++)
423 per_cpu(vector_irq
, cpu
)[IRQ0_VECTOR
+ irq
] = irq
;
425 __setup_vector_irq(cpu
);
428 int apic_retrigger_irq(struct irq_data
*data
)
430 struct irq_cfg
*cfg
= irqd_cfg(data
);
434 raw_spin_lock_irqsave(&vector_lock
, flags
);
435 cpu
= cpumask_first_and(cfg
->domain
, cpu_online_mask
);
436 apic
->send_IPI_mask(cpumask_of(cpu
), cfg
->vector
);
437 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
442 void apic_ack_edge(struct irq_data
*data
)
444 irq_complete_move(irqd_cfg(data
));
450 * Either sets data->affinity to a valid value, and returns
451 * ->cpu_mask_to_apicid of that in dest_id, or returns -1 and
452 * leaves data->affinity untouched.
454 int apic_set_affinity(struct irq_data
*data
, const struct cpumask
*mask
,
455 unsigned int *dest_id
)
457 struct irq_cfg
*cfg
= irqd_cfg(data
);
458 unsigned int irq
= data
->irq
;
461 if (!config_enabled(CONFIG_SMP
))
464 if (!cpumask_intersects(mask
, cpu_online_mask
))
467 err
= assign_irq_vector(irq
, cfg
, mask
);
471 err
= apic
->cpu_mask_to_apicid_and(mask
, cfg
->domain
, dest_id
);
473 if (assign_irq_vector(irq
, cfg
, data
->affinity
))
474 pr_err("Failed to recover vector for irq %d\n", irq
);
478 cpumask_copy(data
->affinity
, mask
);
483 static int vector_set_affinity(struct irq_data
*irq_data
,
484 const struct cpumask
*dest
, bool force
)
486 struct irq_cfg
*cfg
= irq_data
->chip_data
;
487 int err
, irq
= irq_data
->irq
;
489 if (!config_enabled(CONFIG_SMP
))
492 if (!cpumask_intersects(dest
, cpu_online_mask
))
495 err
= assign_irq_vector(irq
, cfg
, dest
);
497 struct irq_data
*top
= irq_get_irq_data(irq
);
499 if (assign_irq_vector(irq
, cfg
, top
->affinity
))
500 pr_err("Failed to recover vector for irq %d\n", irq
);
504 return IRQ_SET_MASK_OK
;
507 static struct irq_chip lapic_controller
= {
508 .irq_ack
= apic_ack_edge
,
509 .irq_set_affinity
= vector_set_affinity
,
510 .irq_retrigger
= apic_retrigger_irq
,
514 void send_cleanup_vector(struct irq_cfg
*cfg
)
516 cpumask_var_t cleanup_mask
;
518 if (unlikely(!alloc_cpumask_var(&cleanup_mask
, GFP_ATOMIC
))) {
521 for_each_cpu_and(i
, cfg
->old_domain
, cpu_online_mask
)
522 apic
->send_IPI_mask(cpumask_of(i
),
523 IRQ_MOVE_CLEANUP_VECTOR
);
525 cpumask_and(cleanup_mask
, cfg
->old_domain
, cpu_online_mask
);
526 apic
->send_IPI_mask(cleanup_mask
, IRQ_MOVE_CLEANUP_VECTOR
);
527 free_cpumask_var(cleanup_mask
);
529 cfg
->move_in_progress
= 0;
532 asmlinkage __visible
void smp_irq_move_cleanup_interrupt(void)
540 me
= smp_processor_id();
541 for (vector
= FIRST_EXTERNAL_VECTOR
; vector
< NR_VECTORS
; vector
++) {
544 struct irq_desc
*desc
;
547 irq
= __this_cpu_read(vector_irq
[vector
]);
549 if (irq
<= VECTOR_UNDEFINED
)
552 desc
= irq_to_desc(irq
);
560 raw_spin_lock(&desc
->lock
);
563 * Check if the irq migration is in progress. If so, we
564 * haven't received the cleanup request yet for this irq.
566 if (cfg
->move_in_progress
)
569 if (vector
== cfg
->vector
&& cpumask_test_cpu(me
, cfg
->domain
))
572 irr
= apic_read(APIC_IRR
+ (vector
/ 32 * 0x10));
574 * Check if the vector that needs to be cleanedup is
575 * registered at the cpu's IRR. If so, then this is not
576 * the best time to clean it up. Lets clean it up in the
577 * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR
580 if (irr
& (1 << (vector
% 32))) {
581 apic
->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR
);
584 __this_cpu_write(vector_irq
[vector
], VECTOR_UNDEFINED
);
586 raw_spin_unlock(&desc
->lock
);
592 static void __irq_complete_move(struct irq_cfg
*cfg
, unsigned vector
)
596 if (likely(!cfg
->move_in_progress
))
599 me
= smp_processor_id();
601 if (vector
== cfg
->vector
&& cpumask_test_cpu(me
, cfg
->domain
))
602 send_cleanup_vector(cfg
);
605 void irq_complete_move(struct irq_cfg
*cfg
)
607 __irq_complete_move(cfg
, ~get_irq_regs()->orig_ax
);
610 void irq_force_complete_move(int irq
)
612 struct irq_cfg
*cfg
= irq_cfg(irq
);
617 __irq_complete_move(cfg
, cfg
->vector
);
622 * Dynamic irq allocate and deallocation. Should be replaced by irq domains!
624 int arch_setup_hwirq(unsigned int irq
, int node
)
630 cfg
= alloc_irq_cfg(node
);
634 raw_spin_lock_irqsave(&vector_lock
, flags
);
635 ret
= __assign_irq_vector(irq
, cfg
, apic
->target_cpus());
636 raw_spin_unlock_irqrestore(&vector_lock
, flags
);
639 irq_set_chip_data(irq
, cfg
);
645 void arch_teardown_hwirq(unsigned int irq
)
647 struct irq_cfg
*cfg
= irq_cfg(irq
);
649 free_remapped_irq(irq
);
650 clear_irq_vector(irq
, cfg
);
651 irq_set_chip_data(irq
, NULL
);
655 static void __init
print_APIC_field(int base
)
661 for (i
= 0; i
< 8; i
++)
662 pr_cont("%08x", apic_read(base
+ i
*0x10));
667 static void __init
print_local_APIC(void *dummy
)
669 unsigned int i
, v
, ver
, maxlvt
;
672 pr_debug("printing local APIC contents on CPU#%d/%d:\n",
673 smp_processor_id(), hard_smp_processor_id());
674 v
= apic_read(APIC_ID
);
675 pr_info("... APIC ID: %08x (%01x)\n", v
, read_apic_id());
676 v
= apic_read(APIC_LVR
);
677 pr_info("... APIC VERSION: %08x\n", v
);
678 ver
= GET_APIC_VERSION(v
);
679 maxlvt
= lapic_get_maxlvt();
681 v
= apic_read(APIC_TASKPRI
);
682 pr_debug("... APIC TASKPRI: %08x (%02x)\n", v
, v
& APIC_TPRI_MASK
);
685 if (APIC_INTEGRATED(ver
)) {
686 if (!APIC_XAPIC(ver
)) {
687 v
= apic_read(APIC_ARBPRI
);
688 pr_debug("... APIC ARBPRI: %08x (%02x)\n",
689 v
, v
& APIC_ARBPRI_MASK
);
691 v
= apic_read(APIC_PROCPRI
);
692 pr_debug("... APIC PROCPRI: %08x\n", v
);
696 * Remote read supported only in the 82489DX and local APIC for
697 * Pentium processors.
699 if (!APIC_INTEGRATED(ver
) || maxlvt
== 3) {
700 v
= apic_read(APIC_RRR
);
701 pr_debug("... APIC RRR: %08x\n", v
);
704 v
= apic_read(APIC_LDR
);
705 pr_debug("... APIC LDR: %08x\n", v
);
706 if (!x2apic_enabled()) {
707 v
= apic_read(APIC_DFR
);
708 pr_debug("... APIC DFR: %08x\n", v
);
710 v
= apic_read(APIC_SPIV
);
711 pr_debug("... APIC SPIV: %08x\n", v
);
713 pr_debug("... APIC ISR field:\n");
714 print_APIC_field(APIC_ISR
);
715 pr_debug("... APIC TMR field:\n");
716 print_APIC_field(APIC_TMR
);
717 pr_debug("... APIC IRR field:\n");
718 print_APIC_field(APIC_IRR
);
721 if (APIC_INTEGRATED(ver
)) {
722 /* Due to the Pentium erratum 3AP. */
724 apic_write(APIC_ESR
, 0);
726 v
= apic_read(APIC_ESR
);
727 pr_debug("... APIC ESR: %08x\n", v
);
730 icr
= apic_icr_read();
731 pr_debug("... APIC ICR: %08x\n", (u32
)icr
);
732 pr_debug("... APIC ICR2: %08x\n", (u32
)(icr
>> 32));
734 v
= apic_read(APIC_LVTT
);
735 pr_debug("... APIC LVTT: %08x\n", v
);
739 v
= apic_read(APIC_LVTPC
);
740 pr_debug("... APIC LVTPC: %08x\n", v
);
742 v
= apic_read(APIC_LVT0
);
743 pr_debug("... APIC LVT0: %08x\n", v
);
744 v
= apic_read(APIC_LVT1
);
745 pr_debug("... APIC LVT1: %08x\n", v
);
749 v
= apic_read(APIC_LVTERR
);
750 pr_debug("... APIC LVTERR: %08x\n", v
);
753 v
= apic_read(APIC_TMICT
);
754 pr_debug("... APIC TMICT: %08x\n", v
);
755 v
= apic_read(APIC_TMCCT
);
756 pr_debug("... APIC TMCCT: %08x\n", v
);
757 v
= apic_read(APIC_TDCR
);
758 pr_debug("... APIC TDCR: %08x\n", v
);
760 if (boot_cpu_has(X86_FEATURE_EXTAPIC
)) {
761 v
= apic_read(APIC_EFEAT
);
762 maxlvt
= (v
>> 16) & 0xff;
763 pr_debug("... APIC EFEAT: %08x\n", v
);
764 v
= apic_read(APIC_ECTRL
);
765 pr_debug("... APIC ECTRL: %08x\n", v
);
766 for (i
= 0; i
< maxlvt
; i
++) {
767 v
= apic_read(APIC_EILVTn(i
));
768 pr_debug("... APIC EILVT%d: %08x\n", i
, v
);
774 static void __init
print_local_APICs(int maxcpu
)
782 for_each_online_cpu(cpu
) {
785 smp_call_function_single(cpu
, print_local_APIC
, NULL
, 1);
790 static void __init
print_PIC(void)
795 if (!nr_legacy_irqs())
798 pr_debug("\nprinting PIC contents\n");
800 raw_spin_lock_irqsave(&i8259A_lock
, flags
);
802 v
= inb(0xa1) << 8 | inb(0x21);
803 pr_debug("... PIC IMR: %04x\n", v
);
805 v
= inb(0xa0) << 8 | inb(0x20);
806 pr_debug("... PIC IRR: %04x\n", v
);
810 v
= inb(0xa0) << 8 | inb(0x20);
814 raw_spin_unlock_irqrestore(&i8259A_lock
, flags
);
816 pr_debug("... PIC ISR: %04x\n", v
);
818 v
= inb(0x4d1) << 8 | inb(0x4d0);
819 pr_debug("... PIC ELCR: %04x\n", v
);
822 static int show_lapic __initdata
= 1;
823 static __init
int setup_show_lapic(char *arg
)
827 if (strcmp(arg
, "all") == 0) {
828 show_lapic
= CONFIG_NR_CPUS
;
830 get_option(&arg
, &num
);
837 __setup("show_lapic=", setup_show_lapic
);
839 static int __init
print_ICs(void)
841 if (apic_verbosity
== APIC_QUIET
)
846 /* don't print out if apic is not there */
847 if (!cpu_has_apic
&& !apic_from_smp_config())
850 print_local_APICs(show_lapic
);
856 late_initcall(print_ICs
);