[deliverable/linux.git] / arch / x86 / kernel / irq.c

/*
 * Common interrupt code for 32 and 64 bit
 */
#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/of.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/ftrace.h>
#include <linux/delay.h>
#include <linux/export.h>

#include <asm/apic.h>
#include <asm/io_apic.h>
#include <asm/irq.h>
#include <asm/idle.h>
#include <asm/mce.h>
#include <asm/hw_irq.h>

#define CREATE_TRACE_POINTS
#include <asm/trace/irq_vectors.h>

atomic_t irq_err_count;

/* Function pointer for generic interrupt vector handling */
void (*x86_platform_ipi_callback)(void) = NULL;

/*
 * 'what should we do if we get a hw irq event on an illegal vector'.
 * each architecture has to answer this themselves.
 */
void ack_bad_irq(unsigned int irq)
{
	if (printk_ratelimit())
		pr_err("unexpected IRQ trap at vector %02x\n", irq);

	/*
	 * Currently unexpected vectors happen only on SMP and APIC.
	 * We _must_ ack these because every local APIC has only N
	 * irq slots per priority level, and a 'hanging, unacked' IRQ
	 * holds up an irq slot - in excessive cases (when multiple
	 * unexpected vectors occur) that might lock up the APIC
	 * completely.
	 * But only ack when the APIC is enabled -AK
	 */
	ack_APIC_irq();
}

#define irq_stats(x)		(&per_cpu(irq_stat, x))
/*
 * /proc/interrupts printing for arch specific interrupts
 */
int arch_show_interrupts(struct seq_file *p, int prec)
{
	int j;

	seq_printf(p, "%*s: ", prec, "NMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
	seq_printf(p, "  Non-maskable interrupts\n");
#ifdef CONFIG_X86_LOCAL_APIC
	seq_printf(p, "%*s: ", prec, "LOC");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
	seq_printf(p, "  Local timer interrupts\n");

	seq_printf(p, "%*s: ", prec, "SPU");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
	seq_printf(p, "  Spurious interrupts\n");
	seq_printf(p, "%*s: ", prec, "PMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
	seq_printf(p, "  Performance monitoring interrupts\n");
	seq_printf(p, "%*s: ", prec, "IWI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
	seq_printf(p, "  IRQ work interrupts\n");
	seq_printf(p, "%*s: ", prec, "RTR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
	seq_printf(p, "  APIC ICR read retries\n");
#endif
	if (x86_platform_ipi_callback) {
		seq_printf(p, "%*s: ", prec, "PLT");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
		seq_printf(p, "  Platform interrupts\n");
	}
#ifdef CONFIG_SMP
	seq_printf(p, "%*s: ", prec, "RES");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
	seq_printf(p, "  Rescheduling interrupts\n");
	seq_printf(p, "%*s: ", prec, "CAL");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_call_count -
					irq_stats(j)->irq_tlb_count);
	seq_printf(p, "  Function call interrupts\n");
	seq_printf(p, "%*s: ", prec, "TLB");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
	seq_printf(p, "  TLB shootdowns\n");
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	seq_printf(p, "%*s: ", prec, "TRM");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
	seq_printf(p, "  Thermal event interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	seq_printf(p, "%*s: ", prec, "THR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
	seq_printf(p, "  Threshold APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE
	seq_printf(p, "%*s: ", prec, "MCE");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
	seq_printf(p, "  Machine check exceptions\n");
	seq_printf(p, "%*s: ", prec, "MCP");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
	seq_printf(p, "  Machine check polls\n");
#endif
#if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN)
	seq_printf(p, "%*s: ", prec, "THR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_hv_callback_count);
	seq_printf(p, "  Hypervisor callback interrupts\n");
#endif
	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
#endif
	return 0;
}

/*
 * /proc/stat helpers
 */
u64 arch_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = irq_stats(cpu)->__nmi_count;

#ifdef CONFIG_X86_LOCAL_APIC
	sum += irq_stats(cpu)->apic_timer_irqs;
	sum += irq_stats(cpu)->irq_spurious_count;
	sum += irq_stats(cpu)->apic_perf_irqs;
	sum += irq_stats(cpu)->apic_irq_work_irqs;
	sum += irq_stats(cpu)->icr_read_retry_count;
#endif
	if (x86_platform_ipi_callback)
		sum += irq_stats(cpu)->x86_platform_ipis;
#ifdef CONFIG_SMP
	sum += irq_stats(cpu)->irq_resched_count;
	sum += irq_stats(cpu)->irq_call_count;
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	sum += irq_stats(cpu)->irq_thermal_count;
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	sum += irq_stats(cpu)->irq_threshold_count;
#endif
#ifdef CONFIG_X86_MCE
	sum += per_cpu(mce_exception_count, cpu);
	sum += per_cpu(mce_poll_count, cpu);
#endif
	return sum;
}

u64 arch_irq_stat(void)
{
	u64 sum = atomic_read(&irq_err_count);
	return sum;
}


/*
 * do_IRQ handles all normal device IRQ's (the special
 * SMP cross-CPU interrupts have their own specific
 * handlers).
 */
__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	/* high bit used in ret_from_ code  */
	unsigned vector = ~regs->orig_ax;
	unsigned irq;

	irq_enter();
	exit_idle();

	irq = __this_cpu_read(vector_irq[vector]);

	if (!handle_irq(irq, regs)) {
		ack_APIC_irq();

		if (irq != VECTOR_RETRIGGERED) {
			pr_emerg_ratelimited("%s: %d.%d No irq handler for vector (irq %d)\n",
					     __func__, smp_processor_id(),
					     vector, irq);
		} else {
			__this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED);
		}
	}

	irq_exit();

	set_irq_regs(old_regs);
	return 1;
}

/*
 * Handler for X86_PLATFORM_IPI_VECTOR.
 */
void __smp_x86_platform_ipi(void)
{
	inc_irq_stat(x86_platform_ipis);

	if (x86_platform_ipi_callback)
		x86_platform_ipi_callback();
}

__visible void smp_x86_platform_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	__smp_x86_platform_ipi();
	exiting_irq();
	set_irq_regs(old_regs);
}

#ifdef CONFIG_HAVE_KVM
/*
 * Handler for POSTED_INTERRUPT_VECTOR.
 */
__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	ack_APIC_irq();

	irq_enter();

	exit_idle();

	inc_irq_stat(kvm_posted_intr_ipis);

	irq_exit();

	set_irq_regs(old_regs);
}
#endif

__visible void smp_trace_x86_platform_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	__smp_x86_platform_ipi();
	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
	exiting_irq();
	set_irq_regs(old_regs);
}

EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);

#ifdef CONFIG_HOTPLUG_CPU

/* These two declarations are only used in check_irq_vectors_for_cpu_disable()
 * below, which is protected by stop_machine().  Putting them on the stack
 * results in a stack frame overflow.  Dynamically allocating could result in a
 * failure so declare these two cpumasks as global.
 */
static struct cpumask affinity_new, online_new;

/*
 * This cpu is going to be removed and its vectors migrated to the remaining
 * online cpus.  Check to see if there are enough vectors in the remaining cpus.
 * This function is protected by stop_machine().
 */
int check_irq_vectors_for_cpu_disable(void)
{
	int irq, cpu;
	unsigned int this_cpu, vector, this_count, count;
	struct irq_desc *desc;
	struct irq_data *data;

	this_cpu = smp_processor_id();
	cpumask_copy(&online_new, cpu_online_mask);
	cpu_clear(this_cpu, online_new);

	this_count = 0;
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		irq = __this_cpu_read(vector_irq[vector]);
		if (irq >= 0) {
			desc = irq_to_desc(irq);
			data = irq_desc_get_irq_data(desc);
			cpumask_copy(&affinity_new, data->affinity);
			cpu_clear(this_cpu, affinity_new);

			/* Do not count inactive or per-cpu irqs. */
			if (!irq_has_action(irq) || irqd_is_per_cpu(data))
				continue;

			/*
			 * A single irq may be mapped to multiple
			 * cpu's vector_irq[] (for example IOAPIC cluster
			 * mode).  In this case we have two
			 * possibilities:
			 *
			 * 1) the resulting affinity mask is empty; that is
			 * this the down'd cpu is the last cpu in the irq's
			 * affinity mask, or
			 *
			 * 2) the resulting affinity mask is no longer
			 * a subset of the online cpus but the affinity
			 * mask is not zero; that is the down'd cpu is the
			 * last online cpu in a user set affinity mask.
			 */
			if (cpumask_empty(&affinity_new) ||
			    !cpumask_subset(&affinity_new, &online_new))
				this_count++;
		}
	}

	count = 0;
	for_each_online_cpu(cpu) {
		if (cpu == this_cpu)
			continue;
		for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
		     vector++) {
			if (per_cpu(vector_irq, cpu)[vector] < 0)
				count++;
		}
	}

	if (count < this_count) {
		pr_warn("CPU %d disable failed: CPU has %u vectors assigned and there are only %u available.\n",
			this_cpu, this_count, count);
		return -ERANGE;
	}
	return 0;
}

/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
void fixup_irqs(void)
{
	unsigned int irq, vector;
	static int warned;
	struct irq_desc *desc;
	struct irq_data *data;
	struct irq_chip *chip;

	for_each_irq_desc(irq, desc) {
		int break_affinity = 0;
		int set_affinity = 1;
		const struct cpumask *affinity;

		if (!desc)
			continue;
		if (irq == 2)
			continue;

		/* interrupt's are disabled at this point */
		raw_spin_lock(&desc->lock);

		data = irq_desc_get_irq_data(desc);
		affinity = data->affinity;
		if (!irq_has_action(irq) || irqd_is_per_cpu(data) ||
		    cpumask_subset(affinity, cpu_online_mask)) {
			raw_spin_unlock(&desc->lock);
			continue;
		}

		/*
		 * Complete the irq move. This cpu is going down and for
		 * non intr-remapping case, we can't wait till this interrupt
		 * arrives at this cpu before completing the irq move.
		 */
		irq_force_complete_move(irq);

		if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
			break_affinity = 1;
			affinity = cpu_online_mask;
		}

		chip = irq_data_get_irq_chip(data);
		if (!irqd_can_move_in_process_context(data) && chip->irq_mask)
			chip->irq_mask(data);

		if (chip->irq_set_affinity)
			chip->irq_set_affinity(data, affinity, true);
		else if (!(warned++))
			set_affinity = 0;

		/*
		 * We unmask if the irq was not marked masked by the
		 * core code. That respects the lazy irq disable
		 * behaviour.
		 */
		if (!irqd_can_move_in_process_context(data) &&
		    !irqd_irq_masked(data) && chip->irq_unmask)
			chip->irq_unmask(data);

		raw_spin_unlock(&desc->lock);

		if (break_affinity && set_affinity)
			pr_notice("Broke affinity for irq %i\n", irq);
		else if (!set_affinity)
			pr_notice("Cannot set affinity for irq %i\n", irq);
	}

	/*
	 * We can remove mdelay() and then send spuriuous interrupts to
	 * new cpu targets for all the irqs that were handled previously by
	 * this cpu. While it works, I have seen spurious interrupt messages
	 * (nothing wrong but still...).
	 *
	 * So for now, retain mdelay(1) and check the IRR and then send those
	 * interrupts to new targets as this cpu is already offlined...
	 */
	mdelay(1);

	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		unsigned int irr;

		if (__this_cpu_read(vector_irq[vector]) <= VECTOR_UNDEFINED)
			continue;

		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr  & (1 << (vector % 32))) {
			irq = __this_cpu_read(vector_irq[vector]);

			desc = irq_to_desc(irq);
			data = irq_desc_get_irq_data(desc);
			chip = irq_data_get_irq_chip(data);
			raw_spin_lock(&desc->lock);
			if (chip->irq_retrigger) {
				chip->irq_retrigger(data);
				__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
			}
			raw_spin_unlock(&desc->lock);
		}
		if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
			__this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED);
	}
}
#endif
Commit	Line	Data
6b39ba77 TG	1	/*
	2	* Common interrupt code for 32 and 64 bit
	3	*/
	4	#include <linux/cpu.h>
	5	#include <linux/interrupt.h>
	6	#include <linux/kernel_stat.h>
4722d194	7	#include <linux/of.h>
6b39ba77	8	#include <linux/seq_file.h>
6a02e710	9	#include <linux/smp.h>
7c1d7cdc	10	#include <linux/ftrace.h>
ca444564	11	#include <linux/delay.h>
69c60c88	12	#include <linux/export.h>
6b39ba77	13
7b6aa335	14	#include <asm/apic.h>
6b39ba77	15	#include <asm/io_apic.h>
c3d80000	16	#include <asm/irq.h>
7c1d7cdc	17	#include <asm/idle.h>
01ca79f1	18	#include <asm/mce.h>
2c1b284e	19	#include <asm/hw_irq.h>
83ab8514 SRRH	20
83ab8514 SRRH	21	#define CREATE_TRACE_POINTS
cf910e83	22	#include <asm/trace/irq_vectors.h>
6b39ba77 TG	23
	24	atomic_t irq_err_count;
	25
acaabe79	26	/* Function pointer for generic interrupt vector handling */
4a4de9c7	27	void (*x86_platform_ipi_callback)(void) = NULL;
acaabe79	28
249f6d9e TG	29	/*
	30	* 'what should we do if we get a hw irq event on an illegal vector'.
	31	* each architecture has to answer this themselves.
	32	*/
	33	void ack_bad_irq(unsigned int irq)
	34	{
edea7148 CG	35	if (printk_ratelimit())
edea7148 CG	36	pr_err("unexpected IRQ trap at vector %02x\n", irq);
249f6d9e	37
249f6d9e TG	38	/*
	39	* Currently unexpected vectors happen only on SMP and APIC.
	40	* We _must_ ack these because every local APIC has only N
	41	* irq slots per priority level, and a 'hanging, unacked' IRQ
	42	* holds up an irq slot - in excessive cases (when multiple
	43	* unexpected vectors occur) that might lock up the APIC
	44	* completely.
	45	* But only ack when the APIC is enabled -AK
	46	*/
08306ce6	47	ack_APIC_irq();
249f6d9e TG	48	}
249f6d9e TG	49
1b437c8c	50	#define irq_stats(x) (&per_cpu(irq_stat, x))
6b39ba77	51	/*
517e4981	52	* /proc/interrupts printing for arch specific interrupts
6b39ba77	53	*/
517e4981	54	int arch_show_interrupts(struct seq_file *p, int prec)
6b39ba77 TG	55	{
	56	int j;
	57
7a81d9a7	58	seq_printf(p, "%*s: ", prec, "NMI");
6b39ba77 TG	59	for_each_online_cpu(j)
	60	seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
	61	seq_printf(p, " Non-maskable interrupts\n");
	62	#ifdef CONFIG_X86_LOCAL_APIC
7a81d9a7	63	seq_printf(p, "%*s: ", prec, "LOC");
6b39ba77 TG	64	for_each_online_cpu(j)
	65	seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
	66	seq_printf(p, " Local timer interrupts\n");
474e56b8 JSR	67
	68	seq_printf(p, "%*s: ", prec, "SPU");
	69	for_each_online_cpu(j)
	70	seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
	71	seq_printf(p, " Spurious interrupts\n");
89ccf465	72	seq_printf(p, "%*s: ", prec, "PMI");
241771ef IM	73	for_each_online_cpu(j)
241771ef IM	74	seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
89ccf465	75	seq_printf(p, " Performance monitoring interrupts\n");
e360adbe	76	seq_printf(p, "%*s: ", prec, "IWI");
b6276f35	77	for_each_online_cpu(j)
e360adbe PZ	78	seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
e360adbe PZ	79	seq_printf(p, " IRQ work interrupts\n");
346b46be FLVC	80	seq_printf(p, "%*s: ", prec, "RTR");
346b46be FLVC	81	for_each_online_cpu(j)
b49d7d87	82	seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
346b46be	83	seq_printf(p, " APIC ICR read retries\n");
6b39ba77	84	#endif
4a4de9c7	85	if (x86_platform_ipi_callback) {
59d13812	86	seq_printf(p, "%*s: ", prec, "PLT");
acaabe79	87	for_each_online_cpu(j)
4a4de9c7	88	seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
acaabe79 DS	89	seq_printf(p, " Platform interrupts\n");
acaabe79 DS	90	}
6b39ba77	91	#ifdef CONFIG_SMP
7a81d9a7	92	seq_printf(p, "%*s: ", prec, "RES");
6b39ba77 TG	93	for_each_online_cpu(j)
	94	seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
	95	seq_printf(p, " Rescheduling interrupts\n");
7a81d9a7	96	seq_printf(p, "%*s: ", prec, "CAL");
6b39ba77	97	for_each_online_cpu(j)
fd0f5869 TS	98	seq_printf(p, "%10u ", irq_stats(j)->irq_call_count -
fd0f5869 TS	99	irq_stats(j)->irq_tlb_count);
6b39ba77	100	seq_printf(p, " Function call interrupts\n");
7a81d9a7	101	seq_printf(p, "%*s: ", prec, "TLB");
6b39ba77 TG	102	for_each_online_cpu(j)
	103	seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
	104	seq_printf(p, " TLB shootdowns\n");
	105	#endif
0444c9bd	106	#ifdef CONFIG_X86_THERMAL_VECTOR
7a81d9a7	107	seq_printf(p, "%*s: ", prec, "TRM");
6b39ba77 TG	108	for_each_online_cpu(j)
	109	seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
	110	seq_printf(p, " Thermal event interrupts\n");
0444c9bd JB	111	#endif
0444c9bd JB	112	#ifdef CONFIG_X86_MCE_THRESHOLD
7a81d9a7	113	seq_printf(p, "%*s: ", prec, "THR");
6b39ba77 TG	114	for_each_online_cpu(j)
	115	seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
	116	seq_printf(p, " Threshold APIC interrupts\n");
01ca79f1	117	#endif
c1ebf835	118	#ifdef CONFIG_X86_MCE
01ca79f1 AK	119	seq_printf(p, "%*s: ", prec, "MCE");
	120	for_each_online_cpu(j)
	121	seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
	122	seq_printf(p, " Machine check exceptions\n");
ca84f696 AK	123	seq_printf(p, "%*s: ", prec, "MCP");
	124	for_each_online_cpu(j)
	125	seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
	126	seq_printf(p, " Machine check polls\n");
6b39ba77	127	#endif
f704a7d7	128	#if IS_ENABLED(CONFIG_HYPERV) \|\| defined(CONFIG_XEN)
929320e4 TG	129	seq_printf(p, "%*s: ", prec, "THR");
	130	for_each_online_cpu(j)
	131	seq_printf(p, "%10u ", irq_stats(j)->irq_hv_callback_count);
	132	seq_printf(p, " Hypervisor callback interrupts\n");
	133	#endif
7a81d9a7	134	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
6b39ba77	135	#if defined(CONFIG_X86_IO_APIC)
7a81d9a7	136	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
6b39ba77 TG	137	#endif
	138	return 0;
	139	}
	140
6b39ba77 TG	141	/*
	142	* /proc/stat helpers
	143	*/
	144	u64 arch_irq_stat_cpu(unsigned int cpu)
	145	{
	146	u64 sum = irq_stats(cpu)->__nmi_count;
	147
	148	#ifdef CONFIG_X86_LOCAL_APIC
	149	sum += irq_stats(cpu)->apic_timer_irqs;
474e56b8	150	sum += irq_stats(cpu)->irq_spurious_count;
241771ef	151	sum += irq_stats(cpu)->apic_perf_irqs;
e360adbe	152	sum += irq_stats(cpu)->apic_irq_work_irqs;
b49d7d87	153	sum += irq_stats(cpu)->icr_read_retry_count;
6b39ba77	154	#endif
4a4de9c7 DS	155	if (x86_platform_ipi_callback)
4a4de9c7 DS	156	sum += irq_stats(cpu)->x86_platform_ipis;
6b39ba77 TG	157	#ifdef CONFIG_SMP
	158	sum += irq_stats(cpu)->irq_resched_count;
	159	sum += irq_stats(cpu)->irq_call_count;
6b39ba77	160	#endif
0444c9bd	161	#ifdef CONFIG_X86_THERMAL_VECTOR
6b39ba77	162	sum += irq_stats(cpu)->irq_thermal_count;
0444c9bd JB	163	#endif
0444c9bd JB	164	#ifdef CONFIG_X86_MCE_THRESHOLD
6b39ba77	165	sum += irq_stats(cpu)->irq_threshold_count;
8051dbd2	166	#endif
c1ebf835	167	#ifdef CONFIG_X86_MCE
8051dbd2 HS	168	sum += per_cpu(mce_exception_count, cpu);
8051dbd2 HS	169	sum += per_cpu(mce_poll_count, cpu);
6b39ba77 TG	170	#endif
	171	return sum;
	172	}
	173
	174	u64 arch_irq_stat(void)
	175	{
	176	u64 sum = atomic_read(&irq_err_count);
6b39ba77 TG	177	return sum;
6b39ba77 TG	178	}
c3d80000	179
7c1d7cdc JF	180
	181	/*
	182	* do_IRQ handles all normal device IRQ's (the special
	183	* SMP cross-CPU interrupts have their own specific
	184	* handlers).
	185	*/
1d9090e2	186	__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
7c1d7cdc JF	187	{
	188	struct pt_regs *old_regs = set_irq_regs(regs);
	189
	190	/* high bit used in ret_from_ code */
	191	unsigned vector = ~regs->orig_ax;
	192	unsigned irq;
	193
7c1d7cdc	194	irq_enter();
98ad1cc1	195	exit_idle();
7c1d7cdc	196
0a3aee0d	197	irq = __this_cpu_read(vector_irq[vector]);
7c1d7cdc JF	198
7c1d7cdc JF	199	if (!handle_irq(irq, regs)) {
08306ce6	200	ack_APIC_irq();
7c1d7cdc	201
9345005f PB	202	if (irq != VECTOR_RETRIGGERED) {
	203	pr_emerg_ratelimited("%s: %d.%d No irq handler for vector (irq %d)\n",
	204	__func__, smp_processor_id(),
	205	vector, irq);
	206	} else {
	207	__this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED);
	208	}
7c1d7cdc JF	209	}
	210
	211	irq_exit();
	212
	213	set_irq_regs(old_regs);
	214	return 1;
	215	}
	216
acaabe79	217	/*
4a4de9c7	218	* Handler for X86_PLATFORM_IPI_VECTOR.
acaabe79	219	*/
eddc0e92	220	void __smp_x86_platform_ipi(void)
acaabe79	221	{
4a4de9c7	222	inc_irq_stat(x86_platform_ipis);
acaabe79	223
4a4de9c7 DS	224	if (x86_platform_ipi_callback)
4a4de9c7 DS	225	x86_platform_ipi_callback();
eddc0e92	226	}
acaabe79	227
1d9090e2	228	__visible void smp_x86_platform_ipi(struct pt_regs *regs)
eddc0e92 SA	229	{
eddc0e92 SA	230	struct pt_regs *old_regs = set_irq_regs(regs);
acaabe79	231
eddc0e92 SA	232	entering_ack_irq();
	233	__smp_x86_platform_ipi();
	234	exiting_irq();
acaabe79 DS	235	set_irq_regs(old_regs);
	236	}
	237
d78f2664 YZ	238	#ifdef CONFIG_HAVE_KVM
	239	/*
	240	* Handler for POSTED_INTERRUPT_VECTOR.
	241	*/
1d9090e2	242	__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
d78f2664 YZ	243	{
	244	struct pt_regs *old_regs = set_irq_regs(regs);
	245
	246	ack_APIC_irq();
	247
	248	irq_enter();
	249
	250	exit_idle();
	251
	252	inc_irq_stat(kvm_posted_intr_ipis);
	253
	254	irq_exit();
	255
	256	set_irq_regs(old_regs);
	257	}
	258	#endif
	259
1d9090e2	260	__visible void smp_trace_x86_platform_ipi(struct pt_regs *regs)
cf910e83 SA	261	{
	262	struct pt_regs *old_regs = set_irq_regs(regs);
	263
	264	entering_ack_irq();
	265	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	266	__smp_x86_platform_ipi();
	267	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
	268	exiting_irq();
	269	set_irq_regs(old_regs);
	270	}
	271
c3d80000	272	EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
7a7732bc SS	273
7a7732bc SS	274	#ifdef CONFIG_HOTPLUG_CPU
39424e89 PB	275
	276	/* These two declarations are only used in check_irq_vectors_for_cpu_disable()
	277	* below, which is protected by stop_machine(). Putting them on the stack
	278	* results in a stack frame overflow. Dynamically allocating could result in a
	279	* failure so declare these two cpumasks as global.
	280	*/
	281	static struct cpumask affinity_new, online_new;
	282
da6139e4 PB	283	/*
	284	* This cpu is going to be removed and its vectors migrated to the remaining
	285	* online cpus. Check to see if there are enough vectors in the remaining cpus.
	286	* This function is protected by stop_machine().
	287	*/
	288	int check_irq_vectors_for_cpu_disable(void)
	289	{
	290	int irq, cpu;
	291	unsigned int this_cpu, vector, this_count, count;
	292	struct irq_desc *desc;
	293	struct irq_data *data;
da6139e4 PB	294
	295	this_cpu = smp_processor_id();
	296	cpumask_copy(&online_new, cpu_online_mask);
	297	cpu_clear(this_cpu, online_new);
	298
	299	this_count = 0;
	300	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
	301	irq = __this_cpu_read(vector_irq[vector]);
	302	if (irq >= 0) {
	303	desc = irq_to_desc(irq);
	304	data = irq_desc_get_irq_data(desc);
	305	cpumask_copy(&affinity_new, data->affinity);
	306	cpu_clear(this_cpu, affinity_new);
	307
	308	/* Do not count inactive or per-cpu irqs. */
	309	if (!irq_has_action(irq) \|\| irqd_is_per_cpu(data))
	310	continue;
	311
	312	/*
	313	* A single irq may be mapped to multiple
	314	* cpu's vector_irq[] (for example IOAPIC cluster
	315	* mode). In this case we have two
	316	* possibilities:
	317	*
	318	* 1) the resulting affinity mask is empty; that is
	319	* this the down'd cpu is the last cpu in the irq's
	320	* affinity mask, or
	321	*
	322	* 2) the resulting affinity mask is no longer
	323	* a subset of the online cpus but the affinity
	324	* mask is not zero; that is the down'd cpu is the
	325	* last online cpu in a user set affinity mask.
	326	*/
	327	if (cpumask_empty(&affinity_new) \|\|
	328	!cpumask_subset(&affinity_new, &online_new))
	329	this_count++;
	330	}
	331	}
	332
	333	count = 0;
	334	for_each_online_cpu(cpu) {
	335	if (cpu == this_cpu)
	336	continue;
	337	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
	338	vector++) {
	339	if (per_cpu(vector_irq, cpu)[vector] < 0)
	340	count++;
	341	}
	342	}
	343
	344	if (count < this_count) {
	345	pr_warn("CPU %d disable failed: CPU has %u vectors assigned and there are only %u available.\n",
	346	this_cpu, this_count, count);
	347	return -ERANGE;
	348	}
	349	return 0;
	350	}
	351
7a7732bc SS	352	/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
	353	void fixup_irqs(void)
	354	{
5231a686	355	unsigned int irq, vector;
7a7732bc SS	356	static int warned;
7a7732bc SS	357	struct irq_desc *desc;
a3c08e5d	358	struct irq_data *data;
51c43ac6	359	struct irq_chip *chip;
7a7732bc SS	360
	361	for_each_irq_desc(irq, desc) {
	362	int break_affinity = 0;
	363	int set_affinity = 1;
	364	const struct cpumask *affinity;
	365
	366	if (!desc)
	367	continue;
	368	if (irq == 2)
	369	continue;
	370
	371	/* interrupt's are disabled at this point */
239007b8	372	raw_spin_lock(&desc->lock);
7a7732bc	373
51c43ac6	374	data = irq_desc_get_irq_data(desc);
a3c08e5d	375	affinity = data->affinity;
b87ba87c	376	if (!irq_has_action(irq) \|\| irqd_is_per_cpu(data) \|\|
58bff947	377	cpumask_subset(affinity, cpu_online_mask)) {
239007b8	378	raw_spin_unlock(&desc->lock);
7a7732bc SS	379	continue;
	380	}
	381
a5e74b84 SS	382	/*
	383	* Complete the irq move. This cpu is going down and for
	384	* non intr-remapping case, we can't wait till this interrupt
	385	* arrives at this cpu before completing the irq move.
	386	*/
	387	irq_force_complete_move(irq);
	388
7a7732bc SS	389	if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
7a7732bc SS	390	break_affinity = 1;
2530cd4f	391	affinity = cpu_online_mask;
7a7732bc SS	392	}
7a7732bc SS	393
51c43ac6 TG	394	chip = irq_data_get_irq_chip(data);
	395	if (!irqd_can_move_in_process_context(data) && chip->irq_mask)
	396	chip->irq_mask(data);
7a7732bc	397
51c43ac6 TG	398	if (chip->irq_set_affinity)
51c43ac6 TG	399	chip->irq_set_affinity(data, affinity, true);
7a7732bc SS	400	else if (!(warned++))
	401	set_affinity = 0;
	402
99dd5497 LC	403	/*
	404	* We unmask if the irq was not marked masked by the
	405	* core code. That respects the lazy irq disable
	406	* behaviour.
	407	*/
983bbf1a	408	if (!irqd_can_move_in_process_context(data) &&
99dd5497	409	!irqd_irq_masked(data) && chip->irq_unmask)
51c43ac6	410	chip->irq_unmask(data);
7a7732bc	411
239007b8	412	raw_spin_unlock(&desc->lock);
7a7732bc SS	413
7a7732bc SS	414	if (break_affinity && set_affinity)
c767a54b	415	pr_notice("Broke affinity for irq %i\n", irq);
7a7732bc	416	else if (!set_affinity)
c767a54b	417	pr_notice("Cannot set affinity for irq %i\n", irq);
7a7732bc SS	418	}
7a7732bc SS	419
5231a686 SS	420	/*
	421	* We can remove mdelay() and then send spuriuous interrupts to
	422	* new cpu targets for all the irqs that were handled previously by
	423	* this cpu. While it works, I have seen spurious interrupt messages
	424	* (nothing wrong but still...).
	425	*
	426	* So for now, retain mdelay(1) and check the IRR and then send those
	427	* interrupts to new targets as this cpu is already offlined...
	428	*/
7a7732bc	429	mdelay(1);
5231a686 SS	430
	431	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
	432	unsigned int irr;
	433
9345005f	434	if (__this_cpu_read(vector_irq[vector]) <= VECTOR_UNDEFINED)
5231a686 SS	435	continue;
	436
	437	irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
	438	if (irr & (1 << (vector % 32))) {
0a3aee0d	439	irq = __this_cpu_read(vector_irq[vector]);
5231a686	440
5117348d	441	desc = irq_to_desc(irq);
51c43ac6 TG	442	data = irq_desc_get_irq_data(desc);
51c43ac6 TG	443	chip = irq_data_get_irq_chip(data);
239007b8	444	raw_spin_lock(&desc->lock);
9345005f	445	if (chip->irq_retrigger) {
51c43ac6	446	chip->irq_retrigger(data);
9345005f PB	447	__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
9345005f PB	448	}
239007b8	449	raw_spin_unlock(&desc->lock);
5231a686	450	}
9345005f PB	451	if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
9345005f PB	452	__this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED);
5231a686	453	}
7a7732bc SS	454	}
7a7732bc SS	455	#endif