[deliverable/linux.git] / arch / x86 / kernel / nmi.c

/*
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
 *  Copyright (C) 2011	Don Zickus Red Hat, Inc.
 *
 *  Pentium III FXSR, SSE support
 *	Gareth Hughes <gareth@valinux.com>, May 2000
 */

/*
 * Handle hardware traps and faults.
 */
#include <linux/spinlock.h>
#include <linux/kprobes.h>
#include <linux/kdebug.h>
#include <linux/nmi.h>
#include <linux/delay.h>
#include <linux/hardirq.h>
#include <linux/slab.h>
#include <linux/export.h>

#include <linux/mca.h>

#if defined(CONFIG_EDAC)
#include <linux/edac.h>
#endif

#include <linux/atomic.h>
#include <asm/traps.h>
#include <asm/mach_traps.h>
#include <asm/nmi.h>

#define NMI_MAX_NAMELEN	16
struct nmiaction {
	struct list_head list;
	nmi_handler_t handler;
	unsigned int flags;
	char *name;
};

struct nmi_desc {
	spinlock_t lock;
	struct list_head head;
};

static struct nmi_desc nmi_desc[NMI_MAX] = 
{
	{
		.lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
		.head = LIST_HEAD_INIT(nmi_desc[0].head),
	},
	{
		.lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
		.head = LIST_HEAD_INIT(nmi_desc[1].head),
	},

};

struct nmi_stats {
	unsigned int normal;
	unsigned int unknown;
	unsigned int external;
	unsigned int swallow;
};

static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);

static int ignore_nmis;

int unknown_nmi_panic;
/*
 * Prevent NMI reason port (0x61) being accessed simultaneously, can
 * only be used in NMI handler.
 */
static DEFINE_RAW_SPINLOCK(nmi_reason_lock);

static int __init setup_unknown_nmi_panic(char *str)
{
	unknown_nmi_panic = 1;
	return 1;
}
__setup("unknown_nmi_panic", setup_unknown_nmi_panic);

#define nmi_to_desc(type) (&nmi_desc[type])

static int notrace __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
{
	struct nmi_desc *desc = nmi_to_desc(type);
	struct nmiaction *a;
	int handled=0;

	rcu_read_lock();

	/*
	 * NMIs are edge-triggered, which means if you have enough
	 * of them concurrently, you can lose some because only one
	 * can be latched at any given time.  Walk the whole list
	 * to handle those situations.
	 */
	list_for_each_entry_rcu(a, &desc->head, list)
		handled += a->handler(type, regs);

	rcu_read_unlock();

	/* return total number of NMI events handled */
	return handled;
}

static int __setup_nmi(unsigned int type, struct nmiaction *action)
{
	struct nmi_desc *desc = nmi_to_desc(type);
	unsigned long flags;

	spin_lock_irqsave(&desc->lock, flags);

	/*
	 * most handlers of type NMI_UNKNOWN never return because
	 * they just assume the NMI is theirs.  Just a sanity check
	 * to manage expectations
	 */
	WARN_ON_ONCE(type == NMI_UNKNOWN && !list_empty(&desc->head));

	/*
	 * some handlers need to be executed first otherwise a fake
	 * event confuses some handlers (kdump uses this flag)
	 */
	if (action->flags & NMI_FLAG_FIRST)
		list_add_rcu(&action->list, &desc->head);
	else
		list_add_tail_rcu(&action->list, &desc->head);
	
	spin_unlock_irqrestore(&desc->lock, flags);
	return 0;
}

static struct nmiaction *__free_nmi(unsigned int type, const char *name)
{
	struct nmi_desc *desc = nmi_to_desc(type);
	struct nmiaction *n;
	unsigned long flags;

	spin_lock_irqsave(&desc->lock, flags);

	list_for_each_entry_rcu(n, &desc->head, list) {
		/*
		 * the name passed in to describe the nmi handler
		 * is used as the lookup key
		 */
		if (!strcmp(n->name, name)) {
			WARN(in_nmi(),
				"Trying to free NMI (%s) from NMI context!\n", n->name);
			list_del_rcu(&n->list);
			break;
		}
	}

	spin_unlock_irqrestore(&desc->lock, flags);
	synchronize_rcu();
	return (n);
}

int register_nmi_handler(unsigned int type, nmi_handler_t handler,
			unsigned long nmiflags, const char *devname)
{
	struct nmiaction *action;
	int retval = -ENOMEM;

	if (!handler)
		return -EINVAL;

	action = kzalloc(sizeof(struct nmiaction), GFP_KERNEL);
	if (!action)
		goto fail_action;

	action->handler = handler;
	action->flags = nmiflags;
	action->name = kstrndup(devname, NMI_MAX_NAMELEN, GFP_KERNEL);
	if (!action->name)
		goto fail_action_name;

	retval = __setup_nmi(type, action);

	if (retval)
		goto fail_setup_nmi;

	return retval;

fail_setup_nmi:
	kfree(action->name);
fail_action_name:
	kfree(action);
fail_action:	

	return retval;
}
EXPORT_SYMBOL_GPL(register_nmi_handler);

void unregister_nmi_handler(unsigned int type, const char *name)
{
	struct nmiaction *a;

	a = __free_nmi(type, name);
	if (a) {
		kfree(a->name);
		kfree(a);
	}
}

EXPORT_SYMBOL_GPL(unregister_nmi_handler);

static notrace __kprobes void
pci_serr_error(unsigned char reason, struct pt_regs *regs)
{
	pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
		 reason, smp_processor_id());

	/*
	 * On some machines, PCI SERR line is used to report memory
	 * errors. EDAC makes use of it.
	 */
#if defined(CONFIG_EDAC)
	if (edac_handler_set()) {
		edac_atomic_assert_error();
		return;
	}
#endif

	if (panic_on_unrecovered_nmi)
		panic("NMI: Not continuing");

	pr_emerg("Dazed and confused, but trying to continue\n");

	/* Clear and disable the PCI SERR error line. */
	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
	outb(reason, NMI_REASON_PORT);
}

static notrace __kprobes void
io_check_error(unsigned char reason, struct pt_regs *regs)
{
	unsigned long i;

	pr_emerg(
	"NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n",
		 reason, smp_processor_id());
	show_registers(regs);

	if (panic_on_io_nmi)
		panic("NMI IOCK error: Not continuing");

	/* Re-enable the IOCK line, wait for a few seconds */
	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
	outb(reason, NMI_REASON_PORT);

	i = 20000;
	while (--i) {
		touch_nmi_watchdog();
		udelay(100);
	}

	reason &= ~NMI_REASON_CLEAR_IOCHK;
	outb(reason, NMI_REASON_PORT);
}

static notrace __kprobes void
unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
{
	int handled;

	/*
	 * Use 'false' as back-to-back NMIs are dealt with one level up.
	 * Of course this makes having multiple 'unknown' handlers useless
	 * as only the first one is ever run (unless it can actually determine
	 * if it caused the NMI)
	 */
	handled = nmi_handle(NMI_UNKNOWN, regs, false);
	if (handled) {
		__this_cpu_add(nmi_stats.unknown, handled);
		return;
	}

	__this_cpu_add(nmi_stats.unknown, 1);

#ifdef CONFIG_MCA
	/*
	 * Might actually be able to figure out what the guilty party
	 * is:
	 */
	if (MCA_bus) {
		mca_handle_nmi();
		return;
	}
#endif
	pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
		 reason, smp_processor_id());

	pr_emerg("Do you have a strange power saving mode enabled?\n");
	if (unknown_nmi_panic || panic_on_unrecovered_nmi)
		panic("NMI: Not continuing");

	pr_emerg("Dazed and confused, but trying to continue\n");
}

static DEFINE_PER_CPU(bool, swallow_nmi);
static DEFINE_PER_CPU(unsigned long, last_nmi_rip);

static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
{
	unsigned char reason = 0;
	int handled;
	bool b2b = false;

	/*
	 * CPU-specific NMI must be processed before non-CPU-specific
	 * NMI, otherwise we may lose it, because the CPU-specific
	 * NMI can not be detected/processed on other CPUs.
	 */

	/*
	 * Back-to-back NMIs are interesting because they can either
	 * be two NMI or more than two NMIs (any thing over two is dropped
	 * due to NMI being edge-triggered).  If this is the second half
	 * of the back-to-back NMI, assume we dropped things and process
	 * more handlers.  Otherwise reset the 'swallow' NMI behaviour
	 */
	if (regs->ip == __this_cpu_read(last_nmi_rip))
		b2b = true;
	else
		__this_cpu_write(swallow_nmi, false);

	__this_cpu_write(last_nmi_rip, regs->ip);

	handled = nmi_handle(NMI_LOCAL, regs, b2b);
	__this_cpu_add(nmi_stats.normal, handled);
	if (handled) {
		/*
		 * There are cases when a NMI handler handles multiple
		 * events in the current NMI.  One of these events may
		 * be queued for in the next NMI.  Because the event is
		 * already handled, the next NMI will result in an unknown
		 * NMI.  Instead lets flag this for a potential NMI to
		 * swallow.
		 */
		if (handled > 1)
			__this_cpu_write(swallow_nmi, true);
		return;
	}

	/* Non-CPU-specific NMI: NMI sources can be processed on any CPU */
	raw_spin_lock(&nmi_reason_lock);
	reason = x86_platform.get_nmi_reason();

	if (reason & NMI_REASON_MASK) {
		if (reason & NMI_REASON_SERR)
			pci_serr_error(reason, regs);
		else if (reason & NMI_REASON_IOCHK)
			io_check_error(reason, regs);
#ifdef CONFIG_X86_32
		/*
		 * Reassert NMI in case it became active
		 * meanwhile as it's edge-triggered:
		 */
		reassert_nmi();
#endif
		__this_cpu_add(nmi_stats.external, 1);
		raw_spin_unlock(&nmi_reason_lock);
		return;
	}
	raw_spin_unlock(&nmi_reason_lock);

	/*
	 * Only one NMI can be latched at a time.  To handle
	 * this we may process multiple nmi handlers at once to
	 * cover the case where an NMI is dropped.  The downside
	 * to this approach is we may process an NMI prematurely,
	 * while its real NMI is sitting latched.  This will cause
	 * an unknown NMI on the next run of the NMI processing.
	 *
	 * We tried to flag that condition above, by setting the
	 * swallow_nmi flag when we process more than one event.
	 * This condition is also only present on the second half
	 * of a back-to-back NMI, so we flag that condition too.
	 *
	 * If both are true, we assume we already processed this
	 * NMI previously and we swallow it.  Otherwise we reset
	 * the logic.
	 *
	 * There are scenarios where we may accidentally swallow
	 * a 'real' unknown NMI.  For example, while processing
	 * a perf NMI another perf NMI comes in along with a
	 * 'real' unknown NMI.  These two NMIs get combined into
	 * one (as descibed above).  When the next NMI gets
	 * processed, it will be flagged by perf as handled, but
	 * noone will know that there was a 'real' unknown NMI sent
	 * also.  As a result it gets swallowed.  Or if the first
	 * perf NMI returns two events handled then the second
	 * NMI will get eaten by the logic below, again losing a
	 * 'real' unknown NMI.  But this is the best we can do
	 * for now.
	 */
	if (b2b && __this_cpu_read(swallow_nmi))
		__this_cpu_add(nmi_stats.swallow, 1);
	else
		unknown_nmi_error(reason, regs);
}

dotraplinkage notrace __kprobes void
do_nmi(struct pt_regs *regs, long error_code)
{
	nmi_enter();

	inc_irq_stat(__nmi_count);

	if (!ignore_nmis)
		default_do_nmi(regs);

	nmi_exit();
}

void stop_nmi(void)
{
	ignore_nmis++;
}

void restart_nmi(void)
{
	ignore_nmis--;
}

/* reset the back-to-back NMI logic */
void local_touch_nmi(void)
{
	__this_cpu_write(last_nmi_rip, 0);
}
Commit	Line	Data
1d48922c DZ	1	/*
	2	* Copyright (C) 1991, 1992 Linus Torvalds
	3	* Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
9c48f1c6	4	* Copyright (C) 2011 Don Zickus Red Hat, Inc.
1d48922c DZ	5	*
	6	* Pentium III FXSR, SSE support
	7	* Gareth Hughes <gareth@valinux.com>, May 2000
	8	*/
	9
	10	/*
	11	* Handle hardware traps and faults.
	12	*/
	13	#include <linux/spinlock.h>
	14	#include <linux/kprobes.h>
	15	#include <linux/kdebug.h>
	16	#include <linux/nmi.h>
c9126b2e DZ	17	#include <linux/delay.h>
	18	#include <linux/hardirq.h>
	19	#include <linux/slab.h>
69c60c88	20	#include <linux/export.h>
1d48922c	21
d48b0e17 IM	22	#include <linux/mca.h>
d48b0e17 IM	23
1d48922c DZ	24	#if defined(CONFIG_EDAC)
	25	#include <linux/edac.h>
	26	#endif
	27
	28	#include <linux/atomic.h>
	29	#include <asm/traps.h>
	30	#include <asm/mach_traps.h>
c9126b2e DZ	31	#include <asm/nmi.h>
	32
	33	#define NMI_MAX_NAMELEN 16
	34	struct nmiaction {
	35	struct list_head list;
	36	nmi_handler_t handler;
	37	unsigned int flags;
	38	char *name;
	39	};
	40
	41	struct nmi_desc {
	42	spinlock_t lock;
	43	struct list_head head;
	44	};
	45
	46	static struct nmi_desc nmi_desc[NMI_MAX] =
	47	{
	48	{
	49	.lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
	50	.head = LIST_HEAD_INIT(nmi_desc[0].head),
	51	},
	52	{
	53	.lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
	54	.head = LIST_HEAD_INIT(nmi_desc[1].head),
	55	},
	56
	57	};
1d48922c	58
efc3aac5 DZ	59	struct nmi_stats {
	60	unsigned int normal;
	61	unsigned int unknown;
	62	unsigned int external;
	63	unsigned int swallow;
	64	};
	65
	66	static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
	67
1d48922c DZ	68	static int ignore_nmis;
	69
	70	int unknown_nmi_panic;
	71	/*
	72	* Prevent NMI reason port (0x61) being accessed simultaneously, can
	73	* only be used in NMI handler.
	74	*/
	75	static DEFINE_RAW_SPINLOCK(nmi_reason_lock);
	76
	77	static int __init setup_unknown_nmi_panic(char *str)
	78	{
	79	unknown_nmi_panic = 1;
	80	return 1;
	81	}
	82	__setup("unknown_nmi_panic", setup_unknown_nmi_panic);
	83
c9126b2e DZ	84	#define nmi_to_desc(type) (&nmi_desc[type])
c9126b2e DZ	85
b227e233	86	static int notrace __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
c9126b2e DZ	87	{
	88	struct nmi_desc *desc = nmi_to_desc(type);
	89	struct nmiaction *a;
	90	int handled=0;
	91
	92	rcu_read_lock();
	93
	94	/*
	95	* NMIs are edge-triggered, which means if you have enough
	96	* of them concurrently, you can lose some because only one
	97	* can be latched at any given time. Walk the whole list
	98	* to handle those situations.
	99	*/
b227e233	100	list_for_each_entry_rcu(a, &desc->head, list)
c9126b2e DZ	101	handled += a->handler(type, regs);
c9126b2e DZ	102
c9126b2e DZ	103	rcu_read_unlock();
	104
	105	/* return total number of NMI events handled */
	106	return handled;
	107	}
	108
	109	static int __setup_nmi(unsigned int type, struct nmiaction *action)
	110	{
	111	struct nmi_desc *desc = nmi_to_desc(type);
	112	unsigned long flags;
	113
	114	spin_lock_irqsave(&desc->lock, flags);
	115
b227e233 DZ	116	/*
	117	* most handlers of type NMI_UNKNOWN never return because
	118	* they just assume the NMI is theirs. Just a sanity check
	119	* to manage expectations
	120	*/
	121	WARN_ON_ONCE(type == NMI_UNKNOWN && !list_empty(&desc->head));
	122
c9126b2e DZ	123	/*
	124	* some handlers need to be executed first otherwise a fake
	125	* event confuses some handlers (kdump uses this flag)
	126	*/
	127	if (action->flags & NMI_FLAG_FIRST)
	128	list_add_rcu(&action->list, &desc->head);
	129	else
	130	list_add_tail_rcu(&action->list, &desc->head);
	131
	132	spin_unlock_irqrestore(&desc->lock, flags);
	133	return 0;
	134	}
	135
	136	static struct nmiaction __free_nmi(unsigned int type, const char name)
	137	{
	138	struct nmi_desc *desc = nmi_to_desc(type);
	139	struct nmiaction *n;
	140	unsigned long flags;
	141
	142	spin_lock_irqsave(&desc->lock, flags);
	143
	144	list_for_each_entry_rcu(n, &desc->head, list) {
	145	/*
	146	* the name passed in to describe the nmi handler
	147	* is used as the lookup key
	148	*/
	149	if (!strcmp(n->name, name)) {
	150	WARN(in_nmi(),
	151	"Trying to free NMI (%s) from NMI context!\n", n->name);
	152	list_del_rcu(&n->list);
	153	break;
	154	}
	155	}
	156
	157	spin_unlock_irqrestore(&desc->lock, flags);
	158	synchronize_rcu();
	159	return (n);
	160	}
	161
	162	int register_nmi_handler(unsigned int type, nmi_handler_t handler,
	163	unsigned long nmiflags, const char *devname)
	164	{
	165	struct nmiaction *action;
	166	int retval = -ENOMEM;
	167
	168	if (!handler)
	169	return -EINVAL;
	170
	171	action = kzalloc(sizeof(struct nmiaction), GFP_KERNEL);
	172	if (!action)
	173	goto fail_action;
	174
	175	action->handler = handler;
	176	action->flags = nmiflags;
	177	action->name = kstrndup(devname, NMI_MAX_NAMELEN, GFP_KERNEL);
	178	if (!action->name)
	179	goto fail_action_name;
	180
	181	retval = __setup_nmi(type, action);
	182
	183	if (retval)
	184	goto fail_setup_nmi;
	185
	186	return retval;
187
188	fail_setup_nmi:
189	kfree(action->name);
190	fail_action_name:
191	kfree(action);
192	fail_action:
193
194	return retval;
195	}
196	EXPORT_SYMBOL_GPL(register_nmi_handler);
197
198	void unregister_nmi_handler(unsigned int type, const char *name)
199	{
200	struct nmiaction *a;
201
202	a = __free_nmi(type, name);
203	if (a) {
204	kfree(a->name);
205	kfree(a);
206	}
207	}
208
209	EXPORT_SYMBOL_GPL(unregister_nmi_handler);
210
1d48922c DZ	211	static notrace __kprobes void
	212	pci_serr_error(unsigned char reason, struct pt_regs *regs)
	213	{
	214	pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
	215	reason, smp_processor_id());
	216
	217	/*
	218	* On some machines, PCI SERR line is used to report memory
	219	* errors. EDAC makes use of it.
	220	*/
	221	#if defined(CONFIG_EDAC)
	222	if (edac_handler_set()) {
	223	edac_atomic_assert_error();
	224	return;
	225	}
	226	#endif
	227
	228	if (panic_on_unrecovered_nmi)
	229	panic("NMI: Not continuing");
	230
	231	pr_emerg("Dazed and confused, but trying to continue\n");
	232
	233	/* Clear and disable the PCI SERR error line. */
	234	reason = (reason & NMI_REASON_CLEAR_MASK) \| NMI_REASON_CLEAR_SERR;
	235	outb(reason, NMI_REASON_PORT);
	236	}
	237
	238	static notrace __kprobes void
	239	io_check_error(unsigned char reason, struct pt_regs *regs)
	240	{
	241	unsigned long i;
	242
	243	pr_emerg(
	244	"NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n",
	245	reason, smp_processor_id());
	246	show_registers(regs);
	247
	248	if (panic_on_io_nmi)
	249	panic("NMI IOCK error: Not continuing");
	250
	251	/* Re-enable the IOCK line, wait for a few seconds */
	252	reason = (reason & NMI_REASON_CLEAR_MASK) \| NMI_REASON_CLEAR_IOCHK;
	253	outb(reason, NMI_REASON_PORT);
	254
	255	i = 20000;
	256	while (--i) {
	257	touch_nmi_watchdog();
	258	udelay(100);
	259	}
	260
	261	reason &= ~NMI_REASON_CLEAR_IOCHK;
	262	outb(reason, NMI_REASON_PORT);
	263	}
	264
	265	static notrace __kprobes void
	266	unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
	267	{
9c48f1c6 DZ	268	int handled;
9c48f1c6 DZ	269
b227e233 DZ	270	/*
	271	* Use 'false' as back-to-back NMIs are dealt with one level up.
	272	* Of course this makes having multiple 'unknown' handlers useless
	273	* as only the first one is ever run (unless it can actually determine
	274	* if it caused the NMI)
	275	*/
	276	handled = nmi_handle(NMI_UNKNOWN, regs, false);
efc3aac5 DZ	277	if (handled) {
efc3aac5 DZ	278	__this_cpu_add(nmi_stats.unknown, handled);
1d48922c	279	return;
efc3aac5 DZ	280	}
	281
	282	__this_cpu_add(nmi_stats.unknown, 1);
	283
1d48922c DZ	284	#ifdef CONFIG_MCA
	285	/*
	286	* Might actually be able to figure out what the guilty party
	287	* is:
	288	*/
	289	if (MCA_bus) {
	290	mca_handle_nmi();
	291	return;
	292	}
	293	#endif
	294	pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
	295	reason, smp_processor_id());
	296
	297	pr_emerg("Do you have a strange power saving mode enabled?\n");
	298	if (unknown_nmi_panic \|\| panic_on_unrecovered_nmi)
	299	panic("NMI: Not continuing");
	300
	301	pr_emerg("Dazed and confused, but trying to continue\n");
	302	}
	303
b227e233 DZ	304	static DEFINE_PER_CPU(bool, swallow_nmi);
	305	static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
	306
1d48922c DZ	307	static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
	308	{
	309	unsigned char reason = 0;
9c48f1c6	310	int handled;
b227e233	311	bool b2b = false;
1d48922c DZ	312
	313	/*
	314	* CPU-specific NMI must be processed before non-CPU-specific
	315	* NMI, otherwise we may lose it, because the CPU-specific
	316	* NMI can not be detected/processed on other CPUs.
	317	*/
b227e233 DZ	318
	319	/*
	320	* Back-to-back NMIs are interesting because they can either
	321	* be two NMI or more than two NMIs (any thing over two is dropped
	322	* due to NMI being edge-triggered). If this is the second half
	323	* of the back-to-back NMI, assume we dropped things and process
	324	* more handlers. Otherwise reset the 'swallow' NMI behaviour
	325	*/
	326	if (regs->ip == __this_cpu_read(last_nmi_rip))
	327	b2b = true;
	328	else
	329	__this_cpu_write(swallow_nmi, false);
	330
	331	__this_cpu_write(last_nmi_rip, regs->ip);
	332
	333	handled = nmi_handle(NMI_LOCAL, regs, b2b);
efc3aac5	334	__this_cpu_add(nmi_stats.normal, handled);
b227e233 DZ	335	if (handled) {
	336	/*
	337	* There are cases when a NMI handler handles multiple
	338	* events in the current NMI. One of these events may
	339	* be queued for in the next NMI. Because the event is
	340	* already handled, the next NMI will result in an unknown
	341	* NMI. Instead lets flag this for a potential NMI to
	342	* swallow.
	343	*/
	344	if (handled > 1)
	345	__this_cpu_write(swallow_nmi, true);
1d48922c	346	return;
b227e233	347	}
1d48922c DZ	348
	349	/* Non-CPU-specific NMI: NMI sources can be processed on any CPU */
	350	raw_spin_lock(&nmi_reason_lock);
064a59b6	351	reason = x86_platform.get_nmi_reason();
1d48922c DZ	352
	353	if (reason & NMI_REASON_MASK) {
	354	if (reason & NMI_REASON_SERR)
	355	pci_serr_error(reason, regs);
	356	else if (reason & NMI_REASON_IOCHK)
	357	io_check_error(reason, regs);
	358	#ifdef CONFIG_X86_32
	359	/*
	360	* Reassert NMI in case it became active
	361	* meanwhile as it's edge-triggered:
	362	*/
	363	reassert_nmi();
	364	#endif
efc3aac5	365	__this_cpu_add(nmi_stats.external, 1);
1d48922c DZ	366	raw_spin_unlock(&nmi_reason_lock);
	367	return;
	368	}
	369	raw_spin_unlock(&nmi_reason_lock);
	370
b227e233 DZ	371	/*
	372	* Only one NMI can be latched at a time. To handle
	373	* this we may process multiple nmi handlers at once to
	374	* cover the case where an NMI is dropped. The downside
	375	* to this approach is we may process an NMI prematurely,
	376	* while its real NMI is sitting latched. This will cause
	377	* an unknown NMI on the next run of the NMI processing.
	378	*
	379	* We tried to flag that condition above, by setting the
	380	* swallow_nmi flag when we process more than one event.
	381	* This condition is also only present on the second half
	382	* of a back-to-back NMI, so we flag that condition too.
	383	*
	384	* If both are true, we assume we already processed this
	385	* NMI previously and we swallow it. Otherwise we reset
	386	* the logic.
	387	*
	388	* There are scenarios where we may accidentally swallow
	389	* a 'real' unknown NMI. For example, while processing
	390	* a perf NMI another perf NMI comes in along with a
	391	* 'real' unknown NMI. These two NMIs get combined into
	392	* one (as descibed above). When the next NMI gets
	393	* processed, it will be flagged by perf as handled, but
	394	* noone will know that there was a 'real' unknown NMI sent
	395	* also. As a result it gets swallowed. Or if the first
	396	* perf NMI returns two events handled then the second
	397	* NMI will get eaten by the logic below, again losing a
	398	* 'real' unknown NMI. But this is the best we can do
	399	* for now.
	400	*/
	401	if (b2b && __this_cpu_read(swallow_nmi))
efc3aac5	402	__this_cpu_add(nmi_stats.swallow, 1);
b227e233 DZ	403	else
b227e233 DZ	404	unknown_nmi_error(reason, regs);
1d48922c DZ	405	}
	406
	407	dotraplinkage notrace __kprobes void
	408	do_nmi(struct pt_regs *regs, long error_code)
	409	{
	410	nmi_enter();
	411
	412	inc_irq_stat(__nmi_count);
	413
	414	if (!ignore_nmis)
	415	default_do_nmi(regs);
	416
	417	nmi_exit();
	418	}
	419
	420	void stop_nmi(void)
	421	{
	422	ignore_nmis++;
	423	}
	424
	425	void restart_nmi(void)
	426	{
	427	ignore_nmis--;
	428	}
b227e233 DZ	429
	430	/* reset the back-to-back NMI logic */
	431	void local_touch_nmi(void)
	432	{
	433	__this_cpu_write(last_nmi_rip, 0);
	434	}