[deliverable/linux.git] / arch / x86 / kernel / smp.c

/*
 *	Intel SMP support routines.
 *
 *	(c) 1995 Alan Cox, Building #3 <alan@redhat.com>
 *	(c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
 *      (c) 2002,2003 Andi Kleen, SuSE Labs.
 *
 *	i386 and x86_64 integration by Glauber Costa <gcosta@redhat.com>
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 */

#include <linux/init.h>

#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
#include <linux/kernel_stat.h>
#include <linux/mc146818rtc.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
#include <linux/cpu.h>

#include <asm/mtrr.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/proto.h>
#ifdef CONFIG_X86_32
#include <mach_apic.h>
#include <mach_ipi.h>
#else
#include <asm/mach_apic.h>
#endif
/*
 *	Some notes on x86 processor bugs affecting SMP operation:
 *
 *	Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
 *	The Linux implications for SMP are handled as follows:
 *
 *	Pentium III / [Xeon]
 *		None of the E1AP-E3AP errata are visible to the user.
 *
 *	E1AP.	see PII A1AP
 *	E2AP.	see PII A2AP
 *	E3AP.	see PII A3AP
 *
 *	Pentium II / [Xeon]
 *		None of the A1AP-A3AP errata are visible to the user.
 *
 *	A1AP.	see PPro 1AP
 *	A2AP.	see PPro 2AP
 *	A3AP.	see PPro 7AP
 *
 *	Pentium Pro
 *		None of 1AP-9AP errata are visible to the normal user,
 *	except occasional delivery of 'spurious interrupt' as trap #15.
 *	This is very rare and a non-problem.
 *
 *	1AP.	Linux maps APIC as non-cacheable
 *	2AP.	worked around in hardware
 *	3AP.	fixed in C0 and above steppings microcode update.
 *		Linux does not use excessive STARTUP_IPIs.
 *	4AP.	worked around in hardware
 *	5AP.	symmetric IO mode (normal Linux operation) not affected.
 *		'noapic' mode has vector 0xf filled out properly.
 *	6AP.	'noapic' mode might be affected - fixed in later steppings
 *	7AP.	We do not assume writes to the LVT deassering IRQs
 *	8AP.	We do not enable low power mode (deep sleep) during MP bootup
 *	9AP.	We do not use mixed mode
 *
 *	Pentium
 *		There is a marginal case where REP MOVS on 100MHz SMP
 *	machines with B stepping processors can fail. XXX should provide
 *	an L1cache=Writethrough or L1cache=off option.
 *
 *		B stepping CPUs may hang. There are hardware work arounds
 *	for this. We warn about it in case your board doesn't have the work
 *	arounds. Basically that's so I can tell anyone with a B stepping
 *	CPU and SMP problems "tough".
 *
 *	Specific items [From Pentium Processor Specification Update]
 *
 *	1AP.	Linux doesn't use remote read
 *	2AP.	Linux doesn't trust APIC errors
 *	3AP.	We work around this
 *	4AP.	Linux never generated 3 interrupts of the same priority
 *		to cause a lost local interrupt.
 *	5AP.	Remote read is never used
 *	6AP.	not affected - worked around in hardware
 *	7AP.	not affected - worked around in hardware
 *	8AP.	worked around in hardware - we get explicit CS errors if not
 *	9AP.	only 'noapic' mode affected. Might generate spurious
 *		interrupts, we log only the first one and count the
 *		rest silently.
 *	10AP.	not affected - worked around in hardware
 *	11AP.	Linux reads the APIC between writes to avoid this, as per
 *		the documentation. Make sure you preserve this as it affects
 *		the C stepping chips too.
 *	12AP.	not affected - worked around in hardware
 *	13AP.	not affected - worked around in hardware
 *	14AP.	we always deassert INIT during bootup
 *	15AP.	not affected - worked around in hardware
 *	16AP.	not affected - worked around in hardware
 *	17AP.	not affected - worked around in hardware
 *	18AP.	not affected - worked around in hardware
 *	19AP.	not affected - worked around in BIOS
 *
 *	If this sounds worrying believe me these bugs are either ___RARE___,
 *	or are signal timing bugs worked around in hardware and there's
 *	about nothing of note with C stepping upwards.
 */

/*
 * this function sends a 'reschedule' IPI to another CPU.
 * it goes straight through and wastes no time serializing
 * anything. Worst case is that we lose a reschedule ...
 */
static void native_smp_send_reschedule(int cpu)
{
	WARN_ON(cpu_is_offline(cpu));
	send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
}

/*
 * Structure and data for smp_call_function(). This is designed to minimise
 * static memory requirements. It also looks cleaner.
 */
static DEFINE_SPINLOCK(call_lock);

struct call_data_struct {
	void (*func) (void *info);
	void *info;
	atomic_t started;
	atomic_t finished;
	int wait;
};

void lock_ipi_call_lock(void)
{
	spin_lock_irq(&call_lock);
}

void unlock_ipi_call_lock(void)
{
	spin_unlock_irq(&call_lock);
}

static struct call_data_struct *call_data;

static void __smp_call_function(void (*func) (void *info), void *info,
				int nonatomic, int wait)
{
	struct call_data_struct data;
	int cpus = num_online_cpus() - 1;

	if (!cpus)
		return;

	data.func = func;
	data.info = info;
	atomic_set(&data.started, 0);
	data.wait = wait;
	if (wait)
		atomic_set(&data.finished, 0);

	call_data = &data;
	mb();

	/* Send a message to all other CPUs and wait for them to respond */
	send_IPI_allbutself(CALL_FUNCTION_VECTOR);

	/* Wait for response */
	while (atomic_read(&data.started) != cpus)
		cpu_relax();

	if (wait)
		while (atomic_read(&data.finished) != cpus)
			cpu_relax();
}


/**
 * smp_call_function_mask(): Run a function on a set of other CPUs.
 * @mask: The set of cpus to run on.  Must not include the current cpu.
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait (atomically) until function has completed on other CPUs.
 *
  * Returns 0 on success, else a negative status code.
 *
 * If @wait is true, then returns once @func has returned; otherwise
 * it returns just before the target cpu calls @func.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler.
 */
static int
native_smp_call_function_mask(cpumask_t mask,
			      void (*func)(void *), void *info,
			      int wait)
{
	struct call_data_struct data;
	cpumask_t allbutself;
	int cpus;

	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	/* Holding any lock stops cpus from going down. */
	spin_lock(&call_lock);

	allbutself = cpu_online_map;
	cpu_clear(smp_processor_id(), allbutself);

	cpus_and(mask, mask, allbutself);
	cpus = cpus_weight(mask);

	if (!cpus) {
		spin_unlock(&call_lock);
		return 0;
	}

	data.func = func;
	data.info = info;
	atomic_set(&data.started, 0);
	data.wait = wait;
	if (wait)
		atomic_set(&data.finished, 0);

	call_data = &data;
	wmb();

	/* Send a message to other CPUs */
	if (cpus_equal(mask, allbutself))
		send_IPI_allbutself(CALL_FUNCTION_VECTOR);
	else
		send_IPI_mask(mask, CALL_FUNCTION_VECTOR);

	/* Wait for response */
	while (atomic_read(&data.started) != cpus)
		cpu_relax();

	if (wait)
		while (atomic_read(&data.finished) != cpus)
			cpu_relax();
	spin_unlock(&call_lock);

	return 0;
}

static void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	/*
	 * Remove this CPU:
	 */
	cpu_clear(smp_processor_id(), cpu_online_map);
	disable_local_APIC();
	if (hlt_works(smp_processor_id()))
		for (;;) halt();
	for (;;);
}

/*
 * this function calls the 'stop' function on all other CPUs in the system.
 */

static void native_smp_send_stop(void)
{
	int nolock;
	unsigned long flags;

	if (reboot_force)
		return;

	/* Don't deadlock on the call lock in panic */
	nolock = !spin_trylock(&call_lock);
	local_irq_save(flags);
	__smp_call_function(stop_this_cpu, NULL, 0, 0);
	if (!nolock)
		spin_unlock(&call_lock);
	disable_local_APIC();
	local_irq_restore(flags);
}

/*
 * Reschedule call back. Nothing to do,
 * all the work is done automatically when
 * we return from the interrupt.
 */
void smp_reschedule_interrupt(struct pt_regs *regs)
{
	ack_APIC_irq();
#ifdef CONFIG_X86_32
	__get_cpu_var(irq_stat).irq_resched_count++;
#else
	add_pda(irq_resched_count, 1);
#endif
}

void smp_call_function_interrupt(struct pt_regs *regs)
{
	void (*func) (void *info) = call_data->func;
	void *info = call_data->info;
	int wait = call_data->wait;

	ack_APIC_irq();
	/*
	 * Notify initiating CPU that I've grabbed the data and am
	 * about to execute the function
	 */
	mb();
	atomic_inc(&call_data->started);
	/*
	 * At this point the info structure may be out of scope unless wait==1
	 */
	irq_enter();
	(*func)(info);
#ifdef CONFIG_X86_32
	__get_cpu_var(irq_stat).irq_call_count++;
#else
	add_pda(irq_call_count, 1);
#endif
	irq_exit();

	if (wait) {
		mb();
		atomic_inc(&call_data->finished);
	}
}

struct smp_ops smp_ops = {
	.smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
	.smp_prepare_cpus = native_smp_prepare_cpus,
	.cpu_up = native_cpu_up,
	.smp_cpus_done = native_smp_cpus_done,

	.smp_send_stop = native_smp_send_stop,
	.smp_send_reschedule = native_smp_send_reschedule,
	.smp_call_function_mask = native_smp_call_function_mask,
};
EXPORT_SYMBOL_GPL(smp_ops);
Commit	Line	Data
0941ecb5 GC	1	/*
	2	* Intel SMP support routines.
	3	*
	4	* (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
	5	* (c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
	6	* (c) 2002,2003 Andi Kleen, SuSE Labs.
	7	*
	8	* i386 and x86_64 integration by Glauber Costa <gcosta@redhat.com>
	9	*
	10	* This code is released under the GNU General Public License version 2 or
	11	* later.
	12	*/
	13
f9e47a12 GC	14	#include <linux/init.h>
	15
	16	#include <linux/mm.h>
	17	#include <linux/delay.h>
	18	#include <linux/spinlock.h>
	19	#include <linux/kernel_stat.h>
	20	#include <linux/mc146818rtc.h>
	21	#include <linux/cache.h>
	22	#include <linux/interrupt.h>
	23	#include <linux/cpu.h>
	24
	25	#include <asm/mtrr.h>
	26	#include <asm/tlbflush.h>
	27	#include <asm/mmu_context.h>
	28	#include <asm/proto.h>
	29	#ifdef CONFIG_X86_32
	30	#include <mach_apic.h>
	31	#include <mach_ipi.h>
	32	#else
	33	#include <asm/mach_apic.h>
	34	#endif
0941ecb5 GC	35	/*
	36	* Some notes on x86 processor bugs affecting SMP operation:
	37	*
	38	* Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
	39	* The Linux implications for SMP are handled as follows:
	40	*
	41	* Pentium III / [Xeon]
	42	* None of the E1AP-E3AP errata are visible to the user.
	43	*
	44	* E1AP. see PII A1AP
	45	* E2AP. see PII A2AP
	46	* E3AP. see PII A3AP
	47	*
	48	* Pentium II / [Xeon]
	49	* None of the A1AP-A3AP errata are visible to the user.
	50	*
	51	* A1AP. see PPro 1AP
	52	* A2AP. see PPro 2AP
	53	* A3AP. see PPro 7AP
	54	*
	55	* Pentium Pro
	56	* None of 1AP-9AP errata are visible to the normal user,
	57	* except occasional delivery of 'spurious interrupt' as trap #15.
	58	* This is very rare and a non-problem.
	59	*
	60	* 1AP. Linux maps APIC as non-cacheable
	61	* 2AP. worked around in hardware
	62	* 3AP. fixed in C0 and above steppings microcode update.
	63	* Linux does not use excessive STARTUP_IPIs.
	64	* 4AP. worked around in hardware
	65	* 5AP. symmetric IO mode (normal Linux operation) not affected.
	66	* 'noapic' mode has vector 0xf filled out properly.
	67	* 6AP. 'noapic' mode might be affected - fixed in later steppings
	68	* 7AP. We do not assume writes to the LVT deassering IRQs
	69	* 8AP. We do not enable low power mode (deep sleep) during MP bootup
	70	* 9AP. We do not use mixed mode
	71	*
	72	* Pentium
	73	* There is a marginal case where REP MOVS on 100MHz SMP
	74	* machines with B stepping processors can fail. XXX should provide
	75	* an L1cache=Writethrough or L1cache=off option.
	76	*
	77	* B stepping CPUs may hang. There are hardware work arounds
	78	* for this. We warn about it in case your board doesn't have the work
	79	* arounds. Basically that's so I can tell anyone with a B stepping
	80	* CPU and SMP problems "tough".
	81	*
	82	* Specific items [From Pentium Processor Specification Update]
	83	*
	84	* 1AP. Linux doesn't use remote read
	85	* 2AP. Linux doesn't trust APIC errors
	86	* 3AP. We work around this
	87	* 4AP. Linux never generated 3 interrupts of the same priority
	88	* to cause a lost local interrupt.
	89	* 5AP. Remote read is never used
	90	* 6AP. not affected - worked around in hardware
	91	* 7AP. not affected - worked around in hardware
	92	* 8AP. worked around in hardware - we get explicit CS errors if not
	93	* 9AP. only 'noapic' mode affected. Might generate spurious
	94	* interrupts, we log only the first one and count the
	95	* rest silently.
	96	* 10AP. not affected - worked around in hardware
	97	* 11AP. Linux reads the APIC between writes to avoid this, as per
	98	* the documentation. Make sure you preserve this as it affects
99	* the C stepping chips too.
100	* 12AP. not affected - worked around in hardware
101	* 13AP. not affected - worked around in hardware
102	* 14AP. we always deassert INIT during bootup
103	* 15AP. not affected - worked around in hardware
104	* 16AP. not affected - worked around in hardware
105	* 17AP. not affected - worked around in hardware
106	* 18AP. not affected - worked around in hardware
107	* 19AP. not affected - worked around in BIOS
108	*
109	* If this sounds worrying believe me these bugs are either ___RARE___,
110	* or are signal timing bugs worked around in hardware and there's
111	* about nothing of note with C stepping upwards.
112	*/
f9e47a12 GC	113
	114	/*
	115	* this function sends a 'reschedule' IPI to another CPU.
	116	* it goes straight through and wastes no time serializing
	117	* anything. Worst case is that we lose a reschedule ...
	118	*/
	119	static void native_smp_send_reschedule(int cpu)
	120	{
	121	WARN_ON(cpu_is_offline(cpu));
	122	send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
	123	}
	124
	125	/*
	126	* Structure and data for smp_call_function(). This is designed to minimise
	127	* static memory requirements. It also looks cleaner.
	128	*/
	129	static DEFINE_SPINLOCK(call_lock);
	130
	131	struct call_data_struct {
	132	void (func) (void info);
	133	void *info;
	134	atomic_t started;
	135	atomic_t finished;
	136	int wait;
	137	};
	138
	139	void lock_ipi_call_lock(void)
	140	{
	141	spin_lock_irq(&call_lock);
	142	}
	143
	144	void unlock_ipi_call_lock(void)
	145	{
	146	spin_unlock_irq(&call_lock);
	147	}
	148
	149	static struct call_data_struct *call_data;
	150
	151	static void __smp_call_function(void (func) (void info), void *info,
	152	int nonatomic, int wait)
	153	{
	154	struct call_data_struct data;
	155	int cpus = num_online_cpus() - 1;
	156
	157	if (!cpus)
	158	return;
	159
	160	data.func = func;
	161	data.info = info;
	162	atomic_set(&data.started, 0);
	163	data.wait = wait;
	164	if (wait)
	165	atomic_set(&data.finished, 0);
	166
	167	call_data = &data;
	168	mb();
	169
	170	/* Send a message to all other CPUs and wait for them to respond */
	171	send_IPI_allbutself(CALL_FUNCTION_VECTOR);
	172
	173	/* Wait for response */
	174	while (atomic_read(&data.started) != cpus)
	175	cpu_relax();
	176
177	if (wait)
178	while (atomic_read(&data.finished) != cpus)
179	cpu_relax();
180	}
181
182
183	/**
184	* smp_call_function_mask(): Run a function on a set of other CPUs.
185	* @mask: The set of cpus to run on. Must not include the current cpu.
186	* @func: The function to run. This must be fast and non-blocking.
187	* @info: An arbitrary pointer to pass to the function.
188	* @wait: If true, wait (atomically) until function has completed on other CPUs.
189	*
190	* Returns 0 on success, else a negative status code.
191	*
192	* If @wait is true, then returns once @func has returned; otherwise
193	* it returns just before the target cpu calls @func.
194	*
195	* You must not call this function with disabled interrupts or from a
196	* hardware interrupt handler or from a bottom half handler.
197	*/
198	static int
199	native_smp_call_function_mask(cpumask_t mask,
200	void (func)(void ), void *info,
201	int wait)
202	{
203	struct call_data_struct data;
204	cpumask_t allbutself;
205	int cpus;
206
207	/* Can deadlock when called with interrupts disabled */
208	WARN_ON(irqs_disabled());
209
210	/* Holding any lock stops cpus from going down. */
211	spin_lock(&call_lock);
212
213	allbutself = cpu_online_map;
214	cpu_clear(smp_processor_id(), allbutself);
215
216	cpus_and(mask, mask, allbutself);
217	cpus = cpus_weight(mask);
218
219	if (!cpus) {
220	spin_unlock(&call_lock);
221	return 0;
222	}
223
224	data.func = func;
225	data.info = info;
226	atomic_set(&data.started, 0);
227	data.wait = wait;
228	if (wait)
229	atomic_set(&data.finished, 0);
230
231	call_data = &data;
232	wmb();
233
234	/* Send a message to other CPUs */
235	if (cpus_equal(mask, allbutself))
236	send_IPI_allbutself(CALL_FUNCTION_VECTOR);
237	else
238	send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
239
240	/* Wait for response */
241	while (atomic_read(&data.started) != cpus)
242	cpu_relax();
243
244	if (wait)
245	while (atomic_read(&data.finished) != cpus)
246	cpu_relax();
247	spin_unlock(&call_lock);
248
249	return 0;
250	}
251
252	static void stop_this_cpu(void *dummy)
253	{
254	local_irq_disable();
255	/*
256	* Remove this CPU:
257	*/
258	cpu_clear(smp_processor_id(), cpu_online_map);
259	disable_local_APIC();
260	if (hlt_works(smp_processor_id()))
261	for (;;) halt();
262	for (;;);
263	}
264
265	/*
266	* this function calls the 'stop' function on all other CPUs in the system.
267	*/
268
269	static void native_smp_send_stop(void)
270	{
271	int nolock;
272	unsigned long flags;
273
274	if (reboot_force)
275	return;
276
277	/* Don't deadlock on the call lock in panic */
278	nolock = !spin_trylock(&call_lock);
279	local_irq_save(flags);
280	__smp_call_function(stop_this_cpu, NULL, 0, 0);
281	if (!nolock)
282	spin_unlock(&call_lock);
283	disable_local_APIC();
284	local_irq_restore(flags);
285	}
286
287	/*
288	* Reschedule call back. Nothing to do,
289	* all the work is done automatically when
290	* we return from the interrupt.
291	*/
292	void smp_reschedule_interrupt(struct pt_regs *regs)
293	{
294	ack_APIC_irq();
295	#ifdef CONFIG_X86_32
296	__get_cpu_var(irq_stat).irq_resched_count++;
297	#else
298	add_pda(irq_resched_count, 1);
299	#endif
300	}
301
302	void smp_call_function_interrupt(struct pt_regs *regs)
303	{
304	void (func) (void info) = call_data->func;
305	void *info = call_data->info;
306	int wait = call_data->wait;
307
308	ack_APIC_irq();
309	/*
310	* Notify initiating CPU that I've grabbed the data and am
311	* about to execute the function
312	*/
313	mb();
314	atomic_inc(&call_data->started);
315	/*
316	* At this point the info structure may be out of scope unless wait==1
317	*/
318	irq_enter();
319	(*func)(info);
320	#ifdef CONFIG_X86_32
321	__get_cpu_var(irq_stat).irq_call_count++;
322	#else
323	add_pda(irq_call_count, 1);
324	#endif
325	irq_exit();
326
327	if (wait) {
328	mb();
329	atomic_inc(&call_data->finished);
330	}
331	}
332
333	struct smp_ops smp_ops = {
334	.smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
335	.smp_prepare_cpus = native_smp_prepare_cpus,
336	.cpu_up = native_cpu_up,
337	.smp_cpus_done = native_smp_cpus_done,
338
339	.smp_send_stop = native_smp_send_stop,
340	.smp_send_reschedule = native_smp_send_reschedule,
341	.smp_call_function_mask = native_smp_call_function_mask,
342	};
343	EXPORT_SYMBOL_GPL(smp_ops);
344