[deliverable/linux.git] / arch / mips / kernel / smp.c

/*
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 *
 * Copyright (C) 2000, 2001 Kanoj Sarcar
 * Copyright (C) 2000, 2001 Ralf Baechle
 * Copyright (C) 2000, 2001 Silicon Graphics, Inc.
 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation
 */
#include <linux/cache.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/threads.h>
#include <linux/module.h>
#include <linux/time.h>
#include <linux/timex.h>
#include <linux/sched.h>
#include <linux/cpumask.h>
#include <linux/cpu.h>

#include <asm/atomic.h>
#include <asm/cpu.h>
#include <asm/processor.h>
#include <asm/system.h>
#include <asm/mmu_context.h>
#include <asm/smp.h>

cpumask_t phys_cpu_present_map;		/* Bitmask of available CPUs */
volatile cpumask_t cpu_callin_map;	/* Bitmask of started secondaries */
cpumask_t cpu_online_map;		/* Bitmask of currently online CPUs */
int __cpu_number_map[NR_CPUS];		/* Map physical to logical */
int __cpu_logical_map[NR_CPUS];		/* Map logical to physical */

EXPORT_SYMBOL(phys_cpu_present_map);
EXPORT_SYMBOL(cpu_online_map);

static void smp_tune_scheduling (void)
{
	struct cache_desc *cd = &current_cpu_data.scache;
	unsigned long cachesize;       /* kB   */
	unsigned long cpu_khz;

	/*
	 * Crude estimate until we actually meassure ...
	 */
	cpu_khz = loops_per_jiffy * 2 * HZ / 1000;

	/*
	 * Rough estimation for SMP scheduling, this is the number of
	 * cycles it takes for a fully memory-limited process to flush
	 * the SMP-local cache.
	 *
	 * (For a P5 this pretty much means we will choose another idle
	 *  CPU almost always at wakeup time (this is due to the small
	 *  L1 cache), on PIIs it's around 50-100 usecs, depending on
	 *  the cache size)
	 */
	if (!cpu_khz)
		return;

	cachesize = cd->linesz * cd->sets * cd->ways;
}

extern void __init calibrate_delay(void);
extern ATTRIB_NORET void cpu_idle(void);

/*
 * First C code run on the secondary CPUs after being started up by
 * the master.
 */
asmlinkage void start_secondary(void)
{
	unsigned int cpu;

	cpu_probe();
	cpu_report();
	per_cpu_trap_init();
	prom_init_secondary();

	/*
	 * XXX parity protection should be folded in here when it's converted
	 * to an option instead of something based on .cputype
	 */

	calibrate_delay();
	preempt_disable();
	cpu = smp_processor_id();
	cpu_data[cpu].udelay_val = loops_per_jiffy;

	prom_smp_finish();

	cpu_set(cpu, cpu_callin_map);

	cpu_idle();
}

DEFINE_SPINLOCK(smp_call_lock);

struct call_data_struct *call_data;

/*
 * Run a function on all other CPUs.
 *  <func>      The function to run. This must be fast and non-blocking.
 *  <info>      An arbitrary pointer to pass to the function.
 *  <retry>     If true, keep retrying until ready.
 *  <wait>      If true, wait until function has completed on other CPUs.
 *  [RETURNS]   0 on success, else a negative status code.
 *
 * Does not return until remote CPUs are nearly ready to execute <func>
 * or are or have executed.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler:
 *
 * CPU A                               CPU B
 * Disable interrupts
 *                                     smp_call_function()
 *                                     Take call_lock
 *                                     Send IPIs
 *                                     Wait for all cpus to acknowledge IPI
 *                                     CPU A has not responded, spin waiting
 *                                     for cpu A to respond, holding call_lock
 * smp_call_function()
 * Spin waiting for call_lock
 * Deadlock                            Deadlock
 */
int smp_call_function (void (*func) (void *info), void *info, int retry,
								int wait)
{
	struct call_data_struct data;
	int i, cpus = num_online_cpus() - 1;
	int cpu = smp_processor_id();

	/*
	 * Can die spectacularly if this CPU isn't yet marked online
	 */
	BUG_ON(!cpu_online(cpu));

	if (!cpus)
		return 0;

	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	data.func = func;
	data.info = info;
	atomic_set(&data.started, 0);
	data.wait = wait;
	if (wait)
		atomic_set(&data.finished, 0);

	spin_lock(&smp_call_lock);
	call_data = &data;
	mb();

	/* Send a message to all other CPUs and wait for them to respond */
	for_each_online_cpu(i)
		if (i != cpu)
			core_send_ipi(i, SMP_CALL_FUNCTION);

	/* Wait for response */
	/* FIXME: lock-up detection, backtrace on lock-up */
	while (atomic_read(&data.started) != cpus)
		barrier();

	if (wait)
		while (atomic_read(&data.finished) != cpus)
			barrier();
	spin_unlock(&smp_call_lock);

	return 0;
}

void smp_call_function_interrupt(void)
{
	void (*func) (void *info) = call_data->func;
	void *info = call_data->info;
	int wait = call_data->wait;

	/*
	 * Notify initiating CPU that I've grabbed the data and am
	 * about to execute the function.
	 */
	mb();
	atomic_inc(&call_data->started);

	/*
	 * At this point the info structure may be out of scope unless wait==1.
	 */
	irq_enter();
	(*func)(info);
	irq_exit();

	if (wait) {
		mb();
		atomic_inc(&call_data->finished);
	}
}

static void stop_this_cpu(void *dummy)
{
	/*
	 * Remove this CPU:
	 */
	cpu_clear(smp_processor_id(), cpu_online_map);
	local_irq_enable();	/* May need to service _machine_restart IPI */
	for (;;);		/* Wait if available. */
}

void smp_send_stop(void)
{
	smp_call_function(stop_this_cpu, NULL, 1, 0);
}

void __init smp_cpus_done(unsigned int max_cpus)
{
	prom_cpus_done();
}

/* called from main before smp_init() */
void __init smp_prepare_cpus(unsigned int max_cpus)
{
	init_new_context(current, &init_mm);
	current_thread_info()->cpu = 0;
	smp_tune_scheduling();
	plat_prepare_cpus(max_cpus);
}

/* preload SMP state for boot cpu */
void __devinit smp_prepare_boot_cpu(void)
{
	/*
	 * This assumes that bootup is always handled by the processor
	 * with the logic and physical number 0.
	 */
	__cpu_number_map[0] = 0;
	__cpu_logical_map[0] = 0;
	cpu_set(0, phys_cpu_present_map);
	cpu_set(0, cpu_online_map);
	cpu_set(0, cpu_callin_map);
}

/*
 * Called once for each "cpu_possible(cpu)".  Needs to spin up the cpu
 * and keep control until "cpu_online(cpu)" is set.  Note: cpu is
 * physical, not logical.
 */
int __devinit __cpu_up(unsigned int cpu)
{
	struct task_struct *idle;

	/*
	 * Processor goes to start_secondary(), sets online flag
	 * The following code is purely to make sure
	 * Linux can schedule processes on this slave.
	 */
	idle = fork_idle(cpu);
	if (IS_ERR(idle))
		panic(KERN_ERR "Fork failed for CPU %d", cpu);

	prom_boot_secondary(cpu, idle);

	/*
	 * Trust is futile.  We should really have timeouts ...
	 */
	while (!cpu_isset(cpu, cpu_callin_map))
		udelay(100);

	cpu_set(cpu, cpu_online_map);

	return 0;
}

/* Not really SMP stuff ... */
int setup_profiling_timer(unsigned int multiplier)
{
	return 0;
}

static void flush_tlb_all_ipi(void *info)
{
	local_flush_tlb_all();
}

void flush_tlb_all(void)
{
	on_each_cpu(flush_tlb_all_ipi, 0, 1, 1);
}

static void flush_tlb_mm_ipi(void *mm)
{
	local_flush_tlb_mm((struct mm_struct *)mm);
}

/*
 * The following tlb flush calls are invoked when old translations are
 * being torn down, or pte attributes are changing. For single threaded
 * address spaces, a new context is obtained on the current cpu, and tlb
 * context on other cpus are invalidated to force a new context allocation
 * at switch_mm time, should the mm ever be used on other cpus. For
 * multithreaded address spaces, intercpu interrupts have to be sent.
 * Another case where intercpu interrupts are required is when the target
 * mm might be active on another cpu (eg debuggers doing the flushes on
 * behalf of debugees, kswapd stealing pages from another process etc).
 * Kanoj 07/00.
 */

void flush_tlb_mm(struct mm_struct *mm)
{
	preempt_disable();

	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
		smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1, 1);
	} else {
		int i;
		for (i = 0; i < num_online_cpus(); i++)
			if (smp_processor_id() != i)
				cpu_context(i, mm) = 0;
	}
	local_flush_tlb_mm(mm);

	preempt_enable();
}

struct flush_tlb_data {
	struct vm_area_struct *vma;
	unsigned long addr1;
	unsigned long addr2;
};

static void flush_tlb_range_ipi(void *info)
{
	struct flush_tlb_data *fd = (struct flush_tlb_data *)info;

	local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
}

void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
	struct mm_struct *mm = vma->vm_mm;

	preempt_disable();
	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
		struct flush_tlb_data fd;

		fd.vma = vma;
		fd.addr1 = start;
		fd.addr2 = end;
		smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1, 1);
	} else {
		int i;
		for (i = 0; i < num_online_cpus(); i++)
			if (smp_processor_id() != i)
				cpu_context(i, mm) = 0;
	}
	local_flush_tlb_range(vma, start, end);
	preempt_enable();
}

static void flush_tlb_kernel_range_ipi(void *info)
{
	struct flush_tlb_data *fd = (struct flush_tlb_data *)info;

	local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
}

void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	struct flush_tlb_data fd;

	fd.addr1 = start;
	fd.addr2 = end;
	on_each_cpu(flush_tlb_kernel_range_ipi, (void *)&fd, 1, 1);
}

static void flush_tlb_page_ipi(void *info)
{
	struct flush_tlb_data *fd = (struct flush_tlb_data *)info;

	local_flush_tlb_page(fd->vma, fd->addr1);
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
	preempt_disable();
	if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
		struct flush_tlb_data fd;

		fd.vma = vma;
		fd.addr1 = page;
		smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1, 1);
	} else {
		int i;
		for (i = 0; i < num_online_cpus(); i++)
			if (smp_processor_id() != i)
				cpu_context(i, vma->vm_mm) = 0;
	}
	local_flush_tlb_page(vma, page);
	preempt_enable();
}

static void flush_tlb_one_ipi(void *info)
{
	unsigned long vaddr = (unsigned long) info;

	local_flush_tlb_one(vaddr);
}

void flush_tlb_one(unsigned long vaddr)
{
	smp_call_function(flush_tlb_one_ipi, (void *) vaddr, 1, 1);
	local_flush_tlb_one(vaddr);
}

static DEFINE_PER_CPU(struct cpu, cpu_devices);

static int __init topology_init(void)
{
	int cpu;
	int ret;

	for_each_cpu(cpu) {
		ret = register_cpu(&per_cpu(cpu_devices, cpu), cpu, NULL);
		if (ret)
			printk(KERN_WARNING "topology_init: register_cpu %d "
			       "failed (%d)\n", cpu, ret);
	}

	return 0;
}

subsys_initcall(topology_init);

EXPORT_SYMBOL(flush_tlb_page);
EXPORT_SYMBOL(flush_tlb_one);
EXPORT_SYMBOL(cpu_data);
EXPORT_SYMBOL(synchronize_irq);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This program is free software; you can redistribute it and/or
	3	* modify it under the terms of the GNU General Public License
	4	* as published by the Free Software Foundation; either version 2
	5	* of the License, or (at your option) any later version.
	6	*
	7	* This program is distributed in the hope that it will be useful,
	8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	* GNU General Public License for more details.
	11	*
	12	* You should have received a copy of the GNU General Public License
	13	* along with this program; if not, write to the Free Software
	14	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	15	*
	16	* Copyright (C) 2000, 2001 Kanoj Sarcar
	17	* Copyright (C) 2000, 2001 Ralf Baechle
	18	* Copyright (C) 2000, 2001 Silicon Graphics, Inc.
	19	* Copyright (C) 2000, 2001, 2003 Broadcom Corporation
	20	*/
	21	#include <linux/cache.h>
	22	#include <linux/delay.h>
	23	#include <linux/init.h>
	24	#include <linux/interrupt.h>
	25	#include <linux/spinlock.h>
	26	#include <linux/threads.h>
	27	#include <linux/module.h>
	28	#include <linux/time.h>
	29	#include <linux/timex.h>
	30	#include <linux/sched.h>
	31	#include <linux/cpumask.h>
1e35aaba	32	#include <linux/cpu.h>
1da177e4 LT	33
	34	#include <asm/atomic.h>
	35	#include <asm/cpu.h>
	36	#include <asm/processor.h>
	37	#include <asm/system.h>
	38	#include <asm/mmu_context.h>
	39	#include <asm/smp.h>
	40
	41	cpumask_t phys_cpu_present_map; /* Bitmask of available CPUs */
	42	volatile cpumask_t cpu_callin_map; /* Bitmask of started secondaries */
	43	cpumask_t cpu_online_map; /* Bitmask of currently online CPUs */
	44	int __cpu_number_map[NR_CPUS]; /* Map physical to logical */
	45	int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */
	46
	47	EXPORT_SYMBOL(phys_cpu_present_map);
	48	EXPORT_SYMBOL(cpu_online_map);
	49
	50	static void smp_tune_scheduling (void)
	51	{
	52	struct cache_desc *cd = &current_cpu_data.scache;
	53	unsigned long cachesize; /* kB */
1da177e4 LT	54	unsigned long cpu_khz;
	55
	56	/*
	57	* Crude estimate until we actually meassure ...
	58	*/
	59	cpu_khz = loops_per_jiffy * 2 * HZ / 1000;
	60
	61	/*
	62	* Rough estimation for SMP scheduling, this is the number of
	63	* cycles it takes for a fully memory-limited process to flush
	64	* the SMP-local cache.
	65	*
	66	* (For a P5 this pretty much means we will choose another idle
	67	* CPU almost always at wakeup time (this is due to the small
	68	* L1 cache), on PIIs it's around 50-100 usecs, depending on
	69	* the cache size)
	70	*/
	71	if (!cpu_khz)
	72	return;
	73
	74	cachesize = cd->linesz * cd->sets * cd->ways;
	75	}
	76
	77	extern void __init calibrate_delay(void);
	78	extern ATTRIB_NORET void cpu_idle(void);
	79
	80	/*
	81	* First C code run on the secondary CPUs after being started up by
	82	* the master.
	83	*/
	84	asmlinkage void start_secondary(void)
	85	{
5bfb5d69	86	unsigned int cpu;
1da177e4 LT	87
	88	cpu_probe();
	89	cpu_report();
	90	per_cpu_trap_init();
	91	prom_init_secondary();
	92
	93	/*
	94	* XXX parity protection should be folded in here when it's converted
	95	* to an option instead of something based on .cputype
	96	*/
	97
	98	calibrate_delay();
5bfb5d69 NP	99	preempt_disable();
5bfb5d69 NP	100	cpu = smp_processor_id();
1da177e4 LT	101	cpu_data[cpu].udelay_val = loops_per_jiffy;
	102
	103	prom_smp_finish();
	104
	105	cpu_set(cpu, cpu_callin_map);
	106
	107	cpu_idle();
	108	}
	109
	110	DEFINE_SPINLOCK(smp_call_lock);
	111
	112	struct call_data_struct *call_data;
	113
	114	/*
	115	* Run a function on all other CPUs.
	116	* <func> The function to run. This must be fast and non-blocking.
	117	* <info> An arbitrary pointer to pass to the function.
	118	* <retry> If true, keep retrying until ready.
	119	* <wait> If true, wait until function has completed on other CPUs.
	120	* [RETURNS] 0 on success, else a negative status code.
	121	*
	122	* Does not return until remote CPUs are nearly ready to execute <func>
	123	* or are or have executed.
	124	*
	125	* You must not call this function with disabled interrupts or from a
57f0060b RB	126	* hardware interrupt handler or from a bottom half handler:
	127	*
	128	* CPU A CPU B
	129	* Disable interrupts
	130	* smp_call_function()
	131	* Take call_lock
	132	* Send IPIs
	133	* Wait for all cpus to acknowledge IPI
	134	* CPU A has not responded, spin waiting
	135	* for cpu A to respond, holding call_lock
	136	* smp_call_function()
	137	* Spin waiting for call_lock
	138	* Deadlock Deadlock
1da177e4 LT	139	*/
	140	int smp_call_function (void (func) (void info), void *info, int retry,
	141	int wait)
	142	{
	143	struct call_data_struct data;
	144	int i, cpus = num_online_cpus() - 1;
	145	int cpu = smp_processor_id();
	146
ae1b3d51 RB	147	/*
	148	* Can die spectacularly if this CPU isn't yet marked online
	149	*/
	150	BUG_ON(!cpu_online(cpu));
	151
1da177e4 LT	152	if (!cpus)
	153	return 0;
	154
	155	/* Can deadlock when called with interrupts disabled */
	156	WARN_ON(irqs_disabled());
	157
	158	data.func = func;
	159	data.info = info;
	160	atomic_set(&data.started, 0);
	161	data.wait = wait;
	162	if (wait)
	163	atomic_set(&data.finished, 0);
	164
	165	spin_lock(&smp_call_lock);
	166	call_data = &data;
	167	mb();
	168
	169	/* Send a message to all other CPUs and wait for them to respond */
394e3902 AM	170	for_each_online_cpu(i)
394e3902 AM	171	if (i != cpu)
1da177e4 LT	172	core_send_ipi(i, SMP_CALL_FUNCTION);
	173
	174	/* Wait for response */
	175	/* FIXME: lock-up detection, backtrace on lock-up */
	176	while (atomic_read(&data.started) != cpus)
	177	barrier();
	178
	179	if (wait)
	180	while (atomic_read(&data.finished) != cpus)
	181	barrier();
	182	spin_unlock(&smp_call_lock);
	183
	184	return 0;
	185	}
	186
	187	void smp_call_function_interrupt(void)
	188	{
	189	void (func) (void info) = call_data->func;
	190	void *info = call_data->info;
	191	int wait = call_data->wait;
	192
	193	/*
	194	* Notify initiating CPU that I've grabbed the data and am
	195	* about to execute the function.
	196	*/
	197	mb();
	198	atomic_inc(&call_data->started);
	199
	200	/*
	201	* At this point the info structure may be out of scope unless wait==1.
	202	*/
	203	irq_enter();
	204	(*func)(info);
	205	irq_exit();
	206
	207	if (wait) {
	208	mb();
	209	atomic_inc(&call_data->finished);
	210	}
	211	}
	212
	213	static void stop_this_cpu(void *dummy)
	214	{
	215	/*
	216	* Remove this CPU:
	217	*/
	218	cpu_clear(smp_processor_id(), cpu_online_map);
	219	local_irq_enable(); /* May need to service _machine_restart IPI */
	220	for (;;); /* Wait if available. */
	221	}
	222
	223	void smp_send_stop(void)
	224	{
	225	smp_call_function(stop_this_cpu, NULL, 1, 0);
	226	}
	227
	228	void __init smp_cpus_done(unsigned int max_cpus)
	229	{
	230	prom_cpus_done();
	231	}
	232
	233	/* called from main before smp_init() */
	234	void __init smp_prepare_cpus(unsigned int max_cpus)
	235	{
1da177e4 LT	236	init_new_context(current, &init_mm);
	237	current_thread_info()->cpu = 0;
	238	smp_tune_scheduling();
9b6695a8	239	plat_prepare_cpus(max_cpus);
1da177e4 LT	240	}
	241
	242	/* preload SMP state for boot cpu */
	243	void __devinit smp_prepare_boot_cpu(void)
	244	{
	245	/*
	246	* This assumes that bootup is always handled by the processor
	247	* with the logic and physical number 0.
	248	*/
	249	__cpu_number_map[0] = 0;
	250	__cpu_logical_map[0] = 0;
	251	cpu_set(0, phys_cpu_present_map);
	252	cpu_set(0, cpu_online_map);
	253	cpu_set(0, cpu_callin_map);
	254	}
	255
	256	/*
b727a602 RB	257	* Called once for each "cpu_possible(cpu)". Needs to spin up the cpu
	258	* and keep control until "cpu_online(cpu)" is set. Note: cpu is
	259	* physical, not logical.
1da177e4	260	*/
b727a602	261	int __devinit __cpu_up(unsigned int cpu)
1da177e4 LT	262	{
	263	struct task_struct *idle;
	264
	265	/*
b727a602	266	* Processor goes to start_secondary(), sets online flag
1da177e4 LT	267	* The following code is purely to make sure
	268	* Linux can schedule processes on this slave.
	269	*/
	270	idle = fork_idle(cpu);
	271	if (IS_ERR(idle))
b727a602	272	panic(KERN_ERR "Fork failed for CPU %d", cpu);
1da177e4 LT	273
	274	prom_boot_secondary(cpu, idle);
	275
b727a602 RB	276	/*
	277	* Trust is futile. We should really have timeouts ...
	278	*/
1da177e4 LT	279	while (!cpu_isset(cpu, cpu_callin_map))
	280	udelay(100);
	281
	282	cpu_set(cpu, cpu_online_map);
	283
	284	return 0;
	285	}
	286
1da177e4 LT	287	/* Not really SMP stuff ... */
	288	int setup_profiling_timer(unsigned int multiplier)
	289	{
	290	return 0;
	291	}
	292
	293	static void flush_tlb_all_ipi(void *info)
	294	{
	295	local_flush_tlb_all();
	296	}
	297
	298	void flush_tlb_all(void)
	299	{
	300	on_each_cpu(flush_tlb_all_ipi, 0, 1, 1);
	301	}
	302
	303	static void flush_tlb_mm_ipi(void *mm)
	304	{
	305	local_flush_tlb_mm((struct mm_struct *)mm);
	306	}
	307
	308	/*
	309	* The following tlb flush calls are invoked when old translations are
	310	* being torn down, or pte attributes are changing. For single threaded
	311	* address spaces, a new context is obtained on the current cpu, and tlb
	312	* context on other cpus are invalidated to force a new context allocation
	313	* at switch_mm time, should the mm ever be used on other cpus. For
	314	* multithreaded address spaces, intercpu interrupts have to be sent.
	315	* Another case where intercpu interrupts are required is when the target
	316	* mm might be active on another cpu (eg debuggers doing the flushes on
	317	* behalf of debugees, kswapd stealing pages from another process etc).
	318	* Kanoj 07/00.
	319	*/
	320
	321	void flush_tlb_mm(struct mm_struct *mm)
	322	{
	323	preempt_disable();
	324
	325	if ((atomic_read(&mm->mm_users) != 1) \|\| (current->mm != mm)) {
	326	smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1, 1);
	327	} else {
	328	int i;
	329	for (i = 0; i < num_online_cpus(); i++)
	330	if (smp_processor_id() != i)
	331	cpu_context(i, mm) = 0;
	332	}
	333	local_flush_tlb_mm(mm);
	334
	335	preempt_enable();
	336	}
	337
	338	struct flush_tlb_data {
	339	struct vm_area_struct *vma;
	340	unsigned long addr1;
	341	unsigned long addr2;
	342	};
	343
	344	static void flush_tlb_range_ipi(void *info)
	345	{
	346	struct flush_tlb_data fd = (struct flush_tlb_data )info;
	347
	348	local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
	349	}
	350
351	void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
352	{
353	struct mm_struct *mm = vma->vm_mm;
354
355	preempt_disable();
356	if ((atomic_read(&mm->mm_users) != 1) \|\| (current->mm != mm)) {
357	struct flush_tlb_data fd;
358
359	fd.vma = vma;
360	fd.addr1 = start;
361	fd.addr2 = end;
362	smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1, 1);
363	} else {
364	int i;
365	for (i = 0; i < num_online_cpus(); i++)
366	if (smp_processor_id() != i)
367	cpu_context(i, mm) = 0;
368	}
369	local_flush_tlb_range(vma, start, end);
370	preempt_enable();
371	}
372
373	static void flush_tlb_kernel_range_ipi(void *info)
374	{
375	struct flush_tlb_data fd = (struct flush_tlb_data )info;
376
377	local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
378	}
379
380	void flush_tlb_kernel_range(unsigned long start, unsigned long end)
381	{
382	struct flush_tlb_data fd;
383
384	fd.addr1 = start;
385	fd.addr2 = end;
386	on_each_cpu(flush_tlb_kernel_range_ipi, (void *)&fd, 1, 1);
387	}
388
389	static void flush_tlb_page_ipi(void *info)
390	{
391	struct flush_tlb_data fd = (struct flush_tlb_data )info;
392
393	local_flush_tlb_page(fd->vma, fd->addr1);
394	}
395
396	void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
397	{
398	preempt_disable();
399	if ((atomic_read(&vma->vm_mm->mm_users) != 1) \|\| (current->mm != vma->vm_mm)) {
400	struct flush_tlb_data fd;
401
402	fd.vma = vma;
403	fd.addr1 = page;
404	smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1, 1);
405	} else {
406	int i;
407	for (i = 0; i < num_online_cpus(); i++)
408	if (smp_processor_id() != i)
409	cpu_context(i, vma->vm_mm) = 0;
410	}
411	local_flush_tlb_page(vma, page);
412	preempt_enable();
413	}
414
415	static void flush_tlb_one_ipi(void *info)
416	{
417	unsigned long vaddr = (unsigned long) info;
418
419	local_flush_tlb_one(vaddr);
420	}
421
422	void flush_tlb_one(unsigned long vaddr)
423	{
424	smp_call_function(flush_tlb_one_ipi, (void *) vaddr, 1, 1);
425	local_flush_tlb_one(vaddr);
426	}
427
1e35aaba RI	428	static DEFINE_PER_CPU(struct cpu, cpu_devices);
	429
	430	static int __init topology_init(void)
	431	{
	432	int cpu;
	433	int ret;
	434
	435	for_each_cpu(cpu) {
	436	ret = register_cpu(&per_cpu(cpu_devices, cpu), cpu, NULL);
	437	if (ret)
	438	printk(KERN_WARNING "topology_init: register_cpu %d "
	439	"failed (%d)\n", cpu, ret);
	440	}
	441
	442	return 0;
	443	}
	444
	445	subsys_initcall(topology_init);
	446
1da177e4 LT	447	EXPORT_SYMBOL(flush_tlb_page);
	448	EXPORT_SYMBOL(flush_tlb_one);
	449	EXPORT_SYMBOL(cpu_data);
	450	EXPORT_SYMBOL(synchronize_irq);