[deliverable/linux.git] / arch / alpha / kernel / smp.c

/*
 *	linux/arch/alpha/kernel/smp.c
 *
 *      2001-07-09 Phil Ezolt (Phillip.Ezolt@compaq.com)
 *            Renamed modified smp_call_function to smp_call_function_on_cpu()
 *            Created an function that conforms to the old calling convention
 *            of smp_call_function().
 *
 *            This is helpful for DCPI.
 *
 */

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/threads.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
#include <linux/irq.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/bitops.h>
#include <linux/cpu.h>

#include <asm/hwrpb.h>
#include <asm/ptrace.h>
#include <linux/atomic.h>

#include <asm/io.h>
#include <asm/irq.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>

#include "proto.h"
#include "irq_impl.h"


#define DEBUG_SMP 0
#if DEBUG_SMP
#define DBGS(args)	printk args
#else
#define DBGS(args)
#endif

/* A collection of per-processor data.  */
struct cpuinfo_alpha cpu_data[NR_CPUS];
EXPORT_SYMBOL(cpu_data);

/* A collection of single bit ipi messages.  */
static struct {
	unsigned long bits ____cacheline_aligned;
} ipi_data[NR_CPUS] __cacheline_aligned;

enum ipi_message_type {
	IPI_RESCHEDULE,
	IPI_CALL_FUNC,
	IPI_CALL_FUNC_SINGLE,
	IPI_CPU_STOP,
};

/* Set to a secondary's cpuid when it comes online.  */
static int smp_secondary_alive = 0;

int smp_num_probed;		/* Internal processor count */
int smp_num_cpus = 1;		/* Number that came online.  */
EXPORT_SYMBOL(smp_num_cpus);

/*
 * Called by both boot and secondaries to move global data into
 *  per-processor storage.
 */
static inline void __init
smp_store_cpu_info(int cpuid)
{
	cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
	cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
	cpu_data[cpuid].need_new_asn = 0;
	cpu_data[cpuid].asn_lock = 0;
}

/*
 * Ideally sets up per-cpu profiling hooks.  Doesn't do much now...
 */
static inline void __init
smp_setup_percpu_timer(int cpuid)
{
	cpu_data[cpuid].prof_counter = 1;
	cpu_data[cpuid].prof_multiplier = 1;
}

static void __init
wait_boot_cpu_to_stop(int cpuid)
{
	unsigned long stop = jiffies + 10*HZ;

	while (time_before(jiffies, stop)) {
	        if (!smp_secondary_alive)
			return;
		barrier();
	}

	printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
	for (;;)
		barrier();
}

/*
 * Where secondaries begin a life of C.
 */
void __cpuinit
smp_callin(void)
{
	int cpuid = hard_smp_processor_id();

	if (cpu_online(cpuid)) {
		printk("??, cpu 0x%x already present??\n", cpuid);
		BUG();
	}
	set_cpu_online(cpuid, true);

	/* Turn on machine checks.  */
	wrmces(7);

	/* Set trap vectors.  */
	trap_init();

	/* Set interrupt vector.  */
	wrent(entInt, 0);

	/* Get our local ticker going. */
	smp_setup_percpu_timer(cpuid);

	/* Call platform-specific callin, if specified */
	if (alpha_mv.smp_callin) alpha_mv.smp_callin();

	/* All kernel threads share the same mm context.  */
	atomic_inc(&init_mm.mm_count);
	current->active_mm = &init_mm;

	/* inform the notifiers about the new cpu */
	notify_cpu_starting(cpuid);

	/* Must have completely accurate bogos.  */
	local_irq_enable();

	/* Wait boot CPU to stop with irq enabled before running
	   calibrate_delay. */
	wait_boot_cpu_to_stop(cpuid);
	mb();
	calibrate_delay();

	smp_store_cpu_info(cpuid);
	/* Allow master to continue only after we written loops_per_jiffy.  */
	wmb();
	smp_secondary_alive = 1;

	DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
	      cpuid, current, current->active_mm));

	preempt_disable();
	cpu_startup_entry(CPUHP_ONLINE);
}

/* Wait until hwrpb->txrdy is clear for cpu.  Return -1 on timeout.  */
static int
wait_for_txrdy (unsigned long cpumask)
{
	unsigned long timeout;

	if (!(hwrpb->txrdy & cpumask))
		return 0;

	timeout = jiffies + 10*HZ;
	while (time_before(jiffies, timeout)) {
		if (!(hwrpb->txrdy & cpumask))
			return 0;
		udelay(10);
		barrier();
	}

	return -1;
}

/*
 * Send a message to a secondary's console.  "START" is one such
 * interesting message.  ;-)
 */
static void __cpuinit
send_secondary_console_msg(char *str, int cpuid)
{
	struct percpu_struct *cpu;
	register char *cp1, *cp2;
	unsigned long cpumask;
	size_t len;

	cpu = (struct percpu_struct *)
		((char*)hwrpb
		 + hwrpb->processor_offset
		 + cpuid * hwrpb->processor_size);

	cpumask = (1UL << cpuid);
	if (wait_for_txrdy(cpumask))
		goto timeout;

	cp2 = str;
	len = strlen(cp2);
	*(unsigned int *)&cpu->ipc_buffer[0] = len;
	cp1 = (char *) &cpu->ipc_buffer[1];
	memcpy(cp1, cp2, len);

	/* atomic test and set */
	wmb();
	set_bit(cpuid, &hwrpb->rxrdy);

	if (wait_for_txrdy(cpumask))
		goto timeout;
	return;

 timeout:
	printk("Processor %x not ready\n", cpuid);
}

/*
 * A secondary console wants to send a message.  Receive it.
 */
static void
recv_secondary_console_msg(void)
{
	int mycpu, i, cnt;
	unsigned long txrdy = hwrpb->txrdy;
	char *cp1, *cp2, buf[80];
	struct percpu_struct *cpu;

	DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));

	mycpu = hard_smp_processor_id();

	for (i = 0; i < NR_CPUS; i++) {
		if (!(txrdy & (1UL << i)))
			continue;

		DBGS(("recv_secondary_console_msg: "
		      "TXRDY contains CPU %d.\n", i));

		cpu = (struct percpu_struct *)
		  ((char*)hwrpb
		   + hwrpb->processor_offset
		   + i * hwrpb->processor_size);

 		DBGS(("recv_secondary_console_msg: on %d from %d"
		      " HALT_REASON 0x%lx FLAGS 0x%lx\n",
		      mycpu, i, cpu->halt_reason, cpu->flags));

		cnt = cpu->ipc_buffer[0] >> 32;
		if (cnt <= 0 || cnt >= 80)
			strcpy(buf, "<<< BOGUS MSG >>>");
		else {
			cp1 = (char *) &cpu->ipc_buffer[11];
			cp2 = buf;
			strcpy(cp2, cp1);
			
			while ((cp2 = strchr(cp2, '\r')) != 0) {
				*cp2 = ' ';
				if (cp2[1] == '\n')
					cp2[1] = ' ';
			}
		}

		DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
		      "message is '%s'\n", mycpu, buf));
	}

	hwrpb->txrdy = 0;
}

/*
 * Convince the console to have a secondary cpu begin execution.
 */
static int __cpuinit
secondary_cpu_start(int cpuid, struct task_struct *idle)
{
	struct percpu_struct *cpu;
	struct pcb_struct *hwpcb, *ipcb;
	unsigned long timeout;
	  
	cpu = (struct percpu_struct *)
		((char*)hwrpb
		 + hwrpb->processor_offset
		 + cpuid * hwrpb->processor_size);
	hwpcb = (struct pcb_struct *) cpu->hwpcb;
	ipcb = &task_thread_info(idle)->pcb;

	/* Initialize the CPU's HWPCB to something just good enough for
	   us to get started.  Immediately after starting, we'll swpctx
	   to the target idle task's pcb.  Reuse the stack in the mean
	   time.  Precalculate the target PCBB.  */
	hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
	hwpcb->usp = 0;
	hwpcb->ptbr = ipcb->ptbr;
	hwpcb->pcc = 0;
	hwpcb->asn = 0;
	hwpcb->unique = virt_to_phys(ipcb);
	hwpcb->flags = ipcb->flags;
	hwpcb->res1 = hwpcb->res2 = 0;

#if 0
	DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
	      hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
#endif
	DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
	      cpuid, idle->state, ipcb->flags));

	/* Setup HWRPB fields that SRM uses to activate secondary CPU */
	hwrpb->CPU_restart = __smp_callin;
	hwrpb->CPU_restart_data = (unsigned long) __smp_callin;

	/* Recalculate and update the HWRPB checksum */
	hwrpb_update_checksum(hwrpb);

	/*
	 * Send a "start" command to the specified processor.
	 */

	/* SRM III 3.4.1.3 */
	cpu->flags |= 0x22;	/* turn on Context Valid and Restart Capable */
	cpu->flags &= ~1;	/* turn off Bootstrap In Progress */
	wmb();

	send_secondary_console_msg("START\r\n", cpuid);

	/* Wait 10 seconds for an ACK from the console.  */
	timeout = jiffies + 10*HZ;
	while (time_before(jiffies, timeout)) {
		if (cpu->flags & 1)
			goto started;
		udelay(10);
		barrier();
	}
	printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
	return -1;

 started:
	DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
	return 0;
}

/*
 * Bring one cpu online.
 */
static int __cpuinit
smp_boot_one_cpu(int cpuid, struct task_struct *idle)
{
	unsigned long timeout;

	/* Signal the secondary to wait a moment.  */
	smp_secondary_alive = -1;

	/* Whirrr, whirrr, whirrrrrrrrr... */
	if (secondary_cpu_start(cpuid, idle))
		return -1;

	/* Notify the secondary CPU it can run calibrate_delay.  */
	mb();
	smp_secondary_alive = 0;

	/* We've been acked by the console; wait one second for
	   the task to start up for real.  */
	timeout = jiffies + 1*HZ;
	while (time_before(jiffies, timeout)) {
		if (smp_secondary_alive == 1)
			goto alive;
		udelay(10);
		barrier();
	}

	/* We failed to boot the CPU.  */

	printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
	return -1;

 alive:
	/* Another "Red Snapper". */
	return 0;
}

/*
 * Called from setup_arch.  Detect an SMP system and which processors
 * are present.
 */
void __init
setup_smp(void)
{
	struct percpu_struct *cpubase, *cpu;
	unsigned long i;

	if (boot_cpuid != 0) {
		printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
		       boot_cpuid);
	}

	if (hwrpb->nr_processors > 1) {
		int boot_cpu_palrev;

		DBGS(("setup_smp: nr_processors %ld\n",
		      hwrpb->nr_processors));

		cpubase = (struct percpu_struct *)
			((char*)hwrpb + hwrpb->processor_offset);
		boot_cpu_palrev = cpubase->pal_revision;

		for (i = 0; i < hwrpb->nr_processors; i++) {
			cpu = (struct percpu_struct *)
				((char *)cpubase + i*hwrpb->processor_size);
			if ((cpu->flags & 0x1cc) == 0x1cc) {
				smp_num_probed++;
				set_cpu_possible(i, true);
				set_cpu_present(i, true);
				cpu->pal_revision = boot_cpu_palrev;
			}

			DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
			      i, cpu->flags, cpu->type));
			DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
			      i, cpu->pal_revision));
		}
	} else {
		smp_num_probed = 1;
	}

	printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n",
	       smp_num_probed, cpumask_bits(cpu_present_mask)[0]);
}

/*
 * Called by smp_init prepare the secondaries
 */
void __init
smp_prepare_cpus(unsigned int max_cpus)
{
	/* Take care of some initial bookkeeping.  */
	memset(ipi_data, 0, sizeof(ipi_data));

	current_thread_info()->cpu = boot_cpuid;

	smp_store_cpu_info(boot_cpuid);
	smp_setup_percpu_timer(boot_cpuid);

	/* Nothing to do on a UP box, or when told not to.  */
	if (smp_num_probed == 1 || max_cpus == 0) {
		init_cpu_possible(cpumask_of(boot_cpuid));
		init_cpu_present(cpumask_of(boot_cpuid));
		printk(KERN_INFO "SMP mode deactivated.\n");
		return;
	}

	printk(KERN_INFO "SMP starting up secondaries.\n");

	smp_num_cpus = smp_num_probed;
}

void
smp_prepare_boot_cpu(void)
{
}

int __cpuinit
__cpu_up(unsigned int cpu, struct task_struct *tidle)
{
	smp_boot_one_cpu(cpu, tidle);

	return cpu_online(cpu) ? 0 : -ENOSYS;
}

void __init
smp_cpus_done(unsigned int max_cpus)
{
	int cpu;
	unsigned long bogosum = 0;

	for(cpu = 0; cpu < NR_CPUS; cpu++) 
		if (cpu_online(cpu))
			bogosum += cpu_data[cpu].loops_per_jiffy;
	
	printk(KERN_INFO "SMP: Total of %d processors activated "
	       "(%lu.%02lu BogoMIPS).\n",
	       num_online_cpus(), 
	       (bogosum + 2500) / (500000/HZ),
	       ((bogosum + 2500) / (5000/HZ)) % 100);
}

\f
void
smp_percpu_timer_interrupt(struct pt_regs *regs)
{
	struct pt_regs *old_regs;
	int cpu = smp_processor_id();
	unsigned long user = user_mode(regs);
	struct cpuinfo_alpha *data = &cpu_data[cpu];

	old_regs = set_irq_regs(regs);

	/* Record kernel PC.  */
	profile_tick(CPU_PROFILING);

	if (!--data->prof_counter) {
		/* We need to make like a normal interrupt -- otherwise
		   timer interrupts ignore the global interrupt lock,
		   which would be a Bad Thing.  */
		irq_enter();

		update_process_times(user);

		data->prof_counter = data->prof_multiplier;

		irq_exit();
	}
	set_irq_regs(old_regs);
}

int
setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}

\f
static void
send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
{
	int i;

	mb();
	for_each_cpu(i, to_whom)
		set_bit(operation, &ipi_data[i].bits);

	mb();
	for_each_cpu(i, to_whom)
		wripir(i);
}

void
handle_ipi(struct pt_regs *regs)
{
	int this_cpu = smp_processor_id();
	unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
	unsigned long ops;

#if 0
	DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
	      this_cpu, *pending_ipis, regs->pc));
#endif

	mb();	/* Order interrupt and bit testing. */
	while ((ops = xchg(pending_ipis, 0)) != 0) {
	  mb();	/* Order bit clearing and data access. */
	  do {
		unsigned long which;

		which = ops & -ops;
		ops &= ~which;
		which = __ffs(which);

		switch (which) {
		case IPI_RESCHEDULE:
			scheduler_ipi();
			break;

		case IPI_CALL_FUNC:
			generic_smp_call_function_interrupt();
			break;

		case IPI_CALL_FUNC_SINGLE:
			generic_smp_call_function_single_interrupt();
			break;

		case IPI_CPU_STOP:
			halt();

		default:
			printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
			       this_cpu, which);
			break;
		}
	  } while (ops);

	  mb();	/* Order data access and bit testing. */
	}

	cpu_data[this_cpu].ipi_count++;

	if (hwrpb->txrdy)
		recv_secondary_console_msg();
}

void
smp_send_reschedule(int cpu)
{
#ifdef DEBUG_IPI_MSG
	if (cpu == hard_smp_processor_id())
		printk(KERN_WARNING
		       "smp_send_reschedule: Sending IPI to self.\n");
#endif
	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
}

void
smp_send_stop(void)
{
	cpumask_t to_whom;
	cpumask_copy(&to_whom, cpu_possible_mask);
	cpumask_clear_cpu(smp_processor_id(), &to_whom);
#ifdef DEBUG_IPI_MSG
	if (hard_smp_processor_id() != boot_cpu_id)
		printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
#endif
	send_ipi_message(&to_whom, IPI_CPU_STOP);
}

void arch_send_call_function_ipi_mask(const struct cpumask *mask)
{
	send_ipi_message(mask, IPI_CALL_FUNC);
}

void arch_send_call_function_single_ipi(int cpu)
{
	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
}

static void
ipi_imb(void *ignored)
{
	imb();
}

void
smp_imb(void)
{
	/* Must wait other processors to flush their icache before continue. */
	if (on_each_cpu(ipi_imb, NULL, 1))
		printk(KERN_CRIT "smp_imb: timed out\n");
}
EXPORT_SYMBOL(smp_imb);

static void
ipi_flush_tlb_all(void *ignored)
{
	tbia();
}

void
flush_tlb_all(void)
{
	/* Although we don't have any data to pass, we do want to
	   synchronize with the other processors.  */
	if (on_each_cpu(ipi_flush_tlb_all, NULL, 1)) {
		printk(KERN_CRIT "flush_tlb_all: timed out\n");
	}
}

#define asn_locked() (cpu_data[smp_processor_id()].asn_lock)

static void
ipi_flush_tlb_mm(void *x)
{
	struct mm_struct *mm = (struct mm_struct *) x;
	if (mm == current->active_mm && !asn_locked())
		flush_tlb_current(mm);
	else
		flush_tlb_other(mm);
}

void
flush_tlb_mm(struct mm_struct *mm)
{
	preempt_disable();

	if (mm == current->active_mm) {
		flush_tlb_current(mm);
		if (atomic_read(&mm->mm_users) <= 1) {
			int cpu, this_cpu = smp_processor_id();
			for (cpu = 0; cpu < NR_CPUS; cpu++) {
				if (!cpu_online(cpu) || cpu == this_cpu)
					continue;
				if (mm->context[cpu])
					mm->context[cpu] = 0;
			}
			preempt_enable();
			return;
		}
	}

	if (smp_call_function(ipi_flush_tlb_mm, mm, 1)) {
		printk(KERN_CRIT "flush_tlb_mm: timed out\n");
	}

	preempt_enable();
}
EXPORT_SYMBOL(flush_tlb_mm);

struct flush_tlb_page_struct {
	struct vm_area_struct *vma;
	struct mm_struct *mm;
	unsigned long addr;
};

static void
ipi_flush_tlb_page(void *x)
{
	struct flush_tlb_page_struct *data = (struct flush_tlb_page_struct *)x;
	struct mm_struct * mm = data->mm;

	if (mm == current->active_mm && !asn_locked())
		flush_tlb_current_page(mm, data->vma, data->addr);
	else
		flush_tlb_other(mm);
}

void
flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
{
	struct flush_tlb_page_struct data;
	struct mm_struct *mm = vma->vm_mm;

	preempt_disable();

	if (mm == current->active_mm) {
		flush_tlb_current_page(mm, vma, addr);
		if (atomic_read(&mm->mm_users) <= 1) {
			int cpu, this_cpu = smp_processor_id();
			for (cpu = 0; cpu < NR_CPUS; cpu++) {
				if (!cpu_online(cpu) || cpu == this_cpu)
					continue;
				if (mm->context[cpu])
					mm->context[cpu] = 0;
			}
			preempt_enable();
			return;
		}
	}

	data.vma = vma;
	data.mm = mm;
	data.addr = addr;

	if (smp_call_function(ipi_flush_tlb_page, &data, 1)) {
		printk(KERN_CRIT "flush_tlb_page: timed out\n");
	}

	preempt_enable();
}
EXPORT_SYMBOL(flush_tlb_page);

void
flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
	/* On the Alpha we always flush the whole user tlb.  */
	flush_tlb_mm(vma->vm_mm);
}
EXPORT_SYMBOL(flush_tlb_range);

static void
ipi_flush_icache_page(void *x)
{
	struct mm_struct *mm = (struct mm_struct *) x;
	if (mm == current->active_mm && !asn_locked())
		__load_new_mm_context(mm);
	else
		flush_tlb_other(mm);
}

void
flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
			unsigned long addr, int len)
{
	struct mm_struct *mm = vma->vm_mm;

	if ((vma->vm_flags & VM_EXEC) == 0)
		return;

	preempt_disable();

	if (mm == current->active_mm) {
		__load_new_mm_context(mm);
		if (atomic_read(&mm->mm_users) <= 1) {
			int cpu, this_cpu = smp_processor_id();
			for (cpu = 0; cpu < NR_CPUS; cpu++) {
				if (!cpu_online(cpu) || cpu == this_cpu)
					continue;
				if (mm->context[cpu])
					mm->context[cpu] = 0;
			}
			preempt_enable();
			return;
		}
	}

	if (smp_call_function(ipi_flush_icache_page, mm, 1)) {
		printk(KERN_CRIT "flush_icache_page: timed out\n");
	}

	preempt_enable();
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/arch/alpha/kernel/smp.c
	3	*
	4	* 2001-07-09 Phil Ezolt (Phillip.Ezolt@compaq.com)
	5	* Renamed modified smp_call_function to smp_call_function_on_cpu()
	6	* Created an function that conforms to the old calling convention
	7	* of smp_call_function().
	8	*
	9	* This is helpful for DCPI.
	10	*
	11	*/
	12
	13	#include <linux/errno.h>
	14	#include <linux/kernel.h>
	15	#include <linux/kernel_stat.h>
	16	#include <linux/module.h>
	17	#include <linux/sched.h>
	18	#include <linux/mm.h>
4e950f6f	19	#include <linux/err.h>
1da177e4 LT	20	#include <linux/threads.h>
1da177e4 LT	21	#include <linux/smp.h>
1da177e4 LT	22	#include <linux/interrupt.h>
	23	#include <linux/init.h>
	24	#include <linux/delay.h>
	25	#include <linux/spinlock.h>
	26	#include <linux/irq.h>
	27	#include <linux/cache.h>
	28	#include <linux/profile.h>
	29	#include <linux/bitops.h>
574f34ce	30	#include <linux/cpu.h>
1da177e4 LT	31
	32	#include <asm/hwrpb.h>
	33	#include <asm/ptrace.h>
60063497	34	#include <linux/atomic.h>
1da177e4 LT	35
	36	#include <asm/io.h>
	37	#include <asm/irq.h>
	38	#include <asm/pgtable.h>
	39	#include <asm/pgalloc.h>
	40	#include <asm/mmu_context.h>
	41	#include <asm/tlbflush.h>
	42
	43	#include "proto.h"
	44	#include "irq_impl.h"
	45
	46
	47	#define DEBUG_SMP 0
	48	#if DEBUG_SMP
	49	#define DBGS(args) printk args
	50	#else
	51	#define DBGS(args)
	52	#endif
	53
	54	/* A collection of per-processor data. */
	55	struct cpuinfo_alpha cpu_data[NR_CPUS];
cff52daf	56	EXPORT_SYMBOL(cpu_data);
1da177e4 LT	57
	58	/* A collection of single bit ipi messages. */
	59	static struct {
	60	unsigned long bits ____cacheline_aligned;
	61	} ipi_data[NR_CPUS] __cacheline_aligned;
	62
	63	enum ipi_message_type {
	64	IPI_RESCHEDULE,
	65	IPI_CALL_FUNC,
c524a1d8	66	IPI_CALL_FUNC_SINGLE,
1da177e4 LT	67	IPI_CPU_STOP,
	68	};
	69
	70	/* Set to a secondary's cpuid when it comes online. */
f8d6c8d9	71	static int smp_secondary_alive = 0;
1da177e4	72
1da177e4 LT	73	int smp_num_probed; /* Internal processor count */
1da177e4 LT	74	int smp_num_cpus = 1; /* Number that came online. */
cff52daf	75	EXPORT_SYMBOL(smp_num_cpus);
1da177e4	76
1da177e4 LT	77	/*
	78	* Called by both boot and secondaries to move global data into
	79	* per-processor storage.
	80	*/
	81	static inline void __init
	82	smp_store_cpu_info(int cpuid)
	83	{
	84	cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
	85	cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
	86	cpu_data[cpuid].need_new_asn = 0;
	87	cpu_data[cpuid].asn_lock = 0;
	88	}
	89
	90	/*
	91	* Ideally sets up per-cpu profiling hooks. Doesn't do much now...
	92	*/
	93	static inline void __init
	94	smp_setup_percpu_timer(int cpuid)
	95	{
	96	cpu_data[cpuid].prof_counter = 1;
	97	cpu_data[cpuid].prof_multiplier = 1;
	98	}
	99
	100	static void __init
	101	wait_boot_cpu_to_stop(int cpuid)
	102	{
	103	unsigned long stop = jiffies + 10*HZ;
	104
	105	while (time_before(jiffies, stop)) {
	106	if (!smp_secondary_alive)
	107	return;
	108	barrier();
	109	}
	110
	111	printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
	112	for (;;)
	113	barrier();
	114	}
	115
	116	/*
	117	* Where secondaries begin a life of C.
	118	*/
7d6a8a1c	119	void __cpuinit
1da177e4 LT	120	smp_callin(void)
	121	{
	122	int cpuid = hard_smp_processor_id();
	123
1371be0f	124	if (cpu_online(cpuid)) {
1da177e4 LT	125	printk("??, cpu 0x%x already present??\n", cpuid);
	126	BUG();
	127	}
1371be0f	128	set_cpu_online(cpuid, true);
1da177e4 LT	129
	130	/* Turn on machine checks. */
	131	wrmces(7);
	132
	133	/* Set trap vectors. */
	134	trap_init();
	135
	136	/* Set interrupt vector. */
	137	wrent(entInt, 0);
	138
	139	/* Get our local ticker going. */
	140	smp_setup_percpu_timer(cpuid);
	141
	142	/* Call platform-specific callin, if specified */
	143	if (alpha_mv.smp_callin) alpha_mv.smp_callin();
	144
	145	/* All kernel threads share the same mm context. */
	146	atomic_inc(&init_mm.mm_count);
	147	current->active_mm = &init_mm;
	148
e545a614 MS	149	/* inform the notifiers about the new cpu */
	150	notify_cpu_starting(cpuid);
	151
1da177e4 LT	152	/* Must have completely accurate bogos. */
	153	local_irq_enable();
	154
	155	/* Wait boot CPU to stop with irq enabled before running
	156	calibrate_delay. */
	157	wait_boot_cpu_to_stop(cpuid);
	158	mb();
	159	calibrate_delay();
	160
	161	smp_store_cpu_info(cpuid);
	162	/* Allow master to continue only after we written loops_per_jiffy. */
	163	wmb();
	164	smp_secondary_alive = 1;
	165
	166	DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
	167	cpuid, current, current->active_mm));
	168
6a6c0272	169	preempt_disable();
a123322d	170	cpu_startup_entry(CPUHP_ONLINE);
1da177e4 LT	171	}
	172
	173	/* Wait until hwrpb->txrdy is clear for cpu. Return -1 on timeout. */
f8d6c8d9	174	static int
1da177e4 LT	175	wait_for_txrdy (unsigned long cpumask)
	176	{
	177	unsigned long timeout;
	178
	179	if (!(hwrpb->txrdy & cpumask))
	180	return 0;
	181
	182	timeout = jiffies + 10*HZ;
	183	while (time_before(jiffies, timeout)) {
	184	if (!(hwrpb->txrdy & cpumask))
	185	return 0;
	186	udelay(10);
	187	barrier();
	188	}
	189
	190	return -1;
	191	}
	192
	193	/*
	194	* Send a message to a secondary's console. "START" is one such
	195	* interesting message. ;-)
	196	*/
7d6a8a1c	197	static void __cpuinit
1da177e4 LT	198	send_secondary_console_msg(char *str, int cpuid)
	199	{
	200	struct percpu_struct *cpu;
	201	register char cp1, cp2;
	202	unsigned long cpumask;
	203	size_t len;
	204
	205	cpu = (struct percpu_struct *)
	206	((char*)hwrpb
	207	+ hwrpb->processor_offset
	208	+ cpuid * hwrpb->processor_size);
	209
	210	cpumask = (1UL << cpuid);
	211	if (wait_for_txrdy(cpumask))
	212	goto timeout;
	213
	214	cp2 = str;
	215	len = strlen(cp2);
	216	(unsigned int )&cpu->ipc_buffer[0] = len;
	217	cp1 = (char *) &cpu->ipc_buffer[1];
	218	memcpy(cp1, cp2, len);
	219
	220	/* atomic test and set */
	221	wmb();
	222	set_bit(cpuid, &hwrpb->rxrdy);
	223
	224	if (wait_for_txrdy(cpumask))
	225	goto timeout;
	226	return;
	227
	228	timeout:
	229	printk("Processor %x not ready\n", cpuid);
	230	}
	231
	232	/*
	233	* A secondary console wants to send a message. Receive it.
	234	*/
	235	static void
	236	recv_secondary_console_msg(void)
	237	{
	238	int mycpu, i, cnt;
	239	unsigned long txrdy = hwrpb->txrdy;
	240	char cp1, cp2, buf[80];
	241	struct percpu_struct *cpu;
	242
	243	DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));
	244
	245	mycpu = hard_smp_processor_id();
	246
	247	for (i = 0; i < NR_CPUS; i++) {
	248	if (!(txrdy & (1UL << i)))
	249	continue;
	250
	251	DBGS(("recv_secondary_console_msg: "
	252	"TXRDY contains CPU %d.\n", i));
	253
	254	cpu = (struct percpu_struct *)
	255	((char*)hwrpb
	256	+ hwrpb->processor_offset
	257	+ i * hwrpb->processor_size);
	258
	259	DBGS(("recv_secondary_console_msg: on %d from %d"
	260	" HALT_REASON 0x%lx FLAGS 0x%lx\n",
	261	mycpu, i, cpu->halt_reason, cpu->flags));
262
263	cnt = cpu->ipc_buffer[0] >> 32;
264	if (cnt <= 0 \|\| cnt >= 80)
265	strcpy(buf, "<<< BOGUS MSG >>>");
266	else {
267	cp1 = (char *) &cpu->ipc_buffer[11];
268	cp2 = buf;
269	strcpy(cp2, cp1);
270
271	while ((cp2 = strchr(cp2, '\r')) != 0) {
272	*cp2 = ' ';
273	if (cp2[1] == '\n')
274	cp2[1] = ' ';
275	}
276	}
277
278	DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
279	"message is '%s'\n", mycpu, buf));
280	}
281
282	hwrpb->txrdy = 0;
283	}
284
285	/*
286	* Convince the console to have a secondary cpu begin execution.
287	*/
7d6a8a1c	288	static int __cpuinit
1da177e4 LT	289	secondary_cpu_start(int cpuid, struct task_struct *idle)
	290	{
	291	struct percpu_struct *cpu;
	292	struct pcb_struct hwpcb, ipcb;
	293	unsigned long timeout;
	294
	295	cpu = (struct percpu_struct *)
	296	((char*)hwrpb
	297	+ hwrpb->processor_offset
	298	+ cpuid * hwrpb->processor_size);
	299	hwpcb = (struct pcb_struct *) cpu->hwpcb;
37bfbaf9	300	ipcb = &task_thread_info(idle)->pcb;
1da177e4 LT	301
	302	/* Initialize the CPU's HWPCB to something just good enough for
	303	us to get started. Immediately after starting, we'll swpctx
	304	to the target idle task's pcb. Reuse the stack in the mean
	305	time. Precalculate the target PCBB. */
	306	hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
	307	hwpcb->usp = 0;
	308	hwpcb->ptbr = ipcb->ptbr;
	309	hwpcb->pcc = 0;
	310	hwpcb->asn = 0;
	311	hwpcb->unique = virt_to_phys(ipcb);
	312	hwpcb->flags = ipcb->flags;
	313	hwpcb->res1 = hwpcb->res2 = 0;
	314
	315	#if 0
	316	DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
	317	hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
	318	#endif
	319	DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
	320	cpuid, idle->state, ipcb->flags));
	321
	322	/* Setup HWRPB fields that SRM uses to activate secondary CPU */
	323	hwrpb->CPU_restart = __smp_callin;
	324	hwrpb->CPU_restart_data = (unsigned long) __smp_callin;
	325
	326	/* Recalculate and update the HWRPB checksum */
	327	hwrpb_update_checksum(hwrpb);
	328
	329	/*
	330	* Send a "start" command to the specified processor.
	331	*/
	332
	333	/* SRM III 3.4.1.3 */
	334	cpu->flags \|= 0x22; /* turn on Context Valid and Restart Capable */
	335	cpu->flags &= ~1; /* turn off Bootstrap In Progress */
	336	wmb();
	337
	338	send_secondary_console_msg("START\r\n", cpuid);
	339
	340	/* Wait 10 seconds for an ACK from the console. */
	341	timeout = jiffies + 10*HZ;
	342	while (time_before(jiffies, timeout)) {
	343	if (cpu->flags & 1)
	344	goto started;
	345	udelay(10);
	346	barrier();
	347	}
	348	printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
	349	return -1;
	350
	351	started:
	352	DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
	353	return 0;
	354	}
	355
	356	/*
	357	* Bring one cpu online.
	358	*/
ed5f6561	359	static int __cpuinit
2ec9415c	360	smp_boot_one_cpu(int cpuid, struct task_struct *idle)
1da177e4	361	{
1da177e4 LT	362	unsigned long timeout;
1da177e4 LT	363
1da177e4 LT	364	/* Signal the secondary to wait a moment. */
	365	smp_secondary_alive = -1;
	366
	367	/* Whirrr, whirrr, whirrrrrrrrr... */
	368	if (secondary_cpu_start(cpuid, idle))
	369	return -1;
	370
	371	/* Notify the secondary CPU it can run calibrate_delay. */
	372	mb();
	373	smp_secondary_alive = 0;
	374
	375	/* We've been acked by the console; wait one second for
	376	the task to start up for real. */
	377	timeout = jiffies + 1*HZ;
	378	while (time_before(jiffies, timeout)) {
	379	if (smp_secondary_alive == 1)
	380	goto alive;
	381	udelay(10);
	382	barrier();
	383	}
	384
	385	/* We failed to boot the CPU. */
	386
	387	printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
	388	return -1;
	389
	390	alive:
	391	/* Another "Red Snapper". */
	392	return 0;
	393	}
	394
	395	/*
	396	* Called from setup_arch. Detect an SMP system and which processors
	397	* are present.
	398	*/
	399	void __init
	400	setup_smp(void)
	401	{
	402	struct percpu_struct cpubase, cpu;
	403	unsigned long i;
	404
	405	if (boot_cpuid != 0) {
	406	printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
	407	boot_cpuid);
	408	}
	409
	410	if (hwrpb->nr_processors > 1) {
	411	int boot_cpu_palrev;
	412
	413	DBGS(("setup_smp: nr_processors %ld\n",
	414	hwrpb->nr_processors));
	415
	416	cpubase = (struct percpu_struct *)
	417	((char*)hwrpb + hwrpb->processor_offset);
	418	boot_cpu_palrev = cpubase->pal_revision;
	419
	420	for (i = 0; i < hwrpb->nr_processors; i++) {
	421	cpu = (struct percpu_struct *)
	422	((char )cpubase + ihwrpb->processor_size);
	423	if ((cpu->flags & 0x1cc) == 0x1cc) {
	424	smp_num_probed++;
1371be0f RR	425	set_cpu_possible(i, true);
1371be0f RR	426	set_cpu_present(i, true);
1da177e4 LT	427	cpu->pal_revision = boot_cpu_palrev;
	428	}
	429
	430	DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
	431	i, cpu->flags, cpu->type));
	432	DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
	433	i, cpu->pal_revision));
	434	}
	435	} else {
	436	smp_num_probed = 1;
1da177e4	437	}
1da177e4	438
5f054e31	439	printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n",
81740fc6	440	smp_num_probed, cpumask_bits(cpu_present_mask)[0]);
1da177e4 LT	441	}
	442
	443	/*
	444	* Called by smp_init prepare the secondaries
	445	*/
	446	void __init
	447	smp_prepare_cpus(unsigned int max_cpus)
	448	{
1da177e4 LT	449	/* Take care of some initial bookkeeping. */
	450	memset(ipi_data, 0, sizeof(ipi_data));
	451
	452	current_thread_info()->cpu = boot_cpuid;
	453
	454	smp_store_cpu_info(boot_cpuid);
	455	smp_setup_percpu_timer(boot_cpuid);
	456
	457	/* Nothing to do on a UP box, or when told not to. */
	458	if (smp_num_probed == 1 \|\| max_cpus == 0) {
1371be0f RR	459	init_cpu_possible(cpumask_of(boot_cpuid));
1371be0f RR	460	init_cpu_present(cpumask_of(boot_cpuid));
1da177e4 LT	461	printk(KERN_INFO "SMP mode deactivated.\n");
	462	return;
	463	}
	464
	465	printk(KERN_INFO "SMP starting up secondaries.\n");
	466
328c2a8a	467	smp_num_cpus = smp_num_probed;
1da177e4 LT	468	}
1da177e4 LT	469
f8d6c8d9	470	void
1da177e4 LT	471	smp_prepare_boot_cpu(void)
1da177e4 LT	472	{
1da177e4 LT	473	}
1da177e4 LT	474
ed5f6561	475	int __cpuinit
8239c25f	476	__cpu_up(unsigned int cpu, struct task_struct *tidle)
1da177e4	477	{
2ec9415c	478	smp_boot_one_cpu(cpu, tidle);
1da177e4 LT	479
	480	return cpu_online(cpu) ? 0 : -ENOSYS;
	481	}
	482
	483	void __init
	484	smp_cpus_done(unsigned int max_cpus)
	485	{
	486	int cpu;
	487	unsigned long bogosum = 0;
	488
	489	for(cpu = 0; cpu < NR_CPUS; cpu++)
	490	if (cpu_online(cpu))
	491	bogosum += cpu_data[cpu].loops_per_jiffy;
	492
	493	printk(KERN_INFO "SMP: Total of %d processors activated "
	494	"(%lu.%02lu BogoMIPS).\n",
	495	num_online_cpus(),
	496	(bogosum + 2500) / (500000/HZ),
	497	((bogosum + 2500) / (5000/HZ)) % 100);
	498	}
	499
	500	\f
	501	void
	502	smp_percpu_timer_interrupt(struct pt_regs *regs)
	503	{
8774cb81	504	struct pt_regs *old_regs;
1da177e4 LT	505	int cpu = smp_processor_id();
	506	unsigned long user = user_mode(regs);
	507	struct cpuinfo_alpha *data = &cpu_data[cpu];
	508
8774cb81 AV	509	old_regs = set_irq_regs(regs);
8774cb81 AV	510
1da177e4	511	/* Record kernel PC. */
8774cb81	512	profile_tick(CPU_PROFILING);
1da177e4 LT	513
	514	if (!--data->prof_counter) {
	515	/* We need to make like a normal interrupt -- otherwise
	516	timer interrupts ignore the global interrupt lock,
	517	which would be a Bad Thing. */
	518	irq_enter();
	519
	520	update_process_times(user);
	521
	522	data->prof_counter = data->prof_multiplier;
	523
	524	irq_exit();
	525	}
8774cb81	526	set_irq_regs(old_regs);
1da177e4 LT	527	}
1da177e4 LT	528
ed5f6561	529	int
1da177e4 LT	530	setup_profiling_timer(unsigned int multiplier)
	531	{
	532	return -EINVAL;
	533	}
	534
	535	\f
	536	static void
81065e4f	537	send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
1da177e4 LT	538	{
	539	int i;
	540
	541	mb();
81065e4f	542	for_each_cpu(i, to_whom)
1da177e4 LT	543	set_bit(operation, &ipi_data[i].bits);
	544
	545	mb();
81065e4f	546	for_each_cpu(i, to_whom)
1da177e4 LT	547	wripir(i);
	548	}
	549
1da177e4 LT	550	void
	551	handle_ipi(struct pt_regs *regs)
	552	{
	553	int this_cpu = smp_processor_id();
	554	unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
	555	unsigned long ops;
	556
	557	#if 0
	558	DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
	559	this_cpu, *pending_ipis, regs->pc));
	560	#endif
	561
	562	mb(); /* Order interrupt and bit testing. */
	563	while ((ops = xchg(pending_ipis, 0)) != 0) {
	564	mb(); /* Order bit clearing and data access. */
	565	do {
	566	unsigned long which;
	567
	568	which = ops & -ops;
	569	ops &= ~which;
	570	which = __ffs(which);
	571
	572	switch (which) {
	573	case IPI_RESCHEDULE:
184748cc	574	scheduler_ipi();
1da177e4 LT	575	break;
	576
	577	case IPI_CALL_FUNC:
c524a1d8 JA	578	generic_smp_call_function_interrupt();
	579	break;
	580
	581	case IPI_CALL_FUNC_SINGLE:
	582	generic_smp_call_function_single_interrupt();
1da177e4	583	break;
1da177e4 LT	584
	585	case IPI_CPU_STOP:
	586	halt();
	587
	588	default:
	589	printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
	590	this_cpu, which);
	591	break;
	592	}
	593	} while (ops);
	594
	595	mb(); /* Order data access and bit testing. */
	596	}
	597
	598	cpu_data[this_cpu].ipi_count++;
	599
	600	if (hwrpb->txrdy)
	601	recv_secondary_console_msg();
	602	}
	603
	604	void
	605	smp_send_reschedule(int cpu)
	606	{
	607	#ifdef DEBUG_IPI_MSG
	608	if (cpu == hard_smp_processor_id())
	609	printk(KERN_WARNING
	610	"smp_send_reschedule: Sending IPI to self.\n");
	611	#endif
81065e4f	612	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
1da177e4 LT	613	}
	614
	615	void
	616	smp_send_stop(void)
	617	{
81740fc6 KM	618	cpumask_t to_whom;
	619	cpumask_copy(&to_whom, cpu_possible_mask);
	620	cpumask_clear_cpu(smp_processor_id(), &to_whom);
1da177e4 LT	621	#ifdef DEBUG_IPI_MSG
	622	if (hard_smp_processor_id() != boot_cpu_id)
	623	printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
	624	#endif
81065e4f	625	send_ipi_message(&to_whom, IPI_CPU_STOP);
1da177e4 LT	626	}
1da177e4 LT	627
81065e4f	628	void arch_send_call_function_ipi_mask(const struct cpumask *mask)
1da177e4	629	{
c524a1d8	630	send_ipi_message(mask, IPI_CALL_FUNC);
1da177e4 LT	631	}
1da177e4 LT	632
c524a1d8	633	void arch_send_call_function_single_ipi(int cpu)
1da177e4	634	{
81065e4f	635	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
1da177e4 LT	636	}
	637
	638	static void
	639	ipi_imb(void *ignored)
	640	{
	641	imb();
	642	}
	643
	644	void
	645	smp_imb(void)
	646	{
	647	/* Must wait other processors to flush their icache before continue. */
15c8b6c1	648	if (on_each_cpu(ipi_imb, NULL, 1))
1da177e4 LT	649	printk(KERN_CRIT "smp_imb: timed out\n");
1da177e4 LT	650	}
cff52daf	651	EXPORT_SYMBOL(smp_imb);
1da177e4 LT	652
	653	static void
	654	ipi_flush_tlb_all(void *ignored)
	655	{
	656	tbia();
	657	}
	658
	659	void
	660	flush_tlb_all(void)
	661	{
	662	/* Although we don't have any data to pass, we do want to
	663	synchronize with the other processors. */
15c8b6c1	664	if (on_each_cpu(ipi_flush_tlb_all, NULL, 1)) {
1da177e4 LT	665	printk(KERN_CRIT "flush_tlb_all: timed out\n");
	666	}
	667	}
	668
	669	#define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
	670
	671	static void
	672	ipi_flush_tlb_mm(void *x)
	673	{
	674	struct mm_struct mm = (struct mm_struct ) x;
	675	if (mm == current->active_mm && !asn_locked())
	676	flush_tlb_current(mm);
	677	else
	678	flush_tlb_other(mm);
	679	}
	680
	681	void
	682	flush_tlb_mm(struct mm_struct *mm)
	683	{
	684	preempt_disable();
	685
	686	if (mm == current->active_mm) {
	687	flush_tlb_current(mm);
	688	if (atomic_read(&mm->mm_users) <= 1) {
	689	int cpu, this_cpu = smp_processor_id();
	690	for (cpu = 0; cpu < NR_CPUS; cpu++) {
	691	if (!cpu_online(cpu) \|\| cpu == this_cpu)
	692	continue;
	693	if (mm->context[cpu])
	694	mm->context[cpu] = 0;
	695	}
	696	preempt_enable();
	697	return;
	698	}
	699	}
	700
8691e5a8	701	if (smp_call_function(ipi_flush_tlb_mm, mm, 1)) {
1da177e4 LT	702	printk(KERN_CRIT "flush_tlb_mm: timed out\n");
	703	}
	704
	705	preempt_enable();
	706	}
cff52daf	707	EXPORT_SYMBOL(flush_tlb_mm);
1da177e4 LT	708
	709	struct flush_tlb_page_struct {
	710	struct vm_area_struct *vma;
	711	struct mm_struct *mm;
	712	unsigned long addr;
	713	};
	714
	715	static void
	716	ipi_flush_tlb_page(void *x)
	717	{
	718	struct flush_tlb_page_struct data = (struct flush_tlb_page_struct )x;
	719	struct mm_struct * mm = data->mm;
	720
	721	if (mm == current->active_mm && !asn_locked())
	722	flush_tlb_current_page(mm, data->vma, data->addr);
	723	else
	724	flush_tlb_other(mm);
	725	}
	726
	727	void
	728	flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
	729	{
	730	struct flush_tlb_page_struct data;
	731	struct mm_struct *mm = vma->vm_mm;
	732
	733	preempt_disable();
	734
	735	if (mm == current->active_mm) {
	736	flush_tlb_current_page(mm, vma, addr);
	737	if (atomic_read(&mm->mm_users) <= 1) {
	738	int cpu, this_cpu = smp_processor_id();
	739	for (cpu = 0; cpu < NR_CPUS; cpu++) {
	740	if (!cpu_online(cpu) \|\| cpu == this_cpu)
	741	continue;
	742	if (mm->context[cpu])
	743	mm->context[cpu] = 0;
	744	}
	745	preempt_enable();
	746	return;
	747	}
	748	}
	749
	750	data.vma = vma;
	751	data.mm = mm;
	752	data.addr = addr;
	753
8691e5a8	754	if (smp_call_function(ipi_flush_tlb_page, &data, 1)) {
1da177e4 LT	755	printk(KERN_CRIT "flush_tlb_page: timed out\n");
	756	}
	757
	758	preempt_enable();
	759	}
cff52daf	760	EXPORT_SYMBOL(flush_tlb_page);
1da177e4 LT	761
	762	void
	763	flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
	764	{
	765	/* On the Alpha we always flush the whole user tlb. */
	766	flush_tlb_mm(vma->vm_mm);
	767	}
cff52daf	768	EXPORT_SYMBOL(flush_tlb_range);
1da177e4 LT	769
	770	static void
	771	ipi_flush_icache_page(void *x)
	772	{
	773	struct mm_struct mm = (struct mm_struct ) x;
	774	if (mm == current->active_mm && !asn_locked())
	775	__load_new_mm_context(mm);
	776	else
	777	flush_tlb_other(mm);
	778	}
	779
	780	void
	781	flush_icache_user_range(struct vm_area_struct vma, struct page page,
	782	unsigned long addr, int len)
	783	{
	784	struct mm_struct *mm = vma->vm_mm;
	785
	786	if ((vma->vm_flags & VM_EXEC) == 0)
	787	return;
	788
	789	preempt_disable();
	790
	791	if (mm == current->active_mm) {
	792	__load_new_mm_context(mm);
	793	if (atomic_read(&mm->mm_users) <= 1) {
	794	int cpu, this_cpu = smp_processor_id();
	795	for (cpu = 0; cpu < NR_CPUS; cpu++) {
	796	if (!cpu_online(cpu) \|\| cpu == this_cpu)
	797	continue;
	798	if (mm->context[cpu])
	799	mm->context[cpu] = 0;
	800	}
	801	preempt_enable();
	802	return;
	803	}
	804	}
	805
8691e5a8	806	if (smp_call_function(ipi_flush_icache_page, mm, 1)) {
1da177e4 LT	807	printk(KERN_CRIT "flush_icache_page: timed out\n");
	808	}
	809
	810	preempt_enable();
	811	}