[deliverable/linux.git] / arch / mips / mm / tlb-r3k.c

/*
 * r2300.c: R2000 and R3000 specific mmu/cache code.
 *
 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
 *
 * with a lot of changes to make this thing work for R3000s
 * Tx39XX R4k style caches added. HK
 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
 * Copyright (C) 2002  Ralf Baechle
 * Copyright (C) 2002  Maciej W. Rozycki
 */
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>

#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/mmu_context.h>
#include <asm/tlbmisc.h>
#include <asm/isadep.h>
#include <asm/io.h>
#include <asm/bootinfo.h>
#include <asm/cpu.h>

#undef DEBUG_TLB

extern void build_tlb_refill_handler(void);

/* CP0 hazard avoidance. */
#define BARRIER				\
	__asm__ __volatile__(		\
		".set	push\n\t"	\
		".set	noreorder\n\t"	\
		"nop\n\t"		\
		".set	pop\n\t")

int r3k_have_wired_reg;			/* Should be in cpu_data? */

/* TLB operations. */
static void local_flush_tlb_from(int entry)
{
	unsigned long old_ctx;

	old_ctx = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
	write_c0_entrylo0(0);
	while (entry < current_cpu_data.tlbsize) {
		write_c0_index(entry << 8);
		write_c0_entryhi((entry | 0x80000) << 12);
		entry++;				/* BARRIER */
		tlb_write_indexed();
	}
	write_c0_entryhi(old_ctx);
}

void local_flush_tlb_all(void)
{
	unsigned long flags;

#ifdef DEBUG_TLB
	printk("[tlball]");
#endif
	local_irq_save(flags);
	local_flush_tlb_from(r3k_have_wired_reg ? read_c0_wired() : 8);
	local_irq_restore(flags);
}

void local_flush_tlb_mm(struct mm_struct *mm)
{
	int cpu = smp_processor_id();

	if (cpu_context(cpu, mm) != 0) {
#ifdef DEBUG_TLB
		printk("[tlbmm<%lu>]", (unsigned long)cpu_context(cpu, mm));
#endif
		drop_mmu_context(mm, cpu);
	}
}

void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
			   unsigned long end)
{
	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
	struct mm_struct *mm = vma->vm_mm;
	int cpu = smp_processor_id();

	if (cpu_context(cpu, mm) != 0) {
		unsigned long size, flags;

#ifdef DEBUG_TLB
		printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
			cpu_context(cpu, mm) & asid_mask, start, end);
#endif
		local_irq_save(flags);
		size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
		if (size <= current_cpu_data.tlbsize) {
			int oldpid = read_c0_entryhi() & asid_mask;
			int newpid = cpu_context(cpu, mm) & asid_mask;

			start &= PAGE_MASK;
			end += PAGE_SIZE - 1;
			end &= PAGE_MASK;
			while (start < end) {
				int idx;

				write_c0_entryhi(start | newpid);
				start += PAGE_SIZE;	/* BARRIER */
				tlb_probe();
				idx = read_c0_index();
				write_c0_entrylo0(0);
				write_c0_entryhi(KSEG0);
				if (idx < 0)		/* BARRIER */
					continue;
				tlb_write_indexed();
			}
			write_c0_entryhi(oldpid);
		} else {
			drop_mmu_context(mm, cpu);
		}
		local_irq_restore(flags);
	}
}

void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	unsigned long size, flags;

#ifdef DEBUG_TLB
	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
#endif
	local_irq_save(flags);
	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	if (size <= current_cpu_data.tlbsize) {
		int pid = read_c0_entryhi();

		start &= PAGE_MASK;
		end += PAGE_SIZE - 1;
		end &= PAGE_MASK;

		while (start < end) {
			int idx;

			write_c0_entryhi(start);
			start += PAGE_SIZE;		/* BARRIER */
			tlb_probe();
			idx = read_c0_index();
			write_c0_entrylo0(0);
			write_c0_entryhi(KSEG0);
			if (idx < 0)			/* BARRIER */
				continue;
			tlb_write_indexed();
		}
		write_c0_entryhi(pid);
	} else {
		local_flush_tlb_all();
	}
	local_irq_restore(flags);
}

void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
	int cpu = smp_processor_id();

	if (cpu_context(cpu, vma->vm_mm) != 0) {
		unsigned long flags;
		int oldpid, newpid, idx;

#ifdef DEBUG_TLB
		printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
#endif
		newpid = cpu_context(cpu, vma->vm_mm) & asid_mask;
		page &= PAGE_MASK;
		local_irq_save(flags);
		oldpid = read_c0_entryhi() & asid_mask;
		write_c0_entryhi(page | newpid);
		BARRIER;
		tlb_probe();
		idx = read_c0_index();
		write_c0_entrylo0(0);
		write_c0_entryhi(KSEG0);
		if (idx < 0)				/* BARRIER */
			goto finish;
		tlb_write_indexed();

finish:
		write_c0_entryhi(oldpid);
		local_irq_restore(flags);
	}
}

void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
{
	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
	unsigned long flags;
	int idx, pid;

	/*
	 * Handle debugger faulting in for debugee.
	 */
	if (current->active_mm != vma->vm_mm)
		return;

	pid = read_c0_entryhi() & asid_mask;

#ifdef DEBUG_TLB
	if ((pid != (cpu_context(cpu, vma->vm_mm) & asid_mask)) || (cpu_context(cpu, vma->vm_mm) == 0)) {
		printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
		       (cpu_context(cpu, vma->vm_mm)), pid);
	}
#endif

	local_irq_save(flags);
	address &= PAGE_MASK;
	write_c0_entryhi(address | pid);
	BARRIER;
	tlb_probe();
	idx = read_c0_index();
	write_c0_entrylo0(pte_val(pte));
	write_c0_entryhi(address | pid);
	if (idx < 0) {					/* BARRIER */
		tlb_write_random();
	} else {
		tlb_write_indexed();
	}
	write_c0_entryhi(pid);
	local_irq_restore(flags);
}

void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
		     unsigned long entryhi, unsigned long pagemask)
{
	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
	unsigned long flags;
	unsigned long old_ctx;
	static unsigned long wired = 0;

	if (r3k_have_wired_reg) {			/* TX39XX */
		unsigned long old_pagemask;
		unsigned long w;

#ifdef DEBUG_TLB
		printk("[tlbwired<entry lo0 %8x, hi %8x\n, pagemask %8x>]\n",
		       entrylo0, entryhi, pagemask);
#endif

		local_irq_save(flags);
		/* Save old context and create impossible VPN2 value */
		old_ctx = read_c0_entryhi() & asid_mask;
		old_pagemask = read_c0_pagemask();
		w = read_c0_wired();
		write_c0_wired(w + 1);
		write_c0_index(w << 8);
		write_c0_pagemask(pagemask);
		write_c0_entryhi(entryhi);
		write_c0_entrylo0(entrylo0);
		BARRIER;
		tlb_write_indexed();

		write_c0_entryhi(old_ctx);
		write_c0_pagemask(old_pagemask);
		local_flush_tlb_all();
		local_irq_restore(flags);

	} else if (wired < 8) {
#ifdef DEBUG_TLB
		printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
		       entrylo0, entryhi);
#endif

		local_irq_save(flags);
		old_ctx = read_c0_entryhi() & asid_mask;
		write_c0_entrylo0(entrylo0);
		write_c0_entryhi(entryhi);
		write_c0_index(wired);
		wired++;				/* BARRIER */
		tlb_write_indexed();
		write_c0_entryhi(old_ctx);
		local_flush_tlb_all();
		local_irq_restore(flags);
	}
}

void tlb_init(void)
{
	switch (current_cpu_type()) {
	case CPU_TX3922:
	case CPU_TX3927:
		r3k_have_wired_reg = 1;
		write_c0_wired(0);		/* Set to 8 on reset... */
		break;
	}
	local_flush_tlb_from(0);
	build_tlb_refill_handler();
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* r2300.c: R2000 and R3000 specific mmu/cache code.
	3	*
79add627	4	* Copyright (C) 1996 David S. Miller (davem@davemloft.net)
1da177e4 LT	5	*
	6	* with a lot of changes to make this thing work for R3000s
	7	* Tx39XX R4k style caches added. HK
	8	* Copyright (C) 1998, 1999, 2000 Harald Koerfgen
	9	* Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
	10	* Copyright (C) 2002 Ralf Baechle
	11	* Copyright (C) 2002 Maciej W. Rozycki
	12	*/
1da177e4 LT	13	#include <linux/kernel.h>
1da177e4 LT	14	#include <linux/sched.h>
631330f5	15	#include <linux/smp.h>
1da177e4 LT	16	#include <linux/mm.h>
	17
	18	#include <asm/page.h>
	19	#include <asm/pgtable.h>
	20	#include <asm/mmu_context.h>
3d18c983	21	#include <asm/tlbmisc.h>
1da177e4 LT	22	#include <asm/isadep.h>
	23	#include <asm/io.h>
	24	#include <asm/bootinfo.h>
	25	#include <asm/cpu.h>
	26
	27	#undef DEBUG_TLB
	28
	29	extern void build_tlb_refill_handler(void);
	30
	31	/* CP0 hazard avoidance. */
	32	#define BARRIER \
	33	__asm__ __volatile__( \
	34	".set push\n\t" \
	35	".set noreorder\n\t" \
	36	"nop\n\t" \
	37	".set pop\n\t")
	38
3c865dd9	39	int r3k_have_wired_reg; /* Should be in cpu_data? */
1da177e4 LT	40
1da177e4 LT	41	/* TLB operations. */
dbfbf60f	42	static void local_flush_tlb_from(int entry)
1da177e4	43	{
1da177e4	44	unsigned long old_ctx;
1da177e4	45
4edf00a4	46	old_ctx = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
1da177e4	47	write_c0_entrylo0(0);
9a20b092	48	while (entry < current_cpu_data.tlbsize) {
1da177e4 LT	49	write_c0_index(entry << 8);
1da177e4 LT	50	write_c0_entryhi((entry \| 0x80000) << 12);
9a20b092	51	entry++; /* BARRIER */
1da177e4 LT	52	tlb_write_indexed();
	53	}
	54	write_c0_entryhi(old_ctx);
dbfbf60f MR	55	}
	56
	57	void local_flush_tlb_all(void)
	58	{
	59	unsigned long flags;
	60
	61	#ifdef DEBUG_TLB
	62	printk("[tlball]");
	63	#endif
	64	local_irq_save(flags);
	65	local_flush_tlb_from(r3k_have_wired_reg ? read_c0_wired() : 8);
1da177e4 LT	66	local_irq_restore(flags);
	67	}
	68
	69	void local_flush_tlb_mm(struct mm_struct *mm)
	70	{
	71	int cpu = smp_processor_id();
	72
	73	if (cpu_context(cpu, mm) != 0) {
	74	#ifdef DEBUG_TLB
	75	printk("[tlbmm<%lu>]", (unsigned long)cpu_context(cpu, mm));
	76	#endif
	77	drop_mmu_context(mm, cpu);
	78	}
	79	}
	80
	81	void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	82	unsigned long end)
	83	{
4edf00a4	84	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
1da177e4 LT	85	struct mm_struct *mm = vma->vm_mm;
	86	int cpu = smp_processor_id();
	87
	88	if (cpu_context(cpu, mm) != 0) {
a5e696e5	89	unsigned long size, flags;
1da177e4 LT	90
	91	#ifdef DEBUG_TLB
	92	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
4edf00a4	93	cpu_context(cpu, mm) & asid_mask, start, end);
1da177e4 LT	94	#endif
	95	local_irq_save(flags);
	96	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	97	if (size <= current_cpu_data.tlbsize) {
4edf00a4 PB	98	int oldpid = read_c0_entryhi() & asid_mask;
4edf00a4 PB	99	int newpid = cpu_context(cpu, mm) & asid_mask;
1da177e4 LT	100
	101	start &= PAGE_MASK;
	102	end += PAGE_SIZE - 1;
	103	end &= PAGE_MASK;
	104	while (start < end) {
	105	int idx;
	106
	107	write_c0_entryhi(start \| newpid);
	108	start += PAGE_SIZE; /* BARRIER */
	109	tlb_probe();
	110	idx = read_c0_index();
	111	write_c0_entrylo0(0);
	112	write_c0_entryhi(KSEG0);
	113	if (idx < 0) /* BARRIER */
	114	continue;
	115	tlb_write_indexed();
	116	}
	117	write_c0_entryhi(oldpid);
	118	} else {
	119	drop_mmu_context(mm, cpu);
	120	}
	121	local_irq_restore(flags);
	122	}
	123	}
	124
	125	void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
	126	{
a5e696e5	127	unsigned long size, flags;
1da177e4 LT	128
	129	#ifdef DEBUG_TLB
	130	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
	131	#endif
	132	local_irq_save(flags);
	133	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	134	if (size <= current_cpu_data.tlbsize) {
	135	int pid = read_c0_entryhi();
	136
	137	start &= PAGE_MASK;
	138	end += PAGE_SIZE - 1;
	139	end &= PAGE_MASK;
	140
	141	while (start < end) {
	142	int idx;
	143
	144	write_c0_entryhi(start);
	145	start += PAGE_SIZE; /* BARRIER */
	146	tlb_probe();
	147	idx = read_c0_index();
	148	write_c0_entrylo0(0);
	149	write_c0_entryhi(KSEG0);
	150	if (idx < 0) /* BARRIER */
	151	continue;
	152	tlb_write_indexed();
	153	}
	154	write_c0_entryhi(pid);
	155	} else {
	156	local_flush_tlb_all();
	157	}
	158	local_irq_restore(flags);
	159	}
	160
	161	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
	162	{
4edf00a4	163	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
1da177e4 LT	164	int cpu = smp_processor_id();
1da177e4 LT	165
e5cd534a	166	if (cpu_context(cpu, vma->vm_mm) != 0) {
1da177e4 LT	167	unsigned long flags;
	168	int oldpid, newpid, idx;
	169
	170	#ifdef DEBUG_TLB
	171	printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
	172	#endif
4edf00a4	173	newpid = cpu_context(cpu, vma->vm_mm) & asid_mask;
1da177e4 LT	174	page &= PAGE_MASK;
1da177e4 LT	175	local_irq_save(flags);
4edf00a4	176	oldpid = read_c0_entryhi() & asid_mask;
1da177e4 LT	177	write_c0_entryhi(page \| newpid);
	178	BARRIER;
	179	tlb_probe();
	180	idx = read_c0_index();
	181	write_c0_entrylo0(0);
	182	write_c0_entryhi(KSEG0);
	183	if (idx < 0) /* BARRIER */
	184	goto finish;
	185	tlb_write_indexed();
	186
	187	finish:
	188	write_c0_entryhi(oldpid);
	189	local_irq_restore(flags);
	190	}
	191	}
	192
	193	void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
	194	{
4edf00a4	195	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
1da177e4 LT	196	unsigned long flags;
	197	int idx, pid;
	198
	199	/*
	200	* Handle debugger faulting in for debugee.
	201	*/
	202	if (current->active_mm != vma->vm_mm)
	203	return;
	204
4edf00a4	205	pid = read_c0_entryhi() & asid_mask;
1da177e4 LT	206
1da177e4 LT	207	#ifdef DEBUG_TLB
4edf00a4	208	if ((pid != (cpu_context(cpu, vma->vm_mm) & asid_mask)) \|\| (cpu_context(cpu, vma->vm_mm) == 0)) {
1da177e4 LT	209	printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
	210	(cpu_context(cpu, vma->vm_mm)), pid);
	211	}
	212	#endif
	213
	214	local_irq_save(flags);
	215	address &= PAGE_MASK;
	216	write_c0_entryhi(address \| pid);
	217	BARRIER;
	218	tlb_probe();
	219	idx = read_c0_index();
	220	write_c0_entrylo0(pte_val(pte));
	221	write_c0_entryhi(address \| pid);
	222	if (idx < 0) { /* BARRIER */
	223	tlb_write_random();
	224	} else {
	225	tlb_write_indexed();
	226	}
	227	write_c0_entryhi(pid);
	228	local_irq_restore(flags);
	229	}
	230
694b8c35 ML	231	void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
694b8c35 ML	232	unsigned long entryhi, unsigned long pagemask)
1da177e4	233	{
4edf00a4	234	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
1da177e4 LT	235	unsigned long flags;
	236	unsigned long old_ctx;
	237	static unsigned long wired = 0;
	238
	239	if (r3k_have_wired_reg) { /* TX39XX */
	240	unsigned long old_pagemask;
	241	unsigned long w;
	242
	243	#ifdef DEBUG_TLB
	244	printk("[tlbwired<entry lo0 %8x, hi %8x\n, pagemask %8x>]\n",
	245	entrylo0, entryhi, pagemask);
	246	#endif
	247
	248	local_irq_save(flags);
	249	/* Save old context and create impossible VPN2 value */
4edf00a4	250	old_ctx = read_c0_entryhi() & asid_mask;
1da177e4 LT	251	old_pagemask = read_c0_pagemask();
	252	w = read_c0_wired();
	253	write_c0_wired(w + 1);
1da177e4 LT	254	write_c0_index(w << 8);
	255	write_c0_pagemask(pagemask);
	256	write_c0_entryhi(entryhi);
	257	write_c0_entrylo0(entrylo0);
	258	BARRIER;
	259	tlb_write_indexed();
	260
	261	write_c0_entryhi(old_ctx);
	262	write_c0_pagemask(old_pagemask);
	263	local_flush_tlb_all();
	264	local_irq_restore(flags);
	265
	266	} else if (wired < 8) {
	267	#ifdef DEBUG_TLB
	268	printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
	269	entrylo0, entryhi);
	270	#endif
	271
	272	local_irq_save(flags);
4edf00a4	273	old_ctx = read_c0_entryhi() & asid_mask;
1da177e4 LT	274	write_c0_entrylo0(entrylo0);
	275	write_c0_entryhi(entryhi);
	276	write_c0_index(wired);
	277	wired++; /* BARRIER */
	278	tlb_write_indexed();
	279	write_c0_entryhi(old_ctx);
	280	local_flush_tlb_all();
	281	local_irq_restore(flags);
	282	}
	283	}
	284
078a55fc	285	void tlb_init(void)
1da177e4	286	{
3bcb03f3 MR	287	switch (current_cpu_type()) {
	288	case CPU_TX3922:
	289	case CPU_TX3927:
	290	r3k_have_wired_reg = 1;
	291	write_c0_wired(0); /* Set to 8 on reset... */
	292	break;
	293	}
dbfbf60f	294	local_flush_tlb_from(0);
1da177e4 LT	295	build_tlb_refill_handler();
1da177e4 LT	296	}