[deliverable/linux.git] / arch / mips / mm / tlb-r3k.c

/*
 * r2300.c: R2000 and R3000 specific mmu/cache code.
 *
 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
 *
 * with a lot of changes to make this thing work for R3000s
 * Tx39XX R4k style caches added. HK
 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
 * Copyright (C) 2002  Ralf Baechle
 * Copyright (C) 2002  Maciej W. Rozycki
 */
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>

#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/mmu_context.h>
#include <asm/tlbmisc.h>
#include <asm/isadep.h>
#include <asm/io.h>
#include <asm/bootinfo.h>
#include <asm/cpu.h>

#undef DEBUG_TLB

extern void build_tlb_refill_handler(void);

/* CP0 hazard avoidance. */
#define BARRIER				\
	__asm__ __volatile__(		\
		".set	push\n\t"	\
		".set	noreorder\n\t"	\
		"nop\n\t"		\
		".set	pop\n\t")

static int r3k_have_wired_reg;			/* Should be in cpu_data? */

/* TLB operations. */
static void local_flush_tlb_from(int entry)
{
	unsigned long old_ctx;

	old_ctx = read_c0_entryhi() & ASID_MASK;
	write_c0_entrylo0(0);
	while (entry < current_cpu_data.tlbsize) {
		write_c0_index(entry << 8);
		write_c0_entryhi((entry | 0x80000) << 12);
		entry++;				/* BARRIER */
		tlb_write_indexed();
	}
	write_c0_entryhi(old_ctx);
}

void local_flush_tlb_all(void)
{
	unsigned long flags;

#ifdef DEBUG_TLB
	printk("[tlball]");
#endif
	local_irq_save(flags);
	local_flush_tlb_from(r3k_have_wired_reg ? read_c0_wired() : 8);
	local_irq_restore(flags);
}

void local_flush_tlb_mm(struct mm_struct *mm)
{
	int cpu = smp_processor_id();

	if (cpu_context(cpu, mm) != 0) {
#ifdef DEBUG_TLB
		printk("[tlbmm<%lu>]", (unsigned long)cpu_context(cpu, mm));
#endif
		drop_mmu_context(mm, cpu);
	}
}

void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
			   unsigned long end)
{
	struct mm_struct *mm = vma->vm_mm;
	int cpu = smp_processor_id();

	if (cpu_context(cpu, mm) != 0) {
		unsigned long size, flags;

#ifdef DEBUG_TLB
		printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
			cpu_context(cpu, mm) & ASID_MASK, start, end);
#endif
		local_irq_save(flags);
		size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
		if (size <= current_cpu_data.tlbsize) {
			int oldpid = read_c0_entryhi() & ASID_MASK;
			int newpid = cpu_context(cpu, mm) & ASID_MASK;

			start &= PAGE_MASK;
			end += PAGE_SIZE - 1;
			end &= PAGE_MASK;
			while (start < end) {
				int idx;

				write_c0_entryhi(start | newpid);
				start += PAGE_SIZE;	/* BARRIER */
				tlb_probe();
				idx = read_c0_index();
				write_c0_entrylo0(0);
				write_c0_entryhi(KSEG0);
				if (idx < 0)		/* BARRIER */
					continue;
				tlb_write_indexed();
			}
			write_c0_entryhi(oldpid);
		} else {
			drop_mmu_context(mm, cpu);
		}
		local_irq_restore(flags);
	}
}

void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	unsigned long size, flags;

#ifdef DEBUG_TLB
	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
#endif
	local_irq_save(flags);
	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	if (size <= current_cpu_data.tlbsize) {
		int pid = read_c0_entryhi();

		start &= PAGE_MASK;
		end += PAGE_SIZE - 1;
		end &= PAGE_MASK;

		while (start < end) {
			int idx;

			write_c0_entryhi(start);
			start += PAGE_SIZE;		/* BARRIER */
			tlb_probe();
			idx = read_c0_index();
			write_c0_entrylo0(0);
			write_c0_entryhi(KSEG0);
			if (idx < 0)			/* BARRIER */
				continue;
			tlb_write_indexed();
		}
		write_c0_entryhi(pid);
	} else {
		local_flush_tlb_all();
	}
	local_irq_restore(flags);
}

void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
	int cpu = smp_processor_id();

	if (cpu_context(cpu, vma->vm_mm) != 0) {
		unsigned long flags;
		int oldpid, newpid, idx;

#ifdef DEBUG_TLB
		printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
#endif
		newpid = cpu_context(cpu, vma->vm_mm) & ASID_MASK;
		page &= PAGE_MASK;
		local_irq_save(flags);
		oldpid = read_c0_entryhi() & ASID_MASK;
		write_c0_entryhi(page | newpid);
		BARRIER;
		tlb_probe();
		idx = read_c0_index();
		write_c0_entrylo0(0);
		write_c0_entryhi(KSEG0);
		if (idx < 0)				/* BARRIER */
			goto finish;
		tlb_write_indexed();

finish:
		write_c0_entryhi(oldpid);
		local_irq_restore(flags);
	}
}

void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
{
	unsigned long flags;
	int idx, pid;

	/*
	 * Handle debugger faulting in for debugee.
	 */
	if (current->active_mm != vma->vm_mm)
		return;

	pid = read_c0_entryhi() & ASID_MASK;

#ifdef DEBUG_TLB
	if ((pid != (cpu_context(cpu, vma->vm_mm) & ASID_MASK)) || (cpu_context(cpu, vma->vm_mm) == 0)) {
		printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
		       (cpu_context(cpu, vma->vm_mm)), pid);
	}
#endif

	local_irq_save(flags);
	address &= PAGE_MASK;
	write_c0_entryhi(address | pid);
	BARRIER;
	tlb_probe();
	idx = read_c0_index();
	write_c0_entrylo0(pte_val(pte));
	write_c0_entryhi(address | pid);
	if (idx < 0) {					/* BARRIER */
		tlb_write_random();
	} else {
		tlb_write_indexed();
	}
	write_c0_entryhi(pid);
	local_irq_restore(flags);
}

void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
		     unsigned long entryhi, unsigned long pagemask)
{
	unsigned long flags;
	unsigned long old_ctx;
	static unsigned long wired = 0;

	if (r3k_have_wired_reg) {			/* TX39XX */
		unsigned long old_pagemask;
		unsigned long w;

#ifdef DEBUG_TLB
		printk("[tlbwired<entry lo0 %8x, hi %8x\n, pagemask %8x>]\n",
		       entrylo0, entryhi, pagemask);
#endif

		local_irq_save(flags);
		/* Save old context and create impossible VPN2 value */
		old_ctx = read_c0_entryhi() & ASID_MASK;
		old_pagemask = read_c0_pagemask();
		w = read_c0_wired();
		write_c0_wired(w + 1);
		write_c0_index(w << 8);
		write_c0_pagemask(pagemask);
		write_c0_entryhi(entryhi);
		write_c0_entrylo0(entrylo0);
		BARRIER;
		tlb_write_indexed();

		write_c0_entryhi(old_ctx);
		write_c0_pagemask(old_pagemask);
		local_flush_tlb_all();
		local_irq_restore(flags);

	} else if (wired < 8) {
#ifdef DEBUG_TLB
		printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
		       entrylo0, entryhi);
#endif

		local_irq_save(flags);
		old_ctx = read_c0_entryhi() & ASID_MASK;
		write_c0_entrylo0(entrylo0);
		write_c0_entryhi(entryhi);
		write_c0_index(wired);
		wired++;				/* BARRIER */
		tlb_write_indexed();
		write_c0_entryhi(old_ctx);
		local_flush_tlb_all();
		local_irq_restore(flags);
	}
}

void tlb_init(void)
{
	switch (current_cpu_type()) {
	case CPU_TX3922:
	case CPU_TX3927:
		r3k_have_wired_reg = 1;
		write_c0_wired(0);		/* Set to 8 on reset... */
		break;
	}
	local_flush_tlb_from(0);
	build_tlb_refill_handler();
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* r2300.c: R2000 and R3000 specific mmu/cache code.
	3	*
79add627	4	* Copyright (C) 1996 David S. Miller (davem@davemloft.net)
1da177e4 LT	5	*
	6	* with a lot of changes to make this thing work for R3000s
	7	* Tx39XX R4k style caches added. HK
	8	* Copyright (C) 1998, 1999, 2000 Harald Koerfgen
	9	* Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
	10	* Copyright (C) 2002 Ralf Baechle
	11	* Copyright (C) 2002 Maciej W. Rozycki
	12	*/
1da177e4 LT	13	#include <linux/kernel.h>
1da177e4 LT	14	#include <linux/sched.h>
631330f5	15	#include <linux/smp.h>
1da177e4 LT	16	#include <linux/mm.h>
	17
	18	#include <asm/page.h>
	19	#include <asm/pgtable.h>
	20	#include <asm/mmu_context.h>
3d18c983	21	#include <asm/tlbmisc.h>
1da177e4 LT	22	#include <asm/isadep.h>
	23	#include <asm/io.h>
	24	#include <asm/bootinfo.h>
	25	#include <asm/cpu.h>
	26
	27	#undef DEBUG_TLB
	28
	29	extern void build_tlb_refill_handler(void);
	30
	31	/* CP0 hazard avoidance. */
	32	#define BARRIER \
	33	__asm__ __volatile__( \
	34	".set push\n\t" \
	35	".set noreorder\n\t" \
	36	"nop\n\t" \
	37	".set pop\n\t")
	38
3bcb03f3	39	static int r3k_have_wired_reg; /* Should be in cpu_data? */
1da177e4 LT	40
1da177e4 LT	41	/* TLB operations. */
dbfbf60f	42	static void local_flush_tlb_from(int entry)
1da177e4	43	{
1da177e4	44	unsigned long old_ctx;
1da177e4	45
48c4ac97	46	old_ctx = read_c0_entryhi() & ASID_MASK;
1da177e4	47	write_c0_entrylo0(0);
9a20b092	48	while (entry < current_cpu_data.tlbsize) {
1da177e4 LT	49	write_c0_index(entry << 8);
1da177e4 LT	50	write_c0_entryhi((entry \| 0x80000) << 12);
9a20b092	51	entry++; /* BARRIER */
1da177e4 LT	52	tlb_write_indexed();
	53	}
	54	write_c0_entryhi(old_ctx);
dbfbf60f MR	55	}
	56
	57	void local_flush_tlb_all(void)
	58	{
	59	unsigned long flags;
	60
	61	#ifdef DEBUG_TLB
	62	printk("[tlball]");
	63	#endif
	64	local_irq_save(flags);
	65	local_flush_tlb_from(r3k_have_wired_reg ? read_c0_wired() : 8);
1da177e4 LT	66	local_irq_restore(flags);
	67	}
	68
	69	void local_flush_tlb_mm(struct mm_struct *mm)
	70	{
	71	int cpu = smp_processor_id();
	72
	73	if (cpu_context(cpu, mm) != 0) {
	74	#ifdef DEBUG_TLB
	75	printk("[tlbmm<%lu>]", (unsigned long)cpu_context(cpu, mm));
	76	#endif
	77	drop_mmu_context(mm, cpu);
	78	}
	79	}
	80
	81	void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	82	unsigned long end)
	83	{
	84	struct mm_struct *mm = vma->vm_mm;
	85	int cpu = smp_processor_id();
	86
	87	if (cpu_context(cpu, mm) != 0) {
a5e696e5	88	unsigned long size, flags;
1da177e4 LT	89
	90	#ifdef DEBUG_TLB
	91	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
48c4ac97	92	cpu_context(cpu, mm) & ASID_MASK, start, end);
1da177e4 LT	93	#endif
	94	local_irq_save(flags);
	95	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	96	if (size <= current_cpu_data.tlbsize) {
48c4ac97 DD	97	int oldpid = read_c0_entryhi() & ASID_MASK;
48c4ac97 DD	98	int newpid = cpu_context(cpu, mm) & ASID_MASK;
1da177e4 LT	99
	100	start &= PAGE_MASK;
	101	end += PAGE_SIZE - 1;
	102	end &= PAGE_MASK;
	103	while (start < end) {
	104	int idx;
	105
	106	write_c0_entryhi(start \| newpid);
	107	start += PAGE_SIZE; /* BARRIER */
	108	tlb_probe();
	109	idx = read_c0_index();
	110	write_c0_entrylo0(0);
	111	write_c0_entryhi(KSEG0);
	112	if (idx < 0) /* BARRIER */
	113	continue;
	114	tlb_write_indexed();
	115	}
	116	write_c0_entryhi(oldpid);
	117	} else {
	118	drop_mmu_context(mm, cpu);
	119	}
	120	local_irq_restore(flags);
	121	}
	122	}
	123
	124	void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
	125	{
a5e696e5	126	unsigned long size, flags;
1da177e4 LT	127
	128	#ifdef DEBUG_TLB
	129	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
	130	#endif
	131	local_irq_save(flags);
	132	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	133	if (size <= current_cpu_data.tlbsize) {
	134	int pid = read_c0_entryhi();
	135
	136	start &= PAGE_MASK;
	137	end += PAGE_SIZE - 1;
	138	end &= PAGE_MASK;
	139
	140	while (start < end) {
	141	int idx;
	142
	143	write_c0_entryhi(start);
	144	start += PAGE_SIZE; /* BARRIER */
	145	tlb_probe();
	146	idx = read_c0_index();
	147	write_c0_entrylo0(0);
	148	write_c0_entryhi(KSEG0);
	149	if (idx < 0) /* BARRIER */
	150	continue;
	151	tlb_write_indexed();
	152	}
	153	write_c0_entryhi(pid);
	154	} else {
	155	local_flush_tlb_all();
	156	}
	157	local_irq_restore(flags);
	158	}
	159
	160	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
	161	{
	162	int cpu = smp_processor_id();
	163
e5cd534a	164	if (cpu_context(cpu, vma->vm_mm) != 0) {
1da177e4 LT	165	unsigned long flags;
	166	int oldpid, newpid, idx;
	167
	168	#ifdef DEBUG_TLB
	169	printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
	170	#endif
48c4ac97	171	newpid = cpu_context(cpu, vma->vm_mm) & ASID_MASK;
1da177e4 LT	172	page &= PAGE_MASK;
1da177e4 LT	173	local_irq_save(flags);
48c4ac97	174	oldpid = read_c0_entryhi() & ASID_MASK;
1da177e4 LT	175	write_c0_entryhi(page \| newpid);
	176	BARRIER;
	177	tlb_probe();
	178	idx = read_c0_index();
	179	write_c0_entrylo0(0);
	180	write_c0_entryhi(KSEG0);
	181	if (idx < 0) /* BARRIER */
	182	goto finish;
	183	tlb_write_indexed();
	184
	185	finish:
	186	write_c0_entryhi(oldpid);
	187	local_irq_restore(flags);
	188	}
	189	}
	190
	191	void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
	192	{
	193	unsigned long flags;
	194	int idx, pid;
	195
	196	/*
	197	* Handle debugger faulting in for debugee.
	198	*/
	199	if (current->active_mm != vma->vm_mm)
	200	return;
	201
48c4ac97	202	pid = read_c0_entryhi() & ASID_MASK;
1da177e4 LT	203
1da177e4 LT	204	#ifdef DEBUG_TLB
48c4ac97	205	if ((pid != (cpu_context(cpu, vma->vm_mm) & ASID_MASK)) \|\| (cpu_context(cpu, vma->vm_mm) == 0)) {
1da177e4 LT	206	printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
	207	(cpu_context(cpu, vma->vm_mm)), pid);
	208	}
	209	#endif
	210
	211	local_irq_save(flags);
	212	address &= PAGE_MASK;
	213	write_c0_entryhi(address \| pid);
	214	BARRIER;
	215	tlb_probe();
	216	idx = read_c0_index();
	217	write_c0_entrylo0(pte_val(pte));
	218	write_c0_entryhi(address \| pid);
	219	if (idx < 0) { /* BARRIER */
	220	tlb_write_random();
	221	} else {
	222	tlb_write_indexed();
	223	}
	224	write_c0_entryhi(pid);
	225	local_irq_restore(flags);
	226	}
	227
694b8c35 ML	228	void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
694b8c35 ML	229	unsigned long entryhi, unsigned long pagemask)
1da177e4 LT	230	{
	231	unsigned long flags;
	232	unsigned long old_ctx;
	233	static unsigned long wired = 0;
	234
	235	if (r3k_have_wired_reg) { /* TX39XX */
	236	unsigned long old_pagemask;
	237	unsigned long w;
	238
	239	#ifdef DEBUG_TLB
	240	printk("[tlbwired<entry lo0 %8x, hi %8x\n, pagemask %8x>]\n",
	241	entrylo0, entryhi, pagemask);
	242	#endif
	243
	244	local_irq_save(flags);
	245	/* Save old context and create impossible VPN2 value */
48c4ac97	246	old_ctx = read_c0_entryhi() & ASID_MASK;
1da177e4 LT	247	old_pagemask = read_c0_pagemask();
	248	w = read_c0_wired();
	249	write_c0_wired(w + 1);
1da177e4 LT	250	write_c0_index(w << 8);
	251	write_c0_pagemask(pagemask);
	252	write_c0_entryhi(entryhi);
	253	write_c0_entrylo0(entrylo0);
	254	BARRIER;
	255	tlb_write_indexed();
	256
	257	write_c0_entryhi(old_ctx);
	258	write_c0_pagemask(old_pagemask);
	259	local_flush_tlb_all();
	260	local_irq_restore(flags);
	261
	262	} else if (wired < 8) {
	263	#ifdef DEBUG_TLB
	264	printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
	265	entrylo0, entryhi);
	266	#endif
	267
	268	local_irq_save(flags);
48c4ac97	269	old_ctx = read_c0_entryhi() & ASID_MASK;
1da177e4 LT	270	write_c0_entrylo0(entrylo0);
	271	write_c0_entryhi(entryhi);
	272	write_c0_index(wired);
	273	wired++; /* BARRIER */
	274	tlb_write_indexed();
	275	write_c0_entryhi(old_ctx);
	276	local_flush_tlb_all();
	277	local_irq_restore(flags);
	278	}
	279	}
	280
078a55fc	281	void tlb_init(void)
1da177e4	282	{
3bcb03f3 MR	283	switch (current_cpu_type()) {
	284	case CPU_TX3922:
	285	case CPU_TX3927:
	286	r3k_have_wired_reg = 1;
	287	write_c0_wired(0); /* Set to 8 on reset... */
	288	break;
	289	}
dbfbf60f	290	local_flush_tlb_from(0);
1da177e4 LT	291	build_tlb_refill_handler();
1da177e4 LT	292	}