[deliverable/linux.git] / arch / mips / kvm / tlb.c

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
 * TLB handlers run from KSEG0
 *
 * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
 * Authors: Sanjay Lal <sanjayl@kymasys.com>
 */

#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/kvm_host.h>
#include <linux/srcu.h>

#include <asm/cpu.h>
#include <asm/bootinfo.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/cacheflush.h>
#include <asm/tlb.h>
#include <asm/tlbdebug.h>

#undef CONFIG_MIPS_MT
#include <asm/r4kcache.h>
#define CONFIG_MIPS_MT

#define KVM_GUEST_PC_TLB    0
#define KVM_GUEST_SP_TLB    1

atomic_t kvm_mips_instance;
EXPORT_SYMBOL_GPL(kvm_mips_instance);

static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
{
	int cpu = smp_processor_id();

	return vcpu->arch.guest_kernel_asid[cpu] &
			cpu_asid_mask(&cpu_data[cpu]);
}

static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
{
	int cpu = smp_processor_id();

	return vcpu->arch.guest_user_asid[cpu] &
			cpu_asid_mask(&cpu_data[cpu]);
}

inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.commpage_tlb;
}

/* Structure defining an tlb entry data set. */

void kvm_mips_dump_host_tlbs(void)
{
	unsigned long flags;

	local_irq_save(flags);

	kvm_info("HOST TLBs:\n");
	dump_tlb_regs();
	pr_info("\n");
	dump_tlb_all();

	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);

void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;
	struct kvm_mips_tlb tlb;
	int i;

	kvm_info("Guest TLBs:\n");
	kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));

	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
		tlb = vcpu->arch.guest_tlb[i];
		kvm_info("TLB%c%3d Hi 0x%08lx ",
			 (tlb.tlb_lo[0] | tlb.tlb_lo[1]) & ENTRYLO_V
							? ' ' : '*',
			 i, tlb.tlb_hi);
		kvm_info("Lo0=0x%09llx %c%c attr %lx ",
			 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]),
			 (tlb.tlb_lo[0] & ENTRYLO_D) ? 'D' : ' ',
			 (tlb.tlb_lo[0] & ENTRYLO_G) ? 'G' : ' ',
			 (tlb.tlb_lo[0] & ENTRYLO_C) >> ENTRYLO_C_SHIFT);
		kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
			 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]),
			 (tlb.tlb_lo[1] & ENTRYLO_D) ? 'D' : ' ',
			 (tlb.tlb_lo[1] & ENTRYLO_G) ? 'G' : ' ',
			 (tlb.tlb_lo[1] & ENTRYLO_C) >> ENTRYLO_C_SHIFT,
			 tlb.tlb_mask);
	}
}
EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);

/* XXXKYMA: Must be called with interrupts disabled */
/* set flush_dcache_mask == 0 if no dcache flush required */
int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi,
			    unsigned long entrylo0, unsigned long entrylo1,
			    int flush_dcache_mask)
{
	unsigned long flags;
	unsigned long old_entryhi;
	int idx;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();
	write_c0_entryhi(entryhi);
	mtc0_tlbw_hazard();

	tlb_probe();
	tlb_probe_hazard();
	idx = read_c0_index();

	if (idx > current_cpu_data.tlbsize) {
		kvm_err("%s: Invalid Index: %d\n", __func__, idx);
		kvm_mips_dump_host_tlbs();
		local_irq_restore(flags);
		return -1;
	}

	write_c0_entrylo0(entrylo0);
	write_c0_entrylo1(entrylo1);
	mtc0_tlbw_hazard();

	if (idx < 0)
		tlb_write_random();
	else
		tlb_write_indexed();
	tlbw_use_hazard();

	kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n",
		  vcpu->arch.pc, idx, read_c0_entryhi(),
		  read_c0_entrylo0(), read_c0_entrylo1());

	/* Flush D-cache */
	if (flush_dcache_mask) {
		if (entrylo0 & ENTRYLO_V) {
			++vcpu->stat.flush_dcache_exits;
			flush_data_cache_page((entryhi & VPN2_MASK) &
					      ~flush_dcache_mask);
		}
		if (entrylo1 & ENTRYLO_V) {
			++vcpu->stat.flush_dcache_exits;
			flush_data_cache_page(((entryhi & VPN2_MASK) &
					       ~flush_dcache_mask) |
					      (0x1 << PAGE_SHIFT));
		}
	}

	/* Restore old ASID */
	write_c0_entryhi(old_entryhi);
	mtc0_tlbw_hazard();
	local_irq_restore(flags);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_write);

int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
	struct kvm_vcpu *vcpu)
{
	kvm_pfn_t pfn;
	unsigned long flags, old_entryhi = 0, vaddr = 0;
	unsigned long entrylo[2] = { 0, 0 };
	unsigned int pair_idx;

	pfn = PFN_DOWN(virt_to_phys(vcpu->arch.kseg0_commpage));
	pair_idx = (badvaddr >> PAGE_SHIFT) & 1;
	entrylo[pair_idx] = mips3_paddr_to_tlbpfn(pfn << PAGE_SHIFT) |
		((_page_cachable_default >> _CACHE_SHIFT) << ENTRYLO_C_SHIFT) |
		ENTRYLO_D | ENTRYLO_V;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();
	vaddr = badvaddr & (PAGE_MASK << 1);
	write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu));
	write_c0_entrylo0(entrylo[0]);
	write_c0_entrylo1(entrylo[1]);
	write_c0_index(kvm_mips_get_commpage_asid(vcpu));
	mtc0_tlbw_hazard();
	tlb_write_indexed();
	tlbw_use_hazard();

	kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n",
		  vcpu->arch.pc, read_c0_index(), read_c0_entryhi(),
		  read_c0_entrylo0(), read_c0_entrylo1());

	/* Restore old ASID */
	write_c0_entryhi(old_entryhi);
	mtc0_tlbw_hazard();
	local_irq_restore(flags);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault);

int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
{
	int i;
	int index = -1;
	struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;

	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
		if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
		    TLB_HI_ASID_HIT(tlb[i], entryhi)) {
			index = i;
			break;
		}
	}

	kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
		  __func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]);

	return index;
}
EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);

int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr)
{
	unsigned long old_entryhi, flags;
	int idx;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();

	if (KVM_GUEST_KERNEL_MODE(vcpu))
		write_c0_entryhi((vaddr & VPN2_MASK) |
				 kvm_mips_get_kernel_asid(vcpu));
	else {
		write_c0_entryhi((vaddr & VPN2_MASK) |
				 kvm_mips_get_user_asid(vcpu));
	}

	mtc0_tlbw_hazard();

	tlb_probe();
	tlb_probe_hazard();
	idx = read_c0_index();

	/* Restore old ASID */
	write_c0_entryhi(old_entryhi);
	mtc0_tlbw_hazard();

	local_irq_restore(flags);

	kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx);

	return idx;
}
EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup);

int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
{
	int idx;
	unsigned long flags, old_entryhi;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();

	write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu));
	mtc0_tlbw_hazard();

	tlb_probe();
	tlb_probe_hazard();
	idx = read_c0_index();

	if (idx >= current_cpu_data.tlbsize)
		BUG();

	if (idx > 0) {
		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
		write_c0_entrylo0(0);
		write_c0_entrylo1(0);
		mtc0_tlbw_hazard();

		tlb_write_indexed();
		tlbw_use_hazard();
	}

	write_c0_entryhi(old_entryhi);
	mtc0_tlbw_hazard();

	local_irq_restore(flags);

	if (idx > 0)
		kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__,
			  (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);

void kvm_mips_flush_host_tlb(int skip_kseg0)
{
	unsigned long flags;
	unsigned long old_entryhi, entryhi;
	unsigned long old_pagemask;
	int entry = 0;
	int maxentry = current_cpu_data.tlbsize;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();
	old_pagemask = read_c0_pagemask();

	/* Blast 'em all away. */
	for (entry = 0; entry < maxentry; entry++) {
		write_c0_index(entry);

		if (skip_kseg0) {
			mtc0_tlbr_hazard();
			tlb_read();
			tlb_read_hazard();

			entryhi = read_c0_entryhi();

			/* Don't blow away guest kernel entries */
			if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0)
				continue;

			write_c0_pagemask(old_pagemask);
		}

		/* Make sure all entries differ. */
		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
		write_c0_entrylo0(0);
		write_c0_entrylo1(0);
		mtc0_tlbw_hazard();

		tlb_write_indexed();
		tlbw_use_hazard();
	}

	write_c0_entryhi(old_entryhi);
	write_c0_pagemask(old_pagemask);
	mtc0_tlbw_hazard();

	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb);

void kvm_local_flush_tlb_all(void)
{
	unsigned long flags;
	unsigned long old_ctx;
	int entry = 0;

	local_irq_save(flags);
	/* Save old context and create impossible VPN2 value */
	old_ctx = read_c0_entryhi();
	write_c0_entrylo0(0);
	write_c0_entrylo1(0);

	/* Blast 'em all away. */
	while (entry < current_cpu_data.tlbsize) {
		/* Make sure all entries differ. */
		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
		write_c0_index(entry);
		mtc0_tlbw_hazard();
		tlb_write_indexed();
		tlbw_use_hazard();
		entry++;
	}
	write_c0_entryhi(old_ctx);
	mtc0_tlbw_hazard();

	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all);
Commit	Line	Data
858dd5d4	1	/*
d116e812 DCZ	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
	7	* TLB handlers run from KSEG0
	8	*
	9	* Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved.
	10	* Authors: Sanjay Lal <sanjayl@kymasys.com>
	11	*/
858dd5d4	12
858dd5d4 SL	13	#include <linux/sched.h>
	14	#include <linux/smp.h>
	15	#include <linux/mm.h>
	16	#include <linux/delay.h>
403015b3	17	#include <linux/export.h>
858dd5d4	18	#include <linux/kvm_host.h>
6d17c0d1 SL	19	#include <linux/srcu.h>
6d17c0d1 SL	20
858dd5d4 SL	21	#include <asm/cpu.h>
	22	#include <asm/bootinfo.h>
	23	#include <asm/mmu_context.h>
	24	#include <asm/pgtable.h>
	25	#include <asm/cacheflush.h>
e36059e5	26	#include <asm/tlb.h>
e922a4cb	27	#include <asm/tlbdebug.h>
858dd5d4 SL	28
	29	#undef CONFIG_MIPS_MT
	30	#include <asm/r4kcache.h>
	31	#define CONFIG_MIPS_MT
	32
	33	#define KVM_GUEST_PC_TLB 0
	34	#define KVM_GUEST_SP_TLB 1
	35
858dd5d4	36	atomic_t kvm_mips_instance;
cb1b447f	37	EXPORT_SYMBOL_GPL(kvm_mips_instance);
858dd5d4	38
403015b3	39	static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
858dd5d4	40	{
4edf00a4 PB	41	int cpu = smp_processor_id();
	42
	43	return vcpu->arch.guest_kernel_asid[cpu] &
	44	cpu_asid_mask(&cpu_data[cpu]);
858dd5d4 SL	45	}
858dd5d4 SL	46
403015b3	47	static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
858dd5d4	48	{
4edf00a4 PB	49	int cpu = smp_processor_id();
	50
	51	return vcpu->arch.guest_user_asid[cpu] &
	52	cpu_asid_mask(&cpu_data[cpu]);
858dd5d4 SL	53	}
858dd5d4 SL	54
bdb7ed86	55	inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu)
858dd5d4 SL	56	{
	57	return vcpu->kvm->arch.commpage_tlb;
	58	}
	59
d116e812	60	/* Structure defining an tlb entry data set. */
858dd5d4 SL	61
	62	void kvm_mips_dump_host_tlbs(void)
	63	{
858dd5d4	64	unsigned long flags;
858dd5d4 SL	65
	66	local_irq_save(flags);
	67
6ad78a5c	68	kvm_info("HOST TLBs:\n");
e922a4cb JH	69	dump_tlb_regs();
	70	pr_info("\n");
	71	dump_tlb_all();
	72
858dd5d4 SL	73	local_irq_restore(flags);
858dd5d4 SL	74	}
cb1b447f	75	EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);
858dd5d4 SL	76
	77	void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
	78	{
	79	struct mips_coproc *cop0 = vcpu->arch.cop0;
	80	struct kvm_mips_tlb tlb;
	81	int i;
	82
6ad78a5c DCZ	83	kvm_info("Guest TLBs:\n");
6ad78a5c DCZ	84	kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));
858dd5d4 SL	85
	86	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
	87	tlb = vcpu->arch.guest_tlb[i];
6ad78a5c	88	kvm_info("TLB%c%3d Hi 0x%08lx ",
e6207bbe	89	(tlb.tlb_lo[0] \| tlb.tlb_lo[1]) & ENTRYLO_V
9fbfb06a	90	? ' ' : '*',
6ad78a5c	91	i, tlb.tlb_hi);
8cffd197	92	kvm_info("Lo0=0x%09llx %c%c attr %lx ",
9fbfb06a	93	(u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]),
e6207bbe JH	94	(tlb.tlb_lo[0] & ENTRYLO_D) ? 'D' : ' ',
	95	(tlb.tlb_lo[0] & ENTRYLO_G) ? 'G' : ' ',
	96	(tlb.tlb_lo[0] & ENTRYLO_C) >> ENTRYLO_C_SHIFT);
8cffd197	97	kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
9fbfb06a	98	(u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]),
e6207bbe JH	99	(tlb.tlb_lo[1] & ENTRYLO_D) ? 'D' : ' ',
	100	(tlb.tlb_lo[1] & ENTRYLO_G) ? 'G' : ' ',
	101	(tlb.tlb_lo[1] & ENTRYLO_C) >> ENTRYLO_C_SHIFT,
	102	tlb.tlb_mask);
858dd5d4 SL	103	}
858dd5d4 SL	104	}
cb1b447f	105	EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);
858dd5d4	106
858dd5d4 SL	107	/* XXXKYMA: Must be called with interrupts disabled */
858dd5d4 SL	108	/* set flush_dcache_mask == 0 if no dcache flush required */
d116e812 DCZ	109	int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi,
	110	unsigned long entrylo0, unsigned long entrylo1,
	111	int flush_dcache_mask)
858dd5d4 SL	112	{
	113	unsigned long flags;
	114	unsigned long old_entryhi;
b045c406	115	int idx;
858dd5d4 SL	116
	117	local_irq_save(flags);
	118
858dd5d4 SL	119	old_entryhi = read_c0_entryhi();
	120	write_c0_entryhi(entryhi);
	121	mtc0_tlbw_hazard();
	122
	123	tlb_probe();
	124	tlb_probe_hazard();
	125	idx = read_c0_index();
	126
	127	if (idx > current_cpu_data.tlbsize) {
	128	kvm_err("%s: Invalid Index: %d\n", __func__, idx);
	129	kvm_mips_dump_host_tlbs();
cfec0e75	130	local_irq_restore(flags);
858dd5d4 SL	131	return -1;
	132	}
	133
858dd5d4 SL	134	write_c0_entrylo0(entrylo0);
	135	write_c0_entrylo1(entrylo1);
	136	mtc0_tlbw_hazard();
	137
b5dfc6c1 JH	138	if (idx < 0)
	139	tlb_write_random();
	140	else
	141	tlb_write_indexed();
858dd5d4 SL	142	tlbw_use_hazard();
858dd5d4 SL	143
3d654833 JH	144	kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n",
	145	vcpu->arch.pc, idx, read_c0_entryhi(),
	146	read_c0_entrylo0(), read_c0_entrylo1());
858dd5d4 SL	147
	148	/* Flush D-cache */
	149	if (flush_dcache_mask) {
e6207bbe	150	if (entrylo0 & ENTRYLO_V) {
858dd5d4	151	++vcpu->stat.flush_dcache_exits;
d116e812 DCZ	152	flush_data_cache_page((entryhi & VPN2_MASK) &
d116e812 DCZ	153	~flush_dcache_mask);
858dd5d4	154	}
e6207bbe	155	if (entrylo1 & ENTRYLO_V) {
858dd5d4	156	++vcpu->stat.flush_dcache_exits;
d116e812 DCZ	157	flush_data_cache_page(((entryhi & VPN2_MASK) &
	158	~flush_dcache_mask) \|
	159	(0x1 << PAGE_SHIFT));
858dd5d4 SL	160	}
	161	}
	162
	163	/* Restore old ASID */
	164	write_c0_entryhi(old_entryhi);
	165	mtc0_tlbw_hazard();
858dd5d4 SL	166	local_irq_restore(flags);
	167	return 0;
	168	}
403015b3	169	EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_write);
858dd5d4 SL	170
	171	int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
	172	struct kvm_vcpu *vcpu)
	173	{
42aa12e7	174	kvm_pfn_t pfn;
858dd5d4	175	unsigned long flags, old_entryhi = 0, vaddr = 0;
42aa12e7 JH	176	unsigned long entrylo[2] = { 0, 0 };
42aa12e7 JH	177	unsigned int pair_idx;
858dd5d4	178
cfacaced	179	pfn = PFN_DOWN(virt_to_phys(vcpu->arch.kseg0_commpage));
42aa12e7 JH	180	pair_idx = (badvaddr >> PAGE_SHIFT) & 1;
42aa12e7 JH	181	entrylo[pair_idx] = mips3_paddr_to_tlbpfn(pfn << PAGE_SHIFT) \|
7414d2f6 JH	182	((_page_cachable_default >> _CACHE_SHIFT) << ENTRYLO_C_SHIFT) \|
7414d2f6 JH	183	ENTRYLO_D \| ENTRYLO_V;
858dd5d4 SL	184
	185	local_irq_save(flags);
	186
	187	old_entryhi = read_c0_entryhi();
	188	vaddr = badvaddr & (PAGE_MASK << 1);
	189	write_c0_entryhi(vaddr \| kvm_mips_get_kernel_asid(vcpu));
42aa12e7 JH	190	write_c0_entrylo0(entrylo[0]);
42aa12e7 JH	191	write_c0_entrylo1(entrylo[1]);
858dd5d4 SL	192	write_c0_index(kvm_mips_get_commpage_asid(vcpu));
	193	mtc0_tlbw_hazard();
	194	tlb_write_indexed();
858dd5d4 SL	195	tlbw_use_hazard();
858dd5d4 SL	196
d116e812 DCZ	197	kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n",
	198	vcpu->arch.pc, read_c0_index(), read_c0_entryhi(),
	199	read_c0_entrylo0(), read_c0_entrylo1());
858dd5d4 SL	200
	201	/* Restore old ASID */
	202	write_c0_entryhi(old_entryhi);
	203	mtc0_tlbw_hazard();
858dd5d4 SL	204	local_irq_restore(flags);
	205
	206	return 0;
	207	}
cb1b447f	208	EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault);
858dd5d4	209
858dd5d4 SL	210	int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
	211	{
	212	int i;
	213	int index = -1;
	214	struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;
	215
858dd5d4	216	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
d116e812 DCZ	217	if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
d116e812 DCZ	218	TLB_HI_ASID_HIT(tlb[i], entryhi)) {
858dd5d4 SL	219	index = i;
	220	break;
	221	}
	222	}
	223
858dd5d4	224	kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
9fbfb06a	225	__func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]);
858dd5d4 SL	226
	227	return index;
	228	}
cb1b447f	229	EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);
858dd5d4 SL	230
	231	int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr)
	232	{
	233	unsigned long old_entryhi, flags;
b045c406	234	int idx;
858dd5d4	235
858dd5d4 SL	236	local_irq_save(flags);
	237
	238	old_entryhi = read_c0_entryhi();
	239
	240	if (KVM_GUEST_KERNEL_MODE(vcpu))
d116e812 DCZ	241	write_c0_entryhi((vaddr & VPN2_MASK) \|
d116e812 DCZ	242	kvm_mips_get_kernel_asid(vcpu));
858dd5d4	243	else {
d116e812 DCZ	244	write_c0_entryhi((vaddr & VPN2_MASK) \|
d116e812 DCZ	245	kvm_mips_get_user_asid(vcpu));
858dd5d4 SL	246	}
	247
	248	mtc0_tlbw_hazard();
	249
	250	tlb_probe();
	251	tlb_probe_hazard();
	252	idx = read_c0_index();
	253
	254	/* Restore old ASID */
	255	write_c0_entryhi(old_entryhi);
	256	mtc0_tlbw_hazard();
858dd5d4 SL	257
	258	local_irq_restore(flags);
	259
858dd5d4	260	kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx);
858dd5d4 SL	261
	262	return idx;
	263	}
cb1b447f	264	EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup);
858dd5d4 SL	265
	266	int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
	267	{
	268	int idx;
	269	unsigned long flags, old_entryhi;
	270
	271	local_irq_save(flags);
	272
858dd5d4 SL	273	old_entryhi = read_c0_entryhi();
	274
	275	write_c0_entryhi((va & VPN2_MASK) \| kvm_mips_get_user_asid(vcpu));
	276	mtc0_tlbw_hazard();
	277
	278	tlb_probe();
	279	tlb_probe_hazard();
	280	idx = read_c0_index();
	281
	282	if (idx >= current_cpu_data.tlbsize)
	283	BUG();
	284
	285	if (idx > 0) {
	286	write_c0_entryhi(UNIQUE_ENTRYHI(idx));
858dd5d4	287	write_c0_entrylo0(0);
858dd5d4 SL	288	write_c0_entrylo1(0);
	289	mtc0_tlbw_hazard();
	290
	291	tlb_write_indexed();
138f7ad9	292	tlbw_use_hazard();
858dd5d4 SL	293	}
	294
	295	write_c0_entryhi(old_entryhi);
	296	mtc0_tlbw_hazard();
858dd5d4 SL	297
	298	local_irq_restore(flags);
	299
3d654833	300	if (idx > 0)
858dd5d4	301	kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__,
3d654833	302	(va & VPN2_MASK) \| kvm_mips_get_user_asid(vcpu), idx);
858dd5d4 SL	303
	304	return 0;
	305	}
cb1b447f	306	EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);
858dd5d4 SL	307
	308	void kvm_mips_flush_host_tlb(int skip_kseg0)
	309	{
	310	unsigned long flags;
	311	unsigned long old_entryhi, entryhi;
	312	unsigned long old_pagemask;
	313	int entry = 0;
	314	int maxentry = current_cpu_data.tlbsize;
	315
858dd5d4 SL	316	local_irq_save(flags);
	317
	318	old_entryhi = read_c0_entryhi();
	319	old_pagemask = read_c0_pagemask();
	320
	321	/* Blast 'em all away. */
	322	for (entry = 0; entry < maxentry; entry++) {
858dd5d4	323	write_c0_index(entry);
858dd5d4 SL	324
858dd5d4 SL	325	if (skip_kseg0) {
138f7ad9	326	mtc0_tlbr_hazard();
858dd5d4	327	tlb_read();
138f7ad9	328	tlb_read_hazard();
858dd5d4 SL	329
	330	entryhi = read_c0_entryhi();
	331
	332	/* Don't blow away guest kernel entries */
d116e812	333	if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0)
858dd5d4	334	continue;
a700434d JH	335
a700434d JH	336	write_c0_pagemask(old_pagemask);
858dd5d4 SL	337	}
	338
	339	/* Make sure all entries differ. */
	340	write_c0_entryhi(UNIQUE_ENTRYHI(entry));
858dd5d4	341	write_c0_entrylo0(0);
858dd5d4 SL	342	write_c0_entrylo1(0);
	343	mtc0_tlbw_hazard();
	344
	345	tlb_write_indexed();
138f7ad9	346	tlbw_use_hazard();
858dd5d4 SL	347	}
858dd5d4 SL	348
858dd5d4 SL	349	write_c0_entryhi(old_entryhi);
	350	write_c0_pagemask(old_pagemask);
	351	mtc0_tlbw_hazard();
858dd5d4 SL	352
	353	local_irq_restore(flags);
	354	}
cb1b447f	355	EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb);
858dd5d4	356
858dd5d4 SL	357	void kvm_local_flush_tlb_all(void)
	358	{
	359	unsigned long flags;
	360	unsigned long old_ctx;
	361	int entry = 0;
	362
	363	local_irq_save(flags);
	364	/* Save old context and create impossible VPN2 value */
	365	old_ctx = read_c0_entryhi();
	366	write_c0_entrylo0(0);
	367	write_c0_entrylo1(0);
	368
	369	/* Blast 'em all away. */
	370	while (entry < current_cpu_data.tlbsize) {
	371	/* Make sure all entries differ. */
	372	write_c0_entryhi(UNIQUE_ENTRYHI(entry));
	373	write_c0_index(entry);
	374	mtc0_tlbw_hazard();
	375	tlb_write_indexed();
138f7ad9	376	tlbw_use_hazard();
858dd5d4 SL	377	entry++;
858dd5d4 SL	378	}
858dd5d4 SL	379	write_c0_entryhi(old_ctx);
	380	mtc0_tlbw_hazard();
	381
	382	local_irq_restore(flags);
	383	}
cb1b447f	384	EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all);