[deliverable/linux.git] / arch / mips / kvm / tlb.c

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
 * TLB handlers run from KSEG0
 *
 * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
 * Authors: Sanjay Lal <sanjayl@kymasys.com>
 */

#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/kvm_host.h>
#include <linux/srcu.h>

#include <asm/cpu.h>
#include <asm/bootinfo.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/cacheflush.h>
#include <asm/tlb.h>

#undef CONFIG_MIPS_MT
#include <asm/r4kcache.h>
#define CONFIG_MIPS_MT

#define KVM_GUEST_PC_TLB    0
#define KVM_GUEST_SP_TLB    1

atomic_t kvm_mips_instance;
EXPORT_SYMBOL_GPL(kvm_mips_instance);

/* These function pointers are initialized once the KVM module is loaded */
kvm_pfn_t (*kvm_mips_gfn_to_pfn)(struct kvm *kvm, gfn_t gfn);
EXPORT_SYMBOL_GPL(kvm_mips_gfn_to_pfn);

void (*kvm_mips_release_pfn_clean)(kvm_pfn_t pfn);
EXPORT_SYMBOL_GPL(kvm_mips_release_pfn_clean);

bool (*kvm_mips_is_error_pfn)(kvm_pfn_t pfn);
EXPORT_SYMBOL_GPL(kvm_mips_is_error_pfn);

static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
{
	int cpu = smp_processor_id();

	return vcpu->arch.guest_kernel_asid[cpu] &
			cpu_asid_mask(&cpu_data[cpu]);
}

static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
{
	int cpu = smp_processor_id();

	return vcpu->arch.guest_user_asid[cpu] &
			cpu_asid_mask(&cpu_data[cpu]);
}

inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu)
{
	return vcpu->kvm->arch.commpage_tlb;
}

/* Structure defining an tlb entry data set. */

void kvm_mips_dump_host_tlbs(void)
{
	unsigned long old_entryhi;
	unsigned long old_pagemask;
	struct kvm_mips_tlb tlb;
	unsigned long flags;
	int i;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();
	old_pagemask = read_c0_pagemask();

	kvm_info("HOST TLBs:\n");
	kvm_info("ASID: %#lx\n", read_c0_entryhi() &
		 cpu_asid_mask(&current_cpu_data));

	for (i = 0; i < current_cpu_data.tlbsize; i++) {
		write_c0_index(i);
		mtc0_tlbw_hazard();

		tlb_read();
		tlbw_use_hazard();

		tlb.tlb_hi = read_c0_entryhi();
		tlb.tlb_lo0 = read_c0_entrylo0();
		tlb.tlb_lo1 = read_c0_entrylo1();
		tlb.tlb_mask = read_c0_pagemask();

		kvm_info("TLB%c%3d Hi 0x%08lx ",
			 (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*',
			 i, tlb.tlb_hi);
		kvm_info("Lo0=0x%09llx %c%c attr %lx ",
			 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo0),
			 (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ',
			 (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ',
			 (tlb.tlb_lo0 >> 3) & 7);
		kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
			 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo1),
			 (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ',
			 (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ',
			 (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask);
	}
	write_c0_entryhi(old_entryhi);
	write_c0_pagemask(old_pagemask);
	mtc0_tlbw_hazard();
	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);

void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
{
	struct mips_coproc *cop0 = vcpu->arch.cop0;
	struct kvm_mips_tlb tlb;
	int i;

	kvm_info("Guest TLBs:\n");
	kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));

	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
		tlb = vcpu->arch.guest_tlb[i];
		kvm_info("TLB%c%3d Hi 0x%08lx ",
			 (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*',
			 i, tlb.tlb_hi);
		kvm_info("Lo0=0x%09llx %c%c attr %lx ",
			 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo0),
			 (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ',
			 (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ',
			 (tlb.tlb_lo0 >> 3) & 7);
		kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
			 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo1),
			 (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ',
			 (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ',
			 (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask);
	}
}
EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);

/* XXXKYMA: Must be called with interrupts disabled */
/* set flush_dcache_mask == 0 if no dcache flush required */
int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi,
			    unsigned long entrylo0, unsigned long entrylo1,
			    int flush_dcache_mask)
{
	unsigned long flags;
	unsigned long old_entryhi;
	int idx;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();
	write_c0_entryhi(entryhi);
	mtc0_tlbw_hazard();

	tlb_probe();
	tlb_probe_hazard();
	idx = read_c0_index();

	if (idx > current_cpu_data.tlbsize) {
		kvm_err("%s: Invalid Index: %d\n", __func__, idx);
		kvm_mips_dump_host_tlbs();
		local_irq_restore(flags);
		return -1;
	}

	write_c0_entrylo0(entrylo0);
	write_c0_entrylo1(entrylo1);
	mtc0_tlbw_hazard();

	if (idx < 0)
		tlb_write_random();
	else
		tlb_write_indexed();
	tlbw_use_hazard();

	kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n",
		  vcpu->arch.pc, idx, read_c0_entryhi(),
		  read_c0_entrylo0(), read_c0_entrylo1());

	/* Flush D-cache */
	if (flush_dcache_mask) {
		if (entrylo0 & MIPS3_PG_V) {
			++vcpu->stat.flush_dcache_exits;
			flush_data_cache_page((entryhi & VPN2_MASK) &
					      ~flush_dcache_mask);
		}
		if (entrylo1 & MIPS3_PG_V) {
			++vcpu->stat.flush_dcache_exits;
			flush_data_cache_page(((entryhi & VPN2_MASK) &
					       ~flush_dcache_mask) |
					      (0x1 << PAGE_SHIFT));
		}
	}

	/* Restore old ASID */
	write_c0_entryhi(old_entryhi);
	mtc0_tlbw_hazard();
	tlbw_use_hazard();
	local_irq_restore(flags);
	return 0;
}
EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_write);

int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
	struct kvm_vcpu *vcpu)
{
	kvm_pfn_t pfn0, pfn1;
	unsigned long flags, old_entryhi = 0, vaddr = 0;
	unsigned long entrylo0 = 0, entrylo1 = 0;

	pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT;
	pfn1 = 0;
	entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) |
		   (1 << 2) | (0x1 << 1);
	entrylo1 = 0;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();
	vaddr = badvaddr & (PAGE_MASK << 1);
	write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu));
	mtc0_tlbw_hazard();
	write_c0_entrylo0(entrylo0);
	mtc0_tlbw_hazard();
	write_c0_entrylo1(entrylo1);
	mtc0_tlbw_hazard();
	write_c0_index(kvm_mips_get_commpage_asid(vcpu));
	mtc0_tlbw_hazard();
	tlb_write_indexed();
	mtc0_tlbw_hazard();
	tlbw_use_hazard();

	kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n",
		  vcpu->arch.pc, read_c0_index(), read_c0_entryhi(),
		  read_c0_entrylo0(), read_c0_entrylo1());

	/* Restore old ASID */
	write_c0_entryhi(old_entryhi);
	mtc0_tlbw_hazard();
	tlbw_use_hazard();
	local_irq_restore(flags);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault);

int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
{
	int i;
	int index = -1;
	struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;

	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
		if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
		    TLB_HI_ASID_HIT(tlb[i], entryhi)) {
			index = i;
			break;
		}
	}

	kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
		  __func__, entryhi, index, tlb[i].tlb_lo0, tlb[i].tlb_lo1);

	return index;
}
EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);

int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr)
{
	unsigned long old_entryhi, flags;
	int idx;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();

	if (KVM_GUEST_KERNEL_MODE(vcpu))
		write_c0_entryhi((vaddr & VPN2_MASK) |
				 kvm_mips_get_kernel_asid(vcpu));
	else {
		write_c0_entryhi((vaddr & VPN2_MASK) |
				 kvm_mips_get_user_asid(vcpu));
	}

	mtc0_tlbw_hazard();

	tlb_probe();
	tlb_probe_hazard();
	idx = read_c0_index();

	/* Restore old ASID */
	write_c0_entryhi(old_entryhi);
	mtc0_tlbw_hazard();
	tlbw_use_hazard();

	local_irq_restore(flags);

	kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx);

	return idx;
}
EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup);

int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
{
	int idx;
	unsigned long flags, old_entryhi;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();

	write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu));
	mtc0_tlbw_hazard();

	tlb_probe();
	tlb_probe_hazard();
	idx = read_c0_index();

	if (idx >= current_cpu_data.tlbsize)
		BUG();

	if (idx > 0) {
		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
		mtc0_tlbw_hazard();

		write_c0_entrylo0(0);
		mtc0_tlbw_hazard();

		write_c0_entrylo1(0);
		mtc0_tlbw_hazard();

		tlb_write_indexed();
		mtc0_tlbw_hazard();
	}

	write_c0_entryhi(old_entryhi);
	mtc0_tlbw_hazard();
	tlbw_use_hazard();

	local_irq_restore(flags);

	if (idx > 0)
		kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__,
			  (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx);

	return 0;
}
EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);

void kvm_mips_flush_host_tlb(int skip_kseg0)
{
	unsigned long flags;
	unsigned long old_entryhi, entryhi;
	unsigned long old_pagemask;
	int entry = 0;
	int maxentry = current_cpu_data.tlbsize;

	local_irq_save(flags);

	old_entryhi = read_c0_entryhi();
	old_pagemask = read_c0_pagemask();

	/* Blast 'em all away. */
	for (entry = 0; entry < maxentry; entry++) {
		write_c0_index(entry);
		mtc0_tlbw_hazard();

		if (skip_kseg0) {
			tlb_read();
			tlbw_use_hazard();

			entryhi = read_c0_entryhi();

			/* Don't blow away guest kernel entries */
			if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0)
				continue;
		}

		/* Make sure all entries differ. */
		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
		mtc0_tlbw_hazard();
		write_c0_entrylo0(0);
		mtc0_tlbw_hazard();
		write_c0_entrylo1(0);
		mtc0_tlbw_hazard();

		tlb_write_indexed();
		mtc0_tlbw_hazard();
	}

	tlbw_use_hazard();

	write_c0_entryhi(old_entryhi);
	write_c0_pagemask(old_pagemask);
	mtc0_tlbw_hazard();
	tlbw_use_hazard();

	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb);

void kvm_local_flush_tlb_all(void)
{
	unsigned long flags;
	unsigned long old_ctx;
	int entry = 0;

	local_irq_save(flags);
	/* Save old context and create impossible VPN2 value */
	old_ctx = read_c0_entryhi();
	write_c0_entrylo0(0);
	write_c0_entrylo1(0);

	/* Blast 'em all away. */
	while (entry < current_cpu_data.tlbsize) {
		/* Make sure all entries differ. */
		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
		write_c0_index(entry);
		mtc0_tlbw_hazard();
		tlb_write_indexed();
		entry++;
	}
	tlbw_use_hazard();
	write_c0_entryhi(old_ctx);
	mtc0_tlbw_hazard();

	local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all);
Commit	Line	Data
858dd5d4	1	/*
d116e812 DCZ	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
	7	* TLB handlers run from KSEG0
	8	*
	9	* Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved.
	10	* Authors: Sanjay Lal <sanjayl@kymasys.com>
	11	*/
858dd5d4	12
858dd5d4 SL	13	#include <linux/sched.h>
	14	#include <linux/smp.h>
	15	#include <linux/mm.h>
	16	#include <linux/delay.h>
403015b3	17	#include <linux/export.h>
858dd5d4	18	#include <linux/kvm_host.h>
6d17c0d1 SL	19	#include <linux/srcu.h>
6d17c0d1 SL	20
858dd5d4 SL	21	#include <asm/cpu.h>
	22	#include <asm/bootinfo.h>
	23	#include <asm/mmu_context.h>
	24	#include <asm/pgtable.h>
	25	#include <asm/cacheflush.h>
e36059e5	26	#include <asm/tlb.h>
858dd5d4 SL	27
	28	#undef CONFIG_MIPS_MT
	29	#include <asm/r4kcache.h>
	30	#define CONFIG_MIPS_MT
	31
	32	#define KVM_GUEST_PC_TLB 0
	33	#define KVM_GUEST_SP_TLB 1
	34
858dd5d4	35	atomic_t kvm_mips_instance;
cb1b447f	36	EXPORT_SYMBOL_GPL(kvm_mips_instance);
858dd5d4 SL	37
858dd5d4 SL	38	/* These function pointers are initialized once the KVM module is loaded */
ba049e93	39	kvm_pfn_t (kvm_mips_gfn_to_pfn)(struct kvm kvm, gfn_t gfn);
cb1b447f	40	EXPORT_SYMBOL_GPL(kvm_mips_gfn_to_pfn);
858dd5d4	41
ba049e93	42	void (*kvm_mips_release_pfn_clean)(kvm_pfn_t pfn);
cb1b447f	43	EXPORT_SYMBOL_GPL(kvm_mips_release_pfn_clean);
858dd5d4	44
ba049e93	45	bool (*kvm_mips_is_error_pfn)(kvm_pfn_t pfn);
cb1b447f	46	EXPORT_SYMBOL_GPL(kvm_mips_is_error_pfn);
858dd5d4	47
403015b3	48	static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
858dd5d4	49	{
4edf00a4 PB	50	int cpu = smp_processor_id();
	51
	52	return vcpu->arch.guest_kernel_asid[cpu] &
	53	cpu_asid_mask(&cpu_data[cpu]);
858dd5d4 SL	54	}
858dd5d4 SL	55
403015b3	56	static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
858dd5d4	57	{
4edf00a4 PB	58	int cpu = smp_processor_id();
	59
	60	return vcpu->arch.guest_user_asid[cpu] &
	61	cpu_asid_mask(&cpu_data[cpu]);
858dd5d4 SL	62	}
858dd5d4 SL	63
bdb7ed86	64	inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu)
858dd5d4 SL	65	{
	66	return vcpu->kvm->arch.commpage_tlb;
	67	}
	68
d116e812	69	/* Structure defining an tlb entry data set. */
858dd5d4 SL	70
	71	void kvm_mips_dump_host_tlbs(void)
	72	{
	73	unsigned long old_entryhi;
	74	unsigned long old_pagemask;
	75	struct kvm_mips_tlb tlb;
	76	unsigned long flags;
	77	int i;
	78
	79	local_irq_save(flags);
	80
	81	old_entryhi = read_c0_entryhi();
	82	old_pagemask = read_c0_pagemask();
	83
6ad78a5c	84	kvm_info("HOST TLBs:\n");
4edf00a4 PB	85	kvm_info("ASID: %#lx\n", read_c0_entryhi() &
4edf00a4 PB	86	cpu_asid_mask(&current_cpu_data));
858dd5d4 SL	87
	88	for (i = 0; i < current_cpu_data.tlbsize; i++) {
	89	write_c0_index(i);
	90	mtc0_tlbw_hazard();
	91
	92	tlb_read();
	93	tlbw_use_hazard();
	94
	95	tlb.tlb_hi = read_c0_entryhi();
	96	tlb.tlb_lo0 = read_c0_entrylo0();
	97	tlb.tlb_lo1 = read_c0_entrylo1();
	98	tlb.tlb_mask = read_c0_pagemask();
	99
6ad78a5c DCZ	100	kvm_info("TLB%c%3d Hi 0x%08lx ",
	101	(tlb.tlb_lo0 \| tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*',
	102	i, tlb.tlb_hi);
8cffd197 JH	103	kvm_info("Lo0=0x%09llx %c%c attr %lx ",
8cffd197 JH	104	(u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo0),
6ad78a5c DCZ	105	(tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ',
	106	(tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ',
	107	(tlb.tlb_lo0 >> 3) & 7);
8cffd197 JH	108	kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
8cffd197 JH	109	(u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo1),
6ad78a5c DCZ	110	(tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ',
	111	(tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ',
	112	(tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask);
858dd5d4 SL	113	}
	114	write_c0_entryhi(old_entryhi);
	115	write_c0_pagemask(old_pagemask);
	116	mtc0_tlbw_hazard();
	117	local_irq_restore(flags);
	118	}
cb1b447f	119	EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);
858dd5d4 SL	120
	121	void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
	122	{
	123	struct mips_coproc *cop0 = vcpu->arch.cop0;
	124	struct kvm_mips_tlb tlb;
	125	int i;
	126
6ad78a5c DCZ	127	kvm_info("Guest TLBs:\n");
6ad78a5c DCZ	128	kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));
858dd5d4 SL	129
	130	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
	131	tlb = vcpu->arch.guest_tlb[i];
6ad78a5c DCZ	132	kvm_info("TLB%c%3d Hi 0x%08lx ",
	133	(tlb.tlb_lo0 \| tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*',
	134	i, tlb.tlb_hi);
8cffd197 JH	135	kvm_info("Lo0=0x%09llx %c%c attr %lx ",
8cffd197 JH	136	(u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo0),
6ad78a5c DCZ	137	(tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ',
	138	(tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ',
	139	(tlb.tlb_lo0 >> 3) & 7);
8cffd197 JH	140	kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
8cffd197 JH	141	(u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo1),
6ad78a5c DCZ	142	(tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ',
	143	(tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ',
	144	(tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask);
858dd5d4 SL	145	}
858dd5d4 SL	146	}
cb1b447f	147	EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);
858dd5d4	148
858dd5d4 SL	149	/* XXXKYMA: Must be called with interrupts disabled */
858dd5d4 SL	150	/* set flush_dcache_mask == 0 if no dcache flush required */
d116e812 DCZ	151	int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi,
	152	unsigned long entrylo0, unsigned long entrylo1,
	153	int flush_dcache_mask)
858dd5d4 SL	154	{
	155	unsigned long flags;
	156	unsigned long old_entryhi;
b045c406	157	int idx;
858dd5d4 SL	158
	159	local_irq_save(flags);
	160
858dd5d4 SL	161	old_entryhi = read_c0_entryhi();
	162	write_c0_entryhi(entryhi);
	163	mtc0_tlbw_hazard();
	164
	165	tlb_probe();
	166	tlb_probe_hazard();
	167	idx = read_c0_index();
	168
	169	if (idx > current_cpu_data.tlbsize) {
	170	kvm_err("%s: Invalid Index: %d\n", __func__, idx);
	171	kvm_mips_dump_host_tlbs();
cfec0e75	172	local_irq_restore(flags);
858dd5d4 SL	173	return -1;
	174	}
	175
858dd5d4 SL	176	write_c0_entrylo0(entrylo0);
	177	write_c0_entrylo1(entrylo1);
	178	mtc0_tlbw_hazard();
	179
b5dfc6c1 JH	180	if (idx < 0)
	181	tlb_write_random();
	182	else
	183	tlb_write_indexed();
858dd5d4 SL	184	tlbw_use_hazard();
858dd5d4 SL	185
3d654833 JH	186	kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n",
	187	vcpu->arch.pc, idx, read_c0_entryhi(),
	188	read_c0_entrylo0(), read_c0_entrylo1());
858dd5d4 SL	189
	190	/* Flush D-cache */
	191	if (flush_dcache_mask) {
	192	if (entrylo0 & MIPS3_PG_V) {
	193	++vcpu->stat.flush_dcache_exits;
d116e812 DCZ	194	flush_data_cache_page((entryhi & VPN2_MASK) &
d116e812 DCZ	195	~flush_dcache_mask);
858dd5d4 SL	196	}
	197	if (entrylo1 & MIPS3_PG_V) {
	198	++vcpu->stat.flush_dcache_exits;
d116e812 DCZ	199	flush_data_cache_page(((entryhi & VPN2_MASK) &
	200	~flush_dcache_mask) \|
	201	(0x1 << PAGE_SHIFT));
858dd5d4 SL	202	}
	203	}
	204
	205	/* Restore old ASID */
	206	write_c0_entryhi(old_entryhi);
	207	mtc0_tlbw_hazard();
	208	tlbw_use_hazard();
	209	local_irq_restore(flags);
	210	return 0;
	211	}
403015b3	212	EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_write);
858dd5d4 SL	213
	214	int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
	215	struct kvm_vcpu *vcpu)
	216	{
ba049e93	217	kvm_pfn_t pfn0, pfn1;
858dd5d4 SL	218	unsigned long flags, old_entryhi = 0, vaddr = 0;
	219	unsigned long entrylo0 = 0, entrylo1 = 0;
	220
858dd5d4 SL	221	pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT;
858dd5d4 SL	222	pfn1 = 0;
d116e812 DCZ	223	entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) \| (0x3 << 3) \|
d116e812 DCZ	224	(1 << 2) \| (0x1 << 1);
858dd5d4 SL	225	entrylo1 = 0;
	226
	227	local_irq_save(flags);
	228
	229	old_entryhi = read_c0_entryhi();
	230	vaddr = badvaddr & (PAGE_MASK << 1);
	231	write_c0_entryhi(vaddr \| kvm_mips_get_kernel_asid(vcpu));
	232	mtc0_tlbw_hazard();
	233	write_c0_entrylo0(entrylo0);
	234	mtc0_tlbw_hazard();
	235	write_c0_entrylo1(entrylo1);
	236	mtc0_tlbw_hazard();
	237	write_c0_index(kvm_mips_get_commpage_asid(vcpu));
	238	mtc0_tlbw_hazard();
	239	tlb_write_indexed();
	240	mtc0_tlbw_hazard();
	241	tlbw_use_hazard();
	242
d116e812 DCZ	243	kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n",
	244	vcpu->arch.pc, read_c0_index(), read_c0_entryhi(),
	245	read_c0_entrylo0(), read_c0_entrylo1());
858dd5d4 SL	246
	247	/* Restore old ASID */
	248	write_c0_entryhi(old_entryhi);
	249	mtc0_tlbw_hazard();
	250	tlbw_use_hazard();
	251	local_irq_restore(flags);
	252
	253	return 0;
	254	}
cb1b447f	255	EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault);
858dd5d4	256
858dd5d4 SL	257	int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
	258	{
	259	int i;
	260	int index = -1;
	261	struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;
	262
858dd5d4	263	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
d116e812 DCZ	264	if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
d116e812 DCZ	265	TLB_HI_ASID_HIT(tlb[i], entryhi)) {
858dd5d4 SL	266	index = i;
	267	break;
	268	}
	269	}
	270
858dd5d4 SL	271	kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
858dd5d4 SL	272	__func__, entryhi, index, tlb[i].tlb_lo0, tlb[i].tlb_lo1);
858dd5d4 SL	273
	274	return index;
	275	}
cb1b447f	276	EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);
858dd5d4 SL	277
	278	int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr)
	279	{
	280	unsigned long old_entryhi, flags;
b045c406	281	int idx;
858dd5d4	282
858dd5d4 SL	283	local_irq_save(flags);
	284
	285	old_entryhi = read_c0_entryhi();
	286
	287	if (KVM_GUEST_KERNEL_MODE(vcpu))
d116e812 DCZ	288	write_c0_entryhi((vaddr & VPN2_MASK) \|
d116e812 DCZ	289	kvm_mips_get_kernel_asid(vcpu));
858dd5d4	290	else {
d116e812 DCZ	291	write_c0_entryhi((vaddr & VPN2_MASK) \|
d116e812 DCZ	292	kvm_mips_get_user_asid(vcpu));
858dd5d4 SL	293	}
	294
	295	mtc0_tlbw_hazard();
	296
	297	tlb_probe();
	298	tlb_probe_hazard();
	299	idx = read_c0_index();
	300
	301	/* Restore old ASID */
	302	write_c0_entryhi(old_entryhi);
	303	mtc0_tlbw_hazard();
	304	tlbw_use_hazard();
	305
	306	local_irq_restore(flags);
	307
858dd5d4	308	kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx);
858dd5d4 SL	309
	310	return idx;
	311	}
cb1b447f	312	EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup);
858dd5d4 SL	313
	314	int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
	315	{
	316	int idx;
	317	unsigned long flags, old_entryhi;
	318
	319	local_irq_save(flags);
	320
858dd5d4 SL	321	old_entryhi = read_c0_entryhi();
	322
	323	write_c0_entryhi((va & VPN2_MASK) \| kvm_mips_get_user_asid(vcpu));
	324	mtc0_tlbw_hazard();
	325
	326	tlb_probe();
	327	tlb_probe_hazard();
	328	idx = read_c0_index();
	329
	330	if (idx >= current_cpu_data.tlbsize)
	331	BUG();
	332
	333	if (idx > 0) {
	334	write_c0_entryhi(UNIQUE_ENTRYHI(idx));
	335	mtc0_tlbw_hazard();
	336
	337	write_c0_entrylo0(0);
	338	mtc0_tlbw_hazard();
	339
	340	write_c0_entrylo1(0);
	341	mtc0_tlbw_hazard();
	342
	343	tlb_write_indexed();
	344	mtc0_tlbw_hazard();
	345	}
	346
	347	write_c0_entryhi(old_entryhi);
	348	mtc0_tlbw_hazard();
	349	tlbw_use_hazard();
	350
	351	local_irq_restore(flags);
	352
3d654833	353	if (idx > 0)
858dd5d4	354	kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__,
3d654833	355	(va & VPN2_MASK) \| kvm_mips_get_user_asid(vcpu), idx);
858dd5d4 SL	356
	357	return 0;
	358	}
cb1b447f	359	EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);
858dd5d4 SL	360
	361	void kvm_mips_flush_host_tlb(int skip_kseg0)
	362	{
	363	unsigned long flags;
	364	unsigned long old_entryhi, entryhi;
	365	unsigned long old_pagemask;
	366	int entry = 0;
	367	int maxentry = current_cpu_data.tlbsize;
	368
858dd5d4 SL	369	local_irq_save(flags);
	370
	371	old_entryhi = read_c0_entryhi();
	372	old_pagemask = read_c0_pagemask();
	373
	374	/* Blast 'em all away. */
	375	for (entry = 0; entry < maxentry; entry++) {
858dd5d4 SL	376	write_c0_index(entry);
	377	mtc0_tlbw_hazard();
	378
	379	if (skip_kseg0) {
	380	tlb_read();
	381	tlbw_use_hazard();
	382
	383	entryhi = read_c0_entryhi();
	384
	385	/* Don't blow away guest kernel entries */
d116e812	386	if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0)
858dd5d4	387	continue;
858dd5d4 SL	388	}
	389
	390	/* Make sure all entries differ. */
	391	write_c0_entryhi(UNIQUE_ENTRYHI(entry));
	392	mtc0_tlbw_hazard();
	393	write_c0_entrylo0(0);
	394	mtc0_tlbw_hazard();
	395	write_c0_entrylo1(0);
	396	mtc0_tlbw_hazard();
	397
	398	tlb_write_indexed();
	399	mtc0_tlbw_hazard();
	400	}
	401
	402	tlbw_use_hazard();
	403
	404	write_c0_entryhi(old_entryhi);
	405	write_c0_pagemask(old_pagemask);
	406	mtc0_tlbw_hazard();
	407	tlbw_use_hazard();
	408
	409	local_irq_restore(flags);
	410	}
cb1b447f	411	EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb);
858dd5d4	412
858dd5d4 SL	413	void kvm_local_flush_tlb_all(void)
	414	{
	415	unsigned long flags;
	416	unsigned long old_ctx;
	417	int entry = 0;
	418
	419	local_irq_save(flags);
	420	/* Save old context and create impossible VPN2 value */
	421	old_ctx = read_c0_entryhi();
	422	write_c0_entrylo0(0);
	423	write_c0_entrylo1(0);
	424
	425	/* Blast 'em all away. */
	426	while (entry < current_cpu_data.tlbsize) {
	427	/* Make sure all entries differ. */
	428	write_c0_entryhi(UNIQUE_ENTRYHI(entry));
	429	write_c0_index(entry);
	430	mtc0_tlbw_hazard();
	431	tlb_write_indexed();
	432	entry++;
	433	}
	434	tlbw_use_hazard();
	435	write_c0_entryhi(old_ctx);
	436	mtc0_tlbw_hazard();
	437
	438	local_irq_restore(flags);
	439	}
cb1b447f	440	EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all);