[deliverable/linux.git] / arch / powerpc / platforms / cell / beat_htab.c

/*
 * "Cell Reference Set" HTAB support.
 *
 * (C) Copyright 2006-2007 TOSHIBA CORPORATION
 *
 * This code is based on arch/powerpc/platforms/pseries/lpar.c:
 *  Copyright (C) 2001 Todd Inglett, IBM Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#undef DEBUG_LOW

#include <linux/kernel.h>
#include <linux/spinlock.h>

#include <asm/mmu.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/machdep.h>
#include <asm/udbg.h>

#include "beat_wrapper.h"

#ifdef DEBUG_LOW
#define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0)
#else
#define DBG_LOW(fmt...) do { } while (0)
#endif

static DEFINE_RAW_SPINLOCK(beat_htab_lock);

static inline unsigned int beat_read_mask(unsigned hpte_group)
{
	unsigned long rmask = 0;
	u64 hpte_v[5];

	beat_read_htab_entries(0, hpte_group + 0, hpte_v);
	if (!(hpte_v[0] & HPTE_V_BOLTED))
		rmask |= 0x8000;
	if (!(hpte_v[1] & HPTE_V_BOLTED))
		rmask |= 0x4000;
	if (!(hpte_v[2] & HPTE_V_BOLTED))
		rmask |= 0x2000;
	if (!(hpte_v[3] & HPTE_V_BOLTED))
		rmask |= 0x1000;
	beat_read_htab_entries(0, hpte_group + 4, hpte_v);
	if (!(hpte_v[0] & HPTE_V_BOLTED))
		rmask |= 0x0800;
	if (!(hpte_v[1] & HPTE_V_BOLTED))
		rmask |= 0x0400;
	if (!(hpte_v[2] & HPTE_V_BOLTED))
		rmask |= 0x0200;
	if (!(hpte_v[3] & HPTE_V_BOLTED))
		rmask |= 0x0100;
	hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP);
	beat_read_htab_entries(0, hpte_group + 0, hpte_v);
	if (!(hpte_v[0] & HPTE_V_BOLTED))
		rmask |= 0x80;
	if (!(hpte_v[1] & HPTE_V_BOLTED))
		rmask |= 0x40;
	if (!(hpte_v[2] & HPTE_V_BOLTED))
		rmask |= 0x20;
	if (!(hpte_v[3] & HPTE_V_BOLTED))
		rmask |= 0x10;
	beat_read_htab_entries(0, hpte_group + 4, hpte_v);
	if (!(hpte_v[0] & HPTE_V_BOLTED))
		rmask |= 0x08;
	if (!(hpte_v[1] & HPTE_V_BOLTED))
		rmask |= 0x04;
	if (!(hpte_v[2] & HPTE_V_BOLTED))
		rmask |= 0x02;
	if (!(hpte_v[3] & HPTE_V_BOLTED))
		rmask |= 0x01;
	return rmask;
}

static long beat_lpar_hpte_insert(unsigned long hpte_group,
				  unsigned long vpn, unsigned long pa,
				  unsigned long rflags, unsigned long vflags,
				  int psize, int apsize, int ssize)
{
	unsigned long lpar_rc;
	u64 hpte_v, hpte_r, slot;

	if (vflags & HPTE_V_SECONDARY)
		return -1;

	if (!(vflags & HPTE_V_BOLTED))
		DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
			"rflags=%lx, vflags=%lx, psize=%d)\n",
		hpte_group, va, pa, rflags, vflags, psize);

	hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) |
		vflags | HPTE_V_VALID;
	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;

	if (!(vflags & HPTE_V_BOLTED))
		DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);

	if (rflags & _PAGE_NO_CACHE)
		hpte_r &= ~_PAGE_COHERENT;

	raw_spin_lock(&beat_htab_lock);
	lpar_rc = beat_read_mask(hpte_group);
	if (lpar_rc == 0) {
		if (!(vflags & HPTE_V_BOLTED))
			DBG_LOW(" full\n");
		raw_spin_unlock(&beat_htab_lock);
		return -1;
	}

	lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48,
		hpte_v, hpte_r, &slot);
	raw_spin_unlock(&beat_htab_lock);

	/*
	 * Since we try and ioremap PHBs we don't own, the pte insert
	 * will fail. However we must catch the failure in hash_page
	 * or we will loop forever, so return -2 in this case.
	 */
	if (unlikely(lpar_rc != 0)) {
		if (!(vflags & HPTE_V_BOLTED))
			DBG_LOW(" lpar err %lx\n", lpar_rc);
		return -2;
	}
	if (!(vflags & HPTE_V_BOLTED))
		DBG_LOW(" -> slot: %lx\n", slot);

	/* We have to pass down the secondary bucket bit here as well */
	return (slot ^ hpte_group) & 15;
}

static long beat_lpar_hpte_remove(unsigned long hpte_group)
{
	DBG_LOW("hpte_remove(group=%lx)\n", hpte_group);
	return -1;
}

static unsigned long beat_lpar_hpte_getword0(unsigned long slot)
{
	unsigned long dword0;
	unsigned long lpar_rc;
	u64 dword[5];

	lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword);

	dword0 = dword[slot&3];

	BUG_ON(lpar_rc != 0);

	return dword0;
}

static void beat_lpar_hptab_clear(void)
{
	unsigned long size_bytes = 1UL << ppc64_pft_size;
	unsigned long hpte_count = size_bytes >> 4;
	int i;
	u64 dummy0, dummy1;

	/* TODO: Use bulk call */
	for (i = 0; i < hpte_count; i++)
		beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1);
}

/*
 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
 * the low 3 bits of flags happen to line up.  So no transform is needed.
 * We can probably optimize here and assume the high bits of newpp are
 * already zero.  For now I am paranoid.
 */
static long beat_lpar_hpte_updatepp(unsigned long slot,
				    unsigned long newpp,
				    unsigned long vpn,
				    int psize, int apsize,
				    int ssize, int local)
{
	unsigned long lpar_rc;
	u64 dummy0, dummy1;
	unsigned long want_v;

	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);

	DBG_LOW("    update: "
		"avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
		want_v & HPTE_V_AVPN, slot, psize, newpp);

	raw_spin_lock(&beat_htab_lock);
	dummy0 = beat_lpar_hpte_getword0(slot);
	if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) {
		DBG_LOW("not found !\n");
		raw_spin_unlock(&beat_htab_lock);
		return -1;
	}

	lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0,
					&dummy1);
	raw_spin_unlock(&beat_htab_lock);
	if (lpar_rc != 0 || dummy0 == 0) {
		DBG_LOW("not found !\n");
		return -1;
	}

	DBG_LOW("ok %lx %lx\n", dummy0, dummy1);

	BUG_ON(lpar_rc != 0);

	return 0;
}

static long beat_lpar_hpte_find(unsigned long vpn, int psize)
{
	unsigned long hash;
	unsigned long i, j;
	long slot;
	unsigned long want_v, hpte_v;

	hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);

	for (j = 0; j < 2; j++) {
		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		for (i = 0; i < HPTES_PER_GROUP; i++) {
			hpte_v = beat_lpar_hpte_getword0(slot);

			if (HPTE_V_COMPARE(hpte_v, want_v)
			    && (hpte_v & HPTE_V_VALID)
			    && (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
				/* HPTE matches */
				if (j)
					slot = -slot;
				return slot;
			}
			++slot;
		}
		hash = ~hash;
	}

	return -1;
}

static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
					  unsigned long ea,
					  int psize, int ssize)
{
	unsigned long vpn;
	unsigned long lpar_rc, slot, vsid;
	u64 dummy0, dummy1;

	vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
	vpn = hpt_vpn(ea, vsid, MMU_SEGSIZE_256M);

	raw_spin_lock(&beat_htab_lock);
	slot = beat_lpar_hpte_find(vpn, psize);
	BUG_ON(slot == -1);

	lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7,
		&dummy0, &dummy1);
	raw_spin_unlock(&beat_htab_lock);

	BUG_ON(lpar_rc != 0);
}

static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
				      int psize, int apsize,
				      int ssize, int local)
{
	unsigned long want_v;
	unsigned long lpar_rc;
	u64 dummy1, dummy2;
	unsigned long flags;

	DBG_LOW("    inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
		slot, va, psize, local);
	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);

	raw_spin_lock_irqsave(&beat_htab_lock, flags);
	dummy1 = beat_lpar_hpte_getword0(slot);

	if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) {
		DBG_LOW("not found !\n");
		raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
		return;
	}

	lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0,
		&dummy1, &dummy2);
	raw_spin_unlock_irqrestore(&beat_htab_lock, flags);

	BUG_ON(lpar_rc != 0);
}

void __init hpte_init_beat(void)
{
	ppc_md.hpte_invalidate	= beat_lpar_hpte_invalidate;
	ppc_md.hpte_updatepp	= beat_lpar_hpte_updatepp;
	ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
	ppc_md.hpte_insert	= beat_lpar_hpte_insert;
	ppc_md.hpte_remove	= beat_lpar_hpte_remove;
	ppc_md.hpte_clear_all	= beat_lpar_hptab_clear;
}

static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
				  unsigned long vpn, unsigned long pa,
				  unsigned long rflags, unsigned long vflags,
				  int psize, int apsize, int ssize)
{
	unsigned long lpar_rc;
	u64 hpte_v, hpte_r, slot;

	if (vflags & HPTE_V_SECONDARY)
		return -1;

	if (!(vflags & HPTE_V_BOLTED))
		DBG_LOW("hpte_insert(group=%lx, vpn=%016lx, pa=%016lx, "
			"rflags=%lx, vflags=%lx, psize=%d)\n",
		hpte_group, vpn, pa, rflags, vflags, psize);

	hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) |
		vflags | HPTE_V_VALID;
	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;

	if (!(vflags & HPTE_V_BOLTED))
		DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);

	if (rflags & _PAGE_NO_CACHE)
		hpte_r &= ~_PAGE_COHERENT;

	/* insert into not-volted entry */
	lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
		HPTE_V_BOLTED, 0, &slot);
	/*
	 * Since we try and ioremap PHBs we don't own, the pte insert
	 * will fail. However we must catch the failure in hash_page
	 * or we will loop forever, so return -2 in this case.
	 */
	if (unlikely(lpar_rc != 0)) {
		if (!(vflags & HPTE_V_BOLTED))
			DBG_LOW(" lpar err %lx\n", lpar_rc);
		return -2;
	}
	if (!(vflags & HPTE_V_BOLTED))
		DBG_LOW(" -> slot: %lx\n", slot);

	/* We have to pass down the secondary bucket bit here as well */
	return (slot ^ hpte_group) & 15;
}

/*
 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
 * the low 3 bits of flags happen to line up.  So no transform is needed.
 * We can probably optimize here and assume the high bits of newpp are
 * already zero.  For now I am paranoid.
 */
static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
				       unsigned long newpp,
				       unsigned long vpn,
				       int psize, int apsize,
				       int ssize, int local)
{
	unsigned long lpar_rc;
	unsigned long want_v;
	unsigned long pss;

	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
	pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];

	DBG_LOW("    update: "
		"avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
		want_v & HPTE_V_AVPN, slot, psize, newpp);

	lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp);

	if (lpar_rc == 0xfffffff7) {
		DBG_LOW("not found !\n");
		return -1;
	}

	DBG_LOW("ok\n");

	BUG_ON(lpar_rc != 0);

	return 0;
}

static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,
					 int psize, int apsize,
					 int ssize, int local)
{
	unsigned long want_v;
	unsigned long lpar_rc;
	unsigned long pss;

	DBG_LOW("    inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
		slot, vpn, psize, local);
	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
	pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];

	lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);

	/* E_busy can be valid output: page may be already replaced */
	BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7);
}

static int64_t _beat_lpar_hptab_clear_v3(void)
{
	return beat_clear_htab3(0);
}

static void beat_lpar_hptab_clear_v3(void)
{
	_beat_lpar_hptab_clear_v3();
}

void __init hpte_init_beat_v3(void)
{
	if (_beat_lpar_hptab_clear_v3() == 0) {
		ppc_md.hpte_invalidate	= beat_lpar_hpte_invalidate_v3;
		ppc_md.hpte_updatepp	= beat_lpar_hpte_updatepp_v3;
		ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
		ppc_md.hpte_insert	= beat_lpar_hpte_insert_v3;
		ppc_md.hpte_remove	= beat_lpar_hpte_remove;
		ppc_md.hpte_clear_all	= beat_lpar_hptab_clear_v3;
	} else {
		ppc_md.hpte_invalidate	= beat_lpar_hpte_invalidate;
		ppc_md.hpte_updatepp	= beat_lpar_hpte_updatepp;
		ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
		ppc_md.hpte_insert	= beat_lpar_hpte_insert;
		ppc_md.hpte_remove	= beat_lpar_hpte_remove;
		ppc_md.hpte_clear_all	= beat_lpar_hptab_clear;
	}
}
Commit	Line	Data
fe4a0cf1 IK	1	/*
	2	* "Cell Reference Set" HTAB support.
	3	*
	4	* (C) Copyright 2006-2007 TOSHIBA CORPORATION
	5	*
	6	* This code is based on arch/powerpc/platforms/pseries/lpar.c:
	7	* Copyright (C) 2001 Todd Inglett, IBM Corporation
	8	*
	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License as published by
	11	* the Free Software Foundation; either version 2 of the License, or
	12	* (at your option) any later version.
	13	*
	14	* This program is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	* GNU General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU General Public License along
	20	* with this program; if not, write to the Free Software Foundation, Inc.,
	21	* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	22	*/
	23
	24	#undef DEBUG_LOW
	25
	26	#include <linux/kernel.h>
	27	#include <linux/spinlock.h>
	28
	29	#include <asm/mmu.h>
	30	#include <asm/page.h>
	31	#include <asm/pgtable.h>
	32	#include <asm/machdep.h>
	33	#include <asm/udbg.h>
	34
8ae6e30d	35	#include "beat_wrapper.h"
fe4a0cf1 IK	36
fe4a0cf1 IK	37	#ifdef DEBUG_LOW
2fe37a6e	38	#define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0)
fe4a0cf1	39	#else
2fe37a6e	40	#define DBG_LOW(fmt...) do { } while (0)
fe4a0cf1 IK	41	#endif
fe4a0cf1 IK	42
7cc8a5e3	43	static DEFINE_RAW_SPINLOCK(beat_htab_lock);
fe4a0cf1 IK	44
	45	static inline unsigned int beat_read_mask(unsigned hpte_group)
	46	{
fe4a0cf1	47	unsigned long rmask = 0;
19b0bd02	48	u64 hpte_v[5];
fe4a0cf1 IK	49
	50	beat_read_htab_entries(0, hpte_group + 0, hpte_v);
	51	if (!(hpte_v[0] & HPTE_V_BOLTED))
	52	rmask \|= 0x8000;
	53	if (!(hpte_v[1] & HPTE_V_BOLTED))
	54	rmask \|= 0x4000;
	55	if (!(hpte_v[2] & HPTE_V_BOLTED))
	56	rmask \|= 0x2000;
	57	if (!(hpte_v[3] & HPTE_V_BOLTED))
	58	rmask \|= 0x1000;
	59	beat_read_htab_entries(0, hpte_group + 4, hpte_v);
	60	if (!(hpte_v[0] & HPTE_V_BOLTED))
	61	rmask \|= 0x0800;
	62	if (!(hpte_v[1] & HPTE_V_BOLTED))
	63	rmask \|= 0x0400;
	64	if (!(hpte_v[2] & HPTE_V_BOLTED))
	65	rmask \|= 0x0200;
	66	if (!(hpte_v[3] & HPTE_V_BOLTED))
	67	rmask \|= 0x0100;
	68	hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP);
	69	beat_read_htab_entries(0, hpte_group + 0, hpte_v);
	70	if (!(hpte_v[0] & HPTE_V_BOLTED))
	71	rmask \|= 0x80;
	72	if (!(hpte_v[1] & HPTE_V_BOLTED))
	73	rmask \|= 0x40;
	74	if (!(hpte_v[2] & HPTE_V_BOLTED))
	75	rmask \|= 0x20;
	76	if (!(hpte_v[3] & HPTE_V_BOLTED))
	77	rmask \|= 0x10;
	78	beat_read_htab_entries(0, hpte_group + 4, hpte_v);
	79	if (!(hpte_v[0] & HPTE_V_BOLTED))
	80	rmask \|= 0x08;
	81	if (!(hpte_v[1] & HPTE_V_BOLTED))
	82	rmask \|= 0x04;
	83	if (!(hpte_v[2] & HPTE_V_BOLTED))
	84	rmask \|= 0x02;
	85	if (!(hpte_v[3] & HPTE_V_BOLTED))
	86	rmask \|= 0x01;
	87	return rmask;
	88	}
	89
	90	static long beat_lpar_hpte_insert(unsigned long hpte_group,
5524a27d	91	unsigned long vpn, unsigned long pa,
fe4a0cf1	92	unsigned long rflags, unsigned long vflags,
b1022fbd	93	int psize, int apsize, int ssize)
fe4a0cf1 IK	94	{
fe4a0cf1 IK	95	unsigned long lpar_rc;
19b0bd02	96	u64 hpte_v, hpte_r, slot;
fe4a0cf1	97
fe4a0cf1 IK	98	if (vflags & HPTE_V_SECONDARY)
	99	return -1;
	100
	101	if (!(vflags & HPTE_V_BOLTED))
	102	DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, "
	103	"rflags=%lx, vflags=%lx, psize=%d)\n",
	104	hpte_group, va, pa, rflags, vflags, psize);
	105
b1022fbd	106	hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) \|
1189be65	107	vflags \| HPTE_V_VALID;
b1022fbd	108	hpte_r = hpte_encode_r(pa, psize, apsize) \| rflags;
fe4a0cf1 IK	109
	110	if (!(vflags & HPTE_V_BOLTED))
	111	DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
	112
443dcac4	113	if (rflags & _PAGE_NO_CACHE)
fe4a0cf1 IK	114	hpte_r &= ~_PAGE_COHERENT;
fe4a0cf1 IK	115
7cc8a5e3	116	raw_spin_lock(&beat_htab_lock);
2fe37a6e IK	117	lpar_rc = beat_read_mask(hpte_group);
2fe37a6e IK	118	if (lpar_rc == 0) {
fe4a0cf1 IK	119	if (!(vflags & HPTE_V_BOLTED))
fe4a0cf1 IK	120	DBG_LOW(" full\n");
7cc8a5e3	121	raw_spin_unlock(&beat_htab_lock);
fe4a0cf1 IK	122	return -1;
	123	}
	124
	125	lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48,
	126	hpte_v, hpte_r, &slot);
7cc8a5e3	127	raw_spin_unlock(&beat_htab_lock);
fe4a0cf1 IK	128
	129	/*
	130	* Since we try and ioremap PHBs we don't own, the pte insert
	131	* will fail. However we must catch the failure in hash_page
	132	* or we will loop forever, so return -2 in this case.
	133	*/
	134	if (unlikely(lpar_rc != 0)) {
	135	if (!(vflags & HPTE_V_BOLTED))
	136	DBG_LOW(" lpar err %lx\n", lpar_rc);
	137	return -2;
	138	}
	139	if (!(vflags & HPTE_V_BOLTED))
	140	DBG_LOW(" -> slot: %lx\n", slot);
	141
	142	/* We have to pass down the secondary bucket bit here as well */
	143	return (slot ^ hpte_group) & 15;
	144	}
	145
	146	static long beat_lpar_hpte_remove(unsigned long hpte_group)
	147	{
	148	DBG_LOW("hpte_remove(group=%lx)\n", hpte_group);
	149	return -1;
	150	}
	151
	152	static unsigned long beat_lpar_hpte_getword0(unsigned long slot)
	153	{
19b0bd02	154	unsigned long dword0;
fe4a0cf1	155	unsigned long lpar_rc;
19b0bd02	156	u64 dword[5];
fe4a0cf1 IK	157
	158	lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword);
	159
	160	dword0 = dword[slot&3];
	161
	162	BUG_ON(lpar_rc != 0);
	163
	164	return dword0;
	165	}
	166
	167	static void beat_lpar_hptab_clear(void)
	168	{
	169	unsigned long size_bytes = 1UL << ppc64_pft_size;
	170	unsigned long hpte_count = size_bytes >> 4;
	171	int i;
19b0bd02	172	u64 dummy0, dummy1;
fe4a0cf1 IK	173
	174	/* TODO: Use bulk call */
	175	for (i = 0; i < hpte_count; i++)
	176	beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1);
	177	}
	178
	179	/*
	180	* NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
	181	* the low 3 bits of flags happen to line up. So no transform is needed.
	182	* We can probably optimize here and assume the high bits of newpp are
	183	* already zero. For now I am paranoid.
	184	*/
	185	static long beat_lpar_hpte_updatepp(unsigned long slot,
	186	unsigned long newpp,
5524a27d	187	unsigned long vpn,
db3d8534 AK	188	int psize, int apsize,
db3d8534 AK	189	int ssize, int local)
fe4a0cf1 IK	190	{
fe4a0cf1 IK	191	unsigned long lpar_rc;
19b0bd02 IM	192	u64 dummy0, dummy1;
19b0bd02 IM	193	unsigned long want_v;
fe4a0cf1	194
74f227b2	195	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
fe4a0cf1 IK	196
	197	DBG_LOW(" update: "
	198	"avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
	199	want_v & HPTE_V_AVPN, slot, psize, newpp);
	200
7cc8a5e3	201	raw_spin_lock(&beat_htab_lock);
fe4a0cf1 IK	202	dummy0 = beat_lpar_hpte_getword0(slot);
	203	if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) {
	204	DBG_LOW("not found !\n");
7cc8a5e3	205	raw_spin_unlock(&beat_htab_lock);
fe4a0cf1 IK	206	return -1;
	207	}
	208
	209	lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0,
	210	&dummy1);
7cc8a5e3	211	raw_spin_unlock(&beat_htab_lock);
fe4a0cf1 IK	212	if (lpar_rc != 0 \|\| dummy0 == 0) {
	213	DBG_LOW("not found !\n");
	214	return -1;
	215	}
	216
	217	DBG_LOW("ok %lx %lx\n", dummy0, dummy1);
	218
	219	BUG_ON(lpar_rc != 0);
	220
	221	return 0;
	222	}
	223
5524a27d	224	static long beat_lpar_hpte_find(unsigned long vpn, int psize)
fe4a0cf1 IK	225	{
	226	unsigned long hash;
	227	unsigned long i, j;
	228	long slot;
	229	unsigned long want_v, hpte_v;
	230
5524a27d	231	hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
74f227b2	232	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
fe4a0cf1 IK	233
	234	for (j = 0; j < 2; j++) {
	235	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	236	for (i = 0; i < HPTES_PER_GROUP; i++) {
	237	hpte_v = beat_lpar_hpte_getword0(slot);
	238
	239	if (HPTE_V_COMPARE(hpte_v, want_v)
	240	&& (hpte_v & HPTE_V_VALID)
	241	&& (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
	242	/* HPTE matches */
	243	if (j)
	244	slot = -slot;
	245	return slot;
	246	}
	247	++slot;
	248	}
	249	hash = ~hash;
	250	}
	251
	252	return -1;
	253	}
	254
	255	static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
	256	unsigned long ea,
1189be65	257	int psize, int ssize)
fe4a0cf1	258	{
5524a27d AK	259	unsigned long vpn;
5524a27d AK	260	unsigned long lpar_rc, slot, vsid;
19b0bd02	261	u64 dummy0, dummy1;
fe4a0cf1	262
1189be65	263	vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
5524a27d	264	vpn = hpt_vpn(ea, vsid, MMU_SEGSIZE_256M);
fe4a0cf1	265
7cc8a5e3	266	raw_spin_lock(&beat_htab_lock);
5524a27d	267	slot = beat_lpar_hpte_find(vpn, psize);
fe4a0cf1 IK	268	BUG_ON(slot == -1);
	269
	270	lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7,
	271	&dummy0, &dummy1);
7cc8a5e3	272	raw_spin_unlock(&beat_htab_lock);
fe4a0cf1 IK	273
	274	BUG_ON(lpar_rc != 0);
	275	}
	276
5524a27d	277	static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
db3d8534 AK	278	int psize, int apsize,
db3d8534 AK	279	int ssize, int local)
fe4a0cf1 IK	280	{
	281	unsigned long want_v;
	282	unsigned long lpar_rc;
19b0bd02	283	u64 dummy1, dummy2;
fe4a0cf1 IK	284	unsigned long flags;
	285
	286	DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
	287	slot, va, psize, local);
74f227b2	288	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
fe4a0cf1	289
7cc8a5e3	290	raw_spin_lock_irqsave(&beat_htab_lock, flags);
fe4a0cf1 IK	291	dummy1 = beat_lpar_hpte_getword0(slot);
	292
	293	if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) {
	294	DBG_LOW("not found !\n");
7cc8a5e3	295	raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
fe4a0cf1 IK	296	return;
	297	}
	298
	299	lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0,
	300	&dummy1, &dummy2);
7cc8a5e3	301	raw_spin_unlock_irqrestore(&beat_htab_lock, flags);
fe4a0cf1 IK	302
	303	BUG_ON(lpar_rc != 0);
	304	}
	305
	306	void __init hpte_init_beat(void)
	307	{
	308	ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate;
	309	ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp;
	310	ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
	311	ppc_md.hpte_insert = beat_lpar_hpte_insert;
	312	ppc_md.hpte_remove = beat_lpar_hpte_remove;
	313	ppc_md.hpte_clear_all = beat_lpar_hptab_clear;
	314	}
7f2c8577 IK	315
7f2c8577 IK	316	static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
5524a27d	317	unsigned long vpn, unsigned long pa,
7f2c8577	318	unsigned long rflags, unsigned long vflags,
b1022fbd	319	int psize, int apsize, int ssize)
7f2c8577 IK	320	{
7f2c8577 IK	321	unsigned long lpar_rc;
19b0bd02	322	u64 hpte_v, hpte_r, slot;
7f2c8577	323
7f2c8577 IK	324	if (vflags & HPTE_V_SECONDARY)
	325	return -1;
	326
	327	if (!(vflags & HPTE_V_BOLTED))
5524a27d	328	DBG_LOW("hpte_insert(group=%lx, vpn=%016lx, pa=%016lx, "
7f2c8577	329	"rflags=%lx, vflags=%lx, psize=%d)\n",
5524a27d	330	hpte_group, vpn, pa, rflags, vflags, psize);
7f2c8577	331
b1022fbd	332	hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) \|
1189be65	333	vflags \| HPTE_V_VALID;
b1022fbd	334	hpte_r = hpte_encode_r(pa, psize, apsize) \| rflags;
7f2c8577 IK	335
	336	if (!(vflags & HPTE_V_BOLTED))
	337	DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
	338
443dcac4	339	if (rflags & _PAGE_NO_CACHE)
7f2c8577 IK	340	hpte_r &= ~_PAGE_COHERENT;
	341
	342	/* insert into not-volted entry */
	343	lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r,
	344	HPTE_V_BOLTED, 0, &slot);
	345	/*
	346	* Since we try and ioremap PHBs we don't own, the pte insert
	347	* will fail. However we must catch the failure in hash_page
	348	* or we will loop forever, so return -2 in this case.
	349	*/
	350	if (unlikely(lpar_rc != 0)) {
	351	if (!(vflags & HPTE_V_BOLTED))
	352	DBG_LOW(" lpar err %lx\n", lpar_rc);
	353	return -2;
	354	}
	355	if (!(vflags & HPTE_V_BOLTED))
	356	DBG_LOW(" -> slot: %lx\n", slot);
	357
	358	/* We have to pass down the secondary bucket bit here as well */
	359	return (slot ^ hpte_group) & 15;
	360	}
	361
	362	/*
	363	* NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
	364	* the low 3 bits of flags happen to line up. So no transform is needed.
	365	* We can probably optimize here and assume the high bits of newpp are
	366	* already zero. For now I am paranoid.
	367	*/
	368	static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
db3d8534 AK	369	unsigned long newpp,
	370	unsigned long vpn,
	371	int psize, int apsize,
	372	int ssize, int local)
7f2c8577 IK	373	{
	374	unsigned long lpar_rc;
	375	unsigned long want_v;
	376	unsigned long pss;
	377
74f227b2	378	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
b1022fbd	379	pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];
7f2c8577 IK	380
	381	DBG_LOW(" update: "
	382	"avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
	383	want_v & HPTE_V_AVPN, slot, psize, newpp);
	384
	385	lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp);
	386
	387	if (lpar_rc == 0xfffffff7) {
	388	DBG_LOW("not found !\n");
	389	return -1;
	390	}
	391
	392	DBG_LOW("ok\n");
	393
	394	BUG_ON(lpar_rc != 0);
	395
	396	return 0;
	397	}
	398
5524a27d	399	static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,
db3d8534 AK	400	int psize, int apsize,
db3d8534 AK	401	int ssize, int local)
7f2c8577 IK	402	{
	403	unsigned long want_v;
	404	unsigned long lpar_rc;
	405	unsigned long pss;
	406
5524a27d AK	407	DBG_LOW(" inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n",
5524a27d AK	408	slot, vpn, psize, local);
74f227b2	409	want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M);
b1022fbd	410	pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize];
7f2c8577 IK	411
	412	lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);
	413
	414	/* E_busy can be valid output: page may be already replaced */
	415	BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7);
	416	}
	417
	418	static int64_t _beat_lpar_hptab_clear_v3(void)
	419	{
	420	return beat_clear_htab3(0);
	421	}
	422
	423	static void beat_lpar_hptab_clear_v3(void)
	424	{
	425	_beat_lpar_hptab_clear_v3();
	426	}
	427
	428	void __init hpte_init_beat_v3(void)
	429	{
	430	if (_beat_lpar_hptab_clear_v3() == 0) {
	431	ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate_v3;
	432	ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp_v3;
	433	ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
	434	ppc_md.hpte_insert = beat_lpar_hpte_insert_v3;
	435	ppc_md.hpte_remove = beat_lpar_hpte_remove;
	436	ppc_md.hpte_clear_all = beat_lpar_hptab_clear_v3;
	437	} else {
	438	ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate;
	439	ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp;
	440	ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp;
	441	ppc_md.hpte_insert = beat_lpar_hpte_insert;
	442	ppc_md.hpte_remove = beat_lpar_hpte_remove;
	443	ppc_md.hpte_clear_all = beat_lpar_hptab_clear;
	444	}
	445	}