[deliverable/linux.git] / arch / powerpc / mm / tlb_nohash.c

/*
 * This file contains the routines for TLB flushing.
 * On machines where the MMU does not use a hash table to store virtual to
 * physical translations (ie, SW loaded TLBs or Book3E compilant processors,
 * this does -not- include 603 however which shares the implementation with
 * hash based processors)
 *
 *  -- BenH
 *
 * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
 *                     IBM Corp.
 *
 *  Derived from arch/ppc/mm/init.c:
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
 *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
 *    Copyright (C) 1996 Paul Mackerras
 *
 *  Derived from "arch/i386/mm/init.c"
 *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version
 *  2 of the License, or (at your option) any later version.
 *
 */

#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/preempt.h>
#include <linux/spinlock.h>
#include <linux/memblock.h>
#include <linux/of_fdt.h>
#include <linux/hugetlb.h>

#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include <asm/code-patching.h>
#include <asm/hugetlb.h>

#include "mmu_decl.h"

/*
 * This struct lists the sw-supported page sizes.  The hardawre MMU may support
 * other sizes not listed here.   The .ind field is only used on MMUs that have
 * indirect page table entries.
 */
#ifdef CONFIG_PPC_BOOK3E_MMU
#ifdef CONFIG_PPC_FSL_BOOK3E
struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	[MMU_PAGE_4K] = {
		.shift	= 12,
		.enc	= BOOK3E_PAGESZ_4K,
	},
	[MMU_PAGE_4M] = {
		.shift	= 22,
		.enc	= BOOK3E_PAGESZ_4M,
	},
	[MMU_PAGE_16M] = {
		.shift	= 24,
		.enc	= BOOK3E_PAGESZ_16M,
	},
	[MMU_PAGE_64M] = {
		.shift	= 26,
		.enc	= BOOK3E_PAGESZ_64M,
	},
	[MMU_PAGE_256M] = {
		.shift	= 28,
		.enc	= BOOK3E_PAGESZ_256M,
	},
	[MMU_PAGE_1G] = {
		.shift	= 30,
		.enc	= BOOK3E_PAGESZ_1GB,
	},
};
#else
struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	[MMU_PAGE_4K] = {
		.shift	= 12,
		.ind	= 20,
		.enc	= BOOK3E_PAGESZ_4K,
	},
	[MMU_PAGE_16K] = {
		.shift	= 14,
		.enc	= BOOK3E_PAGESZ_16K,
	},
	[MMU_PAGE_64K] = {
		.shift	= 16,
		.ind	= 28,
		.enc	= BOOK3E_PAGESZ_64K,
	},
	[MMU_PAGE_1M] = {
		.shift	= 20,
		.enc	= BOOK3E_PAGESZ_1M,
	},
	[MMU_PAGE_16M] = {
		.shift	= 24,
		.ind	= 36,
		.enc	= BOOK3E_PAGESZ_16M,
	},
	[MMU_PAGE_256M] = {
		.shift	= 28,
		.enc	= BOOK3E_PAGESZ_256M,
	},
	[MMU_PAGE_1G] = {
		.shift	= 30,
		.enc	= BOOK3E_PAGESZ_1GB,
	},
};
#endif /* CONFIG_FSL_BOOKE */

static inline int mmu_get_tsize(int psize)
{
	return mmu_psize_defs[psize].enc;
}
#else
static inline int mmu_get_tsize(int psize)
{
	/* This isn't used on !Book3E for now */
	return 0;
}
#endif /* CONFIG_PPC_BOOK3E_MMU */

/* The variables below are currently only used on 64-bit Book3E
 * though this will probably be made common with other nohash
 * implementations at some point
 */
#ifdef CONFIG_PPC64

int mmu_linear_psize;		/* Page size used for the linear mapping */
int mmu_pte_psize;		/* Page size used for PTE pages */
int mmu_vmemmap_psize;		/* Page size used for the virtual mem map */
int book3e_htw_enabled;		/* Is HW tablewalk enabled ? */
unsigned long linear_map_top;	/* Top of linear mapping */

#endif /* CONFIG_PPC64 */

#ifdef CONFIG_PPC_FSL_BOOK3E
/* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
DEFINE_PER_CPU(int, next_tlbcam_idx);
EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
#endif

/*
 * Base TLB flushing operations:
 *
 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 *  - flush_tlb_page(vma, vmaddr) flushes one page
 *  - flush_tlb_range(vma, start, end) flushes a range of pages
 *  - flush_tlb_kernel_range(start, end) flushes kernel pages
 *
 *  - local_* variants of page and mm only apply to the current
 *    processor
 */

/*
 * These are the base non-SMP variants of page and mm flushing
 */
void local_flush_tlb_mm(struct mm_struct *mm)
{
	unsigned int pid;

	preempt_disable();
	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
		_tlbil_pid(pid);
	preempt_enable();
}
EXPORT_SYMBOL(local_flush_tlb_mm);

void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
			    int tsize, int ind)
{
	unsigned int pid;

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (pid != MMU_NO_CONTEXT)
		_tlbil_va(vmaddr, pid, tsize, ind);
	preempt_enable();
}

void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
	__local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
			       mmu_get_tsize(mmu_virtual_psize), 0);
}
EXPORT_SYMBOL(local_flush_tlb_page);

/*
 * And here are the SMP non-local implementations
 */
#ifdef CONFIG_SMP

static DEFINE_RAW_SPINLOCK(tlbivax_lock);

static int mm_is_core_local(struct mm_struct *mm)
{
	return cpumask_subset(mm_cpumask(mm),
			      topology_thread_cpumask(smp_processor_id()));
}

struct tlb_flush_param {
	unsigned long addr;
	unsigned int pid;
	unsigned int tsize;
	unsigned int ind;
};

static void do_flush_tlb_mm_ipi(void *param)
{
	struct tlb_flush_param *p = param;

	_tlbil_pid(p ? p->pid : 0);
}

static void do_flush_tlb_page_ipi(void *param)
{
	struct tlb_flush_param *p = param;

	_tlbil_va(p->addr, p->pid, p->tsize, p->ind);
}


/* Note on invalidations and PID:
 *
 * We snapshot the PID with preempt disabled. At this point, it can still
 * change either because:
 * - our context is being stolen (PID -> NO_CONTEXT) on another CPU
 * - we are invaliating some target that isn't currently running here
 *   and is concurrently acquiring a new PID on another CPU
 * - some other CPU is re-acquiring a lost PID for this mm
 * etc...
 *
 * However, this shouldn't be a problem as we only guarantee
 * invalidation of TLB entries present prior to this call, so we
 * don't care about the PID changing, and invalidating a stale PID
 * is generally harmless.
 */

void flush_tlb_mm(struct mm_struct *mm)
{
	unsigned int pid;

	preempt_disable();
	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto no_context;
	if (!mm_is_core_local(mm)) {
		struct tlb_flush_param p = { .pid = pid };
		/* Ignores smp_processor_id() even if set. */
		smp_call_function_many(mm_cpumask(mm),
				       do_flush_tlb_mm_ipi, &p, 1);
	}
	_tlbil_pid(pid);
 no_context:
	preempt_enable();
}
EXPORT_SYMBOL(flush_tlb_mm);

void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
		      int tsize, int ind)
{
	struct cpumask *cpu_mask;
	unsigned int pid;

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto bail;
	cpu_mask = mm_cpumask(mm);
	if (!mm_is_core_local(mm)) {
		/* If broadcast tlbivax is supported, use it */
		if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
			int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
			if (lock)
				raw_spin_lock(&tlbivax_lock);
			_tlbivax_bcast(vmaddr, pid, tsize, ind);
			if (lock)
				raw_spin_unlock(&tlbivax_lock);
			goto bail;
		} else {
			struct tlb_flush_param p = {
				.pid = pid,
				.addr = vmaddr,
				.tsize = tsize,
				.ind = ind,
			};
			/* Ignores smp_processor_id() even if set in cpu_mask */
			smp_call_function_many(cpu_mask,
					       do_flush_tlb_page_ipi, &p, 1);
		}
	}
	_tlbil_va(vmaddr, pid, tsize, ind);
 bail:
	preempt_enable();
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
#ifdef CONFIG_HUGETLB_PAGE
	if (is_vm_hugetlb_page(vma))
		flush_hugetlb_page(vma, vmaddr);
#endif

	__flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
			 mmu_get_tsize(mmu_virtual_psize), 0);
}
EXPORT_SYMBOL(flush_tlb_page);

#endif /* CONFIG_SMP */

#ifdef CONFIG_PPC_47x
void __init early_init_mmu_47x(void)
{
#ifdef CONFIG_SMP
	unsigned long root = of_get_flat_dt_root();
	if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
		mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
#endif /* CONFIG_SMP */
}
#endif /* CONFIG_PPC_47x */

/*
 * Flush kernel TLB entries in the given range
 */
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
#ifdef CONFIG_SMP
	preempt_disable();
	smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
	_tlbil_pid(0);
	preempt_enable();
#else
	_tlbil_pid(0);
#endif
}
EXPORT_SYMBOL(flush_tlb_kernel_range);

/*
 * Currently, for range flushing, we just do a full mm flush. This should
 * be optimized based on a threshold on the size of the range, since
 * some implementation can stack multiple tlbivax before a tlbsync but
 * for now, we keep it that way
 */
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
		     unsigned long end)

{
	flush_tlb_mm(vma->vm_mm);
}
EXPORT_SYMBOL(flush_tlb_range);

void tlb_flush(struct mmu_gather *tlb)
{
	flush_tlb_mm(tlb->mm);
}

/*
 * Below are functions specific to the 64-bit variant of Book3E though that
 * may change in the future
 */

#ifdef CONFIG_PPC64

/*
 * Handling of virtual linear page tables or indirect TLB entries
 * flushing when PTE pages are freed
 */
void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
{
	int tsize = mmu_psize_defs[mmu_pte_psize].enc;

	if (book3e_htw_enabled) {
		unsigned long start = address & PMD_MASK;
		unsigned long end = address + PMD_SIZE;
		unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;

		/* This isn't the most optimal, ideally we would factor out the
		 * while preempt & CPU mask mucking around, or even the IPI but
		 * it will do for now
		 */
		while (start < end) {
			__flush_tlb_page(tlb->mm, start, tsize, 1);
			start += size;
		}
	} else {
		unsigned long rmask = 0xf000000000000000ul;
		unsigned long rid = (address & rmask) | 0x1000000000000000ul;
		unsigned long vpte = address & ~rmask;

#ifdef CONFIG_PPC_64K_PAGES
		vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
#else
		vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
#endif
		vpte |= rid;
		__flush_tlb_page(tlb->mm, vpte, tsize, 0);
	}
}

static void setup_page_sizes(void)
{
	unsigned int tlb0cfg;
	unsigned int tlb0ps;
	unsigned int eptcfg;
	int i, psize;

#ifdef CONFIG_PPC_FSL_BOOK3E
	unsigned int mmucfg = mfspr(SPRN_MMUCFG);
	int fsl_mmu = mmu_has_feature(MMU_FTR_TYPE_FSL_E);

	if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) {
		unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
		unsigned int min_pg, max_pg;

		min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
		max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;

		for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
			struct mmu_psize_def *def;
			unsigned int shift;

			def = &mmu_psize_defs[psize];
			shift = def->shift;

			if (shift == 0)
				continue;

			/* adjust to be in terms of 4^shift Kb */
			shift = (shift - 10) >> 1;

			if ((shift >= min_pg) && (shift <= max_pg))
				def->flags |= MMU_PAGE_SIZE_DIRECT;
		}

		goto no_indirect;
	}

	if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V2) {
		u32 tlb1ps = mfspr(SPRN_TLB1PS);

		for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
			struct mmu_psize_def *def = &mmu_psize_defs[psize];

			if (tlb1ps & (1U << (def->shift - 10))) {
				def->flags |= MMU_PAGE_SIZE_DIRECT;
			}
		}

		goto no_indirect;
	}
#endif

	tlb0cfg = mfspr(SPRN_TLB0CFG);
	tlb0ps = mfspr(SPRN_TLB0PS);
	eptcfg = mfspr(SPRN_EPTCFG);

	/* Look for supported direct sizes */
	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
		struct mmu_psize_def *def = &mmu_psize_defs[psize];

		if (tlb0ps & (1U << (def->shift - 10)))
			def->flags |= MMU_PAGE_SIZE_DIRECT;
	}

	/* Indirect page sizes supported ? */
	if ((tlb0cfg & TLBnCFG_IND) == 0)
		goto no_indirect;

	/* Now, we only deal with one IND page size for each
	 * direct size. Hopefully all implementations today are
	 * unambiguous, but we might want to be careful in the
	 * future.
	 */
	for (i = 0; i < 3; i++) {
		unsigned int ps, sps;

		sps = eptcfg & 0x1f;
		eptcfg >>= 5;
		ps = eptcfg & 0x1f;
		eptcfg >>= 5;
		if (!ps || !sps)
			continue;
		for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
			struct mmu_psize_def *def = &mmu_psize_defs[psize];

			if (ps == (def->shift - 10))
				def->flags |= MMU_PAGE_SIZE_INDIRECT;
			if (sps == (def->shift - 10))
				def->ind = ps + 10;
		}
	}
 no_indirect:

	/* Cleanup array and print summary */
	pr_info("MMU: Supported page sizes\n");
	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
		struct mmu_psize_def *def = &mmu_psize_defs[psize];
		const char *__page_type_names[] = {
			"unsupported",
			"direct",
			"indirect",
			"direct & indirect"
		};
		if (def->flags == 0) {
			def->shift = 0;	
			continue;
		}
		pr_info("  %8ld KB as %s\n", 1ul << (def->shift - 10),
			__page_type_names[def->flags & 0x3]);
	}
}

static void __patch_exception(int exc, unsigned long addr)
{
	extern unsigned int interrupt_base_book3e;
 	unsigned int *ibase = &interrupt_base_book3e;
 
	/* Our exceptions vectors start with a NOP and -then- a branch
	 * to deal with single stepping from userspace which stops on
	 * the second instruction. Thus we need to patch the second
	 * instruction of the exception, not the first one
	 */

	patch_branch(ibase + (exc / 4) + 1, addr, 0);
}

#define patch_exception(exc, name) do { \
	extern unsigned int name; \
	__patch_exception((exc), (unsigned long)&name); \
} while (0)

static void setup_mmu_htw(void)
{
	/* Check if HW tablewalk is present, and if yes, enable it by:
	 *
	 * - patching the TLB miss handlers to branch to the
	 *   one dedicates to it
	 *
	 * - setting the global book3e_htw_enabled
       	 */
	unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG);

	if ((tlb0cfg & TLBnCFG_IND) &&
	    (tlb0cfg & TLBnCFG_PT)) {
		patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
		patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
		book3e_htw_enabled = 1;
	}
	pr_info("MMU: Book3E HW tablewalk %s\n",
		book3e_htw_enabled ? "enabled" : "not supported");
}

/*
 * Early initialization of the MMU TLB code
 */
static void __early_init_mmu(int boot_cpu)
{
	unsigned int mas4;

	/* XXX This will have to be decided at runtime, but right
	 * now our boot and TLB miss code hard wires it. Ideally
	 * we should find out a suitable page size and patch the
	 * TLB miss code (either that or use the PACA to store
	 * the value we want)
	 */
	mmu_linear_psize = MMU_PAGE_1G;

	/* XXX This should be decided at runtime based on supported
	 * page sizes in the TLB, but for now let's assume 16M is
	 * always there and a good fit (which it probably is)
	 */
	mmu_vmemmap_psize = MMU_PAGE_16M;

	/* XXX This code only checks for TLB 0 capabilities and doesn't
	 *     check what page size combos are supported by the HW. It
	 *     also doesn't handle the case where a separate array holds
	 *     the IND entries from the array loaded by the PT.
	 */
	if (boot_cpu) {
		/* Look for supported page sizes */
		setup_page_sizes();

		/* Look for HW tablewalk support */
		setup_mmu_htw();
	}

	/* Set MAS4 based on page table setting */

	mas4 = 0x4 << MAS4_WIMGED_SHIFT;
	if (book3e_htw_enabled) {
		mas4 |= mas4 | MAS4_INDD;
#ifdef CONFIG_PPC_64K_PAGES
		mas4 |=	BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
		mmu_pte_psize = MMU_PAGE_256M;
#else
		mas4 |=	BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
		mmu_pte_psize = MMU_PAGE_1M;
#endif
	} else {
#ifdef CONFIG_PPC_64K_PAGES
		mas4 |=	BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
#else
		mas4 |=	BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
#endif
		mmu_pte_psize = mmu_virtual_psize;
	}
	mtspr(SPRN_MAS4, mas4);

	/* Set the global containing the top of the linear mapping
	 * for use by the TLB miss code
	 */
	linear_map_top = memblock_end_of_DRAM();

#ifdef CONFIG_PPC_FSL_BOOK3E
	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
		unsigned int num_cams;

		/* use a quarter of the TLBCAM for bolted linear map */
		num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
		linear_map_top = map_mem_in_cams(linear_map_top, num_cams);

		/* limit memory so we dont have linear faults */
		memblock_enforce_memory_limit(linear_map_top);

		patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
		patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e);
	}
#endif

	/* A sync won't hurt us after mucking around with
	 * the MMU configuration
	 */
	mb();

	memblock_set_current_limit(linear_map_top);
}

void __init early_init_mmu(void)
{
	__early_init_mmu(1);
}

void __cpuinit early_init_mmu_secondary(void)
{
	__early_init_mmu(0);
}

void setup_initial_memory_limit(phys_addr_t first_memblock_base,
				phys_addr_t first_memblock_size)
{
	/* On non-FSL Embedded 64-bit, we adjust the RMA size to match
	 * the bolted TLB entry. We know for now that only 1G
	 * entries are supported though that may eventually
	 * change.
	 *
	 * on FSL Embedded 64-bit, we adjust the RMA size to match the
	 * first bolted TLB entry size.  We still limit max to 1G even if
	 * the TLB could cover more.  This is due to what the early init
	 * code is setup to do.
	 *
	 * We crop it to the size of the first MEMBLOCK to
	 * avoid going over total available memory just in case...
	 */
#ifdef CONFIG_PPC_FSL_BOOK3E
	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
		unsigned long linear_sz;
		linear_sz = calc_cam_sz(first_memblock_size, PAGE_OFFSET,
					first_memblock_base);
		ppc64_rma_size = min_t(u64, linear_sz, 0x40000000);
	} else
#endif
		ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);

	/* Finally limit subsequent allocations */
	memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
}
#else /* ! CONFIG_PPC64 */
void __init early_init_mmu(void)
{
#ifdef CONFIG_PPC_47x
	early_init_mmu_47x();
#endif
}
#endif /* CONFIG_PPC64 */
Commit	Line	Data
f048aace BH	1	/*
	2	* This file contains the routines for TLB flushing.
	3	* On machines where the MMU does not use a hash table to store virtual to
	4	* physical translations (ie, SW loaded TLBs or Book3E compilant processors,
	5	* this does -not- include 603 however which shares the implementation with
	6	* hash based processors)
	7	*
	8	* -- BenH
	9	*
25d21ad6 BH	10	* Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
25d21ad6 BH	11	* IBM Corp.
f048aace BH	12	*
	13	* Derived from arch/ppc/mm/init.c:
	14	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
	15	*
	16	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
	17	* and Cort Dougan (PReP) (cort@cs.nmt.edu)
	18	* Copyright (C) 1996 Paul Mackerras
	19	*
	20	* Derived from "arch/i386/mm/init.c"
	21	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
	22	*
	23	* This program is free software; you can redistribute it and/or
	24	* modify it under the terms of the GNU General Public License
	25	* as published by the Free Software Foundation; either version
	26	* 2 of the License, or (at your option) any later version.
	27	*
	28	*/
	29
	30	#include <linux/kernel.h>
93087948	31	#include <linux/export.h>
f048aace BH	32	#include <linux/mm.h>
	33	#include <linux/init.h>
	34	#include <linux/highmem.h>
	35	#include <linux/pagemap.h>
	36	#include <linux/preempt.h>
	37	#include <linux/spinlock.h>
95f72d1e	38	#include <linux/memblock.h>
91b191c7	39	#include <linux/of_fdt.h>
41151e77	40	#include <linux/hugetlb.h>
f048aace BH	41
	42	#include <asm/tlbflush.h>
	43	#include <asm/tlb.h>
25d21ad6	44	#include <asm/code-patching.h>
41151e77	45	#include <asm/hugetlb.h>
f048aace BH	46
	47	#include "mmu_decl.h"
	48
41151e77 BB	49	/*
	50	* This struct lists the sw-supported page sizes. The hardawre MMU may support
	51	* other sizes not listed here. The .ind field is only used on MMUs that have
	52	* indirect page table entries.
	53	*/
	54	#ifdef CONFIG_PPC_BOOK3E_MMU
881fde1d	55	#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77 BB	56	struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	57	[MMU_PAGE_4K] = {
	58	.shift = 12,
	59	.enc = BOOK3E_PAGESZ_4K,
	60	},
	61	[MMU_PAGE_4M] = {
	62	.shift = 22,
	63	.enc = BOOK3E_PAGESZ_4M,
	64	},
	65	[MMU_PAGE_16M] = {
	66	.shift = 24,
	67	.enc = BOOK3E_PAGESZ_16M,
	68	},
	69	[MMU_PAGE_64M] = {
	70	.shift = 26,
	71	.enc = BOOK3E_PAGESZ_64M,
	72	},
	73	[MMU_PAGE_256M] = {
	74	.shift = 28,
	75	.enc = BOOK3E_PAGESZ_256M,
	76	},
	77	[MMU_PAGE_1G] = {
	78	.shift = 30,
	79	.enc = BOOK3E_PAGESZ_1GB,
	80	},
	81	};
	82	#else
25d21ad6 BH	83	struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	84	[MMU_PAGE_4K] = {
	85	.shift = 12,
f2b26c92	86	.ind = 20,
25d21ad6 BH	87	.enc = BOOK3E_PAGESZ_4K,
	88	},
	89	[MMU_PAGE_16K] = {
	90	.shift = 14,
	91	.enc = BOOK3E_PAGESZ_16K,
	92	},
	93	[MMU_PAGE_64K] = {
	94	.shift = 16,
f2b26c92	95	.ind = 28,
25d21ad6 BH	96	.enc = BOOK3E_PAGESZ_64K,
	97	},
	98	[MMU_PAGE_1M] = {
	99	.shift = 20,
	100	.enc = BOOK3E_PAGESZ_1M,
	101	},
	102	[MMU_PAGE_16M] = {
	103	.shift = 24,
f2b26c92	104	.ind = 36,
25d21ad6 BH	105	.enc = BOOK3E_PAGESZ_16M,
	106	},
	107	[MMU_PAGE_256M] = {
	108	.shift = 28,
	109	.enc = BOOK3E_PAGESZ_256M,
	110	},
	111	[MMU_PAGE_1G] = {
	112	.shift = 30,
	113	.enc = BOOK3E_PAGESZ_1GB,
	114	},
	115	};
41151e77 BB	116	#endif /* CONFIG_FSL_BOOKE */
41151e77 BB	117
25d21ad6 BH	118	static inline int mmu_get_tsize(int psize)
	119	{
	120	return mmu_psize_defs[psize].enc;
	121	}
	122	#else
	123	static inline int mmu_get_tsize(int psize)
	124	{
	125	/* This isn't used on !Book3E for now */
	126	return 0;
	127	}
41151e77	128	#endif /* CONFIG_PPC_BOOK3E_MMU */
25d21ad6 BH	129
	130	/* The variables below are currently only used on 64-bit Book3E
	131	* though this will probably be made common with other nohash
	132	* implementations at some point
	133	*/
	134	#ifdef CONFIG_PPC64
	135
	136	int mmu_linear_psize; /* Page size used for the linear mapping */
	137	int mmu_pte_psize; /* Page size used for PTE pages */
32a74949	138	int mmu_vmemmap_psize; /* Page size used for the virtual mem map */
25d21ad6 BH	139	int book3e_htw_enabled; /* Is HW tablewalk enabled ? */
	140	unsigned long linear_map_top; /* Top of linear mapping */
	141
	142	#endif /* CONFIG_PPC64 */
	143
3160b097 BB	144	#ifdef CONFIG_PPC_FSL_BOOK3E
	145	/* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
	146	DEFINE_PER_CPU(int, next_tlbcam_idx);
	147	EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
	148	#endif
	149
f048aace BH	150	/*
	151	* Base TLB flushing operations:
	152	*
	153	* - flush_tlb_mm(mm) flushes the specified mm context TLB's
	154	* - flush_tlb_page(vma, vmaddr) flushes one page
	155	* - flush_tlb_range(vma, start, end) flushes a range of pages
	156	* - flush_tlb_kernel_range(start, end) flushes kernel pages
	157	*
	158	* - local_* variants of page and mm only apply to the current
	159	* processor
	160	*/
	161
	162	/*
	163	* These are the base non-SMP variants of page and mm flushing
	164	*/
	165	void local_flush_tlb_mm(struct mm_struct *mm)
	166	{
	167	unsigned int pid;
	168
	169	preempt_disable();
	170	pid = mm->context.id;
	171	if (pid != MMU_NO_CONTEXT)
	172	_tlbil_pid(pid);
	173	preempt_enable();
	174	}
	175	EXPORT_SYMBOL(local_flush_tlb_mm);
	176
d4e167da BH	177	void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
d4e167da BH	178	int tsize, int ind)
f048aace BH	179	{
	180	unsigned int pid;
	181
	182	preempt_disable();
d4e167da	183	pid = mm ? mm->context.id : 0;
f048aace	184	if (pid != MMU_NO_CONTEXT)
d4e167da	185	_tlbil_va(vmaddr, pid, tsize, ind);
f048aace BH	186	preempt_enable();
f048aace BH	187	}
f048aace	188
d4e167da BH	189	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	190	{
	191	__local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
25d21ad6	192	mmu_get_tsize(mmu_virtual_psize), 0);
d4e167da BH	193	}
d4e167da BH	194	EXPORT_SYMBOL(local_flush_tlb_page);
f048aace BH	195
	196	/*
	197	* And here are the SMP non-local implementations
	198	*/
	199	#ifdef CONFIG_SMP
	200
3eb93c55	201	static DEFINE_RAW_SPINLOCK(tlbivax_lock);
f048aace	202
fcce8109 BH	203	static int mm_is_core_local(struct mm_struct *mm)
	204	{
	205	return cpumask_subset(mm_cpumask(mm),
	206	topology_thread_cpumask(smp_processor_id()));
	207	}
	208
f048aace BH	209	struct tlb_flush_param {
	210	unsigned long addr;
	211	unsigned int pid;
d4e167da BH	212	unsigned int tsize;
d4e167da BH	213	unsigned int ind;
f048aace BH	214	};
	215
	216	static void do_flush_tlb_mm_ipi(void *param)
	217	{
	218	struct tlb_flush_param *p = param;
	219
	220	_tlbil_pid(p ? p->pid : 0);
	221	}
	222
	223	static void do_flush_tlb_page_ipi(void *param)
	224	{
	225	struct tlb_flush_param *p = param;
	226
d4e167da	227	_tlbil_va(p->addr, p->pid, p->tsize, p->ind);
f048aace BH	228	}
	229
	230
	231	/* Note on invalidations and PID:
	232	*
	233	* We snapshot the PID with preempt disabled. At this point, it can still
	234	* change either because:
	235	* - our context is being stolen (PID -> NO_CONTEXT) on another CPU
	236	* - we are invaliating some target that isn't currently running here
	237	* and is concurrently acquiring a new PID on another CPU
	238	* - some other CPU is re-acquiring a lost PID for this mm
	239	* etc...
	240	*
	241	* However, this shouldn't be a problem as we only guarantee
	242	* invalidation of TLB entries present prior to this call, so we
	243	* don't care about the PID changing, and invalidating a stale PID
	244	* is generally harmless.
	245	*/
	246
	247	void flush_tlb_mm(struct mm_struct *mm)
	248	{
f048aace BH	249	unsigned int pid;
	250
	251	preempt_disable();
	252	pid = mm->context.id;
	253	if (unlikely(pid == MMU_NO_CONTEXT))
	254	goto no_context;
fcce8109	255	if (!mm_is_core_local(mm)) {
f048aace	256	struct tlb_flush_param p = { .pid = pid };
56aa4129 RR	257	/* Ignores smp_processor_id() even if set. */
	258	smp_call_function_many(mm_cpumask(mm),
	259	do_flush_tlb_mm_ipi, &p, 1);
f048aace BH	260	}
	261	_tlbil_pid(pid);
	262	no_context:
	263	preempt_enable();
	264	}
	265	EXPORT_SYMBOL(flush_tlb_mm);
	266
d4e167da BH	267	void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
d4e167da BH	268	int tsize, int ind)
f048aace	269	{
56aa4129	270	struct cpumask *cpu_mask;
f048aace BH	271	unsigned int pid;
	272
	273	preempt_disable();
d4e167da	274	pid = mm ? mm->context.id : 0;
f048aace BH	275	if (unlikely(pid == MMU_NO_CONTEXT))
f048aace BH	276	goto bail;
d4e167da	277	cpu_mask = mm_cpumask(mm);
fcce8109	278	if (!mm_is_core_local(mm)) {
f048aace BH	279	/* If broadcast tlbivax is supported, use it */
	280	if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
	281	int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
	282	if (lock)
3eb93c55	283	raw_spin_lock(&tlbivax_lock);
d4e167da	284	_tlbivax_bcast(vmaddr, pid, tsize, ind);
f048aace	285	if (lock)
3eb93c55	286	raw_spin_unlock(&tlbivax_lock);
f048aace BH	287	goto bail;
f048aace BH	288	} else {
d4e167da BH	289	struct tlb_flush_param p = {
	290	.pid = pid,
	291	.addr = vmaddr,
	292	.tsize = tsize,
	293	.ind = ind,
	294	};
56aa4129 RR	295	/* Ignores smp_processor_id() even if set in cpu_mask */
56aa4129 RR	296	smp_call_function_many(cpu_mask,
f048aace BH	297	do_flush_tlb_page_ipi, &p, 1);
	298	}
	299	}
d4e167da	300	_tlbil_va(vmaddr, pid, tsize, ind);
f048aace BH	301	bail:
	302	preempt_enable();
	303	}
d4e167da BH	304
	305	void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	306	{
41151e77 BB	307	#ifdef CONFIG_HUGETLB_PAGE
	308	if (is_vm_hugetlb_page(vma))
	309	flush_hugetlb_page(vma, vmaddr);
	310	#endif
	311
d4e167da	312	__flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
25d21ad6	313	mmu_get_tsize(mmu_virtual_psize), 0);
d4e167da	314	}
f048aace BH	315	EXPORT_SYMBOL(flush_tlb_page);
	316
	317	#endif /* CONFIG_SMP */
	318
91b191c7 DK	319	#ifdef CONFIG_PPC_47x
	320	void __init early_init_mmu_47x(void)
	321	{
	322	#ifdef CONFIG_SMP
	323	unsigned long root = of_get_flat_dt_root();
	324	if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
	325	mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
	326	#endif /* CONFIG_SMP */
	327	}
	328	#endif /* CONFIG_PPC_47x */
	329
f048aace BH	330	/*
	331	* Flush kernel TLB entries in the given range
	332	*/
	333	void flush_tlb_kernel_range(unsigned long start, unsigned long end)
	334	{
	335	#ifdef CONFIG_SMP
	336	preempt_disable();
	337	smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
	338	_tlbil_pid(0);
	339	preempt_enable();
d6a09e0c	340	#else
f048aace	341	_tlbil_pid(0);
d6a09e0c	342	#endif
f048aace BH	343	}
	344	EXPORT_SYMBOL(flush_tlb_kernel_range);
	345
	346	/*
	347	* Currently, for range flushing, we just do a full mm flush. This should
	348	* be optimized based on a threshold on the size of the range, since
	349	* some implementation can stack multiple tlbivax before a tlbsync but
	350	* for now, we keep it that way
	351	*/
	352	void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	353	unsigned long end)
	354
	355	{
	356	flush_tlb_mm(vma->vm_mm);
	357	}
	358	EXPORT_SYMBOL(flush_tlb_range);
c7cc58a1 BH	359
	360	void tlb_flush(struct mmu_gather *tlb)
	361	{
	362	flush_tlb_mm(tlb->mm);
c7cc58a1	363	}
25d21ad6 BH	364
	365	/*
	366	* Below are functions specific to the 64-bit variant of Book3E though that
	367	* may change in the future
	368	*/
	369
	370	#ifdef CONFIG_PPC64
	371
	372	/*
	373	* Handling of virtual linear page tables or indirect TLB entries
	374	* flushing when PTE pages are freed
	375	*/
	376	void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
	377	{
	378	int tsize = mmu_psize_defs[mmu_pte_psize].enc;
	379
	380	if (book3e_htw_enabled) {
	381	unsigned long start = address & PMD_MASK;
	382	unsigned long end = address + PMD_SIZE;
	383	unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;
	384
	385	/* This isn't the most optimal, ideally we would factor out the
	386	* while preempt & CPU mask mucking around, or even the IPI but
	387	* it will do for now
	388	*/
	389	while (start < end) {
	390	__flush_tlb_page(tlb->mm, start, tsize, 1);
	391	start += size;
	392	}
	393	} else {
	394	unsigned long rmask = 0xf000000000000000ul;
	395	unsigned long rid = (address & rmask) \| 0x1000000000000000ul;
	396	unsigned long vpte = address & ~rmask;
	397
	398	#ifdef CONFIG_PPC_64K_PAGES
	399	vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
	400	#else
	401	vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
	402	#endif
	403	vpte \|= rid;
	404	__flush_tlb_page(tlb->mm, vpte, tsize, 0);
	405	}
	406	}
	407
f2b26c92 BH	408	static void setup_page_sizes(void)
f2b26c92 BH	409	{
988cf86d KG	410	unsigned int tlb0cfg;
	411	unsigned int tlb0ps;
	412	unsigned int eptcfg;
f2b26c92 BH	413	int i, psize;
f2b26c92 BH	414
988cf86d KG	415	#ifdef CONFIG_PPC_FSL_BOOK3E
988cf86d KG	416	unsigned int mmucfg = mfspr(SPRN_MMUCFG);
1b291873	417	int fsl_mmu = mmu_has_feature(MMU_FTR_TYPE_FSL_E);
988cf86d	418
1b291873	419	if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) {
988cf86d KG	420	unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
	421	unsigned int min_pg, max_pg;
	422
	423	min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
	424	max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;
	425
	426	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	427	struct mmu_psize_def *def;
	428	unsigned int shift;
	429
	430	def = &mmu_psize_defs[psize];
	431	shift = def->shift;
	432
	433	if (shift == 0)
	434	continue;
	435
	436	/* adjust to be in terms of 4^shift Kb */
	437	shift = (shift - 10) >> 1;
	438
	439	if ((shift >= min_pg) && (shift <= max_pg))
	440	def->flags \|= MMU_PAGE_SIZE_DIRECT;
	441	}
	442
	443	goto no_indirect;
	444	}
1b291873 KG	445
	446	if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V2) {
	447	u32 tlb1ps = mfspr(SPRN_TLB1PS);
	448
	449	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	450	struct mmu_psize_def *def = &mmu_psize_defs[psize];
	451
	452	if (tlb1ps & (1U << (def->shift - 10))) {
	453	def->flags \|= MMU_PAGE_SIZE_DIRECT;
	454	}
	455	}
	456
	457	goto no_indirect;
	458	}
988cf86d KG	459	#endif
	460
	461	tlb0cfg = mfspr(SPRN_TLB0CFG);
	462	tlb0ps = mfspr(SPRN_TLB0PS);
	463	eptcfg = mfspr(SPRN_EPTCFG);
	464
f2b26c92 BH	465	/* Look for supported direct sizes */
	466	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	467	struct mmu_psize_def *def = &mmu_psize_defs[psize];
	468
	469	if (tlb0ps & (1U << (def->shift - 10)))
	470	def->flags \|= MMU_PAGE_SIZE_DIRECT;
	471	}
	472
	473	/* Indirect page sizes supported ? */
	474	if ((tlb0cfg & TLBnCFG_IND) == 0)
	475	goto no_indirect;
	476
	477	/* Now, we only deal with one IND page size for each
	478	* direct size. Hopefully all implementations today are
	479	* unambiguous, but we might want to be careful in the
	480	* future.
	481	*/
	482	for (i = 0; i < 3; i++) {
	483	unsigned int ps, sps;
	484
	485	sps = eptcfg & 0x1f;
	486	eptcfg >>= 5;
	487	ps = eptcfg & 0x1f;
	488	eptcfg >>= 5;
	489	if (!ps \|\| !sps)
	490	continue;
	491	for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
	492	struct mmu_psize_def *def = &mmu_psize_defs[psize];
	493
	494	if (ps == (def->shift - 10))
	495	def->flags \|= MMU_PAGE_SIZE_INDIRECT;
	496	if (sps == (def->shift - 10))
	497	def->ind = ps + 10;
	498	}
	499	}
	500	no_indirect:
	501
	502	/* Cleanup array and print summary */
	503	pr_info("MMU: Supported page sizes\n");
	504	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	505	struct mmu_psize_def *def = &mmu_psize_defs[psize];
	506	const char *__page_type_names[] = {
	507	"unsupported",
	508	"direct",
	509	"indirect",
	510	"direct & indirect"
	511	};
	512	if (def->flags == 0) {
	513	def->shift = 0;
	514	continue;
	515	}
	516	pr_info(" %8ld KB as %s\n", 1ul << (def->shift - 10),
	517	__page_type_names[def->flags & 0x3]);
	518	}
	519	}
	520
f67f4ef5	521	static void __patch_exception(int exc, unsigned long addr)
25d21ad6 BH	522	{
25d21ad6 BH	523	extern unsigned int interrupt_base_book3e;
f67f4ef5 SW	524	unsigned int *ibase = &interrupt_base_book3e;
	525
	526	/* Our exceptions vectors start with a NOP and -then- a branch
	527	* to deal with single stepping from userspace which stops on
	528	* the second instruction. Thus we need to patch the second
	529	* instruction of the exception, not the first one
	530	*/
	531
	532	patch_branch(ibase + (exc / 4) + 1, addr, 0);
	533	}
25d21ad6	534
f67f4ef5 SW	535	#define patch_exception(exc, name) do { \
	536	extern unsigned int name; \
	537	__patch_exception((exc), (unsigned long)&name); \
	538	} while (0)
f2b26c92	539
f67f4ef5 SW	540	static void setup_mmu_htw(void)
f67f4ef5 SW	541	{
f2b26c92 BH	542	/* Check if HW tablewalk is present, and if yes, enable it by:
	543	*
	544	* - patching the TLB miss handlers to branch to the
	545	* one dedicates to it
	546	*
	547	* - setting the global book3e_htw_enabled
	548	*/
	549	unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG);
	550
	551	if ((tlb0cfg & TLBnCFG_IND) &&
	552	(tlb0cfg & TLBnCFG_PT)) {
f67f4ef5 SW	553	patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
f67f4ef5 SW	554	patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
f2b26c92 BH	555	book3e_htw_enabled = 1;
f2b26c92 BH	556	}
32d206eb KG	557	pr_info("MMU: Book3E HW tablewalk %s\n",
32d206eb KG	558	book3e_htw_enabled ? "enabled" : "not supported");
f2b26c92 BH	559	}
	560
	561	/*
	562	* Early initialization of the MMU TLB code
	563	*/
	564	static void __early_init_mmu(int boot_cpu)
	565	{
25d21ad6 BH	566	unsigned int mas4;
	567
	568	/* XXX This will have to be decided at runtime, but right
32a74949 BH	569	* now our boot and TLB miss code hard wires it. Ideally
	570	* we should find out a suitable page size and patch the
	571	* TLB miss code (either that or use the PACA to store
	572	* the value we want)
25d21ad6 BH	573	*/
	574	mmu_linear_psize = MMU_PAGE_1G;
	575
32a74949 BH	576	/* XXX This should be decided at runtime based on supported
	577	* page sizes in the TLB, but for now let's assume 16M is
	578	* always there and a good fit (which it probably is)
	579	*/
	580	mmu_vmemmap_psize = MMU_PAGE_16M;
25d21ad6	581
25d21ad6 BH	582	/* XXX This code only checks for TLB 0 capabilities and doesn't
	583	* check what page size combos are supported by the HW. It
	584	* also doesn't handle the case where a separate array holds
	585	* the IND entries from the array loaded by the PT.
	586	*/
	587	if (boot_cpu) {
f2b26c92 BH	588	/* Look for supported page sizes */
f2b26c92 BH	589	setup_page_sizes();
25d21ad6	590
f2b26c92 BH	591	/* Look for HW tablewalk support */
f2b26c92 BH	592	setup_mmu_htw();
25d21ad6 BH	593	}
	594
	595	/* Set MAS4 based on page table setting */
	596
	597	mas4 = 0x4 << MAS4_WIMGED_SHIFT;
	598	if (book3e_htw_enabled) {
	599	mas4 \|= mas4 \| MAS4_INDD;
	600	#ifdef CONFIG_PPC_64K_PAGES
	601	mas4 \|= BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
	602	mmu_pte_psize = MMU_PAGE_256M;
	603	#else
	604	mas4 \|= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
	605	mmu_pte_psize = MMU_PAGE_1M;
	606	#endif
	607	} else {
	608	#ifdef CONFIG_PPC_64K_PAGES
	609	mas4 \|= BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
	610	#else
	611	mas4 \|= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
	612	#endif
	613	mmu_pte_psize = mmu_virtual_psize;
	614	}
	615	mtspr(SPRN_MAS4, mas4);
	616
	617	/* Set the global containing the top of the linear mapping
	618	* for use by the TLB miss code
	619	*/
95f72d1e	620	linear_map_top = memblock_end_of_DRAM();
25d21ad6	621
55fd766b KG	622	#ifdef CONFIG_PPC_FSL_BOOK3E
	623	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
	624	unsigned int num_cams;
	625
	626	/* use a quarter of the TLBCAM for bolted linear map */
	627	num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
	628	linear_map_top = map_mem_in_cams(linear_map_top, num_cams);
	629
	630	/* limit memory so we dont have linear faults */
	631	memblock_enforce_memory_limit(linear_map_top);
f67f4ef5 SW	632
	633	patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
	634	patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e);
55fd766b KG	635	}
	636	#endif
	637
25d21ad6 BH	638	/* A sync won't hurt us after mucking around with
	639	* the MMU configuration
	640	*/
	641	mb();
e63075a3 BH	642
e63075a3 BH	643	memblock_set_current_limit(linear_map_top);
25d21ad6 BH	644	}
	645
	646	void __init early_init_mmu(void)
	647	{
	648	__early_init_mmu(1);
	649	}
	650
	651	void __cpuinit early_init_mmu_secondary(void)
	652	{
	653	__early_init_mmu(0);
	654	}
	655
cd3db0c4 BH	656	void setup_initial_memory_limit(phys_addr_t first_memblock_base,
	657	phys_addr_t first_memblock_size)
	658	{
1dc91c3e	659	/* On non-FSL Embedded 64-bit, we adjust the RMA size to match
cd3db0c4 BH	660	* the bolted TLB entry. We know for now that only 1G
cd3db0c4 BH	661	* entries are supported though that may eventually
1dc91c3e KG	662	* change.
	663	*
	664	* on FSL Embedded 64-bit, we adjust the RMA size to match the
	665	* first bolted TLB entry size. We still limit max to 1G even if
	666	* the TLB could cover more. This is due to what the early init
	667	* code is setup to do.
	668	*
	669	* We crop it to the size of the first MEMBLOCK to
cd3db0c4 BH	670	* avoid going over total available memory just in case...
cd3db0c4 BH	671	*/
1dc91c3e KG	672	#ifdef CONFIG_PPC_FSL_BOOK3E
	673	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
	674	unsigned long linear_sz;
	675	linear_sz = calc_cam_sz(first_memblock_size, PAGE_OFFSET,
	676	first_memblock_base);
	677	ppc64_rma_size = min_t(u64, linear_sz, 0x40000000);
	678	} else
	679	#endif
	680	ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
cd3db0c4 BH	681
cd3db0c4 BH	682	/* Finally limit subsequent allocations */
4a89261b	683	memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
cd3db0c4	684	}
91b191c7 DK	685	#else /* ! CONFIG_PPC64 */
	686	void __init early_init_mmu(void)
	687	{
	688	#ifdef CONFIG_PPC_47x
	689	early_init_mmu_47x();
	690	#endif
	691	}
25d21ad6	692	#endif /* CONFIG_PPC64 */