[deliverable/linux.git] / arch / powerpc / mm / hugetlbpage-book3e.c

/*
 * PPC Huge TLB Page Support for Book3E MMU
 *
 * Copyright (C) 2009 David Gibson, IBM Corporation.
 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
 *
 */
#include <linux/mm.h>
#include <linux/hugetlb.h>

#ifdef CONFIG_PPC_FSL_BOOK3E
#ifdef CONFIG_PPC64
static inline int tlb1_next(void)
{
	struct paca_struct *paca = get_paca();
	struct tlb_core_data *tcd;
	int this, next;

	tcd = paca->tcd_ptr;
	this = tcd->esel_next;

	next = this + 1;
	if (next >= tcd->esel_max)
		next = tcd->esel_first;

	tcd->esel_next = next;
	return this;
}
#else
static inline int tlb1_next(void)
{
	int index, ncams;

	ncams = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY;

	index = this_cpu_read(next_tlbcam_idx);

	/* Just round-robin the entries and wrap when we hit the end */
	if (unlikely(index == ncams - 1))
		__this_cpu_write(next_tlbcam_idx, tlbcam_index);
	else
		__this_cpu_inc(next_tlbcam_idx);

	return index;
}
#endif /* !PPC64 */
#endif /* FSL */

static inline int mmu_get_tsize(int psize)
{
	return mmu_psize_defs[psize].enc;
}

static inline int book3e_tlb_exists(unsigned long ea, unsigned long pid)
{
	int found = 0;

	mtspr(SPRN_MAS6, pid << 16);
	if (mmu_has_feature(MMU_FTR_USE_TLBRSRV)) {
		asm volatile(
			"li	%0,0\n"
			"tlbsx.	0,%1\n"
			"bne	1f\n"
			"li	%0,1\n"
			"1:\n"
			: "=&r"(found) : "r"(ea));
	} else {
		asm volatile(
			"tlbsx	0,%1\n"
			"mfspr	%0,0x271\n"
			"srwi	%0,%0,31\n"
			: "=&r"(found) : "r"(ea));
	}

	return found;
}

void book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea,
			    pte_t pte)
{
	unsigned long mas1, mas2;
	u64 mas7_3;
	unsigned long psize, tsize, shift;
	unsigned long flags;
	struct mm_struct *mm;

#ifdef CONFIG_PPC_FSL_BOOK3E
	int index;
#endif

	if (unlikely(is_kernel_addr(ea)))
		return;

	mm = vma->vm_mm;

#ifdef CONFIG_PPC_MM_SLICES
	psize = get_slice_psize(mm, ea);
	tsize = mmu_get_tsize(psize);
	shift = mmu_psize_defs[psize].shift;
#else
	psize = vma_mmu_pagesize(vma);
	shift = __ilog2(psize);
	tsize = shift - 10;
#endif

	/*
	 * We can't be interrupted while we're setting up the MAS
	 * regusters or after we've confirmed that no tlb exists.
	 */
	local_irq_save(flags);

	if (unlikely(book3e_tlb_exists(ea, mm->context.id))) {
		local_irq_restore(flags);
		return;
	}

#ifdef CONFIG_PPC_FSL_BOOK3E
	/* We have to use the CAM(TLB1) on FSL parts for hugepages */
	index = tlb1_next();
	mtspr(SPRN_MAS0, MAS0_ESEL(index) | MAS0_TLBSEL(1));
#endif

	mas1 = MAS1_VALID | MAS1_TID(mm->context.id) | MAS1_TSIZE(tsize);
	mas2 = ea & ~((1UL << shift) - 1);
	mas2 |= (pte_val(pte) >> PTE_WIMGE_SHIFT) & MAS2_WIMGE_MASK;
	mas7_3 = (u64)pte_pfn(pte) << PAGE_SHIFT;
	mas7_3 |= (pte_val(pte) >> PTE_BAP_SHIFT) & MAS3_BAP_MASK;
	if (!pte_dirty(pte))
		mas7_3 &= ~(MAS3_SW|MAS3_UW);

	mtspr(SPRN_MAS1, mas1);
	mtspr(SPRN_MAS2, mas2);

	if (mmu_has_feature(MMU_FTR_USE_PAIRED_MAS)) {
		mtspr(SPRN_MAS7_MAS3, mas7_3);
	} else {
		if (mmu_has_feature(MMU_FTR_BIG_PHYS))
			mtspr(SPRN_MAS7, upper_32_bits(mas7_3));
		mtspr(SPRN_MAS3, lower_32_bits(mas7_3));
	}

	asm volatile ("tlbwe");

	local_irq_restore(flags);
}

void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
	struct hstate *hstate = hstate_file(vma->vm_file);
	unsigned long tsize = huge_page_shift(hstate) - 10;

	__flush_tlb_page(vma->vm_mm, vmaddr, tsize, 0);
}
Commit	Line	Data
41151e77 BB	1	/*
	2	* PPC Huge TLB Page Support for Book3E MMU
	3	*
	4	* Copyright (C) 2009 David Gibson, IBM Corporation.
	5	* Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
	6	*
	7	*/
	8	#include <linux/mm.h>
	9	#include <linux/hugetlb.h>
	10
bbead78c SW	11	#ifdef CONFIG_PPC_FSL_BOOK3E
	12	#ifdef CONFIG_PPC64
	13	static inline int tlb1_next(void)
	14	{
	15	struct paca_struct *paca = get_paca();
	16	struct tlb_core_data *tcd;
	17	int this, next;
	18
	19	tcd = paca->tcd_ptr;
	20	this = tcd->esel_next;
	21
	22	next = this + 1;
	23	if (next >= tcd->esel_max)
	24	next = tcd->esel_first;
	25
	26	tcd->esel_next = next;
	27	return this;
	28	}
	29	#else
	30	static inline int tlb1_next(void)
	31	{
	32	int index, ncams;
	33
	34	ncams = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY;
	35
69111bac	36	index = this_cpu_read(next_tlbcam_idx);
bbead78c SW	37
	38	/* Just round-robin the entries and wrap when we hit the end */
	39	if (unlikely(index == ncams - 1))
69111bac	40	__this_cpu_write(next_tlbcam_idx, tlbcam_index);
bbead78c	41	else
69111bac	42	__this_cpu_inc(next_tlbcam_idx);
bbead78c SW	43
	44	return index;
	45	}
	46	#endif /* !PPC64 */
	47	#endif /* FSL */
	48
41151e77 BB	49	static inline int mmu_get_tsize(int psize)
	50	{
	51	return mmu_psize_defs[psize].enc;
	52	}
	53
	54	static inline int book3e_tlb_exists(unsigned long ea, unsigned long pid)
	55	{
	56	int found = 0;
	57
	58	mtspr(SPRN_MAS6, pid << 16);
	59	if (mmu_has_feature(MMU_FTR_USE_TLBRSRV)) {
	60	asm volatile(
	61	"li %0,0\n"
	62	"tlbsx. 0,%1\n"
	63	"bne 1f\n"
	64	"li %0,1\n"
	65	"1:\n"
	66	: "=&r"(found) : "r"(ea));
	67	} else {
	68	asm volatile(
	69	"tlbsx 0,%1\n"
	70	"mfspr %0,0x271\n"
	71	"srwi %0,%0,31\n"
	72	: "=&r"(found) : "r"(ea));
	73	}
	74
	75	return found;
	76	}
	77
d93e4d7d BB	78	void book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea,
d93e4d7d BB	79	pte_t pte)
41151e77 BB	80	{
	81	unsigned long mas1, mas2;
	82	u64 mas7_3;
	83	unsigned long psize, tsize, shift;
	84	unsigned long flags;
d93e4d7d	85	struct mm_struct *mm;
41151e77 BB	86
41151e77 BB	87	#ifdef CONFIG_PPC_FSL_BOOK3E
bbead78c	88	int index;
41151e77 BB	89	#endif
	90
	91	if (unlikely(is_kernel_addr(ea)))
	92	return;
	93
d93e4d7d BB	94	mm = vma->vm_mm;
d93e4d7d BB	95
25c29f9e	96	#ifdef CONFIG_PPC_MM_SLICES
8c1674de BB	97	psize = get_slice_psize(mm, ea);
8c1674de BB	98	tsize = mmu_get_tsize(psize);
41151e77 BB	99	shift = mmu_psize_defs[psize].shift;
41151e77 BB	100	#else
d93e4d7d	101	psize = vma_mmu_pagesize(vma);
8c1674de BB	102	shift = __ilog2(psize);
8c1674de BB	103	tsize = shift - 10;
41151e77 BB	104	#endif
	105
	106	/*
	107	* We can't be interrupted while we're setting up the MAS
	108	* regusters or after we've confirmed that no tlb exists.
	109	*/
	110	local_irq_save(flags);
	111
	112	if (unlikely(book3e_tlb_exists(ea, mm->context.id))) {
	113	local_irq_restore(flags);
	114	return;
	115	}
	116
	117	#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77	118	/* We have to use the CAM(TLB1) on FSL parts for hugepages */
bbead78c	119	index = tlb1_next();
41151e77	120	mtspr(SPRN_MAS0, MAS0_ESEL(index) \| MAS0_TLBSEL(1));
41151e77	121	#endif
bbead78c	122
41151e77 BB	123	mas1 = MAS1_VALID \| MAS1_TID(mm->context.id) \| MAS1_TSIZE(tsize);
	124	mas2 = ea & ~((1UL << shift) - 1);
	125	mas2 \|= (pte_val(pte) >> PTE_WIMGE_SHIFT) & MAS2_WIMGE_MASK;
	126	mas7_3 = (u64)pte_pfn(pte) << PAGE_SHIFT;
	127	mas7_3 \|= (pte_val(pte) >> PTE_BAP_SHIFT) & MAS3_BAP_MASK;
	128	if (!pte_dirty(pte))
	129	mas7_3 &= ~(MAS3_SW\|MAS3_UW);
	130
	131	mtspr(SPRN_MAS1, mas1);
	132	mtspr(SPRN_MAS2, mas2);
	133
	134	if (mmu_has_feature(MMU_FTR_USE_PAIRED_MAS)) {
	135	mtspr(SPRN_MAS7_MAS3, mas7_3);
	136	} else {
7c732cba KH	137	if (mmu_has_feature(MMU_FTR_BIG_PHYS))
7c732cba KH	138	mtspr(SPRN_MAS7, upper_32_bits(mas7_3));
41151e77 BB	139	mtspr(SPRN_MAS3, lower_32_bits(mas7_3));
	140	}
	141
	142	asm volatile ("tlbwe");
	143
	144	local_irq_restore(flags);
	145	}
	146
	147	void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	148	{
	149	struct hstate *hstate = hstate_file(vma->vm_file);
	150	unsigned long tsize = huge_page_shift(hstate) - 10;
	151
d742aa15	152	__flush_tlb_page(vma->vm_mm, vmaddr, tsize, 0);
41151e77	153	}