[deliverable/linux.git] / arch / x86 / mm / pgtable.c

#include <linux/mm.h>
#include <linux/gfp.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/tlb.h>
#include <asm/fixmap.h>

#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO

#ifdef CONFIG_HIGHPTE
#define PGALLOC_USER_GFP __GFP_HIGHMEM
#else
#define PGALLOC_USER_GFP 0
#endif

gfp_t __userpte_alloc_gfp = PGALLOC_GFP | PGALLOC_USER_GFP;

pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
	return (pte_t *)__get_free_page(PGALLOC_GFP);
}

pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
	struct page *pte;

	pte = alloc_pages(__userpte_alloc_gfp, 0);
	if (!pte)
		return NULL;
	if (!pgtable_page_ctor(pte)) {
		__free_page(pte);
		return NULL;
	}
	return pte;
}

static int __init setup_userpte(char *arg)
{
	if (!arg)
		return -EINVAL;

	/*
	 * "userpte=nohigh" disables allocation of user pagetables in
	 * high memory.
	 */
	if (strcmp(arg, "nohigh") == 0)
		__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
	else
		return -EINVAL;
	return 0;
}
early_param("userpte", setup_userpte);

void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
	pgtable_page_dtor(pte);
	paravirt_release_pte(page_to_pfn(pte));
	tlb_remove_page(tlb, pte);
}

#if PAGETABLE_LEVELS > 2
void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
{
	struct page *page = virt_to_page(pmd);
	paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
	/*
	 * NOTE! For PAE, any changes to the top page-directory-pointer-table
	 * entries need a full cr3 reload to flush.
	 */
#ifdef CONFIG_X86_PAE
	tlb->need_flush_all = 1;
#endif
	pgtable_pmd_page_dtor(page);
	tlb_remove_page(tlb, page);
}

#if PAGETABLE_LEVELS > 3
void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
{
	paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
	tlb_remove_page(tlb, virt_to_page(pud));
}
#endif	/* PAGETABLE_LEVELS > 3 */
#endif	/* PAGETABLE_LEVELS > 2 */

static inline void pgd_list_add(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);

	list_add(&page->lru, &pgd_list);
}

static inline void pgd_list_del(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);

	list_del(&page->lru);
}

#define UNSHARED_PTRS_PER_PGD				\
	(SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)


static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
{
	BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
	virt_to_page(pgd)->index = (pgoff_t)mm;
}

struct mm_struct *pgd_page_get_mm(struct page *page)
{
	return (struct mm_struct *)page->index;
}

static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
{
	/* If the pgd points to a shared pagetable level (either the
	   ptes in non-PAE, or shared PMD in PAE), then just copy the
	   references from swapper_pg_dir. */
	if (PAGETABLE_LEVELS == 2 ||
	    (PAGETABLE_LEVELS == 3 && SHARED_KERNEL_PMD) ||
	    PAGETABLE_LEVELS == 4) {
		clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
				swapper_pg_dir + KERNEL_PGD_BOUNDARY,
				KERNEL_PGD_PTRS);
	}

	/* list required to sync kernel mapping updates */
	if (!SHARED_KERNEL_PMD) {
		pgd_set_mm(pgd, mm);
		pgd_list_add(pgd);
	}
}

static void pgd_dtor(pgd_t *pgd)
{
	if (SHARED_KERNEL_PMD)
		return;

	spin_lock(&pgd_lock);
	pgd_list_del(pgd);
	spin_unlock(&pgd_lock);
}

/*
 * List of all pgd's needed for non-PAE so it can invalidate entries
 * in both cached and uncached pgd's; not needed for PAE since the
 * kernel pmd is shared. If PAE were not to share the pmd a similar
 * tactic would be needed. This is essentially codepath-based locking
 * against pageattr.c; it is the unique case in which a valid change
 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
 * vmalloc faults work because attached pagetables are never freed.
 * -- nyc
 */

#ifdef CONFIG_X86_PAE
/*
 * In PAE mode, we need to do a cr3 reload (=tlb flush) when
 * updating the top-level pagetable entries to guarantee the
 * processor notices the update.  Since this is expensive, and
 * all 4 top-level entries are used almost immediately in a
 * new process's life, we just pre-populate them here.
 *
 * Also, if we're in a paravirt environment where the kernel pmd is
 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
 * and initialize the kernel pmds here.
 */
#define PREALLOCATED_PMDS	UNSHARED_PTRS_PER_PGD

void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
{
	paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);

	/* Note: almost everything apart from _PAGE_PRESENT is
	   reserved at the pmd (PDPT) level. */
	set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));

	/*
	 * According to Intel App note "TLBs, Paging-Structure Caches,
	 * and Their Invalidation", April 2007, document 317080-001,
	 * section 8.1: in PAE mode we explicitly have to flush the
	 * TLB via cr3 if the top-level pgd is changed...
	 */
	flush_tlb_mm(mm);
}
#else  /* !CONFIG_X86_PAE */

/* No need to prepopulate any pagetable entries in non-PAE modes. */
#define PREALLOCATED_PMDS	0

#endif	/* CONFIG_X86_PAE */

static void free_pmds(struct mm_struct *mm, pmd_t *pmds[])
{
	int i;

	for(i = 0; i < PREALLOCATED_PMDS; i++)
		if (pmds[i]) {
			pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
			free_page((unsigned long)pmds[i]);
			mm_dec_nr_pmds(mm);
		}
}

static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[])
{
	int i;
	bool failed = false;

	for(i = 0; i < PREALLOCATED_PMDS; i++) {
		pmd_t *pmd = (pmd_t *)__get_free_page(PGALLOC_GFP);
		if (!pmd)
			failed = true;
		if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
			free_page((unsigned long)pmd);
			pmd = NULL;
			failed = true;
		}
		if (pmd)
			mm_inc_nr_pmds(mm);
		pmds[i] = pmd;
	}

	if (failed) {
		free_pmds(mm, pmds);
		return -ENOMEM;
	}

	return 0;
}

/*
 * Mop up any pmd pages which may still be attached to the pgd.
 * Normally they will be freed by munmap/exit_mmap, but any pmd we
 * preallocate which never got a corresponding vma will need to be
 * freed manually.
 */
static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
{
	int i;

	for(i = 0; i < PREALLOCATED_PMDS; i++) {
		pgd_t pgd = pgdp[i];

		if (pgd_val(pgd) != 0) {
			pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);

			pgdp[i] = native_make_pgd(0);

			paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
			pmd_free(mm, pmd);
			mm_dec_nr_pmds(mm);
		}
	}
}

static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
{
	pud_t *pud;
	int i;

	if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
		return;

	pud = pud_offset(pgd, 0);

	for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
		pmd_t *pmd = pmds[i];

		if (i >= KERNEL_PGD_BOUNDARY)
			memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
			       sizeof(pmd_t) * PTRS_PER_PMD);

		pud_populate(mm, pud, pmd);
	}
}

pgd_t *pgd_alloc(struct mm_struct *mm)
{
	pgd_t *pgd;
	pmd_t *pmds[PREALLOCATED_PMDS];

	pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);

	if (pgd == NULL)
		goto out;

	mm->pgd = pgd;

	if (preallocate_pmds(mm, pmds) != 0)
		goto out_free_pgd;

	if (paravirt_pgd_alloc(mm) != 0)
		goto out_free_pmds;

	/*
	 * Make sure that pre-populating the pmds is atomic with
	 * respect to anything walking the pgd_list, so that they
	 * never see a partially populated pgd.
	 */
	spin_lock(&pgd_lock);

	pgd_ctor(mm, pgd);
	pgd_prepopulate_pmd(mm, pgd, pmds);

	spin_unlock(&pgd_lock);

	return pgd;

out_free_pmds:
	free_pmds(mm, pmds);
out_free_pgd:
	free_page((unsigned long)pgd);
out:
	return NULL;
}

void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
	pgd_mop_up_pmds(mm, pgd);
	pgd_dtor(pgd);
	paravirt_pgd_free(mm, pgd);
	free_page((unsigned long)pgd);
}

/*
 * Used to set accessed or dirty bits in the page table entries
 * on other architectures. On x86, the accessed and dirty bits
 * are tracked by hardware. However, do_wp_page calls this function
 * to also make the pte writeable at the same time the dirty bit is
 * set. In that case we do actually need to write the PTE.
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pte_t *ptep,
			  pte_t entry, int dirty)
{
	int changed = !pte_same(*ptep, entry);

	if (changed && dirty) {
		*ptep = entry;
		pte_update_defer(vma->vm_mm, address, ptep);
	}

	return changed;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp,
			  pmd_t entry, int dirty)
{
	int changed = !pmd_same(*pmdp, entry);

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	if (changed && dirty) {
		*pmdp = entry;
		pmd_update_defer(vma->vm_mm, address, pmdp);
		/*
		 * We had a write-protection fault here and changed the pmd
		 * to to more permissive. No need to flush the TLB for that,
		 * #PF is architecturally guaranteed to do that and in the
		 * worst-case we'll generate a spurious fault.
		 */
	}

	return changed;
}
#endif

int ptep_test_and_clear_young(struct vm_area_struct *vma,
			      unsigned long addr, pte_t *ptep)
{
	int ret = 0;

	if (pte_young(*ptep))
		ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
					 (unsigned long *) &ptep->pte);

	if (ret)
		pte_update(vma->vm_mm, addr, ptep);

	return ret;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_test_and_clear_young(struct vm_area_struct *vma,
			      unsigned long addr, pmd_t *pmdp)
{
	int ret = 0;

	if (pmd_young(*pmdp))
		ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
					 (unsigned long *)pmdp);

	if (ret)
		pmd_update(vma->vm_mm, addr, pmdp);

	return ret;
}
#endif

int ptep_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pte_t *ptep)
{
	/*
	 * On x86 CPUs, clearing the accessed bit without a TLB flush
	 * doesn't cause data corruption. [ It could cause incorrect
	 * page aging and the (mistaken) reclaim of hot pages, but the
	 * chance of that should be relatively low. ]
	 *
	 * So as a performance optimization don't flush the TLB when
	 * clearing the accessed bit, it will eventually be flushed by
	 * a context switch or a VM operation anyway. [ In the rare
	 * event of it not getting flushed for a long time the delay
	 * shouldn't really matter because there's no real memory
	 * pressure for swapout to react to. ]
	 */
	return ptep_test_and_clear_young(vma, address, ptep);
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pmd_t *pmdp)
{
	int young;

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	young = pmdp_test_and_clear_young(vma, address, pmdp);
	if (young)
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);

	return young;
}

void pmdp_splitting_flush(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp)
{
	int set;
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
				(unsigned long *)pmdp);
	if (set) {
		pmd_update(vma->vm_mm, address, pmdp);
		/* need tlb flush only to serialize against gup-fast */
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	}
}
#endif

/**
 * reserve_top_address - reserves a hole in the top of kernel address space
 * @reserve - size of hole to reserve
 *
 * Can be used to relocate the fixmap area and poke a hole in the top
 * of kernel address space to make room for a hypervisor.
 */
void __init reserve_top_address(unsigned long reserve)
{
#ifdef CONFIG_X86_32
	BUG_ON(fixmaps_set > 0);
	__FIXADDR_TOP = round_down(-reserve, 1 << PMD_SHIFT) - PAGE_SIZE;
	printk(KERN_INFO "Reserving virtual address space above 0x%08lx (rounded to 0x%08lx)\n",
	       -reserve, __FIXADDR_TOP + PAGE_SIZE);
#endif
}

int fixmaps_set;

void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
{
	unsigned long address = __fix_to_virt(idx);

	if (idx >= __end_of_fixed_addresses) {
		BUG();
		return;
	}
	set_pte_vaddr(address, pte);
	fixmaps_set++;
}

void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
		       pgprot_t flags)
{
	__native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
}
Commit	Line	Data
4f76cd38	1	#include <linux/mm.h>
5a0e3ad6	2	#include <linux/gfp.h>
4f76cd38	3	#include <asm/pgalloc.h>
ee5aa8d3	4	#include <asm/pgtable.h>
4f76cd38	5	#include <asm/tlb.h>
a1d5a869	6	#include <asm/fixmap.h>
4f76cd38	7
9e730237 VN	8	#define PGALLOC_GFP GFP_KERNEL \| __GFP_NOTRACK \| __GFP_REPEAT \| __GFP_ZERO
9e730237 VN	9
14315592 IC	10	#ifdef CONFIG_HIGHPTE
	11	#define PGALLOC_USER_GFP __GFP_HIGHMEM
	12	#else
	13	#define PGALLOC_USER_GFP 0
	14	#endif
	15
	16	gfp_t __userpte_alloc_gfp = PGALLOC_GFP \| PGALLOC_USER_GFP;
	17
4f76cd38 JF	18	pte_t pte_alloc_one_kernel(struct mm_struct mm, unsigned long address)
4f76cd38 JF	19	{
9e730237	20	return (pte_t *)__get_free_page(PGALLOC_GFP);
4f76cd38 JF	21	}
	22
	23	pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
	24	{
	25	struct page *pte;
	26
14315592	27	pte = alloc_pages(__userpte_alloc_gfp, 0);
cecbd1b5 KS	28	if (!pte)
	29	return NULL;
	30	if (!pgtable_page_ctor(pte)) {
	31	__free_page(pte);
	32	return NULL;
	33	}
4f76cd38 JF	34	return pte;
	35	}
	36
14315592 IC	37	static int __init setup_userpte(char *arg)
	38	{
	39	if (!arg)
	40	return -EINVAL;
	41
	42	/*
	43	* "userpte=nohigh" disables allocation of user pagetables in
	44	* high memory.
	45	*/
	46	if (strcmp(arg, "nohigh") == 0)
	47	__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
	48	else
	49	return -EINVAL;
	50	return 0;
	51	}
	52	early_param("userpte", setup_userpte);
	53
9e1b32ca	54	void ___pte_free_tlb(struct mmu_gather tlb, struct page pte)
397f687a JF	55	{
397f687a JF	56	pgtable_page_dtor(pte);
6944a9c8	57	paravirt_release_pte(page_to_pfn(pte));
397f687a JF	58	tlb_remove_page(tlb, pte);
	59	}
	60
170fdff7	61	#if PAGETABLE_LEVELS > 2
9e1b32ca	62	void ___pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd)
170fdff7	63	{
c283610e	64	struct page *page = virt_to_page(pmd);
6944a9c8	65	paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
1de14c3c DH	66	/*
	67	* NOTE! For PAE, any changes to the top page-directory-pointer-table
	68	* entries need a full cr3 reload to flush.
	69	*/
	70	#ifdef CONFIG_X86_PAE
	71	tlb->need_flush_all = 1;
	72	#endif
c283610e KS	73	pgtable_pmd_page_dtor(page);
c283610e KS	74	tlb_remove_page(tlb, page);
170fdff7	75	}
5a5f8f42 JF	76
5a5f8f42 JF	77	#if PAGETABLE_LEVELS > 3
9e1b32ca	78	void ___pud_free_tlb(struct mmu_gather tlb, pud_t pud)
5a5f8f42	79	{
2761fa09	80	paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
5a5f8f42 JF	81	tlb_remove_page(tlb, virt_to_page(pud));
	82	}
	83	#endif /* PAGETABLE_LEVELS > 3 */
170fdff7 JF	84	#endif /* PAGETABLE_LEVELS > 2 */
170fdff7 JF	85
4f76cd38 JF	86	static inline void pgd_list_add(pgd_t *pgd)
	87	{
	88	struct page *page = virt_to_page(pgd);
4f76cd38	89
4f76cd38	90	list_add(&page->lru, &pgd_list);
4f76cd38 JF	91	}
	92
	93	static inline void pgd_list_del(pgd_t *pgd)
	94	{
	95	struct page *page = virt_to_page(pgd);
4f76cd38	96
4f76cd38	97	list_del(&page->lru);
4f76cd38 JF	98	}
4f76cd38 JF	99
4f76cd38	100	#define UNSHARED_PTRS_PER_PGD \
68db065c	101	(SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
4f76cd38	102
617d34d9 JF	103
	104	static void pgd_set_mm(pgd_t pgd, struct mm_struct mm)
	105	{
	106	BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
	107	virt_to_page(pgd)->index = (pgoff_t)mm;
	108	}
	109
	110	struct mm_struct pgd_page_get_mm(struct page page)
	111	{
	112	return (struct mm_struct *)page->index;
	113	}
	114
	115	static void pgd_ctor(struct mm_struct mm, pgd_t pgd)
4f76cd38	116	{
4f76cd38 JF	117	/* If the pgd points to a shared pagetable level (either the
	118	ptes in non-PAE, or shared PMD in PAE), then just copy the
	119	references from swapper_pg_dir. */
	120	if (PAGETABLE_LEVELS == 2 \|\|
85958b46 JF	121	(PAGETABLE_LEVELS == 3 && SHARED_KERNEL_PMD) \|\|
85958b46 JF	122	PAGETABLE_LEVELS == 4) {
68db065c JF	123	clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
68db065c JF	124	swapper_pg_dir + KERNEL_PGD_BOUNDARY,
4f76cd38	125	KERNEL_PGD_PTRS);
4f76cd38 JF	126	}
	127
	128	/* list required to sync kernel mapping updates */
617d34d9 JF	129	if (!SHARED_KERNEL_PMD) {
617d34d9 JF	130	pgd_set_mm(pgd, mm);
4f76cd38	131	pgd_list_add(pgd);
617d34d9	132	}
4f76cd38 JF	133	}
4f76cd38 JF	134
17b74627	135	static void pgd_dtor(pgd_t *pgd)
4f76cd38	136	{
4f76cd38 JF	137	if (SHARED_KERNEL_PMD)
	138	return;
	139
a79e53d8	140	spin_lock(&pgd_lock);
4f76cd38	141	pgd_list_del(pgd);
a79e53d8	142	spin_unlock(&pgd_lock);
4f76cd38 JF	143	}
4f76cd38 JF	144
85958b46 JF	145	/*
	146	* List of all pgd's needed for non-PAE so it can invalidate entries
	147	* in both cached and uncached pgd's; not needed for PAE since the
	148	* kernel pmd is shared. If PAE were not to share the pmd a similar
	149	* tactic would be needed. This is essentially codepath-based locking
	150	* against pageattr.c; it is the unique case in which a valid change
	151	* of kernel pagetables can't be lazily synchronized by vmalloc faults.
	152	* vmalloc faults work because attached pagetables are never freed.
6d49e352	153	* -- nyc
85958b46 JF	154	*/
85958b46 JF	155
4f76cd38	156	#ifdef CONFIG_X86_PAE
d8d5900e JF	157	/*
	158	* In PAE mode, we need to do a cr3 reload (=tlb flush) when
	159	* updating the top-level pagetable entries to guarantee the
	160	* processor notices the update. Since this is expensive, and
	161	* all 4 top-level entries are used almost immediately in a
	162	* new process's life, we just pre-populate them here.
	163	*
	164	* Also, if we're in a paravirt environment where the kernel pmd is
	165	* not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
	166	* and initialize the kernel pmds here.
	167	*/
	168	#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
	169
	170	void pud_populate(struct mm_struct mm, pud_t pudp, pmd_t *pmd)
	171	{
	172	paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
	173
	174	/* Note: almost everything apart from _PAGE_PRESENT is
	175	reserved at the pmd (PDPT) level. */
	176	set_pud(pudp, __pud(__pa(pmd) \| _PAGE_PRESENT));
	177
	178	/*
	179	* According to Intel App note "TLBs, Paging-Structure Caches,
	180	* and Their Invalidation", April 2007, document 317080-001,
	181	* section 8.1: in PAE mode we explicitly have to flush the
	182	* TLB via cr3 if the top-level pgd is changed...
	183	*/
4981d01e	184	flush_tlb_mm(mm);
d8d5900e JF	185	}
	186	#else /* !CONFIG_X86_PAE */
	187
	188	/* No need to prepopulate any pagetable entries in non-PAE modes. */
	189	#define PREALLOCATED_PMDS 0
	190
	191	#endif /* CONFIG_X86_PAE */
	192
dc6c9a35	193	static void free_pmds(struct mm_struct mm, pmd_t pmds[])
d8d5900e JF	194	{
	195	int i;
	196
	197	for(i = 0; i < PREALLOCATED_PMDS; i++)
09ef4939 KS	198	if (pmds[i]) {
09ef4939 KS	199	pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
d8d5900e	200	free_page((unsigned long)pmds[i]);
dc6c9a35	201	mm_dec_nr_pmds(mm);
09ef4939	202	}
d8d5900e JF	203	}
d8d5900e JF	204
dc6c9a35	205	static int preallocate_pmds(struct mm_struct mm, pmd_t pmds[])
d8d5900e JF	206	{
	207	int i;
	208	bool failed = false;
	209
	210	for(i = 0; i < PREALLOCATED_PMDS; i++) {
9e730237	211	pmd_t pmd = (pmd_t )__get_free_page(PGALLOC_GFP);
09ef4939	212	if (!pmd)
d8d5900e	213	failed = true;
09ef4939	214	if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
2a46eed5	215	free_page((unsigned long)pmd);
09ef4939 KS	216	pmd = NULL;
	217	failed = true;
	218	}
dc6c9a35 KS	219	if (pmd)
dc6c9a35 KS	220	mm_inc_nr_pmds(mm);
d8d5900e JF	221	pmds[i] = pmd;
	222	}
	223
	224	if (failed) {
dc6c9a35	225	free_pmds(mm, pmds);
d8d5900e JF	226	return -ENOMEM;
	227	}
	228
	229	return 0;
	230	}
	231
4f76cd38 JF	232	/*
	233	* Mop up any pmd pages which may still be attached to the pgd.
	234	* Normally they will be freed by munmap/exit_mmap, but any pmd we
	235	* preallocate which never got a corresponding vma will need to be
	236	* freed manually.
	237	*/
	238	static void pgd_mop_up_pmds(struct mm_struct mm, pgd_t pgdp)
	239	{
	240	int i;
	241
d8d5900e	242	for(i = 0; i < PREALLOCATED_PMDS; i++) {
4f76cd38 JF	243	pgd_t pgd = pgdp[i];
	244
	245	if (pgd_val(pgd) != 0) {
	246	pmd_t pmd = (pmd_t )pgd_page_vaddr(pgd);
	247
	248	pgdp[i] = native_make_pgd(0);
	249
6944a9c8	250	paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
4f76cd38	251	pmd_free(mm, pmd);
dc6c9a35	252	mm_dec_nr_pmds(mm);
4f76cd38 JF	253	}
	254	}
	255	}
	256
d8d5900e	257	static void pgd_prepopulate_pmd(struct mm_struct mm, pgd_t pgd, pmd_t *pmds[])
4f76cd38 JF	258	{
4f76cd38 JF	259	pud_t *pud;
4f76cd38 JF	260	int i;
4f76cd38 JF	261
cf3e5050 JF	262	if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
	263	return;
	264
4f76cd38	265	pud = pud_offset(pgd, 0);
4f76cd38	266
73b44ff4	267	for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
d8d5900e	268	pmd_t *pmd = pmds[i];
4f76cd38	269
68db065c	270	if (i >= KERNEL_PGD_BOUNDARY)
4f76cd38 JF	271	memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
	272	sizeof(pmd_t) * PTRS_PER_PMD);
	273
	274	pud_populate(mm, pud, pmd);
	275	}
4f76cd38	276	}
1ec1fe73	277
d8d5900e	278	pgd_t pgd_alloc(struct mm_struct mm)
1ec1fe73	279	{
d8d5900e JF	280	pgd_t *pgd;
d8d5900e JF	281	pmd_t *pmds[PREALLOCATED_PMDS];
1ec1fe73	282
9e730237	283	pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);
d8d5900e JF	284
	285	if (pgd == NULL)
	286	goto out;
	287
	288	mm->pgd = pgd;
	289
dc6c9a35	290	if (preallocate_pmds(mm, pmds) != 0)
d8d5900e JF	291	goto out_free_pgd;
	292
	293	if (paravirt_pgd_alloc(mm) != 0)
	294	goto out_free_pmds;
1ec1fe73 IM	295
1ec1fe73 IM	296	/*
d8d5900e JF	297	* Make sure that pre-populating the pmds is atomic with
	298	* respect to anything walking the pgd_list, so that they
	299	* never see a partially populated pgd.
1ec1fe73	300	*/
a79e53d8	301	spin_lock(&pgd_lock);
4f76cd38	302
617d34d9	303	pgd_ctor(mm, pgd);
d8d5900e	304	pgd_prepopulate_pmd(mm, pgd, pmds);
4f76cd38	305
a79e53d8	306	spin_unlock(&pgd_lock);
4f76cd38 JF	307
4f76cd38 JF	308	return pgd;
d8d5900e JF	309
d8d5900e JF	310	out_free_pmds:
dc6c9a35	311	free_pmds(mm, pmds);
d8d5900e JF	312	out_free_pgd:
	313	free_page((unsigned long)pgd);
	314	out:
	315	return NULL;
4f76cd38 JF	316	}
	317
	318	void pgd_free(struct mm_struct mm, pgd_t pgd)
	319	{
	320	pgd_mop_up_pmds(mm, pgd);
	321	pgd_dtor(pgd);
eba0045f	322	paravirt_pgd_free(mm, pgd);
4f76cd38 JF	323	free_page((unsigned long)pgd);
4f76cd38 JF	324	}
ee5aa8d3	325
0f9a921c RR	326	/*
	327	* Used to set accessed or dirty bits in the page table entries
	328	* on other architectures. On x86, the accessed and dirty bits
	329	* are tracked by hardware. However, do_wp_page calls this function
	330	* to also make the pte writeable at the same time the dirty bit is
	331	* set. In that case we do actually need to write the PTE.
	332	*/
ee5aa8d3 JF	333	int ptep_set_access_flags(struct vm_area_struct *vma,
	334	unsigned long address, pte_t *ptep,
	335	pte_t entry, int dirty)
	336	{
	337	int changed = !pte_same(*ptep, entry);
	338
	339	if (changed && dirty) {
	340	*ptep = entry;
	341	pte_update_defer(vma->vm_mm, address, ptep);
ee5aa8d3 JF	342	}
	343
	344	return changed;
	345	}
f9fbf1a3	346
db3eb96f AA	347	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	348	int pmdp_set_access_flags(struct vm_area_struct *vma,
	349	unsigned long address, pmd_t *pmdp,
	350	pmd_t entry, int dirty)
	351	{
	352	int changed = !pmd_same(*pmdp, entry);
	353
	354	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	355
	356	if (changed && dirty) {
	357	*pmdp = entry;
	358	pmd_update_defer(vma->vm_mm, address, pmdp);
5e4bf1a5 IM	359	/*
	360	* We had a write-protection fault here and changed the pmd
	361	* to to more permissive. No need to flush the TLB for that,
	362	* #PF is architecturally guaranteed to do that and in the
	363	* worst-case we'll generate a spurious fault.
	364	*/
db3eb96f AA	365	}
	366
	367	return changed;
	368	}
	369	#endif
	370
f9fbf1a3 JF	371	int ptep_test_and_clear_young(struct vm_area_struct *vma,
	372	unsigned long addr, pte_t *ptep)
	373	{
	374	int ret = 0;
	375
	376	if (pte_young(*ptep))
	377	ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
48e23957	378	(unsigned long *) &ptep->pte);
f9fbf1a3 JF	379
	380	if (ret)
	381	pte_update(vma->vm_mm, addr, ptep);
	382
	383	return ret;
	384	}
c20311e1	385
db3eb96f AA	386	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	387	int pmdp_test_and_clear_young(struct vm_area_struct *vma,
	388	unsigned long addr, pmd_t *pmdp)
	389	{
	390	int ret = 0;
	391
	392	if (pmd_young(*pmdp))
	393	ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
f2d6bfe9	394	(unsigned long *)pmdp);
db3eb96f AA	395
	396	if (ret)
	397	pmd_update(vma->vm_mm, addr, pmdp);
	398
	399	return ret;
	400	}
	401	#endif
	402
c20311e1 JF	403	int ptep_clear_flush_young(struct vm_area_struct *vma,
	404	unsigned long address, pte_t *ptep)
	405	{
b13b1d2d SL	406	/*
	407	* On x86 CPUs, clearing the accessed bit without a TLB flush
	408	* doesn't cause data corruption. [ It could cause incorrect
	409	* page aging and the (mistaken) reclaim of hot pages, but the
	410	* chance of that should be relatively low. ]
	411	*
	412	* So as a performance optimization don't flush the TLB when
	413	* clearing the accessed bit, it will eventually be flushed by
	414	* a context switch or a VM operation anyway. [ In the rare
	415	* event of it not getting flushed for a long time the delay
	416	* shouldn't really matter because there's no real memory
	417	* pressure for swapout to react to. ]
	418	*/
	419	return ptep_test_and_clear_young(vma, address, ptep);
c20311e1	420	}
7c7e6e07	421
db3eb96f AA	422	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	423	int pmdp_clear_flush_young(struct vm_area_struct *vma,
	424	unsigned long address, pmd_t *pmdp)
	425	{
	426	int young;
	427
	428	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	429
	430	young = pmdp_test_and_clear_young(vma, address, pmdp);
	431	if (young)
	432	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	433
	434	return young;
	435	}
	436
	437	void pmdp_splitting_flush(struct vm_area_struct *vma,
	438	unsigned long address, pmd_t *pmdp)
	439	{
	440	int set;
	441	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	442	set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
f2d6bfe9	443	(unsigned long *)pmdp);
db3eb96f AA	444	if (set) {
	445	pmd_update(vma->vm_mm, address, pmdp);
	446	/* need tlb flush only to serialize against gup-fast */
	447	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	448	}
	449	}
	450	#endif
	451
fd862dde GP	452	/**
	453	* reserve_top_address - reserves a hole in the top of kernel address space
	454	* @reserve - size of hole to reserve
	455	*
	456	* Can be used to relocate the fixmap area and poke a hole in the top
	457	* of kernel address space to make room for a hypervisor.
	458	*/
	459	void __init reserve_top_address(unsigned long reserve)
	460	{
	461	#ifdef CONFIG_X86_32
	462	BUG_ON(fixmaps_set > 0);
73159fdc AL	463	__FIXADDR_TOP = round_down(-reserve, 1 << PMD_SHIFT) - PAGE_SIZE;
	464	printk(KERN_INFO "Reserving virtual address space above 0x%08lx (rounded to 0x%08lx)\n",
	465	-reserve, __FIXADDR_TOP + PAGE_SIZE);
fd862dde GP	466	#endif
	467	}
	468
7c7e6e07 JF	469	int fixmaps_set;
7c7e6e07 JF	470
aeaaa59c	471	void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
7c7e6e07 JF	472	{
	473	unsigned long address = __fix_to_virt(idx);
	474
	475	if (idx >= __end_of_fixed_addresses) {
	476	BUG();
	477	return;
	478	}
aeaaa59c	479	set_pte_vaddr(address, pte);
7c7e6e07 JF	480	fixmaps_set++;
7c7e6e07 JF	481	}
aeaaa59c	482
3b3809ac MH	483	void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
3b3809ac MH	484	pgprot_t flags)
aeaaa59c JF	485	{
	486	__native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
	487	}