[deliverable/linux.git] / arch / x86 / mm / pageattr.c

/*
 * Copyright 2002 Andi Kleen, SuSE Labs.
 * Thanks to Ben LaHaise for precious feedback.
 */
#include <linux/highmem.h>
#include <linux/bootmem.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/mm.h>

#include <asm/e820.h>
#include <asm/processor.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
#include <asm/uaccess.h>
#include <asm/pgalloc.h>

static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
	return addr >= start && addr < end;
}

/*
 * Flushing functions
 */
void clflush_cache_range(void *addr, int size)
{
	int i;

	for (i = 0; i < size; i += boot_cpu_data.x86_clflush_size)
		clflush(addr+i);
}

static void flush_kernel_map(void *arg)
{
	/*
	 * Flush all to work around Errata in early athlons regarding
	 * large page flushing.
	 */
	__flush_tlb_all();

	if (boot_cpu_data.x86_model >= 4)
		wbinvd();
}

static void global_flush_tlb(void)
{
	BUG_ON(irqs_disabled());

	on_each_cpu(flush_kernel_map, NULL, 1, 1);
}

/*
 * Certain areas of memory on x86 require very specific protection flags,
 * for example the BIOS area or kernel text. Callers don't always get this
 * right (again, ioremap() on BIOS memory is not uncommon) so this function
 * checks and fixes these known static required protection bits.
 */
static inline pgprot_t static_protections(pgprot_t prot, unsigned long address)
{
	pgprot_t forbidden = __pgprot(0);

	/*
	 * The BIOS area between 640k and 1Mb needs to be executable for
	 * PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
	 */
	if (within(__pa(address), BIOS_BEGIN, BIOS_END))
		pgprot_val(forbidden) |= _PAGE_NX;

	/*
	 * The kernel text needs to be executable for obvious reasons
	 * Does not cover __inittext since that is gone later on
	 */
	if (within(address, (unsigned long)_text, (unsigned long)_etext))
		pgprot_val(forbidden) |= _PAGE_NX;

#ifdef CONFIG_DEBUG_RODATA
	/* The .rodata section needs to be read-only */
	if (within(address, (unsigned long)__start_rodata,
				(unsigned long)__end_rodata))
		pgprot_val(forbidden) |= _PAGE_RW;
#endif

	prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));

	return prot;
}

pte_t *lookup_address(unsigned long address, int *level)
{
	pgd_t *pgd = pgd_offset_k(address);
	pud_t *pud;
	pmd_t *pmd;

	*level = PG_LEVEL_NONE;

	if (pgd_none(*pgd))
		return NULL;
	pud = pud_offset(pgd, address);
	if (pud_none(*pud))
		return NULL;
	pmd = pmd_offset(pud, address);
	if (pmd_none(*pmd))
		return NULL;

	*level = PG_LEVEL_2M;
	if (pmd_large(*pmd))
		return (pte_t *)pmd;

	*level = PG_LEVEL_4K;
	return pte_offset_kernel(pmd, address);
}

static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
{
	/* change init_mm */
	set_pte_atomic(kpte, pte);
#ifdef CONFIG_X86_32
	if (!SHARED_KERNEL_PMD) {
		struct page *page;

		for (page = pgd_list; page; page = (struct page *)page->index) {
			pgd_t *pgd;
			pud_t *pud;
			pmd_t *pmd;

			pgd = (pgd_t *)page_address(page) + pgd_index(address);
			pud = pud_offset(pgd, address);
			pmd = pmd_offset(pud, address);
			set_pte_atomic((pte_t *)pmd, pte);
		}
	}
#endif
}

static int split_large_page(pte_t *kpte, unsigned long address)
{
	pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte));
	gfp_t gfp_flags = GFP_KERNEL;
	unsigned long flags;
	unsigned long addr;
	pte_t *pbase, *tmp;
	struct page *base;
	int i, level;

#ifdef CONFIG_DEBUG_PAGEALLOC
	gfp_flags = GFP_ATOMIC;
#endif
	base = alloc_pages(gfp_flags, 0);
	if (!base)
		return -ENOMEM;

	spin_lock_irqsave(&pgd_lock, flags);
	/*
	 * Check for races, another CPU might have split this page
	 * up for us already:
	 */
	tmp = lookup_address(address, &level);
	if (tmp != kpte) {
		WARN_ON_ONCE(1);
		goto out_unlock;
	}

	address = __pa(address);
	addr = address & LARGE_PAGE_MASK;
	pbase = (pte_t *)page_address(base);
#ifdef CONFIG_X86_32
	paravirt_alloc_pt(&init_mm, page_to_pfn(base));
#endif

	for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE)
		set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, ref_prot));

	/*
	 * Install the new, split up pagetable. Important detail here:
	 *
	 * On Intel the NX bit of all levels must be cleared to make a
	 * page executable. See section 4.13.2 of Intel 64 and IA-32
	 * Architectures Software Developer's Manual).
	 */
	ref_prot = pte_pgprot(pte_mkexec(pte_clrhuge(*kpte)));
	__set_pmd_pte(kpte, address, mk_pte(base, ref_prot));
	base = NULL;

out_unlock:
	spin_unlock_irqrestore(&pgd_lock, flags);

	if (base)
		__free_pages(base, 0);

	return 0;
}

static int
__change_page_attr(unsigned long address, unsigned long pfn, pgprot_t prot)
{
	struct page *kpte_page;
	int level, err = 0;
	pte_t *kpte;

#ifdef CONFIG_X86_32
	BUG_ON(pfn > max_low_pfn);
#endif

repeat:
	kpte = lookup_address(address, &level);
	if (!kpte)
		return -EINVAL;

	kpte_page = virt_to_page(kpte);
	BUG_ON(PageLRU(kpte_page));
	BUG_ON(PageCompound(kpte_page));

	prot = static_protections(prot, address);

	if (level == PG_LEVEL_4K) {
		set_pte_atomic(kpte, pfn_pte(pfn, canon_pgprot(prot)));
	} else {
		err = split_large_page(kpte, address);
		if (!err)
			goto repeat;
	}
	return err;
}

/**
 * change_page_attr_addr - Change page table attributes in linear mapping
 * @address: Virtual address in linear mapping.
 * @numpages: Number of pages to change
 * @prot:    New page table attribute (PAGE_*)
 *
 * Change page attributes of a page in the direct mapping. This is a variant
 * of change_page_attr() that also works on memory holes that do not have
 * mem_map entry (pfn_valid() is false).
 *
 * See change_page_attr() documentation for more details.
 *
 * Modules and drivers should use the set_memory_* APIs instead.
 */

static int change_page_attr_addr(unsigned long address, int numpages,
								pgprot_t prot)
{
	int err = 0, kernel_map = 0, i;

#ifdef CONFIG_X86_64
	if (address >= __START_KERNEL_map &&
			address < __START_KERNEL_map + KERNEL_TEXT_SIZE) {

		address = (unsigned long)__va(__pa(address));
		kernel_map = 1;
	}
#endif

	for (i = 0; i < numpages; i++, address += PAGE_SIZE) {
		unsigned long pfn = __pa(address) >> PAGE_SHIFT;

		if (!kernel_map || pte_present(pfn_pte(0, prot))) {
			err = __change_page_attr(address, pfn, prot);
			if (err)
				break;
		}
#ifdef CONFIG_X86_64
		/*
		 * Handle kernel mapping too which aliases part of
		 * lowmem:
		 */
		if (__pa(address) < KERNEL_TEXT_SIZE) {
			unsigned long addr2;
			pgprot_t prot2;

			addr2 = __START_KERNEL_map + __pa(address);
			/* Make sure the kernel mappings stay executable */
			prot2 = pte_pgprot(pte_mkexec(pfn_pte(0, prot)));
			err = __change_page_attr(addr2, pfn, prot2);
		}
#endif
	}

	return err;
}

/**
 * change_page_attr_set - Change page table attributes in the linear mapping.
 * @addr: Virtual address in linear mapping.
 * @numpages: Number of pages to change
 * @prot: Protection/caching type bits to set (PAGE_*)
 *
 * Returns 0 on success, otherwise a negated errno.
 *
 * This should be used when a page is mapped with a different caching policy
 * than write-back somewhere - some CPUs do not like it when mappings with
 * different caching policies exist. This changes the page attributes of the
 * in kernel linear mapping too.
 *
 * The caller needs to ensure that there are no conflicting mappings elsewhere
 * (e.g. in user space) * This function only deals with the kernel linear map.
 *
 * This function is different from change_page_attr() in that only selected bits
 * are impacted, all other bits remain as is.
 */
static int change_page_attr_set(unsigned long addr, int numpages,
								pgprot_t prot)
{
	pgprot_t current_prot;
	int level;
	pte_t *pte;

	pte = lookup_address(addr, &level);
	if (pte)
		current_prot = pte_pgprot(*pte);
	else
		pgprot_val(current_prot) = 0;

	pgprot_val(prot) = pgprot_val(current_prot) | pgprot_val(prot);

	return change_page_attr_addr(addr, numpages, prot);
}

/**
 * change_page_attr_clear - Change page table attributes in the linear mapping.
 * @addr: Virtual address in linear mapping.
 * @numpages: Number of pages to change
 * @prot: Protection/caching type bits to clear (PAGE_*)
 *
 * Returns 0 on success, otherwise a negated errno.
 *
 * This should be used when a page is mapped with a different caching policy
 * than write-back somewhere - some CPUs do not like it when mappings with
 * different caching policies exist. This changes the page attributes of the
 * in kernel linear mapping too.
 *
 * The caller needs to ensure that there are no conflicting mappings elsewhere
 * (e.g. in user space) * This function only deals with the kernel linear map.
 *
 * This function is different from change_page_attr() in that only selected bits
 * are impacted, all other bits remain as is.
 */
static int change_page_attr_clear(unsigned long addr, int numpages,
								pgprot_t prot)
{
	pgprot_t current_prot;
	int level;
	pte_t *pte;

	pte = lookup_address(addr, &level);
	if (pte)
		current_prot = pte_pgprot(*pte);
	else
		pgprot_val(current_prot) = 0;

	pgprot_val(prot) = pgprot_val(current_prot) & ~pgprot_val(prot);

	return change_page_attr_addr(addr, numpages, prot);
}

int set_memory_uc(unsigned long addr, int numpages)
{
	int err;

	err = change_page_attr_set(addr, numpages,
				__pgprot(_PAGE_PCD | _PAGE_PWT));
	global_flush_tlb();
	return err;
}
EXPORT_SYMBOL(set_memory_uc);

int set_memory_wb(unsigned long addr, int numpages)
{
	int err;

	err = change_page_attr_clear(addr, numpages,
				__pgprot(_PAGE_PCD | _PAGE_PWT));
	global_flush_tlb();
	return err;
}
EXPORT_SYMBOL(set_memory_wb);

int set_memory_x(unsigned long addr, int numpages)
{
	int err;

	err = change_page_attr_clear(addr, numpages,
				__pgprot(_PAGE_NX));
	global_flush_tlb();
	return err;
}
EXPORT_SYMBOL(set_memory_x);

int set_memory_nx(unsigned long addr, int numpages)
{
	int err;

	err = change_page_attr_set(addr, numpages,
				__pgprot(_PAGE_NX));
	global_flush_tlb();
	return err;
}
EXPORT_SYMBOL(set_memory_nx);

int set_memory_ro(unsigned long addr, int numpages)
{
	int err;

	err = change_page_attr_clear(addr, numpages,
				__pgprot(_PAGE_RW));
	global_flush_tlb();
	return err;
}

int set_memory_rw(unsigned long addr, int numpages)
{
	int err;

	err = change_page_attr_set(addr, numpages,
				__pgprot(_PAGE_RW));
	global_flush_tlb();
	return err;
}

int set_memory_np(unsigned long addr, int numpages)
{
	int err;

	err = change_page_attr_clear(addr, numpages,
				__pgprot(_PAGE_PRESENT));
	global_flush_tlb();
	return err;
}

int set_pages_uc(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

	return set_memory_uc(addr, numpages);
}
EXPORT_SYMBOL(set_pages_uc);

int set_pages_wb(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

	return set_memory_wb(addr, numpages);
}
EXPORT_SYMBOL(set_pages_wb);

int set_pages_x(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

	return set_memory_x(addr, numpages);
}
EXPORT_SYMBOL(set_pages_x);

int set_pages_nx(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

	return set_memory_nx(addr, numpages);
}
EXPORT_SYMBOL(set_pages_nx);

int set_pages_ro(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

	return set_memory_ro(addr, numpages);
}

int set_pages_rw(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);

	return set_memory_rw(addr, numpages);
}


#ifdef CONFIG_DEBUG_PAGEALLOC

static int __set_pages_p(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);
	return change_page_attr_set(addr, numpages,
				__pgprot(_PAGE_PRESENT | _PAGE_RW));
}

static int __set_pages_np(struct page *page, int numpages)
{
	unsigned long addr = (unsigned long)page_address(page);
	return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_PRESENT));
}

void kernel_map_pages(struct page *page, int numpages, int enable)
{
	if (PageHighMem(page))
		return;
	if (!enable) {
		debug_check_no_locks_freed(page_address(page),
					   numpages * PAGE_SIZE);
	}

	/*
	 * If page allocator is not up yet then do not call c_p_a():
	 */
	if (!debug_pagealloc_enabled)
		return;

	/*
	 * The return value is ignored - the calls cannot fail,
	 * large pages are disabled at boot time:
	 */
	if (enable)
		__set_pages_p(page, numpages);
	else
		__set_pages_np(page, numpages);

	/*
	 * We should perform an IPI and flush all tlbs,
	 * but that can deadlock->flush only current cpu:
	 */
	__flush_tlb_all();
}
#endif

/*
 * The testcases use internal knowledge of the implementation that shouldn't
 * be exposed to the rest of the kernel. Include these directly here.
 */
#ifdef CONFIG_CPA_DEBUG
#include "pageattr-test.c"
#endif
Commit	Line	Data
9f4c815c IM	1	/*
9f4c815c IM	2	* Copyright 2002 Andi Kleen, SuSE Labs.
1da177e4	3	* Thanks to Ben LaHaise for precious feedback.
9f4c815c	4	*/
1da177e4	5	#include <linux/highmem.h>
8192206d	6	#include <linux/bootmem.h>
1da177e4	7	#include <linux/module.h>
9f4c815c	8	#include <linux/sched.h>
1da177e4	9	#include <linux/slab.h>
9f4c815c IM	10	#include <linux/mm.h>
9f4c815c IM	11
950f9d95	12	#include <asm/e820.h>
1da177e4 LT	13	#include <asm/processor.h>
1da177e4 LT	14	#include <asm/tlbflush.h>
f8af095d	15	#include <asm/sections.h>
9f4c815c IM	16	#include <asm/uaccess.h>
9f4c815c IM	17	#include <asm/pgalloc.h>
1da177e4	18
ed724be6 AV	19	static inline int
ed724be6 AV	20	within(unsigned long addr, unsigned long start, unsigned long end)
687c4825	21	{
ed724be6 AV	22	return addr >= start && addr < end;
	23	}
	24
d7c8f21a TG	25	/*
	26	* Flushing functions
	27	*/
	28	void clflush_cache_range(void *addr, int size)
	29	{
	30	int i;
	31
	32	for (i = 0; i < size; i += boot_cpu_data.x86_clflush_size)
	33	clflush(addr+i);
	34	}
	35
	36	static void flush_kernel_map(void *arg)
	37	{
	38	/*
	39	* Flush all to work around Errata in early athlons regarding
	40	* large page flushing.
	41	*/
	42	__flush_tlb_all();
	43
	44	if (boot_cpu_data.x86_model >= 4)
	45	wbinvd();
	46	}
	47
	48	static void global_flush_tlb(void)
	49	{
	50	BUG_ON(irqs_disabled());
	51
	52	on_each_cpu(flush_kernel_map, NULL, 1, 1);
	53	}
	54
ed724be6 AV	55	/*
	56	* Certain areas of memory on x86 require very specific protection flags,
	57	* for example the BIOS area or kernel text. Callers don't always get this
	58	* right (again, ioremap() on BIOS memory is not uncommon) so this function
	59	* checks and fixes these known static required protection bits.
	60	*/
	61	static inline pgprot_t static_protections(pgprot_t prot, unsigned long address)
	62	{
	63	pgprot_t forbidden = __pgprot(0);
	64
687c4825	65	/*
ed724be6 AV	66	* The BIOS area between 640k and 1Mb needs to be executable for
ed724be6 AV	67	* PCI BIOS based config access (CONFIG_PCI_GOBIOS) support.
687c4825	68	*/
ed724be6 AV	69	if (within(__pa(address), BIOS_BEGIN, BIOS_END))
	70	pgprot_val(forbidden) \|= _PAGE_NX;
	71
	72	/*
	73	* The kernel text needs to be executable for obvious reasons
	74	* Does not cover __inittext since that is gone later on
	75	*/
	76	if (within(address, (unsigned long)_text, (unsigned long)_etext))
	77	pgprot_val(forbidden) \|= _PAGE_NX;
	78
	79	#ifdef CONFIG_DEBUG_RODATA
	80	/* The .rodata section needs to be read-only */
	81	if (within(address, (unsigned long)__start_rodata,
	82	(unsigned long)__end_rodata))
	83	pgprot_val(forbidden) \|= _PAGE_RW;
	84	#endif
	85
	86	prot = __pgprot(pgprot_val(prot) & ~pgprot_val(forbidden));
687c4825 IM	87
	88	return prot;
	89	}
	90
f0646e43	91	pte_t lookup_address(unsigned long address, int level)
9f4c815c	92	{
1da177e4 LT	93	pgd_t *pgd = pgd_offset_k(address);
	94	pud_t *pud;
	95	pmd_t *pmd;
9f4c815c	96
30551bb3 TG	97	*level = PG_LEVEL_NONE;
30551bb3 TG	98
1da177e4 LT	99	if (pgd_none(*pgd))
	100	return NULL;
	101	pud = pud_offset(pgd, address);
	102	if (pud_none(*pud))
	103	return NULL;
	104	pmd = pmd_offset(pud, address);
	105	if (pmd_none(*pmd))
	106	return NULL;
30551bb3 TG	107
30551bb3 TG	108	*level = PG_LEVEL_2M;
1da177e4 LT	109	if (pmd_large(*pmd))
1da177e4 LT	110	return (pte_t *)pmd;
1da177e4	111
30551bb3	112	*level = PG_LEVEL_4K;
9f4c815c IM	113	return pte_offset_kernel(pmd, address);
	114	}
	115
9a3dc780	116	static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
9f4c815c	117	{
9f4c815c IM	118	/* change init_mm */
9f4c815c IM	119	set_pte_atomic(kpte, pte);
44af6c41	120	#ifdef CONFIG_X86_32
e4b71dcf	121	if (!SHARED_KERNEL_PMD) {
44af6c41 IM	122	struct page *page;
	123
	124	for (page = pgd_list; page; page = (struct page *)page->index) {
	125	pgd_t *pgd;
	126	pud_t *pud;
	127	pmd_t *pmd;
	128
	129	pgd = (pgd_t *)page_address(page) + pgd_index(address);
	130	pud = pud_offset(pgd, address);
	131	pmd = pmd_offset(pud, address);
	132	set_pte_atomic((pte_t *)pmd, pte);
	133	}
1da177e4	134	}
44af6c41	135	#endif
1da177e4 LT	136	}
1da177e4 LT	137
7afe15b9	138	static int split_large_page(pte_t *kpte, unsigned long address)
bb5c2dbd	139	{
7afe15b9	140	pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte));
12d6f21e	141	gfp_t gfp_flags = GFP_KERNEL;
9a3dc780	142	unsigned long flags;
bb5c2dbd IM	143	unsigned long addr;
	144	pte_t pbase, tmp;
	145	struct page *base;
7afe15b9	146	int i, level;
bb5c2dbd	147
12d6f21e IM	148	#ifdef CONFIG_DEBUG_PAGEALLOC
	149	gfp_flags = GFP_ATOMIC;
	150	#endif
	151	base = alloc_pages(gfp_flags, 0);
bb5c2dbd IM	152	if (!base)
	153	return -ENOMEM;
	154
9a3dc780	155	spin_lock_irqsave(&pgd_lock, flags);
bb5c2dbd IM	156	/*
	157	* Check for races, another CPU might have split this page
	158	* up for us already:
	159	*/
	160	tmp = lookup_address(address, &level);
5508a748 IM	161	if (tmp != kpte) {
5508a748 IM	162	WARN_ON_ONCE(1);
bb5c2dbd	163	goto out_unlock;
5508a748	164	}
bb5c2dbd IM	165
	166	address = __pa(address);
	167	addr = address & LARGE_PAGE_MASK;
	168	pbase = (pte_t *)page_address(base);
44af6c41	169	#ifdef CONFIG_X86_32
bb5c2dbd	170	paravirt_alloc_pt(&init_mm, page_to_pfn(base));
44af6c41	171	#endif
bb5c2dbd IM	172
	173	for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE)
	174	set_pte(&pbase[i], pfn_pte(addr >> PAGE_SHIFT, ref_prot));
	175
	176	/*
4c881ca1 HY	177	* Install the new, split up pagetable. Important detail here:
	178	*
	179	* On Intel the NX bit of all levels must be cleared to make a
	180	* page executable. See section 4.13.2 of Intel 64 and IA-32
	181	* Architectures Software Developer's Manual).
bb5c2dbd	182	*/
4c881ca1	183	ref_prot = pte_pgprot(pte_mkexec(pte_clrhuge(*kpte)));
9a3dc780	184	__set_pmd_pte(kpte, address, mk_pte(base, ref_prot));
bb5c2dbd IM	185	base = NULL;
	186
	187	out_unlock:
9a3dc780	188	spin_unlock_irqrestore(&pgd_lock, flags);
bb5c2dbd IM	189
	190	if (base)
	191	__free_pages(base, 0);
	192
	193	return 0;
	194	}
	195
44af6c41	196	static int
8192206d	197	__change_page_attr(unsigned long address, unsigned long pfn, pgprot_t prot)
9f4c815c	198	{
1da177e4	199	struct page *kpte_page;
bb5c2dbd	200	int level, err = 0;
9f4c815c	201	pte_t *kpte;
1da177e4	202
8192206d IM	203	#ifdef CONFIG_X86_32
	204	BUG_ON(pfn > max_low_pfn);
	205	#endif
1da177e4	206
97f99fed	207	repeat:
f0646e43	208	kpte = lookup_address(address, &level);
1da177e4 LT	209	if (!kpte)
1da177e4 LT	210	return -EINVAL;
9f4c815c	211
1da177e4	212	kpte_page = virt_to_page(kpte);
65d2f0bc AK	213	BUG_ON(PageLRU(kpte_page));
	214	BUG_ON(PageCompound(kpte_page));
	215
ed724be6	216	prot = static_protections(prot, address);
65d2f0bc	217
30551bb3	218	if (level == PG_LEVEL_4K) {
8192206d	219	set_pte_atomic(kpte, pfn_pte(pfn, canon_pgprot(prot)));
78c94aba	220	} else {
7afe15b9	221	err = split_large_page(kpte, address);
bb5c2dbd IM	222	if (!err)
bb5c2dbd IM	223	goto repeat;
1da177e4	224	}
bb5c2dbd	225	return err;
9f4c815c	226	}
1da177e4	227
44af6c41 IM	228	/**
	229	* change_page_attr_addr - Change page table attributes in linear mapping
	230	* @address: Virtual address in linear mapping.
	231	* @numpages: Number of pages to change
	232	* @prot: New page table attribute (PAGE_*)
1da177e4	233	*
44af6c41 IM	234	* Change page attributes of a page in the direct mapping. This is a variant
	235	* of change_page_attr() that also works on memory holes that do not have
	236	* mem_map entry (pfn_valid() is false).
9f4c815c	237	*
44af6c41	238	* See change_page_attr() documentation for more details.
75cbade8 AV	239	*
75cbade8 AV	240	* Modules and drivers should use the set_memory_* APIs instead.
1da177e4	241	*/
44af6c41	242
d1028a15 AV	243	static int change_page_attr_addr(unsigned long address, int numpages,
d1028a15 AV	244	pgprot_t prot)
1da177e4	245	{
44af6c41 IM	246	int err = 0, kernel_map = 0, i;
	247
	248	#ifdef CONFIG_X86_64
	249	if (address >= __START_KERNEL_map &&
	250	address < __START_KERNEL_map + KERNEL_TEXT_SIZE) {
1da177e4	251
44af6c41 IM	252	address = (unsigned long)__va(__pa(address));
	253	kernel_map = 1;
	254	}
	255	#endif
	256
	257	for (i = 0; i < numpages; i++, address += PAGE_SIZE) {
	258	unsigned long pfn = __pa(address) >> PAGE_SHIFT;
	259
	260	if (!kernel_map \|\| pte_present(pfn_pte(0, prot))) {
8192206d	261	err = __change_page_attr(address, pfn, prot);
44af6c41 IM	262	if (err)
	263	break;
	264	}
	265	#ifdef CONFIG_X86_64
	266	/*
	267	* Handle kernel mapping too which aliases part of
	268	* lowmem:
	269	*/
	270	if (__pa(address) < KERNEL_TEXT_SIZE) {
	271	unsigned long addr2;
	272	pgprot_t prot2;
	273
	274	addr2 = __START_KERNEL_map + __pa(address);
	275	/* Make sure the kernel mappings stay executable */
	276	prot2 = pte_pgprot(pte_mkexec(pfn_pte(0, prot)));
8192206d	277	err = __change_page_attr(addr2, pfn, prot2);
44af6c41 IM	278	}
44af6c41 IM	279	#endif
9f4c815c	280	}
9f4c815c	281
1da177e4 LT	282	return err;
	283	}
	284
75cbade8 AV	285	/**
	286	* change_page_attr_set - Change page table attributes in the linear mapping.
	287	* @addr: Virtual address in linear mapping.
	288	* @numpages: Number of pages to change
	289	* @prot: Protection/caching type bits to set (PAGE_*)
	290	*
	291	* Returns 0 on success, otherwise a negated errno.
	292	*
	293	* This should be used when a page is mapped with a different caching policy
	294	* than write-back somewhere - some CPUs do not like it when mappings with
	295	* different caching policies exist. This changes the page attributes of the
	296	* in kernel linear mapping too.
	297	*
75cbade8 AV	298	* The caller needs to ensure that there are no conflicting mappings elsewhere
	299	* (e.g. in user space) * This function only deals with the kernel linear map.
	300	*
	301	* This function is different from change_page_attr() in that only selected bits
	302	* are impacted, all other bits remain as is.
	303	*/
d1028a15 AV	304	static int change_page_attr_set(unsigned long addr, int numpages,
d1028a15 AV	305	pgprot_t prot)
75cbade8 AV	306	{
	307	pgprot_t current_prot;
	308	int level;
	309	pte_t *pte;
	310
	311	pte = lookup_address(addr, &level);
	312	if (pte)
	313	current_prot = pte_pgprot(*pte);
	314	else
	315	pgprot_val(current_prot) = 0;
	316
	317	pgprot_val(prot) = pgprot_val(current_prot) \| pgprot_val(prot);
	318
	319	return change_page_attr_addr(addr, numpages, prot);
	320	}
	321
	322	/**
	323	* change_page_attr_clear - Change page table attributes in the linear mapping.
	324	* @addr: Virtual address in linear mapping.
	325	* @numpages: Number of pages to change
	326	* @prot: Protection/caching type bits to clear (PAGE_*)
	327	*
	328	* Returns 0 on success, otherwise a negated errno.
	329	*
	330	* This should be used when a page is mapped with a different caching policy
	331	* than write-back somewhere - some CPUs do not like it when mappings with
	332	* different caching policies exist. This changes the page attributes of the
	333	* in kernel linear mapping too.
	334	*
75cbade8 AV	335	* The caller needs to ensure that there are no conflicting mappings elsewhere
	336	* (e.g. in user space) * This function only deals with the kernel linear map.
	337	*
	338	* This function is different from change_page_attr() in that only selected bits
	339	* are impacted, all other bits remain as is.
	340	*/
d1028a15 AV	341	static int change_page_attr_clear(unsigned long addr, int numpages,
d1028a15 AV	342	pgprot_t prot)
75cbade8 AV	343	{
	344	pgprot_t current_prot;
	345	int level;
	346	pte_t *pte;
	347
	348	pte = lookup_address(addr, &level);
	349	if (pte)
	350	current_prot = pte_pgprot(*pte);
	351	else
	352	pgprot_val(current_prot) = 0;
	353
	354	pgprot_val(prot) = pgprot_val(current_prot) & ~pgprot_val(prot);
	355
	356	return change_page_attr_addr(addr, numpages, prot);
	357	}
	358
75cbade8 AV	359	int set_memory_uc(unsigned long addr, int numpages)
75cbade8 AV	360	{
d7c8f21a	361	int err;
75cbade8	362
d7c8f21a TG	363	err = change_page_attr_set(addr, numpages,
	364	__pgprot(_PAGE_PCD \| _PAGE_PWT));
	365	global_flush_tlb();
	366	return err;
75cbade8 AV	367	}
	368	EXPORT_SYMBOL(set_memory_uc);
	369
	370	int set_memory_wb(unsigned long addr, int numpages)
	371	{
d7c8f21a	372	int err;
75cbade8	373
d7c8f21a TG	374	err = change_page_attr_clear(addr, numpages,
	375	__pgprot(_PAGE_PCD \| _PAGE_PWT));
	376	global_flush_tlb();
	377	return err;
75cbade8 AV	378	}
	379	EXPORT_SYMBOL(set_memory_wb);
	380
	381	int set_memory_x(unsigned long addr, int numpages)
	382	{
d7c8f21a	383	int err;
75cbade8	384
d7c8f21a TG	385	err = change_page_attr_clear(addr, numpages,
	386	__pgprot(_PAGE_NX));
	387	global_flush_tlb();
	388	return err;
75cbade8 AV	389	}
	390	EXPORT_SYMBOL(set_memory_x);
	391
	392	int set_memory_nx(unsigned long addr, int numpages)
	393	{
d7c8f21a	394	int err;
75cbade8	395
d7c8f21a TG	396	err = change_page_attr_set(addr, numpages,
	397	__pgprot(_PAGE_NX));
	398	global_flush_tlb();
	399	return err;
75cbade8 AV	400	}
	401	EXPORT_SYMBOL(set_memory_nx);
	402
	403	int set_memory_ro(unsigned long addr, int numpages)
	404	{
d7c8f21a	405	int err;
75cbade8	406
d7c8f21a TG	407	err = change_page_attr_clear(addr, numpages,
	408	__pgprot(_PAGE_RW));
	409	global_flush_tlb();
	410	return err;
75cbade8	411	}
75cbade8 AV	412
	413	int set_memory_rw(unsigned long addr, int numpages)
	414	{
d7c8f21a	415	int err;
75cbade8	416
d7c8f21a TG	417	err = change_page_attr_set(addr, numpages,
	418	__pgprot(_PAGE_RW));
	419	global_flush_tlb();
	420	return err;
75cbade8	421	}
f62d0f00 IM	422
	423	int set_memory_np(unsigned long addr, int numpages)
	424	{
d7c8f21a	425	int err;
f62d0f00	426
d7c8f21a TG	427	err = change_page_attr_clear(addr, numpages,
	428	__pgprot(_PAGE_PRESENT));
	429	global_flush_tlb();
	430	return err;
f62d0f00	431	}
75cbade8 AV	432
	433	int set_pages_uc(struct page *page, int numpages)
	434	{
	435	unsigned long addr = (unsigned long)page_address(page);
75cbade8	436
d7c8f21a	437	return set_memory_uc(addr, numpages);
75cbade8 AV	438	}
	439	EXPORT_SYMBOL(set_pages_uc);
	440
	441	int set_pages_wb(struct page *page, int numpages)
	442	{
	443	unsigned long addr = (unsigned long)page_address(page);
75cbade8	444
d7c8f21a	445	return set_memory_wb(addr, numpages);
75cbade8 AV	446	}
	447	EXPORT_SYMBOL(set_pages_wb);
	448
	449	int set_pages_x(struct page *page, int numpages)
	450	{
	451	unsigned long addr = (unsigned long)page_address(page);
75cbade8	452
d7c8f21a	453	return set_memory_x(addr, numpages);
75cbade8 AV	454	}
	455	EXPORT_SYMBOL(set_pages_x);
	456
	457	int set_pages_nx(struct page *page, int numpages)
	458	{
	459	unsigned long addr = (unsigned long)page_address(page);
75cbade8	460
d7c8f21a	461	return set_memory_nx(addr, numpages);
75cbade8 AV	462	}
	463	EXPORT_SYMBOL(set_pages_nx);
	464
	465	int set_pages_ro(struct page *page, int numpages)
	466	{
	467	unsigned long addr = (unsigned long)page_address(page);
75cbade8	468
d7c8f21a	469	return set_memory_ro(addr, numpages);
75cbade8	470	}
75cbade8 AV	471
	472	int set_pages_rw(struct page *page, int numpages)
	473	{
	474	unsigned long addr = (unsigned long)page_address(page);
e81d5dc4	475
d7c8f21a	476	return set_memory_rw(addr, numpages);
78c94aba IM	477	}
78c94aba IM	478
1da177e4 LT	479
1da177e4 LT	480	#ifdef CONFIG_DEBUG_PAGEALLOC
f62d0f00 IM	481
	482	static int __set_pages_p(struct page *page, int numpages)
	483	{
	484	unsigned long addr = (unsigned long)page_address(page);
	485	return change_page_attr_set(addr, numpages,
	486	__pgprot(_PAGE_PRESENT \| _PAGE_RW));
	487	}
	488
	489	static int __set_pages_np(struct page *page, int numpages)
	490	{
	491	unsigned long addr = (unsigned long)page_address(page);
	492	return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_PRESENT));
	493	}
	494
1da177e4 LT	495	void kernel_map_pages(struct page *page, int numpages, int enable)
	496	{
	497	if (PageHighMem(page))
	498	return;
9f4c815c	499	if (!enable) {
f9b8404c IM	500	debug_check_no_locks_freed(page_address(page),
f9b8404c IM	501	numpages * PAGE_SIZE);
9f4c815c	502	}
de5097c2	503
12d6f21e IM	504	/*
	505	* If page allocator is not up yet then do not call c_p_a():
	506	*/
	507	if (!debug_pagealloc_enabled)
	508	return;
	509
9f4c815c	510	/*
e4b71dcf IM	511	* The return value is ignored - the calls cannot fail,
e4b71dcf IM	512	* large pages are disabled at boot time:
1da177e4	513	*/
f62d0f00 IM	514	if (enable)
	515	__set_pages_p(page, numpages);
	516	else
	517	__set_pages_np(page, numpages);
9f4c815c IM	518
9f4c815c IM	519	/*
e4b71dcf IM	520	* We should perform an IPI and flush all tlbs,
e4b71dcf IM	521	* but that can deadlock->flush only current cpu:
1da177e4 LT	522	*/
	523	__flush_tlb_all();
	524	}
	525	#endif
d1028a15 AV	526
	527	/*
	528	* The testcases use internal knowledge of the implementation that shouldn't
	529	* be exposed to the rest of the kernel. Include these directly here.
	530	*/
	531	#ifdef CONFIG_CPA_DEBUG
	532	#include "pageattr-test.c"
	533	#endif