[deliverable/linux.git] / arch / x86 / xen / mmu.c

/*
 * Xen mmu operations
 *
 * This file contains the various mmu fetch and update operations.
 * The most important job they must perform is the mapping between the
 * domain's pfn and the overall machine mfns.
 *
 * Xen allows guests to directly update the pagetable, in a controlled
 * fashion.  In other words, the guest modifies the same pagetable
 * that the CPU actually uses, which eliminates the overhead of having
 * a separate shadow pagetable.
 *
 * In order to allow this, it falls on the guest domain to map its
 * notion of a "physical" pfn - which is just a domain-local linear
 * address - into a real "machine address" which the CPU's MMU can
 * use.
 *
 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
 * inserted directly into the pagetable.  When creating a new
 * pte/pmd/pgd, it converts the passed pfn into an mfn.  Conversely,
 * when reading the content back with __(pgd|pmd|pte)_val, it converts
 * the mfn back into a pfn.
 *
 * The other constraint is that all pages which make up a pagetable
 * must be mapped read-only in the guest.  This prevents uncontrolled
 * guest updates to the pagetable.  Xen strictly enforces this, and
 * will disallow any pagetable update which will end up mapping a
 * pagetable page RW, and will disallow using any writable page as a
 * pagetable.
 *
 * Naively, when loading %cr3 with the base of a new pagetable, Xen
 * would need to validate the whole pagetable before going on.
 * Naturally, this is quite slow.  The solution is to "pin" a
 * pagetable, which enforces all the constraints on the pagetable even
 * when it is not actively in use.  This menas that Xen can be assured
 * that it is still valid when you do load it into %cr3, and doesn't
 * need to revalidate it.
 *
 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 */
#include <linux/sched.h>
#include <linux/highmem.h>
#include <linux/bug.h>

#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/paravirt.h>

#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>

#include <xen/page.h>
#include <xen/interface/xen.h>

#include "multicalls.h"
#include "mmu.h"

xmaddr_t arbitrary_virt_to_machine(unsigned long address)
{
	pte_t *pte = lookup_address(address);
	unsigned offset = address & PAGE_MASK;

	BUG_ON(pte == NULL);

	return XMADDR((pte_mfn(*pte) << PAGE_SHIFT) + offset);
}

void make_lowmem_page_readonly(void *vaddr)
{
	pte_t *pte, ptev;
	unsigned long address = (unsigned long)vaddr;

	pte = lookup_address(address);
	BUG_ON(pte == NULL);

	ptev = pte_wrprotect(*pte);

	if (HYPERVISOR_update_va_mapping(address, ptev, 0))
		BUG();
}

void make_lowmem_page_readwrite(void *vaddr)
{
	pte_t *pte, ptev;
	unsigned long address = (unsigned long)vaddr;

	pte = lookup_address(address);
	BUG_ON(pte == NULL);

	ptev = pte_mkwrite(*pte);

	if (HYPERVISOR_update_va_mapping(address, ptev, 0))
		BUG();
}


void xen_set_pmd(pmd_t *ptr, pmd_t val)
{
	struct multicall_space mcs;
	struct mmu_update *u;

	preempt_disable();

	mcs = xen_mc_entry(sizeof(*u));
	u = mcs.args;
	u->ptr = virt_to_machine(ptr).maddr;
	u->val = pmd_val_ma(val);
	MULTI_mmu_update(mcs.mc, u, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_MMU);

	preempt_enable();
}

/*
 * Associate a virtual page frame with a given physical page frame
 * and protection flags for that frame.
 */
void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	pgd = swapper_pg_dir + pgd_index(vaddr);
	if (pgd_none(*pgd)) {
		BUG();
		return;
	}
	pud = pud_offset(pgd, vaddr);
	if (pud_none(*pud)) {
		BUG();
		return;
	}
	pmd = pmd_offset(pud, vaddr);
	if (pmd_none(*pmd)) {
		BUG();
		return;
	}
	pte = pte_offset_kernel(pmd, vaddr);
	/* <mfn,flags> stored as-is, to permit clearing entries */
	xen_set_pte(pte, mfn_pte(mfn, flags));

	/*
	 * It's enough to flush this one mapping.
	 * (PGE mappings get flushed as well)
	 */
	__flush_tlb_one(vaddr);
}

void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
		    pte_t *ptep, pte_t pteval)
{
	if (mm == current->mm || mm == &init_mm) {
		if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
			struct multicall_space mcs;
			mcs = xen_mc_entry(0);

			MULTI_update_va_mapping(mcs.mc, addr, pteval, 0);
			xen_mc_issue(PARAVIRT_LAZY_MMU);
			return;
		} else
			if (HYPERVISOR_update_va_mapping(addr, pteval, 0) == 0)
				return;
	}
	xen_set_pte(ptep, pteval);
}

#ifdef CONFIG_X86_PAE
void xen_set_pud(pud_t *ptr, pud_t val)
{
	struct multicall_space mcs;
	struct mmu_update *u;

	preempt_disable();

	mcs = xen_mc_entry(sizeof(*u));
	u = mcs.args;
	u->ptr = virt_to_machine(ptr).maddr;
	u->val = pud_val_ma(val);
	MULTI_mmu_update(mcs.mc, u, 1, NULL, DOMID_SELF);

	xen_mc_issue(PARAVIRT_LAZY_MMU);

	preempt_enable();
}

void xen_set_pte(pte_t *ptep, pte_t pte)
{
	ptep->pte_high = pte.pte_high;
	smp_wmb();
	ptep->pte_low = pte.pte_low;
}

void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
{
	set_64bit((u64 *)ptep, pte_val_ma(pte));
}

void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
{
	ptep->pte_low = 0;
	smp_wmb();		/* make sure low gets written first */
	ptep->pte_high = 0;
}

void xen_pmd_clear(pmd_t *pmdp)
{
	xen_set_pmd(pmdp, __pmd(0));
}

unsigned long long xen_pte_val(pte_t pte)
{
	unsigned long long ret = 0;

	if (pte.pte_low) {
		ret = ((unsigned long long)pte.pte_high << 32) | pte.pte_low;
		ret = machine_to_phys(XMADDR(ret)).paddr | 1;
	}

	return ret;
}

unsigned long long xen_pmd_val(pmd_t pmd)
{
	unsigned long long ret = pmd.pmd;
	if (ret)
		ret = machine_to_phys(XMADDR(ret)).paddr | 1;
	return ret;
}

unsigned long long xen_pgd_val(pgd_t pgd)
{
	unsigned long long ret = pgd.pgd;
	if (ret)
		ret = machine_to_phys(XMADDR(ret)).paddr | 1;
	return ret;
}

pte_t xen_make_pte(unsigned long long pte)
{
	if (pte & 1)
		pte = phys_to_machine(XPADDR(pte)).maddr;

	return (pte_t){ pte, pte >> 32 };
}

pmd_t xen_make_pmd(unsigned long long pmd)
{
	if (pmd & 1)
		pmd = phys_to_machine(XPADDR(pmd)).maddr;

	return (pmd_t){ pmd };
}

pgd_t xen_make_pgd(unsigned long long pgd)
{
	if (pgd & _PAGE_PRESENT)
		pgd = phys_to_machine(XPADDR(pgd)).maddr;

	return (pgd_t){ pgd };
}
#else  /* !PAE */
void xen_set_pte(pte_t *ptep, pte_t pte)
{
	*ptep = pte;
}

unsigned long xen_pte_val(pte_t pte)
{
	unsigned long ret = pte.pte_low;

	if (ret & _PAGE_PRESENT)
		ret = machine_to_phys(XMADDR(ret)).paddr;

	return ret;
}

unsigned long xen_pgd_val(pgd_t pgd)
{
	unsigned long ret = pgd.pgd;
	if (ret)
		ret = machine_to_phys(XMADDR(ret)).paddr | 1;
	return ret;
}

pte_t xen_make_pte(unsigned long pte)
{
	if (pte & _PAGE_PRESENT)
		pte = phys_to_machine(XPADDR(pte)).maddr;

	return (pte_t){ pte };
}

pgd_t xen_make_pgd(unsigned long pgd)
{
	if (pgd & _PAGE_PRESENT)
		pgd = phys_to_machine(XPADDR(pgd)).maddr;

	return (pgd_t){ pgd };
}
#endif	/* CONFIG_X86_PAE */


/*
  (Yet another) pagetable walker.  This one is intended for pinning a
  pagetable.  This means that it walks a pagetable and calls the
  callback function on each page it finds making up the page table,
  at every level.  It walks the entire pagetable, but it only bothers
  pinning pte pages which are below pte_limit.  In the normal case
  this will be TASK_SIZE, but at boot we need to pin up to
  FIXADDR_TOP.  But the important bit is that we don't pin beyond
  there, because then we start getting into Xen's ptes.
*/
static int pgd_walk(pgd_t *pgd_base, int (*func)(struct page *, unsigned),
		    unsigned long limit)
{
	pgd_t *pgd = pgd_base;
	int flush = 0;
	unsigned long addr = 0;
	unsigned long pgd_next;

	BUG_ON(limit > FIXADDR_TOP);

	if (xen_feature(XENFEAT_auto_translated_physmap))
		return 0;

	for (; addr != FIXADDR_TOP; pgd++, addr = pgd_next) {
		pud_t *pud;
		unsigned long pud_limit, pud_next;

		pgd_next = pud_limit = pgd_addr_end(addr, FIXADDR_TOP);

		if (!pgd_val(*pgd))
			continue;

		pud = pud_offset(pgd, 0);

		if (PTRS_PER_PUD > 1) /* not folded */
			flush |= (*func)(virt_to_page(pud), 0);

		for (; addr != pud_limit; pud++, addr = pud_next) {
			pmd_t *pmd;
			unsigned long pmd_limit;

			pud_next = pud_addr_end(addr, pud_limit);

			if (pud_next < limit)
				pmd_limit = pud_next;
			else
				pmd_limit = limit;

			if (pud_none(*pud))
				continue;

			pmd = pmd_offset(pud, 0);

			if (PTRS_PER_PMD > 1) /* not folded */
				flush |= (*func)(virt_to_page(pmd), 0);

			for (; addr != pmd_limit; pmd++) {
				addr += (PAGE_SIZE * PTRS_PER_PTE);
				if ((pmd_limit-1) < (addr-1)) {
					addr = pmd_limit;
					break;
				}

				if (pmd_none(*pmd))
					continue;

				flush |= (*func)(pmd_page(*pmd), 0);
			}
		}
	}

	flush |= (*func)(virt_to_page(pgd_base), UVMF_TLB_FLUSH);

	return flush;
}

static int pin_page(struct page *page, unsigned flags)
{
	unsigned pgfl = test_and_set_bit(PG_pinned, &page->flags);
	int flush;

	if (pgfl)
		flush = 0;		/* already pinned */
	else if (PageHighMem(page))
		/* kmaps need flushing if we found an unpinned
		   highpage */
		flush = 1;
	else {
		void *pt = lowmem_page_address(page);
		unsigned long pfn = page_to_pfn(page);
		struct multicall_space mcs = __xen_mc_entry(0);

		flush = 0;

		MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
					pfn_pte(pfn, PAGE_KERNEL_RO),
					flags);
	}

	return flush;
}

/* This is called just after a mm has been created, but it has not
   been used yet.  We need to make sure that its pagetable is all
   read-only, and can be pinned. */
void xen_pgd_pin(pgd_t *pgd)
{
	struct multicall_space mcs;
	struct mmuext_op *op;

	xen_mc_batch();

	if (pgd_walk(pgd, pin_page, TASK_SIZE)) {
		/* re-enable interrupts for kmap_flush_unused */
		xen_mc_issue(0);
		kmap_flush_unused();
		xen_mc_batch();
	}

	mcs = __xen_mc_entry(sizeof(*op));
	op = mcs.args;

#ifdef CONFIG_X86_PAE
	op->cmd = MMUEXT_PIN_L3_TABLE;
#else
	op->cmd = MMUEXT_PIN_L2_TABLE;
#endif
	op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd)));
	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);

	xen_mc_issue(0);
}

/* The init_mm pagetable is really pinned as soon as its created, but
   that's before we have page structures to store the bits.  So do all
   the book-keeping now. */
static __init int mark_pinned(struct page *page, unsigned flags)
{
	SetPagePinned(page);
	return 0;
}

void __init xen_mark_init_mm_pinned(void)
{
	pgd_walk(init_mm.pgd, mark_pinned, FIXADDR_TOP);
}

static int unpin_page(struct page *page, unsigned flags)
{
	unsigned pgfl = test_and_clear_bit(PG_pinned, &page->flags);

	if (pgfl && !PageHighMem(page)) {
		void *pt = lowmem_page_address(page);
		unsigned long pfn = page_to_pfn(page);
		struct multicall_space mcs = __xen_mc_entry(0);

		MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
					pfn_pte(pfn, PAGE_KERNEL),
					flags);
	}

	return 0;		/* never need to flush on unpin */
}

/* Release a pagetables pages back as normal RW */
static void xen_pgd_unpin(pgd_t *pgd)
{
	struct mmuext_op *op;
	struct multicall_space mcs;

	xen_mc_batch();

	mcs = __xen_mc_entry(sizeof(*op));

	op = mcs.args;
	op->cmd = MMUEXT_UNPIN_TABLE;
	op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd)));

	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);

	pgd_walk(pgd, unpin_page, TASK_SIZE);

	xen_mc_issue(0);
}

void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
{
	spin_lock(&next->page_table_lock);
	xen_pgd_pin(next->pgd);
	spin_unlock(&next->page_table_lock);
}

void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
{
	spin_lock(&mm->page_table_lock);
	xen_pgd_pin(mm->pgd);
	spin_unlock(&mm->page_table_lock);
}


#ifdef CONFIG_SMP
/* Another cpu may still have their %cr3 pointing at the pagetable, so
   we need to repoint it somewhere else before we can unpin it. */
static void drop_other_mm_ref(void *info)
{
	struct mm_struct *mm = info;

	if (__get_cpu_var(cpu_tlbstate).active_mm == mm)
		leave_mm(smp_processor_id());
}

static void drop_mm_ref(struct mm_struct *mm)
{
	if (current->active_mm == mm) {
		if (current->mm == mm)
			load_cr3(swapper_pg_dir);
		else
			leave_mm(smp_processor_id());
	}

	if (!cpus_empty(mm->cpu_vm_mask))
		xen_smp_call_function_mask(mm->cpu_vm_mask, drop_other_mm_ref,
					   mm, 1);
}
#else
static void drop_mm_ref(struct mm_struct *mm)
{
	if (current->active_mm == mm)
		load_cr3(swapper_pg_dir);
}
#endif

/*
 * While a process runs, Xen pins its pagetables, which means that the
 * hypervisor forces it to be read-only, and it controls all updates
 * to it.  This means that all pagetable updates have to go via the
 * hypervisor, which is moderately expensive.
 *
 * Since we're pulling the pagetable down, we switch to use init_mm,
 * unpin old process pagetable and mark it all read-write, which
 * allows further operations on it to be simple memory accesses.
 *
 * The only subtle point is that another CPU may be still using the
 * pagetable because of lazy tlb flushing.  This means we need need to
 * switch all CPUs off this pagetable before we can unpin it.
 */
void xen_exit_mmap(struct mm_struct *mm)
{
	get_cpu();		/* make sure we don't move around */
	drop_mm_ref(mm);
	put_cpu();

	spin_lock(&mm->page_table_lock);

	/* pgd may not be pinned in the error exit path of execve */
	if (PagePinned(virt_to_page(mm->pgd)))
		xen_pgd_unpin(mm->pgd);
	spin_unlock(&mm->page_table_lock);
}
Commit	Line	Data
3b827c1b JF	1	/*
	2	* Xen mmu operations
	3	*
	4	* This file contains the various mmu fetch and update operations.
	5	* The most important job they must perform is the mapping between the
	6	* domain's pfn and the overall machine mfns.
	7	*
	8	* Xen allows guests to directly update the pagetable, in a controlled
	9	* fashion. In other words, the guest modifies the same pagetable
	10	* that the CPU actually uses, which eliminates the overhead of having
	11	* a separate shadow pagetable.
	12	*
	13	* In order to allow this, it falls on the guest domain to map its
	14	* notion of a "physical" pfn - which is just a domain-local linear
	15	* address - into a real "machine address" which the CPU's MMU can
	16	* use.
	17	*
	18	* A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
	19	* inserted directly into the pagetable. When creating a new
	20	* pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
	21	* when reading the content back with __(pgd\|pmd\|pte)_val, it converts
	22	* the mfn back into a pfn.
	23	*
	24	* The other constraint is that all pages which make up a pagetable
	25	* must be mapped read-only in the guest. This prevents uncontrolled
	26	* guest updates to the pagetable. Xen strictly enforces this, and
	27	* will disallow any pagetable update which will end up mapping a
	28	* pagetable page RW, and will disallow using any writable page as a
	29	* pagetable.
	30	*
	31	* Naively, when loading %cr3 with the base of a new pagetable, Xen
	32	* would need to validate the whole pagetable before going on.
	33	* Naturally, this is quite slow. The solution is to "pin" a
	34	* pagetable, which enforces all the constraints on the pagetable even
	35	* when it is not actively in use. This menas that Xen can be assured
	36	* that it is still valid when you do load it into %cr3, and doesn't
	37	* need to revalidate it.
	38	*
	39	* Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
	40	*/
f120f13e	41	#include <linux/sched.h>
f4f97b3e	42	#include <linux/highmem.h>
3b827c1b	43	#include <linux/bug.h>
3b827c1b JF	44
	45	#include <asm/pgtable.h>
	46	#include <asm/tlbflush.h>
	47	#include <asm/mmu_context.h>
f4f97b3e	48	#include <asm/paravirt.h>
3b827c1b JF	49
3b827c1b JF	50	#include <asm/xen/hypercall.h>
f4f97b3e	51	#include <asm/xen/hypervisor.h>
3b827c1b JF	52
	53	#include <xen/page.h>
	54	#include <xen/interface/xen.h>
	55
f4f97b3e	56	#include "multicalls.h"
3b827c1b JF	57	#include "mmu.h"
	58
	59	xmaddr_t arbitrary_virt_to_machine(unsigned long address)
	60	{
	61	pte_t *pte = lookup_address(address);
	62	unsigned offset = address & PAGE_MASK;
	63
	64	BUG_ON(pte == NULL);
	65
	66	return XMADDR((pte_mfn(*pte) << PAGE_SHIFT) + offset);
	67	}
	68
	69	void make_lowmem_page_readonly(void *vaddr)
	70	{
	71	pte_t *pte, ptev;
	72	unsigned long address = (unsigned long)vaddr;
	73
	74	pte = lookup_address(address);
	75	BUG_ON(pte == NULL);
	76
	77	ptev = pte_wrprotect(*pte);
	78
	79	if (HYPERVISOR_update_va_mapping(address, ptev, 0))
	80	BUG();
	81	}
	82
	83	void make_lowmem_page_readwrite(void *vaddr)
	84	{
	85	pte_t *pte, ptev;
	86	unsigned long address = (unsigned long)vaddr;
	87
	88	pte = lookup_address(address);
	89	BUG_ON(pte == NULL);
	90
	91	ptev = pte_mkwrite(*pte);
	92
	93	if (HYPERVISOR_update_va_mapping(address, ptev, 0))
	94	BUG();
	95	}
	96
	97
3b827c1b JF	98	void xen_set_pmd(pmd_t *ptr, pmd_t val)
3b827c1b JF	99	{
d66bf8fc JF	100	struct multicall_space mcs;
d66bf8fc JF	101	struct mmu_update *u;
3b827c1b	102
d66bf8fc JF	103	preempt_disable();
	104
	105	mcs = xen_mc_entry(sizeof(*u));
	106	u = mcs.args;
	107	u->ptr = virt_to_machine(ptr).maddr;
	108	u->val = pmd_val_ma(val);
	109	MULTI_mmu_update(mcs.mc, u, 1, NULL, DOMID_SELF);
	110
	111	xen_mc_issue(PARAVIRT_LAZY_MMU);
	112
	113	preempt_enable();
3b827c1b JF	114	}
3b827c1b JF	115
3b827c1b JF	116	/*
	117	* Associate a virtual page frame with a given physical page frame
	118	* and protection flags for that frame.
	119	*/
	120	void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
	121	{
	122	pgd_t *pgd;
	123	pud_t *pud;
	124	pmd_t *pmd;
	125	pte_t *pte;
	126
	127	pgd = swapper_pg_dir + pgd_index(vaddr);
	128	if (pgd_none(*pgd)) {
	129	BUG();
	130	return;
	131	}
	132	pud = pud_offset(pgd, vaddr);
	133	if (pud_none(*pud)) {
	134	BUG();
	135	return;
	136	}
	137	pmd = pmd_offset(pud, vaddr);
	138	if (pmd_none(*pmd)) {
	139	BUG();
	140	return;
	141	}
	142	pte = pte_offset_kernel(pmd, vaddr);
	143	/* <mfn,flags> stored as-is, to permit clearing entries */
	144	xen_set_pte(pte, mfn_pte(mfn, flags));
	145
	146	/*
	147	* It's enough to flush this one mapping.
	148	* (PGE mappings get flushed as well)
	149	*/
	150	__flush_tlb_one(vaddr);
	151	}
	152
	153	void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
	154	pte_t *ptep, pte_t pteval)
	155	{
d66bf8fc	156	if (mm == current->mm \|\| mm == &init_mm) {
8965c1c0	157	if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
d66bf8fc JF	158	struct multicall_space mcs;
	159	mcs = xen_mc_entry(0);
	160
	161	MULTI_update_va_mapping(mcs.mc, addr, pteval, 0);
	162	xen_mc_issue(PARAVIRT_LAZY_MMU);
	163	return;
	164	} else
	165	if (HYPERVISOR_update_va_mapping(addr, pteval, 0) == 0)
	166	return;
	167	}
	168	xen_set_pte(ptep, pteval);
3b827c1b JF	169	}
	170
	171	#ifdef CONFIG_X86_PAE
f4f97b3e JF	172	void xen_set_pud(pud_t *ptr, pud_t val)
f4f97b3e JF	173	{
d66bf8fc JF	174	struct multicall_space mcs;
d66bf8fc JF	175	struct mmu_update *u;
f4f97b3e	176
d66bf8fc JF	177	preempt_disable();
	178
	179	mcs = xen_mc_entry(sizeof(*u));
	180	u = mcs.args;
	181	u->ptr = virt_to_machine(ptr).maddr;
	182	u->val = pud_val_ma(val);
	183	MULTI_mmu_update(mcs.mc, u, 1, NULL, DOMID_SELF);
	184
	185	xen_mc_issue(PARAVIRT_LAZY_MMU);
	186
	187	preempt_enable();
f4f97b3e JF	188	}
	189
	190	void xen_set_pte(pte_t *ptep, pte_t pte)
	191	{
	192	ptep->pte_high = pte.pte_high;
	193	smp_wmb();
	194	ptep->pte_low = pte.pte_low;
	195	}
	196
3b827c1b JF	197	void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
	198	{
	199	set_64bit((u64 *)ptep, pte_val_ma(pte));
	200	}
	201
	202	void xen_pte_clear(struct mm_struct mm, unsigned long addr, pte_t ptep)
	203	{
	204	ptep->pte_low = 0;
	205	smp_wmb(); /* make sure low gets written first */
	206	ptep->pte_high = 0;
	207	}
	208
	209	void xen_pmd_clear(pmd_t *pmdp)
	210	{
	211	xen_set_pmd(pmdp, __pmd(0));
	212	}
	213
	214	unsigned long long xen_pte_val(pte_t pte)
	215	{
	216	unsigned long long ret = 0;
	217
	218	if (pte.pte_low) {
	219	ret = ((unsigned long long)pte.pte_high << 32) \| pte.pte_low;
	220	ret = machine_to_phys(XMADDR(ret)).paddr \| 1;
	221	}
	222
	223	return ret;
	224	}
	225
	226	unsigned long long xen_pmd_val(pmd_t pmd)
	227	{
	228	unsigned long long ret = pmd.pmd;
	229	if (ret)
	230	ret = machine_to_phys(XMADDR(ret)).paddr \| 1;
	231	return ret;
	232	}
	233
	234	unsigned long long xen_pgd_val(pgd_t pgd)
	235	{
	236	unsigned long long ret = pgd.pgd;
	237	if (ret)
	238	ret = machine_to_phys(XMADDR(ret)).paddr \| 1;
	239	return ret;
	240	}
	241
	242	pte_t xen_make_pte(unsigned long long pte)
	243	{
	244	if (pte & 1)
	245	pte = phys_to_machine(XPADDR(pte)).maddr;
	246
	247	return (pte_t){ pte, pte >> 32 };
	248	}
	249
	250	pmd_t xen_make_pmd(unsigned long long pmd)
	251	{
	252	if (pmd & 1)
	253	pmd = phys_to_machine(XPADDR(pmd)).maddr;
	254
	255	return (pmd_t){ pmd };
	256	}
	257
	258	pgd_t xen_make_pgd(unsigned long long pgd)
	259	{
	260	if (pgd & _PAGE_PRESENT)
261	pgd = phys_to_machine(XPADDR(pgd)).maddr;
262
263	return (pgd_t){ pgd };
264	}
265	#else /* !PAE */
f4f97b3e JF	266	void xen_set_pte(pte_t *ptep, pte_t pte)
	267	{
	268	*ptep = pte;
	269	}
	270
3b827c1b JF	271	unsigned long xen_pte_val(pte_t pte)
	272	{
	273	unsigned long ret = pte.pte_low;
	274
	275	if (ret & _PAGE_PRESENT)
	276	ret = machine_to_phys(XMADDR(ret)).paddr;
	277
	278	return ret;
	279	}
	280
3b827c1b JF	281	unsigned long xen_pgd_val(pgd_t pgd)
	282	{
	283	unsigned long ret = pgd.pgd;
	284	if (ret)
	285	ret = machine_to_phys(XMADDR(ret)).paddr \| 1;
	286	return ret;
	287	}
	288
	289	pte_t xen_make_pte(unsigned long pte)
	290	{
	291	if (pte & _PAGE_PRESENT)
	292	pte = phys_to_machine(XPADDR(pte)).maddr;
	293
	294	return (pte_t){ pte };
	295	}
	296
3b827c1b JF	297	pgd_t xen_make_pgd(unsigned long pgd)
	298	{
	299	if (pgd & _PAGE_PRESENT)
	300	pgd = phys_to_machine(XPADDR(pgd)).maddr;
	301
	302	return (pgd_t){ pgd };
	303	}
	304	#endif /* CONFIG_X86_PAE */
	305
	306
	307
f4f97b3e JF	308	/*
	309	(Yet another) pagetable walker. This one is intended for pinning a
	310	pagetable. This means that it walks a pagetable and calls the
	311	callback function on each page it finds making up the page table,
	312	at every level. It walks the entire pagetable, but it only bothers
	313	pinning pte pages which are below pte_limit. In the normal case
	314	this will be TASK_SIZE, but at boot we need to pin up to
	315	FIXADDR_TOP. But the important bit is that we don't pin beyond
	316	there, because then we start getting into Xen's ptes.
	317	*/
	318	static int pgd_walk(pgd_t pgd_base, int (func)(struct page *, unsigned),
	319	unsigned long limit)
3b827c1b JF	320	{
3b827c1b JF	321	pgd_t *pgd = pgd_base;
f4f97b3e JF	322	int flush = 0;
	323	unsigned long addr = 0;
	324	unsigned long pgd_next;
	325
	326	BUG_ON(limit > FIXADDR_TOP);
3b827c1b JF	327
3b827c1b JF	328	if (xen_feature(XENFEAT_auto_translated_physmap))
f4f97b3e JF	329	return 0;
	330
	331	for (; addr != FIXADDR_TOP; pgd++, addr = pgd_next) {
	332	pud_t *pud;
	333	unsigned long pud_limit, pud_next;
3b827c1b	334
f4f97b3e JF	335	pgd_next = pud_limit = pgd_addr_end(addr, FIXADDR_TOP);
	336
	337	if (!pgd_val(*pgd))
3b827c1b	338	continue;
f4f97b3e	339
3b827c1b JF	340	pud = pud_offset(pgd, 0);
	341
	342	if (PTRS_PER_PUD > 1) /* not folded */
f4f97b3e JF	343	flush \|= (*func)(virt_to_page(pud), 0);
	344
	345	for (; addr != pud_limit; pud++, addr = pud_next) {
	346	pmd_t *pmd;
	347	unsigned long pmd_limit;
	348
	349	pud_next = pud_addr_end(addr, pud_limit);
	350
	351	if (pud_next < limit)
	352	pmd_limit = pud_next;
	353	else
	354	pmd_limit = limit;
3b827c1b	355
3b827c1b JF	356	if (pud_none(*pud))
3b827c1b JF	357	continue;
f4f97b3e	358
3b827c1b JF	359	pmd = pmd_offset(pud, 0);
	360
	361	if (PTRS_PER_PMD > 1) /* not folded */
f4f97b3e JF	362	flush \|= (*func)(virt_to_page(pmd), 0);
	363
	364	for (; addr != pmd_limit; pmd++) {
	365	addr += (PAGE_SIZE * PTRS_PER_PTE);
	366	if ((pmd_limit-1) < (addr-1)) {
	367	addr = pmd_limit;
	368	break;
	369	}
3b827c1b	370
3b827c1b JF	371	if (pmd_none(*pmd))
	372	continue;
	373
f4f97b3e	374	flush \|= (func)(pmd_page(pmd), 0);
3b827c1b JF	375	}
	376	}
	377	}
	378
f4f97b3e JF	379	flush \|= (*func)(virt_to_page(pgd_base), UVMF_TLB_FLUSH);
	380
	381	return flush;
3b827c1b JF	382	}
3b827c1b JF	383
f4f97b3e JF	384	static int pin_page(struct page *page, unsigned flags)
	385	{
	386	unsigned pgfl = test_and_set_bit(PG_pinned, &page->flags);
	387	int flush;
	388
	389	if (pgfl)
	390	flush = 0; /* already pinned */
	391	else if (PageHighMem(page))
	392	/* kmaps need flushing if we found an unpinned
	393	highpage */
	394	flush = 1;
	395	else {
	396	void *pt = lowmem_page_address(page);
	397	unsigned long pfn = page_to_pfn(page);
	398	struct multicall_space mcs = __xen_mc_entry(0);
	399
	400	flush = 0;
	401
	402	MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
	403	pfn_pte(pfn, PAGE_KERNEL_RO),
	404	flags);
	405	}
	406
	407	return flush;
	408	}
3b827c1b	409
f4f97b3e JF	410	/* This is called just after a mm has been created, but it has not
	411	been used yet. We need to make sure that its pagetable is all
	412	read-only, and can be pinned. */
3b827c1b JF	413	void xen_pgd_pin(pgd_t *pgd)
3b827c1b JF	414	{
f4f97b3e JF	415	struct multicall_space mcs;
f4f97b3e JF	416	struct mmuext_op *op;
3b827c1b	417
f4f97b3e	418	xen_mc_batch();
3b827c1b	419
f87e4cac JF	420	if (pgd_walk(pgd, pin_page, TASK_SIZE)) {
	421	/* re-enable interrupts for kmap_flush_unused */
	422	xen_mc_issue(0);
f4f97b3e	423	kmap_flush_unused();
f87e4cac JF	424	xen_mc_batch();
f87e4cac JF	425	}
f4f97b3e JF	426
	427	mcs = __xen_mc_entry(sizeof(*op));
	428	op = mcs.args;
	429
	430	#ifdef CONFIG_X86_PAE
	431	op->cmd = MMUEXT_PIN_L3_TABLE;
3b827c1b	432	#else
f4f97b3e	433	op->cmd = MMUEXT_PIN_L2_TABLE;
3b827c1b	434	#endif
f4f97b3e JF	435	op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd)));
	436	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
	437
	438	xen_mc_issue(0);
3b827c1b JF	439	}
3b827c1b JF	440
f4f97b3e JF	441	/* The init_mm pagetable is really pinned as soon as its created, but
	442	that's before we have page structures to store the bits. So do all
	443	the book-keeping now. */
	444	static __init int mark_pinned(struct page *page, unsigned flags)
3b827c1b	445	{
f4f97b3e JF	446	SetPagePinned(page);
	447	return 0;
	448	}
3b827c1b	449
f4f97b3e JF	450	void __init xen_mark_init_mm_pinned(void)
	451	{
	452	pgd_walk(init_mm.pgd, mark_pinned, FIXADDR_TOP);
	453	}
3b827c1b	454
f4f97b3e JF	455	static int unpin_page(struct page *page, unsigned flags)
	456	{
	457	unsigned pgfl = test_and_clear_bit(PG_pinned, &page->flags);
3b827c1b	458
f4f97b3e JF	459	if (pgfl && !PageHighMem(page)) {
	460	void *pt = lowmem_page_address(page);
	461	unsigned long pfn = page_to_pfn(page);
	462	struct multicall_space mcs = __xen_mc_entry(0);
	463
	464	MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
	465	pfn_pte(pfn, PAGE_KERNEL),
	466	flags);
	467	}
	468
	469	return 0; /* never need to flush on unpin */
3b827c1b JF	470	}
3b827c1b JF	471
f4f97b3e JF	472	/* Release a pagetables pages back as normal RW */
	473	static void xen_pgd_unpin(pgd_t *pgd)
	474	{
	475	struct mmuext_op *op;
	476	struct multicall_space mcs;
	477
	478	xen_mc_batch();
	479
	480	mcs = __xen_mc_entry(sizeof(*op));
	481
	482	op = mcs.args;
	483	op->cmd = MMUEXT_UNPIN_TABLE;
	484	op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd)));
	485
	486	MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
	487
	488	pgd_walk(pgd, unpin_page, TASK_SIZE);
	489
	490	xen_mc_issue(0);
	491	}
3b827c1b JF	492
	493	void xen_activate_mm(struct mm_struct prev, struct mm_struct next)
	494	{
f4f97b3e	495	spin_lock(&next->page_table_lock);
3b827c1b	496	xen_pgd_pin(next->pgd);
f4f97b3e	497	spin_unlock(&next->page_table_lock);
3b827c1b JF	498	}
	499
	500	void xen_dup_mmap(struct mm_struct oldmm, struct mm_struct mm)
	501	{
f4f97b3e	502	spin_lock(&mm->page_table_lock);
3b827c1b	503	xen_pgd_pin(mm->pgd);
f4f97b3e	504	spin_unlock(&mm->page_table_lock);
3b827c1b JF	505	}
3b827c1b JF	506
3b827c1b	507
f87e4cac JF	508	#ifdef CONFIG_SMP
	509	/* Another cpu may still have their %cr3 pointing at the pagetable, so
	510	we need to repoint it somewhere else before we can unpin it. */
	511	static void drop_other_mm_ref(void *info)
	512	{
	513	struct mm_struct *mm = info;
3b827c1b	514
f87e4cac JF	515	if (__get_cpu_var(cpu_tlbstate).active_mm == mm)
	516	leave_mm(smp_processor_id());
	517	}
3b827c1b	518
f87e4cac JF	519	static void drop_mm_ref(struct mm_struct *mm)
	520	{
	521	if (current->active_mm == mm) {
	522	if (current->mm == mm)
	523	load_cr3(swapper_pg_dir);
	524	else
	525	leave_mm(smp_processor_id());
3b827c1b JF	526	}
3b827c1b JF	527
f87e4cac JF	528	if (!cpus_empty(mm->cpu_vm_mask))
	529	xen_smp_call_function_mask(mm->cpu_vm_mask, drop_other_mm_ref,
	530	mm, 1);
	531	}
	532	#else
	533	static void drop_mm_ref(struct mm_struct *mm)
	534	{
	535	if (current->active_mm == mm)
	536	load_cr3(swapper_pg_dir);
	537	}
	538	#endif
	539
	540	/*
	541	* While a process runs, Xen pins its pagetables, which means that the
	542	* hypervisor forces it to be read-only, and it controls all updates
	543	* to it. This means that all pagetable updates have to go via the
	544	* hypervisor, which is moderately expensive.
	545	*
	546	* Since we're pulling the pagetable down, we switch to use init_mm,
	547	* unpin old process pagetable and mark it all read-write, which
	548	* allows further operations on it to be simple memory accesses.
	549	*
	550	* The only subtle point is that another CPU may be still using the
	551	* pagetable because of lazy tlb flushing. This means we need need to
	552	* switch all CPUs off this pagetable before we can unpin it.
	553	*/
	554	void xen_exit_mmap(struct mm_struct *mm)
	555	{
	556	get_cpu(); /* make sure we don't move around */
	557	drop_mm_ref(mm);
	558	put_cpu();
3b827c1b	559
f120f13e	560	spin_lock(&mm->page_table_lock);
df912ea4 JF	561
	562	/* pgd may not be pinned in the error exit path of execve */
	563	if (PagePinned(virt_to_page(mm->pgd)))
	564	xen_pgd_unpin(mm->pgd);
f120f13e	565	spin_unlock(&mm->page_table_lock);
3b827c1b	566	}