[deliverable/linux.git] / arch / s390 / mm / pgtable.c

/*
 *    Copyright IBM Corp. 2007, 2011
 *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
 */

#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/rcupdate.h>
#include <linux/slab.h>
#include <linux/swapops.h>
#include <linux/sysctl.h>
#include <linux/ksm.h>
#include <linux/mman.h>

#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>

static inline pte_t ptep_flush_direct(struct mm_struct *mm,
				      unsigned long addr, pte_t *ptep)
{
	int active, count;
	pte_t old;

	old = *ptep;
	if (unlikely(pte_val(old) & _PAGE_INVALID))
		return old;
	active = (mm == current->active_mm) ? 1 : 0;
	count = atomic_add_return(0x10000, &mm->context.attach_count);
	if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
	    cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
		__ptep_ipte_local(addr, ptep);
	else
		__ptep_ipte(addr, ptep);
	atomic_sub(0x10000, &mm->context.attach_count);
	return old;
}

static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
				    unsigned long addr, pte_t *ptep)
{
	int active, count;
	pte_t old;

	old = *ptep;
	if (unlikely(pte_val(old) & _PAGE_INVALID))
		return old;
	active = (mm == current->active_mm) ? 1 : 0;
	count = atomic_add_return(0x10000, &mm->context.attach_count);
	if ((count & 0xffff) <= active) {
		pte_val(*ptep) |= _PAGE_INVALID;
		mm->context.flush_mm = 1;
	} else
		__ptep_ipte(addr, ptep);
	atomic_sub(0x10000, &mm->context.attach_count);
	return old;
}

static inline pgste_t pgste_get_lock(pte_t *ptep)
{
	unsigned long new = 0;
#ifdef CONFIG_PGSTE
	unsigned long old;

	preempt_disable();
	asm(
		"	lg	%0,%2\n"
		"0:	lgr	%1,%0\n"
		"	nihh	%0,0xff7f\n"	/* clear PCL bit in old */
		"	oihh	%1,0x0080\n"	/* set PCL bit in new */
		"	csg	%0,%1,%2\n"
		"	jl	0b\n"
		: "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
		: "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
#endif
	return __pgste(new);
}

static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
{
#ifdef CONFIG_PGSTE
	asm(
		"	nihh	%1,0xff7f\n"	/* clear PCL bit */
		"	stg	%1,%0\n"
		: "=Q" (ptep[PTRS_PER_PTE])
		: "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
		: "cc", "memory");
	preempt_enable();
#endif
}

static inline pgste_t pgste_get(pte_t *ptep)
{
	unsigned long pgste = 0;
#ifdef CONFIG_PGSTE
	pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
#endif
	return __pgste(pgste);
}

static inline void pgste_set(pte_t *ptep, pgste_t pgste)
{
#ifdef CONFIG_PGSTE
	*(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
#endif
}

static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
				       struct mm_struct *mm)
{
#ifdef CONFIG_PGSTE
	unsigned long address, bits, skey;

	if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID)
		return pgste;
	address = pte_val(pte) & PAGE_MASK;
	skey = (unsigned long) page_get_storage_key(address);
	bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
	/* Transfer page changed & referenced bit to guest bits in pgste */
	pgste_val(pgste) |= bits << 48;		/* GR bit & GC bit */
	/* Copy page access key and fetch protection bit to pgste */
	pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
	pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
#endif
	return pgste;

}

static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
				 struct mm_struct *mm)
{
#ifdef CONFIG_PGSTE
	unsigned long address;
	unsigned long nkey;

	if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID)
		return;
	VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
	address = pte_val(entry) & PAGE_MASK;
	/*
	 * Set page access key and fetch protection bit from pgste.
	 * The guest C/R information is still in the PGSTE, set real
	 * key C/R to 0.
	 */
	nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
	nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
	page_set_storage_key(address, nkey, 0);
#endif
}

static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
{
#ifdef CONFIG_PGSTE
	if ((pte_val(entry) & _PAGE_PRESENT) &&
	    (pte_val(entry) & _PAGE_WRITE) &&
	    !(pte_val(entry) & _PAGE_INVALID)) {
		if (!MACHINE_HAS_ESOP) {
			/*
			 * Without enhanced suppression-on-protection force
			 * the dirty bit on for all writable ptes.
			 */
			pte_val(entry) |= _PAGE_DIRTY;
			pte_val(entry) &= ~_PAGE_PROTECT;
		}
		if (!(pte_val(entry) & _PAGE_PROTECT))
			/* This pte allows write access, set user-dirty */
			pgste_val(pgste) |= PGSTE_UC_BIT;
	}
#endif
	*ptep = entry;
	return pgste;
}

static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,
					unsigned long addr,
					pte_t *ptep, pgste_t pgste)
{
#ifdef CONFIG_PGSTE
	if (pgste_val(pgste) & PGSTE_IN_BIT) {
		pgste_val(pgste) &= ~PGSTE_IN_BIT;
		ptep_notify(mm, addr, ptep);
	}
#endif
	return pgste;
}

static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
				      unsigned long addr, pte_t *ptep)
{
	pgste_t pgste = __pgste(0);

	if (mm_has_pgste(mm)) {
		pgste = pgste_get_lock(ptep);
		pgste = pgste_ipte_notify(mm, addr, ptep, pgste);
	}
	return pgste;
}

static inline void ptep_xchg_commit(struct mm_struct *mm,
				    unsigned long addr, pte_t *ptep,
				    pgste_t pgste, pte_t old, pte_t new)
{
	if (mm_has_pgste(mm)) {
		if (pte_val(old) & _PAGE_INVALID)
			pgste_set_key(ptep, pgste, new, mm);
		if (pte_val(new) & _PAGE_INVALID) {
			pgste = pgste_update_all(old, pgste, mm);
			if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
			    _PGSTE_GPS_USAGE_UNUSED)
				pte_val(old) |= _PAGE_UNUSED;
		}
		pgste = pgste_set_pte(ptep, pgste, new);
		pgste_set_unlock(ptep, pgste);
	} else {
		*ptep = new;
	}
}

pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
		       pte_t *ptep, pte_t new)
{
	pgste_t pgste;
	pte_t old;

	pgste = ptep_xchg_start(mm, addr, ptep);
	old = ptep_flush_direct(mm, addr, ptep);
	ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
	return old;
}
EXPORT_SYMBOL(ptep_xchg_direct);

pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, pte_t new)
{
	pgste_t pgste;
	pte_t old;

	pgste = ptep_xchg_start(mm, addr, ptep);
	old = ptep_flush_lazy(mm, addr, ptep);
	ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
	return old;
}
EXPORT_SYMBOL(ptep_xchg_lazy);

pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
			     pte_t *ptep)
{
	pgste_t pgste;
	pte_t old;

	pgste = ptep_xchg_start(mm, addr, ptep);
	old = ptep_flush_lazy(mm, addr, ptep);
	if (mm_has_pgste(mm)) {
		pgste = pgste_update_all(old, pgste, mm);
		pgste_set(ptep, pgste);
	}
	return old;
}
EXPORT_SYMBOL(ptep_modify_prot_start);

void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
			     pte_t *ptep, pte_t pte)
{
	pgste_t pgste;

	if (mm_has_pgste(mm)) {
		pgste = pgste_get(ptep);
		pgste_set_key(ptep, pgste, pte, mm);
		pgste = pgste_set_pte(ptep, pgste, pte);
		pgste_set_unlock(ptep, pgste);
	} else {
		*ptep = pte;
	}
}
EXPORT_SYMBOL(ptep_modify_prot_commit);

static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
				      unsigned long addr, pmd_t *pmdp)
{
	int active, count;
	pmd_t old;

	old = *pmdp;
	if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
		return old;
	if (!MACHINE_HAS_IDTE) {
		__pmdp_csp(pmdp);
		return old;
	}
	active = (mm == current->active_mm) ? 1 : 0;
	count = atomic_add_return(0x10000, &mm->context.attach_count);
	if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
	    cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
		__pmdp_idte_local(addr, pmdp);
	else
		__pmdp_idte(addr, pmdp);
	atomic_sub(0x10000, &mm->context.attach_count);
	return old;
}

static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
				    unsigned long addr, pmd_t *pmdp)
{
	int active, count;
	pmd_t old;

	old = *pmdp;
	if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
		return old;
	active = (mm == current->active_mm) ? 1 : 0;
	count = atomic_add_return(0x10000, &mm->context.attach_count);
	if ((count & 0xffff) <= active) {
		pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
		mm->context.flush_mm = 1;
	} else if (MACHINE_HAS_IDTE)
		__pmdp_idte(addr, pmdp);
	else
		__pmdp_csp(pmdp);
	atomic_sub(0x10000, &mm->context.attach_count);
	return old;
}

pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
		       pmd_t *pmdp, pmd_t new)
{
	pmd_t old;

	old = pmdp_flush_direct(mm, addr, pmdp);
	*pmdp = new;
	return old;
}
EXPORT_SYMBOL(pmdp_xchg_direct);

pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
		     pmd_t *pmdp, pmd_t new)
{
	pmd_t old;

	old = pmdp_flush_lazy(mm, addr, pmdp);
	*pmdp = new;
	return old;
}
EXPORT_SYMBOL(pmdp_xchg_lazy);

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
				pgtable_t pgtable)
{
	struct list_head *lh = (struct list_head *) pgtable;

	assert_spin_locked(pmd_lockptr(mm, pmdp));

	/* FIFO */
	if (!pmd_huge_pte(mm, pmdp))
		INIT_LIST_HEAD(lh);
	else
		list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
	pmd_huge_pte(mm, pmdp) = pgtable;
}

pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
	struct list_head *lh;
	pgtable_t pgtable;
	pte_t *ptep;

	assert_spin_locked(pmd_lockptr(mm, pmdp));

	/* FIFO */
	pgtable = pmd_huge_pte(mm, pmdp);
	lh = (struct list_head *) pgtable;
	if (list_empty(lh))
		pmd_huge_pte(mm, pmdp) = NULL;
	else {
		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
		list_del(lh);
	}
	ptep = (pte_t *) pgtable;
	pte_val(*ptep) = _PAGE_INVALID;
	ptep++;
	pte_val(*ptep) = _PAGE_INVALID;
	return pgtable;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#ifdef CONFIG_PGSTE
void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, pte_t entry)
{
	pgste_t pgste;

	/* the mm_has_pgste() check is done in set_pte_at() */
	pgste = pgste_get_lock(ptep);
	pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
	pgste_set_key(ptep, pgste, entry, mm);
	pgste = pgste_set_pte(ptep, pgste, entry);
	pgste_set_unlock(ptep, pgste);
}

void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
{
	pgste_t pgste;

	pgste = pgste_get_lock(ptep);
	pgste_val(pgste) |= PGSTE_IN_BIT;
	pgste_set_unlock(ptep, pgste);
}

static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry)
{
	if (!non_swap_entry(entry))
		dec_mm_counter(mm, MM_SWAPENTS);
	else if (is_migration_entry(entry)) {
		struct page *page = migration_entry_to_page(entry);

		dec_mm_counter(mm, mm_counter(page));
	}
	free_swap_and_cache(entry);
}

void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
		     pte_t *ptep, int reset)
{
	unsigned long pgstev;
	pgste_t pgste;
	pte_t pte;

	/* Zap unused and logically-zero pages */
	pgste = pgste_get_lock(ptep);
	pgstev = pgste_val(pgste);
	pte = *ptep;
	if (pte_swap(pte) &&
	    ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
	     (pgstev & _PGSTE_GPS_ZERO))) {
		ptep_zap_swap_entry(mm, pte_to_swp_entry(pte));
		pte_clear(mm, addr, ptep);
	}
	if (reset)
		pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK;
	pgste_set_unlock(ptep, pgste);
}

void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
{
	unsigned long ptev;
	pgste_t pgste;

	/* Clear storage key */
	pgste = pgste_get_lock(ptep);
	pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT |
			      PGSTE_GR_BIT | PGSTE_GC_BIT);
	ptev = pte_val(*ptep);
	if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
		page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1);
	pgste_set_unlock(ptep, pgste);
}

/*
 * Test and reset if a guest page is dirty
 */
bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
{
	spinlock_t *ptl;
	pgste_t pgste;
	pte_t *ptep;
	pte_t pte;
	bool dirty;

	ptep = get_locked_pte(mm, addr, &ptl);
	if (unlikely(!ptep))
		return false;

	pgste = pgste_get_lock(ptep);
	dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
	pgste_val(pgste) &= ~PGSTE_UC_BIT;
	pte = *ptep;
	if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
		pgste = pgste_ipte_notify(mm, addr, ptep, pgste);
		__ptep_ipte(addr, ptep);
		if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
			pte_val(pte) |= _PAGE_PROTECT;
		else
			pte_val(pte) |= _PAGE_INVALID;
		*ptep = pte;
	}
	pgste_set_unlock(ptep, pgste);

	spin_unlock(ptl);
	return dirty;
}
EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty);

int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
			  unsigned char key, bool nq)
{
	unsigned long keyul;
	spinlock_t *ptl;
	pgste_t old, new;
	pte_t *ptep;

	down_read(&mm->mmap_sem);
	ptep = get_locked_pte(mm, addr, &ptl);
	if (unlikely(!ptep)) {
		up_read(&mm->mmap_sem);
		return -EFAULT;
	}

	new = old = pgste_get_lock(ptep);
	pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT |
			    PGSTE_ACC_BITS | PGSTE_FP_BIT);
	keyul = (unsigned long) key;
	pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
	pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
	if (!(pte_val(*ptep) & _PAGE_INVALID)) {
		unsigned long address, bits, skey;

		address = pte_val(*ptep) & PAGE_MASK;
		skey = (unsigned long) page_get_storage_key(address);
		bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
		skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
		/* Set storage key ACC and FP */
		page_set_storage_key(address, skey, !nq);
		/* Merge host changed & referenced into pgste  */
		pgste_val(new) |= bits << 52;
	}
	/* changing the guest storage key is considered a change of the page */
	if ((pgste_val(new) ^ pgste_val(old)) &
	    (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
		pgste_val(new) |= PGSTE_UC_BIT;

	pgste_set_unlock(ptep, new);
	pte_unmap_unlock(ptep, ptl);
	up_read(&mm->mmap_sem);
	return 0;
}
EXPORT_SYMBOL(set_guest_storage_key);

unsigned char get_guest_storage_key(struct mm_struct *mm, unsigned long addr)
{
	unsigned char key;
	spinlock_t *ptl;
	pgste_t pgste;
	pte_t *ptep;

	down_read(&mm->mmap_sem);
	ptep = get_locked_pte(mm, addr, &ptl);
	if (unlikely(!ptep)) {
		up_read(&mm->mmap_sem);
		return -EFAULT;
	}
	pgste = pgste_get_lock(ptep);

	if (pte_val(*ptep) & _PAGE_INVALID) {
		key  = (pgste_val(pgste) & PGSTE_ACC_BITS) >> 56;
		key |= (pgste_val(pgste) & PGSTE_FP_BIT) >> 56;
		key |= (pgste_val(pgste) & PGSTE_GR_BIT) >> 48;
		key |= (pgste_val(pgste) & PGSTE_GC_BIT) >> 48;
	} else {
		key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK);

		/* Reflect guest's logical view, not physical */
		if (pgste_val(pgste) & PGSTE_GR_BIT)
			key |= _PAGE_REFERENCED;
		if (pgste_val(pgste) & PGSTE_GC_BIT)
			key |= _PAGE_CHANGED;
	}

	pgste_set_unlock(ptep, pgste);
	pte_unmap_unlock(ptep, ptl);
	up_read(&mm->mmap_sem);
	return key;
}
EXPORT_SYMBOL(get_guest_storage_key);
#endif
Commit	Line	Data
3610cce8	1	/*
a53c8fab	2	* Copyright IBM Corp. 2007, 2011
3610cce8 MS	3	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
	4	*/
	5
	6	#include <linux/sched.h>
	7	#include <linux/kernel.h>
	8	#include <linux/errno.h>
5a0e3ad6	9	#include <linux/gfp.h>
3610cce8 MS	10	#include <linux/mm.h>
	11	#include <linux/swap.h>
	12	#include <linux/smp.h>
3610cce8	13	#include <linux/spinlock.h>
80217147	14	#include <linux/rcupdate.h>
e5992f2e	15	#include <linux/slab.h>
b31288fa	16	#include <linux/swapops.h>
0b46e0a3	17	#include <linux/sysctl.h>
3ac8e380 DD	18	#include <linux/ksm.h>
3ac8e380 DD	19	#include <linux/mman.h>
3610cce8	20
3610cce8 MS	21	#include <asm/pgtable.h>
	22	#include <asm/pgalloc.h>
	23	#include <asm/tlb.h>
	24	#include <asm/tlbflush.h>
6252d702	25	#include <asm/mmu_context.h>
3610cce8	26
ebde765c MS	27	static inline pte_t ptep_flush_direct(struct mm_struct *mm,
	28	unsigned long addr, pte_t *ptep)
	29	{
	30	int active, count;
	31	pte_t old;
	32
	33	old = *ptep;
	34	if (unlikely(pte_val(old) & _PAGE_INVALID))
	35	return old;
	36	active = (mm == current->active_mm) ? 1 : 0;
	37	count = atomic_add_return(0x10000, &mm->context.attach_count);
	38	if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
	39	cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
	40	__ptep_ipte_local(addr, ptep);
	41	else
	42	__ptep_ipte(addr, ptep);
	43	atomic_sub(0x10000, &mm->context.attach_count);
	44	return old;
	45	}
	46
	47	static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
	48	unsigned long addr, pte_t *ptep)
	49	{
	50	int active, count;
	51	pte_t old;
	52
	53	old = *ptep;
	54	if (unlikely(pte_val(old) & _PAGE_INVALID))
	55	return old;
	56	active = (mm == current->active_mm) ? 1 : 0;
	57	count = atomic_add_return(0x10000, &mm->context.attach_count);
	58	if ((count & 0xffff) <= active) {
	59	pte_val(*ptep) \|= _PAGE_INVALID;
	60	mm->context.flush_mm = 1;
	61	} else
	62	__ptep_ipte(addr, ptep);
	63	atomic_sub(0x10000, &mm->context.attach_count);
	64	return old;
	65	}
	66
1e133ab2 MS	67	static inline pgste_t pgste_get_lock(pte_t *ptep)
	68	{
	69	unsigned long new = 0;
	70	#ifdef CONFIG_PGSTE
	71	unsigned long old;
	72
	73	preempt_disable();
	74	asm(
	75	" lg %0,%2\n"
	76	"0: lgr %1,%0\n"
	77	" nihh %0,0xff7f\n" /* clear PCL bit in old */
	78	" oihh %1,0x0080\n" /* set PCL bit in new */
	79	" csg %0,%1,%2\n"
	80	" jl 0b\n"
	81	: "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
	82	: "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
	83	#endif
	84	return __pgste(new);
	85	}
	86
	87	static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
	88	{
	89	#ifdef CONFIG_PGSTE
	90	asm(
	91	" nihh %1,0xff7f\n" /* clear PCL bit */
	92	" stg %1,%0\n"
	93	: "=Q" (ptep[PTRS_PER_PTE])
	94	: "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
	95	: "cc", "memory");
	96	preempt_enable();
	97	#endif
	98	}
	99
	100	static inline pgste_t pgste_get(pte_t *ptep)
	101	{
	102	unsigned long pgste = 0;
	103	#ifdef CONFIG_PGSTE
	104	pgste = (unsigned long )(ptep + PTRS_PER_PTE);
	105	#endif
	106	return __pgste(pgste);
	107	}
	108
	109	static inline void pgste_set(pte_t *ptep, pgste_t pgste)
	110	{
	111	#ifdef CONFIG_PGSTE
	112	(pgste_t )(ptep + PTRS_PER_PTE) = pgste;
	113	#endif
	114	}
	115
ebde765c MS	116	static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
	117	struct mm_struct *mm)
	118	{
	119	#ifdef CONFIG_PGSTE
	120	unsigned long address, bits, skey;
	121
	122	if (!mm_use_skey(mm) \|\| pte_val(pte) & _PAGE_INVALID)
	123	return pgste;
	124	address = pte_val(pte) & PAGE_MASK;
	125	skey = (unsigned long) page_get_storage_key(address);
	126	bits = skey & (_PAGE_CHANGED \| _PAGE_REFERENCED);
	127	/* Transfer page changed & referenced bit to guest bits in pgste */
	128	pgste_val(pgste) \|= bits << 48; /* GR bit & GC bit */
	129	/* Copy page access key and fetch protection bit to pgste */
	130	pgste_val(pgste) &= ~(PGSTE_ACC_BITS \| PGSTE_FP_BIT);
	131	pgste_val(pgste) \|= (skey & (_PAGE_ACC_BITS \| _PAGE_FP_BIT)) << 56;
	132	#endif
	133	return pgste;
	134
	135	}
	136
	137	static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
	138	struct mm_struct *mm)
	139	{
	140	#ifdef CONFIG_PGSTE
	141	unsigned long address;
	142	unsigned long nkey;
	143
	144	if (!mm_use_skey(mm) \|\| pte_val(entry) & _PAGE_INVALID)
	145	return;
	146	VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
	147	address = pte_val(entry) & PAGE_MASK;
	148	/*
	149	* Set page access key and fetch protection bit from pgste.
	150	* The guest C/R information is still in the PGSTE, set real
	151	* key C/R to 0.
	152	*/
	153	nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS \| PGSTE_FP_BIT)) >> 56;
	154	nkey \|= (pgste_val(pgste) & (PGSTE_GR_BIT \| PGSTE_GC_BIT)) >> 48;
	155	page_set_storage_key(address, nkey, 0);
	156	#endif
	157	}
	158
	159	static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
	160	{
	161	#ifdef CONFIG_PGSTE
	162	if ((pte_val(entry) & _PAGE_PRESENT) &&
	163	(pte_val(entry) & _PAGE_WRITE) &&
	164	!(pte_val(entry) & _PAGE_INVALID)) {
	165	if (!MACHINE_HAS_ESOP) {
	166	/*
	167	* Without enhanced suppression-on-protection force
	168	* the dirty bit on for all writable ptes.
	169	*/
	170	pte_val(entry) \|= _PAGE_DIRTY;
	171	pte_val(entry) &= ~_PAGE_PROTECT;
	172	}
	173	if (!(pte_val(entry) & _PAGE_PROTECT))
	174	/* This pte allows write access, set user-dirty */
	175	pgste_val(pgste) \|= PGSTE_UC_BIT;
	176	}
	177	#endif
	178	*ptep = entry;
	179	return pgste;
180	}
181
182	static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,
183	unsigned long addr,
184	pte_t *ptep, pgste_t pgste)
185	{
186	#ifdef CONFIG_PGSTE
187	if (pgste_val(pgste) & PGSTE_IN_BIT) {
188	pgste_val(pgste) &= ~PGSTE_IN_BIT;
1e133ab2	189	ptep_notify(mm, addr, ptep);
ebde765c MS	190	}
	191	#endif
	192	return pgste;
	193	}
	194
ebde765c MS	195	static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
	196	unsigned long addr, pte_t *ptep)
	197	{
	198	pgste_t pgste = __pgste(0);
	199
	200	if (mm_has_pgste(mm)) {
	201	pgste = pgste_get_lock(ptep);
	202	pgste = pgste_ipte_notify(mm, addr, ptep, pgste);
	203	}
	204	return pgste;
	205	}
	206
	207	static inline void ptep_xchg_commit(struct mm_struct *mm,
	208	unsigned long addr, pte_t *ptep,
	209	pgste_t pgste, pte_t old, pte_t new)
	210	{
	211	if (mm_has_pgste(mm)) {
	212	if (pte_val(old) & _PAGE_INVALID)
	213	pgste_set_key(ptep, pgste, new, mm);
	214	if (pte_val(new) & _PAGE_INVALID) {
	215	pgste = pgste_update_all(old, pgste, mm);
	216	if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
	217	_PGSTE_GPS_USAGE_UNUSED)
	218	pte_val(old) \|= _PAGE_UNUSED;
	219	}
	220	pgste = pgste_set_pte(ptep, pgste, new);
	221	pgste_set_unlock(ptep, pgste);
	222	} else {
	223	*ptep = new;
	224	}
	225	}
	226
	227	pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
	228	pte_t *ptep, pte_t new)
	229	{
	230	pgste_t pgste;
	231	pte_t old;
	232
	233	pgste = ptep_xchg_start(mm, addr, ptep);
	234	old = ptep_flush_direct(mm, addr, ptep);
	235	ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
	236	return old;
	237	}
	238	EXPORT_SYMBOL(ptep_xchg_direct);
	239
	240	pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
	241	pte_t *ptep, pte_t new)
	242	{
	243	pgste_t pgste;
	244	pte_t old;
	245
	246	pgste = ptep_xchg_start(mm, addr, ptep);
	247	old = ptep_flush_lazy(mm, addr, ptep);
	248	ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
	249	return old;
	250	}
	251	EXPORT_SYMBOL(ptep_xchg_lazy);
	252
	253	pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
	254	pte_t *ptep)
	255	{
	256	pgste_t pgste;
	257	pte_t old;
	258
259	pgste = ptep_xchg_start(mm, addr, ptep);
260	old = ptep_flush_lazy(mm, addr, ptep);
261	if (mm_has_pgste(mm)) {
262	pgste = pgste_update_all(old, pgste, mm);
263	pgste_set(ptep, pgste);
264	}
265	return old;
266	}
267	EXPORT_SYMBOL(ptep_modify_prot_start);
268
269	void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
270	pte_t *ptep, pte_t pte)
271	{
272	pgste_t pgste;
273
274	if (mm_has_pgste(mm)) {
275	pgste = pgste_get(ptep);
276	pgste_set_key(ptep, pgste, pte, mm);
277	pgste = pgste_set_pte(ptep, pgste, pte);
278	pgste_set_unlock(ptep, pgste);
279	} else {
280	*ptep = pte;
281	}
282	}
283	EXPORT_SYMBOL(ptep_modify_prot_commit);
284
227be799 MS	285	static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
	286	unsigned long addr, pmd_t *pmdp)
	287	{
	288	int active, count;
	289	pmd_t old;
	290
	291	old = *pmdp;
	292	if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
	293	return old;
	294	if (!MACHINE_HAS_IDTE) {
	295	__pmdp_csp(pmdp);
	296	return old;
	297	}
	298	active = (mm == current->active_mm) ? 1 : 0;
	299	count = atomic_add_return(0x10000, &mm->context.attach_count);
	300	if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
	301	cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
	302	__pmdp_idte_local(addr, pmdp);
	303	else
	304	__pmdp_idte(addr, pmdp);
	305	atomic_sub(0x10000, &mm->context.attach_count);
	306	return old;
	307	}
	308
	309	static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
	310	unsigned long addr, pmd_t *pmdp)
	311	{
	312	int active, count;
	313	pmd_t old;
	314
	315	old = *pmdp;
	316	if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
	317	return old;
	318	active = (mm == current->active_mm) ? 1 : 0;
	319	count = atomic_add_return(0x10000, &mm->context.attach_count);
	320	if ((count & 0xffff) <= active) {
	321	pmd_val(*pmdp) \|= _SEGMENT_ENTRY_INVALID;
	322	mm->context.flush_mm = 1;
	323	} else if (MACHINE_HAS_IDTE)
	324	__pmdp_idte(addr, pmdp);
	325	else
	326	__pmdp_csp(pmdp);
	327	atomic_sub(0x10000, &mm->context.attach_count);
	328	return old;
	329	}
	330
	331	pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
	332	pmd_t *pmdp, pmd_t new)
	333	{
	334	pmd_t old;
	335
	336	old = pmdp_flush_direct(mm, addr, pmdp);
	337	*pmdp = new;
	338	return old;
	339	}
	340	EXPORT_SYMBOL(pmdp_xchg_direct);
	341
	342	pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
	343	pmd_t *pmdp, pmd_t new)
	344	{
	345	pmd_t old;
	346
	347	old = pmdp_flush_lazy(mm, addr, pmdp);
	348	*pmdp = new;
349	return old;
350	}
351	EXPORT_SYMBOL(pmdp_xchg_lazy);
352
75077afb	353	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6b0b50b0 AK	354	void pgtable_trans_huge_deposit(struct mm_struct mm, pmd_t pmdp,
6b0b50b0 AK	355	pgtable_t pgtable)
9501d09f GS	356	{
	357	struct list_head lh = (struct list_head ) pgtable;
	358
ec66ad66	359	assert_spin_locked(pmd_lockptr(mm, pmdp));
9501d09f GS	360
9501d09f GS	361	/* FIFO */
c389a250	362	if (!pmd_huge_pte(mm, pmdp))
9501d09f GS	363	INIT_LIST_HEAD(lh);
9501d09f GS	364	else
c389a250 KS	365	list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
c389a250 KS	366	pmd_huge_pte(mm, pmdp) = pgtable;
9501d09f GS	367	}
9501d09f GS	368
6b0b50b0	369	pgtable_t pgtable_trans_huge_withdraw(struct mm_struct mm, pmd_t pmdp)
9501d09f GS	370	{
	371	struct list_head *lh;
	372	pgtable_t pgtable;
	373	pte_t *ptep;
	374
ec66ad66	375	assert_spin_locked(pmd_lockptr(mm, pmdp));
9501d09f GS	376
9501d09f GS	377	/* FIFO */
c389a250	378	pgtable = pmd_huge_pte(mm, pmdp);
9501d09f GS	379	lh = (struct list_head *) pgtable;
9501d09f GS	380	if (list_empty(lh))
c389a250	381	pmd_huge_pte(mm, pmdp) = NULL;
9501d09f	382	else {
c389a250	383	pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
9501d09f GS	384	list_del(lh);
	385	}
	386	ptep = (pte_t *) pgtable;
e5098611	387	pte_val(*ptep) = _PAGE_INVALID;
9501d09f	388	ptep++;
e5098611	389	pte_val(*ptep) = _PAGE_INVALID;
9501d09f GS	390	return pgtable;
9501d09f GS	391	}
75077afb	392	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1e133ab2 MS	393
	394	#ifdef CONFIG_PGSTE
	395	void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
	396	pte_t *ptep, pte_t entry)
	397	{
	398	pgste_t pgste;
	399
	400	/* the mm_has_pgste() check is done in set_pte_at() */
	401	pgste = pgste_get_lock(ptep);
	402	pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
	403	pgste_set_key(ptep, pgste, entry, mm);
	404	pgste = pgste_set_pte(ptep, pgste, entry);
	405	pgste_set_unlock(ptep, pgste);
	406	}
	407
	408	void ptep_set_notify(struct mm_struct mm, unsigned long addr, pte_t ptep)
	409	{
	410	pgste_t pgste;
	411
	412	pgste = pgste_get_lock(ptep);
	413	pgste_val(pgste) \|= PGSTE_IN_BIT;
	414	pgste_set_unlock(ptep, pgste);
	415	}
	416
	417	static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry)
	418	{
	419	if (!non_swap_entry(entry))
	420	dec_mm_counter(mm, MM_SWAPENTS);
	421	else if (is_migration_entry(entry)) {
	422	struct page *page = migration_entry_to_page(entry);
	423
	424	dec_mm_counter(mm, mm_counter(page));
	425	}
	426	free_swap_and_cache(entry);
	427	}
	428
	429	void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
	430	pte_t *ptep, int reset)
	431	{
	432	unsigned long pgstev;
	433	pgste_t pgste;
	434	pte_t pte;
	435
	436	/* Zap unused and logically-zero pages */
	437	pgste = pgste_get_lock(ptep);
	438	pgstev = pgste_val(pgste);
	439	pte = *ptep;
	440	if (pte_swap(pte) &&
	441	((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED \|\|
	442	(pgstev & _PGSTE_GPS_ZERO))) {
	443	ptep_zap_swap_entry(mm, pte_to_swp_entry(pte));
	444	pte_clear(mm, addr, ptep);
	445	}
	446	if (reset)
	447	pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK;
	448	pgste_set_unlock(ptep, pgste);
	449	}
	450
	451	void ptep_zap_key(struct mm_struct mm, unsigned long addr, pte_t ptep)
	452	{
	453	unsigned long ptev;
	454	pgste_t pgste;
	455
	456	/* Clear storage key */
457	pgste = pgste_get_lock(ptep);
458	pgste_val(pgste) &= ~(PGSTE_ACC_BITS \| PGSTE_FP_BIT \|
459	PGSTE_GR_BIT \| PGSTE_GC_BIT);
460	ptev = pte_val(*ptep);
461	if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
462	page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1);
463	pgste_set_unlock(ptep, pgste);
464	}
465
466	/*
467	* Test and reset if a guest page is dirty
468	*/
469	bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
470	{
471	spinlock_t *ptl;
472	pgste_t pgste;
473	pte_t *ptep;
474	pte_t pte;
475	bool dirty;
476
477	ptep = get_locked_pte(mm, addr, &ptl);
478	if (unlikely(!ptep))
479	return false;
480
481	pgste = pgste_get_lock(ptep);
482	dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
483	pgste_val(pgste) &= ~PGSTE_UC_BIT;
484	pte = *ptep;
485	if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
486	pgste = pgste_ipte_notify(mm, addr, ptep, pgste);
487	__ptep_ipte(addr, ptep);
488	if (MACHINE_HAS_ESOP \|\| !(pte_val(pte) & _PAGE_WRITE))
489	pte_val(pte) \|= _PAGE_PROTECT;
490	else
491	pte_val(pte) \|= _PAGE_INVALID;
492	*ptep = pte;
493	}
494	pgste_set_unlock(ptep, pgste);
495
496	spin_unlock(ptl);
497	return dirty;
498	}
499	EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty);
500
501	int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
502	unsigned char key, bool nq)
503	{
504	unsigned long keyul;
505	spinlock_t *ptl;
506	pgste_t old, new;
507	pte_t *ptep;
508
509	down_read(&mm->mmap_sem);
510	ptep = get_locked_pte(mm, addr, &ptl);
511	if (unlikely(!ptep)) {
512	up_read(&mm->mmap_sem);
513	return -EFAULT;
514	}
515
516	new = old = pgste_get_lock(ptep);
517	pgste_val(new) &= ~(PGSTE_GR_BIT \| PGSTE_GC_BIT \|
518	PGSTE_ACC_BITS \| PGSTE_FP_BIT);
519	keyul = (unsigned long) key;
520	pgste_val(new) \|= (keyul & (_PAGE_CHANGED \| _PAGE_REFERENCED)) << 48;
521	pgste_val(new) \|= (keyul & (_PAGE_ACC_BITS \| _PAGE_FP_BIT)) << 56;
522	if (!(pte_val(*ptep) & _PAGE_INVALID)) {
523	unsigned long address, bits, skey;
524
525	address = pte_val(*ptep) & PAGE_MASK;
526	skey = (unsigned long) page_get_storage_key(address);
527	bits = skey & (_PAGE_CHANGED \| _PAGE_REFERENCED);
528	skey = key & (_PAGE_ACC_BITS \| _PAGE_FP_BIT);
529	/* Set storage key ACC and FP */
530	page_set_storage_key(address, skey, !nq);
531	/* Merge host changed & referenced into pgste */
532	pgste_val(new) \|= bits << 52;
533	}
534	/* changing the guest storage key is considered a change of the page */
535	if ((pgste_val(new) ^ pgste_val(old)) &
536	(PGSTE_ACC_BITS \| PGSTE_FP_BIT \| PGSTE_GR_BIT \| PGSTE_GC_BIT))
537	pgste_val(new) \|= PGSTE_UC_BIT;
538
539	pgste_set_unlock(ptep, new);
540	pte_unmap_unlock(ptep, ptl);
541	up_read(&mm->mmap_sem);
542	return 0;
543	}
544	EXPORT_SYMBOL(set_guest_storage_key);
545
546	unsigned char get_guest_storage_key(struct mm_struct *mm, unsigned long addr)
547	{
548	unsigned char key;
549	spinlock_t *ptl;
550	pgste_t pgste;
551	pte_t *ptep;
552
553	down_read(&mm->mmap_sem);
554	ptep = get_locked_pte(mm, addr, &ptl);
555	if (unlikely(!ptep)) {
556	up_read(&mm->mmap_sem);
557	return -EFAULT;
558	}
559	pgste = pgste_get_lock(ptep);
560
561	if (pte_val(*ptep) & _PAGE_INVALID) {
562	key = (pgste_val(pgste) & PGSTE_ACC_BITS) >> 56;
563	key \|= (pgste_val(pgste) & PGSTE_FP_BIT) >> 56;
564	key \|= (pgste_val(pgste) & PGSTE_GR_BIT) >> 48;
565	key \|= (pgste_val(pgste) & PGSTE_GC_BIT) >> 48;
566	} else {
567	key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK);
568
569	/* Reflect guest's logical view, not physical */
570	if (pgste_val(pgste) & PGSTE_GR_BIT)
571	key \|= _PAGE_REFERENCED;
572	if (pgste_val(pgste) & PGSTE_GC_BIT)
573	key \|= _PAGE_CHANGED;
574	}
575
576	pgste_set_unlock(ptep, pgste);
577	pte_unmap_unlock(ptep, ptl);
578	up_read(&mm->mmap_sem);
579	return key;
580	}
581	EXPORT_SYMBOL(get_guest_storage_key);
582	#endif