[deliverable/linux.git] / arch / tile / mm / homecache.c

/*
 * Copyright 2010 Tilera Corporation. All Rights Reserved.
 *
 *   This program is free software; you can redistribute it and/or
 *   modify it under the terms of the GNU General Public License
 *   as published by the Free Software Foundation, version 2.
 *
 *   This program is distributed in the hope that it will be useful, but
 *   WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 *   NON INFRINGEMENT.  See the GNU General Public License for
 *   more details.
 *
 * This code maintains the "home" for each page in the system.
 */

#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/bootmem.h>
#include <linux/rmap.h>
#include <linux/pagemap.h>
#include <linux/mutex.h>
#include <linux/interrupt.h>
#include <linux/sysctl.h>
#include <linux/pagevec.h>
#include <linux/ptrace.h>
#include <linux/timex.h>
#include <linux/cache.h>
#include <linux/smp.h>
#include <linux/module.h>

#include <asm/page.h>
#include <asm/sections.h>
#include <asm/tlbflush.h>
#include <asm/pgalloc.h>
#include <asm/homecache.h>

#include <arch/sim.h>

#include "migrate.h"


#if CHIP_HAS_COHERENT_LOCAL_CACHE()

/*
 * The noallocl2 option suppresses all use of the L2 cache to cache
 * locally from a remote home.  There's no point in using it if we
 * don't have coherent local caching, though.
 */
static int __write_once noallocl2;
static int __init set_noallocl2(char *str)
{
	noallocl2 = 1;
	return 0;
}
early_param("noallocl2", set_noallocl2);

#else

#define noallocl2 0

#endif

/* Provide no-op versions of these routines to keep flush_remote() cleaner. */
#define mark_caches_evicted_start() 0
#define mark_caches_evicted_finish(mask, timestamp) do {} while (0)


/*
 * Update the irq_stat for cpus that we are going to interrupt
 * with TLB or cache flushes.  Also handle removing dataplane cpus
 * from the TLB flush set, and setting dataplane_tlb_state instead.
 */
static void hv_flush_update(const struct cpumask *cache_cpumask,
			    struct cpumask *tlb_cpumask,
			    unsigned long tlb_va, unsigned long tlb_length,
			    HV_Remote_ASID *asids, int asidcount)
{
	struct cpumask mask;
	int i, cpu;

	cpumask_clear(&mask);
	if (cache_cpumask)
		cpumask_or(&mask, &mask, cache_cpumask);
	if (tlb_cpumask && tlb_length) {
		cpumask_or(&mask, &mask, tlb_cpumask);
	}

	for (i = 0; i < asidcount; ++i)
		cpumask_set_cpu(asids[i].y * smp_width + asids[i].x, &mask);

	/*
	 * Don't bother to update atomically; losing a count
	 * here is not that critical.
	 */
	for_each_cpu(cpu, &mask)
		++per_cpu(irq_stat, cpu).irq_hv_flush_count;
}

/*
 * This wrapper function around hv_flush_remote() does several things:
 *
 *  - Provides a return value error-checking panic path, since
 *    there's never any good reason for hv_flush_remote() to fail.
 *  - Accepts a 32-bit PFN rather than a 64-bit PA, which generally
 *    is the type that Linux wants to pass around anyway.
 *  - Centralizes the mark_caches_evicted() handling.
 *  - Canonicalizes that lengths of zero make cpumasks NULL.
 *  - Handles deferring TLB flushes for dataplane tiles.
 *  - Tracks remote interrupts in the per-cpu irq_cpustat_t.
 *
 * Note that we have to wait until the cache flush completes before
 * updating the per-cpu last_cache_flush word, since otherwise another
 * concurrent flush can race, conclude the flush has already
 * completed, and start to use the page while it's still dirty
 * remotely (running concurrently with the actual evict, presumably).
 */
void flush_remote(unsigned long cache_pfn, unsigned long cache_control,
		  const struct cpumask *cache_cpumask_orig,
		  HV_VirtAddr tlb_va, unsigned long tlb_length,
		  unsigned long tlb_pgsize,
		  const struct cpumask *tlb_cpumask_orig,
		  HV_Remote_ASID *asids, int asidcount)
{
	int rc;
	int timestamp = 0;  /* happy compiler */
	struct cpumask cache_cpumask_copy, tlb_cpumask_copy;
	struct cpumask *cache_cpumask, *tlb_cpumask;
	HV_PhysAddr cache_pa;
	char cache_buf[NR_CPUS*5], tlb_buf[NR_CPUS*5];

	mb();   /* provided just to simplify "magic hypervisor" mode */

	/*
	 * Canonicalize and copy the cpumasks.
	 */
	if (cache_cpumask_orig && cache_control) {
		cpumask_copy(&cache_cpumask_copy, cache_cpumask_orig);
		cache_cpumask = &cache_cpumask_copy;
	} else {
		cpumask_clear(&cache_cpumask_copy);
		cache_cpumask = NULL;
	}
	if (cache_cpumask == NULL)
		cache_control = 0;
	if (tlb_cpumask_orig && tlb_length) {
		cpumask_copy(&tlb_cpumask_copy, tlb_cpumask_orig);
		tlb_cpumask = &tlb_cpumask_copy;
	} else {
		cpumask_clear(&tlb_cpumask_copy);
		tlb_cpumask = NULL;
	}

	hv_flush_update(cache_cpumask, tlb_cpumask, tlb_va, tlb_length,
			asids, asidcount);
	cache_pa = (HV_PhysAddr)cache_pfn << PAGE_SHIFT;
	if (cache_control & HV_FLUSH_EVICT_L2)
		timestamp = mark_caches_evicted_start();
	rc = hv_flush_remote(cache_pa, cache_control,
			     cpumask_bits(cache_cpumask),
			     tlb_va, tlb_length, tlb_pgsize,
			     cpumask_bits(tlb_cpumask),
			     asids, asidcount);
	if (cache_control & HV_FLUSH_EVICT_L2)
		mark_caches_evicted_finish(cache_cpumask, timestamp);
	if (rc == 0)
		return;
	cpumask_scnprintf(cache_buf, sizeof(cache_buf), &cache_cpumask_copy);
	cpumask_scnprintf(tlb_buf, sizeof(tlb_buf), &tlb_cpumask_copy);

	pr_err("hv_flush_remote(%#llx, %#lx, %p [%s],"
	       " %#lx, %#lx, %#lx, %p [%s], %p, %d) = %d\n",
	       cache_pa, cache_control, cache_cpumask, cache_buf,
	       (unsigned long)tlb_va, tlb_length, tlb_pgsize,
	       tlb_cpumask, tlb_buf,
	       asids, asidcount, rc);
	panic("Unsafe to continue.");
}

void homecache_evict(const struct cpumask *mask)
{
	flush_remote(0, HV_FLUSH_EVICT_L2, mask, 0, 0, 0, NULL, NULL, 0);
}

/* Return a mask of the cpus whose caches currently own these pages. */
static void homecache_mask(struct page *page, int pages,
			   struct cpumask *home_mask)
{
	int i;
	cpumask_clear(home_mask);
	for (i = 0; i < pages; ++i) {
		int home = page_home(&page[i]);
		if (home == PAGE_HOME_IMMUTABLE ||
		    home == PAGE_HOME_INCOHERENT) {
			cpumask_copy(home_mask, cpu_possible_mask);
			return;
		}
#if CHIP_HAS_CBOX_HOME_MAP()
		if (home == PAGE_HOME_HASH) {
			cpumask_or(home_mask, home_mask, &hash_for_home_map);
			continue;
		}
#endif
		if (home == PAGE_HOME_UNCACHED)
			continue;
		BUG_ON(home < 0 || home >= NR_CPUS);
		cpumask_set_cpu(home, home_mask);
	}
}

/*
 * Return the passed length, or zero if it's long enough that we
 * believe we should evict the whole L2 cache.
 */
static unsigned long cache_flush_length(unsigned long length)
{
	return (length >= CHIP_L2_CACHE_SIZE()) ? HV_FLUSH_EVICT_L2 : length;
}

/* Flush a page out of whatever cache(s) it is in. */
void homecache_flush_cache(struct page *page, int order)
{
	int pages = 1 << order;
	int length = cache_flush_length(pages * PAGE_SIZE);
	unsigned long pfn = page_to_pfn(page);
	struct cpumask home_mask;

	homecache_mask(page, pages, &home_mask);
	flush_remote(pfn, length, &home_mask, 0, 0, 0, NULL, NULL, 0);
	sim_validate_lines_evicted(PFN_PHYS(pfn), pages * PAGE_SIZE);
}


/* Report the home corresponding to a given PTE. */
static int pte_to_home(pte_t pte)
{
	if (hv_pte_get_nc(pte))
		return PAGE_HOME_IMMUTABLE;
	switch (hv_pte_get_mode(pte)) {
	case HV_PTE_MODE_CACHE_TILE_L3:
		return get_remote_cache_cpu(pte);
	case HV_PTE_MODE_CACHE_NO_L3:
		return PAGE_HOME_INCOHERENT;
	case HV_PTE_MODE_UNCACHED:
		return PAGE_HOME_UNCACHED;
#if CHIP_HAS_CBOX_HOME_MAP()
	case HV_PTE_MODE_CACHE_HASH_L3:
		return PAGE_HOME_HASH;
#endif
	}
	panic("Bad PTE %#llx\n", pte.val);
}

/* Update the home of a PTE if necessary (can also be used for a pgprot_t). */
pte_t pte_set_home(pte_t pte, int home)
{
	/* Check for non-linear file mapping "PTEs" and pass them through. */
	if (pte_file(pte))
		return pte;

#if CHIP_HAS_MMIO()
	/* Check for MMIO mappings and pass them through. */
	if (hv_pte_get_mode(pte) == HV_PTE_MODE_MMIO)
		return pte;
#endif


	/*
	 * Only immutable pages get NC mappings.  If we have a
	 * non-coherent PTE, but the underlying page is not
	 * immutable, it's likely the result of a forced
	 * caching setting running up against ptrace setting
	 * the page to be writable underneath.  In this case,
	 * just keep the PTE coherent.
	 */
	if (hv_pte_get_nc(pte) && home != PAGE_HOME_IMMUTABLE) {
		pte = hv_pte_clear_nc(pte);
		pr_err("non-immutable page incoherently referenced: %#llx\n",
		       pte.val);
	}

	switch (home) {

	case PAGE_HOME_UNCACHED:
		pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
		break;

	case PAGE_HOME_INCOHERENT:
		pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
		break;

	case PAGE_HOME_IMMUTABLE:
		/*
		 * We could home this page anywhere, since it's immutable,
		 * but by default just home it to follow "hash_default".
		 */
		BUG_ON(hv_pte_get_writable(pte));
		if (pte_get_forcecache(pte)) {
			/* Upgrade "force any cpu" to "No L3" for immutable. */
			if (hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_TILE_L3
			    && pte_get_anyhome(pte)) {
				pte = hv_pte_set_mode(pte,
						      HV_PTE_MODE_CACHE_NO_L3);
			}
		} else
#if CHIP_HAS_CBOX_HOME_MAP()
		if (hash_default)
			pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
		else
#endif
			pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
		pte = hv_pte_set_nc(pte);
		break;

#if CHIP_HAS_CBOX_HOME_MAP()
	case PAGE_HOME_HASH:
		pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
		break;
#endif

	default:
		BUG_ON(home < 0 || home >= NR_CPUS ||
		       !cpu_is_valid_lotar(home));
		pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_TILE_L3);
		pte = set_remote_cache_cpu(pte, home);
		break;
	}

#if CHIP_HAS_NC_AND_NOALLOC_BITS()
	if (noallocl2)
		pte = hv_pte_set_no_alloc_l2(pte);

	/* Simplify "no local and no l3" to "uncached" */
	if (hv_pte_get_no_alloc_l2(pte) && hv_pte_get_no_alloc_l1(pte) &&
	    hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_NO_L3) {
		pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
	}
#endif

	/* Checking this case here gives a better panic than from the hv. */
	BUG_ON(hv_pte_get_mode(pte) == 0);

	return pte;
}
EXPORT_SYMBOL(pte_set_home);

/*
 * The routines in this section are the "static" versions of the normal
 * dynamic homecaching routines; they just set the home cache
 * of a kernel page once, and require a full-chip cache/TLB flush,
 * so they're not suitable for anything but infrequent use.
 */

#if CHIP_HAS_CBOX_HOME_MAP()
static inline int initial_page_home(void) { return PAGE_HOME_HASH; }
#else
static inline int initial_page_home(void) { return 0; }
#endif

int page_home(struct page *page)
{
	if (PageHighMem(page)) {
		return initial_page_home();
	} else {
		unsigned long kva = (unsigned long)page_address(page);
		return pte_to_home(*virt_to_pte(NULL, kva));
	}
}

void homecache_change_page_home(struct page *page, int order, int home)
{
	int i, pages = (1 << order);
	unsigned long kva;

	BUG_ON(PageHighMem(page));
	BUG_ON(page_count(page) > 1);
	BUG_ON(page_mapcount(page) != 0);
	kva = (unsigned long) page_address(page);
	flush_remote(0, HV_FLUSH_EVICT_L2, &cpu_cacheable_map,
		     kva, pages * PAGE_SIZE, PAGE_SIZE, cpu_online_mask,
		     NULL, 0);

	for (i = 0; i < pages; ++i, kva += PAGE_SIZE) {
		pte_t *ptep = virt_to_pte(NULL, kva);
		pte_t pteval = *ptep;
		BUG_ON(!pte_present(pteval) || pte_huge(pteval));
		*ptep = pte_set_home(pteval, home);
	}
}

struct page *homecache_alloc_pages(gfp_t gfp_mask,
				   unsigned int order, int home)
{
	struct page *page;
	BUG_ON(gfp_mask & __GFP_HIGHMEM);   /* must be lowmem */
	page = alloc_pages(gfp_mask, order);
	if (page)
		homecache_change_page_home(page, order, home);
	return page;
}
EXPORT_SYMBOL(homecache_alloc_pages);

struct page *homecache_alloc_pages_node(int nid, gfp_t gfp_mask,
					unsigned int order, int home)
{
	struct page *page;
	BUG_ON(gfp_mask & __GFP_HIGHMEM);   /* must be lowmem */
	page = alloc_pages_node(nid, gfp_mask, order);
	if (page)
		homecache_change_page_home(page, order, home);
	return page;
}

void homecache_free_pages(unsigned long addr, unsigned int order)
{
	struct page *page;

	if (addr == 0)
		return;

	VM_BUG_ON(!virt_addr_valid((void *)addr));
	page = virt_to_page((void *)addr);
	if (put_page_testzero(page)) {
		int pages = (1 << order);
		homecache_change_page_home(page, order, initial_page_home());
		while (pages--)
			__free_page(page++);
	}
}
Commit	Line	Data
867e359b CM	1	/*
	2	* Copyright 2010 Tilera Corporation. All Rights Reserved.
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation, version 2.
	7	*
	8	* This program is distributed in the hope that it will be useful, but
	9	* WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
	11	* NON INFRINGEMENT. See the GNU General Public License for
	12	* more details.
	13	*
	14	* This code maintains the "home" for each page in the system.
	15	*/
	16
	17	#include <linux/kernel.h>
	18	#include <linux/mm.h>
	19	#include <linux/spinlock.h>
	20	#include <linux/list.h>
	21	#include <linux/bootmem.h>
	22	#include <linux/rmap.h>
	23	#include <linux/pagemap.h>
	24	#include <linux/mutex.h>
	25	#include <linux/interrupt.h>
	26	#include <linux/sysctl.h>
	27	#include <linux/pagevec.h>
	28	#include <linux/ptrace.h>
	29	#include <linux/timex.h>
	30	#include <linux/cache.h>
	31	#include <linux/smp.h>
c745a8a1	32	#include <linux/module.h>
867e359b CM	33
	34	#include <asm/page.h>
	35	#include <asm/sections.h>
	36	#include <asm/tlbflush.h>
	37	#include <asm/pgalloc.h>
	38	#include <asm/homecache.h>
	39
bf65e440 CM	40	#include <arch/sim.h>
bf65e440 CM	41
867e359b CM	42	#include "migrate.h"
	43
	44
	45	#if CHIP_HAS_COHERENT_LOCAL_CACHE()
	46
	47	/*
	48	* The noallocl2 option suppresses all use of the L2 cache to cache
	49	* locally from a remote home. There's no point in using it if we
	50	* don't have coherent local caching, though.
	51	*/
0707ad30	52	static int __write_once noallocl2;
867e359b CM	53	static int __init set_noallocl2(char *str)
	54	{
	55	noallocl2 = 1;
	56	return 0;
	57	}
	58	early_param("noallocl2", set_noallocl2);
	59
	60	#else
	61
	62	#define noallocl2 0
	63
	64	#endif
	65
867e359b CM	66	/* Provide no-op versions of these routines to keep flush_remote() cleaner. */
	67	#define mark_caches_evicted_start() 0
	68	#define mark_caches_evicted_finish(mask, timestamp) do {} while (0)
	69
	70
867e359b CM	71	/*
	72	* Update the irq_stat for cpus that we are going to interrupt
	73	* with TLB or cache flushes. Also handle removing dataplane cpus
	74	* from the TLB flush set, and setting dataplane_tlb_state instead.
	75	*/
	76	static void hv_flush_update(const struct cpumask *cache_cpumask,
	77	struct cpumask *tlb_cpumask,
	78	unsigned long tlb_va, unsigned long tlb_length,
	79	HV_Remote_ASID *asids, int asidcount)
	80	{
	81	struct cpumask mask;
	82	int i, cpu;
	83
	84	cpumask_clear(&mask);
	85	if (cache_cpumask)
	86	cpumask_or(&mask, &mask, cache_cpumask);
	87	if (tlb_cpumask && tlb_length) {
	88	cpumask_or(&mask, &mask, tlb_cpumask);
	89	}
	90
	91	for (i = 0; i < asidcount; ++i)
	92	cpumask_set_cpu(asids[i].y * smp_width + asids[i].x, &mask);
	93
	94	/*
	95	* Don't bother to update atomically; losing a count
	96	* here is not that critical.
	97	*/
	98	for_each_cpu(cpu, &mask)
	99	++per_cpu(irq_stat, cpu).irq_hv_flush_count;
	100	}
	101
	102	/*
	103	* This wrapper function around hv_flush_remote() does several things:
	104	*
	105	* - Provides a return value error-checking panic path, since
	106	* there's never any good reason for hv_flush_remote() to fail.
	107	* - Accepts a 32-bit PFN rather than a 64-bit PA, which generally
	108	* is the type that Linux wants to pass around anyway.
	109	* - Centralizes the mark_caches_evicted() handling.
	110	* - Canonicalizes that lengths of zero make cpumasks NULL.
	111	* - Handles deferring TLB flushes for dataplane tiles.
	112	* - Tracks remote interrupts in the per-cpu irq_cpustat_t.
	113	*
	114	* Note that we have to wait until the cache flush completes before
	115	* updating the per-cpu last_cache_flush word, since otherwise another
	116	* concurrent flush can race, conclude the flush has already
	117	* completed, and start to use the page while it's still dirty
	118	* remotely (running concurrently with the actual evict, presumably).
	119	*/
	120	void flush_remote(unsigned long cache_pfn, unsigned long cache_control,
	121	const struct cpumask *cache_cpumask_orig,
	122	HV_VirtAddr tlb_va, unsigned long tlb_length,
	123	unsigned long tlb_pgsize,
	124	const struct cpumask *tlb_cpumask_orig,
	125	HV_Remote_ASID *asids, int asidcount)
	126	{
	127	int rc;
	128	int timestamp = 0; /* happy compiler */
	129	struct cpumask cache_cpumask_copy, tlb_cpumask_copy;
	130	struct cpumask cache_cpumask, tlb_cpumask;
	131	HV_PhysAddr cache_pa;
	132	char cache_buf[NR_CPUS5], tlb_buf[NR_CPUS5];
	133
	134	mb(); /* provided just to simplify "magic hypervisor" mode */
135
136	/*
137	* Canonicalize and copy the cpumasks.
138	*/
139	if (cache_cpumask_orig && cache_control) {
140	cpumask_copy(&cache_cpumask_copy, cache_cpumask_orig);
141	cache_cpumask = &cache_cpumask_copy;
142	} else {
143	cpumask_clear(&cache_cpumask_copy);
144	cache_cpumask = NULL;
145	}
146	if (cache_cpumask == NULL)
147	cache_control = 0;
148	if (tlb_cpumask_orig && tlb_length) {
149	cpumask_copy(&tlb_cpumask_copy, tlb_cpumask_orig);
150	tlb_cpumask = &tlb_cpumask_copy;
151	} else {
152	cpumask_clear(&tlb_cpumask_copy);
153	tlb_cpumask = NULL;
154	}
155
156	hv_flush_update(cache_cpumask, tlb_cpumask, tlb_va, tlb_length,
157	asids, asidcount);
158	cache_pa = (HV_PhysAddr)cache_pfn << PAGE_SHIFT;
159	if (cache_control & HV_FLUSH_EVICT_L2)
160	timestamp = mark_caches_evicted_start();
161	rc = hv_flush_remote(cache_pa, cache_control,
162	cpumask_bits(cache_cpumask),
163	tlb_va, tlb_length, tlb_pgsize,
164	cpumask_bits(tlb_cpumask),
165	asids, asidcount);
166	if (cache_control & HV_FLUSH_EVICT_L2)
167	mark_caches_evicted_finish(cache_cpumask, timestamp);
168	if (rc == 0)
169	return;
170	cpumask_scnprintf(cache_buf, sizeof(cache_buf), &cache_cpumask_copy);
171	cpumask_scnprintf(tlb_buf, sizeof(tlb_buf), &tlb_cpumask_copy);
172
0707ad30	173	pr_err("hv_flush_remote(%#llx, %#lx, %p [%s],"
867e359b CM	174	" %#lx, %#lx, %#lx, %p [%s], %p, %d) = %d\n",
	175	cache_pa, cache_control, cache_cpumask, cache_buf,
	176	(unsigned long)tlb_va, tlb_length, tlb_pgsize,
	177	tlb_cpumask, tlb_buf,
	178	asids, asidcount, rc);
867e359b CM	179	panic("Unsafe to continue.");
	180	}
	181
	182	void homecache_evict(const struct cpumask *mask)
	183	{
	184	flush_remote(0, HV_FLUSH_EVICT_L2, mask, 0, 0, 0, NULL, NULL, 0);
	185	}
	186
	187	/* Return a mask of the cpus whose caches currently own these pages. */
	188	static void homecache_mask(struct page *page, int pages,
	189	struct cpumask *home_mask)
	190	{
	191	int i;
	192	cpumask_clear(home_mask);
	193	for (i = 0; i < pages; ++i) {
	194	int home = page_home(&page[i]);
	195	if (home == PAGE_HOME_IMMUTABLE \|\|
	196	home == PAGE_HOME_INCOHERENT) {
	197	cpumask_copy(home_mask, cpu_possible_mask);
	198	return;
	199	}
	200	#if CHIP_HAS_CBOX_HOME_MAP()
	201	if (home == PAGE_HOME_HASH) {
	202	cpumask_or(home_mask, home_mask, &hash_for_home_map);
	203	continue;
	204	}
	205	#endif
	206	if (home == PAGE_HOME_UNCACHED)
	207	continue;
	208	BUG_ON(home < 0 \|\| home >= NR_CPUS);
	209	cpumask_set_cpu(home, home_mask);
	210	}
	211	}
	212
	213	/*
	214	* Return the passed length, or zero if it's long enough that we
	215	* believe we should evict the whole L2 cache.
	216	*/
	217	static unsigned long cache_flush_length(unsigned long length)
	218	{
	219	return (length >= CHIP_L2_CACHE_SIZE()) ? HV_FLUSH_EVICT_L2 : length;
	220	}
	221
867e359b CM	222	/* Flush a page out of whatever cache(s) it is in. */
	223	void homecache_flush_cache(struct page *page, int order)
	224	{
	225	int pages = 1 << order;
	226	int length = cache_flush_length(pages * PAGE_SIZE);
	227	unsigned long pfn = page_to_pfn(page);
	228	struct cpumask home_mask;
	229
	230	homecache_mask(page, pages, &home_mask);
	231	flush_remote(pfn, length, &home_mask, 0, 0, 0, NULL, NULL, 0);
bf65e440	232	sim_validate_lines_evicted(PFN_PHYS(pfn), pages * PAGE_SIZE);
867e359b CM	233	}
	234
	235
	236	/* Report the home corresponding to a given PTE. */
	237	static int pte_to_home(pte_t pte)
	238	{
	239	if (hv_pte_get_nc(pte))
	240	return PAGE_HOME_IMMUTABLE;
	241	switch (hv_pte_get_mode(pte)) {
	242	case HV_PTE_MODE_CACHE_TILE_L3:
	243	return get_remote_cache_cpu(pte);
	244	case HV_PTE_MODE_CACHE_NO_L3:
	245	return PAGE_HOME_INCOHERENT;
	246	case HV_PTE_MODE_UNCACHED:
	247	return PAGE_HOME_UNCACHED;
	248	#if CHIP_HAS_CBOX_HOME_MAP()
	249	case HV_PTE_MODE_CACHE_HASH_L3:
	250	return PAGE_HOME_HASH;
	251	#endif
	252	}
	253	panic("Bad PTE %#llx\n", pte.val);
	254	}
	255
	256	/* Update the home of a PTE if necessary (can also be used for a pgprot_t). */
	257	pte_t pte_set_home(pte_t pte, int home)
	258	{
	259	/* Check for non-linear file mapping "PTEs" and pass them through. */
	260	if (pte_file(pte))
	261	return pte;
	262
	263	#if CHIP_HAS_MMIO()
	264	/* Check for MMIO mappings and pass them through. */
	265	if (hv_pte_get_mode(pte) == HV_PTE_MODE_MMIO)
	266	return pte;
	267	#endif
	268
	269
	270	/*
	271	* Only immutable pages get NC mappings. If we have a
	272	* non-coherent PTE, but the underlying page is not
	273	* immutable, it's likely the result of a forced
	274	* caching setting running up against ptrace setting
	275	* the page to be writable underneath. In this case,
	276	* just keep the PTE coherent.
	277	*/
	278	if (hv_pte_get_nc(pte) && home != PAGE_HOME_IMMUTABLE) {
	279	pte = hv_pte_clear_nc(pte);
0707ad30	280	pr_err("non-immutable page incoherently referenced: %#llx\n",
867e359b CM	281	pte.val);
	282	}
	283
	284	switch (home) {
	285
	286	case PAGE_HOME_UNCACHED:
	287	pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
	288	break;
	289
	290	case PAGE_HOME_INCOHERENT:
	291	pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
	292	break;
	293
	294	case PAGE_HOME_IMMUTABLE:
	295	/*
	296	* We could home this page anywhere, since it's immutable,
	297	* but by default just home it to follow "hash_default".
	298	*/
	299	BUG_ON(hv_pte_get_writable(pte));
	300	if (pte_get_forcecache(pte)) {
	301	/* Upgrade "force any cpu" to "No L3" for immutable. */
	302	if (hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_TILE_L3
	303	&& pte_get_anyhome(pte)) {
	304	pte = hv_pte_set_mode(pte,
	305	HV_PTE_MODE_CACHE_NO_L3);
	306	}
	307	} else
	308	#if CHIP_HAS_CBOX_HOME_MAP()
	309	if (hash_default)
	310	pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
	311	else
	312	#endif
	313	pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
	314	pte = hv_pte_set_nc(pte);
	315	break;
	316
	317	#if CHIP_HAS_CBOX_HOME_MAP()
	318	case PAGE_HOME_HASH:
	319	pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
	320	break;
	321	#endif
	322
	323	default:
	324	BUG_ON(home < 0 \|\| home >= NR_CPUS \|\|
	325	!cpu_is_valid_lotar(home));
	326	pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_TILE_L3);
	327	pte = set_remote_cache_cpu(pte, home);
	328	break;
	329	}
	330
	331	#if CHIP_HAS_NC_AND_NOALLOC_BITS()
	332	if (noallocl2)
	333	pte = hv_pte_set_no_alloc_l2(pte);
	334
	335	/* Simplify "no local and no l3" to "uncached" */
	336	if (hv_pte_get_no_alloc_l2(pte) && hv_pte_get_no_alloc_l1(pte) &&
	337	hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_NO_L3) {
	338	pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
	339	}
	340	#endif
	341
	342	/* Checking this case here gives a better panic than from the hv. */
	343	BUG_ON(hv_pte_get_mode(pte) == 0);
	344
345	return pte;
346	}
c745a8a1	347	EXPORT_SYMBOL(pte_set_home);
867e359b CM	348
	349	/*
	350	* The routines in this section are the "static" versions of the normal
	351	* dynamic homecaching routines; they just set the home cache
	352	* of a kernel page once, and require a full-chip cache/TLB flush,
	353	* so they're not suitable for anything but infrequent use.
	354	*/
	355
	356	#if CHIP_HAS_CBOX_HOME_MAP()
	357	static inline int initial_page_home(void) { return PAGE_HOME_HASH; }
	358	#else
	359	static inline int initial_page_home(void) { return 0; }
	360	#endif
	361
	362	int page_home(struct page *page)
	363	{
	364	if (PageHighMem(page)) {
	365	return initial_page_home();
	366	} else {
	367	unsigned long kva = (unsigned long)page_address(page);
	368	return pte_to_home(*virt_to_pte(NULL, kva));
	369	}
	370	}
	371
	372	void homecache_change_page_home(struct page *page, int order, int home)
	373	{
	374	int i, pages = (1 << order);
	375	unsigned long kva;
	376
	377	BUG_ON(PageHighMem(page));
	378	BUG_ON(page_count(page) > 1);
	379	BUG_ON(page_mapcount(page) != 0);
	380	kva = (unsigned long) page_address(page);
	381	flush_remote(0, HV_FLUSH_EVICT_L2, &cpu_cacheable_map,
	382	kva, pages * PAGE_SIZE, PAGE_SIZE, cpu_online_mask,
	383	NULL, 0);
	384
	385	for (i = 0; i < pages; ++i, kva += PAGE_SIZE) {
	386	pte_t *ptep = virt_to_pte(NULL, kva);
	387	pte_t pteval = *ptep;
	388	BUG_ON(!pte_present(pteval) \|\| pte_huge(pteval));
	389	*ptep = pte_set_home(pteval, home);
	390	}
	391	}
	392
	393	struct page *homecache_alloc_pages(gfp_t gfp_mask,
	394	unsigned int order, int home)
	395	{
	396	struct page *page;
	397	BUG_ON(gfp_mask & __GFP_HIGHMEM); /* must be lowmem */
	398	page = alloc_pages(gfp_mask, order);
	399	if (page)
	400	homecache_change_page_home(page, order, home);
	401	return page;
	402	}
c745a8a1	403	EXPORT_SYMBOL(homecache_alloc_pages);
867e359b CM	404
	405	struct page *homecache_alloc_pages_node(int nid, gfp_t gfp_mask,
	406	unsigned int order, int home)
	407	{
	408	struct page *page;
	409	BUG_ON(gfp_mask & __GFP_HIGHMEM); /* must be lowmem */
	410	page = alloc_pages_node(nid, gfp_mask, order);
	411	if (page)
	412	homecache_change_page_home(page, order, home);
	413	return page;
	414	}
	415
	416	void homecache_free_pages(unsigned long addr, unsigned int order)
	417	{
	418	struct page *page;
	419
	420	if (addr == 0)
	421	return;
	422
	423	VM_BUG_ON(!virt_addr_valid((void *)addr));
	424	page = virt_to_page((void *)addr);
	425	if (put_page_testzero(page)) {
	426	int pages = (1 << order);
	427	homecache_change_page_home(page, order, initial_page_home());
	428	while (pages--)
	429	__free_page(page++);
	430	}
	431	}