[deliverable/linux.git] / arch / s390 / numa / mode_emu.c

/*
 * NUMA support for s390
 *
 * NUMA emulation (aka fake NUMA) distributes the available memory to nodes
 * without using real topology information about the physical memory of the
 * machine.
 *
 * It distributes the available CPUs to nodes while respecting the original
 * machine topology information. This is done by trying to avoid to separate
 * CPUs which reside on the same book or even on the same MC.
 *
 * Because the current Linux scheduler code requires a stable cpu to node
 * mapping, cores are pinned to nodes when the first CPU thread is set online.
 *
 * Copyright IBM Corp. 2015
 */

#define KMSG_COMPONENT "numa_emu"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/kernel.h>
#include <linux/cpumask.h>
#include <linux/memblock.h>
#include <linux/node.h>
#include <linux/memory.h>
#include <linux/slab.h>
#include <asm/smp.h>
#include <asm/topology.h>
#include "numa_mode.h"
#include "toptree.h"

/* Distances between the different system components */
#define DIST_EMPTY	0
#define DIST_CORE	1
#define DIST_MC		2
#define DIST_BOOK	3
#define DIST_MAX	4

/* Node distance reported to common code */
#define EMU_NODE_DIST	10

/* Node ID for free (not yet pinned) cores */
#define NODE_ID_FREE	-1

/* Different levels of toptree */
enum toptree_level {CORE, MC, BOOK, NODE, TOPOLOGY};

/* The two toptree IDs */
enum {TOPTREE_ID_PHYS, TOPTREE_ID_NUMA};

/* Number of NUMA nodes */
static int emu_nodes = 1;
/* NUMA stripe size */
static unsigned long emu_size;

/*
 * Node to core pinning information updates are protected by
 * "sched_domains_mutex".
 */
static struct {
	s32 to_node_id[CONFIG_NR_CPUS];	/* Pinned core to node mapping */
	int total;			/* Total number of pinned cores */
	int per_node_target;		/* Cores per node without extra cores */
	int per_node[MAX_NUMNODES];	/* Number of cores pinned to node */
} *emu_cores;

/*
 * Pin a core to a node
 */
static void pin_core_to_node(int core_id, int node_id)
{
	if (emu_cores->to_node_id[core_id] == NODE_ID_FREE) {
		emu_cores->per_node[node_id]++;
		emu_cores->to_node_id[core_id] = node_id;
		emu_cores->total++;
	} else {
		WARN_ON(emu_cores->to_node_id[core_id] != node_id);
	}
}

/*
 * Number of pinned cores of a node
 */
static int cores_pinned(struct toptree *node)
{
	return emu_cores->per_node[node->id];
}

/*
 * ID of the node where the core is pinned (or NODE_ID_FREE)
 */
static int core_pinned_to_node_id(struct toptree *core)
{
	return emu_cores->to_node_id[core->id];
}

/*
 * Number of cores in the tree that are not yet pinned
 */
static int cores_free(struct toptree *tree)
{
	struct toptree *core;
	int count = 0;

	toptree_for_each(core, tree, CORE) {
		if (core_pinned_to_node_id(core) == NODE_ID_FREE)
			count++;
	}
	return count;
}

/*
 * Return node of core
 */
static struct toptree *core_node(struct toptree *core)
{
	return core->parent->parent->parent;
}

/*
 * Return book of core
 */
static struct toptree *core_book(struct toptree *core)
{
	return core->parent->parent;
}

/*
 * Return mc of core
 */
static struct toptree *core_mc(struct toptree *core)
{
	return core->parent;
}

/*
 * Distance between two cores
 */
static int dist_core_to_core(struct toptree *core1, struct toptree *core2)
{
	if (core_book(core1)->id != core_book(core2)->id)
		return DIST_BOOK;
	if (core_mc(core1)->id != core_mc(core2)->id)
		return DIST_MC;
	/* Same core or sibling on same MC */
	return DIST_CORE;
}

/*
 * Distance of a node to a core
 */
static int dist_node_to_core(struct toptree *node, struct toptree *core)
{
	struct toptree *core_node;
	int dist_min = DIST_MAX;

	toptree_for_each(core_node, node, CORE)
		dist_min = min(dist_min, dist_core_to_core(core_node, core));
	return dist_min == DIST_MAX ? DIST_EMPTY : dist_min;
}

/*
 * Unify will delete empty nodes, therefore recreate nodes.
 */
static void toptree_unify_tree(struct toptree *tree)
{
	int nid;

	toptree_unify(tree);
	for (nid = 0; nid < emu_nodes; nid++)
		toptree_get_child(tree, nid);
}

/*
 * Find the best/nearest node for a given core and ensure that no node
 * gets more than "emu_cores->per_node_target + extra" cores.
 */
static struct toptree *node_for_core(struct toptree *numa, struct toptree *core,
				     int extra)
{
	struct toptree *node, *node_best = NULL;
	int dist_cur, dist_best, cores_target;

	cores_target = emu_cores->per_node_target + extra;
	dist_best = DIST_MAX;
	node_best = NULL;
	toptree_for_each(node, numa, NODE) {
		/* Already pinned cores must use their nodes */
		if (core_pinned_to_node_id(core) == node->id) {
			node_best = node;
			break;
		}
		/* Skip nodes that already have enough cores */
		if (cores_pinned(node) >= cores_target)
			continue;
		dist_cur = dist_node_to_core(node, core);
		if (dist_cur < dist_best) {
			dist_best = dist_cur;
			node_best = node;
		}
	}
	return node_best;
}

/*
 * Find the best node for each core with respect to "extra" core count
 */
static void toptree_to_numa_single(struct toptree *numa, struct toptree *phys,
				   int extra)
{
	struct toptree *node, *core, *tmp;

	toptree_for_each_safe(core, tmp, phys, CORE) {
		node = node_for_core(numa, core, extra);
		if (!node)
			return;
		toptree_move(core, node);
		pin_core_to_node(core->id, node->id);
	}
}

/*
 * Move structures of given level to specified NUMA node
 */
static void move_level_to_numa_node(struct toptree *node, struct toptree *phys,
				    enum toptree_level level, bool perfect)
{
	int cores_free, cores_target = emu_cores->per_node_target;
	struct toptree *cur, *tmp;

	toptree_for_each_safe(cur, tmp, phys, level) {
		cores_free = cores_target - toptree_count(node, CORE);
		if (perfect) {
			if (cores_free == toptree_count(cur, CORE))
				toptree_move(cur, node);
		} else {
			if (cores_free >= toptree_count(cur, CORE))
				toptree_move(cur, node);
		}
	}
}

/*
 * Move structures of a given level to NUMA nodes. If "perfect" is specified
 * move only perfectly fitting structures. Otherwise move also smaller
 * than needed structures.
 */
static void move_level_to_numa(struct toptree *numa, struct toptree *phys,
			       enum toptree_level level, bool perfect)
{
	struct toptree *node;

	toptree_for_each(node, numa, NODE)
		move_level_to_numa_node(node, phys, level, perfect);
}

/*
 * For the first run try to move the big structures
 */
static void toptree_to_numa_first(struct toptree *numa, struct toptree *phys)
{
	struct toptree *core;

	/* Always try to move perfectly fitting structures first */
	move_level_to_numa(numa, phys, BOOK, true);
	move_level_to_numa(numa, phys, BOOK, false);
	move_level_to_numa(numa, phys, MC, true);
	move_level_to_numa(numa, phys, MC, false);
	/* Now pin all the moved cores */
	toptree_for_each(core, numa, CORE)
		pin_core_to_node(core->id, core_node(core)->id);
}

/*
 * Allocate new topology and create required nodes
 */
static struct toptree *toptree_new(int id, int nodes)
{
	struct toptree *tree;
	int nid;

	tree = toptree_alloc(TOPOLOGY, id);
	if (!tree)
		goto fail;
	for (nid = 0; nid < nodes; nid++) {
		if (!toptree_get_child(tree, nid))
			goto fail;
	}
	return tree;
fail:
	panic("NUMA emulation could not allocate topology");
}

/*
 * Allocate and initialize core to node mapping
 */
static void create_core_to_node_map(void)
{
	int i;

	emu_cores = kzalloc(sizeof(*emu_cores), GFP_KERNEL);
	if (emu_cores == NULL)
		panic("Could not allocate cores to node memory");
	for (i = 0; i < ARRAY_SIZE(emu_cores->to_node_id); i++)
		emu_cores->to_node_id[i] = NODE_ID_FREE;
}

/*
 * Move cores from physical topology into NUMA target topology
 * and try to keep as much of the physical topology as possible.
 */
static struct toptree *toptree_to_numa(struct toptree *phys)
{
	static int first = 1;
	struct toptree *numa;
	int cores_total;

	cores_total = emu_cores->total + cores_free(phys);
	emu_cores->per_node_target = cores_total / emu_nodes;
	numa = toptree_new(TOPTREE_ID_NUMA, emu_nodes);
	if (first) {
		toptree_to_numa_first(numa, phys);
		first = 0;
	}
	toptree_to_numa_single(numa, phys, 0);
	toptree_to_numa_single(numa, phys, 1);
	toptree_unify_tree(numa);

	WARN_ON(cpumask_weight(&phys->mask));
	return numa;
}

/*
 * Create a toptree out of the physical topology that we got from the hypervisor
 */
static struct toptree *toptree_from_topology(void)
{
	struct toptree *phys, *node, *book, *mc, *core;
	struct cpu_topology_s390 *top;
	int cpu;

	phys = toptree_new(TOPTREE_ID_PHYS, 1);

	for_each_online_cpu(cpu) {
		top = &per_cpu(cpu_topology, cpu);
		node = toptree_get_child(phys, 0);
		book = toptree_get_child(node, top->book_id);
		mc = toptree_get_child(book, top->socket_id);
		core = toptree_get_child(mc, top->core_id);
		if (!book || !mc || !core)
			panic("NUMA emulation could not allocate memory");
		cpumask_set_cpu(cpu, &core->mask);
		toptree_update_mask(mc);
	}
	return phys;
}

/*
 * Add toptree core to topology and create correct CPU masks
 */
static void topology_add_core(struct toptree *core)
{
	struct cpu_topology_s390 *top;
	int cpu;

	for_each_cpu(cpu, &core->mask) {
		top = &per_cpu(cpu_topology, cpu);
		cpumask_copy(&top->thread_mask, &core->mask);
		cpumask_copy(&top->core_mask, &core_mc(core)->mask);
		cpumask_copy(&top->book_mask, &core_book(core)->mask);
		cpumask_set_cpu(cpu, &node_to_cpumask_map[core_node(core)->id]);
		top->node_id = core_node(core)->id;
	}
}

/*
 * Apply toptree to topology and create CPU masks
 */
static void toptree_to_topology(struct toptree *numa)
{
	struct toptree *core;
	int i;

	/* Clear all node masks */
	for (i = 0; i < MAX_NUMNODES; i++)
		cpumask_clear(&node_to_cpumask_map[i]);

	/* Rebuild all masks */
	toptree_for_each(core, numa, CORE)
		topology_add_core(core);
}

/*
 * Show the node to core mapping
 */
static void print_node_to_core_map(void)
{
	int nid, cid;

	if (!numa_debug_enabled)
		return;
	printk(KERN_DEBUG "NUMA node to core mapping\n");
	for (nid = 0; nid < emu_nodes; nid++) {
		printk(KERN_DEBUG "  node %3d: ", nid);
		for (cid = 0; cid < ARRAY_SIZE(emu_cores->to_node_id); cid++) {
			if (emu_cores->to_node_id[cid] == nid)
				printk(KERN_CONT "%d ", cid);
		}
		printk(KERN_CONT "\n");
	}
}

/*
 * Transfer physical topology into a NUMA topology and modify CPU masks
 * according to the NUMA topology.
 *
 * Must be called with "sched_domains_mutex" lock held.
 */
static void emu_update_cpu_topology(void)
{
	struct toptree *phys, *numa;

	if (emu_cores == NULL)
		create_core_to_node_map();
	phys = toptree_from_topology();
	numa = toptree_to_numa(phys);
	toptree_free(phys);
	toptree_to_topology(numa);
	toptree_free(numa);
	print_node_to_core_map();
}

/*
 * If emu_size is not set, use CONFIG_EMU_SIZE. Then round to minimum
 * alignment (needed for memory hotplug).
 */
static unsigned long emu_setup_size_adjust(unsigned long size)
{
	unsigned long size_new;

	size = size ? : CONFIG_EMU_SIZE;
	size_new = roundup(size, memory_block_size_bytes());
	if (size_new == size)
		return size;
	pr_warn("Increasing memory stripe size from %ld MB to %ld MB\n",
		size >> 20, size_new >> 20);
	return size_new;
}

/*
 * If we have not enough memory for the specified nodes, reduce the node count.
 */
static int emu_setup_nodes_adjust(int nodes)
{
	int nodes_max;

	nodes_max = memblock.memory.total_size / emu_size;
	nodes_max = max(nodes_max, 1);
	if (nodes_max >= nodes)
		return nodes;
	pr_warn("Not enough memory for %d nodes, reducing node count\n", nodes);
	return nodes_max;
}

/*
 * Early emu setup
 */
static void emu_setup(void)
{
	emu_size = emu_setup_size_adjust(emu_size);
	emu_nodes = emu_setup_nodes_adjust(emu_nodes);
	pr_info("Creating %d nodes with memory stripe size %ld MB\n",
		emu_nodes, emu_size >> 20);
}

/*
 * Return node id for given page number
 */
static int emu_pfn_to_nid(unsigned long pfn)
{
	return (pfn / (emu_size >> PAGE_SHIFT)) % emu_nodes;
}

/*
 * Return stripe size
 */
static unsigned long emu_align(void)
{
	return emu_size;
}

/*
 * Return distance between two nodes
 */
static int emu_distance(int node1, int node2)
{
	return (node1 != node2) * EMU_NODE_DIST;
}

/*
 * Define callbacks for generic s390 NUMA infrastructure
 */
const struct numa_mode numa_mode_emu = {
	.name = "emu",
	.setup = emu_setup,
	.update_cpu_topology = emu_update_cpu_topology,
	.__pfn_to_nid = emu_pfn_to_nid,
	.align = emu_align,
	.distance = emu_distance,
};

/*
 * Kernel parameter: emu_nodes=<n>
 */
static int __init early_parse_emu_nodes(char *p)
{
	int count;

	if (kstrtoint(p, 0, &count) != 0 || count <= 0)
		return 0;
	if (count <= 0)
		return 0;
	emu_nodes = min(count, MAX_NUMNODES);
	return 0;
}
early_param("emu_nodes", early_parse_emu_nodes);

/*
 * Kernel parameter: emu_size=[<n>[k|M|G|T]]
 */
static int __init early_parse_emu_size(char *p)
{
	emu_size = memparse(p, NULL);
	return 0;
}
early_param("emu_size", early_parse_emu_size);
Commit	Line	Data
c29a7baf MH	1	/*
	2	* NUMA support for s390
	3	*
	4	* NUMA emulation (aka fake NUMA) distributes the available memory to nodes
	5	* without using real topology information about the physical memory of the
	6	* machine.
	7	*
	8	* It distributes the available CPUs to nodes while respecting the original
	9	* machine topology information. This is done by trying to avoid to separate
	10	* CPUs which reside on the same book or even on the same MC.
	11	*
	12	* Because the current Linux scheduler code requires a stable cpu to node
	13	* mapping, cores are pinned to nodes when the first CPU thread is set online.
	14	*
	15	* Copyright IBM Corp. 2015
	16	*/
	17
	18	#define KMSG_COMPONENT "numa_emu"
	19	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	20
	21	#include <linux/kernel.h>
	22	#include <linux/cpumask.h>
	23	#include <linux/memblock.h>
	24	#include <linux/node.h>
	25	#include <linux/memory.h>
7cde4910	26	#include <linux/slab.h>
c29a7baf MH	27	#include <asm/smp.h>
	28	#include <asm/topology.h>
	29	#include "numa_mode.h"
	30	#include "toptree.h"
	31
	32	/* Distances between the different system components */
	33	#define DIST_EMPTY 0
	34	#define DIST_CORE 1
	35	#define DIST_MC 2
	36	#define DIST_BOOK 3
	37	#define DIST_MAX 4
	38
	39	/* Node distance reported to common code */
	40	#define EMU_NODE_DIST 10
	41
	42	/* Node ID for free (not yet pinned) cores */
	43	#define NODE_ID_FREE -1
	44
	45	/* Different levels of toptree */
	46	enum toptree_level {CORE, MC, BOOK, NODE, TOPOLOGY};
	47
	48	/* The two toptree IDs */
	49	enum {TOPTREE_ID_PHYS, TOPTREE_ID_NUMA};
	50
	51	/* Number of NUMA nodes */
	52	static int emu_nodes = 1;
	53	/* NUMA stripe size */
	54	static unsigned long emu_size;
3a3814c2 MH	55
	56	/*
	57	* Node to core pinning information updates are protected by
	58	* "sched_domains_mutex".
	59	*/
7cde4910 MH	60	static struct {
	61	s32 to_node_id[CONFIG_NR_CPUS]; /* Pinned core to node mapping */
	62	int total; /* Total number of pinned cores */
	63	int per_node_target; /* Cores per node without extra cores */
	64	int per_node[MAX_NUMNODES]; /* Number of cores pinned to node */
	65	} *emu_cores;
c29a7baf MH	66
	67	/*
	68	* Pin a core to a node
	69	*/
	70	static void pin_core_to_node(int core_id, int node_id)
	71	{
7cde4910 MH	72	if (emu_cores->to_node_id[core_id] == NODE_ID_FREE) {
	73	emu_cores->per_node[node_id]++;
	74	emu_cores->to_node_id[core_id] = node_id;
	75	emu_cores->total++;
c29a7baf	76	} else {
7cde4910	77	WARN_ON(emu_cores->to_node_id[core_id] != node_id);
c29a7baf MH	78	}
	79	}
	80
	81	/*
	82	* Number of pinned cores of a node
	83	*/
	84	static int cores_pinned(struct toptree *node)
	85	{
7cde4910	86	return emu_cores->per_node[node->id];
c29a7baf MH	87	}
	88
	89	/*
	90	* ID of the node where the core is pinned (or NODE_ID_FREE)
	91	*/
	92	static int core_pinned_to_node_id(struct toptree *core)
	93	{
7cde4910	94	return emu_cores->to_node_id[core->id];
c29a7baf MH	95	}
	96
	97	/*
	98	* Number of cores in the tree that are not yet pinned
	99	*/
	100	static int cores_free(struct toptree *tree)
	101	{
	102	struct toptree *core;
	103	int count = 0;
	104
	105	toptree_for_each(core, tree, CORE) {
	106	if (core_pinned_to_node_id(core) == NODE_ID_FREE)
	107	count++;
	108	}
	109	return count;
	110	}
	111
	112	/*
	113	* Return node of core
	114	*/
	115	static struct toptree core_node(struct toptree core)
	116	{
	117	return core->parent->parent->parent;
	118	}
	119
	120	/*
	121	* Return book of core
	122	*/
	123	static struct toptree core_book(struct toptree core)
	124	{
	125	return core->parent->parent;
	126	}
	127
	128	/*
	129	* Return mc of core
	130	*/
	131	static struct toptree core_mc(struct toptree core)
	132	{
	133	return core->parent;
	134	}
	135
	136	/*
	137	* Distance between two cores
	138	*/
	139	static int dist_core_to_core(struct toptree core1, struct toptree core2)
	140	{
	141	if (core_book(core1)->id != core_book(core2)->id)
	142	return DIST_BOOK;
	143	if (core_mc(core1)->id != core_mc(core2)->id)
	144	return DIST_MC;
	145	/* Same core or sibling on same MC */
	146	return DIST_CORE;
	147	}
	148
	149	/*
	150	* Distance of a node to a core
	151	*/
	152	static int dist_node_to_core(struct toptree node, struct toptree core)
	153	{
	154	struct toptree *core_node;
	155	int dist_min = DIST_MAX;
	156
	157	toptree_for_each(core_node, node, CORE)
	158	dist_min = min(dist_min, dist_core_to_core(core_node, core));
159	return dist_min == DIST_MAX ? DIST_EMPTY : dist_min;
160	}
161
162	/*
163	* Unify will delete empty nodes, therefore recreate nodes.
164	*/
165	static void toptree_unify_tree(struct toptree *tree)
166	{
167	int nid;
168
169	toptree_unify(tree);
170	for (nid = 0; nid < emu_nodes; nid++)
171	toptree_get_child(tree, nid);
172	}
173
174	/*
175	* Find the best/nearest node for a given core and ensure that no node
7cde4910	176	* gets more than "emu_cores->per_node_target + extra" cores.
c29a7baf MH	177	*/
	178	static struct toptree node_for_core(struct toptree numa, struct toptree *core,
	179	int extra)
	180	{
	181	struct toptree node, node_best = NULL;
7cde4910	182	int dist_cur, dist_best, cores_target;
c29a7baf	183
7cde4910	184	cores_target = emu_cores->per_node_target + extra;
c29a7baf MH	185	dist_best = DIST_MAX;
	186	node_best = NULL;
	187	toptree_for_each(node, numa, NODE) {
	188	/* Already pinned cores must use their nodes */
	189	if (core_pinned_to_node_id(core) == node->id) {
	190	node_best = node;
	191	break;
	192	}
	193	/* Skip nodes that already have enough cores */
7cde4910	194	if (cores_pinned(node) >= cores_target)
c29a7baf MH	195	continue;
	196	dist_cur = dist_node_to_core(node, core);
	197	if (dist_cur < dist_best) {
	198	dist_best = dist_cur;
	199	node_best = node;
	200	}
	201	}
	202	return node_best;
	203	}
	204
	205	/*
	206	* Find the best node for each core with respect to "extra" core count
	207	*/
	208	static void toptree_to_numa_single(struct toptree numa, struct toptree phys,
	209	int extra)
	210	{
	211	struct toptree node, core, *tmp;
	212
	213	toptree_for_each_safe(core, tmp, phys, CORE) {
	214	node = node_for_core(numa, core, extra);
	215	if (!node)
	216	return;
	217	toptree_move(core, node);
	218	pin_core_to_node(core->id, node->id);
	219	}
	220	}
	221
	222	/*
	223	* Move structures of given level to specified NUMA node
	224	*/
	225	static void move_level_to_numa_node(struct toptree node, struct toptree phys,
	226	enum toptree_level level, bool perfect)
	227	{
7cde4910	228	int cores_free, cores_target = emu_cores->per_node_target;
c29a7baf	229	struct toptree cur, tmp;
c29a7baf MH	230
c29a7baf MH	231	toptree_for_each_safe(cur, tmp, phys, level) {
7cde4910	232	cores_free = cores_target - toptree_count(node, CORE);
c29a7baf MH	233	if (perfect) {
	234	if (cores_free == toptree_count(cur, CORE))
	235	toptree_move(cur, node);
	236	} else {
	237	if (cores_free >= toptree_count(cur, CORE))
	238	toptree_move(cur, node);
	239	}
	240	}
	241	}
	242
	243	/*
	244	* Move structures of a given level to NUMA nodes. If "perfect" is specified
	245	* move only perfectly fitting structures. Otherwise move also smaller
	246	* than needed structures.
	247	*/
	248	static void move_level_to_numa(struct toptree numa, struct toptree phys,
	249	enum toptree_level level, bool perfect)
	250	{
	251	struct toptree *node;
	252
	253	toptree_for_each(node, numa, NODE)
	254	move_level_to_numa_node(node, phys, level, perfect);
	255	}
	256
	257	/*
	258	* For the first run try to move the big structures
	259	*/
	260	static void toptree_to_numa_first(struct toptree numa, struct toptree phys)
	261	{
	262	struct toptree *core;
	263
	264	/* Always try to move perfectly fitting structures first */
	265	move_level_to_numa(numa, phys, BOOK, true);
	266	move_level_to_numa(numa, phys, BOOK, false);
	267	move_level_to_numa(numa, phys, MC, true);
	268	move_level_to_numa(numa, phys, MC, false);
	269	/* Now pin all the moved cores */
	270	toptree_for_each(core, numa, CORE)
	271	pin_core_to_node(core->id, core_node(core)->id);
	272	}
	273
	274	/*
	275	* Allocate new topology and create required nodes
	276	*/
	277	static struct toptree *toptree_new(int id, int nodes)
	278	{
	279	struct toptree *tree;
	280	int nid;
	281
	282	tree = toptree_alloc(TOPOLOGY, id);
	283	if (!tree)
	284	goto fail;
	285	for (nid = 0; nid < nodes; nid++) {
	286	if (!toptree_get_child(tree, nid))
	287	goto fail;
	288	}
	289	return tree;
	290	fail:
	291	panic("NUMA emulation could not allocate topology");
	292	}
	293
7cde4910 MH	294	/*
	295	* Allocate and initialize core to node mapping
	296	*/
	297	static void create_core_to_node_map(void)
	298	{
	299	int i;
	300
	301	emu_cores = kzalloc(sizeof(*emu_cores), GFP_KERNEL);
	302	if (emu_cores == NULL)
	303	panic("Could not allocate cores to node memory");
	304	for (i = 0; i < ARRAY_SIZE(emu_cores->to_node_id); i++)
	305	emu_cores->to_node_id[i] = NODE_ID_FREE;
	306	}
	307
c29a7baf MH	308	/*
	309	* Move cores from physical topology into NUMA target topology
	310	* and try to keep as much of the physical topology as possible.
	311	*/
	312	static struct toptree toptree_to_numa(struct toptree phys)
	313	{
	314	static int first = 1;
	315	struct toptree *numa;
7cde4910	316	int cores_total;
c29a7baf	317
7cde4910 MH	318	cores_total = emu_cores->total + cores_free(phys);
7cde4910 MH	319	emu_cores->per_node_target = cores_total / emu_nodes;
c29a7baf MH	320	numa = toptree_new(TOPTREE_ID_NUMA, emu_nodes);
	321	if (first) {
	322	toptree_to_numa_first(numa, phys);
	323	first = 0;
	324	}
	325	toptree_to_numa_single(numa, phys, 0);
	326	toptree_to_numa_single(numa, phys, 1);
	327	toptree_unify_tree(numa);
	328
	329	WARN_ON(cpumask_weight(&phys->mask));
	330	return numa;
	331	}
	332
	333	/*
	334	* Create a toptree out of the physical topology that we got from the hypervisor
	335	*/
	336	static struct toptree *toptree_from_topology(void)
	337	{
	338	struct toptree phys, node, book, mc, *core;
	339	struct cpu_topology_s390 *top;
	340	int cpu;
	341
	342	phys = toptree_new(TOPTREE_ID_PHYS, 1);
	343
	344	for_each_online_cpu(cpu) {
	345	top = &per_cpu(cpu_topology, cpu);
	346	node = toptree_get_child(phys, 0);
	347	book = toptree_get_child(node, top->book_id);
	348	mc = toptree_get_child(book, top->socket_id);
	349	core = toptree_get_child(mc, top->core_id);
	350	if (!book \|\| !mc \|\| !core)
	351	panic("NUMA emulation could not allocate memory");
	352	cpumask_set_cpu(cpu, &core->mask);
	353	toptree_update_mask(mc);
	354	}
	355	return phys;
	356	}
	357
	358	/*
	359	* Add toptree core to topology and create correct CPU masks
	360	*/
	361	static void topology_add_core(struct toptree *core)
	362	{
	363	struct cpu_topology_s390 *top;
	364	int cpu;
	365
	366	for_each_cpu(cpu, &core->mask) {
	367	top = &per_cpu(cpu_topology, cpu);
	368	cpumask_copy(&top->thread_mask, &core->mask);
	369	cpumask_copy(&top->core_mask, &core_mc(core)->mask);
	370	cpumask_copy(&top->book_mask, &core_book(core)->mask);
22be9cd9	371	cpumask_set_cpu(cpu, &node_to_cpumask_map[core_node(core)->id]);
c29a7baf MH	372	top->node_id = core_node(core)->id;
	373	}
	374	}
	375
	376	/*
	377	* Apply toptree to topology and create CPU masks
	378	*/
	379	static void toptree_to_topology(struct toptree *numa)
	380	{
	381	struct toptree *core;
	382	int i;
	383
	384	/* Clear all node masks */
	385	for (i = 0; i < MAX_NUMNODES; i++)
22be9cd9	386	cpumask_clear(&node_to_cpumask_map[i]);
c29a7baf MH	387
	388	/* Rebuild all masks */
	389	toptree_for_each(core, numa, CORE)
	390	topology_add_core(core);
	391	}
	392
	393	/*
	394	* Show the node to core mapping
	395	*/
	396	static void print_node_to_core_map(void)
	397	{
	398	int nid, cid;
	399
	400	if (!numa_debug_enabled)
	401	return;
	402	printk(KERN_DEBUG "NUMA node to core mapping\n");
	403	for (nid = 0; nid < emu_nodes; nid++) {
	404	printk(KERN_DEBUG " node %3d: ", nid);
7cde4910 MH	405	for (cid = 0; cid < ARRAY_SIZE(emu_cores->to_node_id); cid++) {
7cde4910 MH	406	if (emu_cores->to_node_id[cid] == nid)
c29a7baf MH	407	printk(KERN_CONT "%d ", cid);
	408	}
	409	printk(KERN_CONT "\n");
	410	}
	411	}
	412
	413	/*
	414	* Transfer physical topology into a NUMA topology and modify CPU masks
	415	* according to the NUMA topology.
	416	*
3a3814c2	417	* Must be called with "sched_domains_mutex" lock held.
c29a7baf MH	418	*/
	419	static void emu_update_cpu_topology(void)
	420	{
	421	struct toptree phys, numa;
	422
7cde4910 MH	423	if (emu_cores == NULL)
7cde4910 MH	424	create_core_to_node_map();
c29a7baf MH	425	phys = toptree_from_topology();
	426	numa = toptree_to_numa(phys);
	427	toptree_free(phys);
	428	toptree_to_topology(numa);
	429	toptree_free(numa);
	430	print_node_to_core_map();
	431	}
	432
	433	/*
	434	* If emu_size is not set, use CONFIG_EMU_SIZE. Then round to minimum
	435	* alignment (needed for memory hotplug).
	436	*/
	437	static unsigned long emu_setup_size_adjust(unsigned long size)
	438	{
b02064a9 MH	439	unsigned long size_new;
b02064a9 MH	440
c29a7baf	441	size = size ? : CONFIG_EMU_SIZE;
b02064a9 MH	442	size_new = roundup(size, memory_block_size_bytes());
	443	if (size_new == size)
	444	return size;
	445	pr_warn("Increasing memory stripe size from %ld MB to %ld MB\n",
	446	size >> 20, size_new >> 20);
	447	return size_new;
c29a7baf MH	448	}
	449
	450	/*
	451	* If we have not enough memory for the specified nodes, reduce the node count.
	452	*/
	453	static int emu_setup_nodes_adjust(int nodes)
	454	{
	455	int nodes_max;
	456
	457	nodes_max = memblock.memory.total_size / emu_size;
	458	nodes_max = max(nodes_max, 1);
	459	if (nodes_max >= nodes)
	460	return nodes;
	461	pr_warn("Not enough memory for %d nodes, reducing node count\n", nodes);
	462	return nodes_max;
	463	}
	464
	465	/*
	466	* Early emu setup
	467	*/
	468	static void emu_setup(void)
	469	{
c29a7baf MH	470	emu_size = emu_setup_size_adjust(emu_size);
c29a7baf MH	471	emu_nodes = emu_setup_nodes_adjust(emu_nodes);
c29a7baf MH	472	pr_info("Creating %d nodes with memory stripe size %ld MB\n",
	473	emu_nodes, emu_size >> 20);
	474	}
	475
	476	/*
	477	* Return node id for given page number
	478	*/
	479	static int emu_pfn_to_nid(unsigned long pfn)
	480	{
	481	return (pfn / (emu_size >> PAGE_SHIFT)) % emu_nodes;
	482	}
	483
	484	/*
	485	* Return stripe size
	486	*/
	487	static unsigned long emu_align(void)
	488	{
	489	return emu_size;
	490	}
	491
	492	/*
	493	* Return distance between two nodes
	494	*/
	495	static int emu_distance(int node1, int node2)
	496	{
	497	return (node1 != node2) * EMU_NODE_DIST;
	498	}
	499
	500	/*
	501	* Define callbacks for generic s390 NUMA infrastructure
	502	*/
	503	const struct numa_mode numa_mode_emu = {
	504	.name = "emu",
	505	.setup = emu_setup,
	506	.update_cpu_topology = emu_update_cpu_topology,
	507	.__pfn_to_nid = emu_pfn_to_nid,
	508	.align = emu_align,
	509	.distance = emu_distance,
	510	};
	511
	512	/*
	513	* Kernel parameter: emu_nodes=<n>
	514	*/
	515	static int __init early_parse_emu_nodes(char *p)
	516	{
	517	int count;
	518
	519	if (kstrtoint(p, 0, &count) != 0 \|\| count <= 0)
	520	return 0;
	521	if (count <= 0)
	522	return 0;
	523	emu_nodes = min(count, MAX_NUMNODES);
	524	return 0;
	525	}
	526	early_param("emu_nodes", early_parse_emu_nodes);
	527
	528	/*
	529	* Kernel parameter: emu_size=[<n>[k\|M\|G\|T]]
	530	*/
	531	static int __init early_parse_emu_size(char *p)
	532	{
	533	emu_size = memparse(p, NULL);
	534	return 0;
	535	}
536	early_param("emu_size", early_parse_emu_size);