[deliverable/linux.git] / arch / x86 / events / amd / uncore.c

/*
 * Copyright (C) 2013 Advanced Micro Devices, Inc.
 *
 * Author: Jacob Shin <jacob.shin@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/perf_event.h>
#include <linux/percpu.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>

#include <asm/cpufeature.h>
#include <asm/perf_event.h>
#include <asm/msr.h>

#define NUM_COUNTERS_NB		4
#define NUM_COUNTERS_L2		4
#define MAX_COUNTERS		NUM_COUNTERS_NB

#define RDPMC_BASE_NB		6
#define RDPMC_BASE_L2		10

#define COUNTER_SHIFT		16

struct amd_uncore {
	int id;
	int refcnt;
	int cpu;
	int num_counters;
	int rdpmc_base;
	u32 msr_base;
	cpumask_t *active_mask;
	struct pmu *pmu;
	struct perf_event *events[MAX_COUNTERS];
	struct amd_uncore *free_when_cpu_online;
};

static struct amd_uncore * __percpu *amd_uncore_nb;
static struct amd_uncore * __percpu *amd_uncore_l2;

static struct pmu amd_nb_pmu;
static struct pmu amd_l2_pmu;

static cpumask_t amd_nb_active_mask;
static cpumask_t amd_l2_active_mask;

static bool is_nb_event(struct perf_event *event)
{
	return event->pmu->type == amd_nb_pmu.type;
}

static bool is_l2_event(struct perf_event *event)
{
	return event->pmu->type == amd_l2_pmu.type;
}

static struct amd_uncore *event_to_amd_uncore(struct perf_event *event)
{
	if (is_nb_event(event) && amd_uncore_nb)
		return *per_cpu_ptr(amd_uncore_nb, event->cpu);
	else if (is_l2_event(event) && amd_uncore_l2)
		return *per_cpu_ptr(amd_uncore_l2, event->cpu);

	return NULL;
}

static void amd_uncore_read(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	u64 prev, new;
	s64 delta;

	/*
	 * since we do not enable counter overflow interrupts,
	 * we do not have to worry about prev_count changing on us
	 */

	prev = local64_read(&hwc->prev_count);
	rdpmcl(hwc->event_base_rdpmc, new);
	local64_set(&hwc->prev_count, new);
	delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
	delta >>= COUNTER_SHIFT;
	local64_add(delta, &event->count);
}

static void amd_uncore_start(struct perf_event *event, int flags)
{
	struct hw_perf_event *hwc = &event->hw;

	if (flags & PERF_EF_RELOAD)
		wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));

	hwc->state = 0;
	wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
	perf_event_update_userpage(event);
}

static void amd_uncore_stop(struct perf_event *event, int flags)
{
	struct hw_perf_event *hwc = &event->hw;

	wrmsrl(hwc->config_base, hwc->config);
	hwc->state |= PERF_HES_STOPPED;

	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
		amd_uncore_read(event);
		hwc->state |= PERF_HES_UPTODATE;
	}
}

static int amd_uncore_add(struct perf_event *event, int flags)
{
	int i;
	struct amd_uncore *uncore = event_to_amd_uncore(event);
	struct hw_perf_event *hwc = &event->hw;

	/* are we already assigned? */
	if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
		goto out;

	for (i = 0; i < uncore->num_counters; i++) {
		if (uncore->events[i] == event) {
			hwc->idx = i;
			goto out;
		}
	}

	/* if not, take the first available counter */
	hwc->idx = -1;
	for (i = 0; i < uncore->num_counters; i++) {
		if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
			hwc->idx = i;
			break;
		}
	}

out:
	if (hwc->idx == -1)
		return -EBUSY;

	hwc->config_base = uncore->msr_base + (2 * hwc->idx);
	hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
	hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;

	if (flags & PERF_EF_START)
		amd_uncore_start(event, PERF_EF_RELOAD);

	return 0;
}

static void amd_uncore_del(struct perf_event *event, int flags)
{
	int i;
	struct amd_uncore *uncore = event_to_amd_uncore(event);
	struct hw_perf_event *hwc = &event->hw;

	amd_uncore_stop(event, PERF_EF_UPDATE);

	for (i = 0; i < uncore->num_counters; i++) {
		if (cmpxchg(&uncore->events[i], event, NULL) == event)
			break;
	}

	hwc->idx = -1;
}

static int amd_uncore_event_init(struct perf_event *event)
{
	struct amd_uncore *uncore;
	struct hw_perf_event *hwc = &event->hw;

	if (event->attr.type != event->pmu->type)
		return -ENOENT;

	/*
	 * NB and L2 counters (MSRs) are shared across all cores that share the
	 * same NB / L2 cache. Interrupts can be directed to a single target
	 * core, however, event counts generated by processes running on other
	 * cores cannot be masked out. So we do not support sampling and
	 * per-thread events.
	 */
	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
		return -EINVAL;

	/* NB and L2 counters do not have usr/os/guest/host bits */
	if (event->attr.exclude_user || event->attr.exclude_kernel ||
	    event->attr.exclude_host || event->attr.exclude_guest)
		return -EINVAL;

	/* and we do not enable counter overflow interrupts */
	hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
	hwc->idx = -1;

	if (event->cpu < 0)
		return -EINVAL;

	uncore = event_to_amd_uncore(event);
	if (!uncore)
		return -ENODEV;

	/*
	 * since request can come in to any of the shared cores, we will remap
	 * to a single common cpu.
	 */
	event->cpu = uncore->cpu;

	return 0;
}

static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
					    struct device_attribute *attr,
					    char *buf)
{
	cpumask_t *active_mask;
	struct pmu *pmu = dev_get_drvdata(dev);

	if (pmu->type == amd_nb_pmu.type)
		active_mask = &amd_nb_active_mask;
	else if (pmu->type == amd_l2_pmu.type)
		active_mask = &amd_l2_active_mask;
	else
		return 0;

	return cpumap_print_to_pagebuf(true, buf, active_mask);
}
static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);

static struct attribute *amd_uncore_attrs[] = {
	&dev_attr_cpumask.attr,
	NULL,
};

static struct attribute_group amd_uncore_attr_group = {
	.attrs = amd_uncore_attrs,
};

PMU_FORMAT_ATTR(event, "config:0-7,32-35");
PMU_FORMAT_ATTR(umask, "config:8-15");

static struct attribute *amd_uncore_format_attr[] = {
	&format_attr_event.attr,
	&format_attr_umask.attr,
	NULL,
};

static struct attribute_group amd_uncore_format_group = {
	.name = "format",
	.attrs = amd_uncore_format_attr,
};

static const struct attribute_group *amd_uncore_attr_groups[] = {
	&amd_uncore_attr_group,
	&amd_uncore_format_group,
	NULL,
};

static struct pmu amd_nb_pmu = {
	.task_ctx_nr	= perf_invalid_context,
	.attr_groups	= amd_uncore_attr_groups,
	.name		= "amd_nb",
	.event_init	= amd_uncore_event_init,
	.add		= amd_uncore_add,
	.del		= amd_uncore_del,
	.start		= amd_uncore_start,
	.stop		= amd_uncore_stop,
	.read		= amd_uncore_read,
};

static struct pmu amd_l2_pmu = {
	.task_ctx_nr	= perf_invalid_context,
	.attr_groups	= amd_uncore_attr_groups,
	.name		= "amd_l2",
	.event_init	= amd_uncore_event_init,
	.add		= amd_uncore_add,
	.del		= amd_uncore_del,
	.start		= amd_uncore_start,
	.stop		= amd_uncore_stop,
	.read		= amd_uncore_read,
};

static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
{
	return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
			cpu_to_node(cpu));
}

static int amd_uncore_cpu_up_prepare(unsigned int cpu)
{
	struct amd_uncore *uncore_nb = NULL, *uncore_l2;

	if (amd_uncore_nb) {
		uncore_nb = amd_uncore_alloc(cpu);
		if (!uncore_nb)
			goto fail;
		uncore_nb->cpu = cpu;
		uncore_nb->num_counters = NUM_COUNTERS_NB;
		uncore_nb->rdpmc_base = RDPMC_BASE_NB;
		uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
		uncore_nb->active_mask = &amd_nb_active_mask;
		uncore_nb->pmu = &amd_nb_pmu;
		*per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
	}

	if (amd_uncore_l2) {
		uncore_l2 = amd_uncore_alloc(cpu);
		if (!uncore_l2)
			goto fail;
		uncore_l2->cpu = cpu;
		uncore_l2->num_counters = NUM_COUNTERS_L2;
		uncore_l2->rdpmc_base = RDPMC_BASE_L2;
		uncore_l2->msr_base = MSR_F16H_L2I_PERF_CTL;
		uncore_l2->active_mask = &amd_l2_active_mask;
		uncore_l2->pmu = &amd_l2_pmu;
		*per_cpu_ptr(amd_uncore_l2, cpu) = uncore_l2;
	}

	return 0;

fail:
	if (amd_uncore_nb)
		*per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
	kfree(uncore_nb);
	return -ENOMEM;
}

static struct amd_uncore *
amd_uncore_find_online_sibling(struct amd_uncore *this,
			       struct amd_uncore * __percpu *uncores)
{
	unsigned int cpu;
	struct amd_uncore *that;

	for_each_online_cpu(cpu) {
		that = *per_cpu_ptr(uncores, cpu);

		if (!that)
			continue;

		if (this == that)
			continue;

		if (this->id == that->id) {
			that->free_when_cpu_online = this;
			this = that;
			break;
		}
	}

	this->refcnt++;
	return this;
}

static int amd_uncore_cpu_starting(unsigned int cpu)
{
	unsigned int eax, ebx, ecx, edx;
	struct amd_uncore *uncore;

	if (amd_uncore_nb) {
		uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
		uncore->id = ecx & 0xff;

		uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
		*per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
	}

	if (amd_uncore_l2) {
		unsigned int apicid = cpu_data(cpu).apicid;
		unsigned int nshared;

		uncore = *per_cpu_ptr(amd_uncore_l2, cpu);
		cpuid_count(0x8000001d, 2, &eax, &ebx, &ecx, &edx);
		nshared = ((eax >> 14) & 0xfff) + 1;
		uncore->id = apicid - (apicid % nshared);

		uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_l2);
		*per_cpu_ptr(amd_uncore_l2, cpu) = uncore;
	}

	return 0;
}

static void uncore_online(unsigned int cpu,
			  struct amd_uncore * __percpu *uncores)
{
	struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);

	kfree(uncore->free_when_cpu_online);
	uncore->free_when_cpu_online = NULL;

	if (cpu == uncore->cpu)
		cpumask_set_cpu(cpu, uncore->active_mask);
}

static int amd_uncore_cpu_online(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_online(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_online(cpu, amd_uncore_l2);

	return 0;
}

static void uncore_down_prepare(unsigned int cpu,
				struct amd_uncore * __percpu *uncores)
{
	unsigned int i;
	struct amd_uncore *this = *per_cpu_ptr(uncores, cpu);

	if (this->cpu != cpu)
		return;

	/* this cpu is going down, migrate to a shared sibling if possible */
	for_each_online_cpu(i) {
		struct amd_uncore *that = *per_cpu_ptr(uncores, i);

		if (cpu == i)
			continue;

		if (this == that) {
			perf_pmu_migrate_context(this->pmu, cpu, i);
			cpumask_clear_cpu(cpu, that->active_mask);
			cpumask_set_cpu(i, that->active_mask);
			that->cpu = i;
			break;
		}
	}
}

static int amd_uncore_cpu_down_prepare(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_down_prepare(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_down_prepare(cpu, amd_uncore_l2);

	return 0;
}

static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
{
	struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);

	if (cpu == uncore->cpu)
		cpumask_clear_cpu(cpu, uncore->active_mask);

	if (!--uncore->refcnt)
		kfree(uncore);
	*per_cpu_ptr(uncores, cpu) = NULL;
}

static int amd_uncore_cpu_dead(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_dead(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_dead(cpu, amd_uncore_l2);

	return 0;
}

static int __init amd_uncore_init(void)
{
	int ret = -ENODEV;

	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
		goto fail_nodev;

	if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
		goto fail_nodev;

	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
		amd_uncore_nb = alloc_percpu(struct amd_uncore *);
		if (!amd_uncore_nb) {
			ret = -ENOMEM;
			goto fail_nb;
		}
		ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
		if (ret)
			goto fail_nb;

		pr_info("perf: AMD NB counters detected\n");
		ret = 0;
	}

	if (boot_cpu_has(X86_FEATURE_PERFCTR_L2)) {
		amd_uncore_l2 = alloc_percpu(struct amd_uncore *);
		if (!amd_uncore_l2) {
			ret = -ENOMEM;
			goto fail_l2;
		}
		ret = perf_pmu_register(&amd_l2_pmu, amd_l2_pmu.name, -1);
		if (ret)
			goto fail_l2;

		pr_info("perf: AMD L2I counters detected\n");
		ret = 0;
	}

	/*
	 * Install callbacks. Core will call them for each online cpu.
	 */
	if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
			      "PERF_X86_AMD_UNCORE_PREP",
			      amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead))
		goto fail_l2;

	if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
			      "AP_PERF_X86_AMD_UNCORE_STARTING",
			      amd_uncore_cpu_starting, NULL))
		goto fail_prep;
	if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
			      "AP_PERF_X86_AMD_UNCORE_ONLINE",
			      amd_uncore_cpu_online,
			      amd_uncore_cpu_down_prepare))
		goto fail_start;
	return 0;

fail_start:
	cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
fail_prep:
	cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
fail_l2:
	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
		perf_pmu_unregister(&amd_nb_pmu);
	if (amd_uncore_l2)
		free_percpu(amd_uncore_l2);
fail_nb:
	if (amd_uncore_nb)
		free_percpu(amd_uncore_nb);

fail_nodev:
	return ret;
}
device_initcall(amd_uncore_init);
Commit	Line	Data
c43ca509 JS	1	/*
	2	* Copyright (C) 2013 Advanced Micro Devices, Inc.
	3	*
	4	* Author: Jacob Shin <jacob.shin@amd.com>
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*/
	10
	11	#include <linux/perf_event.h>
	12	#include <linux/percpu.h>
	13	#include <linux/types.h>
	14	#include <linux/slab.h>
	15	#include <linux/init.h>
	16	#include <linux/cpu.h>
	17	#include <linux/cpumask.h>
	18
	19	#include <asm/cpufeature.h>
	20	#include <asm/perf_event.h>
	21	#include <asm/msr.h>
	22
	23	#define NUM_COUNTERS_NB 4
	24	#define NUM_COUNTERS_L2 4
	25	#define MAX_COUNTERS NUM_COUNTERS_NB
	26
	27	#define RDPMC_BASE_NB 6
	28	#define RDPMC_BASE_L2 10
	29
	30	#define COUNTER_SHIFT 16
	31
	32	struct amd_uncore {
	33	int id;
	34	int refcnt;
	35	int cpu;
	36	int num_counters;
	37	int rdpmc_base;
	38	u32 msr_base;
	39	cpumask_t *active_mask;
	40	struct pmu *pmu;
	41	struct perf_event *events[MAX_COUNTERS];
	42	struct amd_uncore *free_when_cpu_online;
	43	};
	44
	45	static struct amd_uncore * __percpu *amd_uncore_nb;
	46	static struct amd_uncore * __percpu *amd_uncore_l2;
	47
	48	static struct pmu amd_nb_pmu;
	49	static struct pmu amd_l2_pmu;
	50
	51	static cpumask_t amd_nb_active_mask;
	52	static cpumask_t amd_l2_active_mask;
	53
	54	static bool is_nb_event(struct perf_event *event)
	55	{
	56	return event->pmu->type == amd_nb_pmu.type;
	57	}
	58
	59	static bool is_l2_event(struct perf_event *event)
	60	{
	61	return event->pmu->type == amd_l2_pmu.type;
	62	}
	63
	64	static struct amd_uncore event_to_amd_uncore(struct perf_event event)
65	{
66	if (is_nb_event(event) && amd_uncore_nb)
67	return *per_cpu_ptr(amd_uncore_nb, event->cpu);
68	else if (is_l2_event(event) && amd_uncore_l2)
69	return *per_cpu_ptr(amd_uncore_l2, event->cpu);
70
71	return NULL;
72	}
73
74	static void amd_uncore_read(struct perf_event *event)
75	{
76	struct hw_perf_event *hwc = &event->hw;
77	u64 prev, new;
78	s64 delta;
79
80	/*
81	* since we do not enable counter overflow interrupts,
82	* we do not have to worry about prev_count changing on us
83	*/
84
85	prev = local64_read(&hwc->prev_count);
86	rdpmcl(hwc->event_base_rdpmc, new);
87	local64_set(&hwc->prev_count, new);
88	delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
89	delta >>= COUNTER_SHIFT;
90	local64_add(delta, &event->count);
91	}
92
93	static void amd_uncore_start(struct perf_event *event, int flags)
94	{
95	struct hw_perf_event *hwc = &event->hw;
96
97	if (flags & PERF_EF_RELOAD)
98	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
99
100	hwc->state = 0;
101	wrmsrl(hwc->config_base, (hwc->config \| ARCH_PERFMON_EVENTSEL_ENABLE));
102	perf_event_update_userpage(event);
103	}
104
105	static void amd_uncore_stop(struct perf_event *event, int flags)
106	{
107	struct hw_perf_event *hwc = &event->hw;
108
109	wrmsrl(hwc->config_base, hwc->config);
110	hwc->state \|= PERF_HES_STOPPED;
111
112	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
113	amd_uncore_read(event);
114	hwc->state \|= PERF_HES_UPTODATE;
115	}
116	}
117
118	static int amd_uncore_add(struct perf_event *event, int flags)
119	{
120	int i;
121	struct amd_uncore *uncore = event_to_amd_uncore(event);
122	struct hw_perf_event *hwc = &event->hw;
123
124	/* are we already assigned? */
125	if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
126	goto out;
127
128	for (i = 0; i < uncore->num_counters; i++) {
129	if (uncore->events[i] == event) {
130	hwc->idx = i;
131	goto out;
132	}
133	}
134
135	/* if not, take the first available counter */
136	hwc->idx = -1;
137	for (i = 0; i < uncore->num_counters; i++) {
138	if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
139	hwc->idx = i;
140	break;
141	}
142	}
143
144	out:
145	if (hwc->idx == -1)
146	return -EBUSY;
147
148	hwc->config_base = uncore->msr_base + (2 * hwc->idx);
149	hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
150	hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
151	hwc->state = PERF_HES_UPTODATE \| PERF_HES_STOPPED;
152
153	if (flags & PERF_EF_START)
154	amd_uncore_start(event, PERF_EF_RELOAD);
155
156	return 0;
157	}
158
159	static void amd_uncore_del(struct perf_event *event, int flags)
160	{
161	int i;
162	struct amd_uncore *uncore = event_to_amd_uncore(event);
163	struct hw_perf_event *hwc = &event->hw;
164
165	amd_uncore_stop(event, PERF_EF_UPDATE);
166
167	for (i = 0; i < uncore->num_counters; i++) {
168	if (cmpxchg(&uncore->events[i], event, NULL) == event)
169	break;
170	}
171
172	hwc->idx = -1;
173	}
174
175	static int amd_uncore_event_init(struct perf_event *event)
176	{
177	struct amd_uncore *uncore;
178	struct hw_perf_event *hwc = &event->hw;
179
180	if (event->attr.type != event->pmu->type)
181	return -ENOENT;
182
183	/*
184	* NB and L2 counters (MSRs) are shared across all cores that share the
185	* same NB / L2 cache. Interrupts can be directed to a single target
186	* core, however, event counts generated by processes running on other
187	* cores cannot be masked out. So we do not support sampling and
188	* per-thread events.
189	*/
190	if (is_sampling_event(event) \|\| event->attach_state & PERF_ATTACH_TASK)
191	return -EINVAL;
192
193	/* NB and L2 counters do not have usr/os/guest/host bits */
194	if (event->attr.exclude_user \|\| event->attr.exclude_kernel \|\|
195	event->attr.exclude_host \|\| event->attr.exclude_guest)
196	return -EINVAL;
197
198	/* and we do not enable counter overflow interrupts */
199	hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
200	hwc->idx = -1;
201
202	if (event->cpu < 0)
203	return -EINVAL;
204
205	uncore = event_to_amd_uncore(event);
206	if (!uncore)
207	return -ENODEV;
208
209	/*
210	* since request can come in to any of the shared cores, we will remap
211	* to a single common cpu.
212	*/
213	event->cpu = uncore->cpu;
214
215	return 0;
216	}
217
218	static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
219	struct device_attribute *attr,
220	char *buf)
221	{
c43ca509 JS	222	cpumask_t *active_mask;
	223	struct pmu *pmu = dev_get_drvdata(dev);
	224
	225	if (pmu->type == amd_nb_pmu.type)
	226	active_mask = &amd_nb_active_mask;
	227	else if (pmu->type == amd_l2_pmu.type)
	228	active_mask = &amd_l2_active_mask;
	229	else
	230	return 0;
	231
5aaba363	232	return cpumap_print_to_pagebuf(true, buf, active_mask);
c43ca509 JS	233	}
	234	static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
	235
	236	static struct attribute *amd_uncore_attrs[] = {
	237	&dev_attr_cpumask.attr,
	238	NULL,
	239	};
	240
	241	static struct attribute_group amd_uncore_attr_group = {
	242	.attrs = amd_uncore_attrs,
	243	};
	244
	245	PMU_FORMAT_ATTR(event, "config:0-7,32-35");
	246	PMU_FORMAT_ATTR(umask, "config:8-15");
	247
	248	static struct attribute *amd_uncore_format_attr[] = {
	249	&format_attr_event.attr,
	250	&format_attr_umask.attr,
	251	NULL,
	252	};
	253
	254	static struct attribute_group amd_uncore_format_group = {
	255	.name = "format",
	256	.attrs = amd_uncore_format_attr,
	257	};
	258
	259	static const struct attribute_group *amd_uncore_attr_groups[] = {
	260	&amd_uncore_attr_group,
	261	&amd_uncore_format_group,
	262	NULL,
	263	};
	264
	265	static struct pmu amd_nb_pmu = {
31d50c55	266	.task_ctx_nr = perf_invalid_context,
c43ca509 JS	267	.attr_groups = amd_uncore_attr_groups,
	268	.name = "amd_nb",
	269	.event_init = amd_uncore_event_init,
	270	.add = amd_uncore_add,
	271	.del = amd_uncore_del,
	272	.start = amd_uncore_start,
	273	.stop = amd_uncore_stop,
	274	.read = amd_uncore_read,
	275	};
	276
	277	static struct pmu amd_l2_pmu = {
31d50c55	278	.task_ctx_nr = perf_invalid_context,
c43ca509 JS	279	.attr_groups = amd_uncore_attr_groups,
	280	.name = "amd_l2",
	281	.event_init = amd_uncore_event_init,
	282	.add = amd_uncore_add,
	283	.del = amd_uncore_del,
	284	.start = amd_uncore_start,
	285	.stop = amd_uncore_stop,
	286	.read = amd_uncore_read,
	287	};
	288
148f9bb8	289	static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
c43ca509 JS	290	{
	291	return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
	292	cpu_to_node(cpu));
	293	}
	294
503d3291	295	static int amd_uncore_cpu_up_prepare(unsigned int cpu)
c43ca509	296	{
503d3291	297	struct amd_uncore uncore_nb = NULL, uncore_l2;
c43ca509 JS	298
c43ca509 JS	299	if (amd_uncore_nb) {
503d3291 ZZ	300	uncore_nb = amd_uncore_alloc(cpu);
	301	if (!uncore_nb)
	302	goto fail;
	303	uncore_nb->cpu = cpu;
	304	uncore_nb->num_counters = NUM_COUNTERS_NB;
	305	uncore_nb->rdpmc_base = RDPMC_BASE_NB;
	306	uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
	307	uncore_nb->active_mask = &amd_nb_active_mask;
	308	uncore_nb->pmu = &amd_nb_pmu;
	309	*per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
c43ca509 JS	310	}
	311
	312	if (amd_uncore_l2) {
503d3291 ZZ	313	uncore_l2 = amd_uncore_alloc(cpu);
	314	if (!uncore_l2)
	315	goto fail;
	316	uncore_l2->cpu = cpu;
	317	uncore_l2->num_counters = NUM_COUNTERS_L2;
	318	uncore_l2->rdpmc_base = RDPMC_BASE_L2;
	319	uncore_l2->msr_base = MSR_F16H_L2I_PERF_CTL;
	320	uncore_l2->active_mask = &amd_l2_active_mask;
	321	uncore_l2->pmu = &amd_l2_pmu;
	322	*per_cpu_ptr(amd_uncore_l2, cpu) = uncore_l2;
c43ca509	323	}
503d3291 ZZ	324
	325	return 0;
	326
	327	fail:
8bc9162c TG	328	if (amd_uncore_nb)
8bc9162c TG	329	*per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
503d3291 ZZ	330	kfree(uncore_nb);
503d3291 ZZ	331	return -ENOMEM;
c43ca509 JS	332	}
	333
	334	static struct amd_uncore *
148f9bb8 PG	335	amd_uncore_find_online_sibling(struct amd_uncore *this,
148f9bb8 PG	336	struct amd_uncore * __percpu *uncores)
c43ca509 JS	337	{
	338	unsigned int cpu;
	339	struct amd_uncore *that;
	340
	341	for_each_online_cpu(cpu) {
	342	that = *per_cpu_ptr(uncores, cpu);
	343
	344	if (!that)
	345	continue;
	346
	347	if (this == that)
	348	continue;
	349
	350	if (this->id == that->id) {
	351	that->free_when_cpu_online = this;
	352	this = that;
	353	break;
	354	}
	355	}
	356
	357	this->refcnt++;
	358	return this;
	359	}
	360
96b2bd38	361	static int amd_uncore_cpu_starting(unsigned int cpu)
c43ca509 JS	362	{
	363	unsigned int eax, ebx, ecx, edx;
	364	struct amd_uncore *uncore;
	365
	366	if (amd_uncore_nb) {
	367	uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
	368	cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
	369	uncore->id = ecx & 0xff;
	370
	371	uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
	372	*per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
	373	}
	374
	375	if (amd_uncore_l2) {
	376	unsigned int apicid = cpu_data(cpu).apicid;
	377	unsigned int nshared;
	378
	379	uncore = *per_cpu_ptr(amd_uncore_l2, cpu);
	380	cpuid_count(0x8000001d, 2, &eax, &ebx, &ecx, &edx);
	381	nshared = ((eax >> 14) & 0xfff) + 1;
	382	uncore->id = apicid - (apicid % nshared);
	383
	384	uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_l2);
	385	*per_cpu_ptr(amd_uncore_l2, cpu) = uncore;
	386	}
96b2bd38 RC	387
96b2bd38 RC	388	return 0;
c43ca509 JS	389	}
c43ca509 JS	390
148f9bb8 PG	391	static void uncore_online(unsigned int cpu,
148f9bb8 PG	392	struct amd_uncore * __percpu *uncores)
c43ca509 JS	393	{
	394	struct amd_uncore uncore = per_cpu_ptr(uncores, cpu);
	395
	396	kfree(uncore->free_when_cpu_online);
	397	uncore->free_when_cpu_online = NULL;
	398
	399	if (cpu == uncore->cpu)
	400	cpumask_set_cpu(cpu, uncore->active_mask);
	401	}
	402
96b2bd38	403	static int amd_uncore_cpu_online(unsigned int cpu)
c43ca509 JS	404	{
	405	if (amd_uncore_nb)
	406	uncore_online(cpu, amd_uncore_nb);
	407
	408	if (amd_uncore_l2)
	409	uncore_online(cpu, amd_uncore_l2);
96b2bd38 RC	410
96b2bd38 RC	411	return 0;
c43ca509 JS	412	}
c43ca509 JS	413
148f9bb8 PG	414	static void uncore_down_prepare(unsigned int cpu,
148f9bb8 PG	415	struct amd_uncore * __percpu *uncores)
c43ca509 JS	416	{
	417	unsigned int i;
	418	struct amd_uncore this = per_cpu_ptr(uncores, cpu);
	419
	420	if (this->cpu != cpu)
	421	return;
	422
	423	/* this cpu is going down, migrate to a shared sibling if possible */
	424	for_each_online_cpu(i) {
	425	struct amd_uncore that = per_cpu_ptr(uncores, i);
	426
	427	if (cpu == i)
	428	continue;
	429
	430	if (this == that) {
	431	perf_pmu_migrate_context(this->pmu, cpu, i);
	432	cpumask_clear_cpu(cpu, that->active_mask);
	433	cpumask_set_cpu(i, that->active_mask);
	434	that->cpu = i;
	435	break;
	436	}
	437	}
	438	}
	439
96b2bd38	440	static int amd_uncore_cpu_down_prepare(unsigned int cpu)
c43ca509 JS	441	{
	442	if (amd_uncore_nb)
	443	uncore_down_prepare(cpu, amd_uncore_nb);
	444
	445	if (amd_uncore_l2)
	446	uncore_down_prepare(cpu, amd_uncore_l2);
96b2bd38 RC	447
96b2bd38 RC	448	return 0;
c43ca509 JS	449	}
c43ca509 JS	450
148f9bb8	451	static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
c43ca509 JS	452	{
	453	struct amd_uncore uncore = per_cpu_ptr(uncores, cpu);
	454
	455	if (cpu == uncore->cpu)
	456	cpumask_clear_cpu(cpu, uncore->active_mask);
	457
	458	if (!--uncore->refcnt)
	459	kfree(uncore);
503d3291	460	*per_cpu_ptr(uncores, cpu) = NULL;
c43ca509 JS	461	}
c43ca509 JS	462
96b2bd38	463	static int amd_uncore_cpu_dead(unsigned int cpu)
c43ca509 JS	464	{
	465	if (amd_uncore_nb)
	466	uncore_dead(cpu, amd_uncore_nb);
	467
	468	if (amd_uncore_l2)
	469	uncore_dead(cpu, amd_uncore_l2);
c43ca509	470
96b2bd38	471	return 0;
503d3291 ZZ	472	}
503d3291 ZZ	473
c43ca509 JS	474	static int __init amd_uncore_init(void)
c43ca509 JS	475	{
c43ca509 JS	476	int ret = -ENODEV;
	477
	478	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
503d3291	479	goto fail_nodev;
c43ca509	480
362f924b	481	if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
503d3291	482	goto fail_nodev;
c43ca509	483
362f924b	484	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
c43ca509	485	amd_uncore_nb = alloc_percpu(struct amd_uncore *);
503d3291 ZZ	486	if (!amd_uncore_nb) {
	487	ret = -ENOMEM;
	488	goto fail_nb;
	489	}
	490	ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
	491	if (ret)
	492	goto fail_nb;
c43ca509	493
1b74dde7	494	pr_info("perf: AMD NB counters detected\n");
c43ca509 JS	495	ret = 0;
	496	}
	497
362f924b	498	if (boot_cpu_has(X86_FEATURE_PERFCTR_L2)) {
c43ca509	499	amd_uncore_l2 = alloc_percpu(struct amd_uncore *);
503d3291 ZZ	500	if (!amd_uncore_l2) {
	501	ret = -ENOMEM;
	502	goto fail_l2;
	503	}
	504	ret = perf_pmu_register(&amd_l2_pmu, amd_l2_pmu.name, -1);
	505	if (ret)
	506	goto fail_l2;
c43ca509	507
1b74dde7	508	pr_info("perf: AMD L2I counters detected\n");
c43ca509 JS	509	ret = 0;
	510	}
	511
96b2bd38 RC	512	/*
	513	* Install callbacks. Core will call them for each online cpu.
	514	*/
	515	if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
	516	"PERF_X86_AMD_UNCORE_PREP",
	517	amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead))
	518	goto fail_l2;
	519
	520	if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
	521	"AP_PERF_X86_AMD_UNCORE_STARTING",
	522	amd_uncore_cpu_starting, NULL))
	523	goto fail_prep;
	524	if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
	525	"AP_PERF_X86_AMD_UNCORE_ONLINE",
	526	amd_uncore_cpu_online,
	527	amd_uncore_cpu_down_prepare))
	528	goto fail_start;
c43ca509	529	return 0;
503d3291	530
96b2bd38 RC	531	fail_start:
	532	cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
	533	fail_prep:
	534	cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
503d3291	535	fail_l2:
362f924b	536	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
503d3291 ZZ	537	perf_pmu_unregister(&amd_nb_pmu);
	538	if (amd_uncore_l2)
	539	free_percpu(amd_uncore_l2);
	540	fail_nb:
	541	if (amd_uncore_nb)
	542	free_percpu(amd_uncore_nb);
	543
	544	fail_nodev:
	545	return ret;
c43ca509 JS	546	}
c43ca509 JS	547	device_initcall(amd_uncore_init);