[deliverable/linux.git] / arch / x86 / xen / pmu.c

#include <linux/types.h>
#include <linux/interrupt.h>

#include <asm/xen/hypercall.h>
#include <xen/page.h>
#include <xen/interface/xen.h>
#include <xen/interface/vcpu.h>
#include <xen/interface/xenpmu.h>

#include "xen-ops.h"
#include "pmu.h"

/* x86_pmu.handle_irq definition */
#include "../kernel/cpu/perf_event.h"


/* Shared page between hypervisor and domain */
static DEFINE_PER_CPU(struct xen_pmu_data *, xenpmu_shared);
#define get_xenpmu_data()    per_cpu(xenpmu_shared, smp_processor_id())


/* AMD PMU */
#define F15H_NUM_COUNTERS   6
#define F10H_NUM_COUNTERS   4

static __read_mostly uint32_t amd_counters_base;
static __read_mostly uint32_t amd_ctrls_base;
static __read_mostly int amd_msr_step;
static __read_mostly int k7_counters_mirrored;
static __read_mostly int amd_num_counters;

/* Intel PMU */
#define MSR_TYPE_COUNTER            0
#define MSR_TYPE_CTRL               1
#define MSR_TYPE_GLOBAL             2
#define MSR_TYPE_ARCH_COUNTER       3
#define MSR_TYPE_ARCH_CTRL          4

/* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
#define PMU_GENERAL_NR_SHIFT        8
#define PMU_GENERAL_NR_BITS         8
#define PMU_GENERAL_NR_MASK         (((1 << PMU_GENERAL_NR_BITS) - 1) \
				     << PMU_GENERAL_NR_SHIFT)

/* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
#define PMU_FIXED_NR_SHIFT          0
#define PMU_FIXED_NR_BITS           5
#define PMU_FIXED_NR_MASK           (((1 << PMU_FIXED_NR_BITS) - 1) \
				     << PMU_FIXED_NR_SHIFT)

/* Alias registers (0x4c1) for full-width writes to PMCs */
#define MSR_PMC_ALIAS_MASK          (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))

#define INTEL_PMC_TYPE_SHIFT        30

static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;


static void xen_pmu_arch_init(void)
{
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {

		switch (boot_cpu_data.x86) {
		case 0x15:
			amd_num_counters = F15H_NUM_COUNTERS;
			amd_counters_base = MSR_F15H_PERF_CTR;
			amd_ctrls_base = MSR_F15H_PERF_CTL;
			amd_msr_step = 2;
			k7_counters_mirrored = 1;
			break;
		case 0x10:
		case 0x12:
		case 0x14:
		case 0x16:
		default:
			amd_num_counters = F10H_NUM_COUNTERS;
			amd_counters_base = MSR_K7_PERFCTR0;
			amd_ctrls_base = MSR_K7_EVNTSEL0;
			amd_msr_step = 1;
			k7_counters_mirrored = 0;
			break;
		}
	} else {
		uint32_t eax, ebx, ecx, edx;

		cpuid(0xa, &eax, &ebx, &ecx, &edx);

		intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
			PMU_GENERAL_NR_SHIFT;
		intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
			PMU_FIXED_NR_SHIFT;
	}
}

static inline uint32_t get_fam15h_addr(u32 addr)
{
	switch (addr) {
	case MSR_K7_PERFCTR0:
	case MSR_K7_PERFCTR1:
	case MSR_K7_PERFCTR2:
	case MSR_K7_PERFCTR3:
		return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
	case MSR_K7_EVNTSEL0:
	case MSR_K7_EVNTSEL1:
	case MSR_K7_EVNTSEL2:
	case MSR_K7_EVNTSEL3:
		return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
	default:
		break;
	}

	return addr;
}

static inline bool is_amd_pmu_msr(unsigned int msr)
{
	if ((msr >= MSR_F15H_PERF_CTL &&
	     msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) ||
	    (msr >= MSR_K7_EVNTSEL0 &&
	     msr < MSR_K7_PERFCTR0 + amd_num_counters))
		return true;

	return false;
}

static int is_intel_pmu_msr(u32 msr_index, int *type, int *index)
{
	u32 msr_index_pmc;

	switch (msr_index) {
	case MSR_CORE_PERF_FIXED_CTR_CTRL:
	case MSR_IA32_DS_AREA:
	case MSR_IA32_PEBS_ENABLE:
		*type = MSR_TYPE_CTRL;
		return true;

	case MSR_CORE_PERF_GLOBAL_CTRL:
	case MSR_CORE_PERF_GLOBAL_STATUS:
	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
		*type = MSR_TYPE_GLOBAL;
		return true;

	default:

		if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
		    (msr_index < MSR_CORE_PERF_FIXED_CTR0 +
				 intel_num_fixed_counters)) {
			*index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
			*type = MSR_TYPE_COUNTER;
			return true;
		}

		if ((msr_index >= MSR_P6_EVNTSEL0) &&
		    (msr_index < MSR_P6_EVNTSEL0 +  intel_num_arch_counters)) {
			*index = msr_index - MSR_P6_EVNTSEL0;
			*type = MSR_TYPE_ARCH_CTRL;
			return true;
		}

		msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
		if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
		    (msr_index_pmc < MSR_IA32_PERFCTR0 +
				     intel_num_arch_counters)) {
			*type = MSR_TYPE_ARCH_COUNTER;
			*index = msr_index_pmc - MSR_IA32_PERFCTR0;
			return true;
		}
		return false;
	}
}

bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
{

	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
		if (is_amd_pmu_msr(msr)) {
			*val = native_read_msr_safe(msr, err);
			return true;
		}
	} else {
		int type, index;

		if (is_intel_pmu_msr(msr, &type, &index)) {
			*val = native_read_msr_safe(msr, err);
			return true;
		}
	}

	return false;
}

bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
{
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
		if (is_amd_pmu_msr(msr)) {
			*err = native_write_msr_safe(msr, low, high);
			return true;
		}
	} else {
		int type, index;

		if (is_intel_pmu_msr(msr, &type, &index)) {
			*err = native_write_msr_safe(msr, low, high);
			return true;
		}
	}

	return false;
}

static unsigned long long xen_amd_read_pmc(int counter)
{
	uint32_t msr;
	int err;

	msr = amd_counters_base + (counter * amd_msr_step);
	return native_read_msr_safe(msr, &err);
}

static unsigned long long xen_intel_read_pmc(int counter)
{
	int err;
	uint32_t msr;

	if (counter & (1<<INTEL_PMC_TYPE_SHIFT))
		msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
	else
		msr = MSR_IA32_PERFCTR0 + counter;

	return native_read_msr_safe(msr, &err);
}

unsigned long long xen_read_pmc(int counter)
{
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
		return xen_amd_read_pmc(counter);
	else
		return xen_intel_read_pmc(counter);
}

int pmu_apic_update(uint32_t val)
{
	int ret;
	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return -EINVAL;
	}

	xenpmu_data->pmu.l.lapic_lvtpc = val;
	ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);

	return ret;
}

/* perf callbacks */
static int xen_is_in_guest(void)
{
	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return 0;
	}

	if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
		return 0;

	return 1;
}

static int xen_is_user_mode(void)
{
	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return 0;
	}

	if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV)
		return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER);
	else
		return !!(xenpmu_data->pmu.r.regs.cpl & 3);
}

static unsigned long xen_get_guest_ip(void)
{
	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return 0;
	}

	return xenpmu_data->pmu.r.regs.ip;
}

static struct perf_guest_info_callbacks xen_guest_cbs = {
	.is_in_guest            = xen_is_in_guest,
	.is_user_mode           = xen_is_user_mode,
	.get_guest_ip           = xen_get_guest_ip,
};

/* Convert registers from Xen's format to Linux' */
static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
			     struct pt_regs *regs, uint64_t pmu_flags)
{
	regs->ip = xen_regs->ip;
	regs->cs = xen_regs->cs;
	regs->sp = xen_regs->sp;

	if (pmu_flags & PMU_SAMPLE_PV) {
		if (pmu_flags & PMU_SAMPLE_USER)
			regs->cs |= 3;
		else
			regs->cs &= ~3;
	} else {
		if (xen_regs->cpl)
			regs->cs |= 3;
		else
			regs->cs &= ~3;
	}
}

irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
{
	int err, ret = IRQ_NONE;
	struct pt_regs regs;
	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return ret;
	}

	err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
	if (err) {
		pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
		return ret;
	}

	xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
			 xenpmu_data->pmu.pmu_flags);
	if (x86_pmu.handle_irq(&regs))
		ret = IRQ_HANDLED;

	return ret;
}

bool is_xen_pmu(int cpu)
{
	return (per_cpu(xenpmu_shared, cpu) != NULL);
}

void xen_pmu_init(int cpu)
{
	int err;
	struct xen_pmu_params xp;
	unsigned long pfn;
	struct xen_pmu_data *xenpmu_data;

	BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);

	if (xen_hvm_domain())
		return;

	xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
	if (!xenpmu_data) {
		pr_err("VPMU init: No memory\n");
		return;
	}
	pfn = virt_to_pfn(xenpmu_data);

	xp.val = pfn_to_mfn(pfn);
	xp.vcpu = cpu;
	xp.version.maj = XENPMU_VER_MAJ;
	xp.version.min = XENPMU_VER_MIN;
	err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
	if (err)
		goto fail;

	per_cpu(xenpmu_shared, cpu) = xenpmu_data;

	if (cpu == 0) {
		perf_register_guest_info_callbacks(&xen_guest_cbs);
		xen_pmu_arch_init();
	}

	return;

fail:
	pr_warn_once("Could not initialize VPMU for cpu %d, error %d\n",
		cpu, err);
	free_pages((unsigned long)xenpmu_data, 0);
}

void xen_pmu_finish(int cpu)
{
	struct xen_pmu_params xp;

	if (xen_hvm_domain())
		return;

	xp.vcpu = cpu;
	xp.version.maj = XENPMU_VER_MAJ;
	xp.version.min = XENPMU_VER_MIN;

	(void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);

	free_pages((unsigned long)per_cpu(xenpmu_shared, cpu), 0);
	per_cpu(xenpmu_shared, cpu) = NULL;
}
Commit	Line	Data
65d0cf0b BO	1	#include <linux/types.h>
	2	#include <linux/interrupt.h>
	3
	4	#include <asm/xen/hypercall.h>
	5	#include <xen/page.h>
	6	#include <xen/interface/xen.h>
	7	#include <xen/interface/vcpu.h>
	8	#include <xen/interface/xenpmu.h>
	9
	10	#include "xen-ops.h"
	11	#include "pmu.h"
	12
	13	/* x86_pmu.handle_irq definition */
	14	#include "../kernel/cpu/perf_event.h"
	15
	16
	17	/* Shared page between hypervisor and domain */
	18	static DEFINE_PER_CPU(struct xen_pmu_data *, xenpmu_shared);
	19	#define get_xenpmu_data() per_cpu(xenpmu_shared, smp_processor_id())
	20
e27b72df BO	21
	22	/* AMD PMU */
	23	#define F15H_NUM_COUNTERS 6
	24	#define F10H_NUM_COUNTERS 4
	25
	26	static __read_mostly uint32_t amd_counters_base;
	27	static __read_mostly uint32_t amd_ctrls_base;
	28	static __read_mostly int amd_msr_step;
	29	static __read_mostly int k7_counters_mirrored;
	30	static __read_mostly int amd_num_counters;
	31
	32	/* Intel PMU */
	33	#define MSR_TYPE_COUNTER 0
	34	#define MSR_TYPE_CTRL 1
	35	#define MSR_TYPE_GLOBAL 2
	36	#define MSR_TYPE_ARCH_COUNTER 3
	37	#define MSR_TYPE_ARCH_CTRL 4
	38
	39	/* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
	40	#define PMU_GENERAL_NR_SHIFT 8
	41	#define PMU_GENERAL_NR_BITS 8
	42	#define PMU_GENERAL_NR_MASK (((1 << PMU_GENERAL_NR_BITS) - 1) \
	43	<< PMU_GENERAL_NR_SHIFT)
	44
	45	/* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
	46	#define PMU_FIXED_NR_SHIFT 0
	47	#define PMU_FIXED_NR_BITS 5
	48	#define PMU_FIXED_NR_MASK (((1 << PMU_FIXED_NR_BITS) - 1) \
	49	<< PMU_FIXED_NR_SHIFT)
	50
	51	/* Alias registers (0x4c1) for full-width writes to PMCs */
	52	#define MSR_PMC_ALIAS_MASK (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))
	53
6b08cd63 BO	54	#define INTEL_PMC_TYPE_SHIFT 30
6b08cd63 BO	55
e27b72df BO	56	static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;
	57
	58
	59	static void xen_pmu_arch_init(void)
	60	{
	61	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
	62
	63	switch (boot_cpu_data.x86) {
	64	case 0x15:
	65	amd_num_counters = F15H_NUM_COUNTERS;
	66	amd_counters_base = MSR_F15H_PERF_CTR;
	67	amd_ctrls_base = MSR_F15H_PERF_CTL;
	68	amd_msr_step = 2;
	69	k7_counters_mirrored = 1;
	70	break;
	71	case 0x10:
	72	case 0x12:
	73	case 0x14:
	74	case 0x16:
	75	default:
	76	amd_num_counters = F10H_NUM_COUNTERS;
	77	amd_counters_base = MSR_K7_PERFCTR0;
	78	amd_ctrls_base = MSR_K7_EVNTSEL0;
	79	amd_msr_step = 1;
	80	k7_counters_mirrored = 0;
	81	break;
	82	}
	83	} else {
	84	uint32_t eax, ebx, ecx, edx;
	85
	86	cpuid(0xa, &eax, &ebx, &ecx, &edx);
	87
	88	intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
	89	PMU_GENERAL_NR_SHIFT;
	90	intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
	91	PMU_FIXED_NR_SHIFT;
	92	}
	93	}
	94
	95	static inline uint32_t get_fam15h_addr(u32 addr)
	96	{
	97	switch (addr) {
	98	case MSR_K7_PERFCTR0:
	99	case MSR_K7_PERFCTR1:
	100	case MSR_K7_PERFCTR2:
	101	case MSR_K7_PERFCTR3:
	102	return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
	103	case MSR_K7_EVNTSEL0:
	104	case MSR_K7_EVNTSEL1:
	105	case MSR_K7_EVNTSEL2:
	106	case MSR_K7_EVNTSEL3:
	107	return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
	108	default:
	109	break;
	110	}
	111
	112	return addr;
	113	}
	114
	115	static inline bool is_amd_pmu_msr(unsigned int msr)
	116	{
	117	if ((msr >= MSR_F15H_PERF_CTL &&
	118	msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) \|\|
	119	(msr >= MSR_K7_EVNTSEL0 &&
120	msr < MSR_K7_PERFCTR0 + amd_num_counters))
121	return true;
122
123	return false;
124	}
125
126	static int is_intel_pmu_msr(u32 msr_index, int type, int index)
127	{
128	u32 msr_index_pmc;
129
130	switch (msr_index) {
131	case MSR_CORE_PERF_FIXED_CTR_CTRL:
132	case MSR_IA32_DS_AREA:
133	case MSR_IA32_PEBS_ENABLE:
134	*type = MSR_TYPE_CTRL;
135	return true;
136
137	case MSR_CORE_PERF_GLOBAL_CTRL:
138	case MSR_CORE_PERF_GLOBAL_STATUS:
139	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
140	*type = MSR_TYPE_GLOBAL;
141	return true;
142
143	default:
144
145	if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
146	(msr_index < MSR_CORE_PERF_FIXED_CTR0 +
147	intel_num_fixed_counters)) {
148	*index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
149	*type = MSR_TYPE_COUNTER;
150	return true;
151	}
152
153	if ((msr_index >= MSR_P6_EVNTSEL0) &&
154	(msr_index < MSR_P6_EVNTSEL0 + intel_num_arch_counters)) {
155	*index = msr_index - MSR_P6_EVNTSEL0;
156	*type = MSR_TYPE_ARCH_CTRL;
157	return true;
158	}
159
160	msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
161	if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
162	(msr_index_pmc < MSR_IA32_PERFCTR0 +
163	intel_num_arch_counters)) {
164	*type = MSR_TYPE_ARCH_COUNTER;
165	*index = msr_index_pmc - MSR_IA32_PERFCTR0;
166	return true;
167	}
168	return false;
169	}
170	}
171
6b08cd63 BO	172	bool pmu_msr_read(unsigned int msr, uint64_t val, int err)
	173	{
	174
	175	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
	176	if (is_amd_pmu_msr(msr)) {
	177	*val = native_read_msr_safe(msr, err);
	178	return true;
	179	}
	180	} else {
	181	int type, index;
	182
	183	if (is_intel_pmu_msr(msr, &type, &index)) {
	184	*val = native_read_msr_safe(msr, err);
	185	return true;
	186	}
	187	}
	188
	189	return false;
	190	}
	191
	192	bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
	193	{
	194	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
	195	if (is_amd_pmu_msr(msr)) {
	196	*err = native_write_msr_safe(msr, low, high);
	197	return true;
	198	}
	199	} else {
	200	int type, index;
	201
	202	if (is_intel_pmu_msr(msr, &type, &index)) {
	203	*err = native_write_msr_safe(msr, low, high);
	204	return true;
	205	}
	206	}
	207
	208	return false;
	209	}
	210
	211	static unsigned long long xen_amd_read_pmc(int counter)
	212	{
	213	uint32_t msr;
	214	int err;
	215
	216	msr = amd_counters_base + (counter * amd_msr_step);
	217	return native_read_msr_safe(msr, &err);
	218	}
	219
	220	static unsigned long long xen_intel_read_pmc(int counter)
	221	{
	222	int err;
	223	uint32_t msr;
	224
	225	if (counter & (1<<INTEL_PMC_TYPE_SHIFT))
	226	msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
	227	else
	228	msr = MSR_IA32_PERFCTR0 + counter;
	229
	230	return native_read_msr_safe(msr, &err);
	231	}
	232
	233	unsigned long long xen_read_pmc(int counter)
	234	{
	235	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
236	return xen_amd_read_pmc(counter);
237	else
238	return xen_intel_read_pmc(counter);
239	}
240
241	int pmu_apic_update(uint32_t val)
242	{
243	int ret;
244	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
245
246	if (!xenpmu_data) {
247	pr_warn_once("%s: pmudata not initialized\n", __func__);
248	return -EINVAL;
249	}
250
251	xenpmu_data->pmu.l.lapic_lvtpc = val;
252	ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);
253
254	return ret;
255	}
256
65d0cf0b BO	257	/* perf callbacks */
	258	static int xen_is_in_guest(void)
	259	{
	260	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	261
	262	if (!xenpmu_data) {
	263	pr_warn_once("%s: pmudata not initialized\n", __func__);
	264	return 0;
	265	}
	266
	267	if (!xen_initial_domain() \|\| (xenpmu_data->domain_id >= DOMID_SELF))
	268	return 0;
	269
	270	return 1;
	271	}
	272
	273	static int xen_is_user_mode(void)
	274	{
	275	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	276
	277	if (!xenpmu_data) {
	278	pr_warn_once("%s: pmudata not initialized\n", __func__);
	279	return 0;
	280	}
	281
	282	if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV)
	283	return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER);
	284	else
	285	return !!(xenpmu_data->pmu.r.regs.cpl & 3);
	286	}
	287
	288	static unsigned long xen_get_guest_ip(void)
	289	{
	290	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	291
	292	if (!xenpmu_data) {
	293	pr_warn_once("%s: pmudata not initialized\n", __func__);
	294	return 0;
	295	}
	296
	297	return xenpmu_data->pmu.r.regs.ip;
	298	}
	299
	300	static struct perf_guest_info_callbacks xen_guest_cbs = {
	301	.is_in_guest = xen_is_in_guest,
	302	.is_user_mode = xen_is_user_mode,
	303	.get_guest_ip = xen_get_guest_ip,
	304	};
	305
	306	/* Convert registers from Xen's format to Linux' */
	307	static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
	308	struct pt_regs *regs, uint64_t pmu_flags)
	309	{
	310	regs->ip = xen_regs->ip;
	311	regs->cs = xen_regs->cs;
	312	regs->sp = xen_regs->sp;
	313
	314	if (pmu_flags & PMU_SAMPLE_PV) {
	315	if (pmu_flags & PMU_SAMPLE_USER)
	316	regs->cs \|= 3;
	317	else
	318	regs->cs &= ~3;
	319	} else {
	320	if (xen_regs->cpl)
321	regs->cs \|= 3;
322	else
323	regs->cs &= ~3;
324	}
325	}
326
327	irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
328	{
6b08cd63	329	int err, ret = IRQ_NONE;
65d0cf0b BO	330	struct pt_regs regs;
	331	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	332
	333	if (!xenpmu_data) {
	334	pr_warn_once("%s: pmudata not initialized\n", __func__);
	335	return ret;
	336	}
	337
6b08cd63 BO	338	err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
	339	if (err) {
	340	pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
	341	return ret;
	342	}
	343
65d0cf0b BO	344	xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
	345	xenpmu_data->pmu.pmu_flags);
	346	if (x86_pmu.handle_irq(&regs))
	347	ret = IRQ_HANDLED;
	348
	349	return ret;
	350	}
	351
	352	bool is_xen_pmu(int cpu)
	353	{
	354	return (per_cpu(xenpmu_shared, cpu) != NULL);
	355	}
	356
	357	void xen_pmu_init(int cpu)
	358	{
	359	int err;
	360	struct xen_pmu_params xp;
	361	unsigned long pfn;
	362	struct xen_pmu_data *xenpmu_data;
	363
	364	BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
	365
	366	if (xen_hvm_domain())
	367	return;
	368
	369	xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
	370	if (!xenpmu_data) {
	371	pr_err("VPMU init: No memory\n");
	372	return;
	373	}
	374	pfn = virt_to_pfn(xenpmu_data);
	375
	376	xp.val = pfn_to_mfn(pfn);
	377	xp.vcpu = cpu;
	378	xp.version.maj = XENPMU_VER_MAJ;
	379	xp.version.min = XENPMU_VER_MIN;
	380	err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
	381	if (err)
	382	goto fail;
	383
	384	per_cpu(xenpmu_shared, cpu) = xenpmu_data;
	385
e27b72df	386	if (cpu == 0) {
65d0cf0b	387	perf_register_guest_info_callbacks(&xen_guest_cbs);
e27b72df BO	388	xen_pmu_arch_init();
e27b72df BO	389	}
65d0cf0b BO	390
	391	return;
	392
	393	fail:
	394	pr_warn_once("Could not initialize VPMU for cpu %d, error %d\n",
	395	cpu, err);
	396	free_pages((unsigned long)xenpmu_data, 0);
	397	}
	398
	399	void xen_pmu_finish(int cpu)
	400	{
	401	struct xen_pmu_params xp;
	402
	403	if (xen_hvm_domain())
	404	return;
	405
	406	xp.vcpu = cpu;
	407	xp.version.maj = XENPMU_VER_MAJ;
	408	xp.version.min = XENPMU_VER_MIN;
	409
	410	(void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);
	411
	412	free_pages((unsigned long)per_cpu(xenpmu_shared, cpu), 0);
	413	per_cpu(xenpmu_shared, cpu) = NULL;
	414	}