Commit | Line | Data |
---|---|---|
f5132b01 | 1 | /* |
c7a7062f | 2 | * Kernel-based Virtual Machine -- Performance Monitoring Unit support |
f5132b01 GN |
3 | * |
4 | * Copyright 2011 Red Hat, Inc. and/or its affiliates. | |
5 | * | |
6 | * Authors: | |
7 | * Avi Kivity <avi@redhat.com> | |
8 | * Gleb Natapov <gleb@redhat.com> | |
9 | * | |
10 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
11 | * the COPYING file in the top-level directory. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <linux/types.h> | |
16 | #include <linux/kvm_host.h> | |
17 | #include <linux/perf_event.h> | |
d27aa7f1 | 18 | #include <asm/perf_event.h> |
f5132b01 GN |
19 | #include "x86.h" |
20 | #include "cpuid.h" | |
21 | #include "lapic.h" | |
474a5bb9 | 22 | #include "pmu.h" |
f5132b01 | 23 | |
474a5bb9 | 24 | static struct kvm_event_hw_type_mapping arch_events[] = { |
f5132b01 GN |
25 | /* Index must match CPUID 0x0A.EBX bit vector */ |
26 | [0] = { 0x3c, 0x00, PERF_COUNT_HW_CPU_CYCLES }, | |
27 | [1] = { 0xc0, 0x00, PERF_COUNT_HW_INSTRUCTIONS }, | |
28 | [2] = { 0x3c, 0x01, PERF_COUNT_HW_BUS_CYCLES }, | |
29 | [3] = { 0x2e, 0x4f, PERF_COUNT_HW_CACHE_REFERENCES }, | |
30 | [4] = { 0x2e, 0x41, PERF_COUNT_HW_CACHE_MISSES }, | |
31 | [5] = { 0xc4, 0x00, PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, | |
32 | [6] = { 0xc5, 0x00, PERF_COUNT_HW_BRANCH_MISSES }, | |
62079d8a | 33 | [7] = { 0x00, 0x30, PERF_COUNT_HW_REF_CPU_CYCLES }, |
f5132b01 GN |
34 | }; |
35 | ||
36 | /* mapping between fixed pmc index and arch_events array */ | |
52eb5a6d | 37 | static int fixed_pmc_events[] = {1, 0, 7}; |
f5132b01 GN |
38 | |
39 | static bool pmc_is_gp(struct kvm_pmc *pmc) | |
40 | { | |
41 | return pmc->type == KVM_PMC_GP; | |
42 | } | |
43 | ||
44 | static inline u64 pmc_bitmask(struct kvm_pmc *pmc) | |
45 | { | |
212dba12 | 46 | struct kvm_pmu *pmu = pmc_to_pmu(pmc); |
f5132b01 GN |
47 | |
48 | return pmu->counter_bitmask[pmc->type]; | |
49 | } | |
50 | ||
c6702c9d | 51 | static inline bool pmc_is_enabled(struct kvm_pmc *pmc) |
f5132b01 | 52 | { |
212dba12 | 53 | struct kvm_pmu *pmu = pmc_to_pmu(pmc); |
f5132b01 GN |
54 | return test_bit(pmc->idx, (unsigned long *)&pmu->global_ctrl); |
55 | } | |
56 | ||
57 | static inline struct kvm_pmc *get_gp_pmc(struct kvm_pmu *pmu, u32 msr, | |
58 | u32 base) | |
59 | { | |
60 | if (msr >= base && msr < base + pmu->nr_arch_gp_counters) | |
61 | return &pmu->gp_counters[msr - base]; | |
62 | return NULL; | |
63 | } | |
64 | ||
65 | static inline struct kvm_pmc *get_fixed_pmc(struct kvm_pmu *pmu, u32 msr) | |
66 | { | |
67 | int base = MSR_CORE_PERF_FIXED_CTR0; | |
68 | if (msr >= base && msr < base + pmu->nr_arch_fixed_counters) | |
69 | return &pmu->fixed_counters[msr - base]; | |
70 | return NULL; | |
71 | } | |
72 | ||
73 | static inline struct kvm_pmc *get_fixed_pmc_idx(struct kvm_pmu *pmu, int idx) | |
74 | { | |
75 | return get_fixed_pmc(pmu, MSR_CORE_PERF_FIXED_CTR0 + idx); | |
76 | } | |
77 | ||
78 | static struct kvm_pmc *global_idx_to_pmc(struct kvm_pmu *pmu, int idx) | |
79 | { | |
15c7ad51 | 80 | if (idx < INTEL_PMC_IDX_FIXED) |
f5132b01 GN |
81 | return get_gp_pmc(pmu, MSR_P6_EVNTSEL0 + idx, MSR_P6_EVNTSEL0); |
82 | else | |
15c7ad51 | 83 | return get_fixed_pmc_idx(pmu, idx - INTEL_PMC_IDX_FIXED); |
f5132b01 GN |
84 | } |
85 | ||
c6702c9d | 86 | static void kvm_pmi_trigger_fn(struct irq_work *irq_work) |
f5132b01 | 87 | { |
212dba12 WH |
88 | struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work); |
89 | struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu); | |
f5132b01 | 90 | |
c6702c9d | 91 | kvm_pmu_deliver_pmi(vcpu); |
f5132b01 GN |
92 | } |
93 | ||
94 | static void kvm_perf_overflow(struct perf_event *perf_event, | |
95 | struct perf_sample_data *data, | |
96 | struct pt_regs *regs) | |
97 | { | |
98 | struct kvm_pmc *pmc = perf_event->overflow_handler_context; | |
212dba12 | 99 | struct kvm_pmu *pmu = pmc_to_pmu(pmc); |
e84cfe4c WH |
100 | |
101 | if (!test_and_set_bit(pmc->idx, | |
102 | (unsigned long *)&pmu->reprogram_pmi)) { | |
671bd993 NA |
103 | __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); |
104 | kvm_make_request(KVM_REQ_PMU, pmc->vcpu); | |
105 | } | |
f5132b01 GN |
106 | } |
107 | ||
108 | static void kvm_perf_overflow_intr(struct perf_event *perf_event, | |
e84cfe4c WH |
109 | struct perf_sample_data *data, |
110 | struct pt_regs *regs) | |
f5132b01 GN |
111 | { |
112 | struct kvm_pmc *pmc = perf_event->overflow_handler_context; | |
212dba12 | 113 | struct kvm_pmu *pmu = pmc_to_pmu(pmc); |
e84cfe4c WH |
114 | |
115 | if (!test_and_set_bit(pmc->idx, | |
116 | (unsigned long *)&pmu->reprogram_pmi)) { | |
671bd993 | 117 | __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); |
f5132b01 | 118 | kvm_make_request(KVM_REQ_PMU, pmc->vcpu); |
e84cfe4c | 119 | |
f5132b01 GN |
120 | /* |
121 | * Inject PMI. If vcpu was in a guest mode during NMI PMI | |
122 | * can be ejected on a guest mode re-entry. Otherwise we can't | |
123 | * be sure that vcpu wasn't executing hlt instruction at the | |
e84cfe4c | 124 | * time of vmexit and is not going to re-enter guest mode until |
f5132b01 GN |
125 | * woken up. So we should wake it, but this is impossible from |
126 | * NMI context. Do it from irq work instead. | |
127 | */ | |
128 | if (!kvm_is_in_guest()) | |
212dba12 | 129 | irq_work_queue(&pmc_to_pmu(pmc)->irq_work); |
f5132b01 GN |
130 | else |
131 | kvm_make_request(KVM_REQ_PMI, pmc->vcpu); | |
132 | } | |
133 | } | |
134 | ||
c6702c9d | 135 | static u64 pmc_read_counter(struct kvm_pmc *pmc) |
f5132b01 GN |
136 | { |
137 | u64 counter, enabled, running; | |
138 | ||
139 | counter = pmc->counter; | |
140 | ||
141 | if (pmc->perf_event) | |
142 | counter += perf_event_read_value(pmc->perf_event, | |
143 | &enabled, &running); | |
144 | ||
145 | /* FIXME: Scaling needed? */ | |
146 | ||
147 | return counter & pmc_bitmask(pmc); | |
148 | } | |
149 | ||
c6702c9d | 150 | static void pmc_stop_counter(struct kvm_pmc *pmc) |
f5132b01 GN |
151 | { |
152 | if (pmc->perf_event) { | |
c6702c9d | 153 | pmc->counter = pmc_read_counter(pmc); |
f5132b01 GN |
154 | perf_event_release_kernel(pmc->perf_event); |
155 | pmc->perf_event = NULL; | |
156 | } | |
157 | } | |
158 | ||
c6702c9d | 159 | static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, |
e84cfe4c WH |
160 | unsigned config, bool exclude_user, |
161 | bool exclude_kernel, bool intr, | |
162 | bool in_tx, bool in_tx_cp) | |
f5132b01 GN |
163 | { |
164 | struct perf_event *event; | |
165 | struct perf_event_attr attr = { | |
166 | .type = type, | |
167 | .size = sizeof(attr), | |
168 | .pinned = true, | |
169 | .exclude_idle = true, | |
170 | .exclude_host = 1, | |
171 | .exclude_user = exclude_user, | |
172 | .exclude_kernel = exclude_kernel, | |
173 | .config = config, | |
174 | }; | |
e84cfe4c | 175 | |
103af0a9 AK |
176 | if (in_tx) |
177 | attr.config |= HSW_IN_TX; | |
178 | if (in_tx_cp) | |
179 | attr.config |= HSW_IN_TX_CHECKPOINTED; | |
f5132b01 GN |
180 | |
181 | attr.sample_period = (-pmc->counter) & pmc_bitmask(pmc); | |
182 | ||
183 | event = perf_event_create_kernel_counter(&attr, -1, current, | |
184 | intr ? kvm_perf_overflow_intr : | |
185 | kvm_perf_overflow, pmc); | |
186 | if (IS_ERR(event)) { | |
e84cfe4c WH |
187 | printk_once("kvm_pmu: event creation failed %ld\n", |
188 | PTR_ERR(event)); | |
f5132b01 GN |
189 | return; |
190 | } | |
191 | ||
192 | pmc->perf_event = event; | |
212dba12 | 193 | clear_bit(pmc->idx, (unsigned long*)&pmc_to_pmu(pmc)->reprogram_pmi); |
f5132b01 GN |
194 | } |
195 | ||
196 | static unsigned find_arch_event(struct kvm_pmu *pmu, u8 event_select, | |
197 | u8 unit_mask) | |
198 | { | |
199 | int i; | |
200 | ||
201 | for (i = 0; i < ARRAY_SIZE(arch_events); i++) | |
202 | if (arch_events[i].eventsel == event_select | |
203 | && arch_events[i].unit_mask == unit_mask | |
204 | && (pmu->available_event_types & (1 << i))) | |
205 | break; | |
206 | ||
207 | if (i == ARRAY_SIZE(arch_events)) | |
208 | return PERF_COUNT_HW_MAX; | |
209 | ||
210 | return arch_events[i].event_type; | |
211 | } | |
212 | ||
213 | static void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) | |
214 | { | |
215 | unsigned config, type = PERF_TYPE_RAW; | |
216 | u8 event_select, unit_mask; | |
217 | ||
a7b9d2cc GN |
218 | if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) |
219 | printk_once("kvm pmu: pin control bit is ignored\n"); | |
220 | ||
f5132b01 GN |
221 | pmc->eventsel = eventsel; |
222 | ||
c6702c9d | 223 | pmc_stop_counter(pmc); |
f5132b01 | 224 | |
c6702c9d | 225 | if (!(eventsel & ARCH_PERFMON_EVENTSEL_ENABLE) || !pmc_is_enabled(pmc)) |
f5132b01 GN |
226 | return; |
227 | ||
228 | event_select = eventsel & ARCH_PERFMON_EVENTSEL_EVENT; | |
229 | unit_mask = (eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; | |
230 | ||
fac33683 | 231 | if (!(eventsel & (ARCH_PERFMON_EVENTSEL_EDGE | |
e84cfe4c WH |
232 | ARCH_PERFMON_EVENTSEL_INV | |
233 | ARCH_PERFMON_EVENTSEL_CMASK | | |
234 | HSW_IN_TX | | |
235 | HSW_IN_TX_CHECKPOINTED))) { | |
212dba12 | 236 | config = find_arch_event(pmc_to_pmu(pmc), event_select, |
f5132b01 GN |
237 | unit_mask); |
238 | if (config != PERF_COUNT_HW_MAX) | |
239 | type = PERF_TYPE_HARDWARE; | |
240 | } | |
241 | ||
242 | if (type == PERF_TYPE_RAW) | |
243 | config = eventsel & X86_RAW_EVENT_MASK; | |
244 | ||
c6702c9d | 245 | pmc_reprogram_counter(pmc, type, config, |
e84cfe4c WH |
246 | !(eventsel & ARCH_PERFMON_EVENTSEL_USR), |
247 | !(eventsel & ARCH_PERFMON_EVENTSEL_OS), | |
248 | eventsel & ARCH_PERFMON_EVENTSEL_INT, | |
249 | (eventsel & HSW_IN_TX), | |
250 | (eventsel & HSW_IN_TX_CHECKPOINTED)); | |
f5132b01 GN |
251 | } |
252 | ||
e84cfe4c | 253 | static void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) |
f5132b01 | 254 | { |
e84cfe4c WH |
255 | unsigned en_field = ctrl & 0x3; |
256 | bool pmi = ctrl & 0x8; | |
f5132b01 | 257 | |
c6702c9d | 258 | pmc_stop_counter(pmc); |
f5132b01 | 259 | |
e84cfe4c | 260 | if (!en_field || !pmc_is_enabled(pmc)) |
f5132b01 GN |
261 | return; |
262 | ||
c6702c9d | 263 | pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, |
e84cfe4c WH |
264 | arch_events[fixed_pmc_events[idx]].event_type, |
265 | !(en_field & 0x2), /* exclude user */ | |
266 | !(en_field & 0x1), /* exclude kernel */ | |
267 | pmi, false, false); | |
f5132b01 GN |
268 | } |
269 | ||
c6702c9d | 270 | static inline u8 fixed_ctrl_field(u64 ctrl, int idx) |
f5132b01 GN |
271 | { |
272 | return (ctrl >> (idx * 4)) & 0xf; | |
273 | } | |
274 | ||
275 | static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data) | |
276 | { | |
277 | int i; | |
278 | ||
279 | for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { | |
e84cfe4c WH |
280 | u8 old_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, i); |
281 | u8 new_ctrl = fixed_ctrl_field(data, i); | |
f5132b01 GN |
282 | struct kvm_pmc *pmc = get_fixed_pmc_idx(pmu, i); |
283 | ||
e84cfe4c | 284 | if (old_ctrl == new_ctrl) |
f5132b01 GN |
285 | continue; |
286 | ||
e84cfe4c | 287 | reprogram_fixed_counter(pmc, new_ctrl, i); |
f5132b01 GN |
288 | } |
289 | ||
290 | pmu->fixed_ctr_ctrl = data; | |
291 | } | |
292 | ||
e84cfe4c | 293 | static void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx) |
f5132b01 | 294 | { |
e84cfe4c | 295 | struct kvm_pmc *pmc = global_idx_to_pmc(pmu, pmc_idx); |
f5132b01 GN |
296 | |
297 | if (!pmc) | |
298 | return; | |
299 | ||
300 | if (pmc_is_gp(pmc)) | |
301 | reprogram_gp_counter(pmc, pmc->eventsel); | |
302 | else { | |
e84cfe4c WH |
303 | int idx = pmc_idx - INTEL_PMC_IDX_FIXED; |
304 | u8 ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, idx); | |
305 | ||
306 | reprogram_fixed_counter(pmc, ctrl, idx); | |
f5132b01 GN |
307 | } |
308 | } | |
309 | ||
310 | static void global_ctrl_changed(struct kvm_pmu *pmu, u64 data) | |
311 | { | |
312 | int bit; | |
313 | u64 diff = pmu->global_ctrl ^ data; | |
314 | ||
315 | pmu->global_ctrl = data; | |
316 | ||
317 | for_each_set_bit(bit, (unsigned long *)&diff, X86_PMC_IDX_MAX) | |
c6702c9d | 318 | reprogram_counter(pmu, bit); |
f5132b01 GN |
319 | } |
320 | ||
e5af058a WH |
321 | void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) |
322 | { | |
323 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); | |
324 | u64 bitmask; | |
325 | int bit; | |
326 | ||
327 | bitmask = pmu->reprogram_pmi; | |
328 | ||
329 | for_each_set_bit(bit, (unsigned long *)&bitmask, X86_PMC_IDX_MAX) { | |
330 | struct kvm_pmc *pmc = global_idx_to_pmc(pmu, bit); | |
331 | ||
332 | if (unlikely(!pmc || !pmc->perf_event)) { | |
333 | clear_bit(bit, (unsigned long *)&pmu->reprogram_pmi); | |
334 | continue; | |
335 | } | |
336 | ||
337 | reprogram_counter(pmu, bit); | |
338 | } | |
339 | } | |
340 | ||
341 | /* check if idx is a valid index to access PMU */ | |
342 | int kvm_pmu_is_valid_msr_idx(struct kvm_vcpu *vcpu, unsigned idx) | |
343 | { | |
344 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); | |
345 | bool fixed = idx & (1u << 30); | |
346 | idx &= ~(3u << 30); | |
347 | return (!fixed && idx >= pmu->nr_arch_gp_counters) || | |
348 | (fixed && idx >= pmu->nr_arch_fixed_counters); | |
349 | } | |
350 | ||
41aac14a WH |
351 | static struct kvm_pmc *kvm_pmu_msr_idx_to_pmc(struct kvm_vcpu *vcpu, |
352 | unsigned idx) | |
e5af058a WH |
353 | { |
354 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); | |
e5af058a WH |
355 | bool fixed = idx & (1u << 30); |
356 | struct kvm_pmc *counters; | |
e5af058a WH |
357 | |
358 | idx &= ~(3u << 30); | |
359 | if (!fixed && idx >= pmu->nr_arch_gp_counters) | |
41aac14a | 360 | return NULL; |
e5af058a | 361 | if (fixed && idx >= pmu->nr_arch_fixed_counters) |
41aac14a | 362 | return NULL; |
e5af058a WH |
363 | counters = fixed ? pmu->fixed_counters : pmu->gp_counters; |
364 | ||
41aac14a WH |
365 | return &counters[idx]; |
366 | } | |
367 | ||
368 | int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) | |
369 | { | |
370 | bool fast_mode = idx & (1u << 31); | |
371 | struct kvm_pmc *pmc; | |
372 | u64 ctr_val; | |
373 | ||
374 | pmc = kvm_pmu_msr_idx_to_pmc(vcpu, idx); | |
375 | if (!pmc) | |
376 | return 1; | |
377 | ||
378 | ctr_val = pmc_read_counter(pmc); | |
e5af058a WH |
379 | if (fast_mode) |
380 | ctr_val = (u32)ctr_val; | |
381 | ||
382 | *data = ctr_val; | |
383 | return 0; | |
384 | } | |
385 | ||
386 | void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) | |
387 | { | |
388 | if (vcpu->arch.apic) | |
389 | kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); | |
390 | } | |
391 | ||
c6702c9d | 392 | bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) |
f5132b01 | 393 | { |
212dba12 | 394 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); |
f5132b01 GN |
395 | int ret; |
396 | ||
397 | switch (msr) { | |
398 | case MSR_CORE_PERF_FIXED_CTR_CTRL: | |
399 | case MSR_CORE_PERF_GLOBAL_STATUS: | |
400 | case MSR_CORE_PERF_GLOBAL_CTRL: | |
401 | case MSR_CORE_PERF_GLOBAL_OVF_CTRL: | |
402 | ret = pmu->version > 1; | |
403 | break; | |
404 | default: | |
405 | ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) | |
406 | || get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) | |
407 | || get_fixed_pmc(pmu, msr); | |
408 | break; | |
409 | } | |
410 | return ret; | |
411 | } | |
412 | ||
413 | int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data) | |
414 | { | |
212dba12 | 415 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); |
f5132b01 GN |
416 | struct kvm_pmc *pmc; |
417 | ||
418 | switch (index) { | |
419 | case MSR_CORE_PERF_FIXED_CTR_CTRL: | |
420 | *data = pmu->fixed_ctr_ctrl; | |
421 | return 0; | |
422 | case MSR_CORE_PERF_GLOBAL_STATUS: | |
423 | *data = pmu->global_status; | |
424 | return 0; | |
425 | case MSR_CORE_PERF_GLOBAL_CTRL: | |
426 | *data = pmu->global_ctrl; | |
427 | return 0; | |
428 | case MSR_CORE_PERF_GLOBAL_OVF_CTRL: | |
429 | *data = pmu->global_ovf_ctrl; | |
430 | return 0; | |
431 | default: | |
432 | if ((pmc = get_gp_pmc(pmu, index, MSR_IA32_PERFCTR0)) || | |
433 | (pmc = get_fixed_pmc(pmu, index))) { | |
c6702c9d | 434 | *data = pmc_read_counter(pmc); |
f5132b01 GN |
435 | return 0; |
436 | } else if ((pmc = get_gp_pmc(pmu, index, MSR_P6_EVNTSEL0))) { | |
437 | *data = pmc->eventsel; | |
438 | return 0; | |
439 | } | |
440 | } | |
441 | return 1; | |
442 | } | |
443 | ||
afd80d85 | 444 | int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) |
f5132b01 | 445 | { |
212dba12 | 446 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); |
f5132b01 | 447 | struct kvm_pmc *pmc; |
afd80d85 PB |
448 | u32 index = msr_info->index; |
449 | u64 data = msr_info->data; | |
f5132b01 GN |
450 | |
451 | switch (index) { | |
452 | case MSR_CORE_PERF_FIXED_CTR_CTRL: | |
453 | if (pmu->fixed_ctr_ctrl == data) | |
454 | return 0; | |
fea52953 | 455 | if (!(data & 0xfffffffffffff444ull)) { |
f5132b01 GN |
456 | reprogram_fixed_counters(pmu, data); |
457 | return 0; | |
458 | } | |
459 | break; | |
460 | case MSR_CORE_PERF_GLOBAL_STATUS: | |
afd80d85 PB |
461 | if (msr_info->host_initiated) { |
462 | pmu->global_status = data; | |
463 | return 0; | |
464 | } | |
f5132b01 GN |
465 | break; /* RO MSR */ |
466 | case MSR_CORE_PERF_GLOBAL_CTRL: | |
467 | if (pmu->global_ctrl == data) | |
468 | return 0; | |
469 | if (!(data & pmu->global_ctrl_mask)) { | |
470 | global_ctrl_changed(pmu, data); | |
471 | return 0; | |
472 | } | |
473 | break; | |
474 | case MSR_CORE_PERF_GLOBAL_OVF_CTRL: | |
475 | if (!(data & (pmu->global_ctrl_mask & ~(3ull<<62)))) { | |
afd80d85 PB |
476 | if (!msr_info->host_initiated) |
477 | pmu->global_status &= ~data; | |
f5132b01 GN |
478 | pmu->global_ovf_ctrl = data; |
479 | return 0; | |
480 | } | |
481 | break; | |
482 | default: | |
483 | if ((pmc = get_gp_pmc(pmu, index, MSR_IA32_PERFCTR0)) || | |
484 | (pmc = get_fixed_pmc(pmu, index))) { | |
afd80d85 PB |
485 | if (!msr_info->host_initiated) |
486 | data = (s64)(s32)data; | |
c6702c9d | 487 | pmc->counter += data - pmc_read_counter(pmc); |
f5132b01 GN |
488 | return 0; |
489 | } else if ((pmc = get_gp_pmc(pmu, index, MSR_P6_EVNTSEL0))) { | |
490 | if (data == pmc->eventsel) | |
491 | return 0; | |
103af0a9 | 492 | if (!(data & pmu->reserved_bits)) { |
f5132b01 GN |
493 | reprogram_gp_counter(pmc, data); |
494 | return 0; | |
495 | } | |
496 | } | |
497 | } | |
498 | return 1; | |
499 | } | |
500 | ||
e84cfe4c WH |
501 | /* refresh PMU settings. This function generally is called when underlying |
502 | * settings are changed (such as changes of PMU CPUID by guest VMs), which | |
503 | * should rarely happen. | |
504 | */ | |
c6702c9d | 505 | void kvm_pmu_refresh(struct kvm_vcpu *vcpu) |
f5132b01 | 506 | { |
212dba12 | 507 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); |
f5132b01 | 508 | struct kvm_cpuid_entry2 *entry; |
d27aa7f1 NA |
509 | union cpuid10_eax eax; |
510 | union cpuid10_edx edx; | |
f5132b01 GN |
511 | |
512 | pmu->nr_arch_gp_counters = 0; | |
513 | pmu->nr_arch_fixed_counters = 0; | |
514 | pmu->counter_bitmask[KVM_PMC_GP] = 0; | |
515 | pmu->counter_bitmask[KVM_PMC_FIXED] = 0; | |
516 | pmu->version = 0; | |
103af0a9 | 517 | pmu->reserved_bits = 0xffffffff00200000ull; |
f5132b01 GN |
518 | |
519 | entry = kvm_find_cpuid_entry(vcpu, 0xa, 0); | |
520 | if (!entry) | |
521 | return; | |
d27aa7f1 NA |
522 | eax.full = entry->eax; |
523 | edx.full = entry->edx; | |
f5132b01 | 524 | |
d27aa7f1 | 525 | pmu->version = eax.split.version_id; |
f5132b01 GN |
526 | if (!pmu->version) |
527 | return; | |
528 | ||
d27aa7f1 NA |
529 | pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters, |
530 | INTEL_PMC_MAX_GENERIC); | |
531 | pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << eax.split.bit_width) - 1; | |
532 | pmu->available_event_types = ~entry->ebx & | |
533 | ((1ull << eax.split.mask_length) - 1); | |
f5132b01 GN |
534 | |
535 | if (pmu->version == 1) { | |
f19a0c2c GN |
536 | pmu->nr_arch_fixed_counters = 0; |
537 | } else { | |
d27aa7f1 NA |
538 | pmu->nr_arch_fixed_counters = |
539 | min_t(int, edx.split.num_counters_fixed, | |
15c7ad51 | 540 | INTEL_PMC_MAX_FIXED); |
f19a0c2c | 541 | pmu->counter_bitmask[KVM_PMC_FIXED] = |
d27aa7f1 | 542 | ((u64)1 << edx.split.bit_width_fixed) - 1; |
f5132b01 GN |
543 | } |
544 | ||
f19a0c2c | 545 | pmu->global_ctrl = ((1 << pmu->nr_arch_gp_counters) - 1) | |
15c7ad51 | 546 | (((1ull << pmu->nr_arch_fixed_counters) - 1) << INTEL_PMC_IDX_FIXED); |
f19a0c2c | 547 | pmu->global_ctrl_mask = ~pmu->global_ctrl; |
103af0a9 AK |
548 | |
549 | entry = kvm_find_cpuid_entry(vcpu, 7, 0); | |
550 | if (entry && | |
551 | (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) && | |
552 | (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) | |
553 | pmu->reserved_bits ^= HSW_IN_TX|HSW_IN_TX_CHECKPOINTED; | |
f5132b01 GN |
554 | } |
555 | ||
f5132b01 GN |
556 | void kvm_pmu_reset(struct kvm_vcpu *vcpu) |
557 | { | |
212dba12 | 558 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); |
f5132b01 GN |
559 | int i; |
560 | ||
561 | irq_work_sync(&pmu->irq_work); | |
15c7ad51 | 562 | for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) { |
f5132b01 | 563 | struct kvm_pmc *pmc = &pmu->gp_counters[i]; |
c6702c9d | 564 | pmc_stop_counter(pmc); |
f5132b01 GN |
565 | pmc->counter = pmc->eventsel = 0; |
566 | } | |
567 | ||
15c7ad51 | 568 | for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) |
c6702c9d | 569 | pmc_stop_counter(&pmu->fixed_counters[i]); |
f5132b01 GN |
570 | |
571 | pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = | |
572 | pmu->global_ovf_ctrl = 0; | |
573 | } | |
574 | ||
e5af058a | 575 | void kvm_pmu_init(struct kvm_vcpu *vcpu) |
f5132b01 | 576 | { |
e5af058a | 577 | int i; |
212dba12 | 578 | struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); |
f5132b01 | 579 | |
e5af058a WH |
580 | memset(pmu, 0, sizeof(*pmu)); |
581 | for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) { | |
582 | pmu->gp_counters[i].type = KVM_PMC_GP; | |
583 | pmu->gp_counters[i].vcpu = vcpu; | |
584 | pmu->gp_counters[i].idx = i; | |
f5132b01 | 585 | } |
e5af058a WH |
586 | for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) { |
587 | pmu->fixed_counters[i].type = KVM_PMC_FIXED; | |
588 | pmu->fixed_counters[i].vcpu = vcpu; | |
589 | pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED; | |
590 | } | |
591 | init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn); | |
592 | kvm_pmu_refresh(vcpu); | |
593 | } | |
594 | ||
595 | void kvm_pmu_destroy(struct kvm_vcpu *vcpu) | |
596 | { | |
597 | kvm_pmu_reset(vcpu); | |
f5132b01 | 598 | } |