Commit | Line | Data |
---|---|---|
b7169166 RR |
1 | /* |
2 | * Performance events - AMD IBS | |
3 | * | |
4 | * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter | |
5 | * | |
6 | * For licencing details see kernel-base/COPYING | |
7 | */ | |
8 | ||
9 | #include <linux/perf_event.h> | |
eb008eb6 PG |
10 | #include <linux/init.h> |
11 | #include <linux/export.h> | |
b7169166 | 12 | #include <linux/pci.h> |
d47e8238 | 13 | #include <linux/ptrace.h> |
bee09ed9 | 14 | #include <linux/syscore_ops.h> |
b7169166 RR |
15 | |
16 | #include <asm/apic.h> | |
17 | ||
27f6d22b | 18 | #include "../perf_event.h" |
d07bdfd3 | 19 | |
b7169166 RR |
20 | static u32 ibs_caps; |
21 | ||
22 | #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) | |
23 | ||
b7074f1f RR |
24 | #include <linux/kprobes.h> |
25 | #include <linux/hardirq.h> | |
26 | ||
27 | #include <asm/nmi.h> | |
28 | ||
51041943 RR |
29 | #define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT) |
30 | #define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT | |
31 | ||
85dc6002 PZ |
32 | |
33 | /* | |
34 | * IBS states: | |
35 | * | |
36 | * ENABLED; tracks the pmu::add(), pmu::del() state, when set the counter is taken | |
37 | * and any further add()s must fail. | |
38 | * | |
39 | * STARTED/STOPPING/STOPPED; deal with pmu::start(), pmu::stop() state but are | |
40 | * complicated by the fact that the IBS hardware can send late NMIs (ie. after | |
41 | * we've cleared the EN bit). | |
42 | * | |
43 | * In order to consume these late NMIs we have the STOPPED state, any NMI that | |
44 | * happens after we've cleared the EN state will clear this bit and report the | |
45 | * NMI handled (this is fundamentally racy in the face or multiple NMI sources, | |
46 | * someone else can consume our BIT and our NMI will go unhandled). | |
47 | * | |
48 | * And since we cannot set/clear this separate bit together with the EN bit, | |
49 | * there are races; if we cleared STARTED early, an NMI could land in | |
50 | * between clearing STARTED and clearing the EN bit (in fact multiple NMIs | |
51 | * could happen if the period is small enough), and consume our STOPPED bit | |
52 | * and trigger streams of unhandled NMIs. | |
53 | * | |
54 | * If, however, we clear STARTED late, an NMI can hit between clearing the | |
55 | * EN bit and clearing STARTED, still see STARTED set and process the event. | |
56 | * If this event will have the VALID bit clear, we bail properly, but this | |
57 | * is not a given. With VALID set we can end up calling pmu::stop() again | |
58 | * (the throttle logic) and trigger the WARNs in there. | |
59 | * | |
60 | * So what we do is set STOPPING before clearing EN to avoid the pmu::stop() | |
61 | * nesting, and clear STARTED late, so that we have a well defined state over | |
62 | * the clearing of the EN bit. | |
63 | * | |
64 | * XXX: we could probably be using !atomic bitops for all this. | |
65 | */ | |
66 | ||
4db2e8e6 RR |
67 | enum ibs_states { |
68 | IBS_ENABLED = 0, | |
69 | IBS_STARTED = 1, | |
70 | IBS_STOPPING = 2, | |
85dc6002 | 71 | IBS_STOPPED = 3, |
4db2e8e6 RR |
72 | |
73 | IBS_MAX_STATES, | |
74 | }; | |
75 | ||
76 | struct cpu_perf_ibs { | |
77 | struct perf_event *event; | |
78 | unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)]; | |
79 | }; | |
80 | ||
51041943 | 81 | struct perf_ibs { |
2e132b12 RR |
82 | struct pmu pmu; |
83 | unsigned int msr; | |
84 | u64 config_mask; | |
85 | u64 cnt_mask; | |
86 | u64 enable_mask; | |
87 | u64 valid_mask; | |
88 | u64 max_period; | |
89 | unsigned long offset_mask[1]; | |
90 | int offset_max; | |
91 | struct cpu_perf_ibs __percpu *pcpu; | |
92 | ||
93 | struct attribute **format_attrs; | |
94 | struct attribute_group format_group; | |
95 | const struct attribute_group *attr_groups[2]; | |
96 | ||
97 | u64 (*get_count)(u64 config); | |
b7074f1f RR |
98 | }; |
99 | ||
100 | struct perf_ibs_data { | |
101 | u32 size; | |
102 | union { | |
103 | u32 data[0]; /* data buffer starts here */ | |
104 | u32 caps; | |
105 | }; | |
106 | u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; | |
51041943 RR |
107 | }; |
108 | ||
db98c5fa | 109 | static int |
98112d2e | 110 | perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period) |
db98c5fa RR |
111 | { |
112 | s64 left = local64_read(&hwc->period_left); | |
113 | s64 period = hwc->sample_period; | |
114 | int overflow = 0; | |
115 | ||
116 | /* | |
117 | * If we are way outside a reasonable range then just skip forward: | |
118 | */ | |
119 | if (unlikely(left <= -period)) { | |
120 | left = period; | |
121 | local64_set(&hwc->period_left, left); | |
122 | hwc->last_period = period; | |
123 | overflow = 1; | |
124 | } | |
125 | ||
fc006cf7 | 126 | if (unlikely(left < (s64)min)) { |
db98c5fa RR |
127 | left += period; |
128 | local64_set(&hwc->period_left, left); | |
129 | hwc->last_period = period; | |
130 | overflow = 1; | |
131 | } | |
132 | ||
7caaf4d8 RR |
133 | /* |
134 | * If the hw period that triggers the sw overflow is too short | |
135 | * we might hit the irq handler. This biases the results. | |
136 | * Thus we shorten the next-to-last period and set the last | |
137 | * period to the max period. | |
138 | */ | |
139 | if (left > max) { | |
140 | left -= max; | |
141 | if (left > max) | |
142 | left = max; | |
143 | else if (left < min) | |
144 | left = min; | |
145 | } | |
db98c5fa | 146 | |
98112d2e | 147 | *hw_period = (u64)left; |
db98c5fa RR |
148 | |
149 | return overflow; | |
150 | } | |
151 | ||
152 | static int | |
153 | perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) | |
154 | { | |
155 | struct hw_perf_event *hwc = &event->hw; | |
156 | int shift = 64 - width; | |
157 | u64 prev_raw_count; | |
158 | u64 delta; | |
159 | ||
160 | /* | |
161 | * Careful: an NMI might modify the previous event value. | |
162 | * | |
163 | * Our tactic to handle this is to first atomically read and | |
164 | * exchange a new raw count - then add that new-prev delta | |
165 | * count to the generic event atomically: | |
166 | */ | |
167 | prev_raw_count = local64_read(&hwc->prev_count); | |
168 | if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, | |
169 | new_raw_count) != prev_raw_count) | |
170 | return 0; | |
171 | ||
172 | /* | |
173 | * Now we have the new raw value and have updated the prev | |
174 | * timestamp already. We can now calculate the elapsed delta | |
175 | * (event-)time and add that to the generic event. | |
176 | * | |
177 | * Careful, not all hw sign-extends above the physical width | |
178 | * of the count. | |
179 | */ | |
180 | delta = (new_raw_count << shift) - (prev_raw_count << shift); | |
181 | delta >>= shift; | |
182 | ||
183 | local64_add(delta, &event->count); | |
184 | local64_sub(delta, &hwc->period_left); | |
185 | ||
186 | return 1; | |
187 | } | |
188 | ||
51041943 RR |
189 | static struct perf_ibs perf_ibs_fetch; |
190 | static struct perf_ibs perf_ibs_op; | |
191 | ||
192 | static struct perf_ibs *get_ibs_pmu(int type) | |
193 | { | |
194 | if (perf_ibs_fetch.pmu.type == type) | |
195 | return &perf_ibs_fetch; | |
196 | if (perf_ibs_op.pmu.type == type) | |
197 | return &perf_ibs_op; | |
198 | return NULL; | |
199 | } | |
b7169166 | 200 | |
450bbd49 RR |
201 | /* |
202 | * Use IBS for precise event sampling: | |
203 | * | |
204 | * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count | |
205 | * perf record -a -e r076:p ... # same as -e cpu-cycles:p | |
206 | * perf record -a -e r0C1:p ... # use ibs op counting micro-ops | |
207 | * | |
208 | * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl, | |
209 | * MSRC001_1033) is used to select either cycle or micro-ops counting | |
210 | * mode. | |
211 | * | |
212 | * The rip of IBS samples has skid 0. Thus, IBS supports precise | |
213 | * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the | |
214 | * rip is invalid when IBS was not able to record the rip correctly. | |
215 | * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then. | |
216 | * | |
217 | */ | |
218 | static int perf_ibs_precise_event(struct perf_event *event, u64 *config) | |
219 | { | |
220 | switch (event->attr.precise_ip) { | |
221 | case 0: | |
222 | return -ENOENT; | |
223 | case 1: | |
224 | case 2: | |
225 | break; | |
226 | default: | |
227 | return -EOPNOTSUPP; | |
228 | } | |
229 | ||
230 | switch (event->attr.type) { | |
231 | case PERF_TYPE_HARDWARE: | |
232 | switch (event->attr.config) { | |
233 | case PERF_COUNT_HW_CPU_CYCLES: | |
234 | *config = 0; | |
235 | return 0; | |
236 | } | |
237 | break; | |
238 | case PERF_TYPE_RAW: | |
239 | switch (event->attr.config) { | |
240 | case 0x0076: | |
241 | *config = 0; | |
242 | return 0; | |
243 | case 0x00C1: | |
244 | *config = IBS_OP_CNT_CTL; | |
245 | return 0; | |
246 | } | |
247 | break; | |
248 | default: | |
249 | return -ENOENT; | |
250 | } | |
251 | ||
252 | return -EOPNOTSUPP; | |
253 | } | |
254 | ||
bad9ac2d RR |
255 | static const struct perf_event_attr ibs_notsupp = { |
256 | .exclude_user = 1, | |
257 | .exclude_kernel = 1, | |
258 | .exclude_hv = 1, | |
259 | .exclude_idle = 1, | |
260 | .exclude_host = 1, | |
261 | .exclude_guest = 1, | |
262 | }; | |
263 | ||
b7169166 RR |
264 | static int perf_ibs_init(struct perf_event *event) |
265 | { | |
51041943 RR |
266 | struct hw_perf_event *hwc = &event->hw; |
267 | struct perf_ibs *perf_ibs; | |
268 | u64 max_cnt, config; | |
450bbd49 | 269 | int ret; |
51041943 RR |
270 | |
271 | perf_ibs = get_ibs_pmu(event->attr.type); | |
450bbd49 RR |
272 | if (perf_ibs) { |
273 | config = event->attr.config; | |
274 | } else { | |
275 | perf_ibs = &perf_ibs_op; | |
276 | ret = perf_ibs_precise_event(event, &config); | |
277 | if (ret) | |
278 | return ret; | |
279 | } | |
280 | ||
281 | if (event->pmu != &perf_ibs->pmu) | |
b7169166 | 282 | return -ENOENT; |
51041943 | 283 | |
bad9ac2d RR |
284 | if (perf_flags(&event->attr) & perf_flags(&ibs_notsupp)) |
285 | return -EINVAL; | |
286 | ||
51041943 RR |
287 | if (config & ~perf_ibs->config_mask) |
288 | return -EINVAL; | |
289 | ||
290 | if (hwc->sample_period) { | |
291 | if (config & perf_ibs->cnt_mask) | |
292 | /* raw max_cnt may not be set */ | |
293 | return -EINVAL; | |
6accb9cf RR |
294 | if (!event->attr.sample_freq && hwc->sample_period & 0x0f) |
295 | /* | |
296 | * lower 4 bits can not be set in ibs max cnt, | |
297 | * but allowing it in case we adjust the | |
298 | * sample period to set a frequency. | |
299 | */ | |
51041943 | 300 | return -EINVAL; |
6accb9cf RR |
301 | hwc->sample_period &= ~0x0FULL; |
302 | if (!hwc->sample_period) | |
303 | hwc->sample_period = 0x10; | |
51041943 RR |
304 | } else { |
305 | max_cnt = config & perf_ibs->cnt_mask; | |
db98c5fa | 306 | config &= ~perf_ibs->cnt_mask; |
51041943 RR |
307 | event->attr.sample_period = max_cnt << 4; |
308 | hwc->sample_period = event->attr.sample_period; | |
309 | } | |
310 | ||
db98c5fa | 311 | if (!hwc->sample_period) |
51041943 RR |
312 | return -EINVAL; |
313 | ||
6accb9cf RR |
314 | /* |
315 | * If we modify hwc->sample_period, we also need to update | |
316 | * hwc->last_period and hwc->period_left. | |
317 | */ | |
318 | hwc->last_period = hwc->sample_period; | |
319 | local64_set(&hwc->period_left, hwc->sample_period); | |
320 | ||
51041943 RR |
321 | hwc->config_base = perf_ibs->msr; |
322 | hwc->config = config; | |
323 | ||
b7169166 RR |
324 | return 0; |
325 | } | |
326 | ||
db98c5fa RR |
327 | static int perf_ibs_set_period(struct perf_ibs *perf_ibs, |
328 | struct hw_perf_event *hwc, u64 *period) | |
329 | { | |
98112d2e | 330 | int overflow; |
db98c5fa RR |
331 | |
332 | /* ignore lower 4 bits in min count: */ | |
98112d2e | 333 | overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period); |
db98c5fa RR |
334 | local64_set(&hwc->prev_count, 0); |
335 | ||
98112d2e | 336 | return overflow; |
db98c5fa RR |
337 | } |
338 | ||
339 | static u64 get_ibs_fetch_count(u64 config) | |
340 | { | |
341 | return (config & IBS_FETCH_CNT) >> 12; | |
342 | } | |
343 | ||
344 | static u64 get_ibs_op_count(u64 config) | |
345 | { | |
8b1e1363 RR |
346 | u64 count = 0; |
347 | ||
348 | if (config & IBS_OP_VAL) | |
349 | count += (config & IBS_OP_MAX_CNT) << 4; /* cnt rolled over */ | |
350 | ||
351 | if (ibs_caps & IBS_CAPS_RDWROPCNT) | |
352 | count += (config & IBS_OP_CUR_CNT) >> 32; | |
353 | ||
354 | return count; | |
db98c5fa RR |
355 | } |
356 | ||
357 | static void | |
358 | perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, | |
c9574fe0 | 359 | u64 *config) |
db98c5fa | 360 | { |
c9574fe0 | 361 | u64 count = perf_ibs->get_count(*config); |
db98c5fa | 362 | |
8b1e1363 RR |
363 | /* |
364 | * Set width to 64 since we do not overflow on max width but | |
365 | * instead on max count. In perf_ibs_set_period() we clear | |
366 | * prev count manually on overflow. | |
367 | */ | |
368 | while (!perf_event_try_update(event, count, 64)) { | |
c9574fe0 RR |
369 | rdmsrl(event->hw.config_base, *config); |
370 | count = perf_ibs->get_count(*config); | |
db98c5fa RR |
371 | } |
372 | } | |
373 | ||
c9574fe0 RR |
374 | static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, |
375 | struct hw_perf_event *hwc, u64 config) | |
db98c5fa | 376 | { |
c9574fe0 RR |
377 | wrmsrl(hwc->config_base, hwc->config | config | perf_ibs->enable_mask); |
378 | } | |
379 | ||
380 | /* | |
381 | * Erratum #420 Instruction-Based Sampling Engine May Generate | |
382 | * Interrupt that Cannot Be Cleared: | |
383 | * | |
384 | * Must clear counter mask first, then clear the enable bit. See | |
385 | * Revision Guide for AMD Family 10h Processors, Publication #41322. | |
386 | */ | |
387 | static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, | |
388 | struct hw_perf_event *hwc, u64 config) | |
389 | { | |
390 | config &= ~perf_ibs->cnt_mask; | |
391 | wrmsrl(hwc->config_base, config); | |
392 | config &= ~perf_ibs->enable_mask; | |
393 | wrmsrl(hwc->config_base, config); | |
db98c5fa RR |
394 | } |
395 | ||
396 | /* | |
397 | * We cannot restore the ibs pmu state, so we always needs to update | |
398 | * the event while stopping it and then reset the state when starting | |
399 | * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in | |
400 | * perf_ibs_start()/perf_ibs_stop() and instead always do it. | |
401 | */ | |
4db2e8e6 RR |
402 | static void perf_ibs_start(struct perf_event *event, int flags) |
403 | { | |
404 | struct hw_perf_event *hwc = &event->hw; | |
405 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); | |
406 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | |
c9574fe0 | 407 | u64 period; |
4db2e8e6 | 408 | |
db98c5fa | 409 | if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) |
4db2e8e6 RR |
410 | return; |
411 | ||
db98c5fa RR |
412 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
413 | hwc->state = 0; | |
414 | ||
c9574fe0 | 415 | perf_ibs_set_period(perf_ibs, hwc, &period); |
5a50f529 | 416 | /* |
85dc6002 PZ |
417 | * Set STARTED before enabling the hardware, such that a subsequent NMI |
418 | * must observe it. | |
5a50f529 | 419 | */ |
85dc6002 | 420 | set_bit(IBS_STARTED, pcpu->state); |
5a50f529 | 421 | clear_bit(IBS_STOPPING, pcpu->state); |
c9574fe0 | 422 | perf_ibs_enable_event(perf_ibs, hwc, period >> 4); |
db98c5fa RR |
423 | |
424 | perf_event_update_userpage(event); | |
4db2e8e6 RR |
425 | } |
426 | ||
427 | static void perf_ibs_stop(struct perf_event *event, int flags) | |
428 | { | |
429 | struct hw_perf_event *hwc = &event->hw; | |
430 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); | |
431 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | |
c9574fe0 | 432 | u64 config; |
db98c5fa | 433 | int stopping; |
4db2e8e6 | 434 | |
85dc6002 PZ |
435 | if (test_and_set_bit(IBS_STOPPING, pcpu->state)) |
436 | return; | |
437 | ||
5a50f529 | 438 | stopping = test_bit(IBS_STARTED, pcpu->state); |
4db2e8e6 | 439 | |
db98c5fa RR |
440 | if (!stopping && (hwc->state & PERF_HES_UPTODATE)) |
441 | return; | |
4db2e8e6 | 442 | |
c9574fe0 | 443 | rdmsrl(hwc->config_base, config); |
db98c5fa RR |
444 | |
445 | if (stopping) { | |
5a50f529 | 446 | /* |
85dc6002 | 447 | * Set STOPPED before disabling the hardware, such that it |
5a50f529 PZ |
448 | * must be visible to NMIs the moment we clear the EN bit, |
449 | * at which point we can generate an !VALID sample which | |
450 | * we need to consume. | |
451 | */ | |
85dc6002 | 452 | set_bit(IBS_STOPPED, pcpu->state); |
c9574fe0 | 453 | perf_ibs_disable_event(perf_ibs, hwc, config); |
5a50f529 PZ |
454 | /* |
455 | * Clear STARTED after disabling the hardware; if it were | |
456 | * cleared before an NMI hitting after the clear but before | |
457 | * clearing the EN bit might think it a spurious NMI and not | |
458 | * handle it. | |
459 | * | |
460 | * Clearing it after, however, creates the problem of the NMI | |
461 | * handler seeing STARTED but not having a valid sample. | |
462 | */ | |
463 | clear_bit(IBS_STARTED, pcpu->state); | |
db98c5fa RR |
464 | WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); |
465 | hwc->state |= PERF_HES_STOPPED; | |
466 | } | |
467 | ||
468 | if (hwc->state & PERF_HES_UPTODATE) | |
469 | return; | |
470 | ||
8b1e1363 RR |
471 | /* |
472 | * Clear valid bit to not count rollovers on update, rollovers | |
473 | * are only updated in the irq handler. | |
474 | */ | |
475 | config &= ~perf_ibs->valid_mask; | |
476 | ||
c9574fe0 | 477 | perf_ibs_event_update(perf_ibs, event, &config); |
db98c5fa | 478 | hwc->state |= PERF_HES_UPTODATE; |
4db2e8e6 RR |
479 | } |
480 | ||
b7169166 RR |
481 | static int perf_ibs_add(struct perf_event *event, int flags) |
482 | { | |
4db2e8e6 RR |
483 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); |
484 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | |
485 | ||
486 | if (test_and_set_bit(IBS_ENABLED, pcpu->state)) | |
487 | return -ENOSPC; | |
488 | ||
db98c5fa RR |
489 | event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
490 | ||
4db2e8e6 RR |
491 | pcpu->event = event; |
492 | ||
493 | if (flags & PERF_EF_START) | |
494 | perf_ibs_start(event, PERF_EF_RELOAD); | |
495 | ||
b7169166 RR |
496 | return 0; |
497 | } | |
498 | ||
499 | static void perf_ibs_del(struct perf_event *event, int flags) | |
500 | { | |
4db2e8e6 RR |
501 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); |
502 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | |
503 | ||
504 | if (!test_and_clear_bit(IBS_ENABLED, pcpu->state)) | |
505 | return; | |
506 | ||
db98c5fa | 507 | perf_ibs_stop(event, PERF_EF_UPDATE); |
4db2e8e6 RR |
508 | |
509 | pcpu->event = NULL; | |
db98c5fa RR |
510 | |
511 | perf_event_update_userpage(event); | |
b7169166 RR |
512 | } |
513 | ||
4db2e8e6 RR |
514 | static void perf_ibs_read(struct perf_event *event) { } |
515 | ||
2e132b12 RR |
516 | PMU_FORMAT_ATTR(rand_en, "config:57"); |
517 | PMU_FORMAT_ATTR(cnt_ctl, "config:19"); | |
518 | ||
519 | static struct attribute *ibs_fetch_format_attrs[] = { | |
520 | &format_attr_rand_en.attr, | |
521 | NULL, | |
522 | }; | |
523 | ||
524 | static struct attribute *ibs_op_format_attrs[] = { | |
525 | NULL, /* &format_attr_cnt_ctl.attr if IBS_CAPS_OPCNT */ | |
526 | NULL, | |
527 | }; | |
528 | ||
51041943 RR |
529 | static struct perf_ibs perf_ibs_fetch = { |
530 | .pmu = { | |
531 | .task_ctx_nr = perf_invalid_context, | |
532 | ||
533 | .event_init = perf_ibs_init, | |
534 | .add = perf_ibs_add, | |
535 | .del = perf_ibs_del, | |
4db2e8e6 RR |
536 | .start = perf_ibs_start, |
537 | .stop = perf_ibs_stop, | |
538 | .read = perf_ibs_read, | |
51041943 RR |
539 | }, |
540 | .msr = MSR_AMD64_IBSFETCHCTL, | |
541 | .config_mask = IBS_FETCH_CONFIG_MASK, | |
542 | .cnt_mask = IBS_FETCH_MAX_CNT, | |
543 | .enable_mask = IBS_FETCH_ENABLE, | |
b7074f1f | 544 | .valid_mask = IBS_FETCH_VAL, |
db98c5fa | 545 | .max_period = IBS_FETCH_MAX_CNT << 4, |
b7074f1f RR |
546 | .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, |
547 | .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, | |
2e132b12 | 548 | .format_attrs = ibs_fetch_format_attrs, |
db98c5fa RR |
549 | |
550 | .get_count = get_ibs_fetch_count, | |
51041943 RR |
551 | }; |
552 | ||
553 | static struct perf_ibs perf_ibs_op = { | |
554 | .pmu = { | |
555 | .task_ctx_nr = perf_invalid_context, | |
556 | ||
557 | .event_init = perf_ibs_init, | |
558 | .add = perf_ibs_add, | |
559 | .del = perf_ibs_del, | |
4db2e8e6 RR |
560 | .start = perf_ibs_start, |
561 | .stop = perf_ibs_stop, | |
562 | .read = perf_ibs_read, | |
51041943 RR |
563 | }, |
564 | .msr = MSR_AMD64_IBSOPCTL, | |
565 | .config_mask = IBS_OP_CONFIG_MASK, | |
566 | .cnt_mask = IBS_OP_MAX_CNT, | |
567 | .enable_mask = IBS_OP_ENABLE, | |
b7074f1f | 568 | .valid_mask = IBS_OP_VAL, |
db98c5fa | 569 | .max_period = IBS_OP_MAX_CNT << 4, |
b7074f1f RR |
570 | .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, |
571 | .offset_max = MSR_AMD64_IBSOP_REG_COUNT, | |
2e132b12 | 572 | .format_attrs = ibs_op_format_attrs, |
db98c5fa RR |
573 | |
574 | .get_count = get_ibs_op_count, | |
b7169166 RR |
575 | }; |
576 | ||
b7074f1f RR |
577 | static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) |
578 | { | |
4db2e8e6 RR |
579 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); |
580 | struct perf_event *event = pcpu->event; | |
b7074f1f RR |
581 | struct hw_perf_event *hwc = &event->hw; |
582 | struct perf_sample_data data; | |
583 | struct perf_raw_record raw; | |
584 | struct pt_regs regs; | |
585 | struct perf_ibs_data ibs_data; | |
d47e8238 | 586 | int offset, size, check_rip, offset_max, throttle = 0; |
b7074f1f | 587 | unsigned int msr; |
c9574fe0 | 588 | u64 *buf, *config, period; |
b7074f1f | 589 | |
4db2e8e6 | 590 | if (!test_bit(IBS_STARTED, pcpu->state)) { |
5a50f529 | 591 | fail: |
fc5fb2b5 RR |
592 | /* |
593 | * Catch spurious interrupts after stopping IBS: After | |
d82603c6 | 594 | * disabling IBS there could be still incoming NMIs |
fc5fb2b5 RR |
595 | * with samples that even have the valid bit cleared. |
596 | * Mark all this NMIs as handled. | |
597 | */ | |
85dc6002 | 598 | if (test_and_clear_bit(IBS_STOPPED, pcpu->state)) |
5a50f529 PZ |
599 | return 1; |
600 | ||
601 | return 0; | |
4db2e8e6 RR |
602 | } |
603 | ||
b7074f1f RR |
604 | msr = hwc->config_base; |
605 | buf = ibs_data.regs; | |
606 | rdmsrl(msr, *buf); | |
607 | if (!(*buf++ & perf_ibs->valid_mask)) | |
5a50f529 | 608 | goto fail; |
b7074f1f | 609 | |
c9574fe0 | 610 | config = &ibs_data.regs[0]; |
c75841a3 | 611 | perf_ibs_event_update(perf_ibs, event, config); |
fd0d000b | 612 | perf_sample_data_init(&data, 0, hwc->last_period); |
c9574fe0 | 613 | if (!perf_ibs_set_period(perf_ibs, hwc, &period)) |
d47e8238 RR |
614 | goto out; /* no sw counter overflow */ |
615 | ||
616 | ibs_data.caps = ibs_caps; | |
617 | size = 1; | |
618 | offset = 1; | |
619 | check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); | |
620 | if (event->attr.sample_type & PERF_SAMPLE_RAW) | |
621 | offset_max = perf_ibs->offset_max; | |
622 | else if (check_rip) | |
623 | offset_max = 2; | |
624 | else | |
625 | offset_max = 1; | |
626 | do { | |
627 | rdmsrl(msr + offset, *buf++); | |
628 | size++; | |
629 | offset = find_next_bit(perf_ibs->offset_mask, | |
630 | perf_ibs->offset_max, | |
631 | offset + 1); | |
632 | } while (offset < offset_max); | |
904cb367 AG |
633 | if (event->attr.sample_type & PERF_SAMPLE_RAW) { |
634 | /* | |
635 | * Read IbsBrTarget and IbsOpData4 separately | |
636 | * depending on their availability. | |
637 | * Can't add to offset_max as they are staggered | |
638 | */ | |
639 | if (ibs_caps & IBS_CAPS_BRNTRGT) { | |
640 | rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++); | |
641 | size++; | |
642 | } | |
643 | if (ibs_caps & IBS_CAPS_OPDATA4) { | |
644 | rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++); | |
645 | size++; | |
646 | } | |
647 | } | |
d47e8238 RR |
648 | ibs_data.size = sizeof(u64) * size; |
649 | ||
650 | regs = *iregs; | |
450bbd49 RR |
651 | if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) { |
652 | regs.flags &= ~PERF_EFLAGS_EXACT; | |
653 | } else { | |
d07bdfd3 | 654 | set_linear_ip(®s, ibs_data.regs[1]); |
450bbd49 RR |
655 | regs.flags |= PERF_EFLAGS_EXACT; |
656 | } | |
c75841a3 | 657 | |
b7074f1f | 658 | if (event->attr.sample_type & PERF_SAMPLE_RAW) { |
d47e8238 | 659 | raw.size = sizeof(u32) + ibs_data.size; |
b7074f1f RR |
660 | raw.data = ibs_data.data; |
661 | data.raw = &raw; | |
662 | } | |
663 | ||
d47e8238 RR |
664 | throttle = perf_event_overflow(event, &data, ®s); |
665 | out: | |
c9574fe0 | 666 | if (throttle) |
0158b83f | 667 | perf_ibs_stop(event, 0); |
c9574fe0 RR |
668 | else |
669 | perf_ibs_enable_event(perf_ibs, hwc, period >> 4); | |
db98c5fa RR |
670 | |
671 | perf_event_update_userpage(event); | |
b7074f1f RR |
672 | |
673 | return 1; | |
674 | } | |
675 | ||
9326638c | 676 | static int |
b7074f1f RR |
677 | perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) |
678 | { | |
c2872d38 | 679 | u64 stamp = sched_clock(); |
b7074f1f RR |
680 | int handled = 0; |
681 | ||
682 | handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs); | |
683 | handled += perf_ibs_handle_irq(&perf_ibs_op, regs); | |
684 | ||
685 | if (handled) | |
686 | inc_irq_stat(apic_perf_irqs); | |
687 | ||
c2872d38 PZ |
688 | perf_sample_event_took(sched_clock() - stamp); |
689 | ||
b7074f1f RR |
690 | return handled; |
691 | } | |
9326638c | 692 | NOKPROBE_SYMBOL(perf_ibs_nmi_handler); |
b7074f1f | 693 | |
4db2e8e6 RR |
694 | static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) |
695 | { | |
696 | struct cpu_perf_ibs __percpu *pcpu; | |
697 | int ret; | |
698 | ||
699 | pcpu = alloc_percpu(struct cpu_perf_ibs); | |
700 | if (!pcpu) | |
701 | return -ENOMEM; | |
702 | ||
703 | perf_ibs->pcpu = pcpu; | |
704 | ||
2e132b12 RR |
705 | /* register attributes */ |
706 | if (perf_ibs->format_attrs[0]) { | |
707 | memset(&perf_ibs->format_group, 0, sizeof(perf_ibs->format_group)); | |
708 | perf_ibs->format_group.name = "format"; | |
709 | perf_ibs->format_group.attrs = perf_ibs->format_attrs; | |
710 | ||
711 | memset(&perf_ibs->attr_groups, 0, sizeof(perf_ibs->attr_groups)); | |
712 | perf_ibs->attr_groups[0] = &perf_ibs->format_group; | |
713 | perf_ibs->pmu.attr_groups = perf_ibs->attr_groups; | |
714 | } | |
715 | ||
4db2e8e6 RR |
716 | ret = perf_pmu_register(&perf_ibs->pmu, name, -1); |
717 | if (ret) { | |
718 | perf_ibs->pcpu = NULL; | |
719 | free_percpu(pcpu); | |
720 | } | |
721 | ||
722 | return ret; | |
723 | } | |
724 | ||
b7169166 RR |
725 | static __init int perf_event_ibs_init(void) |
726 | { | |
2e132b12 RR |
727 | struct attribute **attr = ibs_op_format_attrs; |
728 | ||
b7169166 RR |
729 | if (!ibs_caps) |
730 | return -ENODEV; /* ibs not supported by the cpu */ | |
731 | ||
4db2e8e6 | 732 | perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch"); |
2e132b12 RR |
733 | |
734 | if (ibs_caps & IBS_CAPS_OPCNT) { | |
7bf35238 | 735 | perf_ibs_op.config_mask |= IBS_OP_CNT_CTL; |
2e132b12 RR |
736 | *attr++ = &format_attr_cnt_ctl.attr; |
737 | } | |
4db2e8e6 | 738 | perf_ibs_pmu_init(&perf_ibs_op, "ibs_op"); |
2e132b12 | 739 | |
fab06992 | 740 | register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs"); |
1b74dde7 | 741 | pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps); |
b7169166 RR |
742 | |
743 | return 0; | |
744 | } | |
745 | ||
746 | #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */ | |
747 | ||
748 | static __init int perf_event_ibs_init(void) { return 0; } | |
749 | ||
750 | #endif | |
751 | ||
752 | /* IBS - apic initialization, for perf and oprofile */ | |
753 | ||
754 | static __init u32 __get_ibs_caps(void) | |
755 | { | |
756 | u32 caps; | |
757 | unsigned int max_level; | |
758 | ||
759 | if (!boot_cpu_has(X86_FEATURE_IBS)) | |
760 | return 0; | |
761 | ||
762 | /* check IBS cpuid feature flags */ | |
763 | max_level = cpuid_eax(0x80000000); | |
764 | if (max_level < IBS_CPUID_FEATURES) | |
765 | return IBS_CAPS_DEFAULT; | |
766 | ||
767 | caps = cpuid_eax(IBS_CPUID_FEATURES); | |
768 | if (!(caps & IBS_CAPS_AVAIL)) | |
769 | /* cpuid flags not valid */ | |
770 | return IBS_CAPS_DEFAULT; | |
771 | ||
772 | return caps; | |
773 | } | |
774 | ||
775 | u32 get_ibs_caps(void) | |
776 | { | |
777 | return ibs_caps; | |
778 | } | |
779 | ||
780 | EXPORT_SYMBOL(get_ibs_caps); | |
781 | ||
782 | static inline int get_eilvt(int offset) | |
783 | { | |
784 | return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1); | |
785 | } | |
786 | ||
787 | static inline int put_eilvt(int offset) | |
788 | { | |
789 | return !setup_APIC_eilvt(offset, 0, 0, 1); | |
790 | } | |
791 | ||
792 | /* | |
793 | * Check and reserve APIC extended interrupt LVT offset for IBS if available. | |
794 | */ | |
795 | static inline int ibs_eilvt_valid(void) | |
796 | { | |
797 | int offset; | |
798 | u64 val; | |
799 | int valid = 0; | |
800 | ||
801 | preempt_disable(); | |
802 | ||
803 | rdmsrl(MSR_AMD64_IBSCTL, val); | |
804 | offset = val & IBSCTL_LVT_OFFSET_MASK; | |
805 | ||
806 | if (!(val & IBSCTL_LVT_OFFSET_VALID)) { | |
807 | pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n", | |
808 | smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); | |
809 | goto out; | |
810 | } | |
811 | ||
812 | if (!get_eilvt(offset)) { | |
813 | pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n", | |
814 | smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); | |
815 | goto out; | |
816 | } | |
817 | ||
818 | valid = 1; | |
819 | out: | |
820 | preempt_enable(); | |
821 | ||
822 | return valid; | |
823 | } | |
824 | ||
825 | static int setup_ibs_ctl(int ibs_eilvt_off) | |
826 | { | |
827 | struct pci_dev *cpu_cfg; | |
828 | int nodes; | |
829 | u32 value = 0; | |
830 | ||
831 | nodes = 0; | |
832 | cpu_cfg = NULL; | |
833 | do { | |
834 | cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD, | |
835 | PCI_DEVICE_ID_AMD_10H_NB_MISC, | |
836 | cpu_cfg); | |
837 | if (!cpu_cfg) | |
838 | break; | |
839 | ++nodes; | |
840 | pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off | |
841 | | IBSCTL_LVT_OFFSET_VALID); | |
842 | pci_read_config_dword(cpu_cfg, IBSCTL, &value); | |
843 | if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) { | |
844 | pci_dev_put(cpu_cfg); | |
1b74dde7 CY |
845 | pr_debug("Failed to setup IBS LVT offset, IBSCTL = 0x%08x\n", |
846 | value); | |
b7169166 RR |
847 | return -EINVAL; |
848 | } | |
849 | } while (1); | |
850 | ||
851 | if (!nodes) { | |
1b74dde7 | 852 | pr_debug("No CPU node configured for IBS\n"); |
b7169166 RR |
853 | return -ENODEV; |
854 | } | |
855 | ||
856 | return 0; | |
857 | } | |
858 | ||
859 | /* | |
860 | * This runs only on the current cpu. We try to find an LVT offset and | |
861 | * setup the local APIC. For this we must disable preemption. On | |
862 | * success we initialize all nodes with this offset. This updates then | |
863 | * the offset in the IBS_CTL per-node msr. The per-core APIC setup of | |
864 | * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that | |
865 | * is using the new offset. | |
866 | */ | |
c796b205 | 867 | static void force_ibs_eilvt_setup(void) |
b7169166 RR |
868 | { |
869 | int offset; | |
870 | int ret; | |
871 | ||
872 | preempt_disable(); | |
873 | /* find the next free available EILVT entry, skip offset 0 */ | |
874 | for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) { | |
875 | if (get_eilvt(offset)) | |
876 | break; | |
877 | } | |
878 | preempt_enable(); | |
879 | ||
880 | if (offset == APIC_EILVT_NR_MAX) { | |
1b74dde7 | 881 | pr_debug("No EILVT entry available\n"); |
c796b205 | 882 | return; |
b7169166 RR |
883 | } |
884 | ||
885 | ret = setup_ibs_ctl(offset); | |
886 | if (ret) | |
887 | goto out; | |
888 | ||
c796b205 | 889 | if (!ibs_eilvt_valid()) |
b7169166 | 890 | goto out; |
b7169166 | 891 | |
16e5294e | 892 | pr_info("IBS: LVT offset %d assigned\n", offset); |
b7169166 | 893 | |
c796b205 | 894 | return; |
b7169166 RR |
895 | out: |
896 | preempt_disable(); | |
897 | put_eilvt(offset); | |
898 | preempt_enable(); | |
c796b205 | 899 | return; |
b7169166 RR |
900 | } |
901 | ||
bee09ed9 RR |
902 | static void ibs_eilvt_setup(void) |
903 | { | |
904 | /* | |
905 | * Force LVT offset assignment for family 10h: The offsets are | |
906 | * not assigned by the BIOS for this family, so the OS is | |
907 | * responsible for doing it. If the OS assignment fails, fall | |
908 | * back to BIOS settings and try to setup this. | |
909 | */ | |
910 | if (boot_cpu_data.x86 == 0x10) | |
911 | force_ibs_eilvt_setup(); | |
912 | } | |
913 | ||
b7169166 RR |
914 | static inline int get_ibs_lvt_offset(void) |
915 | { | |
916 | u64 val; | |
917 | ||
918 | rdmsrl(MSR_AMD64_IBSCTL, val); | |
919 | if (!(val & IBSCTL_LVT_OFFSET_VALID)) | |
920 | return -EINVAL; | |
921 | ||
922 | return val & IBSCTL_LVT_OFFSET_MASK; | |
923 | } | |
924 | ||
925 | static void setup_APIC_ibs(void *dummy) | |
926 | { | |
927 | int offset; | |
928 | ||
929 | offset = get_ibs_lvt_offset(); | |
930 | if (offset < 0) | |
931 | goto failed; | |
932 | ||
933 | if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0)) | |
934 | return; | |
935 | failed: | |
936 | pr_warn("perf: IBS APIC setup failed on cpu #%d\n", | |
937 | smp_processor_id()); | |
938 | } | |
939 | ||
940 | static void clear_APIC_ibs(void *dummy) | |
941 | { | |
942 | int offset; | |
943 | ||
944 | offset = get_ibs_lvt_offset(); | |
945 | if (offset >= 0) | |
946 | setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1); | |
947 | } | |
948 | ||
bee09ed9 RR |
949 | #ifdef CONFIG_PM |
950 | ||
951 | static int perf_ibs_suspend(void) | |
952 | { | |
953 | clear_APIC_ibs(NULL); | |
954 | return 0; | |
955 | } | |
956 | ||
957 | static void perf_ibs_resume(void) | |
958 | { | |
959 | ibs_eilvt_setup(); | |
960 | setup_APIC_ibs(NULL); | |
961 | } | |
962 | ||
963 | static struct syscore_ops perf_ibs_syscore_ops = { | |
964 | .resume = perf_ibs_resume, | |
965 | .suspend = perf_ibs_suspend, | |
966 | }; | |
967 | ||
968 | static void perf_ibs_pm_init(void) | |
969 | { | |
970 | register_syscore_ops(&perf_ibs_syscore_ops); | |
971 | } | |
972 | ||
973 | #else | |
974 | ||
975 | static inline void perf_ibs_pm_init(void) { } | |
976 | ||
977 | #endif | |
978 | ||
148f9bb8 | 979 | static int |
b7169166 RR |
980 | perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) |
981 | { | |
982 | switch (action & ~CPU_TASKS_FROZEN) { | |
983 | case CPU_STARTING: | |
984 | setup_APIC_ibs(NULL); | |
985 | break; | |
986 | case CPU_DYING: | |
987 | clear_APIC_ibs(NULL); | |
988 | break; | |
989 | default: | |
990 | break; | |
991 | } | |
992 | ||
993 | return NOTIFY_OK; | |
994 | } | |
995 | ||
996 | static __init int amd_ibs_init(void) | |
997 | { | |
998 | u32 caps; | |
16e5294e | 999 | int ret = -EINVAL; |
b7169166 RR |
1000 | |
1001 | caps = __get_ibs_caps(); | |
1002 | if (!caps) | |
1003 | return -ENODEV; /* ibs not supported by the cpu */ | |
1004 | ||
bee09ed9 | 1005 | ibs_eilvt_setup(); |
16e5294e RR |
1006 | |
1007 | if (!ibs_eilvt_valid()) | |
1008 | goto out; | |
b7169166 | 1009 | |
bee09ed9 | 1010 | perf_ibs_pm_init(); |
047868ce | 1011 | cpu_notifier_register_begin(); |
b7169166 RR |
1012 | ibs_caps = caps; |
1013 | /* make ibs_caps visible to other cpus: */ | |
1014 | smp_mb(); | |
b7169166 | 1015 | smp_call_function(setup_APIC_ibs, NULL, 1); |
047868ce SB |
1016 | __perf_cpu_notifier(perf_ibs_cpu_notifier); |
1017 | cpu_notifier_register_done(); | |
b7169166 | 1018 | |
16e5294e RR |
1019 | ret = perf_event_ibs_init(); |
1020 | out: | |
1021 | if (ret) | |
1022 | pr_err("Failed to setup IBS, %d\n", ret); | |
1023 | return ret; | |
b7169166 RR |
1024 | } |
1025 | ||
1026 | /* Since we need the pci subsystem to init ibs we can't do this earlier: */ | |
1027 | device_initcall(amd_ibs_init); |