Commit | Line | Data |
---|---|---|
b7169166 RR |
1 | /* |
2 | * Performance events - AMD IBS | |
3 | * | |
4 | * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter | |
5 | * | |
6 | * For licencing details see kernel-base/COPYING | |
7 | */ | |
8 | ||
9 | #include <linux/perf_event.h> | |
10 | #include <linux/module.h> | |
11 | #include <linux/pci.h> | |
d47e8238 | 12 | #include <linux/ptrace.h> |
bee09ed9 | 13 | #include <linux/syscore_ops.h> |
b7169166 RR |
14 | |
15 | #include <asm/apic.h> | |
16 | ||
27f6d22b | 17 | #include "../perf_event.h" |
d07bdfd3 | 18 | |
b7169166 RR |
19 | static u32 ibs_caps; |
20 | ||
21 | #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) | |
22 | ||
b7074f1f RR |
23 | #include <linux/kprobes.h> |
24 | #include <linux/hardirq.h> | |
25 | ||
26 | #include <asm/nmi.h> | |
27 | ||
51041943 RR |
28 | #define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT) |
29 | #define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT | |
30 | ||
85dc6002 PZ |
31 | |
32 | /* | |
33 | * IBS states: | |
34 | * | |
35 | * ENABLED; tracks the pmu::add(), pmu::del() state, when set the counter is taken | |
36 | * and any further add()s must fail. | |
37 | * | |
38 | * STARTED/STOPPING/STOPPED; deal with pmu::start(), pmu::stop() state but are | |
39 | * complicated by the fact that the IBS hardware can send late NMIs (ie. after | |
40 | * we've cleared the EN bit). | |
41 | * | |
42 | * In order to consume these late NMIs we have the STOPPED state, any NMI that | |
43 | * happens after we've cleared the EN state will clear this bit and report the | |
44 | * NMI handled (this is fundamentally racy in the face or multiple NMI sources, | |
45 | * someone else can consume our BIT and our NMI will go unhandled). | |
46 | * | |
47 | * And since we cannot set/clear this separate bit together with the EN bit, | |
48 | * there are races; if we cleared STARTED early, an NMI could land in | |
49 | * between clearing STARTED and clearing the EN bit (in fact multiple NMIs | |
50 | * could happen if the period is small enough), and consume our STOPPED bit | |
51 | * and trigger streams of unhandled NMIs. | |
52 | * | |
53 | * If, however, we clear STARTED late, an NMI can hit between clearing the | |
54 | * EN bit and clearing STARTED, still see STARTED set and process the event. | |
55 | * If this event will have the VALID bit clear, we bail properly, but this | |
56 | * is not a given. With VALID set we can end up calling pmu::stop() again | |
57 | * (the throttle logic) and trigger the WARNs in there. | |
58 | * | |
59 | * So what we do is set STOPPING before clearing EN to avoid the pmu::stop() | |
60 | * nesting, and clear STARTED late, so that we have a well defined state over | |
61 | * the clearing of the EN bit. | |
62 | * | |
63 | * XXX: we could probably be using !atomic bitops for all this. | |
64 | */ | |
65 | ||
4db2e8e6 RR |
66 | enum ibs_states { |
67 | IBS_ENABLED = 0, | |
68 | IBS_STARTED = 1, | |
69 | IBS_STOPPING = 2, | |
85dc6002 | 70 | IBS_STOPPED = 3, |
4db2e8e6 RR |
71 | |
72 | IBS_MAX_STATES, | |
73 | }; | |
74 | ||
75 | struct cpu_perf_ibs { | |
76 | struct perf_event *event; | |
77 | unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)]; | |
78 | }; | |
79 | ||
51041943 | 80 | struct perf_ibs { |
2e132b12 RR |
81 | struct pmu pmu; |
82 | unsigned int msr; | |
83 | u64 config_mask; | |
84 | u64 cnt_mask; | |
85 | u64 enable_mask; | |
86 | u64 valid_mask; | |
87 | u64 max_period; | |
88 | unsigned long offset_mask[1]; | |
89 | int offset_max; | |
90 | struct cpu_perf_ibs __percpu *pcpu; | |
91 | ||
92 | struct attribute **format_attrs; | |
93 | struct attribute_group format_group; | |
94 | const struct attribute_group *attr_groups[2]; | |
95 | ||
96 | u64 (*get_count)(u64 config); | |
b7074f1f RR |
97 | }; |
98 | ||
99 | struct perf_ibs_data { | |
100 | u32 size; | |
101 | union { | |
102 | u32 data[0]; /* data buffer starts here */ | |
103 | u32 caps; | |
104 | }; | |
105 | u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; | |
51041943 RR |
106 | }; |
107 | ||
db98c5fa | 108 | static int |
98112d2e | 109 | perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period) |
db98c5fa RR |
110 | { |
111 | s64 left = local64_read(&hwc->period_left); | |
112 | s64 period = hwc->sample_period; | |
113 | int overflow = 0; | |
114 | ||
115 | /* | |
116 | * If we are way outside a reasonable range then just skip forward: | |
117 | */ | |
118 | if (unlikely(left <= -period)) { | |
119 | left = period; | |
120 | local64_set(&hwc->period_left, left); | |
121 | hwc->last_period = period; | |
122 | overflow = 1; | |
123 | } | |
124 | ||
fc006cf7 | 125 | if (unlikely(left < (s64)min)) { |
db98c5fa RR |
126 | left += period; |
127 | local64_set(&hwc->period_left, left); | |
128 | hwc->last_period = period; | |
129 | overflow = 1; | |
130 | } | |
131 | ||
7caaf4d8 RR |
132 | /* |
133 | * If the hw period that triggers the sw overflow is too short | |
134 | * we might hit the irq handler. This biases the results. | |
135 | * Thus we shorten the next-to-last period and set the last | |
136 | * period to the max period. | |
137 | */ | |
138 | if (left > max) { | |
139 | left -= max; | |
140 | if (left > max) | |
141 | left = max; | |
142 | else if (left < min) | |
143 | left = min; | |
144 | } | |
db98c5fa | 145 | |
98112d2e | 146 | *hw_period = (u64)left; |
db98c5fa RR |
147 | |
148 | return overflow; | |
149 | } | |
150 | ||
151 | static int | |
152 | perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) | |
153 | { | |
154 | struct hw_perf_event *hwc = &event->hw; | |
155 | int shift = 64 - width; | |
156 | u64 prev_raw_count; | |
157 | u64 delta; | |
158 | ||
159 | /* | |
160 | * Careful: an NMI might modify the previous event value. | |
161 | * | |
162 | * Our tactic to handle this is to first atomically read and | |
163 | * exchange a new raw count - then add that new-prev delta | |
164 | * count to the generic event atomically: | |
165 | */ | |
166 | prev_raw_count = local64_read(&hwc->prev_count); | |
167 | if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, | |
168 | new_raw_count) != prev_raw_count) | |
169 | return 0; | |
170 | ||
171 | /* | |
172 | * Now we have the new raw value and have updated the prev | |
173 | * timestamp already. We can now calculate the elapsed delta | |
174 | * (event-)time and add that to the generic event. | |
175 | * | |
176 | * Careful, not all hw sign-extends above the physical width | |
177 | * of the count. | |
178 | */ | |
179 | delta = (new_raw_count << shift) - (prev_raw_count << shift); | |
180 | delta >>= shift; | |
181 | ||
182 | local64_add(delta, &event->count); | |
183 | local64_sub(delta, &hwc->period_left); | |
184 | ||
185 | return 1; | |
186 | } | |
187 | ||
51041943 RR |
188 | static struct perf_ibs perf_ibs_fetch; |
189 | static struct perf_ibs perf_ibs_op; | |
190 | ||
191 | static struct perf_ibs *get_ibs_pmu(int type) | |
192 | { | |
193 | if (perf_ibs_fetch.pmu.type == type) | |
194 | return &perf_ibs_fetch; | |
195 | if (perf_ibs_op.pmu.type == type) | |
196 | return &perf_ibs_op; | |
197 | return NULL; | |
198 | } | |
b7169166 | 199 | |
450bbd49 RR |
200 | /* |
201 | * Use IBS for precise event sampling: | |
202 | * | |
203 | * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count | |
204 | * perf record -a -e r076:p ... # same as -e cpu-cycles:p | |
205 | * perf record -a -e r0C1:p ... # use ibs op counting micro-ops | |
206 | * | |
207 | * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl, | |
208 | * MSRC001_1033) is used to select either cycle or micro-ops counting | |
209 | * mode. | |
210 | * | |
211 | * The rip of IBS samples has skid 0. Thus, IBS supports precise | |
212 | * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the | |
213 | * rip is invalid when IBS was not able to record the rip correctly. | |
214 | * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then. | |
215 | * | |
216 | */ | |
217 | static int perf_ibs_precise_event(struct perf_event *event, u64 *config) | |
218 | { | |
219 | switch (event->attr.precise_ip) { | |
220 | case 0: | |
221 | return -ENOENT; | |
222 | case 1: | |
223 | case 2: | |
224 | break; | |
225 | default: | |
226 | return -EOPNOTSUPP; | |
227 | } | |
228 | ||
229 | switch (event->attr.type) { | |
230 | case PERF_TYPE_HARDWARE: | |
231 | switch (event->attr.config) { | |
232 | case PERF_COUNT_HW_CPU_CYCLES: | |
233 | *config = 0; | |
234 | return 0; | |
235 | } | |
236 | break; | |
237 | case PERF_TYPE_RAW: | |
238 | switch (event->attr.config) { | |
239 | case 0x0076: | |
240 | *config = 0; | |
241 | return 0; | |
242 | case 0x00C1: | |
243 | *config = IBS_OP_CNT_CTL; | |
244 | return 0; | |
245 | } | |
246 | break; | |
247 | default: | |
248 | return -ENOENT; | |
249 | } | |
250 | ||
251 | return -EOPNOTSUPP; | |
252 | } | |
253 | ||
bad9ac2d RR |
254 | static const struct perf_event_attr ibs_notsupp = { |
255 | .exclude_user = 1, | |
256 | .exclude_kernel = 1, | |
257 | .exclude_hv = 1, | |
258 | .exclude_idle = 1, | |
259 | .exclude_host = 1, | |
260 | .exclude_guest = 1, | |
261 | }; | |
262 | ||
b7169166 RR |
263 | static int perf_ibs_init(struct perf_event *event) |
264 | { | |
51041943 RR |
265 | struct hw_perf_event *hwc = &event->hw; |
266 | struct perf_ibs *perf_ibs; | |
267 | u64 max_cnt, config; | |
450bbd49 | 268 | int ret; |
51041943 RR |
269 | |
270 | perf_ibs = get_ibs_pmu(event->attr.type); | |
450bbd49 RR |
271 | if (perf_ibs) { |
272 | config = event->attr.config; | |
273 | } else { | |
274 | perf_ibs = &perf_ibs_op; | |
275 | ret = perf_ibs_precise_event(event, &config); | |
276 | if (ret) | |
277 | return ret; | |
278 | } | |
279 | ||
280 | if (event->pmu != &perf_ibs->pmu) | |
b7169166 | 281 | return -ENOENT; |
51041943 | 282 | |
bad9ac2d RR |
283 | if (perf_flags(&event->attr) & perf_flags(&ibs_notsupp)) |
284 | return -EINVAL; | |
285 | ||
51041943 RR |
286 | if (config & ~perf_ibs->config_mask) |
287 | return -EINVAL; | |
288 | ||
289 | if (hwc->sample_period) { | |
290 | if (config & perf_ibs->cnt_mask) | |
291 | /* raw max_cnt may not be set */ | |
292 | return -EINVAL; | |
6accb9cf RR |
293 | if (!event->attr.sample_freq && hwc->sample_period & 0x0f) |
294 | /* | |
295 | * lower 4 bits can not be set in ibs max cnt, | |
296 | * but allowing it in case we adjust the | |
297 | * sample period to set a frequency. | |
298 | */ | |
51041943 | 299 | return -EINVAL; |
6accb9cf RR |
300 | hwc->sample_period &= ~0x0FULL; |
301 | if (!hwc->sample_period) | |
302 | hwc->sample_period = 0x10; | |
51041943 RR |
303 | } else { |
304 | max_cnt = config & perf_ibs->cnt_mask; | |
db98c5fa | 305 | config &= ~perf_ibs->cnt_mask; |
51041943 RR |
306 | event->attr.sample_period = max_cnt << 4; |
307 | hwc->sample_period = event->attr.sample_period; | |
308 | } | |
309 | ||
db98c5fa | 310 | if (!hwc->sample_period) |
51041943 RR |
311 | return -EINVAL; |
312 | ||
6accb9cf RR |
313 | /* |
314 | * If we modify hwc->sample_period, we also need to update | |
315 | * hwc->last_period and hwc->period_left. | |
316 | */ | |
317 | hwc->last_period = hwc->sample_period; | |
318 | local64_set(&hwc->period_left, hwc->sample_period); | |
319 | ||
51041943 RR |
320 | hwc->config_base = perf_ibs->msr; |
321 | hwc->config = config; | |
322 | ||
b7169166 RR |
323 | return 0; |
324 | } | |
325 | ||
db98c5fa RR |
326 | static int perf_ibs_set_period(struct perf_ibs *perf_ibs, |
327 | struct hw_perf_event *hwc, u64 *period) | |
328 | { | |
98112d2e | 329 | int overflow; |
db98c5fa RR |
330 | |
331 | /* ignore lower 4 bits in min count: */ | |
98112d2e | 332 | overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period); |
db98c5fa RR |
333 | local64_set(&hwc->prev_count, 0); |
334 | ||
98112d2e | 335 | return overflow; |
db98c5fa RR |
336 | } |
337 | ||
338 | static u64 get_ibs_fetch_count(u64 config) | |
339 | { | |
340 | return (config & IBS_FETCH_CNT) >> 12; | |
341 | } | |
342 | ||
343 | static u64 get_ibs_op_count(u64 config) | |
344 | { | |
8b1e1363 RR |
345 | u64 count = 0; |
346 | ||
347 | if (config & IBS_OP_VAL) | |
348 | count += (config & IBS_OP_MAX_CNT) << 4; /* cnt rolled over */ | |
349 | ||
350 | if (ibs_caps & IBS_CAPS_RDWROPCNT) | |
351 | count += (config & IBS_OP_CUR_CNT) >> 32; | |
352 | ||
353 | return count; | |
db98c5fa RR |
354 | } |
355 | ||
356 | static void | |
357 | perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, | |
c9574fe0 | 358 | u64 *config) |
db98c5fa | 359 | { |
c9574fe0 | 360 | u64 count = perf_ibs->get_count(*config); |
db98c5fa | 361 | |
8b1e1363 RR |
362 | /* |
363 | * Set width to 64 since we do not overflow on max width but | |
364 | * instead on max count. In perf_ibs_set_period() we clear | |
365 | * prev count manually on overflow. | |
366 | */ | |
367 | while (!perf_event_try_update(event, count, 64)) { | |
c9574fe0 RR |
368 | rdmsrl(event->hw.config_base, *config); |
369 | count = perf_ibs->get_count(*config); | |
db98c5fa RR |
370 | } |
371 | } | |
372 | ||
c9574fe0 RR |
373 | static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, |
374 | struct hw_perf_event *hwc, u64 config) | |
db98c5fa | 375 | { |
c9574fe0 RR |
376 | wrmsrl(hwc->config_base, hwc->config | config | perf_ibs->enable_mask); |
377 | } | |
378 | ||
379 | /* | |
380 | * Erratum #420 Instruction-Based Sampling Engine May Generate | |
381 | * Interrupt that Cannot Be Cleared: | |
382 | * | |
383 | * Must clear counter mask first, then clear the enable bit. See | |
384 | * Revision Guide for AMD Family 10h Processors, Publication #41322. | |
385 | */ | |
386 | static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, | |
387 | struct hw_perf_event *hwc, u64 config) | |
388 | { | |
389 | config &= ~perf_ibs->cnt_mask; | |
390 | wrmsrl(hwc->config_base, config); | |
391 | config &= ~perf_ibs->enable_mask; | |
392 | wrmsrl(hwc->config_base, config); | |
db98c5fa RR |
393 | } |
394 | ||
395 | /* | |
396 | * We cannot restore the ibs pmu state, so we always needs to update | |
397 | * the event while stopping it and then reset the state when starting | |
398 | * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in | |
399 | * perf_ibs_start()/perf_ibs_stop() and instead always do it. | |
400 | */ | |
4db2e8e6 RR |
401 | static void perf_ibs_start(struct perf_event *event, int flags) |
402 | { | |
403 | struct hw_perf_event *hwc = &event->hw; | |
404 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); | |
405 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | |
c9574fe0 | 406 | u64 period; |
4db2e8e6 | 407 | |
db98c5fa | 408 | if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) |
4db2e8e6 RR |
409 | return; |
410 | ||
db98c5fa RR |
411 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
412 | hwc->state = 0; | |
413 | ||
c9574fe0 | 414 | perf_ibs_set_period(perf_ibs, hwc, &period); |
5a50f529 | 415 | /* |
85dc6002 PZ |
416 | * Set STARTED before enabling the hardware, such that a subsequent NMI |
417 | * must observe it. | |
5a50f529 | 418 | */ |
85dc6002 | 419 | set_bit(IBS_STARTED, pcpu->state); |
5a50f529 | 420 | clear_bit(IBS_STOPPING, pcpu->state); |
c9574fe0 | 421 | perf_ibs_enable_event(perf_ibs, hwc, period >> 4); |
db98c5fa RR |
422 | |
423 | perf_event_update_userpage(event); | |
4db2e8e6 RR |
424 | } |
425 | ||
426 | static void perf_ibs_stop(struct perf_event *event, int flags) | |
427 | { | |
428 | struct hw_perf_event *hwc = &event->hw; | |
429 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); | |
430 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | |
c9574fe0 | 431 | u64 config; |
db98c5fa | 432 | int stopping; |
4db2e8e6 | 433 | |
85dc6002 PZ |
434 | if (test_and_set_bit(IBS_STOPPING, pcpu->state)) |
435 | return; | |
436 | ||
5a50f529 | 437 | stopping = test_bit(IBS_STARTED, pcpu->state); |
4db2e8e6 | 438 | |
db98c5fa RR |
439 | if (!stopping && (hwc->state & PERF_HES_UPTODATE)) |
440 | return; | |
4db2e8e6 | 441 | |
c9574fe0 | 442 | rdmsrl(hwc->config_base, config); |
db98c5fa RR |
443 | |
444 | if (stopping) { | |
5a50f529 | 445 | /* |
85dc6002 | 446 | * Set STOPPED before disabling the hardware, such that it |
5a50f529 PZ |
447 | * must be visible to NMIs the moment we clear the EN bit, |
448 | * at which point we can generate an !VALID sample which | |
449 | * we need to consume. | |
450 | */ | |
85dc6002 | 451 | set_bit(IBS_STOPPED, pcpu->state); |
c9574fe0 | 452 | perf_ibs_disable_event(perf_ibs, hwc, config); |
5a50f529 PZ |
453 | /* |
454 | * Clear STARTED after disabling the hardware; if it were | |
455 | * cleared before an NMI hitting after the clear but before | |
456 | * clearing the EN bit might think it a spurious NMI and not | |
457 | * handle it. | |
458 | * | |
459 | * Clearing it after, however, creates the problem of the NMI | |
460 | * handler seeing STARTED but not having a valid sample. | |
461 | */ | |
462 | clear_bit(IBS_STARTED, pcpu->state); | |
db98c5fa RR |
463 | WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); |
464 | hwc->state |= PERF_HES_STOPPED; | |
465 | } | |
466 | ||
467 | if (hwc->state & PERF_HES_UPTODATE) | |
468 | return; | |
469 | ||
8b1e1363 RR |
470 | /* |
471 | * Clear valid bit to not count rollovers on update, rollovers | |
472 | * are only updated in the irq handler. | |
473 | */ | |
474 | config &= ~perf_ibs->valid_mask; | |
475 | ||
c9574fe0 | 476 | perf_ibs_event_update(perf_ibs, event, &config); |
db98c5fa | 477 | hwc->state |= PERF_HES_UPTODATE; |
4db2e8e6 RR |
478 | } |
479 | ||
b7169166 RR |
480 | static int perf_ibs_add(struct perf_event *event, int flags) |
481 | { | |
4db2e8e6 RR |
482 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); |
483 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | |
484 | ||
485 | if (test_and_set_bit(IBS_ENABLED, pcpu->state)) | |
486 | return -ENOSPC; | |
487 | ||
db98c5fa RR |
488 | event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
489 | ||
4db2e8e6 RR |
490 | pcpu->event = event; |
491 | ||
492 | if (flags & PERF_EF_START) | |
493 | perf_ibs_start(event, PERF_EF_RELOAD); | |
494 | ||
b7169166 RR |
495 | return 0; |
496 | } | |
497 | ||
498 | static void perf_ibs_del(struct perf_event *event, int flags) | |
499 | { | |
4db2e8e6 RR |
500 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); |
501 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | |
502 | ||
503 | if (!test_and_clear_bit(IBS_ENABLED, pcpu->state)) | |
504 | return; | |
505 | ||
db98c5fa | 506 | perf_ibs_stop(event, PERF_EF_UPDATE); |
4db2e8e6 RR |
507 | |
508 | pcpu->event = NULL; | |
db98c5fa RR |
509 | |
510 | perf_event_update_userpage(event); | |
b7169166 RR |
511 | } |
512 | ||
4db2e8e6 RR |
513 | static void perf_ibs_read(struct perf_event *event) { } |
514 | ||
2e132b12 RR |
515 | PMU_FORMAT_ATTR(rand_en, "config:57"); |
516 | PMU_FORMAT_ATTR(cnt_ctl, "config:19"); | |
517 | ||
518 | static struct attribute *ibs_fetch_format_attrs[] = { | |
519 | &format_attr_rand_en.attr, | |
520 | NULL, | |
521 | }; | |
522 | ||
523 | static struct attribute *ibs_op_format_attrs[] = { | |
524 | NULL, /* &format_attr_cnt_ctl.attr if IBS_CAPS_OPCNT */ | |
525 | NULL, | |
526 | }; | |
527 | ||
51041943 RR |
528 | static struct perf_ibs perf_ibs_fetch = { |
529 | .pmu = { | |
530 | .task_ctx_nr = perf_invalid_context, | |
531 | ||
532 | .event_init = perf_ibs_init, | |
533 | .add = perf_ibs_add, | |
534 | .del = perf_ibs_del, | |
4db2e8e6 RR |
535 | .start = perf_ibs_start, |
536 | .stop = perf_ibs_stop, | |
537 | .read = perf_ibs_read, | |
51041943 RR |
538 | }, |
539 | .msr = MSR_AMD64_IBSFETCHCTL, | |
540 | .config_mask = IBS_FETCH_CONFIG_MASK, | |
541 | .cnt_mask = IBS_FETCH_MAX_CNT, | |
542 | .enable_mask = IBS_FETCH_ENABLE, | |
b7074f1f | 543 | .valid_mask = IBS_FETCH_VAL, |
db98c5fa | 544 | .max_period = IBS_FETCH_MAX_CNT << 4, |
b7074f1f RR |
545 | .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, |
546 | .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, | |
2e132b12 | 547 | .format_attrs = ibs_fetch_format_attrs, |
db98c5fa RR |
548 | |
549 | .get_count = get_ibs_fetch_count, | |
51041943 RR |
550 | }; |
551 | ||
552 | static struct perf_ibs perf_ibs_op = { | |
553 | .pmu = { | |
554 | .task_ctx_nr = perf_invalid_context, | |
555 | ||
556 | .event_init = perf_ibs_init, | |
557 | .add = perf_ibs_add, | |
558 | .del = perf_ibs_del, | |
4db2e8e6 RR |
559 | .start = perf_ibs_start, |
560 | .stop = perf_ibs_stop, | |
561 | .read = perf_ibs_read, | |
51041943 RR |
562 | }, |
563 | .msr = MSR_AMD64_IBSOPCTL, | |
564 | .config_mask = IBS_OP_CONFIG_MASK, | |
565 | .cnt_mask = IBS_OP_MAX_CNT, | |
566 | .enable_mask = IBS_OP_ENABLE, | |
b7074f1f | 567 | .valid_mask = IBS_OP_VAL, |
db98c5fa | 568 | .max_period = IBS_OP_MAX_CNT << 4, |
b7074f1f RR |
569 | .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, |
570 | .offset_max = MSR_AMD64_IBSOP_REG_COUNT, | |
2e132b12 | 571 | .format_attrs = ibs_op_format_attrs, |
db98c5fa RR |
572 | |
573 | .get_count = get_ibs_op_count, | |
b7169166 RR |
574 | }; |
575 | ||
b7074f1f RR |
576 | static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) |
577 | { | |
4db2e8e6 RR |
578 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); |
579 | struct perf_event *event = pcpu->event; | |
b7074f1f RR |
580 | struct hw_perf_event *hwc = &event->hw; |
581 | struct perf_sample_data data; | |
582 | struct perf_raw_record raw; | |
583 | struct pt_regs regs; | |
584 | struct perf_ibs_data ibs_data; | |
d47e8238 | 585 | int offset, size, check_rip, offset_max, throttle = 0; |
b7074f1f | 586 | unsigned int msr; |
c9574fe0 | 587 | u64 *buf, *config, period; |
b7074f1f | 588 | |
4db2e8e6 | 589 | if (!test_bit(IBS_STARTED, pcpu->state)) { |
5a50f529 | 590 | fail: |
fc5fb2b5 RR |
591 | /* |
592 | * Catch spurious interrupts after stopping IBS: After | |
d82603c6 | 593 | * disabling IBS there could be still incoming NMIs |
fc5fb2b5 RR |
594 | * with samples that even have the valid bit cleared. |
595 | * Mark all this NMIs as handled. | |
596 | */ | |
85dc6002 | 597 | if (test_and_clear_bit(IBS_STOPPED, pcpu->state)) |
5a50f529 PZ |
598 | return 1; |
599 | ||
600 | return 0; | |
4db2e8e6 RR |
601 | } |
602 | ||
b7074f1f RR |
603 | msr = hwc->config_base; |
604 | buf = ibs_data.regs; | |
605 | rdmsrl(msr, *buf); | |
606 | if (!(*buf++ & perf_ibs->valid_mask)) | |
5a50f529 | 607 | goto fail; |
b7074f1f | 608 | |
c9574fe0 | 609 | config = &ibs_data.regs[0]; |
c75841a3 | 610 | perf_ibs_event_update(perf_ibs, event, config); |
fd0d000b | 611 | perf_sample_data_init(&data, 0, hwc->last_period); |
c9574fe0 | 612 | if (!perf_ibs_set_period(perf_ibs, hwc, &period)) |
d47e8238 RR |
613 | goto out; /* no sw counter overflow */ |
614 | ||
615 | ibs_data.caps = ibs_caps; | |
616 | size = 1; | |
617 | offset = 1; | |
618 | check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); | |
619 | if (event->attr.sample_type & PERF_SAMPLE_RAW) | |
620 | offset_max = perf_ibs->offset_max; | |
621 | else if (check_rip) | |
622 | offset_max = 2; | |
623 | else | |
624 | offset_max = 1; | |
625 | do { | |
626 | rdmsrl(msr + offset, *buf++); | |
627 | size++; | |
628 | offset = find_next_bit(perf_ibs->offset_mask, | |
629 | perf_ibs->offset_max, | |
630 | offset + 1); | |
631 | } while (offset < offset_max); | |
904cb367 AG |
632 | if (event->attr.sample_type & PERF_SAMPLE_RAW) { |
633 | /* | |
634 | * Read IbsBrTarget and IbsOpData4 separately | |
635 | * depending on their availability. | |
636 | * Can't add to offset_max as they are staggered | |
637 | */ | |
638 | if (ibs_caps & IBS_CAPS_BRNTRGT) { | |
639 | rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++); | |
640 | size++; | |
641 | } | |
642 | if (ibs_caps & IBS_CAPS_OPDATA4) { | |
643 | rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++); | |
644 | size++; | |
645 | } | |
646 | } | |
d47e8238 RR |
647 | ibs_data.size = sizeof(u64) * size; |
648 | ||
649 | regs = *iregs; | |
450bbd49 RR |
650 | if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) { |
651 | regs.flags &= ~PERF_EFLAGS_EXACT; | |
652 | } else { | |
d07bdfd3 | 653 | set_linear_ip(®s, ibs_data.regs[1]); |
450bbd49 RR |
654 | regs.flags |= PERF_EFLAGS_EXACT; |
655 | } | |
c75841a3 | 656 | |
b7074f1f | 657 | if (event->attr.sample_type & PERF_SAMPLE_RAW) { |
d47e8238 | 658 | raw.size = sizeof(u32) + ibs_data.size; |
b7074f1f RR |
659 | raw.data = ibs_data.data; |
660 | data.raw = &raw; | |
661 | } | |
662 | ||
d47e8238 RR |
663 | throttle = perf_event_overflow(event, &data, ®s); |
664 | out: | |
c9574fe0 | 665 | if (throttle) |
0158b83f | 666 | perf_ibs_stop(event, 0); |
c9574fe0 RR |
667 | else |
668 | perf_ibs_enable_event(perf_ibs, hwc, period >> 4); | |
db98c5fa RR |
669 | |
670 | perf_event_update_userpage(event); | |
b7074f1f RR |
671 | |
672 | return 1; | |
673 | } | |
674 | ||
9326638c | 675 | static int |
b7074f1f RR |
676 | perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) |
677 | { | |
c2872d38 | 678 | u64 stamp = sched_clock(); |
b7074f1f RR |
679 | int handled = 0; |
680 | ||
681 | handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs); | |
682 | handled += perf_ibs_handle_irq(&perf_ibs_op, regs); | |
683 | ||
684 | if (handled) | |
685 | inc_irq_stat(apic_perf_irqs); | |
686 | ||
c2872d38 PZ |
687 | perf_sample_event_took(sched_clock() - stamp); |
688 | ||
b7074f1f RR |
689 | return handled; |
690 | } | |
9326638c | 691 | NOKPROBE_SYMBOL(perf_ibs_nmi_handler); |
b7074f1f | 692 | |
4db2e8e6 RR |
693 | static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) |
694 | { | |
695 | struct cpu_perf_ibs __percpu *pcpu; | |
696 | int ret; | |
697 | ||
698 | pcpu = alloc_percpu(struct cpu_perf_ibs); | |
699 | if (!pcpu) | |
700 | return -ENOMEM; | |
701 | ||
702 | perf_ibs->pcpu = pcpu; | |
703 | ||
2e132b12 RR |
704 | /* register attributes */ |
705 | if (perf_ibs->format_attrs[0]) { | |
706 | memset(&perf_ibs->format_group, 0, sizeof(perf_ibs->format_group)); | |
707 | perf_ibs->format_group.name = "format"; | |
708 | perf_ibs->format_group.attrs = perf_ibs->format_attrs; | |
709 | ||
710 | memset(&perf_ibs->attr_groups, 0, sizeof(perf_ibs->attr_groups)); | |
711 | perf_ibs->attr_groups[0] = &perf_ibs->format_group; | |
712 | perf_ibs->pmu.attr_groups = perf_ibs->attr_groups; | |
713 | } | |
714 | ||
4db2e8e6 RR |
715 | ret = perf_pmu_register(&perf_ibs->pmu, name, -1); |
716 | if (ret) { | |
717 | perf_ibs->pcpu = NULL; | |
718 | free_percpu(pcpu); | |
719 | } | |
720 | ||
721 | return ret; | |
722 | } | |
723 | ||
b7169166 RR |
724 | static __init int perf_event_ibs_init(void) |
725 | { | |
2e132b12 RR |
726 | struct attribute **attr = ibs_op_format_attrs; |
727 | ||
b7169166 RR |
728 | if (!ibs_caps) |
729 | return -ENODEV; /* ibs not supported by the cpu */ | |
730 | ||
4db2e8e6 | 731 | perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch"); |
2e132b12 RR |
732 | |
733 | if (ibs_caps & IBS_CAPS_OPCNT) { | |
7bf35238 | 734 | perf_ibs_op.config_mask |= IBS_OP_CNT_CTL; |
2e132b12 RR |
735 | *attr++ = &format_attr_cnt_ctl.attr; |
736 | } | |
4db2e8e6 | 737 | perf_ibs_pmu_init(&perf_ibs_op, "ibs_op"); |
2e132b12 | 738 | |
fab06992 | 739 | register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs"); |
1b74dde7 | 740 | pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps); |
b7169166 RR |
741 | |
742 | return 0; | |
743 | } | |
744 | ||
745 | #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */ | |
746 | ||
747 | static __init int perf_event_ibs_init(void) { return 0; } | |
748 | ||
749 | #endif | |
750 | ||
751 | /* IBS - apic initialization, for perf and oprofile */ | |
752 | ||
753 | static __init u32 __get_ibs_caps(void) | |
754 | { | |
755 | u32 caps; | |
756 | unsigned int max_level; | |
757 | ||
758 | if (!boot_cpu_has(X86_FEATURE_IBS)) | |
759 | return 0; | |
760 | ||
761 | /* check IBS cpuid feature flags */ | |
762 | max_level = cpuid_eax(0x80000000); | |
763 | if (max_level < IBS_CPUID_FEATURES) | |
764 | return IBS_CAPS_DEFAULT; | |
765 | ||
766 | caps = cpuid_eax(IBS_CPUID_FEATURES); | |
767 | if (!(caps & IBS_CAPS_AVAIL)) | |
768 | /* cpuid flags not valid */ | |
769 | return IBS_CAPS_DEFAULT; | |
770 | ||
771 | return caps; | |
772 | } | |
773 | ||
774 | u32 get_ibs_caps(void) | |
775 | { | |
776 | return ibs_caps; | |
777 | } | |
778 | ||
779 | EXPORT_SYMBOL(get_ibs_caps); | |
780 | ||
781 | static inline int get_eilvt(int offset) | |
782 | { | |
783 | return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1); | |
784 | } | |
785 | ||
786 | static inline int put_eilvt(int offset) | |
787 | { | |
788 | return !setup_APIC_eilvt(offset, 0, 0, 1); | |
789 | } | |
790 | ||
791 | /* | |
792 | * Check and reserve APIC extended interrupt LVT offset for IBS if available. | |
793 | */ | |
794 | static inline int ibs_eilvt_valid(void) | |
795 | { | |
796 | int offset; | |
797 | u64 val; | |
798 | int valid = 0; | |
799 | ||
800 | preempt_disable(); | |
801 | ||
802 | rdmsrl(MSR_AMD64_IBSCTL, val); | |
803 | offset = val & IBSCTL_LVT_OFFSET_MASK; | |
804 | ||
805 | if (!(val & IBSCTL_LVT_OFFSET_VALID)) { | |
806 | pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n", | |
807 | smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); | |
808 | goto out; | |
809 | } | |
810 | ||
811 | if (!get_eilvt(offset)) { | |
812 | pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n", | |
813 | smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); | |
814 | goto out; | |
815 | } | |
816 | ||
817 | valid = 1; | |
818 | out: | |
819 | preempt_enable(); | |
820 | ||
821 | return valid; | |
822 | } | |
823 | ||
824 | static int setup_ibs_ctl(int ibs_eilvt_off) | |
825 | { | |
826 | struct pci_dev *cpu_cfg; | |
827 | int nodes; | |
828 | u32 value = 0; | |
829 | ||
830 | nodes = 0; | |
831 | cpu_cfg = NULL; | |
832 | do { | |
833 | cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD, | |
834 | PCI_DEVICE_ID_AMD_10H_NB_MISC, | |
835 | cpu_cfg); | |
836 | if (!cpu_cfg) | |
837 | break; | |
838 | ++nodes; | |
839 | pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off | |
840 | | IBSCTL_LVT_OFFSET_VALID); | |
841 | pci_read_config_dword(cpu_cfg, IBSCTL, &value); | |
842 | if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) { | |
843 | pci_dev_put(cpu_cfg); | |
1b74dde7 CY |
844 | pr_debug("Failed to setup IBS LVT offset, IBSCTL = 0x%08x\n", |
845 | value); | |
b7169166 RR |
846 | return -EINVAL; |
847 | } | |
848 | } while (1); | |
849 | ||
850 | if (!nodes) { | |
1b74dde7 | 851 | pr_debug("No CPU node configured for IBS\n"); |
b7169166 RR |
852 | return -ENODEV; |
853 | } | |
854 | ||
855 | return 0; | |
856 | } | |
857 | ||
858 | /* | |
859 | * This runs only on the current cpu. We try to find an LVT offset and | |
860 | * setup the local APIC. For this we must disable preemption. On | |
861 | * success we initialize all nodes with this offset. This updates then | |
862 | * the offset in the IBS_CTL per-node msr. The per-core APIC setup of | |
863 | * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that | |
864 | * is using the new offset. | |
865 | */ | |
c796b205 | 866 | static void force_ibs_eilvt_setup(void) |
b7169166 RR |
867 | { |
868 | int offset; | |
869 | int ret; | |
870 | ||
871 | preempt_disable(); | |
872 | /* find the next free available EILVT entry, skip offset 0 */ | |
873 | for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) { | |
874 | if (get_eilvt(offset)) | |
875 | break; | |
876 | } | |
877 | preempt_enable(); | |
878 | ||
879 | if (offset == APIC_EILVT_NR_MAX) { | |
1b74dde7 | 880 | pr_debug("No EILVT entry available\n"); |
c796b205 | 881 | return; |
b7169166 RR |
882 | } |
883 | ||
884 | ret = setup_ibs_ctl(offset); | |
885 | if (ret) | |
886 | goto out; | |
887 | ||
c796b205 | 888 | if (!ibs_eilvt_valid()) |
b7169166 | 889 | goto out; |
b7169166 | 890 | |
16e5294e | 891 | pr_info("IBS: LVT offset %d assigned\n", offset); |
b7169166 | 892 | |
c796b205 | 893 | return; |
b7169166 RR |
894 | out: |
895 | preempt_disable(); | |
896 | put_eilvt(offset); | |
897 | preempt_enable(); | |
c796b205 | 898 | return; |
b7169166 RR |
899 | } |
900 | ||
bee09ed9 RR |
901 | static void ibs_eilvt_setup(void) |
902 | { | |
903 | /* | |
904 | * Force LVT offset assignment for family 10h: The offsets are | |
905 | * not assigned by the BIOS for this family, so the OS is | |
906 | * responsible for doing it. If the OS assignment fails, fall | |
907 | * back to BIOS settings and try to setup this. | |
908 | */ | |
909 | if (boot_cpu_data.x86 == 0x10) | |
910 | force_ibs_eilvt_setup(); | |
911 | } | |
912 | ||
b7169166 RR |
913 | static inline int get_ibs_lvt_offset(void) |
914 | { | |
915 | u64 val; | |
916 | ||
917 | rdmsrl(MSR_AMD64_IBSCTL, val); | |
918 | if (!(val & IBSCTL_LVT_OFFSET_VALID)) | |
919 | return -EINVAL; | |
920 | ||
921 | return val & IBSCTL_LVT_OFFSET_MASK; | |
922 | } | |
923 | ||
924 | static void setup_APIC_ibs(void *dummy) | |
925 | { | |
926 | int offset; | |
927 | ||
928 | offset = get_ibs_lvt_offset(); | |
929 | if (offset < 0) | |
930 | goto failed; | |
931 | ||
932 | if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0)) | |
933 | return; | |
934 | failed: | |
935 | pr_warn("perf: IBS APIC setup failed on cpu #%d\n", | |
936 | smp_processor_id()); | |
937 | } | |
938 | ||
939 | static void clear_APIC_ibs(void *dummy) | |
940 | { | |
941 | int offset; | |
942 | ||
943 | offset = get_ibs_lvt_offset(); | |
944 | if (offset >= 0) | |
945 | setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1); | |
946 | } | |
947 | ||
bee09ed9 RR |
948 | #ifdef CONFIG_PM |
949 | ||
950 | static int perf_ibs_suspend(void) | |
951 | { | |
952 | clear_APIC_ibs(NULL); | |
953 | return 0; | |
954 | } | |
955 | ||
956 | static void perf_ibs_resume(void) | |
957 | { | |
958 | ibs_eilvt_setup(); | |
959 | setup_APIC_ibs(NULL); | |
960 | } | |
961 | ||
962 | static struct syscore_ops perf_ibs_syscore_ops = { | |
963 | .resume = perf_ibs_resume, | |
964 | .suspend = perf_ibs_suspend, | |
965 | }; | |
966 | ||
967 | static void perf_ibs_pm_init(void) | |
968 | { | |
969 | register_syscore_ops(&perf_ibs_syscore_ops); | |
970 | } | |
971 | ||
972 | #else | |
973 | ||
974 | static inline void perf_ibs_pm_init(void) { } | |
975 | ||
976 | #endif | |
977 | ||
148f9bb8 | 978 | static int |
b7169166 RR |
979 | perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) |
980 | { | |
981 | switch (action & ~CPU_TASKS_FROZEN) { | |
982 | case CPU_STARTING: | |
983 | setup_APIC_ibs(NULL); | |
984 | break; | |
985 | case CPU_DYING: | |
986 | clear_APIC_ibs(NULL); | |
987 | break; | |
988 | default: | |
989 | break; | |
990 | } | |
991 | ||
992 | return NOTIFY_OK; | |
993 | } | |
994 | ||
995 | static __init int amd_ibs_init(void) | |
996 | { | |
997 | u32 caps; | |
16e5294e | 998 | int ret = -EINVAL; |
b7169166 RR |
999 | |
1000 | caps = __get_ibs_caps(); | |
1001 | if (!caps) | |
1002 | return -ENODEV; /* ibs not supported by the cpu */ | |
1003 | ||
bee09ed9 | 1004 | ibs_eilvt_setup(); |
16e5294e RR |
1005 | |
1006 | if (!ibs_eilvt_valid()) | |
1007 | goto out; | |
b7169166 | 1008 | |
bee09ed9 | 1009 | perf_ibs_pm_init(); |
047868ce | 1010 | cpu_notifier_register_begin(); |
b7169166 RR |
1011 | ibs_caps = caps; |
1012 | /* make ibs_caps visible to other cpus: */ | |
1013 | smp_mb(); | |
b7169166 | 1014 | smp_call_function(setup_APIC_ibs, NULL, 1); |
047868ce SB |
1015 | __perf_cpu_notifier(perf_ibs_cpu_notifier); |
1016 | cpu_notifier_register_done(); | |
b7169166 | 1017 | |
16e5294e RR |
1018 | ret = perf_event_ibs_init(); |
1019 | out: | |
1020 | if (ret) | |
1021 | pr_err("Failed to setup IBS, %d\n", ret); | |
1022 | return ret; | |
b7169166 RR |
1023 | } |
1024 | ||
1025 | /* Since we need the pci subsystem to init ibs we can't do this earlier: */ | |
1026 | device_initcall(amd_ibs_init); |