Merge tag 'pinctrl-v4.8-1' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw...
[deliverable/linux.git] / arch / x86 / events / amd / ibs.c
CommitLineData
b7169166
RR
1/*
2 * Performance events - AMD IBS
3 *
4 * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter
5 *
6 * For licencing details see kernel-base/COPYING
7 */
8
9#include <linux/perf_event.h>
10#include <linux/module.h>
11#include <linux/pci.h>
d47e8238 12#include <linux/ptrace.h>
bee09ed9 13#include <linux/syscore_ops.h>
b7169166
RR
14
15#include <asm/apic.h>
16
27f6d22b 17#include "../perf_event.h"
d07bdfd3 18
b7169166
RR
19static u32 ibs_caps;
20
21#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD)
22
b7074f1f
RR
23#include <linux/kprobes.h>
24#include <linux/hardirq.h>
25
26#include <asm/nmi.h>
27
51041943
RR
28#define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT)
29#define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT
30
85dc6002
PZ
31
32/*
33 * IBS states:
34 *
35 * ENABLED; tracks the pmu::add(), pmu::del() state, when set the counter is taken
36 * and any further add()s must fail.
37 *
38 * STARTED/STOPPING/STOPPED; deal with pmu::start(), pmu::stop() state but are
39 * complicated by the fact that the IBS hardware can send late NMIs (ie. after
40 * we've cleared the EN bit).
41 *
42 * In order to consume these late NMIs we have the STOPPED state, any NMI that
43 * happens after we've cleared the EN state will clear this bit and report the
44 * NMI handled (this is fundamentally racy in the face or multiple NMI sources,
45 * someone else can consume our BIT and our NMI will go unhandled).
46 *
47 * And since we cannot set/clear this separate bit together with the EN bit,
48 * there are races; if we cleared STARTED early, an NMI could land in
49 * between clearing STARTED and clearing the EN bit (in fact multiple NMIs
50 * could happen if the period is small enough), and consume our STOPPED bit
51 * and trigger streams of unhandled NMIs.
52 *
53 * If, however, we clear STARTED late, an NMI can hit between clearing the
54 * EN bit and clearing STARTED, still see STARTED set and process the event.
55 * If this event will have the VALID bit clear, we bail properly, but this
56 * is not a given. With VALID set we can end up calling pmu::stop() again
57 * (the throttle logic) and trigger the WARNs in there.
58 *
59 * So what we do is set STOPPING before clearing EN to avoid the pmu::stop()
60 * nesting, and clear STARTED late, so that we have a well defined state over
61 * the clearing of the EN bit.
62 *
63 * XXX: we could probably be using !atomic bitops for all this.
64 */
65
4db2e8e6
RR
66enum ibs_states {
67 IBS_ENABLED = 0,
68 IBS_STARTED = 1,
69 IBS_STOPPING = 2,
85dc6002 70 IBS_STOPPED = 3,
4db2e8e6
RR
71
72 IBS_MAX_STATES,
73};
74
75struct cpu_perf_ibs {
76 struct perf_event *event;
77 unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)];
78};
79
51041943 80struct perf_ibs {
2e132b12
RR
81 struct pmu pmu;
82 unsigned int msr;
83 u64 config_mask;
84 u64 cnt_mask;
85 u64 enable_mask;
86 u64 valid_mask;
87 u64 max_period;
88 unsigned long offset_mask[1];
89 int offset_max;
90 struct cpu_perf_ibs __percpu *pcpu;
91
92 struct attribute **format_attrs;
93 struct attribute_group format_group;
94 const struct attribute_group *attr_groups[2];
95
96 u64 (*get_count)(u64 config);
b7074f1f
RR
97};
98
99struct perf_ibs_data {
100 u32 size;
101 union {
102 u32 data[0]; /* data buffer starts here */
103 u32 caps;
104 };
105 u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX];
51041943
RR
106};
107
db98c5fa 108static int
98112d2e 109perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period)
db98c5fa
RR
110{
111 s64 left = local64_read(&hwc->period_left);
112 s64 period = hwc->sample_period;
113 int overflow = 0;
114
115 /*
116 * If we are way outside a reasonable range then just skip forward:
117 */
118 if (unlikely(left <= -period)) {
119 left = period;
120 local64_set(&hwc->period_left, left);
121 hwc->last_period = period;
122 overflow = 1;
123 }
124
fc006cf7 125 if (unlikely(left < (s64)min)) {
db98c5fa
RR
126 left += period;
127 local64_set(&hwc->period_left, left);
128 hwc->last_period = period;
129 overflow = 1;
130 }
131
7caaf4d8
RR
132 /*
133 * If the hw period that triggers the sw overflow is too short
134 * we might hit the irq handler. This biases the results.
135 * Thus we shorten the next-to-last period and set the last
136 * period to the max period.
137 */
138 if (left > max) {
139 left -= max;
140 if (left > max)
141 left = max;
142 else if (left < min)
143 left = min;
144 }
db98c5fa 145
98112d2e 146 *hw_period = (u64)left;
db98c5fa
RR
147
148 return overflow;
149}
150
151static int
152perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width)
153{
154 struct hw_perf_event *hwc = &event->hw;
155 int shift = 64 - width;
156 u64 prev_raw_count;
157 u64 delta;
158
159 /*
160 * Careful: an NMI might modify the previous event value.
161 *
162 * Our tactic to handle this is to first atomically read and
163 * exchange a new raw count - then add that new-prev delta
164 * count to the generic event atomically:
165 */
166 prev_raw_count = local64_read(&hwc->prev_count);
167 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
168 new_raw_count) != prev_raw_count)
169 return 0;
170
171 /*
172 * Now we have the new raw value and have updated the prev
173 * timestamp already. We can now calculate the elapsed delta
174 * (event-)time and add that to the generic event.
175 *
176 * Careful, not all hw sign-extends above the physical width
177 * of the count.
178 */
179 delta = (new_raw_count << shift) - (prev_raw_count << shift);
180 delta >>= shift;
181
182 local64_add(delta, &event->count);
183 local64_sub(delta, &hwc->period_left);
184
185 return 1;
186}
187
51041943
RR
188static struct perf_ibs perf_ibs_fetch;
189static struct perf_ibs perf_ibs_op;
190
191static struct perf_ibs *get_ibs_pmu(int type)
192{
193 if (perf_ibs_fetch.pmu.type == type)
194 return &perf_ibs_fetch;
195 if (perf_ibs_op.pmu.type == type)
196 return &perf_ibs_op;
197 return NULL;
198}
b7169166 199
450bbd49
RR
200/*
201 * Use IBS for precise event sampling:
202 *
203 * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count
204 * perf record -a -e r076:p ... # same as -e cpu-cycles:p
205 * perf record -a -e r0C1:p ... # use ibs op counting micro-ops
206 *
207 * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl,
208 * MSRC001_1033) is used to select either cycle or micro-ops counting
209 * mode.
210 *
211 * The rip of IBS samples has skid 0. Thus, IBS supports precise
212 * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the
213 * rip is invalid when IBS was not able to record the rip correctly.
214 * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then.
215 *
216 */
217static int perf_ibs_precise_event(struct perf_event *event, u64 *config)
218{
219 switch (event->attr.precise_ip) {
220 case 0:
221 return -ENOENT;
222 case 1:
223 case 2:
224 break;
225 default:
226 return -EOPNOTSUPP;
227 }
228
229 switch (event->attr.type) {
230 case PERF_TYPE_HARDWARE:
231 switch (event->attr.config) {
232 case PERF_COUNT_HW_CPU_CYCLES:
233 *config = 0;
234 return 0;
235 }
236 break;
237 case PERF_TYPE_RAW:
238 switch (event->attr.config) {
239 case 0x0076:
240 *config = 0;
241 return 0;
242 case 0x00C1:
243 *config = IBS_OP_CNT_CTL;
244 return 0;
245 }
246 break;
247 default:
248 return -ENOENT;
249 }
250
251 return -EOPNOTSUPP;
252}
253
bad9ac2d
RR
254static const struct perf_event_attr ibs_notsupp = {
255 .exclude_user = 1,
256 .exclude_kernel = 1,
257 .exclude_hv = 1,
258 .exclude_idle = 1,
259 .exclude_host = 1,
260 .exclude_guest = 1,
261};
262
b7169166
RR
263static int perf_ibs_init(struct perf_event *event)
264{
51041943
RR
265 struct hw_perf_event *hwc = &event->hw;
266 struct perf_ibs *perf_ibs;
267 u64 max_cnt, config;
450bbd49 268 int ret;
51041943
RR
269
270 perf_ibs = get_ibs_pmu(event->attr.type);
450bbd49
RR
271 if (perf_ibs) {
272 config = event->attr.config;
273 } else {
274 perf_ibs = &perf_ibs_op;
275 ret = perf_ibs_precise_event(event, &config);
276 if (ret)
277 return ret;
278 }
279
280 if (event->pmu != &perf_ibs->pmu)
b7169166 281 return -ENOENT;
51041943 282
bad9ac2d
RR
283 if (perf_flags(&event->attr) & perf_flags(&ibs_notsupp))
284 return -EINVAL;
285
51041943
RR
286 if (config & ~perf_ibs->config_mask)
287 return -EINVAL;
288
289 if (hwc->sample_period) {
290 if (config & perf_ibs->cnt_mask)
291 /* raw max_cnt may not be set */
292 return -EINVAL;
6accb9cf
RR
293 if (!event->attr.sample_freq && hwc->sample_period & 0x0f)
294 /*
295 * lower 4 bits can not be set in ibs max cnt,
296 * but allowing it in case we adjust the
297 * sample period to set a frequency.
298 */
51041943 299 return -EINVAL;
6accb9cf
RR
300 hwc->sample_period &= ~0x0FULL;
301 if (!hwc->sample_period)
302 hwc->sample_period = 0x10;
51041943
RR
303 } else {
304 max_cnt = config & perf_ibs->cnt_mask;
db98c5fa 305 config &= ~perf_ibs->cnt_mask;
51041943
RR
306 event->attr.sample_period = max_cnt << 4;
307 hwc->sample_period = event->attr.sample_period;
308 }
309
db98c5fa 310 if (!hwc->sample_period)
51041943
RR
311 return -EINVAL;
312
6accb9cf
RR
313 /*
314 * If we modify hwc->sample_period, we also need to update
315 * hwc->last_period and hwc->period_left.
316 */
317 hwc->last_period = hwc->sample_period;
318 local64_set(&hwc->period_left, hwc->sample_period);
319
51041943
RR
320 hwc->config_base = perf_ibs->msr;
321 hwc->config = config;
322
b7169166
RR
323 return 0;
324}
325
db98c5fa
RR
326static int perf_ibs_set_period(struct perf_ibs *perf_ibs,
327 struct hw_perf_event *hwc, u64 *period)
328{
98112d2e 329 int overflow;
db98c5fa
RR
330
331 /* ignore lower 4 bits in min count: */
98112d2e 332 overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period);
db98c5fa
RR
333 local64_set(&hwc->prev_count, 0);
334
98112d2e 335 return overflow;
db98c5fa
RR
336}
337
338static u64 get_ibs_fetch_count(u64 config)
339{
340 return (config & IBS_FETCH_CNT) >> 12;
341}
342
343static u64 get_ibs_op_count(u64 config)
344{
8b1e1363
RR
345 u64 count = 0;
346
347 if (config & IBS_OP_VAL)
348 count += (config & IBS_OP_MAX_CNT) << 4; /* cnt rolled over */
349
350 if (ibs_caps & IBS_CAPS_RDWROPCNT)
351 count += (config & IBS_OP_CUR_CNT) >> 32;
352
353 return count;
db98c5fa
RR
354}
355
356static void
357perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event,
c9574fe0 358 u64 *config)
db98c5fa 359{
c9574fe0 360 u64 count = perf_ibs->get_count(*config);
db98c5fa 361
8b1e1363
RR
362 /*
363 * Set width to 64 since we do not overflow on max width but
364 * instead on max count. In perf_ibs_set_period() we clear
365 * prev count manually on overflow.
366 */
367 while (!perf_event_try_update(event, count, 64)) {
c9574fe0
RR
368 rdmsrl(event->hw.config_base, *config);
369 count = perf_ibs->get_count(*config);
db98c5fa
RR
370 }
371}
372
c9574fe0
RR
373static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,
374 struct hw_perf_event *hwc, u64 config)
db98c5fa 375{
c9574fe0
RR
376 wrmsrl(hwc->config_base, hwc->config | config | perf_ibs->enable_mask);
377}
378
379/*
380 * Erratum #420 Instruction-Based Sampling Engine May Generate
381 * Interrupt that Cannot Be Cleared:
382 *
383 * Must clear counter mask first, then clear the enable bit. See
384 * Revision Guide for AMD Family 10h Processors, Publication #41322.
385 */
386static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs,
387 struct hw_perf_event *hwc, u64 config)
388{
389 config &= ~perf_ibs->cnt_mask;
390 wrmsrl(hwc->config_base, config);
391 config &= ~perf_ibs->enable_mask;
392 wrmsrl(hwc->config_base, config);
db98c5fa
RR
393}
394
395/*
396 * We cannot restore the ibs pmu state, so we always needs to update
397 * the event while stopping it and then reset the state when starting
398 * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in
399 * perf_ibs_start()/perf_ibs_stop() and instead always do it.
400 */
4db2e8e6
RR
401static void perf_ibs_start(struct perf_event *event, int flags)
402{
403 struct hw_perf_event *hwc = &event->hw;
404 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
405 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
c9574fe0 406 u64 period;
4db2e8e6 407
db98c5fa 408 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
4db2e8e6
RR
409 return;
410
db98c5fa
RR
411 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
412 hwc->state = 0;
413
c9574fe0 414 perf_ibs_set_period(perf_ibs, hwc, &period);
5a50f529 415 /*
85dc6002
PZ
416 * Set STARTED before enabling the hardware, such that a subsequent NMI
417 * must observe it.
5a50f529 418 */
85dc6002 419 set_bit(IBS_STARTED, pcpu->state);
5a50f529 420 clear_bit(IBS_STOPPING, pcpu->state);
c9574fe0 421 perf_ibs_enable_event(perf_ibs, hwc, period >> 4);
db98c5fa
RR
422
423 perf_event_update_userpage(event);
4db2e8e6
RR
424}
425
426static void perf_ibs_stop(struct perf_event *event, int flags)
427{
428 struct hw_perf_event *hwc = &event->hw;
429 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
430 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
c9574fe0 431 u64 config;
db98c5fa 432 int stopping;
4db2e8e6 433
85dc6002
PZ
434 if (test_and_set_bit(IBS_STOPPING, pcpu->state))
435 return;
436
5a50f529 437 stopping = test_bit(IBS_STARTED, pcpu->state);
4db2e8e6 438
db98c5fa
RR
439 if (!stopping && (hwc->state & PERF_HES_UPTODATE))
440 return;
4db2e8e6 441
c9574fe0 442 rdmsrl(hwc->config_base, config);
db98c5fa
RR
443
444 if (stopping) {
5a50f529 445 /*
85dc6002 446 * Set STOPPED before disabling the hardware, such that it
5a50f529
PZ
447 * must be visible to NMIs the moment we clear the EN bit,
448 * at which point we can generate an !VALID sample which
449 * we need to consume.
450 */
85dc6002 451 set_bit(IBS_STOPPED, pcpu->state);
c9574fe0 452 perf_ibs_disable_event(perf_ibs, hwc, config);
5a50f529
PZ
453 /*
454 * Clear STARTED after disabling the hardware; if it were
455 * cleared before an NMI hitting after the clear but before
456 * clearing the EN bit might think it a spurious NMI and not
457 * handle it.
458 *
459 * Clearing it after, however, creates the problem of the NMI
460 * handler seeing STARTED but not having a valid sample.
461 */
462 clear_bit(IBS_STARTED, pcpu->state);
db98c5fa
RR
463 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
464 hwc->state |= PERF_HES_STOPPED;
465 }
466
467 if (hwc->state & PERF_HES_UPTODATE)
468 return;
469
8b1e1363
RR
470 /*
471 * Clear valid bit to not count rollovers on update, rollovers
472 * are only updated in the irq handler.
473 */
474 config &= ~perf_ibs->valid_mask;
475
c9574fe0 476 perf_ibs_event_update(perf_ibs, event, &config);
db98c5fa 477 hwc->state |= PERF_HES_UPTODATE;
4db2e8e6
RR
478}
479
b7169166
RR
480static int perf_ibs_add(struct perf_event *event, int flags)
481{
4db2e8e6
RR
482 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
483 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
484
485 if (test_and_set_bit(IBS_ENABLED, pcpu->state))
486 return -ENOSPC;
487
db98c5fa
RR
488 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
489
4db2e8e6
RR
490 pcpu->event = event;
491
492 if (flags & PERF_EF_START)
493 perf_ibs_start(event, PERF_EF_RELOAD);
494
b7169166
RR
495 return 0;
496}
497
498static void perf_ibs_del(struct perf_event *event, int flags)
499{
4db2e8e6
RR
500 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
501 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
502
503 if (!test_and_clear_bit(IBS_ENABLED, pcpu->state))
504 return;
505
db98c5fa 506 perf_ibs_stop(event, PERF_EF_UPDATE);
4db2e8e6
RR
507
508 pcpu->event = NULL;
db98c5fa
RR
509
510 perf_event_update_userpage(event);
b7169166
RR
511}
512
4db2e8e6
RR
513static void perf_ibs_read(struct perf_event *event) { }
514
2e132b12
RR
515PMU_FORMAT_ATTR(rand_en, "config:57");
516PMU_FORMAT_ATTR(cnt_ctl, "config:19");
517
518static struct attribute *ibs_fetch_format_attrs[] = {
519 &format_attr_rand_en.attr,
520 NULL,
521};
522
523static struct attribute *ibs_op_format_attrs[] = {
524 NULL, /* &format_attr_cnt_ctl.attr if IBS_CAPS_OPCNT */
525 NULL,
526};
527
51041943
RR
528static struct perf_ibs perf_ibs_fetch = {
529 .pmu = {
530 .task_ctx_nr = perf_invalid_context,
531
532 .event_init = perf_ibs_init,
533 .add = perf_ibs_add,
534 .del = perf_ibs_del,
4db2e8e6
RR
535 .start = perf_ibs_start,
536 .stop = perf_ibs_stop,
537 .read = perf_ibs_read,
51041943
RR
538 },
539 .msr = MSR_AMD64_IBSFETCHCTL,
540 .config_mask = IBS_FETCH_CONFIG_MASK,
541 .cnt_mask = IBS_FETCH_MAX_CNT,
542 .enable_mask = IBS_FETCH_ENABLE,
b7074f1f 543 .valid_mask = IBS_FETCH_VAL,
db98c5fa 544 .max_period = IBS_FETCH_MAX_CNT << 4,
b7074f1f
RR
545 .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK },
546 .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT,
2e132b12 547 .format_attrs = ibs_fetch_format_attrs,
db98c5fa
RR
548
549 .get_count = get_ibs_fetch_count,
51041943
RR
550};
551
552static struct perf_ibs perf_ibs_op = {
553 .pmu = {
554 .task_ctx_nr = perf_invalid_context,
555
556 .event_init = perf_ibs_init,
557 .add = perf_ibs_add,
558 .del = perf_ibs_del,
4db2e8e6
RR
559 .start = perf_ibs_start,
560 .stop = perf_ibs_stop,
561 .read = perf_ibs_read,
51041943
RR
562 },
563 .msr = MSR_AMD64_IBSOPCTL,
564 .config_mask = IBS_OP_CONFIG_MASK,
565 .cnt_mask = IBS_OP_MAX_CNT,
566 .enable_mask = IBS_OP_ENABLE,
b7074f1f 567 .valid_mask = IBS_OP_VAL,
db98c5fa 568 .max_period = IBS_OP_MAX_CNT << 4,
b7074f1f
RR
569 .offset_mask = { MSR_AMD64_IBSOP_REG_MASK },
570 .offset_max = MSR_AMD64_IBSOP_REG_COUNT,
2e132b12 571 .format_attrs = ibs_op_format_attrs,
db98c5fa
RR
572
573 .get_count = get_ibs_op_count,
b7169166
RR
574};
575
b7074f1f
RR
576static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
577{
4db2e8e6
RR
578 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
579 struct perf_event *event = pcpu->event;
b7074f1f
RR
580 struct hw_perf_event *hwc = &event->hw;
581 struct perf_sample_data data;
582 struct perf_raw_record raw;
583 struct pt_regs regs;
584 struct perf_ibs_data ibs_data;
d47e8238 585 int offset, size, check_rip, offset_max, throttle = 0;
b7074f1f 586 unsigned int msr;
c9574fe0 587 u64 *buf, *config, period;
b7074f1f 588
4db2e8e6 589 if (!test_bit(IBS_STARTED, pcpu->state)) {
5a50f529 590fail:
fc5fb2b5
RR
591 /*
592 * Catch spurious interrupts after stopping IBS: After
d82603c6 593 * disabling IBS there could be still incoming NMIs
fc5fb2b5
RR
594 * with samples that even have the valid bit cleared.
595 * Mark all this NMIs as handled.
596 */
85dc6002 597 if (test_and_clear_bit(IBS_STOPPED, pcpu->state))
5a50f529
PZ
598 return 1;
599
600 return 0;
4db2e8e6
RR
601 }
602
b7074f1f
RR
603 msr = hwc->config_base;
604 buf = ibs_data.regs;
605 rdmsrl(msr, *buf);
606 if (!(*buf++ & perf_ibs->valid_mask))
5a50f529 607 goto fail;
b7074f1f 608
c9574fe0 609 config = &ibs_data.regs[0];
c75841a3 610 perf_ibs_event_update(perf_ibs, event, config);
fd0d000b 611 perf_sample_data_init(&data, 0, hwc->last_period);
c9574fe0 612 if (!perf_ibs_set_period(perf_ibs, hwc, &period))
d47e8238
RR
613 goto out; /* no sw counter overflow */
614
615 ibs_data.caps = ibs_caps;
616 size = 1;
617 offset = 1;
618 check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));
619 if (event->attr.sample_type & PERF_SAMPLE_RAW)
620 offset_max = perf_ibs->offset_max;
621 else if (check_rip)
622 offset_max = 2;
623 else
624 offset_max = 1;
625 do {
626 rdmsrl(msr + offset, *buf++);
627 size++;
628 offset = find_next_bit(perf_ibs->offset_mask,
629 perf_ibs->offset_max,
630 offset + 1);
631 } while (offset < offset_max);
904cb367
AG
632 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
633 /*
634 * Read IbsBrTarget and IbsOpData4 separately
635 * depending on their availability.
636 * Can't add to offset_max as they are staggered
637 */
638 if (ibs_caps & IBS_CAPS_BRNTRGT) {
639 rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++);
640 size++;
641 }
642 if (ibs_caps & IBS_CAPS_OPDATA4) {
643 rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++);
644 size++;
645 }
646 }
d47e8238
RR
647 ibs_data.size = sizeof(u64) * size;
648
649 regs = *iregs;
450bbd49
RR
650 if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) {
651 regs.flags &= ~PERF_EFLAGS_EXACT;
652 } else {
d07bdfd3 653 set_linear_ip(&regs, ibs_data.regs[1]);
450bbd49
RR
654 regs.flags |= PERF_EFLAGS_EXACT;
655 }
c75841a3 656
b7074f1f 657 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
7e3f977e
DB
658 raw = (struct perf_raw_record){
659 .frag = {
660 .size = sizeof(u32) + ibs_data.size,
661 .data = ibs_data.data,
662 },
663 };
b7074f1f
RR
664 data.raw = &raw;
665 }
666
d47e8238
RR
667 throttle = perf_event_overflow(event, &data, &regs);
668out:
c9574fe0 669 if (throttle)
0158b83f 670 perf_ibs_stop(event, 0);
c9574fe0
RR
671 else
672 perf_ibs_enable_event(perf_ibs, hwc, period >> 4);
db98c5fa
RR
673
674 perf_event_update_userpage(event);
b7074f1f
RR
675
676 return 1;
677}
678
9326638c 679static int
b7074f1f
RR
680perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
681{
c2872d38 682 u64 stamp = sched_clock();
b7074f1f
RR
683 int handled = 0;
684
685 handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs);
686 handled += perf_ibs_handle_irq(&perf_ibs_op, regs);
687
688 if (handled)
689 inc_irq_stat(apic_perf_irqs);
690
c2872d38
PZ
691 perf_sample_event_took(sched_clock() - stamp);
692
b7074f1f
RR
693 return handled;
694}
9326638c 695NOKPROBE_SYMBOL(perf_ibs_nmi_handler);
b7074f1f 696
4db2e8e6
RR
697static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
698{
699 struct cpu_perf_ibs __percpu *pcpu;
700 int ret;
701
702 pcpu = alloc_percpu(struct cpu_perf_ibs);
703 if (!pcpu)
704 return -ENOMEM;
705
706 perf_ibs->pcpu = pcpu;
707
2e132b12
RR
708 /* register attributes */
709 if (perf_ibs->format_attrs[0]) {
710 memset(&perf_ibs->format_group, 0, sizeof(perf_ibs->format_group));
711 perf_ibs->format_group.name = "format";
712 perf_ibs->format_group.attrs = perf_ibs->format_attrs;
713
714 memset(&perf_ibs->attr_groups, 0, sizeof(perf_ibs->attr_groups));
715 perf_ibs->attr_groups[0] = &perf_ibs->format_group;
716 perf_ibs->pmu.attr_groups = perf_ibs->attr_groups;
717 }
718
4db2e8e6
RR
719 ret = perf_pmu_register(&perf_ibs->pmu, name, -1);
720 if (ret) {
721 perf_ibs->pcpu = NULL;
722 free_percpu(pcpu);
723 }
724
725 return ret;
726}
727
b7169166
RR
728static __init int perf_event_ibs_init(void)
729{
2e132b12
RR
730 struct attribute **attr = ibs_op_format_attrs;
731
b7169166
RR
732 if (!ibs_caps)
733 return -ENODEV; /* ibs not supported by the cpu */
734
4db2e8e6 735 perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch");
2e132b12
RR
736
737 if (ibs_caps & IBS_CAPS_OPCNT) {
7bf35238 738 perf_ibs_op.config_mask |= IBS_OP_CNT_CTL;
2e132b12
RR
739 *attr++ = &format_attr_cnt_ctl.attr;
740 }
4db2e8e6 741 perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
2e132b12 742
fab06992 743 register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs");
1b74dde7 744 pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps);
b7169166
RR
745
746 return 0;
747}
748
749#else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */
750
751static __init int perf_event_ibs_init(void) { return 0; }
752
753#endif
754
755/* IBS - apic initialization, for perf and oprofile */
756
757static __init u32 __get_ibs_caps(void)
758{
759 u32 caps;
760 unsigned int max_level;
761
762 if (!boot_cpu_has(X86_FEATURE_IBS))
763 return 0;
764
765 /* check IBS cpuid feature flags */
766 max_level = cpuid_eax(0x80000000);
767 if (max_level < IBS_CPUID_FEATURES)
768 return IBS_CAPS_DEFAULT;
769
770 caps = cpuid_eax(IBS_CPUID_FEATURES);
771 if (!(caps & IBS_CAPS_AVAIL))
772 /* cpuid flags not valid */
773 return IBS_CAPS_DEFAULT;
774
775 return caps;
776}
777
778u32 get_ibs_caps(void)
779{
780 return ibs_caps;
781}
782
783EXPORT_SYMBOL(get_ibs_caps);
784
785static inline int get_eilvt(int offset)
786{
787 return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1);
788}
789
790static inline int put_eilvt(int offset)
791{
792 return !setup_APIC_eilvt(offset, 0, 0, 1);
793}
794
795/*
796 * Check and reserve APIC extended interrupt LVT offset for IBS if available.
797 */
798static inline int ibs_eilvt_valid(void)
799{
800 int offset;
801 u64 val;
802 int valid = 0;
803
804 preempt_disable();
805
806 rdmsrl(MSR_AMD64_IBSCTL, val);
807 offset = val & IBSCTL_LVT_OFFSET_MASK;
808
809 if (!(val & IBSCTL_LVT_OFFSET_VALID)) {
810 pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n",
811 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
812 goto out;
813 }
814
815 if (!get_eilvt(offset)) {
816 pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n",
817 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
818 goto out;
819 }
820
821 valid = 1;
822out:
823 preempt_enable();
824
825 return valid;
826}
827
828static int setup_ibs_ctl(int ibs_eilvt_off)
829{
830 struct pci_dev *cpu_cfg;
831 int nodes;
832 u32 value = 0;
833
834 nodes = 0;
835 cpu_cfg = NULL;
836 do {
837 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
838 PCI_DEVICE_ID_AMD_10H_NB_MISC,
839 cpu_cfg);
840 if (!cpu_cfg)
841 break;
842 ++nodes;
843 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
844 | IBSCTL_LVT_OFFSET_VALID);
845 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
846 if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) {
847 pci_dev_put(cpu_cfg);
1b74dde7
CY
848 pr_debug("Failed to setup IBS LVT offset, IBSCTL = 0x%08x\n",
849 value);
b7169166
RR
850 return -EINVAL;
851 }
852 } while (1);
853
854 if (!nodes) {
1b74dde7 855 pr_debug("No CPU node configured for IBS\n");
b7169166
RR
856 return -ENODEV;
857 }
858
859 return 0;
860}
861
862/*
863 * This runs only on the current cpu. We try to find an LVT offset and
864 * setup the local APIC. For this we must disable preemption. On
865 * success we initialize all nodes with this offset. This updates then
866 * the offset in the IBS_CTL per-node msr. The per-core APIC setup of
867 * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that
868 * is using the new offset.
869 */
c796b205 870static void force_ibs_eilvt_setup(void)
b7169166
RR
871{
872 int offset;
873 int ret;
874
875 preempt_disable();
876 /* find the next free available EILVT entry, skip offset 0 */
877 for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) {
878 if (get_eilvt(offset))
879 break;
880 }
881 preempt_enable();
882
883 if (offset == APIC_EILVT_NR_MAX) {
1b74dde7 884 pr_debug("No EILVT entry available\n");
c796b205 885 return;
b7169166
RR
886 }
887
888 ret = setup_ibs_ctl(offset);
889 if (ret)
890 goto out;
891
c796b205 892 if (!ibs_eilvt_valid())
b7169166 893 goto out;
b7169166 894
16e5294e 895 pr_info("IBS: LVT offset %d assigned\n", offset);
b7169166 896
c796b205 897 return;
b7169166
RR
898out:
899 preempt_disable();
900 put_eilvt(offset);
901 preempt_enable();
c796b205 902 return;
b7169166
RR
903}
904
bee09ed9
RR
905static void ibs_eilvt_setup(void)
906{
907 /*
908 * Force LVT offset assignment for family 10h: The offsets are
909 * not assigned by the BIOS for this family, so the OS is
910 * responsible for doing it. If the OS assignment fails, fall
911 * back to BIOS settings and try to setup this.
912 */
913 if (boot_cpu_data.x86 == 0x10)
914 force_ibs_eilvt_setup();
915}
916
b7169166
RR
917static inline int get_ibs_lvt_offset(void)
918{
919 u64 val;
920
921 rdmsrl(MSR_AMD64_IBSCTL, val);
922 if (!(val & IBSCTL_LVT_OFFSET_VALID))
923 return -EINVAL;
924
925 return val & IBSCTL_LVT_OFFSET_MASK;
926}
927
928static void setup_APIC_ibs(void *dummy)
929{
930 int offset;
931
932 offset = get_ibs_lvt_offset();
933 if (offset < 0)
934 goto failed;
935
936 if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0))
937 return;
938failed:
939 pr_warn("perf: IBS APIC setup failed on cpu #%d\n",
940 smp_processor_id());
941}
942
943static void clear_APIC_ibs(void *dummy)
944{
945 int offset;
946
947 offset = get_ibs_lvt_offset();
948 if (offset >= 0)
949 setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1);
950}
951
bee09ed9
RR
952#ifdef CONFIG_PM
953
954static int perf_ibs_suspend(void)
955{
956 clear_APIC_ibs(NULL);
957 return 0;
958}
959
960static void perf_ibs_resume(void)
961{
962 ibs_eilvt_setup();
963 setup_APIC_ibs(NULL);
964}
965
966static struct syscore_ops perf_ibs_syscore_ops = {
967 .resume = perf_ibs_resume,
968 .suspend = perf_ibs_suspend,
969};
970
971static void perf_ibs_pm_init(void)
972{
973 register_syscore_ops(&perf_ibs_syscore_ops);
974}
975
976#else
977
978static inline void perf_ibs_pm_init(void) { }
979
980#endif
981
148f9bb8 982static int
b7169166
RR
983perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
984{
985 switch (action & ~CPU_TASKS_FROZEN) {
986 case CPU_STARTING:
987 setup_APIC_ibs(NULL);
988 break;
989 case CPU_DYING:
990 clear_APIC_ibs(NULL);
991 break;
992 default:
993 break;
994 }
995
996 return NOTIFY_OK;
997}
998
999static __init int amd_ibs_init(void)
1000{
1001 u32 caps;
16e5294e 1002 int ret = -EINVAL;
b7169166
RR
1003
1004 caps = __get_ibs_caps();
1005 if (!caps)
1006 return -ENODEV; /* ibs not supported by the cpu */
1007
bee09ed9 1008 ibs_eilvt_setup();
16e5294e
RR
1009
1010 if (!ibs_eilvt_valid())
1011 goto out;
b7169166 1012
bee09ed9 1013 perf_ibs_pm_init();
047868ce 1014 cpu_notifier_register_begin();
b7169166
RR
1015 ibs_caps = caps;
1016 /* make ibs_caps visible to other cpus: */
1017 smp_mb();
b7169166 1018 smp_call_function(setup_APIC_ibs, NULL, 1);
047868ce
SB
1019 __perf_cpu_notifier(perf_ibs_cpu_notifier);
1020 cpu_notifier_register_done();
b7169166 1021
16e5294e
RR
1022 ret = perf_event_ibs_init();
1023out:
1024 if (ret)
1025 pr_err("Failed to setup IBS, %d\n", ret);
1026 return ret;
b7169166
RR
1027}
1028
1029/* Since we need the pci subsystem to init ibs we can't do this earlier: */
1030device_initcall(amd_ibs_init);
This page took 0.238607 seconds and 5 git commands to generate.