projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
perf/x86/amd: Enable northbridge performance counters on AMD family 15h
[deliverable/linux.git] / arch / x86 / kernel / cpu / perf_event_amd.c
1 #include <linux/perf_event.h>
2 #include <linux/export.h>
3 #include <linux/types.h>
4 #include <linux/init.h>
5 #include <linux/slab.h>
6 #include <asm/apicdef.h>
7
8 #include "perf_event.h"
9
10 static __initconst const u64 amd_hw_cache_event_ids
11 [PERF_COUNT_HW_CACHE_MAX]
12 [PERF_COUNT_HW_CACHE_OP_MAX]
13 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
14 {
15 [ C(L1D) ] = {
16 [ C(OP_READ) ] = {
17 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
18 [ C(RESULT_MISS) ] = 0x0141, /* Data Cache Misses */
19 },
20 [ C(OP_WRITE) ] = {
21 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
22 [ C(RESULT_MISS) ] = 0,
23 },
24 [ C(OP_PREFETCH) ] = {
25 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts */
26 [ C(RESULT_MISS) ] = 0x0167, /* Data Prefetcher :cancelled */
27 },
28 },
29 [ C(L1I ) ] = {
30 [ C(OP_READ) ] = {
31 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */
32 [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */
33 },
34 [ C(OP_WRITE) ] = {
35 [ C(RESULT_ACCESS) ] = -1,
36 [ C(RESULT_MISS) ] = -1,
37 },
38 [ C(OP_PREFETCH) ] = {
39 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
40 [ C(RESULT_MISS) ] = 0,
41 },
42 },
43 [ C(LL ) ] = {
44 [ C(OP_READ) ] = {
45 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
46 [ C(RESULT_MISS) ] = 0x037E, /* L2 Cache Misses : IC+DC */
47 },
48 [ C(OP_WRITE) ] = {
49 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback */
50 [ C(RESULT_MISS) ] = 0,
51 },
52 [ C(OP_PREFETCH) ] = {
53 [ C(RESULT_ACCESS) ] = 0,
54 [ C(RESULT_MISS) ] = 0,
55 },
56 },
57 [ C(DTLB) ] = {
58 [ C(OP_READ) ] = {
59 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
60 [ C(RESULT_MISS) ] = 0x0746, /* L1_DTLB_AND_L2_DLTB_MISS.ALL */
61 },
62 [ C(OP_WRITE) ] = {
63 [ C(RESULT_ACCESS) ] = 0,
64 [ C(RESULT_MISS) ] = 0,
65 },
66 [ C(OP_PREFETCH) ] = {
67 [ C(RESULT_ACCESS) ] = 0,
68 [ C(RESULT_MISS) ] = 0,
69 },
70 },
71 [ C(ITLB) ] = {
72 [ C(OP_READ) ] = {
73 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */
74 [ C(RESULT_MISS) ] = 0x0385, /* L1_ITLB_AND_L2_ITLB_MISS.ALL */
75 },
76 [ C(OP_WRITE) ] = {
77 [ C(RESULT_ACCESS) ] = -1,
78 [ C(RESULT_MISS) ] = -1,
79 },
80 [ C(OP_PREFETCH) ] = {
81 [ C(RESULT_ACCESS) ] = -1,
82 [ C(RESULT_MISS) ] = -1,
83 },
84 },
85 [ C(BPU ) ] = {
86 [ C(OP_READ) ] = {
87 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */
88 [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */
89 },
90 [ C(OP_WRITE) ] = {
91 [ C(RESULT_ACCESS) ] = -1,
92 [ C(RESULT_MISS) ] = -1,
93 },
94 [ C(OP_PREFETCH) ] = {
95 [ C(RESULT_ACCESS) ] = -1,
96 [ C(RESULT_MISS) ] = -1,
97 },
98 },
99 [ C(NODE) ] = {
100 [ C(OP_READ) ] = {
101 [ C(RESULT_ACCESS) ] = 0xb8e9, /* CPU Request to Memory, l+r */
102 [ C(RESULT_MISS) ] = 0x98e9, /* CPU Request to Memory, r */
103 },
104 [ C(OP_WRITE) ] = {
105 [ C(RESULT_ACCESS) ] = -1,
106 [ C(RESULT_MISS) ] = -1,
107 },
108 [ C(OP_PREFETCH) ] = {
109 [ C(RESULT_ACCESS) ] = -1,
110 [ C(RESULT_MISS) ] = -1,
111 },
112 },
113 };
114
115 /*
116 * AMD Performance Monitor K7 and later.
117 */
118 static const u64 amd_perfmon_event_map[] =
119 {
120 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076,
121 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
122 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080,
123 [PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
124 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2,
125 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3,
126 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */
127 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x00d1, /* "Dispatch stalls" event */
128 };
129
130 static u64 amd_pmu_event_map(int hw_event)
131 {
132 return amd_perfmon_event_map[hw_event];
133 }
134
135 static struct event_constraint *amd_nb_event_constraint;
136
137 /*
138 * Previously calculated offsets
139 */
140 static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly;
141 static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly;
142 static unsigned int rdpmc_indexes[X86_PMC_IDX_MAX] __read_mostly;
143
144 /*
145 * Legacy CPUs:
146 * 4 counters starting at 0xc0010000 each offset by 1
147 *
148 * CPUs with core performance counter extensions:
149 * 6 counters starting at 0xc0010200 each offset by 2
150 *
151 * CPUs with north bridge performance counter extensions:
152 * 4 additional counters starting at 0xc0010240 each offset by 2
153 * (indexed right above either one of the above core counters)
154 */
155 static inline int amd_pmu_addr_offset(int index, bool eventsel)
156 {
157 int offset, first, base;
158
159 if (!index)
160 return index;
161
162 if (eventsel)
163 offset = event_offsets[index];
164 else
165 offset = count_offsets[index];
166
167 if (offset)
168 return offset;
169
170 if (amd_nb_event_constraint &&
171 test_bit(index, amd_nb_event_constraint->idxmsk)) {
172 /*
173 * calculate the offset of NB counters with respect to
174 * base eventsel or perfctr
175 */
176
177 first = find_first_bit(amd_nb_event_constraint->idxmsk,
178 X86_PMC_IDX_MAX);
179
180 if (eventsel)
181 base = MSR_F15H_NB_PERF_CTL - x86_pmu.eventsel;
182 else
183 base = MSR_F15H_NB_PERF_CTR - x86_pmu.perfctr;
184
185 offset = base + ((index - first) << 1);
186 } else if (!cpu_has_perfctr_core)
187 offset = index;
188 else
189 offset = index << 1;
190
191 if (eventsel)
192 event_offsets[index] = offset;
193 else
194 count_offsets[index] = offset;
195
196 return offset;
197 }
198
199 static inline int amd_pmu_rdpmc_index(int index)
200 {
201 int ret, first;
202
203 if (!index)
204 return index;
205
206 ret = rdpmc_indexes[index];
207
208 if (ret)
209 return ret;
210
211 if (amd_nb_event_constraint &&
212 test_bit(index, amd_nb_event_constraint->idxmsk)) {
213 /*
214 * according to the mnual, ECX value of the NB counters is
215 * the index of the NB counter (0, 1, 2 or 3) plus 6
216 */
217
218 first = find_first_bit(amd_nb_event_constraint->idxmsk,
219 X86_PMC_IDX_MAX);
220 ret = index - first + 6;
221 } else
222 ret = index;
223
224 rdpmc_indexes[index] = ret;
225
226 return ret;
227 }
228
229 static int amd_core_hw_config(struct perf_event *event)
230 {
231 if (event->attr.exclude_host && event->attr.exclude_guest)
232 /*
233 * When HO == GO == 1 the hardware treats that as GO == HO == 0
234 * and will count in both modes. We don't want to count in that
235 * case so we emulate no-counting by setting US = OS = 0.
236 */
237 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
238 ARCH_PERFMON_EVENTSEL_OS);
239 else if (event->attr.exclude_host)
240 event->hw.config |= AMD64_EVENTSEL_GUESTONLY;
241 else if (event->attr.exclude_guest)
242 event->hw.config |= AMD64_EVENTSEL_HOSTONLY;
243
244 return 0;
245 }
246
247 /*
248 * NB counters do not support the following event select bits:
249 * Host/Guest only
250 * Counter mask
251 * Invert counter mask
252 * Edge detect
253 * OS/User mode
254 */
255 static int amd_nb_hw_config(struct perf_event *event)
256 {
257 /* for NB, we only allow system wide counting mode */
258 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
259 return -EINVAL;
260
261 if (event->attr.exclude_user || event->attr.exclude_kernel ||
262 event->attr.exclude_host || event->attr.exclude_guest)
263 return -EINVAL;
264
265 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
266 ARCH_PERFMON_EVENTSEL_OS);
267
268 if (event->hw.config & ~(AMD64_RAW_EVENT_MASK_NB |
269 ARCH_PERFMON_EVENTSEL_INT))
270 return -EINVAL;
271
272 return 0;
273 }
274
275 /*
276 * AMD64 events are detected based on their event codes.
277 */
278 static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc)
279 {
280 return ((hwc->config >> 24) & 0x0f00) | (hwc->config & 0x00ff);
281 }
282
283 static inline int amd_is_nb_event(struct hw_perf_event *hwc)
284 {
285 return (hwc->config & 0xe0) == 0xe0;
286 }
287
288 static inline int amd_is_perfctr_nb_event(struct hw_perf_event *hwc)
289 {
290 return amd_nb_event_constraint && amd_is_nb_event(hwc);
291 }
292
293 static inline int amd_has_nb(struct cpu_hw_events *cpuc)
294 {
295 struct amd_nb *nb = cpuc->amd_nb;
296
297 return nb && nb->nb_id != -1;
298 }
299
300 static int amd_pmu_hw_config(struct perf_event *event)
301 {
302 int ret;
303
304 /* pass precise event sampling to ibs: */
305 if (event->attr.precise_ip && get_ibs_caps())
306 return -ENOENT;
307
308 if (has_branch_stack(event))
309 return -EOPNOTSUPP;
310
311 ret = x86_pmu_hw_config(event);
312 if (ret)
313 return ret;
314
315 if (event->attr.type == PERF_TYPE_RAW)
316 event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
317
318 if (amd_is_perfctr_nb_event(&event->hw))
319 return amd_nb_hw_config(event);
320
321 return amd_core_hw_config(event);
322 }
323
324 static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,
325 struct perf_event *event)
326 {
327 struct amd_nb *nb = cpuc->amd_nb;
328 int i;
329
330 /*
331 * need to scan whole list because event may not have
332 * been assigned during scheduling
333 *
334 * no race condition possible because event can only
335 * be removed on one CPU at a time AND PMU is disabled
336 * when we come here
337 */
338 for (i = 0; i < x86_pmu.num_counters; i++) {
339 if (cmpxchg(nb->owners + i, event, NULL) == event)
340 break;
341 }
342 }
343
344 static void amd_nb_interrupt_hw_config(struct hw_perf_event *hwc)
345 {
346 int core_id = cpu_data(smp_processor_id()).cpu_core_id;
347
348 /* deliver interrupts only to this core */
349 if (hwc->config & ARCH_PERFMON_EVENTSEL_INT) {
350 hwc->config |= AMD64_EVENTSEL_INT_CORE_ENABLE;
351 hwc->config &= ~AMD64_EVENTSEL_INT_CORE_SEL_MASK;
352 hwc->config |= (u64)(core_id) <<
353 AMD64_EVENTSEL_INT_CORE_SEL_SHIFT;
354 }
355 }
356
357 /*
358 * AMD64 NorthBridge events need special treatment because
359 * counter access needs to be synchronized across all cores
360 * of a package. Refer to BKDG section 3.12
361 *
362 * NB events are events measuring L3 cache, Hypertransport
363 * traffic. They are identified by an event code >= 0xe00.
364 * They measure events on the NorthBride which is shared
365 * by all cores on a package. NB events are counted on a
366 * shared set of counters. When a NB event is programmed
367 * in a counter, the data actually comes from a shared
368 * counter. Thus, access to those counters needs to be
369 * synchronized.
370 *
371 * We implement the synchronization such that no two cores
372 * can be measuring NB events using the same counters. Thus,
373 * we maintain a per-NB allocation table. The available slot
374 * is propagated using the event_constraint structure.
375 *
376 * We provide only one choice for each NB event based on
377 * the fact that only NB events have restrictions. Consequently,
378 * if a counter is available, there is a guarantee the NB event
379 * will be assigned to it. If no slot is available, an empty
380 * constraint is returned and scheduling will eventually fail
381 * for this event.
382 *
383 * Note that all cores attached the same NB compete for the same
384 * counters to host NB events, this is why we use atomic ops. Some
385 * multi-chip CPUs may have more than one NB.
386 *
387 * Given that resources are allocated (cmpxchg), they must be
388 * eventually freed for others to use. This is accomplished by
389 * calling __amd_put_nb_event_constraints()
390 *
391 * Non NB events are not impacted by this restriction.
392 */
393 static struct event_constraint *
394 __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
395 struct event_constraint *c)
396 {
397 struct hw_perf_event *hwc = &event->hw;
398 struct amd_nb *nb = cpuc->amd_nb;
399 struct perf_event *old;
400 int idx, new = -1;
401
402 if (!c)
403 c = &unconstrained;
404
405 if (cpuc->is_fake)
406 return c;
407
408 /*
409 * detect if already present, if so reuse
410 *
411 * cannot merge with actual allocation
412 * because of possible holes
413 *
414 * event can already be present yet not assigned (in hwc->idx)
415 * because of successive calls to x86_schedule_events() from
416 * hw_perf_group_sched_in() without hw_perf_enable()
417 */
418 for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) {
419 if (new == -1 || hwc->idx == idx)
420 /* assign free slot, prefer hwc->idx */
421 old = cmpxchg(nb->owners + idx, NULL, event);
422 else if (nb->owners[idx] == event)
423 /* event already present */
424 old = event;
425 else
426 continue;
427
428 if (old && old != event)
429 continue;
430
431 /* reassign to this slot */
432 if (new != -1)
433 cmpxchg(nb->owners + new, event, NULL);
434 new = idx;
435
436 /* already present, reuse */
437 if (old == event)
438 break;
439 }
440
441 if (new == -1)
442 return &emptyconstraint;
443
444 if (amd_is_perfctr_nb_event(hwc))
445 amd_nb_interrupt_hw_config(hwc);
446
447 return &nb->event_constraints[new];
448 }
449
450 static struct amd_nb *amd_alloc_nb(int cpu)
451 {
452 struct amd_nb *nb;
453 int i;
454
455 nb = kmalloc_node(sizeof(struct amd_nb), GFP_KERNEL | __GFP_ZERO,
456 cpu_to_node(cpu));
457 if (!nb)
458 return NULL;
459
460 nb->nb_id = -1;
461
462 /*
463 * initialize all possible NB constraints
464 */
465 for (i = 0; i < x86_pmu.num_counters; i++) {
466 __set_bit(i, nb->event_constraints[i].idxmsk);
467 nb->event_constraints[i].weight = 1;
468 }
469 return nb;
470 }
471
472 static int amd_pmu_cpu_prepare(int cpu)
473 {
474 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
475
476 WARN_ON_ONCE(cpuc->amd_nb);
477
478 if (boot_cpu_data.x86_max_cores < 2)
479 return NOTIFY_OK;
480
481 cpuc->amd_nb = amd_alloc_nb(cpu);
482 if (!cpuc->amd_nb)
483 return NOTIFY_BAD;
484
485 return NOTIFY_OK;
486 }
487
488 static void amd_pmu_cpu_starting(int cpu)
489 {
490 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
491 struct amd_nb *nb;
492 int i, nb_id;
493
494 cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
495
496 if (boot_cpu_data.x86_max_cores < 2)
497 return;
498
499 nb_id = amd_get_nb_id(cpu);
500 WARN_ON_ONCE(nb_id == BAD_APICID);
501
502 for_each_online_cpu(i) {
503 nb = per_cpu(cpu_hw_events, i).amd_nb;
504 if (WARN_ON_ONCE(!nb))
505 continue;
506
507 if (nb->nb_id == nb_id) {
508 cpuc->kfree_on_online = cpuc->amd_nb;
509 cpuc->amd_nb = nb;
510 break;
511 }
512 }
513
514 cpuc->amd_nb->nb_id = nb_id;
515 cpuc->amd_nb->refcnt++;
516 }
517
518 static void amd_pmu_cpu_dead(int cpu)
519 {
520 struct cpu_hw_events *cpuhw;
521
522 if (boot_cpu_data.x86_max_cores < 2)
523 return;
524
525 cpuhw = &per_cpu(cpu_hw_events, cpu);
526
527 if (cpuhw->amd_nb) {
528 struct amd_nb *nb = cpuhw->amd_nb;
529
530 if (nb->nb_id == -1 || --nb->refcnt == 0)
531 kfree(nb);
532
533 cpuhw->amd_nb = NULL;
534 }
535 }
536
537 static struct event_constraint *
538 amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
539 {
540 /*
541 * if not NB event or no NB, then no constraints
542 */
543 if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))
544 return &unconstrained;
545
546 return __amd_get_nb_event_constraints(cpuc, event,
547 amd_nb_event_constraint);
548 }
549
550 static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
551 struct perf_event *event)
552 {
553 if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))
554 __amd_put_nb_event_constraints(cpuc, event);
555 }
556
557 PMU_FORMAT_ATTR(event, "config:0-7,32-35");
558 PMU_FORMAT_ATTR(umask, "config:8-15" );
559 PMU_FORMAT_ATTR(edge, "config:18" );
560 PMU_FORMAT_ATTR(inv, "config:23" );
561 PMU_FORMAT_ATTR(cmask, "config:24-31" );
562
563 static struct attribute *amd_format_attr[] = {
564 &format_attr_event.attr,
565 &format_attr_umask.attr,
566 &format_attr_edge.attr,
567 &format_attr_inv.attr,
568 &format_attr_cmask.attr,
569 NULL,
570 };
571
572 /* AMD Family 15h */
573
574 #define AMD_EVENT_TYPE_MASK 0x000000F0ULL
575
576 #define AMD_EVENT_FP 0x00000000ULL ... 0x00000010ULL
577 #define AMD_EVENT_LS 0x00000020ULL ... 0x00000030ULL
578 #define AMD_EVENT_DC 0x00000040ULL ... 0x00000050ULL
579 #define AMD_EVENT_CU 0x00000060ULL ... 0x00000070ULL
580 #define AMD_EVENT_IC_DE 0x00000080ULL ... 0x00000090ULL
581 #define AMD_EVENT_EX_LS 0x000000C0ULL
582 #define AMD_EVENT_DE 0x000000D0ULL
583 #define AMD_EVENT_NB 0x000000E0ULL ... 0x000000F0ULL
584
585 /*
586 * AMD family 15h event code/PMC mappings:
587 *
588 * type = event_code & 0x0F0:
589 *
590 * 0x000 FP PERF_CTL[5:3]
591 * 0x010 FP PERF_CTL[5:3]
592 * 0x020 LS PERF_CTL[5:0]
593 * 0x030 LS PERF_CTL[5:0]
594 * 0x040 DC PERF_CTL[5:0]
595 * 0x050 DC PERF_CTL[5:0]
596 * 0x060 CU PERF_CTL[2:0]
597 * 0x070 CU PERF_CTL[2:0]
598 * 0x080 IC/DE PERF_CTL[2:0]
599 * 0x090 IC/DE PERF_CTL[2:0]
600 * 0x0A0 ---
601 * 0x0B0 ---
602 * 0x0C0 EX/LS PERF_CTL[5:0]
603 * 0x0D0 DE PERF_CTL[2:0]
604 * 0x0E0 NB NB_PERF_CTL[3:0]
605 * 0x0F0 NB NB_PERF_CTL[3:0]
606 *
607 * Exceptions:
608 *
609 * 0x000 FP PERF_CTL[3], PERF_CTL[5:3] (*)
610 * 0x003 FP PERF_CTL[3]
611 * 0x004 FP PERF_CTL[3], PERF_CTL[5:3] (*)
612 * 0x00B FP PERF_CTL[3]
613 * 0x00D FP PERF_CTL[3]
614 * 0x023 DE PERF_CTL[2:0]
615 * 0x02D LS PERF_CTL[3]
616 * 0x02E LS PERF_CTL[3,0]
617 * 0x031 LS PERF_CTL[2:0] (**)
618 * 0x043 CU PERF_CTL[2:0]
619 * 0x045 CU PERF_CTL[2:0]
620 * 0x046 CU PERF_CTL[2:0]
621 * 0x054 CU PERF_CTL[2:0]
622 * 0x055 CU PERF_CTL[2:0]
623 * 0x08F IC PERF_CTL[0]
624 * 0x187 DE PERF_CTL[0]
625 * 0x188 DE PERF_CTL[0]
626 * 0x0DB EX PERF_CTL[5:0]
627 * 0x0DC LS PERF_CTL[5:0]
628 * 0x0DD LS PERF_CTL[5:0]
629 * 0x0DE LS PERF_CTL[5:0]
630 * 0x0DF LS PERF_CTL[5:0]
631 * 0x1C0 EX PERF_CTL[5:3]
632 * 0x1D6 EX PERF_CTL[5:0]
633 * 0x1D8 EX PERF_CTL[5:0]
634 *
635 * (*) depending on the umask all FPU counters may be used
636 * (**) only one unitmask enabled at a time
637 */
638
639 static struct event_constraint amd_f15_PMC0 = EVENT_CONSTRAINT(0, 0x01, 0);
640 static struct event_constraint amd_f15_PMC20 = EVENT_CONSTRAINT(0, 0x07, 0);
641 static struct event_constraint amd_f15_PMC3 = EVENT_CONSTRAINT(0, 0x08, 0);
642 static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
643 static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
644 static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
645
646 static struct event_constraint amd_NBPMC96 = EVENT_CONSTRAINT(0, 0x3C0, 0);
647 static struct event_constraint amd_NBPMC74 = EVENT_CONSTRAINT(0, 0xF0, 0);
648
649 static struct event_constraint *
650 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *event)
651 {
652 struct hw_perf_event *hwc = &event->hw;
653 unsigned int event_code = amd_get_event_code(hwc);
654
655 switch (event_code & AMD_EVENT_TYPE_MASK) {
656 case AMD_EVENT_FP:
657 switch (event_code) {
658 case 0x000:
659 if (!(hwc->config & 0x0000F000ULL))
660 break;
661 if (!(hwc->config & 0x00000F00ULL))
662 break;
663 return &amd_f15_PMC3;
664 case 0x004:
665 if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
666 break;
667 return &amd_f15_PMC3;
668 case 0x003:
669 case 0x00B:
670 case 0x00D:
671 return &amd_f15_PMC3;
672 }
673 return &amd_f15_PMC53;
674 case AMD_EVENT_LS:
675 case AMD_EVENT_DC:
676 case AMD_EVENT_EX_LS:
677 switch (event_code) {
678 case 0x023:
679 case 0x043:
680 case 0x045:
681 case 0x046:
682 case 0x054:
683 case 0x055:
684 return &amd_f15_PMC20;
685 case 0x02D:
686 return &amd_f15_PMC3;
687 case 0x02E:
688 return &amd_f15_PMC30;
689 case 0x031:
690 if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
691 return &amd_f15_PMC20;
692 return &emptyconstraint;
693 case 0x1C0:
694 return &amd_f15_PMC53;
695 default:
696 return &amd_f15_PMC50;
697 }
698 case AMD_EVENT_CU:
699 case AMD_EVENT_IC_DE:
700 case AMD_EVENT_DE:
701 switch (event_code) {
702 case 0x08F:
703 case 0x187:
704 case 0x188:
705 return &amd_f15_PMC0;
706 case 0x0DB ... 0x0DF:
707 case 0x1D6:
708 case 0x1D8:
709 return &amd_f15_PMC50;
710 default:
711 return &amd_f15_PMC20;
712 }
713 case AMD_EVENT_NB:
714 return __amd_get_nb_event_constraints(cpuc, event,
715 amd_nb_event_constraint);
716 default:
717 return &emptyconstraint;
718 }
719 }
720
721 static ssize_t amd_event_sysfs_show(char *page, u64 config)
722 {
723 u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT) |
724 (config & AMD64_EVENTSEL_EVENT) >> 24;
725
726 return x86_event_sysfs_show(page, config, event);
727 }
728
729 static __initconst const struct x86_pmu amd_pmu = {
730 .name = "AMD",
731 .handle_irq = x86_pmu_handle_irq,
732 .disable_all = x86_pmu_disable_all,
733 .enable_all = x86_pmu_enable_all,
734 .enable = x86_pmu_enable_event,
735 .disable = x86_pmu_disable_event,
736 .hw_config = amd_pmu_hw_config,
737 .schedule_events = x86_schedule_events,
738 .eventsel = MSR_K7_EVNTSEL0,
739 .perfctr = MSR_K7_PERFCTR0,
740 .addr_offset = amd_pmu_addr_offset,
741 .rdpmc_index = amd_pmu_rdpmc_index,
742 .event_map = amd_pmu_event_map,
743 .max_events = ARRAY_SIZE(amd_perfmon_event_map),
744 .num_counters = AMD64_NUM_COUNTERS,
745 .cntval_bits = 48,
746 .cntval_mask = (1ULL << 48) - 1,
747 .apic = 1,
748 /* use highest bit to detect overflow */
749 .max_period = (1ULL << 47) - 1,
750 .get_event_constraints = amd_get_event_constraints,
751 .put_event_constraints = amd_put_event_constraints,
752
753 .format_attrs = amd_format_attr,
754 .events_sysfs_show = amd_event_sysfs_show,
755
756 .cpu_prepare = amd_pmu_cpu_prepare,
757 .cpu_starting = amd_pmu_cpu_starting,
758 .cpu_dead = amd_pmu_cpu_dead,
759 };
760
761 static int setup_event_constraints(void)
762 {
763 if (boot_cpu_data.x86 == 0x15)
764 x86_pmu.get_event_constraints = amd_get_event_constraints_f15h;
765 return 0;
766 }
767
768 static int setup_perfctr_core(void)
769 {
770 if (!cpu_has_perfctr_core) {
771 WARN(x86_pmu.get_event_constraints == amd_get_event_constraints_f15h,
772 KERN_ERR "Odd, counter constraints enabled but no core perfctrs detected!");
773 return -ENODEV;
774 }
775
776 WARN(x86_pmu.get_event_constraints == amd_get_event_constraints,
777 KERN_ERR "hw perf events core counters need constraints handler!");
778
779 /*
780 * If core performance counter extensions exists, we must use
781 * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also
782 * x86_pmu_addr_offset().
783 */
784 x86_pmu.eventsel = MSR_F15H_PERF_CTL;
785 x86_pmu.perfctr = MSR_F15H_PERF_CTR;
786 x86_pmu.num_counters = AMD64_NUM_COUNTERS_CORE;
787
788 printk(KERN_INFO "perf: AMD core performance counters detected\n");
789
790 return 0;
791 }
792
793 static int setup_perfctr_nb(void)
794 {
795 if (!cpu_has_perfctr_nb)
796 return -ENODEV;
797
798 x86_pmu.num_counters += AMD64_NUM_COUNTERS_NB;
799
800 if (cpu_has_perfctr_core)
801 amd_nb_event_constraint = &amd_NBPMC96;
802 else
803 amd_nb_event_constraint = &amd_NBPMC74;
804
805 printk(KERN_INFO "perf: AMD northbridge performance counters detected\n");
806
807 return 0;
808 }
809
810 __init int amd_pmu_init(void)
811 {
812 /* Performance-monitoring supported from K7 and later: */
813 if (boot_cpu_data.x86 < 6)
814 return -ENODEV;
815
816 x86_pmu = amd_pmu;
817
818 setup_event_constraints();
819 setup_perfctr_core();
820 setup_perfctr_nb();
821
822 /* Events are common for all AMDs */
823 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
824 sizeof(hw_cache_event_ids));
825
826 return 0;
827 }
828
829 void amd_pmu_enable_virt(void)
830 {
831 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
832
833 cpuc->perf_ctr_virt_mask = 0;
834
835 /* Reload all events */
836 x86_pmu_disable_all();
837 x86_pmu_enable_all(0);
838 }
839 EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
840
841 void amd_pmu_disable_virt(void)
842 {
843 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
844
845 /*
846 * We only mask out the Host-only bit so that host-only counting works
847 * when SVM is disabled. If someone sets up a guest-only counter when
848 * SVM is disabled the Guest-only bits still gets set and the counter
849 * will not count anything.
850 */
851 cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
852
853 /* Reload all events */
854 x86_pmu_disable_all();
855 x86_pmu_enable_all(0);
856 }
857 EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);
Linux kernel - Deliverables
This page took 0.132201 seconds and 5 git commands to generate.