projects / deliverable / linux.git / blob
? search:


88ae8cebf3c1cbda30cc5a90480b4f5727269057
[deliverable/linux.git] / arch / x86 / kernel / cpu / perf_counter.c
1 /*
2 * Performance counter x86 architecture code
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
9 *
10 * For licencing details see kernel-base/COPYING
11 */
12
13 #include <linux/perf_counter.h>
14 #include <linux/capability.h>
15 #include <linux/notifier.h>
16 #include <linux/hardirq.h>
17 #include <linux/kprobes.h>
18 #include <linux/module.h>
19 #include <linux/kdebug.h>
20 #include <linux/sched.h>
21 #include <linux/uaccess.h>
22
23 #include <asm/apic.h>
24 #include <asm/stacktrace.h>
25 #include <asm/nmi.h>
26
27 static u64 perf_counter_mask __read_mostly;
28
29 struct cpu_hw_counters {
30 struct perf_counter *counters[X86_PMC_IDX_MAX];
31 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
32 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
33 unsigned long interrupts;
34 u64 throttle_ctrl;
35 int enabled;
36 };
37
38 /*
39 * struct x86_pmu - generic x86 pmu
40 */
41 struct x86_pmu {
42 const char *name;
43 int version;
44 int (*handle_irq)(struct pt_regs *, int);
45 u64 (*save_disable_all)(void);
46 void (*restore_all)(u64);
47 void (*enable)(struct hw_perf_counter *, int);
48 void (*disable)(struct hw_perf_counter *, int);
49 unsigned eventsel;
50 unsigned perfctr;
51 u64 (*event_map)(int);
52 u64 (*raw_event)(u64);
53 int max_events;
54 int num_counters;
55 int num_counters_fixed;
56 int counter_bits;
57 u64 counter_mask;
58 u64 max_period;
59 };
60
61 static struct x86_pmu x86_pmu __read_mostly;
62
63 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
64 .enabled = 1,
65 };
66
67 /*
68 * Intel PerfMon v3. Used on Core2 and later.
69 */
70 static const u64 intel_perfmon_event_map[] =
71 {
72 [PERF_COUNT_CPU_CYCLES] = 0x003c,
73 [PERF_COUNT_INSTRUCTIONS] = 0x00c0,
74 [PERF_COUNT_CACHE_REFERENCES] = 0x4f2e,
75 [PERF_COUNT_CACHE_MISSES] = 0x412e,
76 [PERF_COUNT_BRANCH_INSTRUCTIONS] = 0x00c4,
77 [PERF_COUNT_BRANCH_MISSES] = 0x00c5,
78 [PERF_COUNT_BUS_CYCLES] = 0x013c,
79 };
80
81 static u64 intel_pmu_event_map(int event)
82 {
83 return intel_perfmon_event_map[event];
84 }
85
86 static u64 intel_pmu_raw_event(u64 event)
87 {
88 #define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL
89 #define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL
90 #define CORE_EVNTSEL_COUNTER_MASK 0xFF000000ULL
91
92 #define CORE_EVNTSEL_MASK \
93 (CORE_EVNTSEL_EVENT_MASK | \
94 CORE_EVNTSEL_UNIT_MASK | \
95 CORE_EVNTSEL_COUNTER_MASK)
96
97 return event & CORE_EVNTSEL_MASK;
98 }
99
100 /*
101 * AMD Performance Monitor K7 and later.
102 */
103 static const u64 amd_perfmon_event_map[] =
104 {
105 [PERF_COUNT_CPU_CYCLES] = 0x0076,
106 [PERF_COUNT_INSTRUCTIONS] = 0x00c0,
107 [PERF_COUNT_CACHE_REFERENCES] = 0x0080,
108 [PERF_COUNT_CACHE_MISSES] = 0x0081,
109 [PERF_COUNT_BRANCH_INSTRUCTIONS] = 0x00c4,
110 [PERF_COUNT_BRANCH_MISSES] = 0x00c5,
111 };
112
113 static u64 amd_pmu_event_map(int event)
114 {
115 return amd_perfmon_event_map[event];
116 }
117
118 static u64 amd_pmu_raw_event(u64 event)
119 {
120 #define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL
121 #define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL
122 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
123
124 #define K7_EVNTSEL_MASK \
125 (K7_EVNTSEL_EVENT_MASK | \
126 K7_EVNTSEL_UNIT_MASK | \
127 K7_EVNTSEL_COUNTER_MASK)
128
129 return event & K7_EVNTSEL_MASK;
130 }
131
132 /*
133 * Propagate counter elapsed time into the generic counter.
134 * Can only be executed on the CPU where the counter is active.
135 * Returns the delta events processed.
136 */
137 static u64
138 x86_perf_counter_update(struct perf_counter *counter,
139 struct hw_perf_counter *hwc, int idx)
140 {
141 int shift = 64 - x86_pmu.counter_bits;
142 u64 prev_raw_count, new_raw_count;
143 s64 delta;
144
145 /*
146 * Careful: an NMI might modify the previous counter value.
147 *
148 * Our tactic to handle this is to first atomically read and
149 * exchange a new raw count - then add that new-prev delta
150 * count to the generic counter atomically:
151 */
152 again:
153 prev_raw_count = atomic64_read(&hwc->prev_count);
154 rdmsrl(hwc->counter_base + idx, new_raw_count);
155
156 if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
157 new_raw_count) != prev_raw_count)
158 goto again;
159
160 /*
161 * Now we have the new raw value and have updated the prev
162 * timestamp already. We can now calculate the elapsed delta
163 * (counter-)time and add that to the generic counter.
164 *
165 * Careful, not all hw sign-extends above the physical width
166 * of the count.
167 */
168 delta = (new_raw_count << shift) - (prev_raw_count << shift);
169 delta >>= shift;
170
171 atomic64_add(delta, &counter->count);
172 atomic64_sub(delta, &hwc->period_left);
173
174 return new_raw_count;
175 }
176
177 static atomic_t active_counters;
178 static DEFINE_MUTEX(pmc_reserve_mutex);
179
180 static bool reserve_pmc_hardware(void)
181 {
182 int i;
183
184 if (nmi_watchdog == NMI_LOCAL_APIC)
185 disable_lapic_nmi_watchdog();
186
187 for (i = 0; i < x86_pmu.num_counters; i++) {
188 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
189 goto perfctr_fail;
190 }
191
192 for (i = 0; i < x86_pmu.num_counters; i++) {
193 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
194 goto eventsel_fail;
195 }
196
197 return true;
198
199 eventsel_fail:
200 for (i--; i >= 0; i--)
201 release_evntsel_nmi(x86_pmu.eventsel + i);
202
203 i = x86_pmu.num_counters;
204
205 perfctr_fail:
206 for (i--; i >= 0; i--)
207 release_perfctr_nmi(x86_pmu.perfctr + i);
208
209 if (nmi_watchdog == NMI_LOCAL_APIC)
210 enable_lapic_nmi_watchdog();
211
212 return false;
213 }
214
215 static void release_pmc_hardware(void)
216 {
217 int i;
218
219 for (i = 0; i < x86_pmu.num_counters; i++) {
220 release_perfctr_nmi(x86_pmu.perfctr + i);
221 release_evntsel_nmi(x86_pmu.eventsel + i);
222 }
223
224 if (nmi_watchdog == NMI_LOCAL_APIC)
225 enable_lapic_nmi_watchdog();
226 }
227
228 static void hw_perf_counter_destroy(struct perf_counter *counter)
229 {
230 if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
231 release_pmc_hardware();
232 mutex_unlock(&pmc_reserve_mutex);
233 }
234 }
235
236 static inline int x86_pmu_initialized(void)
237 {
238 return x86_pmu.handle_irq != NULL;
239 }
240
241 /*
242 * Setup the hardware configuration for a given hw_event_type
243 */
244 static int __hw_perf_counter_init(struct perf_counter *counter)
245 {
246 struct perf_counter_hw_event *hw_event = &counter->hw_event;
247 struct hw_perf_counter *hwc = &counter->hw;
248 int err;
249
250 if (!x86_pmu_initialized())
251 return -ENODEV;
252
253 err = 0;
254 if (!atomic_inc_not_zero(&active_counters)) {
255 mutex_lock(&pmc_reserve_mutex);
256 if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
257 err = -EBUSY;
258 else
259 atomic_inc(&active_counters);
260 mutex_unlock(&pmc_reserve_mutex);
261 }
262 if (err)
263 return err;
264
265 /*
266 * Generate PMC IRQs:
267 * (keep 'enabled' bit clear for now)
268 */
269 hwc->config = ARCH_PERFMON_EVENTSEL_INT;
270
271 /*
272 * Count user and OS events unless requested not to.
273 */
274 if (!hw_event->exclude_user)
275 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
276 if (!hw_event->exclude_kernel)
277 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
278
279 /*
280 * If privileged enough, allow NMI events:
281 */
282 hwc->nmi = 0;
283 if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
284 hwc->nmi = 1;
285
286 hwc->irq_period = hw_event->irq_period;
287 if ((s64)hwc->irq_period <= 0 || hwc->irq_period > x86_pmu.max_period)
288 hwc->irq_period = x86_pmu.max_period;
289
290 atomic64_set(&hwc->period_left, hwc->irq_period);
291
292 /*
293 * Raw event type provide the config in the event structure
294 */
295 if (perf_event_raw(hw_event)) {
296 hwc->config |= x86_pmu.raw_event(perf_event_config(hw_event));
297 } else {
298 if (perf_event_id(hw_event) >= x86_pmu.max_events)
299 return -EINVAL;
300 /*
301 * The generic map:
302 */
303 hwc->config |= x86_pmu.event_map(perf_event_id(hw_event));
304 }
305
306 counter->destroy = hw_perf_counter_destroy;
307
308 return 0;
309 }
310
311 static u64 intel_pmu_save_disable_all(void)
312 {
313 u64 ctrl;
314
315 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
316 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
317
318 return ctrl;
319 }
320
321 static u64 amd_pmu_save_disable_all(void)
322 {
323 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
324 int enabled, idx;
325
326 enabled = cpuc->enabled;
327 cpuc->enabled = 0;
328 /*
329 * ensure we write the disable before we start disabling the
330 * counters proper, so that amd_pmu_enable_counter() does the
331 * right thing.
332 */
333 barrier();
334
335 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
336 u64 val;
337
338 if (!test_bit(idx, cpuc->active_mask))
339 continue;
340 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
341 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
342 continue;
343 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
344 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
345 }
346
347 return enabled;
348 }
349
350 u64 hw_perf_save_disable(void)
351 {
352 if (!x86_pmu_initialized())
353 return 0;
354 return x86_pmu.save_disable_all();
355 }
356 /*
357 * Exported because of ACPI idle
358 */
359 EXPORT_SYMBOL_GPL(hw_perf_save_disable);
360
361 static void intel_pmu_restore_all(u64 ctrl)
362 {
363 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
364 }
365
366 static void amd_pmu_restore_all(u64 ctrl)
367 {
368 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
369 int idx;
370
371 cpuc->enabled = ctrl;
372 barrier();
373 if (!ctrl)
374 return;
375
376 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
377 u64 val;
378
379 if (!test_bit(idx, cpuc->active_mask))
380 continue;
381 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
382 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
383 continue;
384 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
385 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
386 }
387 }
388
389 void hw_perf_restore(u64 ctrl)
390 {
391 if (!x86_pmu_initialized())
392 return;
393 x86_pmu.restore_all(ctrl);
394 }
395 /*
396 * Exported because of ACPI idle
397 */
398 EXPORT_SYMBOL_GPL(hw_perf_restore);
399
400 static inline u64 intel_pmu_get_status(void)
401 {
402 u64 status;
403
404 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
405
406 return status;
407 }
408
409 static inline void intel_pmu_ack_status(u64 ack)
410 {
411 wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
412 }
413
414 static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
415 {
416 int err;
417 err = checking_wrmsrl(hwc->config_base + idx,
418 hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
419 }
420
421 static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
422 {
423 int err;
424 err = checking_wrmsrl(hwc->config_base + idx,
425 hwc->config);
426 }
427
428 static inline void
429 intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
430 {
431 int idx = __idx - X86_PMC_IDX_FIXED;
432 u64 ctrl_val, mask;
433 int err;
434
435 mask = 0xfULL << (idx * 4);
436
437 rdmsrl(hwc->config_base, ctrl_val);
438 ctrl_val &= ~mask;
439 err = checking_wrmsrl(hwc->config_base, ctrl_val);
440 }
441
442 static inline void
443 intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
444 {
445 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
446 intel_pmu_disable_fixed(hwc, idx);
447 return;
448 }
449
450 x86_pmu_disable_counter(hwc, idx);
451 }
452
453 static inline void
454 amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
455 {
456 x86_pmu_disable_counter(hwc, idx);
457 }
458
459 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
460
461 /*
462 * Set the next IRQ period, based on the hwc->period_left value.
463 * To be called with the counter disabled in hw:
464 */
465 static void
466 x86_perf_counter_set_period(struct perf_counter *counter,
467 struct hw_perf_counter *hwc, int idx)
468 {
469 s64 left = atomic64_read(&hwc->period_left);
470 s64 period = hwc->irq_period;
471 int err;
472
473 /*
474 * If we are way outside a reasoable range then just skip forward:
475 */
476 if (unlikely(left <= -period)) {
477 left = period;
478 atomic64_set(&hwc->period_left, left);
479 }
480
481 if (unlikely(left <= 0)) {
482 left += period;
483 atomic64_set(&hwc->period_left, left);
484 }
485
486 per_cpu(prev_left[idx], smp_processor_id()) = left;
487
488 /*
489 * The hw counter starts counting from this counter offset,
490 * mark it to be able to extra future deltas:
491 */
492 atomic64_set(&hwc->prev_count, (u64)-left);
493
494 err = checking_wrmsrl(hwc->counter_base + idx,
495 (u64)(-left) & x86_pmu.counter_mask);
496 }
497
498 static inline void
499 intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
500 {
501 int idx = __idx - X86_PMC_IDX_FIXED;
502 u64 ctrl_val, bits, mask;
503 int err;
504
505 /*
506 * Enable IRQ generation (0x8),
507 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
508 * if requested:
509 */
510 bits = 0x8ULL;
511 if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
512 bits |= 0x2;
513 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
514 bits |= 0x1;
515 bits <<= (idx * 4);
516 mask = 0xfULL << (idx * 4);
517
518 rdmsrl(hwc->config_base, ctrl_val);
519 ctrl_val &= ~mask;
520 ctrl_val |= bits;
521 err = checking_wrmsrl(hwc->config_base, ctrl_val);
522 }
523
524 static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
525 {
526 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
527 intel_pmu_enable_fixed(hwc, idx);
528 return;
529 }
530
531 x86_pmu_enable_counter(hwc, idx);
532 }
533
534 static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
535 {
536 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
537
538 if (cpuc->enabled)
539 x86_pmu_enable_counter(hwc, idx);
540 else
541 x86_pmu_disable_counter(hwc, idx);
542 }
543
544 static int
545 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
546 {
547 unsigned int event;
548
549 if (!x86_pmu.num_counters_fixed)
550 return -1;
551
552 if (unlikely(hwc->nmi))
553 return -1;
554
555 event = hwc->config & ARCH_PERFMON_EVENT_MASK;
556
557 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_INSTRUCTIONS)))
558 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
559 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_CPU_CYCLES)))
560 return X86_PMC_IDX_FIXED_CPU_CYCLES;
561 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_BUS_CYCLES)))
562 return X86_PMC_IDX_FIXED_BUS_CYCLES;
563
564 return -1;
565 }
566
567 /*
568 * Find a PMC slot for the freshly enabled / scheduled in counter:
569 */
570 static int x86_pmu_enable(struct perf_counter *counter)
571 {
572 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
573 struct hw_perf_counter *hwc = &counter->hw;
574 int idx;
575
576 idx = fixed_mode_idx(counter, hwc);
577 if (idx >= 0) {
578 /*
579 * Try to get the fixed counter, if that is already taken
580 * then try to get a generic counter:
581 */
582 if (test_and_set_bit(idx, cpuc->used_mask))
583 goto try_generic;
584
585 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
586 /*
587 * We set it so that counter_base + idx in wrmsr/rdmsr maps to
588 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
589 */
590 hwc->counter_base =
591 MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
592 hwc->idx = idx;
593 } else {
594 idx = hwc->idx;
595 /* Try to get the previous generic counter again */
596 if (test_and_set_bit(idx, cpuc->used_mask)) {
597 try_generic:
598 idx = find_first_zero_bit(cpuc->used_mask,
599 x86_pmu.num_counters);
600 if (idx == x86_pmu.num_counters)
601 return -EAGAIN;
602
603 set_bit(idx, cpuc->used_mask);
604 hwc->idx = idx;
605 }
606 hwc->config_base = x86_pmu.eventsel;
607 hwc->counter_base = x86_pmu.perfctr;
608 }
609
610 perf_counters_lapic_init(hwc->nmi);
611
612 x86_pmu.disable(hwc, idx);
613
614 cpuc->counters[idx] = counter;
615 set_bit(idx, cpuc->active_mask);
616
617 x86_perf_counter_set_period(counter, hwc, idx);
618 x86_pmu.enable(hwc, idx);
619
620 return 0;
621 }
622
623 void perf_counter_print_debug(void)
624 {
625 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
626 struct cpu_hw_counters *cpuc;
627 unsigned long flags;
628 int cpu, idx;
629
630 if (!x86_pmu.num_counters)
631 return;
632
633 local_irq_save(flags);
634
635 cpu = smp_processor_id();
636 cpuc = &per_cpu(cpu_hw_counters, cpu);
637
638 if (x86_pmu.version >= 2) {
639 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
640 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
641 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
642 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
643
644 pr_info("\n");
645 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
646 pr_info("CPU#%d: status: %016llx\n", cpu, status);
647 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
648 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
649 }
650 pr_info("CPU#%d: used: %016llx\n", cpu, *(u64 *)cpuc->used_mask);
651
652 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
653 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
654 rdmsrl(x86_pmu.perfctr + idx, pmc_count);
655
656 prev_left = per_cpu(prev_left[idx], cpu);
657
658 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
659 cpu, idx, pmc_ctrl);
660 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
661 cpu, idx, pmc_count);
662 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
663 cpu, idx, prev_left);
664 }
665 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
666 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
667
668 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
669 cpu, idx, pmc_count);
670 }
671 local_irq_restore(flags);
672 }
673
674 static void x86_pmu_disable(struct perf_counter *counter)
675 {
676 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
677 struct hw_perf_counter *hwc = &counter->hw;
678 int idx = hwc->idx;
679
680 /*
681 * Must be done before we disable, otherwise the nmi handler
682 * could reenable again:
683 */
684 clear_bit(idx, cpuc->active_mask);
685 x86_pmu.disable(hwc, idx);
686
687 /*
688 * Make sure the cleared pointer becomes visible before we
689 * (potentially) free the counter:
690 */
691 barrier();
692
693 /*
694 * Drain the remaining delta count out of a counter
695 * that we are disabling:
696 */
697 x86_perf_counter_update(counter, hwc, idx);
698 cpuc->counters[idx] = NULL;
699 clear_bit(idx, cpuc->used_mask);
700 }
701
702 /*
703 * Save and restart an expired counter. Called by NMI contexts,
704 * so it has to be careful about preempting normal counter ops:
705 */
706 static void intel_pmu_save_and_restart(struct perf_counter *counter)
707 {
708 struct hw_perf_counter *hwc = &counter->hw;
709 int idx = hwc->idx;
710
711 x86_perf_counter_update(counter, hwc, idx);
712 x86_perf_counter_set_period(counter, hwc, idx);
713
714 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
715 intel_pmu_enable_counter(hwc, idx);
716 }
717
718 /*
719 * Maximum interrupt frequency of 100KHz per CPU
720 */
721 #define PERFMON_MAX_INTERRUPTS (100000/HZ)
722
723 /*
724 * This handler is triggered by the local APIC, so the APIC IRQ handling
725 * rules apply:
726 */
727 static int intel_pmu_handle_irq(struct pt_regs *regs, int nmi)
728 {
729 int bit, cpu = smp_processor_id();
730 u64 ack, status;
731 struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
732 int ret = 0;
733
734 cpuc->throttle_ctrl = intel_pmu_save_disable_all();
735
736 status = intel_pmu_get_status();
737 if (!status)
738 goto out;
739
740 ret = 1;
741 again:
742 inc_irq_stat(apic_perf_irqs);
743 ack = status;
744 for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
745 struct perf_counter *counter = cpuc->counters[bit];
746
747 clear_bit(bit, (unsigned long *) &status);
748 if (!test_bit(bit, cpuc->active_mask))
749 continue;
750
751 intel_pmu_save_and_restart(counter);
752 if (perf_counter_overflow(counter, nmi, regs, 0))
753 intel_pmu_disable_counter(&counter->hw, bit);
754 }
755
756 intel_pmu_ack_status(ack);
757
758 /*
759 * Repeat if there is more work to be done:
760 */
761 status = intel_pmu_get_status();
762 if (status)
763 goto again;
764 out:
765 /*
766 * Restore - do not reenable when global enable is off or throttled:
767 */
768 if (cpuc->throttle_ctrl) {
769 if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS) {
770 intel_pmu_restore_all(cpuc->throttle_ctrl);
771 } else {
772 pr_info("CPU#%d: perfcounters: max interrupt rate exceeded! Throttle on.\n", smp_processor_id());
773 }
774 }
775
776 return ret;
777 }
778
779 static int amd_pmu_handle_irq(struct pt_regs *regs, int nmi)
780 {
781 int cpu = smp_processor_id();
782 struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
783 u64 val;
784 int handled = 0;
785 struct perf_counter *counter;
786 struct hw_perf_counter *hwc;
787 int idx;
788
789 ++cpuc->interrupts;
790 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
791 if (!test_bit(idx, cpuc->active_mask))
792 continue;
793 counter = cpuc->counters[idx];
794 hwc = &counter->hw;
795 val = x86_perf_counter_update(counter, hwc, idx);
796 if (val & (1ULL << (x86_pmu.counter_bits - 1)))
797 continue;
798 /* counter overflow */
799 x86_perf_counter_set_period(counter, hwc, idx);
800 handled = 1;
801 inc_irq_stat(apic_perf_irqs);
802 if (perf_counter_overflow(counter, nmi, regs, 0))
803 amd_pmu_disable_counter(hwc, idx);
804 else if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS)
805 /*
806 * do not reenable when throttled, but reload
807 * the register
808 */
809 amd_pmu_disable_counter(hwc, idx);
810 else if (counter->state == PERF_COUNTER_STATE_ACTIVE)
811 amd_pmu_enable_counter(hwc, idx);
812 }
813 return handled;
814 }
815
816 void perf_counter_unthrottle(void)
817 {
818 struct cpu_hw_counters *cpuc;
819
820 if (!x86_pmu_initialized())
821 return;
822
823 cpuc = &__get_cpu_var(cpu_hw_counters);
824 if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
825 pr_info("CPU#%d: perfcounters: throttle off.\n", smp_processor_id());
826
827 /*
828 * Clear them before re-enabling irqs/NMIs again:
829 */
830 cpuc->interrupts = 0;
831 hw_perf_restore(cpuc->throttle_ctrl);
832 } else {
833 cpuc->interrupts = 0;
834 }
835 }
836
837 void smp_perf_counter_interrupt(struct pt_regs *regs)
838 {
839 irq_enter();
840 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
841 ack_APIC_irq();
842 x86_pmu.handle_irq(regs, 0);
843 irq_exit();
844 }
845
846 void smp_perf_pending_interrupt(struct pt_regs *regs)
847 {
848 irq_enter();
849 ack_APIC_irq();
850 inc_irq_stat(apic_pending_irqs);
851 perf_counter_do_pending();
852 irq_exit();
853 }
854
855 void set_perf_counter_pending(void)
856 {
857 apic->send_IPI_self(LOCAL_PENDING_VECTOR);
858 }
859
860 void perf_counters_lapic_init(int nmi)
861 {
862 u32 apic_val;
863
864 if (!x86_pmu_initialized())
865 return;
866
867 /*
868 * Enable the performance counter vector in the APIC LVT:
869 */
870 apic_val = apic_read(APIC_LVTERR);
871
872 apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
873 if (nmi)
874 apic_write(APIC_LVTPC, APIC_DM_NMI);
875 else
876 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
877 apic_write(APIC_LVTERR, apic_val);
878 }
879
880 static int __kprobes
881 perf_counter_nmi_handler(struct notifier_block *self,
882 unsigned long cmd, void *__args)
883 {
884 struct die_args *args = __args;
885 struct pt_regs *regs;
886 int ret;
887
888 if (!atomic_read(&active_counters))
889 return NOTIFY_DONE;
890
891 switch (cmd) {
892 case DIE_NMI:
893 case DIE_NMI_IPI:
894 break;
895
896 default:
897 return NOTIFY_DONE;
898 }
899
900 regs = args->regs;
901
902 apic_write(APIC_LVTPC, APIC_DM_NMI);
903 ret = x86_pmu.handle_irq(regs, 1);
904
905 return ret ? NOTIFY_STOP : NOTIFY_OK;
906 }
907
908 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
909 .notifier_call = perf_counter_nmi_handler,
910 .next = NULL,
911 .priority = 1
912 };
913
914 static struct x86_pmu intel_pmu = {
915 .name = "Intel",
916 .handle_irq = intel_pmu_handle_irq,
917 .save_disable_all = intel_pmu_save_disable_all,
918 .restore_all = intel_pmu_restore_all,
919 .enable = intel_pmu_enable_counter,
920 .disable = intel_pmu_disable_counter,
921 .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
922 .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
923 .event_map = intel_pmu_event_map,
924 .raw_event = intel_pmu_raw_event,
925 .max_events = ARRAY_SIZE(intel_perfmon_event_map),
926 /*
927 * Intel PMCs cannot be accessed sanely above 32 bit width,
928 * so we install an artificial 1<<31 period regardless of
929 * the generic counter period:
930 */
931 .max_period = (1ULL << 31) - 1,
932 };
933
934 static struct x86_pmu amd_pmu = {
935 .name = "AMD",
936 .handle_irq = amd_pmu_handle_irq,
937 .save_disable_all = amd_pmu_save_disable_all,
938 .restore_all = amd_pmu_restore_all,
939 .enable = amd_pmu_enable_counter,
940 .disable = amd_pmu_disable_counter,
941 .eventsel = MSR_K7_EVNTSEL0,
942 .perfctr = MSR_K7_PERFCTR0,
943 .event_map = amd_pmu_event_map,
944 .raw_event = amd_pmu_raw_event,
945 .max_events = ARRAY_SIZE(amd_perfmon_event_map),
946 .num_counters = 4,
947 .counter_bits = 48,
948 .counter_mask = (1ULL << 48) - 1,
949 /* use highest bit to detect overflow */
950 .max_period = (1ULL << 47) - 1,
951 };
952
953 static int intel_pmu_init(void)
954 {
955 union cpuid10_edx edx;
956 union cpuid10_eax eax;
957 unsigned int unused;
958 unsigned int ebx;
959 int version;
960
961 if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
962 return -ENODEV;
963
964 /*
965 * Check whether the Architectural PerfMon supports
966 * Branch Misses Retired Event or not.
967 */
968 cpuid(10, &eax.full, &ebx, &unused, &edx.full);
969 if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
970 return -ENODEV;
971
972 version = eax.split.version_id;
973 if (version < 2)
974 return -ENODEV;
975
976 x86_pmu = intel_pmu;
977 x86_pmu.version = version;
978 x86_pmu.num_counters = eax.split.num_counters;
979
980 /*
981 * Quirk: v2 perfmon does not report fixed-purpose counters, so
982 * assume at least 3 counters:
983 */
984 x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
985
986 x86_pmu.counter_bits = eax.split.bit_width;
987 x86_pmu.counter_mask = (1ULL << eax.split.bit_width) - 1;
988
989 return 0;
990 }
991
992 static int amd_pmu_init(void)
993 {
994 x86_pmu = amd_pmu;
995 return 0;
996 }
997
998 void __init init_hw_perf_counters(void)
999 {
1000 int err;
1001
1002 switch (boot_cpu_data.x86_vendor) {
1003 case X86_VENDOR_INTEL:
1004 err = intel_pmu_init();
1005 break;
1006 case X86_VENDOR_AMD:
1007 err = amd_pmu_init();
1008 break;
1009 default:
1010 return;
1011 }
1012 if (err != 0)
1013 return;
1014
1015 pr_info("%s Performance Monitoring support detected.\n", x86_pmu.name);
1016 pr_info("... version: %d\n", x86_pmu.version);
1017 pr_info("... bit width: %d\n", x86_pmu.counter_bits);
1018
1019 pr_info("... num counters: %d\n", x86_pmu.num_counters);
1020 if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1021 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
1022 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1023 x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
1024 }
1025 perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
1026 perf_max_counters = x86_pmu.num_counters;
1027
1028 pr_info("... value mask: %016Lx\n", x86_pmu.counter_mask);
1029 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
1030
1031 if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1032 x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1033 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1034 x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
1035 }
1036 pr_info("... fixed counters: %d\n", x86_pmu.num_counters_fixed);
1037
1038 perf_counter_mask |=
1039 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1040
1041 pr_info("... counter mask: %016Lx\n", perf_counter_mask);
1042
1043 perf_counters_lapic_init(0);
1044 register_die_notifier(&perf_counter_nmi_notifier);
1045 }
1046
1047 static inline void x86_pmu_read(struct perf_counter *counter)
1048 {
1049 x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1050 }
1051
1052 static const struct pmu pmu = {
1053 .enable = x86_pmu_enable,
1054 .disable = x86_pmu_disable,
1055 .read = x86_pmu_read,
1056 };
1057
1058 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1059 {
1060 int err;
1061
1062 err = __hw_perf_counter_init(counter);
1063 if (err)
1064 return ERR_PTR(err);
1065
1066 return &pmu;
1067 }
1068
1069 /*
1070 * callchain support
1071 */
1072
1073 static inline
1074 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1075 {
1076 if (entry->nr < MAX_STACK_DEPTH)
1077 entry->ip[entry->nr++] = ip;
1078 }
1079
1080 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1081 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1082
1083
1084 static void
1085 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1086 {
1087 /* Ignore warnings */
1088 }
1089
1090 static void backtrace_warning(void *data, char *msg)
1091 {
1092 /* Ignore warnings */
1093 }
1094
1095 static int backtrace_stack(void *data, char *name)
1096 {
1097 /* Don't bother with IRQ stacks for now */
1098 return -1;
1099 }
1100
1101 static void backtrace_address(void *data, unsigned long addr, int reliable)
1102 {
1103 struct perf_callchain_entry *entry = data;
1104
1105 if (reliable)
1106 callchain_store(entry, addr);
1107 }
1108
1109 static const struct stacktrace_ops backtrace_ops = {
1110 .warning = backtrace_warning,
1111 .warning_symbol = backtrace_warning_symbol,
1112 .stack = backtrace_stack,
1113 .address = backtrace_address,
1114 };
1115
1116 static void
1117 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1118 {
1119 unsigned long bp;
1120 char *stack;
1121 int nr = entry->nr;
1122
1123 callchain_store(entry, instruction_pointer(regs));
1124
1125 stack = ((char *)regs + sizeof(struct pt_regs));
1126 #ifdef CONFIG_FRAME_POINTER
1127 bp = frame_pointer(regs);
1128 #else
1129 bp = 0;
1130 #endif
1131
1132 dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1133
1134 entry->kernel = entry->nr - nr;
1135 }
1136
1137
1138 struct stack_frame {
1139 const void __user *next_fp;
1140 unsigned long return_address;
1141 };
1142
1143 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1144 {
1145 int ret;
1146
1147 if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1148 return 0;
1149
1150 ret = 1;
1151 pagefault_disable();
1152 if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1153 ret = 0;
1154 pagefault_enable();
1155
1156 return ret;
1157 }
1158
1159 static void
1160 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1161 {
1162 struct stack_frame frame;
1163 const void __user *fp;
1164 int nr = entry->nr;
1165
1166 regs = (struct pt_regs *)current->thread.sp0 - 1;
1167 fp = (void __user *)regs->bp;
1168
1169 callchain_store(entry, regs->ip);
1170
1171 while (entry->nr < MAX_STACK_DEPTH) {
1172 frame.next_fp = NULL;
1173 frame.return_address = 0;
1174
1175 if (!copy_stack_frame(fp, &frame))
1176 break;
1177
1178 if ((unsigned long)fp < user_stack_pointer(regs))
1179 break;
1180
1181 callchain_store(entry, frame.return_address);
1182 fp = frame.next_fp;
1183 }
1184
1185 entry->user = entry->nr - nr;
1186 }
1187
1188 static void
1189 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1190 {
1191 int is_user;
1192
1193 if (!regs)
1194 return;
1195
1196 is_user = user_mode(regs);
1197
1198 if (!current || current->pid == 0)
1199 return;
1200
1201 if (is_user && current->state != TASK_RUNNING)
1202 return;
1203
1204 if (!is_user)
1205 perf_callchain_kernel(regs, entry);
1206
1207 if (current->mm)
1208 perf_callchain_user(regs, entry);
1209 }
1210
1211 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1212 {
1213 struct perf_callchain_entry *entry;
1214
1215 if (in_nmi())
1216 entry = &__get_cpu_var(nmi_entry);
1217 else
1218 entry = &__get_cpu_var(irq_entry);
1219
1220 entry->nr = 0;
1221 entry->hv = 0;
1222 entry->kernel = 0;
1223 entry->user = 0;
1224
1225 perf_do_callchain(regs, entry);
1226
1227 return entry;
1228 }
Linux kernel - Deliverables
This page took 0.064496 seconds and 4 git commands to generate.