1 #ifdef CONFIG_CPU_SUP_INTEL
3 /* The maximal number of PEBS events: */
4 #define MAX_PEBS_EVENTS 4
6 /* The size of a BTS record in bytes: */
7 #define BTS_RECORD_SIZE 24
9 #define BTS_BUFFER_SIZE (PAGE_SIZE << 4)
10 #define PEBS_BUFFER_SIZE PAGE_SIZE
13 * pebs_record_32 for p4 and core not supported
15 struct pebs_record_32 {
23 struct pebs_record_core
{
28 u64 r12
, r13
, r14
, r15
;
31 struct pebs_record_nhm
{
36 u64 r12
, r13
, r14
, r15
;
37 u64 status
, dla
, dse
, lat
;
41 * Bits in the debugctlmsr controlling branch tracing.
43 #define X86_DEBUGCTL_TR (1 << 6)
44 #define X86_DEBUGCTL_BTS (1 << 7)
45 #define X86_DEBUGCTL_BTINT (1 << 8)
46 #define X86_DEBUGCTL_BTS_OFF_OS (1 << 9)
47 #define X86_DEBUGCTL_BTS_OFF_USR (1 << 10)
50 * A debug store configuration.
52 * We only support architectures that use 64bit fields.
57 u64 bts_absolute_maximum
;
58 u64 bts_interrupt_threshold
;
61 u64 pebs_absolute_maximum
;
62 u64 pebs_interrupt_threshold
;
63 u64 pebs_event_reset
[MAX_PEBS_EVENTS
];
66 static void init_debug_store_on_cpu(int cpu
)
68 struct debug_store
*ds
= per_cpu(cpu_hw_events
, cpu
).ds
;
73 wrmsr_on_cpu(cpu
, MSR_IA32_DS_AREA
,
74 (u32
)((u64
)(unsigned long)ds
),
75 (u32
)((u64
)(unsigned long)ds
>> 32));
78 static void fini_debug_store_on_cpu(int cpu
)
80 if (!per_cpu(cpu_hw_events
, cpu
).ds
)
83 wrmsr_on_cpu(cpu
, MSR_IA32_DS_AREA
, 0, 0);
86 static void release_ds_buffers(void)
90 if (!x86_pmu
.bts
&& !x86_pmu
.pebs
)
95 for_each_online_cpu(cpu
)
96 fini_debug_store_on_cpu(cpu
);
98 for_each_possible_cpu(cpu
) {
99 struct debug_store
*ds
= per_cpu(cpu_hw_events
, cpu
).ds
;
104 per_cpu(cpu_hw_events
, cpu
).ds
= NULL
;
106 kfree((void *)(unsigned long)ds
->pebs_buffer_base
);
107 kfree((void *)(unsigned long)ds
->bts_buffer_base
);
114 static int reserve_ds_buffers(void)
118 if (!x86_pmu
.bts
&& !x86_pmu
.pebs
)
123 for_each_possible_cpu(cpu
) {
124 struct debug_store
*ds
;
129 ds
= kzalloc(sizeof(*ds
), GFP_KERNEL
);
134 per_cpu(cpu_hw_events
, cpu
).ds
= ds
;
137 buffer
= kzalloc(BTS_BUFFER_SIZE
, GFP_KERNEL
);
138 if (unlikely(!buffer
))
141 max
= BTS_BUFFER_SIZE
/ BTS_RECORD_SIZE
;
144 ds
->bts_buffer_base
= (u64
)(unsigned long)buffer
;
145 ds
->bts_index
= ds
->bts_buffer_base
;
146 ds
->bts_absolute_maximum
= ds
->bts_buffer_base
+
147 max
* BTS_RECORD_SIZE
;
148 ds
->bts_interrupt_threshold
= ds
->bts_absolute_maximum
-
149 thresh
* BTS_RECORD_SIZE
;
153 buffer
= kzalloc(PEBS_BUFFER_SIZE
, GFP_KERNEL
);
154 if (unlikely(!buffer
))
157 max
= PEBS_BUFFER_SIZE
/ x86_pmu
.pebs_record_size
;
159 ds
->pebs_buffer_base
= (u64
)(unsigned long)buffer
;
160 ds
->pebs_index
= ds
->pebs_buffer_base
;
161 ds
->pebs_absolute_maximum
= ds
->pebs_buffer_base
+
162 max
* x86_pmu
.pebs_record_size
;
164 * Always use single record PEBS
166 ds
->pebs_interrupt_threshold
= ds
->pebs_buffer_base
+
167 x86_pmu
.pebs_record_size
;
174 release_ds_buffers();
176 for_each_online_cpu(cpu
)
177 init_debug_store_on_cpu(cpu
);
189 static struct event_constraint bts_constraint
=
190 EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS
, 0);
192 static void intel_pmu_enable_bts(u64 config
)
194 unsigned long debugctlmsr
;
196 debugctlmsr
= get_debugctlmsr();
198 debugctlmsr
|= X86_DEBUGCTL_TR
;
199 debugctlmsr
|= X86_DEBUGCTL_BTS
;
200 debugctlmsr
|= X86_DEBUGCTL_BTINT
;
202 if (!(config
& ARCH_PERFMON_EVENTSEL_OS
))
203 debugctlmsr
|= X86_DEBUGCTL_BTS_OFF_OS
;
205 if (!(config
& ARCH_PERFMON_EVENTSEL_USR
))
206 debugctlmsr
|= X86_DEBUGCTL_BTS_OFF_USR
;
208 update_debugctlmsr(debugctlmsr
);
211 static void intel_pmu_disable_bts(void)
213 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
214 unsigned long debugctlmsr
;
219 debugctlmsr
= get_debugctlmsr();
222 ~(X86_DEBUGCTL_TR
| X86_DEBUGCTL_BTS
| X86_DEBUGCTL_BTINT
|
223 X86_DEBUGCTL_BTS_OFF_OS
| X86_DEBUGCTL_BTS_OFF_USR
);
225 update_debugctlmsr(debugctlmsr
);
228 static void intel_pmu_drain_bts_buffer(void)
230 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
231 struct debug_store
*ds
= cpuc
->ds
;
237 struct perf_event
*event
= cpuc
->events
[X86_PMC_IDX_FIXED_BTS
];
238 struct bts_record
*at
, *top
;
239 struct perf_output_handle handle
;
240 struct perf_event_header header
;
241 struct perf_sample_data data
;
250 at
= (struct bts_record
*)(unsigned long)ds
->bts_buffer_base
;
251 top
= (struct bts_record
*)(unsigned long)ds
->bts_index
;
256 ds
->bts_index
= ds
->bts_buffer_base
;
258 perf_sample_data_init(&data
, 0);
259 data
.period
= event
->hw
.last_period
;
263 * Prepare a generic sample, i.e. fill in the invariant fields.
264 * We will overwrite the from and to address before we output
267 perf_prepare_sample(&header
, &data
, event
, ®s
);
269 if (perf_output_begin(&handle
, event
, header
.size
* (top
- at
), 1, 1))
272 for (; at
< top
; at
++) {
276 perf_output_sample(&handle
, &header
, &data
, event
);
279 perf_output_end(&handle
);
281 /* There's new data available. */
282 event
->hw
.interrupts
++;
283 event
->pending_kill
= POLL_IN
;
290 static struct event_constraint intel_core_pebs_events
[] = {
291 PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INSTR_RETIRED.ANY */
292 PEBS_EVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
293 PEBS_EVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
294 PEBS_EVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
295 PEBS_EVENT_CONSTRAINT(0x01cb, 0x1), /* MEM_LOAD_RETIRED.L1D_MISS */
296 PEBS_EVENT_CONSTRAINT(0x02cb, 0x1), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
297 PEBS_EVENT_CONSTRAINT(0x04cb, 0x1), /* MEM_LOAD_RETIRED.L2_MISS */
298 PEBS_EVENT_CONSTRAINT(0x08cb, 0x1), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
299 PEBS_EVENT_CONSTRAINT(0x10cb, 0x1), /* MEM_LOAD_RETIRED.DTLB_MISS */
303 static struct event_constraint intel_nehalem_pebs_events
[] = {
304 PEBS_EVENT_CONSTRAINT(0x00c0, 0xf), /* INSTR_RETIRED.ANY */
305 PEBS_EVENT_CONSTRAINT(0xfec1, 0xf), /* X87_OPS_RETIRED.ANY */
306 PEBS_EVENT_CONSTRAINT(0x00c5, 0xf), /* BR_INST_RETIRED.MISPRED */
307 PEBS_EVENT_CONSTRAINT(0x1fc7, 0xf), /* SIMD_INST_RETURED.ANY */
308 PEBS_EVENT_CONSTRAINT(0x01cb, 0xf), /* MEM_LOAD_RETIRED.L1D_MISS */
309 PEBS_EVENT_CONSTRAINT(0x02cb, 0xf), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
310 PEBS_EVENT_CONSTRAINT(0x04cb, 0xf), /* MEM_LOAD_RETIRED.L2_MISS */
311 PEBS_EVENT_CONSTRAINT(0x08cb, 0xf), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
312 PEBS_EVENT_CONSTRAINT(0x10cb, 0xf), /* MEM_LOAD_RETIRED.DTLB_MISS */
316 static struct event_constraint
*
317 intel_pebs_constraints(struct perf_event
*event
)
319 struct event_constraint
*c
;
321 if (!event
->attr
.precise
)
324 if (x86_pmu
.pebs_constraints
) {
325 for_each_event_constraint(c
, x86_pmu
.pebs_constraints
) {
326 if ((event
->hw
.config
& c
->cmask
) == c
->code
)
331 return &emptyconstraint
;
334 static void intel_pmu_pebs_enable(struct perf_event
*event
)
336 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
337 struct hw_perf_event
*hwc
= &event
->hw
;
338 u64 val
= cpuc
->pebs_enabled
;
340 hwc
->config
&= ~ARCH_PERFMON_EVENTSEL_INT
;
342 val
|= 1ULL << hwc
->idx
;
343 wrmsrl(MSR_IA32_PEBS_ENABLE
, val
);
345 intel_pmu_lbr_enable(event
);
348 static void intel_pmu_pebs_disable(struct perf_event
*event
)
350 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
351 struct hw_perf_event
*hwc
= &event
->hw
;
352 u64 val
= cpuc
->pebs_enabled
;
354 val
&= ~(1ULL << hwc
->idx
);
355 wrmsrl(MSR_IA32_PEBS_ENABLE
, val
);
357 hwc
->config
|= ARCH_PERFMON_EVENTSEL_INT
;
359 intel_pmu_lbr_disable(event
);
362 static void intel_pmu_pebs_enable_all(void)
364 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
366 if (cpuc
->pebs_enabled
)
367 wrmsrl(MSR_IA32_PEBS_ENABLE
, cpuc
->pebs_enabled
);
370 static void intel_pmu_pebs_disable_all(void)
372 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
374 if (cpuc
->pebs_enabled
)
375 wrmsrl(MSR_IA32_PEBS_ENABLE
, 0);
378 #include <asm/insn.h>
380 #define MAX_INSN_SIZE 16
382 static inline bool kernel_ip(unsigned long ip
)
385 return ip
> PAGE_OFFSET
;
391 static int intel_pmu_pebs_fixup_ip(struct pt_regs
*regs
)
393 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
394 unsigned long from
= cpuc
->lbr_entries
[0].from
;
395 unsigned long old_to
, to
= cpuc
->lbr_entries
[0].to
;
396 unsigned long ip
= regs
->ip
;
398 if (!cpuc
->lbr_stack
.nr
|| !from
|| !to
)
405 * We sampled a branch insn, rewind using the LBR stack
414 u8 buf
[MAX_INSN_SIZE
];
418 if (!kernel_ip(ip
)) {
419 int bytes
, size
= min_t(int, MAX_INSN_SIZE
, ip
- to
);
421 bytes
= copy_from_user_nmi(buf
, (void __user
*)to
, size
);
429 kernel_insn_init(&insn
, kaddr
);
430 insn_get_length(&insn
);
442 static int intel_pmu_save_and_restart(struct perf_event
*event
);
443 static void intel_pmu_disable_event(struct perf_event
*event
);
445 static void intel_pmu_drain_pebs_core(struct pt_regs
*iregs
)
447 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
448 struct debug_store
*ds
= cpuc
->ds
;
449 struct perf_event
*event
= cpuc
->events
[0]; /* PMC0 only */
450 struct pebs_record_core
*at
, *top
;
451 struct perf_sample_data data
;
455 if (!event
|| !ds
|| !x86_pmu
.pebs
)
458 intel_pmu_pebs_disable_all();
460 at
= (struct pebs_record_core
*)(unsigned long)ds
->pebs_buffer_base
;
461 top
= (struct pebs_record_core
*)(unsigned long)ds
->pebs_index
;
466 ds
->pebs_index
= ds
->pebs_buffer_base
;
468 if (!intel_pmu_save_and_restart(event
))
471 perf_sample_data_init(&data
, 0);
472 data
.period
= event
->hw
.last_period
;
477 * Should not happen, we program the threshold at 1 and do not
483 * We use the interrupt regs as a base because the PEBS record
484 * does not contain a full regs set, specifically it seems to
485 * lack segment descriptors, which get used by things like
488 * In the simple case fix up only the IP and BP,SP regs, for
489 * PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly.
490 * A possible PERF_SAMPLE_REGS will have to transfer all regs.
497 if (intel_pmu_pebs_fixup_ip(®s
))
498 regs
.flags
|= PERF_EFLAGS_EXACT
;
500 regs
.flags
&= ~PERF_EFLAGS_EXACT
;
502 if (perf_event_overflow(event
, 1, &data
, ®s
))
503 intel_pmu_disable_event(event
);
506 intel_pmu_pebs_enable_all();
509 static void intel_pmu_drain_pebs_nhm(struct pt_regs
*iregs
)
511 struct cpu_hw_events
*cpuc
= &__get_cpu_var(cpu_hw_events
);
512 struct debug_store
*ds
= cpuc
->ds
;
513 struct pebs_record_nhm
*at
, *top
;
514 struct perf_sample_data data
;
515 struct perf_event
*event
= NULL
;
519 if (!ds
|| !x86_pmu
.pebs
)
522 intel_pmu_pebs_disable_all();
524 at
= (struct pebs_record_nhm
*)(unsigned long)ds
->pebs_buffer_base
;
525 top
= (struct pebs_record_nhm
*)(unsigned long)ds
->pebs_index
;
530 ds
->pebs_index
= ds
->pebs_buffer_base
;
535 * Should not happen, we program the threshold at 1 and do not
538 WARN_ON_ONCE(n
> MAX_PEBS_EVENTS
);
540 for ( ; at
< top
; at
++) {
541 for_each_bit(bit
, (unsigned long *)&at
->status
, MAX_PEBS_EVENTS
) {
542 if (!cpuc
->events
[bit
]->attr
.precise
)
545 event
= cpuc
->events
[bit
];
551 if (!intel_pmu_save_and_restart(event
))
554 perf_sample_data_init(&data
, 0);
555 data
.period
= event
->hw
.last_period
;
558 * See the comment in intel_pmu_drain_pebs_core()
565 if (intel_pmu_pebs_fixup_ip(®s
))
566 regs
.flags
|= PERF_EFLAGS_EXACT
;
568 regs
.flags
&= ~PERF_EFLAGS_EXACT
;
570 if (perf_event_overflow(event
, 1, &data
, ®s
))
571 intel_pmu_disable_event(event
);
574 intel_pmu_pebs_enable_all();
578 * BTS, PEBS probe and setup
581 static void intel_ds_init(void)
584 * No support for 32bit formats
586 if (!boot_cpu_has(X86_FEATURE_DTES64
))
589 x86_pmu
.bts
= boot_cpu_has(X86_FEATURE_BTS
);
590 x86_pmu
.pebs
= boot_cpu_has(X86_FEATURE_PEBS
);
594 if (x86_pmu
.version
> 1) {
597 * v2+ has a PEBS format field
599 rdmsrl(MSR_IA32_PERF_CAPABILITIES
, capabilities
);
600 format
= (capabilities
>> 8) & 0xf;
605 printk(KERN_CONT
"PEBS v0, ");
606 x86_pmu
.pebs_record_size
= sizeof(struct pebs_record_core
);
607 x86_pmu
.drain_pebs
= intel_pmu_drain_pebs_core
;
608 x86_pmu
.pebs_constraints
= intel_core_pebs_events
;
612 printk(KERN_CONT
"PEBS v1, ");
613 x86_pmu
.pebs_record_size
= sizeof(struct pebs_record_nhm
);
614 x86_pmu
.drain_pebs
= intel_pmu_drain_pebs_nhm
;
615 x86_pmu
.pebs_constraints
= intel_nehalem_pebs_events
;
619 printk(KERN_CONT
"PEBS unknown format: %d, ", format
);
626 #else /* CONFIG_CPU_SUP_INTEL */
628 static int reseve_ds_buffers(void)
633 static void release_ds_buffers(void)
637 #endif /* CONFIG_CPU_SUP_INTEL */