2 * intel_pt.c: Intel Processor Trace support
3 * Copyright (c) 2013-2015, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19 #include <linux/kernel.h>
20 #include <linux/types.h>
34 #include "thread-stack.h"
36 #include "callchain.h"
43 #include "intel-pt-decoder/intel-pt-log.h"
44 #include "intel-pt-decoder/intel-pt-decoder.h"
45 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
46 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
48 #define MAX_TIMESTAMP (~0ULL)
51 struct auxtrace auxtrace
;
52 struct auxtrace_queues queues
;
53 struct auxtrace_heap heap
;
55 struct perf_session
*session
;
56 struct machine
*machine
;
57 struct perf_evsel
*switch_evsel
;
58 struct thread
*unknown_thread
;
59 bool timeless_decoding
;
68 int have_sched_switch
;
74 struct perf_tsc_conversion tc
;
75 bool cap_user_time_zero
;
77 struct itrace_synth_opts synth_opts
;
79 bool sample_instructions
;
80 u64 instructions_sample_type
;
81 u64 instructions_sample_period
;
86 u64 branches_sample_type
;
89 bool sample_transactions
;
90 u64 transactions_sample_type
;
93 bool synth_needs_swap
;
102 unsigned max_non_turbo_ratio
;
106 INTEL_PT_SS_NOT_TRACING
,
109 INTEL_PT_SS_EXPECTING_SWITCH_EVENT
,
110 INTEL_PT_SS_EXPECTING_SWITCH_IP
,
113 struct intel_pt_queue
{
115 unsigned int queue_nr
;
116 struct auxtrace_buffer
*buffer
;
118 const struct intel_pt_state
*state
;
119 struct ip_callchain
*chain
;
120 struct branch_stack
*last_branch
;
121 struct branch_stack
*last_branch_rb
;
122 size_t last_branch_pos
;
123 union perf_event
*event_buf
;
126 bool step_through_buffers
;
127 bool use_buffer_pid_tid
;
132 struct thread
*thread
;
142 static void intel_pt_dump(struct intel_pt
*pt __maybe_unused
,
143 unsigned char *buf
, size_t len
)
145 struct intel_pt_pkt packet
;
148 char desc
[INTEL_PT_PKT_DESC_MAX
];
149 const char *color
= PERF_COLOR_BLUE
;
151 color_fprintf(stdout
, color
,
152 ". ... Intel Processor Trace data: size %zu bytes\n",
156 ret
= intel_pt_get_packet(buf
, len
, &packet
);
162 color_fprintf(stdout
, color
, " %08x: ", pos
);
163 for (i
= 0; i
< pkt_len
; i
++)
164 color_fprintf(stdout
, color
, " %02x", buf
[i
]);
166 color_fprintf(stdout
, color
, " ");
168 ret
= intel_pt_pkt_desc(&packet
, desc
,
169 INTEL_PT_PKT_DESC_MAX
);
171 color_fprintf(stdout
, color
, " %s\n", desc
);
173 color_fprintf(stdout
, color
, " Bad packet!\n");
181 static void intel_pt_dump_event(struct intel_pt
*pt
, unsigned char *buf
,
185 intel_pt_dump(pt
, buf
, len
);
188 static int intel_pt_do_fix_overlap(struct intel_pt
*pt
, struct auxtrace_buffer
*a
,
189 struct auxtrace_buffer
*b
)
193 start
= intel_pt_find_overlap(a
->data
, a
->size
, b
->data
, b
->size
,
197 b
->use_size
= b
->data
+ b
->size
- start
;
202 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue
*ptq
,
203 struct auxtrace_queue
*queue
,
204 struct auxtrace_buffer
*buffer
)
206 if (queue
->cpu
== -1 && buffer
->cpu
!= -1)
207 ptq
->cpu
= buffer
->cpu
;
209 ptq
->pid
= buffer
->pid
;
210 ptq
->tid
= buffer
->tid
;
212 intel_pt_log("queue %u cpu %d pid %d tid %d\n",
213 ptq
->queue_nr
, ptq
->cpu
, ptq
->pid
, ptq
->tid
);
215 thread__zput(ptq
->thread
);
217 if (ptq
->tid
!= -1) {
219 ptq
->thread
= machine__findnew_thread(ptq
->pt
->machine
,
223 ptq
->thread
= machine__find_thread(ptq
->pt
->machine
, -1,
228 /* This function assumes data is processed sequentially only */
229 static int intel_pt_get_trace(struct intel_pt_buffer
*b
, void *data
)
231 struct intel_pt_queue
*ptq
= data
;
232 struct auxtrace_buffer
*buffer
= ptq
->buffer
, *old_buffer
= buffer
;
233 struct auxtrace_queue
*queue
;
240 queue
= &ptq
->pt
->queues
.queue_array
[ptq
->queue_nr
];
242 buffer
= auxtrace_buffer__next(queue
, buffer
);
245 auxtrace_buffer__drop_data(old_buffer
);
250 ptq
->buffer
= buffer
;
253 int fd
= perf_data_file__fd(ptq
->pt
->session
->file
);
255 buffer
->data
= auxtrace_buffer__get_data(buffer
, fd
);
260 if (ptq
->pt
->snapshot_mode
&& !buffer
->consecutive
&& old_buffer
&&
261 intel_pt_do_fix_overlap(ptq
->pt
, old_buffer
, buffer
))
265 auxtrace_buffer__drop_data(old_buffer
);
267 if (buffer
->use_data
) {
268 b
->len
= buffer
->use_size
;
269 b
->buf
= buffer
->use_data
;
271 b
->len
= buffer
->size
;
272 b
->buf
= buffer
->data
;
274 b
->ref_timestamp
= buffer
->reference
;
276 if (!old_buffer
|| ptq
->pt
->sampling_mode
|| (ptq
->pt
->snapshot_mode
&&
277 !buffer
->consecutive
)) {
278 b
->consecutive
= false;
279 b
->trace_nr
= buffer
->buffer_nr
+ 1;
281 b
->consecutive
= true;
284 if (ptq
->use_buffer_pid_tid
&& (ptq
->pid
!= buffer
->pid
||
285 ptq
->tid
!= buffer
->tid
))
286 intel_pt_use_buffer_pid_tid(ptq
, queue
, buffer
);
288 if (ptq
->step_through_buffers
)
292 return intel_pt_get_trace(b
, data
);
297 struct intel_pt_cache_entry
{
298 struct auxtrace_cache_entry entry
;
301 enum intel_pt_insn_op op
;
302 enum intel_pt_insn_branch branch
;
307 static int intel_pt_config_div(const char *var
, const char *value
, void *data
)
312 if (!strcmp(var
, "intel-pt.cache-divisor")) {
313 val
= strtol(value
, NULL
, 0);
314 if (val
> 0 && val
<= INT_MAX
)
321 static int intel_pt_cache_divisor(void)
328 perf_config(intel_pt_config_div
, &d
);
336 static unsigned int intel_pt_cache_size(struct dso
*dso
,
337 struct machine
*machine
)
341 size
= dso__data_size(dso
, machine
);
342 size
/= intel_pt_cache_divisor();
345 if (size
> (1 << 21))
347 return 32 - __builtin_clz(size
);
350 static struct auxtrace_cache
*intel_pt_cache(struct dso
*dso
,
351 struct machine
*machine
)
353 struct auxtrace_cache
*c
;
356 if (dso
->auxtrace_cache
)
357 return dso
->auxtrace_cache
;
359 bits
= intel_pt_cache_size(dso
, machine
);
361 /* Ignoring cache creation failure */
362 c
= auxtrace_cache__new(bits
, sizeof(struct intel_pt_cache_entry
), 200);
364 dso
->auxtrace_cache
= c
;
369 static int intel_pt_cache_add(struct dso
*dso
, struct machine
*machine
,
370 u64 offset
, u64 insn_cnt
, u64 byte_cnt
,
371 struct intel_pt_insn
*intel_pt_insn
)
373 struct auxtrace_cache
*c
= intel_pt_cache(dso
, machine
);
374 struct intel_pt_cache_entry
*e
;
380 e
= auxtrace_cache__alloc_entry(c
);
384 e
->insn_cnt
= insn_cnt
;
385 e
->byte_cnt
= byte_cnt
;
386 e
->op
= intel_pt_insn
->op
;
387 e
->branch
= intel_pt_insn
->branch
;
388 e
->length
= intel_pt_insn
->length
;
389 e
->rel
= intel_pt_insn
->rel
;
391 err
= auxtrace_cache__add(c
, offset
, &e
->entry
);
393 auxtrace_cache__free_entry(c
, e
);
398 static struct intel_pt_cache_entry
*
399 intel_pt_cache_lookup(struct dso
*dso
, struct machine
*machine
, u64 offset
)
401 struct auxtrace_cache
*c
= intel_pt_cache(dso
, machine
);
406 return auxtrace_cache__lookup(dso
->auxtrace_cache
, offset
);
409 static int intel_pt_walk_next_insn(struct intel_pt_insn
*intel_pt_insn
,
410 uint64_t *insn_cnt_ptr
, uint64_t *ip
,
411 uint64_t to_ip
, uint64_t max_insn_cnt
,
414 struct intel_pt_queue
*ptq
= data
;
415 struct machine
*machine
= ptq
->pt
->machine
;
416 struct thread
*thread
;
417 struct addr_location al
;
418 unsigned char buf
[1024];
423 u64 offset
, start_offset
, start_ip
;
427 if (to_ip
&& *ip
== to_ip
)
430 bufsz
= intel_pt_insn_max_size();
432 if (*ip
>= ptq
->pt
->kernel_start
)
433 cpumode
= PERF_RECORD_MISC_KERNEL
;
435 cpumode
= PERF_RECORD_MISC_USER
;
437 thread
= ptq
->thread
;
439 if (cpumode
!= PERF_RECORD_MISC_KERNEL
)
441 thread
= ptq
->pt
->unknown_thread
;
445 thread__find_addr_map(thread
, cpumode
, MAP__FUNCTION
, *ip
, &al
);
446 if (!al
.map
|| !al
.map
->dso
)
449 if (al
.map
->dso
->data
.status
== DSO_DATA_STATUS_ERROR
&&
450 dso__data_status_seen(al
.map
->dso
,
451 DSO_DATA_STATUS_SEEN_ITRACE
))
454 offset
= al
.map
->map_ip(al
.map
, *ip
);
456 if (!to_ip
&& one_map
) {
457 struct intel_pt_cache_entry
*e
;
459 e
= intel_pt_cache_lookup(al
.map
->dso
, machine
, offset
);
461 (!max_insn_cnt
|| e
->insn_cnt
<= max_insn_cnt
)) {
462 *insn_cnt_ptr
= e
->insn_cnt
;
464 intel_pt_insn
->op
= e
->op
;
465 intel_pt_insn
->branch
= e
->branch
;
466 intel_pt_insn
->length
= e
->length
;
467 intel_pt_insn
->rel
= e
->rel
;
468 intel_pt_log_insn_no_data(intel_pt_insn
, *ip
);
473 start_offset
= offset
;
476 /* Load maps to ensure dso->is_64_bit has been updated */
477 map__load(al
.map
, machine
->symbol_filter
);
479 x86_64
= al
.map
->dso
->is_64_bit
;
482 len
= dso__data_read_offset(al
.map
->dso
, machine
,
487 if (intel_pt_get_insn(buf
, len
, x86_64
, intel_pt_insn
))
490 intel_pt_log_insn(intel_pt_insn
, *ip
);
494 if (intel_pt_insn
->branch
!= INTEL_PT_BR_NO_BRANCH
)
497 if (max_insn_cnt
&& insn_cnt
>= max_insn_cnt
)
500 *ip
+= intel_pt_insn
->length
;
502 if (to_ip
&& *ip
== to_ip
)
505 if (*ip
>= al
.map
->end
)
508 offset
+= intel_pt_insn
->length
;
513 *insn_cnt_ptr
= insn_cnt
;
519 * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
523 struct intel_pt_cache_entry
*e
;
525 e
= intel_pt_cache_lookup(al
.map
->dso
, machine
, start_offset
);
530 /* Ignore cache errors */
531 intel_pt_cache_add(al
.map
->dso
, machine
, start_offset
, insn_cnt
,
532 *ip
- start_ip
, intel_pt_insn
);
537 *insn_cnt_ptr
= insn_cnt
;
541 static bool intel_pt_get_config(struct intel_pt
*pt
,
542 struct perf_event_attr
*attr
, u64
*config
)
544 if (attr
->type
== pt
->pmu_type
) {
546 *config
= attr
->config
;
553 static bool intel_pt_exclude_kernel(struct intel_pt
*pt
)
555 struct perf_evsel
*evsel
;
557 evlist__for_each(pt
->session
->evlist
, evsel
) {
558 if (intel_pt_get_config(pt
, &evsel
->attr
, NULL
) &&
559 !evsel
->attr
.exclude_kernel
)
565 static bool intel_pt_return_compression(struct intel_pt
*pt
)
567 struct perf_evsel
*evsel
;
570 if (!pt
->noretcomp_bit
)
573 evlist__for_each(pt
->session
->evlist
, evsel
) {
574 if (intel_pt_get_config(pt
, &evsel
->attr
, &config
) &&
575 (config
& pt
->noretcomp_bit
))
581 static unsigned int intel_pt_mtc_period(struct intel_pt
*pt
)
583 struct perf_evsel
*evsel
;
587 if (!pt
->mtc_freq_bits
)
590 for (shift
= 0, config
= pt
->mtc_freq_bits
; !(config
& 1); shift
++)
593 evlist__for_each(pt
->session
->evlist
, evsel
) {
594 if (intel_pt_get_config(pt
, &evsel
->attr
, &config
))
595 return (config
& pt
->mtc_freq_bits
) >> shift
;
600 static bool intel_pt_timeless_decoding(struct intel_pt
*pt
)
602 struct perf_evsel
*evsel
;
603 bool timeless_decoding
= true;
606 if (!pt
->tsc_bit
|| !pt
->cap_user_time_zero
)
609 evlist__for_each(pt
->session
->evlist
, evsel
) {
610 if (!(evsel
->attr
.sample_type
& PERF_SAMPLE_TIME
))
612 if (intel_pt_get_config(pt
, &evsel
->attr
, &config
)) {
613 if (config
& pt
->tsc_bit
)
614 timeless_decoding
= false;
619 return timeless_decoding
;
622 static bool intel_pt_tracing_kernel(struct intel_pt
*pt
)
624 struct perf_evsel
*evsel
;
626 evlist__for_each(pt
->session
->evlist
, evsel
) {
627 if (intel_pt_get_config(pt
, &evsel
->attr
, NULL
) &&
628 !evsel
->attr
.exclude_kernel
)
634 static bool intel_pt_have_tsc(struct intel_pt
*pt
)
636 struct perf_evsel
*evsel
;
637 bool have_tsc
= false;
643 evlist__for_each(pt
->session
->evlist
, evsel
) {
644 if (intel_pt_get_config(pt
, &evsel
->attr
, &config
)) {
645 if (config
& pt
->tsc_bit
)
654 static u64
intel_pt_ns_to_ticks(const struct intel_pt
*pt
, u64 ns
)
658 quot
= ns
/ pt
->tc
.time_mult
;
659 rem
= ns
% pt
->tc
.time_mult
;
660 return (quot
<< pt
->tc
.time_shift
) + (rem
<< pt
->tc
.time_shift
) /
664 static struct intel_pt_queue
*intel_pt_alloc_queue(struct intel_pt
*pt
,
665 unsigned int queue_nr
)
667 struct intel_pt_params params
= { .get_trace
= 0, };
668 struct intel_pt_queue
*ptq
;
670 ptq
= zalloc(sizeof(struct intel_pt_queue
));
674 if (pt
->synth_opts
.callchain
) {
675 size_t sz
= sizeof(struct ip_callchain
);
677 sz
+= pt
->synth_opts
.callchain_sz
* sizeof(u64
);
678 ptq
->chain
= zalloc(sz
);
683 if (pt
->synth_opts
.last_branch
) {
684 size_t sz
= sizeof(struct branch_stack
);
686 sz
+= pt
->synth_opts
.last_branch_sz
*
687 sizeof(struct branch_entry
);
688 ptq
->last_branch
= zalloc(sz
);
689 if (!ptq
->last_branch
)
691 ptq
->last_branch_rb
= zalloc(sz
);
692 if (!ptq
->last_branch_rb
)
696 ptq
->event_buf
= malloc(PERF_SAMPLE_MAX_SIZE
);
701 ptq
->queue_nr
= queue_nr
;
702 ptq
->exclude_kernel
= intel_pt_exclude_kernel(pt
);
708 params
.get_trace
= intel_pt_get_trace
;
709 params
.walk_insn
= intel_pt_walk_next_insn
;
711 params
.return_compression
= intel_pt_return_compression(pt
);
712 params
.max_non_turbo_ratio
= pt
->max_non_turbo_ratio
;
713 params
.mtc_period
= intel_pt_mtc_period(pt
);
714 params
.tsc_ctc_ratio_n
= pt
->tsc_ctc_ratio_n
;
715 params
.tsc_ctc_ratio_d
= pt
->tsc_ctc_ratio_d
;
717 if (pt
->synth_opts
.instructions
) {
718 if (pt
->synth_opts
.period
) {
719 switch (pt
->synth_opts
.period_type
) {
720 case PERF_ITRACE_PERIOD_INSTRUCTIONS
:
722 INTEL_PT_PERIOD_INSTRUCTIONS
;
723 params
.period
= pt
->synth_opts
.period
;
725 case PERF_ITRACE_PERIOD_TICKS
:
726 params
.period_type
= INTEL_PT_PERIOD_TICKS
;
727 params
.period
= pt
->synth_opts
.period
;
729 case PERF_ITRACE_PERIOD_NANOSECS
:
730 params
.period_type
= INTEL_PT_PERIOD_TICKS
;
731 params
.period
= intel_pt_ns_to_ticks(pt
,
732 pt
->synth_opts
.period
);
739 if (!params
.period
) {
740 params
.period_type
= INTEL_PT_PERIOD_INSTRUCTIONS
;
745 ptq
->decoder
= intel_pt_decoder_new(¶ms
);
752 zfree(&ptq
->event_buf
);
753 zfree(&ptq
->last_branch
);
754 zfree(&ptq
->last_branch_rb
);
760 static void intel_pt_free_queue(void *priv
)
762 struct intel_pt_queue
*ptq
= priv
;
766 thread__zput(ptq
->thread
);
767 intel_pt_decoder_free(ptq
->decoder
);
768 zfree(&ptq
->event_buf
);
769 zfree(&ptq
->last_branch
);
770 zfree(&ptq
->last_branch_rb
);
775 static void intel_pt_set_pid_tid_cpu(struct intel_pt
*pt
,
776 struct auxtrace_queue
*queue
)
778 struct intel_pt_queue
*ptq
= queue
->priv
;
780 if (queue
->tid
== -1 || pt
->have_sched_switch
) {
781 ptq
->tid
= machine__get_current_tid(pt
->machine
, ptq
->cpu
);
782 thread__zput(ptq
->thread
);
785 if (!ptq
->thread
&& ptq
->tid
!= -1)
786 ptq
->thread
= machine__find_thread(pt
->machine
, -1, ptq
->tid
);
789 ptq
->pid
= ptq
->thread
->pid_
;
790 if (queue
->cpu
== -1)
791 ptq
->cpu
= ptq
->thread
->cpu
;
795 static void intel_pt_sample_flags(struct intel_pt_queue
*ptq
)
797 if (ptq
->state
->flags
& INTEL_PT_ABORT_TX
) {
798 ptq
->flags
= PERF_IP_FLAG_BRANCH
| PERF_IP_FLAG_TX_ABORT
;
799 } else if (ptq
->state
->flags
& INTEL_PT_ASYNC
) {
800 if (ptq
->state
->to_ip
)
801 ptq
->flags
= PERF_IP_FLAG_BRANCH
| PERF_IP_FLAG_CALL
|
803 PERF_IP_FLAG_INTERRUPT
;
805 ptq
->flags
= PERF_IP_FLAG_BRANCH
|
806 PERF_IP_FLAG_TRACE_END
;
809 if (ptq
->state
->from_ip
)
810 ptq
->flags
= intel_pt_insn_type(ptq
->state
->insn_op
);
812 ptq
->flags
= PERF_IP_FLAG_BRANCH
|
813 PERF_IP_FLAG_TRACE_BEGIN
;
814 if (ptq
->state
->flags
& INTEL_PT_IN_TX
)
815 ptq
->flags
|= PERF_IP_FLAG_IN_TX
;
816 ptq
->insn_len
= ptq
->state
->insn_len
;
820 static int intel_pt_setup_queue(struct intel_pt
*pt
,
821 struct auxtrace_queue
*queue
,
822 unsigned int queue_nr
)
824 struct intel_pt_queue
*ptq
= queue
->priv
;
826 if (list_empty(&queue
->head
))
830 ptq
= intel_pt_alloc_queue(pt
, queue_nr
);
835 if (queue
->cpu
!= -1)
836 ptq
->cpu
= queue
->cpu
;
837 ptq
->tid
= queue
->tid
;
839 if (pt
->sampling_mode
) {
840 if (pt
->timeless_decoding
)
841 ptq
->step_through_buffers
= true;
842 if (pt
->timeless_decoding
|| !pt
->have_sched_switch
)
843 ptq
->use_buffer_pid_tid
= true;
849 ptq
->switch_state
!= INTEL_PT_SS_EXPECTING_SWITCH_EVENT
)) {
850 const struct intel_pt_state
*state
;
853 if (pt
->timeless_decoding
)
856 intel_pt_log("queue %u getting timestamp\n", queue_nr
);
857 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
858 queue_nr
, ptq
->cpu
, ptq
->pid
, ptq
->tid
);
860 state
= intel_pt_decode(ptq
->decoder
);
862 if (state
->err
== INTEL_PT_ERR_NODATA
) {
863 intel_pt_log("queue %u has no timestamp\n",
869 if (state
->timestamp
)
873 ptq
->timestamp
= state
->timestamp
;
874 intel_pt_log("queue %u timestamp 0x%" PRIx64
"\n",
875 queue_nr
, ptq
->timestamp
);
877 ptq
->have_sample
= true;
878 intel_pt_sample_flags(ptq
);
879 ret
= auxtrace_heap__add(&pt
->heap
, queue_nr
, ptq
->timestamp
);
888 static int intel_pt_setup_queues(struct intel_pt
*pt
)
893 for (i
= 0; i
< pt
->queues
.nr_queues
; i
++) {
894 ret
= intel_pt_setup_queue(pt
, &pt
->queues
.queue_array
[i
], i
);
901 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue
*ptq
)
903 struct branch_stack
*bs_src
= ptq
->last_branch_rb
;
904 struct branch_stack
*bs_dst
= ptq
->last_branch
;
907 bs_dst
->nr
= bs_src
->nr
;
912 nr
= ptq
->pt
->synth_opts
.last_branch_sz
- ptq
->last_branch_pos
;
913 memcpy(&bs_dst
->entries
[0],
914 &bs_src
->entries
[ptq
->last_branch_pos
],
915 sizeof(struct branch_entry
) * nr
);
917 if (bs_src
->nr
>= ptq
->pt
->synth_opts
.last_branch_sz
) {
918 memcpy(&bs_dst
->entries
[nr
],
920 sizeof(struct branch_entry
) * ptq
->last_branch_pos
);
924 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue
*ptq
)
926 ptq
->last_branch_pos
= 0;
927 ptq
->last_branch_rb
->nr
= 0;
930 static void intel_pt_update_last_branch_rb(struct intel_pt_queue
*ptq
)
932 const struct intel_pt_state
*state
= ptq
->state
;
933 struct branch_stack
*bs
= ptq
->last_branch_rb
;
934 struct branch_entry
*be
;
936 if (!ptq
->last_branch_pos
)
937 ptq
->last_branch_pos
= ptq
->pt
->synth_opts
.last_branch_sz
;
939 ptq
->last_branch_pos
-= 1;
941 be
= &bs
->entries
[ptq
->last_branch_pos
];
942 be
->from
= state
->from_ip
;
943 be
->to
= state
->to_ip
;
944 be
->flags
.abort
= !!(state
->flags
& INTEL_PT_ABORT_TX
);
945 be
->flags
.in_tx
= !!(state
->flags
& INTEL_PT_IN_TX
);
946 /* No support for mispredict */
947 be
->flags
.mispred
= ptq
->pt
->mispred_all
;
949 if (bs
->nr
< ptq
->pt
->synth_opts
.last_branch_sz
)
953 static int intel_pt_inject_event(union perf_event
*event
,
954 struct perf_sample
*sample
, u64 type
,
957 event
->header
.size
= perf_event__sample_event_size(sample
, type
, 0);
958 return perf_event__synthesize_sample(event
, type
, 0, sample
, swapped
);
961 static int intel_pt_synth_branch_sample(struct intel_pt_queue
*ptq
)
964 struct intel_pt
*pt
= ptq
->pt
;
965 union perf_event
*event
= ptq
->event_buf
;
966 struct perf_sample sample
= { .ip
= 0, };
967 struct dummy_branch_stack
{
969 struct branch_entry entries
;
972 if (pt
->branches_filter
&& !(pt
->branches_filter
& ptq
->flags
))
975 event
->sample
.header
.type
= PERF_RECORD_SAMPLE
;
976 event
->sample
.header
.misc
= PERF_RECORD_MISC_USER
;
977 event
->sample
.header
.size
= sizeof(struct perf_event_header
);
979 if (!pt
->timeless_decoding
)
980 sample
.time
= tsc_to_perf_time(ptq
->timestamp
, &pt
->tc
);
982 sample
.cpumode
= PERF_RECORD_MISC_USER
;
983 sample
.ip
= ptq
->state
->from_ip
;
984 sample
.pid
= ptq
->pid
;
985 sample
.tid
= ptq
->tid
;
986 sample
.addr
= ptq
->state
->to_ip
;
987 sample
.id
= ptq
->pt
->branches_id
;
988 sample
.stream_id
= ptq
->pt
->branches_id
;
990 sample
.cpu
= ptq
->cpu
;
991 sample
.flags
= ptq
->flags
;
992 sample
.insn_len
= ptq
->insn_len
;
995 * perf report cannot handle events without a branch stack when using
996 * SORT_MODE__BRANCH so make a dummy one.
998 if (pt
->synth_opts
.last_branch
&& sort__mode
== SORT_MODE__BRANCH
) {
999 dummy_bs
= (struct dummy_branch_stack
){
1006 sample
.branch_stack
= (struct branch_stack
*)&dummy_bs
;
1009 if (pt
->synth_opts
.inject
) {
1010 ret
= intel_pt_inject_event(event
, &sample
,
1011 pt
->branches_sample_type
,
1012 pt
->synth_needs_swap
);
1017 ret
= perf_session__deliver_synth_event(pt
->session
, event
, &sample
);
1019 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1025 static int intel_pt_synth_instruction_sample(struct intel_pt_queue
*ptq
)
1028 struct intel_pt
*pt
= ptq
->pt
;
1029 union perf_event
*event
= ptq
->event_buf
;
1030 struct perf_sample sample
= { .ip
= 0, };
1032 event
->sample
.header
.type
= PERF_RECORD_SAMPLE
;
1033 event
->sample
.header
.misc
= PERF_RECORD_MISC_USER
;
1034 event
->sample
.header
.size
= sizeof(struct perf_event_header
);
1036 if (!pt
->timeless_decoding
)
1037 sample
.time
= tsc_to_perf_time(ptq
->timestamp
, &pt
->tc
);
1039 sample
.cpumode
= PERF_RECORD_MISC_USER
;
1040 sample
.ip
= ptq
->state
->from_ip
;
1041 sample
.pid
= ptq
->pid
;
1042 sample
.tid
= ptq
->tid
;
1043 sample
.addr
= ptq
->state
->to_ip
;
1044 sample
.id
= ptq
->pt
->instructions_id
;
1045 sample
.stream_id
= ptq
->pt
->instructions_id
;
1046 sample
.period
= ptq
->state
->tot_insn_cnt
- ptq
->last_insn_cnt
;
1047 sample
.cpu
= ptq
->cpu
;
1048 sample
.flags
= ptq
->flags
;
1049 sample
.insn_len
= ptq
->insn_len
;
1051 ptq
->last_insn_cnt
= ptq
->state
->tot_insn_cnt
;
1053 if (pt
->synth_opts
.callchain
) {
1054 thread_stack__sample(ptq
->thread
, ptq
->chain
,
1055 pt
->synth_opts
.callchain_sz
, sample
.ip
);
1056 sample
.callchain
= ptq
->chain
;
1059 if (pt
->synth_opts
.last_branch
) {
1060 intel_pt_copy_last_branch_rb(ptq
);
1061 sample
.branch_stack
= ptq
->last_branch
;
1064 if (pt
->synth_opts
.inject
) {
1065 ret
= intel_pt_inject_event(event
, &sample
,
1066 pt
->instructions_sample_type
,
1067 pt
->synth_needs_swap
);
1072 ret
= perf_session__deliver_synth_event(pt
->session
, event
, &sample
);
1074 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1077 if (pt
->synth_opts
.last_branch
)
1078 intel_pt_reset_last_branch_rb(ptq
);
1083 static int intel_pt_synth_transaction_sample(struct intel_pt_queue
*ptq
)
1086 struct intel_pt
*pt
= ptq
->pt
;
1087 union perf_event
*event
= ptq
->event_buf
;
1088 struct perf_sample sample
= { .ip
= 0, };
1090 event
->sample
.header
.type
= PERF_RECORD_SAMPLE
;
1091 event
->sample
.header
.misc
= PERF_RECORD_MISC_USER
;
1092 event
->sample
.header
.size
= sizeof(struct perf_event_header
);
1094 if (!pt
->timeless_decoding
)
1095 sample
.time
= tsc_to_perf_time(ptq
->timestamp
, &pt
->tc
);
1097 sample
.cpumode
= PERF_RECORD_MISC_USER
;
1098 sample
.ip
= ptq
->state
->from_ip
;
1099 sample
.pid
= ptq
->pid
;
1100 sample
.tid
= ptq
->tid
;
1101 sample
.addr
= ptq
->state
->to_ip
;
1102 sample
.id
= ptq
->pt
->transactions_id
;
1103 sample
.stream_id
= ptq
->pt
->transactions_id
;
1105 sample
.cpu
= ptq
->cpu
;
1106 sample
.flags
= ptq
->flags
;
1107 sample
.insn_len
= ptq
->insn_len
;
1109 if (pt
->synth_opts
.callchain
) {
1110 thread_stack__sample(ptq
->thread
, ptq
->chain
,
1111 pt
->synth_opts
.callchain_sz
, sample
.ip
);
1112 sample
.callchain
= ptq
->chain
;
1115 if (pt
->synth_opts
.last_branch
) {
1116 intel_pt_copy_last_branch_rb(ptq
);
1117 sample
.branch_stack
= ptq
->last_branch
;
1120 if (pt
->synth_opts
.inject
) {
1121 ret
= intel_pt_inject_event(event
, &sample
,
1122 pt
->transactions_sample_type
,
1123 pt
->synth_needs_swap
);
1128 ret
= perf_session__deliver_synth_event(pt
->session
, event
, &sample
);
1130 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1133 if (pt
->synth_opts
.last_branch
)
1134 intel_pt_reset_last_branch_rb(ptq
);
1139 static int intel_pt_synth_error(struct intel_pt
*pt
, int code
, int cpu
,
1140 pid_t pid
, pid_t tid
, u64 ip
)
1142 union perf_event event
;
1143 char msg
[MAX_AUXTRACE_ERROR_MSG
];
1146 intel_pt__strerror(code
, msg
, MAX_AUXTRACE_ERROR_MSG
);
1148 auxtrace_synth_error(&event
.auxtrace_error
, PERF_AUXTRACE_ERROR_ITRACE
,
1149 code
, cpu
, pid
, tid
, ip
, msg
);
1151 err
= perf_session__deliver_synth_event(pt
->session
, &event
, NULL
);
1153 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1159 static int intel_pt_next_tid(struct intel_pt
*pt
, struct intel_pt_queue
*ptq
)
1161 struct auxtrace_queue
*queue
;
1162 pid_t tid
= ptq
->next_tid
;
1168 intel_pt_log("switch: cpu %d tid %d\n", ptq
->cpu
, tid
);
1170 err
= machine__set_current_tid(pt
->machine
, ptq
->cpu
, -1, tid
);
1172 queue
= &pt
->queues
.queue_array
[ptq
->queue_nr
];
1173 intel_pt_set_pid_tid_cpu(pt
, queue
);
1180 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue
*ptq
, u64 ip
)
1182 struct intel_pt
*pt
= ptq
->pt
;
1184 return ip
== pt
->switch_ip
&&
1185 (ptq
->flags
& PERF_IP_FLAG_BRANCH
) &&
1186 !(ptq
->flags
& (PERF_IP_FLAG_CONDITIONAL
| PERF_IP_FLAG_ASYNC
|
1187 PERF_IP_FLAG_INTERRUPT
| PERF_IP_FLAG_TX_ABORT
));
1190 static int intel_pt_sample(struct intel_pt_queue
*ptq
)
1192 const struct intel_pt_state
*state
= ptq
->state
;
1193 struct intel_pt
*pt
= ptq
->pt
;
1196 if (!ptq
->have_sample
)
1199 ptq
->have_sample
= false;
1201 if (pt
->sample_instructions
&&
1202 (state
->type
& INTEL_PT_INSTRUCTION
)) {
1203 err
= intel_pt_synth_instruction_sample(ptq
);
1208 if (pt
->sample_transactions
&&
1209 (state
->type
& INTEL_PT_TRANSACTION
)) {
1210 err
= intel_pt_synth_transaction_sample(ptq
);
1215 if (!(state
->type
& INTEL_PT_BRANCH
))
1218 if (pt
->synth_opts
.callchain
)
1219 thread_stack__event(ptq
->thread
, ptq
->flags
, state
->from_ip
,
1220 state
->to_ip
, ptq
->insn_len
,
1223 thread_stack__set_trace_nr(ptq
->thread
, state
->trace_nr
);
1225 if (pt
->sample_branches
) {
1226 err
= intel_pt_synth_branch_sample(ptq
);
1231 if (pt
->synth_opts
.last_branch
)
1232 intel_pt_update_last_branch_rb(ptq
);
1234 if (!pt
->sync_switch
)
1237 if (intel_pt_is_switch_ip(ptq
, state
->to_ip
)) {
1238 switch (ptq
->switch_state
) {
1239 case INTEL_PT_SS_UNKNOWN
:
1240 case INTEL_PT_SS_EXPECTING_SWITCH_IP
:
1241 err
= intel_pt_next_tid(pt
, ptq
);
1244 ptq
->switch_state
= INTEL_PT_SS_TRACING
;
1247 ptq
->switch_state
= INTEL_PT_SS_EXPECTING_SWITCH_EVENT
;
1250 } else if (!state
->to_ip
) {
1251 ptq
->switch_state
= INTEL_PT_SS_NOT_TRACING
;
1252 } else if (ptq
->switch_state
== INTEL_PT_SS_NOT_TRACING
) {
1253 ptq
->switch_state
= INTEL_PT_SS_UNKNOWN
;
1254 } else if (ptq
->switch_state
== INTEL_PT_SS_UNKNOWN
&&
1255 state
->to_ip
== pt
->ptss_ip
&&
1256 (ptq
->flags
& PERF_IP_FLAG_CALL
)) {
1257 ptq
->switch_state
= INTEL_PT_SS_TRACING
;
1263 static u64
intel_pt_switch_ip(struct intel_pt
*pt
, u64
*ptss_ip
)
1265 struct machine
*machine
= pt
->machine
;
1267 struct symbol
*sym
, *start
;
1268 u64 ip
, switch_ip
= 0;
1274 map
= machine__kernel_map(machine
);
1278 if (map__load(map
, machine
->symbol_filter
))
1281 start
= dso__first_symbol(map
->dso
, MAP__FUNCTION
);
1283 for (sym
= start
; sym
; sym
= dso__next_symbol(sym
)) {
1284 if (sym
->binding
== STB_GLOBAL
&&
1285 !strcmp(sym
->name
, "__switch_to")) {
1286 ip
= map
->unmap_ip(map
, sym
->start
);
1287 if (ip
>= map
->start
&& ip
< map
->end
) {
1294 if (!switch_ip
|| !ptss_ip
)
1297 if (pt
->have_sched_switch
== 1)
1298 ptss
= "perf_trace_sched_switch";
1300 ptss
= "__perf_event_task_sched_out";
1302 for (sym
= start
; sym
; sym
= dso__next_symbol(sym
)) {
1303 if (!strcmp(sym
->name
, ptss
)) {
1304 ip
= map
->unmap_ip(map
, sym
->start
);
1305 if (ip
>= map
->start
&& ip
< map
->end
) {
1315 static int intel_pt_run_decoder(struct intel_pt_queue
*ptq
, u64
*timestamp
)
1317 const struct intel_pt_state
*state
= ptq
->state
;
1318 struct intel_pt
*pt
= ptq
->pt
;
1321 if (!pt
->kernel_start
) {
1322 pt
->kernel_start
= machine__kernel_start(pt
->machine
);
1323 if (pt
->per_cpu_mmaps
&&
1324 (pt
->have_sched_switch
== 1 || pt
->have_sched_switch
== 3) &&
1325 !pt
->timeless_decoding
&& intel_pt_tracing_kernel(pt
) &&
1326 !pt
->sampling_mode
) {
1327 pt
->switch_ip
= intel_pt_switch_ip(pt
, &pt
->ptss_ip
);
1328 if (pt
->switch_ip
) {
1329 intel_pt_log("switch_ip: %"PRIx64
" ptss_ip: %"PRIx64
"\n",
1330 pt
->switch_ip
, pt
->ptss_ip
);
1331 pt
->sync_switch
= true;
1336 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1337 ptq
->queue_nr
, ptq
->cpu
, ptq
->pid
, ptq
->tid
);
1339 err
= intel_pt_sample(ptq
);
1343 state
= intel_pt_decode(ptq
->decoder
);
1345 if (state
->err
== INTEL_PT_ERR_NODATA
)
1347 if (pt
->sync_switch
&&
1348 state
->from_ip
>= pt
->kernel_start
) {
1349 pt
->sync_switch
= false;
1350 intel_pt_next_tid(pt
, ptq
);
1352 if (pt
->synth_opts
.errors
) {
1353 err
= intel_pt_synth_error(pt
, state
->err
,
1364 ptq
->have_sample
= true;
1365 intel_pt_sample_flags(ptq
);
1367 /* Use estimated TSC upon return to user space */
1369 (state
->from_ip
>= pt
->kernel_start
|| !state
->from_ip
) &&
1370 state
->to_ip
&& state
->to_ip
< pt
->kernel_start
) {
1371 intel_pt_log("TSC %"PRIx64
" est. TSC %"PRIx64
"\n",
1372 state
->timestamp
, state
->est_timestamp
);
1373 ptq
->timestamp
= state
->est_timestamp
;
1374 /* Use estimated TSC in unknown switch state */
1375 } else if (pt
->sync_switch
&&
1376 ptq
->switch_state
== INTEL_PT_SS_UNKNOWN
&&
1377 intel_pt_is_switch_ip(ptq
, state
->to_ip
) &&
1378 ptq
->next_tid
== -1) {
1379 intel_pt_log("TSC %"PRIx64
" est. TSC %"PRIx64
"\n",
1380 state
->timestamp
, state
->est_timestamp
);
1381 ptq
->timestamp
= state
->est_timestamp
;
1382 } else if (state
->timestamp
> ptq
->timestamp
) {
1383 ptq
->timestamp
= state
->timestamp
;
1386 if (!pt
->timeless_decoding
&& ptq
->timestamp
>= *timestamp
) {
1387 *timestamp
= ptq
->timestamp
;
1394 static inline int intel_pt_update_queues(struct intel_pt
*pt
)
1396 if (pt
->queues
.new_data
) {
1397 pt
->queues
.new_data
= false;
1398 return intel_pt_setup_queues(pt
);
1403 static int intel_pt_process_queues(struct intel_pt
*pt
, u64 timestamp
)
1405 unsigned int queue_nr
;
1410 struct auxtrace_queue
*queue
;
1411 struct intel_pt_queue
*ptq
;
1413 if (!pt
->heap
.heap_cnt
)
1416 if (pt
->heap
.heap_array
[0].ordinal
>= timestamp
)
1419 queue_nr
= pt
->heap
.heap_array
[0].queue_nr
;
1420 queue
= &pt
->queues
.queue_array
[queue_nr
];
1423 intel_pt_log("queue %u processing 0x%" PRIx64
" to 0x%" PRIx64
"\n",
1424 queue_nr
, pt
->heap
.heap_array
[0].ordinal
,
1427 auxtrace_heap__pop(&pt
->heap
);
1429 if (pt
->heap
.heap_cnt
) {
1430 ts
= pt
->heap
.heap_array
[0].ordinal
+ 1;
1437 intel_pt_set_pid_tid_cpu(pt
, queue
);
1439 ret
= intel_pt_run_decoder(ptq
, &ts
);
1442 auxtrace_heap__add(&pt
->heap
, queue_nr
, ts
);
1447 ret
= auxtrace_heap__add(&pt
->heap
, queue_nr
, ts
);
1451 ptq
->on_heap
= false;
1458 static int intel_pt_process_timeless_queues(struct intel_pt
*pt
, pid_t tid
,
1461 struct auxtrace_queues
*queues
= &pt
->queues
;
1465 for (i
= 0; i
< queues
->nr_queues
; i
++) {
1466 struct auxtrace_queue
*queue
= &pt
->queues
.queue_array
[i
];
1467 struct intel_pt_queue
*ptq
= queue
->priv
;
1469 if (ptq
&& (tid
== -1 || ptq
->tid
== tid
)) {
1471 intel_pt_set_pid_tid_cpu(pt
, queue
);
1472 intel_pt_run_decoder(ptq
, &ts
);
1478 static int intel_pt_lost(struct intel_pt
*pt
, struct perf_sample
*sample
)
1480 return intel_pt_synth_error(pt
, INTEL_PT_ERR_LOST
, sample
->cpu
,
1481 sample
->pid
, sample
->tid
, 0);
1484 static struct intel_pt_queue
*intel_pt_cpu_to_ptq(struct intel_pt
*pt
, int cpu
)
1488 if (cpu
< 0 || !pt
->queues
.nr_queues
)
1491 if ((unsigned)cpu
>= pt
->queues
.nr_queues
)
1492 i
= pt
->queues
.nr_queues
- 1;
1496 if (pt
->queues
.queue_array
[i
].cpu
== cpu
)
1497 return pt
->queues
.queue_array
[i
].priv
;
1499 for (j
= 0; i
> 0; j
++) {
1500 if (pt
->queues
.queue_array
[--i
].cpu
== cpu
)
1501 return pt
->queues
.queue_array
[i
].priv
;
1504 for (; j
< pt
->queues
.nr_queues
; j
++) {
1505 if (pt
->queues
.queue_array
[j
].cpu
== cpu
)
1506 return pt
->queues
.queue_array
[j
].priv
;
1512 static int intel_pt_sync_switch(struct intel_pt
*pt
, int cpu
, pid_t tid
,
1515 struct intel_pt_queue
*ptq
;
1518 if (!pt
->sync_switch
)
1521 ptq
= intel_pt_cpu_to_ptq(pt
, cpu
);
1525 switch (ptq
->switch_state
) {
1526 case INTEL_PT_SS_NOT_TRACING
:
1529 case INTEL_PT_SS_UNKNOWN
:
1530 case INTEL_PT_SS_TRACING
:
1531 ptq
->next_tid
= tid
;
1532 ptq
->switch_state
= INTEL_PT_SS_EXPECTING_SWITCH_IP
;
1534 case INTEL_PT_SS_EXPECTING_SWITCH_EVENT
:
1535 if (!ptq
->on_heap
) {
1536 ptq
->timestamp
= perf_time_to_tsc(timestamp
,
1538 err
= auxtrace_heap__add(&pt
->heap
, ptq
->queue_nr
,
1542 ptq
->on_heap
= true;
1544 ptq
->switch_state
= INTEL_PT_SS_TRACING
;
1546 case INTEL_PT_SS_EXPECTING_SWITCH_IP
:
1547 ptq
->next_tid
= tid
;
1548 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu
);
1557 static int intel_pt_process_switch(struct intel_pt
*pt
,
1558 struct perf_sample
*sample
)
1560 struct perf_evsel
*evsel
;
1564 evsel
= perf_evlist__id2evsel(pt
->session
->evlist
, sample
->id
);
1565 if (evsel
!= pt
->switch_evsel
)
1568 tid
= perf_evsel__intval(evsel
, sample
, "next_pid");
1571 intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64
" tsc %#"PRIx64
"\n",
1572 cpu
, tid
, sample
->time
, perf_time_to_tsc(sample
->time
,
1575 ret
= intel_pt_sync_switch(pt
, cpu
, tid
, sample
->time
);
1579 return machine__set_current_tid(pt
->machine
, cpu
, -1, tid
);
1582 static int intel_pt_context_switch(struct intel_pt
*pt
, union perf_event
*event
,
1583 struct perf_sample
*sample
)
1585 bool out
= event
->header
.misc
& PERF_RECORD_MISC_SWITCH_OUT
;
1591 if (pt
->have_sched_switch
== 3) {
1594 if (event
->header
.type
!= PERF_RECORD_SWITCH_CPU_WIDE
) {
1595 pr_err("Expecting CPU-wide context switch event\n");
1598 pid
= event
->context_switch
.next_prev_pid
;
1599 tid
= event
->context_switch
.next_prev_tid
;
1608 pr_err("context_switch event has no tid\n");
1612 intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64
" tsc %#"PRIx64
"\n",
1613 cpu
, pid
, tid
, sample
->time
, perf_time_to_tsc(sample
->time
,
1616 ret
= intel_pt_sync_switch(pt
, cpu
, tid
, sample
->time
);
1620 return machine__set_current_tid(pt
->machine
, cpu
, pid
, tid
);
1623 static int intel_pt_process_itrace_start(struct intel_pt
*pt
,
1624 union perf_event
*event
,
1625 struct perf_sample
*sample
)
1627 if (!pt
->per_cpu_mmaps
)
1630 intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64
" tsc %#"PRIx64
"\n",
1631 sample
->cpu
, event
->itrace_start
.pid
,
1632 event
->itrace_start
.tid
, sample
->time
,
1633 perf_time_to_tsc(sample
->time
, &pt
->tc
));
1635 return machine__set_current_tid(pt
->machine
, sample
->cpu
,
1636 event
->itrace_start
.pid
,
1637 event
->itrace_start
.tid
);
1640 static int intel_pt_process_event(struct perf_session
*session
,
1641 union perf_event
*event
,
1642 struct perf_sample
*sample
,
1643 struct perf_tool
*tool
)
1645 struct intel_pt
*pt
= container_of(session
->auxtrace
, struct intel_pt
,
1653 if (!tool
->ordered_events
) {
1654 pr_err("Intel Processor Trace requires ordered events\n");
1658 if (sample
->time
&& sample
->time
!= (u64
)-1)
1659 timestamp
= perf_time_to_tsc(sample
->time
, &pt
->tc
);
1663 if (timestamp
|| pt
->timeless_decoding
) {
1664 err
= intel_pt_update_queues(pt
);
1669 if (pt
->timeless_decoding
) {
1670 if (event
->header
.type
== PERF_RECORD_EXIT
) {
1671 err
= intel_pt_process_timeless_queues(pt
,
1675 } else if (timestamp
) {
1676 err
= intel_pt_process_queues(pt
, timestamp
);
1681 if (event
->header
.type
== PERF_RECORD_AUX
&&
1682 (event
->aux
.flags
& PERF_AUX_FLAG_TRUNCATED
) &&
1683 pt
->synth_opts
.errors
) {
1684 err
= intel_pt_lost(pt
, sample
);
1689 if (pt
->switch_evsel
&& event
->header
.type
== PERF_RECORD_SAMPLE
)
1690 err
= intel_pt_process_switch(pt
, sample
);
1691 else if (event
->header
.type
== PERF_RECORD_ITRACE_START
)
1692 err
= intel_pt_process_itrace_start(pt
, event
, sample
);
1693 else if (event
->header
.type
== PERF_RECORD_SWITCH
||
1694 event
->header
.type
== PERF_RECORD_SWITCH_CPU_WIDE
)
1695 err
= intel_pt_context_switch(pt
, event
, sample
);
1697 intel_pt_log("event %s (%u): cpu %d time %"PRIu64
" tsc %#"PRIx64
"\n",
1698 perf_event__name(event
->header
.type
), event
->header
.type
,
1699 sample
->cpu
, sample
->time
, timestamp
);
1704 static int intel_pt_flush(struct perf_session
*session
, struct perf_tool
*tool
)
1706 struct intel_pt
*pt
= container_of(session
->auxtrace
, struct intel_pt
,
1713 if (!tool
->ordered_events
)
1716 ret
= intel_pt_update_queues(pt
);
1720 if (pt
->timeless_decoding
)
1721 return intel_pt_process_timeless_queues(pt
, -1,
1724 return intel_pt_process_queues(pt
, MAX_TIMESTAMP
);
1727 static void intel_pt_free_events(struct perf_session
*session
)
1729 struct intel_pt
*pt
= container_of(session
->auxtrace
, struct intel_pt
,
1731 struct auxtrace_queues
*queues
= &pt
->queues
;
1734 for (i
= 0; i
< queues
->nr_queues
; i
++) {
1735 intel_pt_free_queue(queues
->queue_array
[i
].priv
);
1736 queues
->queue_array
[i
].priv
= NULL
;
1738 intel_pt_log_disable();
1739 auxtrace_queues__free(queues
);
1742 static void intel_pt_free(struct perf_session
*session
)
1744 struct intel_pt
*pt
= container_of(session
->auxtrace
, struct intel_pt
,
1747 auxtrace_heap__free(&pt
->heap
);
1748 intel_pt_free_events(session
);
1749 session
->auxtrace
= NULL
;
1750 thread__put(pt
->unknown_thread
);
1754 static int intel_pt_process_auxtrace_event(struct perf_session
*session
,
1755 union perf_event
*event
,
1756 struct perf_tool
*tool __maybe_unused
)
1758 struct intel_pt
*pt
= container_of(session
->auxtrace
, struct intel_pt
,
1761 if (pt
->sampling_mode
)
1764 if (!pt
->data_queued
) {
1765 struct auxtrace_buffer
*buffer
;
1767 int fd
= perf_data_file__fd(session
->file
);
1770 if (perf_data_file__is_pipe(session
->file
)) {
1773 data_offset
= lseek(fd
, 0, SEEK_CUR
);
1774 if (data_offset
== -1)
1778 err
= auxtrace_queues__add_event(&pt
->queues
, session
, event
,
1779 data_offset
, &buffer
);
1783 /* Dump here now we have copied a piped trace out of the pipe */
1785 if (auxtrace_buffer__get_data(buffer
, fd
)) {
1786 intel_pt_dump_event(pt
, buffer
->data
,
1788 auxtrace_buffer__put_data(buffer
);
1796 struct intel_pt_synth
{
1797 struct perf_tool dummy_tool
;
1798 struct perf_session
*session
;
1801 static int intel_pt_event_synth(struct perf_tool
*tool
,
1802 union perf_event
*event
,
1803 struct perf_sample
*sample __maybe_unused
,
1804 struct machine
*machine __maybe_unused
)
1806 struct intel_pt_synth
*intel_pt_synth
=
1807 container_of(tool
, struct intel_pt_synth
, dummy_tool
);
1809 return perf_session__deliver_synth_event(intel_pt_synth
->session
, event
,
1813 static int intel_pt_synth_event(struct perf_session
*session
,
1814 struct perf_event_attr
*attr
, u64 id
)
1816 struct intel_pt_synth intel_pt_synth
;
1818 memset(&intel_pt_synth
, 0, sizeof(struct intel_pt_synth
));
1819 intel_pt_synth
.session
= session
;
1821 return perf_event__synthesize_attr(&intel_pt_synth
.dummy_tool
, attr
, 1,
1822 &id
, intel_pt_event_synth
);
1825 static int intel_pt_synth_events(struct intel_pt
*pt
,
1826 struct perf_session
*session
)
1828 struct perf_evlist
*evlist
= session
->evlist
;
1829 struct perf_evsel
*evsel
;
1830 struct perf_event_attr attr
;
1835 evlist__for_each(evlist
, evsel
) {
1836 if (evsel
->attr
.type
== pt
->pmu_type
&& evsel
->ids
) {
1843 pr_debug("There are no selected events with Intel Processor Trace data\n");
1847 memset(&attr
, 0, sizeof(struct perf_event_attr
));
1848 attr
.size
= sizeof(struct perf_event_attr
);
1849 attr
.type
= PERF_TYPE_HARDWARE
;
1850 attr
.sample_type
= evsel
->attr
.sample_type
& PERF_SAMPLE_MASK
;
1851 attr
.sample_type
|= PERF_SAMPLE_IP
| PERF_SAMPLE_TID
|
1853 if (pt
->timeless_decoding
)
1854 attr
.sample_type
&= ~(u64
)PERF_SAMPLE_TIME
;
1856 attr
.sample_type
|= PERF_SAMPLE_TIME
;
1857 if (!pt
->per_cpu_mmaps
)
1858 attr
.sample_type
&= ~(u64
)PERF_SAMPLE_CPU
;
1859 attr
.exclude_user
= evsel
->attr
.exclude_user
;
1860 attr
.exclude_kernel
= evsel
->attr
.exclude_kernel
;
1861 attr
.exclude_hv
= evsel
->attr
.exclude_hv
;
1862 attr
.exclude_host
= evsel
->attr
.exclude_host
;
1863 attr
.exclude_guest
= evsel
->attr
.exclude_guest
;
1864 attr
.sample_id_all
= evsel
->attr
.sample_id_all
;
1865 attr
.read_format
= evsel
->attr
.read_format
;
1867 id
= evsel
->id
[0] + 1000000000;
1871 if (pt
->synth_opts
.instructions
) {
1872 attr
.config
= PERF_COUNT_HW_INSTRUCTIONS
;
1873 if (pt
->synth_opts
.period_type
== PERF_ITRACE_PERIOD_NANOSECS
)
1874 attr
.sample_period
=
1875 intel_pt_ns_to_ticks(pt
, pt
->synth_opts
.period
);
1877 attr
.sample_period
= pt
->synth_opts
.period
;
1878 pt
->instructions_sample_period
= attr
.sample_period
;
1879 if (pt
->synth_opts
.callchain
)
1880 attr
.sample_type
|= PERF_SAMPLE_CALLCHAIN
;
1881 if (pt
->synth_opts
.last_branch
)
1882 attr
.sample_type
|= PERF_SAMPLE_BRANCH_STACK
;
1883 pr_debug("Synthesizing 'instructions' event with id %" PRIu64
" sample type %#" PRIx64
"\n",
1884 id
, (u64
)attr
.sample_type
);
1885 err
= intel_pt_synth_event(session
, &attr
, id
);
1887 pr_err("%s: failed to synthesize 'instructions' event type\n",
1891 pt
->sample_instructions
= true;
1892 pt
->instructions_sample_type
= attr
.sample_type
;
1893 pt
->instructions_id
= id
;
1897 if (pt
->synth_opts
.transactions
) {
1898 attr
.config
= PERF_COUNT_HW_INSTRUCTIONS
;
1899 attr
.sample_period
= 1;
1900 if (pt
->synth_opts
.callchain
)
1901 attr
.sample_type
|= PERF_SAMPLE_CALLCHAIN
;
1902 if (pt
->synth_opts
.last_branch
)
1903 attr
.sample_type
|= PERF_SAMPLE_BRANCH_STACK
;
1904 pr_debug("Synthesizing 'transactions' event with id %" PRIu64
" sample type %#" PRIx64
"\n",
1905 id
, (u64
)attr
.sample_type
);
1906 err
= intel_pt_synth_event(session
, &attr
, id
);
1908 pr_err("%s: failed to synthesize 'transactions' event type\n",
1912 pt
->sample_transactions
= true;
1913 pt
->transactions_id
= id
;
1915 evlist__for_each(evlist
, evsel
) {
1916 if (evsel
->id
&& evsel
->id
[0] == pt
->transactions_id
) {
1918 zfree(&evsel
->name
);
1919 evsel
->name
= strdup("transactions");
1925 if (pt
->synth_opts
.branches
) {
1926 attr
.config
= PERF_COUNT_HW_BRANCH_INSTRUCTIONS
;
1927 attr
.sample_period
= 1;
1928 attr
.sample_type
|= PERF_SAMPLE_ADDR
;
1929 attr
.sample_type
&= ~(u64
)PERF_SAMPLE_CALLCHAIN
;
1930 attr
.sample_type
&= ~(u64
)PERF_SAMPLE_BRANCH_STACK
;
1931 pr_debug("Synthesizing 'branches' event with id %" PRIu64
" sample type %#" PRIx64
"\n",
1932 id
, (u64
)attr
.sample_type
);
1933 err
= intel_pt_synth_event(session
, &attr
, id
);
1935 pr_err("%s: failed to synthesize 'branches' event type\n",
1939 pt
->sample_branches
= true;
1940 pt
->branches_sample_type
= attr
.sample_type
;
1941 pt
->branches_id
= id
;
1944 pt
->synth_needs_swap
= evsel
->needs_swap
;
1949 static struct perf_evsel
*intel_pt_find_sched_switch(struct perf_evlist
*evlist
)
1951 struct perf_evsel
*evsel
;
1953 evlist__for_each_reverse(evlist
, evsel
) {
1954 const char *name
= perf_evsel__name(evsel
);
1956 if (!strcmp(name
, "sched:sched_switch"))
1963 static bool intel_pt_find_switch(struct perf_evlist
*evlist
)
1965 struct perf_evsel
*evsel
;
1967 evlist__for_each(evlist
, evsel
) {
1968 if (evsel
->attr
.context_switch
)
1975 static int intel_pt_perf_config(const char *var
, const char *value
, void *data
)
1977 struct intel_pt
*pt
= data
;
1979 if (!strcmp(var
, "intel-pt.mispred-all"))
1980 pt
->mispred_all
= perf_config_bool(var
, value
);
1985 static const char * const intel_pt_info_fmts
[] = {
1986 [INTEL_PT_PMU_TYPE
] = " PMU Type %"PRId64
"\n",
1987 [INTEL_PT_TIME_SHIFT
] = " Time Shift %"PRIu64
"\n",
1988 [INTEL_PT_TIME_MULT
] = " Time Muliplier %"PRIu64
"\n",
1989 [INTEL_PT_TIME_ZERO
] = " Time Zero %"PRIu64
"\n",
1990 [INTEL_PT_CAP_USER_TIME_ZERO
] = " Cap Time Zero %"PRId64
"\n",
1991 [INTEL_PT_TSC_BIT
] = " TSC bit %#"PRIx64
"\n",
1992 [INTEL_PT_NORETCOMP_BIT
] = " NoRETComp bit %#"PRIx64
"\n",
1993 [INTEL_PT_HAVE_SCHED_SWITCH
] = " Have sched_switch %"PRId64
"\n",
1994 [INTEL_PT_SNAPSHOT_MODE
] = " Snapshot mode %"PRId64
"\n",
1995 [INTEL_PT_PER_CPU_MMAPS
] = " Per-cpu maps %"PRId64
"\n",
1996 [INTEL_PT_MTC_BIT
] = " MTC bit %#"PRIx64
"\n",
1997 [INTEL_PT_TSC_CTC_N
] = " TSC:CTC numerator %"PRIu64
"\n",
1998 [INTEL_PT_TSC_CTC_D
] = " TSC:CTC denominator %"PRIu64
"\n",
1999 [INTEL_PT_CYC_BIT
] = " CYC bit %#"PRIx64
"\n",
2002 static void intel_pt_print_info(u64
*arr
, int start
, int finish
)
2009 for (i
= start
; i
<= finish
; i
++)
2010 fprintf(stdout
, intel_pt_info_fmts
[i
], arr
[i
]);
2013 int intel_pt_process_auxtrace_info(union perf_event
*event
,
2014 struct perf_session
*session
)
2016 struct auxtrace_info_event
*auxtrace_info
= &event
->auxtrace_info
;
2017 size_t min_sz
= sizeof(u64
) * INTEL_PT_PER_CPU_MMAPS
;
2018 struct intel_pt
*pt
;
2021 if (auxtrace_info
->header
.size
< sizeof(struct auxtrace_info_event
) +
2025 pt
= zalloc(sizeof(struct intel_pt
));
2029 perf_config(intel_pt_perf_config
, pt
);
2031 err
= auxtrace_queues__init(&pt
->queues
);
2035 intel_pt_log_set_name(INTEL_PT_PMU_NAME
);
2037 pt
->session
= session
;
2038 pt
->machine
= &session
->machines
.host
; /* No kvm support */
2039 pt
->auxtrace_type
= auxtrace_info
->type
;
2040 pt
->pmu_type
= auxtrace_info
->priv
[INTEL_PT_PMU_TYPE
];
2041 pt
->tc
.time_shift
= auxtrace_info
->priv
[INTEL_PT_TIME_SHIFT
];
2042 pt
->tc
.time_mult
= auxtrace_info
->priv
[INTEL_PT_TIME_MULT
];
2043 pt
->tc
.time_zero
= auxtrace_info
->priv
[INTEL_PT_TIME_ZERO
];
2044 pt
->cap_user_time_zero
= auxtrace_info
->priv
[INTEL_PT_CAP_USER_TIME_ZERO
];
2045 pt
->tsc_bit
= auxtrace_info
->priv
[INTEL_PT_TSC_BIT
];
2046 pt
->noretcomp_bit
= auxtrace_info
->priv
[INTEL_PT_NORETCOMP_BIT
];
2047 pt
->have_sched_switch
= auxtrace_info
->priv
[INTEL_PT_HAVE_SCHED_SWITCH
];
2048 pt
->snapshot_mode
= auxtrace_info
->priv
[INTEL_PT_SNAPSHOT_MODE
];
2049 pt
->per_cpu_mmaps
= auxtrace_info
->priv
[INTEL_PT_PER_CPU_MMAPS
];
2050 intel_pt_print_info(&auxtrace_info
->priv
[0], INTEL_PT_PMU_TYPE
,
2051 INTEL_PT_PER_CPU_MMAPS
);
2053 if (auxtrace_info
->header
.size
>= sizeof(struct auxtrace_info_event
) +
2054 (sizeof(u64
) * INTEL_PT_CYC_BIT
)) {
2055 pt
->mtc_bit
= auxtrace_info
->priv
[INTEL_PT_MTC_BIT
];
2056 pt
->mtc_freq_bits
= auxtrace_info
->priv
[INTEL_PT_MTC_FREQ_BITS
];
2057 pt
->tsc_ctc_ratio_n
= auxtrace_info
->priv
[INTEL_PT_TSC_CTC_N
];
2058 pt
->tsc_ctc_ratio_d
= auxtrace_info
->priv
[INTEL_PT_TSC_CTC_D
];
2059 pt
->cyc_bit
= auxtrace_info
->priv
[INTEL_PT_CYC_BIT
];
2060 intel_pt_print_info(&auxtrace_info
->priv
[0], INTEL_PT_MTC_BIT
,
2064 pt
->timeless_decoding
= intel_pt_timeless_decoding(pt
);
2065 pt
->have_tsc
= intel_pt_have_tsc(pt
);
2066 pt
->sampling_mode
= false;
2067 pt
->est_tsc
= !pt
->timeless_decoding
;
2069 pt
->unknown_thread
= thread__new(999999999, 999999999);
2070 if (!pt
->unknown_thread
) {
2072 goto err_free_queues
;
2076 * Since this thread will not be kept in any rbtree not in a
2077 * list, initialize its list node so that at thread__put() the
2078 * current thread lifetime assuption is kept and we don't segfault
2079 * at list_del_init().
2081 INIT_LIST_HEAD(&pt
->unknown_thread
->node
);
2083 err
= thread__set_comm(pt
->unknown_thread
, "unknown", 0);
2085 goto err_delete_thread
;
2086 if (thread__init_map_groups(pt
->unknown_thread
, pt
->machine
)) {
2088 goto err_delete_thread
;
2091 pt
->auxtrace
.process_event
= intel_pt_process_event
;
2092 pt
->auxtrace
.process_auxtrace_event
= intel_pt_process_auxtrace_event
;
2093 pt
->auxtrace
.flush_events
= intel_pt_flush
;
2094 pt
->auxtrace
.free_events
= intel_pt_free_events
;
2095 pt
->auxtrace
.free
= intel_pt_free
;
2096 session
->auxtrace
= &pt
->auxtrace
;
2101 if (pt
->have_sched_switch
== 1) {
2102 pt
->switch_evsel
= intel_pt_find_sched_switch(session
->evlist
);
2103 if (!pt
->switch_evsel
) {
2104 pr_err("%s: missing sched_switch event\n", __func__
);
2105 goto err_delete_thread
;
2107 } else if (pt
->have_sched_switch
== 2 &&
2108 !intel_pt_find_switch(session
->evlist
)) {
2109 pr_err("%s: missing context_switch attribute flag\n", __func__
);
2110 goto err_delete_thread
;
2113 if (session
->itrace_synth_opts
&& session
->itrace_synth_opts
->set
) {
2114 pt
->synth_opts
= *session
->itrace_synth_opts
;
2116 itrace_synth_opts__set_default(&pt
->synth_opts
);
2117 if (use_browser
!= -1) {
2118 pt
->synth_opts
.branches
= false;
2119 pt
->synth_opts
.callchain
= true;
2123 if (pt
->synth_opts
.log
)
2124 intel_pt_log_enable();
2126 /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2127 if (pt
->tc
.time_mult
) {
2128 u64 tsc_freq
= intel_pt_ns_to_ticks(pt
, 1000000000);
2130 pt
->max_non_turbo_ratio
= (tsc_freq
+ 50000000) / 100000000;
2131 intel_pt_log("TSC frequency %"PRIu64
"\n", tsc_freq
);
2132 intel_pt_log("Maximum non-turbo ratio %u\n",
2133 pt
->max_non_turbo_ratio
);
2136 if (pt
->synth_opts
.calls
)
2137 pt
->branches_filter
|= PERF_IP_FLAG_CALL
| PERF_IP_FLAG_ASYNC
|
2138 PERF_IP_FLAG_TRACE_END
;
2139 if (pt
->synth_opts
.returns
)
2140 pt
->branches_filter
|= PERF_IP_FLAG_RETURN
|
2141 PERF_IP_FLAG_TRACE_BEGIN
;
2143 if (pt
->synth_opts
.callchain
&& !symbol_conf
.use_callchain
) {
2144 symbol_conf
.use_callchain
= true;
2145 if (callchain_register_param(&callchain_param
) < 0) {
2146 symbol_conf
.use_callchain
= false;
2147 pt
->synth_opts
.callchain
= false;
2151 err
= intel_pt_synth_events(pt
, session
);
2153 goto err_delete_thread
;
2155 err
= auxtrace_queues__process_index(&pt
->queues
, session
);
2157 goto err_delete_thread
;
2159 if (pt
->queues
.populated
)
2160 pt
->data_queued
= true;
2162 if (pt
->timeless_decoding
)
2163 pr_debug2("Intel PT decoding without timestamps\n");
2168 thread__zput(pt
->unknown_thread
);
2170 intel_pt_log_disable();
2171 auxtrace_queues__free(&pt
->queues
);
2172 session
->auxtrace
= NULL
;