25 #include "../builtin.h"
28 struct perf_data_file
*output
;
29 struct perf_session
*session
;
30 struct machine
*machine
;
31 union jr_entry
*entry
;
36 bool needs_bswap
; /* handles cross-endianess */
37 bool use_arch_timestamp
;
39 size_t nr_debug_entries
;
40 uint32_t code_load_count
;
42 struct rb_root code_root
;
46 struct debug_line_info
{
49 /* The filename format is unspecified, absolute path, relative etc. */
50 char const filename
[0];
54 struct perf_tool tool
;
55 struct perf_data_file output
;
56 struct perf_data_file input
;
60 #define hmax(a, b) ((a) > (b) ? (a) : (b))
61 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))
64 jit_emit_elf(char *filename
,
75 fprintf(stderr
, "write ELF image %s\n", filename
);
77 fd
= open(filename
, O_CREAT
|O_TRUNC
|O_WRONLY
, 0644);
79 pr_warning("cannot create jit ELF %s: %s\n", filename
, strerror(errno
));
83 ret
= jit_write_elf(fd
, code_addr
, sym
, (const void *)code
, csize
, debug
, nr_debug_entries
);
94 jit_close(struct jit_buf_desc
*jd
)
104 jit_validate_events(struct perf_session
*session
)
106 struct perf_evsel
*evsel
;
109 * check that all events use CLOCK_MONOTONIC
111 evlist__for_each_entry(session
->evlist
, evsel
) {
112 if (evsel
->attr
.use_clockid
== 0 || evsel
->attr
.clockid
!= CLOCK_MONOTONIC
)
119 jit_open(struct jit_buf_desc
*jd
, const char *name
)
121 struct jitheader header
;
122 struct jr_prefix
*prefix
;
124 void *n
, *buf
= NULL
;
125 int ret
, retval
= -1;
127 jd
->in
= fopen(name
, "r");
131 bsz
= hmax(sizeof(header
), sizeof(*prefix
));
138 * protect from writer modifying the file while we are reading it
142 ret
= fread(buf
, sizeof(header
), 1, jd
->in
);
146 memcpy(&header
, buf
, sizeof(header
));
148 if (header
.magic
!= JITHEADER_MAGIC
) {
149 if (header
.magic
!= JITHEADER_MAGIC_SW
)
151 jd
->needs_bswap
= true;
154 if (jd
->needs_bswap
) {
155 header
.version
= bswap_32(header
.version
);
156 header
.total_size
= bswap_32(header
.total_size
);
157 header
.pid
= bswap_32(header
.pid
);
158 header
.elf_mach
= bswap_32(header
.elf_mach
);
159 header
.timestamp
= bswap_64(header
.timestamp
);
160 header
.flags
= bswap_64(header
.flags
);
163 jd
->use_arch_timestamp
= header
.flags
& JITDUMP_FLAGS_ARCH_TIMESTAMP
;
166 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
169 (unsigned long long)header
.timestamp
,
172 jd
->use_arch_timestamp
);
174 if (header
.flags
& JITDUMP_FLAGS_RESERVED
) {
175 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
176 (unsigned long long)header
.flags
& JITDUMP_FLAGS_RESERVED
);
180 if (jd
->use_arch_timestamp
&& !jd
->session
->time_conv
.time_mult
) {
181 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
186 * validate event is using the correct clockid
188 if (!jd
->use_arch_timestamp
&& jit_validate_events(jd
->session
)) {
189 pr_err("error, jitted code must be sampled with perf record -k 1\n");
193 bs
= header
.total_size
- sizeof(header
);
196 n
= realloc(buf
, bs
);
201 /* read extra we do not know about */
202 ret
= fread(buf
, bs
- bsz
, 1, jd
->in
);
207 * keep dirname for generating files and mmap records
209 strcpy(jd
->dir
, name
);
219 static union jr_entry
*
220 jit_get_next_entry(struct jit_buf_desc
*jd
)
222 struct jr_prefix
*prefix
;
231 if (jd
->buf
== NULL
) {
232 size_t sz
= getpagesize();
233 if (sz
< sizeof(*prefix
))
234 sz
= sizeof(*prefix
);
236 jd
->buf
= malloc(sz
);
246 * file is still locked at this point
248 ret
= fread(prefix
, sizeof(*prefix
), 1, jd
->in
);
252 if (jd
->needs_bswap
) {
253 prefix
->id
= bswap_32(prefix
->id
);
254 prefix
->total_size
= bswap_32(prefix
->total_size
);
255 prefix
->timestamp
= bswap_64(prefix
->timestamp
);
258 size
= prefix
->total_size
;
261 if (bs
< sizeof(*prefix
))
264 if (id
>= JIT_CODE_MAX
) {
265 pr_warning("next_entry: unknown prefix %d, skipping\n", id
);
268 if (bs
> jd
->bufsize
) {
270 n
= realloc(jd
->buf
, bs
);
277 addr
= ((void *)jd
->buf
) + sizeof(*prefix
);
279 ret
= fread(addr
, bs
- sizeof(*prefix
), 1, jd
->in
);
283 jr
= (union jr_entry
*)jd
->buf
;
286 case JIT_CODE_DEBUG_INFO
:
287 if (jd
->needs_bswap
) {
289 jr
->info
.code_addr
= bswap_64(jr
->info
.code_addr
);
290 jr
->info
.nr_entry
= bswap_64(jr
->info
.nr_entry
);
291 for (n
= 0 ; n
< jr
->info
.nr_entry
; n
++) {
292 jr
->info
.entries
[n
].addr
= bswap_64(jr
->info
.entries
[n
].addr
);
293 jr
->info
.entries
[n
].lineno
= bswap_32(jr
->info
.entries
[n
].lineno
);
294 jr
->info
.entries
[n
].discrim
= bswap_32(jr
->info
.entries
[n
].discrim
);
301 if (jd
->needs_bswap
) {
302 jr
->load
.pid
= bswap_32(jr
->load
.pid
);
303 jr
->load
.tid
= bswap_32(jr
->load
.tid
);
304 jr
->load
.vma
= bswap_64(jr
->load
.vma
);
305 jr
->load
.code_addr
= bswap_64(jr
->load
.code_addr
);
306 jr
->load
.code_size
= bswap_64(jr
->load
.code_size
);
307 jr
->load
.code_index
= bswap_64(jr
->load
.code_index
);
309 jd
->code_load_count
++;
312 if (jd
->needs_bswap
) {
313 jr
->move
.pid
= bswap_32(jr
->move
.pid
);
314 jr
->move
.tid
= bswap_32(jr
->move
.tid
);
315 jr
->move
.vma
= bswap_64(jr
->move
.vma
);
316 jr
->move
.old_code_addr
= bswap_64(jr
->move
.old_code_addr
);
317 jr
->move
.new_code_addr
= bswap_64(jr
->move
.new_code_addr
);
318 jr
->move
.code_size
= bswap_64(jr
->move
.code_size
);
319 jr
->move
.code_index
= bswap_64(jr
->move
.code_index
);
330 jit_inject_event(struct jit_buf_desc
*jd
, union perf_event
*event
)
334 size
= perf_data_file__write(jd
->output
, event
, event
->header
.size
);
338 jd
->bytes_written
+= size
;
342 static uint64_t convert_timestamp(struct jit_buf_desc
*jd
, uint64_t timestamp
)
344 struct perf_tsc_conversion tc
;
346 if (!jd
->use_arch_timestamp
)
349 tc
.time_shift
= jd
->session
->time_conv
.time_shift
;
350 tc
.time_mult
= jd
->session
->time_conv
.time_mult
;
351 tc
.time_zero
= jd
->session
->time_conv
.time_zero
;
356 return tsc_to_perf_time(timestamp
, &tc
);
359 static int jit_repipe_code_load(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
361 struct perf_sample sample
;
362 union perf_event
*event
;
363 struct perf_tool
*tool
= jd
->session
->tool
;
381 csize
= jr
->load
.code_size
;
382 addr
= jr
->load
.code_addr
;
383 sym
= (void *)((unsigned long)jr
+ sizeof(jr
->load
));
384 code
= (unsigned long)jr
+ jr
->load
.p
.total_size
- csize
;
385 count
= jr
->load
.code_index
;
386 idr_size
= jd
->machine
->id_hdr_size
;
388 event
= calloc(1, sizeof(*event
) + idr_size
);
392 filename
= event
->mmap2
.filename
;
393 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%u.so",
400 size
= PERF_ALIGN(size
, sizeof(u64
));
401 uaddr
= (uintptr_t)code
;
402 ret
= jit_emit_elf(filename
, sym
, addr
, (const void *)uaddr
, csize
, jd
->debug_data
, jd
->nr_debug_entries
);
404 if (jd
->debug_data
&& jd
->nr_debug_entries
) {
405 free(jd
->debug_data
);
406 jd
->debug_data
= NULL
;
407 jd
->nr_debug_entries
= 0;
414 if (stat(filename
, &st
))
415 memset(&st
, 0, sizeof(st
));
417 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
418 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
419 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
420 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
422 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
423 event
->mmap2
.start
= addr
;
424 event
->mmap2
.len
= csize
;
425 event
->mmap2
.pid
= pid
;
426 event
->mmap2
.tid
= tid
;
427 event
->mmap2
.ino
= st
.st_ino
;
428 event
->mmap2
.maj
= major(st
.st_dev
);
429 event
->mmap2
.min
= minor(st
.st_dev
);
430 event
->mmap2
.prot
= st
.st_mode
;
431 event
->mmap2
.flags
= MAP_SHARED
;
432 event
->mmap2
.ino_generation
= 1;
434 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
435 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
439 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
440 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
443 * create pseudo sample to induce dso hit increment
444 * use first address as sample address
446 memset(&sample
, 0, sizeof(sample
));
447 sample
.cpumode
= PERF_RECORD_MISC_USER
;
450 sample
.time
= id
->time
;
453 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
457 ret
= jit_inject_event(jd
, event
);
459 * mark dso as use to generate buildid in the header
462 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
467 static int jit_repipe_code_move(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
469 struct perf_sample sample
;
470 union perf_event
*event
;
471 struct perf_tool
*tool
= jd
->session
->tool
;
485 idr_size
= jd
->machine
->id_hdr_size
;
488 * +16 to account for sample_id_all (hack)
490 event
= calloc(1, sizeof(*event
) + 16);
494 filename
= event
->mmap2
.filename
;
495 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%"PRIu64
,
498 jr
->move
.code_index
);
502 if (stat(filename
, &st
))
503 memset(&st
, 0, sizeof(st
));
505 size
= PERF_ALIGN(size
, sizeof(u64
));
507 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
508 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
509 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
510 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
511 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
512 event
->mmap2
.start
= jr
->move
.new_code_addr
;
513 event
->mmap2
.len
= jr
->move
.code_size
;
514 event
->mmap2
.pid
= pid
;
515 event
->mmap2
.tid
= tid
;
516 event
->mmap2
.ino
= st
.st_ino
;
517 event
->mmap2
.maj
= major(st
.st_dev
);
518 event
->mmap2
.min
= minor(st
.st_dev
);
519 event
->mmap2
.prot
= st
.st_mode
;
520 event
->mmap2
.flags
= MAP_SHARED
;
521 event
->mmap2
.ino_generation
= 1;
523 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
524 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
528 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
529 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
532 * create pseudo sample to induce dso hit increment
533 * use first address as sample address
535 memset(&sample
, 0, sizeof(sample
));
536 sample
.cpumode
= PERF_RECORD_MISC_USER
;
539 sample
.time
= id
->time
;
540 sample
.ip
= jr
->move
.new_code_addr
;
542 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
546 ret
= jit_inject_event(jd
, event
);
548 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
553 static int jit_repipe_debug_info(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
561 sz
= jr
->prefix
.total_size
- sizeof(jr
->info
);
566 memcpy(data
, &jr
->info
.entries
, sz
);
568 jd
->debug_data
= data
;
571 * we must use nr_entry instead of size here because
572 * we cannot distinguish actual entry from padding otherwise
574 jd
->nr_debug_entries
= jr
->info
.nr_entry
;
580 jit_process_dump(struct jit_buf_desc
*jd
)
585 while ((jr
= jit_get_next_entry(jd
))) {
586 switch(jr
->prefix
.id
) {
588 ret
= jit_repipe_code_load(jd
, jr
);
591 ret
= jit_repipe_code_move(jd
, jr
);
593 case JIT_CODE_DEBUG_INFO
:
594 ret
= jit_repipe_debug_info(jd
, jr
);
605 jit_inject(struct jit_buf_desc
*jd
, char *path
)
610 fprintf(stderr
, "injecting: %s\n", path
);
612 ret
= jit_open(jd
, path
);
616 ret
= jit_process_dump(jd
);
621 fprintf(stderr
, "injected: %s (%d)\n", path
, ret
);
627 * File must be with pattern .../jit-XXXX.dump
628 * where XXXX is the PID of the process which did the mmap()
629 * as captured in the RECORD_MMAP record
632 jit_detect(char *mmap_name
, pid_t pid
)
639 fprintf(stderr
, "jit marker trying : %s\n", mmap_name
);
643 p
= strrchr(mmap_name
, '/');
650 if (strncmp(p
, "/jit-", 5))
659 * must be followed by a pid
664 pid2
= (int)strtol(p
, &end
, 10);
669 * pid does not match mmap pid
670 * pid==0 in system-wide mode (synthesized)
672 if (pid
&& pid2
!= pid
)
677 if (strcmp(end
, ".dump"))
681 fprintf(stderr
, "jit marker found: %s\n", mmap_name
);
687 jit_process(struct perf_session
*session
,
688 struct perf_data_file
*output
,
689 struct machine
*machine
,
694 struct perf_evsel
*first
;
695 struct jit_buf_desc jd
;
699 * first, detect marker mmap (i.e., the jitdump mmap)
701 if (jit_detect(filename
, pid
))
704 memset(&jd
, 0, sizeof(jd
));
706 jd
.session
= session
;
708 jd
.machine
= machine
;
711 * track sample_type to compute id_all layout
712 * perf sets the same sample type to all events as of now
714 first
= perf_evlist__first(session
->evlist
);
715 jd
.sample_type
= first
->attr
.sample_type
;
719 ret
= jit_inject(&jd
, filename
);
721 *nbytes
= jd
.bytes_written
;