2 * Common eBPF ELF object loading operations.
4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
6 * Copyright (C) 2015 Huawei Inc.
17 #include <asm/unistd.h>
18 #include <linux/kernel.h>
19 #include <linux/bpf.h>
20 #include <linux/list.h>
27 #define __printf(a, b) __attribute__((format(printf, a, b)))
30 static int __base_pr(const char *format
, ...)
35 va_start(args
, format
);
36 err
= vfprintf(stderr
, format
, args
);
41 static __printf(1, 2) libbpf_print_fn_t __pr_warning
= __base_pr
;
42 static __printf(1, 2) libbpf_print_fn_t __pr_info
= __base_pr
;
43 static __printf(1, 2) libbpf_print_fn_t __pr_debug
;
45 #define __pr(func, fmt, ...) \
48 (func)("libbpf: " fmt, ##__VA_ARGS__); \
51 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__)
52 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__)
53 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__)
55 void libbpf_set_print(libbpf_print_fn_t warn
,
56 libbpf_print_fn_t info
,
57 libbpf_print_fn_t debug
)
64 /* Copied from tools/perf/util/util.h */
66 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
70 # define zclose(fd) ({ \
73 ___err = close((fd)); \
78 #ifdef HAVE_LIBELF_MMAP_SUPPORT
79 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
81 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
85 * bpf_prog should be a better name but it has been used in
89 /* Index in elf obj file, for relocation use. */
92 struct bpf_insn
*insns
;
103 struct bpf_object
*obj
;
105 bpf_program_clear_priv_t clear_priv
;
108 static LIST_HEAD(bpf_objects_list
);
116 struct bpf_program
*programs
;
120 * This field is required because maps_buf will be freed and
121 * maps_buf_sz will be set to 0 after loaded.
127 * Information when doing elf related work. Only valid if fd
144 * All loaded bpf_object is linked in a list, which is
145 * hidden to caller. bpf_objects__<func> handlers deal with
148 struct list_head list
;
151 #define obj_elf_valid(o) ((o)->efile.elf)
153 static void bpf_program__unload(struct bpf_program
*prog
)
161 static void bpf_program__exit(struct bpf_program
*prog
)
166 if (prog
->clear_priv
)
167 prog
->clear_priv(prog
, prog
->priv
);
170 prog
->clear_priv
= NULL
;
172 bpf_program__unload(prog
);
173 zfree(&prog
->section_name
);
175 zfree(&prog
->reloc_desc
);
183 bpf_program__init(void *data
, size_t size
, char *name
, int idx
,
184 struct bpf_program
*prog
)
186 if (size
< sizeof(struct bpf_insn
)) {
187 pr_warning("corrupted section '%s'\n", name
);
191 bzero(prog
, sizeof(*prog
));
193 prog
->section_name
= strdup(name
);
194 if (!prog
->section_name
) {
195 pr_warning("failed to alloc name for prog %s\n",
200 prog
->insns
= malloc(size
);
202 pr_warning("failed to alloc insns for %s\n", name
);
205 prog
->insns_cnt
= size
/ sizeof(struct bpf_insn
);
206 memcpy(prog
->insns
, data
,
207 prog
->insns_cnt
* sizeof(struct bpf_insn
));
213 bpf_program__exit(prog
);
218 bpf_object__add_program(struct bpf_object
*obj
, void *data
, size_t size
,
221 struct bpf_program prog
, *progs
;
224 err
= bpf_program__init(data
, size
, name
, idx
, &prog
);
228 progs
= obj
->programs
;
229 nr_progs
= obj
->nr_programs
;
231 progs
= realloc(progs
, sizeof(progs
[0]) * (nr_progs
+ 1));
234 * In this case the original obj->programs
235 * is still valid, so don't need special treat for
236 * bpf_close_object().
238 pr_warning("failed to alloc a new program '%s'\n",
240 bpf_program__exit(&prog
);
244 pr_debug("found program %s\n", prog
.section_name
);
245 obj
->programs
= progs
;
246 obj
->nr_programs
= nr_progs
+ 1;
248 progs
[nr_progs
] = prog
;
252 static struct bpf_object
*bpf_object__new(const char *path
,
256 struct bpf_object
*obj
;
258 obj
= calloc(1, sizeof(struct bpf_object
) + strlen(path
) + 1);
260 pr_warning("alloc memory failed for %s\n", path
);
264 strcpy(obj
->path
, path
);
268 * Caller of this function should also calls
269 * bpf_object__elf_finish() after data collection to return
270 * obj_buf to user. If not, we should duplicate the buffer to
271 * avoid user freeing them before elf finish.
273 obj
->efile
.obj_buf
= obj_buf
;
274 obj
->efile
.obj_buf_sz
= obj_buf_sz
;
278 INIT_LIST_HEAD(&obj
->list
);
279 list_add(&obj
->list
, &bpf_objects_list
);
283 static void bpf_object__elf_finish(struct bpf_object
*obj
)
285 if (!obj_elf_valid(obj
))
288 if (obj
->efile
.elf
) {
289 elf_end(obj
->efile
.elf
);
290 obj
->efile
.elf
= NULL
;
292 obj
->efile
.symbols
= NULL
;
294 zfree(&obj
->efile
.reloc
);
295 obj
->efile
.nr_reloc
= 0;
296 zclose(obj
->efile
.fd
);
297 obj
->efile
.obj_buf
= NULL
;
298 obj
->efile
.obj_buf_sz
= 0;
301 static int bpf_object__elf_init(struct bpf_object
*obj
)
306 if (obj_elf_valid(obj
)) {
307 pr_warning("elf init: internal error\n");
311 if (obj
->efile
.obj_buf_sz
> 0) {
313 * obj_buf should have been validated by
314 * bpf_object__open_buffer().
316 obj
->efile
.elf
= elf_memory(obj
->efile
.obj_buf
,
317 obj
->efile
.obj_buf_sz
);
319 obj
->efile
.fd
= open(obj
->path
, O_RDONLY
);
320 if (obj
->efile
.fd
< 0) {
321 pr_warning("failed to open %s: %s\n", obj
->path
,
326 obj
->efile
.elf
= elf_begin(obj
->efile
.fd
,
327 LIBBPF_ELF_C_READ_MMAP
,
331 if (!obj
->efile
.elf
) {
332 pr_warning("failed to open %s as ELF file\n",
338 if (!gelf_getehdr(obj
->efile
.elf
, &obj
->efile
.ehdr
)) {
339 pr_warning("failed to get EHDR from %s\n",
344 ep
= &obj
->efile
.ehdr
;
346 if ((ep
->e_type
!= ET_REL
) || (ep
->e_machine
!= 0)) {
347 pr_warning("%s is not an eBPF object file\n",
355 bpf_object__elf_finish(obj
);
360 bpf_object__check_endianness(struct bpf_object
*obj
)
362 static unsigned int const endian
= 1;
364 switch (obj
->efile
.ehdr
.e_ident
[EI_DATA
]) {
366 /* We are big endian, BPF obj is little endian. */
367 if (*(unsigned char const *)&endian
!= 1)
372 /* We are little endian, BPF obj is big endian. */
373 if (*(unsigned char const *)&endian
!= 0)
383 pr_warning("Error: endianness mismatch.\n");
388 bpf_object__init_license(struct bpf_object
*obj
,
389 void *data
, size_t size
)
391 memcpy(obj
->license
, data
,
392 min(size
, sizeof(obj
->license
) - 1));
393 pr_debug("license of %s is %s\n", obj
->path
, obj
->license
);
398 bpf_object__init_kversion(struct bpf_object
*obj
,
399 void *data
, size_t size
)
403 if (size
!= sizeof(kver
)) {
404 pr_warning("invalid kver section in %s\n", obj
->path
);
407 memcpy(&kver
, data
, sizeof(kver
));
408 obj
->kern_version
= kver
;
409 pr_debug("kernel version of %s is %x\n", obj
->path
,
415 bpf_object__init_maps(struct bpf_object
*obj
, void *data
,
419 pr_debug("%s doesn't need map definition\n",
424 obj
->maps_buf
= malloc(size
);
425 if (!obj
->maps_buf
) {
426 pr_warning("malloc maps failed: %s\n", obj
->path
);
430 obj
->maps_buf_sz
= size
;
431 memcpy(obj
->maps_buf
, data
, size
);
432 pr_debug("maps in %s: %ld bytes\n", obj
->path
, (long)size
);
436 static int bpf_object__elf_collect(struct bpf_object
*obj
)
438 Elf
*elf
= obj
->efile
.elf
;
439 GElf_Ehdr
*ep
= &obj
->efile
.ehdr
;
441 int idx
= 0, err
= 0;
443 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
444 if (!elf_rawdata(elf_getscn(elf
, ep
->e_shstrndx
), NULL
)) {
445 pr_warning("failed to get e_shstrndx from %s\n",
450 while ((scn
= elf_nextscn(elf
, scn
)) != NULL
) {
456 if (gelf_getshdr(scn
, &sh
) != &sh
) {
457 pr_warning("failed to get section header from %s\n",
463 name
= elf_strptr(elf
, ep
->e_shstrndx
, sh
.sh_name
);
465 pr_warning("failed to get section name from %s\n",
471 data
= elf_getdata(scn
, 0);
473 pr_warning("failed to get section data from %s(%s)\n",
478 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n",
479 name
, (unsigned long)data
->d_size
,
480 (int)sh
.sh_link
, (unsigned long)sh
.sh_flags
,
483 if (strcmp(name
, "license") == 0)
484 err
= bpf_object__init_license(obj
,
487 else if (strcmp(name
, "version") == 0)
488 err
= bpf_object__init_kversion(obj
,
491 else if (strcmp(name
, "maps") == 0)
492 err
= bpf_object__init_maps(obj
, data
->d_buf
,
494 else if (sh
.sh_type
== SHT_SYMTAB
) {
495 if (obj
->efile
.symbols
) {
496 pr_warning("bpf: multiple SYMTAB in %s\n",
500 obj
->efile
.symbols
= data
;
501 } else if ((sh
.sh_type
== SHT_PROGBITS
) &&
502 (sh
.sh_flags
& SHF_EXECINSTR
) &&
503 (data
->d_size
> 0)) {
504 err
= bpf_object__add_program(obj
, data
->d_buf
,
505 data
->d_size
, name
, idx
);
508 strerror_r(-err
, errmsg
, sizeof(errmsg
));
509 pr_warning("failed to alloc program %s (%s): %s",
510 name
, obj
->path
, errmsg
);
512 } else if (sh
.sh_type
== SHT_REL
) {
513 void *reloc
= obj
->efile
.reloc
;
514 int nr_reloc
= obj
->efile
.nr_reloc
+ 1;
516 reloc
= realloc(reloc
,
517 sizeof(*obj
->efile
.reloc
) * nr_reloc
);
519 pr_warning("realloc failed\n");
522 int n
= nr_reloc
- 1;
524 obj
->efile
.reloc
= reloc
;
525 obj
->efile
.nr_reloc
= nr_reloc
;
527 obj
->efile
.reloc
[n
].shdr
= sh
;
528 obj
->efile
.reloc
[n
].data
= data
;
538 static struct bpf_program
*
539 bpf_object__find_prog_by_idx(struct bpf_object
*obj
, int idx
)
541 struct bpf_program
*prog
;
544 for (i
= 0; i
< obj
->nr_programs
; i
++) {
545 prog
= &obj
->programs
[i
];
546 if (prog
->idx
== idx
)
553 bpf_program__collect_reloc(struct bpf_program
*prog
,
554 size_t nr_maps
, GElf_Shdr
*shdr
,
555 Elf_Data
*data
, Elf_Data
*symbols
)
559 pr_debug("collecting relocating info for: '%s'\n",
561 nrels
= shdr
->sh_size
/ shdr
->sh_entsize
;
563 prog
->reloc_desc
= malloc(sizeof(*prog
->reloc_desc
) * nrels
);
564 if (!prog
->reloc_desc
) {
565 pr_warning("failed to alloc memory in relocation\n");
568 prog
->nr_reloc
= nrels
;
570 for (i
= 0; i
< nrels
; i
++) {
573 unsigned int insn_idx
;
574 struct bpf_insn
*insns
= prog
->insns
;
577 if (!gelf_getrel(data
, i
, &rel
)) {
578 pr_warning("relocation: failed to get %d reloc\n", i
);
582 insn_idx
= rel
.r_offset
/ sizeof(struct bpf_insn
);
583 pr_debug("relocation: insn_idx=%u\n", insn_idx
);
585 if (!gelf_getsym(symbols
,
586 GELF_R_SYM(rel
.r_info
),
588 pr_warning("relocation: symbol %"PRIx64
" not found\n",
589 GELF_R_SYM(rel
.r_info
));
593 if (insns
[insn_idx
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
)) {
594 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
595 insn_idx
, insns
[insn_idx
].code
);
599 map_idx
= sym
.st_value
/ sizeof(struct bpf_map_def
);
600 if (map_idx
>= nr_maps
) {
601 pr_warning("bpf relocation: map_idx %d large than %d\n",
602 (int)map_idx
, (int)nr_maps
- 1);
606 prog
->reloc_desc
[i
].insn_idx
= insn_idx
;
607 prog
->reloc_desc
[i
].map_idx
= map_idx
;
613 bpf_object__create_maps(struct bpf_object
*obj
)
619 nr_maps
= obj
->maps_buf_sz
/ sizeof(struct bpf_map_def
);
620 if (!obj
->maps_buf
|| !nr_maps
) {
621 pr_debug("don't need create maps for %s\n",
626 obj
->map_fds
= malloc(sizeof(int) * nr_maps
);
628 pr_warning("realloc perf_bpf_map_fds failed\n");
631 obj
->nr_map_fds
= nr_maps
;
633 /* fill all fd with -1 */
634 memset(obj
->map_fds
, -1, sizeof(int) * nr_maps
);
637 for (i
= 0; i
< nr_maps
; i
++) {
638 struct bpf_map_def def
;
640 def
= *(struct bpf_map_def
*)(obj
->maps_buf
+
641 i
* sizeof(struct bpf_map_def
));
643 *pfd
= bpf_create_map(def
.type
,
651 pr_warning("failed to create map: %s\n",
653 for (j
= 0; j
< i
; j
++)
654 zclose(obj
->map_fds
[j
]);
656 zfree(&obj
->map_fds
);
659 pr_debug("create map: fd=%d\n", *pfd
);
663 zfree(&obj
->maps_buf
);
664 obj
->maps_buf_sz
= 0;
669 bpf_program__relocate(struct bpf_program
*prog
, int *map_fds
)
673 if (!prog
|| !prog
->reloc_desc
)
676 for (i
= 0; i
< prog
->nr_reloc
; i
++) {
677 int insn_idx
, map_idx
;
678 struct bpf_insn
*insns
= prog
->insns
;
680 insn_idx
= prog
->reloc_desc
[i
].insn_idx
;
681 map_idx
= prog
->reloc_desc
[i
].map_idx
;
683 if (insn_idx
>= (int)prog
->insns_cnt
) {
684 pr_warning("relocation out of range: '%s'\n",
688 insns
[insn_idx
].src_reg
= BPF_PSEUDO_MAP_FD
;
689 insns
[insn_idx
].imm
= map_fds
[map_idx
];
692 zfree(&prog
->reloc_desc
);
699 bpf_object__relocate(struct bpf_object
*obj
)
701 struct bpf_program
*prog
;
705 for (i
= 0; i
< obj
->nr_programs
; i
++) {
706 prog
= &obj
->programs
[i
];
708 err
= bpf_program__relocate(prog
, obj
->map_fds
);
710 pr_warning("failed to relocate '%s'\n",
718 static int bpf_object__collect_reloc(struct bpf_object
*obj
)
722 if (!obj_elf_valid(obj
)) {
723 pr_warning("Internal error: elf object is closed\n");
727 for (i
= 0; i
< obj
->efile
.nr_reloc
; i
++) {
728 GElf_Shdr
*shdr
= &obj
->efile
.reloc
[i
].shdr
;
729 Elf_Data
*data
= obj
->efile
.reloc
[i
].data
;
730 int idx
= shdr
->sh_info
;
731 struct bpf_program
*prog
;
732 size_t nr_maps
= obj
->maps_buf_sz
/
733 sizeof(struct bpf_map_def
);
735 if (shdr
->sh_type
!= SHT_REL
) {
736 pr_warning("internal error at %d\n", __LINE__
);
740 prog
= bpf_object__find_prog_by_idx(obj
, idx
);
742 pr_warning("relocation failed: no %d section\n",
747 err
= bpf_program__collect_reloc(prog
, nr_maps
,
757 load_program(struct bpf_insn
*insns
, int insns_cnt
,
758 char *license
, u32 kern_version
, int *pfd
)
763 if (!insns
|| !insns_cnt
)
766 log_buf
= malloc(BPF_LOG_BUF_SIZE
);
768 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
770 ret
= bpf_load_program(BPF_PROG_TYPE_KPROBE
, insns
,
771 insns_cnt
, license
, kern_version
,
772 log_buf
, BPF_LOG_BUF_SIZE
);
781 pr_warning("load bpf program failed: %s\n", strerror(errno
));
784 pr_warning("-- BEGIN DUMP LOG ---\n");
785 pr_warning("\n%s\n", log_buf
);
786 pr_warning("-- END LOG --\n");
795 bpf_program__load(struct bpf_program
*prog
,
796 char *license
, u32 kern_version
)
800 err
= load_program(prog
->insns
, prog
->insns_cnt
,
801 license
, kern_version
, &fd
);
806 pr_warning("failed to load program '%s'\n",
814 bpf_object__load_progs(struct bpf_object
*obj
)
819 for (i
= 0; i
< obj
->nr_programs
; i
++) {
820 err
= bpf_program__load(&obj
->programs
[i
],
829 static int bpf_object__validate(struct bpf_object
*obj
)
831 if (obj
->kern_version
== 0) {
832 pr_warning("%s doesn't provide kernel version\n",
839 static struct bpf_object
*
840 __bpf_object__open(const char *path
, void *obj_buf
, size_t obj_buf_sz
)
842 struct bpf_object
*obj
;
844 if (elf_version(EV_CURRENT
) == EV_NONE
) {
845 pr_warning("failed to init libelf for %s\n", path
);
849 obj
= bpf_object__new(path
, obj_buf
, obj_buf_sz
);
853 if (bpf_object__elf_init(obj
))
855 if (bpf_object__check_endianness(obj
))
857 if (bpf_object__elf_collect(obj
))
859 if (bpf_object__collect_reloc(obj
))
861 if (bpf_object__validate(obj
))
864 bpf_object__elf_finish(obj
);
867 bpf_object__close(obj
);
871 struct bpf_object
*bpf_object__open(const char *path
)
873 /* param validation */
877 pr_debug("loading %s\n", path
);
879 return __bpf_object__open(path
, NULL
, 0);
882 struct bpf_object
*bpf_object__open_buffer(void *obj_buf
,
888 /* param validation */
889 if (!obj_buf
|| obj_buf_sz
<= 0)
893 snprintf(tmp_name
, sizeof(tmp_name
), "%lx-%lx",
894 (unsigned long)obj_buf
,
895 (unsigned long)obj_buf_sz
);
896 tmp_name
[sizeof(tmp_name
) - 1] = '\0';
899 pr_debug("loading object '%s' from buffer\n",
902 return __bpf_object__open(name
, obj_buf
, obj_buf_sz
);
905 int bpf_object__unload(struct bpf_object
*obj
)
912 for (i
= 0; i
< obj
->nr_map_fds
; i
++)
913 zclose(obj
->map_fds
[i
]);
914 zfree(&obj
->map_fds
);
917 for (i
= 0; i
< obj
->nr_programs
; i
++)
918 bpf_program__unload(&obj
->programs
[i
]);
923 int bpf_object__load(struct bpf_object
*obj
)
929 pr_warning("object should not be loaded twice\n");
934 if (bpf_object__create_maps(obj
))
936 if (bpf_object__relocate(obj
))
938 if (bpf_object__load_progs(obj
))
943 bpf_object__unload(obj
);
944 pr_warning("failed to load object '%s'\n", obj
->path
);
948 void bpf_object__close(struct bpf_object
*obj
)
955 bpf_object__elf_finish(obj
);
956 bpf_object__unload(obj
);
958 zfree(&obj
->maps_buf
);
960 if (obj
->programs
&& obj
->nr_programs
) {
961 for (i
= 0; i
< obj
->nr_programs
; i
++)
962 bpf_program__exit(&obj
->programs
[i
]);
964 zfree(&obj
->programs
);
966 list_del(&obj
->list
);
971 bpf_object__next(struct bpf_object
*prev
)
973 struct bpf_object
*next
;
976 next
= list_first_entry(&bpf_objects_list
,
980 next
= list_next_entry(prev
, list
);
982 /* Empty list is noticed here so don't need checking on entry. */
983 if (&next
->list
== &bpf_objects_list
)
990 bpf_object__get_name(struct bpf_object
*obj
)
998 bpf_program__next(struct bpf_program
*prev
, struct bpf_object
*obj
)
1006 return &obj
->programs
[0];
1008 if (prev
->obj
!= obj
) {
1009 pr_warning("error: program handler doesn't match object\n");
1013 idx
= (prev
- obj
->programs
) + 1;
1014 if (idx
>= obj
->nr_programs
)
1016 return &obj
->programs
[idx
];
1019 int bpf_program__set_private(struct bpf_program
*prog
,
1021 bpf_program_clear_priv_t clear_priv
)
1023 if (prog
->priv
&& prog
->clear_priv
)
1024 prog
->clear_priv(prog
, prog
->priv
);
1027 prog
->clear_priv
= clear_priv
;
1031 int bpf_program__get_private(struct bpf_program
*prog
, void **ppriv
)
1033 *ppriv
= prog
->priv
;
1037 const char *bpf_program__title(struct bpf_program
*prog
, bool dup
)
1041 title
= prog
->section_name
;
1043 title
= strdup(title
);
1045 pr_warning("failed to strdup program title\n");
1053 int bpf_program__fd(struct bpf_program
*prog
)