2 * Common eBPF ELF object loading operations.
4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
6 * Copyright (C) 2015 Huawei Inc.
17 #include <asm/unistd.h>
18 #include <linux/kernel.h>
19 #include <linux/bpf.h>
26 #define __printf(a, b) __attribute__((format(printf, a, b)))
29 static int __base_pr(const char *format
, ...)
34 va_start(args
, format
);
35 err
= vfprintf(stderr
, format
, args
);
40 static __printf(1, 2) libbpf_print_fn_t __pr_warning
= __base_pr
;
41 static __printf(1, 2) libbpf_print_fn_t __pr_info
= __base_pr
;
42 static __printf(1, 2) libbpf_print_fn_t __pr_debug
;
44 #define __pr(func, fmt, ...) \
47 (func)("libbpf: " fmt, ##__VA_ARGS__); \
50 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__)
51 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__)
52 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__)
54 void libbpf_set_print(libbpf_print_fn_t warn
,
55 libbpf_print_fn_t info
,
56 libbpf_print_fn_t debug
)
63 /* Copied from tools/perf/util/util.h */
65 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
69 # define zclose(fd) ({ \
72 ___err = close((fd)); \
77 #ifdef HAVE_LIBELF_MMAP_SUPPORT
78 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
80 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
84 * bpf_prog should be a better name but it has been used in
88 /* Index in elf obj file, for relocation use. */
91 struct bpf_insn
*insns
;
107 struct bpf_program
*programs
;
111 * This field is required because maps_buf will be freed and
112 * maps_buf_sz will be set to 0 after loaded.
118 * Information when doing elf related work. Only valid if fd
136 #define obj_elf_valid(o) ((o)->efile.elf)
138 static void bpf_program__exit(struct bpf_program
*prog
)
143 zfree(&prog
->section_name
);
145 zfree(&prog
->reloc_desc
);
153 bpf_program__init(void *data
, size_t size
, char *name
, int idx
,
154 struct bpf_program
*prog
)
156 if (size
< sizeof(struct bpf_insn
)) {
157 pr_warning("corrupted section '%s'\n", name
);
161 bzero(prog
, sizeof(*prog
));
163 prog
->section_name
= strdup(name
);
164 if (!prog
->section_name
) {
165 pr_warning("failed to alloc name for prog %s\n",
170 prog
->insns
= malloc(size
);
172 pr_warning("failed to alloc insns for %s\n", name
);
175 prog
->insns_cnt
= size
/ sizeof(struct bpf_insn
);
176 memcpy(prog
->insns
, data
,
177 prog
->insns_cnt
* sizeof(struct bpf_insn
));
182 bpf_program__exit(prog
);
187 bpf_object__add_program(struct bpf_object
*obj
, void *data
, size_t size
,
190 struct bpf_program prog
, *progs
;
193 err
= bpf_program__init(data
, size
, name
, idx
, &prog
);
197 progs
= obj
->programs
;
198 nr_progs
= obj
->nr_programs
;
200 progs
= realloc(progs
, sizeof(progs
[0]) * (nr_progs
+ 1));
203 * In this case the original obj->programs
204 * is still valid, so don't need special treat for
205 * bpf_close_object().
207 pr_warning("failed to alloc a new program '%s'\n",
209 bpf_program__exit(&prog
);
213 pr_debug("found program %s\n", prog
.section_name
);
214 obj
->programs
= progs
;
215 obj
->nr_programs
= nr_progs
+ 1;
216 progs
[nr_progs
] = prog
;
220 static struct bpf_object
*bpf_object__new(const char *path
,
224 struct bpf_object
*obj
;
226 obj
= calloc(1, sizeof(struct bpf_object
) + strlen(path
) + 1);
228 pr_warning("alloc memory failed for %s\n", path
);
232 strcpy(obj
->path
, path
);
236 * Caller of this function should also calls
237 * bpf_object__elf_finish() after data collection to return
238 * obj_buf to user. If not, we should duplicate the buffer to
239 * avoid user freeing them before elf finish.
241 obj
->efile
.obj_buf
= obj_buf
;
242 obj
->efile
.obj_buf_sz
= obj_buf_sz
;
248 static void bpf_object__elf_finish(struct bpf_object
*obj
)
250 if (!obj_elf_valid(obj
))
253 if (obj
->efile
.elf
) {
254 elf_end(obj
->efile
.elf
);
255 obj
->efile
.elf
= NULL
;
257 obj
->efile
.symbols
= NULL
;
259 zfree(&obj
->efile
.reloc
);
260 obj
->efile
.nr_reloc
= 0;
261 zclose(obj
->efile
.fd
);
262 obj
->efile
.obj_buf
= NULL
;
263 obj
->efile
.obj_buf_sz
= 0;
266 static int bpf_object__elf_init(struct bpf_object
*obj
)
271 if (obj_elf_valid(obj
)) {
272 pr_warning("elf init: internal error\n");
276 if (obj
->efile
.obj_buf_sz
> 0) {
278 * obj_buf should have been validated by
279 * bpf_object__open_buffer().
281 obj
->efile
.elf
= elf_memory(obj
->efile
.obj_buf
,
282 obj
->efile
.obj_buf_sz
);
284 obj
->efile
.fd
= open(obj
->path
, O_RDONLY
);
285 if (obj
->efile
.fd
< 0) {
286 pr_warning("failed to open %s: %s\n", obj
->path
,
291 obj
->efile
.elf
= elf_begin(obj
->efile
.fd
,
292 LIBBPF_ELF_C_READ_MMAP
,
296 if (!obj
->efile
.elf
) {
297 pr_warning("failed to open %s as ELF file\n",
303 if (!gelf_getehdr(obj
->efile
.elf
, &obj
->efile
.ehdr
)) {
304 pr_warning("failed to get EHDR from %s\n",
309 ep
= &obj
->efile
.ehdr
;
311 if ((ep
->e_type
!= ET_REL
) || (ep
->e_machine
!= 0)) {
312 pr_warning("%s is not an eBPF object file\n",
320 bpf_object__elf_finish(obj
);
325 bpf_object__check_endianness(struct bpf_object
*obj
)
327 static unsigned int const endian
= 1;
329 switch (obj
->efile
.ehdr
.e_ident
[EI_DATA
]) {
331 /* We are big endian, BPF obj is little endian. */
332 if (*(unsigned char const *)&endian
!= 1)
337 /* We are little endian, BPF obj is big endian. */
338 if (*(unsigned char const *)&endian
!= 0)
348 pr_warning("Error: endianness mismatch.\n");
353 bpf_object__init_license(struct bpf_object
*obj
,
354 void *data
, size_t size
)
356 memcpy(obj
->license
, data
,
357 min(size
, sizeof(obj
->license
) - 1));
358 pr_debug("license of %s is %s\n", obj
->path
, obj
->license
);
363 bpf_object__init_kversion(struct bpf_object
*obj
,
364 void *data
, size_t size
)
368 if (size
!= sizeof(kver
)) {
369 pr_warning("invalid kver section in %s\n", obj
->path
);
372 memcpy(&kver
, data
, sizeof(kver
));
373 obj
->kern_version
= kver
;
374 pr_debug("kernel version of %s is %x\n", obj
->path
,
380 bpf_object__init_maps(struct bpf_object
*obj
, void *data
,
384 pr_debug("%s doesn't need map definition\n",
389 obj
->maps_buf
= malloc(size
);
390 if (!obj
->maps_buf
) {
391 pr_warning("malloc maps failed: %s\n", obj
->path
);
395 obj
->maps_buf_sz
= size
;
396 memcpy(obj
->maps_buf
, data
, size
);
397 pr_debug("maps in %s: %ld bytes\n", obj
->path
, (long)size
);
401 static int bpf_object__elf_collect(struct bpf_object
*obj
)
403 Elf
*elf
= obj
->efile
.elf
;
404 GElf_Ehdr
*ep
= &obj
->efile
.ehdr
;
406 int idx
= 0, err
= 0;
408 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
409 if (!elf_rawdata(elf_getscn(elf
, ep
->e_shstrndx
), NULL
)) {
410 pr_warning("failed to get e_shstrndx from %s\n",
415 while ((scn
= elf_nextscn(elf
, scn
)) != NULL
) {
421 if (gelf_getshdr(scn
, &sh
) != &sh
) {
422 pr_warning("failed to get section header from %s\n",
428 name
= elf_strptr(elf
, ep
->e_shstrndx
, sh
.sh_name
);
430 pr_warning("failed to get section name from %s\n",
436 data
= elf_getdata(scn
, 0);
438 pr_warning("failed to get section data from %s(%s)\n",
443 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n",
444 name
, (unsigned long)data
->d_size
,
445 (int)sh
.sh_link
, (unsigned long)sh
.sh_flags
,
448 if (strcmp(name
, "license") == 0)
449 err
= bpf_object__init_license(obj
,
452 else if (strcmp(name
, "version") == 0)
453 err
= bpf_object__init_kversion(obj
,
456 else if (strcmp(name
, "maps") == 0)
457 err
= bpf_object__init_maps(obj
, data
->d_buf
,
459 else if (sh
.sh_type
== SHT_SYMTAB
) {
460 if (obj
->efile
.symbols
) {
461 pr_warning("bpf: multiple SYMTAB in %s\n",
465 obj
->efile
.symbols
= data
;
466 } else if ((sh
.sh_type
== SHT_PROGBITS
) &&
467 (sh
.sh_flags
& SHF_EXECINSTR
) &&
468 (data
->d_size
> 0)) {
469 err
= bpf_object__add_program(obj
, data
->d_buf
,
470 data
->d_size
, name
, idx
);
473 strerror_r(-err
, errmsg
, sizeof(errmsg
));
474 pr_warning("failed to alloc program %s (%s): %s",
475 name
, obj
->path
, errmsg
);
477 } else if (sh
.sh_type
== SHT_REL
) {
478 void *reloc
= obj
->efile
.reloc
;
479 int nr_reloc
= obj
->efile
.nr_reloc
+ 1;
481 reloc
= realloc(reloc
,
482 sizeof(*obj
->efile
.reloc
) * nr_reloc
);
484 pr_warning("realloc failed\n");
487 int n
= nr_reloc
- 1;
489 obj
->efile
.reloc
= reloc
;
490 obj
->efile
.nr_reloc
= nr_reloc
;
492 obj
->efile
.reloc
[n
].shdr
= sh
;
493 obj
->efile
.reloc
[n
].data
= data
;
503 static struct bpf_program
*
504 bpf_object__find_prog_by_idx(struct bpf_object
*obj
, int idx
)
506 struct bpf_program
*prog
;
509 for (i
= 0; i
< obj
->nr_programs
; i
++) {
510 prog
= &obj
->programs
[i
];
511 if (prog
->idx
== idx
)
518 bpf_program__collect_reloc(struct bpf_program
*prog
,
519 size_t nr_maps
, GElf_Shdr
*shdr
,
520 Elf_Data
*data
, Elf_Data
*symbols
)
524 pr_debug("collecting relocating info for: '%s'\n",
526 nrels
= shdr
->sh_size
/ shdr
->sh_entsize
;
528 prog
->reloc_desc
= malloc(sizeof(*prog
->reloc_desc
) * nrels
);
529 if (!prog
->reloc_desc
) {
530 pr_warning("failed to alloc memory in relocation\n");
533 prog
->nr_reloc
= nrels
;
535 for (i
= 0; i
< nrels
; i
++) {
538 unsigned int insn_idx
;
539 struct bpf_insn
*insns
= prog
->insns
;
542 if (!gelf_getrel(data
, i
, &rel
)) {
543 pr_warning("relocation: failed to get %d reloc\n", i
);
547 insn_idx
= rel
.r_offset
/ sizeof(struct bpf_insn
);
548 pr_debug("relocation: insn_idx=%u\n", insn_idx
);
550 if (!gelf_getsym(symbols
,
551 GELF_R_SYM(rel
.r_info
),
553 pr_warning("relocation: symbol %"PRIx64
" not found\n",
554 GELF_R_SYM(rel
.r_info
));
558 if (insns
[insn_idx
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
)) {
559 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
560 insn_idx
, insns
[insn_idx
].code
);
564 map_idx
= sym
.st_value
/ sizeof(struct bpf_map_def
);
565 if (map_idx
>= nr_maps
) {
566 pr_warning("bpf relocation: map_idx %d large than %d\n",
567 (int)map_idx
, (int)nr_maps
- 1);
571 prog
->reloc_desc
[i
].insn_idx
= insn_idx
;
572 prog
->reloc_desc
[i
].map_idx
= map_idx
;
578 bpf_object__create_maps(struct bpf_object
*obj
)
584 nr_maps
= obj
->maps_buf_sz
/ sizeof(struct bpf_map_def
);
585 if (!obj
->maps_buf
|| !nr_maps
) {
586 pr_debug("don't need create maps for %s\n",
591 obj
->map_fds
= malloc(sizeof(int) * nr_maps
);
593 pr_warning("realloc perf_bpf_map_fds failed\n");
596 obj
->nr_map_fds
= nr_maps
;
598 /* fill all fd with -1 */
599 memset(obj
->map_fds
, -1, sizeof(int) * nr_maps
);
602 for (i
= 0; i
< nr_maps
; i
++) {
603 struct bpf_map_def def
;
605 def
= *(struct bpf_map_def
*)(obj
->maps_buf
+
606 i
* sizeof(struct bpf_map_def
));
608 *pfd
= bpf_create_map(def
.type
,
616 pr_warning("failed to create map: %s\n",
618 for (j
= 0; j
< i
; j
++)
619 zclose(obj
->map_fds
[j
]);
621 zfree(&obj
->map_fds
);
624 pr_debug("create map: fd=%d\n", *pfd
);
628 zfree(&obj
->maps_buf
);
629 obj
->maps_buf_sz
= 0;
633 static int bpf_object__collect_reloc(struct bpf_object
*obj
)
637 if (!obj_elf_valid(obj
)) {
638 pr_warning("Internal error: elf object is closed\n");
642 for (i
= 0; i
< obj
->efile
.nr_reloc
; i
++) {
643 GElf_Shdr
*shdr
= &obj
->efile
.reloc
[i
].shdr
;
644 Elf_Data
*data
= obj
->efile
.reloc
[i
].data
;
645 int idx
= shdr
->sh_info
;
646 struct bpf_program
*prog
;
647 size_t nr_maps
= obj
->maps_buf_sz
/
648 sizeof(struct bpf_map_def
);
650 if (shdr
->sh_type
!= SHT_REL
) {
651 pr_warning("internal error at %d\n", __LINE__
);
655 prog
= bpf_object__find_prog_by_idx(obj
, idx
);
657 pr_warning("relocation failed: no %d section\n",
662 err
= bpf_program__collect_reloc(prog
, nr_maps
,
671 static int bpf_object__validate(struct bpf_object
*obj
)
673 if (obj
->kern_version
== 0) {
674 pr_warning("%s doesn't provide kernel version\n",
681 static struct bpf_object
*
682 __bpf_object__open(const char *path
, void *obj_buf
, size_t obj_buf_sz
)
684 struct bpf_object
*obj
;
686 if (elf_version(EV_CURRENT
) == EV_NONE
) {
687 pr_warning("failed to init libelf for %s\n", path
);
691 obj
= bpf_object__new(path
, obj_buf
, obj_buf_sz
);
695 if (bpf_object__elf_init(obj
))
697 if (bpf_object__check_endianness(obj
))
699 if (bpf_object__elf_collect(obj
))
701 if (bpf_object__collect_reloc(obj
))
703 if (bpf_object__validate(obj
))
706 bpf_object__elf_finish(obj
);
709 bpf_object__close(obj
);
713 struct bpf_object
*bpf_object__open(const char *path
)
715 /* param validation */
719 pr_debug("loading %s\n", path
);
721 return __bpf_object__open(path
, NULL
, 0);
724 struct bpf_object
*bpf_object__open_buffer(void *obj_buf
,
727 /* param validation */
728 if (!obj_buf
|| obj_buf_sz
<= 0)
731 pr_debug("loading object from buffer\n");
733 return __bpf_object__open("[buffer]", obj_buf
, obj_buf_sz
);
736 int bpf_object__unload(struct bpf_object
*obj
)
743 for (i
= 0; i
< obj
->nr_map_fds
; i
++)
744 zclose(obj
->map_fds
[i
]);
745 zfree(&obj
->map_fds
);
751 int bpf_object__load(struct bpf_object
*obj
)
757 pr_warning("object should not be loaded twice\n");
762 if (bpf_object__create_maps(obj
))
767 bpf_object__unload(obj
);
768 pr_warning("failed to load object '%s'\n", obj
->path
);
772 void bpf_object__close(struct bpf_object
*obj
)
779 bpf_object__elf_finish(obj
);
780 bpf_object__unload(obj
);
782 zfree(&obj
->maps_buf
);
784 if (obj
->programs
&& obj
->nr_programs
) {
785 for (i
= 0; i
< obj
->nr_programs
; i
++)
786 bpf_program__exit(&obj
->programs
[i
]);
788 zfree(&obj
->programs
);