2 * Common eBPF ELF object loading operations.
4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
6 * Copyright (C) 2015 Huawei Inc.
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation;
11 * version 2.1 of the License (not later!)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with this program; if not, see <http://www.gnu.org/licenses>
30 #include <asm/unistd.h>
31 #include <linux/kernel.h>
32 #include <linux/bpf.h>
33 #include <linux/list.h>
44 #define __printf(a, b) __attribute__((format(printf, a, b)))
47 static int __base_pr(const char *format
, ...)
52 va_start(args
, format
);
53 err
= vfprintf(stderr
, format
, args
);
58 static __printf(1, 2) libbpf_print_fn_t __pr_warning
= __base_pr
;
59 static __printf(1, 2) libbpf_print_fn_t __pr_info
= __base_pr
;
60 static __printf(1, 2) libbpf_print_fn_t __pr_debug
;
62 #define __pr(func, fmt, ...) \
65 (func)("libbpf: " fmt, ##__VA_ARGS__); \
68 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__)
69 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__)
70 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__)
72 void libbpf_set_print(libbpf_print_fn_t warn
,
73 libbpf_print_fn_t info
,
74 libbpf_print_fn_t debug
)
81 #define STRERR_BUFSIZE 128
83 #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START)
84 #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c)
85 #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START)
87 static const char *libbpf_strerror_table
[NR_ERRNO
] = {
88 [ERRCODE_OFFSET(LIBELF
)] = "Something wrong in libelf",
89 [ERRCODE_OFFSET(FORMAT
)] = "BPF object format invalid",
90 [ERRCODE_OFFSET(KVERSION
)] = "'version' section incorrect or lost",
91 [ERRCODE_OFFSET(ENDIAN
)] = "Endian mismatch",
92 [ERRCODE_OFFSET(INTERNAL
)] = "Internal error in libbpf",
93 [ERRCODE_OFFSET(RELOC
)] = "Relocation failed",
94 [ERRCODE_OFFSET(VERIFY
)] = "Kernel verifier blocks program loading",
95 [ERRCODE_OFFSET(PROG2BIG
)] = "Program too big",
96 [ERRCODE_OFFSET(KVER
)] = "Incorrect kernel version",
97 [ERRCODE_OFFSET(PROGTYPE
)] = "Kernel doesn't support this program type",
100 int libbpf_strerror(int err
, char *buf
, size_t size
)
105 err
= err
> 0 ? err
: -err
;
107 if (err
< __LIBBPF_ERRNO__START
) {
110 ret
= strerror_r(err
, buf
, size
);
111 buf
[size
- 1] = '\0';
115 if (err
< __LIBBPF_ERRNO__END
) {
118 msg
= libbpf_strerror_table
[ERRNO_OFFSET(err
)];
119 snprintf(buf
, size
, "%s", msg
);
120 buf
[size
- 1] = '\0';
124 snprintf(buf
, size
, "Unknown libbpf error %d", err
);
125 buf
[size
- 1] = '\0';
129 #define CHECK_ERR(action, err, out) do { \
136 /* Copied from tools/perf/util/util.h */
138 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
142 # define zclose(fd) ({ \
145 ___err = close((fd)); \
150 #ifdef HAVE_LIBELF_MMAP_SUPPORT
151 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
153 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
157 * bpf_prog should be a better name but it has been used in
161 /* Index in elf obj file, for relocation use. */
164 struct bpf_insn
*insns
;
166 enum bpf_prog_type type
;
178 bpf_program_prep_t preprocessor
;
180 struct bpf_object
*obj
;
182 bpf_program_clear_priv_t clear_priv
;
188 struct bpf_map_def def
;
190 bpf_map_clear_priv_t clear_priv
;
193 static LIST_HEAD(bpf_objects_list
);
199 struct bpf_program
*programs
;
201 struct bpf_map
*maps
;
207 * Information when doing elf related work. Only valid if fd
226 * All loaded bpf_object is linked in a list, which is
227 * hidden to caller. bpf_objects__<func> handlers deal with
230 struct list_head list
;
233 #define obj_elf_valid(o) ((o)->efile.elf)
235 static void bpf_program__unload(struct bpf_program
*prog
)
243 * If the object is opened but the program was never loaded,
244 * it is possible that prog->instances.nr == -1.
246 if (prog
->instances
.nr
> 0) {
247 for (i
= 0; i
< prog
->instances
.nr
; i
++)
248 zclose(prog
->instances
.fds
[i
]);
249 } else if (prog
->instances
.nr
!= -1) {
250 pr_warning("Internal error: instances.nr is %d\n",
254 prog
->instances
.nr
= -1;
255 zfree(&prog
->instances
.fds
);
258 static void bpf_program__exit(struct bpf_program
*prog
)
263 if (prog
->clear_priv
)
264 prog
->clear_priv(prog
, prog
->priv
);
267 prog
->clear_priv
= NULL
;
269 bpf_program__unload(prog
);
270 zfree(&prog
->section_name
);
272 zfree(&prog
->reloc_desc
);
280 bpf_program__init(void *data
, size_t size
, char *name
, int idx
,
281 struct bpf_program
*prog
)
283 if (size
< sizeof(struct bpf_insn
)) {
284 pr_warning("corrupted section '%s'\n", name
);
288 bzero(prog
, sizeof(*prog
));
290 prog
->section_name
= strdup(name
);
291 if (!prog
->section_name
) {
292 pr_warning("failed to alloc name for prog %s\n",
297 prog
->insns
= malloc(size
);
299 pr_warning("failed to alloc insns for %s\n", name
);
302 prog
->insns_cnt
= size
/ sizeof(struct bpf_insn
);
303 memcpy(prog
->insns
, data
,
304 prog
->insns_cnt
* sizeof(struct bpf_insn
));
306 prog
->instances
.fds
= NULL
;
307 prog
->instances
.nr
= -1;
308 prog
->type
= BPF_PROG_TYPE_KPROBE
;
312 bpf_program__exit(prog
);
317 bpf_object__add_program(struct bpf_object
*obj
, void *data
, size_t size
,
320 struct bpf_program prog
, *progs
;
323 err
= bpf_program__init(data
, size
, name
, idx
, &prog
);
327 progs
= obj
->programs
;
328 nr_progs
= obj
->nr_programs
;
330 progs
= realloc(progs
, sizeof(progs
[0]) * (nr_progs
+ 1));
333 * In this case the original obj->programs
334 * is still valid, so don't need special treat for
335 * bpf_close_object().
337 pr_warning("failed to alloc a new program '%s'\n",
339 bpf_program__exit(&prog
);
343 pr_debug("found program %s\n", prog
.section_name
);
344 obj
->programs
= progs
;
345 obj
->nr_programs
= nr_progs
+ 1;
347 progs
[nr_progs
] = prog
;
351 static struct bpf_object
*bpf_object__new(const char *path
,
355 struct bpf_object
*obj
;
357 obj
= calloc(1, sizeof(struct bpf_object
) + strlen(path
) + 1);
359 pr_warning("alloc memory failed for %s\n", path
);
360 return ERR_PTR(-ENOMEM
);
363 strcpy(obj
->path
, path
);
367 * Caller of this function should also calls
368 * bpf_object__elf_finish() after data collection to return
369 * obj_buf to user. If not, we should duplicate the buffer to
370 * avoid user freeing them before elf finish.
372 obj
->efile
.obj_buf
= obj_buf
;
373 obj
->efile
.obj_buf_sz
= obj_buf_sz
;
374 obj
->efile
.maps_shndx
= -1;
378 INIT_LIST_HEAD(&obj
->list
);
379 list_add(&obj
->list
, &bpf_objects_list
);
383 static void bpf_object__elf_finish(struct bpf_object
*obj
)
385 if (!obj_elf_valid(obj
))
388 if (obj
->efile
.elf
) {
389 elf_end(obj
->efile
.elf
);
390 obj
->efile
.elf
= NULL
;
392 obj
->efile
.symbols
= NULL
;
394 zfree(&obj
->efile
.reloc
);
395 obj
->efile
.nr_reloc
= 0;
396 zclose(obj
->efile
.fd
);
397 obj
->efile
.obj_buf
= NULL
;
398 obj
->efile
.obj_buf_sz
= 0;
401 static int bpf_object__elf_init(struct bpf_object
*obj
)
406 if (obj_elf_valid(obj
)) {
407 pr_warning("elf init: internal error\n");
408 return -LIBBPF_ERRNO__LIBELF
;
411 if (obj
->efile
.obj_buf_sz
> 0) {
413 * obj_buf should have been validated by
414 * bpf_object__open_buffer().
416 obj
->efile
.elf
= elf_memory(obj
->efile
.obj_buf
,
417 obj
->efile
.obj_buf_sz
);
419 obj
->efile
.fd
= open(obj
->path
, O_RDONLY
);
420 if (obj
->efile
.fd
< 0) {
421 pr_warning("failed to open %s: %s\n", obj
->path
,
426 obj
->efile
.elf
= elf_begin(obj
->efile
.fd
,
427 LIBBPF_ELF_C_READ_MMAP
,
431 if (!obj
->efile
.elf
) {
432 pr_warning("failed to open %s as ELF file\n",
434 err
= -LIBBPF_ERRNO__LIBELF
;
438 if (!gelf_getehdr(obj
->efile
.elf
, &obj
->efile
.ehdr
)) {
439 pr_warning("failed to get EHDR from %s\n",
441 err
= -LIBBPF_ERRNO__FORMAT
;
444 ep
= &obj
->efile
.ehdr
;
446 /* Old LLVM set e_machine to EM_NONE */
447 if ((ep
->e_type
!= ET_REL
) || (ep
->e_machine
&& (ep
->e_machine
!= EM_BPF
))) {
448 pr_warning("%s is not an eBPF object file\n",
450 err
= -LIBBPF_ERRNO__FORMAT
;
456 bpf_object__elf_finish(obj
);
461 bpf_object__check_endianness(struct bpf_object
*obj
)
463 static unsigned int const endian
= 1;
465 switch (obj
->efile
.ehdr
.e_ident
[EI_DATA
]) {
467 /* We are big endian, BPF obj is little endian. */
468 if (*(unsigned char const *)&endian
!= 1)
473 /* We are little endian, BPF obj is big endian. */
474 if (*(unsigned char const *)&endian
!= 0)
478 return -LIBBPF_ERRNO__ENDIAN
;
484 pr_warning("Error: endianness mismatch.\n");
485 return -LIBBPF_ERRNO__ENDIAN
;
489 bpf_object__init_license(struct bpf_object
*obj
,
490 void *data
, size_t size
)
492 memcpy(obj
->license
, data
,
493 min(size
, sizeof(obj
->license
) - 1));
494 pr_debug("license of %s is %s\n", obj
->path
, obj
->license
);
499 bpf_object__init_kversion(struct bpf_object
*obj
,
500 void *data
, size_t size
)
504 if (size
!= sizeof(kver
)) {
505 pr_warning("invalid kver section in %s\n", obj
->path
);
506 return -LIBBPF_ERRNO__FORMAT
;
508 memcpy(&kver
, data
, sizeof(kver
));
509 obj
->kern_version
= kver
;
510 pr_debug("kernel version of %s is %x\n", obj
->path
,
516 bpf_object__init_maps(struct bpf_object
*obj
, void *data
,
522 nr_maps
= size
/ sizeof(struct bpf_map_def
);
523 if (!data
|| !nr_maps
) {
524 pr_debug("%s doesn't need map definition\n",
529 pr_debug("maps in %s: %zd bytes\n", obj
->path
, size
);
531 obj
->maps
= calloc(nr_maps
, sizeof(obj
->maps
[0]));
533 pr_warning("alloc maps for object failed\n");
536 obj
->nr_maps
= nr_maps
;
538 for (i
= 0; i
< nr_maps
; i
++) {
539 struct bpf_map_def
*def
= &obj
->maps
[i
].def
;
542 * fill all fd with -1 so won't close incorrect
543 * fd (fd=0 is stdin) when failure (zclose won't close
546 obj
->maps
[i
].fd
= -1;
548 /* Save map definition into obj->maps */
549 *def
= ((struct bpf_map_def
*)data
)[i
];
555 bpf_object__init_maps_name(struct bpf_object
*obj
)
558 Elf_Data
*symbols
= obj
->efile
.symbols
;
560 if (!symbols
|| obj
->efile
.maps_shndx
< 0)
563 for (i
= 0; i
< symbols
->d_size
/ sizeof(GElf_Sym
); i
++) {
566 const char *map_name
;
568 if (!gelf_getsym(symbols
, i
, &sym
))
570 if (sym
.st_shndx
!= obj
->efile
.maps_shndx
)
573 map_name
= elf_strptr(obj
->efile
.elf
,
574 obj
->efile
.strtabidx
,
576 map_idx
= sym
.st_value
/ sizeof(struct bpf_map_def
);
577 if (map_idx
>= obj
->nr_maps
) {
578 pr_warning("index of map \"%s\" is buggy: %zu > %zu\n",
579 map_name
, map_idx
, obj
->nr_maps
);
582 obj
->maps
[map_idx
].name
= strdup(map_name
);
583 if (!obj
->maps
[map_idx
].name
) {
584 pr_warning("failed to alloc map name\n");
587 pr_debug("map %zu is \"%s\"\n", map_idx
,
588 obj
->maps
[map_idx
].name
);
593 static int bpf_object__elf_collect(struct bpf_object
*obj
)
595 Elf
*elf
= obj
->efile
.elf
;
596 GElf_Ehdr
*ep
= &obj
->efile
.ehdr
;
598 int idx
= 0, err
= 0;
600 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
601 if (!elf_rawdata(elf_getscn(elf
, ep
->e_shstrndx
), NULL
)) {
602 pr_warning("failed to get e_shstrndx from %s\n",
604 return -LIBBPF_ERRNO__FORMAT
;
607 while ((scn
= elf_nextscn(elf
, scn
)) != NULL
) {
613 if (gelf_getshdr(scn
, &sh
) != &sh
) {
614 pr_warning("failed to get section header from %s\n",
616 err
= -LIBBPF_ERRNO__FORMAT
;
620 name
= elf_strptr(elf
, ep
->e_shstrndx
, sh
.sh_name
);
622 pr_warning("failed to get section name from %s\n",
624 err
= -LIBBPF_ERRNO__FORMAT
;
628 data
= elf_getdata(scn
, 0);
630 pr_warning("failed to get section data from %s(%s)\n",
632 err
= -LIBBPF_ERRNO__FORMAT
;
635 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n",
636 name
, (unsigned long)data
->d_size
,
637 (int)sh
.sh_link
, (unsigned long)sh
.sh_flags
,
640 if (strcmp(name
, "license") == 0)
641 err
= bpf_object__init_license(obj
,
644 else if (strcmp(name
, "version") == 0)
645 err
= bpf_object__init_kversion(obj
,
648 else if (strcmp(name
, "maps") == 0) {
649 err
= bpf_object__init_maps(obj
, data
->d_buf
,
651 obj
->efile
.maps_shndx
= idx
;
652 } else if (sh
.sh_type
== SHT_SYMTAB
) {
653 if (obj
->efile
.symbols
) {
654 pr_warning("bpf: multiple SYMTAB in %s\n",
656 err
= -LIBBPF_ERRNO__FORMAT
;
658 obj
->efile
.symbols
= data
;
659 obj
->efile
.strtabidx
= sh
.sh_link
;
661 } else if ((sh
.sh_type
== SHT_PROGBITS
) &&
662 (sh
.sh_flags
& SHF_EXECINSTR
) &&
663 (data
->d_size
> 0)) {
664 err
= bpf_object__add_program(obj
, data
->d_buf
,
665 data
->d_size
, name
, idx
);
667 char errmsg
[STRERR_BUFSIZE
];
669 strerror_r(-err
, errmsg
, sizeof(errmsg
));
670 pr_warning("failed to alloc program %s (%s): %s",
671 name
, obj
->path
, errmsg
);
673 } else if (sh
.sh_type
== SHT_REL
) {
674 void *reloc
= obj
->efile
.reloc
;
675 int nr_reloc
= obj
->efile
.nr_reloc
+ 1;
677 reloc
= realloc(reloc
,
678 sizeof(*obj
->efile
.reloc
) * nr_reloc
);
680 pr_warning("realloc failed\n");
683 int n
= nr_reloc
- 1;
685 obj
->efile
.reloc
= reloc
;
686 obj
->efile
.nr_reloc
= nr_reloc
;
688 obj
->efile
.reloc
[n
].shdr
= sh
;
689 obj
->efile
.reloc
[n
].data
= data
;
696 if (!obj
->efile
.strtabidx
|| obj
->efile
.strtabidx
>= idx
) {
697 pr_warning("Corrupted ELF file: index of strtab invalid\n");
698 return LIBBPF_ERRNO__FORMAT
;
700 if (obj
->efile
.maps_shndx
>= 0)
701 err
= bpf_object__init_maps_name(obj
);
706 static struct bpf_program
*
707 bpf_object__find_prog_by_idx(struct bpf_object
*obj
, int idx
)
709 struct bpf_program
*prog
;
712 for (i
= 0; i
< obj
->nr_programs
; i
++) {
713 prog
= &obj
->programs
[i
];
714 if (prog
->idx
== idx
)
721 bpf_program__collect_reloc(struct bpf_program
*prog
,
722 size_t nr_maps
, GElf_Shdr
*shdr
,
723 Elf_Data
*data
, Elf_Data
*symbols
,
728 pr_debug("collecting relocating info for: '%s'\n",
730 nrels
= shdr
->sh_size
/ shdr
->sh_entsize
;
732 prog
->reloc_desc
= malloc(sizeof(*prog
->reloc_desc
) * nrels
);
733 if (!prog
->reloc_desc
) {
734 pr_warning("failed to alloc memory in relocation\n");
737 prog
->nr_reloc
= nrels
;
739 for (i
= 0; i
< nrels
; i
++) {
742 unsigned int insn_idx
;
743 struct bpf_insn
*insns
= prog
->insns
;
746 if (!gelf_getrel(data
, i
, &rel
)) {
747 pr_warning("relocation: failed to get %d reloc\n", i
);
748 return -LIBBPF_ERRNO__FORMAT
;
751 if (!gelf_getsym(symbols
,
752 GELF_R_SYM(rel
.r_info
),
754 pr_warning("relocation: symbol %"PRIx64
" not found\n",
755 GELF_R_SYM(rel
.r_info
));
756 return -LIBBPF_ERRNO__FORMAT
;
759 if (sym
.st_shndx
!= maps_shndx
) {
760 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
761 prog
->section_name
, sym
.st_shndx
);
762 return -LIBBPF_ERRNO__RELOC
;
765 insn_idx
= rel
.r_offset
/ sizeof(struct bpf_insn
);
766 pr_debug("relocation: insn_idx=%u\n", insn_idx
);
768 if (insns
[insn_idx
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
)) {
769 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
770 insn_idx
, insns
[insn_idx
].code
);
771 return -LIBBPF_ERRNO__RELOC
;
774 map_idx
= sym
.st_value
/ sizeof(struct bpf_map_def
);
775 if (map_idx
>= nr_maps
) {
776 pr_warning("bpf relocation: map_idx %d large than %d\n",
777 (int)map_idx
, (int)nr_maps
- 1);
778 return -LIBBPF_ERRNO__RELOC
;
781 prog
->reloc_desc
[i
].insn_idx
= insn_idx
;
782 prog
->reloc_desc
[i
].map_idx
= map_idx
;
788 bpf_object__create_maps(struct bpf_object
*obj
)
792 for (i
= 0; i
< obj
->nr_maps
; i
++) {
793 struct bpf_map_def
*def
= &obj
->maps
[i
].def
;
794 int *pfd
= &obj
->maps
[i
].fd
;
796 *pfd
= bpf_create_map(def
->type
,
804 pr_warning("failed to create map: %s\n",
806 for (j
= 0; j
< i
; j
++)
807 zclose(obj
->maps
[j
].fd
);
810 pr_debug("create map: fd=%d\n", *pfd
);
817 bpf_program__relocate(struct bpf_program
*prog
, struct bpf_object
*obj
)
821 if (!prog
|| !prog
->reloc_desc
)
824 for (i
= 0; i
< prog
->nr_reloc
; i
++) {
825 int insn_idx
, map_idx
;
826 struct bpf_insn
*insns
= prog
->insns
;
828 insn_idx
= prog
->reloc_desc
[i
].insn_idx
;
829 map_idx
= prog
->reloc_desc
[i
].map_idx
;
831 if (insn_idx
>= (int)prog
->insns_cnt
) {
832 pr_warning("relocation out of range: '%s'\n",
834 return -LIBBPF_ERRNO__RELOC
;
836 insns
[insn_idx
].src_reg
= BPF_PSEUDO_MAP_FD
;
837 insns
[insn_idx
].imm
= obj
->maps
[map_idx
].fd
;
840 zfree(&prog
->reloc_desc
);
847 bpf_object__relocate(struct bpf_object
*obj
)
849 struct bpf_program
*prog
;
853 for (i
= 0; i
< obj
->nr_programs
; i
++) {
854 prog
= &obj
->programs
[i
];
856 err
= bpf_program__relocate(prog
, obj
);
858 pr_warning("failed to relocate '%s'\n",
866 static int bpf_object__collect_reloc(struct bpf_object
*obj
)
870 if (!obj_elf_valid(obj
)) {
871 pr_warning("Internal error: elf object is closed\n");
872 return -LIBBPF_ERRNO__INTERNAL
;
875 for (i
= 0; i
< obj
->efile
.nr_reloc
; i
++) {
876 GElf_Shdr
*shdr
= &obj
->efile
.reloc
[i
].shdr
;
877 Elf_Data
*data
= obj
->efile
.reloc
[i
].data
;
878 int idx
= shdr
->sh_info
;
879 struct bpf_program
*prog
;
880 size_t nr_maps
= obj
->nr_maps
;
882 if (shdr
->sh_type
!= SHT_REL
) {
883 pr_warning("internal error at %d\n", __LINE__
);
884 return -LIBBPF_ERRNO__INTERNAL
;
887 prog
= bpf_object__find_prog_by_idx(obj
, idx
);
889 pr_warning("relocation failed: no %d section\n",
891 return -LIBBPF_ERRNO__RELOC
;
894 err
= bpf_program__collect_reloc(prog
, nr_maps
,
897 obj
->efile
.maps_shndx
);
905 load_program(enum bpf_prog_type type
, struct bpf_insn
*insns
,
906 int insns_cnt
, char *license
, u32 kern_version
, int *pfd
)
911 if (!insns
|| !insns_cnt
)
914 log_buf
= malloc(BPF_LOG_BUF_SIZE
);
916 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
918 ret
= bpf_load_program(type
, insns
, insns_cnt
, license
,
919 kern_version
, log_buf
, BPF_LOG_BUF_SIZE
);
927 ret
= -LIBBPF_ERRNO__LOAD
;
928 pr_warning("load bpf program failed: %s\n", strerror(errno
));
930 if (log_buf
&& log_buf
[0] != '\0') {
931 ret
= -LIBBPF_ERRNO__VERIFY
;
932 pr_warning("-- BEGIN DUMP LOG ---\n");
933 pr_warning("\n%s\n", log_buf
);
934 pr_warning("-- END LOG --\n");
935 } else if (insns_cnt
>= BPF_MAXINSNS
) {
936 pr_warning("Program too large (%d insns), at most %d insns\n",
937 insns_cnt
, BPF_MAXINSNS
);
938 ret
= -LIBBPF_ERRNO__PROG2BIG
;
940 /* Wrong program type? */
941 if (type
!= BPF_PROG_TYPE_KPROBE
) {
944 fd
= bpf_load_program(BPF_PROG_TYPE_KPROBE
, insns
,
945 insns_cnt
, license
, kern_version
,
949 ret
= -LIBBPF_ERRNO__PROGTYPE
;
955 ret
= -LIBBPF_ERRNO__KVER
;
964 bpf_program__load(struct bpf_program
*prog
,
965 char *license
, u32 kern_version
)
969 if (prog
->instances
.nr
< 0 || !prog
->instances
.fds
) {
970 if (prog
->preprocessor
) {
971 pr_warning("Internal error: can't load program '%s'\n",
973 return -LIBBPF_ERRNO__INTERNAL
;
976 prog
->instances
.fds
= malloc(sizeof(int));
977 if (!prog
->instances
.fds
) {
978 pr_warning("Not enough memory for BPF fds\n");
981 prog
->instances
.nr
= 1;
982 prog
->instances
.fds
[0] = -1;
985 if (!prog
->preprocessor
) {
986 if (prog
->instances
.nr
!= 1) {
987 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
988 prog
->section_name
, prog
->instances
.nr
);
990 err
= load_program(prog
->type
, prog
->insns
, prog
->insns_cnt
,
991 license
, kern_version
, &fd
);
993 prog
->instances
.fds
[0] = fd
;
997 for (i
= 0; i
< prog
->instances
.nr
; i
++) {
998 struct bpf_prog_prep_result result
;
999 bpf_program_prep_t preprocessor
= prog
->preprocessor
;
1001 bzero(&result
, sizeof(result
));
1002 err
= preprocessor(prog
, i
, prog
->insns
,
1003 prog
->insns_cnt
, &result
);
1005 pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1006 i
, prog
->section_name
);
1010 if (!result
.new_insn_ptr
|| !result
.new_insn_cnt
) {
1011 pr_debug("Skip loading the %dth instance of program '%s'\n",
1012 i
, prog
->section_name
);
1013 prog
->instances
.fds
[i
] = -1;
1019 err
= load_program(prog
->type
, result
.new_insn_ptr
,
1020 result
.new_insn_cnt
,
1021 license
, kern_version
, &fd
);
1024 pr_warning("Loading the %dth instance of program '%s' failed\n",
1025 i
, prog
->section_name
);
1031 prog
->instances
.fds
[i
] = fd
;
1035 pr_warning("failed to load program '%s'\n",
1036 prog
->section_name
);
1037 zfree(&prog
->insns
);
1038 prog
->insns_cnt
= 0;
1043 bpf_object__load_progs(struct bpf_object
*obj
)
1048 for (i
= 0; i
< obj
->nr_programs
; i
++) {
1049 err
= bpf_program__load(&obj
->programs
[i
],
1058 static int bpf_object__validate(struct bpf_object
*obj
)
1060 if (obj
->kern_version
== 0) {
1061 pr_warning("%s doesn't provide kernel version\n",
1063 return -LIBBPF_ERRNO__KVERSION
;
1068 static struct bpf_object
*
1069 __bpf_object__open(const char *path
, void *obj_buf
, size_t obj_buf_sz
)
1071 struct bpf_object
*obj
;
1074 if (elf_version(EV_CURRENT
) == EV_NONE
) {
1075 pr_warning("failed to init libelf for %s\n", path
);
1076 return ERR_PTR(-LIBBPF_ERRNO__LIBELF
);
1079 obj
= bpf_object__new(path
, obj_buf
, obj_buf_sz
);
1083 CHECK_ERR(bpf_object__elf_init(obj
), err
, out
);
1084 CHECK_ERR(bpf_object__check_endianness(obj
), err
, out
);
1085 CHECK_ERR(bpf_object__elf_collect(obj
), err
, out
);
1086 CHECK_ERR(bpf_object__collect_reloc(obj
), err
, out
);
1087 CHECK_ERR(bpf_object__validate(obj
), err
, out
);
1089 bpf_object__elf_finish(obj
);
1092 bpf_object__close(obj
);
1093 return ERR_PTR(err
);
1096 struct bpf_object
*bpf_object__open(const char *path
)
1098 /* param validation */
1102 pr_debug("loading %s\n", path
);
1104 return __bpf_object__open(path
, NULL
, 0);
1107 struct bpf_object
*bpf_object__open_buffer(void *obj_buf
,
1113 /* param validation */
1114 if (!obj_buf
|| obj_buf_sz
<= 0)
1118 snprintf(tmp_name
, sizeof(tmp_name
), "%lx-%lx",
1119 (unsigned long)obj_buf
,
1120 (unsigned long)obj_buf_sz
);
1121 tmp_name
[sizeof(tmp_name
) - 1] = '\0';
1124 pr_debug("loading object '%s' from buffer\n",
1127 return __bpf_object__open(name
, obj_buf
, obj_buf_sz
);
1130 int bpf_object__unload(struct bpf_object
*obj
)
1137 for (i
= 0; i
< obj
->nr_maps
; i
++)
1138 zclose(obj
->maps
[i
].fd
);
1140 for (i
= 0; i
< obj
->nr_programs
; i
++)
1141 bpf_program__unload(&obj
->programs
[i
]);
1146 int bpf_object__load(struct bpf_object
*obj
)
1154 pr_warning("object should not be loaded twice\n");
1160 CHECK_ERR(bpf_object__create_maps(obj
), err
, out
);
1161 CHECK_ERR(bpf_object__relocate(obj
), err
, out
);
1162 CHECK_ERR(bpf_object__load_progs(obj
), err
, out
);
1166 bpf_object__unload(obj
);
1167 pr_warning("failed to load object '%s'\n", obj
->path
);
1171 void bpf_object__close(struct bpf_object
*obj
)
1178 bpf_object__elf_finish(obj
);
1179 bpf_object__unload(obj
);
1181 for (i
= 0; i
< obj
->nr_maps
; i
++) {
1182 zfree(&obj
->maps
[i
].name
);
1183 if (obj
->maps
[i
].clear_priv
)
1184 obj
->maps
[i
].clear_priv(&obj
->maps
[i
],
1186 obj
->maps
[i
].priv
= NULL
;
1187 obj
->maps
[i
].clear_priv
= NULL
;
1192 if (obj
->programs
&& obj
->nr_programs
) {
1193 for (i
= 0; i
< obj
->nr_programs
; i
++)
1194 bpf_program__exit(&obj
->programs
[i
]);
1196 zfree(&obj
->programs
);
1198 list_del(&obj
->list
);
1203 bpf_object__next(struct bpf_object
*prev
)
1205 struct bpf_object
*next
;
1208 next
= list_first_entry(&bpf_objects_list
,
1212 next
= list_next_entry(prev
, list
);
1214 /* Empty list is noticed here so don't need checking on entry. */
1215 if (&next
->list
== &bpf_objects_list
)
1221 const char *bpf_object__name(struct bpf_object
*obj
)
1223 return obj
? obj
->path
: ERR_PTR(-EINVAL
);
1226 unsigned int bpf_object__kversion(struct bpf_object
*obj
)
1228 return obj
? obj
->kern_version
: 0;
1231 struct bpf_program
*
1232 bpf_program__next(struct bpf_program
*prev
, struct bpf_object
*obj
)
1240 return &obj
->programs
[0];
1242 if (prev
->obj
!= obj
) {
1243 pr_warning("error: program handler doesn't match object\n");
1247 idx
= (prev
- obj
->programs
) + 1;
1248 if (idx
>= obj
->nr_programs
)
1250 return &obj
->programs
[idx
];
1253 int bpf_program__set_priv(struct bpf_program
*prog
, void *priv
,
1254 bpf_program_clear_priv_t clear_priv
)
1256 if (prog
->priv
&& prog
->clear_priv
)
1257 prog
->clear_priv(prog
, prog
->priv
);
1260 prog
->clear_priv
= clear_priv
;
1264 void *bpf_program__priv(struct bpf_program
*prog
)
1266 return prog
? prog
->priv
: ERR_PTR(-EINVAL
);
1269 const char *bpf_program__title(struct bpf_program
*prog
, bool needs_copy
)
1273 title
= prog
->section_name
;
1275 title
= strdup(title
);
1277 pr_warning("failed to strdup program title\n");
1278 return ERR_PTR(-ENOMEM
);
1285 int bpf_program__fd(struct bpf_program
*prog
)
1287 return bpf_program__nth_fd(prog
, 0);
1290 int bpf_program__set_prep(struct bpf_program
*prog
, int nr_instances
,
1291 bpf_program_prep_t prep
)
1295 if (nr_instances
<= 0 || !prep
)
1298 if (prog
->instances
.nr
> 0 || prog
->instances
.fds
) {
1299 pr_warning("Can't set pre-processor after loading\n");
1303 instances_fds
= malloc(sizeof(int) * nr_instances
);
1304 if (!instances_fds
) {
1305 pr_warning("alloc memory failed for fds\n");
1309 /* fill all fd with -1 */
1310 memset(instances_fds
, -1, sizeof(int) * nr_instances
);
1312 prog
->instances
.nr
= nr_instances
;
1313 prog
->instances
.fds
= instances_fds
;
1314 prog
->preprocessor
= prep
;
1318 int bpf_program__nth_fd(struct bpf_program
*prog
, int n
)
1322 if (n
>= prog
->instances
.nr
|| n
< 0) {
1323 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
1324 n
, prog
->section_name
, prog
->instances
.nr
);
1328 fd
= prog
->instances
.fds
[n
];
1330 pr_warning("%dth instance of program '%s' is invalid\n",
1331 n
, prog
->section_name
);
1338 static void bpf_program__set_type(struct bpf_program
*prog
,
1339 enum bpf_prog_type type
)
1344 int bpf_program__set_tracepoint(struct bpf_program
*prog
)
1348 bpf_program__set_type(prog
, BPF_PROG_TYPE_TRACEPOINT
);
1352 int bpf_program__set_kprobe(struct bpf_program
*prog
)
1356 bpf_program__set_type(prog
, BPF_PROG_TYPE_KPROBE
);
1360 static bool bpf_program__is_type(struct bpf_program
*prog
,
1361 enum bpf_prog_type type
)
1363 return prog
? (prog
->type
== type
) : false;
1366 bool bpf_program__is_tracepoint(struct bpf_program
*prog
)
1368 return bpf_program__is_type(prog
, BPF_PROG_TYPE_TRACEPOINT
);
1371 bool bpf_program__is_kprobe(struct bpf_program
*prog
)
1373 return bpf_program__is_type(prog
, BPF_PROG_TYPE_KPROBE
);
1376 int bpf_map__fd(struct bpf_map
*map
)
1378 return map
? map
->fd
: -EINVAL
;
1381 const struct bpf_map_def
*bpf_map__def(struct bpf_map
*map
)
1383 return map
? &map
->def
: ERR_PTR(-EINVAL
);
1386 const char *bpf_map__name(struct bpf_map
*map
)
1388 return map
? map
->name
: NULL
;
1391 int bpf_map__set_priv(struct bpf_map
*map
, void *priv
,
1392 bpf_map_clear_priv_t clear_priv
)
1398 if (map
->clear_priv
)
1399 map
->clear_priv(map
, map
->priv
);
1403 map
->clear_priv
= clear_priv
;
1407 void *bpf_map__priv(struct bpf_map
*map
)
1409 return map
? map
->priv
: ERR_PTR(-EINVAL
);
1413 bpf_map__next(struct bpf_map
*prev
, struct bpf_object
*obj
)
1416 struct bpf_map
*s
, *e
;
1418 if (!obj
|| !obj
->maps
)
1422 e
= obj
->maps
+ obj
->nr_maps
;
1427 if ((prev
< s
) || (prev
>= e
)) {
1428 pr_warning("error in %s: map handler doesn't belong to object\n",
1433 idx
= (prev
- obj
->maps
) + 1;
1434 if (idx
>= obj
->nr_maps
)
1436 return &obj
->maps
[idx
];
1440 bpf_object__find_map_by_name(struct bpf_object
*obj
, const char *name
)
1442 struct bpf_map
*pos
;
1444 bpf_map__for_each(pos
, obj
) {
1445 if (pos
->name
&& !strcmp(pos
->name
, name
))