Commit | Line | Data |
---|---|---|
1b76c13e WN |
1 | /* |
2 | * Common eBPF ELF object loading operations. | |
3 | * | |
4 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
5 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
6 | * Copyright (C) 2015 Huawei Inc. | |
7 | */ | |
8 | ||
9 | #include <stdlib.h> | |
b3f59d66 WN |
10 | #include <stdio.h> |
11 | #include <stdarg.h> | |
34090915 | 12 | #include <inttypes.h> |
b3f59d66 | 13 | #include <string.h> |
1b76c13e | 14 | #include <unistd.h> |
1a5e3fb1 WN |
15 | #include <fcntl.h> |
16 | #include <errno.h> | |
1b76c13e | 17 | #include <asm/unistd.h> |
cb1e5e96 | 18 | #include <linux/kernel.h> |
1b76c13e | 19 | #include <linux/bpf.h> |
9a208eff | 20 | #include <linux/list.h> |
1a5e3fb1 WN |
21 | #include <libelf.h> |
22 | #include <gelf.h> | |
1b76c13e WN |
23 | |
24 | #include "libbpf.h" | |
52d3352e | 25 | #include "bpf.h" |
b3f59d66 WN |
26 | |
27 | #define __printf(a, b) __attribute__((format(printf, a, b))) | |
28 | ||
29 | __printf(1, 2) | |
30 | static int __base_pr(const char *format, ...) | |
31 | { | |
32 | va_list args; | |
33 | int err; | |
34 | ||
35 | va_start(args, format); | |
36 | err = vfprintf(stderr, format, args); | |
37 | va_end(args); | |
38 | return err; | |
39 | } | |
40 | ||
41 | static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; | |
42 | static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; | |
43 | static __printf(1, 2) libbpf_print_fn_t __pr_debug; | |
44 | ||
45 | #define __pr(func, fmt, ...) \ | |
46 | do { \ | |
47 | if ((func)) \ | |
48 | (func)("libbpf: " fmt, ##__VA_ARGS__); \ | |
49 | } while (0) | |
50 | ||
51 | #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) | |
52 | #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) | |
53 | #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) | |
54 | ||
55 | void libbpf_set_print(libbpf_print_fn_t warn, | |
56 | libbpf_print_fn_t info, | |
57 | libbpf_print_fn_t debug) | |
58 | { | |
59 | __pr_warning = warn; | |
60 | __pr_info = info; | |
61 | __pr_debug = debug; | |
62 | } | |
1a5e3fb1 | 63 | |
6371ca3b WN |
64 | #define STRERR_BUFSIZE 128 |
65 | ||
66 | #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) | |
67 | #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) | |
68 | #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) | |
69 | ||
70 | static const char *libbpf_strerror_table[NR_ERRNO] = { | |
71 | [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", | |
72 | [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", | |
73 | [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", | |
74 | [ERRCODE_OFFSET(ENDIAN)] = "Endian missmatch", | |
75 | [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", | |
76 | [ERRCODE_OFFSET(RELOC)] = "Relocation failed", | |
77 | [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", | |
78 | [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", | |
79 | [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", | |
80 | }; | |
81 | ||
82 | int libbpf_strerror(int err, char *buf, size_t size) | |
83 | { | |
84 | if (!buf || !size) | |
85 | return -1; | |
86 | ||
87 | err = err > 0 ? err : -err; | |
88 | ||
89 | if (err < __LIBBPF_ERRNO__START) { | |
90 | int ret; | |
91 | ||
92 | ret = strerror_r(err, buf, size); | |
93 | buf[size - 1] = '\0'; | |
94 | return ret; | |
95 | } | |
96 | ||
97 | if (err < __LIBBPF_ERRNO__END) { | |
98 | const char *msg; | |
99 | ||
100 | msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; | |
101 | snprintf(buf, size, "%s", msg); | |
102 | buf[size - 1] = '\0'; | |
103 | return 0; | |
104 | } | |
105 | ||
106 | snprintf(buf, size, "Unknown libbpf error %d", err); | |
107 | buf[size - 1] = '\0'; | |
108 | return -1; | |
109 | } | |
110 | ||
111 | #define CHECK_ERR(action, err, out) do { \ | |
112 | err = action; \ | |
113 | if (err) \ | |
114 | goto out; \ | |
115 | } while(0) | |
116 | ||
117 | ||
1a5e3fb1 WN |
118 | /* Copied from tools/perf/util/util.h */ |
119 | #ifndef zfree | |
120 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
121 | #endif | |
122 | ||
123 | #ifndef zclose | |
124 | # define zclose(fd) ({ \ | |
125 | int ___err = 0; \ | |
126 | if ((fd) >= 0) \ | |
127 | ___err = close((fd)); \ | |
128 | fd = -1; \ | |
129 | ___err; }) | |
130 | #endif | |
131 | ||
132 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
133 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
134 | #else | |
135 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
136 | #endif | |
137 | ||
a5b8bd47 WN |
138 | /* |
139 | * bpf_prog should be a better name but it has been used in | |
140 | * linux/filter.h. | |
141 | */ | |
142 | struct bpf_program { | |
143 | /* Index in elf obj file, for relocation use. */ | |
144 | int idx; | |
145 | char *section_name; | |
146 | struct bpf_insn *insns; | |
147 | size_t insns_cnt; | |
34090915 WN |
148 | |
149 | struct { | |
150 | int insn_idx; | |
151 | int map_idx; | |
152 | } *reloc_desc; | |
153 | int nr_reloc; | |
55cffde2 WN |
154 | |
155 | int fd; | |
aa9b1ac3 WN |
156 | |
157 | struct bpf_object *obj; | |
158 | void *priv; | |
159 | bpf_program_clear_priv_t clear_priv; | |
a5b8bd47 WN |
160 | }; |
161 | ||
9a208eff WN |
162 | static LIST_HEAD(bpf_objects_list); |
163 | ||
1a5e3fb1 | 164 | struct bpf_object { |
cb1e5e96 WN |
165 | char license[64]; |
166 | u32 kern_version; | |
0b3d1efa WN |
167 | void *maps_buf; |
168 | size_t maps_buf_sz; | |
169 | ||
a5b8bd47 WN |
170 | struct bpf_program *programs; |
171 | size_t nr_programs; | |
52d3352e WN |
172 | int *map_fds; |
173 | /* | |
174 | * This field is required because maps_buf will be freed and | |
175 | * maps_buf_sz will be set to 0 after loaded. | |
176 | */ | |
177 | size_t nr_map_fds; | |
178 | bool loaded; | |
a5b8bd47 | 179 | |
1a5e3fb1 WN |
180 | /* |
181 | * Information when doing elf related work. Only valid if fd | |
182 | * is valid. | |
183 | */ | |
184 | struct { | |
185 | int fd; | |
6c956392 WN |
186 | void *obj_buf; |
187 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
188 | Elf *elf; |
189 | GElf_Ehdr ehdr; | |
bec7d68c | 190 | Elf_Data *symbols; |
b62f06e8 WN |
191 | struct { |
192 | GElf_Shdr shdr; | |
193 | Elf_Data *data; | |
194 | } *reloc; | |
195 | int nr_reloc; | |
1a5e3fb1 | 196 | } efile; |
9a208eff WN |
197 | /* |
198 | * All loaded bpf_object is linked in a list, which is | |
199 | * hidden to caller. bpf_objects__<func> handlers deal with | |
200 | * all objects. | |
201 | */ | |
202 | struct list_head list; | |
1a5e3fb1 WN |
203 | char path[]; |
204 | }; | |
205 | #define obj_elf_valid(o) ((o)->efile.elf) | |
206 | ||
55cffde2 WN |
207 | static void bpf_program__unload(struct bpf_program *prog) |
208 | { | |
209 | if (!prog) | |
210 | return; | |
211 | ||
212 | zclose(prog->fd); | |
213 | } | |
214 | ||
a5b8bd47 WN |
215 | static void bpf_program__exit(struct bpf_program *prog) |
216 | { | |
217 | if (!prog) | |
218 | return; | |
219 | ||
aa9b1ac3 WN |
220 | if (prog->clear_priv) |
221 | prog->clear_priv(prog, prog->priv); | |
222 | ||
223 | prog->priv = NULL; | |
224 | prog->clear_priv = NULL; | |
225 | ||
55cffde2 | 226 | bpf_program__unload(prog); |
a5b8bd47 WN |
227 | zfree(&prog->section_name); |
228 | zfree(&prog->insns); | |
34090915 WN |
229 | zfree(&prog->reloc_desc); |
230 | ||
231 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
232 | prog->insns_cnt = 0; |
233 | prog->idx = -1; | |
234 | } | |
235 | ||
236 | static int | |
237 | bpf_program__init(void *data, size_t size, char *name, int idx, | |
238 | struct bpf_program *prog) | |
239 | { | |
240 | if (size < sizeof(struct bpf_insn)) { | |
241 | pr_warning("corrupted section '%s'\n", name); | |
242 | return -EINVAL; | |
243 | } | |
244 | ||
245 | bzero(prog, sizeof(*prog)); | |
246 | ||
247 | prog->section_name = strdup(name); | |
248 | if (!prog->section_name) { | |
249 | pr_warning("failed to alloc name for prog %s\n", | |
250 | name); | |
251 | goto errout; | |
252 | } | |
253 | ||
254 | prog->insns = malloc(size); | |
255 | if (!prog->insns) { | |
256 | pr_warning("failed to alloc insns for %s\n", name); | |
257 | goto errout; | |
258 | } | |
259 | prog->insns_cnt = size / sizeof(struct bpf_insn); | |
260 | memcpy(prog->insns, data, | |
261 | prog->insns_cnt * sizeof(struct bpf_insn)); | |
262 | prog->idx = idx; | |
55cffde2 | 263 | prog->fd = -1; |
a5b8bd47 WN |
264 | |
265 | return 0; | |
266 | errout: | |
267 | bpf_program__exit(prog); | |
268 | return -ENOMEM; | |
269 | } | |
270 | ||
271 | static int | |
272 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
273 | char *name, int idx) | |
274 | { | |
275 | struct bpf_program prog, *progs; | |
276 | int nr_progs, err; | |
277 | ||
278 | err = bpf_program__init(data, size, name, idx, &prog); | |
279 | if (err) | |
280 | return err; | |
281 | ||
282 | progs = obj->programs; | |
283 | nr_progs = obj->nr_programs; | |
284 | ||
285 | progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); | |
286 | if (!progs) { | |
287 | /* | |
288 | * In this case the original obj->programs | |
289 | * is still valid, so don't need special treat for | |
290 | * bpf_close_object(). | |
291 | */ | |
292 | pr_warning("failed to alloc a new program '%s'\n", | |
293 | name); | |
294 | bpf_program__exit(&prog); | |
295 | return -ENOMEM; | |
296 | } | |
297 | ||
298 | pr_debug("found program %s\n", prog.section_name); | |
299 | obj->programs = progs; | |
300 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 301 | prog.obj = obj; |
a5b8bd47 WN |
302 | progs[nr_progs] = prog; |
303 | return 0; | |
304 | } | |
305 | ||
6c956392 WN |
306 | static struct bpf_object *bpf_object__new(const char *path, |
307 | void *obj_buf, | |
308 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
309 | { |
310 | struct bpf_object *obj; | |
311 | ||
312 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
313 | if (!obj) { | |
314 | pr_warning("alloc memory failed for %s\n", path); | |
6371ca3b | 315 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
316 | } |
317 | ||
318 | strcpy(obj->path, path); | |
319 | obj->efile.fd = -1; | |
6c956392 WN |
320 | |
321 | /* | |
322 | * Caller of this function should also calls | |
323 | * bpf_object__elf_finish() after data collection to return | |
324 | * obj_buf to user. If not, we should duplicate the buffer to | |
325 | * avoid user freeing them before elf finish. | |
326 | */ | |
327 | obj->efile.obj_buf = obj_buf; | |
328 | obj->efile.obj_buf_sz = obj_buf_sz; | |
329 | ||
52d3352e | 330 | obj->loaded = false; |
9a208eff WN |
331 | |
332 | INIT_LIST_HEAD(&obj->list); | |
333 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
334 | return obj; |
335 | } | |
336 | ||
337 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
338 | { | |
339 | if (!obj_elf_valid(obj)) | |
340 | return; | |
341 | ||
342 | if (obj->efile.elf) { | |
343 | elf_end(obj->efile.elf); | |
344 | obj->efile.elf = NULL; | |
345 | } | |
bec7d68c | 346 | obj->efile.symbols = NULL; |
b62f06e8 WN |
347 | |
348 | zfree(&obj->efile.reloc); | |
349 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 350 | zclose(obj->efile.fd); |
6c956392 WN |
351 | obj->efile.obj_buf = NULL; |
352 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
353 | } |
354 | ||
355 | static int bpf_object__elf_init(struct bpf_object *obj) | |
356 | { | |
357 | int err = 0; | |
358 | GElf_Ehdr *ep; | |
359 | ||
360 | if (obj_elf_valid(obj)) { | |
361 | pr_warning("elf init: internal error\n"); | |
6371ca3b | 362 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
363 | } |
364 | ||
6c956392 WN |
365 | if (obj->efile.obj_buf_sz > 0) { |
366 | /* | |
367 | * obj_buf should have been validated by | |
368 | * bpf_object__open_buffer(). | |
369 | */ | |
370 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
371 | obj->efile.obj_buf_sz); | |
372 | } else { | |
373 | obj->efile.fd = open(obj->path, O_RDONLY); | |
374 | if (obj->efile.fd < 0) { | |
375 | pr_warning("failed to open %s: %s\n", obj->path, | |
376 | strerror(errno)); | |
377 | return -errno; | |
378 | } | |
379 | ||
380 | obj->efile.elf = elf_begin(obj->efile.fd, | |
381 | LIBBPF_ELF_C_READ_MMAP, | |
382 | NULL); | |
1a5e3fb1 WN |
383 | } |
384 | ||
1a5e3fb1 WN |
385 | if (!obj->efile.elf) { |
386 | pr_warning("failed to open %s as ELF file\n", | |
387 | obj->path); | |
6371ca3b | 388 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
389 | goto errout; |
390 | } | |
391 | ||
392 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
393 | pr_warning("failed to get EHDR from %s\n", | |
394 | obj->path); | |
6371ca3b | 395 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
396 | goto errout; |
397 | } | |
398 | ep = &obj->efile.ehdr; | |
399 | ||
400 | if ((ep->e_type != ET_REL) || (ep->e_machine != 0)) { | |
401 | pr_warning("%s is not an eBPF object file\n", | |
402 | obj->path); | |
6371ca3b | 403 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
404 | goto errout; |
405 | } | |
406 | ||
407 | return 0; | |
408 | errout: | |
409 | bpf_object__elf_finish(obj); | |
410 | return err; | |
411 | } | |
412 | ||
cc4228d5 WN |
413 | static int |
414 | bpf_object__check_endianness(struct bpf_object *obj) | |
415 | { | |
416 | static unsigned int const endian = 1; | |
417 | ||
418 | switch (obj->efile.ehdr.e_ident[EI_DATA]) { | |
419 | case ELFDATA2LSB: | |
420 | /* We are big endian, BPF obj is little endian. */ | |
421 | if (*(unsigned char const *)&endian != 1) | |
422 | goto mismatch; | |
423 | break; | |
424 | ||
425 | case ELFDATA2MSB: | |
426 | /* We are little endian, BPF obj is big endian. */ | |
427 | if (*(unsigned char const *)&endian != 0) | |
428 | goto mismatch; | |
429 | break; | |
430 | default: | |
6371ca3b | 431 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
432 | } |
433 | ||
434 | return 0; | |
435 | ||
436 | mismatch: | |
437 | pr_warning("Error: endianness mismatch.\n"); | |
6371ca3b | 438 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
439 | } |
440 | ||
cb1e5e96 WN |
441 | static int |
442 | bpf_object__init_license(struct bpf_object *obj, | |
443 | void *data, size_t size) | |
444 | { | |
445 | memcpy(obj->license, data, | |
446 | min(size, sizeof(obj->license) - 1)); | |
447 | pr_debug("license of %s is %s\n", obj->path, obj->license); | |
448 | return 0; | |
449 | } | |
450 | ||
451 | static int | |
452 | bpf_object__init_kversion(struct bpf_object *obj, | |
453 | void *data, size_t size) | |
454 | { | |
455 | u32 kver; | |
456 | ||
457 | if (size != sizeof(kver)) { | |
458 | pr_warning("invalid kver section in %s\n", obj->path); | |
6371ca3b | 459 | return -LIBBPF_ERRNO__FORMAT; |
cb1e5e96 WN |
460 | } |
461 | memcpy(&kver, data, sizeof(kver)); | |
462 | obj->kern_version = kver; | |
463 | pr_debug("kernel version of %s is %x\n", obj->path, | |
464 | obj->kern_version); | |
465 | return 0; | |
466 | } | |
467 | ||
0b3d1efa WN |
468 | static int |
469 | bpf_object__init_maps(struct bpf_object *obj, void *data, | |
470 | size_t size) | |
471 | { | |
472 | if (size == 0) { | |
473 | pr_debug("%s doesn't need map definition\n", | |
474 | obj->path); | |
475 | return 0; | |
476 | } | |
477 | ||
478 | obj->maps_buf = malloc(size); | |
479 | if (!obj->maps_buf) { | |
480 | pr_warning("malloc maps failed: %s\n", obj->path); | |
481 | return -ENOMEM; | |
482 | } | |
483 | ||
484 | obj->maps_buf_sz = size; | |
485 | memcpy(obj->maps_buf, data, size); | |
486 | pr_debug("maps in %s: %ld bytes\n", obj->path, (long)size); | |
487 | return 0; | |
488 | } | |
489 | ||
29603665 WN |
490 | static int bpf_object__elf_collect(struct bpf_object *obj) |
491 | { | |
492 | Elf *elf = obj->efile.elf; | |
493 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
494 | Elf_Scn *scn = NULL; | |
495 | int idx = 0, err = 0; | |
496 | ||
497 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
498 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
499 | pr_warning("failed to get e_shstrndx from %s\n", | |
500 | obj->path); | |
6371ca3b | 501 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
502 | } |
503 | ||
504 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
505 | char *name; | |
506 | GElf_Shdr sh; | |
507 | Elf_Data *data; | |
508 | ||
509 | idx++; | |
510 | if (gelf_getshdr(scn, &sh) != &sh) { | |
511 | pr_warning("failed to get section header from %s\n", | |
512 | obj->path); | |
6371ca3b | 513 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
514 | goto out; |
515 | } | |
516 | ||
517 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
518 | if (!name) { | |
519 | pr_warning("failed to get section name from %s\n", | |
520 | obj->path); | |
6371ca3b | 521 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
522 | goto out; |
523 | } | |
524 | ||
525 | data = elf_getdata(scn, 0); | |
526 | if (!data) { | |
527 | pr_warning("failed to get section data from %s(%s)\n", | |
528 | name, obj->path); | |
6371ca3b | 529 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
530 | goto out; |
531 | } | |
532 | pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", | |
533 | name, (unsigned long)data->d_size, | |
534 | (int)sh.sh_link, (unsigned long)sh.sh_flags, | |
535 | (int)sh.sh_type); | |
cb1e5e96 WN |
536 | |
537 | if (strcmp(name, "license") == 0) | |
538 | err = bpf_object__init_license(obj, | |
539 | data->d_buf, | |
540 | data->d_size); | |
541 | else if (strcmp(name, "version") == 0) | |
542 | err = bpf_object__init_kversion(obj, | |
543 | data->d_buf, | |
544 | data->d_size); | |
0b3d1efa WN |
545 | else if (strcmp(name, "maps") == 0) |
546 | err = bpf_object__init_maps(obj, data->d_buf, | |
547 | data->d_size); | |
bec7d68c WN |
548 | else if (sh.sh_type == SHT_SYMTAB) { |
549 | if (obj->efile.symbols) { | |
550 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
551 | obj->path); | |
6371ca3b | 552 | err = -LIBBPF_ERRNO__FORMAT; |
bec7d68c WN |
553 | } else |
554 | obj->efile.symbols = data; | |
a5b8bd47 WN |
555 | } else if ((sh.sh_type == SHT_PROGBITS) && |
556 | (sh.sh_flags & SHF_EXECINSTR) && | |
557 | (data->d_size > 0)) { | |
558 | err = bpf_object__add_program(obj, data->d_buf, | |
559 | data->d_size, name, idx); | |
560 | if (err) { | |
6371ca3b WN |
561 | char errmsg[STRERR_BUFSIZE]; |
562 | ||
a5b8bd47 WN |
563 | strerror_r(-err, errmsg, sizeof(errmsg)); |
564 | pr_warning("failed to alloc program %s (%s): %s", | |
565 | name, obj->path, errmsg); | |
566 | } | |
b62f06e8 WN |
567 | } else if (sh.sh_type == SHT_REL) { |
568 | void *reloc = obj->efile.reloc; | |
569 | int nr_reloc = obj->efile.nr_reloc + 1; | |
570 | ||
571 | reloc = realloc(reloc, | |
572 | sizeof(*obj->efile.reloc) * nr_reloc); | |
573 | if (!reloc) { | |
574 | pr_warning("realloc failed\n"); | |
575 | err = -ENOMEM; | |
576 | } else { | |
577 | int n = nr_reloc - 1; | |
578 | ||
579 | obj->efile.reloc = reloc; | |
580 | obj->efile.nr_reloc = nr_reloc; | |
581 | ||
582 | obj->efile.reloc[n].shdr = sh; | |
583 | obj->efile.reloc[n].data = data; | |
584 | } | |
bec7d68c | 585 | } |
cb1e5e96 WN |
586 | if (err) |
587 | goto out; | |
29603665 WN |
588 | } |
589 | out: | |
590 | return err; | |
591 | } | |
592 | ||
34090915 WN |
593 | static struct bpf_program * |
594 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
595 | { | |
596 | struct bpf_program *prog; | |
597 | size_t i; | |
598 | ||
599 | for (i = 0; i < obj->nr_programs; i++) { | |
600 | prog = &obj->programs[i]; | |
601 | if (prog->idx == idx) | |
602 | return prog; | |
603 | } | |
604 | return NULL; | |
605 | } | |
606 | ||
607 | static int | |
608 | bpf_program__collect_reloc(struct bpf_program *prog, | |
609 | size_t nr_maps, GElf_Shdr *shdr, | |
610 | Elf_Data *data, Elf_Data *symbols) | |
611 | { | |
612 | int i, nrels; | |
613 | ||
614 | pr_debug("collecting relocating info for: '%s'\n", | |
615 | prog->section_name); | |
616 | nrels = shdr->sh_size / shdr->sh_entsize; | |
617 | ||
618 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
619 | if (!prog->reloc_desc) { | |
620 | pr_warning("failed to alloc memory in relocation\n"); | |
621 | return -ENOMEM; | |
622 | } | |
623 | prog->nr_reloc = nrels; | |
624 | ||
625 | for (i = 0; i < nrels; i++) { | |
626 | GElf_Sym sym; | |
627 | GElf_Rel rel; | |
628 | unsigned int insn_idx; | |
629 | struct bpf_insn *insns = prog->insns; | |
630 | size_t map_idx; | |
631 | ||
632 | if (!gelf_getrel(data, i, &rel)) { | |
633 | pr_warning("relocation: failed to get %d reloc\n", i); | |
6371ca3b | 634 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
635 | } |
636 | ||
637 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); | |
638 | pr_debug("relocation: insn_idx=%u\n", insn_idx); | |
639 | ||
640 | if (!gelf_getsym(symbols, | |
641 | GELF_R_SYM(rel.r_info), | |
642 | &sym)) { | |
643 | pr_warning("relocation: symbol %"PRIx64" not found\n", | |
644 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 645 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
646 | } |
647 | ||
648 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { | |
649 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
650 | insn_idx, insns[insn_idx].code); | |
6371ca3b | 651 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
652 | } |
653 | ||
654 | map_idx = sym.st_value / sizeof(struct bpf_map_def); | |
655 | if (map_idx >= nr_maps) { | |
656 | pr_warning("bpf relocation: map_idx %d large than %d\n", | |
657 | (int)map_idx, (int)nr_maps - 1); | |
6371ca3b | 658 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
659 | } |
660 | ||
661 | prog->reloc_desc[i].insn_idx = insn_idx; | |
662 | prog->reloc_desc[i].map_idx = map_idx; | |
663 | } | |
664 | return 0; | |
665 | } | |
666 | ||
52d3352e WN |
667 | static int |
668 | bpf_object__create_maps(struct bpf_object *obj) | |
669 | { | |
670 | unsigned int i; | |
671 | size_t nr_maps; | |
672 | int *pfd; | |
673 | ||
674 | nr_maps = obj->maps_buf_sz / sizeof(struct bpf_map_def); | |
675 | if (!obj->maps_buf || !nr_maps) { | |
676 | pr_debug("don't need create maps for %s\n", | |
677 | obj->path); | |
678 | return 0; | |
679 | } | |
680 | ||
681 | obj->map_fds = malloc(sizeof(int) * nr_maps); | |
682 | if (!obj->map_fds) { | |
683 | pr_warning("realloc perf_bpf_map_fds failed\n"); | |
684 | return -ENOMEM; | |
685 | } | |
686 | obj->nr_map_fds = nr_maps; | |
687 | ||
688 | /* fill all fd with -1 */ | |
689 | memset(obj->map_fds, -1, sizeof(int) * nr_maps); | |
690 | ||
691 | pfd = obj->map_fds; | |
692 | for (i = 0; i < nr_maps; i++) { | |
693 | struct bpf_map_def def; | |
694 | ||
695 | def = *(struct bpf_map_def *)(obj->maps_buf + | |
696 | i * sizeof(struct bpf_map_def)); | |
697 | ||
698 | *pfd = bpf_create_map(def.type, | |
699 | def.key_size, | |
700 | def.value_size, | |
701 | def.max_entries); | |
702 | if (*pfd < 0) { | |
703 | size_t j; | |
704 | int err = *pfd; | |
705 | ||
706 | pr_warning("failed to create map: %s\n", | |
707 | strerror(errno)); | |
708 | for (j = 0; j < i; j++) | |
709 | zclose(obj->map_fds[j]); | |
710 | obj->nr_map_fds = 0; | |
711 | zfree(&obj->map_fds); | |
712 | return err; | |
713 | } | |
714 | pr_debug("create map: fd=%d\n", *pfd); | |
715 | pfd++; | |
716 | } | |
717 | ||
718 | zfree(&obj->maps_buf); | |
719 | obj->maps_buf_sz = 0; | |
720 | return 0; | |
721 | } | |
722 | ||
8a47a6c5 WN |
723 | static int |
724 | bpf_program__relocate(struct bpf_program *prog, int *map_fds) | |
725 | { | |
726 | int i; | |
727 | ||
728 | if (!prog || !prog->reloc_desc) | |
729 | return 0; | |
730 | ||
731 | for (i = 0; i < prog->nr_reloc; i++) { | |
732 | int insn_idx, map_idx; | |
733 | struct bpf_insn *insns = prog->insns; | |
734 | ||
735 | insn_idx = prog->reloc_desc[i].insn_idx; | |
736 | map_idx = prog->reloc_desc[i].map_idx; | |
737 | ||
738 | if (insn_idx >= (int)prog->insns_cnt) { | |
739 | pr_warning("relocation out of range: '%s'\n", | |
740 | prog->section_name); | |
6371ca3b | 741 | return -LIBBPF_ERRNO__RELOC; |
8a47a6c5 WN |
742 | } |
743 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
744 | insns[insn_idx].imm = map_fds[map_idx]; | |
745 | } | |
746 | ||
747 | zfree(&prog->reloc_desc); | |
748 | prog->nr_reloc = 0; | |
749 | return 0; | |
750 | } | |
751 | ||
752 | ||
753 | static int | |
754 | bpf_object__relocate(struct bpf_object *obj) | |
755 | { | |
756 | struct bpf_program *prog; | |
757 | size_t i; | |
758 | int err; | |
759 | ||
760 | for (i = 0; i < obj->nr_programs; i++) { | |
761 | prog = &obj->programs[i]; | |
762 | ||
763 | err = bpf_program__relocate(prog, obj->map_fds); | |
764 | if (err) { | |
765 | pr_warning("failed to relocate '%s'\n", | |
766 | prog->section_name); | |
767 | return err; | |
768 | } | |
769 | } | |
770 | return 0; | |
771 | } | |
772 | ||
34090915 WN |
773 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
774 | { | |
775 | int i, err; | |
776 | ||
777 | if (!obj_elf_valid(obj)) { | |
778 | pr_warning("Internal error: elf object is closed\n"); | |
6371ca3b | 779 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
780 | } |
781 | ||
782 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
783 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
784 | Elf_Data *data = obj->efile.reloc[i].data; | |
785 | int idx = shdr->sh_info; | |
786 | struct bpf_program *prog; | |
787 | size_t nr_maps = obj->maps_buf_sz / | |
788 | sizeof(struct bpf_map_def); | |
789 | ||
790 | if (shdr->sh_type != SHT_REL) { | |
791 | pr_warning("internal error at %d\n", __LINE__); | |
6371ca3b | 792 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
793 | } |
794 | ||
795 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
796 | if (!prog) { | |
797 | pr_warning("relocation failed: no %d section\n", | |
798 | idx); | |
6371ca3b | 799 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
800 | } |
801 | ||
802 | err = bpf_program__collect_reloc(prog, nr_maps, | |
803 | shdr, data, | |
804 | obj->efile.symbols); | |
805 | if (err) | |
6371ca3b | 806 | return err; |
34090915 WN |
807 | } |
808 | return 0; | |
809 | } | |
810 | ||
55cffde2 WN |
811 | static int |
812 | load_program(struct bpf_insn *insns, int insns_cnt, | |
813 | char *license, u32 kern_version, int *pfd) | |
814 | { | |
815 | int ret; | |
816 | char *log_buf; | |
817 | ||
818 | if (!insns || !insns_cnt) | |
819 | return -EINVAL; | |
820 | ||
821 | log_buf = malloc(BPF_LOG_BUF_SIZE); | |
822 | if (!log_buf) | |
823 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
824 | ||
825 | ret = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns, | |
826 | insns_cnt, license, kern_version, | |
827 | log_buf, BPF_LOG_BUF_SIZE); | |
828 | ||
829 | if (ret >= 0) { | |
830 | *pfd = ret; | |
831 | ret = 0; | |
832 | goto out; | |
833 | } | |
834 | ||
6371ca3b | 835 | ret = -LIBBPF_ERRNO__LOAD; |
55cffde2 WN |
836 | pr_warning("load bpf program failed: %s\n", strerror(errno)); |
837 | ||
6371ca3b WN |
838 | if (log_buf && log_buf[0] != '\0') { |
839 | ret = -LIBBPF_ERRNO__VERIFY; | |
55cffde2 WN |
840 | pr_warning("-- BEGIN DUMP LOG ---\n"); |
841 | pr_warning("\n%s\n", log_buf); | |
842 | pr_warning("-- END LOG --\n"); | |
6371ca3b WN |
843 | } else { |
844 | if (insns_cnt >= BPF_MAXINSNS) { | |
845 | pr_warning("Program too large (%d insns), at most %d insns\n", | |
846 | insns_cnt, BPF_MAXINSNS); | |
847 | ret = -LIBBPF_ERRNO__PROG2BIG; | |
848 | } else if (log_buf) { | |
849 | pr_warning("log buffer is empty\n"); | |
850 | ret = -LIBBPF_ERRNO__KVER; | |
851 | } | |
55cffde2 WN |
852 | } |
853 | ||
854 | out: | |
855 | free(log_buf); | |
856 | return ret; | |
857 | } | |
858 | ||
859 | static int | |
860 | bpf_program__load(struct bpf_program *prog, | |
861 | char *license, u32 kern_version) | |
862 | { | |
863 | int err, fd; | |
864 | ||
865 | err = load_program(prog->insns, prog->insns_cnt, | |
866 | license, kern_version, &fd); | |
867 | if (!err) | |
868 | prog->fd = fd; | |
869 | ||
870 | if (err) | |
871 | pr_warning("failed to load program '%s'\n", | |
872 | prog->section_name); | |
873 | zfree(&prog->insns); | |
874 | prog->insns_cnt = 0; | |
875 | return err; | |
876 | } | |
877 | ||
878 | static int | |
879 | bpf_object__load_progs(struct bpf_object *obj) | |
880 | { | |
881 | size_t i; | |
882 | int err; | |
883 | ||
884 | for (i = 0; i < obj->nr_programs; i++) { | |
885 | err = bpf_program__load(&obj->programs[i], | |
886 | obj->license, | |
887 | obj->kern_version); | |
888 | if (err) | |
889 | return err; | |
890 | } | |
891 | return 0; | |
892 | } | |
893 | ||
cb1e5e96 WN |
894 | static int bpf_object__validate(struct bpf_object *obj) |
895 | { | |
896 | if (obj->kern_version == 0) { | |
897 | pr_warning("%s doesn't provide kernel version\n", | |
898 | obj->path); | |
6371ca3b | 899 | return -LIBBPF_ERRNO__KVERSION; |
cb1e5e96 WN |
900 | } |
901 | return 0; | |
902 | } | |
903 | ||
1a5e3fb1 | 904 | static struct bpf_object * |
6c956392 | 905 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) |
1a5e3fb1 WN |
906 | { |
907 | struct bpf_object *obj; | |
6371ca3b | 908 | int err; |
1a5e3fb1 WN |
909 | |
910 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
911 | pr_warning("failed to init libelf for %s\n", path); | |
6371ca3b | 912 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
913 | } |
914 | ||
6c956392 | 915 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
6371ca3b WN |
916 | if (IS_ERR(obj)) |
917 | return obj; | |
1a5e3fb1 | 918 | |
6371ca3b WN |
919 | CHECK_ERR(bpf_object__elf_init(obj), err, out); |
920 | CHECK_ERR(bpf_object__check_endianness(obj), err, out); | |
921 | CHECK_ERR(bpf_object__elf_collect(obj), err, out); | |
922 | CHECK_ERR(bpf_object__collect_reloc(obj), err, out); | |
923 | CHECK_ERR(bpf_object__validate(obj), err, out); | |
1a5e3fb1 WN |
924 | |
925 | bpf_object__elf_finish(obj); | |
926 | return obj; | |
927 | out: | |
928 | bpf_object__close(obj); | |
6371ca3b | 929 | return ERR_PTR(err); |
1a5e3fb1 WN |
930 | } |
931 | ||
932 | struct bpf_object *bpf_object__open(const char *path) | |
933 | { | |
934 | /* param validation */ | |
935 | if (!path) | |
936 | return NULL; | |
937 | ||
938 | pr_debug("loading %s\n", path); | |
939 | ||
6c956392 WN |
940 | return __bpf_object__open(path, NULL, 0); |
941 | } | |
942 | ||
943 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
acf860ae WN |
944 | size_t obj_buf_sz, |
945 | const char *name) | |
6c956392 | 946 | { |
acf860ae WN |
947 | char tmp_name[64]; |
948 | ||
6c956392 WN |
949 | /* param validation */ |
950 | if (!obj_buf || obj_buf_sz <= 0) | |
951 | return NULL; | |
952 | ||
acf860ae WN |
953 | if (!name) { |
954 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
955 | (unsigned long)obj_buf, | |
956 | (unsigned long)obj_buf_sz); | |
957 | tmp_name[sizeof(tmp_name) - 1] = '\0'; | |
958 | name = tmp_name; | |
959 | } | |
960 | pr_debug("loading object '%s' from buffer\n", | |
961 | name); | |
6c956392 | 962 | |
acf860ae | 963 | return __bpf_object__open(name, obj_buf, obj_buf_sz); |
1a5e3fb1 WN |
964 | } |
965 | ||
52d3352e WN |
966 | int bpf_object__unload(struct bpf_object *obj) |
967 | { | |
968 | size_t i; | |
969 | ||
970 | if (!obj) | |
971 | return -EINVAL; | |
972 | ||
973 | for (i = 0; i < obj->nr_map_fds; i++) | |
974 | zclose(obj->map_fds[i]); | |
975 | zfree(&obj->map_fds); | |
976 | obj->nr_map_fds = 0; | |
977 | ||
55cffde2 WN |
978 | for (i = 0; i < obj->nr_programs; i++) |
979 | bpf_program__unload(&obj->programs[i]); | |
980 | ||
52d3352e WN |
981 | return 0; |
982 | } | |
983 | ||
984 | int bpf_object__load(struct bpf_object *obj) | |
985 | { | |
6371ca3b WN |
986 | int err; |
987 | ||
52d3352e WN |
988 | if (!obj) |
989 | return -EINVAL; | |
990 | ||
991 | if (obj->loaded) { | |
992 | pr_warning("object should not be loaded twice\n"); | |
993 | return -EINVAL; | |
994 | } | |
995 | ||
996 | obj->loaded = true; | |
6371ca3b WN |
997 | |
998 | CHECK_ERR(bpf_object__create_maps(obj), err, out); | |
999 | CHECK_ERR(bpf_object__relocate(obj), err, out); | |
1000 | CHECK_ERR(bpf_object__load_progs(obj), err, out); | |
52d3352e WN |
1001 | |
1002 | return 0; | |
1003 | out: | |
1004 | bpf_object__unload(obj); | |
1005 | pr_warning("failed to load object '%s'\n", obj->path); | |
6371ca3b | 1006 | return err; |
52d3352e WN |
1007 | } |
1008 | ||
1a5e3fb1 WN |
1009 | void bpf_object__close(struct bpf_object *obj) |
1010 | { | |
a5b8bd47 WN |
1011 | size_t i; |
1012 | ||
1a5e3fb1 WN |
1013 | if (!obj) |
1014 | return; | |
1015 | ||
1016 | bpf_object__elf_finish(obj); | |
52d3352e | 1017 | bpf_object__unload(obj); |
1a5e3fb1 | 1018 | |
0b3d1efa | 1019 | zfree(&obj->maps_buf); |
a5b8bd47 WN |
1020 | |
1021 | if (obj->programs && obj->nr_programs) { | |
1022 | for (i = 0; i < obj->nr_programs; i++) | |
1023 | bpf_program__exit(&obj->programs[i]); | |
1024 | } | |
1025 | zfree(&obj->programs); | |
1026 | ||
9a208eff | 1027 | list_del(&obj->list); |
1a5e3fb1 WN |
1028 | free(obj); |
1029 | } | |
aa9b1ac3 | 1030 | |
9a208eff WN |
1031 | struct bpf_object * |
1032 | bpf_object__next(struct bpf_object *prev) | |
1033 | { | |
1034 | struct bpf_object *next; | |
1035 | ||
1036 | if (!prev) | |
1037 | next = list_first_entry(&bpf_objects_list, | |
1038 | struct bpf_object, | |
1039 | list); | |
1040 | else | |
1041 | next = list_next_entry(prev, list); | |
1042 | ||
1043 | /* Empty list is noticed here so don't need checking on entry. */ | |
1044 | if (&next->list == &bpf_objects_list) | |
1045 | return NULL; | |
1046 | ||
1047 | return next; | |
1048 | } | |
1049 | ||
acf860ae WN |
1050 | const char * |
1051 | bpf_object__get_name(struct bpf_object *obj) | |
1052 | { | |
1053 | if (!obj) | |
6371ca3b | 1054 | return ERR_PTR(-EINVAL); |
acf860ae WN |
1055 | return obj->path; |
1056 | } | |
1057 | ||
45825d8a WN |
1058 | unsigned int |
1059 | bpf_object__get_kversion(struct bpf_object *obj) | |
1060 | { | |
1061 | if (!obj) | |
1062 | return 0; | |
1063 | return obj->kern_version; | |
1064 | } | |
1065 | ||
aa9b1ac3 WN |
1066 | struct bpf_program * |
1067 | bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) | |
1068 | { | |
1069 | size_t idx; | |
1070 | ||
1071 | if (!obj->programs) | |
1072 | return NULL; | |
1073 | /* First handler */ | |
1074 | if (prev == NULL) | |
1075 | return &obj->programs[0]; | |
1076 | ||
1077 | if (prev->obj != obj) { | |
1078 | pr_warning("error: program handler doesn't match object\n"); | |
1079 | return NULL; | |
1080 | } | |
1081 | ||
1082 | idx = (prev - obj->programs) + 1; | |
1083 | if (idx >= obj->nr_programs) | |
1084 | return NULL; | |
1085 | return &obj->programs[idx]; | |
1086 | } | |
1087 | ||
1088 | int bpf_program__set_private(struct bpf_program *prog, | |
1089 | void *priv, | |
1090 | bpf_program_clear_priv_t clear_priv) | |
1091 | { | |
1092 | if (prog->priv && prog->clear_priv) | |
1093 | prog->clear_priv(prog, prog->priv); | |
1094 | ||
1095 | prog->priv = priv; | |
1096 | prog->clear_priv = clear_priv; | |
1097 | return 0; | |
1098 | } | |
1099 | ||
1100 | int bpf_program__get_private(struct bpf_program *prog, void **ppriv) | |
1101 | { | |
1102 | *ppriv = prog->priv; | |
1103 | return 0; | |
1104 | } | |
1105 | ||
715f8db9 | 1106 | const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
1107 | { |
1108 | const char *title; | |
1109 | ||
1110 | title = prog->section_name; | |
715f8db9 | 1111 | if (needs_copy) { |
aa9b1ac3 WN |
1112 | title = strdup(title); |
1113 | if (!title) { | |
1114 | pr_warning("failed to strdup program title\n"); | |
6371ca3b | 1115 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
1116 | } |
1117 | } | |
1118 | ||
1119 | return title; | |
1120 | } | |
1121 | ||
1122 | int bpf_program__fd(struct bpf_program *prog) | |
1123 | { | |
1124 | return prog->fd; | |
1125 | } |