Commit | Line | Data |
---|---|---|
1b76c13e WN |
1 | /* |
2 | * Common eBPF ELF object loading operations. | |
3 | * | |
4 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
5 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
6 | * Copyright (C) 2015 Huawei Inc. | |
7 | */ | |
8 | ||
9 | #include <stdlib.h> | |
b3f59d66 WN |
10 | #include <stdio.h> |
11 | #include <stdarg.h> | |
34090915 | 12 | #include <inttypes.h> |
b3f59d66 | 13 | #include <string.h> |
1b76c13e | 14 | #include <unistd.h> |
1a5e3fb1 WN |
15 | #include <fcntl.h> |
16 | #include <errno.h> | |
1b76c13e | 17 | #include <asm/unistd.h> |
cb1e5e96 | 18 | #include <linux/kernel.h> |
1b76c13e | 19 | #include <linux/bpf.h> |
9a208eff | 20 | #include <linux/list.h> |
1a5e3fb1 WN |
21 | #include <libelf.h> |
22 | #include <gelf.h> | |
1b76c13e WN |
23 | |
24 | #include "libbpf.h" | |
52d3352e | 25 | #include "bpf.h" |
b3f59d66 WN |
26 | |
27 | #define __printf(a, b) __attribute__((format(printf, a, b))) | |
28 | ||
29 | __printf(1, 2) | |
30 | static int __base_pr(const char *format, ...) | |
31 | { | |
32 | va_list args; | |
33 | int err; | |
34 | ||
35 | va_start(args, format); | |
36 | err = vfprintf(stderr, format, args); | |
37 | va_end(args); | |
38 | return err; | |
39 | } | |
40 | ||
41 | static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; | |
42 | static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; | |
43 | static __printf(1, 2) libbpf_print_fn_t __pr_debug; | |
44 | ||
45 | #define __pr(func, fmt, ...) \ | |
46 | do { \ | |
47 | if ((func)) \ | |
48 | (func)("libbpf: " fmt, ##__VA_ARGS__); \ | |
49 | } while (0) | |
50 | ||
51 | #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) | |
52 | #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) | |
53 | #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) | |
54 | ||
55 | void libbpf_set_print(libbpf_print_fn_t warn, | |
56 | libbpf_print_fn_t info, | |
57 | libbpf_print_fn_t debug) | |
58 | { | |
59 | __pr_warning = warn; | |
60 | __pr_info = info; | |
61 | __pr_debug = debug; | |
62 | } | |
1a5e3fb1 WN |
63 | |
64 | /* Copied from tools/perf/util/util.h */ | |
65 | #ifndef zfree | |
66 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
67 | #endif | |
68 | ||
69 | #ifndef zclose | |
70 | # define zclose(fd) ({ \ | |
71 | int ___err = 0; \ | |
72 | if ((fd) >= 0) \ | |
73 | ___err = close((fd)); \ | |
74 | fd = -1; \ | |
75 | ___err; }) | |
76 | #endif | |
77 | ||
78 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
79 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
80 | #else | |
81 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
82 | #endif | |
83 | ||
a5b8bd47 WN |
84 | /* |
85 | * bpf_prog should be a better name but it has been used in | |
86 | * linux/filter.h. | |
87 | */ | |
88 | struct bpf_program { | |
89 | /* Index in elf obj file, for relocation use. */ | |
90 | int idx; | |
91 | char *section_name; | |
92 | struct bpf_insn *insns; | |
93 | size_t insns_cnt; | |
34090915 WN |
94 | |
95 | struct { | |
96 | int insn_idx; | |
97 | int map_idx; | |
98 | } *reloc_desc; | |
99 | int nr_reloc; | |
55cffde2 WN |
100 | |
101 | int fd; | |
aa9b1ac3 WN |
102 | |
103 | struct bpf_object *obj; | |
104 | void *priv; | |
105 | bpf_program_clear_priv_t clear_priv; | |
a5b8bd47 WN |
106 | }; |
107 | ||
9a208eff WN |
108 | static LIST_HEAD(bpf_objects_list); |
109 | ||
1a5e3fb1 | 110 | struct bpf_object { |
cb1e5e96 WN |
111 | char license[64]; |
112 | u32 kern_version; | |
0b3d1efa WN |
113 | void *maps_buf; |
114 | size_t maps_buf_sz; | |
115 | ||
a5b8bd47 WN |
116 | struct bpf_program *programs; |
117 | size_t nr_programs; | |
52d3352e WN |
118 | int *map_fds; |
119 | /* | |
120 | * This field is required because maps_buf will be freed and | |
121 | * maps_buf_sz will be set to 0 after loaded. | |
122 | */ | |
123 | size_t nr_map_fds; | |
124 | bool loaded; | |
a5b8bd47 | 125 | |
1a5e3fb1 WN |
126 | /* |
127 | * Information when doing elf related work. Only valid if fd | |
128 | * is valid. | |
129 | */ | |
130 | struct { | |
131 | int fd; | |
6c956392 WN |
132 | void *obj_buf; |
133 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
134 | Elf *elf; |
135 | GElf_Ehdr ehdr; | |
bec7d68c | 136 | Elf_Data *symbols; |
b62f06e8 WN |
137 | struct { |
138 | GElf_Shdr shdr; | |
139 | Elf_Data *data; | |
140 | } *reloc; | |
141 | int nr_reloc; | |
1a5e3fb1 | 142 | } efile; |
9a208eff WN |
143 | /* |
144 | * All loaded bpf_object is linked in a list, which is | |
145 | * hidden to caller. bpf_objects__<func> handlers deal with | |
146 | * all objects. | |
147 | */ | |
148 | struct list_head list; | |
1a5e3fb1 WN |
149 | char path[]; |
150 | }; | |
151 | #define obj_elf_valid(o) ((o)->efile.elf) | |
152 | ||
55cffde2 WN |
153 | static void bpf_program__unload(struct bpf_program *prog) |
154 | { | |
155 | if (!prog) | |
156 | return; | |
157 | ||
158 | zclose(prog->fd); | |
159 | } | |
160 | ||
a5b8bd47 WN |
161 | static void bpf_program__exit(struct bpf_program *prog) |
162 | { | |
163 | if (!prog) | |
164 | return; | |
165 | ||
aa9b1ac3 WN |
166 | if (prog->clear_priv) |
167 | prog->clear_priv(prog, prog->priv); | |
168 | ||
169 | prog->priv = NULL; | |
170 | prog->clear_priv = NULL; | |
171 | ||
55cffde2 | 172 | bpf_program__unload(prog); |
a5b8bd47 WN |
173 | zfree(&prog->section_name); |
174 | zfree(&prog->insns); | |
34090915 WN |
175 | zfree(&prog->reloc_desc); |
176 | ||
177 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
178 | prog->insns_cnt = 0; |
179 | prog->idx = -1; | |
180 | } | |
181 | ||
182 | static int | |
183 | bpf_program__init(void *data, size_t size, char *name, int idx, | |
184 | struct bpf_program *prog) | |
185 | { | |
186 | if (size < sizeof(struct bpf_insn)) { | |
187 | pr_warning("corrupted section '%s'\n", name); | |
188 | return -EINVAL; | |
189 | } | |
190 | ||
191 | bzero(prog, sizeof(*prog)); | |
192 | ||
193 | prog->section_name = strdup(name); | |
194 | if (!prog->section_name) { | |
195 | pr_warning("failed to alloc name for prog %s\n", | |
196 | name); | |
197 | goto errout; | |
198 | } | |
199 | ||
200 | prog->insns = malloc(size); | |
201 | if (!prog->insns) { | |
202 | pr_warning("failed to alloc insns for %s\n", name); | |
203 | goto errout; | |
204 | } | |
205 | prog->insns_cnt = size / sizeof(struct bpf_insn); | |
206 | memcpy(prog->insns, data, | |
207 | prog->insns_cnt * sizeof(struct bpf_insn)); | |
208 | prog->idx = idx; | |
55cffde2 | 209 | prog->fd = -1; |
a5b8bd47 WN |
210 | |
211 | return 0; | |
212 | errout: | |
213 | bpf_program__exit(prog); | |
214 | return -ENOMEM; | |
215 | } | |
216 | ||
217 | static int | |
218 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
219 | char *name, int idx) | |
220 | { | |
221 | struct bpf_program prog, *progs; | |
222 | int nr_progs, err; | |
223 | ||
224 | err = bpf_program__init(data, size, name, idx, &prog); | |
225 | if (err) | |
226 | return err; | |
227 | ||
228 | progs = obj->programs; | |
229 | nr_progs = obj->nr_programs; | |
230 | ||
231 | progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); | |
232 | if (!progs) { | |
233 | /* | |
234 | * In this case the original obj->programs | |
235 | * is still valid, so don't need special treat for | |
236 | * bpf_close_object(). | |
237 | */ | |
238 | pr_warning("failed to alloc a new program '%s'\n", | |
239 | name); | |
240 | bpf_program__exit(&prog); | |
241 | return -ENOMEM; | |
242 | } | |
243 | ||
244 | pr_debug("found program %s\n", prog.section_name); | |
245 | obj->programs = progs; | |
246 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 247 | prog.obj = obj; |
a5b8bd47 WN |
248 | progs[nr_progs] = prog; |
249 | return 0; | |
250 | } | |
251 | ||
6c956392 WN |
252 | static struct bpf_object *bpf_object__new(const char *path, |
253 | void *obj_buf, | |
254 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
255 | { |
256 | struct bpf_object *obj; | |
257 | ||
258 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
259 | if (!obj) { | |
260 | pr_warning("alloc memory failed for %s\n", path); | |
261 | return NULL; | |
262 | } | |
263 | ||
264 | strcpy(obj->path, path); | |
265 | obj->efile.fd = -1; | |
6c956392 WN |
266 | |
267 | /* | |
268 | * Caller of this function should also calls | |
269 | * bpf_object__elf_finish() after data collection to return | |
270 | * obj_buf to user. If not, we should duplicate the buffer to | |
271 | * avoid user freeing them before elf finish. | |
272 | */ | |
273 | obj->efile.obj_buf = obj_buf; | |
274 | obj->efile.obj_buf_sz = obj_buf_sz; | |
275 | ||
52d3352e | 276 | obj->loaded = false; |
9a208eff WN |
277 | |
278 | INIT_LIST_HEAD(&obj->list); | |
279 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
280 | return obj; |
281 | } | |
282 | ||
283 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
284 | { | |
285 | if (!obj_elf_valid(obj)) | |
286 | return; | |
287 | ||
288 | if (obj->efile.elf) { | |
289 | elf_end(obj->efile.elf); | |
290 | obj->efile.elf = NULL; | |
291 | } | |
bec7d68c | 292 | obj->efile.symbols = NULL; |
b62f06e8 WN |
293 | |
294 | zfree(&obj->efile.reloc); | |
295 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 296 | zclose(obj->efile.fd); |
6c956392 WN |
297 | obj->efile.obj_buf = NULL; |
298 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
299 | } |
300 | ||
301 | static int bpf_object__elf_init(struct bpf_object *obj) | |
302 | { | |
303 | int err = 0; | |
304 | GElf_Ehdr *ep; | |
305 | ||
306 | if (obj_elf_valid(obj)) { | |
307 | pr_warning("elf init: internal error\n"); | |
308 | return -EEXIST; | |
309 | } | |
310 | ||
6c956392 WN |
311 | if (obj->efile.obj_buf_sz > 0) { |
312 | /* | |
313 | * obj_buf should have been validated by | |
314 | * bpf_object__open_buffer(). | |
315 | */ | |
316 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
317 | obj->efile.obj_buf_sz); | |
318 | } else { | |
319 | obj->efile.fd = open(obj->path, O_RDONLY); | |
320 | if (obj->efile.fd < 0) { | |
321 | pr_warning("failed to open %s: %s\n", obj->path, | |
322 | strerror(errno)); | |
323 | return -errno; | |
324 | } | |
325 | ||
326 | obj->efile.elf = elf_begin(obj->efile.fd, | |
327 | LIBBPF_ELF_C_READ_MMAP, | |
328 | NULL); | |
1a5e3fb1 WN |
329 | } |
330 | ||
1a5e3fb1 WN |
331 | if (!obj->efile.elf) { |
332 | pr_warning("failed to open %s as ELF file\n", | |
333 | obj->path); | |
334 | err = -EINVAL; | |
335 | goto errout; | |
336 | } | |
337 | ||
338 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
339 | pr_warning("failed to get EHDR from %s\n", | |
340 | obj->path); | |
341 | err = -EINVAL; | |
342 | goto errout; | |
343 | } | |
344 | ep = &obj->efile.ehdr; | |
345 | ||
346 | if ((ep->e_type != ET_REL) || (ep->e_machine != 0)) { | |
347 | pr_warning("%s is not an eBPF object file\n", | |
348 | obj->path); | |
349 | err = -EINVAL; | |
350 | goto errout; | |
351 | } | |
352 | ||
353 | return 0; | |
354 | errout: | |
355 | bpf_object__elf_finish(obj); | |
356 | return err; | |
357 | } | |
358 | ||
cc4228d5 WN |
359 | static int |
360 | bpf_object__check_endianness(struct bpf_object *obj) | |
361 | { | |
362 | static unsigned int const endian = 1; | |
363 | ||
364 | switch (obj->efile.ehdr.e_ident[EI_DATA]) { | |
365 | case ELFDATA2LSB: | |
366 | /* We are big endian, BPF obj is little endian. */ | |
367 | if (*(unsigned char const *)&endian != 1) | |
368 | goto mismatch; | |
369 | break; | |
370 | ||
371 | case ELFDATA2MSB: | |
372 | /* We are little endian, BPF obj is big endian. */ | |
373 | if (*(unsigned char const *)&endian != 0) | |
374 | goto mismatch; | |
375 | break; | |
376 | default: | |
377 | return -EINVAL; | |
378 | } | |
379 | ||
380 | return 0; | |
381 | ||
382 | mismatch: | |
383 | pr_warning("Error: endianness mismatch.\n"); | |
384 | return -EINVAL; | |
385 | } | |
386 | ||
cb1e5e96 WN |
387 | static int |
388 | bpf_object__init_license(struct bpf_object *obj, | |
389 | void *data, size_t size) | |
390 | { | |
391 | memcpy(obj->license, data, | |
392 | min(size, sizeof(obj->license) - 1)); | |
393 | pr_debug("license of %s is %s\n", obj->path, obj->license); | |
394 | return 0; | |
395 | } | |
396 | ||
397 | static int | |
398 | bpf_object__init_kversion(struct bpf_object *obj, | |
399 | void *data, size_t size) | |
400 | { | |
401 | u32 kver; | |
402 | ||
403 | if (size != sizeof(kver)) { | |
404 | pr_warning("invalid kver section in %s\n", obj->path); | |
405 | return -EINVAL; | |
406 | } | |
407 | memcpy(&kver, data, sizeof(kver)); | |
408 | obj->kern_version = kver; | |
409 | pr_debug("kernel version of %s is %x\n", obj->path, | |
410 | obj->kern_version); | |
411 | return 0; | |
412 | } | |
413 | ||
0b3d1efa WN |
414 | static int |
415 | bpf_object__init_maps(struct bpf_object *obj, void *data, | |
416 | size_t size) | |
417 | { | |
418 | if (size == 0) { | |
419 | pr_debug("%s doesn't need map definition\n", | |
420 | obj->path); | |
421 | return 0; | |
422 | } | |
423 | ||
424 | obj->maps_buf = malloc(size); | |
425 | if (!obj->maps_buf) { | |
426 | pr_warning("malloc maps failed: %s\n", obj->path); | |
427 | return -ENOMEM; | |
428 | } | |
429 | ||
430 | obj->maps_buf_sz = size; | |
431 | memcpy(obj->maps_buf, data, size); | |
432 | pr_debug("maps in %s: %ld bytes\n", obj->path, (long)size); | |
433 | return 0; | |
434 | } | |
435 | ||
29603665 WN |
436 | static int bpf_object__elf_collect(struct bpf_object *obj) |
437 | { | |
438 | Elf *elf = obj->efile.elf; | |
439 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
440 | Elf_Scn *scn = NULL; | |
441 | int idx = 0, err = 0; | |
442 | ||
443 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
444 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
445 | pr_warning("failed to get e_shstrndx from %s\n", | |
446 | obj->path); | |
447 | return -EINVAL; | |
448 | } | |
449 | ||
450 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
451 | char *name; | |
452 | GElf_Shdr sh; | |
453 | Elf_Data *data; | |
454 | ||
455 | idx++; | |
456 | if (gelf_getshdr(scn, &sh) != &sh) { | |
457 | pr_warning("failed to get section header from %s\n", | |
458 | obj->path); | |
459 | err = -EINVAL; | |
460 | goto out; | |
461 | } | |
462 | ||
463 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
464 | if (!name) { | |
465 | pr_warning("failed to get section name from %s\n", | |
466 | obj->path); | |
467 | err = -EINVAL; | |
468 | goto out; | |
469 | } | |
470 | ||
471 | data = elf_getdata(scn, 0); | |
472 | if (!data) { | |
473 | pr_warning("failed to get section data from %s(%s)\n", | |
474 | name, obj->path); | |
475 | err = -EINVAL; | |
476 | goto out; | |
477 | } | |
478 | pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", | |
479 | name, (unsigned long)data->d_size, | |
480 | (int)sh.sh_link, (unsigned long)sh.sh_flags, | |
481 | (int)sh.sh_type); | |
cb1e5e96 WN |
482 | |
483 | if (strcmp(name, "license") == 0) | |
484 | err = bpf_object__init_license(obj, | |
485 | data->d_buf, | |
486 | data->d_size); | |
487 | else if (strcmp(name, "version") == 0) | |
488 | err = bpf_object__init_kversion(obj, | |
489 | data->d_buf, | |
490 | data->d_size); | |
0b3d1efa WN |
491 | else if (strcmp(name, "maps") == 0) |
492 | err = bpf_object__init_maps(obj, data->d_buf, | |
493 | data->d_size); | |
bec7d68c WN |
494 | else if (sh.sh_type == SHT_SYMTAB) { |
495 | if (obj->efile.symbols) { | |
496 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
497 | obj->path); | |
498 | err = -EEXIST; | |
499 | } else | |
500 | obj->efile.symbols = data; | |
a5b8bd47 WN |
501 | } else if ((sh.sh_type == SHT_PROGBITS) && |
502 | (sh.sh_flags & SHF_EXECINSTR) && | |
503 | (data->d_size > 0)) { | |
504 | err = bpf_object__add_program(obj, data->d_buf, | |
505 | data->d_size, name, idx); | |
506 | if (err) { | |
507 | char errmsg[128]; | |
508 | strerror_r(-err, errmsg, sizeof(errmsg)); | |
509 | pr_warning("failed to alloc program %s (%s): %s", | |
510 | name, obj->path, errmsg); | |
511 | } | |
b62f06e8 WN |
512 | } else if (sh.sh_type == SHT_REL) { |
513 | void *reloc = obj->efile.reloc; | |
514 | int nr_reloc = obj->efile.nr_reloc + 1; | |
515 | ||
516 | reloc = realloc(reloc, | |
517 | sizeof(*obj->efile.reloc) * nr_reloc); | |
518 | if (!reloc) { | |
519 | pr_warning("realloc failed\n"); | |
520 | err = -ENOMEM; | |
521 | } else { | |
522 | int n = nr_reloc - 1; | |
523 | ||
524 | obj->efile.reloc = reloc; | |
525 | obj->efile.nr_reloc = nr_reloc; | |
526 | ||
527 | obj->efile.reloc[n].shdr = sh; | |
528 | obj->efile.reloc[n].data = data; | |
529 | } | |
bec7d68c | 530 | } |
cb1e5e96 WN |
531 | if (err) |
532 | goto out; | |
29603665 WN |
533 | } |
534 | out: | |
535 | return err; | |
536 | } | |
537 | ||
34090915 WN |
538 | static struct bpf_program * |
539 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
540 | { | |
541 | struct bpf_program *prog; | |
542 | size_t i; | |
543 | ||
544 | for (i = 0; i < obj->nr_programs; i++) { | |
545 | prog = &obj->programs[i]; | |
546 | if (prog->idx == idx) | |
547 | return prog; | |
548 | } | |
549 | return NULL; | |
550 | } | |
551 | ||
552 | static int | |
553 | bpf_program__collect_reloc(struct bpf_program *prog, | |
554 | size_t nr_maps, GElf_Shdr *shdr, | |
555 | Elf_Data *data, Elf_Data *symbols) | |
556 | { | |
557 | int i, nrels; | |
558 | ||
559 | pr_debug("collecting relocating info for: '%s'\n", | |
560 | prog->section_name); | |
561 | nrels = shdr->sh_size / shdr->sh_entsize; | |
562 | ||
563 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
564 | if (!prog->reloc_desc) { | |
565 | pr_warning("failed to alloc memory in relocation\n"); | |
566 | return -ENOMEM; | |
567 | } | |
568 | prog->nr_reloc = nrels; | |
569 | ||
570 | for (i = 0; i < nrels; i++) { | |
571 | GElf_Sym sym; | |
572 | GElf_Rel rel; | |
573 | unsigned int insn_idx; | |
574 | struct bpf_insn *insns = prog->insns; | |
575 | size_t map_idx; | |
576 | ||
577 | if (!gelf_getrel(data, i, &rel)) { | |
578 | pr_warning("relocation: failed to get %d reloc\n", i); | |
579 | return -EINVAL; | |
580 | } | |
581 | ||
582 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); | |
583 | pr_debug("relocation: insn_idx=%u\n", insn_idx); | |
584 | ||
585 | if (!gelf_getsym(symbols, | |
586 | GELF_R_SYM(rel.r_info), | |
587 | &sym)) { | |
588 | pr_warning("relocation: symbol %"PRIx64" not found\n", | |
589 | GELF_R_SYM(rel.r_info)); | |
590 | return -EINVAL; | |
591 | } | |
592 | ||
593 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { | |
594 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
595 | insn_idx, insns[insn_idx].code); | |
596 | return -EINVAL; | |
597 | } | |
598 | ||
599 | map_idx = sym.st_value / sizeof(struct bpf_map_def); | |
600 | if (map_idx >= nr_maps) { | |
601 | pr_warning("bpf relocation: map_idx %d large than %d\n", | |
602 | (int)map_idx, (int)nr_maps - 1); | |
603 | return -EINVAL; | |
604 | } | |
605 | ||
606 | prog->reloc_desc[i].insn_idx = insn_idx; | |
607 | prog->reloc_desc[i].map_idx = map_idx; | |
608 | } | |
609 | return 0; | |
610 | } | |
611 | ||
52d3352e WN |
612 | static int |
613 | bpf_object__create_maps(struct bpf_object *obj) | |
614 | { | |
615 | unsigned int i; | |
616 | size_t nr_maps; | |
617 | int *pfd; | |
618 | ||
619 | nr_maps = obj->maps_buf_sz / sizeof(struct bpf_map_def); | |
620 | if (!obj->maps_buf || !nr_maps) { | |
621 | pr_debug("don't need create maps for %s\n", | |
622 | obj->path); | |
623 | return 0; | |
624 | } | |
625 | ||
626 | obj->map_fds = malloc(sizeof(int) * nr_maps); | |
627 | if (!obj->map_fds) { | |
628 | pr_warning("realloc perf_bpf_map_fds failed\n"); | |
629 | return -ENOMEM; | |
630 | } | |
631 | obj->nr_map_fds = nr_maps; | |
632 | ||
633 | /* fill all fd with -1 */ | |
634 | memset(obj->map_fds, -1, sizeof(int) * nr_maps); | |
635 | ||
636 | pfd = obj->map_fds; | |
637 | for (i = 0; i < nr_maps; i++) { | |
638 | struct bpf_map_def def; | |
639 | ||
640 | def = *(struct bpf_map_def *)(obj->maps_buf + | |
641 | i * sizeof(struct bpf_map_def)); | |
642 | ||
643 | *pfd = bpf_create_map(def.type, | |
644 | def.key_size, | |
645 | def.value_size, | |
646 | def.max_entries); | |
647 | if (*pfd < 0) { | |
648 | size_t j; | |
649 | int err = *pfd; | |
650 | ||
651 | pr_warning("failed to create map: %s\n", | |
652 | strerror(errno)); | |
653 | for (j = 0; j < i; j++) | |
654 | zclose(obj->map_fds[j]); | |
655 | obj->nr_map_fds = 0; | |
656 | zfree(&obj->map_fds); | |
657 | return err; | |
658 | } | |
659 | pr_debug("create map: fd=%d\n", *pfd); | |
660 | pfd++; | |
661 | } | |
662 | ||
663 | zfree(&obj->maps_buf); | |
664 | obj->maps_buf_sz = 0; | |
665 | return 0; | |
666 | } | |
667 | ||
8a47a6c5 WN |
668 | static int |
669 | bpf_program__relocate(struct bpf_program *prog, int *map_fds) | |
670 | { | |
671 | int i; | |
672 | ||
673 | if (!prog || !prog->reloc_desc) | |
674 | return 0; | |
675 | ||
676 | for (i = 0; i < prog->nr_reloc; i++) { | |
677 | int insn_idx, map_idx; | |
678 | struct bpf_insn *insns = prog->insns; | |
679 | ||
680 | insn_idx = prog->reloc_desc[i].insn_idx; | |
681 | map_idx = prog->reloc_desc[i].map_idx; | |
682 | ||
683 | if (insn_idx >= (int)prog->insns_cnt) { | |
684 | pr_warning("relocation out of range: '%s'\n", | |
685 | prog->section_name); | |
686 | return -ERANGE; | |
687 | } | |
688 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
689 | insns[insn_idx].imm = map_fds[map_idx]; | |
690 | } | |
691 | ||
692 | zfree(&prog->reloc_desc); | |
693 | prog->nr_reloc = 0; | |
694 | return 0; | |
695 | } | |
696 | ||
697 | ||
698 | static int | |
699 | bpf_object__relocate(struct bpf_object *obj) | |
700 | { | |
701 | struct bpf_program *prog; | |
702 | size_t i; | |
703 | int err; | |
704 | ||
705 | for (i = 0; i < obj->nr_programs; i++) { | |
706 | prog = &obj->programs[i]; | |
707 | ||
708 | err = bpf_program__relocate(prog, obj->map_fds); | |
709 | if (err) { | |
710 | pr_warning("failed to relocate '%s'\n", | |
711 | prog->section_name); | |
712 | return err; | |
713 | } | |
714 | } | |
715 | return 0; | |
716 | } | |
717 | ||
34090915 WN |
718 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
719 | { | |
720 | int i, err; | |
721 | ||
722 | if (!obj_elf_valid(obj)) { | |
723 | pr_warning("Internal error: elf object is closed\n"); | |
724 | return -EINVAL; | |
725 | } | |
726 | ||
727 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
728 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
729 | Elf_Data *data = obj->efile.reloc[i].data; | |
730 | int idx = shdr->sh_info; | |
731 | struct bpf_program *prog; | |
732 | size_t nr_maps = obj->maps_buf_sz / | |
733 | sizeof(struct bpf_map_def); | |
734 | ||
735 | if (shdr->sh_type != SHT_REL) { | |
736 | pr_warning("internal error at %d\n", __LINE__); | |
737 | return -EINVAL; | |
738 | } | |
739 | ||
740 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
741 | if (!prog) { | |
742 | pr_warning("relocation failed: no %d section\n", | |
743 | idx); | |
744 | return -ENOENT; | |
745 | } | |
746 | ||
747 | err = bpf_program__collect_reloc(prog, nr_maps, | |
748 | shdr, data, | |
749 | obj->efile.symbols); | |
750 | if (err) | |
751 | return -EINVAL; | |
752 | } | |
753 | return 0; | |
754 | } | |
755 | ||
55cffde2 WN |
756 | static int |
757 | load_program(struct bpf_insn *insns, int insns_cnt, | |
758 | char *license, u32 kern_version, int *pfd) | |
759 | { | |
760 | int ret; | |
761 | char *log_buf; | |
762 | ||
763 | if (!insns || !insns_cnt) | |
764 | return -EINVAL; | |
765 | ||
766 | log_buf = malloc(BPF_LOG_BUF_SIZE); | |
767 | if (!log_buf) | |
768 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
769 | ||
770 | ret = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns, | |
771 | insns_cnt, license, kern_version, | |
772 | log_buf, BPF_LOG_BUF_SIZE); | |
773 | ||
774 | if (ret >= 0) { | |
775 | *pfd = ret; | |
776 | ret = 0; | |
777 | goto out; | |
778 | } | |
779 | ||
780 | ret = -EINVAL; | |
781 | pr_warning("load bpf program failed: %s\n", strerror(errno)); | |
782 | ||
783 | if (log_buf) { | |
784 | pr_warning("-- BEGIN DUMP LOG ---\n"); | |
785 | pr_warning("\n%s\n", log_buf); | |
786 | pr_warning("-- END LOG --\n"); | |
787 | } | |
788 | ||
789 | out: | |
790 | free(log_buf); | |
791 | return ret; | |
792 | } | |
793 | ||
794 | static int | |
795 | bpf_program__load(struct bpf_program *prog, | |
796 | char *license, u32 kern_version) | |
797 | { | |
798 | int err, fd; | |
799 | ||
800 | err = load_program(prog->insns, prog->insns_cnt, | |
801 | license, kern_version, &fd); | |
802 | if (!err) | |
803 | prog->fd = fd; | |
804 | ||
805 | if (err) | |
806 | pr_warning("failed to load program '%s'\n", | |
807 | prog->section_name); | |
808 | zfree(&prog->insns); | |
809 | prog->insns_cnt = 0; | |
810 | return err; | |
811 | } | |
812 | ||
813 | static int | |
814 | bpf_object__load_progs(struct bpf_object *obj) | |
815 | { | |
816 | size_t i; | |
817 | int err; | |
818 | ||
819 | for (i = 0; i < obj->nr_programs; i++) { | |
820 | err = bpf_program__load(&obj->programs[i], | |
821 | obj->license, | |
822 | obj->kern_version); | |
823 | if (err) | |
824 | return err; | |
825 | } | |
826 | return 0; | |
827 | } | |
828 | ||
cb1e5e96 WN |
829 | static int bpf_object__validate(struct bpf_object *obj) |
830 | { | |
831 | if (obj->kern_version == 0) { | |
832 | pr_warning("%s doesn't provide kernel version\n", | |
833 | obj->path); | |
834 | return -EINVAL; | |
835 | } | |
836 | return 0; | |
837 | } | |
838 | ||
1a5e3fb1 | 839 | static struct bpf_object * |
6c956392 | 840 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) |
1a5e3fb1 WN |
841 | { |
842 | struct bpf_object *obj; | |
843 | ||
844 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
845 | pr_warning("failed to init libelf for %s\n", path); | |
846 | return NULL; | |
847 | } | |
848 | ||
6c956392 | 849 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
1a5e3fb1 WN |
850 | if (!obj) |
851 | return NULL; | |
852 | ||
853 | if (bpf_object__elf_init(obj)) | |
854 | goto out; | |
cc4228d5 WN |
855 | if (bpf_object__check_endianness(obj)) |
856 | goto out; | |
29603665 WN |
857 | if (bpf_object__elf_collect(obj)) |
858 | goto out; | |
34090915 WN |
859 | if (bpf_object__collect_reloc(obj)) |
860 | goto out; | |
cb1e5e96 WN |
861 | if (bpf_object__validate(obj)) |
862 | goto out; | |
1a5e3fb1 WN |
863 | |
864 | bpf_object__elf_finish(obj); | |
865 | return obj; | |
866 | out: | |
867 | bpf_object__close(obj); | |
868 | return NULL; | |
869 | } | |
870 | ||
871 | struct bpf_object *bpf_object__open(const char *path) | |
872 | { | |
873 | /* param validation */ | |
874 | if (!path) | |
875 | return NULL; | |
876 | ||
877 | pr_debug("loading %s\n", path); | |
878 | ||
6c956392 WN |
879 | return __bpf_object__open(path, NULL, 0); |
880 | } | |
881 | ||
882 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
acf860ae WN |
883 | size_t obj_buf_sz, |
884 | const char *name) | |
6c956392 | 885 | { |
acf860ae WN |
886 | char tmp_name[64]; |
887 | ||
6c956392 WN |
888 | /* param validation */ |
889 | if (!obj_buf || obj_buf_sz <= 0) | |
890 | return NULL; | |
891 | ||
acf860ae WN |
892 | if (!name) { |
893 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
894 | (unsigned long)obj_buf, | |
895 | (unsigned long)obj_buf_sz); | |
896 | tmp_name[sizeof(tmp_name) - 1] = '\0'; | |
897 | name = tmp_name; | |
898 | } | |
899 | pr_debug("loading object '%s' from buffer\n", | |
900 | name); | |
6c956392 | 901 | |
acf860ae | 902 | return __bpf_object__open(name, obj_buf, obj_buf_sz); |
1a5e3fb1 WN |
903 | } |
904 | ||
52d3352e WN |
905 | int bpf_object__unload(struct bpf_object *obj) |
906 | { | |
907 | size_t i; | |
908 | ||
909 | if (!obj) | |
910 | return -EINVAL; | |
911 | ||
912 | for (i = 0; i < obj->nr_map_fds; i++) | |
913 | zclose(obj->map_fds[i]); | |
914 | zfree(&obj->map_fds); | |
915 | obj->nr_map_fds = 0; | |
916 | ||
55cffde2 WN |
917 | for (i = 0; i < obj->nr_programs; i++) |
918 | bpf_program__unload(&obj->programs[i]); | |
919 | ||
52d3352e WN |
920 | return 0; |
921 | } | |
922 | ||
923 | int bpf_object__load(struct bpf_object *obj) | |
924 | { | |
925 | if (!obj) | |
926 | return -EINVAL; | |
927 | ||
928 | if (obj->loaded) { | |
929 | pr_warning("object should not be loaded twice\n"); | |
930 | return -EINVAL; | |
931 | } | |
932 | ||
933 | obj->loaded = true; | |
934 | if (bpf_object__create_maps(obj)) | |
935 | goto out; | |
8a47a6c5 WN |
936 | if (bpf_object__relocate(obj)) |
937 | goto out; | |
55cffde2 WN |
938 | if (bpf_object__load_progs(obj)) |
939 | goto out; | |
52d3352e WN |
940 | |
941 | return 0; | |
942 | out: | |
943 | bpf_object__unload(obj); | |
944 | pr_warning("failed to load object '%s'\n", obj->path); | |
945 | return -EINVAL; | |
946 | } | |
947 | ||
1a5e3fb1 WN |
948 | void bpf_object__close(struct bpf_object *obj) |
949 | { | |
a5b8bd47 WN |
950 | size_t i; |
951 | ||
1a5e3fb1 WN |
952 | if (!obj) |
953 | return; | |
954 | ||
955 | bpf_object__elf_finish(obj); | |
52d3352e | 956 | bpf_object__unload(obj); |
1a5e3fb1 | 957 | |
0b3d1efa | 958 | zfree(&obj->maps_buf); |
a5b8bd47 WN |
959 | |
960 | if (obj->programs && obj->nr_programs) { | |
961 | for (i = 0; i < obj->nr_programs; i++) | |
962 | bpf_program__exit(&obj->programs[i]); | |
963 | } | |
964 | zfree(&obj->programs); | |
965 | ||
9a208eff | 966 | list_del(&obj->list); |
1a5e3fb1 WN |
967 | free(obj); |
968 | } | |
aa9b1ac3 | 969 | |
9a208eff WN |
970 | struct bpf_object * |
971 | bpf_object__next(struct bpf_object *prev) | |
972 | { | |
973 | struct bpf_object *next; | |
974 | ||
975 | if (!prev) | |
976 | next = list_first_entry(&bpf_objects_list, | |
977 | struct bpf_object, | |
978 | list); | |
979 | else | |
980 | next = list_next_entry(prev, list); | |
981 | ||
982 | /* Empty list is noticed here so don't need checking on entry. */ | |
983 | if (&next->list == &bpf_objects_list) | |
984 | return NULL; | |
985 | ||
986 | return next; | |
987 | } | |
988 | ||
acf860ae WN |
989 | const char * |
990 | bpf_object__get_name(struct bpf_object *obj) | |
991 | { | |
992 | if (!obj) | |
993 | return NULL; | |
994 | return obj->path; | |
995 | } | |
996 | ||
aa9b1ac3 WN |
997 | struct bpf_program * |
998 | bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) | |
999 | { | |
1000 | size_t idx; | |
1001 | ||
1002 | if (!obj->programs) | |
1003 | return NULL; | |
1004 | /* First handler */ | |
1005 | if (prev == NULL) | |
1006 | return &obj->programs[0]; | |
1007 | ||
1008 | if (prev->obj != obj) { | |
1009 | pr_warning("error: program handler doesn't match object\n"); | |
1010 | return NULL; | |
1011 | } | |
1012 | ||
1013 | idx = (prev - obj->programs) + 1; | |
1014 | if (idx >= obj->nr_programs) | |
1015 | return NULL; | |
1016 | return &obj->programs[idx]; | |
1017 | } | |
1018 | ||
1019 | int bpf_program__set_private(struct bpf_program *prog, | |
1020 | void *priv, | |
1021 | bpf_program_clear_priv_t clear_priv) | |
1022 | { | |
1023 | if (prog->priv && prog->clear_priv) | |
1024 | prog->clear_priv(prog, prog->priv); | |
1025 | ||
1026 | prog->priv = priv; | |
1027 | prog->clear_priv = clear_priv; | |
1028 | return 0; | |
1029 | } | |
1030 | ||
1031 | int bpf_program__get_private(struct bpf_program *prog, void **ppriv) | |
1032 | { | |
1033 | *ppriv = prog->priv; | |
1034 | return 0; | |
1035 | } | |
1036 | ||
715f8db9 | 1037 | const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
1038 | { |
1039 | const char *title; | |
1040 | ||
1041 | title = prog->section_name; | |
715f8db9 | 1042 | if (needs_copy) { |
aa9b1ac3 WN |
1043 | title = strdup(title); |
1044 | if (!title) { | |
1045 | pr_warning("failed to strdup program title\n"); | |
1046 | return NULL; | |
1047 | } | |
1048 | } | |
1049 | ||
1050 | return title; | |
1051 | } | |
1052 | ||
1053 | int bpf_program__fd(struct bpf_program *prog) | |
1054 | { | |
1055 | return prog->fd; | |
1056 | } |