Commit | Line | Data |
---|---|---|
1b76c13e WN |
1 | /* |
2 | * Common eBPF ELF object loading operations. | |
3 | * | |
4 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
5 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
6 | * Copyright (C) 2015 Huawei Inc. | |
7 | */ | |
8 | ||
9 | #include <stdlib.h> | |
b3f59d66 WN |
10 | #include <stdio.h> |
11 | #include <stdarg.h> | |
34090915 | 12 | #include <inttypes.h> |
b3f59d66 | 13 | #include <string.h> |
1b76c13e | 14 | #include <unistd.h> |
1a5e3fb1 WN |
15 | #include <fcntl.h> |
16 | #include <errno.h> | |
1b76c13e | 17 | #include <asm/unistd.h> |
cb1e5e96 | 18 | #include <linux/kernel.h> |
1b76c13e | 19 | #include <linux/bpf.h> |
1a5e3fb1 WN |
20 | #include <libelf.h> |
21 | #include <gelf.h> | |
1b76c13e WN |
22 | |
23 | #include "libbpf.h" | |
52d3352e | 24 | #include "bpf.h" |
b3f59d66 WN |
25 | |
26 | #define __printf(a, b) __attribute__((format(printf, a, b))) | |
27 | ||
28 | __printf(1, 2) | |
29 | static int __base_pr(const char *format, ...) | |
30 | { | |
31 | va_list args; | |
32 | int err; | |
33 | ||
34 | va_start(args, format); | |
35 | err = vfprintf(stderr, format, args); | |
36 | va_end(args); | |
37 | return err; | |
38 | } | |
39 | ||
40 | static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; | |
41 | static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; | |
42 | static __printf(1, 2) libbpf_print_fn_t __pr_debug; | |
43 | ||
44 | #define __pr(func, fmt, ...) \ | |
45 | do { \ | |
46 | if ((func)) \ | |
47 | (func)("libbpf: " fmt, ##__VA_ARGS__); \ | |
48 | } while (0) | |
49 | ||
50 | #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) | |
51 | #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) | |
52 | #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) | |
53 | ||
54 | void libbpf_set_print(libbpf_print_fn_t warn, | |
55 | libbpf_print_fn_t info, | |
56 | libbpf_print_fn_t debug) | |
57 | { | |
58 | __pr_warning = warn; | |
59 | __pr_info = info; | |
60 | __pr_debug = debug; | |
61 | } | |
1a5e3fb1 WN |
62 | |
63 | /* Copied from tools/perf/util/util.h */ | |
64 | #ifndef zfree | |
65 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
66 | #endif | |
67 | ||
68 | #ifndef zclose | |
69 | # define zclose(fd) ({ \ | |
70 | int ___err = 0; \ | |
71 | if ((fd) >= 0) \ | |
72 | ___err = close((fd)); \ | |
73 | fd = -1; \ | |
74 | ___err; }) | |
75 | #endif | |
76 | ||
77 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
78 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
79 | #else | |
80 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
81 | #endif | |
82 | ||
a5b8bd47 WN |
83 | /* |
84 | * bpf_prog should be a better name but it has been used in | |
85 | * linux/filter.h. | |
86 | */ | |
87 | struct bpf_program { | |
88 | /* Index in elf obj file, for relocation use. */ | |
89 | int idx; | |
90 | char *section_name; | |
91 | struct bpf_insn *insns; | |
92 | size_t insns_cnt; | |
34090915 WN |
93 | |
94 | struct { | |
95 | int insn_idx; | |
96 | int map_idx; | |
97 | } *reloc_desc; | |
98 | int nr_reloc; | |
55cffde2 WN |
99 | |
100 | int fd; | |
a5b8bd47 WN |
101 | }; |
102 | ||
1a5e3fb1 | 103 | struct bpf_object { |
cb1e5e96 WN |
104 | char license[64]; |
105 | u32 kern_version; | |
0b3d1efa WN |
106 | void *maps_buf; |
107 | size_t maps_buf_sz; | |
108 | ||
a5b8bd47 WN |
109 | struct bpf_program *programs; |
110 | size_t nr_programs; | |
52d3352e WN |
111 | int *map_fds; |
112 | /* | |
113 | * This field is required because maps_buf will be freed and | |
114 | * maps_buf_sz will be set to 0 after loaded. | |
115 | */ | |
116 | size_t nr_map_fds; | |
117 | bool loaded; | |
a5b8bd47 | 118 | |
1a5e3fb1 WN |
119 | /* |
120 | * Information when doing elf related work. Only valid if fd | |
121 | * is valid. | |
122 | */ | |
123 | struct { | |
124 | int fd; | |
6c956392 WN |
125 | void *obj_buf; |
126 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
127 | Elf *elf; |
128 | GElf_Ehdr ehdr; | |
bec7d68c | 129 | Elf_Data *symbols; |
b62f06e8 WN |
130 | struct { |
131 | GElf_Shdr shdr; | |
132 | Elf_Data *data; | |
133 | } *reloc; | |
134 | int nr_reloc; | |
1a5e3fb1 WN |
135 | } efile; |
136 | char path[]; | |
137 | }; | |
138 | #define obj_elf_valid(o) ((o)->efile.elf) | |
139 | ||
55cffde2 WN |
140 | static void bpf_program__unload(struct bpf_program *prog) |
141 | { | |
142 | if (!prog) | |
143 | return; | |
144 | ||
145 | zclose(prog->fd); | |
146 | } | |
147 | ||
a5b8bd47 WN |
148 | static void bpf_program__exit(struct bpf_program *prog) |
149 | { | |
150 | if (!prog) | |
151 | return; | |
152 | ||
55cffde2 | 153 | bpf_program__unload(prog); |
a5b8bd47 WN |
154 | zfree(&prog->section_name); |
155 | zfree(&prog->insns); | |
34090915 WN |
156 | zfree(&prog->reloc_desc); |
157 | ||
158 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
159 | prog->insns_cnt = 0; |
160 | prog->idx = -1; | |
161 | } | |
162 | ||
163 | static int | |
164 | bpf_program__init(void *data, size_t size, char *name, int idx, | |
165 | struct bpf_program *prog) | |
166 | { | |
167 | if (size < sizeof(struct bpf_insn)) { | |
168 | pr_warning("corrupted section '%s'\n", name); | |
169 | return -EINVAL; | |
170 | } | |
171 | ||
172 | bzero(prog, sizeof(*prog)); | |
173 | ||
174 | prog->section_name = strdup(name); | |
175 | if (!prog->section_name) { | |
176 | pr_warning("failed to alloc name for prog %s\n", | |
177 | name); | |
178 | goto errout; | |
179 | } | |
180 | ||
181 | prog->insns = malloc(size); | |
182 | if (!prog->insns) { | |
183 | pr_warning("failed to alloc insns for %s\n", name); | |
184 | goto errout; | |
185 | } | |
186 | prog->insns_cnt = size / sizeof(struct bpf_insn); | |
187 | memcpy(prog->insns, data, | |
188 | prog->insns_cnt * sizeof(struct bpf_insn)); | |
189 | prog->idx = idx; | |
55cffde2 | 190 | prog->fd = -1; |
a5b8bd47 WN |
191 | |
192 | return 0; | |
193 | errout: | |
194 | bpf_program__exit(prog); | |
195 | return -ENOMEM; | |
196 | } | |
197 | ||
198 | static int | |
199 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
200 | char *name, int idx) | |
201 | { | |
202 | struct bpf_program prog, *progs; | |
203 | int nr_progs, err; | |
204 | ||
205 | err = bpf_program__init(data, size, name, idx, &prog); | |
206 | if (err) | |
207 | return err; | |
208 | ||
209 | progs = obj->programs; | |
210 | nr_progs = obj->nr_programs; | |
211 | ||
212 | progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); | |
213 | if (!progs) { | |
214 | /* | |
215 | * In this case the original obj->programs | |
216 | * is still valid, so don't need special treat for | |
217 | * bpf_close_object(). | |
218 | */ | |
219 | pr_warning("failed to alloc a new program '%s'\n", | |
220 | name); | |
221 | bpf_program__exit(&prog); | |
222 | return -ENOMEM; | |
223 | } | |
224 | ||
225 | pr_debug("found program %s\n", prog.section_name); | |
226 | obj->programs = progs; | |
227 | obj->nr_programs = nr_progs + 1; | |
228 | progs[nr_progs] = prog; | |
229 | return 0; | |
230 | } | |
231 | ||
6c956392 WN |
232 | static struct bpf_object *bpf_object__new(const char *path, |
233 | void *obj_buf, | |
234 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
235 | { |
236 | struct bpf_object *obj; | |
237 | ||
238 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
239 | if (!obj) { | |
240 | pr_warning("alloc memory failed for %s\n", path); | |
241 | return NULL; | |
242 | } | |
243 | ||
244 | strcpy(obj->path, path); | |
245 | obj->efile.fd = -1; | |
6c956392 WN |
246 | |
247 | /* | |
248 | * Caller of this function should also calls | |
249 | * bpf_object__elf_finish() after data collection to return | |
250 | * obj_buf to user. If not, we should duplicate the buffer to | |
251 | * avoid user freeing them before elf finish. | |
252 | */ | |
253 | obj->efile.obj_buf = obj_buf; | |
254 | obj->efile.obj_buf_sz = obj_buf_sz; | |
255 | ||
52d3352e | 256 | obj->loaded = false; |
1a5e3fb1 WN |
257 | return obj; |
258 | } | |
259 | ||
260 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
261 | { | |
262 | if (!obj_elf_valid(obj)) | |
263 | return; | |
264 | ||
265 | if (obj->efile.elf) { | |
266 | elf_end(obj->efile.elf); | |
267 | obj->efile.elf = NULL; | |
268 | } | |
bec7d68c | 269 | obj->efile.symbols = NULL; |
b62f06e8 WN |
270 | |
271 | zfree(&obj->efile.reloc); | |
272 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 273 | zclose(obj->efile.fd); |
6c956392 WN |
274 | obj->efile.obj_buf = NULL; |
275 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
276 | } |
277 | ||
278 | static int bpf_object__elf_init(struct bpf_object *obj) | |
279 | { | |
280 | int err = 0; | |
281 | GElf_Ehdr *ep; | |
282 | ||
283 | if (obj_elf_valid(obj)) { | |
284 | pr_warning("elf init: internal error\n"); | |
285 | return -EEXIST; | |
286 | } | |
287 | ||
6c956392 WN |
288 | if (obj->efile.obj_buf_sz > 0) { |
289 | /* | |
290 | * obj_buf should have been validated by | |
291 | * bpf_object__open_buffer(). | |
292 | */ | |
293 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
294 | obj->efile.obj_buf_sz); | |
295 | } else { | |
296 | obj->efile.fd = open(obj->path, O_RDONLY); | |
297 | if (obj->efile.fd < 0) { | |
298 | pr_warning("failed to open %s: %s\n", obj->path, | |
299 | strerror(errno)); | |
300 | return -errno; | |
301 | } | |
302 | ||
303 | obj->efile.elf = elf_begin(obj->efile.fd, | |
304 | LIBBPF_ELF_C_READ_MMAP, | |
305 | NULL); | |
1a5e3fb1 WN |
306 | } |
307 | ||
1a5e3fb1 WN |
308 | if (!obj->efile.elf) { |
309 | pr_warning("failed to open %s as ELF file\n", | |
310 | obj->path); | |
311 | err = -EINVAL; | |
312 | goto errout; | |
313 | } | |
314 | ||
315 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
316 | pr_warning("failed to get EHDR from %s\n", | |
317 | obj->path); | |
318 | err = -EINVAL; | |
319 | goto errout; | |
320 | } | |
321 | ep = &obj->efile.ehdr; | |
322 | ||
323 | if ((ep->e_type != ET_REL) || (ep->e_machine != 0)) { | |
324 | pr_warning("%s is not an eBPF object file\n", | |
325 | obj->path); | |
326 | err = -EINVAL; | |
327 | goto errout; | |
328 | } | |
329 | ||
330 | return 0; | |
331 | errout: | |
332 | bpf_object__elf_finish(obj); | |
333 | return err; | |
334 | } | |
335 | ||
cc4228d5 WN |
336 | static int |
337 | bpf_object__check_endianness(struct bpf_object *obj) | |
338 | { | |
339 | static unsigned int const endian = 1; | |
340 | ||
341 | switch (obj->efile.ehdr.e_ident[EI_DATA]) { | |
342 | case ELFDATA2LSB: | |
343 | /* We are big endian, BPF obj is little endian. */ | |
344 | if (*(unsigned char const *)&endian != 1) | |
345 | goto mismatch; | |
346 | break; | |
347 | ||
348 | case ELFDATA2MSB: | |
349 | /* We are little endian, BPF obj is big endian. */ | |
350 | if (*(unsigned char const *)&endian != 0) | |
351 | goto mismatch; | |
352 | break; | |
353 | default: | |
354 | return -EINVAL; | |
355 | } | |
356 | ||
357 | return 0; | |
358 | ||
359 | mismatch: | |
360 | pr_warning("Error: endianness mismatch.\n"); | |
361 | return -EINVAL; | |
362 | } | |
363 | ||
cb1e5e96 WN |
364 | static int |
365 | bpf_object__init_license(struct bpf_object *obj, | |
366 | void *data, size_t size) | |
367 | { | |
368 | memcpy(obj->license, data, | |
369 | min(size, sizeof(obj->license) - 1)); | |
370 | pr_debug("license of %s is %s\n", obj->path, obj->license); | |
371 | return 0; | |
372 | } | |
373 | ||
374 | static int | |
375 | bpf_object__init_kversion(struct bpf_object *obj, | |
376 | void *data, size_t size) | |
377 | { | |
378 | u32 kver; | |
379 | ||
380 | if (size != sizeof(kver)) { | |
381 | pr_warning("invalid kver section in %s\n", obj->path); | |
382 | return -EINVAL; | |
383 | } | |
384 | memcpy(&kver, data, sizeof(kver)); | |
385 | obj->kern_version = kver; | |
386 | pr_debug("kernel version of %s is %x\n", obj->path, | |
387 | obj->kern_version); | |
388 | return 0; | |
389 | } | |
390 | ||
0b3d1efa WN |
391 | static int |
392 | bpf_object__init_maps(struct bpf_object *obj, void *data, | |
393 | size_t size) | |
394 | { | |
395 | if (size == 0) { | |
396 | pr_debug("%s doesn't need map definition\n", | |
397 | obj->path); | |
398 | return 0; | |
399 | } | |
400 | ||
401 | obj->maps_buf = malloc(size); | |
402 | if (!obj->maps_buf) { | |
403 | pr_warning("malloc maps failed: %s\n", obj->path); | |
404 | return -ENOMEM; | |
405 | } | |
406 | ||
407 | obj->maps_buf_sz = size; | |
408 | memcpy(obj->maps_buf, data, size); | |
409 | pr_debug("maps in %s: %ld bytes\n", obj->path, (long)size); | |
410 | return 0; | |
411 | } | |
412 | ||
29603665 WN |
413 | static int bpf_object__elf_collect(struct bpf_object *obj) |
414 | { | |
415 | Elf *elf = obj->efile.elf; | |
416 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
417 | Elf_Scn *scn = NULL; | |
418 | int idx = 0, err = 0; | |
419 | ||
420 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
421 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
422 | pr_warning("failed to get e_shstrndx from %s\n", | |
423 | obj->path); | |
424 | return -EINVAL; | |
425 | } | |
426 | ||
427 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
428 | char *name; | |
429 | GElf_Shdr sh; | |
430 | Elf_Data *data; | |
431 | ||
432 | idx++; | |
433 | if (gelf_getshdr(scn, &sh) != &sh) { | |
434 | pr_warning("failed to get section header from %s\n", | |
435 | obj->path); | |
436 | err = -EINVAL; | |
437 | goto out; | |
438 | } | |
439 | ||
440 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
441 | if (!name) { | |
442 | pr_warning("failed to get section name from %s\n", | |
443 | obj->path); | |
444 | err = -EINVAL; | |
445 | goto out; | |
446 | } | |
447 | ||
448 | data = elf_getdata(scn, 0); | |
449 | if (!data) { | |
450 | pr_warning("failed to get section data from %s(%s)\n", | |
451 | name, obj->path); | |
452 | err = -EINVAL; | |
453 | goto out; | |
454 | } | |
455 | pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", | |
456 | name, (unsigned long)data->d_size, | |
457 | (int)sh.sh_link, (unsigned long)sh.sh_flags, | |
458 | (int)sh.sh_type); | |
cb1e5e96 WN |
459 | |
460 | if (strcmp(name, "license") == 0) | |
461 | err = bpf_object__init_license(obj, | |
462 | data->d_buf, | |
463 | data->d_size); | |
464 | else if (strcmp(name, "version") == 0) | |
465 | err = bpf_object__init_kversion(obj, | |
466 | data->d_buf, | |
467 | data->d_size); | |
0b3d1efa WN |
468 | else if (strcmp(name, "maps") == 0) |
469 | err = bpf_object__init_maps(obj, data->d_buf, | |
470 | data->d_size); | |
bec7d68c WN |
471 | else if (sh.sh_type == SHT_SYMTAB) { |
472 | if (obj->efile.symbols) { | |
473 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
474 | obj->path); | |
475 | err = -EEXIST; | |
476 | } else | |
477 | obj->efile.symbols = data; | |
a5b8bd47 WN |
478 | } else if ((sh.sh_type == SHT_PROGBITS) && |
479 | (sh.sh_flags & SHF_EXECINSTR) && | |
480 | (data->d_size > 0)) { | |
481 | err = bpf_object__add_program(obj, data->d_buf, | |
482 | data->d_size, name, idx); | |
483 | if (err) { | |
484 | char errmsg[128]; | |
485 | strerror_r(-err, errmsg, sizeof(errmsg)); | |
486 | pr_warning("failed to alloc program %s (%s): %s", | |
487 | name, obj->path, errmsg); | |
488 | } | |
b62f06e8 WN |
489 | } else if (sh.sh_type == SHT_REL) { |
490 | void *reloc = obj->efile.reloc; | |
491 | int nr_reloc = obj->efile.nr_reloc + 1; | |
492 | ||
493 | reloc = realloc(reloc, | |
494 | sizeof(*obj->efile.reloc) * nr_reloc); | |
495 | if (!reloc) { | |
496 | pr_warning("realloc failed\n"); | |
497 | err = -ENOMEM; | |
498 | } else { | |
499 | int n = nr_reloc - 1; | |
500 | ||
501 | obj->efile.reloc = reloc; | |
502 | obj->efile.nr_reloc = nr_reloc; | |
503 | ||
504 | obj->efile.reloc[n].shdr = sh; | |
505 | obj->efile.reloc[n].data = data; | |
506 | } | |
bec7d68c | 507 | } |
cb1e5e96 WN |
508 | if (err) |
509 | goto out; | |
29603665 WN |
510 | } |
511 | out: | |
512 | return err; | |
513 | } | |
514 | ||
34090915 WN |
515 | static struct bpf_program * |
516 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
517 | { | |
518 | struct bpf_program *prog; | |
519 | size_t i; | |
520 | ||
521 | for (i = 0; i < obj->nr_programs; i++) { | |
522 | prog = &obj->programs[i]; | |
523 | if (prog->idx == idx) | |
524 | return prog; | |
525 | } | |
526 | return NULL; | |
527 | } | |
528 | ||
529 | static int | |
530 | bpf_program__collect_reloc(struct bpf_program *prog, | |
531 | size_t nr_maps, GElf_Shdr *shdr, | |
532 | Elf_Data *data, Elf_Data *symbols) | |
533 | { | |
534 | int i, nrels; | |
535 | ||
536 | pr_debug("collecting relocating info for: '%s'\n", | |
537 | prog->section_name); | |
538 | nrels = shdr->sh_size / shdr->sh_entsize; | |
539 | ||
540 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
541 | if (!prog->reloc_desc) { | |
542 | pr_warning("failed to alloc memory in relocation\n"); | |
543 | return -ENOMEM; | |
544 | } | |
545 | prog->nr_reloc = nrels; | |
546 | ||
547 | for (i = 0; i < nrels; i++) { | |
548 | GElf_Sym sym; | |
549 | GElf_Rel rel; | |
550 | unsigned int insn_idx; | |
551 | struct bpf_insn *insns = prog->insns; | |
552 | size_t map_idx; | |
553 | ||
554 | if (!gelf_getrel(data, i, &rel)) { | |
555 | pr_warning("relocation: failed to get %d reloc\n", i); | |
556 | return -EINVAL; | |
557 | } | |
558 | ||
559 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); | |
560 | pr_debug("relocation: insn_idx=%u\n", insn_idx); | |
561 | ||
562 | if (!gelf_getsym(symbols, | |
563 | GELF_R_SYM(rel.r_info), | |
564 | &sym)) { | |
565 | pr_warning("relocation: symbol %"PRIx64" not found\n", | |
566 | GELF_R_SYM(rel.r_info)); | |
567 | return -EINVAL; | |
568 | } | |
569 | ||
570 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { | |
571 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
572 | insn_idx, insns[insn_idx].code); | |
573 | return -EINVAL; | |
574 | } | |
575 | ||
576 | map_idx = sym.st_value / sizeof(struct bpf_map_def); | |
577 | if (map_idx >= nr_maps) { | |
578 | pr_warning("bpf relocation: map_idx %d large than %d\n", | |
579 | (int)map_idx, (int)nr_maps - 1); | |
580 | return -EINVAL; | |
581 | } | |
582 | ||
583 | prog->reloc_desc[i].insn_idx = insn_idx; | |
584 | prog->reloc_desc[i].map_idx = map_idx; | |
585 | } | |
586 | return 0; | |
587 | } | |
588 | ||
52d3352e WN |
589 | static int |
590 | bpf_object__create_maps(struct bpf_object *obj) | |
591 | { | |
592 | unsigned int i; | |
593 | size_t nr_maps; | |
594 | int *pfd; | |
595 | ||
596 | nr_maps = obj->maps_buf_sz / sizeof(struct bpf_map_def); | |
597 | if (!obj->maps_buf || !nr_maps) { | |
598 | pr_debug("don't need create maps for %s\n", | |
599 | obj->path); | |
600 | return 0; | |
601 | } | |
602 | ||
603 | obj->map_fds = malloc(sizeof(int) * nr_maps); | |
604 | if (!obj->map_fds) { | |
605 | pr_warning("realloc perf_bpf_map_fds failed\n"); | |
606 | return -ENOMEM; | |
607 | } | |
608 | obj->nr_map_fds = nr_maps; | |
609 | ||
610 | /* fill all fd with -1 */ | |
611 | memset(obj->map_fds, -1, sizeof(int) * nr_maps); | |
612 | ||
613 | pfd = obj->map_fds; | |
614 | for (i = 0; i < nr_maps; i++) { | |
615 | struct bpf_map_def def; | |
616 | ||
617 | def = *(struct bpf_map_def *)(obj->maps_buf + | |
618 | i * sizeof(struct bpf_map_def)); | |
619 | ||
620 | *pfd = bpf_create_map(def.type, | |
621 | def.key_size, | |
622 | def.value_size, | |
623 | def.max_entries); | |
624 | if (*pfd < 0) { | |
625 | size_t j; | |
626 | int err = *pfd; | |
627 | ||
628 | pr_warning("failed to create map: %s\n", | |
629 | strerror(errno)); | |
630 | for (j = 0; j < i; j++) | |
631 | zclose(obj->map_fds[j]); | |
632 | obj->nr_map_fds = 0; | |
633 | zfree(&obj->map_fds); | |
634 | return err; | |
635 | } | |
636 | pr_debug("create map: fd=%d\n", *pfd); | |
637 | pfd++; | |
638 | } | |
639 | ||
640 | zfree(&obj->maps_buf); | |
641 | obj->maps_buf_sz = 0; | |
642 | return 0; | |
643 | } | |
644 | ||
8a47a6c5 WN |
645 | static int |
646 | bpf_program__relocate(struct bpf_program *prog, int *map_fds) | |
647 | { | |
648 | int i; | |
649 | ||
650 | if (!prog || !prog->reloc_desc) | |
651 | return 0; | |
652 | ||
653 | for (i = 0; i < prog->nr_reloc; i++) { | |
654 | int insn_idx, map_idx; | |
655 | struct bpf_insn *insns = prog->insns; | |
656 | ||
657 | insn_idx = prog->reloc_desc[i].insn_idx; | |
658 | map_idx = prog->reloc_desc[i].map_idx; | |
659 | ||
660 | if (insn_idx >= (int)prog->insns_cnt) { | |
661 | pr_warning("relocation out of range: '%s'\n", | |
662 | prog->section_name); | |
663 | return -ERANGE; | |
664 | } | |
665 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
666 | insns[insn_idx].imm = map_fds[map_idx]; | |
667 | } | |
668 | ||
669 | zfree(&prog->reloc_desc); | |
670 | prog->nr_reloc = 0; | |
671 | return 0; | |
672 | } | |
673 | ||
674 | ||
675 | static int | |
676 | bpf_object__relocate(struct bpf_object *obj) | |
677 | { | |
678 | struct bpf_program *prog; | |
679 | size_t i; | |
680 | int err; | |
681 | ||
682 | for (i = 0; i < obj->nr_programs; i++) { | |
683 | prog = &obj->programs[i]; | |
684 | ||
685 | err = bpf_program__relocate(prog, obj->map_fds); | |
686 | if (err) { | |
687 | pr_warning("failed to relocate '%s'\n", | |
688 | prog->section_name); | |
689 | return err; | |
690 | } | |
691 | } | |
692 | return 0; | |
693 | } | |
694 | ||
34090915 WN |
695 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
696 | { | |
697 | int i, err; | |
698 | ||
699 | if (!obj_elf_valid(obj)) { | |
700 | pr_warning("Internal error: elf object is closed\n"); | |
701 | return -EINVAL; | |
702 | } | |
703 | ||
704 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
705 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
706 | Elf_Data *data = obj->efile.reloc[i].data; | |
707 | int idx = shdr->sh_info; | |
708 | struct bpf_program *prog; | |
709 | size_t nr_maps = obj->maps_buf_sz / | |
710 | sizeof(struct bpf_map_def); | |
711 | ||
712 | if (shdr->sh_type != SHT_REL) { | |
713 | pr_warning("internal error at %d\n", __LINE__); | |
714 | return -EINVAL; | |
715 | } | |
716 | ||
717 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
718 | if (!prog) { | |
719 | pr_warning("relocation failed: no %d section\n", | |
720 | idx); | |
721 | return -ENOENT; | |
722 | } | |
723 | ||
724 | err = bpf_program__collect_reloc(prog, nr_maps, | |
725 | shdr, data, | |
726 | obj->efile.symbols); | |
727 | if (err) | |
728 | return -EINVAL; | |
729 | } | |
730 | return 0; | |
731 | } | |
732 | ||
55cffde2 WN |
733 | static int |
734 | load_program(struct bpf_insn *insns, int insns_cnt, | |
735 | char *license, u32 kern_version, int *pfd) | |
736 | { | |
737 | int ret; | |
738 | char *log_buf; | |
739 | ||
740 | if (!insns || !insns_cnt) | |
741 | return -EINVAL; | |
742 | ||
743 | log_buf = malloc(BPF_LOG_BUF_SIZE); | |
744 | if (!log_buf) | |
745 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
746 | ||
747 | ret = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns, | |
748 | insns_cnt, license, kern_version, | |
749 | log_buf, BPF_LOG_BUF_SIZE); | |
750 | ||
751 | if (ret >= 0) { | |
752 | *pfd = ret; | |
753 | ret = 0; | |
754 | goto out; | |
755 | } | |
756 | ||
757 | ret = -EINVAL; | |
758 | pr_warning("load bpf program failed: %s\n", strerror(errno)); | |
759 | ||
760 | if (log_buf) { | |
761 | pr_warning("-- BEGIN DUMP LOG ---\n"); | |
762 | pr_warning("\n%s\n", log_buf); | |
763 | pr_warning("-- END LOG --\n"); | |
764 | } | |
765 | ||
766 | out: | |
767 | free(log_buf); | |
768 | return ret; | |
769 | } | |
770 | ||
771 | static int | |
772 | bpf_program__load(struct bpf_program *prog, | |
773 | char *license, u32 kern_version) | |
774 | { | |
775 | int err, fd; | |
776 | ||
777 | err = load_program(prog->insns, prog->insns_cnt, | |
778 | license, kern_version, &fd); | |
779 | if (!err) | |
780 | prog->fd = fd; | |
781 | ||
782 | if (err) | |
783 | pr_warning("failed to load program '%s'\n", | |
784 | prog->section_name); | |
785 | zfree(&prog->insns); | |
786 | prog->insns_cnt = 0; | |
787 | return err; | |
788 | } | |
789 | ||
790 | static int | |
791 | bpf_object__load_progs(struct bpf_object *obj) | |
792 | { | |
793 | size_t i; | |
794 | int err; | |
795 | ||
796 | for (i = 0; i < obj->nr_programs; i++) { | |
797 | err = bpf_program__load(&obj->programs[i], | |
798 | obj->license, | |
799 | obj->kern_version); | |
800 | if (err) | |
801 | return err; | |
802 | } | |
803 | return 0; | |
804 | } | |
805 | ||
cb1e5e96 WN |
806 | static int bpf_object__validate(struct bpf_object *obj) |
807 | { | |
808 | if (obj->kern_version == 0) { | |
809 | pr_warning("%s doesn't provide kernel version\n", | |
810 | obj->path); | |
811 | return -EINVAL; | |
812 | } | |
813 | return 0; | |
814 | } | |
815 | ||
1a5e3fb1 | 816 | static struct bpf_object * |
6c956392 | 817 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) |
1a5e3fb1 WN |
818 | { |
819 | struct bpf_object *obj; | |
820 | ||
821 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
822 | pr_warning("failed to init libelf for %s\n", path); | |
823 | return NULL; | |
824 | } | |
825 | ||
6c956392 | 826 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
1a5e3fb1 WN |
827 | if (!obj) |
828 | return NULL; | |
829 | ||
830 | if (bpf_object__elf_init(obj)) | |
831 | goto out; | |
cc4228d5 WN |
832 | if (bpf_object__check_endianness(obj)) |
833 | goto out; | |
29603665 WN |
834 | if (bpf_object__elf_collect(obj)) |
835 | goto out; | |
34090915 WN |
836 | if (bpf_object__collect_reloc(obj)) |
837 | goto out; | |
cb1e5e96 WN |
838 | if (bpf_object__validate(obj)) |
839 | goto out; | |
1a5e3fb1 WN |
840 | |
841 | bpf_object__elf_finish(obj); | |
842 | return obj; | |
843 | out: | |
844 | bpf_object__close(obj); | |
845 | return NULL; | |
846 | } | |
847 | ||
848 | struct bpf_object *bpf_object__open(const char *path) | |
849 | { | |
850 | /* param validation */ | |
851 | if (!path) | |
852 | return NULL; | |
853 | ||
854 | pr_debug("loading %s\n", path); | |
855 | ||
6c956392 WN |
856 | return __bpf_object__open(path, NULL, 0); |
857 | } | |
858 | ||
859 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
860 | size_t obj_buf_sz) | |
861 | { | |
862 | /* param validation */ | |
863 | if (!obj_buf || obj_buf_sz <= 0) | |
864 | return NULL; | |
865 | ||
866 | pr_debug("loading object from buffer\n"); | |
867 | ||
868 | return __bpf_object__open("[buffer]", obj_buf, obj_buf_sz); | |
1a5e3fb1 WN |
869 | } |
870 | ||
52d3352e WN |
871 | int bpf_object__unload(struct bpf_object *obj) |
872 | { | |
873 | size_t i; | |
874 | ||
875 | if (!obj) | |
876 | return -EINVAL; | |
877 | ||
878 | for (i = 0; i < obj->nr_map_fds; i++) | |
879 | zclose(obj->map_fds[i]); | |
880 | zfree(&obj->map_fds); | |
881 | obj->nr_map_fds = 0; | |
882 | ||
55cffde2 WN |
883 | for (i = 0; i < obj->nr_programs; i++) |
884 | bpf_program__unload(&obj->programs[i]); | |
885 | ||
52d3352e WN |
886 | return 0; |
887 | } | |
888 | ||
889 | int bpf_object__load(struct bpf_object *obj) | |
890 | { | |
891 | if (!obj) | |
892 | return -EINVAL; | |
893 | ||
894 | if (obj->loaded) { | |
895 | pr_warning("object should not be loaded twice\n"); | |
896 | return -EINVAL; | |
897 | } | |
898 | ||
899 | obj->loaded = true; | |
900 | if (bpf_object__create_maps(obj)) | |
901 | goto out; | |
8a47a6c5 WN |
902 | if (bpf_object__relocate(obj)) |
903 | goto out; | |
55cffde2 WN |
904 | if (bpf_object__load_progs(obj)) |
905 | goto out; | |
52d3352e WN |
906 | |
907 | return 0; | |
908 | out: | |
909 | bpf_object__unload(obj); | |
910 | pr_warning("failed to load object '%s'\n", obj->path); | |
911 | return -EINVAL; | |
912 | } | |
913 | ||
1a5e3fb1 WN |
914 | void bpf_object__close(struct bpf_object *obj) |
915 | { | |
a5b8bd47 WN |
916 | size_t i; |
917 | ||
1a5e3fb1 WN |
918 | if (!obj) |
919 | return; | |
920 | ||
921 | bpf_object__elf_finish(obj); | |
52d3352e | 922 | bpf_object__unload(obj); |
1a5e3fb1 | 923 | |
0b3d1efa | 924 | zfree(&obj->maps_buf); |
a5b8bd47 WN |
925 | |
926 | if (obj->programs && obj->nr_programs) { | |
927 | for (i = 0; i < obj->nr_programs; i++) | |
928 | bpf_program__exit(&obj->programs[i]); | |
929 | } | |
930 | zfree(&obj->programs); | |
931 | ||
1a5e3fb1 WN |
932 | free(obj); |
933 | } |