Commit | Line | Data |
---|---|---|
69d262a9 WN |
1 | /* |
2 | * bpf-loader.c | |
3 | * | |
4 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
5 | * Copyright (C) 2015 Huawei Inc. | |
6 | */ | |
7 | ||
a08357d8 | 8 | #include <linux/bpf.h> |
69d262a9 WN |
9 | #include <bpf/libbpf.h> |
10 | #include <linux/err.h> | |
03e01f56 | 11 | #include <linux/string.h> |
69d262a9 WN |
12 | #include "perf.h" |
13 | #include "debug.h" | |
14 | #include "bpf-loader.h" | |
a08357d8 WN |
15 | #include "bpf-prologue.h" |
16 | #include "llvm-utils.h" | |
aa3abf30 WN |
17 | #include "probe-event.h" |
18 | #include "probe-finder.h" // for MAX_PROBES | |
d509db04 | 19 | #include "llvm-utils.h" |
69d262a9 WN |
20 | |
21 | #define DEFINE_PRINT_FN(name, level) \ | |
22 | static int libbpf_##name(const char *fmt, ...) \ | |
23 | { \ | |
24 | va_list args; \ | |
25 | int ret; \ | |
26 | \ | |
27 | va_start(args, fmt); \ | |
28 | ret = veprintf(level, verbose, pr_fmt(fmt), args);\ | |
29 | va_end(args); \ | |
30 | return ret; \ | |
31 | } | |
32 | ||
7a011946 WN |
33 | DEFINE_PRINT_FN(warning, 1) |
34 | DEFINE_PRINT_FN(info, 1) | |
69d262a9 WN |
35 | DEFINE_PRINT_FN(debug, 1) |
36 | ||
aa3abf30 WN |
37 | struct bpf_prog_priv { |
38 | struct perf_probe_event pev; | |
a08357d8 WN |
39 | bool need_prologue; |
40 | struct bpf_insn *insns_buf; | |
d35b3289 WN |
41 | int nr_types; |
42 | int *type_mapping; | |
aa3abf30 WN |
43 | }; |
44 | ||
ba1fae43 WN |
45 | static bool libbpf_initialized; |
46 | ||
47 | struct bpf_object * | |
48 | bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name) | |
49 | { | |
50 | struct bpf_object *obj; | |
51 | ||
52 | if (!libbpf_initialized) { | |
53 | libbpf_set_print(libbpf_warning, | |
54 | libbpf_info, | |
55 | libbpf_debug); | |
56 | libbpf_initialized = true; | |
57 | } | |
58 | ||
59 | obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name); | |
60 | if (IS_ERR(obj)) { | |
61 | pr_debug("bpf: failed to load buffer\n"); | |
62 | return ERR_PTR(-EINVAL); | |
63 | } | |
64 | ||
65 | return obj; | |
66 | } | |
67 | ||
d509db04 | 68 | struct bpf_object *bpf__prepare_load(const char *filename, bool source) |
69d262a9 WN |
69 | { |
70 | struct bpf_object *obj; | |
69d262a9 WN |
71 | |
72 | if (!libbpf_initialized) { | |
73 | libbpf_set_print(libbpf_warning, | |
74 | libbpf_info, | |
75 | libbpf_debug); | |
76 | libbpf_initialized = true; | |
77 | } | |
78 | ||
d509db04 WN |
79 | if (source) { |
80 | int err; | |
81 | void *obj_buf; | |
82 | size_t obj_buf_sz; | |
83 | ||
84 | err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz); | |
85 | if (err) | |
d3e0ce39 | 86 | return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE); |
d509db04 WN |
87 | obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename); |
88 | free(obj_buf); | |
89 | } else | |
90 | obj = bpf_object__open(filename); | |
91 | ||
6371ca3b | 92 | if (IS_ERR(obj)) { |
69d262a9 | 93 | pr_debug("bpf: failed to load %s\n", filename); |
6371ca3b | 94 | return obj; |
69d262a9 WN |
95 | } |
96 | ||
97 | return obj; | |
98 | } | |
99 | ||
100 | void bpf__clear(void) | |
101 | { | |
102 | struct bpf_object *obj, *tmp; | |
103 | ||
aa3abf30 WN |
104 | bpf_object__for_each_safe(obj, tmp) { |
105 | bpf__unprobe(obj); | |
69d262a9 | 106 | bpf_object__close(obj); |
aa3abf30 WN |
107 | } |
108 | } | |
109 | ||
110 | static void | |
80cdce76 WN |
111 | clear_prog_priv(struct bpf_program *prog __maybe_unused, |
112 | void *_priv) | |
aa3abf30 WN |
113 | { |
114 | struct bpf_prog_priv *priv = _priv; | |
115 | ||
116 | cleanup_perf_probe_events(&priv->pev, 1); | |
a08357d8 | 117 | zfree(&priv->insns_buf); |
d35b3289 | 118 | zfree(&priv->type_mapping); |
aa3abf30 WN |
119 | free(priv); |
120 | } | |
121 | ||
361f2b1d | 122 | static int |
0bb93490 | 123 | prog_config__exec(const char *value, struct perf_probe_event *pev) |
361f2b1d WN |
124 | { |
125 | pev->uprobes = true; | |
126 | pev->target = strdup(value); | |
127 | if (!pev->target) | |
128 | return -ENOMEM; | |
129 | return 0; | |
130 | } | |
131 | ||
5dbd16c0 | 132 | static int |
0bb93490 | 133 | prog_config__module(const char *value, struct perf_probe_event *pev) |
5dbd16c0 WN |
134 | { |
135 | pev->uprobes = false; | |
136 | pev->target = strdup(value); | |
137 | if (!pev->target) | |
138 | return -ENOMEM; | |
139 | return 0; | |
140 | } | |
141 | ||
03e01f56 | 142 | static int |
0bb93490 | 143 | prog_config__bool(const char *value, bool *pbool, bool invert) |
03e01f56 WN |
144 | { |
145 | int err; | |
146 | bool bool_value; | |
147 | ||
148 | if (!pbool) | |
149 | return -EINVAL; | |
150 | ||
151 | err = strtobool(value, &bool_value); | |
152 | if (err) | |
153 | return err; | |
154 | ||
155 | *pbool = invert ? !bool_value : bool_value; | |
156 | return 0; | |
157 | } | |
158 | ||
159 | static int | |
0bb93490 WN |
160 | prog_config__inlines(const char *value, |
161 | struct perf_probe_event *pev __maybe_unused) | |
03e01f56 | 162 | { |
0bb93490 | 163 | return prog_config__bool(value, &probe_conf.no_inlines, true); |
03e01f56 WN |
164 | } |
165 | ||
166 | static int | |
0bb93490 WN |
167 | prog_config__force(const char *value, |
168 | struct perf_probe_event *pev __maybe_unused) | |
03e01f56 | 169 | { |
0bb93490 | 170 | return prog_config__bool(value, &probe_conf.force_add, false); |
03e01f56 WN |
171 | } |
172 | ||
361f2b1d WN |
173 | static struct { |
174 | const char *key; | |
175 | const char *usage; | |
176 | const char *desc; | |
177 | int (*func)(const char *, struct perf_probe_event *); | |
0bb93490 | 178 | } bpf_prog_config_terms[] = { |
361f2b1d WN |
179 | { |
180 | .key = "exec", | |
181 | .usage = "exec=<full path of file>", | |
182 | .desc = "Set uprobe target", | |
0bb93490 | 183 | .func = prog_config__exec, |
361f2b1d | 184 | }, |
5dbd16c0 WN |
185 | { |
186 | .key = "module", | |
187 | .usage = "module=<module name> ", | |
188 | .desc = "Set kprobe module", | |
0bb93490 | 189 | .func = prog_config__module, |
03e01f56 WN |
190 | }, |
191 | { | |
192 | .key = "inlines", | |
193 | .usage = "inlines=[yes|no] ", | |
194 | .desc = "Probe at inline symbol", | |
0bb93490 | 195 | .func = prog_config__inlines, |
03e01f56 WN |
196 | }, |
197 | { | |
198 | .key = "force", | |
199 | .usage = "force=[yes|no] ", | |
200 | .desc = "Forcibly add events with existing name", | |
0bb93490 | 201 | .func = prog_config__force, |
03e01f56 | 202 | }, |
361f2b1d WN |
203 | }; |
204 | ||
205 | static int | |
0bb93490 WN |
206 | do_prog_config(const char *key, const char *value, |
207 | struct perf_probe_event *pev) | |
361f2b1d WN |
208 | { |
209 | unsigned int i; | |
210 | ||
211 | pr_debug("config bpf program: %s=%s\n", key, value); | |
0bb93490 WN |
212 | for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++) |
213 | if (strcmp(key, bpf_prog_config_terms[i].key) == 0) | |
214 | return bpf_prog_config_terms[i].func(value, pev); | |
361f2b1d | 215 | |
0bb93490 | 216 | pr_debug("BPF: ERROR: invalid program config option: %s=%s\n", |
361f2b1d WN |
217 | key, value); |
218 | ||
0bb93490 WN |
219 | pr_debug("\nHint: Valid options are:\n"); |
220 | for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++) | |
221 | pr_debug("\t%s:\t%s\n", bpf_prog_config_terms[i].usage, | |
222 | bpf_prog_config_terms[i].desc); | |
361f2b1d WN |
223 | pr_debug("\n"); |
224 | ||
0bb93490 | 225 | return -BPF_LOADER_ERRNO__PROGCONF_TERM; |
361f2b1d WN |
226 | } |
227 | ||
228 | static const char * | |
0bb93490 | 229 | parse_prog_config_kvpair(const char *config_str, struct perf_probe_event *pev) |
361f2b1d WN |
230 | { |
231 | char *text = strdup(config_str); | |
232 | char *sep, *line; | |
233 | const char *main_str = NULL; | |
234 | int err = 0; | |
235 | ||
236 | if (!text) { | |
237 | pr_debug("No enough memory: dup config_str failed\n"); | |
238 | return ERR_PTR(-ENOMEM); | |
239 | } | |
240 | ||
241 | line = text; | |
242 | while ((sep = strchr(line, ';'))) { | |
243 | char *equ; | |
244 | ||
245 | *sep = '\0'; | |
246 | equ = strchr(line, '='); | |
247 | if (!equ) { | |
248 | pr_warning("WARNING: invalid config in BPF object: %s\n", | |
249 | line); | |
250 | pr_warning("\tShould be 'key=value'.\n"); | |
251 | goto nextline; | |
252 | } | |
253 | *equ = '\0'; | |
254 | ||
0bb93490 | 255 | err = do_prog_config(line, equ + 1, pev); |
361f2b1d WN |
256 | if (err) |
257 | break; | |
258 | nextline: | |
259 | line = sep + 1; | |
260 | } | |
261 | ||
262 | if (!err) | |
263 | main_str = config_str + (line - text); | |
264 | free(text); | |
265 | ||
266 | return err ? ERR_PTR(err) : main_str; | |
267 | } | |
268 | ||
269 | static int | |
0bb93490 | 270 | parse_prog_config(const char *config_str, struct perf_probe_event *pev) |
361f2b1d WN |
271 | { |
272 | int err; | |
0bb93490 | 273 | const char *main_str = parse_prog_config_kvpair(config_str, pev); |
361f2b1d WN |
274 | |
275 | if (IS_ERR(main_str)) | |
276 | return PTR_ERR(main_str); | |
277 | ||
278 | err = parse_perf_probe_command(main_str, pev); | |
279 | if (err < 0) { | |
280 | pr_debug("bpf: '%s' is not a valid config string\n", | |
281 | config_str); | |
282 | /* parse failed, don't need clear pev. */ | |
283 | return -BPF_LOADER_ERRNO__CONFIG; | |
284 | } | |
285 | return 0; | |
286 | } | |
287 | ||
aa3abf30 WN |
288 | static int |
289 | config_bpf_program(struct bpf_program *prog) | |
290 | { | |
291 | struct perf_probe_event *pev = NULL; | |
292 | struct bpf_prog_priv *priv = NULL; | |
293 | const char *config_str; | |
294 | int err; | |
295 | ||
03e01f56 WN |
296 | /* Initialize per-program probing setting */ |
297 | probe_conf.no_inlines = false; | |
298 | probe_conf.force_add = false; | |
299 | ||
aa3abf30 | 300 | config_str = bpf_program__title(prog, false); |
6371ca3b | 301 | if (IS_ERR(config_str)) { |
aa3abf30 | 302 | pr_debug("bpf: unable to get title for program\n"); |
6371ca3b | 303 | return PTR_ERR(config_str); |
aa3abf30 WN |
304 | } |
305 | ||
306 | priv = calloc(sizeof(*priv), 1); | |
307 | if (!priv) { | |
308 | pr_debug("bpf: failed to alloc priv\n"); | |
309 | return -ENOMEM; | |
310 | } | |
311 | pev = &priv->pev; | |
312 | ||
313 | pr_debug("bpf: config program '%s'\n", config_str); | |
0bb93490 | 314 | err = parse_prog_config(config_str, pev); |
361f2b1d | 315 | if (err) |
aa3abf30 | 316 | goto errout; |
aa3abf30 WN |
317 | |
318 | if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) { | |
319 | pr_debug("bpf: '%s': group for event is set and not '%s'.\n", | |
320 | config_str, PERF_BPF_PROBE_GROUP); | |
d3e0ce39 | 321 | err = -BPF_LOADER_ERRNO__GROUP; |
aa3abf30 WN |
322 | goto errout; |
323 | } else if (!pev->group) | |
324 | pev->group = strdup(PERF_BPF_PROBE_GROUP); | |
325 | ||
326 | if (!pev->group) { | |
327 | pr_debug("bpf: strdup failed\n"); | |
328 | err = -ENOMEM; | |
329 | goto errout; | |
330 | } | |
331 | ||
332 | if (!pev->event) { | |
d3e0ce39 | 333 | pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n", |
aa3abf30 | 334 | config_str); |
d3e0ce39 | 335 | err = -BPF_LOADER_ERRNO__EVENTNAME; |
aa3abf30 WN |
336 | goto errout; |
337 | } | |
338 | pr_debug("bpf: config '%s' is ok\n", config_str); | |
339 | ||
80cdce76 | 340 | err = bpf_program__set_private(prog, priv, clear_prog_priv); |
aa3abf30 WN |
341 | if (err) { |
342 | pr_debug("Failed to set priv for program '%s'\n", config_str); | |
343 | goto errout; | |
344 | } | |
345 | ||
346 | return 0; | |
347 | ||
348 | errout: | |
349 | if (pev) | |
350 | clear_perf_probe_event(pev); | |
351 | free(priv); | |
352 | return err; | |
353 | } | |
354 | ||
355 | static int bpf__prepare_probe(void) | |
356 | { | |
357 | static int err = 0; | |
358 | static bool initialized = false; | |
359 | ||
360 | /* | |
361 | * Make err static, so if init failed the first, bpf__prepare_probe() | |
362 | * fails each time without calling init_probe_symbol_maps multiple | |
363 | * times. | |
364 | */ | |
365 | if (initialized) | |
366 | return err; | |
367 | ||
368 | initialized = true; | |
369 | err = init_probe_symbol_maps(false); | |
370 | if (err < 0) | |
371 | pr_debug("Failed to init_probe_symbol_maps\n"); | |
372 | probe_conf.max_probes = MAX_PROBES; | |
373 | return err; | |
374 | } | |
375 | ||
a08357d8 WN |
376 | static int |
377 | preproc_gen_prologue(struct bpf_program *prog, int n, | |
378 | struct bpf_insn *orig_insns, int orig_insns_cnt, | |
379 | struct bpf_prog_prep_result *res) | |
380 | { | |
381 | struct probe_trace_event *tev; | |
382 | struct perf_probe_event *pev; | |
383 | struct bpf_prog_priv *priv; | |
384 | struct bpf_insn *buf; | |
385 | size_t prologue_cnt = 0; | |
d35b3289 | 386 | int i, err; |
a08357d8 WN |
387 | |
388 | err = bpf_program__get_private(prog, (void **)&priv); | |
389 | if (err || !priv) | |
390 | goto errout; | |
391 | ||
392 | pev = &priv->pev; | |
393 | ||
d35b3289 | 394 | if (n < 0 || n >= priv->nr_types) |
a08357d8 WN |
395 | goto errout; |
396 | ||
d35b3289 WN |
397 | /* Find a tev belongs to that type */ |
398 | for (i = 0; i < pev->ntevs; i++) { | |
399 | if (priv->type_mapping[i] == n) | |
400 | break; | |
401 | } | |
402 | ||
403 | if (i >= pev->ntevs) { | |
404 | pr_debug("Internal error: prologue type %d not found\n", n); | |
405 | return -BPF_LOADER_ERRNO__PROLOGUE; | |
406 | } | |
407 | ||
408 | tev = &pev->tevs[i]; | |
a08357d8 WN |
409 | |
410 | buf = priv->insns_buf; | |
411 | err = bpf__gen_prologue(tev->args, tev->nargs, | |
412 | buf, &prologue_cnt, | |
413 | BPF_MAXINSNS - orig_insns_cnt); | |
414 | if (err) { | |
415 | const char *title; | |
416 | ||
417 | title = bpf_program__title(prog, false); | |
418 | if (!title) | |
419 | title = "[unknown]"; | |
420 | ||
421 | pr_debug("Failed to generate prologue for program %s\n", | |
422 | title); | |
423 | return err; | |
424 | } | |
425 | ||
426 | memcpy(&buf[prologue_cnt], orig_insns, | |
427 | sizeof(struct bpf_insn) * orig_insns_cnt); | |
428 | ||
429 | res->new_insn_ptr = buf; | |
430 | res->new_insn_cnt = prologue_cnt + orig_insns_cnt; | |
431 | res->pfd = NULL; | |
432 | return 0; | |
433 | ||
434 | errout: | |
435 | pr_debug("Internal error in preproc_gen_prologue\n"); | |
436 | return -BPF_LOADER_ERRNO__PROLOGUE; | |
437 | } | |
438 | ||
d35b3289 WN |
439 | /* |
440 | * compare_tev_args is reflexive, transitive and antisymmetric. | |
441 | * I can proof it but this margin is too narrow to contain. | |
442 | */ | |
443 | static int compare_tev_args(const void *ptev1, const void *ptev2) | |
444 | { | |
445 | int i, ret; | |
446 | const struct probe_trace_event *tev1 = | |
447 | *(const struct probe_trace_event **)ptev1; | |
448 | const struct probe_trace_event *tev2 = | |
449 | *(const struct probe_trace_event **)ptev2; | |
450 | ||
451 | ret = tev2->nargs - tev1->nargs; | |
452 | if (ret) | |
453 | return ret; | |
454 | ||
455 | for (i = 0; i < tev1->nargs; i++) { | |
456 | struct probe_trace_arg *arg1, *arg2; | |
457 | struct probe_trace_arg_ref *ref1, *ref2; | |
458 | ||
459 | arg1 = &tev1->args[i]; | |
460 | arg2 = &tev2->args[i]; | |
461 | ||
462 | ret = strcmp(arg1->value, arg2->value); | |
463 | if (ret) | |
464 | return ret; | |
465 | ||
466 | ref1 = arg1->ref; | |
467 | ref2 = arg2->ref; | |
468 | ||
469 | while (ref1 && ref2) { | |
470 | ret = ref2->offset - ref1->offset; | |
471 | if (ret) | |
472 | return ret; | |
473 | ||
474 | ref1 = ref1->next; | |
475 | ref2 = ref2->next; | |
476 | } | |
477 | ||
478 | if (ref1 || ref2) | |
479 | return ref2 ? 1 : -1; | |
480 | } | |
481 | ||
482 | return 0; | |
483 | } | |
484 | ||
485 | /* | |
486 | * Assign a type number to each tevs in a pev. | |
487 | * mapping is an array with same slots as tevs in that pev. | |
488 | * nr_types will be set to number of types. | |
489 | */ | |
490 | static int map_prologue(struct perf_probe_event *pev, int *mapping, | |
491 | int *nr_types) | |
492 | { | |
493 | int i, type = 0; | |
494 | struct probe_trace_event **ptevs; | |
495 | ||
496 | size_t array_sz = sizeof(*ptevs) * pev->ntevs; | |
497 | ||
498 | ptevs = malloc(array_sz); | |
499 | if (!ptevs) { | |
500 | pr_debug("No ehough memory: alloc ptevs failed\n"); | |
501 | return -ENOMEM; | |
502 | } | |
503 | ||
504 | pr_debug("In map_prologue, ntevs=%d\n", pev->ntevs); | |
505 | for (i = 0; i < pev->ntevs; i++) | |
506 | ptevs[i] = &pev->tevs[i]; | |
507 | ||
508 | qsort(ptevs, pev->ntevs, sizeof(*ptevs), | |
509 | compare_tev_args); | |
510 | ||
511 | for (i = 0; i < pev->ntevs; i++) { | |
512 | int n; | |
513 | ||
514 | n = ptevs[i] - pev->tevs; | |
515 | if (i == 0) { | |
516 | mapping[n] = type; | |
517 | pr_debug("mapping[%d]=%d\n", n, type); | |
518 | continue; | |
519 | } | |
520 | ||
521 | if (compare_tev_args(ptevs + i, ptevs + i - 1) == 0) | |
522 | mapping[n] = type; | |
523 | else | |
524 | mapping[n] = ++type; | |
525 | ||
526 | pr_debug("mapping[%d]=%d\n", n, mapping[n]); | |
527 | } | |
528 | free(ptevs); | |
529 | *nr_types = type + 1; | |
530 | ||
531 | return 0; | |
532 | } | |
533 | ||
a08357d8 WN |
534 | static int hook_load_preprocessor(struct bpf_program *prog) |
535 | { | |
536 | struct perf_probe_event *pev; | |
537 | struct bpf_prog_priv *priv; | |
538 | bool need_prologue = false; | |
539 | int err, i; | |
540 | ||
541 | err = bpf_program__get_private(prog, (void **)&priv); | |
542 | if (err || !priv) { | |
543 | pr_debug("Internal error when hook preprocessor\n"); | |
544 | return -BPF_LOADER_ERRNO__INTERNAL; | |
545 | } | |
546 | ||
547 | pev = &priv->pev; | |
548 | for (i = 0; i < pev->ntevs; i++) { | |
549 | struct probe_trace_event *tev = &pev->tevs[i]; | |
550 | ||
551 | if (tev->nargs > 0) { | |
552 | need_prologue = true; | |
553 | break; | |
554 | } | |
555 | } | |
556 | ||
557 | /* | |
558 | * Since all tevs don't have argument, we don't need generate | |
559 | * prologue. | |
560 | */ | |
561 | if (!need_prologue) { | |
562 | priv->need_prologue = false; | |
563 | return 0; | |
564 | } | |
565 | ||
566 | priv->need_prologue = true; | |
567 | priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS); | |
568 | if (!priv->insns_buf) { | |
569 | pr_debug("No enough memory: alloc insns_buf failed\n"); | |
570 | return -ENOMEM; | |
571 | } | |
572 | ||
d35b3289 WN |
573 | priv->type_mapping = malloc(sizeof(int) * pev->ntevs); |
574 | if (!priv->type_mapping) { | |
575 | pr_debug("No enough memory: alloc type_mapping failed\n"); | |
576 | return -ENOMEM; | |
577 | } | |
578 | memset(priv->type_mapping, -1, | |
579 | sizeof(int) * pev->ntevs); | |
580 | ||
581 | err = map_prologue(pev, priv->type_mapping, &priv->nr_types); | |
582 | if (err) | |
583 | return err; | |
584 | ||
585 | err = bpf_program__set_prep(prog, priv->nr_types, | |
a08357d8 WN |
586 | preproc_gen_prologue); |
587 | return err; | |
588 | } | |
589 | ||
aa3abf30 WN |
590 | int bpf__probe(struct bpf_object *obj) |
591 | { | |
592 | int err = 0; | |
593 | struct bpf_program *prog; | |
594 | struct bpf_prog_priv *priv; | |
595 | struct perf_probe_event *pev; | |
596 | ||
597 | err = bpf__prepare_probe(); | |
598 | if (err) { | |
599 | pr_debug("bpf__prepare_probe failed\n"); | |
600 | return err; | |
601 | } | |
602 | ||
603 | bpf_object__for_each_program(prog, obj) { | |
604 | err = config_bpf_program(prog); | |
605 | if (err) | |
606 | goto out; | |
607 | ||
608 | err = bpf_program__get_private(prog, (void **)&priv); | |
609 | if (err || !priv) | |
610 | goto out; | |
611 | pev = &priv->pev; | |
612 | ||
613 | err = convert_perf_probe_events(pev, 1); | |
614 | if (err < 0) { | |
615 | pr_debug("bpf_probe: failed to convert perf probe events"); | |
616 | goto out; | |
617 | } | |
618 | ||
619 | err = apply_perf_probe_events(pev, 1); | |
620 | if (err < 0) { | |
621 | pr_debug("bpf_probe: failed to apply perf probe events"); | |
622 | goto out; | |
623 | } | |
a08357d8 WN |
624 | |
625 | /* | |
626 | * After probing, let's consider prologue, which | |
627 | * adds program fetcher to BPF programs. | |
628 | * | |
629 | * hook_load_preprocessorr() hooks pre-processor | |
630 | * to bpf_program, let it generate prologue | |
631 | * dynamically during loading. | |
632 | */ | |
633 | err = hook_load_preprocessor(prog); | |
634 | if (err) | |
635 | goto out; | |
aa3abf30 WN |
636 | } |
637 | out: | |
638 | return err < 0 ? err : 0; | |
639 | } | |
640 | ||
641 | #define EVENTS_WRITE_BUFSIZE 4096 | |
642 | int bpf__unprobe(struct bpf_object *obj) | |
643 | { | |
644 | int err, ret = 0; | |
645 | struct bpf_program *prog; | |
646 | struct bpf_prog_priv *priv; | |
647 | ||
648 | bpf_object__for_each_program(prog, obj) { | |
649 | int i; | |
650 | ||
651 | err = bpf_program__get_private(prog, (void **)&priv); | |
652 | if (err || !priv) | |
653 | continue; | |
654 | ||
655 | for (i = 0; i < priv->pev.ntevs; i++) { | |
656 | struct probe_trace_event *tev = &priv->pev.tevs[i]; | |
657 | char name_buf[EVENTS_WRITE_BUFSIZE]; | |
658 | struct strfilter *delfilter; | |
659 | ||
660 | snprintf(name_buf, EVENTS_WRITE_BUFSIZE, | |
661 | "%s:%s", tev->group, tev->event); | |
662 | name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0'; | |
663 | ||
664 | delfilter = strfilter__new(name_buf, NULL); | |
665 | if (!delfilter) { | |
666 | pr_debug("Failed to create filter for unprobing\n"); | |
667 | ret = -ENOMEM; | |
668 | continue; | |
669 | } | |
670 | ||
671 | err = del_perf_probe_events(delfilter); | |
672 | strfilter__delete(delfilter); | |
673 | if (err) { | |
674 | pr_debug("Failed to delete %s\n", name_buf); | |
675 | ret = err; | |
676 | continue; | |
677 | } | |
678 | } | |
679 | } | |
680 | return ret; | |
681 | } | |
682 | ||
1e5e3ee8 WN |
683 | int bpf__load(struct bpf_object *obj) |
684 | { | |
685 | int err; | |
686 | ||
687 | err = bpf_object__load(obj); | |
688 | if (err) { | |
689 | pr_debug("bpf: load objects failed\n"); | |
690 | return err; | |
691 | } | |
692 | return 0; | |
693 | } | |
694 | ||
4edf30e3 WN |
695 | int bpf__foreach_tev(struct bpf_object *obj, |
696 | bpf_prog_iter_callback_t func, | |
697 | void *arg) | |
698 | { | |
699 | struct bpf_program *prog; | |
700 | int err; | |
701 | ||
702 | bpf_object__for_each_program(prog, obj) { | |
703 | struct probe_trace_event *tev; | |
704 | struct perf_probe_event *pev; | |
705 | struct bpf_prog_priv *priv; | |
706 | int i, fd; | |
707 | ||
708 | err = bpf_program__get_private(prog, | |
709 | (void **)&priv); | |
710 | if (err || !priv) { | |
711 | pr_debug("bpf: failed to get private field\n"); | |
d3e0ce39 | 712 | return -BPF_LOADER_ERRNO__INTERNAL; |
4edf30e3 WN |
713 | } |
714 | ||
715 | pev = &priv->pev; | |
716 | for (i = 0; i < pev->ntevs; i++) { | |
717 | tev = &pev->tevs[i]; | |
718 | ||
d35b3289 WN |
719 | if (priv->need_prologue) { |
720 | int type = priv->type_mapping[i]; | |
721 | ||
722 | fd = bpf_program__nth_fd(prog, type); | |
723 | } else { | |
a08357d8 | 724 | fd = bpf_program__fd(prog); |
d35b3289 | 725 | } |
a08357d8 | 726 | |
4edf30e3 WN |
727 | if (fd < 0) { |
728 | pr_debug("bpf: failed to get file descriptor\n"); | |
729 | return fd; | |
730 | } | |
731 | ||
732 | err = (*func)(tev, fd, arg); | |
733 | if (err) { | |
734 | pr_debug("bpf: call back failed, stop iterate\n"); | |
735 | return err; | |
736 | } | |
737 | } | |
738 | } | |
739 | return 0; | |
740 | } | |
741 | ||
066dacbf WN |
742 | enum bpf_map_op_type { |
743 | BPF_MAP_OP_SET_VALUE, | |
744 | }; | |
745 | ||
746 | enum bpf_map_key_type { | |
747 | BPF_MAP_KEY_ALL, | |
748 | }; | |
749 | ||
750 | struct bpf_map_op { | |
751 | struct list_head list; | |
752 | enum bpf_map_op_type op_type; | |
753 | enum bpf_map_key_type key_type; | |
754 | union { | |
755 | u64 value; | |
756 | } v; | |
757 | }; | |
758 | ||
759 | struct bpf_map_priv { | |
760 | struct list_head ops_list; | |
761 | }; | |
762 | ||
763 | static void | |
764 | bpf_map_op__delete(struct bpf_map_op *op) | |
765 | { | |
766 | if (!list_empty(&op->list)) | |
767 | list_del(&op->list); | |
768 | free(op); | |
769 | } | |
770 | ||
771 | static void | |
772 | bpf_map_priv__purge(struct bpf_map_priv *priv) | |
773 | { | |
774 | struct bpf_map_op *pos, *n; | |
775 | ||
776 | list_for_each_entry_safe(pos, n, &priv->ops_list, list) { | |
777 | list_del_init(&pos->list); | |
778 | bpf_map_op__delete(pos); | |
779 | } | |
780 | } | |
781 | ||
782 | static void | |
783 | bpf_map_priv__clear(struct bpf_map *map __maybe_unused, | |
784 | void *_priv) | |
785 | { | |
786 | struct bpf_map_priv *priv = _priv; | |
787 | ||
788 | bpf_map_priv__purge(priv); | |
789 | free(priv); | |
790 | } | |
791 | ||
792 | static struct bpf_map_op * | |
793 | bpf_map_op__new(void) | |
794 | { | |
795 | struct bpf_map_op *op; | |
796 | ||
797 | op = zalloc(sizeof(*op)); | |
798 | if (!op) { | |
799 | pr_debug("Failed to alloc bpf_map_op\n"); | |
800 | return ERR_PTR(-ENOMEM); | |
801 | } | |
802 | INIT_LIST_HEAD(&op->list); | |
803 | ||
804 | op->key_type = BPF_MAP_KEY_ALL; | |
805 | return op; | |
806 | } | |
807 | ||
808 | static int | |
809 | bpf_map__add_op(struct bpf_map *map, struct bpf_map_op *op) | |
810 | { | |
811 | struct bpf_map_priv *priv; | |
812 | const char *map_name; | |
813 | int err; | |
814 | ||
815 | map_name = bpf_map__get_name(map); | |
816 | err = bpf_map__get_private(map, (void **)&priv); | |
817 | if (err) { | |
818 | pr_debug("Failed to get private from map %s\n", map_name); | |
819 | return err; | |
820 | } | |
821 | ||
822 | if (!priv) { | |
823 | priv = zalloc(sizeof(*priv)); | |
824 | if (!priv) { | |
825 | pr_debug("No enough memory to alloc map private\n"); | |
826 | return -ENOMEM; | |
827 | } | |
828 | INIT_LIST_HEAD(&priv->ops_list); | |
829 | ||
830 | if (bpf_map__set_private(map, priv, bpf_map_priv__clear)) { | |
831 | free(priv); | |
832 | return -BPF_LOADER_ERRNO__INTERNAL; | |
833 | } | |
834 | } | |
835 | ||
836 | list_add_tail(&op->list, &priv->ops_list); | |
837 | return 0; | |
838 | } | |
839 | ||
840 | static int | |
841 | __bpf_map__config_value(struct bpf_map *map, | |
842 | struct parse_events_term *term) | |
843 | { | |
844 | struct bpf_map_def def; | |
845 | struct bpf_map_op *op; | |
846 | const char *map_name; | |
847 | int err; | |
848 | ||
849 | map_name = bpf_map__get_name(map); | |
850 | ||
851 | err = bpf_map__get_def(map, &def); | |
852 | if (err) { | |
853 | pr_debug("Unable to get map definition from '%s'\n", | |
854 | map_name); | |
855 | return -BPF_LOADER_ERRNO__INTERNAL; | |
856 | } | |
857 | ||
858 | if (def.type != BPF_MAP_TYPE_ARRAY) { | |
859 | pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n", | |
860 | map_name); | |
861 | return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE; | |
862 | } | |
863 | if (def.key_size < sizeof(unsigned int)) { | |
864 | pr_debug("Map %s has incorrect key size\n", map_name); | |
865 | return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE; | |
866 | } | |
867 | switch (def.value_size) { | |
868 | case 1: | |
869 | case 2: | |
870 | case 4: | |
871 | case 8: | |
872 | break; | |
873 | default: | |
874 | pr_debug("Map %s has incorrect value size\n", map_name); | |
875 | return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE; | |
876 | } | |
877 | ||
878 | op = bpf_map_op__new(); | |
879 | if (IS_ERR(op)) | |
880 | return PTR_ERR(op); | |
881 | op->op_type = BPF_MAP_OP_SET_VALUE; | |
882 | op->v.value = term->val.num; | |
883 | ||
884 | err = bpf_map__add_op(map, op); | |
885 | if (err) | |
886 | bpf_map_op__delete(op); | |
887 | return err; | |
888 | } | |
889 | ||
890 | static int | |
891 | bpf_map__config_value(struct bpf_map *map, | |
892 | struct parse_events_term *term, | |
893 | struct perf_evlist *evlist __maybe_unused) | |
894 | { | |
895 | if (!term->err_val) { | |
896 | pr_debug("Config value not set\n"); | |
897 | return -BPF_LOADER_ERRNO__OBJCONF_CONF; | |
898 | } | |
899 | ||
900 | if (term->type_val != PARSE_EVENTS__TERM_TYPE_NUM) { | |
901 | pr_debug("ERROR: wrong value type\n"); | |
902 | return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE; | |
903 | } | |
904 | ||
905 | return __bpf_map__config_value(map, term); | |
906 | } | |
907 | ||
908 | struct bpf_obj_config__map_func { | |
909 | const char *config_opt; | |
910 | int (*config_func)(struct bpf_map *, struct parse_events_term *, | |
911 | struct perf_evlist *); | |
912 | }; | |
913 | ||
914 | struct bpf_obj_config__map_func bpf_obj_config__map_funcs[] = { | |
915 | {"value", bpf_map__config_value}, | |
916 | }; | |
917 | ||
918 | static int | |
919 | bpf__obj_config_map(struct bpf_object *obj, | |
920 | struct parse_events_term *term, | |
921 | struct perf_evlist *evlist, | |
922 | int *key_scan_pos) | |
923 | { | |
924 | /* key is "map:<mapname>.<config opt>" */ | |
925 | char *map_name = strdup(term->config + sizeof("map:") - 1); | |
926 | struct bpf_map *map; | |
927 | int err = -BPF_LOADER_ERRNO__OBJCONF_OPT; | |
928 | char *map_opt; | |
929 | size_t i; | |
930 | ||
931 | if (!map_name) | |
932 | return -ENOMEM; | |
933 | ||
934 | map_opt = strchr(map_name, '.'); | |
935 | if (!map_opt) { | |
936 | pr_debug("ERROR: Invalid map config: %s\n", map_name); | |
937 | goto out; | |
938 | } | |
939 | ||
940 | *map_opt++ = '\0'; | |
941 | if (*map_opt == '\0') { | |
942 | pr_debug("ERROR: Invalid map option: %s\n", term->config); | |
943 | goto out; | |
944 | } | |
945 | ||
946 | map = bpf_object__get_map_by_name(obj, map_name); | |
947 | if (!map) { | |
948 | pr_debug("ERROR: Map %s doesn't exist\n", map_name); | |
949 | err = -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST; | |
950 | goto out; | |
951 | } | |
952 | ||
953 | *key_scan_pos += map_opt - map_name; | |
954 | for (i = 0; i < ARRAY_SIZE(bpf_obj_config__map_funcs); i++) { | |
955 | struct bpf_obj_config__map_func *func = | |
956 | &bpf_obj_config__map_funcs[i]; | |
957 | ||
958 | if (strcmp(map_opt, func->config_opt) == 0) { | |
959 | err = func->config_func(map, term, evlist); | |
960 | goto out; | |
961 | } | |
962 | } | |
963 | ||
964 | pr_debug("ERROR: Invalid map config option '%s'\n", map_opt); | |
965 | err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT; | |
966 | out: | |
967 | free(map_name); | |
968 | if (!err) | |
969 | key_scan_pos += strlen(map_opt); | |
970 | return err; | |
971 | } | |
972 | ||
973 | int bpf__config_obj(struct bpf_object *obj, | |
974 | struct parse_events_term *term, | |
975 | struct perf_evlist *evlist, | |
976 | int *error_pos) | |
977 | { | |
978 | int key_scan_pos = 0; | |
979 | int err; | |
980 | ||
981 | if (!obj || !term || !term->config) | |
982 | return -EINVAL; | |
983 | ||
984 | if (!prefixcmp(term->config, "map:")) { | |
985 | key_scan_pos = sizeof("map:") - 1; | |
986 | err = bpf__obj_config_map(obj, term, evlist, &key_scan_pos); | |
987 | goto out; | |
988 | } | |
989 | err = -BPF_LOADER_ERRNO__OBJCONF_OPT; | |
990 | out: | |
991 | if (error_pos) | |
992 | *error_pos = key_scan_pos; | |
993 | return err; | |
994 | ||
995 | } | |
996 | ||
d3e0ce39 WN |
997 | #define ERRNO_OFFSET(e) ((e) - __BPF_LOADER_ERRNO__START) |
998 | #define ERRCODE_OFFSET(c) ERRNO_OFFSET(BPF_LOADER_ERRNO__##c) | |
999 | #define NR_ERRNO (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START) | |
1000 | ||
1001 | static const char *bpf_loader_strerror_table[NR_ERRNO] = { | |
1002 | [ERRCODE_OFFSET(CONFIG)] = "Invalid config string", | |
1003 | [ERRCODE_OFFSET(GROUP)] = "Invalid group name", | |
1004 | [ERRCODE_OFFSET(EVENTNAME)] = "No event name found in config string", | |
1005 | [ERRCODE_OFFSET(INTERNAL)] = "BPF loader internal error", | |
1006 | [ERRCODE_OFFSET(COMPILE)] = "Error when compiling BPF scriptlet", | |
0bb93490 | 1007 | [ERRCODE_OFFSET(PROGCONF_TERM)] = "Invalid program config term in config string", |
bfc077b4 HK |
1008 | [ERRCODE_OFFSET(PROLOGUE)] = "Failed to generate prologue", |
1009 | [ERRCODE_OFFSET(PROLOGUE2BIG)] = "Prologue too big for program", | |
1010 | [ERRCODE_OFFSET(PROLOGUEOOB)] = "Offset out of bound for prologue", | |
066dacbf WN |
1011 | [ERRCODE_OFFSET(OBJCONF_OPT)] = "Invalid object config option", |
1012 | [ERRCODE_OFFSET(OBJCONF_CONF)] = "Config value not set (missing '=')", | |
1013 | [ERRCODE_OFFSET(OBJCONF_MAP_OPT)] = "Invalid object map config option", | |
1014 | [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST)] = "Target map doesn't exist", | |
1015 | [ERRCODE_OFFSET(OBJCONF_MAP_VALUE)] = "Incorrect value type for map", | |
1016 | [ERRCODE_OFFSET(OBJCONF_MAP_TYPE)] = "Incorrect map type", | |
1017 | [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE)] = "Incorrect map key size", | |
1018 | [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE)] = "Incorrect map value size", | |
d3e0ce39 WN |
1019 | }; |
1020 | ||
6371ca3b WN |
1021 | static int |
1022 | bpf_loader_strerror(int err, char *buf, size_t size) | |
1023 | { | |
1024 | char sbuf[STRERR_BUFSIZE]; | |
1025 | const char *msg; | |
1026 | ||
1027 | if (!buf || !size) | |
1028 | return -1; | |
1029 | ||
1030 | err = err > 0 ? err : -err; | |
1031 | ||
1032 | if (err >= __LIBBPF_ERRNO__START) | |
1033 | return libbpf_strerror(err, buf, size); | |
1034 | ||
d3e0ce39 WN |
1035 | if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) { |
1036 | msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)]; | |
1037 | snprintf(buf, size, "%s", msg); | |
1038 | buf[size - 1] = '\0'; | |
1039 | return 0; | |
1040 | } | |
1041 | ||
1042 | if (err >= __BPF_LOADER_ERRNO__END) | |
1043 | snprintf(buf, size, "Unknown bpf loader error %d", err); | |
1044 | else | |
1045 | snprintf(buf, size, "%s", | |
1046 | strerror_r(err, sbuf, sizeof(sbuf))); | |
1047 | ||
6371ca3b | 1048 | buf[size - 1] = '\0'; |
d3e0ce39 | 1049 | return -1; |
6371ca3b WN |
1050 | } |
1051 | ||
aa3abf30 WN |
1052 | #define bpf__strerror_head(err, buf, size) \ |
1053 | char sbuf[STRERR_BUFSIZE], *emsg;\ | |
1054 | if (!size)\ | |
1055 | return 0;\ | |
1056 | if (err < 0)\ | |
1057 | err = -err;\ | |
6371ca3b WN |
1058 | bpf_loader_strerror(err, sbuf, sizeof(sbuf));\ |
1059 | emsg = sbuf;\ | |
aa3abf30 WN |
1060 | switch (err) {\ |
1061 | default:\ | |
1062 | scnprintf(buf, size, "%s", emsg);\ | |
1063 | break; | |
1064 | ||
1065 | #define bpf__strerror_entry(val, fmt...)\ | |
1066 | case val: {\ | |
1067 | scnprintf(buf, size, fmt);\ | |
1068 | break;\ | |
1069 | } | |
1070 | ||
1071 | #define bpf__strerror_end(buf, size)\ | |
1072 | }\ | |
1073 | buf[size - 1] = '\0'; | |
1074 | ||
d3e0ce39 WN |
1075 | int bpf__strerror_prepare_load(const char *filename, bool source, |
1076 | int err, char *buf, size_t size) | |
1077 | { | |
1078 | size_t n; | |
1079 | int ret; | |
1080 | ||
1081 | n = snprintf(buf, size, "Failed to load %s%s: ", | |
1082 | filename, source ? " from source" : ""); | |
1083 | if (n >= size) { | |
1084 | buf[size - 1] = '\0'; | |
1085 | return 0; | |
1086 | } | |
1087 | buf += n; | |
1088 | size -= n; | |
1089 | ||
1090 | ret = bpf_loader_strerror(err, buf, size); | |
1091 | buf[size - 1] = '\0'; | |
1092 | return ret; | |
1093 | } | |
1094 | ||
aa3abf30 WN |
1095 | int bpf__strerror_probe(struct bpf_object *obj __maybe_unused, |
1096 | int err, char *buf, size_t size) | |
1097 | { | |
1098 | bpf__strerror_head(err, buf, size); | |
0bb93490 | 1099 | case BPF_LOADER_ERRNO__PROGCONF_TERM: { |
361f2b1d WN |
1100 | scnprintf(buf, size, "%s (add -v to see detail)", emsg); |
1101 | break; | |
1102 | } | |
03e01f56 | 1103 | bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'"); |
d3e0ce39 WN |
1104 | bpf__strerror_entry(EACCES, "You need to be root"); |
1105 | bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0"); | |
1106 | bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file"); | |
aa3abf30 WN |
1107 | bpf__strerror_end(buf, size); |
1108 | return 0; | |
69d262a9 | 1109 | } |
1e5e3ee8 | 1110 | |
d3e0ce39 | 1111 | int bpf__strerror_load(struct bpf_object *obj, |
1e5e3ee8 WN |
1112 | int err, char *buf, size_t size) |
1113 | { | |
1114 | bpf__strerror_head(err, buf, size); | |
d3e0ce39 WN |
1115 | case LIBBPF_ERRNO__KVER: { |
1116 | unsigned int obj_kver = bpf_object__get_kversion(obj); | |
1117 | unsigned int real_kver; | |
1118 | ||
1119 | if (fetch_kernel_version(&real_kver, NULL, 0)) { | |
1120 | scnprintf(buf, size, "Unable to fetch kernel version"); | |
1121 | break; | |
1122 | } | |
1123 | ||
1124 | if (obj_kver != real_kver) { | |
1125 | scnprintf(buf, size, | |
1126 | "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")", | |
1127 | KVER_PARAM(obj_kver), | |
1128 | KVER_PARAM(real_kver)); | |
1129 | break; | |
1130 | } | |
1131 | ||
1132 | scnprintf(buf, size, "Failed to load program for unknown reason"); | |
1133 | break; | |
1134 | } | |
1e5e3ee8 WN |
1135 | bpf__strerror_end(buf, size); |
1136 | return 0; | |
1137 | } | |
066dacbf WN |
1138 | |
1139 | int bpf__strerror_config_obj(struct bpf_object *obj __maybe_unused, | |
1140 | struct parse_events_term *term __maybe_unused, | |
1141 | struct perf_evlist *evlist __maybe_unused, | |
1142 | int *error_pos __maybe_unused, int err, | |
1143 | char *buf, size_t size) | |
1144 | { | |
1145 | bpf__strerror_head(err, buf, size); | |
1146 | bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE, | |
1147 | "Can't use this config term with this map type"); | |
1148 | bpf__strerror_end(buf, size); | |
1149 | return 0; | |
1150 | } |