Commit | Line | Data |
---|---|---|
8035e428 IM |
1 | /* |
2 | * builtin-annotate.c | |
3 | * | |
4 | * Builtin annotate command: Analyze the perf.data input file, | |
5 | * look up and read DSOs and symbol information and display | |
6 | * a histogram of results, along various sorting keys. | |
7 | */ | |
8 | #include "builtin.h" | |
9 | ||
10 | #include "util/util.h" | |
11 | ||
12 | #include "util/color.h" | |
5da50258 | 13 | #include <linux/list.h> |
8035e428 | 14 | #include "util/cache.h" |
43cbcd8a | 15 | #include <linux/rbtree.h> |
8035e428 IM |
16 | #include "util/symbol.h" |
17 | #include "util/string.h" | |
18 | ||
19 | #include "perf.h" | |
8f28827a | 20 | #include "util/debug.h" |
8035e428 | 21 | |
62daacb5 | 22 | #include "util/event.h" |
8035e428 IM |
23 | #include "util/parse-options.h" |
24 | #include "util/parse-events.h" | |
6baa0a5a | 25 | #include "util/thread.h" |
dd68ada2 | 26 | #include "util/sort.h" |
3d1d07ec | 27 | #include "util/hist.h" |
94c744b6 | 28 | #include "util/session.h" |
bab81b62 | 29 | #include "util/data_map.h" |
8035e428 | 30 | |
8035e428 | 31 | static char const *input_name = "perf.data"; |
8035e428 | 32 | |
fa6963b2 | 33 | static int force; |
8035e428 | 34 | |
42976487 MG |
35 | static int full_paths; |
36 | ||
301406b9 FW |
37 | static int print_line; |
38 | ||
e4204992 ACM |
39 | struct sym_hist { |
40 | u64 sum; | |
41 | u64 ip[0]; | |
42 | }; | |
43 | ||
301406b9 | 44 | struct sym_ext { |
971738f3 | 45 | struct rb_node node; |
301406b9 FW |
46 | double percent; |
47 | char *path; | |
48 | }; | |
49 | ||
e4204992 ACM |
50 | struct sym_priv { |
51 | struct sym_hist *hist; | |
52 | struct sym_ext *ext; | |
53 | }; | |
54 | ||
b32d133a ACM |
55 | static struct symbol_conf symbol_conf = { |
56 | .priv_size = sizeof(struct sym_priv), | |
57 | .try_vmlinux_path = true, | |
58 | }; | |
59 | ||
e4204992 ACM |
60 | static const char *sym_hist_filter; |
61 | ||
00a192b3 | 62 | static int symbol_filter(struct map *map __used, struct symbol *sym) |
e4204992 | 63 | { |
8f0b0373 ACM |
64 | if (sym_hist_filter == NULL || |
65 | strcmp(sym->name, sym_hist_filter) == 0) { | |
00a192b3 | 66 | struct sym_priv *priv = symbol__priv(sym); |
e4204992 ACM |
67 | const int size = (sizeof(*priv->hist) + |
68 | (sym->end - sym->start) * sizeof(u64)); | |
69 | ||
70 | priv->hist = malloc(size); | |
71 | if (priv->hist) | |
72 | memset(priv->hist, 0, size); | |
73 | return 0; | |
74 | } | |
75 | /* | |
76 | * FIXME: We should really filter it out, as we don't want to go thru symbols | |
77 | * we're not interested, and if a DSO ends up with no symbols, delete it too, | |
78 | * but right now the kernel loading routines in symbol.c bail out if no symbols | |
79 | * are found, fix it later. | |
80 | */ | |
81 | return 0; | |
82 | } | |
8035e428 | 83 | |
0b73da3f IM |
84 | /* |
85 | * collect histogram counts | |
86 | */ | |
9cffa8d5 | 87 | static void hist_hit(struct hist_entry *he, u64 ip) |
8035e428 | 88 | { |
0b73da3f IM |
89 | unsigned int sym_size, offset; |
90 | struct symbol *sym = he->sym; | |
e4204992 ACM |
91 | struct sym_priv *priv; |
92 | struct sym_hist *h; | |
8035e428 | 93 | |
0b73da3f | 94 | he->count++; |
8035e428 | 95 | |
e4204992 ACM |
96 | if (!sym || !he->map) |
97 | return; | |
98 | ||
00a192b3 | 99 | priv = symbol__priv(sym); |
e4204992 | 100 | if (!priv->hist) |
0b73da3f | 101 | return; |
8035e428 | 102 | |
0b73da3f IM |
103 | sym_size = sym->end - sym->start; |
104 | offset = ip - sym->start; | |
8035e428 | 105 | |
ed52ce2e ACM |
106 | if (verbose) |
107 | fprintf(stderr, "%s: ip=%Lx\n", __func__, | |
108 | he->map->unmap_ip(he->map, ip)); | |
109 | ||
0b73da3f IM |
110 | if (offset >= sym_size) |
111 | return; | |
8035e428 | 112 | |
e4204992 ACM |
113 | h = priv->hist; |
114 | h->sum++; | |
115 | h->ip[offset]++; | |
8035e428 | 116 | |
0b73da3f IM |
117 | if (verbose >= 3) |
118 | printf("%p %s: count++ [ip: %p, %08Lx] => %Ld\n", | |
7d37a0cb | 119 | (void *)(unsigned long)he->sym->start, |
0b73da3f | 120 | he->sym->name, |
7d37a0cb | 121 | (void *)(unsigned long)ip, ip - he->sym->start, |
e4204992 | 122 | h->ip[offset]); |
8035e428 IM |
123 | } |
124 | ||
1ed091c4 | 125 | static int hist_entry__add(struct addr_location *al, u64 count) |
8035e428 | 126 | { |
9735abf1 | 127 | bool hit; |
1ed091c4 | 128 | struct hist_entry *he = __hist_entry__add(al, NULL, count, &hit); |
9735abf1 | 129 | if (he == NULL) |
8035e428 | 130 | return -ENOMEM; |
1ed091c4 | 131 | hist_hit(he, al->addr); |
8035e428 IM |
132 | return 0; |
133 | } | |
134 | ||
d8f66248 | 135 | static int process_sample_event(event_t *event, struct perf_session *session __used) |
8035e428 | 136 | { |
1ed091c4 | 137 | struct addr_location al; |
6baa0a5a | 138 | |
62daacb5 | 139 | dump_printf("(IP, %d): %d: %p\n", event->header.misc, |
1ed091c4 | 140 | event->ip.pid, (void *)(long)event->ip.ip); |
8035e428 | 141 | |
1ed091c4 | 142 | if (event__preprocess_sample(event, &al, symbol_filter) < 0) { |
8035e428 IM |
143 | fprintf(stderr, "problem processing %d event, skipping it.\n", |
144 | event->header.type); | |
145 | return -1; | |
146 | } | |
147 | ||
1ed091c4 | 148 | if (hist_entry__add(&al, 1)) { |
ec218fc4 ACM |
149 | fprintf(stderr, "problem incrementing symbol count, " |
150 | "skipping event\n"); | |
151 | return -1; | |
8035e428 | 152 | } |
8035e428 IM |
153 | |
154 | return 0; | |
155 | } | |
156 | ||
ed52ce2e | 157 | static int parse_line(FILE *file, struct hist_entry *he, u64 len) |
0b73da3f | 158 | { |
ed52ce2e | 159 | struct symbol *sym = he->sym; |
0b73da3f | 160 | char *line = NULL, *tmp, *tmp2; |
301406b9 FW |
161 | static const char *prev_line; |
162 | static const char *prev_color; | |
0b73da3f IM |
163 | unsigned int offset; |
164 | size_t line_len; | |
ed52ce2e | 165 | u64 start; |
f37a291c | 166 | s64 line_ip; |
0b73da3f IM |
167 | int ret; |
168 | char *c; | |
169 | ||
170 | if (getline(&line, &line_len, file) < 0) | |
171 | return -1; | |
172 | if (!line) | |
173 | return -1; | |
174 | ||
175 | c = strchr(line, '\n'); | |
176 | if (c) | |
177 | *c = 0; | |
178 | ||
179 | line_ip = -1; | |
180 | offset = 0; | |
181 | ret = -2; | |
182 | ||
183 | /* | |
184 | * Strip leading spaces: | |
185 | */ | |
186 | tmp = line; | |
187 | while (*tmp) { | |
188 | if (*tmp != ' ') | |
189 | break; | |
190 | tmp++; | |
191 | } | |
192 | ||
193 | if (*tmp) { | |
194 | /* | |
195 | * Parse hexa addresses followed by ':' | |
196 | */ | |
197 | line_ip = strtoull(tmp, &tmp2, 16); | |
198 | if (*tmp2 != ':') | |
199 | line_ip = -1; | |
200 | } | |
201 | ||
ed52ce2e ACM |
202 | start = he->map->unmap_ip(he->map, sym->start); |
203 | ||
0b73da3f | 204 | if (line_ip != -1) { |
301406b9 | 205 | const char *path = NULL; |
0b73da3f IM |
206 | unsigned int hits = 0; |
207 | double percent = 0.0; | |
83a0944f | 208 | const char *color; |
00a192b3 | 209 | struct sym_priv *priv = symbol__priv(sym); |
e4204992 ACM |
210 | struct sym_ext *sym_ext = priv->ext; |
211 | struct sym_hist *h = priv->hist; | |
0b73da3f | 212 | |
ed52ce2e | 213 | offset = line_ip - start; |
0b73da3f | 214 | if (offset < len) |
e4204992 | 215 | hits = h->ip[offset]; |
0b73da3f | 216 | |
c17c2db1 | 217 | if (offset < len && sym_ext) { |
301406b9 FW |
218 | path = sym_ext[offset].path; |
219 | percent = sym_ext[offset].percent; | |
e4204992 ACM |
220 | } else if (h->sum) |
221 | percent = 100.0 * hits / h->sum; | |
0b73da3f | 222 | |
1e11fd82 | 223 | color = get_percent_color(percent); |
0b73da3f | 224 | |
301406b9 FW |
225 | /* |
226 | * Also color the filename and line if needed, with | |
227 | * the same color than the percentage. Don't print it | |
228 | * twice for close colored ip with the same filename:line | |
229 | */ | |
230 | if (path) { | |
231 | if (!prev_line || strcmp(prev_line, path) | |
232 | || color != prev_color) { | |
233 | color_fprintf(stdout, color, " %s", path); | |
234 | prev_line = path; | |
235 | prev_color = color; | |
236 | } | |
237 | } | |
238 | ||
0b73da3f IM |
239 | color_fprintf(stdout, color, " %7.2f", percent); |
240 | printf(" : "); | |
241 | color_fprintf(stdout, PERF_COLOR_BLUE, "%s\n", line); | |
242 | } else { | |
243 | if (!*line) | |
244 | printf(" :\n"); | |
245 | else | |
246 | printf(" : %s\n", line); | |
247 | } | |
248 | ||
249 | return 0; | |
250 | } | |
251 | ||
971738f3 FW |
252 | static struct rb_root root_sym_ext; |
253 | ||
254 | static void insert_source_line(struct sym_ext *sym_ext) | |
255 | { | |
256 | struct sym_ext *iter; | |
257 | struct rb_node **p = &root_sym_ext.rb_node; | |
258 | struct rb_node *parent = NULL; | |
259 | ||
260 | while (*p != NULL) { | |
261 | parent = *p; | |
262 | iter = rb_entry(parent, struct sym_ext, node); | |
263 | ||
264 | if (sym_ext->percent > iter->percent) | |
265 | p = &(*p)->rb_left; | |
266 | else | |
267 | p = &(*p)->rb_right; | |
268 | } | |
269 | ||
270 | rb_link_node(&sym_ext->node, parent, p); | |
271 | rb_insert_color(&sym_ext->node, &root_sym_ext); | |
272 | } | |
273 | ||
e4204992 | 274 | static void free_source_line(struct hist_entry *he, int len) |
301406b9 | 275 | { |
00a192b3 | 276 | struct sym_priv *priv = symbol__priv(he->sym); |
e4204992 | 277 | struct sym_ext *sym_ext = priv->ext; |
301406b9 FW |
278 | int i; |
279 | ||
280 | if (!sym_ext) | |
281 | return; | |
282 | ||
283 | for (i = 0; i < len; i++) | |
284 | free(sym_ext[i].path); | |
285 | free(sym_ext); | |
286 | ||
e4204992 | 287 | priv->ext = NULL; |
971738f3 | 288 | root_sym_ext = RB_ROOT; |
301406b9 FW |
289 | } |
290 | ||
291 | /* Get the filename:line for the colored entries */ | |
c17c2db1 | 292 | static void |
ed52ce2e | 293 | get_source_line(struct hist_entry *he, int len, const char *filename) |
301406b9 | 294 | { |
ed52ce2e ACM |
295 | struct symbol *sym = he->sym; |
296 | u64 start; | |
301406b9 FW |
297 | int i; |
298 | char cmd[PATH_MAX * 2]; | |
299 | struct sym_ext *sym_ext; | |
00a192b3 | 300 | struct sym_priv *priv = symbol__priv(sym); |
e4204992 | 301 | struct sym_hist *h = priv->hist; |
301406b9 | 302 | |
e4204992 | 303 | if (!h->sum) |
301406b9 FW |
304 | return; |
305 | ||
e4204992 ACM |
306 | sym_ext = priv->ext = calloc(len, sizeof(struct sym_ext)); |
307 | if (!priv->ext) | |
301406b9 FW |
308 | return; |
309 | ||
ed52ce2e | 310 | start = he->map->unmap_ip(he->map, sym->start); |
301406b9 FW |
311 | |
312 | for (i = 0; i < len; i++) { | |
313 | char *path = NULL; | |
314 | size_t line_len; | |
9cffa8d5 | 315 | u64 offset; |
301406b9 FW |
316 | FILE *fp; |
317 | ||
e4204992 | 318 | sym_ext[i].percent = 100.0 * h->ip[i] / h->sum; |
301406b9 FW |
319 | if (sym_ext[i].percent <= 0.5) |
320 | continue; | |
321 | ||
ed52ce2e | 322 | offset = start + i; |
c17c2db1 | 323 | sprintf(cmd, "addr2line -e %s %016llx", filename, offset); |
301406b9 FW |
324 | fp = popen(cmd, "r"); |
325 | if (!fp) | |
326 | continue; | |
327 | ||
328 | if (getline(&path, &line_len, fp) < 0 || !line_len) | |
329 | goto next; | |
330 | ||
c17c2db1 | 331 | sym_ext[i].path = malloc(sizeof(char) * line_len + 1); |
301406b9 FW |
332 | if (!sym_ext[i].path) |
333 | goto next; | |
334 | ||
335 | strcpy(sym_ext[i].path, path); | |
971738f3 | 336 | insert_source_line(&sym_ext[i]); |
301406b9 FW |
337 | |
338 | next: | |
339 | pclose(fp); | |
340 | } | |
341 | } | |
342 | ||
83a0944f | 343 | static void print_summary(const char *filename) |
971738f3 FW |
344 | { |
345 | struct sym_ext *sym_ext; | |
346 | struct rb_node *node; | |
347 | ||
348 | printf("\nSorted summary for file %s\n", filename); | |
349 | printf("----------------------------------------------\n\n"); | |
350 | ||
351 | if (RB_EMPTY_ROOT(&root_sym_ext)) { | |
352 | printf(" Nothing higher than %1.1f%%\n", MIN_GREEN); | |
353 | return; | |
354 | } | |
355 | ||
356 | node = rb_first(&root_sym_ext); | |
357 | while (node) { | |
358 | double percent; | |
83a0944f | 359 | const char *color; |
971738f3 FW |
360 | char *path; |
361 | ||
362 | sym_ext = rb_entry(node, struct sym_ext, node); | |
363 | percent = sym_ext->percent; | |
1e11fd82 | 364 | color = get_percent_color(percent); |
971738f3 FW |
365 | path = sym_ext->path; |
366 | ||
367 | color_fprintf(stdout, color, " %7.2f %s", percent, path); | |
368 | node = rb_next(node); | |
369 | } | |
370 | } | |
371 | ||
ed52ce2e | 372 | static void annotate_sym(struct hist_entry *he) |
0b73da3f | 373 | { |
ed52ce2e ACM |
374 | struct map *map = he->map; |
375 | struct dso *dso = map->dso; | |
376 | struct symbol *sym = he->sym; | |
439d473b ACM |
377 | const char *filename = dso->long_name, *d_filename; |
378 | u64 len; | |
0b73da3f IM |
379 | char command[PATH_MAX*2]; |
380 | FILE *file; | |
381 | ||
382 | if (!filename) | |
383 | return; | |
439d473b | 384 | |
ed52ce2e ACM |
385 | if (verbose) |
386 | fprintf(stderr, "%s: filename=%s, sym=%s, start=%Lx, end=%Lx\n", | |
387 | __func__, filename, sym->name, | |
388 | map->unmap_ip(map, sym->start), | |
389 | map->unmap_ip(map, sym->end)); | |
390 | ||
42976487 MG |
391 | if (full_paths) |
392 | d_filename = filename; | |
393 | else | |
394 | d_filename = basename(filename); | |
0b73da3f | 395 | |
0b73da3f IM |
396 | len = sym->end - sym->start; |
397 | ||
971738f3 | 398 | if (print_line) { |
ed52ce2e | 399 | get_source_line(he, len, filename); |
971738f3 FW |
400 | print_summary(filename); |
401 | } | |
402 | ||
403 | printf("\n\n------------------------------------------------\n"); | |
42976487 | 404 | printf(" Percent | Source code & Disassembly of %s\n", d_filename); |
971738f3 FW |
405 | printf("------------------------------------------------\n"); |
406 | ||
407 | if (verbose >= 2) | |
439d473b ACM |
408 | printf("annotating [%p] %30s : [%p] %30s\n", |
409 | dso, dso->long_name, sym, sym->name); | |
301406b9 | 410 | |
42976487 | 411 | sprintf(command, "objdump --start-address=0x%016Lx --stop-address=0x%016Lx -dS %s|grep -v %s", |
ed52ce2e ACM |
412 | map->unmap_ip(map, sym->start), map->unmap_ip(map, sym->end), |
413 | filename, filename); | |
0b73da3f IM |
414 | |
415 | if (verbose >= 3) | |
416 | printf("doing: %s\n", command); | |
417 | ||
418 | file = popen(command, "r"); | |
419 | if (!file) | |
420 | return; | |
421 | ||
422 | while (!feof(file)) { | |
ed52ce2e | 423 | if (parse_line(file, he, len) < 0) |
0b73da3f IM |
424 | break; |
425 | } | |
426 | ||
427 | pclose(file); | |
971738f3 | 428 | if (print_line) |
e4204992 | 429 | free_source_line(he, len); |
0b73da3f IM |
430 | } |
431 | ||
432 | static void find_annotations(void) | |
433 | { | |
434 | struct rb_node *nd; | |
0b73da3f | 435 | |
ed52ce2e ACM |
436 | for (nd = rb_first(&output_hists); nd; nd = rb_next(nd)) { |
437 | struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node); | |
e4204992 | 438 | struct sym_priv *priv; |
0b73da3f | 439 | |
e4204992 ACM |
440 | if (he->sym == NULL) |
441 | continue; | |
0b73da3f | 442 | |
00a192b3 | 443 | priv = symbol__priv(he->sym); |
e4204992 ACM |
444 | if (priv->hist == NULL) |
445 | continue; | |
446 | ||
447 | annotate_sym(he); | |
e4204992 ACM |
448 | /* |
449 | * Since we have a hist_entry per IP for the same symbol, free | |
450 | * he->sym->hist to signal we already processed this symbol. | |
451 | */ | |
452 | free(priv->hist); | |
453 | priv->hist = NULL; | |
0b73da3f | 454 | } |
0b73da3f IM |
455 | } |
456 | ||
bab81b62 LZ |
457 | static struct perf_file_handler file_handler = { |
458 | .process_sample_event = process_sample_event, | |
459 | .process_mmap_event = event__process_mmap, | |
460 | .process_comm_event = event__process_comm, | |
461 | .process_fork_event = event__process_task, | |
462 | }; | |
463 | ||
8035e428 IM |
464 | static int __cmd_annotate(void) |
465 | { | |
94c744b6 | 466 | struct perf_session *session = perf_session__new(input_name, O_RDONLY, force); |
bab81b62 LZ |
467 | struct thread *idle; |
468 | int ret; | |
8035e428 | 469 | |
94c744b6 ACM |
470 | if (session == NULL) |
471 | return -ENOMEM; | |
472 | ||
bab81b62 LZ |
473 | idle = register_idle_thread(); |
474 | register_perf_file_handler(&file_handler); | |
8035e428 | 475 | |
94c744b6 | 476 | ret = perf_session__process_events(session, 0, &event__cwdlen, &event__cwd); |
bab81b62 | 477 | if (ret) |
94c744b6 | 478 | goto out_delete; |
8035e428 | 479 | |
62daacb5 ACM |
480 | if (dump_trace) { |
481 | event__print_totals(); | |
94c744b6 | 482 | goto out_delete; |
62daacb5 | 483 | } |
8035e428 | 484 | |
da21d1b5 | 485 | if (verbose > 3) |
d5b889f2 | 486 | threads__fprintf(stdout); |
8035e428 | 487 | |
da21d1b5 | 488 | if (verbose > 2) |
8035e428 IM |
489 | dsos__fprintf(stdout); |
490 | ||
491 | collapse__resort(); | |
62daacb5 | 492 | output__resort(event__total[0]); |
0b73da3f IM |
493 | |
494 | find_annotations(); | |
94c744b6 ACM |
495 | out_delete: |
496 | perf_session__delete(session); | |
8035e428 | 497 | |
bab81b62 | 498 | return ret; |
8035e428 IM |
499 | } |
500 | ||
501 | static const char * const annotate_usage[] = { | |
502 | "perf annotate [<options>] <command>", | |
503 | NULL | |
504 | }; | |
505 | ||
506 | static const struct option options[] = { | |
507 | OPT_STRING('i', "input", &input_name, "file", | |
508 | "input file name"), | |
23b87116 | 509 | OPT_STRING('s', "symbol", &sym_hist_filter, "symbol", |
0b73da3f | 510 | "symbol to annotate"), |
fa6963b2 | 511 | OPT_BOOLEAN('f', "force", &force, "don't complain, do it"), |
8035e428 IM |
512 | OPT_BOOLEAN('v', "verbose", &verbose, |
513 | "be more verbose (show symbol address, etc)"), | |
514 | OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace, | |
515 | "dump raw trace in ASCII"), | |
b32d133a ACM |
516 | OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name, |
517 | "file", "vmlinux pathname"), | |
518 | OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules, | |
42976487 | 519 | "load module symbols - WARNING: use only with -k and LIVE kernel"), |
301406b9 FW |
520 | OPT_BOOLEAN('l', "print-line", &print_line, |
521 | "print matching source lines (may be slow)"), | |
42976487 MG |
522 | OPT_BOOLEAN('P', "full-paths", &full_paths, |
523 | "Don't shorten the displayed pathnames"), | |
8035e428 IM |
524 | OPT_END() |
525 | }; | |
526 | ||
527 | static void setup_sorting(void) | |
528 | { | |
529 | char *tmp, *tok, *str = strdup(sort_order); | |
530 | ||
531 | for (tok = strtok_r(str, ", ", &tmp); | |
532 | tok; tok = strtok_r(NULL, ", ", &tmp)) { | |
533 | if (sort_dimension__add(tok) < 0) { | |
534 | error("Unknown --sort key: `%s'", tok); | |
535 | usage_with_options(annotate_usage, options); | |
536 | } | |
537 | } | |
538 | ||
539 | free(str); | |
540 | } | |
541 | ||
f37a291c | 542 | int cmd_annotate(int argc, const char **argv, const char *prefix __used) |
8035e428 | 543 | { |
b32d133a ACM |
544 | if (symbol__init(&symbol_conf) < 0) |
545 | return -1; | |
8035e428 | 546 | |
8035e428 IM |
547 | argc = parse_options(argc, argv, options, annotate_usage, 0); |
548 | ||
549 | setup_sorting(); | |
550 | ||
0b73da3f IM |
551 | if (argc) { |
552 | /* | |
553 | * Special case: if there's an argument left then assume tha | |
554 | * it's a symbol filter: | |
555 | */ | |
556 | if (argc > 1) | |
557 | usage_with_options(annotate_usage, options); | |
558 | ||
559 | sym_hist_filter = argv[0]; | |
560 | } | |
561 | ||
8035e428 IM |
562 | setup_pager(); |
563 | ||
dd68ada2 JK |
564 | if (field_sep && *field_sep == '.') { |
565 | fputs("'.' is the only non valid --field-separator argument\n", | |
566 | stderr); | |
567 | exit(129); | |
568 | } | |
569 | ||
8035e428 IM |
570 | return __cmd_annotate(); |
571 | } |