Commit | Line | Data |
---|---|---|
36994e58 FW |
1 | /* |
2 | * Memory allocator tracing | |
3 | * | |
4 | * Copyright (C) 2008 Eduard - Gabriel Munteanu | |
5 | * Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi> | |
6 | * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com> | |
7 | */ | |
8 | ||
c826e3cd IM |
9 | #include <linux/tracepoint.h> |
10 | #include <linux/seq_file.h> | |
36994e58 | 11 | #include <linux/debugfs.h> |
c826e3cd | 12 | #include <linux/dcache.h> |
36994e58 | 13 | #include <linux/fs.h> |
c826e3cd | 14 | |
02af61bb | 15 | #include <linux/kmemtrace.h> |
36994e58 | 16 | |
36994e58 | 17 | #include "trace_output.h" |
c826e3cd | 18 | #include "trace.h" |
36994e58 FW |
19 | |
20 | /* Select an alternative, minimalistic output than the original one */ | |
21 | #define TRACE_KMEM_OPT_MINIMAL 0x1 | |
22 | ||
23 | static struct tracer_opt kmem_opts[] = { | |
24 | /* Default disable the minimalistic output */ | |
25 | { TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) }, | |
26 | { } | |
27 | }; | |
28 | ||
29 | static struct tracer_flags kmem_tracer_flags = { | |
c826e3cd IM |
30 | .val = 0, |
31 | .opts = kmem_opts | |
36994e58 FW |
32 | }; |
33 | ||
36994e58 FW |
34 | static struct trace_array *kmemtrace_array; |
35 | ||
ca2b84cb EGM |
36 | /* Trace allocations */ |
37 | static inline void kmemtrace_alloc(enum kmemtrace_type_id type_id, | |
38 | unsigned long call_site, | |
39 | const void *ptr, | |
40 | size_t bytes_req, | |
41 | size_t bytes_alloc, | |
42 | gfp_t gfp_flags, | |
43 | int node) | |
44 | { | |
e1112b4d | 45 | struct ftrace_event_call *call = &event_kmem_alloc; |
ca2b84cb | 46 | struct trace_array *tr = kmemtrace_array; |
c826e3cd IM |
47 | struct kmemtrace_alloc_entry *entry; |
48 | struct ring_buffer_event *event; | |
ca2b84cb EGM |
49 | |
50 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry)); | |
51 | if (!event) | |
52 | return; | |
c826e3cd IM |
53 | |
54 | entry = ring_buffer_event_data(event); | |
ca2b84cb EGM |
55 | tracing_generic_entry_update(&entry->ent, 0, 0); |
56 | ||
c826e3cd IM |
57 | entry->ent.type = TRACE_KMEM_ALLOC; |
58 | entry->type_id = type_id; | |
59 | entry->call_site = call_site; | |
60 | entry->ptr = ptr; | |
61 | entry->bytes_req = bytes_req; | |
62 | entry->bytes_alloc = bytes_alloc; | |
63 | entry->gfp_flags = gfp_flags; | |
64 | entry->node = node; | |
ca2b84cb | 65 | |
eb02ce01 TZ |
66 | if (!filter_check_discard(call, entry, tr->buffer, event)) |
67 | ring_buffer_unlock_commit(tr->buffer, event); | |
ca2b84cb EGM |
68 | |
69 | trace_wake_up(); | |
70 | } | |
71 | ||
72 | static inline void kmemtrace_free(enum kmemtrace_type_id type_id, | |
73 | unsigned long call_site, | |
74 | const void *ptr) | |
75 | { | |
e1112b4d | 76 | struct ftrace_event_call *call = &event_kmem_free; |
ca2b84cb | 77 | struct trace_array *tr = kmemtrace_array; |
c826e3cd IM |
78 | struct kmemtrace_free_entry *entry; |
79 | struct ring_buffer_event *event; | |
ca2b84cb EGM |
80 | |
81 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry)); | |
82 | if (!event) | |
83 | return; | |
84 | entry = ring_buffer_event_data(event); | |
85 | tracing_generic_entry_update(&entry->ent, 0, 0); | |
86 | ||
c826e3cd IM |
87 | entry->ent.type = TRACE_KMEM_FREE; |
88 | entry->type_id = type_id; | |
89 | entry->call_site = call_site; | |
90 | entry->ptr = ptr; | |
ca2b84cb | 91 | |
eb02ce01 TZ |
92 | if (!filter_check_discard(call, entry, tr->buffer, event)) |
93 | ring_buffer_unlock_commit(tr->buffer, event); | |
ca2b84cb EGM |
94 | |
95 | trace_wake_up(); | |
96 | } | |
97 | ||
98 | static void kmemtrace_kmalloc(unsigned long call_site, | |
99 | const void *ptr, | |
100 | size_t bytes_req, | |
101 | size_t bytes_alloc, | |
102 | gfp_t gfp_flags) | |
103 | { | |
104 | kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr, | |
105 | bytes_req, bytes_alloc, gfp_flags, -1); | |
106 | } | |
107 | ||
108 | static void kmemtrace_kmem_cache_alloc(unsigned long call_site, | |
109 | const void *ptr, | |
110 | size_t bytes_req, | |
111 | size_t bytes_alloc, | |
112 | gfp_t gfp_flags) | |
113 | { | |
114 | kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr, | |
115 | bytes_req, bytes_alloc, gfp_flags, -1); | |
116 | } | |
117 | ||
118 | static void kmemtrace_kmalloc_node(unsigned long call_site, | |
119 | const void *ptr, | |
120 | size_t bytes_req, | |
121 | size_t bytes_alloc, | |
122 | gfp_t gfp_flags, | |
123 | int node) | |
124 | { | |
125 | kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr, | |
126 | bytes_req, bytes_alloc, gfp_flags, node); | |
127 | } | |
128 | ||
129 | static void kmemtrace_kmem_cache_alloc_node(unsigned long call_site, | |
130 | const void *ptr, | |
131 | size_t bytes_req, | |
132 | size_t bytes_alloc, | |
133 | gfp_t gfp_flags, | |
134 | int node) | |
135 | { | |
136 | kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr, | |
137 | bytes_req, bytes_alloc, gfp_flags, node); | |
138 | } | |
139 | ||
140 | static void kmemtrace_kfree(unsigned long call_site, const void *ptr) | |
141 | { | |
142 | kmemtrace_free(KMEMTRACE_TYPE_KMALLOC, call_site, ptr); | |
143 | } | |
144 | ||
145 | static void kmemtrace_kmem_cache_free(unsigned long call_site, const void *ptr) | |
146 | { | |
147 | kmemtrace_free(KMEMTRACE_TYPE_CACHE, call_site, ptr); | |
148 | } | |
149 | ||
150 | static int kmemtrace_start_probes(void) | |
151 | { | |
152 | int err; | |
153 | ||
154 | err = register_trace_kmalloc(kmemtrace_kmalloc); | |
155 | if (err) | |
156 | return err; | |
157 | err = register_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | |
158 | if (err) | |
159 | return err; | |
160 | err = register_trace_kmalloc_node(kmemtrace_kmalloc_node); | |
161 | if (err) | |
162 | return err; | |
163 | err = register_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | |
164 | if (err) | |
165 | return err; | |
166 | err = register_trace_kfree(kmemtrace_kfree); | |
167 | if (err) | |
168 | return err; | |
169 | err = register_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | |
170 | ||
171 | return err; | |
172 | } | |
173 | ||
174 | static void kmemtrace_stop_probes(void) | |
175 | { | |
176 | unregister_trace_kmalloc(kmemtrace_kmalloc); | |
177 | unregister_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | |
178 | unregister_trace_kmalloc_node(kmemtrace_kmalloc_node); | |
179 | unregister_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | |
180 | unregister_trace_kfree(kmemtrace_kfree); | |
181 | unregister_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | |
182 | } | |
183 | ||
36994e58 FW |
184 | static int kmem_trace_init(struct trace_array *tr) |
185 | { | |
186 | int cpu; | |
187 | kmemtrace_array = tr; | |
188 | ||
3f237a79 | 189 | for_each_cpu(cpu, cpu_possible_mask) |
36994e58 FW |
190 | tracing_reset(tr, cpu); |
191 | ||
ca2b84cb | 192 | kmemtrace_start_probes(); |
36994e58 FW |
193 | |
194 | return 0; | |
195 | } | |
196 | ||
197 | static void kmem_trace_reset(struct trace_array *tr) | |
198 | { | |
ca2b84cb | 199 | kmemtrace_stop_probes(); |
36994e58 FW |
200 | } |
201 | ||
202 | static void kmemtrace_headers(struct seq_file *s) | |
203 | { | |
204 | /* Don't need headers for the original kmemtrace output */ | |
205 | if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)) | |
206 | return; | |
207 | ||
208 | seq_printf(s, "#\n"); | |
209 | seq_printf(s, "# ALLOC TYPE REQ GIVEN FLAGS " | |
210 | " POINTER NODE CALLER\n"); | |
211 | seq_printf(s, "# FREE | | | | " | |
212 | " | | | |\n"); | |
213 | seq_printf(s, "# |\n\n"); | |
214 | } | |
215 | ||
216 | /* | |
42af9054 EGM |
217 | * The following functions give the original output from kmemtrace, |
218 | * plus the origin CPU, since reordering occurs in-kernel now. | |
36994e58 | 219 | */ |
42af9054 EGM |
220 | |
221 | #define KMEMTRACE_USER_ALLOC 0 | |
222 | #define KMEMTRACE_USER_FREE 1 | |
223 | ||
224 | struct kmemtrace_user_event { | |
c826e3cd IM |
225 | u8 event_id; |
226 | u8 type_id; | |
227 | u16 event_size; | |
228 | u32 cpu; | |
229 | u64 timestamp; | |
230 | unsigned long call_site; | |
231 | unsigned long ptr; | |
42af9054 EGM |
232 | }; |
233 | ||
234 | struct kmemtrace_user_event_alloc { | |
c826e3cd IM |
235 | size_t bytes_req; |
236 | size_t bytes_alloc; | |
237 | unsigned gfp_flags; | |
238 | int node; | |
42af9054 EGM |
239 | }; |
240 | ||
36994e58 | 241 | static enum print_line_t |
ddc1637a LZ |
242 | kmemtrace_print_alloc_user(struct trace_iterator *iter, int flags) |
243 | { | |
244 | struct trace_seq *s = &iter->seq; | |
245 | struct kmemtrace_alloc_entry *entry; | |
246 | int ret; | |
247 | ||
248 | trace_assign_type(entry, iter->ent); | |
249 | ||
250 | ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu " | |
251 | "bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n", | |
252 | entry->type_id, (void *)entry->call_site, (unsigned long)entry->ptr, | |
253 | (unsigned long)entry->bytes_req, (unsigned long)entry->bytes_alloc, | |
254 | (unsigned long)entry->gfp_flags, entry->node); | |
255 | ||
256 | if (!ret) | |
257 | return TRACE_TYPE_PARTIAL_LINE; | |
258 | return TRACE_TYPE_HANDLED; | |
259 | } | |
260 | ||
261 | static enum print_line_t | |
262 | kmemtrace_print_free_user(struct trace_iterator *iter, int flags) | |
36994e58 FW |
263 | { |
264 | struct trace_seq *s = &iter->seq; | |
ddc1637a LZ |
265 | struct kmemtrace_free_entry *entry; |
266 | int ret; | |
267 | ||
268 | trace_assign_type(entry, iter->ent); | |
269 | ||
270 | ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu\n", | |
271 | entry->type_id, (void *)entry->call_site, | |
272 | (unsigned long)entry->ptr); | |
273 | ||
274 | if (!ret) | |
275 | return TRACE_TYPE_PARTIAL_LINE; | |
276 | return TRACE_TYPE_HANDLED; | |
277 | } | |
278 | ||
279 | static enum print_line_t | |
280 | kmemtrace_print_alloc_user_bin(struct trace_iterator *iter, int flags) | |
281 | { | |
282 | struct trace_seq *s = &iter->seq; | |
283 | struct kmemtrace_alloc_entry *entry; | |
42af9054 | 284 | struct kmemtrace_user_event *ev; |
ddc1637a LZ |
285 | struct kmemtrace_user_event_alloc *ev_alloc; |
286 | ||
287 | trace_assign_type(entry, iter->ent); | |
36994e58 | 288 | |
42af9054 EGM |
289 | ev = trace_seq_reserve(s, sizeof(*ev)); |
290 | if (!ev) | |
291 | return TRACE_TYPE_PARTIAL_LINE; | |
c826e3cd IM |
292 | |
293 | ev->event_id = KMEMTRACE_USER_ALLOC; | |
294 | ev->type_id = entry->type_id; | |
295 | ev->event_size = sizeof(*ev) + sizeof(*ev_alloc); | |
296 | ev->cpu = iter->cpu; | |
297 | ev->timestamp = iter->ts; | |
298 | ev->call_site = entry->call_site; | |
299 | ev->ptr = (unsigned long)entry->ptr; | |
42af9054 EGM |
300 | |
301 | ev_alloc = trace_seq_reserve(s, sizeof(*ev_alloc)); | |
302 | if (!ev_alloc) | |
36994e58 | 303 | return TRACE_TYPE_PARTIAL_LINE; |
c826e3cd IM |
304 | |
305 | ev_alloc->bytes_req = entry->bytes_req; | |
306 | ev_alloc->bytes_alloc = entry->bytes_alloc; | |
307 | ev_alloc->gfp_flags = entry->gfp_flags; | |
308 | ev_alloc->node = entry->node; | |
36994e58 FW |
309 | |
310 | return TRACE_TYPE_HANDLED; | |
311 | } | |
312 | ||
313 | static enum print_line_t | |
ddc1637a | 314 | kmemtrace_print_free_user_bin(struct trace_iterator *iter, int flags) |
36994e58 FW |
315 | { |
316 | struct trace_seq *s = &iter->seq; | |
ddc1637a | 317 | struct kmemtrace_free_entry *entry; |
42af9054 | 318 | struct kmemtrace_user_event *ev; |
36994e58 | 319 | |
ddc1637a LZ |
320 | trace_assign_type(entry, iter->ent); |
321 | ||
42af9054 EGM |
322 | ev = trace_seq_reserve(s, sizeof(*ev)); |
323 | if (!ev) | |
36994e58 | 324 | return TRACE_TYPE_PARTIAL_LINE; |
c826e3cd IM |
325 | |
326 | ev->event_id = KMEMTRACE_USER_FREE; | |
327 | ev->type_id = entry->type_id; | |
328 | ev->event_size = sizeof(*ev); | |
329 | ev->cpu = iter->cpu; | |
330 | ev->timestamp = iter->ts; | |
331 | ev->call_site = entry->call_site; | |
332 | ev->ptr = (unsigned long)entry->ptr; | |
36994e58 FW |
333 | |
334 | return TRACE_TYPE_HANDLED; | |
335 | } | |
336 | ||
36994e58 FW |
337 | /* The two other following provide a more minimalistic output */ |
338 | static enum print_line_t | |
ddc1637a | 339 | kmemtrace_print_alloc_compress(struct trace_iterator *iter) |
36994e58 | 340 | { |
ddc1637a | 341 | struct kmemtrace_alloc_entry *entry; |
36994e58 FW |
342 | struct trace_seq *s = &iter->seq; |
343 | int ret; | |
344 | ||
ddc1637a LZ |
345 | trace_assign_type(entry, iter->ent); |
346 | ||
36994e58 FW |
347 | /* Alloc entry */ |
348 | ret = trace_seq_printf(s, " + "); | |
349 | if (!ret) | |
350 | return TRACE_TYPE_PARTIAL_LINE; | |
351 | ||
352 | /* Type */ | |
353 | switch (entry->type_id) { | |
354 | case KMEMTRACE_TYPE_KMALLOC: | |
355 | ret = trace_seq_printf(s, "K "); | |
356 | break; | |
357 | case KMEMTRACE_TYPE_CACHE: | |
358 | ret = trace_seq_printf(s, "C "); | |
359 | break; | |
360 | case KMEMTRACE_TYPE_PAGES: | |
361 | ret = trace_seq_printf(s, "P "); | |
362 | break; | |
363 | default: | |
364 | ret = trace_seq_printf(s, "? "); | |
365 | } | |
366 | ||
367 | if (!ret) | |
368 | return TRACE_TYPE_PARTIAL_LINE; | |
369 | ||
370 | /* Requested */ | |
ecf441b5 | 371 | ret = trace_seq_printf(s, "%4zu ", entry->bytes_req); |
36994e58 FW |
372 | if (!ret) |
373 | return TRACE_TYPE_PARTIAL_LINE; | |
374 | ||
375 | /* Allocated */ | |
ecf441b5 | 376 | ret = trace_seq_printf(s, "%4zu ", entry->bytes_alloc); |
36994e58 FW |
377 | if (!ret) |
378 | return TRACE_TYPE_PARTIAL_LINE; | |
379 | ||
380 | /* Flags | |
381 | * TODO: would be better to see the name of the GFP flag names | |
382 | */ | |
383 | ret = trace_seq_printf(s, "%08x ", entry->gfp_flags); | |
384 | if (!ret) | |
385 | return TRACE_TYPE_PARTIAL_LINE; | |
386 | ||
387 | /* Pointer to allocated */ | |
388 | ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr); | |
389 | if (!ret) | |
390 | return TRACE_TYPE_PARTIAL_LINE; | |
391 | ||
392 | /* Node */ | |
393 | ret = trace_seq_printf(s, "%4d ", entry->node); | |
394 | if (!ret) | |
395 | return TRACE_TYPE_PARTIAL_LINE; | |
396 | ||
397 | /* Call site */ | |
398 | ret = seq_print_ip_sym(s, entry->call_site, 0); | |
399 | if (!ret) | |
400 | return TRACE_TYPE_PARTIAL_LINE; | |
401 | ||
402 | if (!trace_seq_printf(s, "\n")) | |
403 | return TRACE_TYPE_PARTIAL_LINE; | |
404 | ||
405 | return TRACE_TYPE_HANDLED; | |
406 | } | |
407 | ||
408 | static enum print_line_t | |
ddc1637a | 409 | kmemtrace_print_free_compress(struct trace_iterator *iter) |
36994e58 | 410 | { |
ddc1637a | 411 | struct kmemtrace_free_entry *entry; |
36994e58 FW |
412 | struct trace_seq *s = &iter->seq; |
413 | int ret; | |
414 | ||
ddc1637a LZ |
415 | trace_assign_type(entry, iter->ent); |
416 | ||
36994e58 FW |
417 | /* Free entry */ |
418 | ret = trace_seq_printf(s, " - "); | |
419 | if (!ret) | |
420 | return TRACE_TYPE_PARTIAL_LINE; | |
421 | ||
422 | /* Type */ | |
423 | switch (entry->type_id) { | |
424 | case KMEMTRACE_TYPE_KMALLOC: | |
425 | ret = trace_seq_printf(s, "K "); | |
426 | break; | |
427 | case KMEMTRACE_TYPE_CACHE: | |
428 | ret = trace_seq_printf(s, "C "); | |
429 | break; | |
430 | case KMEMTRACE_TYPE_PAGES: | |
431 | ret = trace_seq_printf(s, "P "); | |
432 | break; | |
433 | default: | |
434 | ret = trace_seq_printf(s, "? "); | |
435 | } | |
436 | ||
437 | if (!ret) | |
438 | return TRACE_TYPE_PARTIAL_LINE; | |
439 | ||
440 | /* Skip requested/allocated/flags */ | |
441 | ret = trace_seq_printf(s, " "); | |
442 | if (!ret) | |
443 | return TRACE_TYPE_PARTIAL_LINE; | |
444 | ||
445 | /* Pointer to allocated */ | |
446 | ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr); | |
447 | if (!ret) | |
448 | return TRACE_TYPE_PARTIAL_LINE; | |
449 | ||
450 | /* Skip node */ | |
451 | ret = trace_seq_printf(s, " "); | |
452 | if (!ret) | |
453 | return TRACE_TYPE_PARTIAL_LINE; | |
454 | ||
455 | /* Call site */ | |
456 | ret = seq_print_ip_sym(s, entry->call_site, 0); | |
457 | if (!ret) | |
458 | return TRACE_TYPE_PARTIAL_LINE; | |
459 | ||
460 | if (!trace_seq_printf(s, "\n")) | |
461 | return TRACE_TYPE_PARTIAL_LINE; | |
462 | ||
463 | return TRACE_TYPE_HANDLED; | |
464 | } | |
465 | ||
466 | static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter) | |
467 | { | |
468 | struct trace_entry *entry = iter->ent; | |
469 | ||
ddc1637a LZ |
470 | if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)) |
471 | return TRACE_TYPE_UNHANDLED; | |
36994e58 | 472 | |
ddc1637a LZ |
473 | switch (entry->type) { |
474 | case TRACE_KMEM_ALLOC: | |
475 | return kmemtrace_print_alloc_compress(iter); | |
476 | case TRACE_KMEM_FREE: | |
477 | return kmemtrace_print_free_compress(iter); | |
36994e58 FW |
478 | default: |
479 | return TRACE_TYPE_UNHANDLED; | |
480 | } | |
481 | } | |
482 | ||
ddc1637a LZ |
483 | static struct trace_event kmem_trace_alloc = { |
484 | .type = TRACE_KMEM_ALLOC, | |
485 | .trace = kmemtrace_print_alloc_user, | |
486 | .binary = kmemtrace_print_alloc_user_bin, | |
487 | }; | |
488 | ||
489 | static struct trace_event kmem_trace_free = { | |
490 | .type = TRACE_KMEM_FREE, | |
491 | .trace = kmemtrace_print_free_user, | |
492 | .binary = kmemtrace_print_free_user_bin, | |
493 | }; | |
494 | ||
36994e58 | 495 | static struct tracer kmem_tracer __read_mostly = { |
c826e3cd IM |
496 | .name = "kmemtrace", |
497 | .init = kmem_trace_init, | |
498 | .reset = kmem_trace_reset, | |
499 | .print_line = kmemtrace_print_line, | |
500 | .print_header = kmemtrace_headers, | |
501 | .flags = &kmem_tracer_flags | |
36994e58 FW |
502 | }; |
503 | ||
3e806802 IM |
504 | void kmemtrace_init(void) |
505 | { | |
506 | /* earliest opportunity to start kmem tracing */ | |
507 | } | |
508 | ||
36994e58 FW |
509 | static int __init init_kmem_tracer(void) |
510 | { | |
ddc1637a LZ |
511 | if (!register_ftrace_event(&kmem_trace_alloc)) { |
512 | pr_warning("Warning: could not register kmem events\n"); | |
513 | return 1; | |
514 | } | |
515 | ||
516 | if (!register_ftrace_event(&kmem_trace_free)) { | |
517 | pr_warning("Warning: could not register kmem events\n"); | |
518 | return 1; | |
519 | } | |
520 | ||
521 | if (!register_tracer(&kmem_tracer)) { | |
522 | pr_warning("Warning: could not register the kmem tracer\n"); | |
523 | return 1; | |
524 | } | |
525 | ||
526 | return 0; | |
36994e58 | 527 | } |
36994e58 | 528 | device_initcall(init_kmem_tracer); |