Commit | Line | Data |
---|---|---|
8256e47c MD |
1 | /* |
2 | * Copyright (C) 2007 Mathieu Desnoyers | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | */ | |
18 | #include <linux/module.h> | |
19 | #include <linux/mutex.h> | |
20 | #include <linux/types.h> | |
21 | #include <linux/jhash.h> | |
22 | #include <linux/list.h> | |
23 | #include <linux/rcupdate.h> | |
24 | #include <linux/marker.h> | |
25 | #include <linux/err.h> | |
1aeb272c | 26 | #include <linux/slab.h> |
8256e47c MD |
27 | |
28 | extern struct marker __start___markers[]; | |
29 | extern struct marker __stop___markers[]; | |
30 | ||
fb40bd78 | 31 | /* Set to 1 to enable marker debug output */ |
ab883af5 | 32 | static const int marker_debug; |
fb40bd78 | 33 | |
8256e47c | 34 | /* |
314de8a9 | 35 | * markers_mutex nests inside module_mutex. Markers mutex protects the builtin |
fb40bd78 | 36 | * and module markers and the hash table. |
8256e47c MD |
37 | */ |
38 | static DEFINE_MUTEX(markers_mutex); | |
39 | ||
8256e47c MD |
40 | /* |
41 | * Marker hash table, containing the active markers. | |
42 | * Protected by module_mutex. | |
43 | */ | |
44 | #define MARKER_HASH_BITS 6 | |
45 | #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) | |
46 | ||
fb40bd78 MD |
47 | /* |
48 | * Note about RCU : | |
49 | * It is used to make sure every handler has finished using its private data | |
50 | * between two consecutive operation (add or remove) on a given marker. It is | |
51 | * also used to delay the free of multiple probes array until a quiescent state | |
52 | * is reached. | |
53 | * marker entries modifications are protected by the markers_mutex. | |
54 | */ | |
8256e47c MD |
55 | struct marker_entry { |
56 | struct hlist_node hlist; | |
57 | char *format; | |
fb40bd78 MD |
58 | void (*call)(const struct marker *mdata, /* Probe wrapper */ |
59 | void *call_private, const char *fmt, ...); | |
60 | struct marker_probe_closure single; | |
61 | struct marker_probe_closure *multi; | |
8256e47c | 62 | int refcount; /* Number of times armed. 0 if disarmed. */ |
fb40bd78 MD |
63 | struct rcu_head rcu; |
64 | void *oldptr; | |
de4fc64f HH |
65 | unsigned char rcu_pending:1; |
66 | unsigned char ptype:1; | |
8256e47c MD |
67 | char name[0]; /* Contains name'\0'format'\0' */ |
68 | }; | |
69 | ||
70 | static struct hlist_head marker_table[MARKER_TABLE_SIZE]; | |
71 | ||
72 | /** | |
73 | * __mark_empty_function - Empty probe callback | |
fb40bd78 MD |
74 | * @probe_private: probe private data |
75 | * @call_private: call site private data | |
8256e47c MD |
76 | * @fmt: format string |
77 | * @...: variable argument list | |
78 | * | |
79 | * Empty callback provided as a probe to the markers. By providing this to a | |
80 | * disabled marker, we make sure the execution flow is always valid even | |
81 | * though the function pointer change and the marker enabling are two distinct | |
82 | * operations that modifies the execution flow of preemptible code. | |
83 | */ | |
fb40bd78 MD |
84 | void __mark_empty_function(void *probe_private, void *call_private, |
85 | const char *fmt, va_list *args) | |
8256e47c MD |
86 | { |
87 | } | |
88 | EXPORT_SYMBOL_GPL(__mark_empty_function); | |
89 | ||
fb40bd78 MD |
90 | /* |
91 | * marker_probe_cb Callback that prepares the variable argument list for probes. | |
92 | * @mdata: pointer of type struct marker | |
93 | * @call_private: caller site private data | |
94 | * @fmt: format string | |
95 | * @...: Variable argument list. | |
96 | * | |
97 | * Since we do not use "typical" pointer based RCU in the 1 argument case, we | |
98 | * need to put a full smp_rmb() in this branch. This is why we do not use | |
99 | * rcu_dereference() for the pointer read. | |
100 | */ | |
101 | void marker_probe_cb(const struct marker *mdata, void *call_private, | |
102 | const char *fmt, ...) | |
103 | { | |
104 | va_list args; | |
105 | char ptype; | |
106 | ||
107 | /* | |
fd3c36f8 MD |
108 | * preempt_disable does two things : disabling preemption to make sure |
109 | * the teardown of the callbacks can be done correctly when they are in | |
110 | * modules and they insure RCU read coherency. | |
fb40bd78 MD |
111 | */ |
112 | preempt_disable(); | |
58336114 | 113 | ptype = mdata->ptype; |
fb40bd78 MD |
114 | if (likely(!ptype)) { |
115 | marker_probe_func *func; | |
116 | /* Must read the ptype before ptr. They are not data dependant, | |
117 | * so we put an explicit smp_rmb() here. */ | |
118 | smp_rmb(); | |
58336114 | 119 | func = mdata->single.func; |
fb40bd78 MD |
120 | /* Must read the ptr before private data. They are not data |
121 | * dependant, so we put an explicit smp_rmb() here. */ | |
122 | smp_rmb(); | |
123 | va_start(args, fmt); | |
124 | func(mdata->single.probe_private, call_private, fmt, &args); | |
125 | va_end(args); | |
126 | } else { | |
127 | struct marker_probe_closure *multi; | |
128 | int i; | |
129 | /* | |
130 | * multi points to an array, therefore accessing the array | |
131 | * depends on reading multi. However, even in this case, | |
132 | * we must insure that the pointer is read _before_ the array | |
133 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
134 | * in the fast path, so put the explicit barrier here. | |
135 | */ | |
136 | smp_read_barrier_depends(); | |
58336114 | 137 | multi = mdata->multi; |
fb40bd78 MD |
138 | for (i = 0; multi[i].func; i++) { |
139 | va_start(args, fmt); | |
140 | multi[i].func(multi[i].probe_private, call_private, fmt, | |
141 | &args); | |
142 | va_end(args); | |
143 | } | |
144 | } | |
145 | preempt_enable(); | |
146 | } | |
147 | EXPORT_SYMBOL_GPL(marker_probe_cb); | |
148 | ||
149 | /* | |
150 | * marker_probe_cb Callback that does not prepare the variable argument list. | |
151 | * @mdata: pointer of type struct marker | |
152 | * @call_private: caller site private data | |
153 | * @fmt: format string | |
154 | * @...: Variable argument list. | |
155 | * | |
156 | * Should be connected to markers "MARK_NOARGS". | |
157 | */ | |
158 | void marker_probe_cb_noarg(const struct marker *mdata, | |
159 | void *call_private, const char *fmt, ...) | |
160 | { | |
161 | va_list args; /* not initialized */ | |
162 | char ptype; | |
163 | ||
164 | preempt_disable(); | |
58336114 | 165 | ptype = mdata->ptype; |
fb40bd78 MD |
166 | if (likely(!ptype)) { |
167 | marker_probe_func *func; | |
168 | /* Must read the ptype before ptr. They are not data dependant, | |
169 | * so we put an explicit smp_rmb() here. */ | |
170 | smp_rmb(); | |
58336114 | 171 | func = mdata->single.func; |
fb40bd78 MD |
172 | /* Must read the ptr before private data. They are not data |
173 | * dependant, so we put an explicit smp_rmb() here. */ | |
174 | smp_rmb(); | |
175 | func(mdata->single.probe_private, call_private, fmt, &args); | |
176 | } else { | |
177 | struct marker_probe_closure *multi; | |
178 | int i; | |
179 | /* | |
180 | * multi points to an array, therefore accessing the array | |
181 | * depends on reading multi. However, even in this case, | |
182 | * we must insure that the pointer is read _before_ the array | |
183 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
184 | * in the fast path, so put the explicit barrier here. | |
185 | */ | |
186 | smp_read_barrier_depends(); | |
58336114 | 187 | multi = mdata->multi; |
fb40bd78 MD |
188 | for (i = 0; multi[i].func; i++) |
189 | multi[i].func(multi[i].probe_private, call_private, fmt, | |
190 | &args); | |
191 | } | |
192 | preempt_enable(); | |
193 | } | |
194 | EXPORT_SYMBOL_GPL(marker_probe_cb_noarg); | |
195 | ||
196 | static void free_old_closure(struct rcu_head *head) | |
197 | { | |
198 | struct marker_entry *entry = container_of(head, | |
199 | struct marker_entry, rcu); | |
200 | kfree(entry->oldptr); | |
201 | /* Make sure we free the data before setting the pending flag to 0 */ | |
202 | smp_wmb(); | |
203 | entry->rcu_pending = 0; | |
204 | } | |
205 | ||
206 | static void debug_print_probes(struct marker_entry *entry) | |
207 | { | |
208 | int i; | |
209 | ||
210 | if (!marker_debug) | |
211 | return; | |
212 | ||
213 | if (!entry->ptype) { | |
214 | printk(KERN_DEBUG "Single probe : %p %p\n", | |
215 | entry->single.func, | |
216 | entry->single.probe_private); | |
217 | } else { | |
218 | for (i = 0; entry->multi[i].func; i++) | |
219 | printk(KERN_DEBUG "Multi probe %d : %p %p\n", i, | |
220 | entry->multi[i].func, | |
221 | entry->multi[i].probe_private); | |
222 | } | |
223 | } | |
224 | ||
225 | static struct marker_probe_closure * | |
226 | marker_entry_add_probe(struct marker_entry *entry, | |
227 | marker_probe_func *probe, void *probe_private) | |
228 | { | |
229 | int nr_probes = 0; | |
230 | struct marker_probe_closure *old, *new; | |
231 | ||
232 | WARN_ON(!probe); | |
233 | ||
234 | debug_print_probes(entry); | |
235 | old = entry->multi; | |
236 | if (!entry->ptype) { | |
237 | if (entry->single.func == probe && | |
238 | entry->single.probe_private == probe_private) | |
239 | return ERR_PTR(-EBUSY); | |
240 | if (entry->single.func == __mark_empty_function) { | |
241 | /* 0 -> 1 probes */ | |
242 | entry->single.func = probe; | |
243 | entry->single.probe_private = probe_private; | |
244 | entry->refcount = 1; | |
245 | entry->ptype = 0; | |
246 | debug_print_probes(entry); | |
247 | return NULL; | |
248 | } else { | |
249 | /* 1 -> 2 probes */ | |
250 | nr_probes = 1; | |
251 | old = NULL; | |
252 | } | |
253 | } else { | |
254 | /* (N -> N+1), (N != 0, 1) probes */ | |
255 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) | |
256 | if (old[nr_probes].func == probe | |
257 | && old[nr_probes].probe_private | |
258 | == probe_private) | |
259 | return ERR_PTR(-EBUSY); | |
260 | } | |
261 | /* + 2 : one for new probe, one for NULL func */ | |
262 | new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure), | |
263 | GFP_KERNEL); | |
264 | if (new == NULL) | |
265 | return ERR_PTR(-ENOMEM); | |
266 | if (!old) | |
267 | new[0] = entry->single; | |
268 | else | |
269 | memcpy(new, old, | |
270 | nr_probes * sizeof(struct marker_probe_closure)); | |
271 | new[nr_probes].func = probe; | |
272 | new[nr_probes].probe_private = probe_private; | |
273 | entry->refcount = nr_probes + 1; | |
274 | entry->multi = new; | |
275 | entry->ptype = 1; | |
276 | debug_print_probes(entry); | |
277 | return old; | |
278 | } | |
279 | ||
280 | static struct marker_probe_closure * | |
281 | marker_entry_remove_probe(struct marker_entry *entry, | |
282 | marker_probe_func *probe, void *probe_private) | |
283 | { | |
284 | int nr_probes = 0, nr_del = 0, i; | |
285 | struct marker_probe_closure *old, *new; | |
286 | ||
287 | old = entry->multi; | |
288 | ||
289 | debug_print_probes(entry); | |
290 | if (!entry->ptype) { | |
291 | /* 0 -> N is an error */ | |
292 | WARN_ON(entry->single.func == __mark_empty_function); | |
293 | /* 1 -> 0 probes */ | |
294 | WARN_ON(probe && entry->single.func != probe); | |
295 | WARN_ON(entry->single.probe_private != probe_private); | |
296 | entry->single.func = __mark_empty_function; | |
297 | entry->refcount = 0; | |
298 | entry->ptype = 0; | |
299 | debug_print_probes(entry); | |
300 | return NULL; | |
301 | } else { | |
302 | /* (N -> M), (N > 1, M >= 0) probes */ | |
303 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) { | |
304 | if ((!probe || old[nr_probes].func == probe) | |
305 | && old[nr_probes].probe_private | |
306 | == probe_private) | |
307 | nr_del++; | |
308 | } | |
309 | } | |
310 | ||
311 | if (nr_probes - nr_del == 0) { | |
312 | /* N -> 0, (N > 1) */ | |
313 | entry->single.func = __mark_empty_function; | |
314 | entry->refcount = 0; | |
315 | entry->ptype = 0; | |
316 | } else if (nr_probes - nr_del == 1) { | |
317 | /* N -> 1, (N > 1) */ | |
318 | for (i = 0; old[i].func; i++) | |
319 | if ((probe && old[i].func != probe) || | |
320 | old[i].probe_private != probe_private) | |
321 | entry->single = old[i]; | |
322 | entry->refcount = 1; | |
323 | entry->ptype = 0; | |
324 | } else { | |
325 | int j = 0; | |
326 | /* N -> M, (N > 1, M > 1) */ | |
327 | /* + 1 for NULL */ | |
328 | new = kzalloc((nr_probes - nr_del + 1) | |
329 | * sizeof(struct marker_probe_closure), GFP_KERNEL); | |
330 | if (new == NULL) | |
331 | return ERR_PTR(-ENOMEM); | |
332 | for (i = 0; old[i].func; i++) | |
333 | if ((probe && old[i].func != probe) || | |
334 | old[i].probe_private != probe_private) | |
335 | new[j++] = old[i]; | |
336 | entry->refcount = nr_probes - nr_del; | |
337 | entry->ptype = 1; | |
338 | entry->multi = new; | |
339 | } | |
340 | debug_print_probes(entry); | |
341 | return old; | |
342 | } | |
343 | ||
8256e47c MD |
344 | /* |
345 | * Get marker if the marker is present in the marker hash table. | |
346 | * Must be called with markers_mutex held. | |
347 | * Returns NULL if not present. | |
348 | */ | |
349 | static struct marker_entry *get_marker(const char *name) | |
350 | { | |
351 | struct hlist_head *head; | |
352 | struct hlist_node *node; | |
353 | struct marker_entry *e; | |
354 | u32 hash = jhash(name, strlen(name), 0); | |
355 | ||
356 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
357 | hlist_for_each_entry(e, node, head, hlist) { | |
358 | if (!strcmp(name, e->name)) | |
359 | return e; | |
360 | } | |
361 | return NULL; | |
362 | } | |
363 | ||
364 | /* | |
365 | * Add the marker to the marker hash table. Must be called with markers_mutex | |
366 | * held. | |
367 | */ | |
fb40bd78 | 368 | static struct marker_entry *add_marker(const char *name, const char *format) |
8256e47c MD |
369 | { |
370 | struct hlist_head *head; | |
371 | struct hlist_node *node; | |
372 | struct marker_entry *e; | |
373 | size_t name_len = strlen(name) + 1; | |
374 | size_t format_len = 0; | |
375 | u32 hash = jhash(name, name_len-1, 0); | |
376 | ||
377 | if (format) | |
378 | format_len = strlen(format) + 1; | |
379 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
380 | hlist_for_each_entry(e, node, head, hlist) { | |
381 | if (!strcmp(name, e->name)) { | |
382 | printk(KERN_NOTICE | |
fb40bd78 MD |
383 | "Marker %s busy\n", name); |
384 | return ERR_PTR(-EBUSY); /* Already there */ | |
8256e47c MD |
385 | } |
386 | } | |
387 | /* | |
388 | * Using kmalloc here to allocate a variable length element. Could | |
389 | * cause some memory fragmentation if overused. | |
390 | */ | |
391 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, | |
392 | GFP_KERNEL); | |
393 | if (!e) | |
fb40bd78 | 394 | return ERR_PTR(-ENOMEM); |
8256e47c MD |
395 | memcpy(&e->name[0], name, name_len); |
396 | if (format) { | |
397 | e->format = &e->name[name_len]; | |
398 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
399 | if (strcmp(e->format, MARK_NOARGS) == 0) |
400 | e->call = marker_probe_cb_noarg; | |
401 | else | |
402 | e->call = marker_probe_cb; | |
8256e47c MD |
403 | trace_mark(core_marker_format, "name %s format %s", |
404 | e->name, e->format); | |
fb40bd78 | 405 | } else { |
8256e47c | 406 | e->format = NULL; |
fb40bd78 MD |
407 | e->call = marker_probe_cb; |
408 | } | |
409 | e->single.func = __mark_empty_function; | |
410 | e->single.probe_private = NULL; | |
411 | e->multi = NULL; | |
412 | e->ptype = 0; | |
8256e47c | 413 | e->refcount = 0; |
fb40bd78 | 414 | e->rcu_pending = 0; |
8256e47c | 415 | hlist_add_head(&e->hlist, head); |
fb40bd78 | 416 | return e; |
8256e47c MD |
417 | } |
418 | ||
419 | /* | |
420 | * Remove the marker from the marker hash table. Must be called with mutex_lock | |
421 | * held. | |
422 | */ | |
fb40bd78 | 423 | static int remove_marker(const char *name) |
8256e47c MD |
424 | { |
425 | struct hlist_head *head; | |
426 | struct hlist_node *node; | |
427 | struct marker_entry *e; | |
428 | int found = 0; | |
429 | size_t len = strlen(name) + 1; | |
8256e47c MD |
430 | u32 hash = jhash(name, len-1, 0); |
431 | ||
432 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
433 | hlist_for_each_entry(e, node, head, hlist) { | |
434 | if (!strcmp(name, e->name)) { | |
435 | found = 1; | |
436 | break; | |
437 | } | |
438 | } | |
fb40bd78 MD |
439 | if (!found) |
440 | return -ENOENT; | |
441 | if (e->single.func != __mark_empty_function) | |
442 | return -EBUSY; | |
443 | hlist_del(&e->hlist); | |
444 | /* Make sure the call_rcu has been executed */ | |
445 | if (e->rcu_pending) | |
446 | rcu_barrier(); | |
447 | kfree(e); | |
448 | return 0; | |
8256e47c MD |
449 | } |
450 | ||
451 | /* | |
452 | * Set the mark_entry format to the format found in the element. | |
453 | */ | |
454 | static int marker_set_format(struct marker_entry **entry, const char *format) | |
455 | { | |
456 | struct marker_entry *e; | |
457 | size_t name_len = strlen((*entry)->name) + 1; | |
458 | size_t format_len = strlen(format) + 1; | |
459 | ||
fb40bd78 | 460 | |
8256e47c MD |
461 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, |
462 | GFP_KERNEL); | |
463 | if (!e) | |
464 | return -ENOMEM; | |
465 | memcpy(&e->name[0], (*entry)->name, name_len); | |
466 | e->format = &e->name[name_len]; | |
467 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
468 | if (strcmp(e->format, MARK_NOARGS) == 0) |
469 | e->call = marker_probe_cb_noarg; | |
470 | else | |
471 | e->call = marker_probe_cb; | |
472 | e->single = (*entry)->single; | |
473 | e->multi = (*entry)->multi; | |
474 | e->ptype = (*entry)->ptype; | |
8256e47c | 475 | e->refcount = (*entry)->refcount; |
fb40bd78 | 476 | e->rcu_pending = 0; |
8256e47c MD |
477 | hlist_add_before(&e->hlist, &(*entry)->hlist); |
478 | hlist_del(&(*entry)->hlist); | |
fb40bd78 MD |
479 | /* Make sure the call_rcu has been executed */ |
480 | if ((*entry)->rcu_pending) | |
481 | rcu_barrier(); | |
8256e47c MD |
482 | kfree(*entry); |
483 | *entry = e; | |
484 | trace_mark(core_marker_format, "name %s format %s", | |
485 | e->name, e->format); | |
486 | return 0; | |
487 | } | |
488 | ||
489 | /* | |
490 | * Sets the probe callback corresponding to one marker. | |
491 | */ | |
fb40bd78 MD |
492 | static int set_marker(struct marker_entry **entry, struct marker *elem, |
493 | int active) | |
8256e47c MD |
494 | { |
495 | int ret; | |
496 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); | |
497 | ||
498 | if ((*entry)->format) { | |
499 | if (strcmp((*entry)->format, elem->format) != 0) { | |
500 | printk(KERN_NOTICE | |
501 | "Format mismatch for probe %s " | |
502 | "(%s), marker (%s)\n", | |
503 | (*entry)->name, | |
504 | (*entry)->format, | |
505 | elem->format); | |
506 | return -EPERM; | |
507 | } | |
508 | } else { | |
509 | ret = marker_set_format(entry, elem->format); | |
510 | if (ret) | |
511 | return ret; | |
512 | } | |
fb40bd78 MD |
513 | |
514 | /* | |
515 | * probe_cb setup (statically known) is done here. It is | |
516 | * asynchronous with the rest of execution, therefore we only | |
517 | * pass from a "safe" callback (with argument) to an "unsafe" | |
518 | * callback (does not set arguments). | |
519 | */ | |
520 | elem->call = (*entry)->call; | |
521 | /* | |
522 | * Sanity check : | |
523 | * We only update the single probe private data when the ptr is | |
524 | * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1) | |
525 | */ | |
526 | WARN_ON(elem->single.func != __mark_empty_function | |
527 | && elem->single.probe_private | |
528 | != (*entry)->single.probe_private && | |
529 | !elem->ptype); | |
530 | elem->single.probe_private = (*entry)->single.probe_private; | |
531 | /* | |
532 | * Make sure the private data is valid when we update the | |
533 | * single probe ptr. | |
534 | */ | |
535 | smp_wmb(); | |
536 | elem->single.func = (*entry)->single.func; | |
537 | /* | |
538 | * We also make sure that the new probe callbacks array is consistent | |
539 | * before setting a pointer to it. | |
540 | */ | |
541 | rcu_assign_pointer(elem->multi, (*entry)->multi); | |
542 | /* | |
543 | * Update the function or multi probe array pointer before setting the | |
544 | * ptype. | |
545 | */ | |
546 | smp_wmb(); | |
547 | elem->ptype = (*entry)->ptype; | |
548 | elem->state = active; | |
549 | ||
8256e47c MD |
550 | return 0; |
551 | } | |
552 | ||
553 | /* | |
554 | * Disable a marker and its probe callback. | |
fd3c36f8 MD |
555 | * Note: only waiting an RCU period after setting elem->call to the empty |
556 | * function insures that the original callback is not used anymore. This insured | |
557 | * by preempt_disable around the call site. | |
8256e47c MD |
558 | */ |
559 | static void disable_marker(struct marker *elem) | |
560 | { | |
fb40bd78 | 561 | /* leave "call" as is. It is known statically. */ |
8256e47c | 562 | elem->state = 0; |
fb40bd78 MD |
563 | elem->single.func = __mark_empty_function; |
564 | /* Update the function before setting the ptype */ | |
565 | smp_wmb(); | |
566 | elem->ptype = 0; /* single probe */ | |
8256e47c MD |
567 | /* |
568 | * Leave the private data and id there, because removal is racy and | |
fd3c36f8 MD |
569 | * should be done only after an RCU period. These are never used until |
570 | * the next initialization anyway. | |
8256e47c MD |
571 | */ |
572 | } | |
573 | ||
574 | /** | |
575 | * marker_update_probe_range - Update a probe range | |
576 | * @begin: beginning of the range | |
577 | * @end: end of the range | |
8256e47c MD |
578 | * |
579 | * Updates the probe callback corresponding to a range of markers. | |
8256e47c MD |
580 | */ |
581 | void marker_update_probe_range(struct marker *begin, | |
fb40bd78 | 582 | struct marker *end) |
8256e47c MD |
583 | { |
584 | struct marker *iter; | |
585 | struct marker_entry *mark_entry; | |
586 | ||
314de8a9 | 587 | mutex_lock(&markers_mutex); |
8256e47c MD |
588 | for (iter = begin; iter < end; iter++) { |
589 | mark_entry = get_marker(iter->name); | |
fb40bd78 MD |
590 | if (mark_entry) { |
591 | set_marker(&mark_entry, iter, | |
592 | !!mark_entry->refcount); | |
8256e47c MD |
593 | /* |
594 | * ignore error, continue | |
595 | */ | |
8256e47c MD |
596 | } else { |
597 | disable_marker(iter); | |
598 | } | |
599 | } | |
314de8a9 | 600 | mutex_unlock(&markers_mutex); |
8256e47c MD |
601 | } |
602 | ||
603 | /* | |
604 | * Update probes, removing the faulty probes. | |
fb40bd78 MD |
605 | * |
606 | * Internal callback only changed before the first probe is connected to it. | |
607 | * Single probe private data can only be changed on 0 -> 1 and 2 -> 1 | |
608 | * transitions. All other transitions will leave the old private data valid. | |
609 | * This makes the non-atomicity of the callback/private data updates valid. | |
610 | * | |
611 | * "special case" updates : | |
612 | * 0 -> 1 callback | |
613 | * 1 -> 0 callback | |
614 | * 1 -> 2 callbacks | |
615 | * 2 -> 1 callbacks | |
616 | * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates. | |
617 | * Site effect : marker_set_format may delete the marker entry (creating a | |
618 | * replacement). | |
8256e47c | 619 | */ |
fb40bd78 | 620 | static void marker_update_probes(void) |
8256e47c | 621 | { |
8256e47c | 622 | /* Core kernel markers */ |
fb40bd78 | 623 | marker_update_probe_range(__start___markers, __stop___markers); |
8256e47c | 624 | /* Markers in modules. */ |
fb40bd78 | 625 | module_update_markers(); |
8256e47c MD |
626 | } |
627 | ||
628 | /** | |
629 | * marker_probe_register - Connect a probe to a marker | |
630 | * @name: marker name | |
631 | * @format: format string | |
632 | * @probe: probe handler | |
fb40bd78 | 633 | * @probe_private: probe private data |
8256e47c MD |
634 | * |
635 | * private data must be a valid allocated memory address, or NULL. | |
636 | * Returns 0 if ok, error value on error. | |
fb40bd78 | 637 | * The probe address must at least be aligned on the architecture pointer size. |
8256e47c MD |
638 | */ |
639 | int marker_probe_register(const char *name, const char *format, | |
fb40bd78 | 640 | marker_probe_func *probe, void *probe_private) |
8256e47c MD |
641 | { |
642 | struct marker_entry *entry; | |
314de8a9 | 643 | int ret = 0; |
fb40bd78 | 644 | struct marker_probe_closure *old; |
8256e47c MD |
645 | |
646 | mutex_lock(&markers_mutex); | |
647 | entry = get_marker(name); | |
fb40bd78 MD |
648 | if (!entry) { |
649 | entry = add_marker(name, format); | |
650 | if (IS_ERR(entry)) { | |
651 | ret = PTR_ERR(entry); | |
652 | goto end; | |
653 | } | |
8256e47c | 654 | } |
fb40bd78 MD |
655 | /* |
656 | * If we detect that a call_rcu is pending for this marker, | |
657 | * make sure it's executed now. | |
658 | */ | |
659 | if (entry->rcu_pending) | |
660 | rcu_barrier(); | |
661 | old = marker_entry_add_probe(entry, probe, probe_private); | |
662 | if (IS_ERR(old)) { | |
663 | ret = PTR_ERR(old); | |
8256e47c | 664 | goto end; |
fb40bd78 | 665 | } |
314de8a9 | 666 | mutex_unlock(&markers_mutex); |
fb40bd78 MD |
667 | marker_update_probes(); /* may update entry */ |
668 | mutex_lock(&markers_mutex); | |
669 | entry = get_marker(name); | |
670 | WARN_ON(!entry); | |
671 | entry->oldptr = old; | |
672 | entry->rcu_pending = 1; | |
673 | /* write rcu_pending before calling the RCU callback */ | |
674 | smp_wmb(); | |
6496968e MD |
675 | #ifdef CONFIG_PREEMPT_RCU |
676 | synchronize_sched(); /* Until we have the call_rcu_sched() */ | |
677 | #endif | |
fb40bd78 | 678 | call_rcu(&entry->rcu, free_old_closure); |
8256e47c MD |
679 | end: |
680 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
681 | return ret; |
682 | } | |
683 | EXPORT_SYMBOL_GPL(marker_probe_register); | |
684 | ||
685 | /** | |
686 | * marker_probe_unregister - Disconnect a probe from a marker | |
687 | * @name: marker name | |
fb40bd78 MD |
688 | * @probe: probe function pointer |
689 | * @probe_private: probe private data | |
8256e47c MD |
690 | * |
691 | * Returns the private data given to marker_probe_register, or an ERR_PTR(). | |
fb40bd78 MD |
692 | * We do not need to call a synchronize_sched to make sure the probes have |
693 | * finished running before doing a module unload, because the module unload | |
694 | * itself uses stop_machine(), which insures that every preempt disabled section | |
695 | * have finished. | |
8256e47c | 696 | */ |
fb40bd78 MD |
697 | int marker_probe_unregister(const char *name, |
698 | marker_probe_func *probe, void *probe_private) | |
8256e47c | 699 | { |
8256e47c | 700 | struct marker_entry *entry; |
fb40bd78 | 701 | struct marker_probe_closure *old; |
544adb41 | 702 | int ret = -ENOENT; |
8256e47c MD |
703 | |
704 | mutex_lock(&markers_mutex); | |
705 | entry = get_marker(name); | |
544adb41 | 706 | if (!entry) |
8256e47c | 707 | goto end; |
fb40bd78 MD |
708 | if (entry->rcu_pending) |
709 | rcu_barrier(); | |
710 | old = marker_entry_remove_probe(entry, probe, probe_private); | |
314de8a9 | 711 | mutex_unlock(&markers_mutex); |
fb40bd78 MD |
712 | marker_update_probes(); /* may update entry */ |
713 | mutex_lock(&markers_mutex); | |
714 | entry = get_marker(name); | |
544adb41 JJ |
715 | if (!entry) |
716 | goto end; | |
fb40bd78 MD |
717 | entry->oldptr = old; |
718 | entry->rcu_pending = 1; | |
719 | /* write rcu_pending before calling the RCU callback */ | |
720 | smp_wmb(); | |
6496968e MD |
721 | #ifdef CONFIG_PREEMPT_RCU |
722 | synchronize_sched(); /* Until we have the call_rcu_sched() */ | |
723 | #endif | |
fb40bd78 MD |
724 | call_rcu(&entry->rcu, free_old_closure); |
725 | remove_marker(name); /* Ignore busy error message */ | |
544adb41 | 726 | ret = 0; |
8256e47c MD |
727 | end: |
728 | mutex_unlock(&markers_mutex); | |
fb40bd78 | 729 | return ret; |
8256e47c MD |
730 | } |
731 | EXPORT_SYMBOL_GPL(marker_probe_unregister); | |
732 | ||
fb40bd78 MD |
733 | static struct marker_entry * |
734 | get_marker_from_private_data(marker_probe_func *probe, void *probe_private) | |
8256e47c | 735 | { |
8256e47c | 736 | struct marker_entry *entry; |
8256e47c | 737 | unsigned int i; |
fb40bd78 MD |
738 | struct hlist_head *head; |
739 | struct hlist_node *node; | |
8256e47c | 740 | |
8256e47c MD |
741 | for (i = 0; i < MARKER_TABLE_SIZE; i++) { |
742 | head = &marker_table[i]; | |
743 | hlist_for_each_entry(entry, node, head, hlist) { | |
fb40bd78 MD |
744 | if (!entry->ptype) { |
745 | if (entry->single.func == probe | |
746 | && entry->single.probe_private | |
747 | == probe_private) | |
748 | return entry; | |
749 | } else { | |
750 | struct marker_probe_closure *closure; | |
751 | closure = entry->multi; | |
752 | for (i = 0; closure[i].func; i++) { | |
753 | if (closure[i].func == probe && | |
754 | closure[i].probe_private | |
755 | == probe_private) | |
756 | return entry; | |
757 | } | |
8256e47c MD |
758 | } |
759 | } | |
760 | } | |
fb40bd78 | 761 | return NULL; |
8256e47c | 762 | } |
8256e47c MD |
763 | |
764 | /** | |
fb40bd78 MD |
765 | * marker_probe_unregister_private_data - Disconnect a probe from a marker |
766 | * @probe: probe function | |
767 | * @probe_private: probe private data | |
8256e47c | 768 | * |
fb40bd78 MD |
769 | * Unregister a probe by providing the registered private data. |
770 | * Only removes the first marker found in hash table. | |
771 | * Return 0 on success or error value. | |
772 | * We do not need to call a synchronize_sched to make sure the probes have | |
773 | * finished running before doing a module unload, because the module unload | |
774 | * itself uses stop_machine(), which insures that every preempt disabled section | |
775 | * have finished. | |
8256e47c | 776 | */ |
fb40bd78 MD |
777 | int marker_probe_unregister_private_data(marker_probe_func *probe, |
778 | void *probe_private) | |
8256e47c MD |
779 | { |
780 | struct marker_entry *entry; | |
314de8a9 | 781 | int ret = 0; |
fb40bd78 | 782 | struct marker_probe_closure *old; |
8256e47c MD |
783 | |
784 | mutex_lock(&markers_mutex); | |
fb40bd78 | 785 | entry = get_marker_from_private_data(probe, probe_private); |
8256e47c MD |
786 | if (!entry) { |
787 | ret = -ENOENT; | |
788 | goto end; | |
789 | } | |
fb40bd78 MD |
790 | if (entry->rcu_pending) |
791 | rcu_barrier(); | |
792 | old = marker_entry_remove_probe(entry, NULL, probe_private); | |
8256e47c | 793 | mutex_unlock(&markers_mutex); |
fb40bd78 | 794 | marker_update_probes(); /* may update entry */ |
8256e47c | 795 | mutex_lock(&markers_mutex); |
fb40bd78 MD |
796 | entry = get_marker_from_private_data(probe, probe_private); |
797 | WARN_ON(!entry); | |
798 | entry->oldptr = old; | |
799 | entry->rcu_pending = 1; | |
800 | /* write rcu_pending before calling the RCU callback */ | |
801 | smp_wmb(); | |
6496968e MD |
802 | #ifdef CONFIG_PREEMPT_RCU |
803 | synchronize_sched(); /* Until we have the call_rcu_sched() */ | |
804 | #endif | |
fb40bd78 MD |
805 | call_rcu(&entry->rcu, free_old_closure); |
806 | remove_marker(entry->name); /* Ignore busy error message */ | |
8256e47c MD |
807 | end: |
808 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
809 | return ret; |
810 | } | |
fb40bd78 | 811 | EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); |
8256e47c MD |
812 | |
813 | /** | |
814 | * marker_get_private_data - Get a marker's probe private data | |
815 | * @name: marker name | |
fb40bd78 MD |
816 | * @probe: probe to match |
817 | * @num: get the nth matching probe's private data | |
8256e47c | 818 | * |
fb40bd78 MD |
819 | * Returns the nth private data pointer (starting from 0) matching, or an |
820 | * ERR_PTR. | |
8256e47c MD |
821 | * Returns the private data pointer, or an ERR_PTR. |
822 | * The private data pointer should _only_ be dereferenced if the caller is the | |
823 | * owner of the data, or its content could vanish. This is mostly used to | |
824 | * confirm that a caller is the owner of a registered probe. | |
825 | */ | |
fb40bd78 MD |
826 | void *marker_get_private_data(const char *name, marker_probe_func *probe, |
827 | int num) | |
8256e47c MD |
828 | { |
829 | struct hlist_head *head; | |
830 | struct hlist_node *node; | |
831 | struct marker_entry *e; | |
832 | size_t name_len = strlen(name) + 1; | |
833 | u32 hash = jhash(name, name_len-1, 0); | |
fb40bd78 | 834 | int i; |
8256e47c MD |
835 | |
836 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
837 | hlist_for_each_entry(e, node, head, hlist) { | |
838 | if (!strcmp(name, e->name)) { | |
fb40bd78 MD |
839 | if (!e->ptype) { |
840 | if (num == 0 && e->single.func == probe) | |
841 | return e->single.probe_private; | |
842 | else | |
843 | break; | |
844 | } else { | |
845 | struct marker_probe_closure *closure; | |
846 | int match = 0; | |
847 | closure = e->multi; | |
848 | for (i = 0; closure[i].func; i++) { | |
849 | if (closure[i].func != probe) | |
850 | continue; | |
851 | if (match++ == num) | |
852 | return closure[i].probe_private; | |
853 | } | |
854 | } | |
8256e47c MD |
855 | } |
856 | } | |
857 | return ERR_PTR(-ENOENT); | |
858 | } | |
859 | EXPORT_SYMBOL_GPL(marker_get_private_data); |