Commit | Line | Data |
---|---|---|
8256e47c MD |
1 | /* |
2 | * Copyright (C) 2007 Mathieu Desnoyers | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | */ | |
18 | #include <linux/module.h> | |
19 | #include <linux/mutex.h> | |
20 | #include <linux/types.h> | |
21 | #include <linux/jhash.h> | |
22 | #include <linux/list.h> | |
23 | #include <linux/rcupdate.h> | |
24 | #include <linux/marker.h> | |
25 | #include <linux/err.h> | |
26 | ||
27 | extern struct marker __start___markers[]; | |
28 | extern struct marker __stop___markers[]; | |
29 | ||
30 | /* | |
31 | * module_mutex nests inside markers_mutex. Markers mutex protects the builtin | |
32 | * and module markers, the hash table and deferred_sync. | |
33 | */ | |
34 | static DEFINE_MUTEX(markers_mutex); | |
35 | ||
36 | /* | |
37 | * Marker deferred synchronization. | |
38 | * Upon marker probe_unregister, we delay call to synchronize_sched() to | |
39 | * accelerate mass unregistration (only when there is no more reference to a | |
40 | * given module do we call synchronize_sched()). However, we need to make sure | |
41 | * every critical region has ended before we re-arm a marker that has been | |
42 | * unregistered and then registered back with a different probe data. | |
43 | */ | |
44 | static int deferred_sync; | |
45 | ||
46 | /* | |
47 | * Marker hash table, containing the active markers. | |
48 | * Protected by module_mutex. | |
49 | */ | |
50 | #define MARKER_HASH_BITS 6 | |
51 | #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) | |
52 | ||
53 | struct marker_entry { | |
54 | struct hlist_node hlist; | |
55 | char *format; | |
56 | marker_probe_func *probe; | |
57 | void *private; | |
58 | int refcount; /* Number of times armed. 0 if disarmed. */ | |
59 | char name[0]; /* Contains name'\0'format'\0' */ | |
60 | }; | |
61 | ||
62 | static struct hlist_head marker_table[MARKER_TABLE_SIZE]; | |
63 | ||
64 | /** | |
65 | * __mark_empty_function - Empty probe callback | |
66 | * @mdata: pointer of type const struct marker | |
67 | * @fmt: format string | |
68 | * @...: variable argument list | |
69 | * | |
70 | * Empty callback provided as a probe to the markers. By providing this to a | |
71 | * disabled marker, we make sure the execution flow is always valid even | |
72 | * though the function pointer change and the marker enabling are two distinct | |
73 | * operations that modifies the execution flow of preemptible code. | |
74 | */ | |
75 | void __mark_empty_function(const struct marker *mdata, void *private, | |
76 | const char *fmt, ...) | |
77 | { | |
78 | } | |
79 | EXPORT_SYMBOL_GPL(__mark_empty_function); | |
80 | ||
81 | /* | |
82 | * Get marker if the marker is present in the marker hash table. | |
83 | * Must be called with markers_mutex held. | |
84 | * Returns NULL if not present. | |
85 | */ | |
86 | static struct marker_entry *get_marker(const char *name) | |
87 | { | |
88 | struct hlist_head *head; | |
89 | struct hlist_node *node; | |
90 | struct marker_entry *e; | |
91 | u32 hash = jhash(name, strlen(name), 0); | |
92 | ||
93 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
94 | hlist_for_each_entry(e, node, head, hlist) { | |
95 | if (!strcmp(name, e->name)) | |
96 | return e; | |
97 | } | |
98 | return NULL; | |
99 | } | |
100 | ||
101 | /* | |
102 | * Add the marker to the marker hash table. Must be called with markers_mutex | |
103 | * held. | |
104 | */ | |
105 | static int add_marker(const char *name, const char *format, | |
106 | marker_probe_func *probe, void *private) | |
107 | { | |
108 | struct hlist_head *head; | |
109 | struct hlist_node *node; | |
110 | struct marker_entry *e; | |
111 | size_t name_len = strlen(name) + 1; | |
112 | size_t format_len = 0; | |
113 | u32 hash = jhash(name, name_len-1, 0); | |
114 | ||
115 | if (format) | |
116 | format_len = strlen(format) + 1; | |
117 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
118 | hlist_for_each_entry(e, node, head, hlist) { | |
119 | if (!strcmp(name, e->name)) { | |
120 | printk(KERN_NOTICE | |
121 | "Marker %s busy, probe %p already installed\n", | |
122 | name, e->probe); | |
123 | return -EBUSY; /* Already there */ | |
124 | } | |
125 | } | |
126 | /* | |
127 | * Using kmalloc here to allocate a variable length element. Could | |
128 | * cause some memory fragmentation if overused. | |
129 | */ | |
130 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, | |
131 | GFP_KERNEL); | |
132 | if (!e) | |
133 | return -ENOMEM; | |
134 | memcpy(&e->name[0], name, name_len); | |
135 | if (format) { | |
136 | e->format = &e->name[name_len]; | |
137 | memcpy(e->format, format, format_len); | |
138 | trace_mark(core_marker_format, "name %s format %s", | |
139 | e->name, e->format); | |
140 | } else | |
141 | e->format = NULL; | |
142 | e->probe = probe; | |
143 | e->private = private; | |
144 | e->refcount = 0; | |
145 | hlist_add_head(&e->hlist, head); | |
146 | return 0; | |
147 | } | |
148 | ||
149 | /* | |
150 | * Remove the marker from the marker hash table. Must be called with mutex_lock | |
151 | * held. | |
152 | */ | |
153 | static void *remove_marker(const char *name) | |
154 | { | |
155 | struct hlist_head *head; | |
156 | struct hlist_node *node; | |
157 | struct marker_entry *e; | |
158 | int found = 0; | |
159 | size_t len = strlen(name) + 1; | |
160 | void *private = NULL; | |
161 | u32 hash = jhash(name, len-1, 0); | |
162 | ||
163 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
164 | hlist_for_each_entry(e, node, head, hlist) { | |
165 | if (!strcmp(name, e->name)) { | |
166 | found = 1; | |
167 | break; | |
168 | } | |
169 | } | |
170 | if (found) { | |
171 | private = e->private; | |
172 | hlist_del(&e->hlist); | |
173 | kfree(e); | |
174 | } | |
175 | return private; | |
176 | } | |
177 | ||
178 | /* | |
179 | * Set the mark_entry format to the format found in the element. | |
180 | */ | |
181 | static int marker_set_format(struct marker_entry **entry, const char *format) | |
182 | { | |
183 | struct marker_entry *e; | |
184 | size_t name_len = strlen((*entry)->name) + 1; | |
185 | size_t format_len = strlen(format) + 1; | |
186 | ||
187 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, | |
188 | GFP_KERNEL); | |
189 | if (!e) | |
190 | return -ENOMEM; | |
191 | memcpy(&e->name[0], (*entry)->name, name_len); | |
192 | e->format = &e->name[name_len]; | |
193 | memcpy(e->format, format, format_len); | |
194 | e->probe = (*entry)->probe; | |
195 | e->private = (*entry)->private; | |
196 | e->refcount = (*entry)->refcount; | |
197 | hlist_add_before(&e->hlist, &(*entry)->hlist); | |
198 | hlist_del(&(*entry)->hlist); | |
199 | kfree(*entry); | |
200 | *entry = e; | |
201 | trace_mark(core_marker_format, "name %s format %s", | |
202 | e->name, e->format); | |
203 | return 0; | |
204 | } | |
205 | ||
206 | /* | |
207 | * Sets the probe callback corresponding to one marker. | |
208 | */ | |
209 | static int set_marker(struct marker_entry **entry, struct marker *elem) | |
210 | { | |
211 | int ret; | |
212 | WARN_ON(strcmp((*entry)->name, elem->name) != 0); | |
213 | ||
214 | if ((*entry)->format) { | |
215 | if (strcmp((*entry)->format, elem->format) != 0) { | |
216 | printk(KERN_NOTICE | |
217 | "Format mismatch for probe %s " | |
218 | "(%s), marker (%s)\n", | |
219 | (*entry)->name, | |
220 | (*entry)->format, | |
221 | elem->format); | |
222 | return -EPERM; | |
223 | } | |
224 | } else { | |
225 | ret = marker_set_format(entry, elem->format); | |
226 | if (ret) | |
227 | return ret; | |
228 | } | |
229 | elem->call = (*entry)->probe; | |
230 | elem->private = (*entry)->private; | |
231 | elem->state = 1; | |
232 | return 0; | |
233 | } | |
234 | ||
235 | /* | |
236 | * Disable a marker and its probe callback. | |
237 | * Note: only after a synchronize_sched() issued after setting elem->call to the | |
238 | * empty function insures that the original callback is not used anymore. This | |
239 | * insured by preemption disabling around the call site. | |
240 | */ | |
241 | static void disable_marker(struct marker *elem) | |
242 | { | |
243 | elem->state = 0; | |
244 | elem->call = __mark_empty_function; | |
245 | /* | |
246 | * Leave the private data and id there, because removal is racy and | |
247 | * should be done only after a synchronize_sched(). These are never used | |
248 | * until the next initialization anyway. | |
249 | */ | |
250 | } | |
251 | ||
252 | /** | |
253 | * marker_update_probe_range - Update a probe range | |
254 | * @begin: beginning of the range | |
255 | * @end: end of the range | |
256 | * @probe_module: module address of the probe being updated | |
257 | * @refcount: number of references left to the given probe_module (out) | |
258 | * | |
259 | * Updates the probe callback corresponding to a range of markers. | |
260 | * Must be called with markers_mutex held. | |
261 | */ | |
262 | void marker_update_probe_range(struct marker *begin, | |
263 | struct marker *end, struct module *probe_module, | |
264 | int *refcount) | |
265 | { | |
266 | struct marker *iter; | |
267 | struct marker_entry *mark_entry; | |
268 | ||
269 | for (iter = begin; iter < end; iter++) { | |
270 | mark_entry = get_marker(iter->name); | |
271 | if (mark_entry && mark_entry->refcount) { | |
272 | set_marker(&mark_entry, iter); | |
273 | /* | |
274 | * ignore error, continue | |
275 | */ | |
276 | if (probe_module) | |
277 | if (probe_module == | |
278 | __module_text_address((unsigned long)mark_entry->probe)) | |
279 | (*refcount)++; | |
280 | } else { | |
281 | disable_marker(iter); | |
282 | } | |
283 | } | |
284 | } | |
285 | ||
286 | /* | |
287 | * Update probes, removing the faulty probes. | |
288 | * Issues a synchronize_sched() when no reference to the module passed | |
289 | * as parameter is found in the probes so the probe module can be | |
290 | * safely unloaded from now on. | |
291 | */ | |
292 | static void marker_update_probes(struct module *probe_module) | |
293 | { | |
294 | int refcount = 0; | |
295 | ||
296 | mutex_lock(&markers_mutex); | |
297 | /* Core kernel markers */ | |
298 | marker_update_probe_range(__start___markers, | |
299 | __stop___markers, probe_module, &refcount); | |
300 | /* Markers in modules. */ | |
301 | module_update_markers(probe_module, &refcount); | |
302 | if (probe_module && refcount == 0) { | |
303 | synchronize_sched(); | |
304 | deferred_sync = 0; | |
305 | } | |
306 | mutex_unlock(&markers_mutex); | |
307 | } | |
308 | ||
309 | /** | |
310 | * marker_probe_register - Connect a probe to a marker | |
311 | * @name: marker name | |
312 | * @format: format string | |
313 | * @probe: probe handler | |
314 | * @private: probe private data | |
315 | * | |
316 | * private data must be a valid allocated memory address, or NULL. | |
317 | * Returns 0 if ok, error value on error. | |
318 | */ | |
319 | int marker_probe_register(const char *name, const char *format, | |
320 | marker_probe_func *probe, void *private) | |
321 | { | |
322 | struct marker_entry *entry; | |
323 | int ret = 0, need_update = 0; | |
324 | ||
325 | mutex_lock(&markers_mutex); | |
326 | entry = get_marker(name); | |
327 | if (entry && entry->refcount) { | |
328 | ret = -EBUSY; | |
329 | goto end; | |
330 | } | |
331 | if (deferred_sync) { | |
332 | synchronize_sched(); | |
333 | deferred_sync = 0; | |
334 | } | |
335 | ret = add_marker(name, format, probe, private); | |
336 | if (ret) | |
337 | goto end; | |
338 | need_update = 1; | |
339 | end: | |
340 | mutex_unlock(&markers_mutex); | |
341 | if (need_update) | |
342 | marker_update_probes(NULL); | |
343 | return ret; | |
344 | } | |
345 | EXPORT_SYMBOL_GPL(marker_probe_register); | |
346 | ||
347 | /** | |
348 | * marker_probe_unregister - Disconnect a probe from a marker | |
349 | * @name: marker name | |
350 | * | |
351 | * Returns the private data given to marker_probe_register, or an ERR_PTR(). | |
352 | */ | |
353 | void *marker_probe_unregister(const char *name) | |
354 | { | |
355 | struct module *probe_module; | |
356 | struct marker_entry *entry; | |
357 | void *private; | |
358 | int need_update = 0; | |
359 | ||
360 | mutex_lock(&markers_mutex); | |
361 | entry = get_marker(name); | |
362 | if (!entry) { | |
363 | private = ERR_PTR(-ENOENT); | |
364 | goto end; | |
365 | } | |
366 | entry->refcount = 0; | |
367 | /* In what module is the probe handler ? */ | |
368 | probe_module = __module_text_address((unsigned long)entry->probe); | |
369 | private = remove_marker(name); | |
370 | deferred_sync = 1; | |
371 | need_update = 1; | |
372 | end: | |
373 | mutex_unlock(&markers_mutex); | |
374 | if (need_update) | |
375 | marker_update_probes(probe_module); | |
376 | return private; | |
377 | } | |
378 | EXPORT_SYMBOL_GPL(marker_probe_unregister); | |
379 | ||
380 | /** | |
381 | * marker_probe_unregister_private_data - Disconnect a probe from a marker | |
382 | * @private: probe private data | |
383 | * | |
384 | * Unregister a marker by providing the registered private data. | |
385 | * Returns the private data given to marker_probe_register, or an ERR_PTR(). | |
386 | */ | |
387 | void *marker_probe_unregister_private_data(void *private) | |
388 | { | |
389 | struct module *probe_module; | |
390 | struct hlist_head *head; | |
391 | struct hlist_node *node; | |
392 | struct marker_entry *entry; | |
393 | int found = 0; | |
394 | unsigned int i; | |
395 | int need_update = 0; | |
396 | ||
397 | mutex_lock(&markers_mutex); | |
398 | for (i = 0; i < MARKER_TABLE_SIZE; i++) { | |
399 | head = &marker_table[i]; | |
400 | hlist_for_each_entry(entry, node, head, hlist) { | |
401 | if (entry->private == private) { | |
402 | found = 1; | |
403 | goto iter_end; | |
404 | } | |
405 | } | |
406 | } | |
407 | iter_end: | |
408 | if (!found) { | |
409 | private = ERR_PTR(-ENOENT); | |
410 | goto end; | |
411 | } | |
412 | entry->refcount = 0; | |
413 | /* In what module is the probe handler ? */ | |
414 | probe_module = __module_text_address((unsigned long)entry->probe); | |
415 | private = remove_marker(entry->name); | |
416 | deferred_sync = 1; | |
417 | need_update = 1; | |
418 | end: | |
419 | mutex_unlock(&markers_mutex); | |
420 | if (need_update) | |
421 | marker_update_probes(probe_module); | |
422 | return private; | |
423 | } | |
424 | EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); | |
425 | ||
426 | /** | |
427 | * marker_arm - Arm a marker | |
428 | * @name: marker name | |
429 | * | |
430 | * Activate a marker. It keeps a reference count of the number of | |
431 | * arming/disarming done. | |
432 | * Returns 0 if ok, error value on error. | |
433 | */ | |
434 | int marker_arm(const char *name) | |
435 | { | |
436 | struct marker_entry *entry; | |
437 | int ret = 0, need_update = 0; | |
438 | ||
439 | mutex_lock(&markers_mutex); | |
440 | entry = get_marker(name); | |
441 | if (!entry) { | |
442 | ret = -ENOENT; | |
443 | goto end; | |
444 | } | |
445 | /* | |
446 | * Only need to update probes when refcount passes from 0 to 1. | |
447 | */ | |
448 | if (entry->refcount++) | |
449 | goto end; | |
450 | need_update = 1; | |
451 | end: | |
452 | mutex_unlock(&markers_mutex); | |
453 | if (need_update) | |
454 | marker_update_probes(NULL); | |
455 | return ret; | |
456 | } | |
457 | EXPORT_SYMBOL_GPL(marker_arm); | |
458 | ||
459 | /** | |
460 | * marker_disarm - Disarm a marker | |
461 | * @name: marker name | |
462 | * | |
463 | * Disarm a marker. It keeps a reference count of the number of arming/disarming | |
464 | * done. | |
465 | * Returns 0 if ok, error value on error. | |
466 | */ | |
467 | int marker_disarm(const char *name) | |
468 | { | |
469 | struct marker_entry *entry; | |
470 | int ret = 0, need_update = 0; | |
471 | ||
472 | mutex_lock(&markers_mutex); | |
473 | entry = get_marker(name); | |
474 | if (!entry) { | |
475 | ret = -ENOENT; | |
476 | goto end; | |
477 | } | |
478 | /* | |
479 | * Only permit decrement refcount if higher than 0. | |
480 | * Do probe update only on 1 -> 0 transition. | |
481 | */ | |
482 | if (entry->refcount) { | |
483 | if (--entry->refcount) | |
484 | goto end; | |
485 | } else { | |
486 | ret = -EPERM; | |
487 | goto end; | |
488 | } | |
489 | need_update = 1; | |
490 | end: | |
491 | mutex_unlock(&markers_mutex); | |
492 | if (need_update) | |
493 | marker_update_probes(NULL); | |
494 | return ret; | |
495 | } | |
496 | EXPORT_SYMBOL_GPL(marker_disarm); | |
497 | ||
498 | /** | |
499 | * marker_get_private_data - Get a marker's probe private data | |
500 | * @name: marker name | |
501 | * | |
502 | * Returns the private data pointer, or an ERR_PTR. | |
503 | * The private data pointer should _only_ be dereferenced if the caller is the | |
504 | * owner of the data, or its content could vanish. This is mostly used to | |
505 | * confirm that a caller is the owner of a registered probe. | |
506 | */ | |
507 | void *marker_get_private_data(const char *name) | |
508 | { | |
509 | struct hlist_head *head; | |
510 | struct hlist_node *node; | |
511 | struct marker_entry *e; | |
512 | size_t name_len = strlen(name) + 1; | |
513 | u32 hash = jhash(name, name_len-1, 0); | |
514 | int found = 0; | |
515 | ||
516 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
517 | hlist_for_each_entry(e, node, head, hlist) { | |
518 | if (!strcmp(name, e->name)) { | |
519 | found = 1; | |
520 | return e->private; | |
521 | } | |
522 | } | |
523 | return ERR_PTR(-ENOENT); | |
524 | } | |
525 | EXPORT_SYMBOL_GPL(marker_get_private_data); |