1 // SPDX-License-Identifier: MIT
3 * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 #include <side/trace.h>
13 /* Top 8 bits reserved for kernel tracer use. */
14 #if SIDE_BITS_PER_LONG == 64
15 # define SIDE_EVENT_ENABLED_KERNEL_MASK 0xFF00000000000000ULL
16 # define SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK 0x8000000000000000ULL
18 /* Allow 2^56 tracer references on an event. */
19 # define SIDE_EVENT_ENABLED_USER_MASK 0x00FFFFFFFFFFFFFFULL
21 # define SIDE_EVENT_ENABLED_KERNEL_MASK 0xFF000000UL
22 # define SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK 0x80000000UL
24 /* Allow 2^24 tracer references on an event. */
25 # define SIDE_EVENT_ENABLED_USER_MASK 0x00FFFFFFUL
28 struct side_events_register_handle
{
29 struct side_list_node node
;
30 struct side_event_description
**events
;
34 struct side_tracer_handle
{
35 struct side_list_node node
;
36 void (*cb
)(enum side_tracer_notification notif
,
37 struct side_event_description
**events
, uint32_t nr_events
, void *priv
);
41 struct side_callback
{
43 void (*call
)(const struct side_event_description
*desc
,
44 const struct side_arg_vec
*side_arg_vec
,
46 void (*call_variadic
)(const struct side_event_description
*desc
,
47 const struct side_arg_vec
*side_arg_vec
,
48 const struct side_arg_dynamic_struct
*var_struct
,
55 static struct side_rcu_gp_state rcu_gp
;
58 * Lazy initialization for early use within library constructors.
60 static bool initialized
;
62 * Do not register/unregister any more events after destructor.
64 static bool finalized
;
67 * Recursive mutex to allow tracer callbacks to use the side API.
69 static pthread_mutex_t side_lock
= PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
;
71 static DEFINE_SIDE_LIST_HEAD(side_events_list
);
72 static DEFINE_SIDE_LIST_HEAD(side_tracer_list
);
75 * Callback filter key for state dump.
77 static __thread
void *filter_key
;
80 * The empty callback has a NULL function callback pointer, which stops
81 * iteration on the array of callbacks immediately.
83 const char side_empty_callback
[sizeof(struct side_callback
)];
86 void _side_call(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
, void *key
)
88 struct side_rcu_read_state rcu_read_state
;
89 const struct side_event_state_0
*es0
;
90 const struct side_callback
*side_cb
;
93 if (side_unlikely(finalized
))
95 if (side_unlikely(!initialized
))
97 if (side_unlikely(event_state
->version
!= 0))
99 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
100 assert(!(es0
->desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
));
101 enabled
= __atomic_load_n(&es0
->enabled
, __ATOMIC_RELAXED
);
102 if (side_unlikely(enabled
& SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK
)) {
103 // TODO: call kernel write.
105 side_rcu_read_begin(&rcu_gp
, &rcu_read_state
);
106 for (side_cb
= side_rcu_dereference(es0
->callbacks
); side_cb
->u
.call
!= NULL
; side_cb
++) {
107 /* A NULL key is always a match. */
108 if (key
&& side_cb
->key
&& side_cb
->key
!= key
)
110 side_cb
->u
.call(es0
->desc
, side_arg_vec
, side_cb
->priv
);
112 side_rcu_read_end(&rcu_gp
, &rcu_read_state
);
115 void side_call(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
)
117 _side_call(event_state
, side_arg_vec
, NULL
);
120 void side_call_key(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
)
122 _side_call(event_state
, side_arg_vec
, filter_key
);
126 void _side_call_variadic(const struct side_event_state
*event_state
,
127 const struct side_arg_vec
*side_arg_vec
,
128 const struct side_arg_dynamic_struct
*var_struct
,
131 struct side_rcu_read_state rcu_read_state
;
132 const struct side_event_state_0
*es0
;
133 const struct side_callback
*side_cb
;
136 if (side_unlikely(finalized
))
138 if (side_unlikely(!initialized
))
140 if (side_unlikely(event_state
->version
!= 0))
142 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
143 assert(es0
->desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
);
144 enabled
= __atomic_load_n(&es0
->enabled
, __ATOMIC_RELAXED
);
145 if (side_unlikely(enabled
& SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK
)) {
146 // TODO: call kernel write.
148 side_rcu_read_begin(&rcu_gp
, &rcu_read_state
);
149 for (side_cb
= side_rcu_dereference(es0
->callbacks
); side_cb
->u
.call_variadic
!= NULL
; side_cb
++) {
150 /* A NULL key is always a match. */
151 if (key
&& side_cb
->key
&& side_cb
->key
!= key
)
153 side_cb
->u
.call_variadic(es0
->desc
, side_arg_vec
, var_struct
, side_cb
->priv
);
155 side_rcu_read_end(&rcu_gp
, &rcu_read_state
);
158 void side_call_variadic(const struct side_event_state
*event_state
,
159 const struct side_arg_vec
*side_arg_vec
,
160 const struct side_arg_dynamic_struct
*var_struct
)
162 _side_call_variadic(event_state
, side_arg_vec
, var_struct
, NULL
);
165 void side_call_variadic_key(const struct side_event_state
*event_state
,
166 const struct side_arg_vec
*side_arg_vec
,
167 const struct side_arg_dynamic_struct
*var_struct
)
169 _side_call_variadic(event_state
, side_arg_vec
, var_struct
, filter_key
);
173 const struct side_callback
*side_tracer_callback_lookup(
174 const struct side_event_description
*desc
,
175 void *call
, void *priv
, void *key
)
177 struct side_event_state
*event_state
= side_ptr_get(desc
->state
);
178 const struct side_event_state_0
*es0
;
179 const struct side_callback
*cb
;
181 if (side_unlikely(event_state
->version
!= 0))
183 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
184 for (cb
= es0
->callbacks
; cb
->u
.call
!= NULL
; cb
++) {
185 if ((void *) cb
->u
.call
== call
&& cb
->priv
== priv
&& cb
->key
== key
)
192 int _side_tracer_callback_register(struct side_event_description
*desc
,
193 void *call
, void *priv
, void *key
)
195 struct side_event_state
*event_state
;
196 struct side_callback
*old_cb
, *new_cb
;
197 struct side_event_state_0
*es0
;
198 int ret
= SIDE_ERROR_OK
;
202 return SIDE_ERROR_INVAL
;
204 return SIDE_ERROR_EXITING
;
207 pthread_mutex_lock(&side_lock
);
208 event_state
= side_ptr_get(desc
->state
);
209 if (side_unlikely(event_state
->version
!= 0))
211 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
212 old_nr_cb
= es0
->nr_callbacks
;
213 if (old_nr_cb
== UINT32_MAX
) {
214 ret
= SIDE_ERROR_INVAL
;
217 /* Reject duplicate (call, priv) tuples. */
218 if (side_tracer_callback_lookup(desc
, call
, priv
, key
)) {
219 ret
= SIDE_ERROR_EXIST
;
222 old_cb
= (struct side_callback
*) es0
->callbacks
;
223 /* old_nr_cb + 1 (new cb) + 1 (NULL) */
224 new_cb
= (struct side_callback
*) calloc(old_nr_cb
+ 2, sizeof(struct side_callback
));
226 ret
= SIDE_ERROR_NOMEM
;
229 memcpy(new_cb
, old_cb
, old_nr_cb
);
230 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
231 new_cb
[old_nr_cb
].u
.call_variadic
=
232 (side_tracer_callback_variadic_func
) call
;
234 new_cb
[old_nr_cb
].u
.call
=
235 (side_tracer_callback_func
) call
;
236 new_cb
[old_nr_cb
].priv
= priv
;
237 new_cb
[old_nr_cb
].key
= key
;
238 /* High order bits are already zeroed. */
239 side_rcu_assign_pointer(es0
->callbacks
, new_cb
);
240 side_rcu_wait_grace_period(&rcu_gp
);
244 /* Increment concurrently with kernel setting the top bits. */
246 (void) __atomic_add_fetch(&es0
->enabled
, 1, __ATOMIC_RELAXED
);
248 pthread_mutex_unlock(&side_lock
);
252 int side_tracer_callback_register(struct side_event_description
*desc
,
253 side_tracer_callback_func call
,
254 void *priv
, void *key
)
256 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
257 return SIDE_ERROR_INVAL
;
258 return _side_tracer_callback_register(desc
, (void *) call
, priv
, key
);
261 int side_tracer_callback_variadic_register(struct side_event_description
*desc
,
262 side_tracer_callback_variadic_func call_variadic
,
263 void *priv
, void *key
)
265 if (!(desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
))
266 return SIDE_ERROR_INVAL
;
267 return _side_tracer_callback_register(desc
, (void *) call_variadic
, priv
, key
);
270 static int _side_tracer_callback_unregister(struct side_event_description
*desc
,
271 void *call
, void *priv
, void *key
)
273 struct side_event_state
*event_state
;
274 struct side_callback
*old_cb
, *new_cb
;
275 const struct side_callback
*cb_pos
;
276 struct side_event_state_0
*es0
;
278 int ret
= SIDE_ERROR_OK
;
282 return SIDE_ERROR_INVAL
;
284 return SIDE_ERROR_EXITING
;
287 pthread_mutex_lock(&side_lock
);
288 event_state
= side_ptr_get(desc
->state
);
289 if (side_unlikely(event_state
->version
!= 0))
291 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
292 cb_pos
= side_tracer_callback_lookup(desc
, call
, priv
, key
);
294 ret
= SIDE_ERROR_NOENT
;
297 old_nr_cb
= es0
->nr_callbacks
;
298 old_cb
= (struct side_callback
*) es0
->callbacks
;
299 if (old_nr_cb
== 1) {
300 new_cb
= (struct side_callback
*) &side_empty_callback
;
302 pos_idx
= cb_pos
- es0
->callbacks
;
303 /* Remove entry at pos_idx. */
304 /* old_nr_cb - 1 (removed cb) + 1 (NULL) */
305 new_cb
= (struct side_callback
*) calloc(old_nr_cb
, sizeof(struct side_callback
));
307 ret
= SIDE_ERROR_NOMEM
;
310 memcpy(new_cb
, old_cb
, pos_idx
);
311 memcpy(&new_cb
[pos_idx
], &old_cb
[pos_idx
+ 1], old_nr_cb
- pos_idx
- 1);
313 /* High order bits are already zeroed. */
314 side_rcu_assign_pointer(es0
->callbacks
, new_cb
);
315 side_rcu_wait_grace_period(&rcu_gp
);
318 /* Decrement concurrently with kernel setting the top bits. */
320 (void) __atomic_add_fetch(&es0
->enabled
, -1, __ATOMIC_RELAXED
);
322 pthread_mutex_unlock(&side_lock
);
326 int side_tracer_callback_unregister(struct side_event_description
*desc
,
327 side_tracer_callback_func call
,
328 void *priv
, void *key
)
330 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
331 return SIDE_ERROR_INVAL
;
332 return _side_tracer_callback_unregister(desc
, (void *) call
, priv
, key
);
335 int side_tracer_callback_variadic_unregister(struct side_event_description
*desc
,
336 side_tracer_callback_variadic_func call_variadic
,
337 void *priv
, void *key
)
339 if (!(desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
))
340 return SIDE_ERROR_INVAL
;
341 return _side_tracer_callback_unregister(desc
, (void *) call_variadic
, priv
, key
);
344 struct side_events_register_handle
*side_events_register(struct side_event_description
**events
, uint32_t nr_events
)
346 struct side_events_register_handle
*events_handle
= NULL
;
347 struct side_tracer_handle
*tracer_handle
;
353 events_handle
= (struct side_events_register_handle
*)
354 calloc(1, sizeof(struct side_events_register_handle
));
357 events_handle
->events
= events
;
358 events_handle
->nr_events
= nr_events
;
360 pthread_mutex_lock(&side_lock
);
361 side_list_insert_node_tail(&side_events_list
, &events_handle
->node
);
362 side_list_for_each_entry(tracer_handle
, &side_tracer_list
, node
) {
363 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_INSERT_EVENTS
,
364 events
, nr_events
, tracer_handle
->priv
);
366 pthread_mutex_unlock(&side_lock
);
367 //TODO: call event batch register ioctl
368 return events_handle
;
372 void side_event_remove_callbacks(struct side_event_description
*desc
)
374 struct side_event_state
*event_state
= side_ptr_get(desc
->state
);
375 struct side_event_state_0
*es0
;
376 struct side_callback
*old_cb
;
379 if (side_unlikely(event_state
->version
!= 0))
381 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
382 nr_cb
= es0
->nr_callbacks
;
385 old_cb
= (struct side_callback
*) es0
->callbacks
;
386 (void) __atomic_add_fetch(&es0
->enabled
, -1, __ATOMIC_RELAXED
);
388 * Setting the state back to 0 cb and empty callbacks out of
389 * caution. This should not matter because instrumentation is
392 es0
->nr_callbacks
= 0;
393 side_rcu_assign_pointer(es0
->callbacks
, &side_empty_callback
);
395 * No need to wait for grace period because instrumentation is
402 * Unregister event handle. At this point, all side events in that
403 * handle should be unreachable.
405 void side_events_unregister(struct side_events_register_handle
*events_handle
)
407 struct side_tracer_handle
*tracer_handle
;
416 pthread_mutex_lock(&side_lock
);
417 side_list_remove_node(&events_handle
->node
);
418 side_list_for_each_entry(tracer_handle
, &side_tracer_list
, node
) {
419 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_REMOVE_EVENTS
,
420 events_handle
->events
, events_handle
->nr_events
,
421 tracer_handle
->priv
);
423 for (i
= 0; i
< events_handle
->nr_events
; i
++) {
424 struct side_event_description
*event
= events_handle
->events
[i
];
426 /* Skip NULL pointers */
429 side_event_remove_callbacks(event
);
431 pthread_mutex_unlock(&side_lock
);
432 //TODO: call event batch unregister ioctl
436 struct side_tracer_handle
*side_tracer_event_notification_register(
437 void (*cb
)(enum side_tracer_notification notif
,
438 struct side_event_description
**events
, uint32_t nr_events
, void *priv
),
441 struct side_tracer_handle
*tracer_handle
;
442 struct side_events_register_handle
*events_handle
;
448 tracer_handle
= (struct side_tracer_handle
*)
449 calloc(1, sizeof(struct side_tracer_handle
));
452 pthread_mutex_lock(&side_lock
);
453 tracer_handle
->cb
= cb
;
454 tracer_handle
->priv
= priv
;
455 side_list_insert_node_tail(&side_tracer_list
, &tracer_handle
->node
);
456 side_list_for_each_entry(events_handle
, &side_events_list
, node
) {
457 cb(SIDE_TRACER_NOTIFICATION_INSERT_EVENTS
,
458 events_handle
->events
, events_handle
->nr_events
, priv
);
460 pthread_mutex_unlock(&side_lock
);
461 return tracer_handle
;
464 void side_tracer_event_notification_unregister(struct side_tracer_handle
*tracer_handle
)
466 struct side_events_register_handle
*events_handle
;
472 pthread_mutex_lock(&side_lock
);
473 side_list_for_each_entry(events_handle
, &side_events_list
, node
) {
474 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_REMOVE_EVENTS
,
475 events_handle
->events
, events_handle
->nr_events
,
476 tracer_handle
->priv
);
478 side_list_remove_node(&tracer_handle
->node
);
479 pthread_mutex_unlock(&side_lock
);
487 side_rcu_gp_init(&rcu_gp
);
492 * side_exit() is executed from a library destructor. It can be called
493 * explicitly at application exit as well. Concurrent side API use is
494 * not expected at that point.
498 struct side_events_register_handle
*handle
, *tmp
;
502 side_list_for_each_entry_safe(handle
, tmp
, &side_events_list
, node
)
503 side_events_unregister(handle
);
504 side_rcu_gp_exit(&rcu_gp
);