1 // SPDX-License-Identifier: MIT
3 * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 #include <side/trace.h>
13 /* Top 8 bits reserved for kernel tracer use. */
14 #if SIDE_BITS_PER_LONG == 64
15 # define SIDE_EVENT_ENABLED_KERNEL_MASK 0xFF00000000000000ULL
16 # define SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK 0x8000000000000000ULL
18 /* Allow 2^56 tracer references on an event. */
19 # define SIDE_EVENT_ENABLED_USER_MASK 0x00FFFFFFFFFFFFFFULL
21 # define SIDE_EVENT_ENABLED_KERNEL_MASK 0xFF000000UL
22 # define SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK 0x80000000UL
24 /* Allow 2^24 tracer references on an event. */
25 # define SIDE_EVENT_ENABLED_USER_MASK 0x00FFFFFFUL
28 struct side_events_register_handle
{
29 struct side_list_node node
;
30 struct side_event_description
**events
;
34 struct side_tracer_handle
{
35 struct side_list_node node
;
36 void (*cb
)(enum side_tracer_notification notif
,
37 struct side_event_description
**events
, uint32_t nr_events
, void *priv
);
41 static struct side_rcu_gp_state rcu_gp
;
44 * Lazy initialization for early use within library constructors.
46 static bool initialized
;
48 * Do not register/unregister any more events after destructor.
50 static bool finalized
;
53 * Recursive mutex to allow tracer callbacks to use the side API.
55 static pthread_mutex_t side_lock
= PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
;
57 static DEFINE_SIDE_LIST_HEAD(side_events_list
);
58 static DEFINE_SIDE_LIST_HEAD(side_tracer_list
);
61 * The empty callback has a NULL function callback pointer, which stops
62 * iteration on the array of callbacks immediately.
64 const struct side_callback side_empty_callback
= { };
66 void side_call(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
)
68 struct side_rcu_read_state rcu_read_state
;
69 const struct side_event_state_0
*es0
;
70 const struct side_callback
*side_cb
;
73 if (side_unlikely(finalized
))
75 if (side_unlikely(!initialized
))
77 if (side_unlikely(event_state
->version
!= 0))
79 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
80 assert(!(es0
->desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
));
81 enabled
= __atomic_load_n(&es0
->enabled
, __ATOMIC_RELAXED
);
82 if (side_unlikely(enabled
& SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK
)) {
83 // TODO: call kernel write.
85 side_rcu_read_begin(&rcu_gp
, &rcu_read_state
);
86 for (side_cb
= side_rcu_dereference(es0
->callbacks
); side_cb
->u
.call
!= NULL
; side_cb
++)
87 side_cb
->u
.call(es0
->desc
, side_arg_vec
, side_cb
->priv
);
88 side_rcu_read_end(&rcu_gp
, &rcu_read_state
);
91 void side_call_variadic(const struct side_event_state
*event_state
,
92 const struct side_arg_vec
*side_arg_vec
,
93 const struct side_arg_dynamic_struct
*var_struct
)
95 struct side_rcu_read_state rcu_read_state
;
96 const struct side_event_state_0
*es0
;
97 const struct side_callback
*side_cb
;
100 if (side_unlikely(finalized
))
102 if (side_unlikely(!initialized
))
104 if (side_unlikely(event_state
->version
!= 0))
106 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
107 assert(es0
->desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
);
108 enabled
= __atomic_load_n(&es0
->enabled
, __ATOMIC_RELAXED
);
109 if (side_unlikely(enabled
& SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK
)) {
110 // TODO: call kernel write.
112 side_rcu_read_begin(&rcu_gp
, &rcu_read_state
);
113 for (side_cb
= side_rcu_dereference(es0
->callbacks
); side_cb
->u
.call_variadic
!= NULL
; side_cb
++)
114 side_cb
->u
.call_variadic(es0
->desc
, side_arg_vec
, var_struct
, side_cb
->priv
);
115 side_rcu_read_end(&rcu_gp
, &rcu_read_state
);
119 const struct side_callback
*side_tracer_callback_lookup(
120 const struct side_event_description
*desc
,
121 void *call
, void *priv
)
123 struct side_event_state
*event_state
= side_ptr_get(desc
->state
);
124 const struct side_event_state_0
*es0
;
125 const struct side_callback
*cb
;
127 if (side_unlikely(event_state
->version
!= 0))
129 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
130 for (cb
= es0
->callbacks
; cb
->u
.call
!= NULL
; cb
++) {
131 if ((void *) cb
->u
.call
== call
&& cb
->priv
== priv
)
138 int _side_tracer_callback_register(struct side_event_description
*desc
,
139 void *call
, void *priv
)
141 struct side_event_state
*event_state
;
142 struct side_callback
*old_cb
, *new_cb
;
143 struct side_event_state_0
*es0
;
144 int ret
= SIDE_ERROR_OK
;
148 return SIDE_ERROR_INVAL
;
150 return SIDE_ERROR_EXITING
;
153 pthread_mutex_lock(&side_lock
);
154 event_state
= side_ptr_get(desc
->state
);
155 if (side_unlikely(event_state
->version
!= 0))
157 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
158 old_nr_cb
= es0
->nr_callbacks
;
159 if (old_nr_cb
== UINT32_MAX
) {
160 ret
= SIDE_ERROR_INVAL
;
163 /* Reject duplicate (call, priv) tuples. */
164 if (side_tracer_callback_lookup(desc
, call
, priv
)) {
165 ret
= SIDE_ERROR_EXIST
;
168 old_cb
= (struct side_callback
*) es0
->callbacks
;
169 /* old_nr_cb + 1 (new cb) + 1 (NULL) */
170 new_cb
= (struct side_callback
*) calloc(old_nr_cb
+ 2, sizeof(struct side_callback
));
172 ret
= SIDE_ERROR_NOMEM
;
175 memcpy(new_cb
, old_cb
, old_nr_cb
);
176 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
177 new_cb
[old_nr_cb
].u
.call_variadic
=
178 (side_tracer_callback_variadic_func
) call
;
180 new_cb
[old_nr_cb
].u
.call
=
181 (side_tracer_callback_func
) call
;
182 new_cb
[old_nr_cb
].priv
= priv
;
183 /* High order bits are already zeroed. */
184 side_rcu_assign_pointer(es0
->callbacks
, new_cb
);
185 side_rcu_wait_grace_period(&rcu_gp
);
189 /* Increment concurrently with kernel setting the top bits. */
191 (void) __atomic_add_fetch(&es0
->enabled
, 1, __ATOMIC_RELAXED
);
193 pthread_mutex_unlock(&side_lock
);
197 int side_tracer_callback_register(struct side_event_description
*desc
,
198 side_tracer_callback_func call
,
201 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
202 return SIDE_ERROR_INVAL
;
203 return _side_tracer_callback_register(desc
, (void *) call
, priv
);
206 int side_tracer_callback_variadic_register(struct side_event_description
*desc
,
207 side_tracer_callback_variadic_func call_variadic
,
210 if (!(desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
))
211 return SIDE_ERROR_INVAL
;
212 return _side_tracer_callback_register(desc
, (void *) call_variadic
, priv
);
215 static int _side_tracer_callback_unregister(struct side_event_description
*desc
,
216 void *call
, void *priv
)
218 struct side_event_state
*event_state
;
219 struct side_callback
*old_cb
, *new_cb
;
220 const struct side_callback
*cb_pos
;
221 struct side_event_state_0
*es0
;
223 int ret
= SIDE_ERROR_OK
;
227 return SIDE_ERROR_INVAL
;
229 return SIDE_ERROR_EXITING
;
232 pthread_mutex_lock(&side_lock
);
233 event_state
= side_ptr_get(desc
->state
);
234 if (side_unlikely(event_state
->version
!= 0))
236 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
237 cb_pos
= side_tracer_callback_lookup(desc
, call
, priv
);
239 ret
= SIDE_ERROR_NOENT
;
242 old_nr_cb
= es0
->nr_callbacks
;
243 old_cb
= (struct side_callback
*) es0
->callbacks
;
244 if (old_nr_cb
== 1) {
245 new_cb
= (struct side_callback
*) &side_empty_callback
;
247 pos_idx
= cb_pos
- es0
->callbacks
;
248 /* Remove entry at pos_idx. */
249 /* old_nr_cb - 1 (removed cb) + 1 (NULL) */
250 new_cb
= (struct side_callback
*) calloc(old_nr_cb
, sizeof(struct side_callback
));
252 ret
= SIDE_ERROR_NOMEM
;
255 memcpy(new_cb
, old_cb
, pos_idx
);
256 memcpy(&new_cb
[pos_idx
], &old_cb
[pos_idx
+ 1], old_nr_cb
- pos_idx
- 1);
258 /* High order bits are already zeroed. */
259 side_rcu_assign_pointer(es0
->callbacks
, new_cb
);
260 side_rcu_wait_grace_period(&rcu_gp
);
263 /* Decrement concurrently with kernel setting the top bits. */
265 (void) __atomic_add_fetch(&es0
->enabled
, -1, __ATOMIC_RELAXED
);
267 pthread_mutex_unlock(&side_lock
);
271 int side_tracer_callback_unregister(struct side_event_description
*desc
,
272 side_tracer_callback_func call
,
275 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
276 return SIDE_ERROR_INVAL
;
277 return _side_tracer_callback_unregister(desc
, (void *) call
, priv
);
280 int side_tracer_callback_variadic_unregister(struct side_event_description
*desc
,
281 side_tracer_callback_variadic_func call_variadic
,
284 if (!(desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
))
285 return SIDE_ERROR_INVAL
;
286 return _side_tracer_callback_unregister(desc
, (void *) call_variadic
, priv
);
289 struct side_events_register_handle
*side_events_register(struct side_event_description
**events
, uint32_t nr_events
)
291 struct side_events_register_handle
*events_handle
= NULL
;
292 struct side_tracer_handle
*tracer_handle
;
298 events_handle
= (struct side_events_register_handle
*)
299 calloc(1, sizeof(struct side_events_register_handle
));
302 events_handle
->events
= events
;
303 events_handle
->nr_events
= nr_events
;
305 pthread_mutex_lock(&side_lock
);
306 side_list_insert_node_tail(&side_events_list
, &events_handle
->node
);
307 side_list_for_each_entry(tracer_handle
, &side_tracer_list
, node
) {
308 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_INSERT_EVENTS
,
309 events
, nr_events
, tracer_handle
->priv
);
311 pthread_mutex_unlock(&side_lock
);
312 //TODO: call event batch register ioctl
313 return events_handle
;
317 void side_event_remove_callbacks(struct side_event_description
*desc
)
319 struct side_event_state
*event_state
= side_ptr_get(desc
->state
);
320 struct side_event_state_0
*es0
;
321 struct side_callback
*old_cb
;
324 if (side_unlikely(event_state
->version
!= 0))
326 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
327 nr_cb
= es0
->nr_callbacks
;
330 old_cb
= (struct side_callback
*) es0
->callbacks
;
331 (void) __atomic_add_fetch(&es0
->enabled
, -1, __ATOMIC_RELAXED
);
333 * Setting the state back to 0 cb and empty callbacks out of
334 * caution. This should not matter because instrumentation is
337 es0
->nr_callbacks
= 0;
338 side_rcu_assign_pointer(es0
->callbacks
, &side_empty_callback
);
340 * No need to wait for grace period because instrumentation is
347 * Unregister event handle. At this point, all side events in that
348 * handle should be unreachable.
350 void side_events_unregister(struct side_events_register_handle
*events_handle
)
352 struct side_tracer_handle
*tracer_handle
;
361 pthread_mutex_lock(&side_lock
);
362 side_list_remove_node(&events_handle
->node
);
363 side_list_for_each_entry(tracer_handle
, &side_tracer_list
, node
) {
364 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_REMOVE_EVENTS
,
365 events_handle
->events
, events_handle
->nr_events
,
366 tracer_handle
->priv
);
368 for (i
= 0; i
< events_handle
->nr_events
; i
++) {
369 struct side_event_description
*event
= events_handle
->events
[i
];
371 /* Skip NULL pointers */
374 side_event_remove_callbacks(event
);
376 pthread_mutex_unlock(&side_lock
);
377 //TODO: call event batch unregister ioctl
381 struct side_tracer_handle
*side_tracer_event_notification_register(
382 void (*cb
)(enum side_tracer_notification notif
,
383 struct side_event_description
**events
, uint32_t nr_events
, void *priv
),
386 struct side_tracer_handle
*tracer_handle
;
387 struct side_events_register_handle
*events_handle
;
393 tracer_handle
= (struct side_tracer_handle
*)
394 calloc(1, sizeof(struct side_tracer_handle
));
397 pthread_mutex_lock(&side_lock
);
398 tracer_handle
->cb
= cb
;
399 tracer_handle
->priv
= priv
;
400 side_list_insert_node_tail(&side_tracer_list
, &tracer_handle
->node
);
401 side_list_for_each_entry(events_handle
, &side_events_list
, node
) {
402 cb(SIDE_TRACER_NOTIFICATION_INSERT_EVENTS
,
403 events_handle
->events
, events_handle
->nr_events
, priv
);
405 pthread_mutex_unlock(&side_lock
);
406 return tracer_handle
;
409 void side_tracer_event_notification_unregister(struct side_tracer_handle
*tracer_handle
)
411 struct side_events_register_handle
*events_handle
;
417 pthread_mutex_lock(&side_lock
);
418 side_list_for_each_entry(events_handle
, &side_events_list
, node
) {
419 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_REMOVE_EVENTS
,
420 events_handle
->events
, events_handle
->nr_events
,
421 tracer_handle
->priv
);
423 side_list_remove_node(&tracer_handle
->node
);
424 pthread_mutex_unlock(&side_lock
);
432 side_rcu_gp_init(&rcu_gp
);
437 * side_exit() is executed from a library destructor. It can be called
438 * explicitly at application exit as well. Concurrent side API use is
439 * not expected at that point.
443 struct side_events_register_handle
*handle
, *tmp
;
447 side_list_for_each_entry_safe(handle
, tmp
, &side_events_list
, node
)
448 side_events_unregister(handle
);
449 side_rcu_gp_exit(&rcu_gp
);