1 // SPDX-License-Identifier: MIT
3 * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 #include <side/trace.h>
18 /* Top 8 bits reserved for shared tracer use. */
19 #if SIDE_BITS_PER_LONG == 64
20 # define SIDE_EVENT_ENABLED_SHARED_MASK 0xFF00000000000000ULL
21 # define SIDE_EVENT_ENABLED_SHARED_USER_EVENT_MASK 0x8000000000000000ULL
22 # define SIDE_EVENT_ENABLED_SHARED_PTRACE_MASK 0x4000000000000000ULL
24 /* Allow 2^56 private tracer references on an event. */
25 # define SIDE_EVENT_ENABLED_PRIVATE_MASK 0x00FFFFFFFFFFFFFFULL
27 # define SIDE_EVENT_ENABLED_SHARED_MASK 0xFF000000UL
28 # define SIDE_EVENT_ENABLED_SHARED_USER_EVENT_MASK 0x80000000UL
29 # define SIDE_EVENT_ENABLED_SHARED_PTRACE_MASK 0x40000000UL
31 /* Allow 2^24 private tracer references on an event. */
32 # define SIDE_EVENT_ENABLED_PRIVATE_MASK 0x00FFFFFFUL
35 #define SIDE_KEY_RESERVED_RANGE_END 0x8
37 /* Key 0x0 is reserved to match all. */
38 #define SIDE_KEY_MATCH_ALL 0x0
39 /* Key 0x1 is reserved for user event. */
40 #define SIDE_KEY_USER_EVENT 0x1
41 /* Key 0x2 is reserved for ptrace. */
42 #define SIDE_KEY_PTRACE 0x2
44 #define SIDE_RETRY_BUSY_LOOP_ATTEMPTS 100
45 #define SIDE_RETRY_DELAY_MS 1
47 struct side_events_register_handle
{
48 struct side_list_node node
;
49 struct side_event_description
**events
;
53 struct side_tracer_handle
{
54 struct side_list_node node
;
55 void (*cb
)(enum side_tracer_notification notif
,
56 struct side_event_description
**events
, uint32_t nr_events
, void *priv
);
60 struct side_statedump_notification
{
61 struct side_list_node node
;
65 struct side_statedump_request_handle
{
66 struct side_list_node node
; /* Statedump request RCU list node. */
67 struct side_list_head notification_queue
; /* Queue of struct side_statedump_notification */
68 void (*cb
)(void *statedump_request_key
);
70 enum side_statedump_mode mode
;
73 struct side_callback
{
75 void (*call
)(const struct side_event_description
*desc
,
76 const struct side_arg_vec
*side_arg_vec
,
77 void *priv
, void *caller_addr
);
78 void (*call_variadic
)(const struct side_event_description
*desc
,
79 const struct side_arg_vec
*side_arg_vec
,
80 const struct side_arg_dynamic_struct
*var_struct
,
81 void *priv
, void *caller_addr
);
87 enum agent_thread_state
{
88 AGENT_THREAD_STATE_BLOCKED
= 0,
89 AGENT_THREAD_STATE_HANDLE_REQUEST
= (1 << 0),
90 AGENT_THREAD_STATE_EXIT
= (1 << 1),
91 AGENT_THREAD_STATE_PAUSE
= (1 << 2),
92 AGENT_THREAD_STATE_PAUSE_ACK
= (1 << 3),
95 struct statedump_agent_thread
{
98 enum agent_thread_state state
;
99 pthread_cond_t worker_cond
;
100 pthread_cond_t waiter_cond
;
103 static struct side_rcu_gp_state event_rcu_gp
, statedump_rcu_gp
;
106 * Lazy initialization for early use within library constructors.
108 static bool initialized
;
110 * Do not register/unregister any more events after destructor.
112 static bool finalized
;
115 * Recursive mutex to allow tracer callbacks to use the side API.
117 static pthread_mutex_t side_event_lock
= PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
;
118 static pthread_mutex_t side_statedump_lock
= PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
;
119 static pthread_mutex_t side_key_lock
= PTHREAD_MUTEX_INITIALIZER
;
121 * The side_agent_thread_lock protects the life-time of the agent
122 * thread: reference counting, creation, join. It is not taken by
123 * the agent thread per se so it does not have a circular dependency
125 * The side_statedump_lock nests inside the side_agent_thread_lock.
127 static pthread_mutex_t side_agent_thread_lock
= PTHREAD_MUTEX_INITIALIZER
;
129 /* Dynamic tracer key allocation. */
130 static uint64_t side_key_next
= SIDE_KEY_RESERVED_RANGE_END
;
132 static struct statedump_agent_thread statedump_agent_thread
;
134 static DEFINE_SIDE_LIST_HEAD(side_events_list
);
135 static DEFINE_SIDE_LIST_HEAD(side_tracer_list
);
138 * The statedump request list is a RCU list to allow the agent thread to
139 * iterate over this list with a RCU read-side lock.
141 static DEFINE_SIDE_LIST_HEAD(side_statedump_list
);
144 * The empty callback has a NULL function callback pointer, which stops
145 * iteration on the array of callbacks immediately.
147 const char side_empty_callback
[sizeof(struct side_callback
)];
149 side_static_event(side_statedump_begin
, "side", "statedump_begin",
150 SIDE_LOGLEVEL_INFO
, side_field_list(side_field_string("name")));
151 side_static_event(side_statedump_end
, "side", "statedump_end",
152 SIDE_LOGLEVEL_INFO
, side_field_list(side_field_string("name")));
155 * side_ptrace_hook is a place holder for a debugger breakpoint.
156 * var_struct is NULL if not variadic.
158 void side_ptrace_hook(const struct side_event_state
*event_state
__attribute__((unused
)),
159 const struct side_arg_vec
*side_arg_vec
__attribute__((unused
)),
160 const struct side_arg_dynamic_struct
*var_struct
__attribute__((unused
)),
161 void *caller_addr
__attribute__((unused
)))
162 __attribute__((noinline
));
163 void side_ptrace_hook(const struct side_event_state
*event_state
__attribute__((unused
)),
164 const struct side_arg_vec
*side_arg_vec
__attribute__((unused
)),
165 const struct side_arg_dynamic_struct
*var_struct
__attribute__((unused
)),
166 void *caller_addr
__attribute__((unused
)))
170 static inline __attribute__((always_inline
))
171 void _side_call(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
, uint64_t key
)
173 void *caller_addr
= __builtin_return_address(0);
174 struct side_rcu_read_state rcu_read_state
;
175 const struct side_event_state_0
*es0
;
176 const struct side_callback
*side_cb
;
179 if (side_unlikely(finalized
))
181 if (side_unlikely(!initialized
))
183 if (side_unlikely(event_state
->version
!= 0))
185 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
186 assert(!(es0
->desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
));
187 enabled
= __atomic_load_n(&es0
->enabled
, __ATOMIC_RELAXED
);
188 if (side_unlikely(enabled
& SIDE_EVENT_ENABLED_SHARED_MASK
)) {
189 if ((enabled
& SIDE_EVENT_ENABLED_SHARED_USER_EVENT_MASK
) &&
190 (key
== SIDE_KEY_MATCH_ALL
|| key
== SIDE_KEY_USER_EVENT
)) {
191 // TODO: User event integration: call kernel write.
193 if ((enabled
& SIDE_EVENT_ENABLED_SHARED_PTRACE_MASK
) &&
194 (key
== SIDE_KEY_MATCH_ALL
|| key
== SIDE_KEY_PTRACE
))
195 side_ptrace_hook(event_state
, side_arg_vec
, NULL
, caller_addr
);
197 side_rcu_read_begin(&event_rcu_gp
, &rcu_read_state
);
198 for (side_cb
= side_rcu_dereference(es0
->callbacks
); side_cb
->u
.call
!= NULL
; side_cb
++) {
199 if (key
!= SIDE_KEY_MATCH_ALL
&& side_cb
->key
!= SIDE_KEY_MATCH_ALL
&& side_cb
->key
!= key
)
201 side_cb
->u
.call(es0
->desc
, side_arg_vec
, side_cb
->priv
, caller_addr
);
203 side_rcu_read_end(&event_rcu_gp
, &rcu_read_state
);
206 void side_call(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
)
208 _side_call(event_state
, side_arg_vec
, SIDE_KEY_MATCH_ALL
);
211 void side_statedump_call(const struct side_event_state
*event_state
,
212 const struct side_arg_vec
*side_arg_vec
,
213 void *statedump_request_key
)
215 _side_call(event_state
, side_arg_vec
, *(const uint64_t *) statedump_request_key
);
218 static inline __attribute__((always_inline
))
219 void _side_call_variadic(const struct side_event_state
*event_state
,
220 const struct side_arg_vec
*side_arg_vec
,
221 const struct side_arg_dynamic_struct
*var_struct
,
224 void *caller_addr
= __builtin_return_address(0);
225 struct side_rcu_read_state rcu_read_state
;
226 const struct side_event_state_0
*es0
;
227 const struct side_callback
*side_cb
;
230 if (side_unlikely(finalized
))
232 if (side_unlikely(!initialized
))
234 if (side_unlikely(event_state
->version
!= 0))
236 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
237 assert(es0
->desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
);
238 enabled
= __atomic_load_n(&es0
->enabled
, __ATOMIC_RELAXED
);
239 if (side_unlikely(enabled
& SIDE_EVENT_ENABLED_SHARED_MASK
)) {
240 if ((enabled
& SIDE_EVENT_ENABLED_SHARED_USER_EVENT_MASK
) &&
241 (key
== SIDE_KEY_MATCH_ALL
|| key
== SIDE_KEY_USER_EVENT
)) {
242 // TODO: User event integration: call kernel write.
244 if ((enabled
& SIDE_EVENT_ENABLED_SHARED_PTRACE_MASK
) &&
245 (key
== SIDE_KEY_MATCH_ALL
|| key
== SIDE_KEY_PTRACE
))
246 side_ptrace_hook(event_state
, side_arg_vec
, var_struct
, caller_addr
);
248 side_rcu_read_begin(&event_rcu_gp
, &rcu_read_state
);
249 for (side_cb
= side_rcu_dereference(es0
->callbacks
); side_cb
->u
.call_variadic
!= NULL
; side_cb
++) {
250 if (key
!= SIDE_KEY_MATCH_ALL
&& side_cb
->key
!= SIDE_KEY_MATCH_ALL
&& side_cb
->key
!= key
)
252 side_cb
->u
.call_variadic(es0
->desc
, side_arg_vec
, var_struct
, side_cb
->priv
, caller_addr
);
254 side_rcu_read_end(&event_rcu_gp
, &rcu_read_state
);
257 void side_call_variadic(const struct side_event_state
*event_state
,
258 const struct side_arg_vec
*side_arg_vec
,
259 const struct side_arg_dynamic_struct
*var_struct
)
261 _side_call_variadic(event_state
, side_arg_vec
, var_struct
, SIDE_KEY_MATCH_ALL
);
264 void side_statedump_call_variadic(const struct side_event_state
*event_state
,
265 const struct side_arg_vec
*side_arg_vec
,
266 const struct side_arg_dynamic_struct
*var_struct
,
267 void *statedump_request_key
)
269 _side_call_variadic(event_state
, side_arg_vec
, var_struct
, *(const uint64_t *) statedump_request_key
);
273 const struct side_callback
*side_tracer_callback_lookup(
274 const struct side_event_description
*desc
,
275 void *call
, void *priv
, uint64_t key
)
277 struct side_event_state
*event_state
= side_ptr_get(desc
->state
);
278 const struct side_event_state_0
*es0
;
279 const struct side_callback
*cb
;
281 if (side_unlikely(event_state
->version
!= 0))
283 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
284 for (cb
= es0
->callbacks
; cb
->u
.call
!= NULL
; cb
++) {
285 if ((void *) cb
->u
.call
== call
&& cb
->priv
== priv
&& cb
->key
== key
)
292 int _side_tracer_callback_register(struct side_event_description
*desc
,
293 void *call
, void *priv
, uint64_t key
)
295 struct side_event_state
*event_state
;
296 struct side_callback
*old_cb
, *new_cb
;
297 struct side_event_state_0
*es0
;
298 int ret
= SIDE_ERROR_OK
;
302 return SIDE_ERROR_INVAL
;
304 return SIDE_ERROR_EXITING
;
307 pthread_mutex_lock(&side_event_lock
);
308 event_state
= side_ptr_get(desc
->state
);
309 if (side_unlikely(event_state
->version
!= 0))
311 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
312 old_nr_cb
= es0
->nr_callbacks
;
313 if (old_nr_cb
== UINT32_MAX
) {
314 ret
= SIDE_ERROR_INVAL
;
317 /* Reject duplicate (call, priv) tuples. */
318 if (side_tracer_callback_lookup(desc
, call
, priv
, key
)) {
319 ret
= SIDE_ERROR_EXIST
;
322 old_cb
= (struct side_callback
*) es0
->callbacks
;
323 /* old_nr_cb + 1 (new cb) + 1 (NULL) */
324 new_cb
= (struct side_callback
*) calloc(old_nr_cb
+ 2, sizeof(struct side_callback
));
326 ret
= SIDE_ERROR_NOMEM
;
329 memcpy(new_cb
, old_cb
, old_nr_cb
);
330 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
331 new_cb
[old_nr_cb
].u
.call_variadic
=
332 (side_tracer_callback_variadic_func
) call
;
334 new_cb
[old_nr_cb
].u
.call
=
335 (side_tracer_callback_func
) call
;
336 new_cb
[old_nr_cb
].priv
= priv
;
337 new_cb
[old_nr_cb
].key
= key
;
338 /* High order bits are already zeroed. */
339 side_rcu_assign_pointer(es0
->callbacks
, new_cb
);
340 side_rcu_wait_grace_period(&event_rcu_gp
);
344 /* Increment concurrently with kernel setting the top bits. */
346 (void) __atomic_add_fetch(&es0
->enabled
, 1, __ATOMIC_RELAXED
);
348 pthread_mutex_unlock(&side_event_lock
);
352 int side_tracer_callback_register(struct side_event_description
*desc
,
353 side_tracer_callback_func call
,
354 void *priv
, uint64_t key
)
356 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
357 return SIDE_ERROR_INVAL
;
358 return _side_tracer_callback_register(desc
, (void *) call
, priv
, key
);
361 int side_tracer_callback_variadic_register(struct side_event_description
*desc
,
362 side_tracer_callback_variadic_func call_variadic
,
363 void *priv
, uint64_t key
)
365 if (!(desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
))
366 return SIDE_ERROR_INVAL
;
367 return _side_tracer_callback_register(desc
, (void *) call_variadic
, priv
, key
);
370 static int _side_tracer_callback_unregister(struct side_event_description
*desc
,
371 void *call
, void *priv
, uint64_t key
)
373 struct side_event_state
*event_state
;
374 struct side_callback
*old_cb
, *new_cb
;
375 const struct side_callback
*cb_pos
;
376 struct side_event_state_0
*es0
;
378 int ret
= SIDE_ERROR_OK
;
382 return SIDE_ERROR_INVAL
;
384 return SIDE_ERROR_EXITING
;
387 pthread_mutex_lock(&side_event_lock
);
388 event_state
= side_ptr_get(desc
->state
);
389 if (side_unlikely(event_state
->version
!= 0))
391 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
392 cb_pos
= side_tracer_callback_lookup(desc
, call
, priv
, key
);
394 ret
= SIDE_ERROR_NOENT
;
397 old_nr_cb
= es0
->nr_callbacks
;
398 old_cb
= (struct side_callback
*) es0
->callbacks
;
399 if (old_nr_cb
== 1) {
400 new_cb
= (struct side_callback
*) &side_empty_callback
;
402 pos_idx
= cb_pos
- es0
->callbacks
;
403 /* Remove entry at pos_idx. */
404 /* old_nr_cb - 1 (removed cb) + 1 (NULL) */
405 new_cb
= (struct side_callback
*) calloc(old_nr_cb
, sizeof(struct side_callback
));
407 ret
= SIDE_ERROR_NOMEM
;
410 memcpy(new_cb
, old_cb
, pos_idx
);
411 memcpy(&new_cb
[pos_idx
], &old_cb
[pos_idx
+ 1], old_nr_cb
- pos_idx
- 1);
413 /* High order bits are already zeroed. */
414 side_rcu_assign_pointer(es0
->callbacks
, new_cb
);
415 side_rcu_wait_grace_period(&event_rcu_gp
);
418 /* Decrement concurrently with kernel setting the top bits. */
420 (void) __atomic_add_fetch(&es0
->enabled
, -1, __ATOMIC_RELAXED
);
422 pthread_mutex_unlock(&side_event_lock
);
426 int side_tracer_callback_unregister(struct side_event_description
*desc
,
427 side_tracer_callback_func call
,
428 void *priv
, uint64_t key
)
430 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
431 return SIDE_ERROR_INVAL
;
432 return _side_tracer_callback_unregister(desc
, (void *) call
, priv
, key
);
435 int side_tracer_callback_variadic_unregister(struct side_event_description
*desc
,
436 side_tracer_callback_variadic_func call_variadic
,
437 void *priv
, uint64_t key
)
439 if (!(desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
))
440 return SIDE_ERROR_INVAL
;
441 return _side_tracer_callback_unregister(desc
, (void *) call_variadic
, priv
, key
);
444 struct side_events_register_handle
*side_events_register(struct side_event_description
**events
, uint32_t nr_events
)
446 struct side_events_register_handle
*events_handle
= NULL
;
447 struct side_tracer_handle
*tracer_handle
;
453 events_handle
= (struct side_events_register_handle
*)
454 calloc(1, sizeof(struct side_events_register_handle
));
457 events_handle
->events
= events
;
458 events_handle
->nr_events
= nr_events
;
460 pthread_mutex_lock(&side_event_lock
);
461 side_list_insert_node_tail(&side_events_list
, &events_handle
->node
);
462 side_list_for_each_entry(tracer_handle
, &side_tracer_list
, node
) {
463 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_INSERT_EVENTS
,
464 events
, nr_events
, tracer_handle
->priv
);
466 pthread_mutex_unlock(&side_event_lock
);
467 //TODO: User event integration: call event batch register ioctl
468 return events_handle
;
472 void side_event_remove_callbacks(struct side_event_description
*desc
)
474 struct side_event_state
*event_state
= side_ptr_get(desc
->state
);
475 struct side_event_state_0
*es0
;
476 struct side_callback
*old_cb
;
479 if (side_unlikely(event_state
->version
!= 0))
481 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
482 nr_cb
= es0
->nr_callbacks
;
485 old_cb
= (struct side_callback
*) es0
->callbacks
;
486 (void) __atomic_add_fetch(&es0
->enabled
, -1, __ATOMIC_RELAXED
);
488 * Setting the state back to 0 cb and empty callbacks out of
489 * caution. This should not matter because instrumentation is
492 es0
->nr_callbacks
= 0;
493 side_rcu_assign_pointer(es0
->callbacks
, &side_empty_callback
);
495 * No need to wait for grace period because instrumentation is
502 * Unregister event handle. At this point, all side events in that
503 * handle should be unreachable.
505 void side_events_unregister(struct side_events_register_handle
*events_handle
)
507 struct side_tracer_handle
*tracer_handle
;
516 pthread_mutex_lock(&side_event_lock
);
517 side_list_remove_node(&events_handle
->node
);
518 side_list_for_each_entry(tracer_handle
, &side_tracer_list
, node
) {
519 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_REMOVE_EVENTS
,
520 events_handle
->events
, events_handle
->nr_events
,
521 tracer_handle
->priv
);
523 for (i
= 0; i
< events_handle
->nr_events
; i
++) {
524 struct side_event_description
*event
= events_handle
->events
[i
];
526 /* Skip NULL pointers */
529 side_event_remove_callbacks(event
);
531 pthread_mutex_unlock(&side_event_lock
);
532 //TODO: User event integration: call event batch unregister ioctl
536 struct side_tracer_handle
*side_tracer_event_notification_register(
537 void (*cb
)(enum side_tracer_notification notif
,
538 struct side_event_description
**events
, uint32_t nr_events
, void *priv
),
541 struct side_tracer_handle
*tracer_handle
;
542 struct side_events_register_handle
*events_handle
;
548 tracer_handle
= (struct side_tracer_handle
*)
549 calloc(1, sizeof(struct side_tracer_handle
));
552 pthread_mutex_lock(&side_event_lock
);
553 tracer_handle
->cb
= cb
;
554 tracer_handle
->priv
= priv
;
555 side_list_insert_node_tail(&side_tracer_list
, &tracer_handle
->node
);
556 side_list_for_each_entry(events_handle
, &side_events_list
, node
) {
557 cb(SIDE_TRACER_NOTIFICATION_INSERT_EVENTS
,
558 events_handle
->events
, events_handle
->nr_events
, priv
);
560 pthread_mutex_unlock(&side_event_lock
);
561 return tracer_handle
;
564 void side_tracer_event_notification_unregister(struct side_tracer_handle
*tracer_handle
)
566 struct side_events_register_handle
*events_handle
;
572 pthread_mutex_lock(&side_event_lock
);
573 side_list_for_each_entry(events_handle
, &side_events_list
, node
) {
574 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_REMOVE_EVENTS
,
575 events_handle
->events
, events_handle
->nr_events
,
576 tracer_handle
->priv
);
578 side_list_remove_node(&tracer_handle
->node
);
579 pthread_mutex_unlock(&side_event_lock
);
583 /* Called with side_statedump_lock held. */
585 void queue_statedump_pending(struct side_statedump_request_handle
*handle
, uint64_t key
)
587 struct side_statedump_notification
*notif
;
589 notif
= (struct side_statedump_notification
*) calloc(1, sizeof(struct side_statedump_notification
));
593 side_list_insert_node_tail(&handle
->notification_queue
, ¬if
->node
);
594 if (handle
->mode
== SIDE_STATEDUMP_MODE_AGENT_THREAD
) {
595 (void)__atomic_or_fetch(&statedump_agent_thread
.state
, AGENT_THREAD_STATE_HANDLE_REQUEST
, __ATOMIC_SEQ_CST
);
596 pthread_cond_broadcast(&statedump_agent_thread
.worker_cond
);
600 /* Called with side_statedump_lock held. */
602 void unqueue_statedump_pending(struct side_statedump_request_handle
*handle
, uint64_t key
)
604 struct side_statedump_notification
*notif
, *tmp
;
606 side_list_for_each_entry_safe(notif
, tmp
, &handle
->notification_queue
, node
) {
607 if (key
== SIDE_KEY_MATCH_ALL
|| key
== notif
->key
) {
608 side_list_remove_node(¬if
->node
);
615 void side_statedump_run(struct side_statedump_request_handle
*handle
,
616 struct side_statedump_notification
*notif
)
618 side_statedump_event_call(side_statedump_begin
, ¬if
->key
,
619 side_arg_list(side_arg_string(handle
->name
)));
620 /* Invoke the state dump callback specifically for the tracer key. */
621 handle
->cb(¬if
->key
);
622 side_statedump_event_call(side_statedump_end
, ¬if
->key
,
623 side_arg_list(side_arg_string(handle
->name
)));
627 void _side_statedump_run_pending_requests(struct side_statedump_request_handle
*handle
)
629 struct side_statedump_notification
*notif
, *tmp
;
630 DEFINE_SIDE_LIST_HEAD(tmp_head
);
632 pthread_mutex_lock(&side_statedump_lock
);
633 side_list_splice(&handle
->notification_queue
, &tmp_head
);
634 side_list_head_init(&handle
->notification_queue
);
635 pthread_mutex_unlock(&side_statedump_lock
);
637 /* We are now sole owner of the tmp_head list. */
638 side_list_for_each_entry(notif
, &tmp_head
, node
)
639 side_statedump_run(handle
, notif
);
640 side_list_for_each_entry_safe(notif
, tmp
, &tmp_head
, node
)
643 if (handle
->mode
== SIDE_STATEDUMP_MODE_AGENT_THREAD
) {
644 pthread_mutex_lock(&side_statedump_lock
);
645 pthread_cond_broadcast(&statedump_agent_thread
.waiter_cond
);
646 pthread_mutex_unlock(&side_statedump_lock
);
651 void *statedump_agent_func(void *arg
__attribute__((unused
)))
654 struct side_statedump_request_handle
*handle
;
655 struct side_rcu_read_state rcu_read_state
;
656 enum agent_thread_state state
;
658 pthread_mutex_lock(&side_statedump_lock
);
660 state
= __atomic_load_n(&statedump_agent_thread
.state
, __ATOMIC_SEQ_CST
);
661 if (state
== AGENT_THREAD_STATE_BLOCKED
)
662 pthread_cond_wait(&statedump_agent_thread
.worker_cond
, &side_statedump_lock
);
666 pthread_mutex_unlock(&side_statedump_lock
);
667 if (state
& AGENT_THREAD_STATE_EXIT
)
669 if (state
& AGENT_THREAD_STATE_PAUSE
) {
672 (void)__atomic_or_fetch(&statedump_agent_thread
.state
, AGENT_THREAD_STATE_PAUSE_ACK
, __ATOMIC_SEQ_CST
);
674 state
= __atomic_load_n(&statedump_agent_thread
.state
, __ATOMIC_SEQ_CST
);
675 if (!(state
& AGENT_THREAD_STATE_PAUSE
))
677 if (attempt
> SIDE_RETRY_BUSY_LOOP_ATTEMPTS
) {
678 (void)poll(NULL
, 0, SIDE_RETRY_DELAY_MS
);
686 (void)__atomic_and_fetch(&statedump_agent_thread
.state
, ~AGENT_THREAD_STATE_HANDLE_REQUEST
, __ATOMIC_SEQ_CST
);
687 side_rcu_read_begin(&statedump_rcu_gp
, &rcu_read_state
);
688 side_list_for_each_entry_rcu(handle
, &side_statedump_list
, node
)
689 _side_statedump_run_pending_requests(handle
);
690 side_rcu_read_end(&statedump_rcu_gp
, &rcu_read_state
);
696 void statedump_agent_thread_init(void)
698 pthread_cond_init(&statedump_agent_thread
.worker_cond
, NULL
);
699 pthread_cond_init(&statedump_agent_thread
.waiter_cond
, NULL
);
700 statedump_agent_thread
.state
= AGENT_THREAD_STATE_BLOCKED
;
703 /* Called with side_agent_thread_lock and side_statedump_lock held. */
705 void statedump_agent_thread_get(void)
709 if (statedump_agent_thread
.ref
++)
711 statedump_agent_thread_init();
712 ret
= pthread_create(&statedump_agent_thread
.id
, NULL
,
713 statedump_agent_func
, NULL
);
720 * Called with side_agent_thread_lock and side_statedump_lock held.
721 * Returns true if join for agent thread is needed.
724 bool statedump_agent_thread_put(void)
726 if (--statedump_agent_thread
.ref
)
728 (void)__atomic_or_fetch(&statedump_agent_thread
.state
, AGENT_THREAD_STATE_EXIT
, __ATOMIC_SEQ_CST
);
729 pthread_cond_broadcast(&statedump_agent_thread
.worker_cond
);
734 void statedump_agent_thread_fini(void)
736 statedump_agent_thread
.state
= AGENT_THREAD_STATE_BLOCKED
;
737 if (pthread_cond_destroy(&statedump_agent_thread
.worker_cond
))
739 if (pthread_cond_destroy(&statedump_agent_thread
.waiter_cond
))
743 /* Called with side_agent_thread_lock held. */
745 void statedump_agent_thread_join(void)
750 ret
= pthread_join(statedump_agent_thread
.id
, &retval
);
754 statedump_agent_thread_fini();
757 struct side_statedump_request_handle
*
758 side_statedump_request_notification_register(const char *state_name
,
759 void (*statedump_cb
)(void *statedump_request_key
),
760 enum side_statedump_mode mode
)
762 struct side_statedump_request_handle
*handle
;
769 handle
= (struct side_statedump_request_handle
*)
770 calloc(1, sizeof(struct side_statedump_request_handle
));
773 name
= strdup(state_name
);
776 handle
->cb
= statedump_cb
;
779 side_list_head_init(&handle
->notification_queue
);
781 if (mode
== SIDE_STATEDUMP_MODE_AGENT_THREAD
)
782 pthread_mutex_lock(&side_agent_thread_lock
);
783 pthread_mutex_lock(&side_statedump_lock
);
784 if (mode
== SIDE_STATEDUMP_MODE_AGENT_THREAD
)
785 statedump_agent_thread_get();
786 side_list_insert_node_tail_rcu(&side_statedump_list
, &handle
->node
);
787 /* Queue statedump pending for all tracers. */
788 queue_statedump_pending(handle
, SIDE_KEY_MATCH_ALL
);
789 pthread_mutex_unlock(&side_statedump_lock
);
791 if (mode
== SIDE_STATEDUMP_MODE_AGENT_THREAD
) {
792 pthread_mutex_unlock(&side_agent_thread_lock
);
794 pthread_mutex_lock(&side_statedump_lock
);
795 while (!side_list_empty(&handle
->notification_queue
))
796 pthread_cond_wait(&statedump_agent_thread
.waiter_cond
, &side_statedump_lock
);
797 pthread_mutex_unlock(&side_statedump_lock
);
807 void side_statedump_request_notification_unregister(struct side_statedump_request_handle
*handle
)
816 if (handle
->mode
== SIDE_STATEDUMP_MODE_AGENT_THREAD
)
817 pthread_mutex_lock(&side_agent_thread_lock
);
818 pthread_mutex_lock(&side_statedump_lock
);
819 unqueue_statedump_pending(handle
, SIDE_KEY_MATCH_ALL
);
820 side_list_remove_node_rcu(&handle
->node
);
821 if (handle
->mode
== SIDE_STATEDUMP_MODE_AGENT_THREAD
)
822 join
= statedump_agent_thread_put();
823 pthread_mutex_unlock(&side_statedump_lock
);
825 statedump_agent_thread_join();
826 if (handle
->mode
== SIDE_STATEDUMP_MODE_AGENT_THREAD
)
827 pthread_mutex_unlock(&side_agent_thread_lock
);
829 side_rcu_wait_grace_period(&statedump_rcu_gp
);
834 /* Returns true if the handle has pending statedump requests. */
835 bool side_statedump_poll_pending_requests(struct side_statedump_request_handle
*handle
)
839 if (handle
->mode
!= SIDE_STATEDUMP_MODE_POLLING
)
841 pthread_mutex_lock(&side_statedump_lock
);
842 ret
= !side_list_empty(&handle
->notification_queue
);
843 pthread_mutex_unlock(&side_statedump_lock
);
848 * Only polling mode state dump handles allow application to explicitly handle the
851 int side_statedump_run_pending_requests(struct side_statedump_request_handle
*handle
)
853 if (handle
->mode
!= SIDE_STATEDUMP_MODE_POLLING
)
854 return SIDE_ERROR_INVAL
;
855 _side_statedump_run_pending_requests(handle
);
856 return SIDE_ERROR_OK
;
860 * Request a state dump for tracer callbacks identified with "key".
862 int side_tracer_statedump_request(uint64_t key
)
864 struct side_statedump_request_handle
*handle
;
866 if (key
== SIDE_KEY_MATCH_ALL
)
867 return SIDE_ERROR_INVAL
;
868 pthread_mutex_lock(&side_statedump_lock
);
869 side_list_for_each_entry(handle
, &side_statedump_list
, node
)
870 queue_statedump_pending(handle
, key
);
871 pthread_mutex_lock(&side_statedump_lock
);
872 return SIDE_ERROR_OK
;
876 * Cancel a statedump request.
878 int side_tracer_statedump_request_cancel(uint64_t key
)
880 struct side_statedump_request_handle
*handle
;
882 if (key
== SIDE_KEY_MATCH_ALL
)
883 return SIDE_ERROR_INVAL
;
884 pthread_mutex_lock(&side_statedump_lock
);
885 side_list_for_each_entry(handle
, &side_statedump_list
, node
)
886 unqueue_statedump_pending(handle
, key
);
887 pthread_mutex_lock(&side_statedump_lock
);
888 return SIDE_ERROR_OK
;
892 * Tracer keys are represented on 64-bit. Return SIDE_ERROR_NOMEM on
893 * overflow (which should never happen in practice).
895 int side_tracer_request_key(uint64_t *key
)
897 int ret
= SIDE_ERROR_OK
;
899 pthread_mutex_lock(&side_key_lock
);
900 if (side_key_next
== 0) {
901 ret
= SIDE_ERROR_NOMEM
;
904 *key
= side_key_next
++;
906 pthread_mutex_unlock(&side_key_lock
);
911 * Use of pthread_atfork depends on glibc 2.24 to eliminate hangs when
912 * waiting for the agent thread if the agent thread calls malloc. This
913 * is corrected by GNU libc
914 * commit 8a727af925be63aa6ea0f5f90e16751fd541626b.
915 * Ref. https://bugzilla.redhat.com/show_bug.cgi?id=906468
918 void side_before_fork(void)
922 pthread_mutex_lock(&side_agent_thread_lock
);
923 if (!statedump_agent_thread
.ref
)
925 /* Pause agent thread. */
926 pthread_mutex_lock(&side_statedump_lock
);
927 (void)__atomic_or_fetch(&statedump_agent_thread
.state
, AGENT_THREAD_STATE_PAUSE
, __ATOMIC_SEQ_CST
);
928 pthread_cond_broadcast(&statedump_agent_thread
.worker_cond
);
929 pthread_mutex_unlock(&side_statedump_lock
);
930 /* Wait for agent thread acknowledge. */
931 while (!(__atomic_load_n(&statedump_agent_thread
.state
, __ATOMIC_SEQ_CST
) & AGENT_THREAD_STATE_PAUSE_ACK
)) {
932 if (attempt
> SIDE_RETRY_BUSY_LOOP_ATTEMPTS
) {
933 (void)poll(NULL
, 0, SIDE_RETRY_DELAY_MS
);
942 void side_after_fork_parent(void)
944 if (statedump_agent_thread
.ref
)
945 (void)__atomic_and_fetch(&statedump_agent_thread
.state
,
946 ~(AGENT_THREAD_STATE_PAUSE
| AGENT_THREAD_STATE_PAUSE_ACK
),
948 pthread_mutex_unlock(&side_agent_thread_lock
);
952 * The agent thread does not exist in the child process after a fork.
953 * Re-initialize its data structures and create a new agent thread.
956 void side_after_fork_child(void)
958 if (statedump_agent_thread
.ref
) {
961 statedump_agent_thread_fini();
962 statedump_agent_thread_init();
963 ret
= pthread_create(&statedump_agent_thread
.id
, NULL
,
964 statedump_agent_func
, NULL
);
969 pthread_mutex_unlock(&side_agent_thread_lock
);
976 side_rcu_gp_init(&event_rcu_gp
);
977 side_rcu_gp_init(&statedump_rcu_gp
);
978 if (pthread_atfork(side_before_fork
, side_after_fork_parent
, side_after_fork_child
))
984 * side_exit() is executed from a library destructor. It can be called
985 * explicitly at application exit as well. Concurrent side API use is
986 * not expected at that point.
990 struct side_events_register_handle
*handle
, *tmp
;
994 side_list_for_each_entry_safe(handle
, tmp
, &side_events_list
, node
)
995 side_events_unregister(handle
);
996 side_rcu_gp_exit(&event_rcu_gp
);
997 side_rcu_gp_exit(&statedump_rcu_gp
);