1 // SPDX-License-Identifier: MIT
2 // SPDX-FileCopyrightText: 2016 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
21 #include <linux/auxvec.h>
23 #include <rseq/rseq.h>
25 #ifndef AT_RSEQ_FEATURE_SIZE
26 # define AT_RSEQ_FEATURE_SIZE 27
30 # define AT_RSEQ_ALIGN 28
33 static __attribute__((constructor
))
36 static pthread_mutex_t init_lock
= PTHREAD_MUTEX_INITIALIZER
;
39 static const ptrdiff_t *libc_rseq_offset_p
;
40 static const unsigned int *libc_rseq_size_p
;
41 static const unsigned int *libc_rseq_flags_p
;
43 /* Offset from the thread pointer to the rseq area. */
44 ptrdiff_t rseq_offset
;
47 * Size of the registered rseq area. 0 if the registration was
50 unsigned int rseq_size
= -1U;
52 /* Flags used during rseq registration. */
53 unsigned int rseq_flags
;
56 * rseq feature size supported by the kernel. 0 if the registration was
59 unsigned int rseq_feature_size
= -1U;
61 static int rseq_ownership
;
62 static int rseq_reg_success
; /* At least one rseq registration has succeded. */
64 /* Allocate a large area for the TLS. */
65 #define RSEQ_THREAD_AREA_ALLOC_SIZE 1024
67 /* Original struct rseq feature size is 20 bytes. */
68 #define ORIG_RSEQ_FEATURE_SIZE 20
70 /* Original struct rseq allocation size is 32 bytes. */
71 #define ORIG_RSEQ_ALLOC_SIZE 32
74 * The alignment on RSEQ_THREAD_AREA_ALLOC_SIZE guarantees that the
75 * rseq_abi structure allocated size is at least
76 * RSEQ_THREAD_AREA_ALLOC_SIZE bytes to hold extra space for yet unknown
77 * kernel rseq extensions.
80 __thread
struct rseq_abi __rseq_abi
__attribute__((tls_model("initial-exec"), aligned(RSEQ_THREAD_AREA_ALLOC_SIZE
))) = {
81 .cpu_id
= RSEQ_ABI_CPU_ID_UNINITIALIZED
,
84 static int sys_rseq(struct rseq_abi
*rseq_abi
, uint32_t rseq_len
,
85 int flags
, uint32_t sig
)
87 return syscall(__NR_rseq
, rseq_abi
, rseq_len
, flags
, sig
);
90 static int sys_getcpu(unsigned *cpu
, unsigned *node
)
92 return syscall(__NR_getcpu
, cpu
, node
, NULL
);
95 bool rseq_available(unsigned int query
)
100 case RSEQ_AVAILABLE_QUERY_KERNEL
:
101 rc
= sys_rseq(NULL
, 0, 0, 0);
113 case RSEQ_AVAILABLE_QUERY_LIBC
:
114 if (rseq_size
&& !rseq_ownership
)
123 int rseq_register_current_thread(void)
129 if (!rseq_ownership
) {
130 /* Treat libc's ownership as a successful registration. */
133 rc
= sys_rseq(&__rseq_abi
, rseq_size
, 0, RSEQ_SIG
);
135 if (RSEQ_READ_ONCE(rseq_reg_success
)) {
136 /* Incoherent success/failure within process. */
141 assert(rseq_current_cpu_raw() >= 0);
142 RSEQ_WRITE_ONCE(rseq_reg_success
, 1);
146 int rseq_unregister_current_thread(void)
150 if (!rseq_ownership
) {
151 /* Treat libc's ownership as a successful unregistration. */
154 rc
= sys_rseq(&__rseq_abi
, rseq_size
, RSEQ_ABI_FLAG_UNREGISTER
, RSEQ_SIG
);
161 unsigned int get_rseq_feature_size(void)
163 unsigned long auxv_rseq_feature_size
, auxv_rseq_align
;
165 auxv_rseq_align
= getauxval(AT_RSEQ_ALIGN
);
166 assert(!auxv_rseq_align
|| auxv_rseq_align
<= RSEQ_THREAD_AREA_ALLOC_SIZE
);
168 auxv_rseq_feature_size
= getauxval(AT_RSEQ_FEATURE_SIZE
);
169 assert(!auxv_rseq_feature_size
|| auxv_rseq_feature_size
<= RSEQ_THREAD_AREA_ALLOC_SIZE
);
170 if (auxv_rseq_feature_size
)
171 return auxv_rseq_feature_size
;
173 return ORIG_RSEQ_FEATURE_SIZE
;
177 * Initialize the public symbols for the rseq offset, size, feature size and
178 * flags prior to registering threads. If glibc owns the registration, get the
179 * values from its public symbols.
184 /* Ensure initialization is only done once. */
185 if (RSEQ_READ_ONCE(init_done
))
189 * Take the mutex, check the initialization flag again and atomically
190 * set it to ensure we are the only thread doing the initialization.
192 pthread_mutex_lock(&init_lock
);
195 RSEQ_WRITE_ONCE(init_done
, 1);
198 * Check for glibc rseq support, if the 3 public symbols are found and
199 * the rseq_size is not zero, glibc owns the registration.
201 libc_rseq_offset_p
= dlsym(RTLD_NEXT
, "__rseq_offset");
202 libc_rseq_size_p
= dlsym(RTLD_NEXT
, "__rseq_size");
203 libc_rseq_flags_p
= dlsym(RTLD_NEXT
, "__rseq_flags");
204 if (libc_rseq_size_p
&& libc_rseq_offset_p
&& libc_rseq_flags_p
&&
205 *libc_rseq_size_p
!= 0) {
206 /* rseq registration owned by glibc */
207 rseq_offset
= *libc_rseq_offset_p
;
208 rseq_size
= *libc_rseq_size_p
;
209 rseq_flags
= *libc_rseq_flags_p
;
210 rseq_feature_size
= get_rseq_feature_size();
213 * The registered rseq area could be smaller than the feature
214 * size reported by the kernel auxval. Cap it to the rseq size
215 * so we don't try to access features past the end of the rseq
218 if (rseq_feature_size
> rseq_size
)
219 rseq_feature_size
= rseq_size
;
223 /* librseq owns the registration */
226 /* Calculate the offset of the rseq area from the thread pointer. */
227 rseq_offset
= (uintptr_t)&__rseq_abi
- (uintptr_t)rseq_thread_pointer();
229 /* rseq flags are deprecated, always set to 0. */
233 * Check if the rseq syscall is available, if not set the size and
236 if (!rseq_available(RSEQ_AVAILABLE_QUERY_KERNEL
)) {
238 rseq_feature_size
= 0;
243 * If the feature size matches the original ABI (20), set the size to
244 * match the original ABI allocation (32), otherwise use the allocated
247 rseq_feature_size
= get_rseq_feature_size();
248 if (rseq_feature_size
== ORIG_RSEQ_FEATURE_SIZE
)
249 rseq_size
= ORIG_RSEQ_ALLOC_SIZE
;
251 rseq_size
= RSEQ_THREAD_AREA_ALLOC_SIZE
;
253 pthread_mutex_unlock(&init_lock
);
256 static __attribute__((destructor
))
263 rseq_feature_size
= -1U;
267 int32_t rseq_fallback_current_cpu(void)
271 cpu
= sched_getcpu();
273 perror("sched_getcpu()");
279 int32_t rseq_fallback_current_node(void)
281 uint32_t cpu_id
, node_id
;
284 ret
= sys_getcpu(&cpu_id
, &node_id
);
286 perror("sys_getcpu()");
289 return (int32_t) node_id
;