1 // SPDX-License-Identifier: MIT
2 // SPDX-FileCopyrightText: 2016 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
21 #include <linux/auxvec.h>
23 #include <rseq/rseq.h>
25 #ifndef AT_RSEQ_FEATURE_SIZE
26 # define AT_RSEQ_FEATURE_SIZE 27
30 # define AT_RSEQ_ALIGN 28
33 static __attribute__((constructor
))
36 static pthread_mutex_t init_lock
= PTHREAD_MUTEX_INITIALIZER
;
39 static const ptrdiff_t *libc_rseq_offset_p
;
40 static const unsigned int *libc_rseq_size_p
;
41 static const unsigned int *libc_rseq_flags_p
;
43 /* Offset from the thread pointer to the rseq area. */
44 ptrdiff_t rseq_offset
;
47 * Size of the registered rseq area. 0 if the registration was
50 unsigned int rseq_size
= -1U;
52 /* Flags used during rseq registration. */
53 unsigned int rseq_flags
;
56 * rseq feature size supported by the kernel. 0 if the registration was
59 unsigned int rseq_feature_size
= -1U;
61 static int rseq_ownership
;
62 static int rseq_reg_success
; /* At least one rseq registration has succeded. */
64 /* Allocate a large area for the TLS. */
65 #define RSEQ_THREAD_AREA_ALLOC_SIZE 1024
67 /* Original struct rseq feature size is 20 bytes. */
68 #define ORIG_RSEQ_FEATURE_SIZE 20
70 /* Original struct rseq allocation size is 32 bytes. */
71 #define ORIG_RSEQ_ALLOC_SIZE 32
74 * The alignment on RSEQ_THREAD_AREA_ALLOC_SIZE guarantees that the
75 * rseq_abi structure allocated size is at least
76 * RSEQ_THREAD_AREA_ALLOC_SIZE bytes to hold extra space for yet unknown
77 * kernel rseq extensions.
80 __thread
struct rseq_abi __rseq_abi
__attribute__((tls_model("initial-exec"), aligned(RSEQ_THREAD_AREA_ALLOC_SIZE
))) = {
81 .cpu_id
= RSEQ_ABI_CPU_ID_UNINITIALIZED
,
84 static int sys_rseq(struct rseq_abi
*rseq_abi
, uint32_t rseq_len
,
85 int flags
, uint32_t sig
)
87 return syscall(__NR_rseq
, rseq_abi
, rseq_len
, flags
, sig
);
90 static int sys_getcpu(unsigned *cpu
, unsigned *node
)
92 return syscall(__NR_getcpu
, cpu
, node
, NULL
);
95 bool rseq_available(unsigned int query
)
100 case RSEQ_AVAILABLE_QUERY_KERNEL
:
101 rc
= sys_rseq(NULL
, 0, 0, 0);
113 case RSEQ_AVAILABLE_QUERY_LIBC
:
114 if (rseq_size
&& !rseq_ownership
)
123 int rseq_register_current_thread(void)
129 if (!rseq_ownership
) {
130 /* Treat libc's ownership as a successful registration. */
133 rc
= sys_rseq(&__rseq_abi
, rseq_size
, 0, RSEQ_SIG
);
135 if (RSEQ_READ_ONCE(rseq_reg_success
)) {
136 /* Incoherent success/failure within process. */
141 assert(rseq_current_cpu_raw() >= 0);
142 RSEQ_WRITE_ONCE(rseq_reg_success
, 1);
146 int rseq_unregister_current_thread(void)
150 if (!rseq_ownership
) {
151 /* Treat libc's ownership as a successful unregistration. */
154 rc
= sys_rseq(&__rseq_abi
, rseq_size
, RSEQ_ABI_FLAG_UNREGISTER
, RSEQ_SIG
);
161 unsigned int get_rseq_feature_size(void)
163 unsigned long auxv_rseq_feature_size
, auxv_rseq_align
;
165 auxv_rseq_align
= getauxval(AT_RSEQ_ALIGN
);
166 assert(!auxv_rseq_align
|| auxv_rseq_align
<= RSEQ_THREAD_AREA_ALLOC_SIZE
);
168 auxv_rseq_feature_size
= getauxval(AT_RSEQ_FEATURE_SIZE
);
169 assert(!auxv_rseq_feature_size
|| auxv_rseq_feature_size
<= RSEQ_THREAD_AREA_ALLOC_SIZE
);
170 if (auxv_rseq_feature_size
)
171 return auxv_rseq_feature_size
;
173 return ORIG_RSEQ_FEATURE_SIZE
;
179 if (RSEQ_READ_ONCE(init_done
))
182 pthread_mutex_lock(&init_lock
);
185 RSEQ_WRITE_ONCE(init_done
, 1);
186 libc_rseq_offset_p
= dlsym(RTLD_NEXT
, "__rseq_offset");
187 libc_rseq_size_p
= dlsym(RTLD_NEXT
, "__rseq_size");
188 libc_rseq_flags_p
= dlsym(RTLD_NEXT
, "__rseq_flags");
189 if (libc_rseq_size_p
&& libc_rseq_offset_p
&& libc_rseq_flags_p
&&
190 *libc_rseq_size_p
!= 0) {
191 /* rseq registration owned by glibc */
192 rseq_offset
= *libc_rseq_offset_p
;
193 rseq_size
= *libc_rseq_size_p
;
194 rseq_flags
= *libc_rseq_flags_p
;
195 rseq_feature_size
= get_rseq_feature_size();
196 if (rseq_feature_size
> rseq_size
)
197 rseq_feature_size
= rseq_size
;
201 if (!rseq_available(RSEQ_AVAILABLE_QUERY_KERNEL
)) {
203 rseq_feature_size
= 0;
206 rseq_offset
= (uintptr_t)&__rseq_abi
- (uintptr_t)rseq_thread_pointer();
208 rseq_feature_size
= get_rseq_feature_size();
209 if (rseq_feature_size
== ORIG_RSEQ_FEATURE_SIZE
)
210 rseq_size
= ORIG_RSEQ_ALLOC_SIZE
;
212 rseq_size
= RSEQ_THREAD_AREA_ALLOC_SIZE
;
214 pthread_mutex_unlock(&init_lock
);
217 static __attribute__((destructor
))
224 rseq_feature_size
= -1U;
228 int32_t rseq_fallback_current_cpu(void)
232 cpu
= sched_getcpu();
234 perror("sched_getcpu()");
240 int32_t rseq_fallback_current_node(void)
242 uint32_t cpu_id
, node_id
;
245 ret
= sys_getcpu(&cpu_id
, &node_id
);
247 perror("sys_getcpu()");
250 return (int32_t) node_id
;