include/rseq/rseq.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2016-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 /*
   5  * rseq.h
   6  */
   7
   8 #ifndef RSEQ_H
   9 #define RSEQ_H
  10
  11 #include <stdint.h>
  12 #include <stdbool.h>
  13 #include <pthread.h>
  14 #include <signal.h>
  15 #include <sched.h>
  16 #include <errno.h>
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <stddef.h>
  20 #include <assert.h>
  21 #include <rseq/rseq-abi.h>
  22 #include <rseq/compiler.h>
  23
  24 #ifndef rseq_sizeof_field
  25 #define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
  26 #endif
  27
  28 #ifndef rseq_offsetofend
  29 #define rseq_offsetofend(TYPE, MEMBER) \
  30         (offsetof(TYPE, MEMBER) + rseq_sizeof_field(TYPE, MEMBER))
  31 #endif
  32
  33 /*
  34  * Empty code injection macros, override when testing.
  35  * It is important to consider that the ASM injection macros need to be
  36  * fully reentrant (e.g. do not modify the stack).
  37  */
  38 #ifndef RSEQ_INJECT_ASM
  39 #define RSEQ_INJECT_ASM(n)
  40 #endif
  41
  42 #ifndef RSEQ_INJECT_C
  43 #define RSEQ_INJECT_C(n)
  44 #endif
  45
  46 #ifndef RSEQ_INJECT_INPUT
  47 #define RSEQ_INJECT_INPUT
  48 #endif
  49
  50 #ifndef RSEQ_INJECT_CLOBBER
  51 #define RSEQ_INJECT_CLOBBER
  52 #endif
  53
  54 #ifndef RSEQ_INJECT_FAILED
  55 #define RSEQ_INJECT_FAILED
  56 #endif
  57
  58 /*
  59  * User code can define RSEQ_GET_ABI_OVERRIDE to override the
  60  * rseq_get_abi() implementation, for instance to use glibc's symbols
  61  * directly.
  62  */
  63 #ifndef RSEQ_GET_ABI_OVERRIDE
  64
  65 # include <rseq/rseq-thread-pointer.h>
  66
  67 # ifdef __cplusplus
  68 extern "C" {
  69 # endif
  70
  71 /* Offset from the thread pointer to the rseq area. */
  72 extern ptrdiff_t rseq_offset;
  73
  74 /*
  75  * Size of the registered rseq area. 0 if the registration was
  76  * unsuccessful.
  77  */
  78 extern unsigned int rseq_size;
  79
  80 /* Flags used during rseq registration. */
  81 extern unsigned int rseq_flags;
  82
  83 /*
  84  * rseq feature size supported by the kernel. 0 if the registration was
  85  * unsuccessful.
  86  */
  87 extern unsigned int rseq_feature_size;
  88
  89 static inline struct rseq_abi *rseq_get_abi(void)
  90 {
  91         return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset);
  92 }
  93
  94 # ifdef __cplusplus
  95 }
  96 # endif
  97
  98 #endif /* RSEQ_GET_ABI_OVERRIDE */
  99
 100 #define rseq_likely(x)          __builtin_expect(!!(x), 1)
 101 #define rseq_unlikely(x)        __builtin_expect(!!(x), 0)
 102 #define rseq_barrier()          __asm__ __volatile__("" : : : "memory")
 103
 104 #define RSEQ_ACCESS_ONCE(x)     (*(__volatile__  __typeof__(x) *)&(x))
 105 #define RSEQ_WRITE_ONCE(x, v)   __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); })
 106 #define RSEQ_READ_ONCE(x)       RSEQ_ACCESS_ONCE(x)
 107
 108 #define __rseq_str_1(x) #x
 109 #define __rseq_str(x)           __rseq_str_1(x)
 110
 111 #define rseq_log(fmt, ...)                                                     \
 112         fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \
 113                 ## __VA_ARGS__, __func__)
 114
 115 #define rseq_bug(fmt, ...)              \
 116         do {                            \
 117                 rseq_log(fmt, ## __VA_ARGS__); \
 118                 abort();                \
 119         } while (0)
 120
 121 #if defined(__x86_64__) || defined(__i386__)
 122 #include <rseq/rseq-x86.h>
 123 #elif defined(__ARMEL__) || defined(__ARMEB__)
 124 #include <rseq/rseq-arm.h>
 125 #elif defined (__AARCH64EL__)
 126 #include <rseq/rseq-arm64.h>
 127 #elif defined(__PPC__)
 128 #include <rseq/rseq-ppc.h>
 129 #elif defined(__mips__)
 130 #include <rseq/rseq-mips.h>
 131 #elif defined(__s390__)
 132 #include <rseq/rseq-s390.h>
 133 #elif defined(__riscv)
 134 #include <rseq/rseq-riscv.h>
 135 #else
 136 #error unsupported target
 137 #endif
 138
 139 #ifdef __cplusplus
 140 extern "C" {
 141 #endif
 142
 143 /*
 144  * Register rseq for the current thread. This needs to be called once
 145  * by any thread which uses restartable sequences, before they start
 146  * using restartable sequences, to ensure restartable sequences
 147  * succeed. A restartable sequence executed from a non-registered
 148  * thread will always fail.
 149  */
 150 int rseq_register_current_thread(void);
 151
 152 /*
 153  * Unregister rseq for current thread.
 154  */
 155 int rseq_unregister_current_thread(void);
 156
 157 /*
 158  * Restartable sequence fallback for reading the current CPU number.
 159  */
 160 int32_t rseq_fallback_current_cpu(void);
 161
 162 /*
 163  * Restartable sequence fallback for reading the current node number.
 164  */
 165 int32_t rseq_fallback_current_node(void);
 166
 167 enum rseq_available_query {
 168         RSEQ_AVAILABLE_QUERY_KERNEL = 0,
 169         RSEQ_AVAILABLE_QUERY_LIBC = 1,
 170 };
 171
 172 /*
 173  * Returns true if rseq is supported.
 174  */
 175 bool rseq_available(unsigned int query);
 176
 177 /*
 178  * Values returned can be either the current CPU number, -1 (rseq is
 179  * uninitialized), or -2 (rseq initialization has failed).
 180  */
 181 static inline int32_t rseq_current_cpu_raw(void)
 182 {
 183         return RSEQ_READ_ONCE(rseq_get_abi()->cpu_id);
 184 }
 185
 186 /*
 187  * Returns a possible CPU number, which is typically the current CPU.
 188  * The returned CPU number can be used to prepare for an rseq critical
 189  * section, which will confirm whether the cpu number is indeed the
 190  * current one, and whether rseq is initialized.
 191  *
 192  * The CPU number returned by rseq_cpu_start should always be validated
 193  * by passing it to a rseq asm sequence, or by comparing it to the
 194  * return value of rseq_current_cpu_raw() if the rseq asm sequence
 195  * does not need to be invoked.
 196  */
 197 static inline uint32_t rseq_cpu_start(void)
 198 {
 199         return RSEQ_READ_ONCE(rseq_get_abi()->cpu_id_start);
 200 }
 201
 202 static inline uint32_t rseq_current_cpu(void)
 203 {
 204         int32_t cpu;
 205
 206         cpu = rseq_current_cpu_raw();
 207         if (rseq_unlikely(cpu < 0))
 208                 cpu = rseq_fallback_current_cpu();
 209         return cpu;
 210 }
 211
 212 static inline bool rseq_node_id_available(void)
 213 {
 214         return (int) rseq_feature_size >= (int) rseq_offsetofend(struct rseq_abi, node_id);
 215 }
 216
 217 /*
 218  * Current NUMA node number.
 219  */
 220 static inline uint32_t rseq_current_node_id(void)
 221 {
 222         assert(rseq_node_id_available());
 223         return RSEQ_READ_ONCE(rseq_get_abi()->node_id);
 224 }
 225
 226 static inline bool rseq_mm_cid_available(void)
 227 {
 228         return (int) rseq_feature_size >= (int) rseq_offsetofend(struct rseq_abi, mm_cid);
 229 }
 230
 231 static inline uint32_t rseq_current_mm_cid(void)
 232 {
 233         return RSEQ_READ_ONCE(rseq_get_abi()->mm_cid);
 234 }
 235
 236 static inline void rseq_clear_rseq_cs(void)
 237 {
 238         RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs, 0);
 239 }
 240
 241 /*
 242  * rseq_prepare_unload() should be invoked by each thread executing a rseq
 243  * critical section at least once between their last critical section and
 244  * library unload of the library defining the rseq critical section (struct
 245  * rseq_cs) or the code referred to by the struct rseq_cs start_ip and
 246  * post_commit_offset fields. This also applies to use of rseq in code
 247  * generated by JIT: rseq_prepare_unload() should be invoked at least once by
 248  * each thread executing a rseq critical section before reclaim of the memory
 249  * holding the struct rseq_cs or reclaim of the code pointed to by struct
 250  * rseq_cs start_ip and post_commit_offset fields.
 251  */
 252 static inline void rseq_prepare_unload(void)
 253 {
 254         rseq_clear_rseq_cs();
 255 }
 256
 257 #ifdef __cplusplus
 258 }
 259 #endif
 260
 261 #endif  /* RSEQ_H_ */