include/rseq/rseq.h

   1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
   2 /*
   3  * rseq.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 #ifndef RSEQ_H
   9 #define RSEQ_H
  10
  11 #include <stdint.h>
  12 #include <stdbool.h>
  13 #include <pthread.h>
  14 #include <signal.h>
  15 #include <sched.h>
  16 #include <errno.h>
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <sched.h>
  20 #include <linux/rseq.h>
  21
  22 /*
  23  * Empty code injection macros, override when testing.
  24  * It is important to consider that the ASM injection macros need to be
  25  * fully reentrant (e.g. do not modify the stack).
  26  */
  27 #ifndef RSEQ_INJECT_ASM
  28 #define RSEQ_INJECT_ASM(n)
  29 #endif
  30
  31 #ifndef RSEQ_INJECT_C
  32 #define RSEQ_INJECT_C(n)
  33 #endif
  34
  35 #ifndef RSEQ_INJECT_INPUT
  36 #define RSEQ_INJECT_INPUT
  37 #endif
  38
  39 #ifndef RSEQ_INJECT_CLOBBER
  40 #define RSEQ_INJECT_CLOBBER
  41 #endif
  42
  43 #ifndef RSEQ_INJECT_FAILED
  44 #define RSEQ_INJECT_FAILED
  45 #endif
  46
  47 #ifdef __cplusplus
  48 extern "C" {
  49 #endif
  50
  51 extern __thread struct rseq __rseq_abi;
  52 extern int __rseq_handled;
  53
  54 #ifdef __cplusplus
  55 }
  56 #endif
  57
  58 #define rseq_likely(x)          __builtin_expect(!!(x), 1)
  59 #define rseq_unlikely(x)        __builtin_expect(!!(x), 0)
  60 #define rseq_barrier()          __asm__ __volatile__("" : : : "memory")
  61
  62 #define RSEQ_ACCESS_ONCE(x)     (*(__volatile__  __typeof__(x) *)&(x))
  63 #define RSEQ_WRITE_ONCE(x, v)   __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); })
  64 #define RSEQ_READ_ONCE(x)       RSEQ_ACCESS_ONCE(x)
  65
  66 #define __rseq_str_1(x) #x
  67 #define __rseq_str(x)           __rseq_str_1(x)
  68
  69 #define rseq_log(fmt, args...)                                                 \
  70         fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \
  71                 ## args, __func__)
  72
  73 #define rseq_bug(fmt, args...)          \
  74         do {                            \
  75                 rseq_log(fmt, ##args);  \
  76                 abort();                \
  77         } while (0)
  78
  79 #if defined(__x86_64__) || defined(__i386__)
  80 #include <rseq/rseq-x86.h>
  81 #elif defined(__ARMEL__) || defined(__ARMEB__)
  82 #include <rseq/rseq-arm.h>
  83 #elif defined (__AARCH64EL__)
  84 #include <rseq/rseq-arm64.h>
  85 #elif defined(__PPC__)
  86 #include <rseq/rseq-ppc.h>
  87 #elif defined(__mips__)
  88 #include <rseq/rseq-mips.h>
  89 #elif defined(__s390__)
  90 #include <rseq/rseq-s390.h>
  91 #else
  92 #error unsupported target
  93 #endif
  94
  95 #ifdef __cplusplus
  96 extern "C" {
  97 #endif
  98
  99 /*
 100  * Register rseq for the current thread. This needs to be called once
 101  * by any thread which uses restartable sequences, before they start
 102  * using restartable sequences, to ensure restartable sequences
 103  * succeed. A restartable sequence executed from a non-registered
 104  * thread will always fail.
 105  */
 106 int rseq_register_current_thread(void);
 107
 108 /*
 109  * Unregister rseq for current thread.
 110  */
 111 int rseq_unregister_current_thread(void);
 112
 113 /*
 114  * Restartable sequence fallback for reading the current CPU number.
 115  */
 116 int32_t rseq_fallback_current_cpu(void);
 117
 118 int rseq_available(void);
 119
 120 /*
 121  * Values returned can be either the current CPU number, -1 (rseq is
 122  * uninitialized), or -2 (rseq initialization has failed).
 123  */
 124 static inline int32_t rseq_current_cpu_raw(void)
 125 {
 126         return RSEQ_READ_ONCE(__rseq_abi.cpu_id);
 127 }
 128
 129 /*
 130  * Returns a possible CPU number, which is typically the current CPU.
 131  * The returned CPU number can be used to prepare for an rseq critical
 132  * section, which will confirm whether the cpu number is indeed the
 133  * current one, and whether rseq is initialized.
 134  *
 135  * The CPU number returned by rseq_cpu_start should always be validated
 136  * by passing it to a rseq asm sequence, or by comparing it to the
 137  * return value of rseq_current_cpu_raw() if the rseq asm sequence
 138  * does not need to be invoked.
 139  */
 140 static inline uint32_t rseq_cpu_start(void)
 141 {
 142         return RSEQ_READ_ONCE(__rseq_abi.cpu_id_start);
 143 }
 144
 145 static inline uint32_t rseq_current_cpu(void)
 146 {
 147         int32_t cpu;
 148
 149         cpu = rseq_current_cpu_raw();
 150         if (rseq_unlikely(cpu < 0))
 151                 cpu = rseq_fallback_current_cpu();
 152         return cpu;
 153 }
 154
 155 static inline void rseq_clear_rseq_cs(void)
 156 {
 157 #ifdef __LP64__
 158         RSEQ_WRITE_ONCE(__rseq_abi.rseq_cs.ptr, 0);
 159 #else
 160         RSEQ_WRITE_ONCE(__rseq_abi.rseq_cs.ptr.ptr32, 0);
 161 #endif
 162 }
 163
 164 /*
 165  * rseq_prepare_unload() should be invoked by each thread executing a rseq
 166  * critical section at least once between their last critical section and
 167  * library unload of the library defining the rseq critical section
 168  * (struct rseq_cs). This also applies to use of rseq in code generated by
 169  * JIT: rseq_prepare_unload() should be invoked at least once by each
 170  * thread executing a rseq critical section before reclaim of the memory
 171  * holding the struct rseq_cs.
 172  */
 173 static inline void rseq_prepare_unload(void)
 174 {
 175         rseq_clear_rseq_cs();
 176 }
 177
 178 #ifdef __cplusplus
 179 }
 180 #endif
 181
 182 #endif  /* RSEQ_H_ */