include/rseq/arch/x86.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 /*
   5  * rseq/arch/x86.h
   6  */
   7
   8 #ifndef _RSEQ_RSEQ_H
   9 #error "Never use <rseq/arch/x86.h> directly; include <rseq/rseq.h> instead."
  10 #endif
  11
  12 #include <stdint.h>
  13
  14 /*
  15  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  16  * are not part of the public API.
  17  */
  18
  19 /*
  20  * RSEQ_SIG is used with the following reserved undefined instructions, which
  21  * trap in user-space:
  22  *
  23  * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
  24  * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
  25  */
  26 #define RSEQ_SIG        0x53053053
  27
  28 /*
  29  * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
  30  * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
  31  * address through a "r" input operand.
  32  */
  33
  34 /*
  35  * Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. Those
  36  * are defined explicitly as macros to be used from assembly.
  37  */
  38 #define RSEQ_ASM_CPU_ID_OFFSET          4
  39 #define RSEQ_ASM_CS_OFFSET              8
  40 #define RSEQ_ASM_MM_CID_OFFSET          24
  41
  42 /*
  43  * Refer to the Linux kernel memory model (LKMM) for documentation of
  44  * the memory barriers. Expect all x86 hardware to be x86-TSO (Total
  45  * Store Order).
  46  */
  47
  48 /* CPU memory barrier. */
  49 #define rseq_smp_mb()   \
  50         __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
  51 /* CPU read memory barrier */
  52 #define rseq_smp_rmb()  rseq_barrier()
  53 /* CPU write memory barrier */
  54 #define rseq_smp_wmb()  rseq_barrier()
  55
  56 /* Acquire: One-way permeable barrier. */
  57 #define rseq_smp_load_acquire(p)                                        \
  58 __extension__ ({                                                        \
  59         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  60         rseq_barrier();                                                 \
  61         ____p1;                                                         \
  62 })
  63
  64 /* Acquire barrier after control dependency. */
  65 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  66
  67 /* Release: One-way permeable barrier. */
  68 #define rseq_smp_store_release(p, v)                                    \
  69 do {                                                                    \
  70         rseq_barrier();                                                 \
  71         RSEQ_WRITE_ONCE(*(p), v);                                       \
  72 } while (0)
  73
  74 /* Segment selector for the thread pointer. */
  75 #ifdef RSEQ_ARCH_AMD64
  76 # define RSEQ_ASM_TP_SEGMENT            %%fs
  77 #else
  78 # define RSEQ_ASM_TP_SEGMENT            %%gs
  79 #endif
  80
  81 /*
  82  * Helper macro to define a variable of pointer type stored in a 64-bit
  83  * integer. Only used internally in rseq headers.
  84  */
  85 #ifdef RSEQ_ARCH_AMD64
  86 # define RSEQ_ASM_U64_PTR(x)            ".quad " x
  87 #else
  88 # define RSEQ_ASM_U64_PTR(x)            ".long " x ", 0x0"
  89 #endif
  90
  91 #define RSEQ_ASM_U32(x)                 ".long " x
  92
  93 /* Common architecture support macros. */
  94 #include "rseq/arch/generic/common.h"
  95
  96 /*
  97  * Define a critical section abort handler.
  98  *
  99  *  @label:
 100  *    Local label to the abort handler.
 101  *  @teardown:
 102  *    Sequence of instructions to run on abort.
 103  *  @abort_label:
 104  *    C label to jump to at the end of the sequence.
 105  */
 106 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 107                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 108                 /*                                                      \
 109                  * Disassembler-friendly signature:                     \
 110                  *   x86-32: ud1 <sig>,%edi                             \
 111                  *   x86-64: ud1 <sig>(%rip),%edi                       \
 112                  */                                                     \
 113                 ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 114                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 115                 __rseq_str(label) ":\n\t"                               \
 116                 teardown                                                \
 117                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 118                 ".popsection\n\t"
 119
 120 /*
 121  * Define a critical section teardown handler.
 122  *
 123  *  @label:
 124  *    Local label to the teardown handler.
 125  *  @teardown:
 126  *    Sequence of instructions to run on teardown.
 127  *  @target_label:
 128  *    C label to jump to at the end of the sequence.
 129  */
 130 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)         \
 131                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 132                 __rseq_str(label) ":\n\t"                               \
 133                 teardown                                                \
 134                 "jmp %l[" __rseq_str(target_label) "]\n\t"              \
 135                 ".popsection\n\t"
 136
 137 /*
 138  * Store the address of the critical section descriptor structure at
 139  * @cs_label into the @rseq_cs pointer and emit the label @label, which
 140  * is the beginning of the sequence of consecutive assembly instructions.
 141  *
 142  *  @label:
 143  *    Local label to the beginning of the sequence of consecutive assembly
 144  *    instructions.
 145  *  @cs_label:
 146  *    Source local label to the critical section descriptor structure.
 147  *  @rseq_cs:
 148  *    Destination pointer where to store the address of the critical
 149  *    section descriptor structure.
 150  */
 151 #ifdef RSEQ_ARCH_AMD64
 152 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 153                 RSEQ_INJECT_ASM(1)                                      \
 154                 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
 155                 "movq %%rax, " __rseq_str(rseq_cs) "\n\t"               \
 156                 __rseq_str(label) ":\n\t"
 157 #else
 158 # define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)               \
 159                 RSEQ_INJECT_ASM(1)                                      \
 160                 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"   \
 161                 __rseq_str(label) ":\n\t"
 162 #endif
 163
 164 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 165 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
 166                 RSEQ_INJECT_ASM(2)                                      \
 167                 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
 168                 "jnz " __rseq_str(label) "\n\t"
 169
 170 /* Per-cpu-id indexing. */
 171
 172 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 173 #define RSEQ_TEMPLATE_MO_RELAXED
 174 #include "rseq/arch/x86/bits.h"
 175 #undef RSEQ_TEMPLATE_MO_RELAXED
 176
 177 #define RSEQ_TEMPLATE_MO_RELEASE
 178 #include "rseq/arch/x86/bits.h"
 179 #undef RSEQ_TEMPLATE_MO_RELEASE
 180 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 181
 182 /* Per-mm-cid indexing. */
 183
 184 #define RSEQ_TEMPLATE_INDEX_MM_CID
 185 #define RSEQ_TEMPLATE_MO_RELAXED
 186 #include "rseq/arch/x86/bits.h"
 187 #undef RSEQ_TEMPLATE_MO_RELAXED
 188
 189 #define RSEQ_TEMPLATE_MO_RELEASE
 190 #include "rseq/arch/x86/bits.h"
 191 #undef RSEQ_TEMPLATE_MO_RELEASE
 192 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 193
 194 /* APIs which are not indexed. */
 195
 196 #define RSEQ_TEMPLATE_INDEX_NONE
 197 #define RSEQ_TEMPLATE_MO_RELAXED
 198 #include "rseq/arch/x86/bits.h"
 199 #undef RSEQ_TEMPLATE_MO_RELAXED
 200 #undef RSEQ_TEMPLATE_INDEX_NONE