include/rseq/rseq-x86.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 /*
   5  * rseq-x86.h
   6  */
   7
   8 #ifndef RSEQ_H
   9 #error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
  10 #endif
  11
  12 #include <stdint.h>
  13
  14 /*
  15  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  16  * are not part of the public API.
  17  */
  18
  19 /*
  20  * RSEQ_SIG is used with the following reserved undefined instructions, which
  21  * trap in user-space:
  22  *
  23  * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
  24  * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
  25  */
  26 #define RSEQ_SIG        0x53053053
  27
  28 /*
  29  * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
  30  * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
  31  * address through a "r" input operand.
  32  */
  33
  34 /*
  35  * Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. Those
  36  * are defined explicitly as macros to be used from assembly.
  37  */
  38 #define RSEQ_ASM_CPU_ID_OFFSET          4
  39 #define RSEQ_ASM_CS_OFFSET              8
  40 #define RSEQ_ASM_MM_CID_OFFSET          24
  41
  42 /*
  43  * Refer to the Linux kernel memory model (LKMM) for documentation of
  44  * the memory barriers. Expect all x86 hardware to be x86-TSO (Total
  45  * Store Order).
  46  */
  47
  48 /* CPU memory barrier. */
  49 #define rseq_smp_mb()   \
  50         __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
  51 /* CPU read memory barrier */
  52 #define rseq_smp_rmb()  rseq_barrier()
  53 /* CPU write memory barrier */
  54 #define rseq_smp_wmb()  rseq_barrier()
  55
  56 /* Acquire: One-way permeable barrier. */
  57 #define rseq_smp_load_acquire(p)                                        \
  58 __extension__ ({                                                        \
  59         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  60         rseq_barrier();                                                 \
  61         ____p1;                                                         \
  62 })
  63
  64 /* Acquire barrier after control dependency. */
  65 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  66
  67 /* Release: One-way permeable barrier. */
  68 #define rseq_smp_store_release(p, v)                                    \
  69 do {                                                                    \
  70         rseq_barrier();                                                 \
  71         RSEQ_WRITE_ONCE(*(p), v);                                       \
  72 } while (0)
  73
  74 #ifdef __x86_64__
  75
  76 /* Segment selector for the thread pointer. */
  77 #define RSEQ_ASM_TP_SEGMENT     %%fs
  78
  79 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
  80 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  81                                 start_ip, post_commit_offset, abort_ip) \
  82                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  83                 ".balign 32\n\t"                                        \
  84                 __rseq_str(label) ":\n\t"                               \
  85                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  86                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  87                 ".popsection\n\t"                                       \
  88                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  89                 ".quad " __rseq_str(label) "b\n\t"                      \
  90                 ".popsection\n\t"
  91
  92 /*
  93  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
  94  * assembly instructions at @start_ip.
  95  *
  96  *  @start_ip:
  97  *    Pointer to the first instruction of the sequence of consecutive assembly
  98  *    instructions.
  99  *  @exit_ip:
 100  *    Pointer to an exit point instruction.
 101  *
 102  * Exit points of a rseq critical section consist of all instructions outside
 103  * of the critical section where a critical section can either branch to or
 104  * reach through the normal course of its execution. The abort IP and the
 105  * post-commit IP are already part of the __rseq_cs section and should not be
 106  * explicitly defined as additional exit points. Knowing all exit points is
 107  * useful to assist debuggers stepping over the critical section.
 108  */
 109 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 110                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 111                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
 112                 ".popsection\n\t"
 113
 114 /*
 115  * Store the address of the critical section descriptor structure at
 116  * @cs_label into the @rseq_cs pointer and emit the label @label, which
 117  * is the beginning of the sequence of consecutive assembly instructions.
 118  *
 119  *  @label:
 120  *    Local label to the beginning of the sequence of consecutive assembly
 121  *    instructions.
 122  *  @cs_label:
 123  *    Source local label to the critical section descriptor structure.
 124  *  @rseq_cs:
 125  *    Destination pointer where to store the address of the critical
 126  *    section descriptor structure.
 127  */
 128 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 129                 RSEQ_INJECT_ASM(1)                                      \
 130                 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
 131                 "movq %%rax, " __rseq_str(rseq_cs) "\n\t"               \
 132                 __rseq_str(label) ":\n\t"
 133
 134 #elif defined(__i386__)
 135
 136 /* Segment selector for the thread pointer. */
 137 #define RSEQ_ASM_TP_SEGMENT     %%gs
 138
 139 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
 140 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 141                                 start_ip, post_commit_offset, abort_ip) \
 142                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 143                 ".balign 32\n\t"                                        \
 144                 __rseq_str(label) ":\n\t"                               \
 145                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 146                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 147                 ".popsection\n\t"                                       \
 148                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 149                 ".long " __rseq_str(label) "b, 0x0\n\t"                 \
 150                 ".popsection\n\t"
 151
 152 /*
 153  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 154  * assembly instructions at @start_ip.
 155  *
 156  *  @start_ip:
 157  *    Pointer to the first instruction of the sequence of consecutive assembly
 158  *    instructions.
 159  *  @exit_ip:
 160  *    Pointer to an exit point instruction.
 161  *
 162  * Exit points of a rseq critical section consist of all instructions outside
 163  * of the critical section where a critical section can either branch to or
 164  * reach through the normal course of its execution. The abort IP and the
 165  * post-commit IP are already part of the __rseq_cs section and should not be
 166  * explicitly defined as additional exit points. Knowing all exit points is
 167  * useful to assist debuggers stepping over the critical section.
 168  */
 169 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 170                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 171                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
 172                 ".popsection\n\t"
 173
 174 /*
 175  * Store the address of the critical section descriptor structure at
 176  * @cs_label into the @rseq_cs pointer and emit the label @label, which
 177  * is the beginning of the sequence of consecutive assembly instructions.
 178  *
 179  *  @label:
 180  *    Local label to the beginning of the sequence of consecutive assembly
 181  *    instructions.
 182  *  @cs_label:
 183  *    Source local label to the critical section descriptor structure.
 184  *  @rseq_cs:
 185  *    Destination pointer where to store the address of the critical
 186  *    section descriptor structure.
 187  */
 188 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 189                 RSEQ_INJECT_ASM(1)                                      \
 190                 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"   \
 191                 __rseq_str(label) ":\n\t"
 192
 193 #endif
 194
 195 /*
 196  * Define an rseq critical section structure of version 0 with no flags.
 197  *
 198  *  @label:
 199  *    Local label for the beginning of the critical section descriptor
 200  *    structure.
 201  *  @start_ip:
 202  *    Pointer to the first instruction of the sequence of consecutive assembly
 203  *    instructions.
 204  *  @post_commit_ip:
 205  *    Pointer to the instruction after the last instruction of the sequence of
 206  *    consecutive assembly instructions.
 207  *  @abort_ip:
 208  *    Pointer to the instruction where to move the execution flow in case of
 209  *    abort of the sequence of consecutive assembly instructions.
 210  */
 211 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 212         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 213                                 (post_commit_ip) - (start_ip), abort_ip)
 214
 215 /*
 216  * Define a critical section abort handler.
 217  *
 218  *  @label:
 219  *    Local label to the abort handler.
 220  *  @teardown:
 221  *    Sequence of instructions to run on abort.
 222  *  @abort_label:
 223  *    C label to jump to at the end of the sequence.
 224  */
 225 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 226                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 227                 /*                                                      \
 228                  * Disassembler-friendly signature:                     \
 229                  *   x86-32: ud1 <sig>,%edi                             \
 230                  *   x86-64: ud1 <sig>(%rip),%edi                       \
 231                  */                                                     \
 232                 ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 233                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 234                 __rseq_str(label) ":\n\t"                               \
 235                 teardown                                                \
 236                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 237                 ".popsection\n\t"
 238
 239 /*
 240  * Define a critical section teardown handler.
 241  *
 242  *  @label:
 243  *    Local label to the teardown handler.
 244  *  @teardown:
 245  *    Sequence of instructions to run on teardown.
 246  *  @target_label:
 247  *    C label to jump to at the end of the sequence.
 248  */
 249 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)         \
 250                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 251                 __rseq_str(label) ":\n\t"                               \
 252                 teardown                                                \
 253                 "jmp %l[" __rseq_str(target_label) "]\n\t"              \
 254                 ".popsection\n\t"
 255
 256 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 257 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
 258                 RSEQ_INJECT_ASM(2)                                      \
 259                 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
 260                 "jnz " __rseq_str(label) "\n\t"
 261
 262 /* Per-cpu-id indexing. */
 263
 264 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 265 #define RSEQ_TEMPLATE_MO_RELAXED
 266 #include "rseq-x86-bits.h"
 267 #undef RSEQ_TEMPLATE_MO_RELAXED
 268
 269 #define RSEQ_TEMPLATE_MO_RELEASE
 270 #include "rseq-x86-bits.h"
 271 #undef RSEQ_TEMPLATE_MO_RELEASE
 272 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 273
 274 /* Per-mm-cid indexing. */
 275
 276 #define RSEQ_TEMPLATE_INDEX_MM_CID
 277 #define RSEQ_TEMPLATE_MO_RELAXED
 278 #include "rseq-x86-bits.h"
 279 #undef RSEQ_TEMPLATE_MO_RELAXED
 280
 281 #define RSEQ_TEMPLATE_MO_RELEASE
 282 #include "rseq-x86-bits.h"
 283 #undef RSEQ_TEMPLATE_MO_RELEASE
 284 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 285
 286 /* APIs which are not indexed. */
 287
 288 #define RSEQ_TEMPLATE_INDEX_NONE
 289 #define RSEQ_TEMPLATE_MO_RELAXED
 290 #include "rseq-x86-bits.h"
 291 #undef RSEQ_TEMPLATE_MO_RELAXED
 292 #undef RSEQ_TEMPLATE_INDEX_NONE