include/rseq/rseq-x86.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 /*
   5  * rseq-x86.h
   6  */
   7
   8 #ifndef RSEQ_H
   9 #error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
  10 #endif
  11
  12 #include <stdint.h>
  13
  14 /*
  15  * RSEQ_SIG is used with the following reserved undefined instructions, which
  16  * trap in user-space:
  17  *
  18  * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
  19  * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
  20  */
  21 #define RSEQ_SIG        0x53053053
  22
  23 /*
  24  * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
  25  * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
  26  * address through a "r" input operand.
  27  */
  28
  29 /* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */
  30 #define RSEQ_ASM_CPU_ID_OFFSET          4
  31 #define RSEQ_ASM_CS_OFFSET              8
  32 #define RSEQ_ASM_MM_CID_OFFSET          24
  33
  34 #define rseq_smp_mb()   \
  35         __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
  36 #define rseq_smp_rmb()  rseq_barrier()
  37 #define rseq_smp_wmb()  rseq_barrier()
  38
  39 #define rseq_smp_load_acquire(p)                                        \
  40 __extension__ ({                                                        \
  41         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  42         rseq_barrier();                                                 \
  43         ____p1;                                                         \
  44 })
  45
  46 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  47
  48 #define rseq_smp_store_release(p, v)                                    \
  49 do {                                                                    \
  50         rseq_barrier();                                                 \
  51         RSEQ_WRITE_ONCE(*(p), v);                                       \
  52 } while (0)
  53
  54 #ifdef __x86_64__
  55
  56 #define RSEQ_ASM_TP_SEGMENT     %%fs
  57
  58 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  59                                 start_ip, post_commit_offset, abort_ip) \
  60                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  61                 ".balign 32\n\t"                                        \
  62                 __rseq_str(label) ":\n\t"                               \
  63                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  64                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  65                 ".popsection\n\t"                                       \
  66                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  67                 ".quad " __rseq_str(label) "b\n\t"                      \
  68                 ".popsection\n\t"
  69
  70
  71 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  72         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  73                                 (post_commit_ip - start_ip), abort_ip)
  74
  75 /*
  76  * Exit points of a rseq critical section consist of all instructions outside
  77  * of the critical section where a critical section can either branch to or
  78  * reach through the normal course of its execution. The abort IP and the
  79  * post-commit IP are already part of the __rseq_cs section and should not be
  80  * explicitly defined as additional exit points. Knowing all exit points is
  81  * useful to assist debuggers stepping over the critical section.
  82  */
  83 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  84                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  85                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  86                 ".popsection\n\t"
  87
  88 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  89                 RSEQ_INJECT_ASM(1)                                      \
  90                 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
  91                 "movq %%rax, " __rseq_str(rseq_cs) "\n\t"               \
  92                 __rseq_str(label) ":\n\t"
  93
  94 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
  95                 RSEQ_INJECT_ASM(2)                                      \
  96                 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
  97                 "jnz " __rseq_str(label) "\n\t"
  98
  99 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 100                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 101                 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
 102                 ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 103                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 104                 __rseq_str(label) ":\n\t"                               \
 105                 teardown                                                \
 106                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 107                 ".popsection\n\t"
 108
 109 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)         \
 110                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 111                 __rseq_str(label) ":\n\t"                               \
 112                 teardown                                                \
 113                 "jmp %l[" __rseq_str(target_label) "]\n\t"              \
 114                 ".popsection\n\t"
 115
 116 #elif defined(__i386__)
 117
 118 #define RSEQ_ASM_TP_SEGMENT     %%gs
 119
 120 /*
 121  * Use eax as scratch register and take memory operands as input to
 122  * lessen register pressure. Especially needed when compiling in O0.
 123  */
 124 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 125                                 start_ip, post_commit_offset, abort_ip) \
 126                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 127                 ".balign 32\n\t"                                        \
 128                 __rseq_str(label) ":\n\t"                               \
 129                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 130                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 131                 ".popsection\n\t"                                       \
 132                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 133                 ".long " __rseq_str(label) "b, 0x0\n\t"                 \
 134                 ".popsection\n\t"
 135
 136 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 137         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 138                                 (post_commit_ip - start_ip), abort_ip)
 139
 140 /*
 141  * Exit points of a rseq critical section consist of all instructions outside
 142  * of the critical section where a critical section can either branch to or
 143  * reach through the normal course of its execution. The abort IP and the
 144  * post-commit IP are already part of the __rseq_cs section and should not be
 145  * explicitly defined as additional exit points. Knowing all exit points is
 146  * useful to assist debuggers stepping over the critical section.
 147  */
 148 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 149                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 150                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
 151                 ".popsection\n\t"
 152
 153 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 154                 RSEQ_INJECT_ASM(1)                                      \
 155                 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"   \
 156                 __rseq_str(label) ":\n\t"
 157
 158 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
 159                 RSEQ_INJECT_ASM(2)                                      \
 160                 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
 161                 "jnz " __rseq_str(label) "\n\t"
 162
 163 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 164                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 165                 /* Disassembler-friendly signature: ud1 <sig>,%edi. */  \
 166                 ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 167                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 168                 __rseq_str(label) ":\n\t"                               \
 169                 teardown                                                \
 170                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 171                 ".popsection\n\t"
 172
 173 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)         \
 174                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 175                 __rseq_str(label) ":\n\t"                               \
 176                 teardown                                                \
 177                 "jmp %l[" __rseq_str(target_label) "]\n\t"              \
 178                 ".popsection\n\t"
 179
 180 #endif
 181
 182 /* Per-cpu-id indexing. */
 183
 184 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 185 #define RSEQ_TEMPLATE_MO_RELAXED
 186 #include "rseq-x86-bits.h"
 187 #undef RSEQ_TEMPLATE_MO_RELAXED
 188
 189 #define RSEQ_TEMPLATE_MO_RELEASE
 190 #include "rseq-x86-bits.h"
 191 #undef RSEQ_TEMPLATE_MO_RELEASE
 192 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 193
 194 /* Per-mm-cid indexing. */
 195
 196 #define RSEQ_TEMPLATE_INDEX_MM_CID
 197 #define RSEQ_TEMPLATE_MO_RELAXED
 198 #include "rseq-x86-bits.h"
 199 #undef RSEQ_TEMPLATE_MO_RELAXED
 200
 201 #define RSEQ_TEMPLATE_MO_RELEASE
 202 #include "rseq-x86-bits.h"
 203 #undef RSEQ_TEMPLATE_MO_RELEASE
 204 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 205
 206 /* APIs which are not indexed. */
 207
 208 #define RSEQ_TEMPLATE_INDEX_NONE
 209 #define RSEQ_TEMPLATE_MO_RELAXED
 210 #include "rseq-x86-bits.h"
 211 #undef RSEQ_TEMPLATE_MO_RELAXED
 212 #undef RSEQ_TEMPLATE_INDEX_NONE