include/rseq/rseq-arm64.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3 /* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
   4
   5 /*
   6  * rseq-arm64.h
   7  */
   8
   9 /*
  10  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  11  * are not part of the public API.
  12  */
  13
  14 /*
  15  * aarch64 -mbig-endian generates mixed endianness code vs data:
  16  * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
  17  * matches code endianness.
  18  */
  19 #define RSEQ_SIG_CODE   0xd428bc00      /* BRK #0x45E0.  */
  20
  21 #ifdef __AARCH64EB__
  22 #define RSEQ_SIG_DATA   0x00bc28d4      /* BRK #0x45E0.  */
  23 #else
  24 #define RSEQ_SIG_DATA   RSEQ_SIG_CODE
  25 #endif
  26
  27 #define RSEQ_SIG        RSEQ_SIG_DATA
  28
  29 /*
  30  * Refer to the Linux kernel memory model (LKMM) for documentation of
  31  * the memory barriers.
  32  */
  33
  34 /* CPU memory barrier. */
  35 #define rseq_smp_mb()   __asm__ __volatile__ ("dmb ish" ::: "memory")
  36 /* CPU read memory barrier */
  37 #define rseq_smp_rmb()  __asm__ __volatile__ ("dmb ishld" ::: "memory")
  38 /* CPU write memory barrier */
  39 #define rseq_smp_wmb()  __asm__ __volatile__ ("dmb ishst" ::: "memory")
  40
  41 /* Acquire: One-way permeable barrier. */
  42 #define rseq_smp_load_acquire(p)                                                \
  43 __extension__ ({                                                                \
  44         union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \
  45         switch (sizeof(*(p))) {                                                 \
  46         case 1:                                                                 \
  47                 __asm__ __volatile__ ("ldarb %w0, %1"                           \
  48                         : "=r" (*(__u8 *)__u.__c)                               \
  49                         : "Q" (*(p)) : "memory");                               \
  50                 break;                                                          \
  51         case 2:                                                                 \
  52                 __asm__ __volatile__ ("ldarh %w0, %1"                           \
  53                         : "=r" (*(__u16 *)__u.__c)                              \
  54                         : "Q" (*(p)) : "memory");                               \
  55                 break;                                                          \
  56         case 4:                                                                 \
  57                 __asm__ __volatile__ ("ldar %w0, %1"                            \
  58                         : "=r" (*(__u32 *)__u.__c)                              \
  59                         : "Q" (*(p)) : "memory");                               \
  60                 break;                                                          \
  61         case 8:                                                                 \
  62                 __asm__ __volatile__ ("ldar %0, %1"                             \
  63                         : "=r" (*(__u64 *)__u.__c)                              \
  64                         : "Q" (*(p)) : "memory");                               \
  65                 break;                                                          \
  66         }                                                                       \
  67         (rseq_unqual_scalar_typeof(*(p)))__u.__val;                             \
  68 })
  69
  70 /* Acquire barrier after control dependency. */
  71 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  72
  73 /* Release: One-way permeable barrier. */
  74 #define rseq_smp_store_release(p, v)                                            \
  75 do {                                                                            \
  76         union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \
  77                 { .__val = (rseq_unqual_scalar_typeof(*(p))) (v) };             \
  78         switch (sizeof(*(p))) {                                                 \
  79         case 1:                                                                 \
  80                 __asm__ __volatile__ ("stlrb %w1, %0"                           \
  81                                 : "=Q" (*(p))                                   \
  82                                 : "r" (*(__u8 *)__u.__c)                        \
  83                                 : "memory");                                    \
  84                 break;                                                          \
  85         case 2:                                                                 \
  86                 __asm__ __volatile__ ("stlrh %w1, %0"                           \
  87                                 : "=Q" (*(p))                                   \
  88                                 : "r" (*(__u16 *)__u.__c)                       \
  89                                 : "memory");                                    \
  90                 break;                                                          \
  91         case 4:                                                                 \
  92                 __asm__ __volatile__ ("stlr %w1, %0"                            \
  93                                 : "=Q" (*(p))                                   \
  94                                 : "r" (*(__u32 *)__u.__c)                       \
  95                                 : "memory");                                    \
  96                 break;                                                          \
  97         case 8:                                                                 \
  98                 __asm__ __volatile__ ("stlr %1, %0"                             \
  99                                 : "=Q" (*(p))                                   \
 100                                 : "r" (*(__u64 *)__u.__c)                       \
 101                                 : "memory");                                    \
 102                 break;                                                          \
 103         }                                                                       \
 104 } while (0)
 105
 106 /* Temporary scratch registers. */
 107 #define RSEQ_ASM_TMP_REG32      "w15"
 108 #define RSEQ_ASM_TMP_REG        "x15"
 109 #define RSEQ_ASM_TMP_REG_2      "x14"
 110
 111 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
 112 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,                \
 113                                 post_commit_offset, abort_ip)                   \
 114         "       .pushsection    __rseq_cs, \"aw\"\n"                            \
 115         "       .balign 32\n"                                                   \
 116         __rseq_str(label) ":\n"                                                 \
 117         "       .long   " __rseq_str(version) ", " __rseq_str(flags) "\n"       \
 118         "       .quad   " __rseq_str(start_ip) ", "                             \
 119                           __rseq_str(post_commit_offset) ", "                   \
 120                           __rseq_str(abort_ip) "\n"                             \
 121         "       .popsection\n\t"                                                \
 122         "       .pushsection __rseq_cs_ptr_array, \"aw\"\n"                     \
 123         "       .quad " __rseq_str(label) "b\n"                                 \
 124         "       .popsection\n"
 125
 126 /*
 127  * Define an rseq critical section structure of version 0 with no flags.
 128  *
 129  *  @label:
 130  *    Local label for the beginning of the critical section descriptor
 131  *    structure.
 132  *  @start_ip:
 133  *    Pointer to the first instruction of the sequence of consecutive assembly
 134  *    instructions.
 135  *  @post_commit_ip:
 136  *    Pointer to the instruction after the last instruction of the sequence of
 137  *    consecutive assembly instructions.
 138  *  @abort_ip:
 139  *    Pointer to the instruction where to move the execution flow in case of
 140  *    abort of the sequence of consecutive assembly instructions.
 141  */
 142 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)        \
 143         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,                      \
 144                                 (post_commit_ip - start_ip), abort_ip)
 145
 146 /*
 147  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 148  * assembly instructions at @start_ip.
 149  *
 150  *  @start_ip:
 151  *    Pointer to the first instruction of the sequence of consecutive assembly
 152  *    instructions.
 153  *  @exit_ip:
 154  *    Pointer to an exit point instruction.
 155  *
 156  * Exit points of a rseq critical section consist of all instructions outside
 157  * of the critical section where a critical section can either branch to or
 158  * reach through the normal course of its execution. The abort IP and the
 159  * post-commit IP are already part of the __rseq_cs section and should not be
 160  * explicitly defined as additional exit points. Knowing all exit points is
 161  * useful to assist debuggers stepping over the critical section.
 162  */
 163 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                           \
 164         "       .pushsection __rseq_exit_point_array, \"aw\"\n"                 \
 165         "       .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n"      \
 166         "       .popsection\n"
 167
 168 /*
 169  * Define a critical section abort handler.
 170  *
 171  *  @label:
 172  *    Local label to the abort handler.
 173  *  @teardown:
 174  *    Sequence of instructions to run on abort.
 175  *  @abort_label:
 176  *    C label to jump to at the end of the sequence.
 177  */
 178 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)                     \
 179         "       b       222f\n"                                                 \
 180         "       .inst   "       __rseq_str(RSEQ_SIG_CODE) "\n"                  \
 181         __rseq_str(label) ":\n"                                                 \
 182         teardown                                                                \
 183         "       b       %l[" __rseq_str(abort_label) "]\n"                      \
 184         "222:\n"
 185
 186 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 187 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
 188         RSEQ_INJECT_ASM(1)                                                      \
 189         "       adrp    " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n"       \
 190         "       add     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 191                         ", :lo12:" __rseq_str(cs_label) "\n"                    \
 192         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n"     \
 193         __rseq_str(label) ":\n"
 194
 195 /* Store @value to address @var. */
 196 #define RSEQ_ASM_OP_STORE(value, var)                                           \
 197         "       str     %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 198
 199 /* Store-release @value to address @var. */
 200 #define RSEQ_ASM_OP_STORE_RELEASE(value, var)                                   \
 201         "       stlr    %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 202
 203 /*
 204  * End-of-sequence store of @value to address @var. Emit
 205  * @post_commit_label label after the store instruction.
 206  */
 207 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
 208         RSEQ_ASM_OP_STORE(value, var)                                           \
 209         __rseq_str(post_commit_label) ":\n"
 210
 211 /*
 212  * End-of-sequence store-release of @value to address @var. Emit
 213  * @post_commit_label label after the store instruction.
 214  */
 215 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)          \
 216         RSEQ_ASM_OP_STORE_RELEASE(value, var)                                   \
 217         __rseq_str(post_commit_label) ":\n"
 218
 219 /* Jump to local label @label when @var != @expect. */
 220 #define RSEQ_ASM_OP_CBNE(var, expect, label)                                    \
 221         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 222         "       sub     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 223                         ", %[" __rseq_str(expect) "]\n"                         \
 224         "       cbnz    " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
 225
 226 /*
 227  * Jump to local label @label when @var != @expect (32-bit register
 228  * comparison).
 229  */
 230 #define RSEQ_ASM_OP_CBNE32(var, expect, label)                                  \
 231         "       ldr     " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n"       \
 232         "       sub     " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32            \
 233                         ", %w[" __rseq_str(expect) "]\n"                        \
 234         "       cbnz    " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
 235
 236 /* Jump to local label @label when @var == @expect. */
 237 #define RSEQ_ASM_OP_CBEQ(var, expect, label)                                    \
 238         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 239         "       sub     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 240                         ", %[" __rseq_str(expect) "]\n"                         \
 241         "       cbz     " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
 242
 243 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 244 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)                     \
 245         RSEQ_INJECT_ASM(2)                                                      \
 246         RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)
 247
 248 /* Load @var into temporary register. */
 249 #define RSEQ_ASM_OP_R_LOAD(var)                                                 \
 250         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
 251
 252 /* Store from temporary register into @var. */
 253 #define RSEQ_ASM_OP_R_STORE(var)                                                \
 254         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
 255
 256 /* Load from address in temporary register+@offset into temporary register. */
 257 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)                                          \
 258         "       ldr     " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG               \
 259                         ", %[" __rseq_str(offset) "]]\n"
 260
 261 /* Add @count to temporary register. */
 262 #define RSEQ_ASM_OP_R_ADD(count)                                                \
 263         "       add     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 264                         ", %[" __rseq_str(count) "]\n"
 265
 266 /*
 267  * End-of-sequence store of temporary register to address @var. Emit
 268  * @post_commit_label label after the store instruction.
 269  */
 270 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
 271         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 272         __rseq_str(post_commit_label) ":\n"
 273
 274 /*
 275  * Copy @len bytes from @src to @dst. This is an inefficient bytewise
 276  * copy and could be improved in the future.
 277  */
 278 #define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len)                            \
 279         "       cbz     %[" __rseq_str(len) "], 333f\n"                         \
 280         "       mov     " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n"       \
 281         "222:   sub     " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n"   \
 282         "       ldrb    " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]"        \
 283                         ", " RSEQ_ASM_TMP_REG_2 "]\n"                           \
 284         "       strb    " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]"        \
 285                         ", " RSEQ_ASM_TMP_REG_2 "]\n"                           \
 286         "       cbnz    " RSEQ_ASM_TMP_REG_2 ", 222b\n"                         \
 287         "333:\n"
 288
 289 /* Per-cpu-id indexing. */
 290
 291 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 292 #define RSEQ_TEMPLATE_MO_RELAXED
 293 #include "rseq-arm64-bits.h"
 294 #undef RSEQ_TEMPLATE_MO_RELAXED
 295
 296 #define RSEQ_TEMPLATE_MO_RELEASE
 297 #include "rseq-arm64-bits.h"
 298 #undef RSEQ_TEMPLATE_MO_RELEASE
 299 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 300
 301 /* Per-mm-cid indexing. */
 302
 303 #define RSEQ_TEMPLATE_INDEX_MM_CID
 304 #define RSEQ_TEMPLATE_MO_RELAXED
 305 #include "rseq-arm64-bits.h"
 306 #undef RSEQ_TEMPLATE_MO_RELAXED
 307
 308 #define RSEQ_TEMPLATE_MO_RELEASE
 309 #include "rseq-arm64-bits.h"
 310 #undef RSEQ_TEMPLATE_MO_RELEASE
 311 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 312
 313 /* APIs which are not indexed. */
 314
 315 #define RSEQ_TEMPLATE_INDEX_NONE
 316 #define RSEQ_TEMPLATE_MO_RELAXED
 317 #include "rseq-arm64-bits.h"
 318 #undef RSEQ_TEMPLATE_MO_RELAXED
 319 #undef RSEQ_TEMPLATE_INDEX_NONE