include/rseq/arch/aarch64.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3 /* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
   4
   5 /*
   6  * rseq/arch/aarch64.h
   7  */
   8
   9 #ifndef _RSEQ_RSEQ_H
  10 #error "Never use <rseq/arch/aarch64.h> directly; include <rseq/rseq.h> instead."
  11 #endif
  12
  13 /*
  14  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  15  * are not part of the public API.
  16  */
  17
  18 /*
  19  * aarch64 -mbig-endian generates mixed endianness code vs data:
  20  * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
  21  * matches code endianness.
  22  */
  23 #define RSEQ_SIG_CODE   0xd428bc00      /* BRK #0x45E0.  */
  24
  25 #ifdef __AARCH64EB__    /* Big endian */
  26 # define RSEQ_SIG_DATA  0x00bc28d4      /* BRK #0x45E0.  */
  27 #else                   /* Little endian */
  28 # define RSEQ_SIG_DATA  RSEQ_SIG_CODE
  29 #endif
  30
  31 #define RSEQ_SIG        RSEQ_SIG_DATA
  32
  33 /*
  34  * Refer to the Linux kernel memory model (LKMM) for documentation of
  35  * the memory barriers.
  36  */
  37
  38 /* CPU memory barrier. */
  39 #define rseq_smp_mb()   __asm__ __volatile__ ("dmb ish" ::: "memory")
  40 /* CPU read memory barrier */
  41 #define rseq_smp_rmb()  __asm__ __volatile__ ("dmb ishld" ::: "memory")
  42 /* CPU write memory barrier */
  43 #define rseq_smp_wmb()  __asm__ __volatile__ ("dmb ishst" ::: "memory")
  44
  45 /* Acquire: One-way permeable barrier. */
  46 #define rseq_smp_load_acquire(p)                                                \
  47 __extension__ ({                                                                \
  48         union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \
  49         switch (sizeof(*(p))) {                                                 \
  50         case 1:                                                                 \
  51                 __asm__ __volatile__ ("ldarb %w0, %1"                           \
  52                         : "=r" (*(__u8 *)__u.__c)                               \
  53                         : "Q" (*(p)) : "memory");                               \
  54                 break;                                                          \
  55         case 2:                                                                 \
  56                 __asm__ __volatile__ ("ldarh %w0, %1"                           \
  57                         : "=r" (*(__u16 *)__u.__c)                              \
  58                         : "Q" (*(p)) : "memory");                               \
  59                 break;                                                          \
  60         case 4:                                                                 \
  61                 __asm__ __volatile__ ("ldar %w0, %1"                            \
  62                         : "=r" (*(__u32 *)__u.__c)                              \
  63                         : "Q" (*(p)) : "memory");                               \
  64                 break;                                                          \
  65         case 8:                                                                 \
  66                 __asm__ __volatile__ ("ldar %0, %1"                             \
  67                         : "=r" (*(__u64 *)__u.__c)                              \
  68                         : "Q" (*(p)) : "memory");                               \
  69                 break;                                                          \
  70         }                                                                       \
  71         (rseq_unqual_scalar_typeof(*(p)))__u.__val;                             \
  72 })
  73
  74 /* Acquire barrier after control dependency. */
  75 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  76
  77 /* Release: One-way permeable barrier. */
  78 #define rseq_smp_store_release(p, v)                                            \
  79 do {                                                                            \
  80         union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \
  81                 { .__val = (rseq_unqual_scalar_typeof(*(p))) (v) };             \
  82         switch (sizeof(*(p))) {                                                 \
  83         case 1:                                                                 \
  84                 __asm__ __volatile__ ("stlrb %w1, %0"                           \
  85                                 : "=Q" (*(p))                                   \
  86                                 : "r" (*(__u8 *)__u.__c)                        \
  87                                 : "memory");                                    \
  88                 break;                                                          \
  89         case 2:                                                                 \
  90                 __asm__ __volatile__ ("stlrh %w1, %0"                           \
  91                                 : "=Q" (*(p))                                   \
  92                                 : "r" (*(__u16 *)__u.__c)                       \
  93                                 : "memory");                                    \
  94                 break;                                                          \
  95         case 4:                                                                 \
  96                 __asm__ __volatile__ ("stlr %w1, %0"                            \
  97                                 : "=Q" (*(p))                                   \
  98                                 : "r" (*(__u32 *)__u.__c)                       \
  99                                 : "memory");                                    \
 100                 break;                                                          \
 101         case 8:                                                                 \
 102                 __asm__ __volatile__ ("stlr %1, %0"                             \
 103                                 : "=Q" (*(p))                                   \
 104                                 : "r" (*(__u64 *)__u.__c)                       \
 105                                 : "memory");                                    \
 106                 break;                                                          \
 107         }                                                                       \
 108 } while (0)
 109
 110 #define RSEQ_ASM_U64_PTR(x)     ".quad " x
 111 #define RSEQ_ASM_U32(x)         ".long " x
 112
 113 /* Temporary scratch registers. */
 114 #define RSEQ_ASM_TMP_REG32      "w15"
 115 #define RSEQ_ASM_TMP_REG        "x15"
 116 #define RSEQ_ASM_TMP_REG_2      "x14"
 117
 118 /* Common architecture support macros. */
 119 #include "rseq/arch/generic/common.h"
 120
 121 /*
 122  * Define a critical section abort handler.
 123  *
 124  *  @label:
 125  *    Local label to the abort handler.
 126  *  @teardown:
 127  *    Sequence of instructions to run on abort.
 128  *  @abort_label:
 129  *    C label to jump to at the end of the sequence.
 130  */
 131 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)                     \
 132         "       b       222f\n"                                                 \
 133         "       .inst   "       __rseq_str(RSEQ_SIG_CODE) "\n"                  \
 134         __rseq_str(label) ":\n"                                                 \
 135         teardown                                                                \
 136         "       b       %l[" __rseq_str(abort_label) "]\n"                      \
 137         "222:\n"
 138
 139 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 140 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
 141         RSEQ_INJECT_ASM(1)                                                      \
 142         "       adrp    " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n"       \
 143         "       add     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 144                         ", :lo12:" __rseq_str(cs_label) "\n"                    \
 145         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n"     \
 146         __rseq_str(label) ":\n"
 147
 148 /* Store @value to address @var. */
 149 #define RSEQ_ASM_OP_STORE(value, var)                                           \
 150         "       str     %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 151
 152 /* Store-release @value to address @var. */
 153 #define RSEQ_ASM_OP_STORE_RELEASE(value, var)                                   \
 154         "       stlr    %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 155
 156 /*
 157  * End-of-sequence store of @value to address @var. Emit
 158  * @post_commit_label label after the store instruction.
 159  */
 160 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
 161         RSEQ_ASM_OP_STORE(value, var)                                           \
 162         __rseq_str(post_commit_label) ":\n"
 163
 164 /*
 165  * End-of-sequence store-release of @value to address @var. Emit
 166  * @post_commit_label label after the store instruction.
 167  */
 168 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)          \
 169         RSEQ_ASM_OP_STORE_RELEASE(value, var)                                   \
 170         __rseq_str(post_commit_label) ":\n"
 171
 172 /* Jump to local label @label when @var != @expect. */
 173 #define RSEQ_ASM_OP_CBNE(var, expect, label)                                    \
 174         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 175         "       sub     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 176                         ", %[" __rseq_str(expect) "]\n"                         \
 177         "       cbnz    " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
 178
 179 /*
 180  * Jump to local label @label when @var != @expect (32-bit register
 181  * comparison).
 182  */
 183 #define RSEQ_ASM_OP_CBNE32(var, expect, label)                                  \
 184         "       ldr     " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n"       \
 185         "       sub     " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32            \
 186                         ", %w[" __rseq_str(expect) "]\n"                        \
 187         "       cbnz    " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
 188
 189 /* Jump to local label @label when @var == @expect. */
 190 #define RSEQ_ASM_OP_CBEQ(var, expect, label)                                    \
 191         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 192         "       sub     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 193                         ", %[" __rseq_str(expect) "]\n"                         \
 194         "       cbz     " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
 195
 196 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 197 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)                     \
 198         RSEQ_INJECT_ASM(2)                                                      \
 199         RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)
 200
 201 /* Load @var into temporary register. */
 202 #define RSEQ_ASM_OP_R_LOAD(var)                                                 \
 203         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
 204
 205 /* Store from temporary register into @var. */
 206 #define RSEQ_ASM_OP_R_STORE(var)                                                \
 207         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
 208
 209 /* Load from address in temporary register+@offset into temporary register. */
 210 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)                                          \
 211         "       ldr     " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG               \
 212                         ", %[" __rseq_str(offset) "]]\n"
 213
 214 /* Add @count to temporary register. */
 215 #define RSEQ_ASM_OP_R_ADD(count)                                                \
 216         "       add     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 217                         ", %[" __rseq_str(count) "]\n"
 218
 219 /*
 220  * End-of-sequence store of temporary register to address @var. Emit
 221  * @post_commit_label label after the store instruction.
 222  */
 223 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
 224         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 225         __rseq_str(post_commit_label) ":\n"
 226
 227 /*
 228  * Copy @len bytes from @src to @dst. This is an inefficient bytewise
 229  * copy and could be improved in the future.
 230  */
 231 #define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len)                            \
 232         "       cbz     %[" __rseq_str(len) "], 333f\n"                         \
 233         "       mov     " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n"       \
 234         "222:   sub     " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n"   \
 235         "       ldrb    " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]"        \
 236                         ", " RSEQ_ASM_TMP_REG_2 "]\n"                           \
 237         "       strb    " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]"        \
 238                         ", " RSEQ_ASM_TMP_REG_2 "]\n"                           \
 239         "       cbnz    " RSEQ_ASM_TMP_REG_2 ", 222b\n"                         \
 240         "333:\n"
 241
 242 /* Per-cpu-id indexing. */
 243
 244 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 245 #define RSEQ_TEMPLATE_MO_RELAXED
 246 #include "rseq/arch/aarch64/bits.h"
 247 #undef RSEQ_TEMPLATE_MO_RELAXED
 248
 249 #define RSEQ_TEMPLATE_MO_RELEASE
 250 #include "rseq/arch/aarch64/bits.h"
 251 #undef RSEQ_TEMPLATE_MO_RELEASE
 252 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 253
 254 /* Per-mm-cid indexing. */
 255
 256 #define RSEQ_TEMPLATE_INDEX_MM_CID
 257 #define RSEQ_TEMPLATE_MO_RELAXED
 258 #include "rseq/arch/aarch64/bits.h"
 259 #undef RSEQ_TEMPLATE_MO_RELAXED
 260
 261 #define RSEQ_TEMPLATE_MO_RELEASE
 262 #include "rseq/arch/aarch64/bits.h"
 263 #undef RSEQ_TEMPLATE_MO_RELEASE
 264 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 265
 266 /* APIs which are not indexed. */
 267
 268 #define RSEQ_TEMPLATE_INDEX_NONE
 269 #define RSEQ_TEMPLATE_MO_RELAXED
 270 #include "rseq/arch/aarch64/bits.h"
 271 #undef RSEQ_TEMPLATE_MO_RELAXED
 272 #undef RSEQ_TEMPLATE_INDEX_NONE