include/rseq/arch/riscv.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
   3 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   4
   5 /*
   6  * rseq-riscv.h
   7  */
   8
   9 /*
  10  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  11  * are not part of the public API.
  12  */
  13
  14 #ifndef _RSEQ_RSEQ_H
  15 #error "Never use <rseq/arch/riscv.h> directly; include <rseq/rseq.h> instead."
  16 #endif
  17
  18 /*
  19  * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
  20  * other architectures, the ebreak instruction has no immediate field for
  21  * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
  22  * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
  23  * is an uncommon instruction and will raise an illegal instruction
  24  * exception when executed in all modes.
  25  */
  26 #include <endian.h>
  27
  28 #if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
  29 #define RSEQ_SIG   0xf1401073  /* csrr mhartid, x0 */
  30 #else
  31 #error "Currently, RSEQ only supports Little-Endian version"
  32 #endif
  33
  34 /*
  35  * Instruction selection between 32-bit/64-bit. Used internally in the
  36  * rseq headers.
  37  */
  38 #if __riscv_xlen == 64
  39 #define __RSEQ_ASM_REG_SEL(a, b)        a
  40 #elif __riscv_xlen == 32
  41 #define __RSEQ_ASM_REG_SEL(a, b)        b
  42 #endif
  43
  44 #define RSEQ_ASM_REG_L  __RSEQ_ASM_REG_SEL("ld ", "lw ")
  45 #define RSEQ_ASM_REG_S  __RSEQ_ASM_REG_SEL("sd ", "sw ")
  46
  47 /*
  48  * Refer to the Linux kernel memory model (LKMM) for documentation of
  49  * the memory barriers.
  50  */
  51
  52 /* Only used internally in rseq headers. */
  53 #define RSEQ_ASM_RISCV_FENCE(p, s) \
  54         __asm__ __volatile__ ("fence " #p "," #s : : : "memory")
  55 /* CPU memory barrier. */
  56 #define rseq_smp_mb()   RSEQ_ASM_RISCV_FENCE(rw, rw)
  57 /* CPU read memory barrier */
  58 #define rseq_smp_rmb()  RSEQ_ASM_RISCV_FENCE(r, r)
  59 /* CPU write memory barrier */
  60 #define rseq_smp_wmb()  RSEQ_ASM_RISCV_FENCE(w, w)
  61
  62 /* Acquire: One-way permeable barrier. */
  63 #define rseq_smp_load_acquire(p)                                        \
  64 __extension__ ({                                                        \
  65         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  66         RSEQ_ASM_RISCV_FENCE(r, rw);                                    \
  67         ____p1;                                                         \
  68 })
  69
  70 /* Acquire barrier after control dependency. */
  71 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  72
  73 /* Release: One-way permeable barrier. */
  74 #define rseq_smp_store_release(p, v)                                    \
  75 do {                                                                    \
  76         RSEQ_ASM_RISCV_FENCE(rw, w);                                    \
  77         RSEQ_WRITE_ONCE(*(p), v);                                       \
  78 } while (0)
  79
  80 #define RSEQ_ASM_U64_PTR(x)     ".quad " x
  81 #define RSEQ_ASM_U32(x)         ".long " x
  82
  83 /* Temporary registers. */
  84 #define RSEQ_ASM_TMP_REG_1      "t6"
  85 #define RSEQ_ASM_TMP_REG_2      "t5"
  86 #define RSEQ_ASM_TMP_REG_3      "t4"
  87 #define RSEQ_ASM_TMP_REG_4      "t3"
  88
  89 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
  90 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,        \
  91                                 post_commit_offset, abort_ip)           \
  92         ".pushsection   __rseq_cs, \"aw\"\n"                            \
  93         ".balign        32\n"                                           \
  94         __rseq_str(label) ":\n"                                         \
  95         RSEQ_ASM_U32(__rseq_str(version)) "\n"                          \
  96         RSEQ_ASM_U32(__rseq_str(flags)) "\n"                            \
  97         RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n"                     \
  98         RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n"           \
  99         RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n"                     \
 100         ".popsection\n\t"                                               \
 101         ".pushsection __rseq_cs_ptr_array, \"aw\"\n"                    \
 102         RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n"                    \
 103         ".popsection\n"
 104
 105 /*
 106  * Define an rseq critical section structure of version 0 with no flags.
 107  *
 108  *  @label:
 109  *    Local label for the beginning of the critical section descriptor
 110  *    structure.
 111  *  @start_ip:
 112  *    Pointer to the first instruction of the sequence of consecutive assembly
 113  *    instructions.
 114  *  @post_commit_ip:
 115  *    Pointer to the instruction after the last instruction of the sequence of
 116  *    consecutive assembly instructions.
 117  *  @abort_ip:
 118  *    Pointer to the instruction where to move the execution flow in case of
 119  *    abort of the sequence of consecutive assembly instructions.
 120  */
 121 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 122         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,               \
 123                                 (post_commit_ip) - (start_ip), abort_ip)
 124
 125 /*
 126  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 127  * assembly instructions at @start_ip.
 128  *
 129  *  @start_ip:
 130  *    Pointer to the first instruction of the sequence of consecutive assembly
 131  *    instructions.
 132  *  @exit_ip:
 133  *    Pointer to an exit point instruction.
 134  *
 135  * Exit points of a rseq critical section consist of all instructions outside
 136  * of the critical section where a critical section can either branch to or
 137  * reach through the normal course of its execution. The abort IP and the
 138  * post-commit IP are already part of the __rseq_cs section and should not be
 139  * explicitly defined as additional exit points. Knowing all exit points is
 140  * useful to assist debuggers stepping over the critical section.
 141  */
 142 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 143         ".pushsection __rseq_exit_point_array, \"aw\"\n"                \
 144         RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n"                     \
 145         RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n"                      \
 146         ".popsection\n"
 147
 148 /*
 149  * Define a critical section abort handler.
 150  *
 151  *  @label:
 152  *    Local label to the abort handler.
 153  *  @teardown:
 154  *    Sequence of instructions to run on abort.
 155  *  @abort_label:
 156  *    C label to jump to at the end of the sequence.
 157  */
 158 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 159         "j      222f\n"                                                 \
 160         ".balign        4\n"                                            \
 161         RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n"                         \
 162         __rseq_str(label) ":\n"                                         \
 163         teardown                                                        \
 164         "j      %l[" __rseq_str(abort_label) "]\n"                      \
 165         "222:\n"
 166
 167 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 168 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 169         RSEQ_INJECT_ASM(1)                                              \
 170         "la     " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n"     \
 171         RSEQ_ASM_REG_S  RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \
 172         __rseq_str(label) ":\n"
 173
 174 /* Store @value to address @var. */
 175 #define RSEQ_ASM_OP_STORE(value, var)                                   \
 176         RSEQ_ASM_REG_S  "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 177
 178 /* Jump to local label @label when @var != @expect. */
 179 #define RSEQ_ASM_OP_CBNE(var, expect, label)                            \
 180         RSEQ_ASM_REG_L  RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
 181         "bne    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
 182                   __rseq_str(label) "\n"
 183
 184 /*
 185  * Jump to local label @label when @var != @expect (32-bit register
 186  * comparison).
 187  */
 188 #define RSEQ_ASM_OP_CBNE32(var, expect, label)                          \
 189         "lw     " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"       \
 190         "bne    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
 191                   __rseq_str(label) "\n"
 192
 193 /* Jump to local label @label when @var == @expect. */
 194 #define RSEQ_ASM_OP_CBEQ(var, expect, label)                            \
 195         RSEQ_ASM_REG_L  RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
 196         "beq    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"    \
 197                   __rseq_str(label) "\n"
 198
 199 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 200 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
 201         RSEQ_INJECT_ASM(2)                                              \
 202         RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)
 203
 204 /* Load @var into temporary register. */
 205 #define RSEQ_ASM_OP_R_LOAD(var)                                         \
 206         RSEQ_ASM_REG_L  RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
 207
 208 /* Store from temporary register into @var. */
 209 #define RSEQ_ASM_OP_R_STORE(var)                                        \
 210         RSEQ_ASM_REG_S  RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
 211
 212 /* Load from address in temporary register+@offset into temporary register. */
 213 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)                                  \
 214         "add    " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], "    \
 215                  RSEQ_ASM_TMP_REG_1 "\n"                                \
 216         RSEQ_ASM_REG_L  RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"
 217
 218 /* Add @count to temporary register. */
 219 #define RSEQ_ASM_OP_R_ADD(count)                                        \
 220         "add    " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1            \
 221                 ", %[" __rseq_str(count) "]\n"
 222
 223 /*
 224  * End-of-sequence store of @value to address @var. Emit
 225  * @post_commit_label label after the store instruction.
 226  */
 227 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)          \
 228         RSEQ_ASM_OP_STORE(value, var)                                   \
 229         __rseq_str(post_commit_label) ":\n"
 230
 231 /*
 232  * End-of-sequence store-release of @value to address @var. Emit
 233  * @post_commit_label label after the store instruction.
 234  */
 235 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)  \
 236         "fence  rw, w\n"                                                \
 237         RSEQ_ASM_OP_STORE(value, var)                                   \
 238         __rseq_str(post_commit_label) ":\n"
 239
 240 /*
 241  * End-of-sequence store of temporary register to address @var. Emit
 242  * @post_commit_label label after the store instruction.
 243  */
 244 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)               \
 245         RSEQ_ASM_REG_S  RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
 246         __rseq_str(post_commit_label) ":\n"
 247
 248 /*
 249  * Copy @len bytes from @src to @dst. This is an inefficient bytewise
 250  * copy and could be improved in the future.
 251  */
 252 #define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len)                    \
 253         "beqz   %[" __rseq_str(len) "], 333f\n"                         \
 254         "mv     " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n"       \
 255         "mv     " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n"       \
 256         "mv     " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n"       \
 257         "222:\n"                                                        \
 258         "lb     " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n"    \
 259         "sb     " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n"    \
 260         "addi   " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n"   \
 261         "addi   " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n"    \
 262         "addi   " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n"    \
 263         "bnez   " RSEQ_ASM_TMP_REG_1 ", 222b\n"                         \
 264         "333:\n"
 265
 266 /*
 267  * Load pointer address from @ptr. Add @off to offset from this pointer.
 268  * Add @inc to the resulting address as an end-of-sequence store.
 269  */
 270 #define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, inc, post_commit_label)      \
 271         "mv     " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n"       \
 272         RSEQ_ASM_OP_R_ADD(off)                                          \
 273         RSEQ_ASM_REG_L    RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \
 274         RSEQ_ASM_OP_R_ADD(inc)                                          \
 275         __rseq_str(post_commit_label) ":\n"
 276
 277 /* Per-cpu-id indexing. */
 278
 279 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 280 #define RSEQ_TEMPLATE_MO_RELAXED
 281 #include "rseq/arch/riscv/bits.h"
 282 #undef RSEQ_TEMPLATE_MO_RELAXED
 283
 284 #define RSEQ_TEMPLATE_MO_RELEASE
 285 #include "rseq/arch/riscv/bits.h"
 286 #undef RSEQ_TEMPLATE_MO_RELEASE
 287 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 288
 289 /* Per-mm-cid indexing. */
 290
 291 #define RSEQ_TEMPLATE_INDEX_MM_CID
 292 #define RSEQ_TEMPLATE_MO_RELAXED
 293 #include "rseq/arch/riscv/bits.h"
 294 #undef RSEQ_TEMPLATE_MO_RELAXED
 295
 296 #define RSEQ_TEMPLATE_MO_RELEASE
 297 #include "rseq/arch/riscv/bits.h"
 298 #undef RSEQ_TEMPLATE_MO_RELEASE
 299 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 300
 301 /* APIs which are not indexed. */
 302
 303 #define RSEQ_TEMPLATE_INDEX_NONE
 304 #define RSEQ_TEMPLATE_MO_RELAXED
 305 #include "rseq/arch/riscv/bits.h"
 306 #undef RSEQ_TEMPLATE_MO_RELAXED
 307 #undef RSEQ_TEMPLATE_INDEX_NONE