| 1 | /* SPDX-License-Identifier: MIT */ |
| 2 | /* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */ |
| 3 | |
| 4 | /* |
| 5 | * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike |
| 6 | * other architectures, the ebreak instruction has no immediate field for |
| 7 | * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG. |
| 8 | * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it |
| 9 | * is an uncommon instruction and will raise an illegal instruction |
| 10 | * exception when executed in all modes. |
| 11 | */ |
| 12 | #include <endian.h> |
| 13 | |
| 14 | #if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN) |
| 15 | #define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */ |
| 16 | #else |
| 17 | #error "Currently, RSEQ only supports Little-Endian version" |
| 18 | #endif |
| 19 | |
| 20 | #if __riscv_xlen == 64 |
| 21 | #define __REG_SEL(a, b) a |
| 22 | #elif __riscv_xlen == 32 |
| 23 | #define __REG_SEL(a, b) b |
| 24 | #endif |
| 25 | |
| 26 | #define REG_L __REG_SEL("ld ", "lw ") |
| 27 | #define REG_S __REG_SEL("sd ", "sw ") |
| 28 | |
| 29 | #define RISCV_FENCE(p, s) \ |
| 30 | __asm__ __volatile__ ("fence " #p "," #s : : : "memory") |
| 31 | #define rseq_smp_mb() RISCV_FENCE(rw, rw) |
| 32 | #define rseq_smp_rmb() RISCV_FENCE(r, r) |
| 33 | #define rseq_smp_wmb() RISCV_FENCE(w, w) |
| 34 | #define RSEQ_ASM_TMP_REG_1 "t6" |
| 35 | #define RSEQ_ASM_TMP_REG_2 "t5" |
| 36 | #define RSEQ_ASM_TMP_REG_3 "t4" |
| 37 | #define RSEQ_ASM_TMP_REG_4 "t3" |
| 38 | |
| 39 | #define rseq_smp_load_acquire(p) \ |
| 40 | __extension__ ({ \ |
| 41 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
| 42 | RISCV_FENCE(r, rw); \ |
| 43 | ____p1; \ |
| 44 | }) |
| 45 | |
| 46 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() |
| 47 | |
| 48 | #define rseq_smp_store_release(p, v) \ |
| 49 | do { \ |
| 50 | RISCV_FENCE(rw, w); \ |
| 51 | RSEQ_WRITE_ONCE(*(p), v); \ |
| 52 | } while (0) |
| 53 | |
| 54 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ |
| 55 | post_commit_offset, abort_ip) \ |
| 56 | ".pushsection __rseq_cs, \"aw\"\n" \ |
| 57 | ".balign 32\n" \ |
| 58 | __rseq_str(label) ":\n" \ |
| 59 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ |
| 60 | ".quad " __rseq_str(start_ip) ", " \ |
| 61 | __rseq_str(post_commit_offset) ", " \ |
| 62 | __rseq_str(abort_ip) "\n" \ |
| 63 | ".popsection\n\t" \ |
| 64 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n" \ |
| 65 | ".quad " __rseq_str(label) "b\n" \ |
| 66 | ".popsection\n" |
| 67 | |
| 68 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
| 69 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ |
| 70 | ((post_commit_ip) - (start_ip)), abort_ip) |
| 71 | |
| 72 | /* |
| 73 | * Exit points of a rseq critical section consist of all instructions outside |
| 74 | * of the critical section where a critical section can either branch to or |
| 75 | * reach through the normal course of its execution. The abort IP and the |
| 76 | * post-commit IP are already part of the __rseq_cs section and should not be |
| 77 | * explicitly defined as additional exit points. Knowing all exit points is |
| 78 | * useful to assist debuggers stepping over the critical section. |
| 79 | */ |
| 80 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ |
| 81 | ".pushsection __rseq_exit_point_array, \"aw\"\n" \ |
| 82 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ |
| 83 | ".popsection\n" |
| 84 | |
| 85 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
| 86 | RSEQ_INJECT_ASM(1) \ |
| 87 | "la " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \ |
| 88 | REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \ |
| 89 | __rseq_str(label) ":\n" |
| 90 | |
| 91 | #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ |
| 92 | "j 222f\n" \ |
| 93 | ".balign 4\n" \ |
| 94 | ".long " __rseq_str(RSEQ_SIG) "\n" \ |
| 95 | __rseq_str(label) ":\n" \ |
| 96 | "j %l[" __rseq_str(abort_label) "]\n" \ |
| 97 | "222:\n" |
| 98 | |
| 99 | #define RSEQ_ASM_OP_STORE(value, var) \ |
| 100 | REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" |
| 101 | |
| 102 | #define RSEQ_ASM_OP_CBNE(var, expect, label) \ |
| 103 | REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ |
| 104 | "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ |
| 105 | __rseq_str(label) "\n" |
| 106 | |
| 107 | #define RSEQ_ASM_OP_CBNE32(var, expect, label) \ |
| 108 | "lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ |
| 109 | "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ |
| 110 | __rseq_str(label) "\n" |
| 111 | |
| 112 | #define RSEQ_ASM_OP_CBEQ(var, expect, label) \ |
| 113 | REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ |
| 114 | "beq " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ |
| 115 | __rseq_str(label) "\n" |
| 116 | |
| 117 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ |
| 118 | RSEQ_INJECT_ASM(2) \ |
| 119 | RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label) |
| 120 | |
| 121 | #define RSEQ_ASM_OP_R_LOAD(var) \ |
| 122 | REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" |
| 123 | |
| 124 | #define RSEQ_ASM_OP_R_STORE(var) \ |
| 125 | REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" |
| 126 | |
| 127 | #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ |
| 128 | "add " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \ |
| 129 | RSEQ_ASM_TMP_REG_1 "\n" \ |
| 130 | REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n" |
| 131 | |
| 132 | #define RSEQ_ASM_OP_R_ADD(count) \ |
| 133 | "add " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \ |
| 134 | ", %[" __rseq_str(count) "]\n" |
| 135 | |
| 136 | #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ |
| 137 | RSEQ_ASM_OP_STORE(value, var) \ |
| 138 | __rseq_str(post_commit_label) ":\n" |
| 139 | |
| 140 | #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \ |
| 141 | "fence rw, w\n" \ |
| 142 | RSEQ_ASM_OP_STORE(value, var) \ |
| 143 | __rseq_str(post_commit_label) ":\n" |
| 144 | |
| 145 | #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ |
| 146 | REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ |
| 147 | __rseq_str(post_commit_label) ":\n" |
| 148 | |
| 149 | #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \ |
| 150 | "beqz %[" __rseq_str(len) "], 333f\n" \ |
| 151 | "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \ |
| 152 | "mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \ |
| 153 | "mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \ |
| 154 | "222:\n" \ |
| 155 | "lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \ |
| 156 | "sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \ |
| 157 | "addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \ |
| 158 | "addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \ |
| 159 | "addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \ |
| 160 | "bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \ |
| 161 | "333:\n" |
| 162 | |
| 163 | #define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label) \ |
| 164 | "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \ |
| 165 | RSEQ_ASM_OP_R_ADD(off) \ |
| 166 | REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \ |
| 167 | RSEQ_ASM_OP_R_ADD(inc) \ |
| 168 | __rseq_str(post_commit_label) ":\n" |
| 169 | |
| 170 | /* Per-cpu-id indexing. */ |
| 171 | |
| 172 | #define RSEQ_TEMPLATE_CPU_ID |
| 173 | #define RSEQ_TEMPLATE_MO_RELAXED |
| 174 | #include "rseq-riscv-bits.h" |
| 175 | #undef RSEQ_TEMPLATE_MO_RELAXED |
| 176 | |
| 177 | #define RSEQ_TEMPLATE_MO_RELEASE |
| 178 | #include "rseq-riscv-bits.h" |
| 179 | #undef RSEQ_TEMPLATE_MO_RELEASE |
| 180 | #undef RSEQ_TEMPLATE_CPU_ID |
| 181 | |
| 182 | /* Per-mm-cid indexing. */ |
| 183 | |
| 184 | #define RSEQ_TEMPLATE_MM_CID |
| 185 | #define RSEQ_TEMPLATE_MO_RELAXED |
| 186 | #include "rseq-riscv-bits.h" |
| 187 | #undef RSEQ_TEMPLATE_MO_RELAXED |
| 188 | |
| 189 | #define RSEQ_TEMPLATE_MO_RELEASE |
| 190 | #include "rseq-riscv-bits.h" |
| 191 | #undef RSEQ_TEMPLATE_MO_RELEASE |
| 192 | #undef RSEQ_TEMPLATE_MM_CID |
| 193 | |
| 194 | /* APIs which are not based on cpu ids. */ |
| 195 | |
| 196 | #define RSEQ_TEMPLATE_CPU_ID_NONE |
| 197 | #define RSEQ_TEMPLATE_MO_RELAXED |
| 198 | #include "rseq-riscv-bits.h" |
| 199 | #undef RSEQ_TEMPLATE_MO_RELAXED |
| 200 | #undef RSEQ_TEMPLATE_CPU_ID_NONE |