projects / librseq.git / blob
? search:


7fa1ffc8d3134e510ce0fbff3181d03002b25c2b
[librseq.git] / include / rseq / rseq-riscv-bits.h
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
3
4 #include "rseq-bits-template.h"
5
6 /*
7 * Refer to rseq-pseudocode.h for documentation and pseudo-code of the
8 * rseq critical section helpers.
9 */
10 #include "rseq-pseudocode.h"
11
12 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
13 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
14
15 static inline __attribute__((always_inline))
16 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store__ptr)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
17 {
18 RSEQ_INJECT_C(9)
19
20 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
21 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[ne]")
22 #ifdef RSEQ_COMPARE_TWICE
23 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
24 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
25 #endif
26 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
27 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
28 RSEQ_INJECT_ASM(3)
29 RSEQ_ASM_OP_CBNE(v, expect, "%l[ne]")
30 RSEQ_INJECT_ASM(4)
31 #ifdef RSEQ_COMPARE_TWICE
32 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
33 RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]")
34 #endif
35 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
36 RSEQ_INJECT_ASM(5)
37 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
38 : /* gcc asm goto does not allow outputs */
39 : [cpu_id] "r" (cpu),
40 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
41 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
42 [v] "m" (*v),
43 [expect] "r" (expect),
44 [newv] "r" (newv)
45 RSEQ_INJECT_INPUT
46 : "memory", RSEQ_ASM_TMP_REG_1
47 RSEQ_INJECT_CLOBBER
48 : abort, ne
49 #ifdef RSEQ_COMPARE_TWICE
50 , error1, error2
51 #endif
52 );
53
54 return 0;
55 abort:
56 RSEQ_INJECT_FAILED
57 return -1;
58 ne:
59 return 1;
60 #ifdef RSEQ_COMPARE_TWICE
61 error1:
62 rseq_bug("cpu_id comparison failed");
63 error2:
64 rseq_bug("expected value comparison failed");
65 #endif
66 }
67
68 static inline __attribute__((always_inline))
69 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbeq_store_add_load_store__ptr)(intptr_t *v, intptr_t expectnot,
70 off_t voffp, intptr_t *load, int cpu)
71 {
72 RSEQ_INJECT_C(9)
73
74 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
75 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[eq]")
76 #ifdef RSEQ_COMPARE_TWICE
77 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
78 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
79 #endif
80 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
81 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
82 RSEQ_INJECT_ASM(3)
83 RSEQ_ASM_OP_CBEQ(v, expectnot, "%l[eq]")
84 RSEQ_INJECT_ASM(4)
85 #ifdef RSEQ_COMPARE_TWICE
86 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
87 RSEQ_ASM_OP_CBEQ(v, expectnot, "%l[error2]")
88 #endif
89 RSEQ_ASM_OP_R_LOAD(v)
90 RSEQ_ASM_OP_R_STORE(load)
91 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
92 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
93 RSEQ_INJECT_ASM(5)
94 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
95 : /* gcc asm goto does not allow outputs */
96 : [cpu_id] "r" (cpu),
97 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
98 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
99 [v] "m" (*v),
100 [expectnot] "r" (expectnot),
101 [load] "m" (*load),
102 [voffp] "r" (voffp)
103 RSEQ_INJECT_INPUT
104 : "memory", RSEQ_ASM_TMP_REG_1
105 RSEQ_INJECT_CLOBBER
106 : abort, eq
107 #ifdef RSEQ_COMPARE_TWICE
108 , error1, error2
109 #endif
110 );
111 return 0;
112 abort:
113 RSEQ_INJECT_FAILED
114 return -1;
115 eq:
116 return 1;
117 #ifdef RSEQ_COMPARE_TWICE
118 error1:
119 rseq_bug("cpu_id comparison failed");
120 error2:
121 rseq_bug("expected value comparison failed");
122 #endif
123 }
124
125 static inline __attribute__((always_inline))
126 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_add_store__ptr)(intptr_t *v, intptr_t count, int cpu)
127 {
128 RSEQ_INJECT_C(9)
129
130 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
131 #ifdef RSEQ_COMPARE_TWICE
132 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
133 #endif
134 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
135 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
136 RSEQ_INJECT_ASM(3)
137 #ifdef RSEQ_COMPARE_TWICE
138 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
139 #endif
140 RSEQ_ASM_OP_R_LOAD(v)
141 RSEQ_ASM_OP_R_ADD(count)
142 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
143 RSEQ_INJECT_ASM(4)
144 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
145 : /* gcc asm goto does not allow outputs */
146 : [cpu_id] "r" (cpu),
147 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
148 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
149 [v] "m" (*v),
150 [count] "r" (count)
151 RSEQ_INJECT_INPUT
152 : "memory", RSEQ_ASM_TMP_REG_1
153 RSEQ_INJECT_CLOBBER
154 : abort
155 #ifdef RSEQ_COMPARE_TWICE
156 , error1
157 #endif
158 );
159 return 0;
160 abort:
161 RSEQ_INJECT_FAILED
162 return -1;
163 #ifdef RSEQ_COMPARE_TWICE
164 error1:
165 rseq_bug("cpu_id comparison failed");
166 #endif
167 }
168
169 static inline __attribute__((always_inline))
170 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_load_cbne_store__ptr)(intptr_t *v, intptr_t expect,
171 intptr_t *v2, intptr_t expect2,
172 intptr_t newv, int cpu)
173 {
174 RSEQ_INJECT_C(9)
175
176 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
177 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[ne]")
178 #ifdef RSEQ_COMPARE_TWICE
179 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
180 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
181 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
182 #endif
183 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
184 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
185 RSEQ_INJECT_ASM(3)
186 RSEQ_ASM_OP_CBNE(v, expect, "%l[ne]")
187 RSEQ_INJECT_ASM(4)
188 RSEQ_ASM_OP_CBNE(v2, expect2, "%l[ne]")
189 RSEQ_INJECT_ASM(5)
190 #ifdef RSEQ_COMPARE_TWICE
191 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
192 RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]")
193 RSEQ_ASM_OP_CBNE(v2, expect2, "%l[error3]")
194 #endif
195 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
196 RSEQ_INJECT_ASM(6)
197 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
198 : /* gcc asm goto does not allow outputs */
199 : [cpu_id] "r" (cpu),
200 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
201 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
202 [v] "m" (*v),
203 [expect] "r" (expect),
204 [v2] "m" (*v2),
205 [expect2] "r" (expect2),
206 [newv] "r" (newv)
207 RSEQ_INJECT_INPUT
208 : "memory", RSEQ_ASM_TMP_REG_1
209 RSEQ_INJECT_CLOBBER
210 : abort, ne
211 #ifdef RSEQ_COMPARE_TWICE
212 , error1, error2, error3
213 #endif
214 );
215
216 return 0;
217 abort:
218 RSEQ_INJECT_FAILED
219 return -1;
220 ne:
221 return 1;
222 #ifdef RSEQ_COMPARE_TWICE
223 error1:
224 rseq_bug("cpu_id comparison failed");
225 error2:
226 rseq_bug("expected value comparison failed");
227 error3:
228 rseq_bug("2nd expected value comparison failed");
229 #endif
230 }
231
232 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
233
234 static inline __attribute__((always_inline))
235 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_add_load_add_store__ptr)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
236 {
237 RSEQ_INJECT_C(9)
238
239 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
240 #ifdef RSEQ_COMPARE_TWICE
241 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
242 #endif
243 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
244 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
245 RSEQ_INJECT_ASM(3)
246 #ifdef RSEQ_COMPARE_TWICE
247 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
248 #endif
249 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
250 RSEQ_INJECT_ASM(4)
251 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
252 : /* gcc asm goto does not allow outputs */
253 : [cpu_id] "r" (cpu),
254 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
255 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
256 [ptr] "r" (ptr),
257 [off] "er" (off),
258 [inc] "er" (inc)
259 RSEQ_INJECT_INPUT
260 : "memory", RSEQ_ASM_TMP_REG_1
261 RSEQ_INJECT_CLOBBER
262 : abort
263 #ifdef RSEQ_COMPARE_TWICE
264 , error1
265 #endif
266 );
267 return 0;
268 abort:
269 RSEQ_INJECT_FAILED
270 return -1;
271 #ifdef RSEQ_COMPARE_TWICE
272 error1:
273 rseq_bug("cpu_id comparison failed");
274 #endif
275 }
276
277 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
278 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
279
280 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
281 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
282
283 static inline __attribute__((always_inline))
284 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store_store__ptr)(intptr_t *v, intptr_t expect,
285 intptr_t *v2, intptr_t newv2,
286 intptr_t newv, int cpu)
287 {
288 RSEQ_INJECT_C(9)
289
290 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
291 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[ne]")
292 #ifdef RSEQ_COMPARE_TWICE
293 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
294 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
295 #endif
296 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
297 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
298 RSEQ_INJECT_ASM(3)
299 RSEQ_ASM_OP_CBNE(v, expect, "%l[ne]")
300 RSEQ_INJECT_ASM(4)
301 #ifdef RSEQ_COMPARE_TWICE
302 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
303 RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]")
304 #endif
305 RSEQ_ASM_OP_STORE(newv2, v2)
306 RSEQ_INJECT_ASM(5)
307 #ifdef RSEQ_TEMPLATE_MO_RELEASE
308 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
309 #else
310 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
311 #endif
312 RSEQ_INJECT_ASM(6)
313 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
314 : /* gcc asm goto does not allow outputs */
315 : [cpu_id] "r" (cpu),
316 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
317 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
318 [expect] "r" (expect),
319 [v] "m" (*v),
320 [newv] "r" (newv),
321 [v2] "m" (*v2),
322 [newv2] "r" (newv2)
323 RSEQ_INJECT_INPUT
324 : "memory", RSEQ_ASM_TMP_REG_1
325 RSEQ_INJECT_CLOBBER
326 : abort, ne
327 #ifdef RSEQ_COMPARE_TWICE
328 , error1, error2
329 #endif
330 );
331
332 return 0;
333 abort:
334 RSEQ_INJECT_FAILED
335 return -1;
336 ne:
337 return 1;
338 #ifdef RSEQ_COMPARE_TWICE
339 error1:
340 rseq_bug("cpu_id comparison failed");
341 error2:
342 rseq_bug("expected value comparison failed");
343 #endif
344 }
345
346 static inline __attribute__((always_inline))
347 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intptr_t expect,
348 void *dst, void *src, size_t len,
349 intptr_t newv, int cpu)
350 {
351 RSEQ_INJECT_C(9)
352 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
353 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[ne]")
354 #ifdef RSEQ_COMPARE_TWICE
355 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
356 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
357 #endif
358 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
359 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
360 RSEQ_INJECT_ASM(3)
361 RSEQ_ASM_OP_CBNE(v, expect, "%l[ne]")
362 RSEQ_INJECT_ASM(4)
363 #ifdef RSEQ_COMPARE_TWICE
364 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
365 RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]")
366 #endif
367 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
368 RSEQ_INJECT_ASM(5)
369 #ifdef RSEQ_TEMPLATE_MO_RELEASE
370 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
371 #else
372 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
373 #endif
374 RSEQ_INJECT_ASM(6)
375 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
376 : /* gcc asm goto does not allow outputs */
377 : [cpu_id] "r" (cpu),
378 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
379 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
380 [expect] "r" (expect),
381 [v] "m" (*v),
382 [newv] "r" (newv),
383 [dst] "r" (dst),
384 [src] "r" (src),
385 [len] "r" (len)
386 RSEQ_INJECT_INPUT
387 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
388 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
389 RSEQ_INJECT_CLOBBER
390 : abort, ne
391 #ifdef RSEQ_COMPARE_TWICE
392 , error1, error2
393 #endif
394 );
395
396 return 0;
397 abort:
398 RSEQ_INJECT_FAILED
399 return -1;
400 ne:
401 return 1;
402 #ifdef RSEQ_COMPARE_TWICE
403 error1:
404 rseq_bug("cpu_id comparison failed");
405 error2:
406 rseq_bug("expected value comparison failed");
407 #endif
408 }
409
410 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
411 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
412
413 #include "rseq-bits-reset.h"
Librseq
This page took 0.038876 seconds and 3 git commands to generate.