projects / librseq.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
architecture headers: add extra parentheses around macro parameters
[librseq.git] / include / rseq / rseq-riscv.h
CommitLineData
90702366 1/* SPDX-License-Identifier: MIT */
f2d7b530
MJ		 2/* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
3
074b1077
MJ		 4/*
5 * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
6 * other architectures, the ebreak instruction has no immediate field for
7 * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
8 * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
9 * is an uncommon instruction and will raise an illegal instruction
10 * exception when executed in all modes.
11 */
12#include <endian.h>
13
14#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
15#define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */
16#else
17#error "Currently, RSEQ only supports Little-Endian version"
18#endif
19
20#if __riscv_xlen == 64
21#define __REG_SEL(a, b) a
22#elif __riscv_xlen == 32
23#define __REG_SEL(a, b) b
24#endif
25
26#define REG_L __REG_SEL("ld ", "lw ")
27#define REG_S __REG_SEL("sd ", "sw ")
28
29#define RISCV_FENCE(p, s) \
30 __asm__ __volatile__ ("fence " #p "," #s : : : "memory")
31#define rseq_smp_mb() RISCV_FENCE(rw, rw)
32#define rseq_smp_rmb() RISCV_FENCE(r, r)
33#define rseq_smp_wmb() RISCV_FENCE(w, w)
34#define RSEQ_ASM_TMP_REG_1 "t6"
35#define RSEQ_ASM_TMP_REG_2 "t5"
36#define RSEQ_ASM_TMP_REG_3 "t4"
37#define RSEQ_ASM_TMP_REG_4 "t3"
38
39#define rseq_smp_load_acquire(p) \
40__extension__ ({ \
41 __typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
42 RISCV_FENCE(r, rw); \
43 ____p1; \
44})
45
46#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
47
48#define rseq_smp_store_release(p, v) \
49do { \
50 RISCV_FENCE(rw, w); \
826417f6 51 RSEQ_WRITE_ONCE(*(p), v); \
074b1077
MJ		 52} while (0)
53
54#ifdef RSEQ_SKIP_FASTPATH
55#include "rseq-skip.h"
56#else /* !RSEQ_SKIP_FASTPATH */
57
58#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
59 post_commit_offset, abort_ip) \
60 ".pushsection __rseq_cs, \"aw\"\n" \
61 ".balign 32\n" \
62 __rseq_str(label) ":\n" \
63 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
64 ".quad " __rseq_str(start_ip) ", " \
65 __rseq_str(post_commit_offset) ", " \
66 __rseq_str(abort_ip) "\n" \
67 ".popsection\n\t" \
68 ".pushsection __rseq_cs_ptr_array, \"aw\"\n" \
69 ".quad " __rseq_str(label) "b\n" \
70 ".popsection\n"
71
72#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
73 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
74 ((post_commit_ip) - (start_ip)), abort_ip)
75
76/*
77 * Exit points of a rseq critical section consist of all instructions outside
78 * of the critical section where a critical section can either branch to or
79 * reach through the normal course of its execution. The abort IP and the
80 * post-commit IP are already part of the __rseq_cs section and should not be
81 * explicitly defined as additional exit points. Knowing all exit points is
82 * useful to assist debuggers stepping over the critical section.
83 */
84#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
85 ".pushsection __rseq_exit_point_array, \"aw\"\n" \
86 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
87 ".popsection\n"
88
89#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
90 RSEQ_INJECT_ASM(1) \
91 "la " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \
92 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \
93 __rseq_str(label) ":\n"
94
95#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
96 "j 222f\n" \
97 ".balign 4\n" \
98 ".long " __rseq_str(RSEQ_SIG) "\n" \
99 __rseq_str(label) ":\n" \
100 "j %l[" __rseq_str(abort_label) "]\n" \
101 "222:\n"
102
103#define RSEQ_ASM_OP_STORE(value, var) \
104 REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
105
106#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
107 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
108 "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
109 __rseq_str(label) "\n"
110
111#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
112 "lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
113 "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
114 __rseq_str(label) "\n"
115
116#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
117 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
118 "beq " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
119 __rseq_str(label) "\n"
120
121#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
122 RSEQ_INJECT_ASM(2) \
123 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
124
125#define RSEQ_ASM_OP_R_LOAD(var) \
126 REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
127
128#define RSEQ_ASM_OP_R_STORE(var) \
129 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
130
131#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
132 "add " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \
133 RSEQ_ASM_TMP_REG_1 "\n" \
134 REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"
135
136#define RSEQ_ASM_OP_R_ADD(count) \
137 "add " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \
138 ", %[" __rseq_str(count) "]\n"
139
140#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
141 RSEQ_ASM_OP_STORE(value, var) \
142 __rseq_str(post_commit_label) ":\n"
143
144#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
145 "fence rw, w\n" \
146 RSEQ_ASM_OP_STORE(value, var) \
147 __rseq_str(post_commit_label) ":\n"
148
149#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
150 REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
151 __rseq_str(post_commit_label) ":\n"
152
153#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
154 "beqz %[" __rseq_str(len) "], 333f\n" \
155 "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \
156 "mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \
157 "mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \
158 "222:\n" \
159 "lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \
160 "sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \
161 "addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \
162 "addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \
163 "addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \
164 "bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \
165 "333:\n"
166
167#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label) \
168 "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \
169 RSEQ_ASM_OP_R_ADD(off) \
170 REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \
171 RSEQ_ASM_OP_R_ADD(inc) \
172 __rseq_str(post_commit_label) ":\n"
173
174static inline __attribute__((always_inline))
175int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
176{
177 RSEQ_INJECT_C(9)
178
179 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
180 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
181#ifdef RSEQ_COMPARE_TWICE
182 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
183 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
184#endif
185 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
186 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
187 RSEQ_INJECT_ASM(3)
188 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
189 RSEQ_INJECT_ASM(4)
190#ifdef RSEQ_COMPARE_TWICE
191 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
192 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
193#endif
194 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
195 RSEQ_INJECT_ASM(5)
196 RSEQ_ASM_DEFINE_ABORT(4, abort)
197 : /* gcc asm goto does not allow outputs */
198 : [cpu_id] "r" (cpu),
199 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
200 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
201 [v] "m" (*v),
202 [expect] "r" (expect),
203 [newv] "r" (newv)
204 RSEQ_INJECT_INPUT
205 : "memory", RSEQ_ASM_TMP_REG_1
206 RSEQ_INJECT_CLOBBER
207 : abort, cmpfail
208#ifdef RSEQ_COMPARE_TWICE
209 , error1, error2
210#endif
211 );
212 rseq_after_asm_goto();
213
214 return 0;
215abort:
216 rseq_after_asm_goto();
217 RSEQ_INJECT_FAILED
218 return -1;
219cmpfail:
220 rseq_after_asm_goto();
221 return 1;
222#ifdef RSEQ_COMPARE_TWICE
223error1:
224 rseq_after_asm_goto();
225 rseq_bug("cpu_id comparison failed");
226error2:
227 rseq_after_asm_goto();
228 rseq_bug("expected value comparison failed");
229#endif
230}
231
232static inline __attribute__((always_inline))
233int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
234 off_t voffp, intptr_t *load, int cpu)
235{
236 RSEQ_INJECT_C(9)
237
238 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
239 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
240#ifdef RSEQ_COMPARE_TWICE
241 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
242 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
243#endif
244 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
245 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
246 RSEQ_INJECT_ASM(3)
247 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
248 RSEQ_INJECT_ASM(4)
249#ifdef RSEQ_COMPARE_TWICE
250 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
251 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
252#endif
253 RSEQ_ASM_OP_R_LOAD(v)
254 RSEQ_ASM_OP_R_STORE(load)
255 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
256 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
257 RSEQ_INJECT_ASM(5)
258 RSEQ_ASM_DEFINE_ABORT(4, abort)
259 : /* gcc asm goto does not allow outputs */
260 : [cpu_id] "r" (cpu),
261 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
262 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
263 [v] "m" (*v),
264 [expectnot] "r" (expectnot),
265 [load] "m" (*load),
266 [voffp] "r" (voffp)
267 RSEQ_INJECT_INPUT
268 : "memory", RSEQ_ASM_TMP_REG_1
269 RSEQ_INJECT_CLOBBER
270 : abort, cmpfail
271#ifdef RSEQ_COMPARE_TWICE
272 , error1, error2
273#endif
274 );
275 rseq_after_asm_goto();
276 return 0;
277abort:
278 rseq_after_asm_goto();
279 RSEQ_INJECT_FAILED
280 return -1;
281cmpfail:
282 rseq_after_asm_goto();
283 return 1;
284#ifdef RSEQ_COMPARE_TWICE
285error1:
286 rseq_after_asm_goto();
287 rseq_bug("cpu_id comparison failed");
288error2:
289 rseq_after_asm_goto();
290 rseq_bug("expected value comparison failed");
291#endif
292}
293
294static inline __attribute__((always_inline))
295int rseq_addv(intptr_t *v, intptr_t count, int cpu)
296{
297 RSEQ_INJECT_C(9)
298
299 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
300#ifdef RSEQ_COMPARE_TWICE
301 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
302#endif
303 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
304 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
305 RSEQ_INJECT_ASM(3)
306#ifdef RSEQ_COMPARE_TWICE
307 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
308#endif
309 RSEQ_ASM_OP_R_LOAD(v)
310 RSEQ_ASM_OP_R_ADD(count)
311 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
312 RSEQ_INJECT_ASM(4)
313 RSEQ_ASM_DEFINE_ABORT(4, abort)
314 : /* gcc asm goto does not allow outputs */
315 : [cpu_id] "r" (cpu),
316 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
317 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
318 [v] "m" (*v),
319 [count] "r" (count)
320 RSEQ_INJECT_INPUT
321 : "memory", RSEQ_ASM_TMP_REG_1
322 RSEQ_INJECT_CLOBBER
323 : abort
324#ifdef RSEQ_COMPARE_TWICE
325 , error1
326#endif
327 );
328 rseq_after_asm_goto();
329 return 0;
330abort:
331 rseq_after_asm_goto();
332 RSEQ_INJECT_FAILED
333 return -1;
334#ifdef RSEQ_COMPARE_TWICE
335error1:
336 rseq_after_asm_goto();
337 rseq_bug("cpu_id comparison failed");
338#endif
339}
340
341static inline __attribute__((always_inline))
342int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
343 intptr_t *v2, intptr_t newv2,
344 intptr_t newv, int cpu)
345{
346 RSEQ_INJECT_C(9)
347
348 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
349 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
350#ifdef RSEQ_COMPARE_TWICE
351 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
353#endif
354 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
355 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
356 RSEQ_INJECT_ASM(3)
357 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
358 RSEQ_INJECT_ASM(4)
359#ifdef RSEQ_COMPARE_TWICE
360 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
361 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
362#endif
363 RSEQ_ASM_OP_STORE(newv2, v2)
364 RSEQ_INJECT_ASM(5)
365 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
366 RSEQ_INJECT_ASM(6)
367 RSEQ_ASM_DEFINE_ABORT(4, abort)
368 : /* gcc asm goto does not allow outputs */
369 : [cpu_id] "r" (cpu),
370 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
371 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
372 [expect] "r" (expect),
373 [v] "m" (*v),
374 [newv] "r" (newv),
375 [v2] "m" (*v2),
376 [newv2] "r" (newv2)
377 RSEQ_INJECT_INPUT
378 : "memory", RSEQ_ASM_TMP_REG_1
379 RSEQ_INJECT_CLOBBER
380 : abort, cmpfail
381#ifdef RSEQ_COMPARE_TWICE
382 , error1, error2
383#endif
384 );
385
386 rseq_after_asm_goto();
387 return 0;
388abort:
389 rseq_after_asm_goto();
390 RSEQ_INJECT_FAILED
391 return -1;
392cmpfail:
393 rseq_after_asm_goto();
394 return 1;
395#ifdef RSEQ_COMPARE_TWICE
396error1:
397 rseq_after_asm_goto();
398 rseq_bug("cpu_id comparison failed");
399error2:
400 rseq_after_asm_goto();
401 rseq_bug("expected value comparison failed");
402#endif
403}
404
405static inline __attribute__((always_inline))
406int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
407 intptr_t *v2, intptr_t newv2,
408 intptr_t newv, int cpu)
409{
410 RSEQ_INJECT_C(9)
411
412 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
413 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
414#ifdef RSEQ_COMPARE_TWICE
415 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
416 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
417#endif
418 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
419 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
420 RSEQ_INJECT_ASM(3)
421 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
422 RSEQ_INJECT_ASM(4)
423#ifdef RSEQ_COMPARE_TWICE
424 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
425 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
426#endif
427 RSEQ_ASM_OP_STORE(newv2, v2)
428 RSEQ_INJECT_ASM(5)
429 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
430 RSEQ_INJECT_ASM(6)
431 RSEQ_ASM_DEFINE_ABORT(4, abort)
432 : /* gcc asm goto does not allow outputs */
433 : [cpu_id] "r" (cpu),
434 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
435 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
436 [expect] "r" (expect),
437 [v] "m" (*v),
438 [newv] "r" (newv),
439 [v2] "m" (*v2),
440 [newv2] "r" (newv2)
441 RSEQ_INJECT_INPUT
442 : "memory", RSEQ_ASM_TMP_REG_1
443 RSEQ_INJECT_CLOBBER
444 : abort, cmpfail
445#ifdef RSEQ_COMPARE_TWICE
446 , error1, error2
447#endif
448 );
449
450 rseq_after_asm_goto();
451 return 0;
452abort:
453 rseq_after_asm_goto();
454 RSEQ_INJECT_FAILED
455 return -1;
456cmpfail:
457 rseq_after_asm_goto();
458 return 1;
459#ifdef RSEQ_COMPARE_TWICE
460error1:
461 rseq_after_asm_goto();
462 rseq_bug("cpu_id comparison failed");
463error2:
464 rseq_after_asm_goto();
465 rseq_bug("expected value comparison failed");
466#endif
467}
468
469static inline __attribute__((always_inline))
470int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
471 intptr_t *v2, intptr_t expect2,
472 intptr_t newv, int cpu)
473{
474 RSEQ_INJECT_C(9)
475
476 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
477 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
478#ifdef RSEQ_COMPARE_TWICE
479 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
480 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
481 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
482#endif
483 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
484 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
485 RSEQ_INJECT_ASM(3)
486 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
487 RSEQ_INJECT_ASM(4)
488 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
489 RSEQ_INJECT_ASM(5)
490#ifdef RSEQ_COMPARE_TWICE
491 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
492 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
493 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
494#endif
495 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
496 RSEQ_INJECT_ASM(6)
497 RSEQ_ASM_DEFINE_ABORT(4, abort)
498 : /* gcc asm goto does not allow outputs */
499 : [cpu_id] "r" (cpu),
500 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
501 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
502 [v] "m" (*v),
503 [expect] "r" (expect),
504 [v2] "m" (*v2),
505 [expect2] "r" (expect2),
506 [newv] "r" (newv)
507 RSEQ_INJECT_INPUT
508 : "memory", RSEQ_ASM_TMP_REG_1
509 RSEQ_INJECT_CLOBBER
510 : abort, cmpfail
511#ifdef RSEQ_COMPARE_TWICE
512 , error1, error2, error3
513#endif
514 );
515
516 rseq_after_asm_goto();
517 return 0;
518abort:
519 rseq_after_asm_goto();
520 RSEQ_INJECT_FAILED
521 return -1;
522cmpfail:
523 rseq_after_asm_goto();
524 return 1;
525#ifdef RSEQ_COMPARE_TWICE
526error1:
527 rseq_after_asm_goto();
528 rseq_bug("cpu_id comparison failed");
529error2:
530 rseq_after_asm_goto();
531 rseq_bug("expected value comparison failed");
532error3:
533 rseq_after_asm_goto();
534 rseq_bug("2nd expected value comparison failed");
535#endif
536}
537
538static inline __attribute__((always_inline))
539int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
540 void *dst, void *src, size_t len,
541 intptr_t newv, int cpu)
542{
543 RSEQ_INJECT_C(9)
544 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
545 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
546#ifdef RSEQ_COMPARE_TWICE
547 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
548 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
549#endif
550 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
551 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
552 RSEQ_INJECT_ASM(3)
553 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
554 RSEQ_INJECT_ASM(4)
555#ifdef RSEQ_COMPARE_TWICE
556 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
557 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
558#endif
559 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
560 RSEQ_INJECT_ASM(5)
561 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
562 RSEQ_INJECT_ASM(6)
563 RSEQ_ASM_DEFINE_ABORT(4, abort)
564 : /* gcc asm goto does not allow outputs */
565 : [cpu_id] "r" (cpu),
566 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
567 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
568 [expect] "r" (expect),
569 [v] "m" (*v),
570 [newv] "r" (newv),
571 [dst] "r" (dst),
572 [src] "r" (src),
573 [len] "r" (len)
574 RSEQ_INJECT_INPUT
575 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
576 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
577 RSEQ_INJECT_CLOBBER
578 : abort, cmpfail
579#ifdef RSEQ_COMPARE_TWICE
580 , error1, error2
581#endif
582 );
583
584 rseq_after_asm_goto();
585 return 0;
586abort:
587 rseq_after_asm_goto();
588 RSEQ_INJECT_FAILED
589 return -1;
590cmpfail:
591 rseq_after_asm_goto();
592 return 1;
593#ifdef RSEQ_COMPARE_TWICE
594error1:
595 rseq_after_asm_goto();
596 rseq_bug("cpu_id comparison failed");
597error2:
598 rseq_after_asm_goto();
599 rseq_bug("expected value comparison failed");
600#endif
601}
602
603static inline __attribute__((always_inline))
604int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
605 void *dst, void *src, size_t len,
606 intptr_t newv, int cpu)
607{
608 RSEQ_INJECT_C(9)
609
610 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
611 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
612#ifdef RSEQ_COMPARE_TWICE
613 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
614 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
615#endif
616 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
618 RSEQ_INJECT_ASM(3)
619 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
620 RSEQ_INJECT_ASM(4)
621#ifdef RSEQ_COMPARE_TWICE
622 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
623 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
624#endif
625 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
626 RSEQ_INJECT_ASM(5)
627 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
628 RSEQ_INJECT_ASM(6)
629 RSEQ_ASM_DEFINE_ABORT(4, abort)
630 : /* gcc asm goto does not allow outputs */
631 : [cpu_id] "r" (cpu),
632 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
633 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
634 [expect] "r" (expect),
635 [v] "m" (*v),
636 [newv] "r" (newv),
637 [dst] "r" (dst),
638 [src] "r" (src),
639 [len] "r" (len)
640 RSEQ_INJECT_INPUT
641 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
642 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
643 RSEQ_INJECT_CLOBBER
644 : abort, cmpfail
645#ifdef RSEQ_COMPARE_TWICE
646 , error1, error2
647#endif
648 );
649
650 rseq_after_asm_goto();
651 return 0;
652abort:
653 rseq_after_asm_goto();
654 RSEQ_INJECT_FAILED
655 return -1;
656cmpfail:
657 rseq_after_asm_goto();
658 return 1;
659#ifdef RSEQ_COMPARE_TWICE
660error1:
661 rseq_after_asm_goto();
662 rseq_bug("cpu_id comparison failed");
663error2:
664 rseq_after_asm_goto();
665 rseq_bug("expected value comparison failed");
666#endif
667}
668
669#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
670
671/*
672 * pval = *(ptr+off)
673 * *pval += inc;
674 */
675static inline __attribute__((always_inline))
676int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
677{
678 RSEQ_INJECT_C(9)
679
680 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
681#ifdef RSEQ_COMPARE_TWICE
682 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
683#endif
684 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
685 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
686 RSEQ_INJECT_ASM(3)
687#ifdef RSEQ_COMPARE_TWICE
688 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
689#endif
690 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
691 RSEQ_INJECT_ASM(4)
692 RSEQ_ASM_DEFINE_ABORT(4, abort)
693 : /* gcc asm goto does not allow outputs */
694 : [cpu_id] "r" (cpu),
695 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
696 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
697 [ptr] "r" (ptr),
698 [off] "er" (off),
699 [inc] "er" (inc)
700 RSEQ_INJECT_INPUT
701 : "memory", RSEQ_ASM_TMP_REG_1
702 RSEQ_INJECT_CLOBBER
703 : abort
704#ifdef RSEQ_COMPARE_TWICE
705 , error1
706#endif
707 );
708 rseq_after_asm_goto();
709 return 0;
710abort:
711 rseq_after_asm_goto();
712 RSEQ_INJECT_FAILED
713 return -1;
714#ifdef RSEQ_COMPARE_TWICE
715error1:
716 rseq_after_asm_goto();
717 rseq_bug("cpu_id comparison failed");
718#endif
719}
720
721#endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.051931 seconds and 4 git commands to generate.