projects / librseq.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
compiler: c++: use decltype
[librseq.git] / include / rseq / rseq-arm64.h
CommitLineData
90702366 1/* SPDX-License-Identifier: MIT */
f2d7b530
MJ		 2/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3/* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
4
d78a16c2
MD		 5/*
6 * rseq-arm64.h
d78a16c2
MD		 7 */
8
8b7c64f7
MD		 9/*
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
13 */
14#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
15
16#ifdef __AARCH64EB__
17#define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
18#else
19#define RSEQ_SIG_DATA RSEQ_SIG_CODE
20#endif
21
22#define RSEQ_SIG RSEQ_SIG_DATA
d78a16c2
MD		 23
24#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
27
28#define rseq_smp_load_acquire(p) \
29__extension__ ({ \
18ca65e0
MD		 30 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \
31 switch (sizeof(*(p))) { \
d78a16c2 32 case 1: \
18ca65e0
MD		 33 __asm__ __volatile__ ("ldarb %w0, %1" \
34 : "=r" (*(__u8 *)__u.__c) \
35 : "Q" (*(p)) : "memory"); \
d78a16c2
MD		 36 break; \
37 case 2: \
18ca65e0
MD		 38 __asm__ __volatile__ ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)__u.__c) \
40 : "Q" (*(p)) : "memory"); \
d78a16c2
MD		 41 break; \
42 case 4: \
18ca65e0
MD		 43 __asm__ __volatile__ ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)__u.__c) \
45 : "Q" (*(p)) : "memory"); \
d78a16c2
MD		 46 break; \
47 case 8: \
18ca65e0
MD		 48 __asm__ __volatile__ ("ldar %0, %1" \
49 : "=r" (*(__u64 *)__u.__c) \
50 : "Q" (*(p)) : "memory"); \
d78a16c2
MD		 51 break; \
52 } \
18ca65e0 53 (__typeof__(*(p)))__u.__val; \
d78a16c2
MD		 54})
55
56#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
57
58#define rseq_smp_store_release(p, v) \
59do { \
18ca65e0
MD		 60 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \
61 { .__val = (rseq_unqual_scalar_typeof(*(p))) (v) }; \
62 switch (sizeof(*(p))) { \
d78a16c2 63 case 1: \
18ca65e0
MD		 64 __asm__ __volatile__ ("stlrb %w1, %0" \
65 : "=Q" (*(p)) \
66 : "r" (*(__u8 *)__u.__c) \
d78a16c2
MD		 67 : "memory"); \
68 break; \
69 case 2: \
18ca65e0
MD		 70 __asm__ __volatile__ ("stlrh %w1, %0" \
71 : "=Q" (*(p)) \
72 : "r" (*(__u16 *)__u.__c) \
d78a16c2
MD		 73 : "memory"); \
74 break; \
75 case 4: \
18ca65e0
MD		 76 __asm__ __volatile__ ("stlr %w1, %0" \
77 : "=Q" (*(p)) \
78 : "r" (*(__u32 *)__u.__c) \
d78a16c2
MD		 79 : "memory"); \
80 break; \
81 case 8: \
18ca65e0
MD		 82 __asm__ __volatile__ ("stlr %1, %0" \
83 : "=Q" (*(p)) \
84 : "r" (*(__u64 *)__u.__c) \
d78a16c2
MD		 85 : "memory"); \
86 break; \
87 } \
88} while (0)
89
90#ifdef RSEQ_SKIP_FASTPATH
91#include "rseq-skip.h"
92#else /* !RSEQ_SKIP_FASTPATH */
93
94#define RSEQ_ASM_TMP_REG32 "w15"
95#define RSEQ_ASM_TMP_REG "x15"
96#define RSEQ_ASM_TMP_REG_2 "x14"
97
98#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
99 post_commit_offset, abort_ip) \
dd01d0fb 100 " .pushsection __rseq_cs, \"aw\"\n" \
d78a16c2
MD		 101 " .balign 32\n" \
102 __rseq_str(label) ":\n" \
103 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
104 " .quad " __rseq_str(start_ip) ", " \
105 __rseq_str(post_commit_offset) ", " \
106 __rseq_str(abort_ip) "\n" \
dd01d0fb
MD		 107 " .popsection\n\t" \
108 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
109 " .quad " __rseq_str(label) "b\n" \
d78a16c2
MD		 110 " .popsection\n"
111
112#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
113 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
114 (post_commit_ip - start_ip), abort_ip)
115
90d9876e
MD		 116/*
117 * Exit points of a rseq critical section consist of all instructions outside
fd622cad
MD		 118 * of the critical section where a critical section can either branch to or
119 * reach through the normal course of its execution. The abort IP and the
120 * post-commit IP are already part of the __rseq_cs section and should not be
121 * explicitly defined as additional exit points. Knowing all exit points is
122 * useful to assist debuggers stepping over the critical section.
90d9876e
MD		 123 */
124#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
125 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
126 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
127 " .popsection\n"
128
d78a16c2
MD		 129#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
130 RSEQ_INJECT_ASM(1) \
131 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
132 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
133 ", :lo12:" __rseq_str(cs_label) "\n" \
134 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
135 __rseq_str(label) ":\n"
136
137#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
138 " b 222f\n" \
8b7c64f7 139 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
d78a16c2
MD		 140 __rseq_str(label) ":\n" \
141 " b %l[" __rseq_str(abort_label) "]\n" \
142 "222:\n"
143
144#define RSEQ_ASM_OP_STORE(value, var) \
145 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
146
147#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
148 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
149
150#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
151 RSEQ_ASM_OP_STORE(value, var) \
152 __rseq_str(post_commit_label) ":\n"
153
154#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
155 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
156 __rseq_str(post_commit_label) ":\n"
157
158#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
159 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
160 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
161 ", %[" __rseq_str(expect) "]\n" \
162 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
163
164#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
165 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
166 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
167 ", %w[" __rseq_str(expect) "]\n" \
168 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
169
170#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
171 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
172 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
173 ", %[" __rseq_str(expect) "]\n" \
174 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
175
176#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
177 RSEQ_INJECT_ASM(2) \
178 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
179
180#define RSEQ_ASM_OP_R_LOAD(var) \
181 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
182
183#define RSEQ_ASM_OP_R_STORE(var) \
184 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
185
186#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
187 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
188 ", %[" __rseq_str(offset) "]]\n"
189
190#define RSEQ_ASM_OP_R_ADD(count) \
191 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
192 ", %[" __rseq_str(count) "]\n"
193
194#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
195 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
196 __rseq_str(post_commit_label) ":\n"
197
198#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
199 " cbz %[" __rseq_str(len) "], 333f\n" \
200 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
201 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
202 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
204 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
205 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
206 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
207 "333:\n"
208
209static inline __attribute__((always_inline))
210int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
211{
212 RSEQ_INJECT_C(9)
213
214 __asm__ __volatile__ goto (
215 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
217#ifdef RSEQ_COMPARE_TWICE
218 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
219 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
220#endif
d78a16c2
MD		 221 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
222 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
223 RSEQ_INJECT_ASM(3)
224 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
225 RSEQ_INJECT_ASM(4)
226#ifdef RSEQ_COMPARE_TWICE
227 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
228 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
229#endif
230 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
231 RSEQ_INJECT_ASM(5)
232 RSEQ_ASM_DEFINE_ABORT(4, abort)
233 : /* gcc asm goto does not allow outputs */
234 : [cpu_id] "r" (cpu),
9698c399 235 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 236 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 237 [v] "Qo" (*v),
238 [expect] "r" (expect),
239 [newv] "r" (newv)
240 RSEQ_INJECT_INPUT
241 : "memory", RSEQ_ASM_TMP_REG
242 : abort, cmpfail
243#ifdef RSEQ_COMPARE_TWICE
244 , error1, error2
245#endif
246 );
dd76f2d6 247 rseq_after_asm_goto();
d78a16c2
MD		 248 return 0;
249abort:
dd76f2d6 250 rseq_after_asm_goto();
d78a16c2
MD		 251 RSEQ_INJECT_FAILED
252 return -1;
253cmpfail:
dd76f2d6 254 rseq_after_asm_goto();
d78a16c2
MD		 255 return 1;
256#ifdef RSEQ_COMPARE_TWICE
257error1:
dd76f2d6 258 rseq_after_asm_goto();
d78a16c2
MD		 259 rseq_bug("cpu_id comparison failed");
260error2:
dd76f2d6 261 rseq_after_asm_goto();
d78a16c2
MD		 262 rseq_bug("expected value comparison failed");
263#endif
264}
265
266static inline __attribute__((always_inline))
267int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
d35eae6b 268 long voffp, intptr_t *load, int cpu)
d78a16c2
MD		 269{
270 RSEQ_INJECT_C(9)
271
272 __asm__ __volatile__ goto (
273 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 274 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
275#ifdef RSEQ_COMPARE_TWICE
276 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
277 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
278#endif
d78a16c2
MD		 279 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
280 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
281 RSEQ_INJECT_ASM(3)
282 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
283 RSEQ_INJECT_ASM(4)
284#ifdef RSEQ_COMPARE_TWICE
285 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
286 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
287#endif
288 RSEQ_ASM_OP_R_LOAD(v)
289 RSEQ_ASM_OP_R_STORE(load)
290 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
291 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
292 RSEQ_INJECT_ASM(5)
293 RSEQ_ASM_DEFINE_ABORT(4, abort)
294 : /* gcc asm goto does not allow outputs */
295 : [cpu_id] "r" (cpu),
9698c399 296 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 297 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 298 [v] "Qo" (*v),
299 [expectnot] "r" (expectnot),
300 [load] "Qo" (*load),
301 [voffp] "r" (voffp)
302 RSEQ_INJECT_INPUT
303 : "memory", RSEQ_ASM_TMP_REG
304 : abort, cmpfail
305#ifdef RSEQ_COMPARE_TWICE
306 , error1, error2
307#endif
308 );
dd76f2d6 309 rseq_after_asm_goto();
d78a16c2
MD		 310 return 0;
311abort:
dd76f2d6 312 rseq_after_asm_goto();
d78a16c2
MD		 313 RSEQ_INJECT_FAILED
314 return -1;
315cmpfail:
dd76f2d6 316 rseq_after_asm_goto();
d78a16c2
MD		 317 return 1;
318#ifdef RSEQ_COMPARE_TWICE
319error1:
dd76f2d6 320 rseq_after_asm_goto();
d78a16c2
MD		 321 rseq_bug("cpu_id comparison failed");
322error2:
dd76f2d6 323 rseq_after_asm_goto();
d78a16c2
MD		 324 rseq_bug("expected value comparison failed");
325#endif
326}
327
328static inline __attribute__((always_inline))
329int rseq_addv(intptr_t *v, intptr_t count, int cpu)
330{
331 RSEQ_INJECT_C(9)
332
333 __asm__ __volatile__ goto (
334 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 335#ifdef RSEQ_COMPARE_TWICE
336 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
337#endif
d78a16c2
MD		 338 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
339 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
340 RSEQ_INJECT_ASM(3)
341#ifdef RSEQ_COMPARE_TWICE
342 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
343#endif
344 RSEQ_ASM_OP_R_LOAD(v)
345 RSEQ_ASM_OP_R_ADD(count)
346 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
347 RSEQ_INJECT_ASM(4)
348 RSEQ_ASM_DEFINE_ABORT(4, abort)
349 : /* gcc asm goto does not allow outputs */
350 : [cpu_id] "r" (cpu),
9698c399 351 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 352 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 353 [v] "Qo" (*v),
354 [count] "r" (count)
355 RSEQ_INJECT_INPUT
356 : "memory", RSEQ_ASM_TMP_REG
357 : abort
358#ifdef RSEQ_COMPARE_TWICE
359 , error1
360#endif
361 );
dd76f2d6 362 rseq_after_asm_goto();
d78a16c2
MD		 363 return 0;
364abort:
dd76f2d6 365 rseq_after_asm_goto();
d78a16c2
MD		 366 RSEQ_INJECT_FAILED
367 return -1;
368#ifdef RSEQ_COMPARE_TWICE
369error1:
dd76f2d6 370 rseq_after_asm_goto();
d78a16c2
MD		 371 rseq_bug("cpu_id comparison failed");
372#endif
373}
374
375static inline __attribute__((always_inline))
376int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
377 intptr_t *v2, intptr_t newv2,
378 intptr_t newv, int cpu)
379{
380 RSEQ_INJECT_C(9)
381
382 __asm__ __volatile__ goto (
383 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 384 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
385#ifdef RSEQ_COMPARE_TWICE
386 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
387 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
388#endif
d78a16c2
MD		 389 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
390 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
391 RSEQ_INJECT_ASM(3)
392 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
393 RSEQ_INJECT_ASM(4)
394#ifdef RSEQ_COMPARE_TWICE
395 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
396 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
397#endif
398 RSEQ_ASM_OP_STORE(newv2, v2)
399 RSEQ_INJECT_ASM(5)
400 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
401 RSEQ_INJECT_ASM(6)
402 RSEQ_ASM_DEFINE_ABORT(4, abort)
403 : /* gcc asm goto does not allow outputs */
404 : [cpu_id] "r" (cpu),
9698c399 405 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 406 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 407 [expect] "r" (expect),
408 [v] "Qo" (*v),
409 [newv] "r" (newv),
410 [v2] "Qo" (*v2),
411 [newv2] "r" (newv2)
412 RSEQ_INJECT_INPUT
413 : "memory", RSEQ_ASM_TMP_REG
414 : abort, cmpfail
415#ifdef RSEQ_COMPARE_TWICE
416 , error1, error2
417#endif
418 );
dd76f2d6 419 rseq_after_asm_goto();
d78a16c2
MD		 420 return 0;
421abort:
dd76f2d6 422 rseq_after_asm_goto();
d78a16c2
MD		 423 RSEQ_INJECT_FAILED
424 return -1;
425cmpfail:
dd76f2d6 426 rseq_after_asm_goto();
d78a16c2
MD		 427 return 1;
428#ifdef RSEQ_COMPARE_TWICE
429error1:
dd76f2d6 430 rseq_after_asm_goto();
d78a16c2
MD		 431 rseq_bug("cpu_id comparison failed");
432error2:
dd76f2d6 433 rseq_after_asm_goto();
d78a16c2
MD		 434 rseq_bug("expected value comparison failed");
435#endif
436}
437
438static inline __attribute__((always_inline))
439int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
440 intptr_t *v2, intptr_t newv2,
441 intptr_t newv, int cpu)
442{
443 RSEQ_INJECT_C(9)
444
445 __asm__ __volatile__ goto (
446 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 447 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
448#ifdef RSEQ_COMPARE_TWICE
449 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
450 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
451#endif
d78a16c2
MD		 452 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
453 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
454 RSEQ_INJECT_ASM(3)
455 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
456 RSEQ_INJECT_ASM(4)
457#ifdef RSEQ_COMPARE_TWICE
458 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
459 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
460#endif
461 RSEQ_ASM_OP_STORE(newv2, v2)
462 RSEQ_INJECT_ASM(5)
463 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
464 RSEQ_INJECT_ASM(6)
465 RSEQ_ASM_DEFINE_ABORT(4, abort)
466 : /* gcc asm goto does not allow outputs */
467 : [cpu_id] "r" (cpu),
9698c399 468 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 469 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 470 [expect] "r" (expect),
471 [v] "Qo" (*v),
472 [newv] "r" (newv),
473 [v2] "Qo" (*v2),
474 [newv2] "r" (newv2)
475 RSEQ_INJECT_INPUT
476 : "memory", RSEQ_ASM_TMP_REG
477 : abort, cmpfail
478#ifdef RSEQ_COMPARE_TWICE
479 , error1, error2
480#endif
481 );
dd76f2d6 482 rseq_after_asm_goto();
d78a16c2
MD		 483 return 0;
484abort:
dd76f2d6 485 rseq_after_asm_goto();
d78a16c2
MD		 486 RSEQ_INJECT_FAILED
487 return -1;
488cmpfail:
dd76f2d6 489 rseq_after_asm_goto();
d78a16c2
MD		 490 return 1;
491#ifdef RSEQ_COMPARE_TWICE
492error1:
dd76f2d6 493 rseq_after_asm_goto();
d78a16c2
MD		 494 rseq_bug("cpu_id comparison failed");
495error2:
dd76f2d6 496 rseq_after_asm_goto();
d78a16c2
MD		 497 rseq_bug("expected value comparison failed");
498#endif
499}
500
501static inline __attribute__((always_inline))
502int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
503 intptr_t *v2, intptr_t expect2,
504 intptr_t newv, int cpu)
505{
506 RSEQ_INJECT_C(9)
507
508 __asm__ __volatile__ goto (
509 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 510 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
511#ifdef RSEQ_COMPARE_TWICE
512 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
513 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
514 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
515#endif
d78a16c2
MD		 516 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
517 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
518 RSEQ_INJECT_ASM(3)
519 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
520 RSEQ_INJECT_ASM(4)
521 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
522 RSEQ_INJECT_ASM(5)
523#ifdef RSEQ_COMPARE_TWICE
524 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
525 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
526 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
527#endif
528 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
529 RSEQ_INJECT_ASM(6)
530 RSEQ_ASM_DEFINE_ABORT(4, abort)
531 : /* gcc asm goto does not allow outputs */
532 : [cpu_id] "r" (cpu),
9698c399 533 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 534 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 535 [v] "Qo" (*v),
536 [expect] "r" (expect),
537 [v2] "Qo" (*v2),
538 [expect2] "r" (expect2),
539 [newv] "r" (newv)
540 RSEQ_INJECT_INPUT
541 : "memory", RSEQ_ASM_TMP_REG
542 : abort, cmpfail
543#ifdef RSEQ_COMPARE_TWICE
544 , error1, error2, error3
545#endif
546 );
dd76f2d6 547 rseq_after_asm_goto();
d78a16c2
MD		 548 return 0;
549abort:
dd76f2d6 550 rseq_after_asm_goto();
d78a16c2
MD		 551 RSEQ_INJECT_FAILED
552 return -1;
553cmpfail:
dd76f2d6 554 rseq_after_asm_goto();
d78a16c2
MD		 555 return 1;
556#ifdef RSEQ_COMPARE_TWICE
557error1:
dd76f2d6 558 rseq_after_asm_goto();
d78a16c2
MD		 559 rseq_bug("cpu_id comparison failed");
560error2:
dd76f2d6 561 rseq_after_asm_goto();
d78a16c2
MD		 562 rseq_bug("expected value comparison failed");
563error3:
dd76f2d6 564 rseq_after_asm_goto();
d78a16c2
MD		 565 rseq_bug("2nd expected value comparison failed");
566#endif
567}
568
569static inline __attribute__((always_inline))
570int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
571 void *dst, void *src, size_t len,
572 intptr_t newv, int cpu)
573{
574 RSEQ_INJECT_C(9)
575
576 __asm__ __volatile__ goto (
577 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 578 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
579#ifdef RSEQ_COMPARE_TWICE
580 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
581 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
582#endif
d78a16c2
MD		 583 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
584 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
585 RSEQ_INJECT_ASM(3)
586 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
587 RSEQ_INJECT_ASM(4)
588#ifdef RSEQ_COMPARE_TWICE
589 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
590 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
591#endif
592 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
593 RSEQ_INJECT_ASM(5)
594 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
595 RSEQ_INJECT_ASM(6)
596 RSEQ_ASM_DEFINE_ABORT(4, abort)
597 : /* gcc asm goto does not allow outputs */
598 : [cpu_id] "r" (cpu),
9698c399 599 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 600 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 601 [expect] "r" (expect),
602 [v] "Qo" (*v),
603 [newv] "r" (newv),
604 [dst] "r" (dst),
605 [src] "r" (src),
606 [len] "r" (len)
607 RSEQ_INJECT_INPUT
608 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
609 : abort, cmpfail
610#ifdef RSEQ_COMPARE_TWICE
611 , error1, error2
612#endif
613 );
dd76f2d6 614 rseq_after_asm_goto();
d78a16c2
MD		 615 return 0;
616abort:
dd76f2d6 617 rseq_after_asm_goto();
d78a16c2
MD		 618 RSEQ_INJECT_FAILED
619 return -1;
620cmpfail:
dd76f2d6 621 rseq_after_asm_goto();
d78a16c2
MD		 622 return 1;
623#ifdef RSEQ_COMPARE_TWICE
624error1:
dd76f2d6 625 rseq_after_asm_goto();
d78a16c2
MD		 626 rseq_bug("cpu_id comparison failed");
627error2:
dd76f2d6 628 rseq_after_asm_goto();
d78a16c2
MD		 629 rseq_bug("expected value comparison failed");
630#endif
631}
632
633static inline __attribute__((always_inline))
634int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
635 void *dst, void *src, size_t len,
636 intptr_t newv, int cpu)
637{
638 RSEQ_INJECT_C(9)
639
640 __asm__ __volatile__ goto (
641 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 642 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
643#ifdef RSEQ_COMPARE_TWICE
644 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
645 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
646#endif
d78a16c2
MD		 647 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
648 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
649 RSEQ_INJECT_ASM(3)
650 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
651 RSEQ_INJECT_ASM(4)
652#ifdef RSEQ_COMPARE_TWICE
653 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
654 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
655#endif
656 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
657 RSEQ_INJECT_ASM(5)
658 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
659 RSEQ_INJECT_ASM(6)
660 RSEQ_ASM_DEFINE_ABORT(4, abort)
661 : /* gcc asm goto does not allow outputs */
662 : [cpu_id] "r" (cpu),
9698c399 663 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 664 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 665 [expect] "r" (expect),
666 [v] "Qo" (*v),
667 [newv] "r" (newv),
668 [dst] "r" (dst),
669 [src] "r" (src),
670 [len] "r" (len)
671 RSEQ_INJECT_INPUT
672 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
673 : abort, cmpfail
674#ifdef RSEQ_COMPARE_TWICE
675 , error1, error2
676#endif
677 );
dd76f2d6 678 rseq_after_asm_goto();
d78a16c2
MD		 679 return 0;
680abort:
dd76f2d6 681 rseq_after_asm_goto();
d78a16c2
MD		 682 RSEQ_INJECT_FAILED
683 return -1;
684cmpfail:
dd76f2d6 685 rseq_after_asm_goto();
d78a16c2
MD		 686 return 1;
687#ifdef RSEQ_COMPARE_TWICE
688error1:
dd76f2d6 689 rseq_after_asm_goto();
d78a16c2
MD		 690 rseq_bug("cpu_id comparison failed");
691error2:
dd76f2d6 692 rseq_after_asm_goto();
d78a16c2
MD		 693 rseq_bug("expected value comparison failed");
694#endif
695}
696
697#endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.059743 seconds and 4 git commands to generate.