f7b080f8cc0ff24560d1d5082466fc868f136493
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3 /* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
14 #define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
17 #define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
19 #define RSEQ_SIG_DATA RSEQ_SIG_CODE
22 #define RSEQ_SIG RSEQ_SIG_DATA
24 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
28 #define rseq_smp_load_acquire(p) \
30 __typeof(*p) ____p1; \
31 switch (sizeof(*p)) { \
33 asm volatile ("ldarb %w0, %1" \
35 : "Q" (*p) : "memory"); \
38 asm volatile ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)p) \
40 : "Q" (*p) : "memory"); \
43 asm volatile ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)p) \
45 : "Q" (*p) : "memory"); \
48 asm volatile ("ldar %0, %1" \
49 : "=r" (*(__u64 *)p) \
50 : "Q" (*p) : "memory"); \
56 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
58 #define rseq_smp_store_release(p, v) \
60 switch (sizeof(*p)) { \
62 asm volatile ("stlrb %w1, %0" \
68 asm volatile ("stlrh %w1, %0" \
74 asm volatile ("stlr %w1, %0" \
80 asm volatile ("stlr %1, %0" \
88 #ifdef RSEQ_SKIP_FASTPATH
89 #include "rseq-skip.h"
90 #else /* !RSEQ_SKIP_FASTPATH */
92 #define RSEQ_ASM_TMP_REG32 "w15"
93 #define RSEQ_ASM_TMP_REG "x15"
94 #define RSEQ_ASM_TMP_REG_2 "x14"
96 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
97 post_commit_offset, abort_ip) \
98 " .pushsection __rseq_cs, \"aw\"\n" \
100 __rseq_str(label) ":\n" \
101 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
102 " .quad " __rseq_str(start_ip) ", " \
103 __rseq_str(post_commit_offset) ", " \
104 __rseq_str(abort_ip) "\n" \
106 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
107 " .quad " __rseq_str(label) "b\n" \
110 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
111 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
112 (post_commit_ip - start_ip), abort_ip)
115 * Exit points of a rseq critical section consist of all instructions outside
116 * of the critical section where a critical section can either branch to or
117 * reach through the normal course of its execution. The abort IP and the
118 * post-commit IP are already part of the __rseq_cs section and should not be
119 * explicitly defined as additional exit points. Knowing all exit points is
120 * useful to assist debuggers stepping over the critical section.
122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
124 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
127 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
129 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
130 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
131 ", :lo12:" __rseq_str(cs_label) "\n" \
132 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
133 __rseq_str(label) ":\n"
135 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
137 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
138 __rseq_str(label) ":\n" \
139 " b %l[" __rseq_str(abort_label) "]\n" \
142 #define RSEQ_ASM_OP_STORE(value, var) \
143 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
145 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
146 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
148 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
149 RSEQ_ASM_OP_STORE(value, var) \
150 __rseq_str(post_commit_label) ":\n"
152 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
153 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
154 __rseq_str(post_commit_label) ":\n"
156 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
157 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
158 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
159 ", %[" __rseq_str(expect) "]\n" \
160 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
162 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
163 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
164 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
165 ", %w[" __rseq_str(expect) "]\n" \
166 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
168 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \
169 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
170 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
171 ", %[" __rseq_str(expect) "]\n" \
172 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
174 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
176 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
178 #define RSEQ_ASM_OP_R_LOAD(var) \
179 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
181 #define RSEQ_ASM_OP_R_STORE(var) \
182 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
184 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
185 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
186 ", %[" __rseq_str(offset) "]]\n"
188 #define RSEQ_ASM_OP_R_ADD(count) \
189 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
190 ", %[" __rseq_str(count) "]\n"
192 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
193 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
194 __rseq_str(post_commit_label) ":\n"
196 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
197 " cbz %[" __rseq_str(len) "], 333f\n" \
198 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
199 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
200 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
201 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
202 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
204 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
207 static inline __attribute__((always_inline
))
208 int rseq_cmpeqv_storev(intptr_t *v
, intptr_t expect
, intptr_t newv
, int cpu
)
212 __asm__ __volatile__
goto (
213 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
214 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
215 #ifdef RSEQ_COMPARE_TWICE
216 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
217 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
219 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
222 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
224 #ifdef RSEQ_COMPARE_TWICE
225 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
226 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
228 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
230 RSEQ_ASM_DEFINE_ABORT(4, abort
)
231 : /* gcc asm goto does not allow outputs */
232 : [cpu_id
] "r" (cpu
),
233 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
234 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
236 [expect
] "r" (expect
),
239 : "memory", RSEQ_ASM_TMP_REG
241 #ifdef RSEQ_COMPARE_TWICE
245 rseq_after_asm_goto();
248 rseq_after_asm_goto();
252 rseq_after_asm_goto();
254 #ifdef RSEQ_COMPARE_TWICE
256 rseq_after_asm_goto();
257 rseq_bug("cpu_id comparison failed");
259 rseq_after_asm_goto();
260 rseq_bug("expected value comparison failed");
264 static inline __attribute__((always_inline
))
265 int rseq_cmpnev_storeoffp_load(intptr_t *v
, intptr_t expectnot
,
266 long voffp
, intptr_t *load
, int cpu
)
270 __asm__ __volatile__
goto (
271 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
272 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
273 #ifdef RSEQ_COMPARE_TWICE
274 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
275 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
277 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
278 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
280 RSEQ_ASM_OP_CMPNE(v
, expectnot
, %l
[cmpfail
])
282 #ifdef RSEQ_COMPARE_TWICE
283 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
284 RSEQ_ASM_OP_CMPNE(v
, expectnot
, %l
[error2
])
286 RSEQ_ASM_OP_R_LOAD(v
)
287 RSEQ_ASM_OP_R_STORE(load
)
288 RSEQ_ASM_OP_R_LOAD_OFF(voffp
)
289 RSEQ_ASM_OP_R_FINAL_STORE(v
, 3)
291 RSEQ_ASM_DEFINE_ABORT(4, abort
)
292 : /* gcc asm goto does not allow outputs */
293 : [cpu_id
] "r" (cpu
),
294 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
295 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
297 [expectnot
] "r" (expectnot
),
301 : "memory", RSEQ_ASM_TMP_REG
303 #ifdef RSEQ_COMPARE_TWICE
307 rseq_after_asm_goto();
310 rseq_after_asm_goto();
314 rseq_after_asm_goto();
316 #ifdef RSEQ_COMPARE_TWICE
318 rseq_after_asm_goto();
319 rseq_bug("cpu_id comparison failed");
321 rseq_after_asm_goto();
322 rseq_bug("expected value comparison failed");
326 static inline __attribute__((always_inline
))
327 int rseq_addv(intptr_t *v
, intptr_t count
, int cpu
)
331 __asm__ __volatile__
goto (
332 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
333 #ifdef RSEQ_COMPARE_TWICE
334 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
336 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
337 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
339 #ifdef RSEQ_COMPARE_TWICE
340 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
342 RSEQ_ASM_OP_R_LOAD(v
)
343 RSEQ_ASM_OP_R_ADD(count
)
344 RSEQ_ASM_OP_R_FINAL_STORE(v
, 3)
346 RSEQ_ASM_DEFINE_ABORT(4, abort
)
347 : /* gcc asm goto does not allow outputs */
348 : [cpu_id
] "r" (cpu
),
349 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
350 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
354 : "memory", RSEQ_ASM_TMP_REG
356 #ifdef RSEQ_COMPARE_TWICE
360 rseq_after_asm_goto();
363 rseq_after_asm_goto();
366 #ifdef RSEQ_COMPARE_TWICE
368 rseq_after_asm_goto();
369 rseq_bug("cpu_id comparison failed");
373 static inline __attribute__((always_inline
))
374 int rseq_cmpeqv_trystorev_storev(intptr_t *v
, intptr_t expect
,
375 intptr_t *v2
, intptr_t newv2
,
376 intptr_t newv
, int cpu
)
380 __asm__ __volatile__
goto (
381 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
382 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
383 #ifdef RSEQ_COMPARE_TWICE
384 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
385 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
387 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
388 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
390 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
392 #ifdef RSEQ_COMPARE_TWICE
393 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
394 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
396 RSEQ_ASM_OP_STORE(newv2
, v2
)
398 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
400 RSEQ_ASM_DEFINE_ABORT(4, abort
)
401 : /* gcc asm goto does not allow outputs */
402 : [cpu_id
] "r" (cpu
),
403 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
404 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
405 [expect
] "r" (expect
),
411 : "memory", RSEQ_ASM_TMP_REG
413 #ifdef RSEQ_COMPARE_TWICE
417 rseq_after_asm_goto();
420 rseq_after_asm_goto();
424 rseq_after_asm_goto();
426 #ifdef RSEQ_COMPARE_TWICE
428 rseq_after_asm_goto();
429 rseq_bug("cpu_id comparison failed");
431 rseq_after_asm_goto();
432 rseq_bug("expected value comparison failed");
436 static inline __attribute__((always_inline
))
437 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v
, intptr_t expect
,
438 intptr_t *v2
, intptr_t newv2
,
439 intptr_t newv
, int cpu
)
443 __asm__ __volatile__
goto (
444 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
445 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
446 #ifdef RSEQ_COMPARE_TWICE
447 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
448 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
450 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
451 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
453 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
455 #ifdef RSEQ_COMPARE_TWICE
456 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
457 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
459 RSEQ_ASM_OP_STORE(newv2
, v2
)
461 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv
, v
, 3)
463 RSEQ_ASM_DEFINE_ABORT(4, abort
)
464 : /* gcc asm goto does not allow outputs */
465 : [cpu_id
] "r" (cpu
),
466 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
467 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
468 [expect
] "r" (expect
),
474 : "memory", RSEQ_ASM_TMP_REG
476 #ifdef RSEQ_COMPARE_TWICE
480 rseq_after_asm_goto();
483 rseq_after_asm_goto();
487 rseq_after_asm_goto();
489 #ifdef RSEQ_COMPARE_TWICE
491 rseq_after_asm_goto();
492 rseq_bug("cpu_id comparison failed");
494 rseq_after_asm_goto();
495 rseq_bug("expected value comparison failed");
499 static inline __attribute__((always_inline
))
500 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v
, intptr_t expect
,
501 intptr_t *v2
, intptr_t expect2
,
502 intptr_t newv
, int cpu
)
506 __asm__ __volatile__
goto (
507 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
508 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
509 #ifdef RSEQ_COMPARE_TWICE
510 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
511 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
512 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error3
])
514 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
515 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
517 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
519 RSEQ_ASM_OP_CMPEQ(v2
, expect2
, %l
[cmpfail
])
521 #ifdef RSEQ_COMPARE_TWICE
522 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
523 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
524 RSEQ_ASM_OP_CMPEQ(v2
, expect2
, %l
[error3
])
526 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
528 RSEQ_ASM_DEFINE_ABORT(4, abort
)
529 : /* gcc asm goto does not allow outputs */
530 : [cpu_id
] "r" (cpu
),
531 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
532 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
534 [expect
] "r" (expect
),
536 [expect2
] "r" (expect2
),
539 : "memory", RSEQ_ASM_TMP_REG
541 #ifdef RSEQ_COMPARE_TWICE
542 , error1
, error2
, error3
545 rseq_after_asm_goto();
548 rseq_after_asm_goto();
552 rseq_after_asm_goto();
554 #ifdef RSEQ_COMPARE_TWICE
556 rseq_after_asm_goto();
557 rseq_bug("cpu_id comparison failed");
559 rseq_after_asm_goto();
560 rseq_bug("expected value comparison failed");
562 rseq_after_asm_goto();
563 rseq_bug("2nd expected value comparison failed");
567 static inline __attribute__((always_inline
))
568 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v
, intptr_t expect
,
569 void *dst
, void *src
, size_t len
,
570 intptr_t newv
, int cpu
)
574 __asm__ __volatile__
goto (
575 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
576 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
577 #ifdef RSEQ_COMPARE_TWICE
578 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
579 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
581 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
582 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
584 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
586 #ifdef RSEQ_COMPARE_TWICE
587 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
588 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
590 RSEQ_ASM_OP_R_BAD_MEMCPY(dst
, src
, len
)
592 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
594 RSEQ_ASM_DEFINE_ABORT(4, abort
)
595 : /* gcc asm goto does not allow outputs */
596 : [cpu_id
] "r" (cpu
),
597 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
598 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
599 [expect
] "r" (expect
),
606 : "memory", RSEQ_ASM_TMP_REG
, RSEQ_ASM_TMP_REG_2
608 #ifdef RSEQ_COMPARE_TWICE
612 rseq_after_asm_goto();
615 rseq_after_asm_goto();
619 rseq_after_asm_goto();
621 #ifdef RSEQ_COMPARE_TWICE
623 rseq_after_asm_goto();
624 rseq_bug("cpu_id comparison failed");
626 rseq_after_asm_goto();
627 rseq_bug("expected value comparison failed");
631 static inline __attribute__((always_inline
))
632 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v
, intptr_t expect
,
633 void *dst
, void *src
, size_t len
,
634 intptr_t newv
, int cpu
)
638 __asm__ __volatile__
goto (
639 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
640 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
641 #ifdef RSEQ_COMPARE_TWICE
642 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
643 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
645 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
646 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
648 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
650 #ifdef RSEQ_COMPARE_TWICE
651 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
652 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
654 RSEQ_ASM_OP_R_BAD_MEMCPY(dst
, src
, len
)
656 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv
, v
, 3)
658 RSEQ_ASM_DEFINE_ABORT(4, abort
)
659 : /* gcc asm goto does not allow outputs */
660 : [cpu_id
] "r" (cpu
),
661 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
662 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
663 [expect
] "r" (expect
),
670 : "memory", RSEQ_ASM_TMP_REG
, RSEQ_ASM_TMP_REG_2
672 #ifdef RSEQ_COMPARE_TWICE
676 rseq_after_asm_goto();
679 rseq_after_asm_goto();
683 rseq_after_asm_goto();
685 #ifdef RSEQ_COMPARE_TWICE
687 rseq_after_asm_goto();
688 rseq_bug("cpu_id comparison failed");
690 rseq_after_asm_goto();
691 rseq_bug("expected value comparison failed");
695 #endif /* !RSEQ_SKIP_FASTPATH */
This page took 0.042859 seconds and 3 git commands to generate.