1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3 /* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
14 #define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
17 #define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
19 #define RSEQ_SIG_DATA RSEQ_SIG_CODE
22 #define RSEQ_SIG RSEQ_SIG_DATA
24 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
28 #define rseq_smp_load_acquire(p) \
30 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \
31 switch (sizeof(*(p))) { \
33 __asm__ __volatile__ ("ldarb %w0, %1" \
34 : "=r" (*(__u8 *)__u.__c) \
35 : "Q" (*(p)) : "memory"); \
38 __asm__ __volatile__ ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)__u.__c) \
40 : "Q" (*(p)) : "memory"); \
43 __asm__ __volatile__ ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)__u.__c) \
45 : "Q" (*(p)) : "memory"); \
48 __asm__ __volatile__ ("ldar %0, %1" \
49 : "=r" (*(__u64 *)__u.__c) \
50 : "Q" (*(p)) : "memory"); \
53 (rseq_unqual_scalar_typeof(*(p)))__u.__val; \
56 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
58 #define rseq_smp_store_release(p, v) \
60 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \
61 { .__val = (rseq_unqual_scalar_typeof(*(p))) (v) }; \
62 switch (sizeof(*(p))) { \
64 __asm__ __volatile__ ("stlrb %w1, %0" \
66 : "r" (*(__u8 *)__u.__c) \
70 __asm__ __volatile__ ("stlrh %w1, %0" \
72 : "r" (*(__u16 *)__u.__c) \
76 __asm__ __volatile__ ("stlr %w1, %0" \
78 : "r" (*(__u32 *)__u.__c) \
82 __asm__ __volatile__ ("stlr %1, %0" \
84 : "r" (*(__u64 *)__u.__c) \
90 #ifdef RSEQ_SKIP_FASTPATH
91 #include "rseq-skip.h"
92 #else /* !RSEQ_SKIP_FASTPATH */
94 #define RSEQ_ASM_TMP_REG32 "w15"
95 #define RSEQ_ASM_TMP_REG "x15"
96 #define RSEQ_ASM_TMP_REG_2 "x14"
98 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
99 post_commit_offset, abort_ip) \
100 " .pushsection __rseq_cs, \"aw\"\n" \
102 __rseq_str(label) ":\n" \
103 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
104 " .quad " __rseq_str(start_ip) ", " \
105 __rseq_str(post_commit_offset) ", " \
106 __rseq_str(abort_ip) "\n" \
108 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
109 " .quad " __rseq_str(label) "b\n" \
112 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
113 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
114 (post_commit_ip - start_ip), abort_ip)
117 * Exit points of a rseq critical section consist of all instructions outside
118 * of the critical section where a critical section can either branch to or
119 * reach through the normal course of its execution. The abort IP and the
120 * post-commit IP are already part of the __rseq_cs section and should not be
121 * explicitly defined as additional exit points. Knowing all exit points is
122 * useful to assist debuggers stepping over the critical section.
124 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
125 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
126 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
129 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
131 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
132 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
133 ", :lo12:" __rseq_str(cs_label) "\n" \
134 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
135 __rseq_str(label) ":\n"
137 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
139 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
140 __rseq_str(label) ":\n" \
141 " b %l[" __rseq_str(abort_label) "]\n" \
144 #define RSEQ_ASM_OP_STORE(value, var) \
145 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
147 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
148 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
150 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
151 RSEQ_ASM_OP_STORE(value, var) \
152 __rseq_str(post_commit_label) ":\n"
154 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
155 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
156 __rseq_str(post_commit_label) ":\n"
158 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
159 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
160 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
161 ", %[" __rseq_str(expect) "]\n" \
162 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
164 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
165 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
166 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
167 ", %w[" __rseq_str(expect) "]\n" \
168 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
170 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \
171 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
172 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
173 ", %[" __rseq_str(expect) "]\n" \
174 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
176 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
178 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
180 #define RSEQ_ASM_OP_R_LOAD(var) \
181 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
183 #define RSEQ_ASM_OP_R_STORE(var) \
184 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
186 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
187 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
188 ", %[" __rseq_str(offset) "]]\n"
190 #define RSEQ_ASM_OP_R_ADD(count) \
191 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
192 ", %[" __rseq_str(count) "]\n"
194 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
195 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
196 __rseq_str(post_commit_label) ":\n"
198 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
199 " cbz %[" __rseq_str(len) "], 333f\n" \
200 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
201 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
202 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
204 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
205 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
206 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
209 static inline __attribute__((always_inline
))
210 int rseq_cmpeqv_storev(intptr_t *v
, intptr_t expect
, intptr_t newv
, int cpu
)
214 __asm__ __volatile__
goto (
215 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
216 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
217 #ifdef RSEQ_COMPARE_TWICE
218 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
219 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
221 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
222 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
224 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
226 #ifdef RSEQ_COMPARE_TWICE
227 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
228 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
230 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
232 RSEQ_ASM_DEFINE_ABORT(4, abort
)
233 : /* gcc asm goto does not allow outputs */
234 : [cpu_id
] "r" (cpu
),
235 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
236 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
238 [expect
] "r" (expect
),
241 : "memory", RSEQ_ASM_TMP_REG
243 #ifdef RSEQ_COMPARE_TWICE
247 rseq_after_asm_goto();
250 rseq_after_asm_goto();
254 rseq_after_asm_goto();
256 #ifdef RSEQ_COMPARE_TWICE
258 rseq_after_asm_goto();
259 rseq_bug("cpu_id comparison failed");
261 rseq_after_asm_goto();
262 rseq_bug("expected value comparison failed");
266 static inline __attribute__((always_inline
))
267 int rseq_cmpnev_storeoffp_load(intptr_t *v
, intptr_t expectnot
,
268 long voffp
, intptr_t *load
, int cpu
)
272 __asm__ __volatile__
goto (
273 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
274 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
275 #ifdef RSEQ_COMPARE_TWICE
276 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
277 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
279 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
280 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
282 RSEQ_ASM_OP_CMPNE(v
, expectnot
, %l
[cmpfail
])
284 #ifdef RSEQ_COMPARE_TWICE
285 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
286 RSEQ_ASM_OP_CMPNE(v
, expectnot
, %l
[error2
])
288 RSEQ_ASM_OP_R_LOAD(v
)
289 RSEQ_ASM_OP_R_STORE(load
)
290 RSEQ_ASM_OP_R_LOAD_OFF(voffp
)
291 RSEQ_ASM_OP_R_FINAL_STORE(v
, 3)
293 RSEQ_ASM_DEFINE_ABORT(4, abort
)
294 : /* gcc asm goto does not allow outputs */
295 : [cpu_id
] "r" (cpu
),
296 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
297 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
299 [expectnot
] "r" (expectnot
),
303 : "memory", RSEQ_ASM_TMP_REG
305 #ifdef RSEQ_COMPARE_TWICE
309 rseq_after_asm_goto();
312 rseq_after_asm_goto();
316 rseq_after_asm_goto();
318 #ifdef RSEQ_COMPARE_TWICE
320 rseq_after_asm_goto();
321 rseq_bug("cpu_id comparison failed");
323 rseq_after_asm_goto();
324 rseq_bug("expected value comparison failed");
328 static inline __attribute__((always_inline
))
329 int rseq_addv(intptr_t *v
, intptr_t count
, int cpu
)
333 __asm__ __volatile__
goto (
334 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
335 #ifdef RSEQ_COMPARE_TWICE
336 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
338 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
339 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
341 #ifdef RSEQ_COMPARE_TWICE
342 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
344 RSEQ_ASM_OP_R_LOAD(v
)
345 RSEQ_ASM_OP_R_ADD(count
)
346 RSEQ_ASM_OP_R_FINAL_STORE(v
, 3)
348 RSEQ_ASM_DEFINE_ABORT(4, abort
)
349 : /* gcc asm goto does not allow outputs */
350 : [cpu_id
] "r" (cpu
),
351 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
352 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
356 : "memory", RSEQ_ASM_TMP_REG
358 #ifdef RSEQ_COMPARE_TWICE
362 rseq_after_asm_goto();
365 rseq_after_asm_goto();
368 #ifdef RSEQ_COMPARE_TWICE
370 rseq_after_asm_goto();
371 rseq_bug("cpu_id comparison failed");
375 static inline __attribute__((always_inline
))
376 int rseq_cmpeqv_trystorev_storev(intptr_t *v
, intptr_t expect
,
377 intptr_t *v2
, intptr_t newv2
,
378 intptr_t newv
, int cpu
)
382 __asm__ __volatile__
goto (
383 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
384 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
385 #ifdef RSEQ_COMPARE_TWICE
386 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
387 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
389 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
390 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
392 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
394 #ifdef RSEQ_COMPARE_TWICE
395 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
396 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
398 RSEQ_ASM_OP_STORE(newv2
, v2
)
400 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
402 RSEQ_ASM_DEFINE_ABORT(4, abort
)
403 : /* gcc asm goto does not allow outputs */
404 : [cpu_id
] "r" (cpu
),
405 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
406 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
407 [expect
] "r" (expect
),
413 : "memory", RSEQ_ASM_TMP_REG
415 #ifdef RSEQ_COMPARE_TWICE
419 rseq_after_asm_goto();
422 rseq_after_asm_goto();
426 rseq_after_asm_goto();
428 #ifdef RSEQ_COMPARE_TWICE
430 rseq_after_asm_goto();
431 rseq_bug("cpu_id comparison failed");
433 rseq_after_asm_goto();
434 rseq_bug("expected value comparison failed");
438 static inline __attribute__((always_inline
))
439 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v
, intptr_t expect
,
440 intptr_t *v2
, intptr_t newv2
,
441 intptr_t newv
, int cpu
)
445 __asm__ __volatile__
goto (
446 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
447 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
448 #ifdef RSEQ_COMPARE_TWICE
449 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
450 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
452 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
453 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
455 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
457 #ifdef RSEQ_COMPARE_TWICE
458 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
459 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
461 RSEQ_ASM_OP_STORE(newv2
, v2
)
463 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv
, v
, 3)
465 RSEQ_ASM_DEFINE_ABORT(4, abort
)
466 : /* gcc asm goto does not allow outputs */
467 : [cpu_id
] "r" (cpu
),
468 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
469 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
470 [expect
] "r" (expect
),
476 : "memory", RSEQ_ASM_TMP_REG
478 #ifdef RSEQ_COMPARE_TWICE
482 rseq_after_asm_goto();
485 rseq_after_asm_goto();
489 rseq_after_asm_goto();
491 #ifdef RSEQ_COMPARE_TWICE
493 rseq_after_asm_goto();
494 rseq_bug("cpu_id comparison failed");
496 rseq_after_asm_goto();
497 rseq_bug("expected value comparison failed");
501 static inline __attribute__((always_inline
))
502 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v
, intptr_t expect
,
503 intptr_t *v2
, intptr_t expect2
,
504 intptr_t newv
, int cpu
)
508 __asm__ __volatile__
goto (
509 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
510 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
511 #ifdef RSEQ_COMPARE_TWICE
512 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
513 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
514 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error3
])
516 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
517 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
519 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
521 RSEQ_ASM_OP_CMPEQ(v2
, expect2
, %l
[cmpfail
])
523 #ifdef RSEQ_COMPARE_TWICE
524 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
525 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
526 RSEQ_ASM_OP_CMPEQ(v2
, expect2
, %l
[error3
])
528 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
530 RSEQ_ASM_DEFINE_ABORT(4, abort
)
531 : /* gcc asm goto does not allow outputs */
532 : [cpu_id
] "r" (cpu
),
533 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
534 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
536 [expect
] "r" (expect
),
538 [expect2
] "r" (expect2
),
541 : "memory", RSEQ_ASM_TMP_REG
543 #ifdef RSEQ_COMPARE_TWICE
544 , error1
, error2
, error3
547 rseq_after_asm_goto();
550 rseq_after_asm_goto();
554 rseq_after_asm_goto();
556 #ifdef RSEQ_COMPARE_TWICE
558 rseq_after_asm_goto();
559 rseq_bug("cpu_id comparison failed");
561 rseq_after_asm_goto();
562 rseq_bug("expected value comparison failed");
564 rseq_after_asm_goto();
565 rseq_bug("2nd expected value comparison failed");
569 static inline __attribute__((always_inline
))
570 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v
, intptr_t expect
,
571 void *dst
, void *src
, size_t len
,
572 intptr_t newv
, int cpu
)
576 __asm__ __volatile__
goto (
577 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
578 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
579 #ifdef RSEQ_COMPARE_TWICE
580 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
581 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
583 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
584 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
586 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
588 #ifdef RSEQ_COMPARE_TWICE
589 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
590 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
592 RSEQ_ASM_OP_R_BAD_MEMCPY(dst
, src
, len
)
594 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
596 RSEQ_ASM_DEFINE_ABORT(4, abort
)
597 : /* gcc asm goto does not allow outputs */
598 : [cpu_id
] "r" (cpu
),
599 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
600 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
601 [expect
] "r" (expect
),
608 : "memory", RSEQ_ASM_TMP_REG
, RSEQ_ASM_TMP_REG_2
610 #ifdef RSEQ_COMPARE_TWICE
614 rseq_after_asm_goto();
617 rseq_after_asm_goto();
621 rseq_after_asm_goto();
623 #ifdef RSEQ_COMPARE_TWICE
625 rseq_after_asm_goto();
626 rseq_bug("cpu_id comparison failed");
628 rseq_after_asm_goto();
629 rseq_bug("expected value comparison failed");
633 static inline __attribute__((always_inline
))
634 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v
, intptr_t expect
,
635 void *dst
, void *src
, size_t len
,
636 intptr_t newv
, int cpu
)
640 __asm__ __volatile__
goto (
641 RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
642 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[cmpfail
])
643 #ifdef RSEQ_COMPARE_TWICE
644 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error1
])
645 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, %l
[error2
])
647 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
648 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
650 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
652 #ifdef RSEQ_COMPARE_TWICE
653 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
654 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
656 RSEQ_ASM_OP_R_BAD_MEMCPY(dst
, src
, len
)
658 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv
, v
, 3)
660 RSEQ_ASM_DEFINE_ABORT(4, abort
)
661 : /* gcc asm goto does not allow outputs */
662 : [cpu_id
] "r" (cpu
),
663 [current_cpu_id
] "Qo" (rseq_get_abi()->cpu_id
),
664 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
665 [expect
] "r" (expect
),
672 : "memory", RSEQ_ASM_TMP_REG
, RSEQ_ASM_TMP_REG_2
674 #ifdef RSEQ_COMPARE_TWICE
678 rseq_after_asm_goto();
681 rseq_after_asm_goto();
685 rseq_after_asm_goto();
687 #ifdef RSEQ_COMPARE_TWICE
689 rseq_after_asm_goto();
690 rseq_bug("cpu_id comparison failed");
692 rseq_after_asm_goto();
693 rseq_bug("expected value comparison failed");
697 #endif /* !RSEQ_SKIP_FASTPATH */
This page took 0.0477 seconds and 5 git commands to generate.