projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
rseq-arm64: Fix buggy load-acquire/store-release macros
[librseq.git] / include / rseq / rseq-arm64.h
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3 /* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
4
5 /*
6 * rseq-arm64.h
7 */
8
9 /*
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
13 */
14 #define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
15
16 #ifdef __AARCH64EB__
17 #define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
18 #else
19 #define RSEQ_SIG_DATA RSEQ_SIG_CODE
20 #endif
21
22 #define RSEQ_SIG RSEQ_SIG_DATA
23
24 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
27
28 #define rseq_smp_load_acquire(p) \
29 __extension__ ({ \
30 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \
31 switch (sizeof(*(p))) { \
32 case 1: \
33 __asm__ __volatile__ ("ldarb %w0, %1" \
34 : "=r" (*(__u8 *)__u.__c) \
35 : "Q" (*(p)) : "memory"); \
36 break; \
37 case 2: \
38 __asm__ __volatile__ ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)__u.__c) \
40 : "Q" (*(p)) : "memory"); \
41 break; \
42 case 4: \
43 __asm__ __volatile__ ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)__u.__c) \
45 : "Q" (*(p)) : "memory"); \
46 break; \
47 case 8: \
48 __asm__ __volatile__ ("ldar %0, %1" \
49 : "=r" (*(__u64 *)__u.__c) \
50 : "Q" (*(p)) : "memory"); \
51 break; \
52 } \
53 (__typeof__(*(p)))__u.__val; \
54 })
55
56 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
57
58 #define rseq_smp_store_release(p, v) \
59 do { \
60 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \
61 { .__val = (rseq_unqual_scalar_typeof(*(p))) (v) }; \
62 switch (sizeof(*(p))) { \
63 case 1: \
64 __asm__ __volatile__ ("stlrb %w1, %0" \
65 : "=Q" (*(p)) \
66 : "r" (*(__u8 *)__u.__c) \
67 : "memory"); \
68 break; \
69 case 2: \
70 __asm__ __volatile__ ("stlrh %w1, %0" \
71 : "=Q" (*(p)) \
72 : "r" (*(__u16 *)__u.__c) \
73 : "memory"); \
74 break; \
75 case 4: \
76 __asm__ __volatile__ ("stlr %w1, %0" \
77 : "=Q" (*(p)) \
78 : "r" (*(__u32 *)__u.__c) \
79 : "memory"); \
80 break; \
81 case 8: \
82 __asm__ __volatile__ ("stlr %1, %0" \
83 : "=Q" (*(p)) \
84 : "r" (*(__u64 *)__u.__c) \
85 : "memory"); \
86 break; \
87 } \
88 } while (0)
89
90 #ifdef RSEQ_SKIP_FASTPATH
91 #include "rseq-skip.h"
92 #else /* !RSEQ_SKIP_FASTPATH */
93
94 #define RSEQ_ASM_TMP_REG32 "w15"
95 #define RSEQ_ASM_TMP_REG "x15"
96 #define RSEQ_ASM_TMP_REG_2 "x14"
97
98 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
99 post_commit_offset, abort_ip) \
100 " .pushsection __rseq_cs, \"aw\"\n" \
101 " .balign 32\n" \
102 __rseq_str(label) ":\n" \
103 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
104 " .quad " __rseq_str(start_ip) ", " \
105 __rseq_str(post_commit_offset) ", " \
106 __rseq_str(abort_ip) "\n" \
107 " .popsection\n\t" \
108 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
109 " .quad " __rseq_str(label) "b\n" \
110 " .popsection\n"
111
112 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
113 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
114 (post_commit_ip - start_ip), abort_ip)
115
116 /*
117 * Exit points of a rseq critical section consist of all instructions outside
118 * of the critical section where a critical section can either branch to or
119 * reach through the normal course of its execution. The abort IP and the
120 * post-commit IP are already part of the __rseq_cs section and should not be
121 * explicitly defined as additional exit points. Knowing all exit points is
122 * useful to assist debuggers stepping over the critical section.
123 */
124 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
125 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
126 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
127 " .popsection\n"
128
129 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
130 RSEQ_INJECT_ASM(1) \
131 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
132 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
133 ", :lo12:" __rseq_str(cs_label) "\n" \
134 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
135 __rseq_str(label) ":\n"
136
137 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
138 " b 222f\n" \
139 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
140 __rseq_str(label) ":\n" \
141 " b %l[" __rseq_str(abort_label) "]\n" \
142 "222:\n"
143
144 #define RSEQ_ASM_OP_STORE(value, var) \
145 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
146
147 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
148 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
149
150 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
151 RSEQ_ASM_OP_STORE(value, var) \
152 __rseq_str(post_commit_label) ":\n"
153
154 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
155 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
156 __rseq_str(post_commit_label) ":\n"
157
158 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
159 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
160 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
161 ", %[" __rseq_str(expect) "]\n" \
162 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
163
164 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
165 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
166 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
167 ", %w[" __rseq_str(expect) "]\n" \
168 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
169
170 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \
171 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
172 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
173 ", %[" __rseq_str(expect) "]\n" \
174 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
175
176 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
177 RSEQ_INJECT_ASM(2) \
178 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
179
180 #define RSEQ_ASM_OP_R_LOAD(var) \
181 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
182
183 #define RSEQ_ASM_OP_R_STORE(var) \
184 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
185
186 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
187 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
188 ", %[" __rseq_str(offset) "]]\n"
189
190 #define RSEQ_ASM_OP_R_ADD(count) \
191 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
192 ", %[" __rseq_str(count) "]\n"
193
194 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
195 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
196 __rseq_str(post_commit_label) ":\n"
197
198 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
199 " cbz %[" __rseq_str(len) "], 333f\n" \
200 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
201 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
202 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
204 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
205 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
206 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
207 "333:\n"
208
209 static inline __attribute__((always_inline))
210 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
211 {
212 RSEQ_INJECT_C(9)
213
214 __asm__ __volatile__ goto (
215 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
217 #ifdef RSEQ_COMPARE_TWICE
218 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
219 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
220 #endif
221 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
222 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
223 RSEQ_INJECT_ASM(3)
224 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
225 RSEQ_INJECT_ASM(4)
226 #ifdef RSEQ_COMPARE_TWICE
227 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
228 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
229 #endif
230 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
231 RSEQ_INJECT_ASM(5)
232 RSEQ_ASM_DEFINE_ABORT(4, abort)
233 : /* gcc asm goto does not allow outputs */
234 : [cpu_id] "r" (cpu),
235 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
236 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
237 [v] "Qo" (*v),
238 [expect] "r" (expect),
239 [newv] "r" (newv)
240 RSEQ_INJECT_INPUT
241 : "memory", RSEQ_ASM_TMP_REG
242 : abort, cmpfail
243 #ifdef RSEQ_COMPARE_TWICE
244 , error1, error2
245 #endif
246 );
247 rseq_after_asm_goto();
248 return 0;
249 abort:
250 rseq_after_asm_goto();
251 RSEQ_INJECT_FAILED
252 return -1;
253 cmpfail:
254 rseq_after_asm_goto();
255 return 1;
256 #ifdef RSEQ_COMPARE_TWICE
257 error1:
258 rseq_after_asm_goto();
259 rseq_bug("cpu_id comparison failed");
260 error2:
261 rseq_after_asm_goto();
262 rseq_bug("expected value comparison failed");
263 #endif
264 }
265
266 static inline __attribute__((always_inline))
267 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
268 long voffp, intptr_t *load, int cpu)
269 {
270 RSEQ_INJECT_C(9)
271
272 __asm__ __volatile__ goto (
273 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
274 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
275 #ifdef RSEQ_COMPARE_TWICE
276 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
277 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
278 #endif
279 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
280 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
281 RSEQ_INJECT_ASM(3)
282 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
283 RSEQ_INJECT_ASM(4)
284 #ifdef RSEQ_COMPARE_TWICE
285 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
286 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
287 #endif
288 RSEQ_ASM_OP_R_LOAD(v)
289 RSEQ_ASM_OP_R_STORE(load)
290 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
291 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
292 RSEQ_INJECT_ASM(5)
293 RSEQ_ASM_DEFINE_ABORT(4, abort)
294 : /* gcc asm goto does not allow outputs */
295 : [cpu_id] "r" (cpu),
296 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
297 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
298 [v] "Qo" (*v),
299 [expectnot] "r" (expectnot),
300 [load] "Qo" (*load),
301 [voffp] "r" (voffp)
302 RSEQ_INJECT_INPUT
303 : "memory", RSEQ_ASM_TMP_REG
304 : abort, cmpfail
305 #ifdef RSEQ_COMPARE_TWICE
306 , error1, error2
307 #endif
308 );
309 rseq_after_asm_goto();
310 return 0;
311 abort:
312 rseq_after_asm_goto();
313 RSEQ_INJECT_FAILED
314 return -1;
315 cmpfail:
316 rseq_after_asm_goto();
317 return 1;
318 #ifdef RSEQ_COMPARE_TWICE
319 error1:
320 rseq_after_asm_goto();
321 rseq_bug("cpu_id comparison failed");
322 error2:
323 rseq_after_asm_goto();
324 rseq_bug("expected value comparison failed");
325 #endif
326 }
327
328 static inline __attribute__((always_inline))
329 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
330 {
331 RSEQ_INJECT_C(9)
332
333 __asm__ __volatile__ goto (
334 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
335 #ifdef RSEQ_COMPARE_TWICE
336 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
337 #endif
338 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
339 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
340 RSEQ_INJECT_ASM(3)
341 #ifdef RSEQ_COMPARE_TWICE
342 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
343 #endif
344 RSEQ_ASM_OP_R_LOAD(v)
345 RSEQ_ASM_OP_R_ADD(count)
346 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
347 RSEQ_INJECT_ASM(4)
348 RSEQ_ASM_DEFINE_ABORT(4, abort)
349 : /* gcc asm goto does not allow outputs */
350 : [cpu_id] "r" (cpu),
351 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
352 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
353 [v] "Qo" (*v),
354 [count] "r" (count)
355 RSEQ_INJECT_INPUT
356 : "memory", RSEQ_ASM_TMP_REG
357 : abort
358 #ifdef RSEQ_COMPARE_TWICE
359 , error1
360 #endif
361 );
362 rseq_after_asm_goto();
363 return 0;
364 abort:
365 rseq_after_asm_goto();
366 RSEQ_INJECT_FAILED
367 return -1;
368 #ifdef RSEQ_COMPARE_TWICE
369 error1:
370 rseq_after_asm_goto();
371 rseq_bug("cpu_id comparison failed");
372 #endif
373 }
374
375 static inline __attribute__((always_inline))
376 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
377 intptr_t *v2, intptr_t newv2,
378 intptr_t newv, int cpu)
379 {
380 RSEQ_INJECT_C(9)
381
382 __asm__ __volatile__ goto (
383 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
384 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
385 #ifdef RSEQ_COMPARE_TWICE
386 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
387 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
388 #endif
389 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
390 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
391 RSEQ_INJECT_ASM(3)
392 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
393 RSEQ_INJECT_ASM(4)
394 #ifdef RSEQ_COMPARE_TWICE
395 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
396 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
397 #endif
398 RSEQ_ASM_OP_STORE(newv2, v2)
399 RSEQ_INJECT_ASM(5)
400 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
401 RSEQ_INJECT_ASM(6)
402 RSEQ_ASM_DEFINE_ABORT(4, abort)
403 : /* gcc asm goto does not allow outputs */
404 : [cpu_id] "r" (cpu),
405 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
406 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
407 [expect] "r" (expect),
408 [v] "Qo" (*v),
409 [newv] "r" (newv),
410 [v2] "Qo" (*v2),
411 [newv2] "r" (newv2)
412 RSEQ_INJECT_INPUT
413 : "memory", RSEQ_ASM_TMP_REG
414 : abort, cmpfail
415 #ifdef RSEQ_COMPARE_TWICE
416 , error1, error2
417 #endif
418 );
419 rseq_after_asm_goto();
420 return 0;
421 abort:
422 rseq_after_asm_goto();
423 RSEQ_INJECT_FAILED
424 return -1;
425 cmpfail:
426 rseq_after_asm_goto();
427 return 1;
428 #ifdef RSEQ_COMPARE_TWICE
429 error1:
430 rseq_after_asm_goto();
431 rseq_bug("cpu_id comparison failed");
432 error2:
433 rseq_after_asm_goto();
434 rseq_bug("expected value comparison failed");
435 #endif
436 }
437
438 static inline __attribute__((always_inline))
439 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
440 intptr_t *v2, intptr_t newv2,
441 intptr_t newv, int cpu)
442 {
443 RSEQ_INJECT_C(9)
444
445 __asm__ __volatile__ goto (
446 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
447 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
448 #ifdef RSEQ_COMPARE_TWICE
449 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
450 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
451 #endif
452 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
453 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
454 RSEQ_INJECT_ASM(3)
455 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
456 RSEQ_INJECT_ASM(4)
457 #ifdef RSEQ_COMPARE_TWICE
458 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
459 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
460 #endif
461 RSEQ_ASM_OP_STORE(newv2, v2)
462 RSEQ_INJECT_ASM(5)
463 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
464 RSEQ_INJECT_ASM(6)
465 RSEQ_ASM_DEFINE_ABORT(4, abort)
466 : /* gcc asm goto does not allow outputs */
467 : [cpu_id] "r" (cpu),
468 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
469 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
470 [expect] "r" (expect),
471 [v] "Qo" (*v),
472 [newv] "r" (newv),
473 [v2] "Qo" (*v2),
474 [newv2] "r" (newv2)
475 RSEQ_INJECT_INPUT
476 : "memory", RSEQ_ASM_TMP_REG
477 : abort, cmpfail
478 #ifdef RSEQ_COMPARE_TWICE
479 , error1, error2
480 #endif
481 );
482 rseq_after_asm_goto();
483 return 0;
484 abort:
485 rseq_after_asm_goto();
486 RSEQ_INJECT_FAILED
487 return -1;
488 cmpfail:
489 rseq_after_asm_goto();
490 return 1;
491 #ifdef RSEQ_COMPARE_TWICE
492 error1:
493 rseq_after_asm_goto();
494 rseq_bug("cpu_id comparison failed");
495 error2:
496 rseq_after_asm_goto();
497 rseq_bug("expected value comparison failed");
498 #endif
499 }
500
501 static inline __attribute__((always_inline))
502 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
503 intptr_t *v2, intptr_t expect2,
504 intptr_t newv, int cpu)
505 {
506 RSEQ_INJECT_C(9)
507
508 __asm__ __volatile__ goto (
509 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
510 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
511 #ifdef RSEQ_COMPARE_TWICE
512 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
513 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
514 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
515 #endif
516 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
517 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
518 RSEQ_INJECT_ASM(3)
519 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
520 RSEQ_INJECT_ASM(4)
521 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
522 RSEQ_INJECT_ASM(5)
523 #ifdef RSEQ_COMPARE_TWICE
524 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
525 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
526 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
527 #endif
528 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
529 RSEQ_INJECT_ASM(6)
530 RSEQ_ASM_DEFINE_ABORT(4, abort)
531 : /* gcc asm goto does not allow outputs */
532 : [cpu_id] "r" (cpu),
533 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
534 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
535 [v] "Qo" (*v),
536 [expect] "r" (expect),
537 [v2] "Qo" (*v2),
538 [expect2] "r" (expect2),
539 [newv] "r" (newv)
540 RSEQ_INJECT_INPUT
541 : "memory", RSEQ_ASM_TMP_REG
542 : abort, cmpfail
543 #ifdef RSEQ_COMPARE_TWICE
544 , error1, error2, error3
545 #endif
546 );
547 rseq_after_asm_goto();
548 return 0;
549 abort:
550 rseq_after_asm_goto();
551 RSEQ_INJECT_FAILED
552 return -1;
553 cmpfail:
554 rseq_after_asm_goto();
555 return 1;
556 #ifdef RSEQ_COMPARE_TWICE
557 error1:
558 rseq_after_asm_goto();
559 rseq_bug("cpu_id comparison failed");
560 error2:
561 rseq_after_asm_goto();
562 rseq_bug("expected value comparison failed");
563 error3:
564 rseq_after_asm_goto();
565 rseq_bug("2nd expected value comparison failed");
566 #endif
567 }
568
569 static inline __attribute__((always_inline))
570 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
571 void *dst, void *src, size_t len,
572 intptr_t newv, int cpu)
573 {
574 RSEQ_INJECT_C(9)
575
576 __asm__ __volatile__ goto (
577 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
578 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
579 #ifdef RSEQ_COMPARE_TWICE
580 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
581 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
582 #endif
583 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
584 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
585 RSEQ_INJECT_ASM(3)
586 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
587 RSEQ_INJECT_ASM(4)
588 #ifdef RSEQ_COMPARE_TWICE
589 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
590 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
591 #endif
592 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
593 RSEQ_INJECT_ASM(5)
594 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
595 RSEQ_INJECT_ASM(6)
596 RSEQ_ASM_DEFINE_ABORT(4, abort)
597 : /* gcc asm goto does not allow outputs */
598 : [cpu_id] "r" (cpu),
599 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
600 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
601 [expect] "r" (expect),
602 [v] "Qo" (*v),
603 [newv] "r" (newv),
604 [dst] "r" (dst),
605 [src] "r" (src),
606 [len] "r" (len)
607 RSEQ_INJECT_INPUT
608 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
609 : abort, cmpfail
610 #ifdef RSEQ_COMPARE_TWICE
611 , error1, error2
612 #endif
613 );
614 rseq_after_asm_goto();
615 return 0;
616 abort:
617 rseq_after_asm_goto();
618 RSEQ_INJECT_FAILED
619 return -1;
620 cmpfail:
621 rseq_after_asm_goto();
622 return 1;
623 #ifdef RSEQ_COMPARE_TWICE
624 error1:
625 rseq_after_asm_goto();
626 rseq_bug("cpu_id comparison failed");
627 error2:
628 rseq_after_asm_goto();
629 rseq_bug("expected value comparison failed");
630 #endif
631 }
632
633 static inline __attribute__((always_inline))
634 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
635 void *dst, void *src, size_t len,
636 intptr_t newv, int cpu)
637 {
638 RSEQ_INJECT_C(9)
639
640 __asm__ __volatile__ goto (
641 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
642 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
643 #ifdef RSEQ_COMPARE_TWICE
644 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
645 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
646 #endif
647 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
648 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
649 RSEQ_INJECT_ASM(3)
650 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
651 RSEQ_INJECT_ASM(4)
652 #ifdef RSEQ_COMPARE_TWICE
653 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
654 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
655 #endif
656 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
657 RSEQ_INJECT_ASM(5)
658 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
659 RSEQ_INJECT_ASM(6)
660 RSEQ_ASM_DEFINE_ABORT(4, abort)
661 : /* gcc asm goto does not allow outputs */
662 : [cpu_id] "r" (cpu),
663 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
664 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
665 [expect] "r" (expect),
666 [v] "Qo" (*v),
667 [newv] "r" (newv),
668 [dst] "r" (dst),
669 [src] "r" (src),
670 [len] "r" (len)
671 RSEQ_INJECT_INPUT
672 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
673 : abort, cmpfail
674 #ifdef RSEQ_COMPARE_TWICE
675 , error1, error2
676 #endif
677 );
678 rseq_after_asm_goto();
679 return 0;
680 abort:
681 rseq_after_asm_goto();
682 RSEQ_INJECT_FAILED
683 return -1;
684 cmpfail:
685 rseq_after_asm_goto();
686 return 1;
687 #ifdef RSEQ_COMPARE_TWICE
688 error1:
689 rseq_after_asm_goto();
690 rseq_bug("cpu_id comparison failed");
691 error2:
692 rseq_after_asm_goto();
693 rseq_bug("expected value comparison failed");
694 #endif
695 }
696
697 #endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.060937 seconds and 5 git commands to generate.