projects / librseq.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
arm: Template memory ordering and percpu access mode
[librseq.git] / include / rseq / rseq-arm64.h
CommitLineData
90702366 1/* SPDX-License-Identifier: MIT */
f2d7b530
MJ		 2/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3/* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
4
d78a16c2
MD		 5/*
6 * rseq-arm64.h
d78a16c2
MD		 7 */
8
8b7c64f7
MD		 9/*
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
13 */
14#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
15
16#ifdef __AARCH64EB__
17#define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
18#else
19#define RSEQ_SIG_DATA RSEQ_SIG_CODE
20#endif
21
22#define RSEQ_SIG RSEQ_SIG_DATA
d78a16c2
MD		 23
24#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
27
28#define rseq_smp_load_acquire(p) \
29__extension__ ({ \
18ca65e0
MD		 30 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \
31 switch (sizeof(*(p))) { \
d78a16c2 32 case 1: \
18ca65e0
MD		 33 __asm__ __volatile__ ("ldarb %w0, %1" \
34 : "=r" (*(__u8 *)__u.__c) \
35 : "Q" (*(p)) : "memory"); \
d78a16c2
MD		 36 break; \
37 case 2: \
18ca65e0
MD		 38 __asm__ __volatile__ ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)__u.__c) \
40 : "Q" (*(p)) : "memory"); \
d78a16c2
MD		 41 break; \
42 case 4: \
18ca65e0
MD		 43 __asm__ __volatile__ ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)__u.__c) \
45 : "Q" (*(p)) : "memory"); \
d78a16c2
MD		 46 break; \
47 case 8: \
18ca65e0
MD		 48 __asm__ __volatile__ ("ldar %0, %1" \
49 : "=r" (*(__u64 *)__u.__c) \
50 : "Q" (*(p)) : "memory"); \
d78a16c2
MD		 51 break; \
52 } \
0aac367c 53 (rseq_unqual_scalar_typeof(*(p)))__u.__val; \
d78a16c2
MD		 54})
55
56#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
57
58#define rseq_smp_store_release(p, v) \
59do { \
18ca65e0
MD		 60 union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \
61 { .__val = (rseq_unqual_scalar_typeof(*(p))) (v) }; \
62 switch (sizeof(*(p))) { \
d78a16c2 63 case 1: \
18ca65e0
MD		 64 __asm__ __volatile__ ("stlrb %w1, %0" \
65 : "=Q" (*(p)) \
66 : "r" (*(__u8 *)__u.__c) \
d78a16c2
MD		 67 : "memory"); \
68 break; \
69 case 2: \
18ca65e0
MD		 70 __asm__ __volatile__ ("stlrh %w1, %0" \
71 : "=Q" (*(p)) \
72 : "r" (*(__u16 *)__u.__c) \
d78a16c2
MD		 73 : "memory"); \
74 break; \
75 case 4: \
18ca65e0
MD		 76 __asm__ __volatile__ ("stlr %w1, %0" \
77 : "=Q" (*(p)) \
78 : "r" (*(__u32 *)__u.__c) \
d78a16c2
MD		 79 : "memory"); \
80 break; \
81 case 8: \
18ca65e0
MD		 82 __asm__ __volatile__ ("stlr %1, %0" \
83 : "=Q" (*(p)) \
84 : "r" (*(__u64 *)__u.__c) \
d78a16c2
MD		 85 : "memory"); \
86 break; \
87 } \
88} while (0)
89
d78a16c2
MD		 90#define RSEQ_ASM_TMP_REG32 "w15"
91#define RSEQ_ASM_TMP_REG "x15"
92#define RSEQ_ASM_TMP_REG_2 "x14"
93
94#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
95 post_commit_offset, abort_ip) \
dd01d0fb 96 " .pushsection __rseq_cs, \"aw\"\n" \
d78a16c2
MD		 97 " .balign 32\n" \
98 __rseq_str(label) ":\n" \
99 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
100 " .quad " __rseq_str(start_ip) ", " \
101 __rseq_str(post_commit_offset) ", " \
102 __rseq_str(abort_ip) "\n" \
dd01d0fb
MD		 103 " .popsection\n\t" \
104 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
105 " .quad " __rseq_str(label) "b\n" \
d78a16c2
MD		 106 " .popsection\n"
107
108#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
109 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
110 (post_commit_ip - start_ip), abort_ip)
111
90d9876e
MD		 112/*
113 * Exit points of a rseq critical section consist of all instructions outside
fd622cad
MD		 114 * of the critical section where a critical section can either branch to or
115 * reach through the normal course of its execution. The abort IP and the
116 * post-commit IP are already part of the __rseq_cs section and should not be
117 * explicitly defined as additional exit points. Knowing all exit points is
118 * useful to assist debuggers stepping over the critical section.
90d9876e
MD		 119 */
120#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
121 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
122 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
123 " .popsection\n"
124
d78a16c2
MD		 125#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
126 RSEQ_INJECT_ASM(1) \
127 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
128 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
129 ", :lo12:" __rseq_str(cs_label) "\n" \
130 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
131 __rseq_str(label) ":\n"
132
133#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
134 " b 222f\n" \
8b7c64f7 135 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
d78a16c2
MD		 136 __rseq_str(label) ":\n" \
137 " b %l[" __rseq_str(abort_label) "]\n" \
138 "222:\n"
139
140#define RSEQ_ASM_OP_STORE(value, var) \
141 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
142
143#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
144 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
145
146#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
147 RSEQ_ASM_OP_STORE(value, var) \
148 __rseq_str(post_commit_label) ":\n"
149
150#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
151 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
152 __rseq_str(post_commit_label) ":\n"
153
154#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
155 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
156 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
157 ", %[" __rseq_str(expect) "]\n" \
158 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
159
160#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
161 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
162 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
163 ", %w[" __rseq_str(expect) "]\n" \
164 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
165
166#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
167 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
168 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
169 ", %[" __rseq_str(expect) "]\n" \
170 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
171
172#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
173 RSEQ_INJECT_ASM(2) \
174 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
175
176#define RSEQ_ASM_OP_R_LOAD(var) \
177 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
178
179#define RSEQ_ASM_OP_R_STORE(var) \
180 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
181
182#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
183 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
184 ", %[" __rseq_str(offset) "]]\n"
185
186#define RSEQ_ASM_OP_R_ADD(count) \
187 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
188 ", %[" __rseq_str(count) "]\n"
189
190#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
191 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
192 __rseq_str(post_commit_label) ":\n"
193
194#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
195 " cbz %[" __rseq_str(len) "], 333f\n" \
196 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
197 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
198 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
199 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
200 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
201 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
202 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
203 "333:\n"
204
205static inline __attribute__((always_inline))
206int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
207{
208 RSEQ_INJECT_C(9)
209
210 __asm__ __volatile__ goto (
211 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 212 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
213#ifdef RSEQ_COMPARE_TWICE
214 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
215 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
216#endif
d78a16c2
MD		 217 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
218 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
219 RSEQ_INJECT_ASM(3)
220 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
221 RSEQ_INJECT_ASM(4)
222#ifdef RSEQ_COMPARE_TWICE
223 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
224 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
225#endif
226 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
227 RSEQ_INJECT_ASM(5)
228 RSEQ_ASM_DEFINE_ABORT(4, abort)
229 : /* gcc asm goto does not allow outputs */
230 : [cpu_id] "r" (cpu),
9698c399 231 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 232 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 233 [v] "Qo" (*v),
234 [expect] "r" (expect),
235 [newv] "r" (newv)
236 RSEQ_INJECT_INPUT
237 : "memory", RSEQ_ASM_TMP_REG
238 : abort, cmpfail
239#ifdef RSEQ_COMPARE_TWICE
240 , error1, error2
241#endif
242 );
dd76f2d6 243 rseq_after_asm_goto();
d78a16c2
MD		 244 return 0;
245abort:
dd76f2d6 246 rseq_after_asm_goto();
d78a16c2
MD		 247 RSEQ_INJECT_FAILED
248 return -1;
249cmpfail:
dd76f2d6 250 rseq_after_asm_goto();
d78a16c2
MD		 251 return 1;
252#ifdef RSEQ_COMPARE_TWICE
253error1:
dd76f2d6 254 rseq_after_asm_goto();
d78a16c2
MD		 255 rseq_bug("cpu_id comparison failed");
256error2:
dd76f2d6 257 rseq_after_asm_goto();
d78a16c2
MD		 258 rseq_bug("expected value comparison failed");
259#endif
260}
261
262static inline __attribute__((always_inline))
263int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
d35eae6b 264 long voffp, intptr_t *load, int cpu)
d78a16c2
MD		 265{
266 RSEQ_INJECT_C(9)
267
268 __asm__ __volatile__ goto (
269 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 270 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
271#ifdef RSEQ_COMPARE_TWICE
272 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
273 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
274#endif
d78a16c2
MD		 275 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
276 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
277 RSEQ_INJECT_ASM(3)
278 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
279 RSEQ_INJECT_ASM(4)
280#ifdef RSEQ_COMPARE_TWICE
281 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
282 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
283#endif
284 RSEQ_ASM_OP_R_LOAD(v)
285 RSEQ_ASM_OP_R_STORE(load)
286 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
287 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
288 RSEQ_INJECT_ASM(5)
289 RSEQ_ASM_DEFINE_ABORT(4, abort)
290 : /* gcc asm goto does not allow outputs */
291 : [cpu_id] "r" (cpu),
9698c399 292 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 293 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 294 [v] "Qo" (*v),
295 [expectnot] "r" (expectnot),
296 [load] "Qo" (*load),
297 [voffp] "r" (voffp)
298 RSEQ_INJECT_INPUT
299 : "memory", RSEQ_ASM_TMP_REG
300 : abort, cmpfail
301#ifdef RSEQ_COMPARE_TWICE
302 , error1, error2
303#endif
304 );
dd76f2d6 305 rseq_after_asm_goto();
d78a16c2
MD		 306 return 0;
307abort:
dd76f2d6 308 rseq_after_asm_goto();
d78a16c2
MD		 309 RSEQ_INJECT_FAILED
310 return -1;
311cmpfail:
dd76f2d6 312 rseq_after_asm_goto();
d78a16c2
MD		 313 return 1;
314#ifdef RSEQ_COMPARE_TWICE
315error1:
dd76f2d6 316 rseq_after_asm_goto();
d78a16c2
MD		 317 rseq_bug("cpu_id comparison failed");
318error2:
dd76f2d6 319 rseq_after_asm_goto();
d78a16c2
MD		 320 rseq_bug("expected value comparison failed");
321#endif
322}
323
324static inline __attribute__((always_inline))
325int rseq_addv(intptr_t *v, intptr_t count, int cpu)
326{
327 RSEQ_INJECT_C(9)
328
329 __asm__ __volatile__ goto (
330 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 331#ifdef RSEQ_COMPARE_TWICE
332 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
333#endif
d78a16c2
MD		 334 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
335 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
336 RSEQ_INJECT_ASM(3)
337#ifdef RSEQ_COMPARE_TWICE
338 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
339#endif
340 RSEQ_ASM_OP_R_LOAD(v)
341 RSEQ_ASM_OP_R_ADD(count)
342 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
343 RSEQ_INJECT_ASM(4)
344 RSEQ_ASM_DEFINE_ABORT(4, abort)
345 : /* gcc asm goto does not allow outputs */
346 : [cpu_id] "r" (cpu),
9698c399 347 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 348 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 349 [v] "Qo" (*v),
350 [count] "r" (count)
351 RSEQ_INJECT_INPUT
352 : "memory", RSEQ_ASM_TMP_REG
353 : abort
354#ifdef RSEQ_COMPARE_TWICE
355 , error1
356#endif
357 );
dd76f2d6 358 rseq_after_asm_goto();
d78a16c2
MD		 359 return 0;
360abort:
dd76f2d6 361 rseq_after_asm_goto();
d78a16c2
MD		 362 RSEQ_INJECT_FAILED
363 return -1;
364#ifdef RSEQ_COMPARE_TWICE
365error1:
dd76f2d6 366 rseq_after_asm_goto();
d78a16c2
MD		 367 rseq_bug("cpu_id comparison failed");
368#endif
369}
370
371static inline __attribute__((always_inline))
372int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
373 intptr_t *v2, intptr_t newv2,
374 intptr_t newv, int cpu)
375{
376 RSEQ_INJECT_C(9)
377
378 __asm__ __volatile__ goto (
379 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 380 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
381#ifdef RSEQ_COMPARE_TWICE
382 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
383 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
384#endif
d78a16c2
MD		 385 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
386 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
387 RSEQ_INJECT_ASM(3)
388 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
389 RSEQ_INJECT_ASM(4)
390#ifdef RSEQ_COMPARE_TWICE
391 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
392 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
393#endif
394 RSEQ_ASM_OP_STORE(newv2, v2)
395 RSEQ_INJECT_ASM(5)
396 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
397 RSEQ_INJECT_ASM(6)
398 RSEQ_ASM_DEFINE_ABORT(4, abort)
399 : /* gcc asm goto does not allow outputs */
400 : [cpu_id] "r" (cpu),
9698c399 401 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 402 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 403 [expect] "r" (expect),
404 [v] "Qo" (*v),
405 [newv] "r" (newv),
406 [v2] "Qo" (*v2),
407 [newv2] "r" (newv2)
408 RSEQ_INJECT_INPUT
409 : "memory", RSEQ_ASM_TMP_REG
410 : abort, cmpfail
411#ifdef RSEQ_COMPARE_TWICE
412 , error1, error2
413#endif
414 );
dd76f2d6 415 rseq_after_asm_goto();
d78a16c2
MD		 416 return 0;
417abort:
dd76f2d6 418 rseq_after_asm_goto();
d78a16c2
MD		 419 RSEQ_INJECT_FAILED
420 return -1;
421cmpfail:
dd76f2d6 422 rseq_after_asm_goto();
d78a16c2
MD		 423 return 1;
424#ifdef RSEQ_COMPARE_TWICE
425error1:
dd76f2d6 426 rseq_after_asm_goto();
d78a16c2
MD		 427 rseq_bug("cpu_id comparison failed");
428error2:
dd76f2d6 429 rseq_after_asm_goto();
d78a16c2
MD		 430 rseq_bug("expected value comparison failed");
431#endif
432}
433
434static inline __attribute__((always_inline))
435int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
436 intptr_t *v2, intptr_t newv2,
437 intptr_t newv, int cpu)
438{
439 RSEQ_INJECT_C(9)
440
441 __asm__ __volatile__ goto (
442 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 443 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
444#ifdef RSEQ_COMPARE_TWICE
445 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
446 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
447#endif
d78a16c2
MD		 448 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
449 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
450 RSEQ_INJECT_ASM(3)
451 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
452 RSEQ_INJECT_ASM(4)
453#ifdef RSEQ_COMPARE_TWICE
454 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
455 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
456#endif
457 RSEQ_ASM_OP_STORE(newv2, v2)
458 RSEQ_INJECT_ASM(5)
459 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
460 RSEQ_INJECT_ASM(6)
461 RSEQ_ASM_DEFINE_ABORT(4, abort)
462 : /* gcc asm goto does not allow outputs */
463 : [cpu_id] "r" (cpu),
9698c399 464 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 465 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 466 [expect] "r" (expect),
467 [v] "Qo" (*v),
468 [newv] "r" (newv),
469 [v2] "Qo" (*v2),
470 [newv2] "r" (newv2)
471 RSEQ_INJECT_INPUT
472 : "memory", RSEQ_ASM_TMP_REG
473 : abort, cmpfail
474#ifdef RSEQ_COMPARE_TWICE
475 , error1, error2
476#endif
477 );
dd76f2d6 478 rseq_after_asm_goto();
d78a16c2
MD		 479 return 0;
480abort:
dd76f2d6 481 rseq_after_asm_goto();
d78a16c2
MD		 482 RSEQ_INJECT_FAILED
483 return -1;
484cmpfail:
dd76f2d6 485 rseq_after_asm_goto();
d78a16c2
MD		 486 return 1;
487#ifdef RSEQ_COMPARE_TWICE
488error1:
dd76f2d6 489 rseq_after_asm_goto();
d78a16c2
MD		 490 rseq_bug("cpu_id comparison failed");
491error2:
dd76f2d6 492 rseq_after_asm_goto();
d78a16c2
MD		 493 rseq_bug("expected value comparison failed");
494#endif
495}
496
497static inline __attribute__((always_inline))
498int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
499 intptr_t *v2, intptr_t expect2,
500 intptr_t newv, int cpu)
501{
502 RSEQ_INJECT_C(9)
503
504 __asm__ __volatile__ goto (
505 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 506 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
507#ifdef RSEQ_COMPARE_TWICE
508 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
509 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
510 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
511#endif
d78a16c2
MD		 512 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
513 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
514 RSEQ_INJECT_ASM(3)
515 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
516 RSEQ_INJECT_ASM(4)
517 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
518 RSEQ_INJECT_ASM(5)
519#ifdef RSEQ_COMPARE_TWICE
520 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
521 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
522 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
523#endif
524 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
525 RSEQ_INJECT_ASM(6)
526 RSEQ_ASM_DEFINE_ABORT(4, abort)
527 : /* gcc asm goto does not allow outputs */
528 : [cpu_id] "r" (cpu),
9698c399 529 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 530 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 531 [v] "Qo" (*v),
532 [expect] "r" (expect),
533 [v2] "Qo" (*v2),
534 [expect2] "r" (expect2),
535 [newv] "r" (newv)
536 RSEQ_INJECT_INPUT
537 : "memory", RSEQ_ASM_TMP_REG
538 : abort, cmpfail
539#ifdef RSEQ_COMPARE_TWICE
540 , error1, error2, error3
541#endif
542 );
dd76f2d6 543 rseq_after_asm_goto();
d78a16c2
MD		 544 return 0;
545abort:
dd76f2d6 546 rseq_after_asm_goto();
d78a16c2
MD		 547 RSEQ_INJECT_FAILED
548 return -1;
549cmpfail:
dd76f2d6 550 rseq_after_asm_goto();
d78a16c2
MD		 551 return 1;
552#ifdef RSEQ_COMPARE_TWICE
553error1:
dd76f2d6 554 rseq_after_asm_goto();
d78a16c2
MD		 555 rseq_bug("cpu_id comparison failed");
556error2:
dd76f2d6 557 rseq_after_asm_goto();
d78a16c2
MD		 558 rseq_bug("expected value comparison failed");
559error3:
dd76f2d6 560 rseq_after_asm_goto();
d78a16c2
MD		 561 rseq_bug("2nd expected value comparison failed");
562#endif
563}
564
565static inline __attribute__((always_inline))
566int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
567 void *dst, void *src, size_t len,
568 intptr_t newv, int cpu)
569{
570 RSEQ_INJECT_C(9)
571
572 __asm__ __volatile__ goto (
573 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 574 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
575#ifdef RSEQ_COMPARE_TWICE
576 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
577 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
578#endif
d78a16c2
MD		 579 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
580 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
581 RSEQ_INJECT_ASM(3)
582 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
583 RSEQ_INJECT_ASM(4)
584#ifdef RSEQ_COMPARE_TWICE
585 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
586 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
587#endif
588 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
589 RSEQ_INJECT_ASM(5)
590 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
591 RSEQ_INJECT_ASM(6)
592 RSEQ_ASM_DEFINE_ABORT(4, abort)
593 : /* gcc asm goto does not allow outputs */
594 : [cpu_id] "r" (cpu),
9698c399 595 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 596 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 597 [expect] "r" (expect),
598 [v] "Qo" (*v),
599 [newv] "r" (newv),
600 [dst] "r" (dst),
601 [src] "r" (src),
602 [len] "r" (len)
603 RSEQ_INJECT_INPUT
604 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
605 : abort, cmpfail
606#ifdef RSEQ_COMPARE_TWICE
607 , error1, error2
608#endif
609 );
dd76f2d6 610 rseq_after_asm_goto();
d78a16c2
MD		 611 return 0;
612abort:
dd76f2d6 613 rseq_after_asm_goto();
d78a16c2
MD		 614 RSEQ_INJECT_FAILED
615 return -1;
616cmpfail:
dd76f2d6 617 rseq_after_asm_goto();
d78a16c2
MD		 618 return 1;
619#ifdef RSEQ_COMPARE_TWICE
620error1:
dd76f2d6 621 rseq_after_asm_goto();
d78a16c2
MD		 622 rseq_bug("cpu_id comparison failed");
623error2:
dd76f2d6 624 rseq_after_asm_goto();
d78a16c2
MD		 625 rseq_bug("expected value comparison failed");
626#endif
627}
628
629static inline __attribute__((always_inline))
630int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
631 void *dst, void *src, size_t len,
632 intptr_t newv, int cpu)
633{
634 RSEQ_INJECT_C(9)
635
636 __asm__ __volatile__ goto (
637 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e
MD		 638 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
639#ifdef RSEQ_COMPARE_TWICE
640 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
641 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
642#endif
d78a16c2
MD		 643 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
644 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
645 RSEQ_INJECT_ASM(3)
646 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
647 RSEQ_INJECT_ASM(4)
648#ifdef RSEQ_COMPARE_TWICE
649 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
650 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
651#endif
652 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
653 RSEQ_INJECT_ASM(5)
654 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
655 RSEQ_INJECT_ASM(6)
656 RSEQ_ASM_DEFINE_ABORT(4, abort)
657 : /* gcc asm goto does not allow outputs */
658 : [cpu_id] "r" (cpu),
9698c399 659 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
2d533093 660 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
d78a16c2
MD		 661 [expect] "r" (expect),
662 [v] "Qo" (*v),
663 [newv] "r" (newv),
664 [dst] "r" (dst),
665 [src] "r" (src),
666 [len] "r" (len)
667 RSEQ_INJECT_INPUT
668 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
669 : abort, cmpfail
670#ifdef RSEQ_COMPARE_TWICE
671 , error1, error2
672#endif
673 );
dd76f2d6 674 rseq_after_asm_goto();
d78a16c2
MD		 675 return 0;
676abort:
dd76f2d6 677 rseq_after_asm_goto();
d78a16c2
MD		 678 RSEQ_INJECT_FAILED
679 return -1;
680cmpfail:
dd76f2d6 681 rseq_after_asm_goto();
d78a16c2
MD		 682 return 1;
683#ifdef RSEQ_COMPARE_TWICE
684error1:
dd76f2d6 685 rseq_after_asm_goto();
d78a16c2
MD		 686 rseq_bug("cpu_id comparison failed");
687error2:
dd76f2d6 688 rseq_after_asm_goto();
d78a16c2
MD		 689 rseq_bug("expected value comparison failed");
690#endif
691}
Librseq
This page took 0.069382 seconds and 4 git commands to generate.