projects / librseq.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86_64: Add rseq_offset_deref_addv()
[librseq.git] / include / rseq / rseq-ppc.h
CommitLineData
744d0b8b 1/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
784b0012
MD		 2/*
3 * rseq-ppc.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
0158e666
MD		 9/*
10 * RSEQ_SIG is used with the following trap instruction:
11 *
12 * powerpc-be: 0f e5 00 0b twui r5,11
13 * powerpc64-le: 0b 00 e5 0f twui r5,11
14 * powerpc64-be: 0f e5 00 0b twui r5,11
15 */
16
17#define RSEQ_SIG 0x0fe5000b
784b0012
MD		 18
19#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
20#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
21#define rseq_smp_rmb() rseq_smp_lwsync()
22#define rseq_smp_wmb() rseq_smp_lwsync()
23
24#define rseq_smp_load_acquire(p) \
25__extension__ ({ \
26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
27 rseq_smp_lwsync(); \
28 ____p1; \
29})
30
31#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
32
33#define rseq_smp_store_release(p, v) \
34do { \
35 rseq_smp_lwsync(); \
36 RSEQ_WRITE_ONCE(*p, v); \
37} while (0)
38
39#ifdef RSEQ_SKIP_FASTPATH
40#include "rseq-skip.h"
41#else /* !RSEQ_SKIP_FASTPATH */
42
43/*
dd01d0fb
MD		 44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
45 * better handle single-stepping through the restartable critical sections.
784b0012
MD		 46 */
47
48#ifdef __PPC64__
49
a799a7ba
MD		 50#define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
51#define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
52#define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
53#define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
54#define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */
55#define RSEQ_CMP_LONG "cmpd "
784b0012
MD		 56
57#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
58 start_ip, post_commit_offset, abort_ip) \
dd01d0fb 59 ".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012
MD		 60 ".balign 32\n\t" \
61 __rseq_str(label) ":\n\t" \
62 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
63 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
dd01d0fb
MD		 64 ".popsection\n\t" \
65 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
66 ".quad " __rseq_str(label) "b\n\t" \
784b0012
MD		 67 ".popsection\n\t"
68
69#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
70 RSEQ_INJECT_ASM(1) \
71 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
72 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
73 "rldicr %%r17, %%r17, 32, 31\n\t" \
74 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
75 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
76 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
77 __rseq_str(label) ":\n\t"
78
90d9876e
MD		 79/*
80 * Exit points of a rseq critical section consist of all instructions outside
fd622cad
MD		 81 * of the critical section where a critical section can either branch to or
82 * reach through the normal course of its execution. The abort IP and the
83 * post-commit IP are already part of the __rseq_cs section and should not be
84 * explicitly defined as additional exit points. Knowing all exit points is
85 * useful to assist debuggers stepping over the critical section.
90d9876e
MD		 86 */
87#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
88 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
89 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
90 ".popsection\n\t"
91
784b0012
MD		 92#else /* #ifdef __PPC64__ */
93
a799a7ba
MD		 94#define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
95#define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */
96#define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
97#define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */
98#define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */
99#define RSEQ_CMP_LONG "cmpw "
784b0012
MD		 100
101#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
102 start_ip, post_commit_offset, abort_ip) \
dd01d0fb 103 ".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012
MD		 104 ".balign 32\n\t" \
105 __rseq_str(label) ":\n\t" \
106 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
107 /* 32-bit only supported on BE */ \
108 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
dd01d0fb
MD		 109 ".popsection\n\t" \
110 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
111 ".long 0x0, " __rseq_str(label) "b\n\t" \
784b0012
MD		 112 ".popsection\n\t"
113
90d9876e
MD		 114/*
115 * Exit points of a rseq critical section consist of all instructions outside
fd622cad
MD		 116 * of the critical section where a critical section can either branch to or
117 * reach through the normal course of its execution. The abort IP and the
118 * post-commit IP are already part of the __rseq_cs section and should not be
119 * explicitly defined as additional exit points. Knowing all exit points is
120 * useful to assist debuggers stepping over the critical section.
90d9876e
MD		 121 */
122#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
124 /* 32-bit only supported on BE */ \
125 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
126 ".popsection\n\t"
127
784b0012
MD		 128#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
129 RSEQ_INJECT_ASM(1) \
130 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
131 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
a799a7ba 132 RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
784b0012
MD		 133 __rseq_str(label) ":\n\t"
134
135#endif /* #ifdef __PPC64__ */
136
137#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
138 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
139 (post_commit_ip - start_ip), abort_ip)
140
141#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
142 RSEQ_INJECT_ASM(2) \
a799a7ba 143 RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
784b0012
MD		 144 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
145 "bne- cr7, " __rseq_str(label) "\n\t"
146
147#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
148 ".pushsection __rseq_failure, \"ax\"\n\t" \
149 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
150 __rseq_str(label) ":\n\t" \
151 "b %l[" __rseq_str(abort_label) "]\n\t" \
152 ".popsection\n\t"
153
154/*
155 * RSEQ_ASM_OPs: asm operations for rseq
156 * RSEQ_ASM_OP_R_*: has hard-code registers in it
157 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
158 */
159#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
a799a7ba
MD		 160 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
161 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
784b0012
MD		 162 "bne- cr7, " __rseq_str(label) "\n\t"
163
164#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
a799a7ba
MD		 165 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
166 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
784b0012
MD		 167 "beq- cr7, " __rseq_str(label) "\n\t"
168
169#define RSEQ_ASM_OP_STORE(value, var) \
a799a7ba 170 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
784b0012
MD		 171
172/* Load @var to r17 */
173#define RSEQ_ASM_OP_R_LOAD(var) \
a799a7ba 174 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
784b0012
MD		 175
176/* Store r17 to @var */
177#define RSEQ_ASM_OP_R_STORE(var) \
a799a7ba 178 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
784b0012
MD		 179
180/* Add @count to r17 */
181#define RSEQ_ASM_OP_R_ADD(count) \
182 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
183
184/* Load (r17 + voffp) to r17 */
185#define RSEQ_ASM_OP_R_LOADX(voffp) \
a799a7ba 186 RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
784b0012
MD		 187
188/* TODO: implement a faster memcpy. */
189#define RSEQ_ASM_OP_R_MEMCPY() \
190 "cmpdi %%r19, 0\n\t" \
191 "beq 333f\n\t" \
192 "addi %%r20, %%r20, -1\n\t" \
193 "addi %%r21, %%r21, -1\n\t" \
194 "222:\n\t" \
195 "lbzu %%r18, 1(%%r20)\n\t" \
196 "stbu %%r18, 1(%%r21)\n\t" \
197 "addi %%r19, %%r19, -1\n\t" \
198 "cmpdi %%r19, 0\n\t" \
199 "bne 222b\n\t" \
200 "333:\n\t" \
201
202#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
a799a7ba 203 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
784b0012
MD		 204 __rseq_str(post_commit_label) ":\n\t"
205
206#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
a799a7ba 207 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
784b0012
MD		 208 __rseq_str(post_commit_label) ":\n\t"
209
210static inline __attribute__((always_inline))
211int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
212{
213 RSEQ_INJECT_C(9)
214
215 __asm__ __volatile__ goto (
216 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
218#ifdef RSEQ_COMPARE_TWICE
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
220 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
221#endif
784b0012
MD		 222 /* Start rseq by storing table entry pointer into rseq_cs. */
223 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
224 /* cmp cpuid */
225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
226 RSEQ_INJECT_ASM(3)
227 /* cmp @v equal to @expect */
228 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
229 RSEQ_INJECT_ASM(4)
230#ifdef RSEQ_COMPARE_TWICE
231 /* cmp cpuid */
232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
233 /* cmp @v equal to @expect */
234 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
235#endif
236 /* final store */
237 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
238 RSEQ_INJECT_ASM(5)
239 RSEQ_ASM_DEFINE_ABORT(4, abort)
240 : /* gcc asm goto does not allow outputs */
241 : [cpu_id] "r" (cpu),
9698c399
MD		 242 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
243 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
784b0012
MD		 244 [v] "m" (*v),
245 [expect] "r" (expect),
246 [newv] "r" (newv)
247 RSEQ_INJECT_INPUT
248 : "memory", "cc", "r17"
249 RSEQ_INJECT_CLOBBER
250 : abort, cmpfail
251#ifdef RSEQ_COMPARE_TWICE
252 , error1, error2
253#endif
254 );
dd76f2d6 255 rseq_after_asm_goto();
784b0012
MD		 256 return 0;
257abort:
dd76f2d6 258 rseq_after_asm_goto();
784b0012
MD		 259 RSEQ_INJECT_FAILED
260 return -1;
261cmpfail:
dd76f2d6 262 rseq_after_asm_goto();
784b0012
MD		 263 return 1;
264#ifdef RSEQ_COMPARE_TWICE
265error1:
dd76f2d6 266 rseq_after_asm_goto();
784b0012
MD		 267 rseq_bug("cpu_id comparison failed");
268error2:
dd76f2d6 269 rseq_after_asm_goto();
784b0012
MD		 270 rseq_bug("expected value comparison failed");
271#endif
272}
273
274static inline __attribute__((always_inline))
275int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
276 off_t voffp, intptr_t *load, int cpu)
277{
278 RSEQ_INJECT_C(9)
279
280 __asm__ __volatile__ goto (
281 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 282 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
283#ifdef RSEQ_COMPARE_TWICE
284 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
285 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
286#endif
784b0012
MD		 287 /* Start rseq by storing table entry pointer into rseq_cs. */
288 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
289 /* cmp cpuid */
290 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
291 RSEQ_INJECT_ASM(3)
292 /* cmp @v not equal to @expectnot */
293 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
294 RSEQ_INJECT_ASM(4)
295#ifdef RSEQ_COMPARE_TWICE
296 /* cmp cpuid */
297 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
298 /* cmp @v not equal to @expectnot */
299 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
300#endif
301 /* load the value of @v */
302 RSEQ_ASM_OP_R_LOAD(v)
303 /* store it in @load */
304 RSEQ_ASM_OP_R_STORE(load)
305 /* dereference voffp(v) */
306 RSEQ_ASM_OP_R_LOADX(voffp)
307 /* final store the value at voffp(v) */
308 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
309 RSEQ_INJECT_ASM(5)
310 RSEQ_ASM_DEFINE_ABORT(4, abort)
311 : /* gcc asm goto does not allow outputs */
312 : [cpu_id] "r" (cpu),
9698c399
MD		 313 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
314 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
784b0012
MD		 315 /* final store input */
316 [v] "m" (*v),
317 [expectnot] "r" (expectnot),
318 [voffp] "b" (voffp),
319 [load] "m" (*load)
320 RSEQ_INJECT_INPUT
321 : "memory", "cc", "r17"
322 RSEQ_INJECT_CLOBBER
323 : abort, cmpfail
324#ifdef RSEQ_COMPARE_TWICE
325 , error1, error2
326#endif
327 );
dd76f2d6 328 rseq_after_asm_goto();
784b0012
MD		 329 return 0;
330abort:
dd76f2d6 331 rseq_after_asm_goto();
784b0012
MD		 332 RSEQ_INJECT_FAILED
333 return -1;
334cmpfail:
dd76f2d6 335 rseq_after_asm_goto();
784b0012
MD		 336 return 1;
337#ifdef RSEQ_COMPARE_TWICE
338error1:
dd76f2d6 339 rseq_after_asm_goto();
784b0012
MD		 340 rseq_bug("cpu_id comparison failed");
341error2:
dd76f2d6 342 rseq_after_asm_goto();
784b0012
MD		 343 rseq_bug("expected value comparison failed");
344#endif
345}
346
347static inline __attribute__((always_inline))
348int rseq_addv(intptr_t *v, intptr_t count, int cpu)
349{
350 RSEQ_INJECT_C(9)
351
352 __asm__ __volatile__ goto (
353 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 354#ifdef RSEQ_COMPARE_TWICE
355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
356#endif
784b0012
MD		 357 /* Start rseq by storing table entry pointer into rseq_cs. */
358 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
359 /* cmp cpuid */
360 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
361 RSEQ_INJECT_ASM(3)
362#ifdef RSEQ_COMPARE_TWICE
363 /* cmp cpuid */
364 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
365#endif
366 /* load the value of @v */
367 RSEQ_ASM_OP_R_LOAD(v)
368 /* add @count to it */
369 RSEQ_ASM_OP_R_ADD(count)
370 /* final store */
371 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
372 RSEQ_INJECT_ASM(4)
373 RSEQ_ASM_DEFINE_ABORT(4, abort)
374 : /* gcc asm goto does not allow outputs */
375 : [cpu_id] "r" (cpu),
9698c399
MD		 376 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
377 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
784b0012
MD		 378 /* final store input */
379 [v] "m" (*v),
380 [count] "r" (count)
381 RSEQ_INJECT_INPUT
382 : "memory", "cc", "r17"
383 RSEQ_INJECT_CLOBBER
384 : abort
385#ifdef RSEQ_COMPARE_TWICE
386 , error1
387#endif
388 );
dd76f2d6 389 rseq_after_asm_goto();
784b0012
MD		 390 return 0;
391abort:
dd76f2d6 392 rseq_after_asm_goto();
784b0012
MD		 393 RSEQ_INJECT_FAILED
394 return -1;
395#ifdef RSEQ_COMPARE_TWICE
396error1:
dd76f2d6 397 rseq_after_asm_goto();
784b0012
MD		 398 rseq_bug("cpu_id comparison failed");
399#endif
400}
401
402static inline __attribute__((always_inline))
403int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
404 intptr_t *v2, intptr_t newv2,
405 intptr_t newv, int cpu)
406{
407 RSEQ_INJECT_C(9)
408
409 __asm__ __volatile__ goto (
410 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 411 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
412#ifdef RSEQ_COMPARE_TWICE
413 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
414 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
415#endif
784b0012
MD		 416 /* Start rseq by storing table entry pointer into rseq_cs. */
417 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
418 /* cmp cpuid */
419 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
420 RSEQ_INJECT_ASM(3)
421 /* cmp @v equal to @expect */
422 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
423 RSEQ_INJECT_ASM(4)
424#ifdef RSEQ_COMPARE_TWICE
425 /* cmp cpuid */
426 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
427 /* cmp @v equal to @expect */
428 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
429#endif
430 /* try store */
431 RSEQ_ASM_OP_STORE(newv2, v2)
432 RSEQ_INJECT_ASM(5)
433 /* final store */
434 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
435 RSEQ_INJECT_ASM(6)
436 RSEQ_ASM_DEFINE_ABORT(4, abort)
437 : /* gcc asm goto does not allow outputs */
438 : [cpu_id] "r" (cpu),
9698c399
MD		 439 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
440 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
784b0012
MD		 441 /* try store input */
442 [v2] "m" (*v2),
443 [newv2] "r" (newv2),
444 /* final store input */
445 [v] "m" (*v),
446 [expect] "r" (expect),
447 [newv] "r" (newv)
448 RSEQ_INJECT_INPUT
449 : "memory", "cc", "r17"
450 RSEQ_INJECT_CLOBBER
451 : abort, cmpfail
452#ifdef RSEQ_COMPARE_TWICE
453 , error1, error2
454#endif
455 );
dd76f2d6 456 rseq_after_asm_goto();
784b0012
MD		 457 return 0;
458abort:
dd76f2d6 459 rseq_after_asm_goto();
784b0012
MD		 460 RSEQ_INJECT_FAILED
461 return -1;
462cmpfail:
dd76f2d6 463 rseq_after_asm_goto();
784b0012
MD		 464 return 1;
465#ifdef RSEQ_COMPARE_TWICE
466error1:
dd76f2d6 467 rseq_after_asm_goto();
784b0012
MD		 468 rseq_bug("cpu_id comparison failed");
469error2:
dd76f2d6 470 rseq_after_asm_goto();
784b0012
MD		 471 rseq_bug("expected value comparison failed");
472#endif
473}
474
475static inline __attribute__((always_inline))
476int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
477 intptr_t *v2, intptr_t newv2,
478 intptr_t newv, int cpu)
479{
480 RSEQ_INJECT_C(9)
481
482 __asm__ __volatile__ goto (
483 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 484 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
485#ifdef RSEQ_COMPARE_TWICE
486 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
487 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
488#endif
784b0012
MD		 489 /* Start rseq by storing table entry pointer into rseq_cs. */
490 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
491 /* cmp cpuid */
492 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
493 RSEQ_INJECT_ASM(3)
494 /* cmp @v equal to @expect */
495 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
496 RSEQ_INJECT_ASM(4)
497#ifdef RSEQ_COMPARE_TWICE
498 /* cmp cpuid */
499 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
500 /* cmp @v equal to @expect */
501 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
502#endif
503 /* try store */
504 RSEQ_ASM_OP_STORE(newv2, v2)
505 RSEQ_INJECT_ASM(5)
506 /* for 'release' */
507 "lwsync\n\t"
508 /* final store */
509 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
510 RSEQ_INJECT_ASM(6)
511 RSEQ_ASM_DEFINE_ABORT(4, abort)
512 : /* gcc asm goto does not allow outputs */
513 : [cpu_id] "r" (cpu),
9698c399
MD		 514 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
515 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
784b0012
MD		 516 /* try store input */
517 [v2] "m" (*v2),
518 [newv2] "r" (newv2),
519 /* final store input */
520 [v] "m" (*v),
521 [expect] "r" (expect),
522 [newv] "r" (newv)
523 RSEQ_INJECT_INPUT
524 : "memory", "cc", "r17"
525 RSEQ_INJECT_CLOBBER
526 : abort, cmpfail
527#ifdef RSEQ_COMPARE_TWICE
528 , error1, error2
529#endif
530 );
dd76f2d6 531 rseq_after_asm_goto();
784b0012
MD		 532 return 0;
533abort:
dd76f2d6 534 rseq_after_asm_goto();
784b0012
MD		 535 RSEQ_INJECT_FAILED
536 return -1;
537cmpfail:
dd76f2d6 538 rseq_after_asm_goto();
784b0012
MD		 539 return 1;
540#ifdef RSEQ_COMPARE_TWICE
541error1:
dd76f2d6 542 rseq_after_asm_goto();
784b0012
MD		 543 rseq_bug("cpu_id comparison failed");
544error2:
dd76f2d6 545 rseq_after_asm_goto();
784b0012
MD		 546 rseq_bug("expected value comparison failed");
547#endif
548}
549
550static inline __attribute__((always_inline))
551int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
552 intptr_t *v2, intptr_t expect2,
553 intptr_t newv, int cpu)
554{
555 RSEQ_INJECT_C(9)
556
557 __asm__ __volatile__ goto (
558 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 559 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
560#ifdef RSEQ_COMPARE_TWICE
561 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
562 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
563 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
564#endif
784b0012
MD		 565 /* Start rseq by storing table entry pointer into rseq_cs. */
566 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
567 /* cmp cpuid */
568 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
569 RSEQ_INJECT_ASM(3)
570 /* cmp @v equal to @expect */
571 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
572 RSEQ_INJECT_ASM(4)
573 /* cmp @v2 equal to @expct2 */
574 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
575 RSEQ_INJECT_ASM(5)
576#ifdef RSEQ_COMPARE_TWICE
577 /* cmp cpuid */
578 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
579 /* cmp @v equal to @expect */
580 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
581 /* cmp @v2 equal to @expct2 */
582 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
583#endif
584 /* final store */
585 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
586 RSEQ_INJECT_ASM(6)
587 RSEQ_ASM_DEFINE_ABORT(4, abort)
588 : /* gcc asm goto does not allow outputs */
589 : [cpu_id] "r" (cpu),
9698c399
MD		 590 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
591 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
784b0012
MD		 592 /* cmp2 input */
593 [v2] "m" (*v2),
594 [expect2] "r" (expect2),
595 /* final store input */
596 [v] "m" (*v),
597 [expect] "r" (expect),
598 [newv] "r" (newv)
599 RSEQ_INJECT_INPUT
600 : "memory", "cc", "r17"
601 RSEQ_INJECT_CLOBBER
602 : abort, cmpfail
603#ifdef RSEQ_COMPARE_TWICE
604 , error1, error2, error3
605#endif
606 );
dd76f2d6 607 rseq_after_asm_goto();
784b0012
MD		 608 return 0;
609abort:
dd76f2d6 610 rseq_after_asm_goto();
784b0012
MD		 611 RSEQ_INJECT_FAILED
612 return -1;
613cmpfail:
dd76f2d6 614 rseq_after_asm_goto();
784b0012
MD		 615 return 1;
616#ifdef RSEQ_COMPARE_TWICE
617error1:
dd76f2d6 618 rseq_after_asm_goto();
784b0012
MD		 619 rseq_bug("cpu_id comparison failed");
620error2:
dd76f2d6 621 rseq_after_asm_goto();
784b0012
MD		 622 rseq_bug("1st expected value comparison failed");
623error3:
dd76f2d6 624 rseq_after_asm_goto();
784b0012
MD		 625 rseq_bug("2nd expected value comparison failed");
626#endif
627}
628
629static inline __attribute__((always_inline))
630int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
631 void *dst, void *src, size_t len,
632 intptr_t newv, int cpu)
633{
634 RSEQ_INJECT_C(9)
635
636 __asm__ __volatile__ goto (
637 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 638 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
639#ifdef RSEQ_COMPARE_TWICE
640 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
641 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
642#endif
784b0012
MD		 643 /* setup for mempcy */
644 "mr %%r19, %[len]\n\t"
645 "mr %%r20, %[src]\n\t"
646 "mr %%r21, %[dst]\n\t"
647 /* Start rseq by storing table entry pointer into rseq_cs. */
648 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
649 /* cmp cpuid */
650 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
651 RSEQ_INJECT_ASM(3)
652 /* cmp @v equal to @expect */
653 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
654 RSEQ_INJECT_ASM(4)
655#ifdef RSEQ_COMPARE_TWICE
656 /* cmp cpuid */
657 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
658 /* cmp @v equal to @expect */
659 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
660#endif
661 /* try memcpy */
662 RSEQ_ASM_OP_R_MEMCPY()
663 RSEQ_INJECT_ASM(5)
664 /* final store */
665 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
666 RSEQ_INJECT_ASM(6)
667 /* teardown */
668 RSEQ_ASM_DEFINE_ABORT(4, abort)
669 : /* gcc asm goto does not allow outputs */
670 : [cpu_id] "r" (cpu),
9698c399
MD		 671 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
672 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
784b0012
MD		 673 /* final store input */
674 [v] "m" (*v),
675 [expect] "r" (expect),
676 [newv] "r" (newv),
677 /* try memcpy input */
678 [dst] "r" (dst),
679 [src] "r" (src),
680 [len] "r" (len)
681 RSEQ_INJECT_INPUT
682 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
683 RSEQ_INJECT_CLOBBER
684 : abort, cmpfail
685#ifdef RSEQ_COMPARE_TWICE
686 , error1, error2
687#endif
688 );
dd76f2d6 689 rseq_after_asm_goto();
784b0012
MD		 690 return 0;
691abort:
dd76f2d6 692 rseq_after_asm_goto();
784b0012
MD		 693 RSEQ_INJECT_FAILED
694 return -1;
695cmpfail:
dd76f2d6 696 rseq_after_asm_goto();
784b0012
MD		 697 return 1;
698#ifdef RSEQ_COMPARE_TWICE
699error1:
dd76f2d6 700 rseq_after_asm_goto();
784b0012
MD		 701 rseq_bug("cpu_id comparison failed");
702error2:
dd76f2d6 703 rseq_after_asm_goto();
784b0012
MD		 704 rseq_bug("expected value comparison failed");
705#endif
706}
707
708static inline __attribute__((always_inline))
709int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
710 void *dst, void *src, size_t len,
711 intptr_t newv, int cpu)
712{
713 RSEQ_INJECT_C(9)
714
715 __asm__ __volatile__ goto (
716 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 717 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
718#ifdef RSEQ_COMPARE_TWICE
719 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
720 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
721#endif
784b0012
MD		 722 /* setup for mempcy */
723 "mr %%r19, %[len]\n\t"
724 "mr %%r20, %[src]\n\t"
725 "mr %%r21, %[dst]\n\t"
726 /* Start rseq by storing table entry pointer into rseq_cs. */
727 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
728 /* cmp cpuid */
729 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
730 RSEQ_INJECT_ASM(3)
731 /* cmp @v equal to @expect */
732 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
733 RSEQ_INJECT_ASM(4)
734#ifdef RSEQ_COMPARE_TWICE
735 /* cmp cpuid */
736 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
737 /* cmp @v equal to @expect */
738 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
739#endif
740 /* try memcpy */
741 RSEQ_ASM_OP_R_MEMCPY()
742 RSEQ_INJECT_ASM(5)
743 /* for 'release' */
744 "lwsync\n\t"
745 /* final store */
746 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
747 RSEQ_INJECT_ASM(6)
748 /* teardown */
749 RSEQ_ASM_DEFINE_ABORT(4, abort)
750 : /* gcc asm goto does not allow outputs */
751 : [cpu_id] "r" (cpu),
9698c399
MD		 752 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
753 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
784b0012
MD		 754 /* final store input */
755 [v] "m" (*v),
756 [expect] "r" (expect),
757 [newv] "r" (newv),
758 /* try memcpy input */
759 [dst] "r" (dst),
760 [src] "r" (src),
761 [len] "r" (len)
762 RSEQ_INJECT_INPUT
763 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
764 RSEQ_INJECT_CLOBBER
765 : abort, cmpfail
766#ifdef RSEQ_COMPARE_TWICE
767 , error1, error2
768#endif
769 );
dd76f2d6 770 rseq_after_asm_goto();
784b0012
MD		 771 return 0;
772abort:
dd76f2d6 773 rseq_after_asm_goto();
784b0012
MD		 774 RSEQ_INJECT_FAILED
775 return -1;
776cmpfail:
dd76f2d6 777 rseq_after_asm_goto();
784b0012
MD		 778 return 1;
779#ifdef RSEQ_COMPARE_TWICE
780error1:
dd76f2d6 781 rseq_after_asm_goto();
784b0012
MD		 782 rseq_bug("cpu_id comparison failed");
783error2:
dd76f2d6 784 rseq_after_asm_goto();
784b0012
MD		 785 rseq_bug("expected value comparison failed");
786#endif
787}
788
a799a7ba
MD		 789#undef RSEQ_STORE_LONG
790#undef RSEQ_LOAD_LONG
791#undef RSEQ_LOADX_LONG
792#undef RSEQ_CMP_LONG
784b0012
MD		 793
794#endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.055945 seconds and 4 git commands to generate.