projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Fix: work-around asm goto compiler bugs
[librseq.git] / include / rseq / rseq-ppc.h
1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
2 /*
3 * rseq-ppc.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
9 /*
10 * RSEQ_SIG is used with the following trap instruction:
11 *
12 * powerpc-be: 0f e5 00 0b twui r5,11
13 * powerpc64-le: 0b 00 e5 0f twui r5,11
14 * powerpc64-be: 0f e5 00 0b twui r5,11
15 */
16
17 #define RSEQ_SIG 0x0fe5000b
18
19 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
20 #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
21 #define rseq_smp_rmb() rseq_smp_lwsync()
22 #define rseq_smp_wmb() rseq_smp_lwsync()
23
24 #define rseq_smp_load_acquire(p) \
25 __extension__ ({ \
26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
27 rseq_smp_lwsync(); \
28 ____p1; \
29 })
30
31 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
32
33 #define rseq_smp_store_release(p, v) \
34 do { \
35 rseq_smp_lwsync(); \
36 RSEQ_WRITE_ONCE(*p, v); \
37 } while (0)
38
39 #ifdef RSEQ_SKIP_FASTPATH
40 #include "rseq-skip.h"
41 #else /* !RSEQ_SKIP_FASTPATH */
42
43 /*
44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
45 * better handle single-stepping through the restartable critical sections.
46 */
47
48 #ifdef __PPC64__
49
50 #define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
51 #define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
52 #define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
53 #define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
54 #define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */
55 #define RSEQ_CMP_LONG "cmpd "
56
57 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
58 start_ip, post_commit_offset, abort_ip) \
59 ".pushsection __rseq_cs, \"aw\"\n\t" \
60 ".balign 32\n\t" \
61 __rseq_str(label) ":\n\t" \
62 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
63 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
64 ".popsection\n\t" \
65 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
66 ".quad " __rseq_str(label) "b\n\t" \
67 ".popsection\n\t"
68
69 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
70 RSEQ_INJECT_ASM(1) \
71 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
72 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
73 "rldicr %%r17, %%r17, 32, 31\n\t" \
74 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
75 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
76 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
77 __rseq_str(label) ":\n\t"
78
79 /*
80 * Exit points of a rseq critical section consist of all instructions outside
81 * of the critical section where a critical section can either branch to or
82 * reach through the normal course of its execution. The abort IP and the
83 * post-commit IP are already part of the __rseq_cs section and should not be
84 * explicitly defined as additional exit points. Knowing all exit points is
85 * useful to assist debuggers stepping over the critical section.
86 */
87 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
88 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
89 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
90 ".popsection\n\t"
91
92 #else /* #ifdef __PPC64__ */
93
94 #define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
95 #define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */
96 #define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
97 #define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */
98 #define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */
99 #define RSEQ_CMP_LONG "cmpw "
100
101 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
102 start_ip, post_commit_offset, abort_ip) \
103 ".pushsection __rseq_cs, \"aw\"\n\t" \
104 ".balign 32\n\t" \
105 __rseq_str(label) ":\n\t" \
106 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
107 /* 32-bit only supported on BE */ \
108 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
109 ".popsection\n\t" \
110 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
111 ".long 0x0, " __rseq_str(label) "b\n\t" \
112 ".popsection\n\t"
113
114 /*
115 * Exit points of a rseq critical section consist of all instructions outside
116 * of the critical section where a critical section can either branch to or
117 * reach through the normal course of its execution. The abort IP and the
118 * post-commit IP are already part of the __rseq_cs section and should not be
119 * explicitly defined as additional exit points. Knowing all exit points is
120 * useful to assist debuggers stepping over the critical section.
121 */
122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
124 /* 32-bit only supported on BE */ \
125 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
126 ".popsection\n\t"
127
128 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
129 RSEQ_INJECT_ASM(1) \
130 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
131 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
132 RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
133 __rseq_str(label) ":\n\t"
134
135 #endif /* #ifdef __PPC64__ */
136
137 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
138 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
139 (post_commit_ip - start_ip), abort_ip)
140
141 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
142 RSEQ_INJECT_ASM(2) \
143 RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
144 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
145 "bne- cr7, " __rseq_str(label) "\n\t"
146
147 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
148 ".pushsection __rseq_failure, \"ax\"\n\t" \
149 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
150 __rseq_str(label) ":\n\t" \
151 "b %l[" __rseq_str(abort_label) "]\n\t" \
152 ".popsection\n\t"
153
154 /*
155 * RSEQ_ASM_OPs: asm operations for rseq
156 * RSEQ_ASM_OP_R_*: has hard-code registers in it
157 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
158 */
159 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
160 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
161 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
162 "bne- cr7, " __rseq_str(label) "\n\t"
163
164 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
165 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
166 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
167 "beq- cr7, " __rseq_str(label) "\n\t"
168
169 #define RSEQ_ASM_OP_STORE(value, var) \
170 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
171
172 /* Load @var to r17 */
173 #define RSEQ_ASM_OP_R_LOAD(var) \
174 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
175
176 /* Store r17 to @var */
177 #define RSEQ_ASM_OP_R_STORE(var) \
178 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
179
180 /* Add @count to r17 */
181 #define RSEQ_ASM_OP_R_ADD(count) \
182 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
183
184 /* Load (r17 + voffp) to r17 */
185 #define RSEQ_ASM_OP_R_LOADX(voffp) \
186 RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
187
188 /* TODO: implement a faster memcpy. */
189 #define RSEQ_ASM_OP_R_MEMCPY() \
190 "cmpdi %%r19, 0\n\t" \
191 "beq 333f\n\t" \
192 "addi %%r20, %%r20, -1\n\t" \
193 "addi %%r21, %%r21, -1\n\t" \
194 "222:\n\t" \
195 "lbzu %%r18, 1(%%r20)\n\t" \
196 "stbu %%r18, 1(%%r21)\n\t" \
197 "addi %%r19, %%r19, -1\n\t" \
198 "cmpdi %%r19, 0\n\t" \
199 "bne 222b\n\t" \
200 "333:\n\t" \
201
202 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
203 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
204 __rseq_str(post_commit_label) ":\n\t"
205
206 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
207 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
208 __rseq_str(post_commit_label) ":\n\t"
209
210 static inline __attribute__((always_inline))
211 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
212 {
213 RSEQ_INJECT_C(9)
214
215 __asm__ __volatile__ goto (
216 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
218 #ifdef RSEQ_COMPARE_TWICE
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
220 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
221 #endif
222 /* Start rseq by storing table entry pointer into rseq_cs. */
223 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
224 /* cmp cpuid */
225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
226 RSEQ_INJECT_ASM(3)
227 /* cmp @v equal to @expect */
228 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
229 RSEQ_INJECT_ASM(4)
230 #ifdef RSEQ_COMPARE_TWICE
231 /* cmp cpuid */
232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
233 /* cmp @v equal to @expect */
234 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
235 #endif
236 /* final store */
237 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
238 RSEQ_INJECT_ASM(5)
239 RSEQ_ASM_DEFINE_ABORT(4, abort)
240 : /* gcc asm goto does not allow outputs */
241 : [cpu_id] "r" (cpu),
242 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
243 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
244 [v] "m" (*v),
245 [expect] "r" (expect),
246 [newv] "r" (newv)
247 RSEQ_INJECT_INPUT
248 : "memory", "cc", "r17"
249 RSEQ_INJECT_CLOBBER
250 : abort, cmpfail
251 #ifdef RSEQ_COMPARE_TWICE
252 , error1, error2
253 #endif
254 );
255 rseq_after_asm_goto();
256 return 0;
257 abort:
258 rseq_after_asm_goto();
259 RSEQ_INJECT_FAILED
260 return -1;
261 cmpfail:
262 rseq_after_asm_goto();
263 return 1;
264 #ifdef RSEQ_COMPARE_TWICE
265 error1:
266 rseq_after_asm_goto();
267 rseq_bug("cpu_id comparison failed");
268 error2:
269 rseq_after_asm_goto();
270 rseq_bug("expected value comparison failed");
271 #endif
272 }
273
274 static inline __attribute__((always_inline))
275 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
276 off_t voffp, intptr_t *load, int cpu)
277 {
278 RSEQ_INJECT_C(9)
279
280 __asm__ __volatile__ goto (
281 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
282 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
283 #ifdef RSEQ_COMPARE_TWICE
284 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
285 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
286 #endif
287 /* Start rseq by storing table entry pointer into rseq_cs. */
288 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
289 /* cmp cpuid */
290 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
291 RSEQ_INJECT_ASM(3)
292 /* cmp @v not equal to @expectnot */
293 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
294 RSEQ_INJECT_ASM(4)
295 #ifdef RSEQ_COMPARE_TWICE
296 /* cmp cpuid */
297 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
298 /* cmp @v not equal to @expectnot */
299 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
300 #endif
301 /* load the value of @v */
302 RSEQ_ASM_OP_R_LOAD(v)
303 /* store it in @load */
304 RSEQ_ASM_OP_R_STORE(load)
305 /* dereference voffp(v) */
306 RSEQ_ASM_OP_R_LOADX(voffp)
307 /* final store the value at voffp(v) */
308 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
309 RSEQ_INJECT_ASM(5)
310 RSEQ_ASM_DEFINE_ABORT(4, abort)
311 : /* gcc asm goto does not allow outputs */
312 : [cpu_id] "r" (cpu),
313 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
314 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
315 /* final store input */
316 [v] "m" (*v),
317 [expectnot] "r" (expectnot),
318 [voffp] "b" (voffp),
319 [load] "m" (*load)
320 RSEQ_INJECT_INPUT
321 : "memory", "cc", "r17"
322 RSEQ_INJECT_CLOBBER
323 : abort, cmpfail
324 #ifdef RSEQ_COMPARE_TWICE
325 , error1, error2
326 #endif
327 );
328 rseq_after_asm_goto();
329 return 0;
330 abort:
331 rseq_after_asm_goto();
332 RSEQ_INJECT_FAILED
333 return -1;
334 cmpfail:
335 rseq_after_asm_goto();
336 return 1;
337 #ifdef RSEQ_COMPARE_TWICE
338 error1:
339 rseq_after_asm_goto();
340 rseq_bug("cpu_id comparison failed");
341 error2:
342 rseq_after_asm_goto();
343 rseq_bug("expected value comparison failed");
344 #endif
345 }
346
347 static inline __attribute__((always_inline))
348 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
349 {
350 RSEQ_INJECT_C(9)
351
352 __asm__ __volatile__ goto (
353 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
354 #ifdef RSEQ_COMPARE_TWICE
355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
356 #endif
357 /* Start rseq by storing table entry pointer into rseq_cs. */
358 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
359 /* cmp cpuid */
360 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
361 RSEQ_INJECT_ASM(3)
362 #ifdef RSEQ_COMPARE_TWICE
363 /* cmp cpuid */
364 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
365 #endif
366 /* load the value of @v */
367 RSEQ_ASM_OP_R_LOAD(v)
368 /* add @count to it */
369 RSEQ_ASM_OP_R_ADD(count)
370 /* final store */
371 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
372 RSEQ_INJECT_ASM(4)
373 RSEQ_ASM_DEFINE_ABORT(4, abort)
374 : /* gcc asm goto does not allow outputs */
375 : [cpu_id] "r" (cpu),
376 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
377 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
378 /* final store input */
379 [v] "m" (*v),
380 [count] "r" (count)
381 RSEQ_INJECT_INPUT
382 : "memory", "cc", "r17"
383 RSEQ_INJECT_CLOBBER
384 : abort
385 #ifdef RSEQ_COMPARE_TWICE
386 , error1
387 #endif
388 );
389 rseq_after_asm_goto();
390 return 0;
391 abort:
392 rseq_after_asm_goto();
393 RSEQ_INJECT_FAILED
394 return -1;
395 #ifdef RSEQ_COMPARE_TWICE
396 error1:
397 rseq_after_asm_goto();
398 rseq_bug("cpu_id comparison failed");
399 #endif
400 }
401
402 static inline __attribute__((always_inline))
403 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
404 intptr_t *v2, intptr_t newv2,
405 intptr_t newv, int cpu)
406 {
407 RSEQ_INJECT_C(9)
408
409 __asm__ __volatile__ goto (
410 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
411 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
412 #ifdef RSEQ_COMPARE_TWICE
413 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
414 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
415 #endif
416 /* Start rseq by storing table entry pointer into rseq_cs. */
417 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
418 /* cmp cpuid */
419 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
420 RSEQ_INJECT_ASM(3)
421 /* cmp @v equal to @expect */
422 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
423 RSEQ_INJECT_ASM(4)
424 #ifdef RSEQ_COMPARE_TWICE
425 /* cmp cpuid */
426 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
427 /* cmp @v equal to @expect */
428 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
429 #endif
430 /* try store */
431 RSEQ_ASM_OP_STORE(newv2, v2)
432 RSEQ_INJECT_ASM(5)
433 /* final store */
434 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
435 RSEQ_INJECT_ASM(6)
436 RSEQ_ASM_DEFINE_ABORT(4, abort)
437 : /* gcc asm goto does not allow outputs */
438 : [cpu_id] "r" (cpu),
439 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
440 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
441 /* try store input */
442 [v2] "m" (*v2),
443 [newv2] "r" (newv2),
444 /* final store input */
445 [v] "m" (*v),
446 [expect] "r" (expect),
447 [newv] "r" (newv)
448 RSEQ_INJECT_INPUT
449 : "memory", "cc", "r17"
450 RSEQ_INJECT_CLOBBER
451 : abort, cmpfail
452 #ifdef RSEQ_COMPARE_TWICE
453 , error1, error2
454 #endif
455 );
456 rseq_after_asm_goto();
457 return 0;
458 abort:
459 rseq_after_asm_goto();
460 RSEQ_INJECT_FAILED
461 return -1;
462 cmpfail:
463 rseq_after_asm_goto();
464 return 1;
465 #ifdef RSEQ_COMPARE_TWICE
466 error1:
467 rseq_after_asm_goto();
468 rseq_bug("cpu_id comparison failed");
469 error2:
470 rseq_after_asm_goto();
471 rseq_bug("expected value comparison failed");
472 #endif
473 }
474
475 static inline __attribute__((always_inline))
476 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
477 intptr_t *v2, intptr_t newv2,
478 intptr_t newv, int cpu)
479 {
480 RSEQ_INJECT_C(9)
481
482 __asm__ __volatile__ goto (
483 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
484 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
485 #ifdef RSEQ_COMPARE_TWICE
486 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
487 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
488 #endif
489 /* Start rseq by storing table entry pointer into rseq_cs. */
490 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
491 /* cmp cpuid */
492 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
493 RSEQ_INJECT_ASM(3)
494 /* cmp @v equal to @expect */
495 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
496 RSEQ_INJECT_ASM(4)
497 #ifdef RSEQ_COMPARE_TWICE
498 /* cmp cpuid */
499 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
500 /* cmp @v equal to @expect */
501 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
502 #endif
503 /* try store */
504 RSEQ_ASM_OP_STORE(newv2, v2)
505 RSEQ_INJECT_ASM(5)
506 /* for 'release' */
507 "lwsync\n\t"
508 /* final store */
509 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
510 RSEQ_INJECT_ASM(6)
511 RSEQ_ASM_DEFINE_ABORT(4, abort)
512 : /* gcc asm goto does not allow outputs */
513 : [cpu_id] "r" (cpu),
514 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
515 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
516 /* try store input */
517 [v2] "m" (*v2),
518 [newv2] "r" (newv2),
519 /* final store input */
520 [v] "m" (*v),
521 [expect] "r" (expect),
522 [newv] "r" (newv)
523 RSEQ_INJECT_INPUT
524 : "memory", "cc", "r17"
525 RSEQ_INJECT_CLOBBER
526 : abort, cmpfail
527 #ifdef RSEQ_COMPARE_TWICE
528 , error1, error2
529 #endif
530 );
531 rseq_after_asm_goto();
532 return 0;
533 abort:
534 rseq_after_asm_goto();
535 RSEQ_INJECT_FAILED
536 return -1;
537 cmpfail:
538 rseq_after_asm_goto();
539 return 1;
540 #ifdef RSEQ_COMPARE_TWICE
541 error1:
542 rseq_after_asm_goto();
543 rseq_bug("cpu_id comparison failed");
544 error2:
545 rseq_after_asm_goto();
546 rseq_bug("expected value comparison failed");
547 #endif
548 }
549
550 static inline __attribute__((always_inline))
551 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
552 intptr_t *v2, intptr_t expect2,
553 intptr_t newv, int cpu)
554 {
555 RSEQ_INJECT_C(9)
556
557 __asm__ __volatile__ goto (
558 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
559 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
560 #ifdef RSEQ_COMPARE_TWICE
561 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
562 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
563 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
564 #endif
565 /* Start rseq by storing table entry pointer into rseq_cs. */
566 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
567 /* cmp cpuid */
568 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
569 RSEQ_INJECT_ASM(3)
570 /* cmp @v equal to @expect */
571 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
572 RSEQ_INJECT_ASM(4)
573 /* cmp @v2 equal to @expct2 */
574 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
575 RSEQ_INJECT_ASM(5)
576 #ifdef RSEQ_COMPARE_TWICE
577 /* cmp cpuid */
578 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
579 /* cmp @v equal to @expect */
580 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
581 /* cmp @v2 equal to @expct2 */
582 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
583 #endif
584 /* final store */
585 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
586 RSEQ_INJECT_ASM(6)
587 RSEQ_ASM_DEFINE_ABORT(4, abort)
588 : /* gcc asm goto does not allow outputs */
589 : [cpu_id] "r" (cpu),
590 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
591 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
592 /* cmp2 input */
593 [v2] "m" (*v2),
594 [expect2] "r" (expect2),
595 /* final store input */
596 [v] "m" (*v),
597 [expect] "r" (expect),
598 [newv] "r" (newv)
599 RSEQ_INJECT_INPUT
600 : "memory", "cc", "r17"
601 RSEQ_INJECT_CLOBBER
602 : abort, cmpfail
603 #ifdef RSEQ_COMPARE_TWICE
604 , error1, error2, error3
605 #endif
606 );
607 rseq_after_asm_goto();
608 return 0;
609 abort:
610 rseq_after_asm_goto();
611 RSEQ_INJECT_FAILED
612 return -1;
613 cmpfail:
614 rseq_after_asm_goto();
615 return 1;
616 #ifdef RSEQ_COMPARE_TWICE
617 error1:
618 rseq_after_asm_goto();
619 rseq_bug("cpu_id comparison failed");
620 error2:
621 rseq_after_asm_goto();
622 rseq_bug("1st expected value comparison failed");
623 error3:
624 rseq_after_asm_goto();
625 rseq_bug("2nd expected value comparison failed");
626 #endif
627 }
628
629 static inline __attribute__((always_inline))
630 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
631 void *dst, void *src, size_t len,
632 intptr_t newv, int cpu)
633 {
634 RSEQ_INJECT_C(9)
635
636 __asm__ __volatile__ goto (
637 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
638 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
639 #ifdef RSEQ_COMPARE_TWICE
640 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
641 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
642 #endif
643 /* setup for mempcy */
644 "mr %%r19, %[len]\n\t"
645 "mr %%r20, %[src]\n\t"
646 "mr %%r21, %[dst]\n\t"
647 /* Start rseq by storing table entry pointer into rseq_cs. */
648 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
649 /* cmp cpuid */
650 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
651 RSEQ_INJECT_ASM(3)
652 /* cmp @v equal to @expect */
653 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
654 RSEQ_INJECT_ASM(4)
655 #ifdef RSEQ_COMPARE_TWICE
656 /* cmp cpuid */
657 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
658 /* cmp @v equal to @expect */
659 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
660 #endif
661 /* try memcpy */
662 RSEQ_ASM_OP_R_MEMCPY()
663 RSEQ_INJECT_ASM(5)
664 /* final store */
665 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
666 RSEQ_INJECT_ASM(6)
667 /* teardown */
668 RSEQ_ASM_DEFINE_ABORT(4, abort)
669 : /* gcc asm goto does not allow outputs */
670 : [cpu_id] "r" (cpu),
671 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
672 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
673 /* final store input */
674 [v] "m" (*v),
675 [expect] "r" (expect),
676 [newv] "r" (newv),
677 /* try memcpy input */
678 [dst] "r" (dst),
679 [src] "r" (src),
680 [len] "r" (len)
681 RSEQ_INJECT_INPUT
682 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
683 RSEQ_INJECT_CLOBBER
684 : abort, cmpfail
685 #ifdef RSEQ_COMPARE_TWICE
686 , error1, error2
687 #endif
688 );
689 rseq_after_asm_goto();
690 return 0;
691 abort:
692 rseq_after_asm_goto();
693 RSEQ_INJECT_FAILED
694 return -1;
695 cmpfail:
696 rseq_after_asm_goto();
697 return 1;
698 #ifdef RSEQ_COMPARE_TWICE
699 error1:
700 rseq_after_asm_goto();
701 rseq_bug("cpu_id comparison failed");
702 error2:
703 rseq_after_asm_goto();
704 rseq_bug("expected value comparison failed");
705 #endif
706 }
707
708 static inline __attribute__((always_inline))
709 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
710 void *dst, void *src, size_t len,
711 intptr_t newv, int cpu)
712 {
713 RSEQ_INJECT_C(9)
714
715 __asm__ __volatile__ goto (
716 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
717 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
718 #ifdef RSEQ_COMPARE_TWICE
719 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
720 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
721 #endif
722 /* setup for mempcy */
723 "mr %%r19, %[len]\n\t"
724 "mr %%r20, %[src]\n\t"
725 "mr %%r21, %[dst]\n\t"
726 /* Start rseq by storing table entry pointer into rseq_cs. */
727 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
728 /* cmp cpuid */
729 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
730 RSEQ_INJECT_ASM(3)
731 /* cmp @v equal to @expect */
732 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
733 RSEQ_INJECT_ASM(4)
734 #ifdef RSEQ_COMPARE_TWICE
735 /* cmp cpuid */
736 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
737 /* cmp @v equal to @expect */
738 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
739 #endif
740 /* try memcpy */
741 RSEQ_ASM_OP_R_MEMCPY()
742 RSEQ_INJECT_ASM(5)
743 /* for 'release' */
744 "lwsync\n\t"
745 /* final store */
746 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
747 RSEQ_INJECT_ASM(6)
748 /* teardown */
749 RSEQ_ASM_DEFINE_ABORT(4, abort)
750 : /* gcc asm goto does not allow outputs */
751 : [cpu_id] "r" (cpu),
752 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
753 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
754 /* final store input */
755 [v] "m" (*v),
756 [expect] "r" (expect),
757 [newv] "r" (newv),
758 /* try memcpy input */
759 [dst] "r" (dst),
760 [src] "r" (src),
761 [len] "r" (len)
762 RSEQ_INJECT_INPUT
763 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
764 RSEQ_INJECT_CLOBBER
765 : abort, cmpfail
766 #ifdef RSEQ_COMPARE_TWICE
767 , error1, error2
768 #endif
769 );
770 rseq_after_asm_goto();
771 return 0;
772 abort:
773 rseq_after_asm_goto();
774 RSEQ_INJECT_FAILED
775 return -1;
776 cmpfail:
777 rseq_after_asm_goto();
778 return 1;
779 #ifdef RSEQ_COMPARE_TWICE
780 error1:
781 rseq_after_asm_goto();
782 rseq_bug("cpu_id comparison failed");
783 error2:
784 rseq_after_asm_goto();
785 rseq_bug("expected value comparison failed");
786 #endif
787 }
788
789 #undef RSEQ_STORE_LONG
790 #undef RSEQ_LOAD_LONG
791 #undef RSEQ_LOADX_LONG
792 #undef RSEQ_CMP_LONG
793
794 #endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.046058 seconds and 5 git commands to generate.