projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Remove rseq_deref_loadoffp placeholders
[librseq.git] / include / rseq / rseq-ppc.h
1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
2 /*
3 * rseq-ppc.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
9 /*
10 * RSEQ_SIG is used with the following trap instruction:
11 *
12 * powerpc-be: 0f e5 00 0b twui r5,11
13 * powerpc64-le: 0b 00 e5 0f twui r5,11
14 * powerpc64-be: 0f e5 00 0b twui r5,11
15 */
16
17 #define RSEQ_SIG 0x0fe5000b
18
19 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
20 #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
21 #define rseq_smp_rmb() rseq_smp_lwsync()
22 #define rseq_smp_wmb() rseq_smp_lwsync()
23
24 #define rseq_smp_load_acquire(p) \
25 __extension__ ({ \
26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
27 rseq_smp_lwsync(); \
28 ____p1; \
29 })
30
31 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
32
33 #define rseq_smp_store_release(p, v) \
34 do { \
35 rseq_smp_lwsync(); \
36 RSEQ_WRITE_ONCE(*p, v); \
37 } while (0)
38
39 #ifdef RSEQ_SKIP_FASTPATH
40 #include "rseq-skip.h"
41 #else /* !RSEQ_SKIP_FASTPATH */
42
43 /*
44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
45 * better handle single-stepping through the restartable critical sections.
46 */
47
48 #ifdef __PPC64__
49
50 #define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
51 #define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
52 #define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
53 #define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
54 #define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */
55 #define RSEQ_CMP_LONG "cmpd "
56
57 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
58 start_ip, post_commit_offset, abort_ip) \
59 ".pushsection __rseq_cs, \"aw\"\n\t" \
60 ".balign 32\n\t" \
61 __rseq_str(label) ":\n\t" \
62 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
63 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
64 ".popsection\n\t" \
65 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
66 ".quad " __rseq_str(label) "b\n\t" \
67 ".popsection\n\t"
68
69 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
70 RSEQ_INJECT_ASM(1) \
71 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
72 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
73 "rldicr %%r17, %%r17, 32, 31\n\t" \
74 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
75 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
76 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
77 __rseq_str(label) ":\n\t"
78
79 /*
80 * Exit points of a rseq critical section consist of all instructions outside
81 * of the critical section where a critical section can either branch to or
82 * reach through the normal course of its execution. The abort IP and the
83 * post-commit IP are already part of the __rseq_cs section and should not be
84 * explicitly defined as additional exit points. Knowing all exit points is
85 * useful to assist debuggers stepping over the critical section.
86 */
87 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
88 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
89 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
90 ".popsection\n\t"
91
92 #else /* #ifdef __PPC64__ */
93
94 #define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
95 #define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */
96 #define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
97 #define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */
98 #define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */
99 #define RSEQ_CMP_LONG "cmpw "
100
101 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
102 start_ip, post_commit_offset, abort_ip) \
103 ".pushsection __rseq_cs, \"aw\"\n\t" \
104 ".balign 32\n\t" \
105 __rseq_str(label) ":\n\t" \
106 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
107 /* 32-bit only supported on BE */ \
108 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
109 ".popsection\n\t" \
110 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
111 ".long 0x0, " __rseq_str(label) "b\n\t" \
112 ".popsection\n\t"
113
114 /*
115 * Exit points of a rseq critical section consist of all instructions outside
116 * of the critical section where a critical section can either branch to or
117 * reach through the normal course of its execution. The abort IP and the
118 * post-commit IP are already part of the __rseq_cs section and should not be
119 * explicitly defined as additional exit points. Knowing all exit points is
120 * useful to assist debuggers stepping over the critical section.
121 */
122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
124 /* 32-bit only supported on BE */ \
125 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
126 ".popsection\n\t"
127
128 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
129 RSEQ_INJECT_ASM(1) \
130 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
131 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
132 RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
133 __rseq_str(label) ":\n\t"
134
135 #endif /* #ifdef __PPC64__ */
136
137 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
138 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
139 (post_commit_ip - start_ip), abort_ip)
140
141 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
142 RSEQ_INJECT_ASM(2) \
143 RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
144 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
145 "bne- cr7, " __rseq_str(label) "\n\t"
146
147 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
148 ".pushsection __rseq_failure, \"ax\"\n\t" \
149 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
150 __rseq_str(label) ":\n\t" \
151 "b %l[" __rseq_str(abort_label) "]\n\t" \
152 ".popsection\n\t"
153
154 /*
155 * RSEQ_ASM_OPs: asm operations for rseq
156 * RSEQ_ASM_OP_R_*: has hard-code registers in it
157 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
158 */
159 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
160 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
161 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
162 "bne- cr7, " __rseq_str(label) "\n\t"
163
164 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
165 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
166 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
167 "beq- cr7, " __rseq_str(label) "\n\t"
168
169 #define RSEQ_ASM_OP_STORE(value, var) \
170 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
171
172 /* Load @var to r17 */
173 #define RSEQ_ASM_OP_R_LOAD(var) \
174 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
175
176 /* Store r17 to @var */
177 #define RSEQ_ASM_OP_R_STORE(var) \
178 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
179
180 /* Add @count to r17 */
181 #define RSEQ_ASM_OP_R_ADD(count) \
182 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
183
184 /* Load (r17 + voffp) to r17 */
185 #define RSEQ_ASM_OP_R_LOADX(voffp) \
186 RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
187
188 /* TODO: implement a faster memcpy. */
189 #define RSEQ_ASM_OP_R_MEMCPY() \
190 "cmpdi %%r19, 0\n\t" \
191 "beq 333f\n\t" \
192 "addi %%r20, %%r20, -1\n\t" \
193 "addi %%r21, %%r21, -1\n\t" \
194 "222:\n\t" \
195 "lbzu %%r18, 1(%%r20)\n\t" \
196 "stbu %%r18, 1(%%r21)\n\t" \
197 "addi %%r19, %%r19, -1\n\t" \
198 "cmpdi %%r19, 0\n\t" \
199 "bne 222b\n\t" \
200 "333:\n\t" \
201
202 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
203 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
204 __rseq_str(post_commit_label) ":\n\t"
205
206 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
207 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
208 __rseq_str(post_commit_label) ":\n\t"
209
210 static inline __attribute__((always_inline))
211 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
212 {
213 RSEQ_INJECT_C(9)
214
215 __asm__ __volatile__ goto (
216 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
218 #ifdef RSEQ_COMPARE_TWICE
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
220 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
221 #endif
222 /* Start rseq by storing table entry pointer into rseq_cs. */
223 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
224 /* cmp cpuid */
225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
226 RSEQ_INJECT_ASM(3)
227 /* cmp @v equal to @expect */
228 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
229 RSEQ_INJECT_ASM(4)
230 #ifdef RSEQ_COMPARE_TWICE
231 /* cmp cpuid */
232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
233 /* cmp @v equal to @expect */
234 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
235 #endif
236 /* final store */
237 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
238 RSEQ_INJECT_ASM(5)
239 RSEQ_ASM_DEFINE_ABORT(4, abort)
240 : /* gcc asm goto does not allow outputs */
241 : [cpu_id] "r" (cpu),
242 [current_cpu_id] "m" (__rseq_abi.cpu_id),
243 [rseq_cs] "m" (__rseq_abi.rseq_cs),
244 [v] "m" (*v),
245 [expect] "r" (expect),
246 [newv] "r" (newv)
247 RSEQ_INJECT_INPUT
248 : "memory", "cc", "r17"
249 RSEQ_INJECT_CLOBBER
250 : abort, cmpfail
251 #ifdef RSEQ_COMPARE_TWICE
252 , error1, error2
253 #endif
254 );
255 return 0;
256 abort:
257 RSEQ_INJECT_FAILED
258 return -1;
259 cmpfail:
260 return 1;
261 #ifdef RSEQ_COMPARE_TWICE
262 error1:
263 rseq_bug("cpu_id comparison failed");
264 error2:
265 rseq_bug("expected value comparison failed");
266 #endif
267 }
268
269 static inline __attribute__((always_inline))
270 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
271 off_t voffp, intptr_t *load, int cpu)
272 {
273 RSEQ_INJECT_C(9)
274
275 __asm__ __volatile__ goto (
276 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
277 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
278 #ifdef RSEQ_COMPARE_TWICE
279 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
280 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
281 #endif
282 /* Start rseq by storing table entry pointer into rseq_cs. */
283 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
284 /* cmp cpuid */
285 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
286 RSEQ_INJECT_ASM(3)
287 /* cmp @v not equal to @expectnot */
288 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
289 RSEQ_INJECT_ASM(4)
290 #ifdef RSEQ_COMPARE_TWICE
291 /* cmp cpuid */
292 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
293 /* cmp @v not equal to @expectnot */
294 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
295 #endif
296 /* load the value of @v */
297 RSEQ_ASM_OP_R_LOAD(v)
298 /* store it in @load */
299 RSEQ_ASM_OP_R_STORE(load)
300 /* dereference voffp(v) */
301 RSEQ_ASM_OP_R_LOADX(voffp)
302 /* final store the value at voffp(v) */
303 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
304 RSEQ_INJECT_ASM(5)
305 RSEQ_ASM_DEFINE_ABORT(4, abort)
306 : /* gcc asm goto does not allow outputs */
307 : [cpu_id] "r" (cpu),
308 [current_cpu_id] "m" (__rseq_abi.cpu_id),
309 [rseq_cs] "m" (__rseq_abi.rseq_cs),
310 /* final store input */
311 [v] "m" (*v),
312 [expectnot] "r" (expectnot),
313 [voffp] "b" (voffp),
314 [load] "m" (*load)
315 RSEQ_INJECT_INPUT
316 : "memory", "cc", "r17"
317 RSEQ_INJECT_CLOBBER
318 : abort, cmpfail
319 #ifdef RSEQ_COMPARE_TWICE
320 , error1, error2
321 #endif
322 );
323 return 0;
324 abort:
325 RSEQ_INJECT_FAILED
326 return -1;
327 cmpfail:
328 return 1;
329 #ifdef RSEQ_COMPARE_TWICE
330 error1:
331 rseq_bug("cpu_id comparison failed");
332 error2:
333 rseq_bug("expected value comparison failed");
334 #endif
335 }
336
337 static inline __attribute__((always_inline))
338 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
339 {
340 RSEQ_INJECT_C(9)
341
342 __asm__ __volatile__ goto (
343 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
344 #ifdef RSEQ_COMPARE_TWICE
345 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
346 #endif
347 /* Start rseq by storing table entry pointer into rseq_cs. */
348 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
349 /* cmp cpuid */
350 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
351 RSEQ_INJECT_ASM(3)
352 #ifdef RSEQ_COMPARE_TWICE
353 /* cmp cpuid */
354 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
355 #endif
356 /* load the value of @v */
357 RSEQ_ASM_OP_R_LOAD(v)
358 /* add @count to it */
359 RSEQ_ASM_OP_R_ADD(count)
360 /* final store */
361 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
362 RSEQ_INJECT_ASM(4)
363 RSEQ_ASM_DEFINE_ABORT(4, abort)
364 : /* gcc asm goto does not allow outputs */
365 : [cpu_id] "r" (cpu),
366 [current_cpu_id] "m" (__rseq_abi.cpu_id),
367 [rseq_cs] "m" (__rseq_abi.rseq_cs),
368 /* final store input */
369 [v] "m" (*v),
370 [count] "r" (count)
371 RSEQ_INJECT_INPUT
372 : "memory", "cc", "r17"
373 RSEQ_INJECT_CLOBBER
374 : abort
375 #ifdef RSEQ_COMPARE_TWICE
376 , error1
377 #endif
378 );
379 return 0;
380 abort:
381 RSEQ_INJECT_FAILED
382 return -1;
383 #ifdef RSEQ_COMPARE_TWICE
384 error1:
385 rseq_bug("cpu_id comparison failed");
386 #endif
387 }
388
389 static inline __attribute__((always_inline))
390 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
391 intptr_t *v2, intptr_t newv2,
392 intptr_t newv, int cpu)
393 {
394 RSEQ_INJECT_C(9)
395
396 __asm__ __volatile__ goto (
397 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
398 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
399 #ifdef RSEQ_COMPARE_TWICE
400 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
401 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
402 #endif
403 /* Start rseq by storing table entry pointer into rseq_cs. */
404 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
405 /* cmp cpuid */
406 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
407 RSEQ_INJECT_ASM(3)
408 /* cmp @v equal to @expect */
409 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
410 RSEQ_INJECT_ASM(4)
411 #ifdef RSEQ_COMPARE_TWICE
412 /* cmp cpuid */
413 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
414 /* cmp @v equal to @expect */
415 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
416 #endif
417 /* try store */
418 RSEQ_ASM_OP_STORE(newv2, v2)
419 RSEQ_INJECT_ASM(5)
420 /* final store */
421 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
422 RSEQ_INJECT_ASM(6)
423 RSEQ_ASM_DEFINE_ABORT(4, abort)
424 : /* gcc asm goto does not allow outputs */
425 : [cpu_id] "r" (cpu),
426 [current_cpu_id] "m" (__rseq_abi.cpu_id),
427 [rseq_cs] "m" (__rseq_abi.rseq_cs),
428 /* try store input */
429 [v2] "m" (*v2),
430 [newv2] "r" (newv2),
431 /* final store input */
432 [v] "m" (*v),
433 [expect] "r" (expect),
434 [newv] "r" (newv)
435 RSEQ_INJECT_INPUT
436 : "memory", "cc", "r17"
437 RSEQ_INJECT_CLOBBER
438 : abort, cmpfail
439 #ifdef RSEQ_COMPARE_TWICE
440 , error1, error2
441 #endif
442 );
443 return 0;
444 abort:
445 RSEQ_INJECT_FAILED
446 return -1;
447 cmpfail:
448 return 1;
449 #ifdef RSEQ_COMPARE_TWICE
450 error1:
451 rseq_bug("cpu_id comparison failed");
452 error2:
453 rseq_bug("expected value comparison failed");
454 #endif
455 }
456
457 static inline __attribute__((always_inline))
458 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
459 intptr_t *v2, intptr_t newv2,
460 intptr_t newv, int cpu)
461 {
462 RSEQ_INJECT_C(9)
463
464 __asm__ __volatile__ goto (
465 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
466 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
467 #ifdef RSEQ_COMPARE_TWICE
468 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
469 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
470 #endif
471 /* Start rseq by storing table entry pointer into rseq_cs. */
472 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
473 /* cmp cpuid */
474 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
475 RSEQ_INJECT_ASM(3)
476 /* cmp @v equal to @expect */
477 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
478 RSEQ_INJECT_ASM(4)
479 #ifdef RSEQ_COMPARE_TWICE
480 /* cmp cpuid */
481 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
482 /* cmp @v equal to @expect */
483 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
484 #endif
485 /* try store */
486 RSEQ_ASM_OP_STORE(newv2, v2)
487 RSEQ_INJECT_ASM(5)
488 /* for 'release' */
489 "lwsync\n\t"
490 /* final store */
491 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
492 RSEQ_INJECT_ASM(6)
493 RSEQ_ASM_DEFINE_ABORT(4, abort)
494 : /* gcc asm goto does not allow outputs */
495 : [cpu_id] "r" (cpu),
496 [current_cpu_id] "m" (__rseq_abi.cpu_id),
497 [rseq_cs] "m" (__rseq_abi.rseq_cs),
498 /* try store input */
499 [v2] "m" (*v2),
500 [newv2] "r" (newv2),
501 /* final store input */
502 [v] "m" (*v),
503 [expect] "r" (expect),
504 [newv] "r" (newv)
505 RSEQ_INJECT_INPUT
506 : "memory", "cc", "r17"
507 RSEQ_INJECT_CLOBBER
508 : abort, cmpfail
509 #ifdef RSEQ_COMPARE_TWICE
510 , error1, error2
511 #endif
512 );
513 return 0;
514 abort:
515 RSEQ_INJECT_FAILED
516 return -1;
517 cmpfail:
518 return 1;
519 #ifdef RSEQ_COMPARE_TWICE
520 error1:
521 rseq_bug("cpu_id comparison failed");
522 error2:
523 rseq_bug("expected value comparison failed");
524 #endif
525 }
526
527 static inline __attribute__((always_inline))
528 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
529 intptr_t *v2, intptr_t expect2,
530 intptr_t newv, int cpu)
531 {
532 RSEQ_INJECT_C(9)
533
534 __asm__ __volatile__ goto (
535 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
537 #ifdef RSEQ_COMPARE_TWICE
538 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
539 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
540 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
541 #endif
542 /* Start rseq by storing table entry pointer into rseq_cs. */
543 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
544 /* cmp cpuid */
545 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
546 RSEQ_INJECT_ASM(3)
547 /* cmp @v equal to @expect */
548 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
549 RSEQ_INJECT_ASM(4)
550 /* cmp @v2 equal to @expct2 */
551 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
552 RSEQ_INJECT_ASM(5)
553 #ifdef RSEQ_COMPARE_TWICE
554 /* cmp cpuid */
555 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
556 /* cmp @v equal to @expect */
557 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
558 /* cmp @v2 equal to @expct2 */
559 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
560 #endif
561 /* final store */
562 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
563 RSEQ_INJECT_ASM(6)
564 RSEQ_ASM_DEFINE_ABORT(4, abort)
565 : /* gcc asm goto does not allow outputs */
566 : [cpu_id] "r" (cpu),
567 [current_cpu_id] "m" (__rseq_abi.cpu_id),
568 [rseq_cs] "m" (__rseq_abi.rseq_cs),
569 /* cmp2 input */
570 [v2] "m" (*v2),
571 [expect2] "r" (expect2),
572 /* final store input */
573 [v] "m" (*v),
574 [expect] "r" (expect),
575 [newv] "r" (newv)
576 RSEQ_INJECT_INPUT
577 : "memory", "cc", "r17"
578 RSEQ_INJECT_CLOBBER
579 : abort, cmpfail
580 #ifdef RSEQ_COMPARE_TWICE
581 , error1, error2, error3
582 #endif
583 );
584 return 0;
585 abort:
586 RSEQ_INJECT_FAILED
587 return -1;
588 cmpfail:
589 return 1;
590 #ifdef RSEQ_COMPARE_TWICE
591 error1:
592 rseq_bug("cpu_id comparison failed");
593 error2:
594 rseq_bug("1st expected value comparison failed");
595 error3:
596 rseq_bug("2nd expected value comparison failed");
597 #endif
598 }
599
600 static inline __attribute__((always_inline))
601 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
602 void *dst, void *src, size_t len,
603 intptr_t newv, int cpu)
604 {
605 RSEQ_INJECT_C(9)
606
607 __asm__ __volatile__ goto (
608 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
609 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
610 #ifdef RSEQ_COMPARE_TWICE
611 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
612 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
613 #endif
614 /* setup for mempcy */
615 "mr %%r19, %[len]\n\t"
616 "mr %%r20, %[src]\n\t"
617 "mr %%r21, %[dst]\n\t"
618 /* Start rseq by storing table entry pointer into rseq_cs. */
619 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
620 /* cmp cpuid */
621 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
622 RSEQ_INJECT_ASM(3)
623 /* cmp @v equal to @expect */
624 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
625 RSEQ_INJECT_ASM(4)
626 #ifdef RSEQ_COMPARE_TWICE
627 /* cmp cpuid */
628 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
629 /* cmp @v equal to @expect */
630 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
631 #endif
632 /* try memcpy */
633 RSEQ_ASM_OP_R_MEMCPY()
634 RSEQ_INJECT_ASM(5)
635 /* final store */
636 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
637 RSEQ_INJECT_ASM(6)
638 /* teardown */
639 RSEQ_ASM_DEFINE_ABORT(4, abort)
640 : /* gcc asm goto does not allow outputs */
641 : [cpu_id] "r" (cpu),
642 [current_cpu_id] "m" (__rseq_abi.cpu_id),
643 [rseq_cs] "m" (__rseq_abi.rseq_cs),
644 /* final store input */
645 [v] "m" (*v),
646 [expect] "r" (expect),
647 [newv] "r" (newv),
648 /* try memcpy input */
649 [dst] "r" (dst),
650 [src] "r" (src),
651 [len] "r" (len)
652 RSEQ_INJECT_INPUT
653 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
654 RSEQ_INJECT_CLOBBER
655 : abort, cmpfail
656 #ifdef RSEQ_COMPARE_TWICE
657 , error1, error2
658 #endif
659 );
660 return 0;
661 abort:
662 RSEQ_INJECT_FAILED
663 return -1;
664 cmpfail:
665 return 1;
666 #ifdef RSEQ_COMPARE_TWICE
667 error1:
668 rseq_bug("cpu_id comparison failed");
669 error2:
670 rseq_bug("expected value comparison failed");
671 #endif
672 }
673
674 static inline __attribute__((always_inline))
675 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
676 void *dst, void *src, size_t len,
677 intptr_t newv, int cpu)
678 {
679 RSEQ_INJECT_C(9)
680
681 __asm__ __volatile__ goto (
682 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
683 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
684 #ifdef RSEQ_COMPARE_TWICE
685 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
686 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
687 #endif
688 /* setup for mempcy */
689 "mr %%r19, %[len]\n\t"
690 "mr %%r20, %[src]\n\t"
691 "mr %%r21, %[dst]\n\t"
692 /* Start rseq by storing table entry pointer into rseq_cs. */
693 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
694 /* cmp cpuid */
695 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
696 RSEQ_INJECT_ASM(3)
697 /* cmp @v equal to @expect */
698 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
699 RSEQ_INJECT_ASM(4)
700 #ifdef RSEQ_COMPARE_TWICE
701 /* cmp cpuid */
702 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
703 /* cmp @v equal to @expect */
704 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
705 #endif
706 /* try memcpy */
707 RSEQ_ASM_OP_R_MEMCPY()
708 RSEQ_INJECT_ASM(5)
709 /* for 'release' */
710 "lwsync\n\t"
711 /* final store */
712 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
713 RSEQ_INJECT_ASM(6)
714 /* teardown */
715 RSEQ_ASM_DEFINE_ABORT(4, abort)
716 : /* gcc asm goto does not allow outputs */
717 : [cpu_id] "r" (cpu),
718 [current_cpu_id] "m" (__rseq_abi.cpu_id),
719 [rseq_cs] "m" (__rseq_abi.rseq_cs),
720 /* final store input */
721 [v] "m" (*v),
722 [expect] "r" (expect),
723 [newv] "r" (newv),
724 /* try memcpy input */
725 [dst] "r" (dst),
726 [src] "r" (src),
727 [len] "r" (len)
728 RSEQ_INJECT_INPUT
729 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
730 RSEQ_INJECT_CLOBBER
731 : abort, cmpfail
732 #ifdef RSEQ_COMPARE_TWICE
733 , error1, error2
734 #endif
735 );
736 return 0;
737 abort:
738 RSEQ_INJECT_FAILED
739 return -1;
740 cmpfail:
741 return 1;
742 #ifdef RSEQ_COMPARE_TWICE
743 error1:
744 rseq_bug("cpu_id comparison failed");
745 error2:
746 rseq_bug("expected value comparison failed");
747 #endif
748 }
749
750 #undef RSEQ_STORE_LONG
751 #undef RSEQ_LOAD_LONG
752 #undef RSEQ_LOADX_LONG
753 #undef RSEQ_CMP_LONG
754
755 #endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.045107 seconds and 5 git commands to generate.