projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Fix: ppc32: do not produce cmpdi invalid instruction
[librseq.git] / include / rseq / rseq-ppc.h
1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
2 /*
3 * rseq-ppc.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
9 /*
10 * RSEQ_SIG is used with the following trap instruction:
11 *
12 * powerpc-be: 0f e5 00 0b twui r5,11
13 * powerpc64-le: 0b 00 e5 0f twui r5,11
14 * powerpc64-be: 0f e5 00 0b twui r5,11
15 */
16
17 #define RSEQ_SIG 0x0fe5000b
18
19 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
20 #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
21 #define rseq_smp_rmb() rseq_smp_lwsync()
22 #define rseq_smp_wmb() rseq_smp_lwsync()
23
24 #define rseq_smp_load_acquire(p) \
25 __extension__ ({ \
26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
27 rseq_smp_lwsync(); \
28 ____p1; \
29 })
30
31 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
32
33 #define rseq_smp_store_release(p, v) \
34 do { \
35 rseq_smp_lwsync(); \
36 RSEQ_WRITE_ONCE(*p, v); \
37 } while (0)
38
39 #ifdef RSEQ_SKIP_FASTPATH
40 #include "rseq-skip.h"
41 #else /* !RSEQ_SKIP_FASTPATH */
42
43 /*
44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
45 * better handle single-stepping through the restartable critical sections.
46 */
47
48 #ifdef __PPC64__
49
50 #define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
51 #define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
52 #define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
53 #define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
54 #define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */
55 #define RSEQ_CMP_LONG "cmpd "
56 #define RSEQ_CMP_LONG_INT "cmpdi "
57
58 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
59 start_ip, post_commit_offset, abort_ip) \
60 ".pushsection __rseq_cs, \"aw\"\n\t" \
61 ".balign 32\n\t" \
62 __rseq_str(label) ":\n\t" \
63 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
64 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
65 ".popsection\n\t" \
66 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
67 ".quad " __rseq_str(label) "b\n\t" \
68 ".popsection\n\t"
69
70 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
71 RSEQ_INJECT_ASM(1) \
72 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
73 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
74 "rldicr %%r17, %%r17, 32, 31\n\t" \
75 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
76 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
77 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
78 __rseq_str(label) ":\n\t"
79
80 /*
81 * Exit points of a rseq critical section consist of all instructions outside
82 * of the critical section where a critical section can either branch to or
83 * reach through the normal course of its execution. The abort IP and the
84 * post-commit IP are already part of the __rseq_cs section and should not be
85 * explicitly defined as additional exit points. Knowing all exit points is
86 * useful to assist debuggers stepping over the critical section.
87 */
88 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
89 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
90 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
91 ".popsection\n\t"
92
93 #else /* #ifdef __PPC64__ */
94
95 #define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
96 #define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */
97 #define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
98 #define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */
99 #define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */
100 #define RSEQ_CMP_LONG "cmpw "
101 #define RSEQ_CMP_LONG_INT "cmpwi "
102
103 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
104 start_ip, post_commit_offset, abort_ip) \
105 ".pushsection __rseq_cs, \"aw\"\n\t" \
106 ".balign 32\n\t" \
107 __rseq_str(label) ":\n\t" \
108 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
109 /* 32-bit only supported on BE */ \
110 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
111 ".popsection\n\t" \
112 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
113 ".long 0x0, " __rseq_str(label) "b\n\t" \
114 ".popsection\n\t"
115
116 /*
117 * Exit points of a rseq critical section consist of all instructions outside
118 * of the critical section where a critical section can either branch to or
119 * reach through the normal course of its execution. The abort IP and the
120 * post-commit IP are already part of the __rseq_cs section and should not be
121 * explicitly defined as additional exit points. Knowing all exit points is
122 * useful to assist debuggers stepping over the critical section.
123 */
124 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
125 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
126 /* 32-bit only supported on BE */ \
127 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
128 ".popsection\n\t"
129
130 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
131 RSEQ_INJECT_ASM(1) \
132 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
133 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
134 RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
135 __rseq_str(label) ":\n\t"
136
137 #endif /* #ifdef __PPC64__ */
138
139 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
140 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
141 (post_commit_ip - start_ip), abort_ip)
142
143 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
144 RSEQ_INJECT_ASM(2) \
145 RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
146 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
147 "bne- cr7, " __rseq_str(label) "\n\t"
148
149 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
150 ".pushsection __rseq_failure, \"ax\"\n\t" \
151 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
152 __rseq_str(label) ":\n\t" \
153 "b %l[" __rseq_str(abort_label) "]\n\t" \
154 ".popsection\n\t"
155
156 /*
157 * RSEQ_ASM_OPs: asm operations for rseq
158 * RSEQ_ASM_OP_R_*: has hard-code registers in it
159 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
160 */
161 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
162 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
163 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
164 "bne- cr7, " __rseq_str(label) "\n\t"
165
166 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
167 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
168 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
169 "beq- cr7, " __rseq_str(label) "\n\t"
170
171 #define RSEQ_ASM_OP_STORE(value, var) \
172 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
173
174 /* Load @var to r17 */
175 #define RSEQ_ASM_OP_R_LOAD(var) \
176 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
177
178 /* Store r17 to @var */
179 #define RSEQ_ASM_OP_R_STORE(var) \
180 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
181
182 /* Add @count to r17 */
183 #define RSEQ_ASM_OP_R_ADD(count) \
184 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
185
186 /* Load (r17 + voffp) to r17 */
187 #define RSEQ_ASM_OP_R_LOADX(voffp) \
188 RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
189
190 /* TODO: implement a faster memcpy. */
191 #define RSEQ_ASM_OP_R_MEMCPY() \
192 RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
193 "beq 333f\n\t" \
194 "addi %%r20, %%r20, -1\n\t" \
195 "addi %%r21, %%r21, -1\n\t" \
196 "222:\n\t" \
197 "lbzu %%r18, 1(%%r20)\n\t" \
198 "stbu %%r18, 1(%%r21)\n\t" \
199 "addi %%r19, %%r19, -1\n\t" \
200 RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
201 "bne 222b\n\t" \
202 "333:\n\t" \
203
204 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
205 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
206 __rseq_str(post_commit_label) ":\n\t"
207
208 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
209 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
210 __rseq_str(post_commit_label) ":\n\t"
211
212 static inline __attribute__((always_inline))
213 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
214 {
215 RSEQ_INJECT_C(9)
216
217 __asm__ __volatile__ goto (
218 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
220 #ifdef RSEQ_COMPARE_TWICE
221 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
222 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
223 #endif
224 /* Start rseq by storing table entry pointer into rseq_cs. */
225 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
226 /* cmp cpuid */
227 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
228 RSEQ_INJECT_ASM(3)
229 /* cmp @v equal to @expect */
230 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
231 RSEQ_INJECT_ASM(4)
232 #ifdef RSEQ_COMPARE_TWICE
233 /* cmp cpuid */
234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
235 /* cmp @v equal to @expect */
236 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
237 #endif
238 /* final store */
239 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
240 RSEQ_INJECT_ASM(5)
241 RSEQ_ASM_DEFINE_ABORT(4, abort)
242 : /* gcc asm goto does not allow outputs */
243 : [cpu_id] "r" (cpu),
244 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
245 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
246 [v] "m" (*v),
247 [expect] "r" (expect),
248 [newv] "r" (newv)
249 RSEQ_INJECT_INPUT
250 : "memory", "cc", "r17"
251 RSEQ_INJECT_CLOBBER
252 : abort, cmpfail
253 #ifdef RSEQ_COMPARE_TWICE
254 , error1, error2
255 #endif
256 );
257 rseq_after_asm_goto();
258 return 0;
259 abort:
260 rseq_after_asm_goto();
261 RSEQ_INJECT_FAILED
262 return -1;
263 cmpfail:
264 rseq_after_asm_goto();
265 return 1;
266 #ifdef RSEQ_COMPARE_TWICE
267 error1:
268 rseq_after_asm_goto();
269 rseq_bug("cpu_id comparison failed");
270 error2:
271 rseq_after_asm_goto();
272 rseq_bug("expected value comparison failed");
273 #endif
274 }
275
276 static inline __attribute__((always_inline))
277 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
278 off_t voffp, intptr_t *load, int cpu)
279 {
280 RSEQ_INJECT_C(9)
281
282 __asm__ __volatile__ goto (
283 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
284 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
285 #ifdef RSEQ_COMPARE_TWICE
286 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
287 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
288 #endif
289 /* Start rseq by storing table entry pointer into rseq_cs. */
290 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
291 /* cmp cpuid */
292 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
293 RSEQ_INJECT_ASM(3)
294 /* cmp @v not equal to @expectnot */
295 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
296 RSEQ_INJECT_ASM(4)
297 #ifdef RSEQ_COMPARE_TWICE
298 /* cmp cpuid */
299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
300 /* cmp @v not equal to @expectnot */
301 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
302 #endif
303 /* load the value of @v */
304 RSEQ_ASM_OP_R_LOAD(v)
305 /* store it in @load */
306 RSEQ_ASM_OP_R_STORE(load)
307 /* dereference voffp(v) */
308 RSEQ_ASM_OP_R_LOADX(voffp)
309 /* final store the value at voffp(v) */
310 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
311 RSEQ_INJECT_ASM(5)
312 RSEQ_ASM_DEFINE_ABORT(4, abort)
313 : /* gcc asm goto does not allow outputs */
314 : [cpu_id] "r" (cpu),
315 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
316 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
317 /* final store input */
318 [v] "m" (*v),
319 [expectnot] "r" (expectnot),
320 [voffp] "b" (voffp),
321 [load] "m" (*load)
322 RSEQ_INJECT_INPUT
323 : "memory", "cc", "r17"
324 RSEQ_INJECT_CLOBBER
325 : abort, cmpfail
326 #ifdef RSEQ_COMPARE_TWICE
327 , error1, error2
328 #endif
329 );
330 rseq_after_asm_goto();
331 return 0;
332 abort:
333 rseq_after_asm_goto();
334 RSEQ_INJECT_FAILED
335 return -1;
336 cmpfail:
337 rseq_after_asm_goto();
338 return 1;
339 #ifdef RSEQ_COMPARE_TWICE
340 error1:
341 rseq_after_asm_goto();
342 rseq_bug("cpu_id comparison failed");
343 error2:
344 rseq_after_asm_goto();
345 rseq_bug("expected value comparison failed");
346 #endif
347 }
348
349 static inline __attribute__((always_inline))
350 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
351 {
352 RSEQ_INJECT_C(9)
353
354 __asm__ __volatile__ goto (
355 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
356 #ifdef RSEQ_COMPARE_TWICE
357 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
358 #endif
359 /* Start rseq by storing table entry pointer into rseq_cs. */
360 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
361 /* cmp cpuid */
362 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
363 RSEQ_INJECT_ASM(3)
364 #ifdef RSEQ_COMPARE_TWICE
365 /* cmp cpuid */
366 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
367 #endif
368 /* load the value of @v */
369 RSEQ_ASM_OP_R_LOAD(v)
370 /* add @count to it */
371 RSEQ_ASM_OP_R_ADD(count)
372 /* final store */
373 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
374 RSEQ_INJECT_ASM(4)
375 RSEQ_ASM_DEFINE_ABORT(4, abort)
376 : /* gcc asm goto does not allow outputs */
377 : [cpu_id] "r" (cpu),
378 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
379 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
380 /* final store input */
381 [v] "m" (*v),
382 [count] "r" (count)
383 RSEQ_INJECT_INPUT
384 : "memory", "cc", "r17"
385 RSEQ_INJECT_CLOBBER
386 : abort
387 #ifdef RSEQ_COMPARE_TWICE
388 , error1
389 #endif
390 );
391 rseq_after_asm_goto();
392 return 0;
393 abort:
394 rseq_after_asm_goto();
395 RSEQ_INJECT_FAILED
396 return -1;
397 #ifdef RSEQ_COMPARE_TWICE
398 error1:
399 rseq_after_asm_goto();
400 rseq_bug("cpu_id comparison failed");
401 #endif
402 }
403
404 static inline __attribute__((always_inline))
405 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
406 intptr_t *v2, intptr_t newv2,
407 intptr_t newv, int cpu)
408 {
409 RSEQ_INJECT_C(9)
410
411 __asm__ __volatile__ goto (
412 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
413 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
414 #ifdef RSEQ_COMPARE_TWICE
415 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
416 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
417 #endif
418 /* Start rseq by storing table entry pointer into rseq_cs. */
419 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
420 /* cmp cpuid */
421 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
422 RSEQ_INJECT_ASM(3)
423 /* cmp @v equal to @expect */
424 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
425 RSEQ_INJECT_ASM(4)
426 #ifdef RSEQ_COMPARE_TWICE
427 /* cmp cpuid */
428 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
429 /* cmp @v equal to @expect */
430 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
431 #endif
432 /* try store */
433 RSEQ_ASM_OP_STORE(newv2, v2)
434 RSEQ_INJECT_ASM(5)
435 /* final store */
436 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
437 RSEQ_INJECT_ASM(6)
438 RSEQ_ASM_DEFINE_ABORT(4, abort)
439 : /* gcc asm goto does not allow outputs */
440 : [cpu_id] "r" (cpu),
441 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
442 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
443 /* try store input */
444 [v2] "m" (*v2),
445 [newv2] "r" (newv2),
446 /* final store input */
447 [v] "m" (*v),
448 [expect] "r" (expect),
449 [newv] "r" (newv)
450 RSEQ_INJECT_INPUT
451 : "memory", "cc", "r17"
452 RSEQ_INJECT_CLOBBER
453 : abort, cmpfail
454 #ifdef RSEQ_COMPARE_TWICE
455 , error1, error2
456 #endif
457 );
458 rseq_after_asm_goto();
459 return 0;
460 abort:
461 rseq_after_asm_goto();
462 RSEQ_INJECT_FAILED
463 return -1;
464 cmpfail:
465 rseq_after_asm_goto();
466 return 1;
467 #ifdef RSEQ_COMPARE_TWICE
468 error1:
469 rseq_after_asm_goto();
470 rseq_bug("cpu_id comparison failed");
471 error2:
472 rseq_after_asm_goto();
473 rseq_bug("expected value comparison failed");
474 #endif
475 }
476
477 static inline __attribute__((always_inline))
478 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
479 intptr_t *v2, intptr_t newv2,
480 intptr_t newv, int cpu)
481 {
482 RSEQ_INJECT_C(9)
483
484 __asm__ __volatile__ goto (
485 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
486 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
487 #ifdef RSEQ_COMPARE_TWICE
488 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
489 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
490 #endif
491 /* Start rseq by storing table entry pointer into rseq_cs. */
492 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
493 /* cmp cpuid */
494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495 RSEQ_INJECT_ASM(3)
496 /* cmp @v equal to @expect */
497 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
498 RSEQ_INJECT_ASM(4)
499 #ifdef RSEQ_COMPARE_TWICE
500 /* cmp cpuid */
501 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
502 /* cmp @v equal to @expect */
503 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
504 #endif
505 /* try store */
506 RSEQ_ASM_OP_STORE(newv2, v2)
507 RSEQ_INJECT_ASM(5)
508 /* for 'release' */
509 "lwsync\n\t"
510 /* final store */
511 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
512 RSEQ_INJECT_ASM(6)
513 RSEQ_ASM_DEFINE_ABORT(4, abort)
514 : /* gcc asm goto does not allow outputs */
515 : [cpu_id] "r" (cpu),
516 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
517 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
518 /* try store input */
519 [v2] "m" (*v2),
520 [newv2] "r" (newv2),
521 /* final store input */
522 [v] "m" (*v),
523 [expect] "r" (expect),
524 [newv] "r" (newv)
525 RSEQ_INJECT_INPUT
526 : "memory", "cc", "r17"
527 RSEQ_INJECT_CLOBBER
528 : abort, cmpfail
529 #ifdef RSEQ_COMPARE_TWICE
530 , error1, error2
531 #endif
532 );
533 rseq_after_asm_goto();
534 return 0;
535 abort:
536 rseq_after_asm_goto();
537 RSEQ_INJECT_FAILED
538 return -1;
539 cmpfail:
540 rseq_after_asm_goto();
541 return 1;
542 #ifdef RSEQ_COMPARE_TWICE
543 error1:
544 rseq_after_asm_goto();
545 rseq_bug("cpu_id comparison failed");
546 error2:
547 rseq_after_asm_goto();
548 rseq_bug("expected value comparison failed");
549 #endif
550 }
551
552 static inline __attribute__((always_inline))
553 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
554 intptr_t *v2, intptr_t expect2,
555 intptr_t newv, int cpu)
556 {
557 RSEQ_INJECT_C(9)
558
559 __asm__ __volatile__ goto (
560 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
561 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
562 #ifdef RSEQ_COMPARE_TWICE
563 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
564 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
565 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
566 #endif
567 /* Start rseq by storing table entry pointer into rseq_cs. */
568 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
569 /* cmp cpuid */
570 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
571 RSEQ_INJECT_ASM(3)
572 /* cmp @v equal to @expect */
573 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
574 RSEQ_INJECT_ASM(4)
575 /* cmp @v2 equal to @expct2 */
576 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
577 RSEQ_INJECT_ASM(5)
578 #ifdef RSEQ_COMPARE_TWICE
579 /* cmp cpuid */
580 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
581 /* cmp @v equal to @expect */
582 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
583 /* cmp @v2 equal to @expct2 */
584 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
585 #endif
586 /* final store */
587 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
588 RSEQ_INJECT_ASM(6)
589 RSEQ_ASM_DEFINE_ABORT(4, abort)
590 : /* gcc asm goto does not allow outputs */
591 : [cpu_id] "r" (cpu),
592 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
593 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
594 /* cmp2 input */
595 [v2] "m" (*v2),
596 [expect2] "r" (expect2),
597 /* final store input */
598 [v] "m" (*v),
599 [expect] "r" (expect),
600 [newv] "r" (newv)
601 RSEQ_INJECT_INPUT
602 : "memory", "cc", "r17"
603 RSEQ_INJECT_CLOBBER
604 : abort, cmpfail
605 #ifdef RSEQ_COMPARE_TWICE
606 , error1, error2, error3
607 #endif
608 );
609 rseq_after_asm_goto();
610 return 0;
611 abort:
612 rseq_after_asm_goto();
613 RSEQ_INJECT_FAILED
614 return -1;
615 cmpfail:
616 rseq_after_asm_goto();
617 return 1;
618 #ifdef RSEQ_COMPARE_TWICE
619 error1:
620 rseq_after_asm_goto();
621 rseq_bug("cpu_id comparison failed");
622 error2:
623 rseq_after_asm_goto();
624 rseq_bug("1st expected value comparison failed");
625 error3:
626 rseq_after_asm_goto();
627 rseq_bug("2nd expected value comparison failed");
628 #endif
629 }
630
631 static inline __attribute__((always_inline))
632 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
633 void *dst, void *src, size_t len,
634 intptr_t newv, int cpu)
635 {
636 RSEQ_INJECT_C(9)
637
638 __asm__ __volatile__ goto (
639 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
640 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
641 #ifdef RSEQ_COMPARE_TWICE
642 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
643 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
644 #endif
645 /* setup for mempcy */
646 "mr %%r19, %[len]\n\t"
647 "mr %%r20, %[src]\n\t"
648 "mr %%r21, %[dst]\n\t"
649 /* Start rseq by storing table entry pointer into rseq_cs. */
650 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
651 /* cmp cpuid */
652 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
653 RSEQ_INJECT_ASM(3)
654 /* cmp @v equal to @expect */
655 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
656 RSEQ_INJECT_ASM(4)
657 #ifdef RSEQ_COMPARE_TWICE
658 /* cmp cpuid */
659 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
660 /* cmp @v equal to @expect */
661 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
662 #endif
663 /* try memcpy */
664 RSEQ_ASM_OP_R_MEMCPY()
665 RSEQ_INJECT_ASM(5)
666 /* final store */
667 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
668 RSEQ_INJECT_ASM(6)
669 /* teardown */
670 RSEQ_ASM_DEFINE_ABORT(4, abort)
671 : /* gcc asm goto does not allow outputs */
672 : [cpu_id] "r" (cpu),
673 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
674 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
675 /* final store input */
676 [v] "m" (*v),
677 [expect] "r" (expect),
678 [newv] "r" (newv),
679 /* try memcpy input */
680 [dst] "r" (dst),
681 [src] "r" (src),
682 [len] "r" (len)
683 RSEQ_INJECT_INPUT
684 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
685 RSEQ_INJECT_CLOBBER
686 : abort, cmpfail
687 #ifdef RSEQ_COMPARE_TWICE
688 , error1, error2
689 #endif
690 );
691 rseq_after_asm_goto();
692 return 0;
693 abort:
694 rseq_after_asm_goto();
695 RSEQ_INJECT_FAILED
696 return -1;
697 cmpfail:
698 rseq_after_asm_goto();
699 return 1;
700 #ifdef RSEQ_COMPARE_TWICE
701 error1:
702 rseq_after_asm_goto();
703 rseq_bug("cpu_id comparison failed");
704 error2:
705 rseq_after_asm_goto();
706 rseq_bug("expected value comparison failed");
707 #endif
708 }
709
710 static inline __attribute__((always_inline))
711 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
712 void *dst, void *src, size_t len,
713 intptr_t newv, int cpu)
714 {
715 RSEQ_INJECT_C(9)
716
717 __asm__ __volatile__ goto (
718 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
719 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
720 #ifdef RSEQ_COMPARE_TWICE
721 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
722 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
723 #endif
724 /* setup for mempcy */
725 "mr %%r19, %[len]\n\t"
726 "mr %%r20, %[src]\n\t"
727 "mr %%r21, %[dst]\n\t"
728 /* Start rseq by storing table entry pointer into rseq_cs. */
729 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
730 /* cmp cpuid */
731 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
732 RSEQ_INJECT_ASM(3)
733 /* cmp @v equal to @expect */
734 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
735 RSEQ_INJECT_ASM(4)
736 #ifdef RSEQ_COMPARE_TWICE
737 /* cmp cpuid */
738 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
739 /* cmp @v equal to @expect */
740 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
741 #endif
742 /* try memcpy */
743 RSEQ_ASM_OP_R_MEMCPY()
744 RSEQ_INJECT_ASM(5)
745 /* for 'release' */
746 "lwsync\n\t"
747 /* final store */
748 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
749 RSEQ_INJECT_ASM(6)
750 /* teardown */
751 RSEQ_ASM_DEFINE_ABORT(4, abort)
752 : /* gcc asm goto does not allow outputs */
753 : [cpu_id] "r" (cpu),
754 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
755 [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
756 /* final store input */
757 [v] "m" (*v),
758 [expect] "r" (expect),
759 [newv] "r" (newv),
760 /* try memcpy input */
761 [dst] "r" (dst),
762 [src] "r" (src),
763 [len] "r" (len)
764 RSEQ_INJECT_INPUT
765 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
766 RSEQ_INJECT_CLOBBER
767 : abort, cmpfail
768 #ifdef RSEQ_COMPARE_TWICE
769 , error1, error2
770 #endif
771 );
772 rseq_after_asm_goto();
773 return 0;
774 abort:
775 rseq_after_asm_goto();
776 RSEQ_INJECT_FAILED
777 return -1;
778 cmpfail:
779 rseq_after_asm_goto();
780 return 1;
781 #ifdef RSEQ_COMPARE_TWICE
782 error1:
783 rseq_after_asm_goto();
784 rseq_bug("cpu_id comparison failed");
785 error2:
786 rseq_after_asm_goto();
787 rseq_bug("expected value comparison failed");
788 #endif
789 }
790
791 #undef RSEQ_STORE_LONG
792 #undef RSEQ_LOAD_LONG
793 #undef RSEQ_LOADX_LONG
794 #undef RSEQ_CMP_LONG
795
796 #endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.04613 seconds and 5 git commands to generate.