projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
rseq/selftests: powerpc code signature: generate valid instructions
[librseq.git] / include / rseq / rseq-ppc.h
1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
2 /*
3 * rseq-ppc.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
9 /*
10 * RSEQ_SIG is used with the following trap instruction:
11 *
12 * powerpc-be: 0f e5 00 0b twui r5,11
13 * powerpc64-le: 0b 00 e5 0f twui r5,11
14 * powerpc64-be: 0f e5 00 0b twui r5,11
15 */
16
17 #define RSEQ_SIG 0x0fe5000b
18
19 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
20 #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
21 #define rseq_smp_rmb() rseq_smp_lwsync()
22 #define rseq_smp_wmb() rseq_smp_lwsync()
23
24 #define rseq_smp_load_acquire(p) \
25 __extension__ ({ \
26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
27 rseq_smp_lwsync(); \
28 ____p1; \
29 })
30
31 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
32
33 #define rseq_smp_store_release(p, v) \
34 do { \
35 rseq_smp_lwsync(); \
36 RSEQ_WRITE_ONCE(*p, v); \
37 } while (0)
38
39 #ifdef RSEQ_SKIP_FASTPATH
40 #include "rseq-skip.h"
41 #else /* !RSEQ_SKIP_FASTPATH */
42
43 /*
44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
45 * better handle single-stepping through the restartable critical sections.
46 */
47
48 #ifdef __PPC64__
49
50 #define STORE_WORD "std "
51 #define LOAD_WORD "ld "
52 #define LOADX_WORD "ldx "
53 #define CMP_WORD "cmpd "
54
55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
56 start_ip, post_commit_offset, abort_ip) \
57 ".pushsection __rseq_cs, \"aw\"\n\t" \
58 ".balign 32\n\t" \
59 __rseq_str(label) ":\n\t" \
60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 ".popsection\n\t" \
63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
64 ".quad " __rseq_str(label) "b\n\t" \
65 ".popsection\n\t"
66
67 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
68 RSEQ_INJECT_ASM(1) \
69 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
70 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
71 "rldicr %%r17, %%r17, 32, 31\n\t" \
72 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
73 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
74 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
75 __rseq_str(label) ":\n\t"
76
77 /*
78 * Exit points of a rseq critical section consist of all instructions outside
79 * of the critical section where a critical section can either branch to or
80 * reach through the normal course of its execution. The abort IP and the
81 * post-commit IP are already part of the __rseq_cs section and should not be
82 * explicitly defined as additional exit points. Knowing all exit points is
83 * useful to assist debuggers stepping over the critical section.
84 */
85 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
86 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
87 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
88 ".popsection\n\t"
89
90 #else /* #ifdef __PPC64__ */
91
92 #define STORE_WORD "stw "
93 #define LOAD_WORD "lwz "
94 #define LOADX_WORD "lwzx "
95 #define CMP_WORD "cmpw "
96
97 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
98 start_ip, post_commit_offset, abort_ip) \
99 ".pushsection __rseq_cs, \"aw\"\n\t" \
100 ".balign 32\n\t" \
101 __rseq_str(label) ":\n\t" \
102 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
103 /* 32-bit only supported on BE */ \
104 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
105 ".popsection\n\t" \
106 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
107 ".long 0x0, " __rseq_str(label) "b\n\t" \
108 ".popsection\n\t"
109
110 /*
111 * Exit points of a rseq critical section consist of all instructions outside
112 * of the critical section where a critical section can either branch to or
113 * reach through the normal course of its execution. The abort IP and the
114 * post-commit IP are already part of the __rseq_cs section and should not be
115 * explicitly defined as additional exit points. Knowing all exit points is
116 * useful to assist debuggers stepping over the critical section.
117 */
118 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
119 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
120 /* 32-bit only supported on BE */ \
121 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
122 ".popsection\n\t"
123
124 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
125 RSEQ_INJECT_ASM(1) \
126 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
127 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
128 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
129 __rseq_str(label) ":\n\t"
130
131 #endif /* #ifdef __PPC64__ */
132
133 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
134 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
135 (post_commit_ip - start_ip), abort_ip)
136
137 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
138 RSEQ_INJECT_ASM(2) \
139 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
140 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
141 "bne- cr7, " __rseq_str(label) "\n\t"
142
143 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
144 ".pushsection __rseq_failure, \"ax\"\n\t" \
145 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
146 __rseq_str(label) ":\n\t" \
147 "b %l[" __rseq_str(abort_label) "]\n\t" \
148 ".popsection\n\t"
149
150 /*
151 * RSEQ_ASM_OPs: asm operations for rseq
152 * RSEQ_ASM_OP_R_*: has hard-code registers in it
153 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
154 */
155 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
156 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
157 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
158 "bne- cr7, " __rseq_str(label) "\n\t"
159
160 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
161 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
162 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
163 "beq- cr7, " __rseq_str(label) "\n\t"
164
165 #define RSEQ_ASM_OP_STORE(value, var) \
166 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
167
168 /* Load @var to r17 */
169 #define RSEQ_ASM_OP_R_LOAD(var) \
170 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
171
172 /* Store r17 to @var */
173 #define RSEQ_ASM_OP_R_STORE(var) \
174 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
175
176 /* Add @count to r17 */
177 #define RSEQ_ASM_OP_R_ADD(count) \
178 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
179
180 /* Load (r17 + voffp) to r17 */
181 #define RSEQ_ASM_OP_R_LOADX(voffp) \
182 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
183
184 /* TODO: implement a faster memcpy. */
185 #define RSEQ_ASM_OP_R_MEMCPY() \
186 "cmpdi %%r19, 0\n\t" \
187 "beq 333f\n\t" \
188 "addi %%r20, %%r20, -1\n\t" \
189 "addi %%r21, %%r21, -1\n\t" \
190 "222:\n\t" \
191 "lbzu %%r18, 1(%%r20)\n\t" \
192 "stbu %%r18, 1(%%r21)\n\t" \
193 "addi %%r19, %%r19, -1\n\t" \
194 "cmpdi %%r19, 0\n\t" \
195 "bne 222b\n\t" \
196 "333:\n\t" \
197
198 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
199 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
200 __rseq_str(post_commit_label) ":\n\t"
201
202 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
203 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
204 __rseq_str(post_commit_label) ":\n\t"
205
206 static inline __attribute__((always_inline))
207 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
208 {
209 RSEQ_INJECT_C(9)
210
211 __asm__ __volatile__ goto (
212 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
213 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
214 #ifdef RSEQ_COMPARE_TWICE
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
216 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
217 #endif
218 /* Start rseq by storing table entry pointer into rseq_cs. */
219 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
220 /* cmp cpuid */
221 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
222 RSEQ_INJECT_ASM(3)
223 /* cmp @v equal to @expect */
224 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
225 RSEQ_INJECT_ASM(4)
226 #ifdef RSEQ_COMPARE_TWICE
227 /* cmp cpuid */
228 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
229 /* cmp @v equal to @expect */
230 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
231 #endif
232 /* final store */
233 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
234 RSEQ_INJECT_ASM(5)
235 RSEQ_ASM_DEFINE_ABORT(4, abort)
236 : /* gcc asm goto does not allow outputs */
237 : [cpu_id] "r" (cpu),
238 [current_cpu_id] "m" (__rseq_abi.cpu_id),
239 [rseq_cs] "m" (__rseq_abi.rseq_cs),
240 [v] "m" (*v),
241 [expect] "r" (expect),
242 [newv] "r" (newv)
243 RSEQ_INJECT_INPUT
244 : "memory", "cc", "r17"
245 RSEQ_INJECT_CLOBBER
246 : abort, cmpfail
247 #ifdef RSEQ_COMPARE_TWICE
248 , error1, error2
249 #endif
250 );
251 return 0;
252 abort:
253 RSEQ_INJECT_FAILED
254 return -1;
255 cmpfail:
256 return 1;
257 #ifdef RSEQ_COMPARE_TWICE
258 error1:
259 rseq_bug("cpu_id comparison failed");
260 error2:
261 rseq_bug("expected value comparison failed");
262 #endif
263 }
264
265 static inline __attribute__((always_inline))
266 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
267 off_t voffp, intptr_t *load, int cpu)
268 {
269 RSEQ_INJECT_C(9)
270
271 __asm__ __volatile__ goto (
272 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
274 #ifdef RSEQ_COMPARE_TWICE
275 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
276 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
277 #endif
278 /* Start rseq by storing table entry pointer into rseq_cs. */
279 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
280 /* cmp cpuid */
281 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
282 RSEQ_INJECT_ASM(3)
283 /* cmp @v not equal to @expectnot */
284 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
285 RSEQ_INJECT_ASM(4)
286 #ifdef RSEQ_COMPARE_TWICE
287 /* cmp cpuid */
288 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
289 /* cmp @v not equal to @expectnot */
290 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
291 #endif
292 /* load the value of @v */
293 RSEQ_ASM_OP_R_LOAD(v)
294 /* store it in @load */
295 RSEQ_ASM_OP_R_STORE(load)
296 /* dereference voffp(v) */
297 RSEQ_ASM_OP_R_LOADX(voffp)
298 /* final store the value at voffp(v) */
299 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
300 RSEQ_INJECT_ASM(5)
301 RSEQ_ASM_DEFINE_ABORT(4, abort)
302 : /* gcc asm goto does not allow outputs */
303 : [cpu_id] "r" (cpu),
304 [current_cpu_id] "m" (__rseq_abi.cpu_id),
305 [rseq_cs] "m" (__rseq_abi.rseq_cs),
306 /* final store input */
307 [v] "m" (*v),
308 [expectnot] "r" (expectnot),
309 [voffp] "b" (voffp),
310 [load] "m" (*load)
311 RSEQ_INJECT_INPUT
312 : "memory", "cc", "r17"
313 RSEQ_INJECT_CLOBBER
314 : abort, cmpfail
315 #ifdef RSEQ_COMPARE_TWICE
316 , error1, error2
317 #endif
318 );
319 return 0;
320 abort:
321 RSEQ_INJECT_FAILED
322 return -1;
323 cmpfail:
324 return 1;
325 #ifdef RSEQ_COMPARE_TWICE
326 error1:
327 rseq_bug("cpu_id comparison failed");
328 error2:
329 rseq_bug("expected value comparison failed");
330 #endif
331 }
332
333 static inline __attribute__((always_inline))
334 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
335 {
336 RSEQ_INJECT_C(9)
337
338 __asm__ __volatile__ goto (
339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
340 #ifdef RSEQ_COMPARE_TWICE
341 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
342 #endif
343 /* Start rseq by storing table entry pointer into rseq_cs. */
344 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
345 /* cmp cpuid */
346 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
347 RSEQ_INJECT_ASM(3)
348 #ifdef RSEQ_COMPARE_TWICE
349 /* cmp cpuid */
350 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
351 #endif
352 /* load the value of @v */
353 RSEQ_ASM_OP_R_LOAD(v)
354 /* add @count to it */
355 RSEQ_ASM_OP_R_ADD(count)
356 /* final store */
357 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
358 RSEQ_INJECT_ASM(4)
359 RSEQ_ASM_DEFINE_ABORT(4, abort)
360 : /* gcc asm goto does not allow outputs */
361 : [cpu_id] "r" (cpu),
362 [current_cpu_id] "m" (__rseq_abi.cpu_id),
363 [rseq_cs] "m" (__rseq_abi.rseq_cs),
364 /* final store input */
365 [v] "m" (*v),
366 [count] "r" (count)
367 RSEQ_INJECT_INPUT
368 : "memory", "cc", "r17"
369 RSEQ_INJECT_CLOBBER
370 : abort
371 #ifdef RSEQ_COMPARE_TWICE
372 , error1
373 #endif
374 );
375 return 0;
376 abort:
377 RSEQ_INJECT_FAILED
378 return -1;
379 #ifdef RSEQ_COMPARE_TWICE
380 error1:
381 rseq_bug("cpu_id comparison failed");
382 #endif
383 }
384
385 static inline __attribute__((always_inline))
386 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
387 intptr_t *v2, intptr_t newv2,
388 intptr_t newv, int cpu)
389 {
390 RSEQ_INJECT_C(9)
391
392 __asm__ __volatile__ goto (
393 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
394 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
395 #ifdef RSEQ_COMPARE_TWICE
396 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
397 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
398 #endif
399 /* Start rseq by storing table entry pointer into rseq_cs. */
400 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
401 /* cmp cpuid */
402 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
403 RSEQ_INJECT_ASM(3)
404 /* cmp @v equal to @expect */
405 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
406 RSEQ_INJECT_ASM(4)
407 #ifdef RSEQ_COMPARE_TWICE
408 /* cmp cpuid */
409 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
410 /* cmp @v equal to @expect */
411 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
412 #endif
413 /* try store */
414 RSEQ_ASM_OP_STORE(newv2, v2)
415 RSEQ_INJECT_ASM(5)
416 /* final store */
417 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
418 RSEQ_INJECT_ASM(6)
419 RSEQ_ASM_DEFINE_ABORT(4, abort)
420 : /* gcc asm goto does not allow outputs */
421 : [cpu_id] "r" (cpu),
422 [current_cpu_id] "m" (__rseq_abi.cpu_id),
423 [rseq_cs] "m" (__rseq_abi.rseq_cs),
424 /* try store input */
425 [v2] "m" (*v2),
426 [newv2] "r" (newv2),
427 /* final store input */
428 [v] "m" (*v),
429 [expect] "r" (expect),
430 [newv] "r" (newv)
431 RSEQ_INJECT_INPUT
432 : "memory", "cc", "r17"
433 RSEQ_INJECT_CLOBBER
434 : abort, cmpfail
435 #ifdef RSEQ_COMPARE_TWICE
436 , error1, error2
437 #endif
438 );
439 return 0;
440 abort:
441 RSEQ_INJECT_FAILED
442 return -1;
443 cmpfail:
444 return 1;
445 #ifdef RSEQ_COMPARE_TWICE
446 error1:
447 rseq_bug("cpu_id comparison failed");
448 error2:
449 rseq_bug("expected value comparison failed");
450 #endif
451 }
452
453 static inline __attribute__((always_inline))
454 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
455 intptr_t *v2, intptr_t newv2,
456 intptr_t newv, int cpu)
457 {
458 RSEQ_INJECT_C(9)
459
460 __asm__ __volatile__ goto (
461 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
462 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
463 #ifdef RSEQ_COMPARE_TWICE
464 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
465 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
466 #endif
467 /* Start rseq by storing table entry pointer into rseq_cs. */
468 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
469 /* cmp cpuid */
470 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
471 RSEQ_INJECT_ASM(3)
472 /* cmp @v equal to @expect */
473 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
474 RSEQ_INJECT_ASM(4)
475 #ifdef RSEQ_COMPARE_TWICE
476 /* cmp cpuid */
477 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
478 /* cmp @v equal to @expect */
479 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
480 #endif
481 /* try store */
482 RSEQ_ASM_OP_STORE(newv2, v2)
483 RSEQ_INJECT_ASM(5)
484 /* for 'release' */
485 "lwsync\n\t"
486 /* final store */
487 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
488 RSEQ_INJECT_ASM(6)
489 RSEQ_ASM_DEFINE_ABORT(4, abort)
490 : /* gcc asm goto does not allow outputs */
491 : [cpu_id] "r" (cpu),
492 [current_cpu_id] "m" (__rseq_abi.cpu_id),
493 [rseq_cs] "m" (__rseq_abi.rseq_cs),
494 /* try store input */
495 [v2] "m" (*v2),
496 [newv2] "r" (newv2),
497 /* final store input */
498 [v] "m" (*v),
499 [expect] "r" (expect),
500 [newv] "r" (newv)
501 RSEQ_INJECT_INPUT
502 : "memory", "cc", "r17"
503 RSEQ_INJECT_CLOBBER
504 : abort, cmpfail
505 #ifdef RSEQ_COMPARE_TWICE
506 , error1, error2
507 #endif
508 );
509 return 0;
510 abort:
511 RSEQ_INJECT_FAILED
512 return -1;
513 cmpfail:
514 return 1;
515 #ifdef RSEQ_COMPARE_TWICE
516 error1:
517 rseq_bug("cpu_id comparison failed");
518 error2:
519 rseq_bug("expected value comparison failed");
520 #endif
521 }
522
523 static inline __attribute__((always_inline))
524 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
525 intptr_t *v2, intptr_t expect2,
526 intptr_t newv, int cpu)
527 {
528 RSEQ_INJECT_C(9)
529
530 __asm__ __volatile__ goto (
531 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
532 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
533 #ifdef RSEQ_COMPARE_TWICE
534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
535 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
537 #endif
538 /* Start rseq by storing table entry pointer into rseq_cs. */
539 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
540 /* cmp cpuid */
541 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
542 RSEQ_INJECT_ASM(3)
543 /* cmp @v equal to @expect */
544 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
545 RSEQ_INJECT_ASM(4)
546 /* cmp @v2 equal to @expct2 */
547 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
548 RSEQ_INJECT_ASM(5)
549 #ifdef RSEQ_COMPARE_TWICE
550 /* cmp cpuid */
551 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
552 /* cmp @v equal to @expect */
553 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
554 /* cmp @v2 equal to @expct2 */
555 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
556 #endif
557 /* final store */
558 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
559 RSEQ_INJECT_ASM(6)
560 RSEQ_ASM_DEFINE_ABORT(4, abort)
561 : /* gcc asm goto does not allow outputs */
562 : [cpu_id] "r" (cpu),
563 [current_cpu_id] "m" (__rseq_abi.cpu_id),
564 [rseq_cs] "m" (__rseq_abi.rseq_cs),
565 /* cmp2 input */
566 [v2] "m" (*v2),
567 [expect2] "r" (expect2),
568 /* final store input */
569 [v] "m" (*v),
570 [expect] "r" (expect),
571 [newv] "r" (newv)
572 RSEQ_INJECT_INPUT
573 : "memory", "cc", "r17"
574 RSEQ_INJECT_CLOBBER
575 : abort, cmpfail
576 #ifdef RSEQ_COMPARE_TWICE
577 , error1, error2, error3
578 #endif
579 );
580 return 0;
581 abort:
582 RSEQ_INJECT_FAILED
583 return -1;
584 cmpfail:
585 return 1;
586 #ifdef RSEQ_COMPARE_TWICE
587 error1:
588 rseq_bug("cpu_id comparison failed");
589 error2:
590 rseq_bug("1st expected value comparison failed");
591 error3:
592 rseq_bug("2nd expected value comparison failed");
593 #endif
594 }
595
596 static inline __attribute__((always_inline))
597 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
598 void *dst, void *src, size_t len,
599 intptr_t newv, int cpu)
600 {
601 RSEQ_INJECT_C(9)
602
603 __asm__ __volatile__ goto (
604 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
605 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
606 #ifdef RSEQ_COMPARE_TWICE
607 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
608 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
609 #endif
610 /* setup for mempcy */
611 "mr %%r19, %[len]\n\t"
612 "mr %%r20, %[src]\n\t"
613 "mr %%r21, %[dst]\n\t"
614 /* Start rseq by storing table entry pointer into rseq_cs. */
615 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
616 /* cmp cpuid */
617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
618 RSEQ_INJECT_ASM(3)
619 /* cmp @v equal to @expect */
620 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
621 RSEQ_INJECT_ASM(4)
622 #ifdef RSEQ_COMPARE_TWICE
623 /* cmp cpuid */
624 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
625 /* cmp @v equal to @expect */
626 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
627 #endif
628 /* try memcpy */
629 RSEQ_ASM_OP_R_MEMCPY()
630 RSEQ_INJECT_ASM(5)
631 /* final store */
632 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
633 RSEQ_INJECT_ASM(6)
634 /* teardown */
635 RSEQ_ASM_DEFINE_ABORT(4, abort)
636 : /* gcc asm goto does not allow outputs */
637 : [cpu_id] "r" (cpu),
638 [current_cpu_id] "m" (__rseq_abi.cpu_id),
639 [rseq_cs] "m" (__rseq_abi.rseq_cs),
640 /* final store input */
641 [v] "m" (*v),
642 [expect] "r" (expect),
643 [newv] "r" (newv),
644 /* try memcpy input */
645 [dst] "r" (dst),
646 [src] "r" (src),
647 [len] "r" (len)
648 RSEQ_INJECT_INPUT
649 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
650 RSEQ_INJECT_CLOBBER
651 : abort, cmpfail
652 #ifdef RSEQ_COMPARE_TWICE
653 , error1, error2
654 #endif
655 );
656 return 0;
657 abort:
658 RSEQ_INJECT_FAILED
659 return -1;
660 cmpfail:
661 return 1;
662 #ifdef RSEQ_COMPARE_TWICE
663 error1:
664 rseq_bug("cpu_id comparison failed");
665 error2:
666 rseq_bug("expected value comparison failed");
667 #endif
668 }
669
670 static inline __attribute__((always_inline))
671 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
672 void *dst, void *src, size_t len,
673 intptr_t newv, int cpu)
674 {
675 RSEQ_INJECT_C(9)
676
677 __asm__ __volatile__ goto (
678 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
679 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
680 #ifdef RSEQ_COMPARE_TWICE
681 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
682 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
683 #endif
684 /* setup for mempcy */
685 "mr %%r19, %[len]\n\t"
686 "mr %%r20, %[src]\n\t"
687 "mr %%r21, %[dst]\n\t"
688 /* Start rseq by storing table entry pointer into rseq_cs. */
689 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
690 /* cmp cpuid */
691 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
692 RSEQ_INJECT_ASM(3)
693 /* cmp @v equal to @expect */
694 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
695 RSEQ_INJECT_ASM(4)
696 #ifdef RSEQ_COMPARE_TWICE
697 /* cmp cpuid */
698 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
699 /* cmp @v equal to @expect */
700 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
701 #endif
702 /* try memcpy */
703 RSEQ_ASM_OP_R_MEMCPY()
704 RSEQ_INJECT_ASM(5)
705 /* for 'release' */
706 "lwsync\n\t"
707 /* final store */
708 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
709 RSEQ_INJECT_ASM(6)
710 /* teardown */
711 RSEQ_ASM_DEFINE_ABORT(4, abort)
712 : /* gcc asm goto does not allow outputs */
713 : [cpu_id] "r" (cpu),
714 [current_cpu_id] "m" (__rseq_abi.cpu_id),
715 [rseq_cs] "m" (__rseq_abi.rseq_cs),
716 /* final store input */
717 [v] "m" (*v),
718 [expect] "r" (expect),
719 [newv] "r" (newv),
720 /* try memcpy input */
721 [dst] "r" (dst),
722 [src] "r" (src),
723 [len] "r" (len)
724 RSEQ_INJECT_INPUT
725 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
726 RSEQ_INJECT_CLOBBER
727 : abort, cmpfail
728 #ifdef RSEQ_COMPARE_TWICE
729 , error1, error2
730 #endif
731 );
732 return 0;
733 abort:
734 RSEQ_INJECT_FAILED
735 return -1;
736 cmpfail:
737 return 1;
738 #ifdef RSEQ_COMPARE_TWICE
739 error1:
740 rseq_bug("cpu_id comparison failed");
741 error2:
742 rseq_bug("expected value comparison failed");
743 #endif
744 }
745
746 #undef STORE_WORD
747 #undef LOAD_WORD
748 #undef LOADX_WORD
749 #undef CMP_WORD
750
751 #endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.063284 seconds and 5 git commands to generate.