projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Introduce __rseq_cs_ptr_array, rename __rseq_table to __rseq_cs
[librseq.git] / include / rseq / rseq-s390.h
1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
2
3 #define RSEQ_SIG 0x53053053
4
5 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
6 #define rseq_smp_rmb() rseq_smp_mb()
7 #define rseq_smp_wmb() rseq_smp_mb()
8
9 #define rseq_smp_load_acquire(p) \
10 __extension__ ({ \
11 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
12 rseq_barrier(); \
13 ____p1; \
14 })
15
16 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
17
18 #define rseq_smp_store_release(p, v) \
19 do { \
20 rseq_barrier(); \
21 RSEQ_WRITE_ONCE(*p, v); \
22 } while (0)
23
24 #ifdef RSEQ_SKIP_FASTPATH
25 #include "rseq-skip.h"
26 #else /* !RSEQ_SKIP_FASTPATH */
27
28 #ifdef __s390x__
29
30 #define LONG_L "lg"
31 #define LONG_S "stg"
32 #define LONG_LT_R "ltgr"
33 #define LONG_CMP "cg"
34 #define LONG_CMP_R "cgr"
35 #define LONG_ADDI "aghi"
36 #define LONG_ADD_R "agr"
37
38 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
39 start_ip, post_commit_offset, abort_ip) \
40 ".pushsection __rseq_cs, \"aw\"\n\t" \
41 ".balign 32\n\t" \
42 __rseq_str(label) ":\n\t" \
43 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
45 ".popsection\n\t" \
46 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
47 ".quad " __rseq_str(label) "b\n\t" \
48 ".popsection\n\t"
49
50 /*
51 * Exit points of a rseq critical section consist of all instructions outside
52 * of the critical section where a critical section can branch to. The abort IP
53 * is already part of the __rseq_cs section and should not be explicitly
54 * defined as an additional exit point. Knowing all exit points is useful to
55 * assist debuggers stepping over the critical section.
56 */
57 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
58 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
59 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
60 ".popsection\n\t"
61
62 #elif __s390__
63
64 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
65 start_ip, post_commit_offset, abort_ip) \
66 ".pushsection __rseq_cs, \"aw\"\n\t" \
67 ".balign 32\n\t" \
68 __rseq_str(label) ":\n\t" \
69 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
70 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
71 ".popsection\n\t" \
72 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
73 ".long 0x0, " __rseq_str(label) "b\n\t" \
74 ".popsection\n\t"
75
76 /*
77 * Exit points of a rseq critical section consist of all instructions outside
78 * of the critical section where a critical section can branch to. The abort IP
79 * is already part of the __rseq_cs and should not be explicitly defined as
80 * an additional exit point. Knowing all exit points is useful to assist
81 * debuggers stepping over the critical section.
82 */
83 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
84 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
85 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
86 ".popsection\n\t"
87
88 #define LONG_L "l"
89 #define LONG_S "st"
90 #define LONG_LT_R "ltr"
91 #define LONG_CMP "c"
92 #define LONG_CMP_R "cr"
93 #define LONG_ADDI "ahi"
94 #define LONG_ADD_R "ar"
95
96 #endif
97
98 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
99 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
100 (post_commit_ip - start_ip), abort_ip)
101
102 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
103 RSEQ_INJECT_ASM(1) \
104 "larl %%r0, " __rseq_str(cs_label) "\n\t" \
105 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
106 __rseq_str(label) ":\n\t"
107
108 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
109 RSEQ_INJECT_ASM(2) \
110 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
111 "jnz " __rseq_str(label) "\n\t"
112
113 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
114 ".pushsection __rseq_failure, \"ax\"\n\t" \
115 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
116 __rseq_str(label) ":\n\t" \
117 teardown \
118 "j %l[" __rseq_str(abort_label) "]\n\t" \
119 ".popsection\n\t"
120
121 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
122 ".pushsection __rseq_failure, \"ax\"\n\t" \
123 __rseq_str(label) ":\n\t" \
124 teardown \
125 "j %l[" __rseq_str(cmpfail_label) "]\n\t" \
126 ".popsection\n\t"
127
128 static inline __attribute__((always_inline))
129 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
130 {
131 RSEQ_INJECT_C(9)
132
133 __asm__ __volatile__ goto (
134 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
135 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
136 #ifdef RSEQ_COMPARE_TWICE
137 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
138 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
139 #endif
140 /* Start rseq by storing table entry pointer into rseq_cs. */
141 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
142 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
143 RSEQ_INJECT_ASM(3)
144 LONG_CMP " %[expect], %[v]\n\t"
145 "jnz %l[cmpfail]\n\t"
146 RSEQ_INJECT_ASM(4)
147 #ifdef RSEQ_COMPARE_TWICE
148 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
149 LONG_CMP " %[expect], %[v]\n\t"
150 "jnz %l[error2]\n\t"
151 #endif
152 /* final store */
153 LONG_S " %[newv], %[v]\n\t"
154 "2:\n\t"
155 RSEQ_INJECT_ASM(5)
156 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
157 : /* gcc asm goto does not allow outputs */
158 : [cpu_id] "r" (cpu),
159 [current_cpu_id] "m" (__rseq_abi.cpu_id),
160 [rseq_cs] "m" (__rseq_abi.rseq_cs),
161 [v] "m" (*v),
162 [expect] "r" (expect),
163 [newv] "r" (newv)
164 RSEQ_INJECT_INPUT
165 : "memory", "cc", "r0"
166 RSEQ_INJECT_CLOBBER
167 : abort, cmpfail
168 #ifdef RSEQ_COMPARE_TWICE
169 , error1, error2
170 #endif
171 );
172 return 0;
173 abort:
174 RSEQ_INJECT_FAILED
175 return -1;
176 cmpfail:
177 return 1;
178 #ifdef RSEQ_COMPARE_TWICE
179 error1:
180 rseq_bug("cpu_id comparison failed");
181 error2:
182 rseq_bug("expected value comparison failed");
183 #endif
184 }
185
186 /*
187 * Compare @v against @expectnot. When it does _not_ match, load @v
188 * into @load, and store the content of *@v + voffp into @v.
189 */
190 static inline __attribute__((always_inline))
191 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
192 off_t voffp, intptr_t *load, int cpu)
193 {
194 RSEQ_INJECT_C(9)
195
196 __asm__ __volatile__ goto (
197 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
198 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
199 #ifdef RSEQ_COMPARE_TWICE
200 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
201 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
202 #endif
203 /* Start rseq by storing table entry pointer into rseq_cs. */
204 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
205 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
206 RSEQ_INJECT_ASM(3)
207 LONG_L " %%r1, %[v]\n\t"
208 LONG_CMP_R " %%r1, %[expectnot]\n\t"
209 "je %l[cmpfail]\n\t"
210 RSEQ_INJECT_ASM(4)
211 #ifdef RSEQ_COMPARE_TWICE
212 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
213 LONG_L " %%r1, %[v]\n\t"
214 LONG_CMP_R " %%r1, %[expectnot]\n\t"
215 "je %l[error2]\n\t"
216 #endif
217 LONG_S " %%r1, %[load]\n\t"
218 LONG_ADD_R " %%r1, %[voffp]\n\t"
219 LONG_L " %%r1, 0(%%r1)\n\t"
220 /* final store */
221 LONG_S " %%r1, %[v]\n\t"
222 "2:\n\t"
223 RSEQ_INJECT_ASM(5)
224 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
225 : /* gcc asm goto does not allow outputs */
226 : [cpu_id] "r" (cpu),
227 [current_cpu_id] "m" (__rseq_abi.cpu_id),
228 [rseq_cs] "m" (__rseq_abi.rseq_cs),
229 /* final store input */
230 [v] "m" (*v),
231 [expectnot] "r" (expectnot),
232 [voffp] "r" (voffp),
233 [load] "m" (*load)
234 RSEQ_INJECT_INPUT
235 : "memory", "cc", "r0", "r1"
236 RSEQ_INJECT_CLOBBER
237 : abort, cmpfail
238 #ifdef RSEQ_COMPARE_TWICE
239 , error1, error2
240 #endif
241 );
242 return 0;
243 abort:
244 RSEQ_INJECT_FAILED
245 return -1;
246 cmpfail:
247 return 1;
248 #ifdef RSEQ_COMPARE_TWICE
249 error1:
250 rseq_bug("cpu_id comparison failed");
251 error2:
252 rseq_bug("expected value comparison failed");
253 #endif
254 }
255
256 static inline __attribute__((always_inline))
257 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
258 {
259 RSEQ_INJECT_C(9)
260
261 __asm__ __volatile__ goto (
262 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
263 #ifdef RSEQ_COMPARE_TWICE
264 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
265 #endif
266 /* Start rseq by storing table entry pointer into rseq_cs. */
267 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
268 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
269 RSEQ_INJECT_ASM(3)
270 #ifdef RSEQ_COMPARE_TWICE
271 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
272 #endif
273 LONG_L " %%r0, %[v]\n\t"
274 LONG_ADD_R " %%r0, %[count]\n\t"
275 /* final store */
276 LONG_S " %%r0, %[v]\n\t"
277 "2:\n\t"
278 RSEQ_INJECT_ASM(4)
279 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
280 : /* gcc asm goto does not allow outputs */
281 : [cpu_id] "r" (cpu),
282 [current_cpu_id] "m" (__rseq_abi.cpu_id),
283 [rseq_cs] "m" (__rseq_abi.rseq_cs),
284 /* final store input */
285 [v] "m" (*v),
286 [count] "r" (count)
287 RSEQ_INJECT_INPUT
288 : "memory", "cc", "r0"
289 RSEQ_INJECT_CLOBBER
290 : abort
291 #ifdef RSEQ_COMPARE_TWICE
292 , error1
293 #endif
294 );
295 return 0;
296 abort:
297 RSEQ_INJECT_FAILED
298 return -1;
299 #ifdef RSEQ_COMPARE_TWICE
300 error1:
301 rseq_bug("cpu_id comparison failed");
302 #endif
303 }
304
305 static inline __attribute__((always_inline))
306 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
307 intptr_t *v2, intptr_t newv2,
308 intptr_t newv, int cpu)
309 {
310 RSEQ_INJECT_C(9)
311
312 __asm__ __volatile__ goto (
313 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
314 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
315 #ifdef RSEQ_COMPARE_TWICE
316 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
317 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
318 #endif
319 /* Start rseq by storing table entry pointer into rseq_cs. */
320 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
321 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
322 RSEQ_INJECT_ASM(3)
323 LONG_CMP " %[expect], %[v]\n\t"
324 "jnz %l[cmpfail]\n\t"
325 RSEQ_INJECT_ASM(4)
326 #ifdef RSEQ_COMPARE_TWICE
327 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
328 LONG_CMP " %[expect], %[v]\n\t"
329 "jnz %l[error2]\n\t"
330 #endif
331 /* try store */
332 LONG_S " %[newv2], %[v2]\n\t"
333 RSEQ_INJECT_ASM(5)
334 /* final store */
335 LONG_S " %[newv], %[v]\n\t"
336 "2:\n\t"
337 RSEQ_INJECT_ASM(6)
338 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
339 : /* gcc asm goto does not allow outputs */
340 : [cpu_id] "r" (cpu),
341 [current_cpu_id] "m" (__rseq_abi.cpu_id),
342 [rseq_cs] "m" (__rseq_abi.rseq_cs),
343 /* try store input */
344 [v2] "m" (*v2),
345 [newv2] "r" (newv2),
346 /* final store input */
347 [v] "m" (*v),
348 [expect] "r" (expect),
349 [newv] "r" (newv)
350 RSEQ_INJECT_INPUT
351 : "memory", "cc", "r0"
352 RSEQ_INJECT_CLOBBER
353 : abort, cmpfail
354 #ifdef RSEQ_COMPARE_TWICE
355 , error1, error2
356 #endif
357 );
358 return 0;
359 abort:
360 RSEQ_INJECT_FAILED
361 return -1;
362 cmpfail:
363 return 1;
364 #ifdef RSEQ_COMPARE_TWICE
365 error1:
366 rseq_bug("cpu_id comparison failed");
367 error2:
368 rseq_bug("expected value comparison failed");
369 #endif
370 }
371
372 /* s390 is TSO. */
373 static inline __attribute__((always_inline))
374 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
375 intptr_t *v2, intptr_t newv2,
376 intptr_t newv, int cpu)
377 {
378 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
379 }
380
381 static inline __attribute__((always_inline))
382 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
383 intptr_t *v2, intptr_t expect2,
384 intptr_t newv, int cpu)
385 {
386 RSEQ_INJECT_C(9)
387
388 __asm__ __volatile__ goto (
389 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
390 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
391 #ifdef RSEQ_COMPARE_TWICE
392 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
393 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
394 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
395 #endif
396 /* Start rseq by storing table entry pointer into rseq_cs. */
397 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
398 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
399 RSEQ_INJECT_ASM(3)
400 LONG_CMP " %[expect], %[v]\n\t"
401 "jnz %l[cmpfail]\n\t"
402 RSEQ_INJECT_ASM(4)
403 LONG_CMP " %[expect2], %[v2]\n\t"
404 "jnz %l[cmpfail]\n\t"
405 RSEQ_INJECT_ASM(5)
406 #ifdef RSEQ_COMPARE_TWICE
407 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
408 LONG_CMP " %[expect], %[v]\n\t"
409 "jnz %l[error2]\n\t"
410 LONG_CMP " %[expect2], %[v2]\n\t"
411 "jnz %l[error3]\n\t"
412 #endif
413 /* final store */
414 LONG_S " %[newv], %[v]\n\t"
415 "2:\n\t"
416 RSEQ_INJECT_ASM(6)
417 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
418 : /* gcc asm goto does not allow outputs */
419 : [cpu_id] "r" (cpu),
420 [current_cpu_id] "m" (__rseq_abi.cpu_id),
421 [rseq_cs] "m" (__rseq_abi.rseq_cs),
422 /* cmp2 input */
423 [v2] "m" (*v2),
424 [expect2] "r" (expect2),
425 /* final store input */
426 [v] "m" (*v),
427 [expect] "r" (expect),
428 [newv] "r" (newv)
429 RSEQ_INJECT_INPUT
430 : "memory", "cc", "r0"
431 RSEQ_INJECT_CLOBBER
432 : abort, cmpfail
433 #ifdef RSEQ_COMPARE_TWICE
434 , error1, error2, error3
435 #endif
436 );
437 return 0;
438 abort:
439 RSEQ_INJECT_FAILED
440 return -1;
441 cmpfail:
442 return 1;
443 #ifdef RSEQ_COMPARE_TWICE
444 error1:
445 rseq_bug("cpu_id comparison failed");
446 error2:
447 rseq_bug("1st expected value comparison failed");
448 error3:
449 rseq_bug("2nd expected value comparison failed");
450 #endif
451 }
452
453 static inline __attribute__((always_inline))
454 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
455 void *dst, void *src, size_t len,
456 intptr_t newv, int cpu)
457 {
458 uint64_t rseq_scratch[3];
459
460 RSEQ_INJECT_C(9)
461
462 __asm__ __volatile__ goto (
463 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
464 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
465 #ifdef RSEQ_COMPARE_TWICE
466 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
467 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
468 #endif
469 LONG_S " %[src], %[rseq_scratch0]\n\t"
470 LONG_S " %[dst], %[rseq_scratch1]\n\t"
471 LONG_S " %[len], %[rseq_scratch2]\n\t"
472 /* Start rseq by storing table entry pointer into rseq_cs. */
473 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
474 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
475 RSEQ_INJECT_ASM(3)
476 LONG_CMP " %[expect], %[v]\n\t"
477 "jnz 5f\n\t"
478 RSEQ_INJECT_ASM(4)
479 #ifdef RSEQ_COMPARE_TWICE
480 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
481 LONG_CMP " %[expect], %[v]\n\t"
482 "jnz 7f\n\t"
483 #endif
484 /* try memcpy */
485 LONG_LT_R " %[len], %[len]\n\t"
486 "jz 333f\n\t"
487 "222:\n\t"
488 "ic %%r0,0(%[src])\n\t"
489 "stc %%r0,0(%[dst])\n\t"
490 LONG_ADDI " %[src], 1\n\t"
491 LONG_ADDI " %[dst], 1\n\t"
492 LONG_ADDI " %[len], -1\n\t"
493 "jnz 222b\n\t"
494 "333:\n\t"
495 RSEQ_INJECT_ASM(5)
496 /* final store */
497 LONG_S " %[newv], %[v]\n\t"
498 "2:\n\t"
499 RSEQ_INJECT_ASM(6)
500 /* teardown */
501 LONG_L " %[len], %[rseq_scratch2]\n\t"
502 LONG_L " %[dst], %[rseq_scratch1]\n\t"
503 LONG_L " %[src], %[rseq_scratch0]\n\t"
504 RSEQ_ASM_DEFINE_ABORT(4,
505 LONG_L " %[len], %[rseq_scratch2]\n\t"
506 LONG_L " %[dst], %[rseq_scratch1]\n\t"
507 LONG_L " %[src], %[rseq_scratch0]\n\t",
508 abort)
509 RSEQ_ASM_DEFINE_CMPFAIL(5,
510 LONG_L " %[len], %[rseq_scratch2]\n\t"
511 LONG_L " %[dst], %[rseq_scratch1]\n\t"
512 LONG_L " %[src], %[rseq_scratch0]\n\t",
513 cmpfail)
514 #ifdef RSEQ_COMPARE_TWICE
515 RSEQ_ASM_DEFINE_CMPFAIL(6,
516 LONG_L " %[len], %[rseq_scratch2]\n\t"
517 LONG_L " %[dst], %[rseq_scratch1]\n\t"
518 LONG_L " %[src], %[rseq_scratch0]\n\t",
519 error1)
520 RSEQ_ASM_DEFINE_CMPFAIL(7,
521 LONG_L " %[len], %[rseq_scratch2]\n\t"
522 LONG_L " %[dst], %[rseq_scratch1]\n\t"
523 LONG_L " %[src], %[rseq_scratch0]\n\t",
524 error2)
525 #endif
526 : /* gcc asm goto does not allow outputs */
527 : [cpu_id] "r" (cpu),
528 [current_cpu_id] "m" (__rseq_abi.cpu_id),
529 [rseq_cs] "m" (__rseq_abi.rseq_cs),
530 /* final store input */
531 [v] "m" (*v),
532 [expect] "r" (expect),
533 [newv] "r" (newv),
534 /* try memcpy input */
535 [dst] "r" (dst),
536 [src] "r" (src),
537 [len] "r" (len),
538 [rseq_scratch0] "m" (rseq_scratch[0]),
539 [rseq_scratch1] "m" (rseq_scratch[1]),
540 [rseq_scratch2] "m" (rseq_scratch[2])
541 RSEQ_INJECT_INPUT
542 : "memory", "cc", "r0"
543 RSEQ_INJECT_CLOBBER
544 : abort, cmpfail
545 #ifdef RSEQ_COMPARE_TWICE
546 , error1, error2
547 #endif
548 );
549 return 0;
550 abort:
551 RSEQ_INJECT_FAILED
552 return -1;
553 cmpfail:
554 return 1;
555 #ifdef RSEQ_COMPARE_TWICE
556 error1:
557 rseq_bug("cpu_id comparison failed");
558 error2:
559 rseq_bug("expected value comparison failed");
560 #endif
561 }
562
563 /* s390 is TSO. */
564 static inline __attribute__((always_inline))
565 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
566 void *dst, void *src, size_t len,
567 intptr_t newv, int cpu)
568 {
569 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
570 newv, cpu);
571 }
572 #endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.04304 seconds and 5 git commands to generate.