1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2018 Vasily Gorbik <gor@linux.ibm.com> */
5 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
6 * access-register mode nor the linkage stack this instruction will always
7 * cause a special-operation exception (the trap-enabled bit in the DUCT
8 * is and will stay 0). The instruction pattern is
9 * b2 ff 0f ff trap4 4095(%r0)
11 #define RSEQ_SIG 0xB2FF0FFF
13 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
14 #define rseq_smp_rmb() rseq_smp_mb()
15 #define rseq_smp_wmb() rseq_smp_mb()
17 #define rseq_smp_load_acquire(p) \
19 __typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
24 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
26 #define rseq_smp_store_release(p, v) \
29 RSEQ_WRITE_ONCE(*(p), v); \
32 #ifdef RSEQ_SKIP_FASTPATH
33 #include "rseq-skip.h"
34 #else /* !RSEQ_SKIP_FASTPATH */
40 #define LONG_LT_R "ltgr"
42 #define LONG_CMP_R "cgr"
43 #define LONG_ADDI "aghi"
44 #define LONG_ADD_R "agr"
46 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
47 start_ip, post_commit_offset, abort_ip) \
48 ".pushsection __rseq_cs, \"aw\"\n\t" \
50 __rseq_str(label) ":\n\t" \
51 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
52 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
54 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
55 ".quad " __rseq_str(label) "b\n\t" \
59 * Exit points of a rseq critical section consist of all instructions outside
60 * of the critical section where a critical section can either branch to or
61 * reach through the normal course of its execution. The abort IP and the
62 * post-commit IP are already part of the __rseq_cs section and should not be
63 * explicitly defined as additional exit points. Knowing all exit points is
64 * useful to assist debuggers stepping over the critical section.
66 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
67 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
68 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
73 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
74 start_ip, post_commit_offset, abort_ip) \
75 ".pushsection __rseq_cs, \"aw\"\n\t" \
77 __rseq_str(label) ":\n\t" \
78 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
79 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
81 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
82 ".long 0x0, " __rseq_str(label) "b\n\t" \
86 * Exit points of a rseq critical section consist of all instructions outside
87 * of the critical section where a critical section can either branch to or
88 * reach through the normal course of its execution. The abort IP and the
89 * post-commit IP are already part of the __rseq_cs section and should not be
90 * explicitly defined as additional exit points. Knowing all exit points is
91 * useful to assist debuggers stepping over the critical section.
93 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
94 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
95 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
100 #define LONG_LT_R "ltr"
102 #define LONG_CMP_R "cr"
103 #define LONG_ADDI "ahi"
104 #define LONG_ADD_R "ar"
108 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
109 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
110 (post_commit_ip - start_ip), abort_ip)
112 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
114 "larl %%r0, " __rseq_str(cs_label) "\n\t" \
115 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
116 __rseq_str(label) ":\n\t"
118 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
120 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
121 "jnz " __rseq_str(label) "\n\t"
123 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
124 ".pushsection __rseq_failure, \"ax\"\n\t" \
125 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
126 __rseq_str(label) ":\n\t" \
128 "jg %l[" __rseq_str(abort_label) "]\n\t" \
131 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
132 ".pushsection __rseq_failure, \"ax\"\n\t" \
133 __rseq_str(label) ":\n\t" \
135 "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
138 static inline __attribute__((always_inline
))
139 int rseq_cmpeqv_storev(intptr_t *v
, intptr_t expect
, intptr_t newv
, int cpu
)
143 __asm__ __volatile__
goto (
144 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
145 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
146 #ifdef RSEQ_COMPARE_TWICE
147 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
148 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
150 /* Start rseq by storing table entry pointer into rseq_cs. */
151 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
152 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
154 LONG_CMP
" %[expect], %[v]\n\t"
155 "jnz %l[cmpfail]\n\t"
157 #ifdef RSEQ_COMPARE_TWICE
158 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
159 LONG_CMP
" %[expect], %[v]\n\t"
163 LONG_S
" %[newv], %[v]\n\t"
166 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
167 : /* gcc asm goto does not allow outputs */
168 : [cpu_id
] "r" (cpu
),
169 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
170 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
172 [expect
] "r" (expect
),
175 : "memory", "cc", "r0"
178 #ifdef RSEQ_COMPARE_TWICE
182 rseq_after_asm_goto();
185 rseq_after_asm_goto();
189 rseq_after_asm_goto();
191 #ifdef RSEQ_COMPARE_TWICE
193 rseq_after_asm_goto();
194 rseq_bug("cpu_id comparison failed");
196 rseq_after_asm_goto();
197 rseq_bug("expected value comparison failed");
202 * Compare @v against @expectnot. When it does _not_ match, load @v
203 * into @load, and store the content of *@v + voffp into @v.
205 static inline __attribute__((always_inline
))
206 int rseq_cmpnev_storeoffp_load(intptr_t *v
, intptr_t expectnot
,
207 long voffp
, intptr_t *load
, int cpu
)
211 __asm__ __volatile__
goto (
212 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
213 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
214 #ifdef RSEQ_COMPARE_TWICE
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
216 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
218 /* Start rseq by storing table entry pointer into rseq_cs. */
219 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
222 LONG_L
" %%r1, %[v]\n\t"
223 LONG_CMP_R
" %%r1, %[expectnot]\n\t"
226 #ifdef RSEQ_COMPARE_TWICE
227 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
228 LONG_L
" %%r1, %[v]\n\t"
229 LONG_CMP_R
" %%r1, %[expectnot]\n\t"
232 LONG_S
" %%r1, %[load]\n\t"
233 LONG_ADD_R
" %%r1, %[voffp]\n\t"
234 LONG_L
" %%r1, 0(%%r1)\n\t"
236 LONG_S
" %%r1, %[v]\n\t"
239 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
240 : /* gcc asm goto does not allow outputs */
241 : [cpu_id
] "r" (cpu
),
242 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
243 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
244 /* final store input */
246 [expectnot
] "r" (expectnot
),
250 : "memory", "cc", "r0", "r1"
253 #ifdef RSEQ_COMPARE_TWICE
257 rseq_after_asm_goto();
260 rseq_after_asm_goto();
264 rseq_after_asm_goto();
266 #ifdef RSEQ_COMPARE_TWICE
268 rseq_after_asm_goto();
269 rseq_bug("cpu_id comparison failed");
271 rseq_after_asm_goto();
272 rseq_bug("expected value comparison failed");
276 static inline __attribute__((always_inline
))
277 int rseq_addv(intptr_t *v
, intptr_t count
, int cpu
)
281 __asm__ __volatile__
goto (
282 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
283 #ifdef RSEQ_COMPARE_TWICE
284 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
286 /* Start rseq by storing table entry pointer into rseq_cs. */
287 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
288 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
290 #ifdef RSEQ_COMPARE_TWICE
291 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
293 LONG_L
" %%r0, %[v]\n\t"
294 LONG_ADD_R
" %%r0, %[count]\n\t"
296 LONG_S
" %%r0, %[v]\n\t"
299 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
300 : /* gcc asm goto does not allow outputs */
301 : [cpu_id
] "r" (cpu
),
302 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
303 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
304 /* final store input */
308 : "memory", "cc", "r0"
311 #ifdef RSEQ_COMPARE_TWICE
315 rseq_after_asm_goto();
318 rseq_after_asm_goto();
321 #ifdef RSEQ_COMPARE_TWICE
323 rseq_after_asm_goto();
324 rseq_bug("cpu_id comparison failed");
328 static inline __attribute__((always_inline
))
329 int rseq_cmpeqv_trystorev_storev(intptr_t *v
, intptr_t expect
,
330 intptr_t *v2
, intptr_t newv2
,
331 intptr_t newv
, int cpu
)
335 __asm__ __volatile__
goto (
336 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
337 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
338 #ifdef RSEQ_COMPARE_TWICE
339 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
340 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
342 /* Start rseq by storing table entry pointer into rseq_cs. */
343 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
344 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
346 LONG_CMP
" %[expect], %[v]\n\t"
347 "jnz %l[cmpfail]\n\t"
349 #ifdef RSEQ_COMPARE_TWICE
350 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
351 LONG_CMP
" %[expect], %[v]\n\t"
355 LONG_S
" %[newv2], %[v2]\n\t"
358 LONG_S
" %[newv], %[v]\n\t"
361 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
362 : /* gcc asm goto does not allow outputs */
363 : [cpu_id
] "r" (cpu
),
364 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
365 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
366 /* try store input */
369 /* final store input */
371 [expect
] "r" (expect
),
374 : "memory", "cc", "r0"
377 #ifdef RSEQ_COMPARE_TWICE
381 rseq_after_asm_goto();
384 rseq_after_asm_goto();
388 rseq_after_asm_goto();
390 #ifdef RSEQ_COMPARE_TWICE
392 rseq_after_asm_goto();
393 rseq_bug("cpu_id comparison failed");
395 rseq_after_asm_goto();
396 rseq_bug("expected value comparison failed");
401 static inline __attribute__((always_inline
))
402 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v
, intptr_t expect
,
403 intptr_t *v2
, intptr_t newv2
,
404 intptr_t newv
, int cpu
)
406 return rseq_cmpeqv_trystorev_storev(v
, expect
, v2
, newv2
, newv
, cpu
);
409 static inline __attribute__((always_inline
))
410 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v
, intptr_t expect
,
411 intptr_t *v2
, intptr_t expect2
,
412 intptr_t newv
, int cpu
)
416 __asm__ __volatile__
goto (
417 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
418 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
419 #ifdef RSEQ_COMPARE_TWICE
420 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
421 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
422 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error3
])
424 /* Start rseq by storing table entry pointer into rseq_cs. */
425 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
426 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
428 LONG_CMP
" %[expect], %[v]\n\t"
429 "jnz %l[cmpfail]\n\t"
431 LONG_CMP
" %[expect2], %[v2]\n\t"
432 "jnz %l[cmpfail]\n\t"
434 #ifdef RSEQ_COMPARE_TWICE
435 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
436 LONG_CMP
" %[expect], %[v]\n\t"
438 LONG_CMP
" %[expect2], %[v2]\n\t"
442 LONG_S
" %[newv], %[v]\n\t"
445 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
446 : /* gcc asm goto does not allow outputs */
447 : [cpu_id
] "r" (cpu
),
448 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
449 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
452 [expect2
] "r" (expect2
),
453 /* final store input */
455 [expect
] "r" (expect
),
458 : "memory", "cc", "r0"
461 #ifdef RSEQ_COMPARE_TWICE
462 , error1
, error2
, error3
465 rseq_after_asm_goto();
468 rseq_after_asm_goto();
472 rseq_after_asm_goto();
474 #ifdef RSEQ_COMPARE_TWICE
476 rseq_after_asm_goto();
477 rseq_bug("cpu_id comparison failed");
479 rseq_after_asm_goto();
480 rseq_bug("1st expected value comparison failed");
482 rseq_after_asm_goto();
483 rseq_bug("2nd expected value comparison failed");
487 static inline __attribute__((always_inline
))
488 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v
, intptr_t expect
,
489 void *dst
, void *src
, size_t len
,
490 intptr_t newv
, int cpu
)
492 uint64_t rseq_scratch
[3];
496 __asm__ __volatile__
goto (
497 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
498 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
499 #ifdef RSEQ_COMPARE_TWICE
500 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
501 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
503 LONG_S
" %[src], %[rseq_scratch0]\n\t"
504 LONG_S
" %[dst], %[rseq_scratch1]\n\t"
505 LONG_S
" %[len], %[rseq_scratch2]\n\t"
506 /* Start rseq by storing table entry pointer into rseq_cs. */
507 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
508 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
510 LONG_CMP
" %[expect], %[v]\n\t"
513 #ifdef RSEQ_COMPARE_TWICE
514 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 6f
)
515 LONG_CMP
" %[expect], %[v]\n\t"
519 LONG_LT_R
" %[len], %[len]\n\t"
522 "ic %%r0,0(%[src])\n\t"
523 "stc %%r0,0(%[dst])\n\t"
524 LONG_ADDI
" %[src], 1\n\t"
525 LONG_ADDI
" %[dst], 1\n\t"
526 LONG_ADDI
" %[len], -1\n\t"
531 LONG_S
" %[newv], %[v]\n\t"
535 LONG_L
" %[len], %[rseq_scratch2]\n\t"
536 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
537 LONG_L
" %[src], %[rseq_scratch0]\n\t"
538 RSEQ_ASM_DEFINE_ABORT(4,
539 LONG_L
" %[len], %[rseq_scratch2]\n\t"
540 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
541 LONG_L
" %[src], %[rseq_scratch0]\n\t",
543 RSEQ_ASM_DEFINE_CMPFAIL(5,
544 LONG_L
" %[len], %[rseq_scratch2]\n\t"
545 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
546 LONG_L
" %[src], %[rseq_scratch0]\n\t",
548 #ifdef RSEQ_COMPARE_TWICE
549 RSEQ_ASM_DEFINE_CMPFAIL(6,
550 LONG_L
" %[len], %[rseq_scratch2]\n\t"
551 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
552 LONG_L
" %[src], %[rseq_scratch0]\n\t",
554 RSEQ_ASM_DEFINE_CMPFAIL(7,
555 LONG_L
" %[len], %[rseq_scratch2]\n\t"
556 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
557 LONG_L
" %[src], %[rseq_scratch0]\n\t",
560 : /* gcc asm goto does not allow outputs */
561 : [cpu_id
] "r" (cpu
),
562 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
563 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
564 /* final store input */
566 [expect
] "r" (expect
),
568 /* try memcpy input */
572 [rseq_scratch0
] "m" (rseq_scratch
[0]),
573 [rseq_scratch1
] "m" (rseq_scratch
[1]),
574 [rseq_scratch2
] "m" (rseq_scratch
[2])
576 : "memory", "cc", "r0"
579 #ifdef RSEQ_COMPARE_TWICE
583 rseq_after_asm_goto();
586 rseq_after_asm_goto();
590 rseq_after_asm_goto();
592 #ifdef RSEQ_COMPARE_TWICE
594 rseq_after_asm_goto();
595 rseq_bug("cpu_id comparison failed");
597 rseq_after_asm_goto();
598 rseq_bug("expected value comparison failed");
603 static inline __attribute__((always_inline
))
604 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v
, intptr_t expect
,
605 void *dst
, void *src
, size_t len
,
606 intptr_t newv
, int cpu
)
608 return rseq_cmpeqv_trymemcpy_storev(v
, expect
, dst
, src
, len
,
612 #endif /* !RSEQ_SKIP_FASTPATH */
This page took 0.041573 seconds and 4 git commands to generate.