1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
4 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
5 * access-register mode nor the linkage stack this instruction will always
6 * cause a special-operation exception (the trap-enabled bit in the DUCT
7 * is and will stay 0). The instruction pattern is
8 * b2 ff 0f ff trap4 4095(%r0)
10 #define RSEQ_SIG 0xB2FF0FFF
12 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
13 #define rseq_smp_rmb() rseq_smp_mb()
14 #define rseq_smp_wmb() rseq_smp_mb()
16 #define rseq_smp_load_acquire(p) \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
25 #define rseq_smp_store_release(p, v) \
28 RSEQ_WRITE_ONCE(*p, v); \
31 #ifdef RSEQ_SKIP_FASTPATH
32 #include "rseq-skip.h"
33 #else /* !RSEQ_SKIP_FASTPATH */
37 #define RSEQ_CS_PTR rseq_cs.ptr
41 #define LONG_LT_R "ltgr"
43 #define LONG_CMP_R "cgr"
44 #define LONG_ADDI "aghi"
45 #define LONG_ADD_R "agr"
47 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
48 start_ip, post_commit_offset, abort_ip) \
49 ".pushsection __rseq_cs, \"aw\"\n\t" \
51 __rseq_str(label) ":\n\t" \
52 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
53 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
55 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
56 ".quad " __rseq_str(label) "b\n\t" \
60 * Exit points of a rseq critical section consist of all instructions outside
61 * of the critical section where a critical section can either branch to or
62 * reach through the normal course of its execution. The abort IP and the
63 * post-commit IP are already part of the __rseq_cs section and should not be
64 * explicitly defined as additional exit points. Knowing all exit points is
65 * useful to assist debuggers stepping over the critical section.
67 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
68 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
69 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
74 #define RSEQ_CS_PTR RSEQ_CS_PTR32
76 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
77 start_ip, post_commit_offset, abort_ip) \
78 ".pushsection __rseq_cs, \"aw\"\n\t" \
80 __rseq_str(label) ":\n\t" \
81 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
82 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
84 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
85 ".long 0x0, " __rseq_str(label) "b\n\t" \
89 * Exit points of a rseq critical section consist of all instructions outside
90 * of the critical section where a critical section can either branch to or
91 * reach through the normal course of its execution. The abort IP and the
92 * post-commit IP are already part of the __rseq_cs section and should not be
93 * explicitly defined as additional exit points. Knowing all exit points is
94 * useful to assist debuggers stepping over the critical section.
96 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
97 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
98 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
103 #define LONG_LT_R "ltr"
105 #define LONG_CMP_R "cr"
106 #define LONG_ADDI "ahi"
107 #define LONG_ADD_R "ar"
111 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
112 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
113 (post_commit_ip - start_ip), abort_ip)
115 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
117 "larl %%r0, " __rseq_str(cs_label) "\n\t" \
118 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
119 __rseq_str(label) ":\n\t"
121 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
123 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
124 "jnz " __rseq_str(label) "\n\t"
126 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
127 ".pushsection __rseq_failure, \"ax\"\n\t" \
128 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
129 __rseq_str(label) ":\n\t" \
131 "jg %l[" __rseq_str(abort_label) "]\n\t" \
134 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
135 ".pushsection __rseq_failure, \"ax\"\n\t" \
136 __rseq_str(label) ":\n\t" \
138 "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
141 static inline __attribute__((always_inline
))
142 int rseq_cmpeqv_storev(intptr_t *v
, intptr_t expect
, intptr_t newv
, int cpu
)
146 __asm__ __volatile__
goto (
147 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
148 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
149 #ifdef RSEQ_COMPARE_TWICE
150 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
151 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
153 /* Start rseq by storing table entry pointer into rseq_cs. */
154 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
155 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
157 LONG_CMP
" %[expect], %[v]\n\t"
158 "jnz %l[cmpfail]\n\t"
160 #ifdef RSEQ_COMPARE_TWICE
161 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
162 LONG_CMP
" %[expect], %[v]\n\t"
166 LONG_S
" %[newv], %[v]\n\t"
169 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
170 : /* gcc asm goto does not allow outputs */
171 : [cpu_id
] "r" (cpu
),
172 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
173 [rseq_cs
] "m" (rseq_get_abi()->RSEQ_CS_PTR
),
175 [expect
] "r" (expect
),
178 : "memory", "cc", "r0"
181 #ifdef RSEQ_COMPARE_TWICE
185 rseq_after_asm_goto();
188 rseq_after_asm_goto();
192 rseq_after_asm_goto();
194 #ifdef RSEQ_COMPARE_TWICE
196 rseq_after_asm_goto();
197 rseq_bug("cpu_id comparison failed");
199 rseq_after_asm_goto();
200 rseq_bug("expected value comparison failed");
205 * Compare @v against @expectnot. When it does _not_ match, load @v
206 * into @load, and store the content of *@v + voffp into @v.
208 static inline __attribute__((always_inline
))
209 int rseq_cmpnev_storeoffp_load(intptr_t *v
, intptr_t expectnot
,
210 long voffp
, intptr_t *load
, int cpu
)
214 __asm__ __volatile__
goto (
215 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
216 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
217 #ifdef RSEQ_COMPARE_TWICE
218 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
221 /* Start rseq by storing table entry pointer into rseq_cs. */
222 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
223 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
225 LONG_L
" %%r1, %[v]\n\t"
226 LONG_CMP_R
" %%r1, %[expectnot]\n\t"
229 #ifdef RSEQ_COMPARE_TWICE
230 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
231 LONG_L
" %%r1, %[v]\n\t"
232 LONG_CMP_R
" %%r1, %[expectnot]\n\t"
235 LONG_S
" %%r1, %[load]\n\t"
236 LONG_ADD_R
" %%r1, %[voffp]\n\t"
237 LONG_L
" %%r1, 0(%%r1)\n\t"
239 LONG_S
" %%r1, %[v]\n\t"
242 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
243 : /* gcc asm goto does not allow outputs */
244 : [cpu_id
] "r" (cpu
),
245 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
246 [rseq_cs
] "m" (rseq_get_abi()->RSEQ_CS_PTR
),
247 /* final store input */
249 [expectnot
] "r" (expectnot
),
253 : "memory", "cc", "r0", "r1"
256 #ifdef RSEQ_COMPARE_TWICE
260 rseq_after_asm_goto();
263 rseq_after_asm_goto();
267 rseq_after_asm_goto();
269 #ifdef RSEQ_COMPARE_TWICE
271 rseq_after_asm_goto();
272 rseq_bug("cpu_id comparison failed");
274 rseq_after_asm_goto();
275 rseq_bug("expected value comparison failed");
279 static inline __attribute__((always_inline
))
280 int rseq_addv(intptr_t *v
, intptr_t count
, int cpu
)
284 __asm__ __volatile__
goto (
285 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
286 #ifdef RSEQ_COMPARE_TWICE
287 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
289 /* Start rseq by storing table entry pointer into rseq_cs. */
290 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
291 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
293 #ifdef RSEQ_COMPARE_TWICE
294 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
296 LONG_L
" %%r0, %[v]\n\t"
297 LONG_ADD_R
" %%r0, %[count]\n\t"
299 LONG_S
" %%r0, %[v]\n\t"
302 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
303 : /* gcc asm goto does not allow outputs */
304 : [cpu_id
] "r" (cpu
),
305 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
306 [rseq_cs
] "m" (rseq_get_abi()->RSEQ_CS_PTR
),
307 /* final store input */
311 : "memory", "cc", "r0"
314 #ifdef RSEQ_COMPARE_TWICE
318 rseq_after_asm_goto();
321 rseq_after_asm_goto();
324 #ifdef RSEQ_COMPARE_TWICE
326 rseq_after_asm_goto();
327 rseq_bug("cpu_id comparison failed");
331 static inline __attribute__((always_inline
))
332 int rseq_cmpeqv_trystorev_storev(intptr_t *v
, intptr_t expect
,
333 intptr_t *v2
, intptr_t newv2
,
334 intptr_t newv
, int cpu
)
338 __asm__ __volatile__
goto (
339 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
340 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
341 #ifdef RSEQ_COMPARE_TWICE
342 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
343 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
345 /* Start rseq by storing table entry pointer into rseq_cs. */
346 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
347 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
349 LONG_CMP
" %[expect], %[v]\n\t"
350 "jnz %l[cmpfail]\n\t"
352 #ifdef RSEQ_COMPARE_TWICE
353 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
354 LONG_CMP
" %[expect], %[v]\n\t"
358 LONG_S
" %[newv2], %[v2]\n\t"
361 LONG_S
" %[newv], %[v]\n\t"
364 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
365 : /* gcc asm goto does not allow outputs */
366 : [cpu_id
] "r" (cpu
),
367 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
368 [rseq_cs
] "m" (rseq_get_abi()->RSEQ_CS_PTR
),
369 /* try store input */
372 /* final store input */
374 [expect
] "r" (expect
),
377 : "memory", "cc", "r0"
380 #ifdef RSEQ_COMPARE_TWICE
384 rseq_after_asm_goto();
387 rseq_after_asm_goto();
391 rseq_after_asm_goto();
393 #ifdef RSEQ_COMPARE_TWICE
395 rseq_after_asm_goto();
396 rseq_bug("cpu_id comparison failed");
398 rseq_after_asm_goto();
399 rseq_bug("expected value comparison failed");
404 static inline __attribute__((always_inline
))
405 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v
, intptr_t expect
,
406 intptr_t *v2
, intptr_t newv2
,
407 intptr_t newv
, int cpu
)
409 return rseq_cmpeqv_trystorev_storev(v
, expect
, v2
, newv2
, newv
, cpu
);
412 static inline __attribute__((always_inline
))
413 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v
, intptr_t expect
,
414 intptr_t *v2
, intptr_t expect2
,
415 intptr_t newv
, int cpu
)
419 __asm__ __volatile__
goto (
420 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
421 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
422 #ifdef RSEQ_COMPARE_TWICE
423 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
424 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
425 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error3
])
427 /* Start rseq by storing table entry pointer into rseq_cs. */
428 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
429 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
431 LONG_CMP
" %[expect], %[v]\n\t"
432 "jnz %l[cmpfail]\n\t"
434 LONG_CMP
" %[expect2], %[v2]\n\t"
435 "jnz %l[cmpfail]\n\t"
437 #ifdef RSEQ_COMPARE_TWICE
438 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
439 LONG_CMP
" %[expect], %[v]\n\t"
441 LONG_CMP
" %[expect2], %[v2]\n\t"
445 LONG_S
" %[newv], %[v]\n\t"
448 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
449 : /* gcc asm goto does not allow outputs */
450 : [cpu_id
] "r" (cpu
),
451 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
452 [rseq_cs
] "m" (rseq_get_abi()->RSEQ_CS_PTR
),
455 [expect2
] "r" (expect2
),
456 /* final store input */
458 [expect
] "r" (expect
),
461 : "memory", "cc", "r0"
464 #ifdef RSEQ_COMPARE_TWICE
465 , error1
, error2
, error3
468 rseq_after_asm_goto();
471 rseq_after_asm_goto();
475 rseq_after_asm_goto();
477 #ifdef RSEQ_COMPARE_TWICE
479 rseq_after_asm_goto();
480 rseq_bug("cpu_id comparison failed");
482 rseq_after_asm_goto();
483 rseq_bug("1st expected value comparison failed");
485 rseq_after_asm_goto();
486 rseq_bug("2nd expected value comparison failed");
490 static inline __attribute__((always_inline
))
491 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v
, intptr_t expect
,
492 void *dst
, void *src
, size_t len
,
493 intptr_t newv
, int cpu
)
495 uint64_t rseq_scratch
[3];
499 __asm__ __volatile__
goto (
500 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
501 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
502 #ifdef RSEQ_COMPARE_TWICE
503 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
504 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
506 LONG_S
" %[src], %[rseq_scratch0]\n\t"
507 LONG_S
" %[dst], %[rseq_scratch1]\n\t"
508 LONG_S
" %[len], %[rseq_scratch2]\n\t"
509 /* Start rseq by storing table entry pointer into rseq_cs. */
510 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
511 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
513 LONG_CMP
" %[expect], %[v]\n\t"
516 #ifdef RSEQ_COMPARE_TWICE
517 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 6f
)
518 LONG_CMP
" %[expect], %[v]\n\t"
522 LONG_LT_R
" %[len], %[len]\n\t"
525 "ic %%r0,0(%[src])\n\t"
526 "stc %%r0,0(%[dst])\n\t"
527 LONG_ADDI
" %[src], 1\n\t"
528 LONG_ADDI
" %[dst], 1\n\t"
529 LONG_ADDI
" %[len], -1\n\t"
534 LONG_S
" %[newv], %[v]\n\t"
538 LONG_L
" %[len], %[rseq_scratch2]\n\t"
539 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
540 LONG_L
" %[src], %[rseq_scratch0]\n\t"
541 RSEQ_ASM_DEFINE_ABORT(4,
542 LONG_L
" %[len], %[rseq_scratch2]\n\t"
543 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
544 LONG_L
" %[src], %[rseq_scratch0]\n\t",
546 RSEQ_ASM_DEFINE_CMPFAIL(5,
547 LONG_L
" %[len], %[rseq_scratch2]\n\t"
548 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
549 LONG_L
" %[src], %[rseq_scratch0]\n\t",
551 #ifdef RSEQ_COMPARE_TWICE
552 RSEQ_ASM_DEFINE_CMPFAIL(6,
553 LONG_L
" %[len], %[rseq_scratch2]\n\t"
554 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
555 LONG_L
" %[src], %[rseq_scratch0]\n\t",
557 RSEQ_ASM_DEFINE_CMPFAIL(7,
558 LONG_L
" %[len], %[rseq_scratch2]\n\t"
559 LONG_L
" %[dst], %[rseq_scratch1]\n\t"
560 LONG_L
" %[src], %[rseq_scratch0]\n\t",
563 : /* gcc asm goto does not allow outputs */
564 : [cpu_id
] "r" (cpu
),
565 [current_cpu_id
] "m" (rseq_get_abi()->cpu_id
),
566 [rseq_cs
] "m" (rseq_get_abi()->RSEQ_CS_PTR
),
567 /* final store input */
569 [expect
] "r" (expect
),
571 /* try memcpy input */
575 [rseq_scratch0
] "m" (rseq_scratch
[0]),
576 [rseq_scratch1
] "m" (rseq_scratch
[1]),
577 [rseq_scratch2
] "m" (rseq_scratch
[2])
579 : "memory", "cc", "r0"
582 #ifdef RSEQ_COMPARE_TWICE
586 rseq_after_asm_goto();
589 rseq_after_asm_goto();
593 rseq_after_asm_goto();
595 #ifdef RSEQ_COMPARE_TWICE
597 rseq_after_asm_goto();
598 rseq_bug("cpu_id comparison failed");
600 rseq_after_asm_goto();
601 rseq_bug("expected value comparison failed");
606 static inline __attribute__((always_inline
))
607 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v
, intptr_t expect
,
608 void *dst
, void *src
, size_t len
,
609 intptr_t newv
, int cpu
)
611 return rseq_cmpeqv_trymemcpy_storev(v
, expect
, dst
, src
, len
,
615 #endif /* !RSEQ_SKIP_FASTPATH */
This page took 0.041747 seconds and 5 git commands to generate.