Commit | Line | Data |
---|---|---|
90702366 | 1 | /* SPDX-License-Identifier: MIT */ |
f2d7b530 | 2 | /* SPDX-FileCopyrightText: 2018 Vasily Gorbik <gor@linux.ibm.com> */ |
4969e3fa | 3 | |
9c15f6a7 MS |
4 | /* |
5 | * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the | |
6 | * access-register mode nor the linkage stack this instruction will always | |
7 | * cause a special-operation exception (the trap-enabled bit in the DUCT | |
8 | * is and will stay 0). The instruction pattern is | |
f3b2cb14 | 9 | * b2 ff 0f ff trap4 4095(%r0) |
9c15f6a7 MS |
10 | */ |
11 | #define RSEQ_SIG 0xB2FF0FFF | |
4969e3fa MD |
12 | |
13 | #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory") | |
14 | #define rseq_smp_rmb() rseq_smp_mb() | |
15 | #define rseq_smp_wmb() rseq_smp_mb() | |
16 | ||
17 | #define rseq_smp_load_acquire(p) \ | |
18 | __extension__ ({ \ | |
3664a718 | 19 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
4969e3fa MD |
20 | rseq_barrier(); \ |
21 | ____p1; \ | |
22 | }) | |
23 | ||
24 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
25 | ||
26 | #define rseq_smp_store_release(p, v) \ | |
27 | do { \ | |
28 | rseq_barrier(); \ | |
826417f6 | 29 | RSEQ_WRITE_ONCE(*(p), v); \ |
4969e3fa MD |
30 | } while (0) |
31 | ||
4969e3fa MD |
32 | #ifdef __s390x__ |
33 | ||
34 | #define LONG_L "lg" | |
35 | #define LONG_S "stg" | |
36 | #define LONG_LT_R "ltgr" | |
37 | #define LONG_CMP "cg" | |
38 | #define LONG_CMP_R "cgr" | |
39 | #define LONG_ADDI "aghi" | |
40 | #define LONG_ADD_R "agr" | |
41 | ||
42 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
43 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 44 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
4969e3fa MD |
45 | ".balign 32\n\t" \ |
46 | __rseq_str(label) ":\n\t" \ | |
47 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
48 | ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
49 | ".popsection\n\t" \ |
50 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
51 | ".quad " __rseq_str(label) "b\n\t" \ | |
4969e3fa MD |
52 | ".popsection\n\t" |
53 | ||
90d9876e MD |
54 | /* |
55 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
56 | * of the critical section where a critical section can either branch to or |
57 | * reach through the normal course of its execution. The abort IP and the | |
58 | * post-commit IP are already part of the __rseq_cs section and should not be | |
59 | * explicitly defined as additional exit points. Knowing all exit points is | |
60 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
61 | */ |
62 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
63 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
64 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ | |
65 | ".popsection\n\t" | |
66 | ||
4969e3fa MD |
67 | #elif __s390__ |
68 | ||
69 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
70 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 71 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
4969e3fa MD |
72 | ".balign 32\n\t" \ |
73 | __rseq_str(label) ":\n\t" \ | |
74 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
75 | ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
76 | ".popsection\n\t" \ |
77 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
78 | ".long 0x0, " __rseq_str(label) "b\n\t" \ | |
4969e3fa MD |
79 | ".popsection\n\t" |
80 | ||
90d9876e MD |
81 | /* |
82 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
83 | * of the critical section where a critical section can either branch to or |
84 | * reach through the normal course of its execution. The abort IP and the | |
85 | * post-commit IP are already part of the __rseq_cs section and should not be | |
86 | * explicitly defined as additional exit points. Knowing all exit points is | |
87 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
88 | */ |
89 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
90 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
91 | ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ | |
92 | ".popsection\n\t" | |
93 | ||
4969e3fa MD |
94 | #define LONG_L "l" |
95 | #define LONG_S "st" | |
96 | #define LONG_LT_R "ltr" | |
97 | #define LONG_CMP "c" | |
98 | #define LONG_CMP_R "cr" | |
99 | #define LONG_ADDI "ahi" | |
100 | #define LONG_ADD_R "ar" | |
101 | ||
102 | #endif | |
103 | ||
104 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ | |
105 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
106 | (post_commit_ip - start_ip), abort_ip) | |
107 | ||
108 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ | |
109 | RSEQ_INJECT_ASM(1) \ | |
110 | "larl %%r0, " __rseq_str(cs_label) "\n\t" \ | |
111 | LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \ | |
112 | __rseq_str(label) ":\n\t" | |
113 | ||
769ec9a5 | 114 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ |
4969e3fa MD |
115 | RSEQ_INJECT_ASM(2) \ |
116 | "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ | |
117 | "jnz " __rseq_str(label) "\n\t" | |
118 | ||
119 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
120 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
121 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ | |
122 | __rseq_str(label) ":\n\t" \ | |
123 | teardown \ | |
90b9a2c8 | 124 | "jg %l[" __rseq_str(abort_label) "]\n\t" \ |
4969e3fa MD |
125 | ".popsection\n\t" |
126 | ||
559d824f | 127 | #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \ |
4969e3fa MD |
128 | ".pushsection __rseq_failure, \"ax\"\n\t" \ |
129 | __rseq_str(label) ":\n\t" \ | |
130 | teardown \ | |
559d824f | 131 | "jg %l[" __rseq_str(target_label) "]\n\t" \ |
4969e3fa MD |
132 | ".popsection\n\t" |
133 | ||
f3b2cb14 | 134 | /* Per-cpu-id indexing. */ |
4969e3fa | 135 | |
abf9e855 | 136 | #define RSEQ_TEMPLATE_INDEX_CPU_ID |
f3b2cb14 MD |
137 | #define RSEQ_TEMPLATE_MO_RELAXED |
138 | #include "rseq-s390-bits.h" | |
139 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
4969e3fa | 140 | |
f3b2cb14 MD |
141 | #define RSEQ_TEMPLATE_MO_RELEASE |
142 | #include "rseq-s390-bits.h" | |
143 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
abf9e855 | 144 | #undef RSEQ_TEMPLATE_INDEX_CPU_ID |
4969e3fa | 145 | |
f3b2cb14 | 146 | /* Per-mm-cid indexing. */ |
4969e3fa | 147 | |
abf9e855 | 148 | #define RSEQ_TEMPLATE_INDEX_MM_CID |
f3b2cb14 MD |
149 | #define RSEQ_TEMPLATE_MO_RELAXED |
150 | #include "rseq-s390-bits.h" | |
151 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
4969e3fa | 152 | |
f3b2cb14 MD |
153 | #define RSEQ_TEMPLATE_MO_RELEASE |
154 | #include "rseq-s390-bits.h" | |
155 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
abf9e855 | 156 | #undef RSEQ_TEMPLATE_INDEX_MM_CID |
4969e3fa | 157 | |
abf9e855 | 158 | /* APIs which are not indexed. */ |
4969e3fa | 159 | |
abf9e855 | 160 | #define RSEQ_TEMPLATE_INDEX_NONE |
f3b2cb14 MD |
161 | #define RSEQ_TEMPLATE_MO_RELAXED |
162 | #include "rseq-s390-bits.h" | |
163 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
abf9e855 | 164 | #undef RSEQ_TEMPLATE_INDEX_NONE |