Commit | Line | Data |
---|---|---|
90702366 | 1 | /* SPDX-License-Identifier: MIT */ |
f2d7b530 MJ |
2 | /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ |
3 | /* SPDX-FileCopyrightText: 2016-2018 Boqun Feng <boqun.feng@gmail.com> */ | |
4 | ||
784b0012 MD |
5 | /* |
6 | * rseq-ppc.h | |
784b0012 MD |
7 | */ |
8 | ||
0158e666 MD |
9 | /* |
10 | * RSEQ_SIG is used with the following trap instruction: | |
11 | * | |
12 | * powerpc-be: 0f e5 00 0b twui r5,11 | |
13 | * powerpc64-le: 0b 00 e5 0f twui r5,11 | |
14 | * powerpc64-be: 0f e5 00 0b twui r5,11 | |
15 | */ | |
16 | ||
17 | #define RSEQ_SIG 0x0fe5000b | |
784b0012 MD |
18 | |
19 | #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc") | |
20 | #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc") | |
21 | #define rseq_smp_rmb() rseq_smp_lwsync() | |
22 | #define rseq_smp_wmb() rseq_smp_lwsync() | |
23 | ||
24 | #define rseq_smp_load_acquire(p) \ | |
25 | __extension__ ({ \ | |
3664a718 | 26 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
784b0012 MD |
27 | rseq_smp_lwsync(); \ |
28 | ____p1; \ | |
29 | }) | |
30 | ||
31 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync() | |
32 | ||
33 | #define rseq_smp_store_release(p, v) \ | |
34 | do { \ | |
35 | rseq_smp_lwsync(); \ | |
826417f6 | 36 | RSEQ_WRITE_ONCE(*(p), v); \ |
784b0012 MD |
37 | } while (0) |
38 | ||
784b0012 | 39 | /* |
dd01d0fb MD |
40 | * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to |
41 | * better handle single-stepping through the restartable critical sections. | |
784b0012 MD |
42 | */ |
43 | ||
44 | #ifdef __PPC64__ | |
45 | ||
a799a7ba MD |
46 | #define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ |
47 | #define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ | |
48 | #define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | |
49 | #define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | |
50 | #define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */ | |
51 | #define RSEQ_CMP_LONG "cmpd " | |
d0c21ef2 | 52 | #define RSEQ_CMP_LONG_INT "cmpdi " |
784b0012 MD |
53 | |
54 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
55 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 56 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
57 | ".balign 32\n\t" \ |
58 | __rseq_str(label) ":\n\t" \ | |
59 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
60 | ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
61 | ".popsection\n\t" \ |
62 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
63 | ".quad " __rseq_str(label) "b\n\t" \ | |
784b0012 MD |
64 | ".popsection\n\t" |
65 | ||
66 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ | |
67 | RSEQ_INJECT_ASM(1) \ | |
68 | "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \ | |
69 | "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \ | |
70 | "rldicr %%r17, %%r17, 32, 31\n\t" \ | |
71 | "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \ | |
72 | "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ | |
73 | "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ | |
74 | __rseq_str(label) ":\n\t" | |
75 | ||
90d9876e MD |
76 | /* |
77 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
78 | * of the critical section where a critical section can either branch to or |
79 | * reach through the normal course of its execution. The abort IP and the | |
80 | * post-commit IP are already part of the __rseq_cs section and should not be | |
81 | * explicitly defined as additional exit points. Knowing all exit points is | |
82 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
83 | */ |
84 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
85 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
86 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ | |
87 | ".popsection\n\t" | |
88 | ||
784b0012 MD |
89 | #else /* #ifdef __PPC64__ */ |
90 | ||
a799a7ba MD |
91 | #define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ |
92 | #define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */ | |
93 | #define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | |
94 | #define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */ | |
95 | #define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */ | |
96 | #define RSEQ_CMP_LONG "cmpw " | |
d0c21ef2 | 97 | #define RSEQ_CMP_LONG_INT "cmpwi " |
784b0012 MD |
98 | |
99 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
100 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 101 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
102 | ".balign 32\n\t" \ |
103 | __rseq_str(label) ":\n\t" \ | |
104 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
105 | /* 32-bit only supported on BE */ \ | |
106 | ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
107 | ".popsection\n\t" \ |
108 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
109 | ".long 0x0, " __rseq_str(label) "b\n\t" \ | |
784b0012 MD |
110 | ".popsection\n\t" |
111 | ||
90d9876e MD |
112 | /* |
113 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
114 | * of the critical section where a critical section can either branch to or |
115 | * reach through the normal course of its execution. The abort IP and the | |
116 | * post-commit IP are already part of the __rseq_cs section and should not be | |
117 | * explicitly defined as additional exit points. Knowing all exit points is | |
118 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
119 | */ |
120 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
121 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
122 | /* 32-bit only supported on BE */ \ | |
123 | ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ | |
124 | ".popsection\n\t" | |
125 | ||
784b0012 MD |
126 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
127 | RSEQ_INJECT_ASM(1) \ | |
128 | "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \ | |
129 | "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ | |
a799a7ba | 130 | RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ |
784b0012 MD |
131 | __rseq_str(label) ":\n\t" |
132 | ||
133 | #endif /* #ifdef __PPC64__ */ | |
134 | ||
135 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ | |
136 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
137 | (post_commit_ip - start_ip), abort_ip) | |
138 | ||
769ec9a5 | 139 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ |
784b0012 | 140 | RSEQ_INJECT_ASM(2) \ |
a799a7ba | 141 | RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ |
784b0012 MD |
142 | "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ |
143 | "bne- cr7, " __rseq_str(label) "\n\t" | |
144 | ||
ebd27573 | 145 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ |
784b0012 MD |
146 | ".pushsection __rseq_failure, \"ax\"\n\t" \ |
147 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ | |
148 | __rseq_str(label) ":\n\t" \ | |
ebd27573 | 149 | teardown \ |
784b0012 MD |
150 | "b %l[" __rseq_str(abort_label) "]\n\t" \ |
151 | ".popsection\n\t" | |
152 | ||
153 | /* | |
154 | * RSEQ_ASM_OPs: asm operations for rseq | |
155 | * RSEQ_ASM_OP_R_*: has hard-code registers in it | |
156 | * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7) | |
157 | */ | |
769ec9a5 | 158 | #define RSEQ_ASM_OP_CBNE(var, expect, label) \ |
a799a7ba MD |
159 | RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
160 | RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ | |
784b0012 MD |
161 | "bne- cr7, " __rseq_str(label) "\n\t" |
162 | ||
769ec9a5 | 163 | #define RSEQ_ASM_OP_CBEQ(var, expectnot, label) \ |
a799a7ba MD |
164 | RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
165 | RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ | |
784b0012 MD |
166 | "beq- cr7, " __rseq_str(label) "\n\t" |
167 | ||
168 | #define RSEQ_ASM_OP_STORE(value, var) \ | |
a799a7ba | 169 | RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" |
784b0012 MD |
170 | |
171 | /* Load @var to r17 */ | |
172 | #define RSEQ_ASM_OP_R_LOAD(var) \ | |
a799a7ba | 173 | RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" |
784b0012 MD |
174 | |
175 | /* Store r17 to @var */ | |
176 | #define RSEQ_ASM_OP_R_STORE(var) \ | |
a799a7ba | 177 | RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" |
784b0012 MD |
178 | |
179 | /* Add @count to r17 */ | |
180 | #define RSEQ_ASM_OP_R_ADD(count) \ | |
181 | "add %%r17, %[" __rseq_str(count) "], %%r17\n\t" | |
182 | ||
183 | /* Load (r17 + voffp) to r17 */ | |
184 | #define RSEQ_ASM_OP_R_LOADX(voffp) \ | |
a799a7ba | 185 | RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" |
784b0012 MD |
186 | |
187 | /* TODO: implement a faster memcpy. */ | |
188 | #define RSEQ_ASM_OP_R_MEMCPY() \ | |
d0c21ef2 | 189 | RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ |
784b0012 MD |
190 | "beq 333f\n\t" \ |
191 | "addi %%r20, %%r20, -1\n\t" \ | |
192 | "addi %%r21, %%r21, -1\n\t" \ | |
193 | "222:\n\t" \ | |
194 | "lbzu %%r18, 1(%%r20)\n\t" \ | |
195 | "stbu %%r18, 1(%%r21)\n\t" \ | |
196 | "addi %%r19, %%r19, -1\n\t" \ | |
d0c21ef2 | 197 | RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ |
784b0012 MD |
198 | "bne 222b\n\t" \ |
199 | "333:\n\t" \ | |
200 | ||
201 | #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ | |
a799a7ba | 202 | RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
784b0012 MD |
203 | __rseq_str(post_commit_label) ":\n\t" |
204 | ||
205 | #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ | |
a799a7ba | 206 | RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ |
784b0012 MD |
207 | __rseq_str(post_commit_label) ":\n\t" |
208 | ||
8fad586a MD |
209 | /* Per-cpu-id indexing. */ |
210 | ||
211 | #define RSEQ_TEMPLATE_CPU_ID | |
212 | #define RSEQ_TEMPLATE_MO_RELAXED | |
213 | #include "rseq-ppc-bits.h" | |
214 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
215 | ||
216 | #define RSEQ_TEMPLATE_MO_RELEASE | |
217 | #include "rseq-ppc-bits.h" | |
218 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
219 | #undef RSEQ_TEMPLATE_CPU_ID | |
220 | ||
221 | /* Per-mm-cid indexing. */ | |
222 | ||
223 | #define RSEQ_TEMPLATE_MM_CID | |
224 | #define RSEQ_TEMPLATE_MO_RELAXED | |
225 | #include "rseq-ppc-bits.h" | |
226 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
227 | ||
228 | #define RSEQ_TEMPLATE_MO_RELEASE | |
229 | #include "rseq-ppc-bits.h" | |
230 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
231 | #undef RSEQ_TEMPLATE_MM_CID | |
232 | ||
233 | /* APIs which are not based on cpu ids. */ | |
234 | ||
235 | #define RSEQ_TEMPLATE_CPU_ID_NONE | |
236 | #define RSEQ_TEMPLATE_MO_RELAXED | |
237 | #include "rseq-ppc-bits.h" | |
238 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
239 | #undef RSEQ_TEMPLATE_CPU_ID_NONE |