Commit | Line | Data |
---|---|---|
90702366 | 1 | /* SPDX-License-Identifier: MIT */ |
5ad4f704 | 2 | /* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ |
f2d7b530 MJ |
3 | /* SPDX-FileCopyrightText: 2016-2018 Boqun Feng <boqun.feng@gmail.com> */ |
4 | ||
784b0012 | 5 | /* |
44ec21eb | 6 | * rseq/arch/ppc.h |
784b0012 MD |
7 | */ |
8 | ||
44ec21eb MJ |
9 | #ifndef _RSEQ_RSEQ_H |
10 | #error "Never use <rseq/arch/ppc.h> directly; include <rseq/rseq.h> instead." | |
11 | #endif | |
12 | ||
5ad4f704 MD |
13 | /* |
14 | * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those | |
15 | * are not part of the public API. | |
16 | */ | |
17 | ||
0158e666 MD |
18 | /* |
19 | * RSEQ_SIG is used with the following trap instruction: | |
20 | * | |
21 | * powerpc-be: 0f e5 00 0b twui r5,11 | |
22 | * powerpc64-le: 0b 00 e5 0f twui r5,11 | |
23 | * powerpc64-be: 0f e5 00 0b twui r5,11 | |
24 | */ | |
25 | ||
26 | #define RSEQ_SIG 0x0fe5000b | |
784b0012 | 27 | |
5ad4f704 MD |
28 | /* |
29 | * Refer to the Linux kernel memory model (LKMM) for documentation of | |
30 | * the memory barriers. | |
31 | */ | |
784b0012 | 32 | |
5ad4f704 MD |
33 | /* CPU memory barrier. */ |
34 | #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc") | |
35 | /* Only used internally in this header. */ | |
36 | #define __rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc") | |
37 | /* CPU read memory barrier */ | |
38 | #define rseq_smp_rmb() __rseq_smp_lwsync() | |
39 | /* CPU write memory barrier */ | |
40 | #define rseq_smp_wmb() __rseq_smp_lwsync() | |
41 | ||
42 | /* Acquire: One-way permeable barrier. */ | |
784b0012 MD |
43 | #define rseq_smp_load_acquire(p) \ |
44 | __extension__ ({ \ | |
3664a718 | 45 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
5ad4f704 | 46 | __rseq_smp_lwsync(); \ |
784b0012 MD |
47 | ____p1; \ |
48 | }) | |
49 | ||
5ad4f704 MD |
50 | /* Acquire barrier after control dependency. */ |
51 | #define rseq_smp_acquire__after_ctrl_dep() __rseq_smp_lwsync() | |
784b0012 | 52 | |
5ad4f704 | 53 | /* Release: One-way permeable barrier. */ |
784b0012 MD |
54 | #define rseq_smp_store_release(p, v) \ |
55 | do { \ | |
5ad4f704 | 56 | __rseq_smp_lwsync(); \ |
826417f6 | 57 | RSEQ_WRITE_ONCE(*(p), v); \ |
784b0012 MD |
58 | } while (0) |
59 | ||
f6b00247 MD |
60 | /* |
61 | * Helper macros to define and access a variable of long integer type. | |
62 | * Only used internally in rseq headers. | |
63 | */ | |
44ec21eb | 64 | #ifdef RSEQ_ARCH_PPC64 |
f6b00247 MD |
65 | # define RSEQ_ASM_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ |
66 | # define RSEQ_ASM_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ | |
67 | # define RSEQ_ASM_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | |
68 | # define RSEQ_ASM_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | |
69 | # define RSEQ_ASM_LOADX_LONG "ldx " /* From base register ("b" constraint) */ | |
70 | # define RSEQ_ASM_CMP_LONG "cmpd " /* Register-to-register comparison */ | |
71 | # define RSEQ_ASM_CMP_LONG_INT "cmpdi " /* Register-to-immediate comparison */ | |
72 | #else | |
73 | # define RSEQ_ASM_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ | |
74 | # define RSEQ_ASM_STORE_INT(arg) RSEQ_ASM_STORE_LONG(arg) /* To memory ("m" constraint) */ | |
75 | # define RSEQ_ASM_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ | |
76 | # define RSEQ_ASM_LOAD_INT(arg) RSEQ_ASM_LOAD_LONG(arg) /* From memory ("m" constraint) */ | |
77 | # define RSEQ_ASM_LOADX_LONG "lwzx " /* From base register ("b" constraint) */ | |
78 | # define RSEQ_ASM_CMP_LONG "cmpw " /* Register-to-register comparison */ | |
79 | # define RSEQ_ASM_CMP_LONG_INT "cmpwi " /* Register-to-immediate comparison */ | |
80 | #endif | |
784b0012 | 81 | |
f6b00247 MD |
82 | /* |
83 | * Helper macros to define a variable of pointer type stored in a 64-bit | |
84 | * integer. Only used internally in rseq headers. | |
85 | */ | |
86 | #ifdef RSEQ_ARCH_PPC64 | |
87 | # define RSEQ_ASM_U64_PTR(x) ".quad " x | |
88 | #else | |
89 | /* 32-bit only supported on big endian. */ | |
90 | # define RSEQ_ASM_U64_PTR(x) ".long 0x0, " x | |
91 | #endif | |
784b0012 | 92 | |
d68304d1 MD |
93 | #define RSEQ_ASM_U32(x) ".long " x |
94 | ||
ed21bf6d MD |
95 | /* Common architecture support macros. */ |
96 | #include "rseq/arch/generic/common.h" | |
97 | ||
98 | /* | |
99 | * Define a critical section abort handler. | |
100 | * | |
101 | * @label: | |
102 | * Local label to the abort handler. | |
103 | * @teardown: | |
104 | * Sequence of instructions to run on abort. | |
105 | * @abort_label: | |
106 | * C label to jump to at the end of the sequence. | |
107 | */ | |
108 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
109 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
110 | RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n\t" \ | |
111 | __rseq_str(label) ":\n\t" \ | |
112 | teardown \ | |
113 | "b %l[" __rseq_str(abort_label) "]\n\t" \ | |
114 | ".popsection\n\t" | |
115 | ||
5ad4f704 MD |
116 | /* |
117 | * Store the address of the critical section descriptor structure at | |
118 | * @cs_label into the @rseq_cs pointer and emit the label @label, which | |
119 | * is the beginning of the sequence of consecutive assembly instructions. | |
120 | * | |
121 | * @label: | |
122 | * Local label to the beginning of the sequence of consecutive assembly | |
123 | * instructions. | |
124 | * @cs_label: | |
125 | * Source local label to the critical section descriptor structure. | |
126 | * @rseq_cs: | |
127 | * Destination pointer where to store the address of the critical | |
128 | * section descriptor structure. | |
129 | */ | |
f6b00247 MD |
130 | #ifdef RSEQ_ARCH_PPC64 |
131 | # define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ | |
784b0012 MD |
132 | RSEQ_INJECT_ASM(1) \ |
133 | "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \ | |
134 | "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \ | |
135 | "rldicr %%r17, %%r17, 32, 31\n\t" \ | |
136 | "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \ | |
137 | "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ | |
138 | "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ | |
139 | __rseq_str(label) ":\n\t" | |
f6b00247 MD |
140 | #else |
141 | # define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ | |
142 | RSEQ_INJECT_ASM(1) \ | |
143 | "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \ | |
144 | "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ | |
145 | RSEQ_ASM_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ | |
146 | __rseq_str(label) ":\n\t" | |
147 | #endif | |
784b0012 | 148 | |
5ad4f704 MD |
149 | /* Jump to local label @label when @cpu_id != @current_cpu_id. */ |
150 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ | |
151 | RSEQ_INJECT_ASM(2) \ | |
152 | RSEQ_ASM_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ | |
153 | "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ | |
154 | "bne- cr7, " __rseq_str(label) "\n\t" | |
155 | ||
784b0012 | 156 | /* |
5ad4f704 MD |
157 | * RSEQ_ASM_OPs: asm operations for rseq. Only used internally by rseq headers. |
158 | * RSEQ_ASM_OP_R_*: has hard-coded registers in it | |
159 | * RSEQ_ASM_OP_* (else): doesn't have hard-coded registers(unless cr7) | |
784b0012 | 160 | */ |
5ad4f704 MD |
161 | |
162 | /* Jump to local label @label when @var != @expect. */ | |
769ec9a5 | 163 | #define RSEQ_ASM_OP_CBNE(var, expect, label) \ |
d1d63a2d MD |
164 | RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
165 | RSEQ_ASM_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ | |
784b0012 MD |
166 | "bne- cr7, " __rseq_str(label) "\n\t" |
167 | ||
5ad4f704 MD |
168 | /* Jump to local label @label when @var == @expect. */ |
169 | #define RSEQ_ASM_OP_CBEQ(var, expect, label) \ | |
d1d63a2d | 170 | RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
5ad4f704 | 171 | RSEQ_ASM_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ |
784b0012 MD |
172 | "beq- cr7, " __rseq_str(label) "\n\t" |
173 | ||
5ad4f704 | 174 | /* Store @value to address @var. */ |
784b0012 | 175 | #define RSEQ_ASM_OP_STORE(value, var) \ |
d1d63a2d | 176 | RSEQ_ASM_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" |
784b0012 MD |
177 | |
178 | /* Load @var to r17 */ | |
179 | #define RSEQ_ASM_OP_R_LOAD(var) \ | |
d1d63a2d | 180 | RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" |
784b0012 MD |
181 | |
182 | /* Store r17 to @var */ | |
183 | #define RSEQ_ASM_OP_R_STORE(var) \ | |
d1d63a2d | 184 | RSEQ_ASM_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" |
784b0012 MD |
185 | |
186 | /* Add @count to r17 */ | |
187 | #define RSEQ_ASM_OP_R_ADD(count) \ | |
188 | "add %%r17, %[" __rseq_str(count) "], %%r17\n\t" | |
189 | ||
190 | /* Load (r17 + voffp) to r17 */ | |
191 | #define RSEQ_ASM_OP_R_LOADX(voffp) \ | |
d1d63a2d | 192 | RSEQ_ASM_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" |
784b0012 | 193 | |
5ad4f704 MD |
194 | /* |
195 | * Copy @len bytes from @src to @dst. This is an inefficient bytewise | |
196 | * copy and could be improved in the future. | |
197 | */ | |
08130179 | 198 | #define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY() \ |
d1d63a2d | 199 | RSEQ_ASM_CMP_LONG_INT "%%r19, 0\n\t" \ |
784b0012 MD |
200 | "beq 333f\n\t" \ |
201 | "addi %%r20, %%r20, -1\n\t" \ | |
202 | "addi %%r21, %%r21, -1\n\t" \ | |
203 | "222:\n\t" \ | |
204 | "lbzu %%r18, 1(%%r20)\n\t" \ | |
205 | "stbu %%r18, 1(%%r21)\n\t" \ | |
206 | "addi %%r19, %%r19, -1\n\t" \ | |
d1d63a2d | 207 | RSEQ_ASM_CMP_LONG_INT "%%r19, 0\n\t" \ |
784b0012 MD |
208 | "bne 222b\n\t" \ |
209 | "333:\n\t" \ | |
210 | ||
5ad4f704 MD |
211 | /* |
212 | * End-of-sequence store of r17 to address @var. Emit | |
213 | * @post_commit_label label after the store instruction. | |
214 | */ | |
784b0012 | 215 | #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ |
d1d63a2d | 216 | RSEQ_ASM_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ |
784b0012 MD |
217 | __rseq_str(post_commit_label) ":\n\t" |
218 | ||
5ad4f704 MD |
219 | /* |
220 | * End-of-sequence store of @value to address @var. Emit | |
221 | * @post_commit_label label after the store instruction. | |
222 | */ | |
784b0012 | 223 | #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ |
d1d63a2d | 224 | RSEQ_ASM_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ |
784b0012 MD |
225 | __rseq_str(post_commit_label) ":\n\t" |
226 | ||
8fad586a MD |
227 | /* Per-cpu-id indexing. */ |
228 | ||
abf9e855 | 229 | #define RSEQ_TEMPLATE_INDEX_CPU_ID |
8fad586a | 230 | #define RSEQ_TEMPLATE_MO_RELAXED |
44ec21eb | 231 | #include "rseq/arch/ppc/bits.h" |
8fad586a MD |
232 | #undef RSEQ_TEMPLATE_MO_RELAXED |
233 | ||
234 | #define RSEQ_TEMPLATE_MO_RELEASE | |
44ec21eb | 235 | #include "rseq/arch/ppc/bits.h" |
8fad586a | 236 | #undef RSEQ_TEMPLATE_MO_RELEASE |
abf9e855 | 237 | #undef RSEQ_TEMPLATE_INDEX_CPU_ID |
8fad586a MD |
238 | |
239 | /* Per-mm-cid indexing. */ | |
240 | ||
abf9e855 | 241 | #define RSEQ_TEMPLATE_INDEX_MM_CID |
8fad586a | 242 | #define RSEQ_TEMPLATE_MO_RELAXED |
44ec21eb | 243 | #include "rseq/arch/ppc/bits.h" |
8fad586a MD |
244 | #undef RSEQ_TEMPLATE_MO_RELAXED |
245 | ||
246 | #define RSEQ_TEMPLATE_MO_RELEASE | |
44ec21eb | 247 | #include "rseq/arch/ppc/bits.h" |
8fad586a | 248 | #undef RSEQ_TEMPLATE_MO_RELEASE |
abf9e855 | 249 | #undef RSEQ_TEMPLATE_INDEX_MM_CID |
8fad586a | 250 | |
abf9e855 | 251 | /* APIs which are not indexed. */ |
8fad586a | 252 | |
abf9e855 | 253 | #define RSEQ_TEMPLATE_INDEX_NONE |
8fad586a | 254 | #define RSEQ_TEMPLATE_MO_RELAXED |
44ec21eb | 255 | #include "rseq/arch/ppc/bits.h" |
8fad586a | 256 | #undef RSEQ_TEMPLATE_MO_RELAXED |
abf9e855 | 257 | #undef RSEQ_TEMPLATE_INDEX_NONE |