projects / librseq.git / blob
? search:


6f01a1ce01cadaf15713163aea4092e7ea0c58d9
[librseq.git] / include / rseq / rseq-arm64-bits.h
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3 /* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
4
5 /*
6 * rseq-arm64-bits.h
7 */
8
9 #include "rseq-bits-template.h"
10
11 /*
12 * Refer to rseq-pseudocode.h for documentation and pseudo-code of the
13 * rseq critical section helpers.
14 */
15 #include "rseq-pseudocode.h"
16
17 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
18 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
19
20 static inline __attribute__((always_inline))
21 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store__ptr)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
22 {
23 RSEQ_INJECT_C(9)
24
25 __asm__ __volatile__ goto (
26 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
27 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[ne])
28 #ifdef RSEQ_COMPARE_TWICE
29 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
30 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
31 #endif
32 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
33 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
34 RSEQ_INJECT_ASM(3)
35 RSEQ_ASM_OP_CBNE(v, expect, %l[ne])
36 RSEQ_INJECT_ASM(4)
37 #ifdef RSEQ_COMPARE_TWICE
38 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
39 RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
40 #endif
41 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
42 RSEQ_INJECT_ASM(5)
43 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
44 : /* gcc asm goto does not allow outputs */
45 : [cpu_id] "r" (cpu),
46 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
48 [v] "Qo" (*v),
49 [expect] "r" (expect),
50 [newv] "r" (newv)
51 RSEQ_INJECT_INPUT
52 : "memory", RSEQ_ASM_TMP_REG
53 : abort, ne
54 #ifdef RSEQ_COMPARE_TWICE
55 , error1, error2
56 #endif
57 );
58 rseq_after_asm_goto();
59 return 0;
60 abort:
61 rseq_after_asm_goto();
62 RSEQ_INJECT_FAILED
63 return -1;
64 ne:
65 rseq_after_asm_goto();
66 return 1;
67 #ifdef RSEQ_COMPARE_TWICE
68 error1:
69 rseq_after_asm_goto();
70 rseq_bug("cpu_id comparison failed");
71 error2:
72 rseq_after_asm_goto();
73 rseq_bug("expected value comparison failed");
74 #endif
75 }
76
77 static inline __attribute__((always_inline))
78 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbeq_store_add_load_store__ptr)(intptr_t *v, intptr_t expectnot,
79 long voffp, intptr_t *load, int cpu)
80 {
81 RSEQ_INJECT_C(9)
82
83 __asm__ __volatile__ goto (
84 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
85 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[eq])
86 #ifdef RSEQ_COMPARE_TWICE
87 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
88 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
89 #endif
90 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
91 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
92 RSEQ_INJECT_ASM(3)
93 RSEQ_ASM_OP_CBEQ(v, expectnot, %l[eq])
94 RSEQ_INJECT_ASM(4)
95 #ifdef RSEQ_COMPARE_TWICE
96 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
97 RSEQ_ASM_OP_CBEQ(v, expectnot, %l[error2])
98 #endif
99 RSEQ_ASM_OP_R_LOAD(v)
100 RSEQ_ASM_OP_R_STORE(load)
101 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
102 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
103 RSEQ_INJECT_ASM(5)
104 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
105 : /* gcc asm goto does not allow outputs */
106 : [cpu_id] "r" (cpu),
107 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
108 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
109 [v] "Qo" (*v),
110 [expectnot] "r" (expectnot),
111 [load] "Qo" (*load),
112 [voffp] "r" (voffp)
113 RSEQ_INJECT_INPUT
114 : "memory", RSEQ_ASM_TMP_REG
115 : abort, eq
116 #ifdef RSEQ_COMPARE_TWICE
117 , error1, error2
118 #endif
119 );
120 rseq_after_asm_goto();
121 return 0;
122 abort:
123 rseq_after_asm_goto();
124 RSEQ_INJECT_FAILED
125 return -1;
126 eq:
127 rseq_after_asm_goto();
128 return 1;
129 #ifdef RSEQ_COMPARE_TWICE
130 error1:
131 rseq_after_asm_goto();
132 rseq_bug("cpu_id comparison failed");
133 error2:
134 rseq_after_asm_goto();
135 rseq_bug("expected value comparison failed");
136 #endif
137 }
138
139 static inline __attribute__((always_inline))
140 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_add_store__ptr)(intptr_t *v, intptr_t count, int cpu)
141 {
142 RSEQ_INJECT_C(9)
143
144 __asm__ __volatile__ goto (
145 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
146 #ifdef RSEQ_COMPARE_TWICE
147 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
148 #endif
149 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
150 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
151 RSEQ_INJECT_ASM(3)
152 #ifdef RSEQ_COMPARE_TWICE
153 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
154 #endif
155 RSEQ_ASM_OP_R_LOAD(v)
156 RSEQ_ASM_OP_R_ADD(count)
157 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
158 RSEQ_INJECT_ASM(4)
159 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
160 : /* gcc asm goto does not allow outputs */
161 : [cpu_id] "r" (cpu),
162 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
163 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
164 [v] "Qo" (*v),
165 [count] "r" (count)
166 RSEQ_INJECT_INPUT
167 : "memory", RSEQ_ASM_TMP_REG
168 : abort
169 #ifdef RSEQ_COMPARE_TWICE
170 , error1
171 #endif
172 );
173 rseq_after_asm_goto();
174 return 0;
175 abort:
176 rseq_after_asm_goto();
177 RSEQ_INJECT_FAILED
178 return -1;
179 #ifdef RSEQ_COMPARE_TWICE
180 error1:
181 rseq_after_asm_goto();
182 rseq_bug("cpu_id comparison failed");
183 #endif
184 }
185
186 static inline __attribute__((always_inline))
187 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_load_cbne_store__ptr)(intptr_t *v, intptr_t expect,
188 intptr_t *v2, intptr_t expect2,
189 intptr_t newv, int cpu)
190 {
191 RSEQ_INJECT_C(9)
192
193 __asm__ __volatile__ goto (
194 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
195 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[ne])
196 #ifdef RSEQ_COMPARE_TWICE
197 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
198 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
199 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
200 #endif
201 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
202 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
203 RSEQ_INJECT_ASM(3)
204 RSEQ_ASM_OP_CBNE(v, expect, %l[ne])
205 RSEQ_INJECT_ASM(4)
206 RSEQ_ASM_OP_CBNE(v2, expect2, %l[ne])
207 RSEQ_INJECT_ASM(5)
208 #ifdef RSEQ_COMPARE_TWICE
209 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
210 RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
211 RSEQ_ASM_OP_CBNE(v2, expect2, %l[error3])
212 #endif
213 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
214 RSEQ_INJECT_ASM(6)
215 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
216 : /* gcc asm goto does not allow outputs */
217 : [cpu_id] "r" (cpu),
218 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
219 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
220 [v] "Qo" (*v),
221 [expect] "r" (expect),
222 [v2] "Qo" (*v2),
223 [expect2] "r" (expect2),
224 [newv] "r" (newv)
225 RSEQ_INJECT_INPUT
226 : "memory", RSEQ_ASM_TMP_REG
227 : abort, ne
228 #ifdef RSEQ_COMPARE_TWICE
229 , error1, error2, error3
230 #endif
231 );
232 rseq_after_asm_goto();
233 return 0;
234 abort:
235 rseq_after_asm_goto();
236 RSEQ_INJECT_FAILED
237 return -1;
238 ne:
239 rseq_after_asm_goto();
240 return 1;
241 #ifdef RSEQ_COMPARE_TWICE
242 error1:
243 rseq_after_asm_goto();
244 rseq_bug("cpu_id comparison failed");
245 error2:
246 rseq_after_asm_goto();
247 rseq_bug("expected value comparison failed");
248 error3:
249 rseq_after_asm_goto();
250 rseq_bug("2nd expected value comparison failed");
251 #endif
252 }
253
254 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
255 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
256
257 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
258 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
259
260 static inline __attribute__((always_inline))
261 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store_store__ptr)(intptr_t *v, intptr_t expect,
262 intptr_t *v2, intptr_t newv2,
263 intptr_t newv, int cpu)
264 {
265 RSEQ_INJECT_C(9)
266
267 __asm__ __volatile__ goto (
268 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
269 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[ne])
270 #ifdef RSEQ_COMPARE_TWICE
271 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
272 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
273 #endif
274 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
275 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
276 RSEQ_INJECT_ASM(3)
277 RSEQ_ASM_OP_CBNE(v, expect, %l[ne])
278 RSEQ_INJECT_ASM(4)
279 #ifdef RSEQ_COMPARE_TWICE
280 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
281 RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
282 #endif
283 RSEQ_ASM_OP_STORE(newv2, v2)
284 RSEQ_INJECT_ASM(5)
285 #ifdef RSEQ_TEMPLATE_MO_RELEASE
286 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
287 #else
288 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
289 #endif
290 RSEQ_INJECT_ASM(6)
291 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
292 : /* gcc asm goto does not allow outputs */
293 : [cpu_id] "r" (cpu),
294 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
295 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
296 [expect] "r" (expect),
297 [v] "Qo" (*v),
298 [newv] "r" (newv),
299 [v2] "Qo" (*v2),
300 [newv2] "r" (newv2)
301 RSEQ_INJECT_INPUT
302 : "memory", RSEQ_ASM_TMP_REG
303 : abort, ne
304 #ifdef RSEQ_COMPARE_TWICE
305 , error1, error2
306 #endif
307 );
308 rseq_after_asm_goto();
309 return 0;
310 abort:
311 rseq_after_asm_goto();
312 RSEQ_INJECT_FAILED
313 return -1;
314 ne:
315 rseq_after_asm_goto();
316 return 1;
317 #ifdef RSEQ_COMPARE_TWICE
318 error1:
319 rseq_after_asm_goto();
320 rseq_bug("cpu_id comparison failed");
321 error2:
322 rseq_after_asm_goto();
323 rseq_bug("expected value comparison failed");
324 #endif
325 }
326
327 static inline __attribute__((always_inline))
328 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intptr_t expect,
329 void *dst, void *src, size_t len,
330 intptr_t newv, int cpu)
331 {
332 RSEQ_INJECT_C(9)
333
334 __asm__ __volatile__ goto (
335 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
336 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[ne])
337 #ifdef RSEQ_COMPARE_TWICE
338 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
339 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
340 #endif
341 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
342 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
343 RSEQ_INJECT_ASM(3)
344 RSEQ_ASM_OP_CBNE(v, expect, %l[ne])
345 RSEQ_INJECT_ASM(4)
346 #ifdef RSEQ_COMPARE_TWICE
347 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
348 RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
349 #endif
350 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
351 RSEQ_INJECT_ASM(5)
352 #ifdef RSEQ_TEMPLATE_MO_RELEASE
353 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
354 #else
355 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
356 #endif
357 RSEQ_INJECT_ASM(6)
358 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
359 : /* gcc asm goto does not allow outputs */
360 : [cpu_id] "r" (cpu),
361 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
362 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
363 [expect] "r" (expect),
364 [v] "Qo" (*v),
365 [newv] "r" (newv),
366 [dst] "r" (dst),
367 [src] "r" (src),
368 [len] "r" (len)
369 RSEQ_INJECT_INPUT
370 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
371 : abort, ne
372 #ifdef RSEQ_COMPARE_TWICE
373 , error1, error2
374 #endif
375 );
376 rseq_after_asm_goto();
377 return 0;
378 abort:
379 rseq_after_asm_goto();
380 RSEQ_INJECT_FAILED
381 return -1;
382 ne:
383 rseq_after_asm_goto();
384 return 1;
385 #ifdef RSEQ_COMPARE_TWICE
386 error1:
387 rseq_after_asm_goto();
388 rseq_bug("cpu_id comparison failed");
389 error2:
390 rseq_after_asm_goto();
391 rseq_bug("expected value comparison failed");
392 #endif
393 }
394
395 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
396 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
397
398 #include "rseq-bits-reset.h"
Librseq
This page took 0.051178 seconds and 3 git commands to generate.