projects / librseq.git / blob
? search:


c97784bbced7bb172e2e8b01fbd60af81da9dbee
[librseq.git] / include / rseq / rseq-arm.h
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3
4 /*
5 * rseq-arm.h
6 */
7
8 /*
9 * - ARM little endian
10 *
11 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
12 * value 0x5de3. This traps if user-space reaches this instruction by mistake,
13 * and the uncommon operand ensures the kernel does not move the instruction
14 * pointer to attacker-controlled code on rseq abort.
15 *
16 * The instruction pattern in the A32 instruction set is:
17 *
18 * e7f5def3 udf #24035 ; 0x5de3
19 *
20 * This translates to the following instruction pattern in the T16 instruction
21 * set:
22 *
23 * little endian:
24 * def3 udf #243 ; 0xf3
25 * e7f5 b.n <7f5>
26 *
27 * - ARMv6+ big endian (BE8):
28 *
29 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
30 * code and big-endian data. The data value of the signature needs to have its
31 * byte order reversed to generate the trap instruction:
32 *
33 * Data: 0xf3def5e7
34 *
35 * Translates to this A32 instruction pattern:
36 *
37 * e7f5def3 udf #24035 ; 0x5de3
38 *
39 * Translates to this T16 instruction pattern:
40 *
41 * def3 udf #243 ; 0xf3
42 * e7f5 b.n <7f5>
43 *
44 * - Prior to ARMv6 big endian (BE32):
45 *
46 * Prior to ARMv6, -mbig-endian generates big-endian code and data
47 * (which match), so the endianness of the data representation of the
48 * signature should not be reversed. However, the choice between BE32
49 * and BE8 is done by the linker, so we cannot know whether code and
50 * data endianness will be mixed before the linker is invoked. So rather
51 * than try to play tricks with the linker, the rseq signature is simply
52 * data (not a trap instruction) prior to ARMv6 on big endian. This is
53 * why the signature is expressed as data (.word) rather than as
54 * instruction (.inst) in assembler.
55 */
56
57 #ifdef __ARMEB__
58 #define RSEQ_SIG 0xf3def5e7 /* udf #24035 ; 0x5de3 (ARMv6+) */
59 #else
60 #define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */
61 #endif
62
63 #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
64 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
65 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
66
67 #define rseq_smp_load_acquire(p) \
68 __extension__ ({ \
69 __typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
70 rseq_smp_mb(); \
71 ____p1; \
72 })
73
74 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
75
76 #define rseq_smp_store_release(p, v) \
77 do { \
78 rseq_smp_mb(); \
79 RSEQ_WRITE_ONCE(*(p), v); \
80 } while (0)
81
82 #ifdef RSEQ_SKIP_FASTPATH
83 #include "rseq-skip.h"
84 #else /* !RSEQ_SKIP_FASTPATH */
85
86 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
87 post_commit_offset, abort_ip) \
88 ".pushsection __rseq_cs, \"aw\"\n\t" \
89 ".balign 32\n\t" \
90 __rseq_str(label) ":\n\t" \
91 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
92 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
93 ".popsection\n\t" \
94 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
95 ".word " __rseq_str(label) "b, 0x0\n\t" \
96 ".popsection\n\t"
97
98 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
99 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
100 (post_commit_ip - start_ip), abort_ip)
101
102 /*
103 * Exit points of a rseq critical section consist of all instructions outside
104 * of the critical section where a critical section can either branch to or
105 * reach through the normal course of its execution. The abort IP and the
106 * post-commit IP are already part of the __rseq_cs section and should not be
107 * explicitly defined as additional exit points. Knowing all exit points is
108 * useful to assist debuggers stepping over the critical section.
109 */
110 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
111 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
112 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
113 ".popsection\n\t"
114
115 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
116 RSEQ_INJECT_ASM(1) \
117 "adr r0, " __rseq_str(cs_label) "\n\t" \
118 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \
119 __rseq_str(label) ":\n\t"
120
121 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
122 RSEQ_INJECT_ASM(2) \
123 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
124 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \
125 "bne " __rseq_str(label) "\n\t"
126
127 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
128 abort_label, version, flags, \
129 start_ip, post_commit_offset, abort_ip) \
130 ".balign 32\n\t" \
131 __rseq_str(table_label) ":\n\t" \
132 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
133 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
134 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
135 __rseq_str(label) ":\n\t" \
136 teardown \
137 "b %l[" __rseq_str(abort_label) "]\n\t"
138
139 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
140 start_ip, post_commit_ip, abort_ip) \
141 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
142 abort_label, 0x0, 0x0, start_ip, \
143 (post_commit_ip - start_ip), abort_ip)
144
145 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
146 __rseq_str(label) ":\n\t" \
147 teardown \
148 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
149
150 static inline __attribute__((always_inline))
151 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
152 {
153 RSEQ_INJECT_C(9)
154
155 __asm__ __volatile__ goto (
156 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
158 #ifdef RSEQ_COMPARE_TWICE
159 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
160 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
161 #endif
162 /* Start rseq by storing table entry pointer into rseq_cs. */
163 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
164 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
165 RSEQ_INJECT_ASM(3)
166 "ldr r0, %[v]\n\t"
167 "cmp %[expect], r0\n\t"
168 "bne %l[cmpfail]\n\t"
169 RSEQ_INJECT_ASM(4)
170 #ifdef RSEQ_COMPARE_TWICE
171 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
172 "ldr r0, %[v]\n\t"
173 "cmp %[expect], r0\n\t"
174 "bne %l[error2]\n\t"
175 #endif
176 /* final store */
177 "str %[newv], %[v]\n\t"
178 "2:\n\t"
179 RSEQ_INJECT_ASM(5)
180 "b 5f\n\t"
181 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
182 "5:\n\t"
183 : /* gcc asm goto does not allow outputs */
184 : [cpu_id] "r" (cpu),
185 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
186 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
187 [v] "m" (*v),
188 [expect] "r" (expect),
189 [newv] "r" (newv)
190 RSEQ_INJECT_INPUT
191 : "r0", "memory", "cc"
192 RSEQ_INJECT_CLOBBER
193 : abort, cmpfail
194 #ifdef RSEQ_COMPARE_TWICE
195 , error1, error2
196 #endif
197 );
198 rseq_after_asm_goto();
199 return 0;
200 abort:
201 rseq_after_asm_goto();
202 RSEQ_INJECT_FAILED
203 return -1;
204 cmpfail:
205 rseq_after_asm_goto();
206 return 1;
207 #ifdef RSEQ_COMPARE_TWICE
208 error1:
209 rseq_after_asm_goto();
210 rseq_bug("cpu_id comparison failed");
211 error2:
212 rseq_after_asm_goto();
213 rseq_bug("expected value comparison failed");
214 #endif
215 }
216
217 static inline __attribute__((always_inline))
218 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
219 long voffp, intptr_t *load, int cpu)
220 {
221 RSEQ_INJECT_C(9)
222
223 __asm__ __volatile__ goto (
224 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
225 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
226 #ifdef RSEQ_COMPARE_TWICE
227 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
228 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
229 #endif
230 /* Start rseq by storing table entry pointer into rseq_cs. */
231 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
233 RSEQ_INJECT_ASM(3)
234 "ldr r0, %[v]\n\t"
235 "cmp %[expectnot], r0\n\t"
236 "beq %l[cmpfail]\n\t"
237 RSEQ_INJECT_ASM(4)
238 #ifdef RSEQ_COMPARE_TWICE
239 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
240 "ldr r0, %[v]\n\t"
241 "cmp %[expectnot], r0\n\t"
242 "beq %l[error2]\n\t"
243 #endif
244 "str r0, %[load]\n\t"
245 "add r0, %[voffp]\n\t"
246 "ldr r0, [r0]\n\t"
247 /* final store */
248 "str r0, %[v]\n\t"
249 "2:\n\t"
250 RSEQ_INJECT_ASM(5)
251 "b 5f\n\t"
252 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
253 "5:\n\t"
254 : /* gcc asm goto does not allow outputs */
255 : [cpu_id] "r" (cpu),
256 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
257 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
258 /* final store input */
259 [v] "m" (*v),
260 [expectnot] "r" (expectnot),
261 [voffp] "Ir" (voffp),
262 [load] "m" (*load)
263 RSEQ_INJECT_INPUT
264 : "r0", "memory", "cc"
265 RSEQ_INJECT_CLOBBER
266 : abort, cmpfail
267 #ifdef RSEQ_COMPARE_TWICE
268 , error1, error2
269 #endif
270 );
271 rseq_after_asm_goto();
272 return 0;
273 abort:
274 rseq_after_asm_goto();
275 RSEQ_INJECT_FAILED
276 return -1;
277 cmpfail:
278 rseq_after_asm_goto();
279 return 1;
280 #ifdef RSEQ_COMPARE_TWICE
281 error1:
282 rseq_after_asm_goto();
283 rseq_bug("cpu_id comparison failed");
284 error2:
285 rseq_after_asm_goto();
286 rseq_bug("expected value comparison failed");
287 #endif
288 }
289
290 static inline __attribute__((always_inline))
291 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
292 {
293 RSEQ_INJECT_C(9)
294
295 __asm__ __volatile__ goto (
296 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
297 #ifdef RSEQ_COMPARE_TWICE
298 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
299 #endif
300 /* Start rseq by storing table entry pointer into rseq_cs. */
301 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
302 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
303 RSEQ_INJECT_ASM(3)
304 #ifdef RSEQ_COMPARE_TWICE
305 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
306 #endif
307 "ldr r0, %[v]\n\t"
308 "add r0, %[count]\n\t"
309 /* final store */
310 "str r0, %[v]\n\t"
311 "2:\n\t"
312 RSEQ_INJECT_ASM(4)
313 "b 5f\n\t"
314 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
315 "5:\n\t"
316 : /* gcc asm goto does not allow outputs */
317 : [cpu_id] "r" (cpu),
318 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
319 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
320 [v] "m" (*v),
321 [count] "Ir" (count)
322 RSEQ_INJECT_INPUT
323 : "r0", "memory", "cc"
324 RSEQ_INJECT_CLOBBER
325 : abort
326 #ifdef RSEQ_COMPARE_TWICE
327 , error1
328 #endif
329 );
330 rseq_after_asm_goto();
331 return 0;
332 abort:
333 rseq_after_asm_goto();
334 RSEQ_INJECT_FAILED
335 return -1;
336 #ifdef RSEQ_COMPARE_TWICE
337 error1:
338 rseq_after_asm_goto();
339 rseq_bug("cpu_id comparison failed");
340 #endif
341 }
342
343 static inline __attribute__((always_inline))
344 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
345 intptr_t *v2, intptr_t newv2,
346 intptr_t newv, int cpu)
347 {
348 RSEQ_INJECT_C(9)
349
350 __asm__ __volatile__ goto (
351 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
352 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
353 #ifdef RSEQ_COMPARE_TWICE
354 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
356 #endif
357 /* Start rseq by storing table entry pointer into rseq_cs. */
358 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
359 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
360 RSEQ_INJECT_ASM(3)
361 "ldr r0, %[v]\n\t"
362 "cmp %[expect], r0\n\t"
363 "bne %l[cmpfail]\n\t"
364 RSEQ_INJECT_ASM(4)
365 #ifdef RSEQ_COMPARE_TWICE
366 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
367 "ldr r0, %[v]\n\t"
368 "cmp %[expect], r0\n\t"
369 "bne %l[error2]\n\t"
370 #endif
371 /* try store */
372 "str %[newv2], %[v2]\n\t"
373 RSEQ_INJECT_ASM(5)
374 /* final store */
375 "str %[newv], %[v]\n\t"
376 "2:\n\t"
377 RSEQ_INJECT_ASM(6)
378 "b 5f\n\t"
379 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
380 "5:\n\t"
381 : /* gcc asm goto does not allow outputs */
382 : [cpu_id] "r" (cpu),
383 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
384 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
385 /* try store input */
386 [v2] "m" (*v2),
387 [newv2] "r" (newv2),
388 /* final store input */
389 [v] "m" (*v),
390 [expect] "r" (expect),
391 [newv] "r" (newv)
392 RSEQ_INJECT_INPUT
393 : "r0", "memory", "cc"
394 RSEQ_INJECT_CLOBBER
395 : abort, cmpfail
396 #ifdef RSEQ_COMPARE_TWICE
397 , error1, error2
398 #endif
399 );
400 rseq_after_asm_goto();
401 return 0;
402 abort:
403 rseq_after_asm_goto();
404 RSEQ_INJECT_FAILED
405 return -1;
406 cmpfail:
407 rseq_after_asm_goto();
408 return 1;
409 #ifdef RSEQ_COMPARE_TWICE
410 error1:
411 rseq_after_asm_goto();
412 rseq_bug("cpu_id comparison failed");
413 error2:
414 rseq_after_asm_goto();
415 rseq_bug("expected value comparison failed");
416 #endif
417 }
418
419 static inline __attribute__((always_inline))
420 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
421 intptr_t *v2, intptr_t newv2,
422 intptr_t newv, int cpu)
423 {
424 RSEQ_INJECT_C(9)
425
426 __asm__ __volatile__ goto (
427 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
428 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
429 #ifdef RSEQ_COMPARE_TWICE
430 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
431 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
432 #endif
433 /* Start rseq by storing table entry pointer into rseq_cs. */
434 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
435 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
436 RSEQ_INJECT_ASM(3)
437 "ldr r0, %[v]\n\t"
438 "cmp %[expect], r0\n\t"
439 "bne %l[cmpfail]\n\t"
440 RSEQ_INJECT_ASM(4)
441 #ifdef RSEQ_COMPARE_TWICE
442 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
443 "ldr r0, %[v]\n\t"
444 "cmp %[expect], r0\n\t"
445 "bne %l[error2]\n\t"
446 #endif
447 /* try store */
448 "str %[newv2], %[v2]\n\t"
449 RSEQ_INJECT_ASM(5)
450 "dmb\n\t" /* full mb provides store-release */
451 /* final store */
452 "str %[newv], %[v]\n\t"
453 "2:\n\t"
454 RSEQ_INJECT_ASM(6)
455 "b 5f\n\t"
456 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
457 "5:\n\t"
458 : /* gcc asm goto does not allow outputs */
459 : [cpu_id] "r" (cpu),
460 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
461 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
462 /* try store input */
463 [v2] "m" (*v2),
464 [newv2] "r" (newv2),
465 /* final store input */
466 [v] "m" (*v),
467 [expect] "r" (expect),
468 [newv] "r" (newv)
469 RSEQ_INJECT_INPUT
470 : "r0", "memory", "cc"
471 RSEQ_INJECT_CLOBBER
472 : abort, cmpfail
473 #ifdef RSEQ_COMPARE_TWICE
474 , error1, error2
475 #endif
476 );
477 rseq_after_asm_goto();
478 return 0;
479 abort:
480 rseq_after_asm_goto();
481 RSEQ_INJECT_FAILED
482 return -1;
483 cmpfail:
484 rseq_after_asm_goto();
485 return 1;
486 #ifdef RSEQ_COMPARE_TWICE
487 error1:
488 rseq_after_asm_goto();
489 rseq_bug("cpu_id comparison failed");
490 error2:
491 rseq_after_asm_goto();
492 rseq_bug("expected value comparison failed");
493 #endif
494 }
495
496 static inline __attribute__((always_inline))
497 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
498 intptr_t *v2, intptr_t expect2,
499 intptr_t newv, int cpu)
500 {
501 RSEQ_INJECT_C(9)
502
503 __asm__ __volatile__ goto (
504 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
505 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
506 #ifdef RSEQ_COMPARE_TWICE
507 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
508 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
509 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
510 #endif
511 /* Start rseq by storing table entry pointer into rseq_cs. */
512 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
513 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
514 RSEQ_INJECT_ASM(3)
515 "ldr r0, %[v]\n\t"
516 "cmp %[expect], r0\n\t"
517 "bne %l[cmpfail]\n\t"
518 RSEQ_INJECT_ASM(4)
519 "ldr r0, %[v2]\n\t"
520 "cmp %[expect2], r0\n\t"
521 "bne %l[cmpfail]\n\t"
522 RSEQ_INJECT_ASM(5)
523 #ifdef RSEQ_COMPARE_TWICE
524 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
525 "ldr r0, %[v]\n\t"
526 "cmp %[expect], r0\n\t"
527 "bne %l[error2]\n\t"
528 "ldr r0, %[v2]\n\t"
529 "cmp %[expect2], r0\n\t"
530 "bne %l[error3]\n\t"
531 #endif
532 /* final store */
533 "str %[newv], %[v]\n\t"
534 "2:\n\t"
535 RSEQ_INJECT_ASM(6)
536 "b 5f\n\t"
537 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
538 "5:\n\t"
539 : /* gcc asm goto does not allow outputs */
540 : [cpu_id] "r" (cpu),
541 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
542 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
543 /* cmp2 input */
544 [v2] "m" (*v2),
545 [expect2] "r" (expect2),
546 /* final store input */
547 [v] "m" (*v),
548 [expect] "r" (expect),
549 [newv] "r" (newv)
550 RSEQ_INJECT_INPUT
551 : "r0", "memory", "cc"
552 RSEQ_INJECT_CLOBBER
553 : abort, cmpfail
554 #ifdef RSEQ_COMPARE_TWICE
555 , error1, error2, error3
556 #endif
557 );
558 rseq_after_asm_goto();
559 return 0;
560 abort:
561 rseq_after_asm_goto();
562 RSEQ_INJECT_FAILED
563 return -1;
564 cmpfail:
565 rseq_after_asm_goto();
566 return 1;
567 #ifdef RSEQ_COMPARE_TWICE
568 error1:
569 rseq_after_asm_goto();
570 rseq_bug("cpu_id comparison failed");
571 error2:
572 rseq_after_asm_goto();
573 rseq_bug("1st expected value comparison failed");
574 error3:
575 rseq_after_asm_goto();
576 rseq_bug("2nd expected value comparison failed");
577 #endif
578 }
579
580 static inline __attribute__((always_inline))
581 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
582 void *dst, void *src, size_t len,
583 intptr_t newv, int cpu)
584 {
585 uint32_t rseq_scratch[3];
586
587 RSEQ_INJECT_C(9)
588
589 __asm__ __volatile__ goto (
590 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
591 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
592 #ifdef RSEQ_COMPARE_TWICE
593 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
594 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
595 #endif
596 "str %[src], %[rseq_scratch0]\n\t"
597 "str %[dst], %[rseq_scratch1]\n\t"
598 "str %[len], %[rseq_scratch2]\n\t"
599 /* Start rseq by storing table entry pointer into rseq_cs. */
600 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
601 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
602 RSEQ_INJECT_ASM(3)
603 "ldr r0, %[v]\n\t"
604 "cmp %[expect], r0\n\t"
605 "bne 5f\n\t"
606 RSEQ_INJECT_ASM(4)
607 #ifdef RSEQ_COMPARE_TWICE
608 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
609 "ldr r0, %[v]\n\t"
610 "cmp %[expect], r0\n\t"
611 "bne 7f\n\t"
612 #endif
613 /* try memcpy */
614 "cmp %[len], #0\n\t" \
615 "beq 333f\n\t" \
616 "222:\n\t" \
617 "ldrb %%r0, [%[src]]\n\t" \
618 "strb %%r0, [%[dst]]\n\t" \
619 "adds %[src], #1\n\t" \
620 "adds %[dst], #1\n\t" \
621 "subs %[len], #1\n\t" \
622 "bne 222b\n\t" \
623 "333:\n\t" \
624 RSEQ_INJECT_ASM(5)
625 /* final store */
626 "str %[newv], %[v]\n\t"
627 "2:\n\t"
628 RSEQ_INJECT_ASM(6)
629 /* teardown */
630 "ldr %[len], %[rseq_scratch2]\n\t"
631 "ldr %[dst], %[rseq_scratch1]\n\t"
632 "ldr %[src], %[rseq_scratch0]\n\t"
633 "b 8f\n\t"
634 RSEQ_ASM_DEFINE_ABORT(3, 4,
635 /* teardown */
636 "ldr %[len], %[rseq_scratch2]\n\t"
637 "ldr %[dst], %[rseq_scratch1]\n\t"
638 "ldr %[src], %[rseq_scratch0]\n\t",
639 abort, 1b, 2b, 4f)
640 RSEQ_ASM_DEFINE_CMPFAIL(5,
641 /* teardown */
642 "ldr %[len], %[rseq_scratch2]\n\t"
643 "ldr %[dst], %[rseq_scratch1]\n\t"
644 "ldr %[src], %[rseq_scratch0]\n\t",
645 cmpfail)
646 #ifdef RSEQ_COMPARE_TWICE
647 RSEQ_ASM_DEFINE_CMPFAIL(6,
648 /* teardown */
649 "ldr %[len], %[rseq_scratch2]\n\t"
650 "ldr %[dst], %[rseq_scratch1]\n\t"
651 "ldr %[src], %[rseq_scratch0]\n\t",
652 error1)
653 RSEQ_ASM_DEFINE_CMPFAIL(7,
654 /* teardown */
655 "ldr %[len], %[rseq_scratch2]\n\t"
656 "ldr %[dst], %[rseq_scratch1]\n\t"
657 "ldr %[src], %[rseq_scratch0]\n\t",
658 error2)
659 #endif
660 "8:\n\t"
661 : /* gcc asm goto does not allow outputs */
662 : [cpu_id] "r" (cpu),
663 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
664 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
665 /* final store input */
666 [v] "m" (*v),
667 [expect] "r" (expect),
668 [newv] "r" (newv),
669 /* try memcpy input */
670 [dst] "r" (dst),
671 [src] "r" (src),
672 [len] "r" (len),
673 [rseq_scratch0] "m" (rseq_scratch[0]),
674 [rseq_scratch1] "m" (rseq_scratch[1]),
675 [rseq_scratch2] "m" (rseq_scratch[2])
676 RSEQ_INJECT_INPUT
677 : "r0", "memory", "cc"
678 RSEQ_INJECT_CLOBBER
679 : abort, cmpfail
680 #ifdef RSEQ_COMPARE_TWICE
681 , error1, error2
682 #endif
683 );
684 rseq_after_asm_goto();
685 return 0;
686 abort:
687 rseq_after_asm_goto();
688 RSEQ_INJECT_FAILED
689 return -1;
690 cmpfail:
691 rseq_after_asm_goto();
692 return 1;
693 #ifdef RSEQ_COMPARE_TWICE
694 error1:
695 rseq_after_asm_goto();
696 rseq_bug("cpu_id comparison failed");
697 error2:
698 rseq_after_asm_goto();
699 rseq_bug("expected value comparison failed");
700 #endif
701 }
702
703 static inline __attribute__((always_inline))
704 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
705 void *dst, void *src, size_t len,
706 intptr_t newv, int cpu)
707 {
708 uint32_t rseq_scratch[3];
709
710 RSEQ_INJECT_C(9)
711
712 __asm__ __volatile__ goto (
713 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
714 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
715 #ifdef RSEQ_COMPARE_TWICE
716 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
717 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
718 #endif
719 "str %[src], %[rseq_scratch0]\n\t"
720 "str %[dst], %[rseq_scratch1]\n\t"
721 "str %[len], %[rseq_scratch2]\n\t"
722 /* Start rseq by storing table entry pointer into rseq_cs. */
723 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
724 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
725 RSEQ_INJECT_ASM(3)
726 "ldr r0, %[v]\n\t"
727 "cmp %[expect], r0\n\t"
728 "bne 5f\n\t"
729 RSEQ_INJECT_ASM(4)
730 #ifdef RSEQ_COMPARE_TWICE
731 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
732 "ldr r0, %[v]\n\t"
733 "cmp %[expect], r0\n\t"
734 "bne 7f\n\t"
735 #endif
736 /* try memcpy */
737 "cmp %[len], #0\n\t" \
738 "beq 333f\n\t" \
739 "222:\n\t" \
740 "ldrb %%r0, [%[src]]\n\t" \
741 "strb %%r0, [%[dst]]\n\t" \
742 "adds %[src], #1\n\t" \
743 "adds %[dst], #1\n\t" \
744 "subs %[len], #1\n\t" \
745 "bne 222b\n\t" \
746 "333:\n\t" \
747 RSEQ_INJECT_ASM(5)
748 "dmb\n\t" /* full mb provides store-release */
749 /* final store */
750 "str %[newv], %[v]\n\t"
751 "2:\n\t"
752 RSEQ_INJECT_ASM(6)
753 /* teardown */
754 "ldr %[len], %[rseq_scratch2]\n\t"
755 "ldr %[dst], %[rseq_scratch1]\n\t"
756 "ldr %[src], %[rseq_scratch0]\n\t"
757 "b 8f\n\t"
758 RSEQ_ASM_DEFINE_ABORT(3, 4,
759 /* teardown */
760 "ldr %[len], %[rseq_scratch2]\n\t"
761 "ldr %[dst], %[rseq_scratch1]\n\t"
762 "ldr %[src], %[rseq_scratch0]\n\t",
763 abort, 1b, 2b, 4f)
764 RSEQ_ASM_DEFINE_CMPFAIL(5,
765 /* teardown */
766 "ldr %[len], %[rseq_scratch2]\n\t"
767 "ldr %[dst], %[rseq_scratch1]\n\t"
768 "ldr %[src], %[rseq_scratch0]\n\t",
769 cmpfail)
770 #ifdef RSEQ_COMPARE_TWICE
771 RSEQ_ASM_DEFINE_CMPFAIL(6,
772 /* teardown */
773 "ldr %[len], %[rseq_scratch2]\n\t"
774 "ldr %[dst], %[rseq_scratch1]\n\t"
775 "ldr %[src], %[rseq_scratch0]\n\t",
776 error1)
777 RSEQ_ASM_DEFINE_CMPFAIL(7,
778 /* teardown */
779 "ldr %[len], %[rseq_scratch2]\n\t"
780 "ldr %[dst], %[rseq_scratch1]\n\t"
781 "ldr %[src], %[rseq_scratch0]\n\t",
782 error2)
783 #endif
784 "8:\n\t"
785 : /* gcc asm goto does not allow outputs */
786 : [cpu_id] "r" (cpu),
787 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
788 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
789 /* final store input */
790 [v] "m" (*v),
791 [expect] "r" (expect),
792 [newv] "r" (newv),
793 /* try memcpy input */
794 [dst] "r" (dst),
795 [src] "r" (src),
796 [len] "r" (len),
797 [rseq_scratch0] "m" (rseq_scratch[0]),
798 [rseq_scratch1] "m" (rseq_scratch[1]),
799 [rseq_scratch2] "m" (rseq_scratch[2])
800 RSEQ_INJECT_INPUT
801 : "r0", "memory", "cc"
802 RSEQ_INJECT_CLOBBER
803 : abort, cmpfail
804 #ifdef RSEQ_COMPARE_TWICE
805 , error1, error2
806 #endif
807 );
808 rseq_after_asm_goto();
809 return 0;
810 abort:
811 rseq_after_asm_goto();
812 RSEQ_INJECT_FAILED
813 return -1;
814 cmpfail:
815 rseq_after_asm_goto();
816 return 1;
817 #ifdef RSEQ_COMPARE_TWICE
818 error1:
819 rseq_after_asm_goto();
820 rseq_bug("cpu_id comparison failed");
821 error2:
822 rseq_after_asm_goto();
823 rseq_bug("expected value comparison failed");
824 #endif
825 }
826
827 #endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.047637 seconds and 3 git commands to generate.