projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
All arch: use rseq_unqual_scalar_typeof in load-acquire
[librseq.git] / include / rseq / rseq-mips.h
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2018 MIPS Tech LLC */
3 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
4
5 /*
6 * Author: Paul Burton <paul.burton@mips.com>
7 */
8
9 /*
10 * RSEQ_SIG uses the break instruction. The instruction pattern is:
11 *
12 * On MIPS:
13 * 0350000d break 0x350
14 *
15 * On nanoMIPS:
16 * 00100350 break 0x350
17 *
18 * On microMIPS:
19 * 0000d407 break 0x350
20 *
21 * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit
22 * halfwords, so the signature halfwords need to be swapped accordingly for
23 * little-endian.
24 */
25 #if defined(__nanomips__)
26 # ifdef __MIPSEL__
27 # define RSEQ_SIG 0x03500010
28 # else
29 # define RSEQ_SIG 0x00100350
30 # endif
31 #elif defined(__mips_micromips)
32 # ifdef __MIPSEL__
33 # define RSEQ_SIG 0xd4070000
34 # else
35 # define RSEQ_SIG 0x0000d407
36 # endif
37 #elif defined(__mips__)
38 # define RSEQ_SIG 0x0350000d
39 #else
40 /* Unknown MIPS architecture. */
41 #endif
42
43 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory")
44 #define rseq_smp_rmb() rseq_smp_mb()
45 #define rseq_smp_wmb() rseq_smp_mb()
46
47 #define rseq_smp_load_acquire(p) \
48 __extension__ ({ \
49 rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
50 rseq_smp_mb(); \
51 ____p1; \
52 })
53
54 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
55
56 #define rseq_smp_store_release(p, v) \
57 do { \
58 rseq_smp_mb(); \
59 RSEQ_WRITE_ONCE(*(p), v); \
60 } while (0)
61
62 #ifdef RSEQ_SKIP_FASTPATH
63 #include "rseq-skip.h"
64 #else /* !RSEQ_SKIP_FASTPATH */
65
66 #if _MIPS_SZLONG == 64
67 # define LONG ".dword"
68 # define LONG_LA "dla"
69 # define LONG_L "ld"
70 # define LONG_S "sd"
71 # define LONG_ADDI "daddiu"
72 # define U32_U64_PAD(x) x
73 #elif _MIPS_SZLONG == 32
74 # define LONG ".word"
75 # define LONG_LA "la"
76 # define LONG_L "lw"
77 # define LONG_S "sw"
78 # define LONG_ADDI "addiu"
79 # ifdef __BIG_ENDIAN
80 # define U32_U64_PAD(x) "0x0, " x
81 # else
82 # define U32_U64_PAD(x) x ", 0x0"
83 # endif
84 #else
85 # error unsupported _MIPS_SZLONG
86 #endif
87
88 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
89 post_commit_offset, abort_ip) \
90 ".pushsection __rseq_cs, \"aw\"\n\t" \
91 ".balign 32\n\t" \
92 __rseq_str(label) ":\n\t" \
93 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
94 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
95 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
96 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
97 ".popsection\n\t" \
98 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
99 LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \
100 ".popsection\n\t"
101
102 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
103 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
104 (post_commit_ip - start_ip), abort_ip)
105
106 /*
107 * Exit points of a rseq critical section consist of all instructions outside
108 * of the critical section where a critical section can either branch to or
109 * reach through the normal course of its execution. The abort IP and the
110 * post-commit IP are already part of the __rseq_cs section and should not be
111 * explicitly defined as additional exit points. Knowing all exit points is
112 * useful to assist debuggers stepping over the critical section.
113 */
114 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
115 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
116 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
117 LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \
118 ".popsection\n\t"
119
120 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
121 RSEQ_INJECT_ASM(1) \
122 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
123 LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
124 __rseq_str(label) ":\n\t"
125
126 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
127 RSEQ_INJECT_ASM(2) \
128 "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
129 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
130
131 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
132 abort_label, version, flags, \
133 start_ip, post_commit_offset, abort_ip) \
134 ".balign 32\n\t" \
135 __rseq_str(table_label) ":\n\t" \
136 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
137 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
138 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
139 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
140 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
141 __rseq_str(label) ":\n\t" \
142 teardown \
143 "b %l[" __rseq_str(abort_label) "]\n\t"
144
145 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
146 start_ip, post_commit_ip, abort_ip) \
147 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
148 abort_label, 0x0, 0x0, start_ip, \
149 (post_commit_ip - start_ip), abort_ip)
150
151 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
152 __rseq_str(label) ":\n\t" \
153 teardown \
154 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
155
156 static inline __attribute__((always_inline))
157 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
158 {
159 RSEQ_INJECT_C(9)
160
161 __asm__ __volatile__ goto (
162 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
163 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
164 #ifdef RSEQ_COMPARE_TWICE
165 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
166 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
167 #endif
168 /* Start rseq by storing table entry pointer into rseq_cs. */
169 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
170 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
171 RSEQ_INJECT_ASM(3)
172 LONG_L " $4, %[v]\n\t"
173 "bne $4, %[expect], %l[cmpfail]\n\t"
174 RSEQ_INJECT_ASM(4)
175 #ifdef RSEQ_COMPARE_TWICE
176 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
177 LONG_L " $4, %[v]\n\t"
178 "bne $4, %[expect], %l[error2]\n\t"
179 #endif
180 /* final store */
181 LONG_S " %[newv], %[v]\n\t"
182 "2:\n\t"
183 RSEQ_INJECT_ASM(5)
184 "b 5f\n\t"
185 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
186 "5:\n\t"
187 : /* gcc asm goto does not allow outputs */
188 : [cpu_id] "r" (cpu),
189 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
190 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
191 [v] "m" (*v),
192 [expect] "r" (expect),
193 [newv] "r" (newv)
194 RSEQ_INJECT_INPUT
195 : "$4", "memory"
196 RSEQ_INJECT_CLOBBER
197 : abort, cmpfail
198 #ifdef RSEQ_COMPARE_TWICE
199 , error1, error2
200 #endif
201 );
202 return 0;
203 abort:
204 RSEQ_INJECT_FAILED
205 return -1;
206 cmpfail:
207 return 1;
208 #ifdef RSEQ_COMPARE_TWICE
209 error1:
210 rseq_bug("cpu_id comparison failed");
211 error2:
212 rseq_bug("expected value comparison failed");
213 #endif
214 }
215
216 static inline __attribute__((always_inline))
217 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
218 long voffp, intptr_t *load, int cpu)
219 {
220 RSEQ_INJECT_C(9)
221
222 __asm__ __volatile__ goto (
223 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
224 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
225 #ifdef RSEQ_COMPARE_TWICE
226 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
227 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
228 #endif
229 /* Start rseq by storing table entry pointer into rseq_cs. */
230 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
231 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
232 RSEQ_INJECT_ASM(3)
233 LONG_L " $4, %[v]\n\t"
234 "beq $4, %[expectnot], %l[cmpfail]\n\t"
235 RSEQ_INJECT_ASM(4)
236 #ifdef RSEQ_COMPARE_TWICE
237 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
238 LONG_L " $4, %[v]\n\t"
239 "beq $4, %[expectnot], %l[error2]\n\t"
240 #endif
241 LONG_S " $4, %[load]\n\t"
242 LONG_ADDI " $4, %[voffp]\n\t"
243 LONG_L " $4, 0($4)\n\t"
244 /* final store */
245 LONG_S " $4, %[v]\n\t"
246 "2:\n\t"
247 RSEQ_INJECT_ASM(5)
248 "b 5f\n\t"
249 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
250 "5:\n\t"
251 : /* gcc asm goto does not allow outputs */
252 : [cpu_id] "r" (cpu),
253 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
254 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
255 /* final store input */
256 [v] "m" (*v),
257 [expectnot] "r" (expectnot),
258 [voffp] "Ir" (voffp),
259 [load] "m" (*load)
260 RSEQ_INJECT_INPUT
261 : "$4", "memory"
262 RSEQ_INJECT_CLOBBER
263 : abort, cmpfail
264 #ifdef RSEQ_COMPARE_TWICE
265 , error1, error2
266 #endif
267 );
268 return 0;
269 abort:
270 RSEQ_INJECT_FAILED
271 return -1;
272 cmpfail:
273 return 1;
274 #ifdef RSEQ_COMPARE_TWICE
275 error1:
276 rseq_bug("cpu_id comparison failed");
277 error2:
278 rseq_bug("expected value comparison failed");
279 #endif
280 }
281
282 static inline __attribute__((always_inline))
283 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
284 {
285 RSEQ_INJECT_C(9)
286
287 __asm__ __volatile__ goto (
288 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
289 #ifdef RSEQ_COMPARE_TWICE
290 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
291 #endif
292 /* Start rseq by storing table entry pointer into rseq_cs. */
293 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
294 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
295 RSEQ_INJECT_ASM(3)
296 #ifdef RSEQ_COMPARE_TWICE
297 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
298 #endif
299 LONG_L " $4, %[v]\n\t"
300 LONG_ADDI " $4, %[count]\n\t"
301 /* final store */
302 LONG_S " $4, %[v]\n\t"
303 "2:\n\t"
304 RSEQ_INJECT_ASM(4)
305 "b 5f\n\t"
306 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
307 "5:\n\t"
308 : /* gcc asm goto does not allow outputs */
309 : [cpu_id] "r" (cpu),
310 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
311 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
312 [v] "m" (*v),
313 [count] "Ir" (count)
314 RSEQ_INJECT_INPUT
315 : "$4", "memory"
316 RSEQ_INJECT_CLOBBER
317 : abort
318 #ifdef RSEQ_COMPARE_TWICE
319 , error1
320 #endif
321 );
322 return 0;
323 abort:
324 RSEQ_INJECT_FAILED
325 return -1;
326 #ifdef RSEQ_COMPARE_TWICE
327 error1:
328 rseq_bug("cpu_id comparison failed");
329 #endif
330 }
331
332 static inline __attribute__((always_inline))
333 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
334 intptr_t *v2, intptr_t newv2,
335 intptr_t newv, int cpu)
336 {
337 RSEQ_INJECT_C(9)
338
339 __asm__ __volatile__ goto (
340 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
341 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
342 #ifdef RSEQ_COMPARE_TWICE
343 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
344 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
345 #endif
346 /* Start rseq by storing table entry pointer into rseq_cs. */
347 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
348 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
349 RSEQ_INJECT_ASM(3)
350 LONG_L " $4, %[v]\n\t"
351 "bne $4, %[expect], %l[cmpfail]\n\t"
352 RSEQ_INJECT_ASM(4)
353 #ifdef RSEQ_COMPARE_TWICE
354 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
355 LONG_L " $4, %[v]\n\t"
356 "bne $4, %[expect], %l[error2]\n\t"
357 #endif
358 /* try store */
359 LONG_S " %[newv2], %[v2]\n\t"
360 RSEQ_INJECT_ASM(5)
361 /* final store */
362 LONG_S " %[newv], %[v]\n\t"
363 "2:\n\t"
364 RSEQ_INJECT_ASM(6)
365 "b 5f\n\t"
366 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
367 "5:\n\t"
368 : /* gcc asm goto does not allow outputs */
369 : [cpu_id] "r" (cpu),
370 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
371 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
372 /* try store input */
373 [v2] "m" (*v2),
374 [newv2] "r" (newv2),
375 /* final store input */
376 [v] "m" (*v),
377 [expect] "r" (expect),
378 [newv] "r" (newv)
379 RSEQ_INJECT_INPUT
380 : "$4", "memory"
381 RSEQ_INJECT_CLOBBER
382 : abort, cmpfail
383 #ifdef RSEQ_COMPARE_TWICE
384 , error1, error2
385 #endif
386 );
387 return 0;
388 abort:
389 RSEQ_INJECT_FAILED
390 return -1;
391 cmpfail:
392 return 1;
393 #ifdef RSEQ_COMPARE_TWICE
394 error1:
395 rseq_bug("cpu_id comparison failed");
396 error2:
397 rseq_bug("expected value comparison failed");
398 #endif
399 }
400
401 static inline __attribute__((always_inline))
402 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
403 intptr_t *v2, intptr_t newv2,
404 intptr_t newv, int cpu)
405 {
406 RSEQ_INJECT_C(9)
407
408 __asm__ __volatile__ goto (
409 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
410 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
411 #ifdef RSEQ_COMPARE_TWICE
412 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
413 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
414 #endif
415 /* Start rseq by storing table entry pointer into rseq_cs. */
416 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
417 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
418 RSEQ_INJECT_ASM(3)
419 LONG_L " $4, %[v]\n\t"
420 "bne $4, %[expect], %l[cmpfail]\n\t"
421 RSEQ_INJECT_ASM(4)
422 #ifdef RSEQ_COMPARE_TWICE
423 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
424 LONG_L " $4, %[v]\n\t"
425 "bne $4, %[expect], %l[error2]\n\t"
426 #endif
427 /* try store */
428 LONG_S " %[newv2], %[v2]\n\t"
429 RSEQ_INJECT_ASM(5)
430 "sync\n\t" /* full sync provides store-release */
431 /* final store */
432 LONG_S " %[newv], %[v]\n\t"
433 "2:\n\t"
434 RSEQ_INJECT_ASM(6)
435 "b 5f\n\t"
436 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
437 "5:\n\t"
438 : /* gcc asm goto does not allow outputs */
439 : [cpu_id] "r" (cpu),
440 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
441 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
442 /* try store input */
443 [v2] "m" (*v2),
444 [newv2] "r" (newv2),
445 /* final store input */
446 [v] "m" (*v),
447 [expect] "r" (expect),
448 [newv] "r" (newv)
449 RSEQ_INJECT_INPUT
450 : "$4", "memory"
451 RSEQ_INJECT_CLOBBER
452 : abort, cmpfail
453 #ifdef RSEQ_COMPARE_TWICE
454 , error1, error2
455 #endif
456 );
457 return 0;
458 abort:
459 RSEQ_INJECT_FAILED
460 return -1;
461 cmpfail:
462 return 1;
463 #ifdef RSEQ_COMPARE_TWICE
464 error1:
465 rseq_bug("cpu_id comparison failed");
466 error2:
467 rseq_bug("expected value comparison failed");
468 #endif
469 }
470
471 static inline __attribute__((always_inline))
472 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
473 intptr_t *v2, intptr_t expect2,
474 intptr_t newv, int cpu)
475 {
476 RSEQ_INJECT_C(9)
477
478 __asm__ __volatile__ goto (
479 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
480 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
481 #ifdef RSEQ_COMPARE_TWICE
482 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
483 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
484 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
485 #endif
486 /* Start rseq by storing table entry pointer into rseq_cs. */
487 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
488 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
489 RSEQ_INJECT_ASM(3)
490 LONG_L " $4, %[v]\n\t"
491 "bne $4, %[expect], %l[cmpfail]\n\t"
492 RSEQ_INJECT_ASM(4)
493 LONG_L " $4, %[v2]\n\t"
494 "bne $4, %[expect2], %l[cmpfail]\n\t"
495 RSEQ_INJECT_ASM(5)
496 #ifdef RSEQ_COMPARE_TWICE
497 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
498 LONG_L " $4, %[v]\n\t"
499 "bne $4, %[expect], %l[error2]\n\t"
500 LONG_L " $4, %[v2]\n\t"
501 "bne $4, %[expect2], %l[error3]\n\t"
502 #endif
503 /* final store */
504 LONG_S " %[newv], %[v]\n\t"
505 "2:\n\t"
506 RSEQ_INJECT_ASM(6)
507 "b 5f\n\t"
508 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
509 "5:\n\t"
510 : /* gcc asm goto does not allow outputs */
511 : [cpu_id] "r" (cpu),
512 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
513 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
514 /* cmp2 input */
515 [v2] "m" (*v2),
516 [expect2] "r" (expect2),
517 /* final store input */
518 [v] "m" (*v),
519 [expect] "r" (expect),
520 [newv] "r" (newv)
521 RSEQ_INJECT_INPUT
522 : "$4", "memory"
523 RSEQ_INJECT_CLOBBER
524 : abort, cmpfail
525 #ifdef RSEQ_COMPARE_TWICE
526 , error1, error2, error3
527 #endif
528 );
529 return 0;
530 abort:
531 RSEQ_INJECT_FAILED
532 return -1;
533 cmpfail:
534 return 1;
535 #ifdef RSEQ_COMPARE_TWICE
536 error1:
537 rseq_bug("cpu_id comparison failed");
538 error2:
539 rseq_bug("1st expected value comparison failed");
540 error3:
541 rseq_bug("2nd expected value comparison failed");
542 #endif
543 }
544
545 static inline __attribute__((always_inline))
546 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
547 void *dst, void *src, size_t len,
548 intptr_t newv, int cpu)
549 {
550 uintptr_t rseq_scratch[3];
551
552 RSEQ_INJECT_C(9)
553
554 __asm__ __volatile__ goto (
555 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
556 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
557 #ifdef RSEQ_COMPARE_TWICE
558 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
559 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
560 #endif
561 LONG_S " %[src], %[rseq_scratch0]\n\t"
562 LONG_S " %[dst], %[rseq_scratch1]\n\t"
563 LONG_S " %[len], %[rseq_scratch2]\n\t"
564 /* Start rseq by storing table entry pointer into rseq_cs. */
565 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
566 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
567 RSEQ_INJECT_ASM(3)
568 LONG_L " $4, %[v]\n\t"
569 "bne $4, %[expect], 5f\n\t"
570 RSEQ_INJECT_ASM(4)
571 #ifdef RSEQ_COMPARE_TWICE
572 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
573 LONG_L " $4, %[v]\n\t"
574 "bne $4, %[expect], 7f\n\t"
575 #endif
576 /* try memcpy */
577 "beqz %[len], 333f\n\t" \
578 "222:\n\t" \
579 "lb $4, 0(%[src])\n\t" \
580 "sb $4, 0(%[dst])\n\t" \
581 LONG_ADDI " %[src], 1\n\t" \
582 LONG_ADDI " %[dst], 1\n\t" \
583 LONG_ADDI " %[len], -1\n\t" \
584 "bnez %[len], 222b\n\t" \
585 "333:\n\t" \
586 RSEQ_INJECT_ASM(5)
587 /* final store */
588 LONG_S " %[newv], %[v]\n\t"
589 "2:\n\t"
590 RSEQ_INJECT_ASM(6)
591 /* teardown */
592 LONG_L " %[len], %[rseq_scratch2]\n\t"
593 LONG_L " %[dst], %[rseq_scratch1]\n\t"
594 LONG_L " %[src], %[rseq_scratch0]\n\t"
595 "b 8f\n\t"
596 RSEQ_ASM_DEFINE_ABORT(3, 4,
597 /* teardown */
598 LONG_L " %[len], %[rseq_scratch2]\n\t"
599 LONG_L " %[dst], %[rseq_scratch1]\n\t"
600 LONG_L " %[src], %[rseq_scratch0]\n\t",
601 abort, 1b, 2b, 4f)
602 RSEQ_ASM_DEFINE_CMPFAIL(5,
603 /* teardown */
604 LONG_L " %[len], %[rseq_scratch2]\n\t"
605 LONG_L " %[dst], %[rseq_scratch1]\n\t"
606 LONG_L " %[src], %[rseq_scratch0]\n\t",
607 cmpfail)
608 #ifdef RSEQ_COMPARE_TWICE
609 RSEQ_ASM_DEFINE_CMPFAIL(6,
610 /* teardown */
611 LONG_L " %[len], %[rseq_scratch2]\n\t"
612 LONG_L " %[dst], %[rseq_scratch1]\n\t"
613 LONG_L " %[src], %[rseq_scratch0]\n\t",
614 error1)
615 RSEQ_ASM_DEFINE_CMPFAIL(7,
616 /* teardown */
617 LONG_L " %[len], %[rseq_scratch2]\n\t"
618 LONG_L " %[dst], %[rseq_scratch1]\n\t"
619 LONG_L " %[src], %[rseq_scratch0]\n\t",
620 error2)
621 #endif
622 "8:\n\t"
623 : /* gcc asm goto does not allow outputs */
624 : [cpu_id] "r" (cpu),
625 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
626 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
627 /* final store input */
628 [v] "m" (*v),
629 [expect] "r" (expect),
630 [newv] "r" (newv),
631 /* try memcpy input */
632 [dst] "r" (dst),
633 [src] "r" (src),
634 [len] "r" (len),
635 [rseq_scratch0] "m" (rseq_scratch[0]),
636 [rseq_scratch1] "m" (rseq_scratch[1]),
637 [rseq_scratch2] "m" (rseq_scratch[2])
638 RSEQ_INJECT_INPUT
639 : "$4", "memory"
640 RSEQ_INJECT_CLOBBER
641 : abort, cmpfail
642 #ifdef RSEQ_COMPARE_TWICE
643 , error1, error2
644 #endif
645 );
646 return 0;
647 abort:
648 RSEQ_INJECT_FAILED
649 return -1;
650 cmpfail:
651 return 1;
652 #ifdef RSEQ_COMPARE_TWICE
653 error1:
654 rseq_bug("cpu_id comparison failed");
655 error2:
656 rseq_bug("expected value comparison failed");
657 #endif
658 }
659
660 static inline __attribute__((always_inline))
661 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
662 void *dst, void *src, size_t len,
663 intptr_t newv, int cpu)
664 {
665 uintptr_t rseq_scratch[3];
666
667 RSEQ_INJECT_C(9)
668
669 __asm__ __volatile__ goto (
670 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
671 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
672 #ifdef RSEQ_COMPARE_TWICE
673 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
674 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
675 #endif
676 LONG_S " %[src], %[rseq_scratch0]\n\t"
677 LONG_S " %[dst], %[rseq_scratch1]\n\t"
678 LONG_S " %[len], %[rseq_scratch2]\n\t"
679 /* Start rseq by storing table entry pointer into rseq_cs. */
680 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
681 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
682 RSEQ_INJECT_ASM(3)
683 LONG_L " $4, %[v]\n\t"
684 "bne $4, %[expect], 5f\n\t"
685 RSEQ_INJECT_ASM(4)
686 #ifdef RSEQ_COMPARE_TWICE
687 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
688 LONG_L " $4, %[v]\n\t"
689 "bne $4, %[expect], 7f\n\t"
690 #endif
691 /* try memcpy */
692 "beqz %[len], 333f\n\t" \
693 "222:\n\t" \
694 "lb $4, 0(%[src])\n\t" \
695 "sb $4, 0(%[dst])\n\t" \
696 LONG_ADDI " %[src], 1\n\t" \
697 LONG_ADDI " %[dst], 1\n\t" \
698 LONG_ADDI " %[len], -1\n\t" \
699 "bnez %[len], 222b\n\t" \
700 "333:\n\t" \
701 RSEQ_INJECT_ASM(5)
702 "sync\n\t" /* full sync provides store-release */
703 /* final store */
704 LONG_S " %[newv], %[v]\n\t"
705 "2:\n\t"
706 RSEQ_INJECT_ASM(6)
707 /* teardown */
708 LONG_L " %[len], %[rseq_scratch2]\n\t"
709 LONG_L " %[dst], %[rseq_scratch1]\n\t"
710 LONG_L " %[src], %[rseq_scratch0]\n\t"
711 "b 8f\n\t"
712 RSEQ_ASM_DEFINE_ABORT(3, 4,
713 /* teardown */
714 LONG_L " %[len], %[rseq_scratch2]\n\t"
715 LONG_L " %[dst], %[rseq_scratch1]\n\t"
716 LONG_L " %[src], %[rseq_scratch0]\n\t",
717 abort, 1b, 2b, 4f)
718 RSEQ_ASM_DEFINE_CMPFAIL(5,
719 /* teardown */
720 LONG_L " %[len], %[rseq_scratch2]\n\t"
721 LONG_L " %[dst], %[rseq_scratch1]\n\t"
722 LONG_L " %[src], %[rseq_scratch0]\n\t",
723 cmpfail)
724 #ifdef RSEQ_COMPARE_TWICE
725 RSEQ_ASM_DEFINE_CMPFAIL(6,
726 /* teardown */
727 LONG_L " %[len], %[rseq_scratch2]\n\t"
728 LONG_L " %[dst], %[rseq_scratch1]\n\t"
729 LONG_L " %[src], %[rseq_scratch0]\n\t",
730 error1)
731 RSEQ_ASM_DEFINE_CMPFAIL(7,
732 /* teardown */
733 LONG_L " %[len], %[rseq_scratch2]\n\t"
734 LONG_L " %[dst], %[rseq_scratch1]\n\t"
735 LONG_L " %[src], %[rseq_scratch0]\n\t",
736 error2)
737 #endif
738 "8:\n\t"
739 : /* gcc asm goto does not allow outputs */
740 : [cpu_id] "r" (cpu),
741 [current_cpu_id] "m" (rseq_get_abi()->cpu_id),
742 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
743 /* final store input */
744 [v] "m" (*v),
745 [expect] "r" (expect),
746 [newv] "r" (newv),
747 /* try memcpy input */
748 [dst] "r" (dst),
749 [src] "r" (src),
750 [len] "r" (len),
751 [rseq_scratch0] "m" (rseq_scratch[0]),
752 [rseq_scratch1] "m" (rseq_scratch[1]),
753 [rseq_scratch2] "m" (rseq_scratch[2])
754 RSEQ_INJECT_INPUT
755 : "$4", "memory"
756 RSEQ_INJECT_CLOBBER
757 : abort, cmpfail
758 #ifdef RSEQ_COMPARE_TWICE
759 , error1, error2
760 #endif
761 );
762 return 0;
763 abort:
764 RSEQ_INJECT_FAILED
765 return -1;
766 cmpfail:
767 return 1;
768 #ifdef RSEQ_COMPARE_TWICE
769 error1:
770 rseq_bug("cpu_id comparison failed");
771 error2:
772 rseq_bug("expected value comparison failed");
773 #endif
774 }
775
776 #endif /* !RSEQ_SKIP_FASTPATH */
Librseq
This page took 0.061515 seconds and 4 git commands to generate.