1 /* SPDX-License-Identifier: MIT */
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
17 #define RSEQ_SIG 0x53053053
19 /* Offset of cpu_id and rseq_cs fields in struct rseq_abi. */
20 #define RSEQ_CPU_ID_OFFSET 4
21 #define RSEQ_CS_OFFSET 8
25 #define RSEQ_ASM_TP_SEGMENT %%fs
27 #define rseq_smp_mb() \
28 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
29 #define rseq_smp_rmb() rseq_barrier()
30 #define rseq_smp_wmb() rseq_barrier()
32 #define rseq_smp_load_acquire(p) \
34 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
39 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
41 #define rseq_smp_store_release(p, v) \
44 RSEQ_WRITE_ONCE(*p, v); \
47 #ifdef RSEQ_SKIP_FASTPATH
48 #include "rseq-skip.h"
49 #else /* !RSEQ_SKIP_FASTPATH */
51 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
52 start_ip, post_commit_offset, abort_ip) \
53 ".pushsection __rseq_cs, \"aw\"\n\t" \
55 __rseq_str(label) ":\n\t" \
56 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
57 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
59 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
60 ".quad " __rseq_str(label) "b\n\t" \
64 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
65 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
66 (post_commit_ip - start_ip), abort_ip)
69 * Exit points of a rseq critical section consist of all instructions outside
70 * of the critical section where a critical section can either branch to or
71 * reach through the normal course of its execution. The abort IP and the
72 * post-commit IP are already part of the __rseq_cs section and should not be
73 * explicitly defined as additional exit points. Knowing all exit points is
74 * useful to assist debuggers stepping over the critical section.
76 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
77 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
78 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
81 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
83 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
84 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
85 __rseq_str(label) ":\n\t"
87 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
89 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
90 "jnz " __rseq_str(label) "\n\t"
92 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
93 ".pushsection __rseq_failure, \"ax\"\n\t" \
94 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
95 ".byte 0x0f, 0xb9, 0x3d\n\t" \
96 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
97 __rseq_str(label) ":\n\t" \
99 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
102 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
103 ".pushsection __rseq_failure, \"ax\"\n\t" \
104 __rseq_str(label) ":\n\t" \
106 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
109 static inline __attribute__((always_inline
))
110 int rseq_cmpeqv_storev(intptr_t *v
, intptr_t expect
, intptr_t newv
, int cpu
)
114 __asm__ __volatile__
goto (
115 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
116 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
117 #ifdef RSEQ_COMPARE_TWICE
118 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
119 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
121 /* Start rseq by storing table entry pointer into rseq_cs. */
122 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
123 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
125 "cmpq %[v], %[expect]\n\t"
126 "jnz %l[cmpfail]\n\t"
128 #ifdef RSEQ_COMPARE_TWICE
129 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
130 "cmpq %[v], %[expect]\n\t"
134 "movq %[newv], %[v]\n\t"
137 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
138 : /* gcc asm goto does not allow outputs */
139 : [cpu_id
] "r" (cpu
),
140 [rseq_offset
] "r" (rseq_offset
),
142 [expect
] "r" (expect
),
144 : "memory", "cc", "rax"
147 #ifdef RSEQ_COMPARE_TWICE
151 rseq_after_asm_goto();
154 rseq_after_asm_goto();
158 rseq_after_asm_goto();
160 #ifdef RSEQ_COMPARE_TWICE
162 rseq_after_asm_goto();
163 rseq_bug("cpu_id comparison failed");
165 rseq_after_asm_goto();
166 rseq_bug("expected value comparison failed");
171 * Compare @v against @expectnot. When it does _not_ match, load @v
172 * into @load, and store the content of *@v + voffp into @v.
174 static inline __attribute__((always_inline
))
175 int rseq_cmpnev_storeoffp_load(intptr_t *v
, intptr_t expectnot
,
176 long voffp
, intptr_t *load
, int cpu
)
180 __asm__ __volatile__
goto (
181 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
182 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
183 #ifdef RSEQ_COMPARE_TWICE
184 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
185 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
187 /* Start rseq by storing table entry pointer into rseq_cs. */
188 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
189 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
191 "movq %[v], %%rbx\n\t"
192 "cmpq %%rbx, %[expectnot]\n\t"
195 #ifdef RSEQ_COMPARE_TWICE
196 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
197 "movq %[v], %%rbx\n\t"
198 "cmpq %%rbx, %[expectnot]\n\t"
201 "movq %%rbx, %[load]\n\t"
202 "addq %[voffp], %%rbx\n\t"
203 "movq (%%rbx), %%rbx\n\t"
205 "movq %%rbx, %[v]\n\t"
208 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
209 : /* gcc asm goto does not allow outputs */
210 : [cpu_id
] "r" (cpu
),
211 [rseq_offset
] "r" (rseq_offset
),
212 /* final store input */
214 [expectnot
] "r" (expectnot
),
215 [voffp
] "er" (voffp
),
217 : "memory", "cc", "rax", "rbx"
220 #ifdef RSEQ_COMPARE_TWICE
224 rseq_after_asm_goto();
227 rseq_after_asm_goto();
231 rseq_after_asm_goto();
233 #ifdef RSEQ_COMPARE_TWICE
235 rseq_after_asm_goto();
236 rseq_bug("cpu_id comparison failed");
238 rseq_after_asm_goto();
239 rseq_bug("expected value comparison failed");
243 static inline __attribute__((always_inline
))
244 int rseq_addv(intptr_t *v
, intptr_t count
, int cpu
)
248 __asm__ __volatile__
goto (
249 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
250 #ifdef RSEQ_COMPARE_TWICE
251 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
253 /* Start rseq by storing table entry pointer into rseq_cs. */
254 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
255 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
257 #ifdef RSEQ_COMPARE_TWICE
258 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
261 "addq %[count], %[v]\n\t"
264 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
265 : /* gcc asm goto does not allow outputs */
266 : [cpu_id
] "r" (cpu
),
267 [rseq_offset
] "r" (rseq_offset
),
268 /* final store input */
271 : "memory", "cc", "rax"
274 #ifdef RSEQ_COMPARE_TWICE
278 rseq_after_asm_goto();
281 rseq_after_asm_goto();
284 #ifdef RSEQ_COMPARE_TWICE
286 rseq_after_asm_goto();
287 rseq_bug("cpu_id comparison failed");
291 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
297 static inline __attribute__((always_inline
))
298 int rseq_offset_deref_addv(intptr_t *ptr
, long off
, intptr_t inc
, int cpu
)
302 __asm__ __volatile__
goto (
303 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
304 #ifdef RSEQ_COMPARE_TWICE
305 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
307 /* Start rseq by storing table entry pointer into rseq_cs. */
308 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
309 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
311 #ifdef RSEQ_COMPARE_TWICE
312 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
315 "movq %[ptr], %%rbx\n\t"
316 "addq %[off], %%rbx\n\t"
318 "movq (%%rbx), %%rcx\n\t"
320 "addq %[inc], (%%rcx)\n\t"
323 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
324 : /* gcc asm goto does not allow outputs */
325 : [cpu_id
] "r" (cpu
),
326 [rseq_offset
] "r" (rseq_offset
),
327 /* final store input */
331 : "memory", "cc", "rax", "rbx", "rcx"
334 #ifdef RSEQ_COMPARE_TWICE
338 rseq_after_asm_goto();
341 rseq_after_asm_goto();
344 #ifdef RSEQ_COMPARE_TWICE
346 rseq_after_asm_goto();
347 rseq_bug("cpu_id comparison failed");
351 static inline __attribute__((always_inline
))
352 int rseq_cmpeqv_trystorev_storev(intptr_t *v
, intptr_t expect
,
353 intptr_t *v2
, intptr_t newv2
,
354 intptr_t newv
, int cpu
)
358 __asm__ __volatile__
goto (
359 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
360 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
361 #ifdef RSEQ_COMPARE_TWICE
362 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
363 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
365 /* Start rseq by storing table entry pointer into rseq_cs. */
366 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
367 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
369 "cmpq %[v], %[expect]\n\t"
370 "jnz %l[cmpfail]\n\t"
372 #ifdef RSEQ_COMPARE_TWICE
373 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
374 "cmpq %[v], %[expect]\n\t"
378 "movq %[newv2], %[v2]\n\t"
381 "movq %[newv], %[v]\n\t"
384 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
385 : /* gcc asm goto does not allow outputs */
386 : [cpu_id
] "r" (cpu
),
387 [rseq_offset
] "r" (rseq_offset
),
388 /* try store input */
391 /* final store input */
393 [expect
] "r" (expect
),
395 : "memory", "cc", "rax"
398 #ifdef RSEQ_COMPARE_TWICE
402 rseq_after_asm_goto();
405 rseq_after_asm_goto();
409 rseq_after_asm_goto();
411 #ifdef RSEQ_COMPARE_TWICE
413 rseq_after_asm_goto();
414 rseq_bug("cpu_id comparison failed");
416 rseq_after_asm_goto();
417 rseq_bug("expected value comparison failed");
422 static inline __attribute__((always_inline
))
423 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v
, intptr_t expect
,
424 intptr_t *v2
, intptr_t newv2
,
425 intptr_t newv
, int cpu
)
427 return rseq_cmpeqv_trystorev_storev(v
, expect
, v2
, newv2
, newv
, cpu
);
430 static inline __attribute__((always_inline
))
431 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v
, intptr_t expect
,
432 intptr_t *v2
, intptr_t expect2
,
433 intptr_t newv
, int cpu
)
437 __asm__ __volatile__
goto (
438 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
439 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
440 #ifdef RSEQ_COMPARE_TWICE
441 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
442 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
443 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error3
])
445 /* Start rseq by storing table entry pointer into rseq_cs. */
446 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
447 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
449 "cmpq %[v], %[expect]\n\t"
450 "jnz %l[cmpfail]\n\t"
452 "cmpq %[v2], %[expect2]\n\t"
453 "jnz %l[cmpfail]\n\t"
455 #ifdef RSEQ_COMPARE_TWICE
456 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
457 "cmpq %[v], %[expect]\n\t"
459 "cmpq %[v2], %[expect2]\n\t"
463 "movq %[newv], %[v]\n\t"
466 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
467 : /* gcc asm goto does not allow outputs */
468 : [cpu_id
] "r" (cpu
),
469 [rseq_offset
] "r" (rseq_offset
),
472 [expect2
] "r" (expect2
),
473 /* final store input */
475 [expect
] "r" (expect
),
477 : "memory", "cc", "rax"
480 #ifdef RSEQ_COMPARE_TWICE
481 , error1
, error2
, error3
484 rseq_after_asm_goto();
487 rseq_after_asm_goto();
491 rseq_after_asm_goto();
493 #ifdef RSEQ_COMPARE_TWICE
495 rseq_after_asm_goto();
496 rseq_bug("cpu_id comparison failed");
498 rseq_after_asm_goto();
499 rseq_bug("1st expected value comparison failed");
501 rseq_after_asm_goto();
502 rseq_bug("2nd expected value comparison failed");
506 static inline __attribute__((always_inline
))
507 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v
, intptr_t expect
,
508 void *dst
, void *src
, size_t len
,
509 intptr_t newv
, int cpu
)
511 uint64_t rseq_scratch
[3];
515 __asm__ __volatile__
goto (
516 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
517 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
518 #ifdef RSEQ_COMPARE_TWICE
519 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
520 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
522 "movq %[src], %[rseq_scratch0]\n\t"
523 "movq %[dst], %[rseq_scratch1]\n\t"
524 "movq %[len], %[rseq_scratch2]\n\t"
525 /* Start rseq by storing table entry pointer into rseq_cs. */
526 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
527 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
529 "cmpq %[v], %[expect]\n\t"
532 #ifdef RSEQ_COMPARE_TWICE
533 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 6f
)
534 "cmpq %[v], %[expect]\n\t"
538 "test %[len], %[len]\n\t" \
541 "movb (%[src]), %%al\n\t" \
542 "movb %%al, (%[dst])\n\t" \
550 "movq %[newv], %[v]\n\t"
554 "movq %[rseq_scratch2], %[len]\n\t"
555 "movq %[rseq_scratch1], %[dst]\n\t"
556 "movq %[rseq_scratch0], %[src]\n\t"
557 RSEQ_ASM_DEFINE_ABORT(4,
558 "movq %[rseq_scratch2], %[len]\n\t"
559 "movq %[rseq_scratch1], %[dst]\n\t"
560 "movq %[rseq_scratch0], %[src]\n\t",
562 RSEQ_ASM_DEFINE_CMPFAIL(5,
563 "movq %[rseq_scratch2], %[len]\n\t"
564 "movq %[rseq_scratch1], %[dst]\n\t"
565 "movq %[rseq_scratch0], %[src]\n\t",
567 #ifdef RSEQ_COMPARE_TWICE
568 RSEQ_ASM_DEFINE_CMPFAIL(6,
569 "movq %[rseq_scratch2], %[len]\n\t"
570 "movq %[rseq_scratch1], %[dst]\n\t"
571 "movq %[rseq_scratch0], %[src]\n\t",
573 RSEQ_ASM_DEFINE_CMPFAIL(7,
574 "movq %[rseq_scratch2], %[len]\n\t"
575 "movq %[rseq_scratch1], %[dst]\n\t"
576 "movq %[rseq_scratch0], %[src]\n\t",
579 : /* gcc asm goto does not allow outputs */
580 : [cpu_id
] "r" (cpu
),
581 [rseq_offset
] "r" (rseq_offset
),
582 /* final store input */
584 [expect
] "r" (expect
),
586 /* try memcpy input */
590 [rseq_scratch0
] "m" (rseq_scratch
[0]),
591 [rseq_scratch1
] "m" (rseq_scratch
[1]),
592 [rseq_scratch2
] "m" (rseq_scratch
[2])
593 : "memory", "cc", "rax"
596 #ifdef RSEQ_COMPARE_TWICE
600 rseq_after_asm_goto();
603 rseq_after_asm_goto();
607 rseq_after_asm_goto();
609 #ifdef RSEQ_COMPARE_TWICE
611 rseq_after_asm_goto();
612 rseq_bug("cpu_id comparison failed");
614 rseq_after_asm_goto();
615 rseq_bug("expected value comparison failed");
620 static inline __attribute__((always_inline
))
621 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v
, intptr_t expect
,
622 void *dst
, void *src
, size_t len
,
623 intptr_t newv
, int cpu
)
625 return rseq_cmpeqv_trymemcpy_storev(v
, expect
, dst
, src
, len
,
630 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
633 static inline __attribute__((always_inline
))
634 int rseq_deref_loadoffp(intptr_t *p
, long voffp
, intptr_t *load
, int cpu
)
638 __asm__ __volatile__
goto (
639 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
640 #ifdef RSEQ_COMPARE_TWICE
641 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
643 /* Start rseq by storing table entry pointer into rseq_cs. */
644 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
645 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
647 "movq %[p], %%rbx\n\t"
649 #ifdef RSEQ_COMPARE_TWICE
650 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
652 "addq %[voffp], %%rbx\n\t"
653 "movq (%%rbx), %%rbx\n\t"
654 "movq %%rbx, %[load]\n\t"
657 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
658 : /* gcc asm goto does not allow outputs */
659 : [cpu_id
] "r" (cpu
),
660 [rseq_offset
] "r" (rseq_offset
),
661 /* final store input */
663 [voffp
] "er" (voffp
),
665 : "memory", "cc", "rax", "rbx"
668 #ifdef RSEQ_COMPARE_TWICE
672 rseq_after_asm_goto();
675 rseq_after_asm_goto();
678 #ifdef RSEQ_COMPARE_TWICE
680 rseq_after_asm_goto();
681 rseq_bug("cpu_id comparison failed");
685 #endif /* !RSEQ_SKIP_FASTPATH */
687 #elif defined(__i386__)
689 #define RSEQ_ASM_TP_SEGMENT %%gs
691 #define rseq_smp_mb() \
692 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
693 #define rseq_smp_rmb() \
694 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
695 #define rseq_smp_wmb() \
696 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
698 #define rseq_smp_load_acquire(p) \
700 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
705 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
707 #define rseq_smp_store_release(p, v) \
710 RSEQ_WRITE_ONCE(*p, v); \
713 #ifdef RSEQ_SKIP_FASTPATH
714 #include "rseq-skip.h"
715 #else /* !RSEQ_SKIP_FASTPATH */
718 * Use eax as scratch register and take memory operands as input to
719 * lessen register pressure. Especially needed when compiling in O0.
721 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
722 start_ip, post_commit_offset, abort_ip) \
723 ".pushsection __rseq_cs, \"aw\"\n\t" \
725 __rseq_str(label) ":\n\t" \
726 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
727 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
729 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
730 ".long " __rseq_str(label) "b, 0x0\n\t" \
733 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
734 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
735 (post_commit_ip - start_ip), abort_ip)
738 * Exit points of a rseq critical section consist of all instructions outside
739 * of the critical section where a critical section can either branch to or
740 * reach through the normal course of its execution. The abort IP and the
741 * post-commit IP are already part of the __rseq_cs section and should not be
742 * explicitly defined as additional exit points. Knowing all exit points is
743 * useful to assist debuggers stepping over the critical section.
745 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
746 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
747 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
750 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
752 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
753 __rseq_str(label) ":\n\t"
755 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
757 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
758 "jnz " __rseq_str(label) "\n\t"
760 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
761 ".pushsection __rseq_failure, \"ax\"\n\t" \
762 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
763 ".byte 0x0f, 0xb9, 0x3d\n\t" \
764 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
765 __rseq_str(label) ":\n\t" \
767 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
770 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
771 ".pushsection __rseq_failure, \"ax\"\n\t" \
772 __rseq_str(label) ":\n\t" \
774 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
777 static inline __attribute__((always_inline
))
778 int rseq_cmpeqv_storev(intptr_t *v
, intptr_t expect
, intptr_t newv
, int cpu
)
782 __asm__ __volatile__
goto (
783 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
784 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
785 #ifdef RSEQ_COMPARE_TWICE
786 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
787 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
789 /* Start rseq by storing table entry pointer into rseq_cs. */
790 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
791 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
793 "cmpl %[v], %[expect]\n\t"
794 "jnz %l[cmpfail]\n\t"
796 #ifdef RSEQ_COMPARE_TWICE
797 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
798 "cmpl %[v], %[expect]\n\t"
802 "movl %[newv], %[v]\n\t"
805 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
806 : /* gcc asm goto does not allow outputs */
807 : [cpu_id
] "r" (cpu
),
808 [rseq_offset
] "r" (rseq_offset
),
810 [expect
] "r" (expect
),
812 : "memory", "cc", "eax"
815 #ifdef RSEQ_COMPARE_TWICE
819 rseq_after_asm_goto();
822 rseq_after_asm_goto();
826 rseq_after_asm_goto();
828 #ifdef RSEQ_COMPARE_TWICE
830 rseq_after_asm_goto();
831 rseq_bug("cpu_id comparison failed");
833 rseq_after_asm_goto();
834 rseq_bug("expected value comparison failed");
839 * Compare @v against @expectnot. When it does _not_ match, load @v
840 * into @load, and store the content of *@v + voffp into @v.
842 static inline __attribute__((always_inline
))
843 int rseq_cmpnev_storeoffp_load(intptr_t *v
, intptr_t expectnot
,
844 long voffp
, intptr_t *load
, int cpu
)
848 __asm__ __volatile__
goto (
849 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
850 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
851 #ifdef RSEQ_COMPARE_TWICE
852 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
853 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
855 /* Start rseq by storing table entry pointer into rseq_cs. */
856 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
857 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
859 "movl %[v], %%ebx\n\t"
860 "cmpl %%ebx, %[expectnot]\n\t"
863 #ifdef RSEQ_COMPARE_TWICE
864 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
865 "movl %[v], %%ebx\n\t"
866 "cmpl %%ebx, %[expectnot]\n\t"
869 "movl %%ebx, %[load]\n\t"
870 "addl %[voffp], %%ebx\n\t"
871 "movl (%%ebx), %%ebx\n\t"
873 "movl %%ebx, %[v]\n\t"
876 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
877 : /* gcc asm goto does not allow outputs */
878 : [cpu_id
] "r" (cpu
),
879 [rseq_offset
] "r" (rseq_offset
),
880 /* final store input */
882 [expectnot
] "r" (expectnot
),
883 [voffp
] "ir" (voffp
),
885 : "memory", "cc", "eax", "ebx"
888 #ifdef RSEQ_COMPARE_TWICE
892 rseq_after_asm_goto();
895 rseq_after_asm_goto();
899 rseq_after_asm_goto();
901 #ifdef RSEQ_COMPARE_TWICE
903 rseq_after_asm_goto();
904 rseq_bug("cpu_id comparison failed");
906 rseq_after_asm_goto();
907 rseq_bug("expected value comparison failed");
911 static inline __attribute__((always_inline
))
912 int rseq_addv(intptr_t *v
, intptr_t count
, int cpu
)
916 __asm__ __volatile__
goto (
917 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
918 #ifdef RSEQ_COMPARE_TWICE
919 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
921 /* Start rseq by storing table entry pointer into rseq_cs. */
922 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
923 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
925 #ifdef RSEQ_COMPARE_TWICE
926 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
929 "addl %[count], %[v]\n\t"
932 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
933 : /* gcc asm goto does not allow outputs */
934 : [cpu_id
] "r" (cpu
),
935 [rseq_offset
] "r" (rseq_offset
),
936 /* final store input */
939 : "memory", "cc", "eax"
942 #ifdef RSEQ_COMPARE_TWICE
946 rseq_after_asm_goto();
949 rseq_after_asm_goto();
952 #ifdef RSEQ_COMPARE_TWICE
954 rseq_after_asm_goto();
955 rseq_bug("cpu_id comparison failed");
959 static inline __attribute__((always_inline
))
960 int rseq_cmpeqv_trystorev_storev(intptr_t *v
, intptr_t expect
,
961 intptr_t *v2
, intptr_t newv2
,
962 intptr_t newv
, int cpu
)
966 __asm__ __volatile__
goto (
967 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
968 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
969 #ifdef RSEQ_COMPARE_TWICE
970 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
971 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
973 /* Start rseq by storing table entry pointer into rseq_cs. */
974 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
975 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
977 "cmpl %[v], %[expect]\n\t"
978 "jnz %l[cmpfail]\n\t"
980 #ifdef RSEQ_COMPARE_TWICE
981 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
982 "cmpl %[v], %[expect]\n\t"
986 "movl %[newv2], %%eax\n\t"
987 "movl %%eax, %[v2]\n\t"
990 "movl %[newv], %[v]\n\t"
993 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
994 : /* gcc asm goto does not allow outputs */
995 : [cpu_id
] "r" (cpu
),
996 [rseq_offset
] "r" (rseq_offset
),
997 /* try store input */
1000 /* final store input */
1002 [expect
] "r" (expect
),
1004 : "memory", "cc", "eax"
1007 #ifdef RSEQ_COMPARE_TWICE
1011 rseq_after_asm_goto();
1014 rseq_after_asm_goto();
1018 rseq_after_asm_goto();
1020 #ifdef RSEQ_COMPARE_TWICE
1022 rseq_after_asm_goto();
1023 rseq_bug("cpu_id comparison failed");
1025 rseq_after_asm_goto();
1026 rseq_bug("expected value comparison failed");
1030 static inline __attribute__((always_inline
))
1031 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v
, intptr_t expect
,
1032 intptr_t *v2
, intptr_t newv2
,
1033 intptr_t newv
, int cpu
)
1037 __asm__ __volatile__
goto (
1038 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
1039 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
1040 #ifdef RSEQ_COMPARE_TWICE
1041 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
1042 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
1044 /* Start rseq by storing table entry pointer into rseq_cs. */
1045 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
1046 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
1048 "movl %[expect], %%eax\n\t"
1049 "cmpl %[v], %%eax\n\t"
1050 "jnz %l[cmpfail]\n\t"
1052 #ifdef RSEQ_COMPARE_TWICE
1053 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
1054 "movl %[expect], %%eax\n\t"
1055 "cmpl %[v], %%eax\n\t"
1056 "jnz %l[error2]\n\t"
1059 "movl %[newv2], %[v2]\n\t"
1061 "lock; addl $0,-128(%%esp)\n\t"
1063 "movl %[newv], %[v]\n\t"
1066 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
1067 : /* gcc asm goto does not allow outputs */
1068 : [cpu_id
] "r" (cpu
),
1069 [rseq_offset
] "r" (rseq_offset
),
1070 /* try store input */
1072 [newv2
] "r" (newv2
),
1073 /* final store input */
1075 [expect
] "m" (expect
),
1077 : "memory", "cc", "eax"
1080 #ifdef RSEQ_COMPARE_TWICE
1084 rseq_after_asm_goto();
1087 rseq_after_asm_goto();
1091 rseq_after_asm_goto();
1093 #ifdef RSEQ_COMPARE_TWICE
1095 rseq_after_asm_goto();
1096 rseq_bug("cpu_id comparison failed");
1098 rseq_after_asm_goto();
1099 rseq_bug("expected value comparison failed");
1104 static inline __attribute__((always_inline
))
1105 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v
, intptr_t expect
,
1106 intptr_t *v2
, intptr_t expect2
,
1107 intptr_t newv
, int cpu
)
1111 __asm__ __volatile__
goto (
1112 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
1113 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
1114 #ifdef RSEQ_COMPARE_TWICE
1115 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
1116 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
1117 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error3
])
1119 /* Start rseq by storing table entry pointer into rseq_cs. */
1120 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
1121 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
1123 "cmpl %[v], %[expect]\n\t"
1124 "jnz %l[cmpfail]\n\t"
1126 "cmpl %[expect2], %[v2]\n\t"
1127 "jnz %l[cmpfail]\n\t"
1129 #ifdef RSEQ_COMPARE_TWICE
1130 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
1131 "cmpl %[v], %[expect]\n\t"
1132 "jnz %l[error2]\n\t"
1133 "cmpl %[expect2], %[v2]\n\t"
1134 "jnz %l[error3]\n\t"
1136 "movl %[newv], %%eax\n\t"
1138 "movl %%eax, %[v]\n\t"
1141 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
1142 : /* gcc asm goto does not allow outputs */
1143 : [cpu_id
] "r" (cpu
),
1144 [rseq_offset
] "r" (rseq_offset
),
1147 [expect2
] "r" (expect2
),
1148 /* final store input */
1150 [expect
] "r" (expect
),
1152 : "memory", "cc", "eax"
1155 #ifdef RSEQ_COMPARE_TWICE
1156 , error1
, error2
, error3
1159 rseq_after_asm_goto();
1162 rseq_after_asm_goto();
1166 rseq_after_asm_goto();
1168 #ifdef RSEQ_COMPARE_TWICE
1170 rseq_after_asm_goto();
1171 rseq_bug("cpu_id comparison failed");
1173 rseq_after_asm_goto();
1174 rseq_bug("1st expected value comparison failed");
1176 rseq_after_asm_goto();
1177 rseq_bug("2nd expected value comparison failed");
1181 /* TODO: implement a faster memcpy. */
1182 static inline __attribute__((always_inline
))
1183 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v
, intptr_t expect
,
1184 void *dst
, void *src
, size_t len
,
1185 intptr_t newv
, int cpu
)
1187 uint32_t rseq_scratch
[3];
1191 __asm__ __volatile__
goto (
1192 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
1193 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
1194 #ifdef RSEQ_COMPARE_TWICE
1195 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
1196 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
1198 "movl %[src], %[rseq_scratch0]\n\t"
1199 "movl %[dst], %[rseq_scratch1]\n\t"
1200 "movl %[len], %[rseq_scratch2]\n\t"
1201 /* Start rseq by storing table entry pointer into rseq_cs. */
1202 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
1203 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
1205 "movl %[expect], %%eax\n\t"
1206 "cmpl %%eax, %[v]\n\t"
1209 #ifdef RSEQ_COMPARE_TWICE
1210 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 6f
)
1211 "movl %[expect], %%eax\n\t"
1212 "cmpl %%eax, %[v]\n\t"
1216 "test %[len], %[len]\n\t" \
1219 "movb (%[src]), %%al\n\t" \
1220 "movb %%al, (%[dst])\n\t" \
1227 "movl %[newv], %%eax\n\t"
1229 "movl %%eax, %[v]\n\t"
1233 "movl %[rseq_scratch2], %[len]\n\t"
1234 "movl %[rseq_scratch1], %[dst]\n\t"
1235 "movl %[rseq_scratch0], %[src]\n\t"
1236 RSEQ_ASM_DEFINE_ABORT(4,
1237 "movl %[rseq_scratch2], %[len]\n\t"
1238 "movl %[rseq_scratch1], %[dst]\n\t"
1239 "movl %[rseq_scratch0], %[src]\n\t",
1241 RSEQ_ASM_DEFINE_CMPFAIL(5,
1242 "movl %[rseq_scratch2], %[len]\n\t"
1243 "movl %[rseq_scratch1], %[dst]\n\t"
1244 "movl %[rseq_scratch0], %[src]\n\t",
1246 #ifdef RSEQ_COMPARE_TWICE
1247 RSEQ_ASM_DEFINE_CMPFAIL(6,
1248 "movl %[rseq_scratch2], %[len]\n\t"
1249 "movl %[rseq_scratch1], %[dst]\n\t"
1250 "movl %[rseq_scratch0], %[src]\n\t",
1252 RSEQ_ASM_DEFINE_CMPFAIL(7,
1253 "movl %[rseq_scratch2], %[len]\n\t"
1254 "movl %[rseq_scratch1], %[dst]\n\t"
1255 "movl %[rseq_scratch0], %[src]\n\t",
1258 : /* gcc asm goto does not allow outputs */
1259 : [cpu_id
] "r" (cpu
),
1260 [rseq_offset
] "r" (rseq_offset
),
1261 /* final store input */
1263 [expect
] "m" (expect
),
1265 /* try memcpy input */
1269 [rseq_scratch0
] "m" (rseq_scratch
[0]),
1270 [rseq_scratch1
] "m" (rseq_scratch
[1]),
1271 [rseq_scratch2
] "m" (rseq_scratch
[2])
1272 : "memory", "cc", "eax"
1275 #ifdef RSEQ_COMPARE_TWICE
1279 rseq_after_asm_goto();
1282 rseq_after_asm_goto();
1286 rseq_after_asm_goto();
1288 #ifdef RSEQ_COMPARE_TWICE
1290 rseq_after_asm_goto();
1291 rseq_bug("cpu_id comparison failed");
1293 rseq_after_asm_goto();
1294 rseq_bug("expected value comparison failed");
1298 /* TODO: implement a faster memcpy. */
1299 static inline __attribute__((always_inline
))
1300 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v
, intptr_t expect
,
1301 void *dst
, void *src
, size_t len
,
1302 intptr_t newv
, int cpu
)
1304 uint32_t rseq_scratch
[3];
1308 __asm__ __volatile__
goto (
1309 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
1310 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
1311 #ifdef RSEQ_COMPARE_TWICE
1312 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
1313 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
1315 "movl %[src], %[rseq_scratch0]\n\t"
1316 "movl %[dst], %[rseq_scratch1]\n\t"
1317 "movl %[len], %[rseq_scratch2]\n\t"
1318 /* Start rseq by storing table entry pointer into rseq_cs. */
1319 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
1320 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
1322 "movl %[expect], %%eax\n\t"
1323 "cmpl %%eax, %[v]\n\t"
1326 #ifdef RSEQ_COMPARE_TWICE
1327 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 6f
)
1328 "movl %[expect], %%eax\n\t"
1329 "cmpl %%eax, %[v]\n\t"
1333 "test %[len], %[len]\n\t" \
1336 "movb (%[src]), %%al\n\t" \
1337 "movb %%al, (%[dst])\n\t" \
1344 "lock; addl $0,-128(%%esp)\n\t"
1345 "movl %[newv], %%eax\n\t"
1347 "movl %%eax, %[v]\n\t"
1351 "movl %[rseq_scratch2], %[len]\n\t"
1352 "movl %[rseq_scratch1], %[dst]\n\t"
1353 "movl %[rseq_scratch0], %[src]\n\t"
1354 RSEQ_ASM_DEFINE_ABORT(4,
1355 "movl %[rseq_scratch2], %[len]\n\t"
1356 "movl %[rseq_scratch1], %[dst]\n\t"
1357 "movl %[rseq_scratch0], %[src]\n\t",
1359 RSEQ_ASM_DEFINE_CMPFAIL(5,
1360 "movl %[rseq_scratch2], %[len]\n\t"
1361 "movl %[rseq_scratch1], %[dst]\n\t"
1362 "movl %[rseq_scratch0], %[src]\n\t",
1364 #ifdef RSEQ_COMPARE_TWICE
1365 RSEQ_ASM_DEFINE_CMPFAIL(6,
1366 "movl %[rseq_scratch2], %[len]\n\t"
1367 "movl %[rseq_scratch1], %[dst]\n\t"
1368 "movl %[rseq_scratch0], %[src]\n\t",
1370 RSEQ_ASM_DEFINE_CMPFAIL(7,
1371 "movl %[rseq_scratch2], %[len]\n\t"
1372 "movl %[rseq_scratch1], %[dst]\n\t"
1373 "movl %[rseq_scratch0], %[src]\n\t",
1376 : /* gcc asm goto does not allow outputs */
1377 : [cpu_id
] "r" (cpu
),
1378 [rseq_offset
] "r" (rseq_offset
),
1379 /* final store input */
1381 [expect
] "m" (expect
),
1383 /* try memcpy input */
1387 [rseq_scratch0
] "m" (rseq_scratch
[0]),
1388 [rseq_scratch1
] "m" (rseq_scratch
[1]),
1389 [rseq_scratch2
] "m" (rseq_scratch
[2])
1390 : "memory", "cc", "eax"
1393 #ifdef RSEQ_COMPARE_TWICE
1397 rseq_after_asm_goto();
1400 rseq_after_asm_goto();
1404 rseq_after_asm_goto();
1406 #ifdef RSEQ_COMPARE_TWICE
1408 rseq_after_asm_goto();
1409 rseq_bug("cpu_id comparison failed");
1411 rseq_after_asm_goto();
1412 rseq_bug("expected value comparison failed");
1417 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
1420 static inline __attribute__((always_inline
))
1421 int rseq_deref_loadoffp(intptr_t *p
, long voffp
, intptr_t *load
, int cpu
)
1425 __asm__ __volatile__
goto (
1426 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
1427 #ifdef RSEQ_COMPARE_TWICE
1428 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
1430 /* Start rseq by storing table entry pointer into rseq_cs. */
1431 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CS_OFFSET(%[rseq_offset
]))
1432 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), 4f
)
1434 "movl %[p], %%ebx\n\t"
1436 #ifdef RSEQ_COMPARE_TWICE
1437 RSEQ_ASM_CMP_CPU_ID(cpu_id
, RSEQ_ASM_TP_SEGMENT
:RSEQ_CPU_ID_OFFSET(%[rseq_offset
]), %l
[error1
])
1439 "addl %[voffp], %%ebx\n\t"
1440 "movl (%%ebx), %%ebx\n\t"
1441 "movl %%ebx, %[load]\n\t"
1444 RSEQ_ASM_DEFINE_ABORT(4, "", abort
)
1445 : /* gcc asm goto does not allow outputs */
1446 : [cpu_id
] "r" (cpu
),
1447 [rseq_offset
] "r" (rseq_offset
),
1448 /* final store input */
1450 [voffp
] "ir" (voffp
),
1452 : "memory", "cc", "eax", "ebx"
1455 #ifdef RSEQ_COMPARE_TWICE
1459 rseq_after_asm_goto();
1462 rseq_after_asm_goto();
1465 #ifdef RSEQ_COMPARE_TWICE
1467 rseq_after_asm_goto();
1468 rseq_bug("cpu_id comparison failed");
1472 #endif /* !RSEQ_SKIP_FASTPATH */
This page took 0.10939 seconds and 5 git commands to generate.