projects / librseq.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Implement the REUSE specification for licensing and copyright
[librseq.git] / include / rseq / rseq-x86.h
CommitLineData
90702366 1/* SPDX-License-Identifier: MIT */
f2d7b530
MJ		 2/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3
784b0012
MD		 4/*
5 * rseq-x86.h
784b0012
MD		 6 */
7
8#include <stdint.h>
9
a5fe9e95
MD		 10/*
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
12 * trap in user-space:
13 *
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
16 */
784b0012
MD		 17#define RSEQ_SIG 0x53053053
18
2d533093 19/* Offset of cpu_id and rseq_cs fields in struct rseq_abi. */
05ebd3f9
MD		 20#define RSEQ_CPU_ID_OFFSET 4
21#define RSEQ_CS_OFFSET 8
22
784b0012
MD		 23#ifdef __x86_64__
24
7dc4bc29
MD		 25#define RSEQ_ASM_TP_SEGMENT %%fs
26
784b0012
MD		 27#define rseq_smp_mb() \
28 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
29#define rseq_smp_rmb() rseq_barrier()
30#define rseq_smp_wmb() rseq_barrier()
31
32#define rseq_smp_load_acquire(p) \
33__extension__ ({ \
34 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
35 rseq_barrier(); \
36 ____p1; \
37})
38
39#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
40
41#define rseq_smp_store_release(p, v) \
42do { \
43 rseq_barrier(); \
44 RSEQ_WRITE_ONCE(*p, v); \
45} while (0)
46
47#ifdef RSEQ_SKIP_FASTPATH
48#include "rseq-skip.h"
49#else /* !RSEQ_SKIP_FASTPATH */
50
51#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
52 start_ip, post_commit_offset, abort_ip) \
dd01d0fb 53 ".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012
MD		 54 ".balign 32\n\t" \
55 __rseq_str(label) ":\n\t" \
56 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
57 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
dd01d0fb
MD		 58 ".popsection\n\t" \
59 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
60 ".quad " __rseq_str(label) "b\n\t" \
784b0012
MD		 61 ".popsection\n\t"
62
dd01d0fb 63
784b0012
MD		 64#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
65 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
66 (post_commit_ip - start_ip), abort_ip)
67
90d9876e
MD		 68/*
69 * Exit points of a rseq critical section consist of all instructions outside
fd622cad
MD		 70 * of the critical section where a critical section can either branch to or
71 * reach through the normal course of its execution. The abort IP and the
72 * post-commit IP are already part of the __rseq_cs section and should not be
73 * explicitly defined as additional exit points. Knowing all exit points is
74 * useful to assist debuggers stepping over the critical section.
90d9876e
MD		 75 */
76#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
77 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
78 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
79 ".popsection\n\t"
80
784b0012
MD		 81#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
82 RSEQ_INJECT_ASM(1) \
83 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
05ebd3f9 84 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
784b0012
MD		 85 __rseq_str(label) ":\n\t"
86
87#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
88 RSEQ_INJECT_ASM(2) \
05ebd3f9 89 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
784b0012
MD		 90 "jnz " __rseq_str(label) "\n\t"
91
92#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
93 ".pushsection __rseq_failure, \"ax\"\n\t" \
a5fe9e95
MD		 94 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
95 ".byte 0x0f, 0xb9, 0x3d\n\t" \
784b0012
MD		 96 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
97 __rseq_str(label) ":\n\t" \
98 teardown \
99 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
100 ".popsection\n\t"
101
102#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
103 ".pushsection __rseq_failure, \"ax\"\n\t" \
104 __rseq_str(label) ":\n\t" \
105 teardown \
106 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
107 ".popsection\n\t"
108
109static inline __attribute__((always_inline))
110int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
111{
112 RSEQ_INJECT_C(9)
113
114 __asm__ __volatile__ goto (
115 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 116 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
117#ifdef RSEQ_COMPARE_TWICE
118 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
119 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
120#endif
784b0012 121 /* Start rseq by storing table entry pointer into rseq_cs. */
7dc4bc29
MD		 122 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
123 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 124 RSEQ_INJECT_ASM(3)
125 "cmpq %[v], %[expect]\n\t"
126 "jnz %l[cmpfail]\n\t"
127 RSEQ_INJECT_ASM(4)
128#ifdef RSEQ_COMPARE_TWICE
7dc4bc29 129 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 130 "cmpq %[v], %[expect]\n\t"
131 "jnz %l[error2]\n\t"
132#endif
133 /* final store */
134 "movq %[newv], %[v]\n\t"
135 "2:\n\t"
136 RSEQ_INJECT_ASM(5)
137 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
138 : /* gcc asm goto does not allow outputs */
139 : [cpu_id] "r" (cpu),
170f840b 140 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 141 [v] "m" (*v),
142 [expect] "r" (expect),
143 [newv] "r" (newv)
144 : "memory", "cc", "rax"
145 RSEQ_INJECT_CLOBBER
146 : abort, cmpfail
147#ifdef RSEQ_COMPARE_TWICE
148 , error1, error2
149#endif
150 );
dd76f2d6 151 rseq_after_asm_goto();
784b0012
MD		 152 return 0;
153abort:
dd76f2d6 154 rseq_after_asm_goto();
784b0012
MD		 155 RSEQ_INJECT_FAILED
156 return -1;
157cmpfail:
dd76f2d6 158 rseq_after_asm_goto();
784b0012
MD		 159 return 1;
160#ifdef RSEQ_COMPARE_TWICE
161error1:
dd76f2d6 162 rseq_after_asm_goto();
784b0012
MD		 163 rseq_bug("cpu_id comparison failed");
164error2:
dd76f2d6 165 rseq_after_asm_goto();
784b0012
MD		 166 rseq_bug("expected value comparison failed");
167#endif
168}
169
170/*
171 * Compare @v against @expectnot. When it does _not_ match, load @v
172 * into @load, and store the content of *@v + voffp into @v.
173 */
174static inline __attribute__((always_inline))
175int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
d35eae6b 176 long voffp, intptr_t *load, int cpu)
784b0012
MD		 177{
178 RSEQ_INJECT_C(9)
179
180 __asm__ __volatile__ goto (
181 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 182 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
183#ifdef RSEQ_COMPARE_TWICE
184 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
185 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
186#endif
784b0012 187 /* Start rseq by storing table entry pointer into rseq_cs. */
7dc4bc29
MD		 188 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
189 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 190 RSEQ_INJECT_ASM(3)
191 "movq %[v], %%rbx\n\t"
192 "cmpq %%rbx, %[expectnot]\n\t"
193 "je %l[cmpfail]\n\t"
194 RSEQ_INJECT_ASM(4)
195#ifdef RSEQ_COMPARE_TWICE
7dc4bc29 196 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 197 "movq %[v], %%rbx\n\t"
198 "cmpq %%rbx, %[expectnot]\n\t"
199 "je %l[error2]\n\t"
200#endif
201 "movq %%rbx, %[load]\n\t"
202 "addq %[voffp], %%rbx\n\t"
203 "movq (%%rbx), %%rbx\n\t"
204 /* final store */
205 "movq %%rbx, %[v]\n\t"
206 "2:\n\t"
207 RSEQ_INJECT_ASM(5)
208 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
209 : /* gcc asm goto does not allow outputs */
210 : [cpu_id] "r" (cpu),
170f840b 211 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 212 /* final store input */
213 [v] "m" (*v),
214 [expectnot] "r" (expectnot),
215 [voffp] "er" (voffp),
216 [load] "m" (*load)
217 : "memory", "cc", "rax", "rbx"
218 RSEQ_INJECT_CLOBBER
219 : abort, cmpfail
220#ifdef RSEQ_COMPARE_TWICE
221 , error1, error2
222#endif
223 );
dd76f2d6 224 rseq_after_asm_goto();
784b0012
MD		 225 return 0;
226abort:
dd76f2d6 227 rseq_after_asm_goto();
784b0012
MD		 228 RSEQ_INJECT_FAILED
229 return -1;
230cmpfail:
dd76f2d6 231 rseq_after_asm_goto();
784b0012
MD		 232 return 1;
233#ifdef RSEQ_COMPARE_TWICE
234error1:
dd76f2d6 235 rseq_after_asm_goto();
784b0012
MD		 236 rseq_bug("cpu_id comparison failed");
237error2:
dd76f2d6 238 rseq_after_asm_goto();
784b0012
MD		 239 rseq_bug("expected value comparison failed");
240#endif
241}
242
243static inline __attribute__((always_inline))
244int rseq_addv(intptr_t *v, intptr_t count, int cpu)
245{
246 RSEQ_INJECT_C(9)
247
248 __asm__ __volatile__ goto (
249 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 250#ifdef RSEQ_COMPARE_TWICE
251 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
252#endif
784b0012 253 /* Start rseq by storing table entry pointer into rseq_cs. */
7dc4bc29
MD		 254 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
255 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 256 RSEQ_INJECT_ASM(3)
257#ifdef RSEQ_COMPARE_TWICE
7dc4bc29 258 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 259#endif
260 /* final store */
261 "addq %[count], %[v]\n\t"
262 "2:\n\t"
263 RSEQ_INJECT_ASM(4)
264 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
265 : /* gcc asm goto does not allow outputs */
266 : [cpu_id] "r" (cpu),
170f840b 267 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 268 /* final store input */
269 [v] "m" (*v),
270 [count] "er" (count)
271 : "memory", "cc", "rax"
272 RSEQ_INJECT_CLOBBER
273 : abort
274#ifdef RSEQ_COMPARE_TWICE
275 , error1
276#endif
277 );
dd76f2d6 278 rseq_after_asm_goto();
784b0012
MD		 279 return 0;
280abort:
dd76f2d6 281 rseq_after_asm_goto();
784b0012
MD		 282 RSEQ_INJECT_FAILED
283 return -1;
284#ifdef RSEQ_COMPARE_TWICE
285error1:
dd76f2d6 286 rseq_after_asm_goto();
784b0012
MD		 287 rseq_bug("cpu_id comparison failed");
288#endif
289}
290
7c02c61b
MD		 291#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
292
293/*
294 * pval = *(ptr+off)
295 * *pval += inc;
296 */
297static inline __attribute__((always_inline))
d35eae6b 298int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
7c02c61b
MD		 299{
300 RSEQ_INJECT_C(9)
301
302 __asm__ __volatile__ goto (
303 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
304#ifdef RSEQ_COMPARE_TWICE
305 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
306#endif
307 /* Start rseq by storing table entry pointer into rseq_cs. */
7dc4bc29
MD		 308 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
309 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
7c02c61b
MD		 310 RSEQ_INJECT_ASM(3)
311#ifdef RSEQ_COMPARE_TWICE
7dc4bc29 312 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
7c02c61b
MD		 313#endif
314 /* get p+v */
315 "movq %[ptr], %%rbx\n\t"
316 "addq %[off], %%rbx\n\t"
317 /* get pv */
318 "movq (%%rbx), %%rcx\n\t"
319 /* *pv += inc */
320 "addq %[inc], (%%rcx)\n\t"
321 "2:\n\t"
322 RSEQ_INJECT_ASM(4)
323 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
324 : /* gcc asm goto does not allow outputs */
325 : [cpu_id] "r" (cpu),
170f840b 326 [rseq_offset] "r" (rseq_offset),
7c02c61b
MD		 327 /* final store input */
328 [ptr] "m" (*ptr),
329 [off] "er" (off),
330 [inc] "er" (inc)
331 : "memory", "cc", "rax", "rbx", "rcx"
332 RSEQ_INJECT_CLOBBER
333 : abort
334#ifdef RSEQ_COMPARE_TWICE
335 , error1
336#endif
337 );
338 rseq_after_asm_goto();
339 return 0;
340abort:
341 rseq_after_asm_goto();
342 RSEQ_INJECT_FAILED
343 return -1;
344#ifdef RSEQ_COMPARE_TWICE
345error1:
346 rseq_after_asm_goto();
347 rseq_bug("cpu_id comparison failed");
348#endif
349}
350
784b0012
MD		 351static inline __attribute__((always_inline))
352int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
353 intptr_t *v2, intptr_t newv2,
354 intptr_t newv, int cpu)
355{
356 RSEQ_INJECT_C(9)
357
358 __asm__ __volatile__ goto (
359 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 360 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
361#ifdef RSEQ_COMPARE_TWICE
362 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
363 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
364#endif
784b0012 365 /* Start rseq by storing table entry pointer into rseq_cs. */
7dc4bc29
MD		 366 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
367 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 368 RSEQ_INJECT_ASM(3)
369 "cmpq %[v], %[expect]\n\t"
370 "jnz %l[cmpfail]\n\t"
371 RSEQ_INJECT_ASM(4)
372#ifdef RSEQ_COMPARE_TWICE
7dc4bc29 373 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 374 "cmpq %[v], %[expect]\n\t"
375 "jnz %l[error2]\n\t"
376#endif
377 /* try store */
378 "movq %[newv2], %[v2]\n\t"
379 RSEQ_INJECT_ASM(5)
380 /* final store */
381 "movq %[newv], %[v]\n\t"
382 "2:\n\t"
383 RSEQ_INJECT_ASM(6)
384 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
385 : /* gcc asm goto does not allow outputs */
386 : [cpu_id] "r" (cpu),
170f840b 387 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 388 /* try store input */
389 [v2] "m" (*v2),
390 [newv2] "r" (newv2),
391 /* final store input */
392 [v] "m" (*v),
393 [expect] "r" (expect),
394 [newv] "r" (newv)
395 : "memory", "cc", "rax"
396 RSEQ_INJECT_CLOBBER
397 : abort, cmpfail
398#ifdef RSEQ_COMPARE_TWICE
399 , error1, error2
400#endif
401 );
dd76f2d6 402 rseq_after_asm_goto();
784b0012
MD		 403 return 0;
404abort:
dd76f2d6 405 rseq_after_asm_goto();
784b0012
MD		 406 RSEQ_INJECT_FAILED
407 return -1;
408cmpfail:
dd76f2d6 409 rseq_after_asm_goto();
784b0012
MD		 410 return 1;
411#ifdef RSEQ_COMPARE_TWICE
412error1:
dd76f2d6 413 rseq_after_asm_goto();
784b0012
MD		 414 rseq_bug("cpu_id comparison failed");
415error2:
dd76f2d6 416 rseq_after_asm_goto();
784b0012
MD		 417 rseq_bug("expected value comparison failed");
418#endif
419}
420
421/* x86-64 is TSO. */
422static inline __attribute__((always_inline))
423int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
424 intptr_t *v2, intptr_t newv2,
425 intptr_t newv, int cpu)
426{
427 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
428}
429
430static inline __attribute__((always_inline))
431int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
432 intptr_t *v2, intptr_t expect2,
433 intptr_t newv, int cpu)
434{
435 RSEQ_INJECT_C(9)
436
437 __asm__ __volatile__ goto (
438 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 439 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
440#ifdef RSEQ_COMPARE_TWICE
441 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
442 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
443 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
444#endif
784b0012 445 /* Start rseq by storing table entry pointer into rseq_cs. */
7dc4bc29
MD		 446 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
447 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 448 RSEQ_INJECT_ASM(3)
449 "cmpq %[v], %[expect]\n\t"
450 "jnz %l[cmpfail]\n\t"
451 RSEQ_INJECT_ASM(4)
452 "cmpq %[v2], %[expect2]\n\t"
453 "jnz %l[cmpfail]\n\t"
454 RSEQ_INJECT_ASM(5)
455#ifdef RSEQ_COMPARE_TWICE
7dc4bc29 456 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 457 "cmpq %[v], %[expect]\n\t"
458 "jnz %l[error2]\n\t"
459 "cmpq %[v2], %[expect2]\n\t"
460 "jnz %l[error3]\n\t"
461#endif
462 /* final store */
463 "movq %[newv], %[v]\n\t"
464 "2:\n\t"
465 RSEQ_INJECT_ASM(6)
466 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
467 : /* gcc asm goto does not allow outputs */
468 : [cpu_id] "r" (cpu),
170f840b 469 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 470 /* cmp2 input */
471 [v2] "m" (*v2),
472 [expect2] "r" (expect2),
473 /* final store input */
474 [v] "m" (*v),
475 [expect] "r" (expect),
476 [newv] "r" (newv)
477 : "memory", "cc", "rax"
478 RSEQ_INJECT_CLOBBER
479 : abort, cmpfail
480#ifdef RSEQ_COMPARE_TWICE
481 , error1, error2, error3
482#endif
483 );
dd76f2d6 484 rseq_after_asm_goto();
784b0012
MD		 485 return 0;
486abort:
dd76f2d6 487 rseq_after_asm_goto();
784b0012
MD		 488 RSEQ_INJECT_FAILED
489 return -1;
490cmpfail:
dd76f2d6 491 rseq_after_asm_goto();
784b0012
MD		 492 return 1;
493#ifdef RSEQ_COMPARE_TWICE
494error1:
dd76f2d6 495 rseq_after_asm_goto();
784b0012
MD		 496 rseq_bug("cpu_id comparison failed");
497error2:
dd76f2d6 498 rseq_after_asm_goto();
784b0012
MD		 499 rseq_bug("1st expected value comparison failed");
500error3:
dd76f2d6 501 rseq_after_asm_goto();
784b0012
MD		 502 rseq_bug("2nd expected value comparison failed");
503#endif
504}
505
506static inline __attribute__((always_inline))
507int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
508 void *dst, void *src, size_t len,
509 intptr_t newv, int cpu)
510{
511 uint64_t rseq_scratch[3];
512
513 RSEQ_INJECT_C(9)
514
515 __asm__ __volatile__ goto (
516 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 517 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
518#ifdef RSEQ_COMPARE_TWICE
519 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
520 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
521#endif
784b0012
MD		 522 "movq %[src], %[rseq_scratch0]\n\t"
523 "movq %[dst], %[rseq_scratch1]\n\t"
524 "movq %[len], %[rseq_scratch2]\n\t"
525 /* Start rseq by storing table entry pointer into rseq_cs. */
7dc4bc29
MD		 526 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
527 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 528 RSEQ_INJECT_ASM(3)
529 "cmpq %[v], %[expect]\n\t"
530 "jnz 5f\n\t"
531 RSEQ_INJECT_ASM(4)
532#ifdef RSEQ_COMPARE_TWICE
7dc4bc29 533 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
784b0012
MD		 534 "cmpq %[v], %[expect]\n\t"
535 "jnz 7f\n\t"
536#endif
537 /* try memcpy */
538 "test %[len], %[len]\n\t" \
539 "jz 333f\n\t" \
540 "222:\n\t" \
541 "movb (%[src]), %%al\n\t" \
542 "movb %%al, (%[dst])\n\t" \
543 "inc %[src]\n\t" \
544 "inc %[dst]\n\t" \
545 "dec %[len]\n\t" \
546 "jnz 222b\n\t" \
547 "333:\n\t" \
548 RSEQ_INJECT_ASM(5)
549 /* final store */
550 "movq %[newv], %[v]\n\t"
551 "2:\n\t"
552 RSEQ_INJECT_ASM(6)
553 /* teardown */
554 "movq %[rseq_scratch2], %[len]\n\t"
555 "movq %[rseq_scratch1], %[dst]\n\t"
556 "movq %[rseq_scratch0], %[src]\n\t"
557 RSEQ_ASM_DEFINE_ABORT(4,
558 "movq %[rseq_scratch2], %[len]\n\t"
559 "movq %[rseq_scratch1], %[dst]\n\t"
560 "movq %[rseq_scratch0], %[src]\n\t",
561 abort)
562 RSEQ_ASM_DEFINE_CMPFAIL(5,
563 "movq %[rseq_scratch2], %[len]\n\t"
564 "movq %[rseq_scratch1], %[dst]\n\t"
565 "movq %[rseq_scratch0], %[src]\n\t",
566 cmpfail)
567#ifdef RSEQ_COMPARE_TWICE
568 RSEQ_ASM_DEFINE_CMPFAIL(6,
569 "movq %[rseq_scratch2], %[len]\n\t"
570 "movq %[rseq_scratch1], %[dst]\n\t"
571 "movq %[rseq_scratch0], %[src]\n\t",
572 error1)
573 RSEQ_ASM_DEFINE_CMPFAIL(7,
574 "movq %[rseq_scratch2], %[len]\n\t"
575 "movq %[rseq_scratch1], %[dst]\n\t"
576 "movq %[rseq_scratch0], %[src]\n\t",
577 error2)
578#endif
579 : /* gcc asm goto does not allow outputs */
580 : [cpu_id] "r" (cpu),
170f840b 581 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 582 /* final store input */
583 [v] "m" (*v),
584 [expect] "r" (expect),
585 [newv] "r" (newv),
586 /* try memcpy input */
587 [dst] "r" (dst),
588 [src] "r" (src),
589 [len] "r" (len),
590 [rseq_scratch0] "m" (rseq_scratch[0]),
591 [rseq_scratch1] "m" (rseq_scratch[1]),
592 [rseq_scratch2] "m" (rseq_scratch[2])
593 : "memory", "cc", "rax"
594 RSEQ_INJECT_CLOBBER
595 : abort, cmpfail
596#ifdef RSEQ_COMPARE_TWICE
597 , error1, error2
598#endif
599 );
dd76f2d6 600 rseq_after_asm_goto();
784b0012
MD		 601 return 0;
602abort:
dd76f2d6 603 rseq_after_asm_goto();
784b0012
MD		 604 RSEQ_INJECT_FAILED
605 return -1;
606cmpfail:
dd76f2d6 607 rseq_after_asm_goto();
784b0012
MD		 608 return 1;
609#ifdef RSEQ_COMPARE_TWICE
610error1:
dd76f2d6 611 rseq_after_asm_goto();
784b0012
MD		 612 rseq_bug("cpu_id comparison failed");
613error2:
dd76f2d6 614 rseq_after_asm_goto();
784b0012
MD		 615 rseq_bug("expected value comparison failed");
616#endif
617}
618
619/* x86-64 is TSO. */
620static inline __attribute__((always_inline))
621int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
622 void *dst, void *src, size_t len,
623 intptr_t newv, int cpu)
624{
625 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
626 newv, cpu);
627}
628
de28c254
MD		 629/*
630 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
631 * into @load.
632 */
633static inline __attribute__((always_inline))
d35eae6b 634int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu)
de28c254
MD		 635{
636 RSEQ_INJECT_C(9)
637
638 __asm__ __volatile__ goto (
639 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
640#ifdef RSEQ_COMPARE_TWICE
641 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
642#endif
643 /* Start rseq by storing table entry pointer into rseq_cs. */
7dc4bc29
MD		 644 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
645 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
de28c254
MD		 646 RSEQ_INJECT_ASM(3)
647 "movq %[p], %%rbx\n\t"
648 RSEQ_INJECT_ASM(4)
649#ifdef RSEQ_COMPARE_TWICE
7dc4bc29 650 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
de28c254
MD		 651#endif
652 "addq %[voffp], %%rbx\n\t"
653 "movq (%%rbx), %%rbx\n\t"
654 "movq %%rbx, %[load]\n\t"
655 "2:\n\t"
656 RSEQ_INJECT_ASM(5)
657 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
658 : /* gcc asm goto does not allow outputs */
659 : [cpu_id] "r" (cpu),
170f840b 660 [rseq_offset] "r" (rseq_offset),
de28c254
MD		 661 /* final store input */
662 [p] "m" (*p),
663 [voffp] "er" (voffp),
664 [load] "m" (*load)
665 : "memory", "cc", "rax", "rbx"
666 RSEQ_INJECT_CLOBBER
667 : abort
668#ifdef RSEQ_COMPARE_TWICE
669 , error1
670#endif
671 );
dd76f2d6 672 rseq_after_asm_goto();
de28c254
MD		 673 return 0;
674abort:
dd76f2d6 675 rseq_after_asm_goto();
de28c254
MD		 676 RSEQ_INJECT_FAILED
677 return -1;
678#ifdef RSEQ_COMPARE_TWICE
679error1:
dd76f2d6 680 rseq_after_asm_goto();
de28c254
MD		 681 rseq_bug("cpu_id comparison failed");
682#endif
683}
684
784b0012
MD		 685#endif /* !RSEQ_SKIP_FASTPATH */
686
f1c6b55b 687#elif defined(__i386__)
784b0012 688
53831176
MD		 689#define RSEQ_ASM_TP_SEGMENT %%gs
690
784b0012
MD		 691#define rseq_smp_mb() \
692 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
693#define rseq_smp_rmb() \
694 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
695#define rseq_smp_wmb() \
696 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
697
698#define rseq_smp_load_acquire(p) \
699__extension__ ({ \
700 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
701 rseq_smp_mb(); \
702 ____p1; \
703})
704
705#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
706
707#define rseq_smp_store_release(p, v) \
708do { \
709 rseq_smp_mb(); \
710 RSEQ_WRITE_ONCE(*p, v); \
711} while (0)
712
713#ifdef RSEQ_SKIP_FASTPATH
714#include "rseq-skip.h"
715#else /* !RSEQ_SKIP_FASTPATH */
716
717/*
718 * Use eax as scratch register and take memory operands as input to
719 * lessen register pressure. Especially needed when compiling in O0.
720 */
721#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
722 start_ip, post_commit_offset, abort_ip) \
dd01d0fb 723 ".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012
MD		 724 ".balign 32\n\t" \
725 __rseq_str(label) ":\n\t" \
726 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
727 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
dd01d0fb
MD		 728 ".popsection\n\t" \
729 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
730 ".long " __rseq_str(label) "b, 0x0\n\t" \
784b0012
MD		 731 ".popsection\n\t"
732
733#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
734 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
735 (post_commit_ip - start_ip), abort_ip)
736
90d9876e
MD		 737/*
738 * Exit points of a rseq critical section consist of all instructions outside
fd622cad
MD		 739 * of the critical section where a critical section can either branch to or
740 * reach through the normal course of its execution. The abort IP and the
741 * post-commit IP are already part of the __rseq_cs section and should not be
742 * explicitly defined as additional exit points. Knowing all exit points is
743 * useful to assist debuggers stepping over the critical section.
90d9876e
MD		 744 */
745#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
746 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
747 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
748 ".popsection\n\t"
749
784b0012
MD		 750#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
751 RSEQ_INJECT_ASM(1) \
05ebd3f9 752 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
784b0012
MD		 753 __rseq_str(label) ":\n\t"
754
755#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
756 RSEQ_INJECT_ASM(2) \
05ebd3f9 757 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
784b0012
MD		 758 "jnz " __rseq_str(label) "\n\t"
759
760#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
761 ".pushsection __rseq_failure, \"ax\"\n\t" \
a5fe9e95
MD		 762 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
763 ".byte 0x0f, 0xb9, 0x3d\n\t" \
784b0012
MD		 764 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
765 __rseq_str(label) ":\n\t" \
766 teardown \
767 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
768 ".popsection\n\t"
769
770#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
771 ".pushsection __rseq_failure, \"ax\"\n\t" \
772 __rseq_str(label) ":\n\t" \
773 teardown \
774 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
775 ".popsection\n\t"
776
777static inline __attribute__((always_inline))
778int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
779{
780 RSEQ_INJECT_C(9)
781
782 __asm__ __volatile__ goto (
783 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 784 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
785#ifdef RSEQ_COMPARE_TWICE
786 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
787 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
788#endif
784b0012 789 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 790 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
791 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 792 RSEQ_INJECT_ASM(3)
793 "cmpl %[v], %[expect]\n\t"
794 "jnz %l[cmpfail]\n\t"
795 RSEQ_INJECT_ASM(4)
796#ifdef RSEQ_COMPARE_TWICE
53831176 797 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 798 "cmpl %[v], %[expect]\n\t"
799 "jnz %l[error2]\n\t"
800#endif
801 /* final store */
802 "movl %[newv], %[v]\n\t"
803 "2:\n\t"
804 RSEQ_INJECT_ASM(5)
805 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
806 : /* gcc asm goto does not allow outputs */
807 : [cpu_id] "r" (cpu),
53831176 808 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 809 [v] "m" (*v),
810 [expect] "r" (expect),
811 [newv] "r" (newv)
812 : "memory", "cc", "eax"
813 RSEQ_INJECT_CLOBBER
814 : abort, cmpfail
815#ifdef RSEQ_COMPARE_TWICE
816 , error1, error2
817#endif
818 );
dd76f2d6 819 rseq_after_asm_goto();
784b0012
MD		 820 return 0;
821abort:
dd76f2d6 822 rseq_after_asm_goto();
784b0012
MD		 823 RSEQ_INJECT_FAILED
824 return -1;
825cmpfail:
dd76f2d6 826 rseq_after_asm_goto();
784b0012
MD		 827 return 1;
828#ifdef RSEQ_COMPARE_TWICE
829error1:
dd76f2d6 830 rseq_after_asm_goto();
784b0012
MD		 831 rseq_bug("cpu_id comparison failed");
832error2:
dd76f2d6 833 rseq_after_asm_goto();
784b0012
MD		 834 rseq_bug("expected value comparison failed");
835#endif
836}
837
838/*
839 * Compare @v against @expectnot. When it does _not_ match, load @v
840 * into @load, and store the content of *@v + voffp into @v.
841 */
842static inline __attribute__((always_inline))
843int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
d35eae6b 844 long voffp, intptr_t *load, int cpu)
784b0012
MD		 845{
846 RSEQ_INJECT_C(9)
847
848 __asm__ __volatile__ goto (
849 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 850 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
851#ifdef RSEQ_COMPARE_TWICE
852 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
853 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
854#endif
784b0012 855 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 856 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
857 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 858 RSEQ_INJECT_ASM(3)
859 "movl %[v], %%ebx\n\t"
860 "cmpl %%ebx, %[expectnot]\n\t"
861 "je %l[cmpfail]\n\t"
862 RSEQ_INJECT_ASM(4)
863#ifdef RSEQ_COMPARE_TWICE
53831176 864 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 865 "movl %[v], %%ebx\n\t"
866 "cmpl %%ebx, %[expectnot]\n\t"
867 "je %l[error2]\n\t"
868#endif
869 "movl %%ebx, %[load]\n\t"
870 "addl %[voffp], %%ebx\n\t"
871 "movl (%%ebx), %%ebx\n\t"
872 /* final store */
873 "movl %%ebx, %[v]\n\t"
874 "2:\n\t"
875 RSEQ_INJECT_ASM(5)
876 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
877 : /* gcc asm goto does not allow outputs */
878 : [cpu_id] "r" (cpu),
53831176 879 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 880 /* final store input */
881 [v] "m" (*v),
882 [expectnot] "r" (expectnot),
883 [voffp] "ir" (voffp),
884 [load] "m" (*load)
885 : "memory", "cc", "eax", "ebx"
886 RSEQ_INJECT_CLOBBER
887 : abort, cmpfail
888#ifdef RSEQ_COMPARE_TWICE
889 , error1, error2
890#endif
891 );
dd76f2d6 892 rseq_after_asm_goto();
784b0012
MD		 893 return 0;
894abort:
dd76f2d6 895 rseq_after_asm_goto();
784b0012
MD		 896 RSEQ_INJECT_FAILED
897 return -1;
898cmpfail:
dd76f2d6 899 rseq_after_asm_goto();
784b0012
MD		 900 return 1;
901#ifdef RSEQ_COMPARE_TWICE
902error1:
dd76f2d6 903 rseq_after_asm_goto();
784b0012
MD		 904 rseq_bug("cpu_id comparison failed");
905error2:
dd76f2d6 906 rseq_after_asm_goto();
784b0012
MD		 907 rseq_bug("expected value comparison failed");
908#endif
909}
910
911static inline __attribute__((always_inline))
912int rseq_addv(intptr_t *v, intptr_t count, int cpu)
913{
914 RSEQ_INJECT_C(9)
915
916 __asm__ __volatile__ goto (
917 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 918#ifdef RSEQ_COMPARE_TWICE
919 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
920#endif
784b0012 921 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 922 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
923 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 924 RSEQ_INJECT_ASM(3)
925#ifdef RSEQ_COMPARE_TWICE
53831176 926 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 927#endif
928 /* final store */
929 "addl %[count], %[v]\n\t"
930 "2:\n\t"
931 RSEQ_INJECT_ASM(4)
932 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
933 : /* gcc asm goto does not allow outputs */
934 : [cpu_id] "r" (cpu),
53831176 935 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 936 /* final store input */
937 [v] "m" (*v),
938 [count] "ir" (count)
939 : "memory", "cc", "eax"
940 RSEQ_INJECT_CLOBBER
941 : abort
942#ifdef RSEQ_COMPARE_TWICE
943 , error1
944#endif
945 );
dd76f2d6 946 rseq_after_asm_goto();
784b0012
MD		 947 return 0;
948abort:
dd76f2d6 949 rseq_after_asm_goto();
784b0012
MD		 950 RSEQ_INJECT_FAILED
951 return -1;
952#ifdef RSEQ_COMPARE_TWICE
953error1:
dd76f2d6 954 rseq_after_asm_goto();
784b0012
MD		 955 rseq_bug("cpu_id comparison failed");
956#endif
957}
958
959static inline __attribute__((always_inline))
960int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
961 intptr_t *v2, intptr_t newv2,
962 intptr_t newv, int cpu)
963{
964 RSEQ_INJECT_C(9)
965
966 __asm__ __volatile__ goto (
967 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 968 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
969#ifdef RSEQ_COMPARE_TWICE
970 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
971 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
972#endif
784b0012 973 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 974 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
975 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 976 RSEQ_INJECT_ASM(3)
977 "cmpl %[v], %[expect]\n\t"
978 "jnz %l[cmpfail]\n\t"
979 RSEQ_INJECT_ASM(4)
980#ifdef RSEQ_COMPARE_TWICE
53831176 981 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 982 "cmpl %[v], %[expect]\n\t"
983 "jnz %l[error2]\n\t"
984#endif
985 /* try store */
986 "movl %[newv2], %%eax\n\t"
987 "movl %%eax, %[v2]\n\t"
988 RSEQ_INJECT_ASM(5)
989 /* final store */
990 "movl %[newv], %[v]\n\t"
991 "2:\n\t"
992 RSEQ_INJECT_ASM(6)
993 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
994 : /* gcc asm goto does not allow outputs */
995 : [cpu_id] "r" (cpu),
53831176 996 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 997 /* try store input */
998 [v2] "m" (*v2),
999 [newv2] "m" (newv2),
1000 /* final store input */
1001 [v] "m" (*v),
1002 [expect] "r" (expect),
1003 [newv] "r" (newv)
1004 : "memory", "cc", "eax"
1005 RSEQ_INJECT_CLOBBER
1006 : abort, cmpfail
1007#ifdef RSEQ_COMPARE_TWICE
1008 , error1, error2
1009#endif
1010 );
dd76f2d6 1011 rseq_after_asm_goto();
784b0012
MD		 1012 return 0;
1013abort:
dd76f2d6 1014 rseq_after_asm_goto();
784b0012
MD		 1015 RSEQ_INJECT_FAILED
1016 return -1;
1017cmpfail:
dd76f2d6 1018 rseq_after_asm_goto();
784b0012
MD		 1019 return 1;
1020#ifdef RSEQ_COMPARE_TWICE
1021error1:
dd76f2d6 1022 rseq_after_asm_goto();
784b0012
MD		 1023 rseq_bug("cpu_id comparison failed");
1024error2:
dd76f2d6 1025 rseq_after_asm_goto();
784b0012
MD		 1026 rseq_bug("expected value comparison failed");
1027#endif
1028}
1029
1030static inline __attribute__((always_inline))
1031int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
1032 intptr_t *v2, intptr_t newv2,
1033 intptr_t newv, int cpu)
1034{
1035 RSEQ_INJECT_C(9)
1036
1037 __asm__ __volatile__ goto (
1038 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 1039 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1040#ifdef RSEQ_COMPARE_TWICE
1041 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1042 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1043#endif
784b0012 1044 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 1045 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1046 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 1047 RSEQ_INJECT_ASM(3)
1048 "movl %[expect], %%eax\n\t"
1049 "cmpl %[v], %%eax\n\t"
1050 "jnz %l[cmpfail]\n\t"
1051 RSEQ_INJECT_ASM(4)
1052#ifdef RSEQ_COMPARE_TWICE
53831176 1053 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 1054 "movl %[expect], %%eax\n\t"
1055 "cmpl %[v], %%eax\n\t"
1056 "jnz %l[error2]\n\t"
1057#endif
1058 /* try store */
1059 "movl %[newv2], %[v2]\n\t"
1060 RSEQ_INJECT_ASM(5)
1061 "lock; addl $0,-128(%%esp)\n\t"
1062 /* final store */
1063 "movl %[newv], %[v]\n\t"
1064 "2:\n\t"
1065 RSEQ_INJECT_ASM(6)
1066 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1067 : /* gcc asm goto does not allow outputs */
1068 : [cpu_id] "r" (cpu),
53831176 1069 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 1070 /* try store input */
1071 [v2] "m" (*v2),
1072 [newv2] "r" (newv2),
1073 /* final store input */
1074 [v] "m" (*v),
1075 [expect] "m" (expect),
1076 [newv] "r" (newv)
1077 : "memory", "cc", "eax"
1078 RSEQ_INJECT_CLOBBER
1079 : abort, cmpfail
1080#ifdef RSEQ_COMPARE_TWICE
1081 , error1, error2
1082#endif
1083 );
dd76f2d6 1084 rseq_after_asm_goto();
784b0012
MD		 1085 return 0;
1086abort:
dd76f2d6 1087 rseq_after_asm_goto();
784b0012
MD		 1088 RSEQ_INJECT_FAILED
1089 return -1;
1090cmpfail:
dd76f2d6 1091 rseq_after_asm_goto();
784b0012
MD		 1092 return 1;
1093#ifdef RSEQ_COMPARE_TWICE
1094error1:
dd76f2d6 1095 rseq_after_asm_goto();
784b0012
MD		 1096 rseq_bug("cpu_id comparison failed");
1097error2:
dd76f2d6 1098 rseq_after_asm_goto();
784b0012
MD		 1099 rseq_bug("expected value comparison failed");
1100#endif
1101
1102}
1103
1104static inline __attribute__((always_inline))
1105int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
1106 intptr_t *v2, intptr_t expect2,
1107 intptr_t newv, int cpu)
1108{
1109 RSEQ_INJECT_C(9)
1110
1111 __asm__ __volatile__ goto (
1112 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 1113 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1114#ifdef RSEQ_COMPARE_TWICE
1115 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1116 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1117 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1118#endif
784b0012 1119 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 1120 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1121 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 1122 RSEQ_INJECT_ASM(3)
1123 "cmpl %[v], %[expect]\n\t"
1124 "jnz %l[cmpfail]\n\t"
1125 RSEQ_INJECT_ASM(4)
1126 "cmpl %[expect2], %[v2]\n\t"
1127 "jnz %l[cmpfail]\n\t"
1128 RSEQ_INJECT_ASM(5)
1129#ifdef RSEQ_COMPARE_TWICE
53831176 1130 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
784b0012
MD		 1131 "cmpl %[v], %[expect]\n\t"
1132 "jnz %l[error2]\n\t"
1133 "cmpl %[expect2], %[v2]\n\t"
1134 "jnz %l[error3]\n\t"
1135#endif
1136 "movl %[newv], %%eax\n\t"
1137 /* final store */
1138 "movl %%eax, %[v]\n\t"
1139 "2:\n\t"
1140 RSEQ_INJECT_ASM(6)
1141 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1142 : /* gcc asm goto does not allow outputs */
1143 : [cpu_id] "r" (cpu),
53831176 1144 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 1145 /* cmp2 input */
1146 [v2] "m" (*v2),
1147 [expect2] "r" (expect2),
1148 /* final store input */
1149 [v] "m" (*v),
1150 [expect] "r" (expect),
1151 [newv] "m" (newv)
1152 : "memory", "cc", "eax"
1153 RSEQ_INJECT_CLOBBER
1154 : abort, cmpfail
1155#ifdef RSEQ_COMPARE_TWICE
1156 , error1, error2, error3
1157#endif
1158 );
dd76f2d6 1159 rseq_after_asm_goto();
784b0012
MD		 1160 return 0;
1161abort:
dd76f2d6 1162 rseq_after_asm_goto();
784b0012
MD		 1163 RSEQ_INJECT_FAILED
1164 return -1;
1165cmpfail:
dd76f2d6 1166 rseq_after_asm_goto();
784b0012
MD		 1167 return 1;
1168#ifdef RSEQ_COMPARE_TWICE
1169error1:
dd76f2d6 1170 rseq_after_asm_goto();
784b0012
MD		 1171 rseq_bug("cpu_id comparison failed");
1172error2:
dd76f2d6 1173 rseq_after_asm_goto();
784b0012
MD		 1174 rseq_bug("1st expected value comparison failed");
1175error3:
dd76f2d6 1176 rseq_after_asm_goto();
784b0012
MD		 1177 rseq_bug("2nd expected value comparison failed");
1178#endif
1179}
1180
1181/* TODO: implement a faster memcpy. */
1182static inline __attribute__((always_inline))
1183int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1184 void *dst, void *src, size_t len,
1185 intptr_t newv, int cpu)
1186{
1187 uint32_t rseq_scratch[3];
1188
1189 RSEQ_INJECT_C(9)
1190
1191 __asm__ __volatile__ goto (
1192 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 1193 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1194#ifdef RSEQ_COMPARE_TWICE
1195 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1196 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1197#endif
784b0012
MD		 1198 "movl %[src], %[rseq_scratch0]\n\t"
1199 "movl %[dst], %[rseq_scratch1]\n\t"
1200 "movl %[len], %[rseq_scratch2]\n\t"
1201 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 1202 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1203 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 1204 RSEQ_INJECT_ASM(3)
1205 "movl %[expect], %%eax\n\t"
1206 "cmpl %%eax, %[v]\n\t"
1207 "jnz 5f\n\t"
1208 RSEQ_INJECT_ASM(4)
1209#ifdef RSEQ_COMPARE_TWICE
53831176 1210 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
784b0012
MD		 1211 "movl %[expect], %%eax\n\t"
1212 "cmpl %%eax, %[v]\n\t"
1213 "jnz 7f\n\t"
1214#endif
1215 /* try memcpy */
1216 "test %[len], %[len]\n\t" \
1217 "jz 333f\n\t" \
1218 "222:\n\t" \
1219 "movb (%[src]), %%al\n\t" \
1220 "movb %%al, (%[dst])\n\t" \
1221 "inc %[src]\n\t" \
1222 "inc %[dst]\n\t" \
1223 "dec %[len]\n\t" \
1224 "jnz 222b\n\t" \
1225 "333:\n\t" \
1226 RSEQ_INJECT_ASM(5)
1227 "movl %[newv], %%eax\n\t"
1228 /* final store */
1229 "movl %%eax, %[v]\n\t"
1230 "2:\n\t"
1231 RSEQ_INJECT_ASM(6)
1232 /* teardown */
1233 "movl %[rseq_scratch2], %[len]\n\t"
1234 "movl %[rseq_scratch1], %[dst]\n\t"
1235 "movl %[rseq_scratch0], %[src]\n\t"
1236 RSEQ_ASM_DEFINE_ABORT(4,
1237 "movl %[rseq_scratch2], %[len]\n\t"
1238 "movl %[rseq_scratch1], %[dst]\n\t"
1239 "movl %[rseq_scratch0], %[src]\n\t",
1240 abort)
1241 RSEQ_ASM_DEFINE_CMPFAIL(5,
1242 "movl %[rseq_scratch2], %[len]\n\t"
1243 "movl %[rseq_scratch1], %[dst]\n\t"
1244 "movl %[rseq_scratch0], %[src]\n\t",
1245 cmpfail)
1246#ifdef RSEQ_COMPARE_TWICE
1247 RSEQ_ASM_DEFINE_CMPFAIL(6,
1248 "movl %[rseq_scratch2], %[len]\n\t"
1249 "movl %[rseq_scratch1], %[dst]\n\t"
1250 "movl %[rseq_scratch0], %[src]\n\t",
1251 error1)
1252 RSEQ_ASM_DEFINE_CMPFAIL(7,
1253 "movl %[rseq_scratch2], %[len]\n\t"
1254 "movl %[rseq_scratch1], %[dst]\n\t"
1255 "movl %[rseq_scratch0], %[src]\n\t",
1256 error2)
1257#endif
1258 : /* gcc asm goto does not allow outputs */
1259 : [cpu_id] "r" (cpu),
53831176 1260 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 1261 /* final store input */
1262 [v] "m" (*v),
1263 [expect] "m" (expect),
1264 [newv] "m" (newv),
1265 /* try memcpy input */
1266 [dst] "r" (dst),
1267 [src] "r" (src),
1268 [len] "r" (len),
1269 [rseq_scratch0] "m" (rseq_scratch[0]),
1270 [rseq_scratch1] "m" (rseq_scratch[1]),
1271 [rseq_scratch2] "m" (rseq_scratch[2])
1272 : "memory", "cc", "eax"
1273 RSEQ_INJECT_CLOBBER
1274 : abort, cmpfail
1275#ifdef RSEQ_COMPARE_TWICE
1276 , error1, error2
1277#endif
1278 );
dd76f2d6 1279 rseq_after_asm_goto();
784b0012
MD		 1280 return 0;
1281abort:
dd76f2d6 1282 rseq_after_asm_goto();
784b0012
MD		 1283 RSEQ_INJECT_FAILED
1284 return -1;
1285cmpfail:
dd76f2d6 1286 rseq_after_asm_goto();
784b0012
MD		 1287 return 1;
1288#ifdef RSEQ_COMPARE_TWICE
1289error1:
dd76f2d6 1290 rseq_after_asm_goto();
784b0012
MD		 1291 rseq_bug("cpu_id comparison failed");
1292error2:
dd76f2d6 1293 rseq_after_asm_goto();
784b0012
MD		 1294 rseq_bug("expected value comparison failed");
1295#endif
1296}
1297
1298/* TODO: implement a faster memcpy. */
1299static inline __attribute__((always_inline))
1300int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1301 void *dst, void *src, size_t len,
1302 intptr_t newv, int cpu)
1303{
1304 uint32_t rseq_scratch[3];
1305
1306 RSEQ_INJECT_C(9)
1307
1308 __asm__ __volatile__ goto (
1309 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 1310 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1311#ifdef RSEQ_COMPARE_TWICE
1312 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1313 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1314#endif
784b0012
MD		 1315 "movl %[src], %[rseq_scratch0]\n\t"
1316 "movl %[dst], %[rseq_scratch1]\n\t"
1317 "movl %[len], %[rseq_scratch2]\n\t"
1318 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 1319 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1320 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
784b0012
MD		 1321 RSEQ_INJECT_ASM(3)
1322 "movl %[expect], %%eax\n\t"
1323 "cmpl %%eax, %[v]\n\t"
1324 "jnz 5f\n\t"
1325 RSEQ_INJECT_ASM(4)
1326#ifdef RSEQ_COMPARE_TWICE
53831176 1327 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
784b0012
MD		 1328 "movl %[expect], %%eax\n\t"
1329 "cmpl %%eax, %[v]\n\t"
1330 "jnz 7f\n\t"
1331#endif
1332 /* try memcpy */
1333 "test %[len], %[len]\n\t" \
1334 "jz 333f\n\t" \
1335 "222:\n\t" \
1336 "movb (%[src]), %%al\n\t" \
1337 "movb %%al, (%[dst])\n\t" \
1338 "inc %[src]\n\t" \
1339 "inc %[dst]\n\t" \
1340 "dec %[len]\n\t" \
1341 "jnz 222b\n\t" \
1342 "333:\n\t" \
1343 RSEQ_INJECT_ASM(5)
1344 "lock; addl $0,-128(%%esp)\n\t"
1345 "movl %[newv], %%eax\n\t"
1346 /* final store */
1347 "movl %%eax, %[v]\n\t"
1348 "2:\n\t"
1349 RSEQ_INJECT_ASM(6)
1350 /* teardown */
1351 "movl %[rseq_scratch2], %[len]\n\t"
1352 "movl %[rseq_scratch1], %[dst]\n\t"
1353 "movl %[rseq_scratch0], %[src]\n\t"
1354 RSEQ_ASM_DEFINE_ABORT(4,
1355 "movl %[rseq_scratch2], %[len]\n\t"
1356 "movl %[rseq_scratch1], %[dst]\n\t"
1357 "movl %[rseq_scratch0], %[src]\n\t",
1358 abort)
1359 RSEQ_ASM_DEFINE_CMPFAIL(5,
1360 "movl %[rseq_scratch2], %[len]\n\t"
1361 "movl %[rseq_scratch1], %[dst]\n\t"
1362 "movl %[rseq_scratch0], %[src]\n\t",
1363 cmpfail)
1364#ifdef RSEQ_COMPARE_TWICE
1365 RSEQ_ASM_DEFINE_CMPFAIL(6,
1366 "movl %[rseq_scratch2], %[len]\n\t"
1367 "movl %[rseq_scratch1], %[dst]\n\t"
1368 "movl %[rseq_scratch0], %[src]\n\t",
1369 error1)
1370 RSEQ_ASM_DEFINE_CMPFAIL(7,
1371 "movl %[rseq_scratch2], %[len]\n\t"
1372 "movl %[rseq_scratch1], %[dst]\n\t"
1373 "movl %[rseq_scratch0], %[src]\n\t",
1374 error2)
1375#endif
1376 : /* gcc asm goto does not allow outputs */
1377 : [cpu_id] "r" (cpu),
53831176 1378 [rseq_offset] "r" (rseq_offset),
784b0012
MD		 1379 /* final store input */
1380 [v] "m" (*v),
1381 [expect] "m" (expect),
1382 [newv] "m" (newv),
1383 /* try memcpy input */
1384 [dst] "r" (dst),
1385 [src] "r" (src),
1386 [len] "r" (len),
1387 [rseq_scratch0] "m" (rseq_scratch[0]),
1388 [rseq_scratch1] "m" (rseq_scratch[1]),
1389 [rseq_scratch2] "m" (rseq_scratch[2])
1390 : "memory", "cc", "eax"
1391 RSEQ_INJECT_CLOBBER
1392 : abort, cmpfail
1393#ifdef RSEQ_COMPARE_TWICE
1394 , error1, error2
1395#endif
1396 );
dd76f2d6 1397 rseq_after_asm_goto();
784b0012
MD		 1398 return 0;
1399abort:
dd76f2d6 1400 rseq_after_asm_goto();
784b0012
MD		 1401 RSEQ_INJECT_FAILED
1402 return -1;
1403cmpfail:
dd76f2d6 1404 rseq_after_asm_goto();
784b0012
MD		 1405 return 1;
1406#ifdef RSEQ_COMPARE_TWICE
1407error1:
dd76f2d6 1408 rseq_after_asm_goto();
784b0012
MD		 1409 rseq_bug("cpu_id comparison failed");
1410error2:
dd76f2d6 1411 rseq_after_asm_goto();
784b0012
MD		 1412 rseq_bug("expected value comparison failed");
1413#endif
1414}
1415
de28c254
MD		 1416/*
1417 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
1418 * into @load.
1419 */
1420static inline __attribute__((always_inline))
d35eae6b 1421int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu)
de28c254
MD		 1422{
1423 RSEQ_INJECT_C(9)
1424
1425 __asm__ __volatile__ goto (
1426 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1427#ifdef RSEQ_COMPARE_TWICE
1428 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1429#endif
1430 /* Start rseq by storing table entry pointer into rseq_cs. */
53831176
MD		 1431 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1432 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
de28c254
MD		 1433 RSEQ_INJECT_ASM(3)
1434 "movl %[p], %%ebx\n\t"
1435 RSEQ_INJECT_ASM(4)
1436#ifdef RSEQ_COMPARE_TWICE
53831176 1437 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
de28c254
MD		 1438#endif
1439 "addl %[voffp], %%ebx\n\t"
1440 "movl (%%ebx), %%ebx\n\t"
1441 "movl %%ebx, %[load]\n\t"
1442 "2:\n\t"
1443 RSEQ_INJECT_ASM(5)
1444 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1445 : /* gcc asm goto does not allow outputs */
1446 : [cpu_id] "r" (cpu),
53831176 1447 [rseq_offset] "r" (rseq_offset),
de28c254
MD		 1448 /* final store input */
1449 [p] "m" (*p),
1450 [voffp] "ir" (voffp),
1451 [load] "m" (*load)
1452 : "memory", "cc", "eax", "ebx"
1453 RSEQ_INJECT_CLOBBER
1454 : abort
1455#ifdef RSEQ_COMPARE_TWICE
1456 , error1
1457#endif
1458 );
dd76f2d6 1459 rseq_after_asm_goto();
de28c254
MD		 1460 return 0;
1461abort:
dd76f2d6 1462 rseq_after_asm_goto();
de28c254
MD		 1463 RSEQ_INJECT_FAILED
1464 return -1;
1465#ifdef RSEQ_COMPARE_TWICE
1466error1:
dd76f2d6 1467 rseq_after_asm_goto();
de28c254
MD		 1468 rseq_bug("cpu_id comparison failed");
1469#endif
1470}
1471
784b0012
MD		 1472#endif /* !RSEQ_SKIP_FASTPATH */
1473
1474#endif
Librseq
This page took 0.095497 seconds and 4 git commands to generate.