projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Fix: use long rather than int for offsets
[librseq.git] / include / rseq / rseq-x86.h
1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
2 /*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #include <stdint.h>
9
10 /*
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
12 * trap in user-space:
13 *
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
16 */
17 #define RSEQ_SIG 0x53053053
18
19 /*
20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22 * address through a "r" input operand.
23 * (TODO: revisit after migration to glibc's ABI)
24 */
25
26 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
27 #define RSEQ_CPU_ID_OFFSET 4
28 #define RSEQ_CS_OFFSET 8
29
30 #ifdef __x86_64__
31
32 #define rseq_smp_mb() \
33 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
34 #define rseq_smp_rmb() rseq_barrier()
35 #define rseq_smp_wmb() rseq_barrier()
36
37 #define rseq_smp_load_acquire(p) \
38 __extension__ ({ \
39 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
40 rseq_barrier(); \
41 ____p1; \
42 })
43
44 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
45
46 #define rseq_smp_store_release(p, v) \
47 do { \
48 rseq_barrier(); \
49 RSEQ_WRITE_ONCE(*p, v); \
50 } while (0)
51
52 #ifdef RSEQ_SKIP_FASTPATH
53 #include "rseq-skip.h"
54 #else /* !RSEQ_SKIP_FASTPATH */
55
56 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
57 start_ip, post_commit_offset, abort_ip) \
58 ".pushsection __rseq_cs, \"aw\"\n\t" \
59 ".balign 32\n\t" \
60 __rseq_str(label) ":\n\t" \
61 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
62 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
63 ".popsection\n\t" \
64 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
65 ".quad " __rseq_str(label) "b\n\t" \
66 ".popsection\n\t"
67
68
69 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
70 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
71 (post_commit_ip - start_ip), abort_ip)
72
73 /*
74 * Exit points of a rseq critical section consist of all instructions outside
75 * of the critical section where a critical section can either branch to or
76 * reach through the normal course of its execution. The abort IP and the
77 * post-commit IP are already part of the __rseq_cs section and should not be
78 * explicitly defined as additional exit points. Knowing all exit points is
79 * useful to assist debuggers stepping over the critical section.
80 */
81 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
82 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
83 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
84 ".popsection\n\t"
85
86 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
87 RSEQ_INJECT_ASM(1) \
88 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
89 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
90 __rseq_str(label) ":\n\t"
91
92 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
93 RSEQ_INJECT_ASM(2) \
94 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
95 "jnz " __rseq_str(label) "\n\t"
96
97 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
98 ".pushsection __rseq_failure, \"ax\"\n\t" \
99 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
100 ".byte 0x0f, 0xb9, 0x3d\n\t" \
101 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
102 __rseq_str(label) ":\n\t" \
103 teardown \
104 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
105 ".popsection\n\t"
106
107 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
108 ".pushsection __rseq_failure, \"ax\"\n\t" \
109 __rseq_str(label) ":\n\t" \
110 teardown \
111 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
112 ".popsection\n\t"
113
114 static inline __attribute__((always_inline))
115 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
116 {
117 RSEQ_INJECT_C(9)
118
119 __asm__ __volatile__ goto (
120 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
121 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
122 #ifdef RSEQ_COMPARE_TWICE
123 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
124 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
125 #endif
126 /* Start rseq by storing table entry pointer into rseq_cs. */
127 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
128 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
129 RSEQ_INJECT_ASM(3)
130 "cmpq %[v], %[expect]\n\t"
131 "jnz %l[cmpfail]\n\t"
132 RSEQ_INJECT_ASM(4)
133 #ifdef RSEQ_COMPARE_TWICE
134 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
135 "cmpq %[v], %[expect]\n\t"
136 "jnz %l[error2]\n\t"
137 #endif
138 /* final store */
139 "movq %[newv], %[v]\n\t"
140 "2:\n\t"
141 RSEQ_INJECT_ASM(5)
142 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
143 : /* gcc asm goto does not allow outputs */
144 : [cpu_id] "r" (cpu),
145 [rseq_abi] "r" (rseq_get_abi()),
146 [v] "m" (*v),
147 [expect] "r" (expect),
148 [newv] "r" (newv)
149 : "memory", "cc", "rax"
150 RSEQ_INJECT_CLOBBER
151 : abort, cmpfail
152 #ifdef RSEQ_COMPARE_TWICE
153 , error1, error2
154 #endif
155 );
156 rseq_after_asm_goto();
157 return 0;
158 abort:
159 rseq_after_asm_goto();
160 RSEQ_INJECT_FAILED
161 return -1;
162 cmpfail:
163 rseq_after_asm_goto();
164 return 1;
165 #ifdef RSEQ_COMPARE_TWICE
166 error1:
167 rseq_after_asm_goto();
168 rseq_bug("cpu_id comparison failed");
169 error2:
170 rseq_after_asm_goto();
171 rseq_bug("expected value comparison failed");
172 #endif
173 }
174
175 /*
176 * Compare @v against @expectnot. When it does _not_ match, load @v
177 * into @load, and store the content of *@v + voffp into @v.
178 */
179 static inline __attribute__((always_inline))
180 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
181 long voffp, intptr_t *load, int cpu)
182 {
183 RSEQ_INJECT_C(9)
184
185 __asm__ __volatile__ goto (
186 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
187 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
188 #ifdef RSEQ_COMPARE_TWICE
189 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
190 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
191 #endif
192 /* Start rseq by storing table entry pointer into rseq_cs. */
193 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
194 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
195 RSEQ_INJECT_ASM(3)
196 "movq %[v], %%rbx\n\t"
197 "cmpq %%rbx, %[expectnot]\n\t"
198 "je %l[cmpfail]\n\t"
199 RSEQ_INJECT_ASM(4)
200 #ifdef RSEQ_COMPARE_TWICE
201 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
202 "movq %[v], %%rbx\n\t"
203 "cmpq %%rbx, %[expectnot]\n\t"
204 "je %l[error2]\n\t"
205 #endif
206 "movq %%rbx, %[load]\n\t"
207 "addq %[voffp], %%rbx\n\t"
208 "movq (%%rbx), %%rbx\n\t"
209 /* final store */
210 "movq %%rbx, %[v]\n\t"
211 "2:\n\t"
212 RSEQ_INJECT_ASM(5)
213 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
214 : /* gcc asm goto does not allow outputs */
215 : [cpu_id] "r" (cpu),
216 [rseq_abi] "r" (rseq_get_abi()),
217 /* final store input */
218 [v] "m" (*v),
219 [expectnot] "r" (expectnot),
220 [voffp] "er" (voffp),
221 [load] "m" (*load)
222 : "memory", "cc", "rax", "rbx"
223 RSEQ_INJECT_CLOBBER
224 : abort, cmpfail
225 #ifdef RSEQ_COMPARE_TWICE
226 , error1, error2
227 #endif
228 );
229 rseq_after_asm_goto();
230 return 0;
231 abort:
232 rseq_after_asm_goto();
233 RSEQ_INJECT_FAILED
234 return -1;
235 cmpfail:
236 rseq_after_asm_goto();
237 return 1;
238 #ifdef RSEQ_COMPARE_TWICE
239 error1:
240 rseq_after_asm_goto();
241 rseq_bug("cpu_id comparison failed");
242 error2:
243 rseq_after_asm_goto();
244 rseq_bug("expected value comparison failed");
245 #endif
246 }
247
248 static inline __attribute__((always_inline))
249 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
250 {
251 RSEQ_INJECT_C(9)
252
253 __asm__ __volatile__ goto (
254 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
255 #ifdef RSEQ_COMPARE_TWICE
256 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
257 #endif
258 /* Start rseq by storing table entry pointer into rseq_cs. */
259 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
260 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
261 RSEQ_INJECT_ASM(3)
262 #ifdef RSEQ_COMPARE_TWICE
263 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
264 #endif
265 /* final store */
266 "addq %[count], %[v]\n\t"
267 "2:\n\t"
268 RSEQ_INJECT_ASM(4)
269 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
270 : /* gcc asm goto does not allow outputs */
271 : [cpu_id] "r" (cpu),
272 [rseq_abi] "r" (rseq_get_abi()),
273 /* final store input */
274 [v] "m" (*v),
275 [count] "er" (count)
276 : "memory", "cc", "rax"
277 RSEQ_INJECT_CLOBBER
278 : abort
279 #ifdef RSEQ_COMPARE_TWICE
280 , error1
281 #endif
282 );
283 rseq_after_asm_goto();
284 return 0;
285 abort:
286 rseq_after_asm_goto();
287 RSEQ_INJECT_FAILED
288 return -1;
289 #ifdef RSEQ_COMPARE_TWICE
290 error1:
291 rseq_after_asm_goto();
292 rseq_bug("cpu_id comparison failed");
293 #endif
294 }
295
296 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
297
298 /*
299 * pval = *(ptr+off)
300 * *pval += inc;
301 */
302 static inline __attribute__((always_inline))
303 int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
304 {
305 RSEQ_INJECT_C(9)
306
307 __asm__ __volatile__ goto (
308 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
309 #ifdef RSEQ_COMPARE_TWICE
310 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
311 #endif
312 /* Start rseq by storing table entry pointer into rseq_cs. */
313 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
314 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
315 RSEQ_INJECT_ASM(3)
316 #ifdef RSEQ_COMPARE_TWICE
317 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
318 #endif
319 /* get p+v */
320 "movq %[ptr], %%rbx\n\t"
321 "addq %[off], %%rbx\n\t"
322 /* get pv */
323 "movq (%%rbx), %%rcx\n\t"
324 /* *pv += inc */
325 "addq %[inc], (%%rcx)\n\t"
326 "2:\n\t"
327 RSEQ_INJECT_ASM(4)
328 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
329 : /* gcc asm goto does not allow outputs */
330 : [cpu_id] "r" (cpu),
331 [rseq_abi] "r" (rseq_get_abi()),
332 /* final store input */
333 [ptr] "m" (*ptr),
334 [off] "er" (off),
335 [inc] "er" (inc)
336 : "memory", "cc", "rax", "rbx", "rcx"
337 RSEQ_INJECT_CLOBBER
338 : abort
339 #ifdef RSEQ_COMPARE_TWICE
340 , error1
341 #endif
342 );
343 rseq_after_asm_goto();
344 return 0;
345 abort:
346 rseq_after_asm_goto();
347 RSEQ_INJECT_FAILED
348 return -1;
349 #ifdef RSEQ_COMPARE_TWICE
350 error1:
351 rseq_after_asm_goto();
352 rseq_bug("cpu_id comparison failed");
353 #endif
354 }
355
356 static inline __attribute__((always_inline))
357 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
358 intptr_t *v2, intptr_t newv2,
359 intptr_t newv, int cpu)
360 {
361 RSEQ_INJECT_C(9)
362
363 __asm__ __volatile__ goto (
364 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
365 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
366 #ifdef RSEQ_COMPARE_TWICE
367 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
368 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
369 #endif
370 /* Start rseq by storing table entry pointer into rseq_cs. */
371 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
372 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
373 RSEQ_INJECT_ASM(3)
374 "cmpq %[v], %[expect]\n\t"
375 "jnz %l[cmpfail]\n\t"
376 RSEQ_INJECT_ASM(4)
377 #ifdef RSEQ_COMPARE_TWICE
378 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
379 "cmpq %[v], %[expect]\n\t"
380 "jnz %l[error2]\n\t"
381 #endif
382 /* try store */
383 "movq %[newv2], %[v2]\n\t"
384 RSEQ_INJECT_ASM(5)
385 /* final store */
386 "movq %[newv], %[v]\n\t"
387 "2:\n\t"
388 RSEQ_INJECT_ASM(6)
389 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
390 : /* gcc asm goto does not allow outputs */
391 : [cpu_id] "r" (cpu),
392 [rseq_abi] "r" (rseq_get_abi()),
393 /* try store input */
394 [v2] "m" (*v2),
395 [newv2] "r" (newv2),
396 /* final store input */
397 [v] "m" (*v),
398 [expect] "r" (expect),
399 [newv] "r" (newv)
400 : "memory", "cc", "rax"
401 RSEQ_INJECT_CLOBBER
402 : abort, cmpfail
403 #ifdef RSEQ_COMPARE_TWICE
404 , error1, error2
405 #endif
406 );
407 rseq_after_asm_goto();
408 return 0;
409 abort:
410 rseq_after_asm_goto();
411 RSEQ_INJECT_FAILED
412 return -1;
413 cmpfail:
414 rseq_after_asm_goto();
415 return 1;
416 #ifdef RSEQ_COMPARE_TWICE
417 error1:
418 rseq_after_asm_goto();
419 rseq_bug("cpu_id comparison failed");
420 error2:
421 rseq_after_asm_goto();
422 rseq_bug("expected value comparison failed");
423 #endif
424 }
425
426 /* x86-64 is TSO. */
427 static inline __attribute__((always_inline))
428 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
429 intptr_t *v2, intptr_t newv2,
430 intptr_t newv, int cpu)
431 {
432 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
433 }
434
435 static inline __attribute__((always_inline))
436 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
437 intptr_t *v2, intptr_t expect2,
438 intptr_t newv, int cpu)
439 {
440 RSEQ_INJECT_C(9)
441
442 __asm__ __volatile__ goto (
443 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
444 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
445 #ifdef RSEQ_COMPARE_TWICE
446 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
447 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
448 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
449 #endif
450 /* Start rseq by storing table entry pointer into rseq_cs. */
451 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
452 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
453 RSEQ_INJECT_ASM(3)
454 "cmpq %[v], %[expect]\n\t"
455 "jnz %l[cmpfail]\n\t"
456 RSEQ_INJECT_ASM(4)
457 "cmpq %[v2], %[expect2]\n\t"
458 "jnz %l[cmpfail]\n\t"
459 RSEQ_INJECT_ASM(5)
460 #ifdef RSEQ_COMPARE_TWICE
461 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
462 "cmpq %[v], %[expect]\n\t"
463 "jnz %l[error2]\n\t"
464 "cmpq %[v2], %[expect2]\n\t"
465 "jnz %l[error3]\n\t"
466 #endif
467 /* final store */
468 "movq %[newv], %[v]\n\t"
469 "2:\n\t"
470 RSEQ_INJECT_ASM(6)
471 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
472 : /* gcc asm goto does not allow outputs */
473 : [cpu_id] "r" (cpu),
474 [rseq_abi] "r" (rseq_get_abi()),
475 /* cmp2 input */
476 [v2] "m" (*v2),
477 [expect2] "r" (expect2),
478 /* final store input */
479 [v] "m" (*v),
480 [expect] "r" (expect),
481 [newv] "r" (newv)
482 : "memory", "cc", "rax"
483 RSEQ_INJECT_CLOBBER
484 : abort, cmpfail
485 #ifdef RSEQ_COMPARE_TWICE
486 , error1, error2, error3
487 #endif
488 );
489 rseq_after_asm_goto();
490 return 0;
491 abort:
492 rseq_after_asm_goto();
493 RSEQ_INJECT_FAILED
494 return -1;
495 cmpfail:
496 rseq_after_asm_goto();
497 return 1;
498 #ifdef RSEQ_COMPARE_TWICE
499 error1:
500 rseq_after_asm_goto();
501 rseq_bug("cpu_id comparison failed");
502 error2:
503 rseq_after_asm_goto();
504 rseq_bug("1st expected value comparison failed");
505 error3:
506 rseq_after_asm_goto();
507 rseq_bug("2nd expected value comparison failed");
508 #endif
509 }
510
511 static inline __attribute__((always_inline))
512 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
513 void *dst, void *src, size_t len,
514 intptr_t newv, int cpu)
515 {
516 uint64_t rseq_scratch[3];
517
518 RSEQ_INJECT_C(9)
519
520 __asm__ __volatile__ goto (
521 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
522 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
523 #ifdef RSEQ_COMPARE_TWICE
524 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
525 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
526 #endif
527 "movq %[src], %[rseq_scratch0]\n\t"
528 "movq %[dst], %[rseq_scratch1]\n\t"
529 "movq %[len], %[rseq_scratch2]\n\t"
530 /* Start rseq by storing table entry pointer into rseq_cs. */
531 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
532 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
533 RSEQ_INJECT_ASM(3)
534 "cmpq %[v], %[expect]\n\t"
535 "jnz 5f\n\t"
536 RSEQ_INJECT_ASM(4)
537 #ifdef RSEQ_COMPARE_TWICE
538 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
539 "cmpq %[v], %[expect]\n\t"
540 "jnz 7f\n\t"
541 #endif
542 /* try memcpy */
543 "test %[len], %[len]\n\t" \
544 "jz 333f\n\t" \
545 "222:\n\t" \
546 "movb (%[src]), %%al\n\t" \
547 "movb %%al, (%[dst])\n\t" \
548 "inc %[src]\n\t" \
549 "inc %[dst]\n\t" \
550 "dec %[len]\n\t" \
551 "jnz 222b\n\t" \
552 "333:\n\t" \
553 RSEQ_INJECT_ASM(5)
554 /* final store */
555 "movq %[newv], %[v]\n\t"
556 "2:\n\t"
557 RSEQ_INJECT_ASM(6)
558 /* teardown */
559 "movq %[rseq_scratch2], %[len]\n\t"
560 "movq %[rseq_scratch1], %[dst]\n\t"
561 "movq %[rseq_scratch0], %[src]\n\t"
562 RSEQ_ASM_DEFINE_ABORT(4,
563 "movq %[rseq_scratch2], %[len]\n\t"
564 "movq %[rseq_scratch1], %[dst]\n\t"
565 "movq %[rseq_scratch0], %[src]\n\t",
566 abort)
567 RSEQ_ASM_DEFINE_CMPFAIL(5,
568 "movq %[rseq_scratch2], %[len]\n\t"
569 "movq %[rseq_scratch1], %[dst]\n\t"
570 "movq %[rseq_scratch0], %[src]\n\t",
571 cmpfail)
572 #ifdef RSEQ_COMPARE_TWICE
573 RSEQ_ASM_DEFINE_CMPFAIL(6,
574 "movq %[rseq_scratch2], %[len]\n\t"
575 "movq %[rseq_scratch1], %[dst]\n\t"
576 "movq %[rseq_scratch0], %[src]\n\t",
577 error1)
578 RSEQ_ASM_DEFINE_CMPFAIL(7,
579 "movq %[rseq_scratch2], %[len]\n\t"
580 "movq %[rseq_scratch1], %[dst]\n\t"
581 "movq %[rseq_scratch0], %[src]\n\t",
582 error2)
583 #endif
584 : /* gcc asm goto does not allow outputs */
585 : [cpu_id] "r" (cpu),
586 [rseq_abi] "r" (rseq_get_abi()),
587 /* final store input */
588 [v] "m" (*v),
589 [expect] "r" (expect),
590 [newv] "r" (newv),
591 /* try memcpy input */
592 [dst] "r" (dst),
593 [src] "r" (src),
594 [len] "r" (len),
595 [rseq_scratch0] "m" (rseq_scratch[0]),
596 [rseq_scratch1] "m" (rseq_scratch[1]),
597 [rseq_scratch2] "m" (rseq_scratch[2])
598 : "memory", "cc", "rax"
599 RSEQ_INJECT_CLOBBER
600 : abort, cmpfail
601 #ifdef RSEQ_COMPARE_TWICE
602 , error1, error2
603 #endif
604 );
605 rseq_after_asm_goto();
606 return 0;
607 abort:
608 rseq_after_asm_goto();
609 RSEQ_INJECT_FAILED
610 return -1;
611 cmpfail:
612 rseq_after_asm_goto();
613 return 1;
614 #ifdef RSEQ_COMPARE_TWICE
615 error1:
616 rseq_after_asm_goto();
617 rseq_bug("cpu_id comparison failed");
618 error2:
619 rseq_after_asm_goto();
620 rseq_bug("expected value comparison failed");
621 #endif
622 }
623
624 /* x86-64 is TSO. */
625 static inline __attribute__((always_inline))
626 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
627 void *dst, void *src, size_t len,
628 intptr_t newv, int cpu)
629 {
630 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
631 newv, cpu);
632 }
633
634 /*
635 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
636 * into @load.
637 */
638 static inline __attribute__((always_inline))
639 int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu)
640 {
641 RSEQ_INJECT_C(9)
642
643 __asm__ __volatile__ goto (
644 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
645 #ifdef RSEQ_COMPARE_TWICE
646 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
647 #endif
648 /* Start rseq by storing table entry pointer into rseq_cs. */
649 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
650 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
651 RSEQ_INJECT_ASM(3)
652 "movq %[p], %%rbx\n\t"
653 RSEQ_INJECT_ASM(4)
654 #ifdef RSEQ_COMPARE_TWICE
655 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
656 #endif
657 "addq %[voffp], %%rbx\n\t"
658 "movq (%%rbx), %%rbx\n\t"
659 "movq %%rbx, %[load]\n\t"
660 "2:\n\t"
661 RSEQ_INJECT_ASM(5)
662 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
663 : /* gcc asm goto does not allow outputs */
664 : [cpu_id] "r" (cpu),
665 [rseq_abi] "r" (rseq_get_abi()),
666 /* final store input */
667 [p] "m" (*p),
668 [voffp] "er" (voffp),
669 [load] "m" (*load)
670 : "memory", "cc", "rax", "rbx"
671 RSEQ_INJECT_CLOBBER
672 : abort
673 #ifdef RSEQ_COMPARE_TWICE
674 , error1
675 #endif
676 );
677 rseq_after_asm_goto();
678 return 0;
679 abort:
680 rseq_after_asm_goto();
681 RSEQ_INJECT_FAILED
682 return -1;
683 #ifdef RSEQ_COMPARE_TWICE
684 error1:
685 rseq_after_asm_goto();
686 rseq_bug("cpu_id comparison failed");
687 #endif
688 }
689
690 #endif /* !RSEQ_SKIP_FASTPATH */
691
692 #elif defined(__i386__)
693
694 #define rseq_smp_mb() \
695 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
696 #define rseq_smp_rmb() \
697 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
698 #define rseq_smp_wmb() \
699 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
700
701 #define rseq_smp_load_acquire(p) \
702 __extension__ ({ \
703 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
704 rseq_smp_mb(); \
705 ____p1; \
706 })
707
708 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
709
710 #define rseq_smp_store_release(p, v) \
711 do { \
712 rseq_smp_mb(); \
713 RSEQ_WRITE_ONCE(*p, v); \
714 } while (0)
715
716 #ifdef RSEQ_SKIP_FASTPATH
717 #include "rseq-skip.h"
718 #else /* !RSEQ_SKIP_FASTPATH */
719
720 /*
721 * Use eax as scratch register and take memory operands as input to
722 * lessen register pressure. Especially needed when compiling in O0.
723 */
724 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
725 start_ip, post_commit_offset, abort_ip) \
726 ".pushsection __rseq_cs, \"aw\"\n\t" \
727 ".balign 32\n\t" \
728 __rseq_str(label) ":\n\t" \
729 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
730 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
731 ".popsection\n\t" \
732 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
733 ".long " __rseq_str(label) "b, 0x0\n\t" \
734 ".popsection\n\t"
735
736 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
737 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
738 (post_commit_ip - start_ip), abort_ip)
739
740 /*
741 * Exit points of a rseq critical section consist of all instructions outside
742 * of the critical section where a critical section can either branch to or
743 * reach through the normal course of its execution. The abort IP and the
744 * post-commit IP are already part of the __rseq_cs section and should not be
745 * explicitly defined as additional exit points. Knowing all exit points is
746 * useful to assist debuggers stepping over the critical section.
747 */
748 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
749 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
750 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
751 ".popsection\n\t"
752
753 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
754 RSEQ_INJECT_ASM(1) \
755 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
756 __rseq_str(label) ":\n\t"
757
758 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
759 RSEQ_INJECT_ASM(2) \
760 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
761 "jnz " __rseq_str(label) "\n\t"
762
763 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
764 ".pushsection __rseq_failure, \"ax\"\n\t" \
765 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
766 ".byte 0x0f, 0xb9, 0x3d\n\t" \
767 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
768 __rseq_str(label) ":\n\t" \
769 teardown \
770 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
771 ".popsection\n\t"
772
773 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
774 ".pushsection __rseq_failure, \"ax\"\n\t" \
775 __rseq_str(label) ":\n\t" \
776 teardown \
777 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
778 ".popsection\n\t"
779
780 static inline __attribute__((always_inline))
781 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
782 {
783 RSEQ_INJECT_C(9)
784
785 __asm__ __volatile__ goto (
786 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
787 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
788 #ifdef RSEQ_COMPARE_TWICE
789 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
790 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
791 #endif
792 /* Start rseq by storing table entry pointer into rseq_cs. */
793 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
794 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
795 RSEQ_INJECT_ASM(3)
796 "cmpl %[v], %[expect]\n\t"
797 "jnz %l[cmpfail]\n\t"
798 RSEQ_INJECT_ASM(4)
799 #ifdef RSEQ_COMPARE_TWICE
800 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
801 "cmpl %[v], %[expect]\n\t"
802 "jnz %l[error2]\n\t"
803 #endif
804 /* final store */
805 "movl %[newv], %[v]\n\t"
806 "2:\n\t"
807 RSEQ_INJECT_ASM(5)
808 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
809 : /* gcc asm goto does not allow outputs */
810 : [cpu_id] "r" (cpu),
811 [rseq_abi] "r" (rseq_get_abi()),
812 [v] "m" (*v),
813 [expect] "r" (expect),
814 [newv] "r" (newv)
815 : "memory", "cc", "eax"
816 RSEQ_INJECT_CLOBBER
817 : abort, cmpfail
818 #ifdef RSEQ_COMPARE_TWICE
819 , error1, error2
820 #endif
821 );
822 rseq_after_asm_goto();
823 return 0;
824 abort:
825 rseq_after_asm_goto();
826 RSEQ_INJECT_FAILED
827 return -1;
828 cmpfail:
829 rseq_after_asm_goto();
830 return 1;
831 #ifdef RSEQ_COMPARE_TWICE
832 error1:
833 rseq_after_asm_goto();
834 rseq_bug("cpu_id comparison failed");
835 error2:
836 rseq_after_asm_goto();
837 rseq_bug("expected value comparison failed");
838 #endif
839 }
840
841 /*
842 * Compare @v against @expectnot. When it does _not_ match, load @v
843 * into @load, and store the content of *@v + voffp into @v.
844 */
845 static inline __attribute__((always_inline))
846 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
847 long voffp, intptr_t *load, int cpu)
848 {
849 RSEQ_INJECT_C(9)
850
851 __asm__ __volatile__ goto (
852 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
853 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
854 #ifdef RSEQ_COMPARE_TWICE
855 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
856 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
857 #endif
858 /* Start rseq by storing table entry pointer into rseq_cs. */
859 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
860 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
861 RSEQ_INJECT_ASM(3)
862 "movl %[v], %%ebx\n\t"
863 "cmpl %%ebx, %[expectnot]\n\t"
864 "je %l[cmpfail]\n\t"
865 RSEQ_INJECT_ASM(4)
866 #ifdef RSEQ_COMPARE_TWICE
867 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
868 "movl %[v], %%ebx\n\t"
869 "cmpl %%ebx, %[expectnot]\n\t"
870 "je %l[error2]\n\t"
871 #endif
872 "movl %%ebx, %[load]\n\t"
873 "addl %[voffp], %%ebx\n\t"
874 "movl (%%ebx), %%ebx\n\t"
875 /* final store */
876 "movl %%ebx, %[v]\n\t"
877 "2:\n\t"
878 RSEQ_INJECT_ASM(5)
879 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
880 : /* gcc asm goto does not allow outputs */
881 : [cpu_id] "r" (cpu),
882 [rseq_abi] "r" (rseq_get_abi()),
883 /* final store input */
884 [v] "m" (*v),
885 [expectnot] "r" (expectnot),
886 [voffp] "ir" (voffp),
887 [load] "m" (*load)
888 : "memory", "cc", "eax", "ebx"
889 RSEQ_INJECT_CLOBBER
890 : abort, cmpfail
891 #ifdef RSEQ_COMPARE_TWICE
892 , error1, error2
893 #endif
894 );
895 rseq_after_asm_goto();
896 return 0;
897 abort:
898 rseq_after_asm_goto();
899 RSEQ_INJECT_FAILED
900 return -1;
901 cmpfail:
902 rseq_after_asm_goto();
903 return 1;
904 #ifdef RSEQ_COMPARE_TWICE
905 error1:
906 rseq_after_asm_goto();
907 rseq_bug("cpu_id comparison failed");
908 error2:
909 rseq_after_asm_goto();
910 rseq_bug("expected value comparison failed");
911 #endif
912 }
913
914 static inline __attribute__((always_inline))
915 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
916 {
917 RSEQ_INJECT_C(9)
918
919 __asm__ __volatile__ goto (
920 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
921 #ifdef RSEQ_COMPARE_TWICE
922 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
923 #endif
924 /* Start rseq by storing table entry pointer into rseq_cs. */
925 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
926 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
927 RSEQ_INJECT_ASM(3)
928 #ifdef RSEQ_COMPARE_TWICE
929 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
930 #endif
931 /* final store */
932 "addl %[count], %[v]\n\t"
933 "2:\n\t"
934 RSEQ_INJECT_ASM(4)
935 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
936 : /* gcc asm goto does not allow outputs */
937 : [cpu_id] "r" (cpu),
938 [rseq_abi] "r" (rseq_get_abi()),
939 /* final store input */
940 [v] "m" (*v),
941 [count] "ir" (count)
942 : "memory", "cc", "eax"
943 RSEQ_INJECT_CLOBBER
944 : abort
945 #ifdef RSEQ_COMPARE_TWICE
946 , error1
947 #endif
948 );
949 rseq_after_asm_goto();
950 return 0;
951 abort:
952 rseq_after_asm_goto();
953 RSEQ_INJECT_FAILED
954 return -1;
955 #ifdef RSEQ_COMPARE_TWICE
956 error1:
957 rseq_after_asm_goto();
958 rseq_bug("cpu_id comparison failed");
959 #endif
960 }
961
962 static inline __attribute__((always_inline))
963 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
964 intptr_t *v2, intptr_t newv2,
965 intptr_t newv, int cpu)
966 {
967 RSEQ_INJECT_C(9)
968
969 __asm__ __volatile__ goto (
970 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
971 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
972 #ifdef RSEQ_COMPARE_TWICE
973 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
974 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
975 #endif
976 /* Start rseq by storing table entry pointer into rseq_cs. */
977 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
978 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
979 RSEQ_INJECT_ASM(3)
980 "cmpl %[v], %[expect]\n\t"
981 "jnz %l[cmpfail]\n\t"
982 RSEQ_INJECT_ASM(4)
983 #ifdef RSEQ_COMPARE_TWICE
984 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
985 "cmpl %[v], %[expect]\n\t"
986 "jnz %l[error2]\n\t"
987 #endif
988 /* try store */
989 "movl %[newv2], %%eax\n\t"
990 "movl %%eax, %[v2]\n\t"
991 RSEQ_INJECT_ASM(5)
992 /* final store */
993 "movl %[newv], %[v]\n\t"
994 "2:\n\t"
995 RSEQ_INJECT_ASM(6)
996 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
997 : /* gcc asm goto does not allow outputs */
998 : [cpu_id] "r" (cpu),
999 [rseq_abi] "r" (rseq_get_abi()),
1000 /* try store input */
1001 [v2] "m" (*v2),
1002 [newv2] "m" (newv2),
1003 /* final store input */
1004 [v] "m" (*v),
1005 [expect] "r" (expect),
1006 [newv] "r" (newv)
1007 : "memory", "cc", "eax"
1008 RSEQ_INJECT_CLOBBER
1009 : abort, cmpfail
1010 #ifdef RSEQ_COMPARE_TWICE
1011 , error1, error2
1012 #endif
1013 );
1014 rseq_after_asm_goto();
1015 return 0;
1016 abort:
1017 rseq_after_asm_goto();
1018 RSEQ_INJECT_FAILED
1019 return -1;
1020 cmpfail:
1021 rseq_after_asm_goto();
1022 return 1;
1023 #ifdef RSEQ_COMPARE_TWICE
1024 error1:
1025 rseq_after_asm_goto();
1026 rseq_bug("cpu_id comparison failed");
1027 error2:
1028 rseq_after_asm_goto();
1029 rseq_bug("expected value comparison failed");
1030 #endif
1031 }
1032
1033 static inline __attribute__((always_inline))
1034 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
1035 intptr_t *v2, intptr_t newv2,
1036 intptr_t newv, int cpu)
1037 {
1038 RSEQ_INJECT_C(9)
1039
1040 __asm__ __volatile__ goto (
1041 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1042 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1043 #ifdef RSEQ_COMPARE_TWICE
1044 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1045 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1046 #endif
1047 /* Start rseq by storing table entry pointer into rseq_cs. */
1048 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1049 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1050 RSEQ_INJECT_ASM(3)
1051 "movl %[expect], %%eax\n\t"
1052 "cmpl %[v], %%eax\n\t"
1053 "jnz %l[cmpfail]\n\t"
1054 RSEQ_INJECT_ASM(4)
1055 #ifdef RSEQ_COMPARE_TWICE
1056 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1057 "movl %[expect], %%eax\n\t"
1058 "cmpl %[v], %%eax\n\t"
1059 "jnz %l[error2]\n\t"
1060 #endif
1061 /* try store */
1062 "movl %[newv2], %[v2]\n\t"
1063 RSEQ_INJECT_ASM(5)
1064 "lock; addl $0,-128(%%esp)\n\t"
1065 /* final store */
1066 "movl %[newv], %[v]\n\t"
1067 "2:\n\t"
1068 RSEQ_INJECT_ASM(6)
1069 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1070 : /* gcc asm goto does not allow outputs */
1071 : [cpu_id] "r" (cpu),
1072 [rseq_abi] "r" (rseq_get_abi()),
1073 /* try store input */
1074 [v2] "m" (*v2),
1075 [newv2] "r" (newv2),
1076 /* final store input */
1077 [v] "m" (*v),
1078 [expect] "m" (expect),
1079 [newv] "r" (newv)
1080 : "memory", "cc", "eax"
1081 RSEQ_INJECT_CLOBBER
1082 : abort, cmpfail
1083 #ifdef RSEQ_COMPARE_TWICE
1084 , error1, error2
1085 #endif
1086 );
1087 rseq_after_asm_goto();
1088 return 0;
1089 abort:
1090 rseq_after_asm_goto();
1091 RSEQ_INJECT_FAILED
1092 return -1;
1093 cmpfail:
1094 rseq_after_asm_goto();
1095 return 1;
1096 #ifdef RSEQ_COMPARE_TWICE
1097 error1:
1098 rseq_after_asm_goto();
1099 rseq_bug("cpu_id comparison failed");
1100 error2:
1101 rseq_after_asm_goto();
1102 rseq_bug("expected value comparison failed");
1103 #endif
1104
1105 }
1106
1107 static inline __attribute__((always_inline))
1108 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
1109 intptr_t *v2, intptr_t expect2,
1110 intptr_t newv, int cpu)
1111 {
1112 RSEQ_INJECT_C(9)
1113
1114 __asm__ __volatile__ goto (
1115 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1116 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1117 #ifdef RSEQ_COMPARE_TWICE
1118 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1119 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1120 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1121 #endif
1122 /* Start rseq by storing table entry pointer into rseq_cs. */
1123 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1124 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1125 RSEQ_INJECT_ASM(3)
1126 "cmpl %[v], %[expect]\n\t"
1127 "jnz %l[cmpfail]\n\t"
1128 RSEQ_INJECT_ASM(4)
1129 "cmpl %[expect2], %[v2]\n\t"
1130 "jnz %l[cmpfail]\n\t"
1131 RSEQ_INJECT_ASM(5)
1132 #ifdef RSEQ_COMPARE_TWICE
1133 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1134 "cmpl %[v], %[expect]\n\t"
1135 "jnz %l[error2]\n\t"
1136 "cmpl %[expect2], %[v2]\n\t"
1137 "jnz %l[error3]\n\t"
1138 #endif
1139 "movl %[newv], %%eax\n\t"
1140 /* final store */
1141 "movl %%eax, %[v]\n\t"
1142 "2:\n\t"
1143 RSEQ_INJECT_ASM(6)
1144 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1145 : /* gcc asm goto does not allow outputs */
1146 : [cpu_id] "r" (cpu),
1147 [rseq_abi] "r" (rseq_get_abi()),
1148 /* cmp2 input */
1149 [v2] "m" (*v2),
1150 [expect2] "r" (expect2),
1151 /* final store input */
1152 [v] "m" (*v),
1153 [expect] "r" (expect),
1154 [newv] "m" (newv)
1155 : "memory", "cc", "eax"
1156 RSEQ_INJECT_CLOBBER
1157 : abort, cmpfail
1158 #ifdef RSEQ_COMPARE_TWICE
1159 , error1, error2, error3
1160 #endif
1161 );
1162 rseq_after_asm_goto();
1163 return 0;
1164 abort:
1165 rseq_after_asm_goto();
1166 RSEQ_INJECT_FAILED
1167 return -1;
1168 cmpfail:
1169 rseq_after_asm_goto();
1170 return 1;
1171 #ifdef RSEQ_COMPARE_TWICE
1172 error1:
1173 rseq_after_asm_goto();
1174 rseq_bug("cpu_id comparison failed");
1175 error2:
1176 rseq_after_asm_goto();
1177 rseq_bug("1st expected value comparison failed");
1178 error3:
1179 rseq_after_asm_goto();
1180 rseq_bug("2nd expected value comparison failed");
1181 #endif
1182 }
1183
1184 /* TODO: implement a faster memcpy. */
1185 static inline __attribute__((always_inline))
1186 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1187 void *dst, void *src, size_t len,
1188 intptr_t newv, int cpu)
1189 {
1190 uint32_t rseq_scratch[3];
1191
1192 RSEQ_INJECT_C(9)
1193
1194 __asm__ __volatile__ goto (
1195 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1196 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1197 #ifdef RSEQ_COMPARE_TWICE
1198 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1199 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1200 #endif
1201 "movl %[src], %[rseq_scratch0]\n\t"
1202 "movl %[dst], %[rseq_scratch1]\n\t"
1203 "movl %[len], %[rseq_scratch2]\n\t"
1204 /* Start rseq by storing table entry pointer into rseq_cs. */
1205 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1206 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1207 RSEQ_INJECT_ASM(3)
1208 "movl %[expect], %%eax\n\t"
1209 "cmpl %%eax, %[v]\n\t"
1210 "jnz 5f\n\t"
1211 RSEQ_INJECT_ASM(4)
1212 #ifdef RSEQ_COMPARE_TWICE
1213 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1214 "movl %[expect], %%eax\n\t"
1215 "cmpl %%eax, %[v]\n\t"
1216 "jnz 7f\n\t"
1217 #endif
1218 /* try memcpy */
1219 "test %[len], %[len]\n\t" \
1220 "jz 333f\n\t" \
1221 "222:\n\t" \
1222 "movb (%[src]), %%al\n\t" \
1223 "movb %%al, (%[dst])\n\t" \
1224 "inc %[src]\n\t" \
1225 "inc %[dst]\n\t" \
1226 "dec %[len]\n\t" \
1227 "jnz 222b\n\t" \
1228 "333:\n\t" \
1229 RSEQ_INJECT_ASM(5)
1230 "movl %[newv], %%eax\n\t"
1231 /* final store */
1232 "movl %%eax, %[v]\n\t"
1233 "2:\n\t"
1234 RSEQ_INJECT_ASM(6)
1235 /* teardown */
1236 "movl %[rseq_scratch2], %[len]\n\t"
1237 "movl %[rseq_scratch1], %[dst]\n\t"
1238 "movl %[rseq_scratch0], %[src]\n\t"
1239 RSEQ_ASM_DEFINE_ABORT(4,
1240 "movl %[rseq_scratch2], %[len]\n\t"
1241 "movl %[rseq_scratch1], %[dst]\n\t"
1242 "movl %[rseq_scratch0], %[src]\n\t",
1243 abort)
1244 RSEQ_ASM_DEFINE_CMPFAIL(5,
1245 "movl %[rseq_scratch2], %[len]\n\t"
1246 "movl %[rseq_scratch1], %[dst]\n\t"
1247 "movl %[rseq_scratch0], %[src]\n\t",
1248 cmpfail)
1249 #ifdef RSEQ_COMPARE_TWICE
1250 RSEQ_ASM_DEFINE_CMPFAIL(6,
1251 "movl %[rseq_scratch2], %[len]\n\t"
1252 "movl %[rseq_scratch1], %[dst]\n\t"
1253 "movl %[rseq_scratch0], %[src]\n\t",
1254 error1)
1255 RSEQ_ASM_DEFINE_CMPFAIL(7,
1256 "movl %[rseq_scratch2], %[len]\n\t"
1257 "movl %[rseq_scratch1], %[dst]\n\t"
1258 "movl %[rseq_scratch0], %[src]\n\t",
1259 error2)
1260 #endif
1261 : /* gcc asm goto does not allow outputs */
1262 : [cpu_id] "r" (cpu),
1263 [rseq_abi] "r" (rseq_get_abi()),
1264 /* final store input */
1265 [v] "m" (*v),
1266 [expect] "m" (expect),
1267 [newv] "m" (newv),
1268 /* try memcpy input */
1269 [dst] "r" (dst),
1270 [src] "r" (src),
1271 [len] "r" (len),
1272 [rseq_scratch0] "m" (rseq_scratch[0]),
1273 [rseq_scratch1] "m" (rseq_scratch[1]),
1274 [rseq_scratch2] "m" (rseq_scratch[2])
1275 : "memory", "cc", "eax"
1276 RSEQ_INJECT_CLOBBER
1277 : abort, cmpfail
1278 #ifdef RSEQ_COMPARE_TWICE
1279 , error1, error2
1280 #endif
1281 );
1282 rseq_after_asm_goto();
1283 return 0;
1284 abort:
1285 rseq_after_asm_goto();
1286 RSEQ_INJECT_FAILED
1287 return -1;
1288 cmpfail:
1289 rseq_after_asm_goto();
1290 return 1;
1291 #ifdef RSEQ_COMPARE_TWICE
1292 error1:
1293 rseq_after_asm_goto();
1294 rseq_bug("cpu_id comparison failed");
1295 error2:
1296 rseq_after_asm_goto();
1297 rseq_bug("expected value comparison failed");
1298 #endif
1299 }
1300
1301 /* TODO: implement a faster memcpy. */
1302 static inline __attribute__((always_inline))
1303 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1304 void *dst, void *src, size_t len,
1305 intptr_t newv, int cpu)
1306 {
1307 uint32_t rseq_scratch[3];
1308
1309 RSEQ_INJECT_C(9)
1310
1311 __asm__ __volatile__ goto (
1312 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1313 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1314 #ifdef RSEQ_COMPARE_TWICE
1315 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1316 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1317 #endif
1318 "movl %[src], %[rseq_scratch0]\n\t"
1319 "movl %[dst], %[rseq_scratch1]\n\t"
1320 "movl %[len], %[rseq_scratch2]\n\t"
1321 /* Start rseq by storing table entry pointer into rseq_cs. */
1322 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1323 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1324 RSEQ_INJECT_ASM(3)
1325 "movl %[expect], %%eax\n\t"
1326 "cmpl %%eax, %[v]\n\t"
1327 "jnz 5f\n\t"
1328 RSEQ_INJECT_ASM(4)
1329 #ifdef RSEQ_COMPARE_TWICE
1330 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1331 "movl %[expect], %%eax\n\t"
1332 "cmpl %%eax, %[v]\n\t"
1333 "jnz 7f\n\t"
1334 #endif
1335 /* try memcpy */
1336 "test %[len], %[len]\n\t" \
1337 "jz 333f\n\t" \
1338 "222:\n\t" \
1339 "movb (%[src]), %%al\n\t" \
1340 "movb %%al, (%[dst])\n\t" \
1341 "inc %[src]\n\t" \
1342 "inc %[dst]\n\t" \
1343 "dec %[len]\n\t" \
1344 "jnz 222b\n\t" \
1345 "333:\n\t" \
1346 RSEQ_INJECT_ASM(5)
1347 "lock; addl $0,-128(%%esp)\n\t"
1348 "movl %[newv], %%eax\n\t"
1349 /* final store */
1350 "movl %%eax, %[v]\n\t"
1351 "2:\n\t"
1352 RSEQ_INJECT_ASM(6)
1353 /* teardown */
1354 "movl %[rseq_scratch2], %[len]\n\t"
1355 "movl %[rseq_scratch1], %[dst]\n\t"
1356 "movl %[rseq_scratch0], %[src]\n\t"
1357 RSEQ_ASM_DEFINE_ABORT(4,
1358 "movl %[rseq_scratch2], %[len]\n\t"
1359 "movl %[rseq_scratch1], %[dst]\n\t"
1360 "movl %[rseq_scratch0], %[src]\n\t",
1361 abort)
1362 RSEQ_ASM_DEFINE_CMPFAIL(5,
1363 "movl %[rseq_scratch2], %[len]\n\t"
1364 "movl %[rseq_scratch1], %[dst]\n\t"
1365 "movl %[rseq_scratch0], %[src]\n\t",
1366 cmpfail)
1367 #ifdef RSEQ_COMPARE_TWICE
1368 RSEQ_ASM_DEFINE_CMPFAIL(6,
1369 "movl %[rseq_scratch2], %[len]\n\t"
1370 "movl %[rseq_scratch1], %[dst]\n\t"
1371 "movl %[rseq_scratch0], %[src]\n\t",
1372 error1)
1373 RSEQ_ASM_DEFINE_CMPFAIL(7,
1374 "movl %[rseq_scratch2], %[len]\n\t"
1375 "movl %[rseq_scratch1], %[dst]\n\t"
1376 "movl %[rseq_scratch0], %[src]\n\t",
1377 error2)
1378 #endif
1379 : /* gcc asm goto does not allow outputs */
1380 : [cpu_id] "r" (cpu),
1381 [rseq_abi] "r" (rseq_get_abi()),
1382 /* final store input */
1383 [v] "m" (*v),
1384 [expect] "m" (expect),
1385 [newv] "m" (newv),
1386 /* try memcpy input */
1387 [dst] "r" (dst),
1388 [src] "r" (src),
1389 [len] "r" (len),
1390 [rseq_scratch0] "m" (rseq_scratch[0]),
1391 [rseq_scratch1] "m" (rseq_scratch[1]),
1392 [rseq_scratch2] "m" (rseq_scratch[2])
1393 : "memory", "cc", "eax"
1394 RSEQ_INJECT_CLOBBER
1395 : abort, cmpfail
1396 #ifdef RSEQ_COMPARE_TWICE
1397 , error1, error2
1398 #endif
1399 );
1400 rseq_after_asm_goto();
1401 return 0;
1402 abort:
1403 rseq_after_asm_goto();
1404 RSEQ_INJECT_FAILED
1405 return -1;
1406 cmpfail:
1407 rseq_after_asm_goto();
1408 return 1;
1409 #ifdef RSEQ_COMPARE_TWICE
1410 error1:
1411 rseq_after_asm_goto();
1412 rseq_bug("cpu_id comparison failed");
1413 error2:
1414 rseq_after_asm_goto();
1415 rseq_bug("expected value comparison failed");
1416 #endif
1417 }
1418
1419 /*
1420 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
1421 * into @load.
1422 */
1423 static inline __attribute__((always_inline))
1424 int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu)
1425 {
1426 RSEQ_INJECT_C(9)
1427
1428 __asm__ __volatile__ goto (
1429 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1430 #ifdef RSEQ_COMPARE_TWICE
1431 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1432 #endif
1433 /* Start rseq by storing table entry pointer into rseq_cs. */
1434 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1435 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1436 RSEQ_INJECT_ASM(3)
1437 "movl %[p], %%ebx\n\t"
1438 RSEQ_INJECT_ASM(4)
1439 #ifdef RSEQ_COMPARE_TWICE
1440 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1441 #endif
1442 "addl %[voffp], %%ebx\n\t"
1443 "movl (%%ebx), %%ebx\n\t"
1444 "movl %%ebx, %[load]\n\t"
1445 "2:\n\t"
1446 RSEQ_INJECT_ASM(5)
1447 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1448 : /* gcc asm goto does not allow outputs */
1449 : [cpu_id] "r" (cpu),
1450 [rseq_abi] "r" (rseq_get_abi()),
1451 /* final store input */
1452 [p] "m" (*p),
1453 [voffp] "ir" (voffp),
1454 [load] "m" (*load)
1455 : "memory", "cc", "eax", "ebx"
1456 RSEQ_INJECT_CLOBBER
1457 : abort
1458 #ifdef RSEQ_COMPARE_TWICE
1459 , error1
1460 #endif
1461 );
1462 rseq_after_asm_goto();
1463 return 0;
1464 abort:
1465 rseq_after_asm_goto();
1466 RSEQ_INJECT_FAILED
1467 return -1;
1468 #ifdef RSEQ_COMPARE_TWICE
1469 error1:
1470 rseq_after_asm_goto();
1471 rseq_bug("cpu_id comparison failed");
1472 #endif
1473 }
1474
1475 #endif /* !RSEQ_SKIP_FASTPATH */
1476
1477 #endif
Librseq
This page took 0.082574 seconds and 4 git commands to generate.