projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Introduce rseq-generic-thread-pointer.h
[librseq.git] / include / rseq / rseq-x86.h
1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
2 /*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #include <stdint.h>
9
10 /*
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
12 * trap in user-space:
13 *
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
16 */
17 #define RSEQ_SIG 0x53053053
18
19 /*
20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22 * address through a "r" input operand.
23 * (TODO: revisit after migration to glibc's ABI)
24 */
25
26 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
27 #define RSEQ_CPU_ID_OFFSET 4
28 #define RSEQ_CS_OFFSET 8
29
30 #ifdef __x86_64__
31
32 #define rseq_smp_mb() \
33 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
34 #define rseq_smp_rmb() rseq_barrier()
35 #define rseq_smp_wmb() rseq_barrier()
36
37 #define rseq_smp_load_acquire(p) \
38 __extension__ ({ \
39 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
40 rseq_barrier(); \
41 ____p1; \
42 })
43
44 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
45
46 #define rseq_smp_store_release(p, v) \
47 do { \
48 rseq_barrier(); \
49 RSEQ_WRITE_ONCE(*p, v); \
50 } while (0)
51
52 #ifdef RSEQ_SKIP_FASTPATH
53 #include "rseq-skip.h"
54 #else /* !RSEQ_SKIP_FASTPATH */
55
56 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
57 start_ip, post_commit_offset, abort_ip) \
58 ".pushsection __rseq_cs, \"aw\"\n\t" \
59 ".balign 32\n\t" \
60 __rseq_str(label) ":\n\t" \
61 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
62 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
63 ".popsection\n\t" \
64 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
65 ".quad " __rseq_str(label) "b\n\t" \
66 ".popsection\n\t"
67
68
69 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
70 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
71 (post_commit_ip - start_ip), abort_ip)
72
73 /*
74 * Exit points of a rseq critical section consist of all instructions outside
75 * of the critical section where a critical section can either branch to or
76 * reach through the normal course of its execution. The abort IP and the
77 * post-commit IP are already part of the __rseq_cs section and should not be
78 * explicitly defined as additional exit points. Knowing all exit points is
79 * useful to assist debuggers stepping over the critical section.
80 */
81 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
82 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
83 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
84 ".popsection\n\t"
85
86 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
87 RSEQ_INJECT_ASM(1) \
88 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
89 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
90 __rseq_str(label) ":\n\t"
91
92 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
93 RSEQ_INJECT_ASM(2) \
94 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
95 "jnz " __rseq_str(label) "\n\t"
96
97 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
98 ".pushsection __rseq_failure, \"ax\"\n\t" \
99 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
100 ".byte 0x0f, 0xb9, 0x3d\n\t" \
101 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
102 __rseq_str(label) ":\n\t" \
103 teardown \
104 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
105 ".popsection\n\t"
106
107 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
108 ".pushsection __rseq_failure, \"ax\"\n\t" \
109 __rseq_str(label) ":\n\t" \
110 teardown \
111 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
112 ".popsection\n\t"
113
114 static inline __attribute__((always_inline))
115 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
116 {
117 RSEQ_INJECT_C(9)
118
119 __asm__ __volatile__ goto (
120 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
121 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
122 #ifdef RSEQ_COMPARE_TWICE
123 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
124 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
125 #endif
126 /* Start rseq by storing table entry pointer into rseq_cs. */
127 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
128 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
129 RSEQ_INJECT_ASM(3)
130 "cmpq %[v], %[expect]\n\t"
131 "jnz %l[cmpfail]\n\t"
132 RSEQ_INJECT_ASM(4)
133 #ifdef RSEQ_COMPARE_TWICE
134 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
135 "cmpq %[v], %[expect]\n\t"
136 "jnz %l[error2]\n\t"
137 #endif
138 /* final store */
139 "movq %[newv], %[v]\n\t"
140 "2:\n\t"
141 RSEQ_INJECT_ASM(5)
142 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
143 : /* gcc asm goto does not allow outputs */
144 : [cpu_id] "r" (cpu),
145 [rseq_abi] "r" (rseq_get_abi()),
146 [v] "m" (*v),
147 [expect] "r" (expect),
148 [newv] "r" (newv)
149 : "memory", "cc", "rax"
150 RSEQ_INJECT_CLOBBER
151 : abort, cmpfail
152 #ifdef RSEQ_COMPARE_TWICE
153 , error1, error2
154 #endif
155 );
156 return 0;
157 abort:
158 RSEQ_INJECT_FAILED
159 return -1;
160 cmpfail:
161 return 1;
162 #ifdef RSEQ_COMPARE_TWICE
163 error1:
164 rseq_bug("cpu_id comparison failed");
165 error2:
166 rseq_bug("expected value comparison failed");
167 #endif
168 }
169
170 /*
171 * Compare @v against @expectnot. When it does _not_ match, load @v
172 * into @load, and store the content of *@v + voffp into @v.
173 */
174 static inline __attribute__((always_inline))
175 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
176 off_t voffp, intptr_t *load, int cpu)
177 {
178 RSEQ_INJECT_C(9)
179
180 __asm__ __volatile__ goto (
181 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
182 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
183 #ifdef RSEQ_COMPARE_TWICE
184 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
185 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
186 #endif
187 /* Start rseq by storing table entry pointer into rseq_cs. */
188 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
189 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
190 RSEQ_INJECT_ASM(3)
191 "movq %[v], %%rbx\n\t"
192 "cmpq %%rbx, %[expectnot]\n\t"
193 "je %l[cmpfail]\n\t"
194 RSEQ_INJECT_ASM(4)
195 #ifdef RSEQ_COMPARE_TWICE
196 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
197 "movq %[v], %%rbx\n\t"
198 "cmpq %%rbx, %[expectnot]\n\t"
199 "je %l[error2]\n\t"
200 #endif
201 "movq %%rbx, %[load]\n\t"
202 "addq %[voffp], %%rbx\n\t"
203 "movq (%%rbx), %%rbx\n\t"
204 /* final store */
205 "movq %%rbx, %[v]\n\t"
206 "2:\n\t"
207 RSEQ_INJECT_ASM(5)
208 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
209 : /* gcc asm goto does not allow outputs */
210 : [cpu_id] "r" (cpu),
211 [rseq_abi] "r" (rseq_get_abi()),
212 /* final store input */
213 [v] "m" (*v),
214 [expectnot] "r" (expectnot),
215 [voffp] "er" (voffp),
216 [load] "m" (*load)
217 : "memory", "cc", "rax", "rbx"
218 RSEQ_INJECT_CLOBBER
219 : abort, cmpfail
220 #ifdef RSEQ_COMPARE_TWICE
221 , error1, error2
222 #endif
223 );
224 return 0;
225 abort:
226 RSEQ_INJECT_FAILED
227 return -1;
228 cmpfail:
229 return 1;
230 #ifdef RSEQ_COMPARE_TWICE
231 error1:
232 rseq_bug("cpu_id comparison failed");
233 error2:
234 rseq_bug("expected value comparison failed");
235 #endif
236 }
237
238 static inline __attribute__((always_inline))
239 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
240 {
241 RSEQ_INJECT_C(9)
242
243 __asm__ __volatile__ goto (
244 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
245 #ifdef RSEQ_COMPARE_TWICE
246 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
247 #endif
248 /* Start rseq by storing table entry pointer into rseq_cs. */
249 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
250 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
251 RSEQ_INJECT_ASM(3)
252 #ifdef RSEQ_COMPARE_TWICE
253 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
254 #endif
255 /* final store */
256 "addq %[count], %[v]\n\t"
257 "2:\n\t"
258 RSEQ_INJECT_ASM(4)
259 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
260 : /* gcc asm goto does not allow outputs */
261 : [cpu_id] "r" (cpu),
262 [rseq_abi] "r" (rseq_get_abi()),
263 /* final store input */
264 [v] "m" (*v),
265 [count] "er" (count)
266 : "memory", "cc", "rax"
267 RSEQ_INJECT_CLOBBER
268 : abort
269 #ifdef RSEQ_COMPARE_TWICE
270 , error1
271 #endif
272 );
273 return 0;
274 abort:
275 RSEQ_INJECT_FAILED
276 return -1;
277 #ifdef RSEQ_COMPARE_TWICE
278 error1:
279 rseq_bug("cpu_id comparison failed");
280 #endif
281 }
282
283 static inline __attribute__((always_inline))
284 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
285 intptr_t *v2, intptr_t newv2,
286 intptr_t newv, int cpu)
287 {
288 RSEQ_INJECT_C(9)
289
290 __asm__ __volatile__ goto (
291 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
292 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
293 #ifdef RSEQ_COMPARE_TWICE
294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
295 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
296 #endif
297 /* Start rseq by storing table entry pointer into rseq_cs. */
298 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
299 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
300 RSEQ_INJECT_ASM(3)
301 "cmpq %[v], %[expect]\n\t"
302 "jnz %l[cmpfail]\n\t"
303 RSEQ_INJECT_ASM(4)
304 #ifdef RSEQ_COMPARE_TWICE
305 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
306 "cmpq %[v], %[expect]\n\t"
307 "jnz %l[error2]\n\t"
308 #endif
309 /* try store */
310 "movq %[newv2], %[v2]\n\t"
311 RSEQ_INJECT_ASM(5)
312 /* final store */
313 "movq %[newv], %[v]\n\t"
314 "2:\n\t"
315 RSEQ_INJECT_ASM(6)
316 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
317 : /* gcc asm goto does not allow outputs */
318 : [cpu_id] "r" (cpu),
319 [rseq_abi] "r" (rseq_get_abi()),
320 /* try store input */
321 [v2] "m" (*v2),
322 [newv2] "r" (newv2),
323 /* final store input */
324 [v] "m" (*v),
325 [expect] "r" (expect),
326 [newv] "r" (newv)
327 : "memory", "cc", "rax"
328 RSEQ_INJECT_CLOBBER
329 : abort, cmpfail
330 #ifdef RSEQ_COMPARE_TWICE
331 , error1, error2
332 #endif
333 );
334 return 0;
335 abort:
336 RSEQ_INJECT_FAILED
337 return -1;
338 cmpfail:
339 return 1;
340 #ifdef RSEQ_COMPARE_TWICE
341 error1:
342 rseq_bug("cpu_id comparison failed");
343 error2:
344 rseq_bug("expected value comparison failed");
345 #endif
346 }
347
348 /* x86-64 is TSO. */
349 static inline __attribute__((always_inline))
350 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
351 intptr_t *v2, intptr_t newv2,
352 intptr_t newv, int cpu)
353 {
354 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
355 }
356
357 static inline __attribute__((always_inline))
358 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
359 intptr_t *v2, intptr_t expect2,
360 intptr_t newv, int cpu)
361 {
362 RSEQ_INJECT_C(9)
363
364 __asm__ __volatile__ goto (
365 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
366 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
367 #ifdef RSEQ_COMPARE_TWICE
368 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
369 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
370 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
371 #endif
372 /* Start rseq by storing table entry pointer into rseq_cs. */
373 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
374 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
375 RSEQ_INJECT_ASM(3)
376 "cmpq %[v], %[expect]\n\t"
377 "jnz %l[cmpfail]\n\t"
378 RSEQ_INJECT_ASM(4)
379 "cmpq %[v2], %[expect2]\n\t"
380 "jnz %l[cmpfail]\n\t"
381 RSEQ_INJECT_ASM(5)
382 #ifdef RSEQ_COMPARE_TWICE
383 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
384 "cmpq %[v], %[expect]\n\t"
385 "jnz %l[error2]\n\t"
386 "cmpq %[v2], %[expect2]\n\t"
387 "jnz %l[error3]\n\t"
388 #endif
389 /* final store */
390 "movq %[newv], %[v]\n\t"
391 "2:\n\t"
392 RSEQ_INJECT_ASM(6)
393 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
394 : /* gcc asm goto does not allow outputs */
395 : [cpu_id] "r" (cpu),
396 [rseq_abi] "r" (rseq_get_abi()),
397 /* cmp2 input */
398 [v2] "m" (*v2),
399 [expect2] "r" (expect2),
400 /* final store input */
401 [v] "m" (*v),
402 [expect] "r" (expect),
403 [newv] "r" (newv)
404 : "memory", "cc", "rax"
405 RSEQ_INJECT_CLOBBER
406 : abort, cmpfail
407 #ifdef RSEQ_COMPARE_TWICE
408 , error1, error2, error3
409 #endif
410 );
411 return 0;
412 abort:
413 RSEQ_INJECT_FAILED
414 return -1;
415 cmpfail:
416 return 1;
417 #ifdef RSEQ_COMPARE_TWICE
418 error1:
419 rseq_bug("cpu_id comparison failed");
420 error2:
421 rseq_bug("1st expected value comparison failed");
422 error3:
423 rseq_bug("2nd expected value comparison failed");
424 #endif
425 }
426
427 static inline __attribute__((always_inline))
428 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
429 void *dst, void *src, size_t len,
430 intptr_t newv, int cpu)
431 {
432 uint64_t rseq_scratch[3];
433
434 RSEQ_INJECT_C(9)
435
436 __asm__ __volatile__ goto (
437 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
438 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
439 #ifdef RSEQ_COMPARE_TWICE
440 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
441 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
442 #endif
443 "movq %[src], %[rseq_scratch0]\n\t"
444 "movq %[dst], %[rseq_scratch1]\n\t"
445 "movq %[len], %[rseq_scratch2]\n\t"
446 /* Start rseq by storing table entry pointer into rseq_cs. */
447 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
448 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
449 RSEQ_INJECT_ASM(3)
450 "cmpq %[v], %[expect]\n\t"
451 "jnz 5f\n\t"
452 RSEQ_INJECT_ASM(4)
453 #ifdef RSEQ_COMPARE_TWICE
454 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
455 "cmpq %[v], %[expect]\n\t"
456 "jnz 7f\n\t"
457 #endif
458 /* try memcpy */
459 "test %[len], %[len]\n\t" \
460 "jz 333f\n\t" \
461 "222:\n\t" \
462 "movb (%[src]), %%al\n\t" \
463 "movb %%al, (%[dst])\n\t" \
464 "inc %[src]\n\t" \
465 "inc %[dst]\n\t" \
466 "dec %[len]\n\t" \
467 "jnz 222b\n\t" \
468 "333:\n\t" \
469 RSEQ_INJECT_ASM(5)
470 /* final store */
471 "movq %[newv], %[v]\n\t"
472 "2:\n\t"
473 RSEQ_INJECT_ASM(6)
474 /* teardown */
475 "movq %[rseq_scratch2], %[len]\n\t"
476 "movq %[rseq_scratch1], %[dst]\n\t"
477 "movq %[rseq_scratch0], %[src]\n\t"
478 RSEQ_ASM_DEFINE_ABORT(4,
479 "movq %[rseq_scratch2], %[len]\n\t"
480 "movq %[rseq_scratch1], %[dst]\n\t"
481 "movq %[rseq_scratch0], %[src]\n\t",
482 abort)
483 RSEQ_ASM_DEFINE_CMPFAIL(5,
484 "movq %[rseq_scratch2], %[len]\n\t"
485 "movq %[rseq_scratch1], %[dst]\n\t"
486 "movq %[rseq_scratch0], %[src]\n\t",
487 cmpfail)
488 #ifdef RSEQ_COMPARE_TWICE
489 RSEQ_ASM_DEFINE_CMPFAIL(6,
490 "movq %[rseq_scratch2], %[len]\n\t"
491 "movq %[rseq_scratch1], %[dst]\n\t"
492 "movq %[rseq_scratch0], %[src]\n\t",
493 error1)
494 RSEQ_ASM_DEFINE_CMPFAIL(7,
495 "movq %[rseq_scratch2], %[len]\n\t"
496 "movq %[rseq_scratch1], %[dst]\n\t"
497 "movq %[rseq_scratch0], %[src]\n\t",
498 error2)
499 #endif
500 : /* gcc asm goto does not allow outputs */
501 : [cpu_id] "r" (cpu),
502 [rseq_abi] "r" (rseq_get_abi()),
503 /* final store input */
504 [v] "m" (*v),
505 [expect] "r" (expect),
506 [newv] "r" (newv),
507 /* try memcpy input */
508 [dst] "r" (dst),
509 [src] "r" (src),
510 [len] "r" (len),
511 [rseq_scratch0] "m" (rseq_scratch[0]),
512 [rseq_scratch1] "m" (rseq_scratch[1]),
513 [rseq_scratch2] "m" (rseq_scratch[2])
514 : "memory", "cc", "rax"
515 RSEQ_INJECT_CLOBBER
516 : abort, cmpfail
517 #ifdef RSEQ_COMPARE_TWICE
518 , error1, error2
519 #endif
520 );
521 return 0;
522 abort:
523 RSEQ_INJECT_FAILED
524 return -1;
525 cmpfail:
526 return 1;
527 #ifdef RSEQ_COMPARE_TWICE
528 error1:
529 rseq_bug("cpu_id comparison failed");
530 error2:
531 rseq_bug("expected value comparison failed");
532 #endif
533 }
534
535 /* x86-64 is TSO. */
536 static inline __attribute__((always_inline))
537 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
538 void *dst, void *src, size_t len,
539 intptr_t newv, int cpu)
540 {
541 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
542 newv, cpu);
543 }
544
545 /*
546 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
547 * into @load.
548 */
549 static inline __attribute__((always_inline))
550 int rseq_deref_loadoffp(intptr_t *p, off_t voffp, intptr_t *load, int cpu)
551 {
552 RSEQ_INJECT_C(9)
553
554 __asm__ __volatile__ goto (
555 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
556 #ifdef RSEQ_COMPARE_TWICE
557 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
558 #endif
559 /* Start rseq by storing table entry pointer into rseq_cs. */
560 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
561 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
562 RSEQ_INJECT_ASM(3)
563 "movq %[p], %%rbx\n\t"
564 RSEQ_INJECT_ASM(4)
565 #ifdef RSEQ_COMPARE_TWICE
566 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
567 #endif
568 "addq %[voffp], %%rbx\n\t"
569 "movq (%%rbx), %%rbx\n\t"
570 "movq %%rbx, %[load]\n\t"
571 "2:\n\t"
572 RSEQ_INJECT_ASM(5)
573 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
574 : /* gcc asm goto does not allow outputs */
575 : [cpu_id] "r" (cpu),
576 [rseq_abi] "r" (rseq_get_abi()),
577 /* final store input */
578 [p] "m" (*p),
579 [voffp] "er" (voffp),
580 [load] "m" (*load)
581 : "memory", "cc", "rax", "rbx"
582 RSEQ_INJECT_CLOBBER
583 : abort
584 #ifdef RSEQ_COMPARE_TWICE
585 , error1
586 #endif
587 );
588 return 0;
589 abort:
590 RSEQ_INJECT_FAILED
591 return -1;
592 #ifdef RSEQ_COMPARE_TWICE
593 error1:
594 rseq_bug("cpu_id comparison failed");
595 #endif
596 }
597
598 #endif /* !RSEQ_SKIP_FASTPATH */
599
600 #elif __i386__
601
602 #define rseq_smp_mb() \
603 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
604 #define rseq_smp_rmb() \
605 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
606 #define rseq_smp_wmb() \
607 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
608
609 #define rseq_smp_load_acquire(p) \
610 __extension__ ({ \
611 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
612 rseq_smp_mb(); \
613 ____p1; \
614 })
615
616 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
617
618 #define rseq_smp_store_release(p, v) \
619 do { \
620 rseq_smp_mb(); \
621 RSEQ_WRITE_ONCE(*p, v); \
622 } while (0)
623
624 #ifdef RSEQ_SKIP_FASTPATH
625 #include "rseq-skip.h"
626 #else /* !RSEQ_SKIP_FASTPATH */
627
628 /*
629 * Use eax as scratch register and take memory operands as input to
630 * lessen register pressure. Especially needed when compiling in O0.
631 */
632 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
633 start_ip, post_commit_offset, abort_ip) \
634 ".pushsection __rseq_cs, \"aw\"\n\t" \
635 ".balign 32\n\t" \
636 __rseq_str(label) ":\n\t" \
637 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
638 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
639 ".popsection\n\t" \
640 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
641 ".long " __rseq_str(label) "b, 0x0\n\t" \
642 ".popsection\n\t"
643
644 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
645 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
646 (post_commit_ip - start_ip), abort_ip)
647
648 /*
649 * Exit points of a rseq critical section consist of all instructions outside
650 * of the critical section where a critical section can either branch to or
651 * reach through the normal course of its execution. The abort IP and the
652 * post-commit IP are already part of the __rseq_cs section and should not be
653 * explicitly defined as additional exit points. Knowing all exit points is
654 * useful to assist debuggers stepping over the critical section.
655 */
656 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
657 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
658 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
659 ".popsection\n\t"
660
661 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
662 RSEQ_INJECT_ASM(1) \
663 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
664 __rseq_str(label) ":\n\t"
665
666 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
667 RSEQ_INJECT_ASM(2) \
668 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
669 "jnz " __rseq_str(label) "\n\t"
670
671 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
672 ".pushsection __rseq_failure, \"ax\"\n\t" \
673 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
674 ".byte 0x0f, 0xb9, 0x3d\n\t" \
675 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
676 __rseq_str(label) ":\n\t" \
677 teardown \
678 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
679 ".popsection\n\t"
680
681 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
682 ".pushsection __rseq_failure, \"ax\"\n\t" \
683 __rseq_str(label) ":\n\t" \
684 teardown \
685 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
686 ".popsection\n\t"
687
688 static inline __attribute__((always_inline))
689 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
690 {
691 RSEQ_INJECT_C(9)
692
693 __asm__ __volatile__ goto (
694 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
695 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
696 #ifdef RSEQ_COMPARE_TWICE
697 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
698 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
699 #endif
700 /* Start rseq by storing table entry pointer into rseq_cs. */
701 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
702 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
703 RSEQ_INJECT_ASM(3)
704 "cmpl %[v], %[expect]\n\t"
705 "jnz %l[cmpfail]\n\t"
706 RSEQ_INJECT_ASM(4)
707 #ifdef RSEQ_COMPARE_TWICE
708 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
709 "cmpl %[v], %[expect]\n\t"
710 "jnz %l[error2]\n\t"
711 #endif
712 /* final store */
713 "movl %[newv], %[v]\n\t"
714 "2:\n\t"
715 RSEQ_INJECT_ASM(5)
716 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
717 : /* gcc asm goto does not allow outputs */
718 : [cpu_id] "r" (cpu),
719 [rseq_abi] "r" (rseq_get_abi()),
720 [v] "m" (*v),
721 [expect] "r" (expect),
722 [newv] "r" (newv)
723 : "memory", "cc", "eax"
724 RSEQ_INJECT_CLOBBER
725 : abort, cmpfail
726 #ifdef RSEQ_COMPARE_TWICE
727 , error1, error2
728 #endif
729 );
730 return 0;
731 abort:
732 RSEQ_INJECT_FAILED
733 return -1;
734 cmpfail:
735 return 1;
736 #ifdef RSEQ_COMPARE_TWICE
737 error1:
738 rseq_bug("cpu_id comparison failed");
739 error2:
740 rseq_bug("expected value comparison failed");
741 #endif
742 }
743
744 /*
745 * Compare @v against @expectnot. When it does _not_ match, load @v
746 * into @load, and store the content of *@v + voffp into @v.
747 */
748 static inline __attribute__((always_inline))
749 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
750 off_t voffp, intptr_t *load, int cpu)
751 {
752 RSEQ_INJECT_C(9)
753
754 __asm__ __volatile__ goto (
755 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
756 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
757 #ifdef RSEQ_COMPARE_TWICE
758 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
759 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
760 #endif
761 /* Start rseq by storing table entry pointer into rseq_cs. */
762 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
763 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
764 RSEQ_INJECT_ASM(3)
765 "movl %[v], %%ebx\n\t"
766 "cmpl %%ebx, %[expectnot]\n\t"
767 "je %l[cmpfail]\n\t"
768 RSEQ_INJECT_ASM(4)
769 #ifdef RSEQ_COMPARE_TWICE
770 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
771 "movl %[v], %%ebx\n\t"
772 "cmpl %%ebx, %[expectnot]\n\t"
773 "je %l[error2]\n\t"
774 #endif
775 "movl %%ebx, %[load]\n\t"
776 "addl %[voffp], %%ebx\n\t"
777 "movl (%%ebx), %%ebx\n\t"
778 /* final store */
779 "movl %%ebx, %[v]\n\t"
780 "2:\n\t"
781 RSEQ_INJECT_ASM(5)
782 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
783 : /* gcc asm goto does not allow outputs */
784 : [cpu_id] "r" (cpu),
785 [rseq_abi] "r" (rseq_get_abi()),
786 /* final store input */
787 [v] "m" (*v),
788 [expectnot] "r" (expectnot),
789 [voffp] "ir" (voffp),
790 [load] "m" (*load)
791 : "memory", "cc", "eax", "ebx"
792 RSEQ_INJECT_CLOBBER
793 : abort, cmpfail
794 #ifdef RSEQ_COMPARE_TWICE
795 , error1, error2
796 #endif
797 );
798 return 0;
799 abort:
800 RSEQ_INJECT_FAILED
801 return -1;
802 cmpfail:
803 return 1;
804 #ifdef RSEQ_COMPARE_TWICE
805 error1:
806 rseq_bug("cpu_id comparison failed");
807 error2:
808 rseq_bug("expected value comparison failed");
809 #endif
810 }
811
812 static inline __attribute__((always_inline))
813 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
814 {
815 RSEQ_INJECT_C(9)
816
817 __asm__ __volatile__ goto (
818 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
819 #ifdef RSEQ_COMPARE_TWICE
820 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
821 #endif
822 /* Start rseq by storing table entry pointer into rseq_cs. */
823 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
824 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
825 RSEQ_INJECT_ASM(3)
826 #ifdef RSEQ_COMPARE_TWICE
827 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
828 #endif
829 /* final store */
830 "addl %[count], %[v]\n\t"
831 "2:\n\t"
832 RSEQ_INJECT_ASM(4)
833 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
834 : /* gcc asm goto does not allow outputs */
835 : [cpu_id] "r" (cpu),
836 [rseq_abi] "r" (rseq_get_abi()),
837 /* final store input */
838 [v] "m" (*v),
839 [count] "ir" (count)
840 : "memory", "cc", "eax"
841 RSEQ_INJECT_CLOBBER
842 : abort
843 #ifdef RSEQ_COMPARE_TWICE
844 , error1
845 #endif
846 );
847 return 0;
848 abort:
849 RSEQ_INJECT_FAILED
850 return -1;
851 #ifdef RSEQ_COMPARE_TWICE
852 error1:
853 rseq_bug("cpu_id comparison failed");
854 #endif
855 }
856
857 static inline __attribute__((always_inline))
858 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
859 intptr_t *v2, intptr_t newv2,
860 intptr_t newv, int cpu)
861 {
862 RSEQ_INJECT_C(9)
863
864 __asm__ __volatile__ goto (
865 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
866 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
867 #ifdef RSEQ_COMPARE_TWICE
868 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
869 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
870 #endif
871 /* Start rseq by storing table entry pointer into rseq_cs. */
872 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
873 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
874 RSEQ_INJECT_ASM(3)
875 "cmpl %[v], %[expect]\n\t"
876 "jnz %l[cmpfail]\n\t"
877 RSEQ_INJECT_ASM(4)
878 #ifdef RSEQ_COMPARE_TWICE
879 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
880 "cmpl %[v], %[expect]\n\t"
881 "jnz %l[error2]\n\t"
882 #endif
883 /* try store */
884 "movl %[newv2], %%eax\n\t"
885 "movl %%eax, %[v2]\n\t"
886 RSEQ_INJECT_ASM(5)
887 /* final store */
888 "movl %[newv], %[v]\n\t"
889 "2:\n\t"
890 RSEQ_INJECT_ASM(6)
891 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
892 : /* gcc asm goto does not allow outputs */
893 : [cpu_id] "r" (cpu),
894 [rseq_abi] "r" (rseq_get_abi()),
895 /* try store input */
896 [v2] "m" (*v2),
897 [newv2] "m" (newv2),
898 /* final store input */
899 [v] "m" (*v),
900 [expect] "r" (expect),
901 [newv] "r" (newv)
902 : "memory", "cc", "eax"
903 RSEQ_INJECT_CLOBBER
904 : abort, cmpfail
905 #ifdef RSEQ_COMPARE_TWICE
906 , error1, error2
907 #endif
908 );
909 return 0;
910 abort:
911 RSEQ_INJECT_FAILED
912 return -1;
913 cmpfail:
914 return 1;
915 #ifdef RSEQ_COMPARE_TWICE
916 error1:
917 rseq_bug("cpu_id comparison failed");
918 error2:
919 rseq_bug("expected value comparison failed");
920 #endif
921 }
922
923 static inline __attribute__((always_inline))
924 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
925 intptr_t *v2, intptr_t newv2,
926 intptr_t newv, int cpu)
927 {
928 RSEQ_INJECT_C(9)
929
930 __asm__ __volatile__ goto (
931 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
932 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
933 #ifdef RSEQ_COMPARE_TWICE
934 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
935 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
936 #endif
937 /* Start rseq by storing table entry pointer into rseq_cs. */
938 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
939 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
940 RSEQ_INJECT_ASM(3)
941 "movl %[expect], %%eax\n\t"
942 "cmpl %[v], %%eax\n\t"
943 "jnz %l[cmpfail]\n\t"
944 RSEQ_INJECT_ASM(4)
945 #ifdef RSEQ_COMPARE_TWICE
946 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
947 "movl %[expect], %%eax\n\t"
948 "cmpl %[v], %%eax\n\t"
949 "jnz %l[error2]\n\t"
950 #endif
951 /* try store */
952 "movl %[newv2], %[v2]\n\t"
953 RSEQ_INJECT_ASM(5)
954 "lock; addl $0,-128(%%esp)\n\t"
955 /* final store */
956 "movl %[newv], %[v]\n\t"
957 "2:\n\t"
958 RSEQ_INJECT_ASM(6)
959 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
960 : /* gcc asm goto does not allow outputs */
961 : [cpu_id] "r" (cpu),
962 [rseq_abi] "r" (rseq_get_abi()),
963 /* try store input */
964 [v2] "m" (*v2),
965 [newv2] "r" (newv2),
966 /* final store input */
967 [v] "m" (*v),
968 [expect] "m" (expect),
969 [newv] "r" (newv)
970 : "memory", "cc", "eax"
971 RSEQ_INJECT_CLOBBER
972 : abort, cmpfail
973 #ifdef RSEQ_COMPARE_TWICE
974 , error1, error2
975 #endif
976 );
977 return 0;
978 abort:
979 RSEQ_INJECT_FAILED
980 return -1;
981 cmpfail:
982 return 1;
983 #ifdef RSEQ_COMPARE_TWICE
984 error1:
985 rseq_bug("cpu_id comparison failed");
986 error2:
987 rseq_bug("expected value comparison failed");
988 #endif
989
990 }
991
992 static inline __attribute__((always_inline))
993 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
994 intptr_t *v2, intptr_t expect2,
995 intptr_t newv, int cpu)
996 {
997 RSEQ_INJECT_C(9)
998
999 __asm__ __volatile__ goto (
1000 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1001 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1002 #ifdef RSEQ_COMPARE_TWICE
1003 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1004 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1005 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1006 #endif
1007 /* Start rseq by storing table entry pointer into rseq_cs. */
1008 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1009 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1010 RSEQ_INJECT_ASM(3)
1011 "cmpl %[v], %[expect]\n\t"
1012 "jnz %l[cmpfail]\n\t"
1013 RSEQ_INJECT_ASM(4)
1014 "cmpl %[expect2], %[v2]\n\t"
1015 "jnz %l[cmpfail]\n\t"
1016 RSEQ_INJECT_ASM(5)
1017 #ifdef RSEQ_COMPARE_TWICE
1018 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1019 "cmpl %[v], %[expect]\n\t"
1020 "jnz %l[error2]\n\t"
1021 "cmpl %[expect2], %[v2]\n\t"
1022 "jnz %l[error3]\n\t"
1023 #endif
1024 "movl %[newv], %%eax\n\t"
1025 /* final store */
1026 "movl %%eax, %[v]\n\t"
1027 "2:\n\t"
1028 RSEQ_INJECT_ASM(6)
1029 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1030 : /* gcc asm goto does not allow outputs */
1031 : [cpu_id] "r" (cpu),
1032 [rseq_abi] "r" (rseq_get_abi()),
1033 /* cmp2 input */
1034 [v2] "m" (*v2),
1035 [expect2] "r" (expect2),
1036 /* final store input */
1037 [v] "m" (*v),
1038 [expect] "r" (expect),
1039 [newv] "m" (newv)
1040 : "memory", "cc", "eax"
1041 RSEQ_INJECT_CLOBBER
1042 : abort, cmpfail
1043 #ifdef RSEQ_COMPARE_TWICE
1044 , error1, error2, error3
1045 #endif
1046 );
1047 return 0;
1048 abort:
1049 RSEQ_INJECT_FAILED
1050 return -1;
1051 cmpfail:
1052 return 1;
1053 #ifdef RSEQ_COMPARE_TWICE
1054 error1:
1055 rseq_bug("cpu_id comparison failed");
1056 error2:
1057 rseq_bug("1st expected value comparison failed");
1058 error3:
1059 rseq_bug("2nd expected value comparison failed");
1060 #endif
1061 }
1062
1063 /* TODO: implement a faster memcpy. */
1064 static inline __attribute__((always_inline))
1065 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1066 void *dst, void *src, size_t len,
1067 intptr_t newv, int cpu)
1068 {
1069 uint32_t rseq_scratch[3];
1070
1071 RSEQ_INJECT_C(9)
1072
1073 __asm__ __volatile__ goto (
1074 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1075 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1076 #ifdef RSEQ_COMPARE_TWICE
1077 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1078 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1079 #endif
1080 "movl %[src], %[rseq_scratch0]\n\t"
1081 "movl %[dst], %[rseq_scratch1]\n\t"
1082 "movl %[len], %[rseq_scratch2]\n\t"
1083 /* Start rseq by storing table entry pointer into rseq_cs. */
1084 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1085 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1086 RSEQ_INJECT_ASM(3)
1087 "movl %[expect], %%eax\n\t"
1088 "cmpl %%eax, %[v]\n\t"
1089 "jnz 5f\n\t"
1090 RSEQ_INJECT_ASM(4)
1091 #ifdef RSEQ_COMPARE_TWICE
1092 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1093 "movl %[expect], %%eax\n\t"
1094 "cmpl %%eax, %[v]\n\t"
1095 "jnz 7f\n\t"
1096 #endif
1097 /* try memcpy */
1098 "test %[len], %[len]\n\t" \
1099 "jz 333f\n\t" \
1100 "222:\n\t" \
1101 "movb (%[src]), %%al\n\t" \
1102 "movb %%al, (%[dst])\n\t" \
1103 "inc %[src]\n\t" \
1104 "inc %[dst]\n\t" \
1105 "dec %[len]\n\t" \
1106 "jnz 222b\n\t" \
1107 "333:\n\t" \
1108 RSEQ_INJECT_ASM(5)
1109 "movl %[newv], %%eax\n\t"
1110 /* final store */
1111 "movl %%eax, %[v]\n\t"
1112 "2:\n\t"
1113 RSEQ_INJECT_ASM(6)
1114 /* teardown */
1115 "movl %[rseq_scratch2], %[len]\n\t"
1116 "movl %[rseq_scratch1], %[dst]\n\t"
1117 "movl %[rseq_scratch0], %[src]\n\t"
1118 RSEQ_ASM_DEFINE_ABORT(4,
1119 "movl %[rseq_scratch2], %[len]\n\t"
1120 "movl %[rseq_scratch1], %[dst]\n\t"
1121 "movl %[rseq_scratch0], %[src]\n\t",
1122 abort)
1123 RSEQ_ASM_DEFINE_CMPFAIL(5,
1124 "movl %[rseq_scratch2], %[len]\n\t"
1125 "movl %[rseq_scratch1], %[dst]\n\t"
1126 "movl %[rseq_scratch0], %[src]\n\t",
1127 cmpfail)
1128 #ifdef RSEQ_COMPARE_TWICE
1129 RSEQ_ASM_DEFINE_CMPFAIL(6,
1130 "movl %[rseq_scratch2], %[len]\n\t"
1131 "movl %[rseq_scratch1], %[dst]\n\t"
1132 "movl %[rseq_scratch0], %[src]\n\t",
1133 error1)
1134 RSEQ_ASM_DEFINE_CMPFAIL(7,
1135 "movl %[rseq_scratch2], %[len]\n\t"
1136 "movl %[rseq_scratch1], %[dst]\n\t"
1137 "movl %[rseq_scratch0], %[src]\n\t",
1138 error2)
1139 #endif
1140 : /* gcc asm goto does not allow outputs */
1141 : [cpu_id] "r" (cpu),
1142 [rseq_abi] "r" (rseq_get_abi()),
1143 /* final store input */
1144 [v] "m" (*v),
1145 [expect] "m" (expect),
1146 [newv] "m" (newv),
1147 /* try memcpy input */
1148 [dst] "r" (dst),
1149 [src] "r" (src),
1150 [len] "r" (len),
1151 [rseq_scratch0] "m" (rseq_scratch[0]),
1152 [rseq_scratch1] "m" (rseq_scratch[1]),
1153 [rseq_scratch2] "m" (rseq_scratch[2])
1154 : "memory", "cc", "eax"
1155 RSEQ_INJECT_CLOBBER
1156 : abort, cmpfail
1157 #ifdef RSEQ_COMPARE_TWICE
1158 , error1, error2
1159 #endif
1160 );
1161 return 0;
1162 abort:
1163 RSEQ_INJECT_FAILED
1164 return -1;
1165 cmpfail:
1166 return 1;
1167 #ifdef RSEQ_COMPARE_TWICE
1168 error1:
1169 rseq_bug("cpu_id comparison failed");
1170 error2:
1171 rseq_bug("expected value comparison failed");
1172 #endif
1173 }
1174
1175 /* TODO: implement a faster memcpy. */
1176 static inline __attribute__((always_inline))
1177 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1178 void *dst, void *src, size_t len,
1179 intptr_t newv, int cpu)
1180 {
1181 uint32_t rseq_scratch[3];
1182
1183 RSEQ_INJECT_C(9)
1184
1185 __asm__ __volatile__ goto (
1186 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1187 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1188 #ifdef RSEQ_COMPARE_TWICE
1189 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1190 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1191 #endif
1192 "movl %[src], %[rseq_scratch0]\n\t"
1193 "movl %[dst], %[rseq_scratch1]\n\t"
1194 "movl %[len], %[rseq_scratch2]\n\t"
1195 /* Start rseq by storing table entry pointer into rseq_cs. */
1196 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1197 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1198 RSEQ_INJECT_ASM(3)
1199 "movl %[expect], %%eax\n\t"
1200 "cmpl %%eax, %[v]\n\t"
1201 "jnz 5f\n\t"
1202 RSEQ_INJECT_ASM(4)
1203 #ifdef RSEQ_COMPARE_TWICE
1204 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1205 "movl %[expect], %%eax\n\t"
1206 "cmpl %%eax, %[v]\n\t"
1207 "jnz 7f\n\t"
1208 #endif
1209 /* try memcpy */
1210 "test %[len], %[len]\n\t" \
1211 "jz 333f\n\t" \
1212 "222:\n\t" \
1213 "movb (%[src]), %%al\n\t" \
1214 "movb %%al, (%[dst])\n\t" \
1215 "inc %[src]\n\t" \
1216 "inc %[dst]\n\t" \
1217 "dec %[len]\n\t" \
1218 "jnz 222b\n\t" \
1219 "333:\n\t" \
1220 RSEQ_INJECT_ASM(5)
1221 "lock; addl $0,-128(%%esp)\n\t"
1222 "movl %[newv], %%eax\n\t"
1223 /* final store */
1224 "movl %%eax, %[v]\n\t"
1225 "2:\n\t"
1226 RSEQ_INJECT_ASM(6)
1227 /* teardown */
1228 "movl %[rseq_scratch2], %[len]\n\t"
1229 "movl %[rseq_scratch1], %[dst]\n\t"
1230 "movl %[rseq_scratch0], %[src]\n\t"
1231 RSEQ_ASM_DEFINE_ABORT(4,
1232 "movl %[rseq_scratch2], %[len]\n\t"
1233 "movl %[rseq_scratch1], %[dst]\n\t"
1234 "movl %[rseq_scratch0], %[src]\n\t",
1235 abort)
1236 RSEQ_ASM_DEFINE_CMPFAIL(5,
1237 "movl %[rseq_scratch2], %[len]\n\t"
1238 "movl %[rseq_scratch1], %[dst]\n\t"
1239 "movl %[rseq_scratch0], %[src]\n\t",
1240 cmpfail)
1241 #ifdef RSEQ_COMPARE_TWICE
1242 RSEQ_ASM_DEFINE_CMPFAIL(6,
1243 "movl %[rseq_scratch2], %[len]\n\t"
1244 "movl %[rseq_scratch1], %[dst]\n\t"
1245 "movl %[rseq_scratch0], %[src]\n\t",
1246 error1)
1247 RSEQ_ASM_DEFINE_CMPFAIL(7,
1248 "movl %[rseq_scratch2], %[len]\n\t"
1249 "movl %[rseq_scratch1], %[dst]\n\t"
1250 "movl %[rseq_scratch0], %[src]\n\t",
1251 error2)
1252 #endif
1253 : /* gcc asm goto does not allow outputs */
1254 : [cpu_id] "r" (cpu),
1255 [rseq_abi] "r" (rseq_get_abi()),
1256 /* final store input */
1257 [v] "m" (*v),
1258 [expect] "m" (expect),
1259 [newv] "m" (newv),
1260 /* try memcpy input */
1261 [dst] "r" (dst),
1262 [src] "r" (src),
1263 [len] "r" (len),
1264 [rseq_scratch0] "m" (rseq_scratch[0]),
1265 [rseq_scratch1] "m" (rseq_scratch[1]),
1266 [rseq_scratch2] "m" (rseq_scratch[2])
1267 : "memory", "cc", "eax"
1268 RSEQ_INJECT_CLOBBER
1269 : abort, cmpfail
1270 #ifdef RSEQ_COMPARE_TWICE
1271 , error1, error2
1272 #endif
1273 );
1274 return 0;
1275 abort:
1276 RSEQ_INJECT_FAILED
1277 return -1;
1278 cmpfail:
1279 return 1;
1280 #ifdef RSEQ_COMPARE_TWICE
1281 error1:
1282 rseq_bug("cpu_id comparison failed");
1283 error2:
1284 rseq_bug("expected value comparison failed");
1285 #endif
1286 }
1287
1288 /*
1289 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
1290 * into @load.
1291 */
1292 static inline __attribute__((always_inline))
1293 int rseq_deref_loadoffp(intptr_t *p, off_t voffp, intptr_t *load, int cpu)
1294 {
1295 RSEQ_INJECT_C(9)
1296
1297 __asm__ __volatile__ goto (
1298 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1299 #ifdef RSEQ_COMPARE_TWICE
1300 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1301 #endif
1302 /* Start rseq by storing table entry pointer into rseq_cs. */
1303 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1304 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1305 RSEQ_INJECT_ASM(3)
1306 "movl %[p], %%ebx\n\t"
1307 RSEQ_INJECT_ASM(4)
1308 #ifdef RSEQ_COMPARE_TWICE
1309 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1310 #endif
1311 "addl %[voffp], %%ebx\n\t"
1312 "movl (%%ebx), %%ebx\n\t"
1313 "movl %%ebx, %[load]\n\t"
1314 "2:\n\t"
1315 RSEQ_INJECT_ASM(5)
1316 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1317 : /* gcc asm goto does not allow outputs */
1318 : [cpu_id] "r" (cpu),
1319 [rseq_abi] "r" (rseq_get_abi()),
1320 /* final store input */
1321 [p] "m" (*p),
1322 [voffp] "ir" (voffp),
1323 [load] "m" (*load)
1324 : "memory", "cc", "eax", "ebx"
1325 RSEQ_INJECT_CLOBBER
1326 : abort
1327 #ifdef RSEQ_COMPARE_TWICE
1328 , error1
1329 #endif
1330 );
1331 return 0;
1332 abort:
1333 RSEQ_INJECT_FAILED
1334 return -1;
1335 #ifdef RSEQ_COMPARE_TWICE
1336 error1:
1337 rseq_bug("cpu_id comparison failed");
1338 #endif
1339 }
1340
1341 #endif /* !RSEQ_SKIP_FASTPATH */
1342
1343 #endif
Librseq
This page took 0.062068 seconds and 4 git commands to generate.