projects / librseq.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Add __rseq_exit_point_array section for debuggers
[librseq.git] / include / rseq / rseq-x86.h
CommitLineData
744d0b8b 1/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
784b0012
MD		 2/*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8#include <stdint.h>
9
10#define RSEQ_SIG 0x53053053
11
12#ifdef __x86_64__
13
14#define rseq_smp_mb() \
15 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
16#define rseq_smp_rmb() rseq_barrier()
17#define rseq_smp_wmb() rseq_barrier()
18
19#define rseq_smp_load_acquire(p) \
20__extension__ ({ \
21 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
22 rseq_barrier(); \
23 ____p1; \
24})
25
26#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
27
28#define rseq_smp_store_release(p, v) \
29do { \
30 rseq_barrier(); \
31 RSEQ_WRITE_ONCE(*p, v); \
32} while (0)
33
34#ifdef RSEQ_SKIP_FASTPATH
35#include "rseq-skip.h"
36#else /* !RSEQ_SKIP_FASTPATH */
37
38#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
39 start_ip, post_commit_offset, abort_ip) \
40 ".pushsection __rseq_table, \"aw\"\n\t" \
41 ".balign 32\n\t" \
42 __rseq_str(label) ":\n\t" \
43 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
45 ".popsection\n\t"
46
47#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
48 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
49 (post_commit_ip - start_ip), abort_ip)
50
90d9876e
MD		 51/*
52 * Exit points of a rseq critical section consist of all instructions outside
53 * of the critical section where a critical section can branch to. The abort IP
54 * is already part of the __rseq_table and should not be explicitly defined as
55 * an additional exit point. Knowing all exit points is useful to assist
56 * debuggers stepping over the critical section.
57 */
58#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
59 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
60 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
61 ".popsection\n\t"
62
784b0012
MD		 63#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
64 RSEQ_INJECT_ASM(1) \
65 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
66 "movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t" \
67 __rseq_str(label) ":\n\t"
68
69#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
70 RSEQ_INJECT_ASM(2) \
71 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
72 "jnz " __rseq_str(label) "\n\t"
73
74#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
75 ".pushsection __rseq_failure, \"ax\"\n\t" \
76 /* Disassembler-friendly signature: nopl <sig>(%rip). */\
77 ".byte 0x0f, 0x1f, 0x05\n\t" \
78 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
79 __rseq_str(label) ":\n\t" \
80 teardown \
81 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
82 ".popsection\n\t"
83
84#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
85 ".pushsection __rseq_failure, \"ax\"\n\t" \
86 __rseq_str(label) ":\n\t" \
87 teardown \
88 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
89 ".popsection\n\t"
90
91static inline __attribute__((always_inline))
92int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
93{
94 RSEQ_INJECT_C(9)
95
96 __asm__ __volatile__ goto (
97 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 98 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
99#ifdef RSEQ_COMPARE_TWICE
100 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
101 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
102#endif
784b0012
MD		 103 /* Start rseq by storing table entry pointer into rseq_cs. */
104 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
105 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
106 RSEQ_INJECT_ASM(3)
107 "cmpq %[v], %[expect]\n\t"
108 "jnz %l[cmpfail]\n\t"
109 RSEQ_INJECT_ASM(4)
110#ifdef RSEQ_COMPARE_TWICE
111 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
112 "cmpq %[v], %[expect]\n\t"
113 "jnz %l[error2]\n\t"
114#endif
115 /* final store */
116 "movq %[newv], %[v]\n\t"
117 "2:\n\t"
118 RSEQ_INJECT_ASM(5)
119 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
120 : /* gcc asm goto does not allow outputs */
121 : [cpu_id] "r" (cpu),
122 [current_cpu_id] "m" (__rseq_abi.cpu_id),
123 [rseq_cs] "m" (__rseq_abi.rseq_cs),
124 [v] "m" (*v),
125 [expect] "r" (expect),
126 [newv] "r" (newv)
127 : "memory", "cc", "rax"
128 RSEQ_INJECT_CLOBBER
129 : abort, cmpfail
130#ifdef RSEQ_COMPARE_TWICE
131 , error1, error2
132#endif
133 );
134 return 0;
135abort:
136 RSEQ_INJECT_FAILED
137 return -1;
138cmpfail:
139 return 1;
140#ifdef RSEQ_COMPARE_TWICE
141error1:
142 rseq_bug("cpu_id comparison failed");
143error2:
144 rseq_bug("expected value comparison failed");
145#endif
146}
147
148/*
149 * Compare @v against @expectnot. When it does _not_ match, load @v
150 * into @load, and store the content of *@v + voffp into @v.
151 */
152static inline __attribute__((always_inline))
153int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
154 off_t voffp, intptr_t *load, int cpu)
155{
156 RSEQ_INJECT_C(9)
157
158 __asm__ __volatile__ goto (
159 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 160 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
161#ifdef RSEQ_COMPARE_TWICE
162 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
163 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
164#endif
784b0012
MD		 165 /* Start rseq by storing table entry pointer into rseq_cs. */
166 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
167 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
168 RSEQ_INJECT_ASM(3)
169 "movq %[v], %%rbx\n\t"
170 "cmpq %%rbx, %[expectnot]\n\t"
171 "je %l[cmpfail]\n\t"
172 RSEQ_INJECT_ASM(4)
173#ifdef RSEQ_COMPARE_TWICE
174 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
175 "movq %[v], %%rbx\n\t"
176 "cmpq %%rbx, %[expectnot]\n\t"
177 "je %l[error2]\n\t"
178#endif
179 "movq %%rbx, %[load]\n\t"
180 "addq %[voffp], %%rbx\n\t"
181 "movq (%%rbx), %%rbx\n\t"
182 /* final store */
183 "movq %%rbx, %[v]\n\t"
184 "2:\n\t"
185 RSEQ_INJECT_ASM(5)
186 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
187 : /* gcc asm goto does not allow outputs */
188 : [cpu_id] "r" (cpu),
189 [current_cpu_id] "m" (__rseq_abi.cpu_id),
190 [rseq_cs] "m" (__rseq_abi.rseq_cs),
191 /* final store input */
192 [v] "m" (*v),
193 [expectnot] "r" (expectnot),
194 [voffp] "er" (voffp),
195 [load] "m" (*load)
196 : "memory", "cc", "rax", "rbx"
197 RSEQ_INJECT_CLOBBER
198 : abort, cmpfail
199#ifdef RSEQ_COMPARE_TWICE
200 , error1, error2
201#endif
202 );
203 return 0;
204abort:
205 RSEQ_INJECT_FAILED
206 return -1;
207cmpfail:
208 return 1;
209#ifdef RSEQ_COMPARE_TWICE
210error1:
211 rseq_bug("cpu_id comparison failed");
212error2:
213 rseq_bug("expected value comparison failed");
214#endif
215}
216
217static inline __attribute__((always_inline))
218int rseq_addv(intptr_t *v, intptr_t count, int cpu)
219{
220 RSEQ_INJECT_C(9)
221
222 __asm__ __volatile__ goto (
223 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 224#ifdef RSEQ_COMPARE_TWICE
225 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
226#endif
784b0012
MD		 227 /* Start rseq by storing table entry pointer into rseq_cs. */
228 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
229 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
230 RSEQ_INJECT_ASM(3)
231#ifdef RSEQ_COMPARE_TWICE
232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
233#endif
234 /* final store */
235 "addq %[count], %[v]\n\t"
236 "2:\n\t"
237 RSEQ_INJECT_ASM(4)
238 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
239 : /* gcc asm goto does not allow outputs */
240 : [cpu_id] "r" (cpu),
241 [current_cpu_id] "m" (__rseq_abi.cpu_id),
242 [rseq_cs] "m" (__rseq_abi.rseq_cs),
243 /* final store input */
244 [v] "m" (*v),
245 [count] "er" (count)
246 : "memory", "cc", "rax"
247 RSEQ_INJECT_CLOBBER
248 : abort
249#ifdef RSEQ_COMPARE_TWICE
250 , error1
251#endif
252 );
253 return 0;
254abort:
255 RSEQ_INJECT_FAILED
256 return -1;
257#ifdef RSEQ_COMPARE_TWICE
258error1:
259 rseq_bug("cpu_id comparison failed");
260#endif
261}
262
263static inline __attribute__((always_inline))
264int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
265 intptr_t *v2, intptr_t newv2,
266 intptr_t newv, int cpu)
267{
268 RSEQ_INJECT_C(9)
269
270 __asm__ __volatile__ goto (
271 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 272 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
273#ifdef RSEQ_COMPARE_TWICE
274 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
275 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
276#endif
784b0012
MD		 277 /* Start rseq by storing table entry pointer into rseq_cs. */
278 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
280 RSEQ_INJECT_ASM(3)
281 "cmpq %[v], %[expect]\n\t"
282 "jnz %l[cmpfail]\n\t"
283 RSEQ_INJECT_ASM(4)
284#ifdef RSEQ_COMPARE_TWICE
285 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
286 "cmpq %[v], %[expect]\n\t"
287 "jnz %l[error2]\n\t"
288#endif
289 /* try store */
290 "movq %[newv2], %[v2]\n\t"
291 RSEQ_INJECT_ASM(5)
292 /* final store */
293 "movq %[newv], %[v]\n\t"
294 "2:\n\t"
295 RSEQ_INJECT_ASM(6)
296 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
297 : /* gcc asm goto does not allow outputs */
298 : [cpu_id] "r" (cpu),
299 [current_cpu_id] "m" (__rseq_abi.cpu_id),
300 [rseq_cs] "m" (__rseq_abi.rseq_cs),
301 /* try store input */
302 [v2] "m" (*v2),
303 [newv2] "r" (newv2),
304 /* final store input */
305 [v] "m" (*v),
306 [expect] "r" (expect),
307 [newv] "r" (newv)
308 : "memory", "cc", "rax"
309 RSEQ_INJECT_CLOBBER
310 : abort, cmpfail
311#ifdef RSEQ_COMPARE_TWICE
312 , error1, error2
313#endif
314 );
315 return 0;
316abort:
317 RSEQ_INJECT_FAILED
318 return -1;
319cmpfail:
320 return 1;
321#ifdef RSEQ_COMPARE_TWICE
322error1:
323 rseq_bug("cpu_id comparison failed");
324error2:
325 rseq_bug("expected value comparison failed");
326#endif
327}
328
329/* x86-64 is TSO. */
330static inline __attribute__((always_inline))
331int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
332 intptr_t *v2, intptr_t newv2,
333 intptr_t newv, int cpu)
334{
335 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
336}
337
338static inline __attribute__((always_inline))
339int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
340 intptr_t *v2, intptr_t expect2,
341 intptr_t newv, int cpu)
342{
343 RSEQ_INJECT_C(9)
344
345 __asm__ __volatile__ goto (
346 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 347 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
348#ifdef RSEQ_COMPARE_TWICE
349 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
350 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
351 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
352#endif
784b0012
MD		 353 /* Start rseq by storing table entry pointer into rseq_cs. */
354 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
355 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
356 RSEQ_INJECT_ASM(3)
357 "cmpq %[v], %[expect]\n\t"
358 "jnz %l[cmpfail]\n\t"
359 RSEQ_INJECT_ASM(4)
360 "cmpq %[v2], %[expect2]\n\t"
361 "jnz %l[cmpfail]\n\t"
362 RSEQ_INJECT_ASM(5)
363#ifdef RSEQ_COMPARE_TWICE
364 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
365 "cmpq %[v], %[expect]\n\t"
366 "jnz %l[error2]\n\t"
367 "cmpq %[v2], %[expect2]\n\t"
368 "jnz %l[error3]\n\t"
369#endif
370 /* final store */
371 "movq %[newv], %[v]\n\t"
372 "2:\n\t"
373 RSEQ_INJECT_ASM(6)
374 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
375 : /* gcc asm goto does not allow outputs */
376 : [cpu_id] "r" (cpu),
377 [current_cpu_id] "m" (__rseq_abi.cpu_id),
378 [rseq_cs] "m" (__rseq_abi.rseq_cs),
379 /* cmp2 input */
380 [v2] "m" (*v2),
381 [expect2] "r" (expect2),
382 /* final store input */
383 [v] "m" (*v),
384 [expect] "r" (expect),
385 [newv] "r" (newv)
386 : "memory", "cc", "rax"
387 RSEQ_INJECT_CLOBBER
388 : abort, cmpfail
389#ifdef RSEQ_COMPARE_TWICE
390 , error1, error2, error3
391#endif
392 );
393 return 0;
394abort:
395 RSEQ_INJECT_FAILED
396 return -1;
397cmpfail:
398 return 1;
399#ifdef RSEQ_COMPARE_TWICE
400error1:
401 rseq_bug("cpu_id comparison failed");
402error2:
403 rseq_bug("1st expected value comparison failed");
404error3:
405 rseq_bug("2nd expected value comparison failed");
406#endif
407}
408
409static inline __attribute__((always_inline))
410int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
411 void *dst, void *src, size_t len,
412 intptr_t newv, int cpu)
413{
414 uint64_t rseq_scratch[3];
415
416 RSEQ_INJECT_C(9)
417
418 __asm__ __volatile__ goto (
419 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 420 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
421#ifdef RSEQ_COMPARE_TWICE
422 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
423 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
424#endif
784b0012
MD		 425 "movq %[src], %[rseq_scratch0]\n\t"
426 "movq %[dst], %[rseq_scratch1]\n\t"
427 "movq %[len], %[rseq_scratch2]\n\t"
428 /* Start rseq by storing table entry pointer into rseq_cs. */
429 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
430 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
431 RSEQ_INJECT_ASM(3)
432 "cmpq %[v], %[expect]\n\t"
433 "jnz 5f\n\t"
434 RSEQ_INJECT_ASM(4)
435#ifdef RSEQ_COMPARE_TWICE
436 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
437 "cmpq %[v], %[expect]\n\t"
438 "jnz 7f\n\t"
439#endif
440 /* try memcpy */
441 "test %[len], %[len]\n\t" \
442 "jz 333f\n\t" \
443 "222:\n\t" \
444 "movb (%[src]), %%al\n\t" \
445 "movb %%al, (%[dst])\n\t" \
446 "inc %[src]\n\t" \
447 "inc %[dst]\n\t" \
448 "dec %[len]\n\t" \
449 "jnz 222b\n\t" \
450 "333:\n\t" \
451 RSEQ_INJECT_ASM(5)
452 /* final store */
453 "movq %[newv], %[v]\n\t"
454 "2:\n\t"
455 RSEQ_INJECT_ASM(6)
456 /* teardown */
457 "movq %[rseq_scratch2], %[len]\n\t"
458 "movq %[rseq_scratch1], %[dst]\n\t"
459 "movq %[rseq_scratch0], %[src]\n\t"
460 RSEQ_ASM_DEFINE_ABORT(4,
461 "movq %[rseq_scratch2], %[len]\n\t"
462 "movq %[rseq_scratch1], %[dst]\n\t"
463 "movq %[rseq_scratch0], %[src]\n\t",
464 abort)
465 RSEQ_ASM_DEFINE_CMPFAIL(5,
466 "movq %[rseq_scratch2], %[len]\n\t"
467 "movq %[rseq_scratch1], %[dst]\n\t"
468 "movq %[rseq_scratch0], %[src]\n\t",
469 cmpfail)
470#ifdef RSEQ_COMPARE_TWICE
471 RSEQ_ASM_DEFINE_CMPFAIL(6,
472 "movq %[rseq_scratch2], %[len]\n\t"
473 "movq %[rseq_scratch1], %[dst]\n\t"
474 "movq %[rseq_scratch0], %[src]\n\t",
475 error1)
476 RSEQ_ASM_DEFINE_CMPFAIL(7,
477 "movq %[rseq_scratch2], %[len]\n\t"
478 "movq %[rseq_scratch1], %[dst]\n\t"
479 "movq %[rseq_scratch0], %[src]\n\t",
480 error2)
481#endif
482 : /* gcc asm goto does not allow outputs */
483 : [cpu_id] "r" (cpu),
484 [current_cpu_id] "m" (__rseq_abi.cpu_id),
485 [rseq_cs] "m" (__rseq_abi.rseq_cs),
486 /* final store input */
487 [v] "m" (*v),
488 [expect] "r" (expect),
489 [newv] "r" (newv),
490 /* try memcpy input */
491 [dst] "r" (dst),
492 [src] "r" (src),
493 [len] "r" (len),
494 [rseq_scratch0] "m" (rseq_scratch[0]),
495 [rseq_scratch1] "m" (rseq_scratch[1]),
496 [rseq_scratch2] "m" (rseq_scratch[2])
497 : "memory", "cc", "rax"
498 RSEQ_INJECT_CLOBBER
499 : abort, cmpfail
500#ifdef RSEQ_COMPARE_TWICE
501 , error1, error2
502#endif
503 );
504 return 0;
505abort:
506 RSEQ_INJECT_FAILED
507 return -1;
508cmpfail:
509 return 1;
510#ifdef RSEQ_COMPARE_TWICE
511error1:
512 rseq_bug("cpu_id comparison failed");
513error2:
514 rseq_bug("expected value comparison failed");
515#endif
516}
517
518/* x86-64 is TSO. */
519static inline __attribute__((always_inline))
520int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
521 void *dst, void *src, size_t len,
522 intptr_t newv, int cpu)
523{
524 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
525 newv, cpu);
526}
527
528#endif /* !RSEQ_SKIP_FASTPATH */
529
530#elif __i386__
531
532#define rseq_smp_mb() \
533 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
534#define rseq_smp_rmb() \
535 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
536#define rseq_smp_wmb() \
537 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
538
539#define rseq_smp_load_acquire(p) \
540__extension__ ({ \
541 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
542 rseq_smp_mb(); \
543 ____p1; \
544})
545
546#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
547
548#define rseq_smp_store_release(p, v) \
549do { \
550 rseq_smp_mb(); \
551 RSEQ_WRITE_ONCE(*p, v); \
552} while (0)
553
554#ifdef RSEQ_SKIP_FASTPATH
555#include "rseq-skip.h"
556#else /* !RSEQ_SKIP_FASTPATH */
557
558/*
559 * Use eax as scratch register and take memory operands as input to
560 * lessen register pressure. Especially needed when compiling in O0.
561 */
562#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
563 start_ip, post_commit_offset, abort_ip) \
564 ".pushsection __rseq_table, \"aw\"\n\t" \
565 ".balign 32\n\t" \
566 __rseq_str(label) ":\n\t" \
567 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
568 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
569 ".popsection\n\t"
570
571#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
572 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
573 (post_commit_ip - start_ip), abort_ip)
574
90d9876e
MD		 575/*
576 * Exit points of a rseq critical section consist of all instructions outside
577 * of the critical section where a critical section can branch to. The abort IP
578 * is already part of the __rseq_table and should not be explicitly defined as
579 * an additional exit point. Knowing all exit points is useful to assist
580 * debuggers stepping over the critical section.
581 */
582#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
583 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
584 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
585 ".popsection\n\t"
586
784b0012
MD		 587#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
588 RSEQ_INJECT_ASM(1) \
589 "movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t" \
590 __rseq_str(label) ":\n\t"
591
592#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
593 RSEQ_INJECT_ASM(2) \
594 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
595 "jnz " __rseq_str(label) "\n\t"
596
597#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
598 ".pushsection __rseq_failure, \"ax\"\n\t" \
599 /* Disassembler-friendly signature: nopl <sig>. */ \
600 ".byte 0x0f, 0x1f, 0x05\n\t" \
601 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
602 __rseq_str(label) ":\n\t" \
603 teardown \
604 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
605 ".popsection\n\t"
606
607#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
608 ".pushsection __rseq_failure, \"ax\"\n\t" \
609 __rseq_str(label) ":\n\t" \
610 teardown \
611 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
612 ".popsection\n\t"
613
614static inline __attribute__((always_inline))
615int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
616{
617 RSEQ_INJECT_C(9)
618
619 __asm__ __volatile__ goto (
620 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 621 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
622#ifdef RSEQ_COMPARE_TWICE
623 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
624 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
625#endif
784b0012
MD		 626 /* Start rseq by storing table entry pointer into rseq_cs. */
627 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
628 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
629 RSEQ_INJECT_ASM(3)
630 "cmpl %[v], %[expect]\n\t"
631 "jnz %l[cmpfail]\n\t"
632 RSEQ_INJECT_ASM(4)
633#ifdef RSEQ_COMPARE_TWICE
634 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
635 "cmpl %[v], %[expect]\n\t"
636 "jnz %l[error2]\n\t"
637#endif
638 /* final store */
639 "movl %[newv], %[v]\n\t"
640 "2:\n\t"
641 RSEQ_INJECT_ASM(5)
642 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
643 : /* gcc asm goto does not allow outputs */
644 : [cpu_id] "r" (cpu),
645 [current_cpu_id] "m" (__rseq_abi.cpu_id),
646 [rseq_cs] "m" (__rseq_abi.rseq_cs),
647 [v] "m" (*v),
648 [expect] "r" (expect),
649 [newv] "r" (newv)
650 : "memory", "cc", "eax"
651 RSEQ_INJECT_CLOBBER
652 : abort, cmpfail
653#ifdef RSEQ_COMPARE_TWICE
654 , error1, error2
655#endif
656 );
657 return 0;
658abort:
659 RSEQ_INJECT_FAILED
660 return -1;
661cmpfail:
662 return 1;
663#ifdef RSEQ_COMPARE_TWICE
664error1:
665 rseq_bug("cpu_id comparison failed");
666error2:
667 rseq_bug("expected value comparison failed");
668#endif
669}
670
671/*
672 * Compare @v against @expectnot. When it does _not_ match, load @v
673 * into @load, and store the content of *@v + voffp into @v.
674 */
675static inline __attribute__((always_inline))
676int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
677 off_t voffp, intptr_t *load, int cpu)
678{
679 RSEQ_INJECT_C(9)
680
681 __asm__ __volatile__ goto (
682 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 683 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
684#ifdef RSEQ_COMPARE_TWICE
685 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
686 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
687#endif
784b0012
MD		 688 /* Start rseq by storing table entry pointer into rseq_cs. */
689 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
690 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
691 RSEQ_INJECT_ASM(3)
692 "movl %[v], %%ebx\n\t"
693 "cmpl %%ebx, %[expectnot]\n\t"
694 "je %l[cmpfail]\n\t"
695 RSEQ_INJECT_ASM(4)
696#ifdef RSEQ_COMPARE_TWICE
697 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
698 "movl %[v], %%ebx\n\t"
699 "cmpl %%ebx, %[expectnot]\n\t"
700 "je %l[error2]\n\t"
701#endif
702 "movl %%ebx, %[load]\n\t"
703 "addl %[voffp], %%ebx\n\t"
704 "movl (%%ebx), %%ebx\n\t"
705 /* final store */
706 "movl %%ebx, %[v]\n\t"
707 "2:\n\t"
708 RSEQ_INJECT_ASM(5)
709 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
710 : /* gcc asm goto does not allow outputs */
711 : [cpu_id] "r" (cpu),
712 [current_cpu_id] "m" (__rseq_abi.cpu_id),
713 [rseq_cs] "m" (__rseq_abi.rseq_cs),
714 /* final store input */
715 [v] "m" (*v),
716 [expectnot] "r" (expectnot),
717 [voffp] "ir" (voffp),
718 [load] "m" (*load)
719 : "memory", "cc", "eax", "ebx"
720 RSEQ_INJECT_CLOBBER
721 : abort, cmpfail
722#ifdef RSEQ_COMPARE_TWICE
723 , error1, error2
724#endif
725 );
726 return 0;
727abort:
728 RSEQ_INJECT_FAILED
729 return -1;
730cmpfail:
731 return 1;
732#ifdef RSEQ_COMPARE_TWICE
733error1:
734 rseq_bug("cpu_id comparison failed");
735error2:
736 rseq_bug("expected value comparison failed");
737#endif
738}
739
740static inline __attribute__((always_inline))
741int rseq_addv(intptr_t *v, intptr_t count, int cpu)
742{
743 RSEQ_INJECT_C(9)
744
745 __asm__ __volatile__ goto (
746 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 747#ifdef RSEQ_COMPARE_TWICE
748 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
749#endif
784b0012
MD		 750 /* Start rseq by storing table entry pointer into rseq_cs. */
751 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
752 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
753 RSEQ_INJECT_ASM(3)
754#ifdef RSEQ_COMPARE_TWICE
755 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
756#endif
757 /* final store */
758 "addl %[count], %[v]\n\t"
759 "2:\n\t"
760 RSEQ_INJECT_ASM(4)
761 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
762 : /* gcc asm goto does not allow outputs */
763 : [cpu_id] "r" (cpu),
764 [current_cpu_id] "m" (__rseq_abi.cpu_id),
765 [rseq_cs] "m" (__rseq_abi.rseq_cs),
766 /* final store input */
767 [v] "m" (*v),
768 [count] "ir" (count)
769 : "memory", "cc", "eax"
770 RSEQ_INJECT_CLOBBER
771 : abort
772#ifdef RSEQ_COMPARE_TWICE
773 , error1
774#endif
775 );
776 return 0;
777abort:
778 RSEQ_INJECT_FAILED
779 return -1;
780#ifdef RSEQ_COMPARE_TWICE
781error1:
782 rseq_bug("cpu_id comparison failed");
783#endif
784}
785
786static inline __attribute__((always_inline))
787int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
788 intptr_t *v2, intptr_t newv2,
789 intptr_t newv, int cpu)
790{
791 RSEQ_INJECT_C(9)
792
793 __asm__ __volatile__ goto (
794 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 795 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
796#ifdef RSEQ_COMPARE_TWICE
797 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
798 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
799#endif
784b0012
MD		 800 /* Start rseq by storing table entry pointer into rseq_cs. */
801 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
802 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
803 RSEQ_INJECT_ASM(3)
804 "cmpl %[v], %[expect]\n\t"
805 "jnz %l[cmpfail]\n\t"
806 RSEQ_INJECT_ASM(4)
807#ifdef RSEQ_COMPARE_TWICE
808 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
809 "cmpl %[v], %[expect]\n\t"
810 "jnz %l[error2]\n\t"
811#endif
812 /* try store */
813 "movl %[newv2], %%eax\n\t"
814 "movl %%eax, %[v2]\n\t"
815 RSEQ_INJECT_ASM(5)
816 /* final store */
817 "movl %[newv], %[v]\n\t"
818 "2:\n\t"
819 RSEQ_INJECT_ASM(6)
820 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
821 : /* gcc asm goto does not allow outputs */
822 : [cpu_id] "r" (cpu),
823 [current_cpu_id] "m" (__rseq_abi.cpu_id),
824 [rseq_cs] "m" (__rseq_abi.rseq_cs),
825 /* try store input */
826 [v2] "m" (*v2),
827 [newv2] "m" (newv2),
828 /* final store input */
829 [v] "m" (*v),
830 [expect] "r" (expect),
831 [newv] "r" (newv)
832 : "memory", "cc", "eax"
833 RSEQ_INJECT_CLOBBER
834 : abort, cmpfail
835#ifdef RSEQ_COMPARE_TWICE
836 , error1, error2
837#endif
838 );
839 return 0;
840abort:
841 RSEQ_INJECT_FAILED
842 return -1;
843cmpfail:
844 return 1;
845#ifdef RSEQ_COMPARE_TWICE
846error1:
847 rseq_bug("cpu_id comparison failed");
848error2:
849 rseq_bug("expected value comparison failed");
850#endif
851}
852
853static inline __attribute__((always_inline))
854int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
855 intptr_t *v2, intptr_t newv2,
856 intptr_t newv, int cpu)
857{
858 RSEQ_INJECT_C(9)
859
860 __asm__ __volatile__ goto (
861 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 862 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
863#ifdef RSEQ_COMPARE_TWICE
864 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
865 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
866#endif
784b0012
MD		 867 /* Start rseq by storing table entry pointer into rseq_cs. */
868 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
869 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
870 RSEQ_INJECT_ASM(3)
871 "movl %[expect], %%eax\n\t"
872 "cmpl %[v], %%eax\n\t"
873 "jnz %l[cmpfail]\n\t"
874 RSEQ_INJECT_ASM(4)
875#ifdef RSEQ_COMPARE_TWICE
876 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
877 "movl %[expect], %%eax\n\t"
878 "cmpl %[v], %%eax\n\t"
879 "jnz %l[error2]\n\t"
880#endif
881 /* try store */
882 "movl %[newv2], %[v2]\n\t"
883 RSEQ_INJECT_ASM(5)
884 "lock; addl $0,-128(%%esp)\n\t"
885 /* final store */
886 "movl %[newv], %[v]\n\t"
887 "2:\n\t"
888 RSEQ_INJECT_ASM(6)
889 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
890 : /* gcc asm goto does not allow outputs */
891 : [cpu_id] "r" (cpu),
892 [current_cpu_id] "m" (__rseq_abi.cpu_id),
893 [rseq_cs] "m" (__rseq_abi.rseq_cs),
894 /* try store input */
895 [v2] "m" (*v2),
896 [newv2] "r" (newv2),
897 /* final store input */
898 [v] "m" (*v),
899 [expect] "m" (expect),
900 [newv] "r" (newv)
901 : "memory", "cc", "eax"
902 RSEQ_INJECT_CLOBBER
903 : abort, cmpfail
904#ifdef RSEQ_COMPARE_TWICE
905 , error1, error2
906#endif
907 );
908 return 0;
909abort:
910 RSEQ_INJECT_FAILED
911 return -1;
912cmpfail:
913 return 1;
914#ifdef RSEQ_COMPARE_TWICE
915error1:
916 rseq_bug("cpu_id comparison failed");
917error2:
918 rseq_bug("expected value comparison failed");
919#endif
920
921}
922
923static inline __attribute__((always_inline))
924int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
925 intptr_t *v2, intptr_t expect2,
926 intptr_t newv, int cpu)
927{
928 RSEQ_INJECT_C(9)
929
930 __asm__ __volatile__ goto (
931 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 932 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
933#ifdef RSEQ_COMPARE_TWICE
934 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
935 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
936 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
937#endif
784b0012
MD		 938 /* Start rseq by storing table entry pointer into rseq_cs. */
939 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
940 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
941 RSEQ_INJECT_ASM(3)
942 "cmpl %[v], %[expect]\n\t"
943 "jnz %l[cmpfail]\n\t"
944 RSEQ_INJECT_ASM(4)
945 "cmpl %[expect2], %[v2]\n\t"
946 "jnz %l[cmpfail]\n\t"
947 RSEQ_INJECT_ASM(5)
948#ifdef RSEQ_COMPARE_TWICE
949 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
950 "cmpl %[v], %[expect]\n\t"
951 "jnz %l[error2]\n\t"
952 "cmpl %[expect2], %[v2]\n\t"
953 "jnz %l[error3]\n\t"
954#endif
955 "movl %[newv], %%eax\n\t"
956 /* final store */
957 "movl %%eax, %[v]\n\t"
958 "2:\n\t"
959 RSEQ_INJECT_ASM(6)
960 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
961 : /* gcc asm goto does not allow outputs */
962 : [cpu_id] "r" (cpu),
963 [current_cpu_id] "m" (__rseq_abi.cpu_id),
964 [rseq_cs] "m" (__rseq_abi.rseq_cs),
965 /* cmp2 input */
966 [v2] "m" (*v2),
967 [expect2] "r" (expect2),
968 /* final store input */
969 [v] "m" (*v),
970 [expect] "r" (expect),
971 [newv] "m" (newv)
972 : "memory", "cc", "eax"
973 RSEQ_INJECT_CLOBBER
974 : abort, cmpfail
975#ifdef RSEQ_COMPARE_TWICE
976 , error1, error2, error3
977#endif
978 );
979 return 0;
980abort:
981 RSEQ_INJECT_FAILED
982 return -1;
983cmpfail:
984 return 1;
985#ifdef RSEQ_COMPARE_TWICE
986error1:
987 rseq_bug("cpu_id comparison failed");
988error2:
989 rseq_bug("1st expected value comparison failed");
990error3:
991 rseq_bug("2nd expected value comparison failed");
992#endif
993}
994
995/* TODO: implement a faster memcpy. */
996static inline __attribute__((always_inline))
997int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
998 void *dst, void *src, size_t len,
999 intptr_t newv, int cpu)
1000{
1001 uint32_t rseq_scratch[3];
1002
1003 RSEQ_INJECT_C(9)
1004
1005 __asm__ __volatile__ goto (
1006 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 1007 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1008#ifdef RSEQ_COMPARE_TWICE
1009 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1010 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1011#endif
784b0012
MD		 1012 "movl %[src], %[rseq_scratch0]\n\t"
1013 "movl %[dst], %[rseq_scratch1]\n\t"
1014 "movl %[len], %[rseq_scratch2]\n\t"
1015 /* Start rseq by storing table entry pointer into rseq_cs. */
1016 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
1017 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1018 RSEQ_INJECT_ASM(3)
1019 "movl %[expect], %%eax\n\t"
1020 "cmpl %%eax, %[v]\n\t"
1021 "jnz 5f\n\t"
1022 RSEQ_INJECT_ASM(4)
1023#ifdef RSEQ_COMPARE_TWICE
1024 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
1025 "movl %[expect], %%eax\n\t"
1026 "cmpl %%eax, %[v]\n\t"
1027 "jnz 7f\n\t"
1028#endif
1029 /* try memcpy */
1030 "test %[len], %[len]\n\t" \
1031 "jz 333f\n\t" \
1032 "222:\n\t" \
1033 "movb (%[src]), %%al\n\t" \
1034 "movb %%al, (%[dst])\n\t" \
1035 "inc %[src]\n\t" \
1036 "inc %[dst]\n\t" \
1037 "dec %[len]\n\t" \
1038 "jnz 222b\n\t" \
1039 "333:\n\t" \
1040 RSEQ_INJECT_ASM(5)
1041 "movl %[newv], %%eax\n\t"
1042 /* final store */
1043 "movl %%eax, %[v]\n\t"
1044 "2:\n\t"
1045 RSEQ_INJECT_ASM(6)
1046 /* teardown */
1047 "movl %[rseq_scratch2], %[len]\n\t"
1048 "movl %[rseq_scratch1], %[dst]\n\t"
1049 "movl %[rseq_scratch0], %[src]\n\t"
1050 RSEQ_ASM_DEFINE_ABORT(4,
1051 "movl %[rseq_scratch2], %[len]\n\t"
1052 "movl %[rseq_scratch1], %[dst]\n\t"
1053 "movl %[rseq_scratch0], %[src]\n\t",
1054 abort)
1055 RSEQ_ASM_DEFINE_CMPFAIL(5,
1056 "movl %[rseq_scratch2], %[len]\n\t"
1057 "movl %[rseq_scratch1], %[dst]\n\t"
1058 "movl %[rseq_scratch0], %[src]\n\t",
1059 cmpfail)
1060#ifdef RSEQ_COMPARE_TWICE
1061 RSEQ_ASM_DEFINE_CMPFAIL(6,
1062 "movl %[rseq_scratch2], %[len]\n\t"
1063 "movl %[rseq_scratch1], %[dst]\n\t"
1064 "movl %[rseq_scratch0], %[src]\n\t",
1065 error1)
1066 RSEQ_ASM_DEFINE_CMPFAIL(7,
1067 "movl %[rseq_scratch2], %[len]\n\t"
1068 "movl %[rseq_scratch1], %[dst]\n\t"
1069 "movl %[rseq_scratch0], %[src]\n\t",
1070 error2)
1071#endif
1072 : /* gcc asm goto does not allow outputs */
1073 : [cpu_id] "r" (cpu),
1074 [current_cpu_id] "m" (__rseq_abi.cpu_id),
1075 [rseq_cs] "m" (__rseq_abi.rseq_cs),
1076 /* final store input */
1077 [v] "m" (*v),
1078 [expect] "m" (expect),
1079 [newv] "m" (newv),
1080 /* try memcpy input */
1081 [dst] "r" (dst),
1082 [src] "r" (src),
1083 [len] "r" (len),
1084 [rseq_scratch0] "m" (rseq_scratch[0]),
1085 [rseq_scratch1] "m" (rseq_scratch[1]),
1086 [rseq_scratch2] "m" (rseq_scratch[2])
1087 : "memory", "cc", "eax"
1088 RSEQ_INJECT_CLOBBER
1089 : abort, cmpfail
1090#ifdef RSEQ_COMPARE_TWICE
1091 , error1, error2
1092#endif
1093 );
1094 return 0;
1095abort:
1096 RSEQ_INJECT_FAILED
1097 return -1;
1098cmpfail:
1099 return 1;
1100#ifdef RSEQ_COMPARE_TWICE
1101error1:
1102 rseq_bug("cpu_id comparison failed");
1103error2:
1104 rseq_bug("expected value comparison failed");
1105#endif
1106}
1107
1108/* TODO: implement a faster memcpy. */
1109static inline __attribute__((always_inline))
1110int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1111 void *dst, void *src, size_t len,
1112 intptr_t newv, int cpu)
1113{
1114 uint32_t rseq_scratch[3];
1115
1116 RSEQ_INJECT_C(9)
1117
1118 __asm__ __volatile__ goto (
1119 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e
MD		 1120 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1121#ifdef RSEQ_COMPARE_TWICE
1122 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1123 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1124#endif
784b0012
MD		 1125 "movl %[src], %[rseq_scratch0]\n\t"
1126 "movl %[dst], %[rseq_scratch1]\n\t"
1127 "movl %[len], %[rseq_scratch2]\n\t"
1128 /* Start rseq by storing table entry pointer into rseq_cs. */
1129 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
1130 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1131 RSEQ_INJECT_ASM(3)
1132 "movl %[expect], %%eax\n\t"
1133 "cmpl %%eax, %[v]\n\t"
1134 "jnz 5f\n\t"
1135 RSEQ_INJECT_ASM(4)
1136#ifdef RSEQ_COMPARE_TWICE
1137 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
1138 "movl %[expect], %%eax\n\t"
1139 "cmpl %%eax, %[v]\n\t"
1140 "jnz 7f\n\t"
1141#endif
1142 /* try memcpy */
1143 "test %[len], %[len]\n\t" \
1144 "jz 333f\n\t" \
1145 "222:\n\t" \
1146 "movb (%[src]), %%al\n\t" \
1147 "movb %%al, (%[dst])\n\t" \
1148 "inc %[src]\n\t" \
1149 "inc %[dst]\n\t" \
1150 "dec %[len]\n\t" \
1151 "jnz 222b\n\t" \
1152 "333:\n\t" \
1153 RSEQ_INJECT_ASM(5)
1154 "lock; addl $0,-128(%%esp)\n\t"
1155 "movl %[newv], %%eax\n\t"
1156 /* final store */
1157 "movl %%eax, %[v]\n\t"
1158 "2:\n\t"
1159 RSEQ_INJECT_ASM(6)
1160 /* teardown */
1161 "movl %[rseq_scratch2], %[len]\n\t"
1162 "movl %[rseq_scratch1], %[dst]\n\t"
1163 "movl %[rseq_scratch0], %[src]\n\t"
1164 RSEQ_ASM_DEFINE_ABORT(4,
1165 "movl %[rseq_scratch2], %[len]\n\t"
1166 "movl %[rseq_scratch1], %[dst]\n\t"
1167 "movl %[rseq_scratch0], %[src]\n\t",
1168 abort)
1169 RSEQ_ASM_DEFINE_CMPFAIL(5,
1170 "movl %[rseq_scratch2], %[len]\n\t"
1171 "movl %[rseq_scratch1], %[dst]\n\t"
1172 "movl %[rseq_scratch0], %[src]\n\t",
1173 cmpfail)
1174#ifdef RSEQ_COMPARE_TWICE
1175 RSEQ_ASM_DEFINE_CMPFAIL(6,
1176 "movl %[rseq_scratch2], %[len]\n\t"
1177 "movl %[rseq_scratch1], %[dst]\n\t"
1178 "movl %[rseq_scratch0], %[src]\n\t",
1179 error1)
1180 RSEQ_ASM_DEFINE_CMPFAIL(7,
1181 "movl %[rseq_scratch2], %[len]\n\t"
1182 "movl %[rseq_scratch1], %[dst]\n\t"
1183 "movl %[rseq_scratch0], %[src]\n\t",
1184 error2)
1185#endif
1186 : /* gcc asm goto does not allow outputs */
1187 : [cpu_id] "r" (cpu),
1188 [current_cpu_id] "m" (__rseq_abi.cpu_id),
1189 [rseq_cs] "m" (__rseq_abi.rseq_cs),
1190 /* final store input */
1191 [v] "m" (*v),
1192 [expect] "m" (expect),
1193 [newv] "m" (newv),
1194 /* try memcpy input */
1195 [dst] "r" (dst),
1196 [src] "r" (src),
1197 [len] "r" (len),
1198 [rseq_scratch0] "m" (rseq_scratch[0]),
1199 [rseq_scratch1] "m" (rseq_scratch[1]),
1200 [rseq_scratch2] "m" (rseq_scratch[2])
1201 : "memory", "cc", "eax"
1202 RSEQ_INJECT_CLOBBER
1203 : abort, cmpfail
1204#ifdef RSEQ_COMPARE_TWICE
1205 , error1, error2
1206#endif
1207 );
1208 return 0;
1209abort:
1210 RSEQ_INJECT_FAILED
1211 return -1;
1212cmpfail:
1213 return 1;
1214#ifdef RSEQ_COMPARE_TWICE
1215error1:
1216 rseq_bug("cpu_id comparison failed");
1217error2:
1218 rseq_bug("expected value comparison failed");
1219#endif
1220}
1221
1222#endif /* !RSEQ_SKIP_FASTPATH */
1223
1224#endif
Librseq
This page took 0.363423 seconds and 4 git commands to generate.