projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Use ptrdiff_t for rseq_offset
[librseq.git] / include / rseq / rseq-x86.h
1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
2 /*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #include <stdint.h>
9
10 /*
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
12 * trap in user-space:
13 *
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
16 */
17 #define RSEQ_SIG 0x53053053
18
19 /* Offset of cpu_id and rseq_cs fields in struct rseq_abi. */
20 #define RSEQ_CPU_ID_OFFSET 4
21 #define RSEQ_CS_OFFSET 8
22
23 #ifdef __x86_64__
24
25 #define RSEQ_ASM_TP_SEGMENT %%fs
26
27 #define rseq_smp_mb() \
28 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
29 #define rseq_smp_rmb() rseq_barrier()
30 #define rseq_smp_wmb() rseq_barrier()
31
32 #define rseq_smp_load_acquire(p) \
33 __extension__ ({ \
34 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
35 rseq_barrier(); \
36 ____p1; \
37 })
38
39 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
40
41 #define rseq_smp_store_release(p, v) \
42 do { \
43 rseq_barrier(); \
44 RSEQ_WRITE_ONCE(*p, v); \
45 } while (0)
46
47 #ifdef RSEQ_SKIP_FASTPATH
48 #include "rseq-skip.h"
49 #else /* !RSEQ_SKIP_FASTPATH */
50
51 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
52 start_ip, post_commit_offset, abort_ip) \
53 ".pushsection __rseq_cs, \"aw\"\n\t" \
54 ".balign 32\n\t" \
55 __rseq_str(label) ":\n\t" \
56 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
57 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
58 ".popsection\n\t" \
59 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
60 ".quad " __rseq_str(label) "b\n\t" \
61 ".popsection\n\t"
62
63
64 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
65 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
66 (post_commit_ip - start_ip), abort_ip)
67
68 /*
69 * Exit points of a rseq critical section consist of all instructions outside
70 * of the critical section where a critical section can either branch to or
71 * reach through the normal course of its execution. The abort IP and the
72 * post-commit IP are already part of the __rseq_cs section and should not be
73 * explicitly defined as additional exit points. Knowing all exit points is
74 * useful to assist debuggers stepping over the critical section.
75 */
76 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
77 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
78 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
79 ".popsection\n\t"
80
81 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
82 RSEQ_INJECT_ASM(1) \
83 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
84 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
85 __rseq_str(label) ":\n\t"
86
87 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
88 RSEQ_INJECT_ASM(2) \
89 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
90 "jnz " __rseq_str(label) "\n\t"
91
92 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
93 ".pushsection __rseq_failure, \"ax\"\n\t" \
94 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
95 ".byte 0x0f, 0xb9, 0x3d\n\t" \
96 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
97 __rseq_str(label) ":\n\t" \
98 teardown \
99 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
100 ".popsection\n\t"
101
102 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
103 ".pushsection __rseq_failure, \"ax\"\n\t" \
104 __rseq_str(label) ":\n\t" \
105 teardown \
106 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
107 ".popsection\n\t"
108
109 static inline __attribute__((always_inline))
110 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
111 {
112 RSEQ_INJECT_C(9)
113
114 __asm__ __volatile__ goto (
115 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
116 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
117 #ifdef RSEQ_COMPARE_TWICE
118 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
119 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
120 #endif
121 /* Start rseq by storing table entry pointer into rseq_cs. */
122 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
123 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
124 RSEQ_INJECT_ASM(3)
125 "cmpq %[v], %[expect]\n\t"
126 "jnz %l[cmpfail]\n\t"
127 RSEQ_INJECT_ASM(4)
128 #ifdef RSEQ_COMPARE_TWICE
129 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
130 "cmpq %[v], %[expect]\n\t"
131 "jnz %l[error2]\n\t"
132 #endif
133 /* final store */
134 "movq %[newv], %[v]\n\t"
135 "2:\n\t"
136 RSEQ_INJECT_ASM(5)
137 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
138 : /* gcc asm goto does not allow outputs */
139 : [cpu_id] "r" (cpu),
140 [rseq_offset] "r" (rseq_offset),
141 [v] "m" (*v),
142 [expect] "r" (expect),
143 [newv] "r" (newv)
144 : "memory", "cc", "rax"
145 RSEQ_INJECT_CLOBBER
146 : abort, cmpfail
147 #ifdef RSEQ_COMPARE_TWICE
148 , error1, error2
149 #endif
150 );
151 rseq_after_asm_goto();
152 return 0;
153 abort:
154 rseq_after_asm_goto();
155 RSEQ_INJECT_FAILED
156 return -1;
157 cmpfail:
158 rseq_after_asm_goto();
159 return 1;
160 #ifdef RSEQ_COMPARE_TWICE
161 error1:
162 rseq_after_asm_goto();
163 rseq_bug("cpu_id comparison failed");
164 error2:
165 rseq_after_asm_goto();
166 rseq_bug("expected value comparison failed");
167 #endif
168 }
169
170 /*
171 * Compare @v against @expectnot. When it does _not_ match, load @v
172 * into @load, and store the content of *@v + voffp into @v.
173 */
174 static inline __attribute__((always_inline))
175 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
176 long voffp, intptr_t *load, int cpu)
177 {
178 RSEQ_INJECT_C(9)
179
180 __asm__ __volatile__ goto (
181 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
182 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
183 #ifdef RSEQ_COMPARE_TWICE
184 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
185 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
186 #endif
187 /* Start rseq by storing table entry pointer into rseq_cs. */
188 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
189 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
190 RSEQ_INJECT_ASM(3)
191 "movq %[v], %%rbx\n\t"
192 "cmpq %%rbx, %[expectnot]\n\t"
193 "je %l[cmpfail]\n\t"
194 RSEQ_INJECT_ASM(4)
195 #ifdef RSEQ_COMPARE_TWICE
196 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
197 "movq %[v], %%rbx\n\t"
198 "cmpq %%rbx, %[expectnot]\n\t"
199 "je %l[error2]\n\t"
200 #endif
201 "movq %%rbx, %[load]\n\t"
202 "addq %[voffp], %%rbx\n\t"
203 "movq (%%rbx), %%rbx\n\t"
204 /* final store */
205 "movq %%rbx, %[v]\n\t"
206 "2:\n\t"
207 RSEQ_INJECT_ASM(5)
208 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
209 : /* gcc asm goto does not allow outputs */
210 : [cpu_id] "r" (cpu),
211 [rseq_offset] "r" (rseq_offset),
212 /* final store input */
213 [v] "m" (*v),
214 [expectnot] "r" (expectnot),
215 [voffp] "er" (voffp),
216 [load] "m" (*load)
217 : "memory", "cc", "rax", "rbx"
218 RSEQ_INJECT_CLOBBER
219 : abort, cmpfail
220 #ifdef RSEQ_COMPARE_TWICE
221 , error1, error2
222 #endif
223 );
224 rseq_after_asm_goto();
225 return 0;
226 abort:
227 rseq_after_asm_goto();
228 RSEQ_INJECT_FAILED
229 return -1;
230 cmpfail:
231 rseq_after_asm_goto();
232 return 1;
233 #ifdef RSEQ_COMPARE_TWICE
234 error1:
235 rseq_after_asm_goto();
236 rseq_bug("cpu_id comparison failed");
237 error2:
238 rseq_after_asm_goto();
239 rseq_bug("expected value comparison failed");
240 #endif
241 }
242
243 static inline __attribute__((always_inline))
244 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
245 {
246 RSEQ_INJECT_C(9)
247
248 __asm__ __volatile__ goto (
249 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
250 #ifdef RSEQ_COMPARE_TWICE
251 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
252 #endif
253 /* Start rseq by storing table entry pointer into rseq_cs. */
254 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
255 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
256 RSEQ_INJECT_ASM(3)
257 #ifdef RSEQ_COMPARE_TWICE
258 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
259 #endif
260 /* final store */
261 "addq %[count], %[v]\n\t"
262 "2:\n\t"
263 RSEQ_INJECT_ASM(4)
264 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
265 : /* gcc asm goto does not allow outputs */
266 : [cpu_id] "r" (cpu),
267 [rseq_offset] "r" (rseq_offset),
268 /* final store input */
269 [v] "m" (*v),
270 [count] "er" (count)
271 : "memory", "cc", "rax"
272 RSEQ_INJECT_CLOBBER
273 : abort
274 #ifdef RSEQ_COMPARE_TWICE
275 , error1
276 #endif
277 );
278 rseq_after_asm_goto();
279 return 0;
280 abort:
281 rseq_after_asm_goto();
282 RSEQ_INJECT_FAILED
283 return -1;
284 #ifdef RSEQ_COMPARE_TWICE
285 error1:
286 rseq_after_asm_goto();
287 rseq_bug("cpu_id comparison failed");
288 #endif
289 }
290
291 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
292
293 /*
294 * pval = *(ptr+off)
295 * *pval += inc;
296 */
297 static inline __attribute__((always_inline))
298 int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
299 {
300 RSEQ_INJECT_C(9)
301
302 __asm__ __volatile__ goto (
303 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
304 #ifdef RSEQ_COMPARE_TWICE
305 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
306 #endif
307 /* Start rseq by storing table entry pointer into rseq_cs. */
308 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
309 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
310 RSEQ_INJECT_ASM(3)
311 #ifdef RSEQ_COMPARE_TWICE
312 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
313 #endif
314 /* get p+v */
315 "movq %[ptr], %%rbx\n\t"
316 "addq %[off], %%rbx\n\t"
317 /* get pv */
318 "movq (%%rbx), %%rcx\n\t"
319 /* *pv += inc */
320 "addq %[inc], (%%rcx)\n\t"
321 "2:\n\t"
322 RSEQ_INJECT_ASM(4)
323 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
324 : /* gcc asm goto does not allow outputs */
325 : [cpu_id] "r" (cpu),
326 [rseq_offset] "r" (rseq_offset),
327 /* final store input */
328 [ptr] "m" (*ptr),
329 [off] "er" (off),
330 [inc] "er" (inc)
331 : "memory", "cc", "rax", "rbx", "rcx"
332 RSEQ_INJECT_CLOBBER
333 : abort
334 #ifdef RSEQ_COMPARE_TWICE
335 , error1
336 #endif
337 );
338 rseq_after_asm_goto();
339 return 0;
340 abort:
341 rseq_after_asm_goto();
342 RSEQ_INJECT_FAILED
343 return -1;
344 #ifdef RSEQ_COMPARE_TWICE
345 error1:
346 rseq_after_asm_goto();
347 rseq_bug("cpu_id comparison failed");
348 #endif
349 }
350
351 static inline __attribute__((always_inline))
352 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
353 intptr_t *v2, intptr_t newv2,
354 intptr_t newv, int cpu)
355 {
356 RSEQ_INJECT_C(9)
357
358 __asm__ __volatile__ goto (
359 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
360 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
361 #ifdef RSEQ_COMPARE_TWICE
362 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
363 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
364 #endif
365 /* Start rseq by storing table entry pointer into rseq_cs. */
366 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
367 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
368 RSEQ_INJECT_ASM(3)
369 "cmpq %[v], %[expect]\n\t"
370 "jnz %l[cmpfail]\n\t"
371 RSEQ_INJECT_ASM(4)
372 #ifdef RSEQ_COMPARE_TWICE
373 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
374 "cmpq %[v], %[expect]\n\t"
375 "jnz %l[error2]\n\t"
376 #endif
377 /* try store */
378 "movq %[newv2], %[v2]\n\t"
379 RSEQ_INJECT_ASM(5)
380 /* final store */
381 "movq %[newv], %[v]\n\t"
382 "2:\n\t"
383 RSEQ_INJECT_ASM(6)
384 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
385 : /* gcc asm goto does not allow outputs */
386 : [cpu_id] "r" (cpu),
387 [rseq_offset] "r" (rseq_offset),
388 /* try store input */
389 [v2] "m" (*v2),
390 [newv2] "r" (newv2),
391 /* final store input */
392 [v] "m" (*v),
393 [expect] "r" (expect),
394 [newv] "r" (newv)
395 : "memory", "cc", "rax"
396 RSEQ_INJECT_CLOBBER
397 : abort, cmpfail
398 #ifdef RSEQ_COMPARE_TWICE
399 , error1, error2
400 #endif
401 );
402 rseq_after_asm_goto();
403 return 0;
404 abort:
405 rseq_after_asm_goto();
406 RSEQ_INJECT_FAILED
407 return -1;
408 cmpfail:
409 rseq_after_asm_goto();
410 return 1;
411 #ifdef RSEQ_COMPARE_TWICE
412 error1:
413 rseq_after_asm_goto();
414 rseq_bug("cpu_id comparison failed");
415 error2:
416 rseq_after_asm_goto();
417 rseq_bug("expected value comparison failed");
418 #endif
419 }
420
421 /* x86-64 is TSO. */
422 static inline __attribute__((always_inline))
423 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
424 intptr_t *v2, intptr_t newv2,
425 intptr_t newv, int cpu)
426 {
427 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
428 }
429
430 static inline __attribute__((always_inline))
431 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
432 intptr_t *v2, intptr_t expect2,
433 intptr_t newv, int cpu)
434 {
435 RSEQ_INJECT_C(9)
436
437 __asm__ __volatile__ goto (
438 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
439 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
440 #ifdef RSEQ_COMPARE_TWICE
441 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
442 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
443 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
444 #endif
445 /* Start rseq by storing table entry pointer into rseq_cs. */
446 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
447 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
448 RSEQ_INJECT_ASM(3)
449 "cmpq %[v], %[expect]\n\t"
450 "jnz %l[cmpfail]\n\t"
451 RSEQ_INJECT_ASM(4)
452 "cmpq %[v2], %[expect2]\n\t"
453 "jnz %l[cmpfail]\n\t"
454 RSEQ_INJECT_ASM(5)
455 #ifdef RSEQ_COMPARE_TWICE
456 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
457 "cmpq %[v], %[expect]\n\t"
458 "jnz %l[error2]\n\t"
459 "cmpq %[v2], %[expect2]\n\t"
460 "jnz %l[error3]\n\t"
461 #endif
462 /* final store */
463 "movq %[newv], %[v]\n\t"
464 "2:\n\t"
465 RSEQ_INJECT_ASM(6)
466 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
467 : /* gcc asm goto does not allow outputs */
468 : [cpu_id] "r" (cpu),
469 [rseq_offset] "r" (rseq_offset),
470 /* cmp2 input */
471 [v2] "m" (*v2),
472 [expect2] "r" (expect2),
473 /* final store input */
474 [v] "m" (*v),
475 [expect] "r" (expect),
476 [newv] "r" (newv)
477 : "memory", "cc", "rax"
478 RSEQ_INJECT_CLOBBER
479 : abort, cmpfail
480 #ifdef RSEQ_COMPARE_TWICE
481 , error1, error2, error3
482 #endif
483 );
484 rseq_after_asm_goto();
485 return 0;
486 abort:
487 rseq_after_asm_goto();
488 RSEQ_INJECT_FAILED
489 return -1;
490 cmpfail:
491 rseq_after_asm_goto();
492 return 1;
493 #ifdef RSEQ_COMPARE_TWICE
494 error1:
495 rseq_after_asm_goto();
496 rseq_bug("cpu_id comparison failed");
497 error2:
498 rseq_after_asm_goto();
499 rseq_bug("1st expected value comparison failed");
500 error3:
501 rseq_after_asm_goto();
502 rseq_bug("2nd expected value comparison failed");
503 #endif
504 }
505
506 static inline __attribute__((always_inline))
507 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
508 void *dst, void *src, size_t len,
509 intptr_t newv, int cpu)
510 {
511 uint64_t rseq_scratch[3];
512
513 RSEQ_INJECT_C(9)
514
515 __asm__ __volatile__ goto (
516 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
517 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
518 #ifdef RSEQ_COMPARE_TWICE
519 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
520 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
521 #endif
522 "movq %[src], %[rseq_scratch0]\n\t"
523 "movq %[dst], %[rseq_scratch1]\n\t"
524 "movq %[len], %[rseq_scratch2]\n\t"
525 /* Start rseq by storing table entry pointer into rseq_cs. */
526 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
527 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
528 RSEQ_INJECT_ASM(3)
529 "cmpq %[v], %[expect]\n\t"
530 "jnz 5f\n\t"
531 RSEQ_INJECT_ASM(4)
532 #ifdef RSEQ_COMPARE_TWICE
533 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
534 "cmpq %[v], %[expect]\n\t"
535 "jnz 7f\n\t"
536 #endif
537 /* try memcpy */
538 "test %[len], %[len]\n\t" \
539 "jz 333f\n\t" \
540 "222:\n\t" \
541 "movb (%[src]), %%al\n\t" \
542 "movb %%al, (%[dst])\n\t" \
543 "inc %[src]\n\t" \
544 "inc %[dst]\n\t" \
545 "dec %[len]\n\t" \
546 "jnz 222b\n\t" \
547 "333:\n\t" \
548 RSEQ_INJECT_ASM(5)
549 /* final store */
550 "movq %[newv], %[v]\n\t"
551 "2:\n\t"
552 RSEQ_INJECT_ASM(6)
553 /* teardown */
554 "movq %[rseq_scratch2], %[len]\n\t"
555 "movq %[rseq_scratch1], %[dst]\n\t"
556 "movq %[rseq_scratch0], %[src]\n\t"
557 RSEQ_ASM_DEFINE_ABORT(4,
558 "movq %[rseq_scratch2], %[len]\n\t"
559 "movq %[rseq_scratch1], %[dst]\n\t"
560 "movq %[rseq_scratch0], %[src]\n\t",
561 abort)
562 RSEQ_ASM_DEFINE_CMPFAIL(5,
563 "movq %[rseq_scratch2], %[len]\n\t"
564 "movq %[rseq_scratch1], %[dst]\n\t"
565 "movq %[rseq_scratch0], %[src]\n\t",
566 cmpfail)
567 #ifdef RSEQ_COMPARE_TWICE
568 RSEQ_ASM_DEFINE_CMPFAIL(6,
569 "movq %[rseq_scratch2], %[len]\n\t"
570 "movq %[rseq_scratch1], %[dst]\n\t"
571 "movq %[rseq_scratch0], %[src]\n\t",
572 error1)
573 RSEQ_ASM_DEFINE_CMPFAIL(7,
574 "movq %[rseq_scratch2], %[len]\n\t"
575 "movq %[rseq_scratch1], %[dst]\n\t"
576 "movq %[rseq_scratch0], %[src]\n\t",
577 error2)
578 #endif
579 : /* gcc asm goto does not allow outputs */
580 : [cpu_id] "r" (cpu),
581 [rseq_offset] "r" (rseq_offset),
582 /* final store input */
583 [v] "m" (*v),
584 [expect] "r" (expect),
585 [newv] "r" (newv),
586 /* try memcpy input */
587 [dst] "r" (dst),
588 [src] "r" (src),
589 [len] "r" (len),
590 [rseq_scratch0] "m" (rseq_scratch[0]),
591 [rseq_scratch1] "m" (rseq_scratch[1]),
592 [rseq_scratch2] "m" (rseq_scratch[2])
593 : "memory", "cc", "rax"
594 RSEQ_INJECT_CLOBBER
595 : abort, cmpfail
596 #ifdef RSEQ_COMPARE_TWICE
597 , error1, error2
598 #endif
599 );
600 rseq_after_asm_goto();
601 return 0;
602 abort:
603 rseq_after_asm_goto();
604 RSEQ_INJECT_FAILED
605 return -1;
606 cmpfail:
607 rseq_after_asm_goto();
608 return 1;
609 #ifdef RSEQ_COMPARE_TWICE
610 error1:
611 rseq_after_asm_goto();
612 rseq_bug("cpu_id comparison failed");
613 error2:
614 rseq_after_asm_goto();
615 rseq_bug("expected value comparison failed");
616 #endif
617 }
618
619 /* x86-64 is TSO. */
620 static inline __attribute__((always_inline))
621 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
622 void *dst, void *src, size_t len,
623 intptr_t newv, int cpu)
624 {
625 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
626 newv, cpu);
627 }
628
629 /*
630 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
631 * into @load.
632 */
633 static inline __attribute__((always_inline))
634 int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu)
635 {
636 RSEQ_INJECT_C(9)
637
638 __asm__ __volatile__ goto (
639 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
640 #ifdef RSEQ_COMPARE_TWICE
641 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
642 #endif
643 /* Start rseq by storing table entry pointer into rseq_cs. */
644 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
645 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
646 RSEQ_INJECT_ASM(3)
647 "movq %[p], %%rbx\n\t"
648 RSEQ_INJECT_ASM(4)
649 #ifdef RSEQ_COMPARE_TWICE
650 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
651 #endif
652 "addq %[voffp], %%rbx\n\t"
653 "movq (%%rbx), %%rbx\n\t"
654 "movq %%rbx, %[load]\n\t"
655 "2:\n\t"
656 RSEQ_INJECT_ASM(5)
657 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
658 : /* gcc asm goto does not allow outputs */
659 : [cpu_id] "r" (cpu),
660 [rseq_offset] "r" (rseq_offset),
661 /* final store input */
662 [p] "m" (*p),
663 [voffp] "er" (voffp),
664 [load] "m" (*load)
665 : "memory", "cc", "rax", "rbx"
666 RSEQ_INJECT_CLOBBER
667 : abort
668 #ifdef RSEQ_COMPARE_TWICE
669 , error1
670 #endif
671 );
672 rseq_after_asm_goto();
673 return 0;
674 abort:
675 rseq_after_asm_goto();
676 RSEQ_INJECT_FAILED
677 return -1;
678 #ifdef RSEQ_COMPARE_TWICE
679 error1:
680 rseq_after_asm_goto();
681 rseq_bug("cpu_id comparison failed");
682 #endif
683 }
684
685 #endif /* !RSEQ_SKIP_FASTPATH */
686
687 #elif defined(__i386__)
688
689 #define RSEQ_ASM_TP_SEGMENT %%gs
690
691 #define rseq_smp_mb() \
692 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
693 #define rseq_smp_rmb() \
694 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
695 #define rseq_smp_wmb() \
696 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
697
698 #define rseq_smp_load_acquire(p) \
699 __extension__ ({ \
700 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
701 rseq_smp_mb(); \
702 ____p1; \
703 })
704
705 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
706
707 #define rseq_smp_store_release(p, v) \
708 do { \
709 rseq_smp_mb(); \
710 RSEQ_WRITE_ONCE(*p, v); \
711 } while (0)
712
713 #ifdef RSEQ_SKIP_FASTPATH
714 #include "rseq-skip.h"
715 #else /* !RSEQ_SKIP_FASTPATH */
716
717 /*
718 * Use eax as scratch register and take memory operands as input to
719 * lessen register pressure. Especially needed when compiling in O0.
720 */
721 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
722 start_ip, post_commit_offset, abort_ip) \
723 ".pushsection __rseq_cs, \"aw\"\n\t" \
724 ".balign 32\n\t" \
725 __rseq_str(label) ":\n\t" \
726 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
727 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
728 ".popsection\n\t" \
729 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
730 ".long " __rseq_str(label) "b, 0x0\n\t" \
731 ".popsection\n\t"
732
733 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
734 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
735 (post_commit_ip - start_ip), abort_ip)
736
737 /*
738 * Exit points of a rseq critical section consist of all instructions outside
739 * of the critical section where a critical section can either branch to or
740 * reach through the normal course of its execution. The abort IP and the
741 * post-commit IP are already part of the __rseq_cs section and should not be
742 * explicitly defined as additional exit points. Knowing all exit points is
743 * useful to assist debuggers stepping over the critical section.
744 */
745 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
746 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
747 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
748 ".popsection\n\t"
749
750 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
751 RSEQ_INJECT_ASM(1) \
752 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
753 __rseq_str(label) ":\n\t"
754
755 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
756 RSEQ_INJECT_ASM(2) \
757 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
758 "jnz " __rseq_str(label) "\n\t"
759
760 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
761 ".pushsection __rseq_failure, \"ax\"\n\t" \
762 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
763 ".byte 0x0f, 0xb9, 0x3d\n\t" \
764 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
765 __rseq_str(label) ":\n\t" \
766 teardown \
767 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
768 ".popsection\n\t"
769
770 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
771 ".pushsection __rseq_failure, \"ax\"\n\t" \
772 __rseq_str(label) ":\n\t" \
773 teardown \
774 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
775 ".popsection\n\t"
776
777 static inline __attribute__((always_inline))
778 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
779 {
780 RSEQ_INJECT_C(9)
781
782 __asm__ __volatile__ goto (
783 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
784 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
785 #ifdef RSEQ_COMPARE_TWICE
786 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
787 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
788 #endif
789 /* Start rseq by storing table entry pointer into rseq_cs. */
790 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
791 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
792 RSEQ_INJECT_ASM(3)
793 "cmpl %[v], %[expect]\n\t"
794 "jnz %l[cmpfail]\n\t"
795 RSEQ_INJECT_ASM(4)
796 #ifdef RSEQ_COMPARE_TWICE
797 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
798 "cmpl %[v], %[expect]\n\t"
799 "jnz %l[error2]\n\t"
800 #endif
801 /* final store */
802 "movl %[newv], %[v]\n\t"
803 "2:\n\t"
804 RSEQ_INJECT_ASM(5)
805 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
806 : /* gcc asm goto does not allow outputs */
807 : [cpu_id] "r" (cpu),
808 [rseq_offset] "r" (rseq_offset),
809 [v] "m" (*v),
810 [expect] "r" (expect),
811 [newv] "r" (newv)
812 : "memory", "cc", "eax"
813 RSEQ_INJECT_CLOBBER
814 : abort, cmpfail
815 #ifdef RSEQ_COMPARE_TWICE
816 , error1, error2
817 #endif
818 );
819 rseq_after_asm_goto();
820 return 0;
821 abort:
822 rseq_after_asm_goto();
823 RSEQ_INJECT_FAILED
824 return -1;
825 cmpfail:
826 rseq_after_asm_goto();
827 return 1;
828 #ifdef RSEQ_COMPARE_TWICE
829 error1:
830 rseq_after_asm_goto();
831 rseq_bug("cpu_id comparison failed");
832 error2:
833 rseq_after_asm_goto();
834 rseq_bug("expected value comparison failed");
835 #endif
836 }
837
838 /*
839 * Compare @v against @expectnot. When it does _not_ match, load @v
840 * into @load, and store the content of *@v + voffp into @v.
841 */
842 static inline __attribute__((always_inline))
843 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
844 long voffp, intptr_t *load, int cpu)
845 {
846 RSEQ_INJECT_C(9)
847
848 __asm__ __volatile__ goto (
849 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
850 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
851 #ifdef RSEQ_COMPARE_TWICE
852 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
853 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
854 #endif
855 /* Start rseq by storing table entry pointer into rseq_cs. */
856 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
857 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
858 RSEQ_INJECT_ASM(3)
859 "movl %[v], %%ebx\n\t"
860 "cmpl %%ebx, %[expectnot]\n\t"
861 "je %l[cmpfail]\n\t"
862 RSEQ_INJECT_ASM(4)
863 #ifdef RSEQ_COMPARE_TWICE
864 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
865 "movl %[v], %%ebx\n\t"
866 "cmpl %%ebx, %[expectnot]\n\t"
867 "je %l[error2]\n\t"
868 #endif
869 "movl %%ebx, %[load]\n\t"
870 "addl %[voffp], %%ebx\n\t"
871 "movl (%%ebx), %%ebx\n\t"
872 /* final store */
873 "movl %%ebx, %[v]\n\t"
874 "2:\n\t"
875 RSEQ_INJECT_ASM(5)
876 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
877 : /* gcc asm goto does not allow outputs */
878 : [cpu_id] "r" (cpu),
879 [rseq_offset] "r" (rseq_offset),
880 /* final store input */
881 [v] "m" (*v),
882 [expectnot] "r" (expectnot),
883 [voffp] "ir" (voffp),
884 [load] "m" (*load)
885 : "memory", "cc", "eax", "ebx"
886 RSEQ_INJECT_CLOBBER
887 : abort, cmpfail
888 #ifdef RSEQ_COMPARE_TWICE
889 , error1, error2
890 #endif
891 );
892 rseq_after_asm_goto();
893 return 0;
894 abort:
895 rseq_after_asm_goto();
896 RSEQ_INJECT_FAILED
897 return -1;
898 cmpfail:
899 rseq_after_asm_goto();
900 return 1;
901 #ifdef RSEQ_COMPARE_TWICE
902 error1:
903 rseq_after_asm_goto();
904 rseq_bug("cpu_id comparison failed");
905 error2:
906 rseq_after_asm_goto();
907 rseq_bug("expected value comparison failed");
908 #endif
909 }
910
911 static inline __attribute__((always_inline))
912 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
913 {
914 RSEQ_INJECT_C(9)
915
916 __asm__ __volatile__ goto (
917 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
918 #ifdef RSEQ_COMPARE_TWICE
919 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
920 #endif
921 /* Start rseq by storing table entry pointer into rseq_cs. */
922 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
923 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
924 RSEQ_INJECT_ASM(3)
925 #ifdef RSEQ_COMPARE_TWICE
926 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
927 #endif
928 /* final store */
929 "addl %[count], %[v]\n\t"
930 "2:\n\t"
931 RSEQ_INJECT_ASM(4)
932 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
933 : /* gcc asm goto does not allow outputs */
934 : [cpu_id] "r" (cpu),
935 [rseq_offset] "r" (rseq_offset),
936 /* final store input */
937 [v] "m" (*v),
938 [count] "ir" (count)
939 : "memory", "cc", "eax"
940 RSEQ_INJECT_CLOBBER
941 : abort
942 #ifdef RSEQ_COMPARE_TWICE
943 , error1
944 #endif
945 );
946 rseq_after_asm_goto();
947 return 0;
948 abort:
949 rseq_after_asm_goto();
950 RSEQ_INJECT_FAILED
951 return -1;
952 #ifdef RSEQ_COMPARE_TWICE
953 error1:
954 rseq_after_asm_goto();
955 rseq_bug("cpu_id comparison failed");
956 #endif
957 }
958
959 static inline __attribute__((always_inline))
960 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
961 intptr_t *v2, intptr_t newv2,
962 intptr_t newv, int cpu)
963 {
964 RSEQ_INJECT_C(9)
965
966 __asm__ __volatile__ goto (
967 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
968 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
969 #ifdef RSEQ_COMPARE_TWICE
970 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
971 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
972 #endif
973 /* Start rseq by storing table entry pointer into rseq_cs. */
974 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
975 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
976 RSEQ_INJECT_ASM(3)
977 "cmpl %[v], %[expect]\n\t"
978 "jnz %l[cmpfail]\n\t"
979 RSEQ_INJECT_ASM(4)
980 #ifdef RSEQ_COMPARE_TWICE
981 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
982 "cmpl %[v], %[expect]\n\t"
983 "jnz %l[error2]\n\t"
984 #endif
985 /* try store */
986 "movl %[newv2], %%eax\n\t"
987 "movl %%eax, %[v2]\n\t"
988 RSEQ_INJECT_ASM(5)
989 /* final store */
990 "movl %[newv], %[v]\n\t"
991 "2:\n\t"
992 RSEQ_INJECT_ASM(6)
993 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
994 : /* gcc asm goto does not allow outputs */
995 : [cpu_id] "r" (cpu),
996 [rseq_offset] "r" (rseq_offset),
997 /* try store input */
998 [v2] "m" (*v2),
999 [newv2] "m" (newv2),
1000 /* final store input */
1001 [v] "m" (*v),
1002 [expect] "r" (expect),
1003 [newv] "r" (newv)
1004 : "memory", "cc", "eax"
1005 RSEQ_INJECT_CLOBBER
1006 : abort, cmpfail
1007 #ifdef RSEQ_COMPARE_TWICE
1008 , error1, error2
1009 #endif
1010 );
1011 rseq_after_asm_goto();
1012 return 0;
1013 abort:
1014 rseq_after_asm_goto();
1015 RSEQ_INJECT_FAILED
1016 return -1;
1017 cmpfail:
1018 rseq_after_asm_goto();
1019 return 1;
1020 #ifdef RSEQ_COMPARE_TWICE
1021 error1:
1022 rseq_after_asm_goto();
1023 rseq_bug("cpu_id comparison failed");
1024 error2:
1025 rseq_after_asm_goto();
1026 rseq_bug("expected value comparison failed");
1027 #endif
1028 }
1029
1030 static inline __attribute__((always_inline))
1031 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
1032 intptr_t *v2, intptr_t newv2,
1033 intptr_t newv, int cpu)
1034 {
1035 RSEQ_INJECT_C(9)
1036
1037 __asm__ __volatile__ goto (
1038 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1039 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1040 #ifdef RSEQ_COMPARE_TWICE
1041 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1042 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1043 #endif
1044 /* Start rseq by storing table entry pointer into rseq_cs. */
1045 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1046 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1047 RSEQ_INJECT_ASM(3)
1048 "movl %[expect], %%eax\n\t"
1049 "cmpl %[v], %%eax\n\t"
1050 "jnz %l[cmpfail]\n\t"
1051 RSEQ_INJECT_ASM(4)
1052 #ifdef RSEQ_COMPARE_TWICE
1053 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1054 "movl %[expect], %%eax\n\t"
1055 "cmpl %[v], %%eax\n\t"
1056 "jnz %l[error2]\n\t"
1057 #endif
1058 /* try store */
1059 "movl %[newv2], %[v2]\n\t"
1060 RSEQ_INJECT_ASM(5)
1061 "lock; addl $0,-128(%%esp)\n\t"
1062 /* final store */
1063 "movl %[newv], %[v]\n\t"
1064 "2:\n\t"
1065 RSEQ_INJECT_ASM(6)
1066 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1067 : /* gcc asm goto does not allow outputs */
1068 : [cpu_id] "r" (cpu),
1069 [rseq_offset] "r" (rseq_offset),
1070 /* try store input */
1071 [v2] "m" (*v2),
1072 [newv2] "r" (newv2),
1073 /* final store input */
1074 [v] "m" (*v),
1075 [expect] "m" (expect),
1076 [newv] "r" (newv)
1077 : "memory", "cc", "eax"
1078 RSEQ_INJECT_CLOBBER
1079 : abort, cmpfail
1080 #ifdef RSEQ_COMPARE_TWICE
1081 , error1, error2
1082 #endif
1083 );
1084 rseq_after_asm_goto();
1085 return 0;
1086 abort:
1087 rseq_after_asm_goto();
1088 RSEQ_INJECT_FAILED
1089 return -1;
1090 cmpfail:
1091 rseq_after_asm_goto();
1092 return 1;
1093 #ifdef RSEQ_COMPARE_TWICE
1094 error1:
1095 rseq_after_asm_goto();
1096 rseq_bug("cpu_id comparison failed");
1097 error2:
1098 rseq_after_asm_goto();
1099 rseq_bug("expected value comparison failed");
1100 #endif
1101
1102 }
1103
1104 static inline __attribute__((always_inline))
1105 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
1106 intptr_t *v2, intptr_t expect2,
1107 intptr_t newv, int cpu)
1108 {
1109 RSEQ_INJECT_C(9)
1110
1111 __asm__ __volatile__ goto (
1112 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1113 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1114 #ifdef RSEQ_COMPARE_TWICE
1115 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1116 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1117 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1118 #endif
1119 /* Start rseq by storing table entry pointer into rseq_cs. */
1120 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1121 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1122 RSEQ_INJECT_ASM(3)
1123 "cmpl %[v], %[expect]\n\t"
1124 "jnz %l[cmpfail]\n\t"
1125 RSEQ_INJECT_ASM(4)
1126 "cmpl %[expect2], %[v2]\n\t"
1127 "jnz %l[cmpfail]\n\t"
1128 RSEQ_INJECT_ASM(5)
1129 #ifdef RSEQ_COMPARE_TWICE
1130 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1131 "cmpl %[v], %[expect]\n\t"
1132 "jnz %l[error2]\n\t"
1133 "cmpl %[expect2], %[v2]\n\t"
1134 "jnz %l[error3]\n\t"
1135 #endif
1136 "movl %[newv], %%eax\n\t"
1137 /* final store */
1138 "movl %%eax, %[v]\n\t"
1139 "2:\n\t"
1140 RSEQ_INJECT_ASM(6)
1141 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1142 : /* gcc asm goto does not allow outputs */
1143 : [cpu_id] "r" (cpu),
1144 [rseq_offset] "r" (rseq_offset),
1145 /* cmp2 input */
1146 [v2] "m" (*v2),
1147 [expect2] "r" (expect2),
1148 /* final store input */
1149 [v] "m" (*v),
1150 [expect] "r" (expect),
1151 [newv] "m" (newv)
1152 : "memory", "cc", "eax"
1153 RSEQ_INJECT_CLOBBER
1154 : abort, cmpfail
1155 #ifdef RSEQ_COMPARE_TWICE
1156 , error1, error2, error3
1157 #endif
1158 );
1159 rseq_after_asm_goto();
1160 return 0;
1161 abort:
1162 rseq_after_asm_goto();
1163 RSEQ_INJECT_FAILED
1164 return -1;
1165 cmpfail:
1166 rseq_after_asm_goto();
1167 return 1;
1168 #ifdef RSEQ_COMPARE_TWICE
1169 error1:
1170 rseq_after_asm_goto();
1171 rseq_bug("cpu_id comparison failed");
1172 error2:
1173 rseq_after_asm_goto();
1174 rseq_bug("1st expected value comparison failed");
1175 error3:
1176 rseq_after_asm_goto();
1177 rseq_bug("2nd expected value comparison failed");
1178 #endif
1179 }
1180
1181 /* TODO: implement a faster memcpy. */
1182 static inline __attribute__((always_inline))
1183 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1184 void *dst, void *src, size_t len,
1185 intptr_t newv, int cpu)
1186 {
1187 uint32_t rseq_scratch[3];
1188
1189 RSEQ_INJECT_C(9)
1190
1191 __asm__ __volatile__ goto (
1192 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1193 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1194 #ifdef RSEQ_COMPARE_TWICE
1195 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1196 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1197 #endif
1198 "movl %[src], %[rseq_scratch0]\n\t"
1199 "movl %[dst], %[rseq_scratch1]\n\t"
1200 "movl %[len], %[rseq_scratch2]\n\t"
1201 /* Start rseq by storing table entry pointer into rseq_cs. */
1202 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1203 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1204 RSEQ_INJECT_ASM(3)
1205 "movl %[expect], %%eax\n\t"
1206 "cmpl %%eax, %[v]\n\t"
1207 "jnz 5f\n\t"
1208 RSEQ_INJECT_ASM(4)
1209 #ifdef RSEQ_COMPARE_TWICE
1210 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
1211 "movl %[expect], %%eax\n\t"
1212 "cmpl %%eax, %[v]\n\t"
1213 "jnz 7f\n\t"
1214 #endif
1215 /* try memcpy */
1216 "test %[len], %[len]\n\t" \
1217 "jz 333f\n\t" \
1218 "222:\n\t" \
1219 "movb (%[src]), %%al\n\t" \
1220 "movb %%al, (%[dst])\n\t" \
1221 "inc %[src]\n\t" \
1222 "inc %[dst]\n\t" \
1223 "dec %[len]\n\t" \
1224 "jnz 222b\n\t" \
1225 "333:\n\t" \
1226 RSEQ_INJECT_ASM(5)
1227 "movl %[newv], %%eax\n\t"
1228 /* final store */
1229 "movl %%eax, %[v]\n\t"
1230 "2:\n\t"
1231 RSEQ_INJECT_ASM(6)
1232 /* teardown */
1233 "movl %[rseq_scratch2], %[len]\n\t"
1234 "movl %[rseq_scratch1], %[dst]\n\t"
1235 "movl %[rseq_scratch0], %[src]\n\t"
1236 RSEQ_ASM_DEFINE_ABORT(4,
1237 "movl %[rseq_scratch2], %[len]\n\t"
1238 "movl %[rseq_scratch1], %[dst]\n\t"
1239 "movl %[rseq_scratch0], %[src]\n\t",
1240 abort)
1241 RSEQ_ASM_DEFINE_CMPFAIL(5,
1242 "movl %[rseq_scratch2], %[len]\n\t"
1243 "movl %[rseq_scratch1], %[dst]\n\t"
1244 "movl %[rseq_scratch0], %[src]\n\t",
1245 cmpfail)
1246 #ifdef RSEQ_COMPARE_TWICE
1247 RSEQ_ASM_DEFINE_CMPFAIL(6,
1248 "movl %[rseq_scratch2], %[len]\n\t"
1249 "movl %[rseq_scratch1], %[dst]\n\t"
1250 "movl %[rseq_scratch0], %[src]\n\t",
1251 error1)
1252 RSEQ_ASM_DEFINE_CMPFAIL(7,
1253 "movl %[rseq_scratch2], %[len]\n\t"
1254 "movl %[rseq_scratch1], %[dst]\n\t"
1255 "movl %[rseq_scratch0], %[src]\n\t",
1256 error2)
1257 #endif
1258 : /* gcc asm goto does not allow outputs */
1259 : [cpu_id] "r" (cpu),
1260 [rseq_offset] "r" (rseq_offset),
1261 /* final store input */
1262 [v] "m" (*v),
1263 [expect] "m" (expect),
1264 [newv] "m" (newv),
1265 /* try memcpy input */
1266 [dst] "r" (dst),
1267 [src] "r" (src),
1268 [len] "r" (len),
1269 [rseq_scratch0] "m" (rseq_scratch[0]),
1270 [rseq_scratch1] "m" (rseq_scratch[1]),
1271 [rseq_scratch2] "m" (rseq_scratch[2])
1272 : "memory", "cc", "eax"
1273 RSEQ_INJECT_CLOBBER
1274 : abort, cmpfail
1275 #ifdef RSEQ_COMPARE_TWICE
1276 , error1, error2
1277 #endif
1278 );
1279 rseq_after_asm_goto();
1280 return 0;
1281 abort:
1282 rseq_after_asm_goto();
1283 RSEQ_INJECT_FAILED
1284 return -1;
1285 cmpfail:
1286 rseq_after_asm_goto();
1287 return 1;
1288 #ifdef RSEQ_COMPARE_TWICE
1289 error1:
1290 rseq_after_asm_goto();
1291 rseq_bug("cpu_id comparison failed");
1292 error2:
1293 rseq_after_asm_goto();
1294 rseq_bug("expected value comparison failed");
1295 #endif
1296 }
1297
1298 /* TODO: implement a faster memcpy. */
1299 static inline __attribute__((always_inline))
1300 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1301 void *dst, void *src, size_t len,
1302 intptr_t newv, int cpu)
1303 {
1304 uint32_t rseq_scratch[3];
1305
1306 RSEQ_INJECT_C(9)
1307
1308 __asm__ __volatile__ goto (
1309 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1310 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1311 #ifdef RSEQ_COMPARE_TWICE
1312 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1313 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1314 #endif
1315 "movl %[src], %[rseq_scratch0]\n\t"
1316 "movl %[dst], %[rseq_scratch1]\n\t"
1317 "movl %[len], %[rseq_scratch2]\n\t"
1318 /* Start rseq by storing table entry pointer into rseq_cs. */
1319 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1320 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1321 RSEQ_INJECT_ASM(3)
1322 "movl %[expect], %%eax\n\t"
1323 "cmpl %%eax, %[v]\n\t"
1324 "jnz 5f\n\t"
1325 RSEQ_INJECT_ASM(4)
1326 #ifdef RSEQ_COMPARE_TWICE
1327 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
1328 "movl %[expect], %%eax\n\t"
1329 "cmpl %%eax, %[v]\n\t"
1330 "jnz 7f\n\t"
1331 #endif
1332 /* try memcpy */
1333 "test %[len], %[len]\n\t" \
1334 "jz 333f\n\t" \
1335 "222:\n\t" \
1336 "movb (%[src]), %%al\n\t" \
1337 "movb %%al, (%[dst])\n\t" \
1338 "inc %[src]\n\t" \
1339 "inc %[dst]\n\t" \
1340 "dec %[len]\n\t" \
1341 "jnz 222b\n\t" \
1342 "333:\n\t" \
1343 RSEQ_INJECT_ASM(5)
1344 "lock; addl $0,-128(%%esp)\n\t"
1345 "movl %[newv], %%eax\n\t"
1346 /* final store */
1347 "movl %%eax, %[v]\n\t"
1348 "2:\n\t"
1349 RSEQ_INJECT_ASM(6)
1350 /* teardown */
1351 "movl %[rseq_scratch2], %[len]\n\t"
1352 "movl %[rseq_scratch1], %[dst]\n\t"
1353 "movl %[rseq_scratch0], %[src]\n\t"
1354 RSEQ_ASM_DEFINE_ABORT(4,
1355 "movl %[rseq_scratch2], %[len]\n\t"
1356 "movl %[rseq_scratch1], %[dst]\n\t"
1357 "movl %[rseq_scratch0], %[src]\n\t",
1358 abort)
1359 RSEQ_ASM_DEFINE_CMPFAIL(5,
1360 "movl %[rseq_scratch2], %[len]\n\t"
1361 "movl %[rseq_scratch1], %[dst]\n\t"
1362 "movl %[rseq_scratch0], %[src]\n\t",
1363 cmpfail)
1364 #ifdef RSEQ_COMPARE_TWICE
1365 RSEQ_ASM_DEFINE_CMPFAIL(6,
1366 "movl %[rseq_scratch2], %[len]\n\t"
1367 "movl %[rseq_scratch1], %[dst]\n\t"
1368 "movl %[rseq_scratch0], %[src]\n\t",
1369 error1)
1370 RSEQ_ASM_DEFINE_CMPFAIL(7,
1371 "movl %[rseq_scratch2], %[len]\n\t"
1372 "movl %[rseq_scratch1], %[dst]\n\t"
1373 "movl %[rseq_scratch0], %[src]\n\t",
1374 error2)
1375 #endif
1376 : /* gcc asm goto does not allow outputs */
1377 : [cpu_id] "r" (cpu),
1378 [rseq_offset] "r" (rseq_offset),
1379 /* final store input */
1380 [v] "m" (*v),
1381 [expect] "m" (expect),
1382 [newv] "m" (newv),
1383 /* try memcpy input */
1384 [dst] "r" (dst),
1385 [src] "r" (src),
1386 [len] "r" (len),
1387 [rseq_scratch0] "m" (rseq_scratch[0]),
1388 [rseq_scratch1] "m" (rseq_scratch[1]),
1389 [rseq_scratch2] "m" (rseq_scratch[2])
1390 : "memory", "cc", "eax"
1391 RSEQ_INJECT_CLOBBER
1392 : abort, cmpfail
1393 #ifdef RSEQ_COMPARE_TWICE
1394 , error1, error2
1395 #endif
1396 );
1397 rseq_after_asm_goto();
1398 return 0;
1399 abort:
1400 rseq_after_asm_goto();
1401 RSEQ_INJECT_FAILED
1402 return -1;
1403 cmpfail:
1404 rseq_after_asm_goto();
1405 return 1;
1406 #ifdef RSEQ_COMPARE_TWICE
1407 error1:
1408 rseq_after_asm_goto();
1409 rseq_bug("cpu_id comparison failed");
1410 error2:
1411 rseq_after_asm_goto();
1412 rseq_bug("expected value comparison failed");
1413 #endif
1414 }
1415
1416 /*
1417 * Dereference @p. Add voffp to the dereferenced pointer, and load its content
1418 * into @load.
1419 */
1420 static inline __attribute__((always_inline))
1421 int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu)
1422 {
1423 RSEQ_INJECT_C(9)
1424
1425 __asm__ __volatile__ goto (
1426 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1427 #ifdef RSEQ_COMPARE_TWICE
1428 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1429 #endif
1430 /* Start rseq by storing table entry pointer into rseq_cs. */
1431 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
1432 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
1433 RSEQ_INJECT_ASM(3)
1434 "movl %[p], %%ebx\n\t"
1435 RSEQ_INJECT_ASM(4)
1436 #ifdef RSEQ_COMPARE_TWICE
1437 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
1438 #endif
1439 "addl %[voffp], %%ebx\n\t"
1440 "movl (%%ebx), %%ebx\n\t"
1441 "movl %%ebx, %[load]\n\t"
1442 "2:\n\t"
1443 RSEQ_INJECT_ASM(5)
1444 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1445 : /* gcc asm goto does not allow outputs */
1446 : [cpu_id] "r" (cpu),
1447 [rseq_offset] "r" (rseq_offset),
1448 /* final store input */
1449 [p] "m" (*p),
1450 [voffp] "ir" (voffp),
1451 [load] "m" (*load)
1452 : "memory", "cc", "eax", "ebx"
1453 RSEQ_INJECT_CLOBBER
1454 : abort
1455 #ifdef RSEQ_COMPARE_TWICE
1456 , error1
1457 #endif
1458 );
1459 rseq_after_asm_goto();
1460 return 0;
1461 abort:
1462 rseq_after_asm_goto();
1463 RSEQ_INJECT_FAILED
1464 return -1;
1465 #ifdef RSEQ_COMPARE_TWICE
1466 error1:
1467 rseq_after_asm_goto();
1468 rseq_bug("cpu_id comparison failed");
1469 #endif
1470 }
1471
1472 #endif /* !RSEQ_SKIP_FASTPATH */
1473
1474 #endif
Librseq
This page took 0.062372 seconds and 5 git commands to generate.