Commit | Line | Data |
---|---|---|
744d0b8b | 1 | /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
784b0012 MD |
2 | /* |
3 | * rseq-x86.h | |
4 | * | |
5 | * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
6 | */ | |
7 | ||
8 | #include <stdint.h> | |
9 | ||
a5fe9e95 MD |
10 | /* |
11 | * RSEQ_SIG is used with the following reserved undefined instructions, which | |
12 | * trap in user-space: | |
13 | * | |
14 | * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi | |
15 | * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi | |
16 | */ | |
784b0012 MD |
17 | #define RSEQ_SIG 0x53053053 |
18 | ||
05ebd3f9 MD |
19 | /* |
20 | * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input | |
21 | * operands, we cannot use "m" input operands, and rather pass the __rseq_abi | |
22 | * address through a "r" input operand. | |
9698c399 | 23 | * (TODO: revisit after migration to glibc's ABI) |
05ebd3f9 MD |
24 | */ |
25 | ||
26 | /* Offset of cpu_id and rseq_cs fields in struct rseq. */ | |
27 | #define RSEQ_CPU_ID_OFFSET 4 | |
28 | #define RSEQ_CS_OFFSET 8 | |
29 | ||
784b0012 MD |
30 | #ifdef __x86_64__ |
31 | ||
32 | #define rseq_smp_mb() \ | |
33 | __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") | |
34 | #define rseq_smp_rmb() rseq_barrier() | |
35 | #define rseq_smp_wmb() rseq_barrier() | |
36 | ||
37 | #define rseq_smp_load_acquire(p) \ | |
38 | __extension__ ({ \ | |
39 | __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ | |
40 | rseq_barrier(); \ | |
41 | ____p1; \ | |
42 | }) | |
43 | ||
44 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
45 | ||
46 | #define rseq_smp_store_release(p, v) \ | |
47 | do { \ | |
48 | rseq_barrier(); \ | |
49 | RSEQ_WRITE_ONCE(*p, v); \ | |
50 | } while (0) | |
51 | ||
52 | #ifdef RSEQ_SKIP_FASTPATH | |
53 | #include "rseq-skip.h" | |
54 | #else /* !RSEQ_SKIP_FASTPATH */ | |
55 | ||
56 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
57 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 58 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
59 | ".balign 32\n\t" \ |
60 | __rseq_str(label) ":\n\t" \ | |
61 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
62 | ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
63 | ".popsection\n\t" \ |
64 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
65 | ".quad " __rseq_str(label) "b\n\t" \ | |
784b0012 MD |
66 | ".popsection\n\t" |
67 | ||
dd01d0fb | 68 | |
784b0012 MD |
69 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
70 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
71 | (post_commit_ip - start_ip), abort_ip) | |
72 | ||
90d9876e MD |
73 | /* |
74 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
75 | * of the critical section where a critical section can either branch to or |
76 | * reach through the normal course of its execution. The abort IP and the | |
77 | * post-commit IP are already part of the __rseq_cs section and should not be | |
78 | * explicitly defined as additional exit points. Knowing all exit points is | |
79 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
80 | */ |
81 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
82 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
83 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ | |
84 | ".popsection\n\t" | |
85 | ||
784b0012 MD |
86 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
87 | RSEQ_INJECT_ASM(1) \ | |
88 | "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ | |
05ebd3f9 | 89 | "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
90 | __rseq_str(label) ":\n\t" |
91 | ||
92 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ | |
93 | RSEQ_INJECT_ASM(2) \ | |
05ebd3f9 | 94 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
95 | "jnz " __rseq_str(label) "\n\t" |
96 | ||
97 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
98 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
99 | /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ |
100 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
101 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
102 | __rseq_str(label) ":\n\t" \ | |
103 | teardown \ | |
104 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
105 | ".popsection\n\t" | |
106 | ||
107 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
108 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
109 | __rseq_str(label) ":\n\t" \ | |
110 | teardown \ | |
111 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ | |
112 | ".popsection\n\t" | |
113 | ||
114 | static inline __attribute__((always_inline)) | |
115 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | |
116 | { | |
117 | RSEQ_INJECT_C(9) | |
118 | ||
119 | __asm__ __volatile__ goto ( | |
120 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
121 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
122 | #ifdef RSEQ_COMPARE_TWICE | |
123 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
124 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
125 | #endif | |
784b0012 | 126 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
127 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
128 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
129 | RSEQ_INJECT_ASM(3) |
130 | "cmpq %[v], %[expect]\n\t" | |
131 | "jnz %l[cmpfail]\n\t" | |
132 | RSEQ_INJECT_ASM(4) | |
133 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 134 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
135 | "cmpq %[v], %[expect]\n\t" |
136 | "jnz %l[error2]\n\t" | |
137 | #endif | |
138 | /* final store */ | |
139 | "movq %[newv], %[v]\n\t" | |
140 | "2:\n\t" | |
141 | RSEQ_INJECT_ASM(5) | |
142 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
143 | : /* gcc asm goto does not allow outputs */ | |
144 | : [cpu_id] "r" (cpu), | |
9698c399 | 145 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
146 | [v] "m" (*v), |
147 | [expect] "r" (expect), | |
148 | [newv] "r" (newv) | |
149 | : "memory", "cc", "rax" | |
150 | RSEQ_INJECT_CLOBBER | |
151 | : abort, cmpfail | |
152 | #ifdef RSEQ_COMPARE_TWICE | |
153 | , error1, error2 | |
154 | #endif | |
155 | ); | |
dd76f2d6 | 156 | rseq_after_asm_goto(); |
784b0012 MD |
157 | return 0; |
158 | abort: | |
dd76f2d6 | 159 | rseq_after_asm_goto(); |
784b0012 MD |
160 | RSEQ_INJECT_FAILED |
161 | return -1; | |
162 | cmpfail: | |
dd76f2d6 | 163 | rseq_after_asm_goto(); |
784b0012 MD |
164 | return 1; |
165 | #ifdef RSEQ_COMPARE_TWICE | |
166 | error1: | |
dd76f2d6 | 167 | rseq_after_asm_goto(); |
784b0012 MD |
168 | rseq_bug("cpu_id comparison failed"); |
169 | error2: | |
dd76f2d6 | 170 | rseq_after_asm_goto(); |
784b0012 MD |
171 | rseq_bug("expected value comparison failed"); |
172 | #endif | |
173 | } | |
174 | ||
175 | /* | |
176 | * Compare @v against @expectnot. When it does _not_ match, load @v | |
177 | * into @load, and store the content of *@v + voffp into @v. | |
178 | */ | |
179 | static inline __attribute__((always_inline)) | |
180 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | |
181 | off_t voffp, intptr_t *load, int cpu) | |
182 | { | |
183 | RSEQ_INJECT_C(9) | |
184 | ||
185 | __asm__ __volatile__ goto ( | |
186 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
187 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
188 | #ifdef RSEQ_COMPARE_TWICE | |
189 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
190 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
191 | #endif | |
784b0012 | 192 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
193 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
194 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
195 | RSEQ_INJECT_ASM(3) |
196 | "movq %[v], %%rbx\n\t" | |
197 | "cmpq %%rbx, %[expectnot]\n\t" | |
198 | "je %l[cmpfail]\n\t" | |
199 | RSEQ_INJECT_ASM(4) | |
200 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 201 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
202 | "movq %[v], %%rbx\n\t" |
203 | "cmpq %%rbx, %[expectnot]\n\t" | |
204 | "je %l[error2]\n\t" | |
205 | #endif | |
206 | "movq %%rbx, %[load]\n\t" | |
207 | "addq %[voffp], %%rbx\n\t" | |
208 | "movq (%%rbx), %%rbx\n\t" | |
209 | /* final store */ | |
210 | "movq %%rbx, %[v]\n\t" | |
211 | "2:\n\t" | |
212 | RSEQ_INJECT_ASM(5) | |
213 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
214 | : /* gcc asm goto does not allow outputs */ | |
215 | : [cpu_id] "r" (cpu), | |
9698c399 | 216 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
217 | /* final store input */ |
218 | [v] "m" (*v), | |
219 | [expectnot] "r" (expectnot), | |
220 | [voffp] "er" (voffp), | |
221 | [load] "m" (*load) | |
222 | : "memory", "cc", "rax", "rbx" | |
223 | RSEQ_INJECT_CLOBBER | |
224 | : abort, cmpfail | |
225 | #ifdef RSEQ_COMPARE_TWICE | |
226 | , error1, error2 | |
227 | #endif | |
228 | ); | |
dd76f2d6 | 229 | rseq_after_asm_goto(); |
784b0012 MD |
230 | return 0; |
231 | abort: | |
dd76f2d6 | 232 | rseq_after_asm_goto(); |
784b0012 MD |
233 | RSEQ_INJECT_FAILED |
234 | return -1; | |
235 | cmpfail: | |
dd76f2d6 | 236 | rseq_after_asm_goto(); |
784b0012 MD |
237 | return 1; |
238 | #ifdef RSEQ_COMPARE_TWICE | |
239 | error1: | |
dd76f2d6 | 240 | rseq_after_asm_goto(); |
784b0012 MD |
241 | rseq_bug("cpu_id comparison failed"); |
242 | error2: | |
dd76f2d6 | 243 | rseq_after_asm_goto(); |
784b0012 MD |
244 | rseq_bug("expected value comparison failed"); |
245 | #endif | |
246 | } | |
247 | ||
248 | static inline __attribute__((always_inline)) | |
249 | int rseq_addv(intptr_t *v, intptr_t count, int cpu) | |
250 | { | |
251 | RSEQ_INJECT_C(9) | |
252 | ||
253 | __asm__ __volatile__ goto ( | |
254 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
255 | #ifdef RSEQ_COMPARE_TWICE |
256 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
257 | #endif | |
784b0012 | 258 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
259 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
260 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
261 | RSEQ_INJECT_ASM(3) |
262 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 263 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
264 | #endif |
265 | /* final store */ | |
266 | "addq %[count], %[v]\n\t" | |
267 | "2:\n\t" | |
268 | RSEQ_INJECT_ASM(4) | |
269 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
270 | : /* gcc asm goto does not allow outputs */ | |
271 | : [cpu_id] "r" (cpu), | |
9698c399 | 272 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
273 | /* final store input */ |
274 | [v] "m" (*v), | |
275 | [count] "er" (count) | |
276 | : "memory", "cc", "rax" | |
277 | RSEQ_INJECT_CLOBBER | |
278 | : abort | |
279 | #ifdef RSEQ_COMPARE_TWICE | |
280 | , error1 | |
281 | #endif | |
282 | ); | |
dd76f2d6 | 283 | rseq_after_asm_goto(); |
784b0012 MD |
284 | return 0; |
285 | abort: | |
dd76f2d6 | 286 | rseq_after_asm_goto(); |
784b0012 MD |
287 | RSEQ_INJECT_FAILED |
288 | return -1; | |
289 | #ifdef RSEQ_COMPARE_TWICE | |
290 | error1: | |
dd76f2d6 | 291 | rseq_after_asm_goto(); |
784b0012 MD |
292 | rseq_bug("cpu_id comparison failed"); |
293 | #endif | |
294 | } | |
295 | ||
7c02c61b MD |
296 | #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV |
297 | ||
298 | /* | |
299 | * pval = *(ptr+off) | |
300 | * *pval += inc; | |
301 | */ | |
302 | static inline __attribute__((always_inline)) | |
303 | int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu) | |
304 | { | |
305 | RSEQ_INJECT_C(9) | |
306 | ||
307 | __asm__ __volatile__ goto ( | |
308 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
309 | #ifdef RSEQ_COMPARE_TWICE | |
310 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
311 | #endif | |
312 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
313 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | |
314 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
315 | RSEQ_INJECT_ASM(3) | |
316 | #ifdef RSEQ_COMPARE_TWICE | |
317 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | |
318 | #endif | |
319 | /* get p+v */ | |
320 | "movq %[ptr], %%rbx\n\t" | |
321 | "addq %[off], %%rbx\n\t" | |
322 | /* get pv */ | |
323 | "movq (%%rbx), %%rcx\n\t" | |
324 | /* *pv += inc */ | |
325 | "addq %[inc], (%%rcx)\n\t" | |
326 | "2:\n\t" | |
327 | RSEQ_INJECT_ASM(4) | |
328 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
329 | : /* gcc asm goto does not allow outputs */ | |
330 | : [cpu_id] "r" (cpu), | |
331 | [rseq_abi] "r" (rseq_get_abi()), | |
332 | /* final store input */ | |
333 | [ptr] "m" (*ptr), | |
334 | [off] "er" (off), | |
335 | [inc] "er" (inc) | |
336 | : "memory", "cc", "rax", "rbx", "rcx" | |
337 | RSEQ_INJECT_CLOBBER | |
338 | : abort | |
339 | #ifdef RSEQ_COMPARE_TWICE | |
340 | , error1 | |
341 | #endif | |
342 | ); | |
343 | rseq_after_asm_goto(); | |
344 | return 0; | |
345 | abort: | |
346 | rseq_after_asm_goto(); | |
347 | RSEQ_INJECT_FAILED | |
348 | return -1; | |
349 | #ifdef RSEQ_COMPARE_TWICE | |
350 | error1: | |
351 | rseq_after_asm_goto(); | |
352 | rseq_bug("cpu_id comparison failed"); | |
353 | #endif | |
354 | } | |
355 | ||
784b0012 MD |
356 | static inline __attribute__((always_inline)) |
357 | int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | |
358 | intptr_t *v2, intptr_t newv2, | |
359 | intptr_t newv, int cpu) | |
360 | { | |
361 | RSEQ_INJECT_C(9) | |
362 | ||
363 | __asm__ __volatile__ goto ( | |
364 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
365 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
366 | #ifdef RSEQ_COMPARE_TWICE | |
367 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
368 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
369 | #endif | |
784b0012 | 370 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
371 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
372 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
373 | RSEQ_INJECT_ASM(3) |
374 | "cmpq %[v], %[expect]\n\t" | |
375 | "jnz %l[cmpfail]\n\t" | |
376 | RSEQ_INJECT_ASM(4) | |
377 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 378 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
379 | "cmpq %[v], %[expect]\n\t" |
380 | "jnz %l[error2]\n\t" | |
381 | #endif | |
382 | /* try store */ | |
383 | "movq %[newv2], %[v2]\n\t" | |
384 | RSEQ_INJECT_ASM(5) | |
385 | /* final store */ | |
386 | "movq %[newv], %[v]\n\t" | |
387 | "2:\n\t" | |
388 | RSEQ_INJECT_ASM(6) | |
389 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
390 | : /* gcc asm goto does not allow outputs */ | |
391 | : [cpu_id] "r" (cpu), | |
9698c399 | 392 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
393 | /* try store input */ |
394 | [v2] "m" (*v2), | |
395 | [newv2] "r" (newv2), | |
396 | /* final store input */ | |
397 | [v] "m" (*v), | |
398 | [expect] "r" (expect), | |
399 | [newv] "r" (newv) | |
400 | : "memory", "cc", "rax" | |
401 | RSEQ_INJECT_CLOBBER | |
402 | : abort, cmpfail | |
403 | #ifdef RSEQ_COMPARE_TWICE | |
404 | , error1, error2 | |
405 | #endif | |
406 | ); | |
dd76f2d6 | 407 | rseq_after_asm_goto(); |
784b0012 MD |
408 | return 0; |
409 | abort: | |
dd76f2d6 | 410 | rseq_after_asm_goto(); |
784b0012 MD |
411 | RSEQ_INJECT_FAILED |
412 | return -1; | |
413 | cmpfail: | |
dd76f2d6 | 414 | rseq_after_asm_goto(); |
784b0012 MD |
415 | return 1; |
416 | #ifdef RSEQ_COMPARE_TWICE | |
417 | error1: | |
dd76f2d6 | 418 | rseq_after_asm_goto(); |
784b0012 MD |
419 | rseq_bug("cpu_id comparison failed"); |
420 | error2: | |
dd76f2d6 | 421 | rseq_after_asm_goto(); |
784b0012 MD |
422 | rseq_bug("expected value comparison failed"); |
423 | #endif | |
424 | } | |
425 | ||
426 | /* x86-64 is TSO. */ | |
427 | static inline __attribute__((always_inline)) | |
428 | int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | |
429 | intptr_t *v2, intptr_t newv2, | |
430 | intptr_t newv, int cpu) | |
431 | { | |
432 | return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); | |
433 | } | |
434 | ||
435 | static inline __attribute__((always_inline)) | |
436 | int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | |
437 | intptr_t *v2, intptr_t expect2, | |
438 | intptr_t newv, int cpu) | |
439 | { | |
440 | RSEQ_INJECT_C(9) | |
441 | ||
442 | __asm__ __volatile__ goto ( | |
443 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
444 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
445 | #ifdef RSEQ_COMPARE_TWICE | |
446 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
447 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
448 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | |
449 | #endif | |
784b0012 | 450 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
451 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
452 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
453 | RSEQ_INJECT_ASM(3) |
454 | "cmpq %[v], %[expect]\n\t" | |
455 | "jnz %l[cmpfail]\n\t" | |
456 | RSEQ_INJECT_ASM(4) | |
457 | "cmpq %[v2], %[expect2]\n\t" | |
458 | "jnz %l[cmpfail]\n\t" | |
459 | RSEQ_INJECT_ASM(5) | |
460 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 461 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
462 | "cmpq %[v], %[expect]\n\t" |
463 | "jnz %l[error2]\n\t" | |
464 | "cmpq %[v2], %[expect2]\n\t" | |
465 | "jnz %l[error3]\n\t" | |
466 | #endif | |
467 | /* final store */ | |
468 | "movq %[newv], %[v]\n\t" | |
469 | "2:\n\t" | |
470 | RSEQ_INJECT_ASM(6) | |
471 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
472 | : /* gcc asm goto does not allow outputs */ | |
473 | : [cpu_id] "r" (cpu), | |
9698c399 | 474 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
475 | /* cmp2 input */ |
476 | [v2] "m" (*v2), | |
477 | [expect2] "r" (expect2), | |
478 | /* final store input */ | |
479 | [v] "m" (*v), | |
480 | [expect] "r" (expect), | |
481 | [newv] "r" (newv) | |
482 | : "memory", "cc", "rax" | |
483 | RSEQ_INJECT_CLOBBER | |
484 | : abort, cmpfail | |
485 | #ifdef RSEQ_COMPARE_TWICE | |
486 | , error1, error2, error3 | |
487 | #endif | |
488 | ); | |
dd76f2d6 | 489 | rseq_after_asm_goto(); |
784b0012 MD |
490 | return 0; |
491 | abort: | |
dd76f2d6 | 492 | rseq_after_asm_goto(); |
784b0012 MD |
493 | RSEQ_INJECT_FAILED |
494 | return -1; | |
495 | cmpfail: | |
dd76f2d6 | 496 | rseq_after_asm_goto(); |
784b0012 MD |
497 | return 1; |
498 | #ifdef RSEQ_COMPARE_TWICE | |
499 | error1: | |
dd76f2d6 | 500 | rseq_after_asm_goto(); |
784b0012 MD |
501 | rseq_bug("cpu_id comparison failed"); |
502 | error2: | |
dd76f2d6 | 503 | rseq_after_asm_goto(); |
784b0012 MD |
504 | rseq_bug("1st expected value comparison failed"); |
505 | error3: | |
dd76f2d6 | 506 | rseq_after_asm_goto(); |
784b0012 MD |
507 | rseq_bug("2nd expected value comparison failed"); |
508 | #endif | |
509 | } | |
510 | ||
511 | static inline __attribute__((always_inline)) | |
512 | int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | |
513 | void *dst, void *src, size_t len, | |
514 | intptr_t newv, int cpu) | |
515 | { | |
516 | uint64_t rseq_scratch[3]; | |
517 | ||
518 | RSEQ_INJECT_C(9) | |
519 | ||
520 | __asm__ __volatile__ goto ( | |
521 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
522 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
523 | #ifdef RSEQ_COMPARE_TWICE | |
524 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
525 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
526 | #endif | |
784b0012 MD |
527 | "movq %[src], %[rseq_scratch0]\n\t" |
528 | "movq %[dst], %[rseq_scratch1]\n\t" | |
529 | "movq %[len], %[rseq_scratch2]\n\t" | |
530 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
05ebd3f9 MD |
531 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
532 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
533 | RSEQ_INJECT_ASM(3) |
534 | "cmpq %[v], %[expect]\n\t" | |
535 | "jnz 5f\n\t" | |
536 | RSEQ_INJECT_ASM(4) | |
537 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 538 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
784b0012 MD |
539 | "cmpq %[v], %[expect]\n\t" |
540 | "jnz 7f\n\t" | |
541 | #endif | |
542 | /* try memcpy */ | |
543 | "test %[len], %[len]\n\t" \ | |
544 | "jz 333f\n\t" \ | |
545 | "222:\n\t" \ | |
546 | "movb (%[src]), %%al\n\t" \ | |
547 | "movb %%al, (%[dst])\n\t" \ | |
548 | "inc %[src]\n\t" \ | |
549 | "inc %[dst]\n\t" \ | |
550 | "dec %[len]\n\t" \ | |
551 | "jnz 222b\n\t" \ | |
552 | "333:\n\t" \ | |
553 | RSEQ_INJECT_ASM(5) | |
554 | /* final store */ | |
555 | "movq %[newv], %[v]\n\t" | |
556 | "2:\n\t" | |
557 | RSEQ_INJECT_ASM(6) | |
558 | /* teardown */ | |
559 | "movq %[rseq_scratch2], %[len]\n\t" | |
560 | "movq %[rseq_scratch1], %[dst]\n\t" | |
561 | "movq %[rseq_scratch0], %[src]\n\t" | |
562 | RSEQ_ASM_DEFINE_ABORT(4, | |
563 | "movq %[rseq_scratch2], %[len]\n\t" | |
564 | "movq %[rseq_scratch1], %[dst]\n\t" | |
565 | "movq %[rseq_scratch0], %[src]\n\t", | |
566 | abort) | |
567 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
568 | "movq %[rseq_scratch2], %[len]\n\t" | |
569 | "movq %[rseq_scratch1], %[dst]\n\t" | |
570 | "movq %[rseq_scratch0], %[src]\n\t", | |
571 | cmpfail) | |
572 | #ifdef RSEQ_COMPARE_TWICE | |
573 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
574 | "movq %[rseq_scratch2], %[len]\n\t" | |
575 | "movq %[rseq_scratch1], %[dst]\n\t" | |
576 | "movq %[rseq_scratch0], %[src]\n\t", | |
577 | error1) | |
578 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
579 | "movq %[rseq_scratch2], %[len]\n\t" | |
580 | "movq %[rseq_scratch1], %[dst]\n\t" | |
581 | "movq %[rseq_scratch0], %[src]\n\t", | |
582 | error2) | |
583 | #endif | |
584 | : /* gcc asm goto does not allow outputs */ | |
585 | : [cpu_id] "r" (cpu), | |
9698c399 | 586 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
587 | /* final store input */ |
588 | [v] "m" (*v), | |
589 | [expect] "r" (expect), | |
590 | [newv] "r" (newv), | |
591 | /* try memcpy input */ | |
592 | [dst] "r" (dst), | |
593 | [src] "r" (src), | |
594 | [len] "r" (len), | |
595 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
596 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
597 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
598 | : "memory", "cc", "rax" | |
599 | RSEQ_INJECT_CLOBBER | |
600 | : abort, cmpfail | |
601 | #ifdef RSEQ_COMPARE_TWICE | |
602 | , error1, error2 | |
603 | #endif | |
604 | ); | |
dd76f2d6 | 605 | rseq_after_asm_goto(); |
784b0012 MD |
606 | return 0; |
607 | abort: | |
dd76f2d6 | 608 | rseq_after_asm_goto(); |
784b0012 MD |
609 | RSEQ_INJECT_FAILED |
610 | return -1; | |
611 | cmpfail: | |
dd76f2d6 | 612 | rseq_after_asm_goto(); |
784b0012 MD |
613 | return 1; |
614 | #ifdef RSEQ_COMPARE_TWICE | |
615 | error1: | |
dd76f2d6 | 616 | rseq_after_asm_goto(); |
784b0012 MD |
617 | rseq_bug("cpu_id comparison failed"); |
618 | error2: | |
dd76f2d6 | 619 | rseq_after_asm_goto(); |
784b0012 MD |
620 | rseq_bug("expected value comparison failed"); |
621 | #endif | |
622 | } | |
623 | ||
624 | /* x86-64 is TSO. */ | |
625 | static inline __attribute__((always_inline)) | |
626 | int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | |
627 | void *dst, void *src, size_t len, | |
628 | intptr_t newv, int cpu) | |
629 | { | |
630 | return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, | |
631 | newv, cpu); | |
632 | } | |
633 | ||
de28c254 MD |
634 | /* |
635 | * Dereference @p. Add voffp to the dereferenced pointer, and load its content | |
636 | * into @load. | |
637 | */ | |
638 | static inline __attribute__((always_inline)) | |
639 | int rseq_deref_loadoffp(intptr_t *p, off_t voffp, intptr_t *load, int cpu) | |
640 | { | |
641 | RSEQ_INJECT_C(9) | |
642 | ||
643 | __asm__ __volatile__ goto ( | |
644 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
645 | #ifdef RSEQ_COMPARE_TWICE | |
646 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
647 | #endif | |
648 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
649 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | |
650 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
651 | RSEQ_INJECT_ASM(3) | |
652 | "movq %[p], %%rbx\n\t" | |
653 | RSEQ_INJECT_ASM(4) | |
654 | #ifdef RSEQ_COMPARE_TWICE | |
655 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | |
656 | #endif | |
657 | "addq %[voffp], %%rbx\n\t" | |
658 | "movq (%%rbx), %%rbx\n\t" | |
659 | "movq %%rbx, %[load]\n\t" | |
660 | "2:\n\t" | |
661 | RSEQ_INJECT_ASM(5) | |
662 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
663 | : /* gcc asm goto does not allow outputs */ | |
664 | : [cpu_id] "r" (cpu), | |
9698c399 | 665 | [rseq_abi] "r" (rseq_get_abi()), |
de28c254 MD |
666 | /* final store input */ |
667 | [p] "m" (*p), | |
668 | [voffp] "er" (voffp), | |
669 | [load] "m" (*load) | |
670 | : "memory", "cc", "rax", "rbx" | |
671 | RSEQ_INJECT_CLOBBER | |
672 | : abort | |
673 | #ifdef RSEQ_COMPARE_TWICE | |
674 | , error1 | |
675 | #endif | |
676 | ); | |
dd76f2d6 | 677 | rseq_after_asm_goto(); |
de28c254 MD |
678 | return 0; |
679 | abort: | |
dd76f2d6 | 680 | rseq_after_asm_goto(); |
de28c254 MD |
681 | RSEQ_INJECT_FAILED |
682 | return -1; | |
683 | #ifdef RSEQ_COMPARE_TWICE | |
684 | error1: | |
dd76f2d6 | 685 | rseq_after_asm_goto(); |
de28c254 MD |
686 | rseq_bug("cpu_id comparison failed"); |
687 | #endif | |
688 | } | |
689 | ||
784b0012 MD |
690 | #endif /* !RSEQ_SKIP_FASTPATH */ |
691 | ||
f1c6b55b | 692 | #elif defined(__i386__) |
784b0012 MD |
693 | |
694 | #define rseq_smp_mb() \ | |
695 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
696 | #define rseq_smp_rmb() \ | |
697 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
698 | #define rseq_smp_wmb() \ | |
699 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
700 | ||
701 | #define rseq_smp_load_acquire(p) \ | |
702 | __extension__ ({ \ | |
703 | __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ | |
704 | rseq_smp_mb(); \ | |
705 | ____p1; \ | |
706 | }) | |
707 | ||
708 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
709 | ||
710 | #define rseq_smp_store_release(p, v) \ | |
711 | do { \ | |
712 | rseq_smp_mb(); \ | |
713 | RSEQ_WRITE_ONCE(*p, v); \ | |
714 | } while (0) | |
715 | ||
716 | #ifdef RSEQ_SKIP_FASTPATH | |
717 | #include "rseq-skip.h" | |
718 | #else /* !RSEQ_SKIP_FASTPATH */ | |
719 | ||
720 | /* | |
721 | * Use eax as scratch register and take memory operands as input to | |
722 | * lessen register pressure. Especially needed when compiling in O0. | |
723 | */ | |
724 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
725 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 726 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
727 | ".balign 32\n\t" \ |
728 | __rseq_str(label) ":\n\t" \ | |
729 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
730 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ | |
dd01d0fb MD |
731 | ".popsection\n\t" \ |
732 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
733 | ".long " __rseq_str(label) "b, 0x0\n\t" \ | |
784b0012 MD |
734 | ".popsection\n\t" |
735 | ||
736 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ | |
737 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
738 | (post_commit_ip - start_ip), abort_ip) | |
739 | ||
90d9876e MD |
740 | /* |
741 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
742 | * of the critical section where a critical section can either branch to or |
743 | * reach through the normal course of its execution. The abort IP and the | |
744 | * post-commit IP are already part of the __rseq_cs section and should not be | |
745 | * explicitly defined as additional exit points. Knowing all exit points is | |
746 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
747 | */ |
748 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
749 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
750 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ | |
751 | ".popsection\n\t" | |
752 | ||
784b0012 MD |
753 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
754 | RSEQ_INJECT_ASM(1) \ | |
05ebd3f9 | 755 | "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
756 | __rseq_str(label) ":\n\t" |
757 | ||
758 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ | |
759 | RSEQ_INJECT_ASM(2) \ | |
05ebd3f9 | 760 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
761 | "jnz " __rseq_str(label) "\n\t" |
762 | ||
763 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
764 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
765 | /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ |
766 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
767 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
768 | __rseq_str(label) ":\n\t" \ | |
769 | teardown \ | |
770 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
771 | ".popsection\n\t" | |
772 | ||
773 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
774 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
775 | __rseq_str(label) ":\n\t" \ | |
776 | teardown \ | |
777 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ | |
778 | ".popsection\n\t" | |
779 | ||
780 | static inline __attribute__((always_inline)) | |
781 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | |
782 | { | |
783 | RSEQ_INJECT_C(9) | |
784 | ||
785 | __asm__ __volatile__ goto ( | |
786 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
787 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
788 | #ifdef RSEQ_COMPARE_TWICE | |
789 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
790 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
791 | #endif | |
784b0012 | 792 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
793 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
794 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
795 | RSEQ_INJECT_ASM(3) |
796 | "cmpl %[v], %[expect]\n\t" | |
797 | "jnz %l[cmpfail]\n\t" | |
798 | RSEQ_INJECT_ASM(4) | |
799 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 800 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
801 | "cmpl %[v], %[expect]\n\t" |
802 | "jnz %l[error2]\n\t" | |
803 | #endif | |
804 | /* final store */ | |
805 | "movl %[newv], %[v]\n\t" | |
806 | "2:\n\t" | |
807 | RSEQ_INJECT_ASM(5) | |
808 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
809 | : /* gcc asm goto does not allow outputs */ | |
810 | : [cpu_id] "r" (cpu), | |
9698c399 | 811 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
812 | [v] "m" (*v), |
813 | [expect] "r" (expect), | |
814 | [newv] "r" (newv) | |
815 | : "memory", "cc", "eax" | |
816 | RSEQ_INJECT_CLOBBER | |
817 | : abort, cmpfail | |
818 | #ifdef RSEQ_COMPARE_TWICE | |
819 | , error1, error2 | |
820 | #endif | |
821 | ); | |
dd76f2d6 | 822 | rseq_after_asm_goto(); |
784b0012 MD |
823 | return 0; |
824 | abort: | |
dd76f2d6 | 825 | rseq_after_asm_goto(); |
784b0012 MD |
826 | RSEQ_INJECT_FAILED |
827 | return -1; | |
828 | cmpfail: | |
dd76f2d6 | 829 | rseq_after_asm_goto(); |
784b0012 MD |
830 | return 1; |
831 | #ifdef RSEQ_COMPARE_TWICE | |
832 | error1: | |
dd76f2d6 | 833 | rseq_after_asm_goto(); |
784b0012 MD |
834 | rseq_bug("cpu_id comparison failed"); |
835 | error2: | |
dd76f2d6 | 836 | rseq_after_asm_goto(); |
784b0012 MD |
837 | rseq_bug("expected value comparison failed"); |
838 | #endif | |
839 | } | |
840 | ||
841 | /* | |
842 | * Compare @v against @expectnot. When it does _not_ match, load @v | |
843 | * into @load, and store the content of *@v + voffp into @v. | |
844 | */ | |
845 | static inline __attribute__((always_inline)) | |
846 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | |
847 | off_t voffp, intptr_t *load, int cpu) | |
848 | { | |
849 | RSEQ_INJECT_C(9) | |
850 | ||
851 | __asm__ __volatile__ goto ( | |
852 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
853 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
854 | #ifdef RSEQ_COMPARE_TWICE | |
855 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
856 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
857 | #endif | |
784b0012 | 858 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
859 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
860 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
861 | RSEQ_INJECT_ASM(3) |
862 | "movl %[v], %%ebx\n\t" | |
863 | "cmpl %%ebx, %[expectnot]\n\t" | |
864 | "je %l[cmpfail]\n\t" | |
865 | RSEQ_INJECT_ASM(4) | |
866 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 867 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
868 | "movl %[v], %%ebx\n\t" |
869 | "cmpl %%ebx, %[expectnot]\n\t" | |
870 | "je %l[error2]\n\t" | |
871 | #endif | |
872 | "movl %%ebx, %[load]\n\t" | |
873 | "addl %[voffp], %%ebx\n\t" | |
874 | "movl (%%ebx), %%ebx\n\t" | |
875 | /* final store */ | |
876 | "movl %%ebx, %[v]\n\t" | |
877 | "2:\n\t" | |
878 | RSEQ_INJECT_ASM(5) | |
879 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
880 | : /* gcc asm goto does not allow outputs */ | |
881 | : [cpu_id] "r" (cpu), | |
9698c399 | 882 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
883 | /* final store input */ |
884 | [v] "m" (*v), | |
885 | [expectnot] "r" (expectnot), | |
886 | [voffp] "ir" (voffp), | |
887 | [load] "m" (*load) | |
888 | : "memory", "cc", "eax", "ebx" | |
889 | RSEQ_INJECT_CLOBBER | |
890 | : abort, cmpfail | |
891 | #ifdef RSEQ_COMPARE_TWICE | |
892 | , error1, error2 | |
893 | #endif | |
894 | ); | |
dd76f2d6 | 895 | rseq_after_asm_goto(); |
784b0012 MD |
896 | return 0; |
897 | abort: | |
dd76f2d6 | 898 | rseq_after_asm_goto(); |
784b0012 MD |
899 | RSEQ_INJECT_FAILED |
900 | return -1; | |
901 | cmpfail: | |
dd76f2d6 | 902 | rseq_after_asm_goto(); |
784b0012 MD |
903 | return 1; |
904 | #ifdef RSEQ_COMPARE_TWICE | |
905 | error1: | |
dd76f2d6 | 906 | rseq_after_asm_goto(); |
784b0012 MD |
907 | rseq_bug("cpu_id comparison failed"); |
908 | error2: | |
dd76f2d6 | 909 | rseq_after_asm_goto(); |
784b0012 MD |
910 | rseq_bug("expected value comparison failed"); |
911 | #endif | |
912 | } | |
913 | ||
914 | static inline __attribute__((always_inline)) | |
915 | int rseq_addv(intptr_t *v, intptr_t count, int cpu) | |
916 | { | |
917 | RSEQ_INJECT_C(9) | |
918 | ||
919 | __asm__ __volatile__ goto ( | |
920 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
921 | #ifdef RSEQ_COMPARE_TWICE |
922 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
923 | #endif | |
784b0012 | 924 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
925 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
926 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
927 | RSEQ_INJECT_ASM(3) |
928 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 929 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
930 | #endif |
931 | /* final store */ | |
932 | "addl %[count], %[v]\n\t" | |
933 | "2:\n\t" | |
934 | RSEQ_INJECT_ASM(4) | |
935 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
936 | : /* gcc asm goto does not allow outputs */ | |
937 | : [cpu_id] "r" (cpu), | |
9698c399 | 938 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
939 | /* final store input */ |
940 | [v] "m" (*v), | |
941 | [count] "ir" (count) | |
942 | : "memory", "cc", "eax" | |
943 | RSEQ_INJECT_CLOBBER | |
944 | : abort | |
945 | #ifdef RSEQ_COMPARE_TWICE | |
946 | , error1 | |
947 | #endif | |
948 | ); | |
dd76f2d6 | 949 | rseq_after_asm_goto(); |
784b0012 MD |
950 | return 0; |
951 | abort: | |
dd76f2d6 | 952 | rseq_after_asm_goto(); |
784b0012 MD |
953 | RSEQ_INJECT_FAILED |
954 | return -1; | |
955 | #ifdef RSEQ_COMPARE_TWICE | |
956 | error1: | |
dd76f2d6 | 957 | rseq_after_asm_goto(); |
784b0012 MD |
958 | rseq_bug("cpu_id comparison failed"); |
959 | #endif | |
960 | } | |
961 | ||
962 | static inline __attribute__((always_inline)) | |
963 | int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | |
964 | intptr_t *v2, intptr_t newv2, | |
965 | intptr_t newv, int cpu) | |
966 | { | |
967 | RSEQ_INJECT_C(9) | |
968 | ||
969 | __asm__ __volatile__ goto ( | |
970 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
971 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
972 | #ifdef RSEQ_COMPARE_TWICE | |
973 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
974 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
975 | #endif | |
784b0012 | 976 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
977 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
978 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
979 | RSEQ_INJECT_ASM(3) |
980 | "cmpl %[v], %[expect]\n\t" | |
981 | "jnz %l[cmpfail]\n\t" | |
982 | RSEQ_INJECT_ASM(4) | |
983 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 984 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
985 | "cmpl %[v], %[expect]\n\t" |
986 | "jnz %l[error2]\n\t" | |
987 | #endif | |
988 | /* try store */ | |
989 | "movl %[newv2], %%eax\n\t" | |
990 | "movl %%eax, %[v2]\n\t" | |
991 | RSEQ_INJECT_ASM(5) | |
992 | /* final store */ | |
993 | "movl %[newv], %[v]\n\t" | |
994 | "2:\n\t" | |
995 | RSEQ_INJECT_ASM(6) | |
996 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
997 | : /* gcc asm goto does not allow outputs */ | |
998 | : [cpu_id] "r" (cpu), | |
9698c399 | 999 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
1000 | /* try store input */ |
1001 | [v2] "m" (*v2), | |
1002 | [newv2] "m" (newv2), | |
1003 | /* final store input */ | |
1004 | [v] "m" (*v), | |
1005 | [expect] "r" (expect), | |
1006 | [newv] "r" (newv) | |
1007 | : "memory", "cc", "eax" | |
1008 | RSEQ_INJECT_CLOBBER | |
1009 | : abort, cmpfail | |
1010 | #ifdef RSEQ_COMPARE_TWICE | |
1011 | , error1, error2 | |
1012 | #endif | |
1013 | ); | |
dd76f2d6 | 1014 | rseq_after_asm_goto(); |
784b0012 MD |
1015 | return 0; |
1016 | abort: | |
dd76f2d6 | 1017 | rseq_after_asm_goto(); |
784b0012 MD |
1018 | RSEQ_INJECT_FAILED |
1019 | return -1; | |
1020 | cmpfail: | |
dd76f2d6 | 1021 | rseq_after_asm_goto(); |
784b0012 MD |
1022 | return 1; |
1023 | #ifdef RSEQ_COMPARE_TWICE | |
1024 | error1: | |
dd76f2d6 | 1025 | rseq_after_asm_goto(); |
784b0012 MD |
1026 | rseq_bug("cpu_id comparison failed"); |
1027 | error2: | |
dd76f2d6 | 1028 | rseq_after_asm_goto(); |
784b0012 MD |
1029 | rseq_bug("expected value comparison failed"); |
1030 | #endif | |
1031 | } | |
1032 | ||
1033 | static inline __attribute__((always_inline)) | |
1034 | int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | |
1035 | intptr_t *v2, intptr_t newv2, | |
1036 | intptr_t newv, int cpu) | |
1037 | { | |
1038 | RSEQ_INJECT_C(9) | |
1039 | ||
1040 | __asm__ __volatile__ goto ( | |
1041 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1042 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1043 | #ifdef RSEQ_COMPARE_TWICE | |
1044 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1045 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1046 | #endif | |
784b0012 | 1047 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
1048 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
1049 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
1050 | RSEQ_INJECT_ASM(3) |
1051 | "movl %[expect], %%eax\n\t" | |
1052 | "cmpl %[v], %%eax\n\t" | |
1053 | "jnz %l[cmpfail]\n\t" | |
1054 | RSEQ_INJECT_ASM(4) | |
1055 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 1056 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
1057 | "movl %[expect], %%eax\n\t" |
1058 | "cmpl %[v], %%eax\n\t" | |
1059 | "jnz %l[error2]\n\t" | |
1060 | #endif | |
1061 | /* try store */ | |
1062 | "movl %[newv2], %[v2]\n\t" | |
1063 | RSEQ_INJECT_ASM(5) | |
1064 | "lock; addl $0,-128(%%esp)\n\t" | |
1065 | /* final store */ | |
1066 | "movl %[newv], %[v]\n\t" | |
1067 | "2:\n\t" | |
1068 | RSEQ_INJECT_ASM(6) | |
1069 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
1070 | : /* gcc asm goto does not allow outputs */ | |
1071 | : [cpu_id] "r" (cpu), | |
9698c399 | 1072 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
1073 | /* try store input */ |
1074 | [v2] "m" (*v2), | |
1075 | [newv2] "r" (newv2), | |
1076 | /* final store input */ | |
1077 | [v] "m" (*v), | |
1078 | [expect] "m" (expect), | |
1079 | [newv] "r" (newv) | |
1080 | : "memory", "cc", "eax" | |
1081 | RSEQ_INJECT_CLOBBER | |
1082 | : abort, cmpfail | |
1083 | #ifdef RSEQ_COMPARE_TWICE | |
1084 | , error1, error2 | |
1085 | #endif | |
1086 | ); | |
dd76f2d6 | 1087 | rseq_after_asm_goto(); |
784b0012 MD |
1088 | return 0; |
1089 | abort: | |
dd76f2d6 | 1090 | rseq_after_asm_goto(); |
784b0012 MD |
1091 | RSEQ_INJECT_FAILED |
1092 | return -1; | |
1093 | cmpfail: | |
dd76f2d6 | 1094 | rseq_after_asm_goto(); |
784b0012 MD |
1095 | return 1; |
1096 | #ifdef RSEQ_COMPARE_TWICE | |
1097 | error1: | |
dd76f2d6 | 1098 | rseq_after_asm_goto(); |
784b0012 MD |
1099 | rseq_bug("cpu_id comparison failed"); |
1100 | error2: | |
dd76f2d6 | 1101 | rseq_after_asm_goto(); |
784b0012 MD |
1102 | rseq_bug("expected value comparison failed"); |
1103 | #endif | |
1104 | ||
1105 | } | |
1106 | ||
1107 | static inline __attribute__((always_inline)) | |
1108 | int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | |
1109 | intptr_t *v2, intptr_t expect2, | |
1110 | intptr_t newv, int cpu) | |
1111 | { | |
1112 | RSEQ_INJECT_C(9) | |
1113 | ||
1114 | __asm__ __volatile__ goto ( | |
1115 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1116 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1117 | #ifdef RSEQ_COMPARE_TWICE | |
1118 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1119 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1120 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | |
1121 | #endif | |
784b0012 | 1122 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
1123 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
1124 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
1125 | RSEQ_INJECT_ASM(3) |
1126 | "cmpl %[v], %[expect]\n\t" | |
1127 | "jnz %l[cmpfail]\n\t" | |
1128 | RSEQ_INJECT_ASM(4) | |
1129 | "cmpl %[expect2], %[v2]\n\t" | |
1130 | "jnz %l[cmpfail]\n\t" | |
1131 | RSEQ_INJECT_ASM(5) | |
1132 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 1133 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
1134 | "cmpl %[v], %[expect]\n\t" |
1135 | "jnz %l[error2]\n\t" | |
1136 | "cmpl %[expect2], %[v2]\n\t" | |
1137 | "jnz %l[error3]\n\t" | |
1138 | #endif | |
1139 | "movl %[newv], %%eax\n\t" | |
1140 | /* final store */ | |
1141 | "movl %%eax, %[v]\n\t" | |
1142 | "2:\n\t" | |
1143 | RSEQ_INJECT_ASM(6) | |
1144 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
1145 | : /* gcc asm goto does not allow outputs */ | |
1146 | : [cpu_id] "r" (cpu), | |
9698c399 | 1147 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
1148 | /* cmp2 input */ |
1149 | [v2] "m" (*v2), | |
1150 | [expect2] "r" (expect2), | |
1151 | /* final store input */ | |
1152 | [v] "m" (*v), | |
1153 | [expect] "r" (expect), | |
1154 | [newv] "m" (newv) | |
1155 | : "memory", "cc", "eax" | |
1156 | RSEQ_INJECT_CLOBBER | |
1157 | : abort, cmpfail | |
1158 | #ifdef RSEQ_COMPARE_TWICE | |
1159 | , error1, error2, error3 | |
1160 | #endif | |
1161 | ); | |
dd76f2d6 | 1162 | rseq_after_asm_goto(); |
784b0012 MD |
1163 | return 0; |
1164 | abort: | |
dd76f2d6 | 1165 | rseq_after_asm_goto(); |
784b0012 MD |
1166 | RSEQ_INJECT_FAILED |
1167 | return -1; | |
1168 | cmpfail: | |
dd76f2d6 | 1169 | rseq_after_asm_goto(); |
784b0012 MD |
1170 | return 1; |
1171 | #ifdef RSEQ_COMPARE_TWICE | |
1172 | error1: | |
dd76f2d6 | 1173 | rseq_after_asm_goto(); |
784b0012 MD |
1174 | rseq_bug("cpu_id comparison failed"); |
1175 | error2: | |
dd76f2d6 | 1176 | rseq_after_asm_goto(); |
784b0012 MD |
1177 | rseq_bug("1st expected value comparison failed"); |
1178 | error3: | |
dd76f2d6 | 1179 | rseq_after_asm_goto(); |
784b0012 MD |
1180 | rseq_bug("2nd expected value comparison failed"); |
1181 | #endif | |
1182 | } | |
1183 | ||
1184 | /* TODO: implement a faster memcpy. */ | |
1185 | static inline __attribute__((always_inline)) | |
1186 | int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | |
1187 | void *dst, void *src, size_t len, | |
1188 | intptr_t newv, int cpu) | |
1189 | { | |
1190 | uint32_t rseq_scratch[3]; | |
1191 | ||
1192 | RSEQ_INJECT_C(9) | |
1193 | ||
1194 | __asm__ __volatile__ goto ( | |
1195 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1196 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1197 | #ifdef RSEQ_COMPARE_TWICE | |
1198 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1199 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1200 | #endif | |
784b0012 MD |
1201 | "movl %[src], %[rseq_scratch0]\n\t" |
1202 | "movl %[dst], %[rseq_scratch1]\n\t" | |
1203 | "movl %[len], %[rseq_scratch2]\n\t" | |
1204 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
05ebd3f9 MD |
1205 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
1206 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
1207 | RSEQ_INJECT_ASM(3) |
1208 | "movl %[expect], %%eax\n\t" | |
1209 | "cmpl %%eax, %[v]\n\t" | |
1210 | "jnz 5f\n\t" | |
1211 | RSEQ_INJECT_ASM(4) | |
1212 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 1213 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
784b0012 MD |
1214 | "movl %[expect], %%eax\n\t" |
1215 | "cmpl %%eax, %[v]\n\t" | |
1216 | "jnz 7f\n\t" | |
1217 | #endif | |
1218 | /* try memcpy */ | |
1219 | "test %[len], %[len]\n\t" \ | |
1220 | "jz 333f\n\t" \ | |
1221 | "222:\n\t" \ | |
1222 | "movb (%[src]), %%al\n\t" \ | |
1223 | "movb %%al, (%[dst])\n\t" \ | |
1224 | "inc %[src]\n\t" \ | |
1225 | "inc %[dst]\n\t" \ | |
1226 | "dec %[len]\n\t" \ | |
1227 | "jnz 222b\n\t" \ | |
1228 | "333:\n\t" \ | |
1229 | RSEQ_INJECT_ASM(5) | |
1230 | "movl %[newv], %%eax\n\t" | |
1231 | /* final store */ | |
1232 | "movl %%eax, %[v]\n\t" | |
1233 | "2:\n\t" | |
1234 | RSEQ_INJECT_ASM(6) | |
1235 | /* teardown */ | |
1236 | "movl %[rseq_scratch2], %[len]\n\t" | |
1237 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1238 | "movl %[rseq_scratch0], %[src]\n\t" | |
1239 | RSEQ_ASM_DEFINE_ABORT(4, | |
1240 | "movl %[rseq_scratch2], %[len]\n\t" | |
1241 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1242 | "movl %[rseq_scratch0], %[src]\n\t", | |
1243 | abort) | |
1244 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
1245 | "movl %[rseq_scratch2], %[len]\n\t" | |
1246 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1247 | "movl %[rseq_scratch0], %[src]\n\t", | |
1248 | cmpfail) | |
1249 | #ifdef RSEQ_COMPARE_TWICE | |
1250 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
1251 | "movl %[rseq_scratch2], %[len]\n\t" | |
1252 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1253 | "movl %[rseq_scratch0], %[src]\n\t", | |
1254 | error1) | |
1255 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
1256 | "movl %[rseq_scratch2], %[len]\n\t" | |
1257 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1258 | "movl %[rseq_scratch0], %[src]\n\t", | |
1259 | error2) | |
1260 | #endif | |
1261 | : /* gcc asm goto does not allow outputs */ | |
1262 | : [cpu_id] "r" (cpu), | |
9698c399 | 1263 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
1264 | /* final store input */ |
1265 | [v] "m" (*v), | |
1266 | [expect] "m" (expect), | |
1267 | [newv] "m" (newv), | |
1268 | /* try memcpy input */ | |
1269 | [dst] "r" (dst), | |
1270 | [src] "r" (src), | |
1271 | [len] "r" (len), | |
1272 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
1273 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
1274 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
1275 | : "memory", "cc", "eax" | |
1276 | RSEQ_INJECT_CLOBBER | |
1277 | : abort, cmpfail | |
1278 | #ifdef RSEQ_COMPARE_TWICE | |
1279 | , error1, error2 | |
1280 | #endif | |
1281 | ); | |
dd76f2d6 | 1282 | rseq_after_asm_goto(); |
784b0012 MD |
1283 | return 0; |
1284 | abort: | |
dd76f2d6 | 1285 | rseq_after_asm_goto(); |
784b0012 MD |
1286 | RSEQ_INJECT_FAILED |
1287 | return -1; | |
1288 | cmpfail: | |
dd76f2d6 | 1289 | rseq_after_asm_goto(); |
784b0012 MD |
1290 | return 1; |
1291 | #ifdef RSEQ_COMPARE_TWICE | |
1292 | error1: | |
dd76f2d6 | 1293 | rseq_after_asm_goto(); |
784b0012 MD |
1294 | rseq_bug("cpu_id comparison failed"); |
1295 | error2: | |
dd76f2d6 | 1296 | rseq_after_asm_goto(); |
784b0012 MD |
1297 | rseq_bug("expected value comparison failed"); |
1298 | #endif | |
1299 | } | |
1300 | ||
1301 | /* TODO: implement a faster memcpy. */ | |
1302 | static inline __attribute__((always_inline)) | |
1303 | int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | |
1304 | void *dst, void *src, size_t len, | |
1305 | intptr_t newv, int cpu) | |
1306 | { | |
1307 | uint32_t rseq_scratch[3]; | |
1308 | ||
1309 | RSEQ_INJECT_C(9) | |
1310 | ||
1311 | __asm__ __volatile__ goto ( | |
1312 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1313 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1314 | #ifdef RSEQ_COMPARE_TWICE | |
1315 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1316 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1317 | #endif | |
784b0012 MD |
1318 | "movl %[src], %[rseq_scratch0]\n\t" |
1319 | "movl %[dst], %[rseq_scratch1]\n\t" | |
1320 | "movl %[len], %[rseq_scratch2]\n\t" | |
1321 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
05ebd3f9 MD |
1322 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
1323 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
1324 | RSEQ_INJECT_ASM(3) |
1325 | "movl %[expect], %%eax\n\t" | |
1326 | "cmpl %%eax, %[v]\n\t" | |
1327 | "jnz 5f\n\t" | |
1328 | RSEQ_INJECT_ASM(4) | |
1329 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 1330 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
784b0012 MD |
1331 | "movl %[expect], %%eax\n\t" |
1332 | "cmpl %%eax, %[v]\n\t" | |
1333 | "jnz 7f\n\t" | |
1334 | #endif | |
1335 | /* try memcpy */ | |
1336 | "test %[len], %[len]\n\t" \ | |
1337 | "jz 333f\n\t" \ | |
1338 | "222:\n\t" \ | |
1339 | "movb (%[src]), %%al\n\t" \ | |
1340 | "movb %%al, (%[dst])\n\t" \ | |
1341 | "inc %[src]\n\t" \ | |
1342 | "inc %[dst]\n\t" \ | |
1343 | "dec %[len]\n\t" \ | |
1344 | "jnz 222b\n\t" \ | |
1345 | "333:\n\t" \ | |
1346 | RSEQ_INJECT_ASM(5) | |
1347 | "lock; addl $0,-128(%%esp)\n\t" | |
1348 | "movl %[newv], %%eax\n\t" | |
1349 | /* final store */ | |
1350 | "movl %%eax, %[v]\n\t" | |
1351 | "2:\n\t" | |
1352 | RSEQ_INJECT_ASM(6) | |
1353 | /* teardown */ | |
1354 | "movl %[rseq_scratch2], %[len]\n\t" | |
1355 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1356 | "movl %[rseq_scratch0], %[src]\n\t" | |
1357 | RSEQ_ASM_DEFINE_ABORT(4, | |
1358 | "movl %[rseq_scratch2], %[len]\n\t" | |
1359 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1360 | "movl %[rseq_scratch0], %[src]\n\t", | |
1361 | abort) | |
1362 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
1363 | "movl %[rseq_scratch2], %[len]\n\t" | |
1364 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1365 | "movl %[rseq_scratch0], %[src]\n\t", | |
1366 | cmpfail) | |
1367 | #ifdef RSEQ_COMPARE_TWICE | |
1368 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
1369 | "movl %[rseq_scratch2], %[len]\n\t" | |
1370 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1371 | "movl %[rseq_scratch0], %[src]\n\t", | |
1372 | error1) | |
1373 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
1374 | "movl %[rseq_scratch2], %[len]\n\t" | |
1375 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1376 | "movl %[rseq_scratch0], %[src]\n\t", | |
1377 | error2) | |
1378 | #endif | |
1379 | : /* gcc asm goto does not allow outputs */ | |
1380 | : [cpu_id] "r" (cpu), | |
9698c399 | 1381 | [rseq_abi] "r" (rseq_get_abi()), |
784b0012 MD |
1382 | /* final store input */ |
1383 | [v] "m" (*v), | |
1384 | [expect] "m" (expect), | |
1385 | [newv] "m" (newv), | |
1386 | /* try memcpy input */ | |
1387 | [dst] "r" (dst), | |
1388 | [src] "r" (src), | |
1389 | [len] "r" (len), | |
1390 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
1391 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
1392 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
1393 | : "memory", "cc", "eax" | |
1394 | RSEQ_INJECT_CLOBBER | |
1395 | : abort, cmpfail | |
1396 | #ifdef RSEQ_COMPARE_TWICE | |
1397 | , error1, error2 | |
1398 | #endif | |
1399 | ); | |
dd76f2d6 | 1400 | rseq_after_asm_goto(); |
784b0012 MD |
1401 | return 0; |
1402 | abort: | |
dd76f2d6 | 1403 | rseq_after_asm_goto(); |
784b0012 MD |
1404 | RSEQ_INJECT_FAILED |
1405 | return -1; | |
1406 | cmpfail: | |
dd76f2d6 | 1407 | rseq_after_asm_goto(); |
784b0012 MD |
1408 | return 1; |
1409 | #ifdef RSEQ_COMPARE_TWICE | |
1410 | error1: | |
dd76f2d6 | 1411 | rseq_after_asm_goto(); |
784b0012 MD |
1412 | rseq_bug("cpu_id comparison failed"); |
1413 | error2: | |
dd76f2d6 | 1414 | rseq_after_asm_goto(); |
784b0012 MD |
1415 | rseq_bug("expected value comparison failed"); |
1416 | #endif | |
1417 | } | |
1418 | ||
de28c254 MD |
1419 | /* |
1420 | * Dereference @p. Add voffp to the dereferenced pointer, and load its content | |
1421 | * into @load. | |
1422 | */ | |
1423 | static inline __attribute__((always_inline)) | |
1424 | int rseq_deref_loadoffp(intptr_t *p, off_t voffp, intptr_t *load, int cpu) | |
1425 | { | |
1426 | RSEQ_INJECT_C(9) | |
1427 | ||
1428 | __asm__ __volatile__ goto ( | |
1429 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
1430 | #ifdef RSEQ_COMPARE_TWICE | |
1431 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1432 | #endif | |
1433 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
1434 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | |
1435 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
1436 | RSEQ_INJECT_ASM(3) | |
1437 | "movl %[p], %%ebx\n\t" | |
1438 | RSEQ_INJECT_ASM(4) | |
1439 | #ifdef RSEQ_COMPARE_TWICE | |
1440 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | |
1441 | #endif | |
1442 | "addl %[voffp], %%ebx\n\t" | |
1443 | "movl (%%ebx), %%ebx\n\t" | |
1444 | "movl %%ebx, %[load]\n\t" | |
1445 | "2:\n\t" | |
1446 | RSEQ_INJECT_ASM(5) | |
1447 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
1448 | : /* gcc asm goto does not allow outputs */ | |
1449 | : [cpu_id] "r" (cpu), | |
9698c399 | 1450 | [rseq_abi] "r" (rseq_get_abi()), |
de28c254 MD |
1451 | /* final store input */ |
1452 | [p] "m" (*p), | |
1453 | [voffp] "ir" (voffp), | |
1454 | [load] "m" (*load) | |
1455 | : "memory", "cc", "eax", "ebx" | |
1456 | RSEQ_INJECT_CLOBBER | |
1457 | : abort | |
1458 | #ifdef RSEQ_COMPARE_TWICE | |
1459 | , error1 | |
1460 | #endif | |
1461 | ); | |
dd76f2d6 | 1462 | rseq_after_asm_goto(); |
de28c254 MD |
1463 | return 0; |
1464 | abort: | |
dd76f2d6 | 1465 | rseq_after_asm_goto(); |
de28c254 MD |
1466 | RSEQ_INJECT_FAILED |
1467 | return -1; | |
1468 | #ifdef RSEQ_COMPARE_TWICE | |
1469 | error1: | |
dd76f2d6 | 1470 | rseq_after_asm_goto(); |
de28c254 MD |
1471 | rseq_bug("cpu_id comparison failed"); |
1472 | #endif | |
1473 | } | |
1474 | ||
784b0012 MD |
1475 | #endif /* !RSEQ_SKIP_FASTPATH */ |
1476 | ||
1477 | #endif |