Commit | Line | Data |
---|---|---|
744d0b8b | 1 | /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
784b0012 MD |
2 | /* |
3 | * rseq-x86.h | |
4 | * | |
5 | * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
6 | */ | |
7 | ||
8 | #include <stdint.h> | |
9 | ||
a5fe9e95 MD |
10 | /* |
11 | * RSEQ_SIG is used with the following reserved undefined instructions, which | |
12 | * trap in user-space: | |
13 | * | |
14 | * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi | |
15 | * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi | |
16 | */ | |
784b0012 MD |
17 | #define RSEQ_SIG 0x53053053 |
18 | ||
05ebd3f9 MD |
19 | /* |
20 | * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input | |
21 | * operands, we cannot use "m" input operands, and rather pass the __rseq_abi | |
22 | * address through a "r" input operand. | |
23 | */ | |
24 | ||
25 | /* Offset of cpu_id and rseq_cs fields in struct rseq. */ | |
26 | #define RSEQ_CPU_ID_OFFSET 4 | |
27 | #define RSEQ_CS_OFFSET 8 | |
28 | ||
784b0012 MD |
29 | #ifdef __x86_64__ |
30 | ||
31 | #define rseq_smp_mb() \ | |
32 | __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") | |
33 | #define rseq_smp_rmb() rseq_barrier() | |
34 | #define rseq_smp_wmb() rseq_barrier() | |
35 | ||
36 | #define rseq_smp_load_acquire(p) \ | |
37 | __extension__ ({ \ | |
38 | __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ | |
39 | rseq_barrier(); \ | |
40 | ____p1; \ | |
41 | }) | |
42 | ||
43 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
44 | ||
45 | #define rseq_smp_store_release(p, v) \ | |
46 | do { \ | |
47 | rseq_barrier(); \ | |
48 | RSEQ_WRITE_ONCE(*p, v); \ | |
49 | } while (0) | |
50 | ||
51 | #ifdef RSEQ_SKIP_FASTPATH | |
52 | #include "rseq-skip.h" | |
53 | #else /* !RSEQ_SKIP_FASTPATH */ | |
54 | ||
55 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
56 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 57 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
58 | ".balign 32\n\t" \ |
59 | __rseq_str(label) ":\n\t" \ | |
60 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
61 | ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
62 | ".popsection\n\t" \ |
63 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
64 | ".quad " __rseq_str(label) "b\n\t" \ | |
784b0012 MD |
65 | ".popsection\n\t" |
66 | ||
dd01d0fb | 67 | |
784b0012 MD |
68 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
69 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
70 | (post_commit_ip - start_ip), abort_ip) | |
71 | ||
90d9876e MD |
72 | /* |
73 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
74 | * of the critical section where a critical section can either branch to or |
75 | * reach through the normal course of its execution. The abort IP and the | |
76 | * post-commit IP are already part of the __rseq_cs section and should not be | |
77 | * explicitly defined as additional exit points. Knowing all exit points is | |
78 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
79 | */ |
80 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
81 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
82 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ | |
83 | ".popsection\n\t" | |
84 | ||
784b0012 MD |
85 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
86 | RSEQ_INJECT_ASM(1) \ | |
87 | "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ | |
05ebd3f9 | 88 | "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
89 | __rseq_str(label) ":\n\t" |
90 | ||
91 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ | |
92 | RSEQ_INJECT_ASM(2) \ | |
05ebd3f9 | 93 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
94 | "jnz " __rseq_str(label) "\n\t" |
95 | ||
96 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
97 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
98 | /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ |
99 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
100 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
101 | __rseq_str(label) ":\n\t" \ | |
102 | teardown \ | |
103 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
104 | ".popsection\n\t" | |
105 | ||
106 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
107 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
108 | __rseq_str(label) ":\n\t" \ | |
109 | teardown \ | |
110 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ | |
111 | ".popsection\n\t" | |
112 | ||
113 | static inline __attribute__((always_inline)) | |
114 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | |
115 | { | |
116 | RSEQ_INJECT_C(9) | |
117 | ||
118 | __asm__ __volatile__ goto ( | |
119 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
120 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
121 | #ifdef RSEQ_COMPARE_TWICE | |
122 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
123 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
124 | #endif | |
784b0012 | 125 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
126 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
127 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
128 | RSEQ_INJECT_ASM(3) |
129 | "cmpq %[v], %[expect]\n\t" | |
130 | "jnz %l[cmpfail]\n\t" | |
131 | RSEQ_INJECT_ASM(4) | |
132 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 133 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
134 | "cmpq %[v], %[expect]\n\t" |
135 | "jnz %l[error2]\n\t" | |
136 | #endif | |
137 | /* final store */ | |
138 | "movq %[newv], %[v]\n\t" | |
139 | "2:\n\t" | |
140 | RSEQ_INJECT_ASM(5) | |
141 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
142 | : /* gcc asm goto does not allow outputs */ | |
143 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 144 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
145 | [v] "m" (*v), |
146 | [expect] "r" (expect), | |
147 | [newv] "r" (newv) | |
148 | : "memory", "cc", "rax" | |
149 | RSEQ_INJECT_CLOBBER | |
150 | : abort, cmpfail | |
151 | #ifdef RSEQ_COMPARE_TWICE | |
152 | , error1, error2 | |
153 | #endif | |
154 | ); | |
155 | return 0; | |
156 | abort: | |
157 | RSEQ_INJECT_FAILED | |
158 | return -1; | |
159 | cmpfail: | |
160 | return 1; | |
161 | #ifdef RSEQ_COMPARE_TWICE | |
162 | error1: | |
163 | rseq_bug("cpu_id comparison failed"); | |
164 | error2: | |
165 | rseq_bug("expected value comparison failed"); | |
166 | #endif | |
167 | } | |
168 | ||
169 | /* | |
170 | * Compare @v against @expectnot. When it does _not_ match, load @v | |
171 | * into @load, and store the content of *@v + voffp into @v. | |
172 | */ | |
173 | static inline __attribute__((always_inline)) | |
174 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | |
175 | off_t voffp, intptr_t *load, int cpu) | |
176 | { | |
177 | RSEQ_INJECT_C(9) | |
178 | ||
179 | __asm__ __volatile__ goto ( | |
180 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
181 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
182 | #ifdef RSEQ_COMPARE_TWICE | |
183 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
184 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
185 | #endif | |
784b0012 | 186 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
187 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
188 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
189 | RSEQ_INJECT_ASM(3) |
190 | "movq %[v], %%rbx\n\t" | |
191 | "cmpq %%rbx, %[expectnot]\n\t" | |
192 | "je %l[cmpfail]\n\t" | |
193 | RSEQ_INJECT_ASM(4) | |
194 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 195 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
196 | "movq %[v], %%rbx\n\t" |
197 | "cmpq %%rbx, %[expectnot]\n\t" | |
198 | "je %l[error2]\n\t" | |
199 | #endif | |
200 | "movq %%rbx, %[load]\n\t" | |
201 | "addq %[voffp], %%rbx\n\t" | |
202 | "movq (%%rbx), %%rbx\n\t" | |
203 | /* final store */ | |
204 | "movq %%rbx, %[v]\n\t" | |
205 | "2:\n\t" | |
206 | RSEQ_INJECT_ASM(5) | |
207 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
208 | : /* gcc asm goto does not allow outputs */ | |
209 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 210 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
211 | /* final store input */ |
212 | [v] "m" (*v), | |
213 | [expectnot] "r" (expectnot), | |
214 | [voffp] "er" (voffp), | |
215 | [load] "m" (*load) | |
216 | : "memory", "cc", "rax", "rbx" | |
217 | RSEQ_INJECT_CLOBBER | |
218 | : abort, cmpfail | |
219 | #ifdef RSEQ_COMPARE_TWICE | |
220 | , error1, error2 | |
221 | #endif | |
222 | ); | |
223 | return 0; | |
224 | abort: | |
225 | RSEQ_INJECT_FAILED | |
226 | return -1; | |
227 | cmpfail: | |
228 | return 1; | |
229 | #ifdef RSEQ_COMPARE_TWICE | |
230 | error1: | |
231 | rseq_bug("cpu_id comparison failed"); | |
232 | error2: | |
233 | rseq_bug("expected value comparison failed"); | |
234 | #endif | |
235 | } | |
236 | ||
237 | static inline __attribute__((always_inline)) | |
238 | int rseq_addv(intptr_t *v, intptr_t count, int cpu) | |
239 | { | |
240 | RSEQ_INJECT_C(9) | |
241 | ||
242 | __asm__ __volatile__ goto ( | |
243 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
244 | #ifdef RSEQ_COMPARE_TWICE |
245 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
246 | #endif | |
784b0012 | 247 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
248 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
249 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
250 | RSEQ_INJECT_ASM(3) |
251 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 252 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
253 | #endif |
254 | /* final store */ | |
255 | "addq %[count], %[v]\n\t" | |
256 | "2:\n\t" | |
257 | RSEQ_INJECT_ASM(4) | |
258 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
259 | : /* gcc asm goto does not allow outputs */ | |
260 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 261 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
262 | /* final store input */ |
263 | [v] "m" (*v), | |
264 | [count] "er" (count) | |
265 | : "memory", "cc", "rax" | |
266 | RSEQ_INJECT_CLOBBER | |
267 | : abort | |
268 | #ifdef RSEQ_COMPARE_TWICE | |
269 | , error1 | |
270 | #endif | |
271 | ); | |
272 | return 0; | |
273 | abort: | |
274 | RSEQ_INJECT_FAILED | |
275 | return -1; | |
276 | #ifdef RSEQ_COMPARE_TWICE | |
277 | error1: | |
278 | rseq_bug("cpu_id comparison failed"); | |
279 | #endif | |
280 | } | |
281 | ||
282 | static inline __attribute__((always_inline)) | |
283 | int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | |
284 | intptr_t *v2, intptr_t newv2, | |
285 | intptr_t newv, int cpu) | |
286 | { | |
287 | RSEQ_INJECT_C(9) | |
288 | ||
289 | __asm__ __volatile__ goto ( | |
290 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
291 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
292 | #ifdef RSEQ_COMPARE_TWICE | |
293 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
294 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
295 | #endif | |
784b0012 | 296 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
297 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
298 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
299 | RSEQ_INJECT_ASM(3) |
300 | "cmpq %[v], %[expect]\n\t" | |
301 | "jnz %l[cmpfail]\n\t" | |
302 | RSEQ_INJECT_ASM(4) | |
303 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 304 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
305 | "cmpq %[v], %[expect]\n\t" |
306 | "jnz %l[error2]\n\t" | |
307 | #endif | |
308 | /* try store */ | |
309 | "movq %[newv2], %[v2]\n\t" | |
310 | RSEQ_INJECT_ASM(5) | |
311 | /* final store */ | |
312 | "movq %[newv], %[v]\n\t" | |
313 | "2:\n\t" | |
314 | RSEQ_INJECT_ASM(6) | |
315 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
316 | : /* gcc asm goto does not allow outputs */ | |
317 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 318 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
319 | /* try store input */ |
320 | [v2] "m" (*v2), | |
321 | [newv2] "r" (newv2), | |
322 | /* final store input */ | |
323 | [v] "m" (*v), | |
324 | [expect] "r" (expect), | |
325 | [newv] "r" (newv) | |
326 | : "memory", "cc", "rax" | |
327 | RSEQ_INJECT_CLOBBER | |
328 | : abort, cmpfail | |
329 | #ifdef RSEQ_COMPARE_TWICE | |
330 | , error1, error2 | |
331 | #endif | |
332 | ); | |
333 | return 0; | |
334 | abort: | |
335 | RSEQ_INJECT_FAILED | |
336 | return -1; | |
337 | cmpfail: | |
338 | return 1; | |
339 | #ifdef RSEQ_COMPARE_TWICE | |
340 | error1: | |
341 | rseq_bug("cpu_id comparison failed"); | |
342 | error2: | |
343 | rseq_bug("expected value comparison failed"); | |
344 | #endif | |
345 | } | |
346 | ||
347 | /* x86-64 is TSO. */ | |
348 | static inline __attribute__((always_inline)) | |
349 | int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | |
350 | intptr_t *v2, intptr_t newv2, | |
351 | intptr_t newv, int cpu) | |
352 | { | |
353 | return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); | |
354 | } | |
355 | ||
356 | static inline __attribute__((always_inline)) | |
357 | int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | |
358 | intptr_t *v2, intptr_t expect2, | |
359 | intptr_t newv, int cpu) | |
360 | { | |
361 | RSEQ_INJECT_C(9) | |
362 | ||
363 | __asm__ __volatile__ goto ( | |
364 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
365 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
366 | #ifdef RSEQ_COMPARE_TWICE | |
367 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
368 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
369 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | |
370 | #endif | |
784b0012 | 371 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
372 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
373 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
374 | RSEQ_INJECT_ASM(3) |
375 | "cmpq %[v], %[expect]\n\t" | |
376 | "jnz %l[cmpfail]\n\t" | |
377 | RSEQ_INJECT_ASM(4) | |
378 | "cmpq %[v2], %[expect2]\n\t" | |
379 | "jnz %l[cmpfail]\n\t" | |
380 | RSEQ_INJECT_ASM(5) | |
381 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 382 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
383 | "cmpq %[v], %[expect]\n\t" |
384 | "jnz %l[error2]\n\t" | |
385 | "cmpq %[v2], %[expect2]\n\t" | |
386 | "jnz %l[error3]\n\t" | |
387 | #endif | |
388 | /* final store */ | |
389 | "movq %[newv], %[v]\n\t" | |
390 | "2:\n\t" | |
391 | RSEQ_INJECT_ASM(6) | |
392 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
393 | : /* gcc asm goto does not allow outputs */ | |
394 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 395 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
396 | /* cmp2 input */ |
397 | [v2] "m" (*v2), | |
398 | [expect2] "r" (expect2), | |
399 | /* final store input */ | |
400 | [v] "m" (*v), | |
401 | [expect] "r" (expect), | |
402 | [newv] "r" (newv) | |
403 | : "memory", "cc", "rax" | |
404 | RSEQ_INJECT_CLOBBER | |
405 | : abort, cmpfail | |
406 | #ifdef RSEQ_COMPARE_TWICE | |
407 | , error1, error2, error3 | |
408 | #endif | |
409 | ); | |
410 | return 0; | |
411 | abort: | |
412 | RSEQ_INJECT_FAILED | |
413 | return -1; | |
414 | cmpfail: | |
415 | return 1; | |
416 | #ifdef RSEQ_COMPARE_TWICE | |
417 | error1: | |
418 | rseq_bug("cpu_id comparison failed"); | |
419 | error2: | |
420 | rseq_bug("1st expected value comparison failed"); | |
421 | error3: | |
422 | rseq_bug("2nd expected value comparison failed"); | |
423 | #endif | |
424 | } | |
425 | ||
426 | static inline __attribute__((always_inline)) | |
427 | int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | |
428 | void *dst, void *src, size_t len, | |
429 | intptr_t newv, int cpu) | |
430 | { | |
431 | uint64_t rseq_scratch[3]; | |
432 | ||
433 | RSEQ_INJECT_C(9) | |
434 | ||
435 | __asm__ __volatile__ goto ( | |
436 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
437 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
438 | #ifdef RSEQ_COMPARE_TWICE | |
439 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
440 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
441 | #endif | |
784b0012 MD |
442 | "movq %[src], %[rseq_scratch0]\n\t" |
443 | "movq %[dst], %[rseq_scratch1]\n\t" | |
444 | "movq %[len], %[rseq_scratch2]\n\t" | |
445 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
05ebd3f9 MD |
446 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
447 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
448 | RSEQ_INJECT_ASM(3) |
449 | "cmpq %[v], %[expect]\n\t" | |
450 | "jnz 5f\n\t" | |
451 | RSEQ_INJECT_ASM(4) | |
452 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 453 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
784b0012 MD |
454 | "cmpq %[v], %[expect]\n\t" |
455 | "jnz 7f\n\t" | |
456 | #endif | |
457 | /* try memcpy */ | |
458 | "test %[len], %[len]\n\t" \ | |
459 | "jz 333f\n\t" \ | |
460 | "222:\n\t" \ | |
461 | "movb (%[src]), %%al\n\t" \ | |
462 | "movb %%al, (%[dst])\n\t" \ | |
463 | "inc %[src]\n\t" \ | |
464 | "inc %[dst]\n\t" \ | |
465 | "dec %[len]\n\t" \ | |
466 | "jnz 222b\n\t" \ | |
467 | "333:\n\t" \ | |
468 | RSEQ_INJECT_ASM(5) | |
469 | /* final store */ | |
470 | "movq %[newv], %[v]\n\t" | |
471 | "2:\n\t" | |
472 | RSEQ_INJECT_ASM(6) | |
473 | /* teardown */ | |
474 | "movq %[rseq_scratch2], %[len]\n\t" | |
475 | "movq %[rseq_scratch1], %[dst]\n\t" | |
476 | "movq %[rseq_scratch0], %[src]\n\t" | |
477 | RSEQ_ASM_DEFINE_ABORT(4, | |
478 | "movq %[rseq_scratch2], %[len]\n\t" | |
479 | "movq %[rseq_scratch1], %[dst]\n\t" | |
480 | "movq %[rseq_scratch0], %[src]\n\t", | |
481 | abort) | |
482 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
483 | "movq %[rseq_scratch2], %[len]\n\t" | |
484 | "movq %[rseq_scratch1], %[dst]\n\t" | |
485 | "movq %[rseq_scratch0], %[src]\n\t", | |
486 | cmpfail) | |
487 | #ifdef RSEQ_COMPARE_TWICE | |
488 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
489 | "movq %[rseq_scratch2], %[len]\n\t" | |
490 | "movq %[rseq_scratch1], %[dst]\n\t" | |
491 | "movq %[rseq_scratch0], %[src]\n\t", | |
492 | error1) | |
493 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
494 | "movq %[rseq_scratch2], %[len]\n\t" | |
495 | "movq %[rseq_scratch1], %[dst]\n\t" | |
496 | "movq %[rseq_scratch0], %[src]\n\t", | |
497 | error2) | |
498 | #endif | |
499 | : /* gcc asm goto does not allow outputs */ | |
500 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 501 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
502 | /* final store input */ |
503 | [v] "m" (*v), | |
504 | [expect] "r" (expect), | |
505 | [newv] "r" (newv), | |
506 | /* try memcpy input */ | |
507 | [dst] "r" (dst), | |
508 | [src] "r" (src), | |
509 | [len] "r" (len), | |
510 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
511 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
512 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
513 | : "memory", "cc", "rax" | |
514 | RSEQ_INJECT_CLOBBER | |
515 | : abort, cmpfail | |
516 | #ifdef RSEQ_COMPARE_TWICE | |
517 | , error1, error2 | |
518 | #endif | |
519 | ); | |
520 | return 0; | |
521 | abort: | |
522 | RSEQ_INJECT_FAILED | |
523 | return -1; | |
524 | cmpfail: | |
525 | return 1; | |
526 | #ifdef RSEQ_COMPARE_TWICE | |
527 | error1: | |
528 | rseq_bug("cpu_id comparison failed"); | |
529 | error2: | |
530 | rseq_bug("expected value comparison failed"); | |
531 | #endif | |
532 | } | |
533 | ||
534 | /* x86-64 is TSO. */ | |
535 | static inline __attribute__((always_inline)) | |
536 | int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | |
537 | void *dst, void *src, size_t len, | |
538 | intptr_t newv, int cpu) | |
539 | { | |
540 | return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, | |
541 | newv, cpu); | |
542 | } | |
543 | ||
de28c254 MD |
544 | /* |
545 | * Dereference @p. Add voffp to the dereferenced pointer, and load its content | |
546 | * into @load. | |
547 | */ | |
548 | static inline __attribute__((always_inline)) | |
549 | int rseq_deref_loadoffp(intptr_t *p, off_t voffp, intptr_t *load, int cpu) | |
550 | { | |
551 | RSEQ_INJECT_C(9) | |
552 | ||
553 | __asm__ __volatile__ goto ( | |
554 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
555 | #ifdef RSEQ_COMPARE_TWICE | |
556 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
557 | #endif | |
558 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
559 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | |
560 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
561 | RSEQ_INJECT_ASM(3) | |
562 | "movq %[p], %%rbx\n\t" | |
563 | RSEQ_INJECT_ASM(4) | |
564 | #ifdef RSEQ_COMPARE_TWICE | |
565 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | |
566 | #endif | |
567 | "addq %[voffp], %%rbx\n\t" | |
568 | "movq (%%rbx), %%rbx\n\t" | |
569 | "movq %%rbx, %[load]\n\t" | |
570 | "2:\n\t" | |
571 | RSEQ_INJECT_ASM(5) | |
572 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
573 | : /* gcc asm goto does not allow outputs */ | |
574 | : [cpu_id] "r" (cpu), | |
575 | [rseq_abi] "r" (&__rseq_abi), | |
576 | /* final store input */ | |
577 | [p] "m" (*p), | |
578 | [voffp] "er" (voffp), | |
579 | [load] "m" (*load) | |
580 | : "memory", "cc", "rax", "rbx" | |
581 | RSEQ_INJECT_CLOBBER | |
582 | : abort | |
583 | #ifdef RSEQ_COMPARE_TWICE | |
584 | , error1 | |
585 | #endif | |
586 | ); | |
587 | return 0; | |
588 | abort: | |
589 | RSEQ_INJECT_FAILED | |
590 | return -1; | |
591 | #ifdef RSEQ_COMPARE_TWICE | |
592 | error1: | |
593 | rseq_bug("cpu_id comparison failed"); | |
594 | #endif | |
595 | } | |
596 | ||
784b0012 MD |
597 | #endif /* !RSEQ_SKIP_FASTPATH */ |
598 | ||
599 | #elif __i386__ | |
600 | ||
601 | #define rseq_smp_mb() \ | |
602 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
603 | #define rseq_smp_rmb() \ | |
604 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
605 | #define rseq_smp_wmb() \ | |
606 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
607 | ||
608 | #define rseq_smp_load_acquire(p) \ | |
609 | __extension__ ({ \ | |
610 | __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ | |
611 | rseq_smp_mb(); \ | |
612 | ____p1; \ | |
613 | }) | |
614 | ||
615 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
616 | ||
617 | #define rseq_smp_store_release(p, v) \ | |
618 | do { \ | |
619 | rseq_smp_mb(); \ | |
620 | RSEQ_WRITE_ONCE(*p, v); \ | |
621 | } while (0) | |
622 | ||
623 | #ifdef RSEQ_SKIP_FASTPATH | |
624 | #include "rseq-skip.h" | |
625 | #else /* !RSEQ_SKIP_FASTPATH */ | |
626 | ||
627 | /* | |
628 | * Use eax as scratch register and take memory operands as input to | |
629 | * lessen register pressure. Especially needed when compiling in O0. | |
630 | */ | |
631 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
632 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 633 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
634 | ".balign 32\n\t" \ |
635 | __rseq_str(label) ":\n\t" \ | |
636 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
637 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ | |
dd01d0fb MD |
638 | ".popsection\n\t" \ |
639 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
640 | ".long " __rseq_str(label) "b, 0x0\n\t" \ | |
784b0012 MD |
641 | ".popsection\n\t" |
642 | ||
643 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ | |
644 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
645 | (post_commit_ip - start_ip), abort_ip) | |
646 | ||
90d9876e MD |
647 | /* |
648 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
649 | * of the critical section where a critical section can either branch to or |
650 | * reach through the normal course of its execution. The abort IP and the | |
651 | * post-commit IP are already part of the __rseq_cs section and should not be | |
652 | * explicitly defined as additional exit points. Knowing all exit points is | |
653 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
654 | */ |
655 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
656 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
657 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ | |
658 | ".popsection\n\t" | |
659 | ||
784b0012 MD |
660 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
661 | RSEQ_INJECT_ASM(1) \ | |
05ebd3f9 | 662 | "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
663 | __rseq_str(label) ":\n\t" |
664 | ||
665 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ | |
666 | RSEQ_INJECT_ASM(2) \ | |
05ebd3f9 | 667 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
668 | "jnz " __rseq_str(label) "\n\t" |
669 | ||
670 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
671 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
672 | /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ |
673 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
674 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
675 | __rseq_str(label) ":\n\t" \ | |
676 | teardown \ | |
677 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
678 | ".popsection\n\t" | |
679 | ||
680 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
681 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
682 | __rseq_str(label) ":\n\t" \ | |
683 | teardown \ | |
684 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ | |
685 | ".popsection\n\t" | |
686 | ||
687 | static inline __attribute__((always_inline)) | |
688 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | |
689 | { | |
690 | RSEQ_INJECT_C(9) | |
691 | ||
692 | __asm__ __volatile__ goto ( | |
693 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
694 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
695 | #ifdef RSEQ_COMPARE_TWICE | |
696 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
697 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
698 | #endif | |
784b0012 | 699 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
700 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
701 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
702 | RSEQ_INJECT_ASM(3) |
703 | "cmpl %[v], %[expect]\n\t" | |
704 | "jnz %l[cmpfail]\n\t" | |
705 | RSEQ_INJECT_ASM(4) | |
706 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 707 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
708 | "cmpl %[v], %[expect]\n\t" |
709 | "jnz %l[error2]\n\t" | |
710 | #endif | |
711 | /* final store */ | |
712 | "movl %[newv], %[v]\n\t" | |
713 | "2:\n\t" | |
714 | RSEQ_INJECT_ASM(5) | |
715 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
716 | : /* gcc asm goto does not allow outputs */ | |
717 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 718 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
719 | [v] "m" (*v), |
720 | [expect] "r" (expect), | |
721 | [newv] "r" (newv) | |
722 | : "memory", "cc", "eax" | |
723 | RSEQ_INJECT_CLOBBER | |
724 | : abort, cmpfail | |
725 | #ifdef RSEQ_COMPARE_TWICE | |
726 | , error1, error2 | |
727 | #endif | |
728 | ); | |
729 | return 0; | |
730 | abort: | |
731 | RSEQ_INJECT_FAILED | |
732 | return -1; | |
733 | cmpfail: | |
734 | return 1; | |
735 | #ifdef RSEQ_COMPARE_TWICE | |
736 | error1: | |
737 | rseq_bug("cpu_id comparison failed"); | |
738 | error2: | |
739 | rseq_bug("expected value comparison failed"); | |
740 | #endif | |
741 | } | |
742 | ||
743 | /* | |
744 | * Compare @v against @expectnot. When it does _not_ match, load @v | |
745 | * into @load, and store the content of *@v + voffp into @v. | |
746 | */ | |
747 | static inline __attribute__((always_inline)) | |
748 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | |
749 | off_t voffp, intptr_t *load, int cpu) | |
750 | { | |
751 | RSEQ_INJECT_C(9) | |
752 | ||
753 | __asm__ __volatile__ goto ( | |
754 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
755 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
756 | #ifdef RSEQ_COMPARE_TWICE | |
757 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
758 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
759 | #endif | |
784b0012 | 760 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
761 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
762 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
763 | RSEQ_INJECT_ASM(3) |
764 | "movl %[v], %%ebx\n\t" | |
765 | "cmpl %%ebx, %[expectnot]\n\t" | |
766 | "je %l[cmpfail]\n\t" | |
767 | RSEQ_INJECT_ASM(4) | |
768 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 769 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
770 | "movl %[v], %%ebx\n\t" |
771 | "cmpl %%ebx, %[expectnot]\n\t" | |
772 | "je %l[error2]\n\t" | |
773 | #endif | |
774 | "movl %%ebx, %[load]\n\t" | |
775 | "addl %[voffp], %%ebx\n\t" | |
776 | "movl (%%ebx), %%ebx\n\t" | |
777 | /* final store */ | |
778 | "movl %%ebx, %[v]\n\t" | |
779 | "2:\n\t" | |
780 | RSEQ_INJECT_ASM(5) | |
781 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
782 | : /* gcc asm goto does not allow outputs */ | |
783 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 784 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
785 | /* final store input */ |
786 | [v] "m" (*v), | |
787 | [expectnot] "r" (expectnot), | |
788 | [voffp] "ir" (voffp), | |
789 | [load] "m" (*load) | |
790 | : "memory", "cc", "eax", "ebx" | |
791 | RSEQ_INJECT_CLOBBER | |
792 | : abort, cmpfail | |
793 | #ifdef RSEQ_COMPARE_TWICE | |
794 | , error1, error2 | |
795 | #endif | |
796 | ); | |
797 | return 0; | |
798 | abort: | |
799 | RSEQ_INJECT_FAILED | |
800 | return -1; | |
801 | cmpfail: | |
802 | return 1; | |
803 | #ifdef RSEQ_COMPARE_TWICE | |
804 | error1: | |
805 | rseq_bug("cpu_id comparison failed"); | |
806 | error2: | |
807 | rseq_bug("expected value comparison failed"); | |
808 | #endif | |
809 | } | |
810 | ||
811 | static inline __attribute__((always_inline)) | |
812 | int rseq_addv(intptr_t *v, intptr_t count, int cpu) | |
813 | { | |
814 | RSEQ_INJECT_C(9) | |
815 | ||
816 | __asm__ __volatile__ goto ( | |
817 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
818 | #ifdef RSEQ_COMPARE_TWICE |
819 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
820 | #endif | |
784b0012 | 821 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
822 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
823 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
824 | RSEQ_INJECT_ASM(3) |
825 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 826 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
827 | #endif |
828 | /* final store */ | |
829 | "addl %[count], %[v]\n\t" | |
830 | "2:\n\t" | |
831 | RSEQ_INJECT_ASM(4) | |
832 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
833 | : /* gcc asm goto does not allow outputs */ | |
834 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 835 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
836 | /* final store input */ |
837 | [v] "m" (*v), | |
838 | [count] "ir" (count) | |
839 | : "memory", "cc", "eax" | |
840 | RSEQ_INJECT_CLOBBER | |
841 | : abort | |
842 | #ifdef RSEQ_COMPARE_TWICE | |
843 | , error1 | |
844 | #endif | |
845 | ); | |
846 | return 0; | |
847 | abort: | |
848 | RSEQ_INJECT_FAILED | |
849 | return -1; | |
850 | #ifdef RSEQ_COMPARE_TWICE | |
851 | error1: | |
852 | rseq_bug("cpu_id comparison failed"); | |
853 | #endif | |
854 | } | |
855 | ||
856 | static inline __attribute__((always_inline)) | |
857 | int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | |
858 | intptr_t *v2, intptr_t newv2, | |
859 | intptr_t newv, int cpu) | |
860 | { | |
861 | RSEQ_INJECT_C(9) | |
862 | ||
863 | __asm__ __volatile__ goto ( | |
864 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
865 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
866 | #ifdef RSEQ_COMPARE_TWICE | |
867 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
868 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
869 | #endif | |
784b0012 | 870 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
871 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
872 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
873 | RSEQ_INJECT_ASM(3) |
874 | "cmpl %[v], %[expect]\n\t" | |
875 | "jnz %l[cmpfail]\n\t" | |
876 | RSEQ_INJECT_ASM(4) | |
877 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 878 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
879 | "cmpl %[v], %[expect]\n\t" |
880 | "jnz %l[error2]\n\t" | |
881 | #endif | |
882 | /* try store */ | |
883 | "movl %[newv2], %%eax\n\t" | |
884 | "movl %%eax, %[v2]\n\t" | |
885 | RSEQ_INJECT_ASM(5) | |
886 | /* final store */ | |
887 | "movl %[newv], %[v]\n\t" | |
888 | "2:\n\t" | |
889 | RSEQ_INJECT_ASM(6) | |
890 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
891 | : /* gcc asm goto does not allow outputs */ | |
892 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 893 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
894 | /* try store input */ |
895 | [v2] "m" (*v2), | |
896 | [newv2] "m" (newv2), | |
897 | /* final store input */ | |
898 | [v] "m" (*v), | |
899 | [expect] "r" (expect), | |
900 | [newv] "r" (newv) | |
901 | : "memory", "cc", "eax" | |
902 | RSEQ_INJECT_CLOBBER | |
903 | : abort, cmpfail | |
904 | #ifdef RSEQ_COMPARE_TWICE | |
905 | , error1, error2 | |
906 | #endif | |
907 | ); | |
908 | return 0; | |
909 | abort: | |
910 | RSEQ_INJECT_FAILED | |
911 | return -1; | |
912 | cmpfail: | |
913 | return 1; | |
914 | #ifdef RSEQ_COMPARE_TWICE | |
915 | error1: | |
916 | rseq_bug("cpu_id comparison failed"); | |
917 | error2: | |
918 | rseq_bug("expected value comparison failed"); | |
919 | #endif | |
920 | } | |
921 | ||
922 | static inline __attribute__((always_inline)) | |
923 | int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | |
924 | intptr_t *v2, intptr_t newv2, | |
925 | intptr_t newv, int cpu) | |
926 | { | |
927 | RSEQ_INJECT_C(9) | |
928 | ||
929 | __asm__ __volatile__ goto ( | |
930 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
931 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
932 | #ifdef RSEQ_COMPARE_TWICE | |
933 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
934 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
935 | #endif | |
784b0012 | 936 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
937 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
938 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
939 | RSEQ_INJECT_ASM(3) |
940 | "movl %[expect], %%eax\n\t" | |
941 | "cmpl %[v], %%eax\n\t" | |
942 | "jnz %l[cmpfail]\n\t" | |
943 | RSEQ_INJECT_ASM(4) | |
944 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 945 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
946 | "movl %[expect], %%eax\n\t" |
947 | "cmpl %[v], %%eax\n\t" | |
948 | "jnz %l[error2]\n\t" | |
949 | #endif | |
950 | /* try store */ | |
951 | "movl %[newv2], %[v2]\n\t" | |
952 | RSEQ_INJECT_ASM(5) | |
953 | "lock; addl $0,-128(%%esp)\n\t" | |
954 | /* final store */ | |
955 | "movl %[newv], %[v]\n\t" | |
956 | "2:\n\t" | |
957 | RSEQ_INJECT_ASM(6) | |
958 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
959 | : /* gcc asm goto does not allow outputs */ | |
960 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 961 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
962 | /* try store input */ |
963 | [v2] "m" (*v2), | |
964 | [newv2] "r" (newv2), | |
965 | /* final store input */ | |
966 | [v] "m" (*v), | |
967 | [expect] "m" (expect), | |
968 | [newv] "r" (newv) | |
969 | : "memory", "cc", "eax" | |
970 | RSEQ_INJECT_CLOBBER | |
971 | : abort, cmpfail | |
972 | #ifdef RSEQ_COMPARE_TWICE | |
973 | , error1, error2 | |
974 | #endif | |
975 | ); | |
976 | return 0; | |
977 | abort: | |
978 | RSEQ_INJECT_FAILED | |
979 | return -1; | |
980 | cmpfail: | |
981 | return 1; | |
982 | #ifdef RSEQ_COMPARE_TWICE | |
983 | error1: | |
984 | rseq_bug("cpu_id comparison failed"); | |
985 | error2: | |
986 | rseq_bug("expected value comparison failed"); | |
987 | #endif | |
988 | ||
989 | } | |
990 | ||
991 | static inline __attribute__((always_inline)) | |
992 | int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | |
993 | intptr_t *v2, intptr_t expect2, | |
994 | intptr_t newv, int cpu) | |
995 | { | |
996 | RSEQ_INJECT_C(9) | |
997 | ||
998 | __asm__ __volatile__ goto ( | |
999 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1000 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1001 | #ifdef RSEQ_COMPARE_TWICE | |
1002 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1003 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1004 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | |
1005 | #endif | |
784b0012 | 1006 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
05ebd3f9 MD |
1007 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
1008 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
1009 | RSEQ_INJECT_ASM(3) |
1010 | "cmpl %[v], %[expect]\n\t" | |
1011 | "jnz %l[cmpfail]\n\t" | |
1012 | RSEQ_INJECT_ASM(4) | |
1013 | "cmpl %[expect2], %[v2]\n\t" | |
1014 | "jnz %l[cmpfail]\n\t" | |
1015 | RSEQ_INJECT_ASM(5) | |
1016 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 1017 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) |
784b0012 MD |
1018 | "cmpl %[v], %[expect]\n\t" |
1019 | "jnz %l[error2]\n\t" | |
1020 | "cmpl %[expect2], %[v2]\n\t" | |
1021 | "jnz %l[error3]\n\t" | |
1022 | #endif | |
1023 | "movl %[newv], %%eax\n\t" | |
1024 | /* final store */ | |
1025 | "movl %%eax, %[v]\n\t" | |
1026 | "2:\n\t" | |
1027 | RSEQ_INJECT_ASM(6) | |
1028 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
1029 | : /* gcc asm goto does not allow outputs */ | |
1030 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 1031 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
1032 | /* cmp2 input */ |
1033 | [v2] "m" (*v2), | |
1034 | [expect2] "r" (expect2), | |
1035 | /* final store input */ | |
1036 | [v] "m" (*v), | |
1037 | [expect] "r" (expect), | |
1038 | [newv] "m" (newv) | |
1039 | : "memory", "cc", "eax" | |
1040 | RSEQ_INJECT_CLOBBER | |
1041 | : abort, cmpfail | |
1042 | #ifdef RSEQ_COMPARE_TWICE | |
1043 | , error1, error2, error3 | |
1044 | #endif | |
1045 | ); | |
1046 | return 0; | |
1047 | abort: | |
1048 | RSEQ_INJECT_FAILED | |
1049 | return -1; | |
1050 | cmpfail: | |
1051 | return 1; | |
1052 | #ifdef RSEQ_COMPARE_TWICE | |
1053 | error1: | |
1054 | rseq_bug("cpu_id comparison failed"); | |
1055 | error2: | |
1056 | rseq_bug("1st expected value comparison failed"); | |
1057 | error3: | |
1058 | rseq_bug("2nd expected value comparison failed"); | |
1059 | #endif | |
1060 | } | |
1061 | ||
1062 | /* TODO: implement a faster memcpy. */ | |
1063 | static inline __attribute__((always_inline)) | |
1064 | int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | |
1065 | void *dst, void *src, size_t len, | |
1066 | intptr_t newv, int cpu) | |
1067 | { | |
1068 | uint32_t rseq_scratch[3]; | |
1069 | ||
1070 | RSEQ_INJECT_C(9) | |
1071 | ||
1072 | __asm__ __volatile__ goto ( | |
1073 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1074 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1075 | #ifdef RSEQ_COMPARE_TWICE | |
1076 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1077 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1078 | #endif | |
784b0012 MD |
1079 | "movl %[src], %[rseq_scratch0]\n\t" |
1080 | "movl %[dst], %[rseq_scratch1]\n\t" | |
1081 | "movl %[len], %[rseq_scratch2]\n\t" | |
1082 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
05ebd3f9 MD |
1083 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
1084 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
1085 | RSEQ_INJECT_ASM(3) |
1086 | "movl %[expect], %%eax\n\t" | |
1087 | "cmpl %%eax, %[v]\n\t" | |
1088 | "jnz 5f\n\t" | |
1089 | RSEQ_INJECT_ASM(4) | |
1090 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 1091 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
784b0012 MD |
1092 | "movl %[expect], %%eax\n\t" |
1093 | "cmpl %%eax, %[v]\n\t" | |
1094 | "jnz 7f\n\t" | |
1095 | #endif | |
1096 | /* try memcpy */ | |
1097 | "test %[len], %[len]\n\t" \ | |
1098 | "jz 333f\n\t" \ | |
1099 | "222:\n\t" \ | |
1100 | "movb (%[src]), %%al\n\t" \ | |
1101 | "movb %%al, (%[dst])\n\t" \ | |
1102 | "inc %[src]\n\t" \ | |
1103 | "inc %[dst]\n\t" \ | |
1104 | "dec %[len]\n\t" \ | |
1105 | "jnz 222b\n\t" \ | |
1106 | "333:\n\t" \ | |
1107 | RSEQ_INJECT_ASM(5) | |
1108 | "movl %[newv], %%eax\n\t" | |
1109 | /* final store */ | |
1110 | "movl %%eax, %[v]\n\t" | |
1111 | "2:\n\t" | |
1112 | RSEQ_INJECT_ASM(6) | |
1113 | /* teardown */ | |
1114 | "movl %[rseq_scratch2], %[len]\n\t" | |
1115 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1116 | "movl %[rseq_scratch0], %[src]\n\t" | |
1117 | RSEQ_ASM_DEFINE_ABORT(4, | |
1118 | "movl %[rseq_scratch2], %[len]\n\t" | |
1119 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1120 | "movl %[rseq_scratch0], %[src]\n\t", | |
1121 | abort) | |
1122 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
1123 | "movl %[rseq_scratch2], %[len]\n\t" | |
1124 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1125 | "movl %[rseq_scratch0], %[src]\n\t", | |
1126 | cmpfail) | |
1127 | #ifdef RSEQ_COMPARE_TWICE | |
1128 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
1129 | "movl %[rseq_scratch2], %[len]\n\t" | |
1130 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1131 | "movl %[rseq_scratch0], %[src]\n\t", | |
1132 | error1) | |
1133 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
1134 | "movl %[rseq_scratch2], %[len]\n\t" | |
1135 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1136 | "movl %[rseq_scratch0], %[src]\n\t", | |
1137 | error2) | |
1138 | #endif | |
1139 | : /* gcc asm goto does not allow outputs */ | |
1140 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 1141 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
1142 | /* final store input */ |
1143 | [v] "m" (*v), | |
1144 | [expect] "m" (expect), | |
1145 | [newv] "m" (newv), | |
1146 | /* try memcpy input */ | |
1147 | [dst] "r" (dst), | |
1148 | [src] "r" (src), | |
1149 | [len] "r" (len), | |
1150 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
1151 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
1152 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
1153 | : "memory", "cc", "eax" | |
1154 | RSEQ_INJECT_CLOBBER | |
1155 | : abort, cmpfail | |
1156 | #ifdef RSEQ_COMPARE_TWICE | |
1157 | , error1, error2 | |
1158 | #endif | |
1159 | ); | |
1160 | return 0; | |
1161 | abort: | |
1162 | RSEQ_INJECT_FAILED | |
1163 | return -1; | |
1164 | cmpfail: | |
1165 | return 1; | |
1166 | #ifdef RSEQ_COMPARE_TWICE | |
1167 | error1: | |
1168 | rseq_bug("cpu_id comparison failed"); | |
1169 | error2: | |
1170 | rseq_bug("expected value comparison failed"); | |
1171 | #endif | |
1172 | } | |
1173 | ||
1174 | /* TODO: implement a faster memcpy. */ | |
1175 | static inline __attribute__((always_inline)) | |
1176 | int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | |
1177 | void *dst, void *src, size_t len, | |
1178 | intptr_t newv, int cpu) | |
1179 | { | |
1180 | uint32_t rseq_scratch[3]; | |
1181 | ||
1182 | RSEQ_INJECT_C(9) | |
1183 | ||
1184 | __asm__ __volatile__ goto ( | |
1185 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1186 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1187 | #ifdef RSEQ_COMPARE_TWICE | |
1188 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1189 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1190 | #endif | |
784b0012 MD |
1191 | "movl %[src], %[rseq_scratch0]\n\t" |
1192 | "movl %[dst], %[rseq_scratch1]\n\t" | |
1193 | "movl %[len], %[rseq_scratch2]\n\t" | |
1194 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
05ebd3f9 MD |
1195 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) |
1196 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
784b0012 MD |
1197 | RSEQ_INJECT_ASM(3) |
1198 | "movl %[expect], %%eax\n\t" | |
1199 | "cmpl %%eax, %[v]\n\t" | |
1200 | "jnz 5f\n\t" | |
1201 | RSEQ_INJECT_ASM(4) | |
1202 | #ifdef RSEQ_COMPARE_TWICE | |
05ebd3f9 | 1203 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) |
784b0012 MD |
1204 | "movl %[expect], %%eax\n\t" |
1205 | "cmpl %%eax, %[v]\n\t" | |
1206 | "jnz 7f\n\t" | |
1207 | #endif | |
1208 | /* try memcpy */ | |
1209 | "test %[len], %[len]\n\t" \ | |
1210 | "jz 333f\n\t" \ | |
1211 | "222:\n\t" \ | |
1212 | "movb (%[src]), %%al\n\t" \ | |
1213 | "movb %%al, (%[dst])\n\t" \ | |
1214 | "inc %[src]\n\t" \ | |
1215 | "inc %[dst]\n\t" \ | |
1216 | "dec %[len]\n\t" \ | |
1217 | "jnz 222b\n\t" \ | |
1218 | "333:\n\t" \ | |
1219 | RSEQ_INJECT_ASM(5) | |
1220 | "lock; addl $0,-128(%%esp)\n\t" | |
1221 | "movl %[newv], %%eax\n\t" | |
1222 | /* final store */ | |
1223 | "movl %%eax, %[v]\n\t" | |
1224 | "2:\n\t" | |
1225 | RSEQ_INJECT_ASM(6) | |
1226 | /* teardown */ | |
1227 | "movl %[rseq_scratch2], %[len]\n\t" | |
1228 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1229 | "movl %[rseq_scratch0], %[src]\n\t" | |
1230 | RSEQ_ASM_DEFINE_ABORT(4, | |
1231 | "movl %[rseq_scratch2], %[len]\n\t" | |
1232 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1233 | "movl %[rseq_scratch0], %[src]\n\t", | |
1234 | abort) | |
1235 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
1236 | "movl %[rseq_scratch2], %[len]\n\t" | |
1237 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1238 | "movl %[rseq_scratch0], %[src]\n\t", | |
1239 | cmpfail) | |
1240 | #ifdef RSEQ_COMPARE_TWICE | |
1241 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
1242 | "movl %[rseq_scratch2], %[len]\n\t" | |
1243 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1244 | "movl %[rseq_scratch0], %[src]\n\t", | |
1245 | error1) | |
1246 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
1247 | "movl %[rseq_scratch2], %[len]\n\t" | |
1248 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1249 | "movl %[rseq_scratch0], %[src]\n\t", | |
1250 | error2) | |
1251 | #endif | |
1252 | : /* gcc asm goto does not allow outputs */ | |
1253 | : [cpu_id] "r" (cpu), | |
05ebd3f9 | 1254 | [rseq_abi] "r" (&__rseq_abi), |
784b0012 MD |
1255 | /* final store input */ |
1256 | [v] "m" (*v), | |
1257 | [expect] "m" (expect), | |
1258 | [newv] "m" (newv), | |
1259 | /* try memcpy input */ | |
1260 | [dst] "r" (dst), | |
1261 | [src] "r" (src), | |
1262 | [len] "r" (len), | |
1263 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
1264 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
1265 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
1266 | : "memory", "cc", "eax" | |
1267 | RSEQ_INJECT_CLOBBER | |
1268 | : abort, cmpfail | |
1269 | #ifdef RSEQ_COMPARE_TWICE | |
1270 | , error1, error2 | |
1271 | #endif | |
1272 | ); | |
1273 | return 0; | |
1274 | abort: | |
1275 | RSEQ_INJECT_FAILED | |
1276 | return -1; | |
1277 | cmpfail: | |
1278 | return 1; | |
1279 | #ifdef RSEQ_COMPARE_TWICE | |
1280 | error1: | |
1281 | rseq_bug("cpu_id comparison failed"); | |
1282 | error2: | |
1283 | rseq_bug("expected value comparison failed"); | |
1284 | #endif | |
1285 | } | |
1286 | ||
de28c254 MD |
1287 | /* |
1288 | * Dereference @p. Add voffp to the dereferenced pointer, and load its content | |
1289 | * into @load. | |
1290 | */ | |
1291 | static inline __attribute__((always_inline)) | |
1292 | int rseq_deref_loadoffp(intptr_t *p, off_t voffp, intptr_t *load, int cpu) | |
1293 | { | |
1294 | RSEQ_INJECT_C(9) | |
1295 | ||
1296 | __asm__ __volatile__ goto ( | |
1297 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
1298 | #ifdef RSEQ_COMPARE_TWICE | |
1299 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1300 | #endif | |
1301 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
1302 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) | |
1303 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) | |
1304 | RSEQ_INJECT_ASM(3) | |
1305 | "movl %[p], %%ebx\n\t" | |
1306 | RSEQ_INJECT_ASM(4) | |
1307 | #ifdef RSEQ_COMPARE_TWICE | |
1308 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) | |
1309 | #endif | |
1310 | "addl %[voffp], %%ebx\n\t" | |
1311 | "movl (%%ebx), %%ebx\n\t" | |
1312 | "movl %%ebx, %[load]\n\t" | |
1313 | "2:\n\t" | |
1314 | RSEQ_INJECT_ASM(5) | |
1315 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
1316 | : /* gcc asm goto does not allow outputs */ | |
1317 | : [cpu_id] "r" (cpu), | |
1318 | [rseq_abi] "r" (&__rseq_abi), | |
1319 | /* final store input */ | |
1320 | [p] "m" (*p), | |
1321 | [voffp] "ir" (voffp), | |
1322 | [load] "m" (*load) | |
1323 | : "memory", "cc", "eax", "ebx" | |
1324 | RSEQ_INJECT_CLOBBER | |
1325 | : abort | |
1326 | #ifdef RSEQ_COMPARE_TWICE | |
1327 | , error1 | |
1328 | #endif | |
1329 | ); | |
1330 | return 0; | |
1331 | abort: | |
1332 | RSEQ_INJECT_FAILED | |
1333 | return -1; | |
1334 | #ifdef RSEQ_COMPARE_TWICE | |
1335 | error1: | |
1336 | rseq_bug("cpu_id comparison failed"); | |
1337 | #endif | |
1338 | } | |
1339 | ||
784b0012 MD |
1340 | #endif /* !RSEQ_SKIP_FASTPATH */ |
1341 | ||
1342 | #endif |