Commit | Line | Data |
---|---|---|
744d0b8b | 1 | /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
784b0012 MD |
2 | /* |
3 | * rseq-x86.h | |
4 | * | |
5 | * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
6 | */ | |
7 | ||
8 | #include <stdint.h> | |
9 | ||
a5fe9e95 MD |
10 | /* |
11 | * RSEQ_SIG is used with the following reserved undefined instructions, which | |
12 | * trap in user-space: | |
13 | * | |
14 | * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi | |
15 | * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi | |
16 | */ | |
784b0012 MD |
17 | #define RSEQ_SIG 0x53053053 |
18 | ||
2d533093 | 19 | /* Offset of cpu_id and rseq_cs fields in struct rseq_abi. */ |
05ebd3f9 MD |
20 | #define RSEQ_CPU_ID_OFFSET 4 |
21 | #define RSEQ_CS_OFFSET 8 | |
22 | ||
784b0012 MD |
23 | #ifdef __x86_64__ |
24 | ||
7dc4bc29 MD |
25 | #define RSEQ_ASM_TP_SEGMENT %%fs |
26 | ||
784b0012 MD |
27 | #define rseq_smp_mb() \ |
28 | __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") | |
29 | #define rseq_smp_rmb() rseq_barrier() | |
30 | #define rseq_smp_wmb() rseq_barrier() | |
31 | ||
32 | #define rseq_smp_load_acquire(p) \ | |
33 | __extension__ ({ \ | |
34 | __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ | |
35 | rseq_barrier(); \ | |
36 | ____p1; \ | |
37 | }) | |
38 | ||
39 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
40 | ||
41 | #define rseq_smp_store_release(p, v) \ | |
42 | do { \ | |
43 | rseq_barrier(); \ | |
44 | RSEQ_WRITE_ONCE(*p, v); \ | |
45 | } while (0) | |
46 | ||
47 | #ifdef RSEQ_SKIP_FASTPATH | |
48 | #include "rseq-skip.h" | |
49 | #else /* !RSEQ_SKIP_FASTPATH */ | |
50 | ||
51 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
52 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 53 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
54 | ".balign 32\n\t" \ |
55 | __rseq_str(label) ":\n\t" \ | |
56 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
57 | ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
58 | ".popsection\n\t" \ |
59 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
60 | ".quad " __rseq_str(label) "b\n\t" \ | |
784b0012 MD |
61 | ".popsection\n\t" |
62 | ||
dd01d0fb | 63 | |
784b0012 MD |
64 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
65 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
66 | (post_commit_ip - start_ip), abort_ip) | |
67 | ||
90d9876e MD |
68 | /* |
69 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
70 | * of the critical section where a critical section can either branch to or |
71 | * reach through the normal course of its execution. The abort IP and the | |
72 | * post-commit IP are already part of the __rseq_cs section and should not be | |
73 | * explicitly defined as additional exit points. Knowing all exit points is | |
74 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
75 | */ |
76 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
77 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
78 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ | |
79 | ".popsection\n\t" | |
80 | ||
784b0012 MD |
81 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
82 | RSEQ_INJECT_ASM(1) \ | |
83 | "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ | |
05ebd3f9 | 84 | "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
85 | __rseq_str(label) ":\n\t" |
86 | ||
87 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ | |
88 | RSEQ_INJECT_ASM(2) \ | |
05ebd3f9 | 89 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
90 | "jnz " __rseq_str(label) "\n\t" |
91 | ||
92 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
93 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
94 | /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ |
95 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
96 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
97 | __rseq_str(label) ":\n\t" \ | |
98 | teardown \ | |
99 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
100 | ".popsection\n\t" | |
101 | ||
102 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
103 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
104 | __rseq_str(label) ":\n\t" \ | |
105 | teardown \ | |
106 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ | |
107 | ".popsection\n\t" | |
108 | ||
109 | static inline __attribute__((always_inline)) | |
110 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | |
111 | { | |
112 | RSEQ_INJECT_C(9) | |
113 | ||
114 | __asm__ __volatile__ goto ( | |
115 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
116 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
117 | #ifdef RSEQ_COMPARE_TWICE | |
118 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
119 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
120 | #endif | |
784b0012 | 121 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
7dc4bc29 MD |
122 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
123 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
124 | RSEQ_INJECT_ASM(3) |
125 | "cmpq %[v], %[expect]\n\t" | |
126 | "jnz %l[cmpfail]\n\t" | |
127 | RSEQ_INJECT_ASM(4) | |
128 | #ifdef RSEQ_COMPARE_TWICE | |
7dc4bc29 | 129 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
130 | "cmpq %[v], %[expect]\n\t" |
131 | "jnz %l[error2]\n\t" | |
132 | #endif | |
133 | /* final store */ | |
134 | "movq %[newv], %[v]\n\t" | |
135 | "2:\n\t" | |
136 | RSEQ_INJECT_ASM(5) | |
137 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
138 | : /* gcc asm goto does not allow outputs */ | |
139 | : [cpu_id] "r" (cpu), | |
170f840b | 140 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
141 | [v] "m" (*v), |
142 | [expect] "r" (expect), | |
143 | [newv] "r" (newv) | |
144 | : "memory", "cc", "rax" | |
145 | RSEQ_INJECT_CLOBBER | |
146 | : abort, cmpfail | |
147 | #ifdef RSEQ_COMPARE_TWICE | |
148 | , error1, error2 | |
149 | #endif | |
150 | ); | |
dd76f2d6 | 151 | rseq_after_asm_goto(); |
784b0012 MD |
152 | return 0; |
153 | abort: | |
dd76f2d6 | 154 | rseq_after_asm_goto(); |
784b0012 MD |
155 | RSEQ_INJECT_FAILED |
156 | return -1; | |
157 | cmpfail: | |
dd76f2d6 | 158 | rseq_after_asm_goto(); |
784b0012 MD |
159 | return 1; |
160 | #ifdef RSEQ_COMPARE_TWICE | |
161 | error1: | |
dd76f2d6 | 162 | rseq_after_asm_goto(); |
784b0012 MD |
163 | rseq_bug("cpu_id comparison failed"); |
164 | error2: | |
dd76f2d6 | 165 | rseq_after_asm_goto(); |
784b0012 MD |
166 | rseq_bug("expected value comparison failed"); |
167 | #endif | |
168 | } | |
169 | ||
170 | /* | |
171 | * Compare @v against @expectnot. When it does _not_ match, load @v | |
172 | * into @load, and store the content of *@v + voffp into @v. | |
173 | */ | |
174 | static inline __attribute__((always_inline)) | |
175 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | |
d35eae6b | 176 | long voffp, intptr_t *load, int cpu) |
784b0012 MD |
177 | { |
178 | RSEQ_INJECT_C(9) | |
179 | ||
180 | __asm__ __volatile__ goto ( | |
181 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
182 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
183 | #ifdef RSEQ_COMPARE_TWICE | |
184 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
185 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
186 | #endif | |
784b0012 | 187 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
7dc4bc29 MD |
188 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
189 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
190 | RSEQ_INJECT_ASM(3) |
191 | "movq %[v], %%rbx\n\t" | |
192 | "cmpq %%rbx, %[expectnot]\n\t" | |
193 | "je %l[cmpfail]\n\t" | |
194 | RSEQ_INJECT_ASM(4) | |
195 | #ifdef RSEQ_COMPARE_TWICE | |
7dc4bc29 | 196 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
197 | "movq %[v], %%rbx\n\t" |
198 | "cmpq %%rbx, %[expectnot]\n\t" | |
199 | "je %l[error2]\n\t" | |
200 | #endif | |
201 | "movq %%rbx, %[load]\n\t" | |
202 | "addq %[voffp], %%rbx\n\t" | |
203 | "movq (%%rbx), %%rbx\n\t" | |
204 | /* final store */ | |
205 | "movq %%rbx, %[v]\n\t" | |
206 | "2:\n\t" | |
207 | RSEQ_INJECT_ASM(5) | |
208 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
209 | : /* gcc asm goto does not allow outputs */ | |
210 | : [cpu_id] "r" (cpu), | |
170f840b | 211 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
212 | /* final store input */ |
213 | [v] "m" (*v), | |
214 | [expectnot] "r" (expectnot), | |
215 | [voffp] "er" (voffp), | |
216 | [load] "m" (*load) | |
217 | : "memory", "cc", "rax", "rbx" | |
218 | RSEQ_INJECT_CLOBBER | |
219 | : abort, cmpfail | |
220 | #ifdef RSEQ_COMPARE_TWICE | |
221 | , error1, error2 | |
222 | #endif | |
223 | ); | |
dd76f2d6 | 224 | rseq_after_asm_goto(); |
784b0012 MD |
225 | return 0; |
226 | abort: | |
dd76f2d6 | 227 | rseq_after_asm_goto(); |
784b0012 MD |
228 | RSEQ_INJECT_FAILED |
229 | return -1; | |
230 | cmpfail: | |
dd76f2d6 | 231 | rseq_after_asm_goto(); |
784b0012 MD |
232 | return 1; |
233 | #ifdef RSEQ_COMPARE_TWICE | |
234 | error1: | |
dd76f2d6 | 235 | rseq_after_asm_goto(); |
784b0012 MD |
236 | rseq_bug("cpu_id comparison failed"); |
237 | error2: | |
dd76f2d6 | 238 | rseq_after_asm_goto(); |
784b0012 MD |
239 | rseq_bug("expected value comparison failed"); |
240 | #endif | |
241 | } | |
242 | ||
243 | static inline __attribute__((always_inline)) | |
244 | int rseq_addv(intptr_t *v, intptr_t count, int cpu) | |
245 | { | |
246 | RSEQ_INJECT_C(9) | |
247 | ||
248 | __asm__ __volatile__ goto ( | |
249 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
250 | #ifdef RSEQ_COMPARE_TWICE |
251 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
252 | #endif | |
784b0012 | 253 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
7dc4bc29 MD |
254 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
255 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
256 | RSEQ_INJECT_ASM(3) |
257 | #ifdef RSEQ_COMPARE_TWICE | |
7dc4bc29 | 258 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
259 | #endif |
260 | /* final store */ | |
261 | "addq %[count], %[v]\n\t" | |
262 | "2:\n\t" | |
263 | RSEQ_INJECT_ASM(4) | |
264 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
265 | : /* gcc asm goto does not allow outputs */ | |
266 | : [cpu_id] "r" (cpu), | |
170f840b | 267 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
268 | /* final store input */ |
269 | [v] "m" (*v), | |
270 | [count] "er" (count) | |
271 | : "memory", "cc", "rax" | |
272 | RSEQ_INJECT_CLOBBER | |
273 | : abort | |
274 | #ifdef RSEQ_COMPARE_TWICE | |
275 | , error1 | |
276 | #endif | |
277 | ); | |
dd76f2d6 | 278 | rseq_after_asm_goto(); |
784b0012 MD |
279 | return 0; |
280 | abort: | |
dd76f2d6 | 281 | rseq_after_asm_goto(); |
784b0012 MD |
282 | RSEQ_INJECT_FAILED |
283 | return -1; | |
284 | #ifdef RSEQ_COMPARE_TWICE | |
285 | error1: | |
dd76f2d6 | 286 | rseq_after_asm_goto(); |
784b0012 MD |
287 | rseq_bug("cpu_id comparison failed"); |
288 | #endif | |
289 | } | |
290 | ||
7c02c61b MD |
291 | #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV |
292 | ||
293 | /* | |
294 | * pval = *(ptr+off) | |
295 | * *pval += inc; | |
296 | */ | |
297 | static inline __attribute__((always_inline)) | |
d35eae6b | 298 | int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu) |
7c02c61b MD |
299 | { |
300 | RSEQ_INJECT_C(9) | |
301 | ||
302 | __asm__ __volatile__ goto ( | |
303 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
304 | #ifdef RSEQ_COMPARE_TWICE | |
305 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
306 | #endif | |
307 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
7dc4bc29 MD |
308 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
309 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
7c02c61b MD |
310 | RSEQ_INJECT_ASM(3) |
311 | #ifdef RSEQ_COMPARE_TWICE | |
7dc4bc29 | 312 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
7c02c61b MD |
313 | #endif |
314 | /* get p+v */ | |
315 | "movq %[ptr], %%rbx\n\t" | |
316 | "addq %[off], %%rbx\n\t" | |
317 | /* get pv */ | |
318 | "movq (%%rbx), %%rcx\n\t" | |
319 | /* *pv += inc */ | |
320 | "addq %[inc], (%%rcx)\n\t" | |
321 | "2:\n\t" | |
322 | RSEQ_INJECT_ASM(4) | |
323 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
324 | : /* gcc asm goto does not allow outputs */ | |
325 | : [cpu_id] "r" (cpu), | |
170f840b | 326 | [rseq_offset] "r" (rseq_offset), |
7c02c61b MD |
327 | /* final store input */ |
328 | [ptr] "m" (*ptr), | |
329 | [off] "er" (off), | |
330 | [inc] "er" (inc) | |
331 | : "memory", "cc", "rax", "rbx", "rcx" | |
332 | RSEQ_INJECT_CLOBBER | |
333 | : abort | |
334 | #ifdef RSEQ_COMPARE_TWICE | |
335 | , error1 | |
336 | #endif | |
337 | ); | |
338 | rseq_after_asm_goto(); | |
339 | return 0; | |
340 | abort: | |
341 | rseq_after_asm_goto(); | |
342 | RSEQ_INJECT_FAILED | |
343 | return -1; | |
344 | #ifdef RSEQ_COMPARE_TWICE | |
345 | error1: | |
346 | rseq_after_asm_goto(); | |
347 | rseq_bug("cpu_id comparison failed"); | |
348 | #endif | |
349 | } | |
350 | ||
784b0012 MD |
351 | static inline __attribute__((always_inline)) |
352 | int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | |
353 | intptr_t *v2, intptr_t newv2, | |
354 | intptr_t newv, int cpu) | |
355 | { | |
356 | RSEQ_INJECT_C(9) | |
357 | ||
358 | __asm__ __volatile__ goto ( | |
359 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
360 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
361 | #ifdef RSEQ_COMPARE_TWICE | |
362 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
363 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
364 | #endif | |
784b0012 | 365 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
7dc4bc29 MD |
366 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
367 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
368 | RSEQ_INJECT_ASM(3) |
369 | "cmpq %[v], %[expect]\n\t" | |
370 | "jnz %l[cmpfail]\n\t" | |
371 | RSEQ_INJECT_ASM(4) | |
372 | #ifdef RSEQ_COMPARE_TWICE | |
7dc4bc29 | 373 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
374 | "cmpq %[v], %[expect]\n\t" |
375 | "jnz %l[error2]\n\t" | |
376 | #endif | |
377 | /* try store */ | |
378 | "movq %[newv2], %[v2]\n\t" | |
379 | RSEQ_INJECT_ASM(5) | |
380 | /* final store */ | |
381 | "movq %[newv], %[v]\n\t" | |
382 | "2:\n\t" | |
383 | RSEQ_INJECT_ASM(6) | |
384 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
385 | : /* gcc asm goto does not allow outputs */ | |
386 | : [cpu_id] "r" (cpu), | |
170f840b | 387 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
388 | /* try store input */ |
389 | [v2] "m" (*v2), | |
390 | [newv2] "r" (newv2), | |
391 | /* final store input */ | |
392 | [v] "m" (*v), | |
393 | [expect] "r" (expect), | |
394 | [newv] "r" (newv) | |
395 | : "memory", "cc", "rax" | |
396 | RSEQ_INJECT_CLOBBER | |
397 | : abort, cmpfail | |
398 | #ifdef RSEQ_COMPARE_TWICE | |
399 | , error1, error2 | |
400 | #endif | |
401 | ); | |
dd76f2d6 | 402 | rseq_after_asm_goto(); |
784b0012 MD |
403 | return 0; |
404 | abort: | |
dd76f2d6 | 405 | rseq_after_asm_goto(); |
784b0012 MD |
406 | RSEQ_INJECT_FAILED |
407 | return -1; | |
408 | cmpfail: | |
dd76f2d6 | 409 | rseq_after_asm_goto(); |
784b0012 MD |
410 | return 1; |
411 | #ifdef RSEQ_COMPARE_TWICE | |
412 | error1: | |
dd76f2d6 | 413 | rseq_after_asm_goto(); |
784b0012 MD |
414 | rseq_bug("cpu_id comparison failed"); |
415 | error2: | |
dd76f2d6 | 416 | rseq_after_asm_goto(); |
784b0012 MD |
417 | rseq_bug("expected value comparison failed"); |
418 | #endif | |
419 | } | |
420 | ||
421 | /* x86-64 is TSO. */ | |
422 | static inline __attribute__((always_inline)) | |
423 | int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | |
424 | intptr_t *v2, intptr_t newv2, | |
425 | intptr_t newv, int cpu) | |
426 | { | |
427 | return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); | |
428 | } | |
429 | ||
430 | static inline __attribute__((always_inline)) | |
431 | int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | |
432 | intptr_t *v2, intptr_t expect2, | |
433 | intptr_t newv, int cpu) | |
434 | { | |
435 | RSEQ_INJECT_C(9) | |
436 | ||
437 | __asm__ __volatile__ goto ( | |
438 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
439 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
440 | #ifdef RSEQ_COMPARE_TWICE | |
441 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
442 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
443 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | |
444 | #endif | |
784b0012 | 445 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
7dc4bc29 MD |
446 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
447 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
448 | RSEQ_INJECT_ASM(3) |
449 | "cmpq %[v], %[expect]\n\t" | |
450 | "jnz %l[cmpfail]\n\t" | |
451 | RSEQ_INJECT_ASM(4) | |
452 | "cmpq %[v2], %[expect2]\n\t" | |
453 | "jnz %l[cmpfail]\n\t" | |
454 | RSEQ_INJECT_ASM(5) | |
455 | #ifdef RSEQ_COMPARE_TWICE | |
7dc4bc29 | 456 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
457 | "cmpq %[v], %[expect]\n\t" |
458 | "jnz %l[error2]\n\t" | |
459 | "cmpq %[v2], %[expect2]\n\t" | |
460 | "jnz %l[error3]\n\t" | |
461 | #endif | |
462 | /* final store */ | |
463 | "movq %[newv], %[v]\n\t" | |
464 | "2:\n\t" | |
465 | RSEQ_INJECT_ASM(6) | |
466 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
467 | : /* gcc asm goto does not allow outputs */ | |
468 | : [cpu_id] "r" (cpu), | |
170f840b | 469 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
470 | /* cmp2 input */ |
471 | [v2] "m" (*v2), | |
472 | [expect2] "r" (expect2), | |
473 | /* final store input */ | |
474 | [v] "m" (*v), | |
475 | [expect] "r" (expect), | |
476 | [newv] "r" (newv) | |
477 | : "memory", "cc", "rax" | |
478 | RSEQ_INJECT_CLOBBER | |
479 | : abort, cmpfail | |
480 | #ifdef RSEQ_COMPARE_TWICE | |
481 | , error1, error2, error3 | |
482 | #endif | |
483 | ); | |
dd76f2d6 | 484 | rseq_after_asm_goto(); |
784b0012 MD |
485 | return 0; |
486 | abort: | |
dd76f2d6 | 487 | rseq_after_asm_goto(); |
784b0012 MD |
488 | RSEQ_INJECT_FAILED |
489 | return -1; | |
490 | cmpfail: | |
dd76f2d6 | 491 | rseq_after_asm_goto(); |
784b0012 MD |
492 | return 1; |
493 | #ifdef RSEQ_COMPARE_TWICE | |
494 | error1: | |
dd76f2d6 | 495 | rseq_after_asm_goto(); |
784b0012 MD |
496 | rseq_bug("cpu_id comparison failed"); |
497 | error2: | |
dd76f2d6 | 498 | rseq_after_asm_goto(); |
784b0012 MD |
499 | rseq_bug("1st expected value comparison failed"); |
500 | error3: | |
dd76f2d6 | 501 | rseq_after_asm_goto(); |
784b0012 MD |
502 | rseq_bug("2nd expected value comparison failed"); |
503 | #endif | |
504 | } | |
505 | ||
506 | static inline __attribute__((always_inline)) | |
507 | int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | |
508 | void *dst, void *src, size_t len, | |
509 | intptr_t newv, int cpu) | |
510 | { | |
511 | uint64_t rseq_scratch[3]; | |
512 | ||
513 | RSEQ_INJECT_C(9) | |
514 | ||
515 | __asm__ __volatile__ goto ( | |
516 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
517 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
518 | #ifdef RSEQ_COMPARE_TWICE | |
519 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
520 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
521 | #endif | |
784b0012 MD |
522 | "movq %[src], %[rseq_scratch0]\n\t" |
523 | "movq %[dst], %[rseq_scratch1]\n\t" | |
524 | "movq %[len], %[rseq_scratch2]\n\t" | |
525 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
7dc4bc29 MD |
526 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
527 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
528 | RSEQ_INJECT_ASM(3) |
529 | "cmpq %[v], %[expect]\n\t" | |
530 | "jnz 5f\n\t" | |
531 | RSEQ_INJECT_ASM(4) | |
532 | #ifdef RSEQ_COMPARE_TWICE | |
7dc4bc29 | 533 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) |
784b0012 MD |
534 | "cmpq %[v], %[expect]\n\t" |
535 | "jnz 7f\n\t" | |
536 | #endif | |
537 | /* try memcpy */ | |
538 | "test %[len], %[len]\n\t" \ | |
539 | "jz 333f\n\t" \ | |
540 | "222:\n\t" \ | |
541 | "movb (%[src]), %%al\n\t" \ | |
542 | "movb %%al, (%[dst])\n\t" \ | |
543 | "inc %[src]\n\t" \ | |
544 | "inc %[dst]\n\t" \ | |
545 | "dec %[len]\n\t" \ | |
546 | "jnz 222b\n\t" \ | |
547 | "333:\n\t" \ | |
548 | RSEQ_INJECT_ASM(5) | |
549 | /* final store */ | |
550 | "movq %[newv], %[v]\n\t" | |
551 | "2:\n\t" | |
552 | RSEQ_INJECT_ASM(6) | |
553 | /* teardown */ | |
554 | "movq %[rseq_scratch2], %[len]\n\t" | |
555 | "movq %[rseq_scratch1], %[dst]\n\t" | |
556 | "movq %[rseq_scratch0], %[src]\n\t" | |
557 | RSEQ_ASM_DEFINE_ABORT(4, | |
558 | "movq %[rseq_scratch2], %[len]\n\t" | |
559 | "movq %[rseq_scratch1], %[dst]\n\t" | |
560 | "movq %[rseq_scratch0], %[src]\n\t", | |
561 | abort) | |
562 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
563 | "movq %[rseq_scratch2], %[len]\n\t" | |
564 | "movq %[rseq_scratch1], %[dst]\n\t" | |
565 | "movq %[rseq_scratch0], %[src]\n\t", | |
566 | cmpfail) | |
567 | #ifdef RSEQ_COMPARE_TWICE | |
568 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
569 | "movq %[rseq_scratch2], %[len]\n\t" | |
570 | "movq %[rseq_scratch1], %[dst]\n\t" | |
571 | "movq %[rseq_scratch0], %[src]\n\t", | |
572 | error1) | |
573 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
574 | "movq %[rseq_scratch2], %[len]\n\t" | |
575 | "movq %[rseq_scratch1], %[dst]\n\t" | |
576 | "movq %[rseq_scratch0], %[src]\n\t", | |
577 | error2) | |
578 | #endif | |
579 | : /* gcc asm goto does not allow outputs */ | |
580 | : [cpu_id] "r" (cpu), | |
170f840b | 581 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
582 | /* final store input */ |
583 | [v] "m" (*v), | |
584 | [expect] "r" (expect), | |
585 | [newv] "r" (newv), | |
586 | /* try memcpy input */ | |
587 | [dst] "r" (dst), | |
588 | [src] "r" (src), | |
589 | [len] "r" (len), | |
590 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
591 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
592 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
593 | : "memory", "cc", "rax" | |
594 | RSEQ_INJECT_CLOBBER | |
595 | : abort, cmpfail | |
596 | #ifdef RSEQ_COMPARE_TWICE | |
597 | , error1, error2 | |
598 | #endif | |
599 | ); | |
dd76f2d6 | 600 | rseq_after_asm_goto(); |
784b0012 MD |
601 | return 0; |
602 | abort: | |
dd76f2d6 | 603 | rseq_after_asm_goto(); |
784b0012 MD |
604 | RSEQ_INJECT_FAILED |
605 | return -1; | |
606 | cmpfail: | |
dd76f2d6 | 607 | rseq_after_asm_goto(); |
784b0012 MD |
608 | return 1; |
609 | #ifdef RSEQ_COMPARE_TWICE | |
610 | error1: | |
dd76f2d6 | 611 | rseq_after_asm_goto(); |
784b0012 MD |
612 | rseq_bug("cpu_id comparison failed"); |
613 | error2: | |
dd76f2d6 | 614 | rseq_after_asm_goto(); |
784b0012 MD |
615 | rseq_bug("expected value comparison failed"); |
616 | #endif | |
617 | } | |
618 | ||
619 | /* x86-64 is TSO. */ | |
620 | static inline __attribute__((always_inline)) | |
621 | int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | |
622 | void *dst, void *src, size_t len, | |
623 | intptr_t newv, int cpu) | |
624 | { | |
625 | return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, | |
626 | newv, cpu); | |
627 | } | |
628 | ||
de28c254 MD |
629 | /* |
630 | * Dereference @p. Add voffp to the dereferenced pointer, and load its content | |
631 | * into @load. | |
632 | */ | |
633 | static inline __attribute__((always_inline)) | |
d35eae6b | 634 | int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu) |
de28c254 MD |
635 | { |
636 | RSEQ_INJECT_C(9) | |
637 | ||
638 | __asm__ __volatile__ goto ( | |
639 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
640 | #ifdef RSEQ_COMPARE_TWICE | |
641 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
642 | #endif | |
643 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
7dc4bc29 MD |
644 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
645 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
de28c254 MD |
646 | RSEQ_INJECT_ASM(3) |
647 | "movq %[p], %%rbx\n\t" | |
648 | RSEQ_INJECT_ASM(4) | |
649 | #ifdef RSEQ_COMPARE_TWICE | |
7dc4bc29 | 650 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
de28c254 MD |
651 | #endif |
652 | "addq %[voffp], %%rbx\n\t" | |
653 | "movq (%%rbx), %%rbx\n\t" | |
654 | "movq %%rbx, %[load]\n\t" | |
655 | "2:\n\t" | |
656 | RSEQ_INJECT_ASM(5) | |
657 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
658 | : /* gcc asm goto does not allow outputs */ | |
659 | : [cpu_id] "r" (cpu), | |
170f840b | 660 | [rseq_offset] "r" (rseq_offset), |
de28c254 MD |
661 | /* final store input */ |
662 | [p] "m" (*p), | |
663 | [voffp] "er" (voffp), | |
664 | [load] "m" (*load) | |
665 | : "memory", "cc", "rax", "rbx" | |
666 | RSEQ_INJECT_CLOBBER | |
667 | : abort | |
668 | #ifdef RSEQ_COMPARE_TWICE | |
669 | , error1 | |
670 | #endif | |
671 | ); | |
dd76f2d6 | 672 | rseq_after_asm_goto(); |
de28c254 MD |
673 | return 0; |
674 | abort: | |
dd76f2d6 | 675 | rseq_after_asm_goto(); |
de28c254 MD |
676 | RSEQ_INJECT_FAILED |
677 | return -1; | |
678 | #ifdef RSEQ_COMPARE_TWICE | |
679 | error1: | |
dd76f2d6 | 680 | rseq_after_asm_goto(); |
de28c254 MD |
681 | rseq_bug("cpu_id comparison failed"); |
682 | #endif | |
683 | } | |
684 | ||
784b0012 MD |
685 | #endif /* !RSEQ_SKIP_FASTPATH */ |
686 | ||
f1c6b55b | 687 | #elif defined(__i386__) |
784b0012 | 688 | |
53831176 MD |
689 | #define RSEQ_ASM_TP_SEGMENT %%gs |
690 | ||
784b0012 MD |
691 | #define rseq_smp_mb() \ |
692 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
693 | #define rseq_smp_rmb() \ | |
694 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
695 | #define rseq_smp_wmb() \ | |
696 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
697 | ||
698 | #define rseq_smp_load_acquire(p) \ | |
699 | __extension__ ({ \ | |
700 | __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ | |
701 | rseq_smp_mb(); \ | |
702 | ____p1; \ | |
703 | }) | |
704 | ||
705 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
706 | ||
707 | #define rseq_smp_store_release(p, v) \ | |
708 | do { \ | |
709 | rseq_smp_mb(); \ | |
710 | RSEQ_WRITE_ONCE(*p, v); \ | |
711 | } while (0) | |
712 | ||
713 | #ifdef RSEQ_SKIP_FASTPATH | |
714 | #include "rseq-skip.h" | |
715 | #else /* !RSEQ_SKIP_FASTPATH */ | |
716 | ||
717 | /* | |
718 | * Use eax as scratch register and take memory operands as input to | |
719 | * lessen register pressure. Especially needed when compiling in O0. | |
720 | */ | |
721 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
722 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 723 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
724 | ".balign 32\n\t" \ |
725 | __rseq_str(label) ":\n\t" \ | |
726 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
727 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ | |
dd01d0fb MD |
728 | ".popsection\n\t" \ |
729 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
730 | ".long " __rseq_str(label) "b, 0x0\n\t" \ | |
784b0012 MD |
731 | ".popsection\n\t" |
732 | ||
733 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ | |
734 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
735 | (post_commit_ip - start_ip), abort_ip) | |
736 | ||
90d9876e MD |
737 | /* |
738 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
739 | * of the critical section where a critical section can either branch to or |
740 | * reach through the normal course of its execution. The abort IP and the | |
741 | * post-commit IP are already part of the __rseq_cs section and should not be | |
742 | * explicitly defined as additional exit points. Knowing all exit points is | |
743 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
744 | */ |
745 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
746 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
747 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ | |
748 | ".popsection\n\t" | |
749 | ||
784b0012 MD |
750 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
751 | RSEQ_INJECT_ASM(1) \ | |
05ebd3f9 | 752 | "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
753 | __rseq_str(label) ":\n\t" |
754 | ||
755 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ | |
756 | RSEQ_INJECT_ASM(2) \ | |
05ebd3f9 | 757 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
758 | "jnz " __rseq_str(label) "\n\t" |
759 | ||
760 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
761 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
762 | /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ |
763 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
764 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
765 | __rseq_str(label) ":\n\t" \ | |
766 | teardown \ | |
767 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
768 | ".popsection\n\t" | |
769 | ||
770 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
771 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
772 | __rseq_str(label) ":\n\t" \ | |
773 | teardown \ | |
774 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ | |
775 | ".popsection\n\t" | |
776 | ||
777 | static inline __attribute__((always_inline)) | |
778 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | |
779 | { | |
780 | RSEQ_INJECT_C(9) | |
781 | ||
782 | __asm__ __volatile__ goto ( | |
783 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
784 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
785 | #ifdef RSEQ_COMPARE_TWICE | |
786 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
787 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
788 | #endif | |
784b0012 | 789 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
53831176 MD |
790 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
791 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
792 | RSEQ_INJECT_ASM(3) |
793 | "cmpl %[v], %[expect]\n\t" | |
794 | "jnz %l[cmpfail]\n\t" | |
795 | RSEQ_INJECT_ASM(4) | |
796 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 797 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
798 | "cmpl %[v], %[expect]\n\t" |
799 | "jnz %l[error2]\n\t" | |
800 | #endif | |
801 | /* final store */ | |
802 | "movl %[newv], %[v]\n\t" | |
803 | "2:\n\t" | |
804 | RSEQ_INJECT_ASM(5) | |
805 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
806 | : /* gcc asm goto does not allow outputs */ | |
807 | : [cpu_id] "r" (cpu), | |
53831176 | 808 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
809 | [v] "m" (*v), |
810 | [expect] "r" (expect), | |
811 | [newv] "r" (newv) | |
812 | : "memory", "cc", "eax" | |
813 | RSEQ_INJECT_CLOBBER | |
814 | : abort, cmpfail | |
815 | #ifdef RSEQ_COMPARE_TWICE | |
816 | , error1, error2 | |
817 | #endif | |
818 | ); | |
dd76f2d6 | 819 | rseq_after_asm_goto(); |
784b0012 MD |
820 | return 0; |
821 | abort: | |
dd76f2d6 | 822 | rseq_after_asm_goto(); |
784b0012 MD |
823 | RSEQ_INJECT_FAILED |
824 | return -1; | |
825 | cmpfail: | |
dd76f2d6 | 826 | rseq_after_asm_goto(); |
784b0012 MD |
827 | return 1; |
828 | #ifdef RSEQ_COMPARE_TWICE | |
829 | error1: | |
dd76f2d6 | 830 | rseq_after_asm_goto(); |
784b0012 MD |
831 | rseq_bug("cpu_id comparison failed"); |
832 | error2: | |
dd76f2d6 | 833 | rseq_after_asm_goto(); |
784b0012 MD |
834 | rseq_bug("expected value comparison failed"); |
835 | #endif | |
836 | } | |
837 | ||
838 | /* | |
839 | * Compare @v against @expectnot. When it does _not_ match, load @v | |
840 | * into @load, and store the content of *@v + voffp into @v. | |
841 | */ | |
842 | static inline __attribute__((always_inline)) | |
843 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | |
d35eae6b | 844 | long voffp, intptr_t *load, int cpu) |
784b0012 MD |
845 | { |
846 | RSEQ_INJECT_C(9) | |
847 | ||
848 | __asm__ __volatile__ goto ( | |
849 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
850 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
851 | #ifdef RSEQ_COMPARE_TWICE | |
852 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
853 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
854 | #endif | |
784b0012 | 855 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
53831176 MD |
856 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
857 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
858 | RSEQ_INJECT_ASM(3) |
859 | "movl %[v], %%ebx\n\t" | |
860 | "cmpl %%ebx, %[expectnot]\n\t" | |
861 | "je %l[cmpfail]\n\t" | |
862 | RSEQ_INJECT_ASM(4) | |
863 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 864 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
865 | "movl %[v], %%ebx\n\t" |
866 | "cmpl %%ebx, %[expectnot]\n\t" | |
867 | "je %l[error2]\n\t" | |
868 | #endif | |
869 | "movl %%ebx, %[load]\n\t" | |
870 | "addl %[voffp], %%ebx\n\t" | |
871 | "movl (%%ebx), %%ebx\n\t" | |
872 | /* final store */ | |
873 | "movl %%ebx, %[v]\n\t" | |
874 | "2:\n\t" | |
875 | RSEQ_INJECT_ASM(5) | |
876 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
877 | : /* gcc asm goto does not allow outputs */ | |
878 | : [cpu_id] "r" (cpu), | |
53831176 | 879 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
880 | /* final store input */ |
881 | [v] "m" (*v), | |
882 | [expectnot] "r" (expectnot), | |
883 | [voffp] "ir" (voffp), | |
884 | [load] "m" (*load) | |
885 | : "memory", "cc", "eax", "ebx" | |
886 | RSEQ_INJECT_CLOBBER | |
887 | : abort, cmpfail | |
888 | #ifdef RSEQ_COMPARE_TWICE | |
889 | , error1, error2 | |
890 | #endif | |
891 | ); | |
dd76f2d6 | 892 | rseq_after_asm_goto(); |
784b0012 MD |
893 | return 0; |
894 | abort: | |
dd76f2d6 | 895 | rseq_after_asm_goto(); |
784b0012 MD |
896 | RSEQ_INJECT_FAILED |
897 | return -1; | |
898 | cmpfail: | |
dd76f2d6 | 899 | rseq_after_asm_goto(); |
784b0012 MD |
900 | return 1; |
901 | #ifdef RSEQ_COMPARE_TWICE | |
902 | error1: | |
dd76f2d6 | 903 | rseq_after_asm_goto(); |
784b0012 MD |
904 | rseq_bug("cpu_id comparison failed"); |
905 | error2: | |
dd76f2d6 | 906 | rseq_after_asm_goto(); |
784b0012 MD |
907 | rseq_bug("expected value comparison failed"); |
908 | #endif | |
909 | } | |
910 | ||
911 | static inline __attribute__((always_inline)) | |
912 | int rseq_addv(intptr_t *v, intptr_t count, int cpu) | |
913 | { | |
914 | RSEQ_INJECT_C(9) | |
915 | ||
916 | __asm__ __volatile__ goto ( | |
917 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
918 | #ifdef RSEQ_COMPARE_TWICE |
919 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
920 | #endif | |
784b0012 | 921 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
53831176 MD |
922 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
923 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
924 | RSEQ_INJECT_ASM(3) |
925 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 926 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
927 | #endif |
928 | /* final store */ | |
929 | "addl %[count], %[v]\n\t" | |
930 | "2:\n\t" | |
931 | RSEQ_INJECT_ASM(4) | |
932 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
933 | : /* gcc asm goto does not allow outputs */ | |
934 | : [cpu_id] "r" (cpu), | |
53831176 | 935 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
936 | /* final store input */ |
937 | [v] "m" (*v), | |
938 | [count] "ir" (count) | |
939 | : "memory", "cc", "eax" | |
940 | RSEQ_INJECT_CLOBBER | |
941 | : abort | |
942 | #ifdef RSEQ_COMPARE_TWICE | |
943 | , error1 | |
944 | #endif | |
945 | ); | |
dd76f2d6 | 946 | rseq_after_asm_goto(); |
784b0012 MD |
947 | return 0; |
948 | abort: | |
dd76f2d6 | 949 | rseq_after_asm_goto(); |
784b0012 MD |
950 | RSEQ_INJECT_FAILED |
951 | return -1; | |
952 | #ifdef RSEQ_COMPARE_TWICE | |
953 | error1: | |
dd76f2d6 | 954 | rseq_after_asm_goto(); |
784b0012 MD |
955 | rseq_bug("cpu_id comparison failed"); |
956 | #endif | |
957 | } | |
958 | ||
959 | static inline __attribute__((always_inline)) | |
960 | int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | |
961 | intptr_t *v2, intptr_t newv2, | |
962 | intptr_t newv, int cpu) | |
963 | { | |
964 | RSEQ_INJECT_C(9) | |
965 | ||
966 | __asm__ __volatile__ goto ( | |
967 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
968 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
969 | #ifdef RSEQ_COMPARE_TWICE | |
970 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
971 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
972 | #endif | |
784b0012 | 973 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
53831176 MD |
974 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
975 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
976 | RSEQ_INJECT_ASM(3) |
977 | "cmpl %[v], %[expect]\n\t" | |
978 | "jnz %l[cmpfail]\n\t" | |
979 | RSEQ_INJECT_ASM(4) | |
980 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 981 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
982 | "cmpl %[v], %[expect]\n\t" |
983 | "jnz %l[error2]\n\t" | |
984 | #endif | |
985 | /* try store */ | |
986 | "movl %[newv2], %%eax\n\t" | |
987 | "movl %%eax, %[v2]\n\t" | |
988 | RSEQ_INJECT_ASM(5) | |
989 | /* final store */ | |
990 | "movl %[newv], %[v]\n\t" | |
991 | "2:\n\t" | |
992 | RSEQ_INJECT_ASM(6) | |
993 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
994 | : /* gcc asm goto does not allow outputs */ | |
995 | : [cpu_id] "r" (cpu), | |
53831176 | 996 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
997 | /* try store input */ |
998 | [v2] "m" (*v2), | |
999 | [newv2] "m" (newv2), | |
1000 | /* final store input */ | |
1001 | [v] "m" (*v), | |
1002 | [expect] "r" (expect), | |
1003 | [newv] "r" (newv) | |
1004 | : "memory", "cc", "eax" | |
1005 | RSEQ_INJECT_CLOBBER | |
1006 | : abort, cmpfail | |
1007 | #ifdef RSEQ_COMPARE_TWICE | |
1008 | , error1, error2 | |
1009 | #endif | |
1010 | ); | |
dd76f2d6 | 1011 | rseq_after_asm_goto(); |
784b0012 MD |
1012 | return 0; |
1013 | abort: | |
dd76f2d6 | 1014 | rseq_after_asm_goto(); |
784b0012 MD |
1015 | RSEQ_INJECT_FAILED |
1016 | return -1; | |
1017 | cmpfail: | |
dd76f2d6 | 1018 | rseq_after_asm_goto(); |
784b0012 MD |
1019 | return 1; |
1020 | #ifdef RSEQ_COMPARE_TWICE | |
1021 | error1: | |
dd76f2d6 | 1022 | rseq_after_asm_goto(); |
784b0012 MD |
1023 | rseq_bug("cpu_id comparison failed"); |
1024 | error2: | |
dd76f2d6 | 1025 | rseq_after_asm_goto(); |
784b0012 MD |
1026 | rseq_bug("expected value comparison failed"); |
1027 | #endif | |
1028 | } | |
1029 | ||
1030 | static inline __attribute__((always_inline)) | |
1031 | int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | |
1032 | intptr_t *v2, intptr_t newv2, | |
1033 | intptr_t newv, int cpu) | |
1034 | { | |
1035 | RSEQ_INJECT_C(9) | |
1036 | ||
1037 | __asm__ __volatile__ goto ( | |
1038 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1039 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1040 | #ifdef RSEQ_COMPARE_TWICE | |
1041 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1042 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1043 | #endif | |
784b0012 | 1044 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
53831176 MD |
1045 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
1046 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
1047 | RSEQ_INJECT_ASM(3) |
1048 | "movl %[expect], %%eax\n\t" | |
1049 | "cmpl %[v], %%eax\n\t" | |
1050 | "jnz %l[cmpfail]\n\t" | |
1051 | RSEQ_INJECT_ASM(4) | |
1052 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 1053 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
1054 | "movl %[expect], %%eax\n\t" |
1055 | "cmpl %[v], %%eax\n\t" | |
1056 | "jnz %l[error2]\n\t" | |
1057 | #endif | |
1058 | /* try store */ | |
1059 | "movl %[newv2], %[v2]\n\t" | |
1060 | RSEQ_INJECT_ASM(5) | |
1061 | "lock; addl $0,-128(%%esp)\n\t" | |
1062 | /* final store */ | |
1063 | "movl %[newv], %[v]\n\t" | |
1064 | "2:\n\t" | |
1065 | RSEQ_INJECT_ASM(6) | |
1066 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
1067 | : /* gcc asm goto does not allow outputs */ | |
1068 | : [cpu_id] "r" (cpu), | |
53831176 | 1069 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
1070 | /* try store input */ |
1071 | [v2] "m" (*v2), | |
1072 | [newv2] "r" (newv2), | |
1073 | /* final store input */ | |
1074 | [v] "m" (*v), | |
1075 | [expect] "m" (expect), | |
1076 | [newv] "r" (newv) | |
1077 | : "memory", "cc", "eax" | |
1078 | RSEQ_INJECT_CLOBBER | |
1079 | : abort, cmpfail | |
1080 | #ifdef RSEQ_COMPARE_TWICE | |
1081 | , error1, error2 | |
1082 | #endif | |
1083 | ); | |
dd76f2d6 | 1084 | rseq_after_asm_goto(); |
784b0012 MD |
1085 | return 0; |
1086 | abort: | |
dd76f2d6 | 1087 | rseq_after_asm_goto(); |
784b0012 MD |
1088 | RSEQ_INJECT_FAILED |
1089 | return -1; | |
1090 | cmpfail: | |
dd76f2d6 | 1091 | rseq_after_asm_goto(); |
784b0012 MD |
1092 | return 1; |
1093 | #ifdef RSEQ_COMPARE_TWICE | |
1094 | error1: | |
dd76f2d6 | 1095 | rseq_after_asm_goto(); |
784b0012 MD |
1096 | rseq_bug("cpu_id comparison failed"); |
1097 | error2: | |
dd76f2d6 | 1098 | rseq_after_asm_goto(); |
784b0012 MD |
1099 | rseq_bug("expected value comparison failed"); |
1100 | #endif | |
1101 | ||
1102 | } | |
1103 | ||
1104 | static inline __attribute__((always_inline)) | |
1105 | int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | |
1106 | intptr_t *v2, intptr_t expect2, | |
1107 | intptr_t newv, int cpu) | |
1108 | { | |
1109 | RSEQ_INJECT_C(9) | |
1110 | ||
1111 | __asm__ __volatile__ goto ( | |
1112 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1113 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1114 | #ifdef RSEQ_COMPARE_TWICE | |
1115 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1116 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1117 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | |
1118 | #endif | |
784b0012 | 1119 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
53831176 MD |
1120 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
1121 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
1122 | RSEQ_INJECT_ASM(3) |
1123 | "cmpl %[v], %[expect]\n\t" | |
1124 | "jnz %l[cmpfail]\n\t" | |
1125 | RSEQ_INJECT_ASM(4) | |
1126 | "cmpl %[expect2], %[v2]\n\t" | |
1127 | "jnz %l[cmpfail]\n\t" | |
1128 | RSEQ_INJECT_ASM(5) | |
1129 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 1130 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
784b0012 MD |
1131 | "cmpl %[v], %[expect]\n\t" |
1132 | "jnz %l[error2]\n\t" | |
1133 | "cmpl %[expect2], %[v2]\n\t" | |
1134 | "jnz %l[error3]\n\t" | |
1135 | #endif | |
1136 | "movl %[newv], %%eax\n\t" | |
1137 | /* final store */ | |
1138 | "movl %%eax, %[v]\n\t" | |
1139 | "2:\n\t" | |
1140 | RSEQ_INJECT_ASM(6) | |
1141 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
1142 | : /* gcc asm goto does not allow outputs */ | |
1143 | : [cpu_id] "r" (cpu), | |
53831176 | 1144 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
1145 | /* cmp2 input */ |
1146 | [v2] "m" (*v2), | |
1147 | [expect2] "r" (expect2), | |
1148 | /* final store input */ | |
1149 | [v] "m" (*v), | |
1150 | [expect] "r" (expect), | |
1151 | [newv] "m" (newv) | |
1152 | : "memory", "cc", "eax" | |
1153 | RSEQ_INJECT_CLOBBER | |
1154 | : abort, cmpfail | |
1155 | #ifdef RSEQ_COMPARE_TWICE | |
1156 | , error1, error2, error3 | |
1157 | #endif | |
1158 | ); | |
dd76f2d6 | 1159 | rseq_after_asm_goto(); |
784b0012 MD |
1160 | return 0; |
1161 | abort: | |
dd76f2d6 | 1162 | rseq_after_asm_goto(); |
784b0012 MD |
1163 | RSEQ_INJECT_FAILED |
1164 | return -1; | |
1165 | cmpfail: | |
dd76f2d6 | 1166 | rseq_after_asm_goto(); |
784b0012 MD |
1167 | return 1; |
1168 | #ifdef RSEQ_COMPARE_TWICE | |
1169 | error1: | |
dd76f2d6 | 1170 | rseq_after_asm_goto(); |
784b0012 MD |
1171 | rseq_bug("cpu_id comparison failed"); |
1172 | error2: | |
dd76f2d6 | 1173 | rseq_after_asm_goto(); |
784b0012 MD |
1174 | rseq_bug("1st expected value comparison failed"); |
1175 | error3: | |
dd76f2d6 | 1176 | rseq_after_asm_goto(); |
784b0012 MD |
1177 | rseq_bug("2nd expected value comparison failed"); |
1178 | #endif | |
1179 | } | |
1180 | ||
1181 | /* TODO: implement a faster memcpy. */ | |
1182 | static inline __attribute__((always_inline)) | |
1183 | int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | |
1184 | void *dst, void *src, size_t len, | |
1185 | intptr_t newv, int cpu) | |
1186 | { | |
1187 | uint32_t rseq_scratch[3]; | |
1188 | ||
1189 | RSEQ_INJECT_C(9) | |
1190 | ||
1191 | __asm__ __volatile__ goto ( | |
1192 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1193 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1194 | #ifdef RSEQ_COMPARE_TWICE | |
1195 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1196 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1197 | #endif | |
784b0012 MD |
1198 | "movl %[src], %[rseq_scratch0]\n\t" |
1199 | "movl %[dst], %[rseq_scratch1]\n\t" | |
1200 | "movl %[len], %[rseq_scratch2]\n\t" | |
1201 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
53831176 MD |
1202 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
1203 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
1204 | RSEQ_INJECT_ASM(3) |
1205 | "movl %[expect], %%eax\n\t" | |
1206 | "cmpl %%eax, %[v]\n\t" | |
1207 | "jnz 5f\n\t" | |
1208 | RSEQ_INJECT_ASM(4) | |
1209 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 1210 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) |
784b0012 MD |
1211 | "movl %[expect], %%eax\n\t" |
1212 | "cmpl %%eax, %[v]\n\t" | |
1213 | "jnz 7f\n\t" | |
1214 | #endif | |
1215 | /* try memcpy */ | |
1216 | "test %[len], %[len]\n\t" \ | |
1217 | "jz 333f\n\t" \ | |
1218 | "222:\n\t" \ | |
1219 | "movb (%[src]), %%al\n\t" \ | |
1220 | "movb %%al, (%[dst])\n\t" \ | |
1221 | "inc %[src]\n\t" \ | |
1222 | "inc %[dst]\n\t" \ | |
1223 | "dec %[len]\n\t" \ | |
1224 | "jnz 222b\n\t" \ | |
1225 | "333:\n\t" \ | |
1226 | RSEQ_INJECT_ASM(5) | |
1227 | "movl %[newv], %%eax\n\t" | |
1228 | /* final store */ | |
1229 | "movl %%eax, %[v]\n\t" | |
1230 | "2:\n\t" | |
1231 | RSEQ_INJECT_ASM(6) | |
1232 | /* teardown */ | |
1233 | "movl %[rseq_scratch2], %[len]\n\t" | |
1234 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1235 | "movl %[rseq_scratch0], %[src]\n\t" | |
1236 | RSEQ_ASM_DEFINE_ABORT(4, | |
1237 | "movl %[rseq_scratch2], %[len]\n\t" | |
1238 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1239 | "movl %[rseq_scratch0], %[src]\n\t", | |
1240 | abort) | |
1241 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
1242 | "movl %[rseq_scratch2], %[len]\n\t" | |
1243 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1244 | "movl %[rseq_scratch0], %[src]\n\t", | |
1245 | cmpfail) | |
1246 | #ifdef RSEQ_COMPARE_TWICE | |
1247 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
1248 | "movl %[rseq_scratch2], %[len]\n\t" | |
1249 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1250 | "movl %[rseq_scratch0], %[src]\n\t", | |
1251 | error1) | |
1252 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
1253 | "movl %[rseq_scratch2], %[len]\n\t" | |
1254 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1255 | "movl %[rseq_scratch0], %[src]\n\t", | |
1256 | error2) | |
1257 | #endif | |
1258 | : /* gcc asm goto does not allow outputs */ | |
1259 | : [cpu_id] "r" (cpu), | |
53831176 | 1260 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
1261 | /* final store input */ |
1262 | [v] "m" (*v), | |
1263 | [expect] "m" (expect), | |
1264 | [newv] "m" (newv), | |
1265 | /* try memcpy input */ | |
1266 | [dst] "r" (dst), | |
1267 | [src] "r" (src), | |
1268 | [len] "r" (len), | |
1269 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
1270 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
1271 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
1272 | : "memory", "cc", "eax" | |
1273 | RSEQ_INJECT_CLOBBER | |
1274 | : abort, cmpfail | |
1275 | #ifdef RSEQ_COMPARE_TWICE | |
1276 | , error1, error2 | |
1277 | #endif | |
1278 | ); | |
dd76f2d6 | 1279 | rseq_after_asm_goto(); |
784b0012 MD |
1280 | return 0; |
1281 | abort: | |
dd76f2d6 | 1282 | rseq_after_asm_goto(); |
784b0012 MD |
1283 | RSEQ_INJECT_FAILED |
1284 | return -1; | |
1285 | cmpfail: | |
dd76f2d6 | 1286 | rseq_after_asm_goto(); |
784b0012 MD |
1287 | return 1; |
1288 | #ifdef RSEQ_COMPARE_TWICE | |
1289 | error1: | |
dd76f2d6 | 1290 | rseq_after_asm_goto(); |
784b0012 MD |
1291 | rseq_bug("cpu_id comparison failed"); |
1292 | error2: | |
dd76f2d6 | 1293 | rseq_after_asm_goto(); |
784b0012 MD |
1294 | rseq_bug("expected value comparison failed"); |
1295 | #endif | |
1296 | } | |
1297 | ||
1298 | /* TODO: implement a faster memcpy. */ | |
1299 | static inline __attribute__((always_inline)) | |
1300 | int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | |
1301 | void *dst, void *src, size_t len, | |
1302 | intptr_t newv, int cpu) | |
1303 | { | |
1304 | uint32_t rseq_scratch[3]; | |
1305 | ||
1306 | RSEQ_INJECT_C(9) | |
1307 | ||
1308 | __asm__ __volatile__ goto ( | |
1309 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
90d9876e MD |
1310 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
1311 | #ifdef RSEQ_COMPARE_TWICE | |
1312 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1313 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
1314 | #endif | |
784b0012 MD |
1315 | "movl %[src], %[rseq_scratch0]\n\t" |
1316 | "movl %[dst], %[rseq_scratch1]\n\t" | |
1317 | "movl %[len], %[rseq_scratch2]\n\t" | |
1318 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
53831176 MD |
1319 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
1320 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
784b0012 MD |
1321 | RSEQ_INJECT_ASM(3) |
1322 | "movl %[expect], %%eax\n\t" | |
1323 | "cmpl %%eax, %[v]\n\t" | |
1324 | "jnz 5f\n\t" | |
1325 | RSEQ_INJECT_ASM(4) | |
1326 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 1327 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) |
784b0012 MD |
1328 | "movl %[expect], %%eax\n\t" |
1329 | "cmpl %%eax, %[v]\n\t" | |
1330 | "jnz 7f\n\t" | |
1331 | #endif | |
1332 | /* try memcpy */ | |
1333 | "test %[len], %[len]\n\t" \ | |
1334 | "jz 333f\n\t" \ | |
1335 | "222:\n\t" \ | |
1336 | "movb (%[src]), %%al\n\t" \ | |
1337 | "movb %%al, (%[dst])\n\t" \ | |
1338 | "inc %[src]\n\t" \ | |
1339 | "inc %[dst]\n\t" \ | |
1340 | "dec %[len]\n\t" \ | |
1341 | "jnz 222b\n\t" \ | |
1342 | "333:\n\t" \ | |
1343 | RSEQ_INJECT_ASM(5) | |
1344 | "lock; addl $0,-128(%%esp)\n\t" | |
1345 | "movl %[newv], %%eax\n\t" | |
1346 | /* final store */ | |
1347 | "movl %%eax, %[v]\n\t" | |
1348 | "2:\n\t" | |
1349 | RSEQ_INJECT_ASM(6) | |
1350 | /* teardown */ | |
1351 | "movl %[rseq_scratch2], %[len]\n\t" | |
1352 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1353 | "movl %[rseq_scratch0], %[src]\n\t" | |
1354 | RSEQ_ASM_DEFINE_ABORT(4, | |
1355 | "movl %[rseq_scratch2], %[len]\n\t" | |
1356 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1357 | "movl %[rseq_scratch0], %[src]\n\t", | |
1358 | abort) | |
1359 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
1360 | "movl %[rseq_scratch2], %[len]\n\t" | |
1361 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1362 | "movl %[rseq_scratch0], %[src]\n\t", | |
1363 | cmpfail) | |
1364 | #ifdef RSEQ_COMPARE_TWICE | |
1365 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
1366 | "movl %[rseq_scratch2], %[len]\n\t" | |
1367 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1368 | "movl %[rseq_scratch0], %[src]\n\t", | |
1369 | error1) | |
1370 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
1371 | "movl %[rseq_scratch2], %[len]\n\t" | |
1372 | "movl %[rseq_scratch1], %[dst]\n\t" | |
1373 | "movl %[rseq_scratch0], %[src]\n\t", | |
1374 | error2) | |
1375 | #endif | |
1376 | : /* gcc asm goto does not allow outputs */ | |
1377 | : [cpu_id] "r" (cpu), | |
53831176 | 1378 | [rseq_offset] "r" (rseq_offset), |
784b0012 MD |
1379 | /* final store input */ |
1380 | [v] "m" (*v), | |
1381 | [expect] "m" (expect), | |
1382 | [newv] "m" (newv), | |
1383 | /* try memcpy input */ | |
1384 | [dst] "r" (dst), | |
1385 | [src] "r" (src), | |
1386 | [len] "r" (len), | |
1387 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
1388 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
1389 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
1390 | : "memory", "cc", "eax" | |
1391 | RSEQ_INJECT_CLOBBER | |
1392 | : abort, cmpfail | |
1393 | #ifdef RSEQ_COMPARE_TWICE | |
1394 | , error1, error2 | |
1395 | #endif | |
1396 | ); | |
dd76f2d6 | 1397 | rseq_after_asm_goto(); |
784b0012 MD |
1398 | return 0; |
1399 | abort: | |
dd76f2d6 | 1400 | rseq_after_asm_goto(); |
784b0012 MD |
1401 | RSEQ_INJECT_FAILED |
1402 | return -1; | |
1403 | cmpfail: | |
dd76f2d6 | 1404 | rseq_after_asm_goto(); |
784b0012 MD |
1405 | return 1; |
1406 | #ifdef RSEQ_COMPARE_TWICE | |
1407 | error1: | |
dd76f2d6 | 1408 | rseq_after_asm_goto(); |
784b0012 MD |
1409 | rseq_bug("cpu_id comparison failed"); |
1410 | error2: | |
dd76f2d6 | 1411 | rseq_after_asm_goto(); |
784b0012 MD |
1412 | rseq_bug("expected value comparison failed"); |
1413 | #endif | |
1414 | } | |
1415 | ||
de28c254 MD |
1416 | /* |
1417 | * Dereference @p. Add voffp to the dereferenced pointer, and load its content | |
1418 | * into @load. | |
1419 | */ | |
1420 | static inline __attribute__((always_inline)) | |
d35eae6b | 1421 | int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu) |
de28c254 MD |
1422 | { |
1423 | RSEQ_INJECT_C(9) | |
1424 | ||
1425 | __asm__ __volatile__ goto ( | |
1426 | RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ | |
1427 | #ifdef RSEQ_COMPARE_TWICE | |
1428 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
1429 | #endif | |
1430 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
53831176 MD |
1431 | RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) |
1432 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) | |
de28c254 MD |
1433 | RSEQ_INJECT_ASM(3) |
1434 | "movl %[p], %%ebx\n\t" | |
1435 | RSEQ_INJECT_ASM(4) | |
1436 | #ifdef RSEQ_COMPARE_TWICE | |
53831176 | 1437 | RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) |
de28c254 MD |
1438 | #endif |
1439 | "addl %[voffp], %%ebx\n\t" | |
1440 | "movl (%%ebx), %%ebx\n\t" | |
1441 | "movl %%ebx, %[load]\n\t" | |
1442 | "2:\n\t" | |
1443 | RSEQ_INJECT_ASM(5) | |
1444 | RSEQ_ASM_DEFINE_ABORT(4, "", abort) | |
1445 | : /* gcc asm goto does not allow outputs */ | |
1446 | : [cpu_id] "r" (cpu), | |
53831176 | 1447 | [rseq_offset] "r" (rseq_offset), |
de28c254 MD |
1448 | /* final store input */ |
1449 | [p] "m" (*p), | |
1450 | [voffp] "ir" (voffp), | |
1451 | [load] "m" (*load) | |
1452 | : "memory", "cc", "eax", "ebx" | |
1453 | RSEQ_INJECT_CLOBBER | |
1454 | : abort | |
1455 | #ifdef RSEQ_COMPARE_TWICE | |
1456 | , error1 | |
1457 | #endif | |
1458 | ); | |
dd76f2d6 | 1459 | rseq_after_asm_goto(); |
de28c254 MD |
1460 | return 0; |
1461 | abort: | |
dd76f2d6 | 1462 | rseq_after_asm_goto(); |
de28c254 MD |
1463 | RSEQ_INJECT_FAILED |
1464 | return -1; | |
1465 | #ifdef RSEQ_COMPARE_TWICE | |
1466 | error1: | |
dd76f2d6 | 1467 | rseq_after_asm_goto(); |
de28c254 MD |
1468 | rseq_bug("cpu_id comparison failed"); |
1469 | #endif | |
1470 | } | |
1471 | ||
784b0012 MD |
1472 | #endif /* !RSEQ_SKIP_FASTPATH */ |
1473 | ||
1474 | #endif |