Commit | Line | Data |
---|---|---|
744d0b8b | 1 | /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ |
784b0012 MD |
2 | /* |
3 | * Author: Paul Burton <paul.burton@mips.com> | |
4 | * (C) Copyright 2018 MIPS Tech LLC | |
5 | * | |
6 | * Based on rseq-arm.h: | |
7 | * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
8 | */ | |
9 | ||
b6bd318f MD |
10 | /* |
11 | * RSEQ_SIG uses the break instruction. The instruction pattern is: | |
12 | * | |
13 | * On MIPS: | |
14 | * 0350000d break 0x350 | |
15 | * | |
16 | * On nanoMIPS: | |
17 | * 00100350 break 0x350 | |
18 | * | |
19 | * On microMIPS: | |
20 | * 0000d407 break 0x350 | |
21 | * | |
22 | * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit | |
23 | * halfwords, so the signature halfwords need to be swapped accordingly for | |
24 | * little-endian. | |
25 | */ | |
26 | #if defined(__nanomips__) | |
27 | # ifdef __MIPSEL__ | |
28 | # define RSEQ_SIG 0x03500010 | |
29 | # else | |
30 | # define RSEQ_SIG 0x00100350 | |
31 | # endif | |
32 | #elif defined(__mips_micromips) | |
33 | # ifdef __MIPSEL__ | |
34 | # define RSEQ_SIG 0xd4070000 | |
35 | # else | |
36 | # define RSEQ_SIG 0x0000d407 | |
37 | # endif | |
38 | #elif defined(__mips__) | |
39 | # define RSEQ_SIG 0x0350000d | |
40 | #else | |
41 | /* Unknown MIPS architecture. */ | |
42 | #endif | |
784b0012 MD |
43 | |
44 | #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory") | |
45 | #define rseq_smp_rmb() rseq_smp_mb() | |
46 | #define rseq_smp_wmb() rseq_smp_mb() | |
47 | ||
48 | #define rseq_smp_load_acquire(p) \ | |
49 | __extension__ ({ \ | |
50 | __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ | |
51 | rseq_smp_mb(); \ | |
52 | ____p1; \ | |
53 | }) | |
54 | ||
55 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
56 | ||
57 | #define rseq_smp_store_release(p, v) \ | |
58 | do { \ | |
59 | rseq_smp_mb(); \ | |
60 | RSEQ_WRITE_ONCE(*p, v); \ | |
61 | } while (0) | |
62 | ||
63 | #ifdef RSEQ_SKIP_FASTPATH | |
64 | #include "rseq-skip.h" | |
65 | #else /* !RSEQ_SKIP_FASTPATH */ | |
66 | ||
67 | #if _MIPS_SZLONG == 64 | |
68 | # define LONG ".dword" | |
69 | # define LONG_LA "dla" | |
70 | # define LONG_L "ld" | |
71 | # define LONG_S "sd" | |
72 | # define LONG_ADDI "daddiu" | |
73 | # define U32_U64_PAD(x) x | |
00fa0940 | 74 | # define RSEQ_CS_PTR rseq_cs.ptr |
784b0012 MD |
75 | #elif _MIPS_SZLONG == 32 |
76 | # define LONG ".word" | |
77 | # define LONG_LA "la" | |
78 | # define LONG_L "lw" | |
79 | # define LONG_S "sw" | |
80 | # define LONG_ADDI "addiu" | |
81 | # ifdef __BIG_ENDIAN | |
82 | # define U32_U64_PAD(x) "0x0, " x | |
83 | # else | |
84 | # define U32_U64_PAD(x) x ", 0x0" | |
85 | # endif | |
2b04cd67 | 86 | # define RSEQ_CS_PTR RSEQ_CS_PTR32 |
784b0012 MD |
87 | #else |
88 | # error unsupported _MIPS_SZLONG | |
89 | #endif | |
90 | ||
dd01d0fb | 91 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ |
784b0012 | 92 | post_commit_offset, abort_ip) \ |
dd01d0fb | 93 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 | 94 | ".balign 32\n\t" \ |
dd01d0fb | 95 | __rseq_str(label) ":\n\t" \ |
784b0012 MD |
96 | ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ |
97 | LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ | |
98 | LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ | |
99 | LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ | |
dd01d0fb MD |
100 | ".popsection\n\t" \ |
101 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
102 | LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \ | |
784b0012 MD |
103 | ".popsection\n\t" |
104 | ||
dd01d0fb MD |
105 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
106 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
784b0012 MD |
107 | (post_commit_ip - start_ip), abort_ip) |
108 | ||
90d9876e MD |
109 | /* |
110 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
111 | * of the critical section where a critical section can either branch to or |
112 | * reach through the normal course of its execution. The abort IP and the | |
113 | * post-commit IP are already part of the __rseq_cs section and should not be | |
114 | * explicitly defined as additional exit points. Knowing all exit points is | |
115 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
116 | */ |
117 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
118 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
119 | LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ | |
120 | LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \ | |
121 | ".popsection\n\t" | |
122 | ||
784b0012 MD |
123 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
124 | RSEQ_INJECT_ASM(1) \ | |
125 | LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \ | |
126 | LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \ | |
127 | __rseq_str(label) ":\n\t" | |
128 | ||
129 | #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ | |
130 | RSEQ_INJECT_ASM(2) \ | |
131 | "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \ | |
132 | "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t" | |
133 | ||
134 | #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ | |
135 | abort_label, version, flags, \ | |
136 | start_ip, post_commit_offset, abort_ip) \ | |
137 | ".balign 32\n\t" \ | |
138 | __rseq_str(table_label) ":\n\t" \ | |
139 | ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
140 | LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ | |
141 | LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ | |
142 | LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ | |
143 | ".word " __rseq_str(RSEQ_SIG) "\n\t" \ | |
144 | __rseq_str(label) ":\n\t" \ | |
145 | teardown \ | |
146 | "b %l[" __rseq_str(abort_label) "]\n\t" | |
147 | ||
148 | #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \ | |
149 | start_ip, post_commit_ip, abort_ip) \ | |
150 | __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \ | |
151 | abort_label, 0x0, 0x0, start_ip, \ | |
152 | (post_commit_ip - start_ip), abort_ip) | |
153 | ||
154 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
155 | __rseq_str(label) ":\n\t" \ | |
156 | teardown \ | |
157 | "b %l[" __rseq_str(cmpfail_label) "]\n\t" | |
158 | ||
784b0012 MD |
159 | static inline __attribute__((always_inline)) |
160 | int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) | |
161 | { | |
162 | RSEQ_INJECT_C(9) | |
163 | ||
784b0012 | 164 | __asm__ __volatile__ goto ( |
dd01d0fb | 165 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
90d9876e MD |
166 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
167 | #ifdef RSEQ_COMPARE_TWICE | |
168 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
169 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
170 | #endif | |
784b0012 MD |
171 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
172 | RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) | |
173 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) | |
174 | RSEQ_INJECT_ASM(3) | |
175 | LONG_L " $4, %[v]\n\t" | |
176 | "bne $4, %[expect], %l[cmpfail]\n\t" | |
177 | RSEQ_INJECT_ASM(4) | |
178 | #ifdef RSEQ_COMPARE_TWICE | |
179 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) | |
180 | LONG_L " $4, %[v]\n\t" | |
181 | "bne $4, %[expect], %l[error2]\n\t" | |
182 | #endif | |
183 | /* final store */ | |
184 | LONG_S " %[newv], %[v]\n\t" | |
185 | "2:\n\t" | |
186 | RSEQ_INJECT_ASM(5) | |
187 | "b 5f\n\t" | |
188 | RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) | |
189 | "5:\n\t" | |
190 | : /* gcc asm goto does not allow outputs */ | |
191 | : [cpu_id] "r" (cpu), | |
9698c399 | 192 | [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
00fa0940 | 193 | [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), |
784b0012 MD |
194 | [v] "m" (*v), |
195 | [expect] "r" (expect), | |
196 | [newv] "r" (newv) | |
197 | RSEQ_INJECT_INPUT | |
198 | : "$4", "memory" | |
199 | RSEQ_INJECT_CLOBBER | |
200 | : abort, cmpfail | |
201 | #ifdef RSEQ_COMPARE_TWICE | |
202 | , error1, error2 | |
203 | #endif | |
204 | ); | |
784b0012 MD |
205 | return 0; |
206 | abort: | |
784b0012 MD |
207 | RSEQ_INJECT_FAILED |
208 | return -1; | |
209 | cmpfail: | |
784b0012 MD |
210 | return 1; |
211 | #ifdef RSEQ_COMPARE_TWICE | |
212 | error1: | |
213 | rseq_bug("cpu_id comparison failed"); | |
214 | error2: | |
215 | rseq_bug("expected value comparison failed"); | |
216 | #endif | |
217 | } | |
218 | ||
219 | static inline __attribute__((always_inline)) | |
220 | int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, | |
d35eae6b | 221 | long voffp, intptr_t *load, int cpu) |
784b0012 MD |
222 | { |
223 | RSEQ_INJECT_C(9) | |
224 | ||
784b0012 | 225 | __asm__ __volatile__ goto ( |
dd01d0fb | 226 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
90d9876e MD |
227 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
228 | #ifdef RSEQ_COMPARE_TWICE | |
229 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
230 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
231 | #endif | |
784b0012 MD |
232 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
233 | RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) | |
234 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) | |
235 | RSEQ_INJECT_ASM(3) | |
236 | LONG_L " $4, %[v]\n\t" | |
237 | "beq $4, %[expectnot], %l[cmpfail]\n\t" | |
238 | RSEQ_INJECT_ASM(4) | |
239 | #ifdef RSEQ_COMPARE_TWICE | |
240 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) | |
241 | LONG_L " $4, %[v]\n\t" | |
242 | "beq $4, %[expectnot], %l[error2]\n\t" | |
243 | #endif | |
244 | LONG_S " $4, %[load]\n\t" | |
245 | LONG_ADDI " $4, %[voffp]\n\t" | |
246 | LONG_L " $4, 0($4)\n\t" | |
247 | /* final store */ | |
248 | LONG_S " $4, %[v]\n\t" | |
249 | "2:\n\t" | |
250 | RSEQ_INJECT_ASM(5) | |
251 | "b 5f\n\t" | |
252 | RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) | |
253 | "5:\n\t" | |
254 | : /* gcc asm goto does not allow outputs */ | |
255 | : [cpu_id] "r" (cpu), | |
9698c399 | 256 | [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
00fa0940 | 257 | [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), |
784b0012 MD |
258 | /* final store input */ |
259 | [v] "m" (*v), | |
260 | [expectnot] "r" (expectnot), | |
261 | [voffp] "Ir" (voffp), | |
262 | [load] "m" (*load) | |
263 | RSEQ_INJECT_INPUT | |
264 | : "$4", "memory" | |
265 | RSEQ_INJECT_CLOBBER | |
266 | : abort, cmpfail | |
267 | #ifdef RSEQ_COMPARE_TWICE | |
268 | , error1, error2 | |
269 | #endif | |
270 | ); | |
784b0012 MD |
271 | return 0; |
272 | abort: | |
784b0012 MD |
273 | RSEQ_INJECT_FAILED |
274 | return -1; | |
275 | cmpfail: | |
784b0012 MD |
276 | return 1; |
277 | #ifdef RSEQ_COMPARE_TWICE | |
278 | error1: | |
279 | rseq_bug("cpu_id comparison failed"); | |
280 | error2: | |
281 | rseq_bug("expected value comparison failed"); | |
282 | #endif | |
283 | } | |
284 | ||
285 | static inline __attribute__((always_inline)) | |
286 | int rseq_addv(intptr_t *v, intptr_t count, int cpu) | |
287 | { | |
288 | RSEQ_INJECT_C(9) | |
289 | ||
784b0012 | 290 | __asm__ __volatile__ goto ( |
dd01d0fb | 291 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
90d9876e MD |
292 | #ifdef RSEQ_COMPARE_TWICE |
293 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
294 | #endif | |
784b0012 MD |
295 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
296 | RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) | |
297 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) | |
298 | RSEQ_INJECT_ASM(3) | |
299 | #ifdef RSEQ_COMPARE_TWICE | |
300 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) | |
301 | #endif | |
302 | LONG_L " $4, %[v]\n\t" | |
303 | LONG_ADDI " $4, %[count]\n\t" | |
304 | /* final store */ | |
305 | LONG_S " $4, %[v]\n\t" | |
306 | "2:\n\t" | |
307 | RSEQ_INJECT_ASM(4) | |
308 | "b 5f\n\t" | |
309 | RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) | |
310 | "5:\n\t" | |
311 | : /* gcc asm goto does not allow outputs */ | |
312 | : [cpu_id] "r" (cpu), | |
9698c399 | 313 | [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
00fa0940 | 314 | [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), |
784b0012 MD |
315 | [v] "m" (*v), |
316 | [count] "Ir" (count) | |
317 | RSEQ_INJECT_INPUT | |
318 | : "$4", "memory" | |
319 | RSEQ_INJECT_CLOBBER | |
320 | : abort | |
321 | #ifdef RSEQ_COMPARE_TWICE | |
322 | , error1 | |
323 | #endif | |
324 | ); | |
784b0012 MD |
325 | return 0; |
326 | abort: | |
784b0012 MD |
327 | RSEQ_INJECT_FAILED |
328 | return -1; | |
329 | #ifdef RSEQ_COMPARE_TWICE | |
330 | error1: | |
331 | rseq_bug("cpu_id comparison failed"); | |
332 | #endif | |
333 | } | |
334 | ||
335 | static inline __attribute__((always_inline)) | |
336 | int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, | |
337 | intptr_t *v2, intptr_t newv2, | |
338 | intptr_t newv, int cpu) | |
339 | { | |
340 | RSEQ_INJECT_C(9) | |
341 | ||
784b0012 | 342 | __asm__ __volatile__ goto ( |
dd01d0fb | 343 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
90d9876e MD |
344 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
345 | #ifdef RSEQ_COMPARE_TWICE | |
346 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
347 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
348 | #endif | |
784b0012 MD |
349 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
350 | RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) | |
351 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) | |
352 | RSEQ_INJECT_ASM(3) | |
353 | LONG_L " $4, %[v]\n\t" | |
354 | "bne $4, %[expect], %l[cmpfail]\n\t" | |
355 | RSEQ_INJECT_ASM(4) | |
356 | #ifdef RSEQ_COMPARE_TWICE | |
357 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) | |
358 | LONG_L " $4, %[v]\n\t" | |
359 | "bne $4, %[expect], %l[error2]\n\t" | |
360 | #endif | |
361 | /* try store */ | |
362 | LONG_S " %[newv2], %[v2]\n\t" | |
363 | RSEQ_INJECT_ASM(5) | |
364 | /* final store */ | |
365 | LONG_S " %[newv], %[v]\n\t" | |
366 | "2:\n\t" | |
367 | RSEQ_INJECT_ASM(6) | |
368 | "b 5f\n\t" | |
369 | RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) | |
370 | "5:\n\t" | |
371 | : /* gcc asm goto does not allow outputs */ | |
372 | : [cpu_id] "r" (cpu), | |
9698c399 | 373 | [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
00fa0940 | 374 | [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), |
784b0012 MD |
375 | /* try store input */ |
376 | [v2] "m" (*v2), | |
377 | [newv2] "r" (newv2), | |
378 | /* final store input */ | |
379 | [v] "m" (*v), | |
380 | [expect] "r" (expect), | |
381 | [newv] "r" (newv) | |
382 | RSEQ_INJECT_INPUT | |
383 | : "$4", "memory" | |
384 | RSEQ_INJECT_CLOBBER | |
385 | : abort, cmpfail | |
386 | #ifdef RSEQ_COMPARE_TWICE | |
387 | , error1, error2 | |
388 | #endif | |
389 | ); | |
784b0012 MD |
390 | return 0; |
391 | abort: | |
784b0012 MD |
392 | RSEQ_INJECT_FAILED |
393 | return -1; | |
394 | cmpfail: | |
784b0012 MD |
395 | return 1; |
396 | #ifdef RSEQ_COMPARE_TWICE | |
397 | error1: | |
398 | rseq_bug("cpu_id comparison failed"); | |
399 | error2: | |
400 | rseq_bug("expected value comparison failed"); | |
401 | #endif | |
402 | } | |
403 | ||
404 | static inline __attribute__((always_inline)) | |
405 | int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, | |
406 | intptr_t *v2, intptr_t newv2, | |
407 | intptr_t newv, int cpu) | |
408 | { | |
409 | RSEQ_INJECT_C(9) | |
410 | ||
784b0012 | 411 | __asm__ __volatile__ goto ( |
dd01d0fb | 412 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
90d9876e MD |
413 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
414 | #ifdef RSEQ_COMPARE_TWICE | |
415 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
416 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
417 | #endif | |
784b0012 MD |
418 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
419 | RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) | |
420 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) | |
421 | RSEQ_INJECT_ASM(3) | |
422 | LONG_L " $4, %[v]\n\t" | |
423 | "bne $4, %[expect], %l[cmpfail]\n\t" | |
424 | RSEQ_INJECT_ASM(4) | |
425 | #ifdef RSEQ_COMPARE_TWICE | |
426 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) | |
427 | LONG_L " $4, %[v]\n\t" | |
428 | "bne $4, %[expect], %l[error2]\n\t" | |
429 | #endif | |
430 | /* try store */ | |
431 | LONG_S " %[newv2], %[v2]\n\t" | |
432 | RSEQ_INJECT_ASM(5) | |
433 | "sync\n\t" /* full sync provides store-release */ | |
434 | /* final store */ | |
435 | LONG_S " %[newv], %[v]\n\t" | |
436 | "2:\n\t" | |
437 | RSEQ_INJECT_ASM(6) | |
438 | "b 5f\n\t" | |
439 | RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) | |
440 | "5:\n\t" | |
441 | : /* gcc asm goto does not allow outputs */ | |
442 | : [cpu_id] "r" (cpu), | |
9698c399 | 443 | [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
00fa0940 | 444 | [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), |
784b0012 MD |
445 | /* try store input */ |
446 | [v2] "m" (*v2), | |
447 | [newv2] "r" (newv2), | |
448 | /* final store input */ | |
449 | [v] "m" (*v), | |
450 | [expect] "r" (expect), | |
451 | [newv] "r" (newv) | |
452 | RSEQ_INJECT_INPUT | |
453 | : "$4", "memory" | |
454 | RSEQ_INJECT_CLOBBER | |
455 | : abort, cmpfail | |
456 | #ifdef RSEQ_COMPARE_TWICE | |
457 | , error1, error2 | |
458 | #endif | |
459 | ); | |
784b0012 MD |
460 | return 0; |
461 | abort: | |
784b0012 MD |
462 | RSEQ_INJECT_FAILED |
463 | return -1; | |
464 | cmpfail: | |
784b0012 MD |
465 | return 1; |
466 | #ifdef RSEQ_COMPARE_TWICE | |
467 | error1: | |
468 | rseq_bug("cpu_id comparison failed"); | |
469 | error2: | |
470 | rseq_bug("expected value comparison failed"); | |
471 | #endif | |
472 | } | |
473 | ||
474 | static inline __attribute__((always_inline)) | |
475 | int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, | |
476 | intptr_t *v2, intptr_t expect2, | |
477 | intptr_t newv, int cpu) | |
478 | { | |
479 | RSEQ_INJECT_C(9) | |
480 | ||
784b0012 | 481 | __asm__ __volatile__ goto ( |
dd01d0fb | 482 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
90d9876e MD |
483 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
484 | #ifdef RSEQ_COMPARE_TWICE | |
485 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
486 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
487 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) | |
488 | #endif | |
784b0012 MD |
489 | /* Start rseq by storing table entry pointer into rseq_cs. */ |
490 | RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) | |
491 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) | |
492 | RSEQ_INJECT_ASM(3) | |
493 | LONG_L " $4, %[v]\n\t" | |
494 | "bne $4, %[expect], %l[cmpfail]\n\t" | |
495 | RSEQ_INJECT_ASM(4) | |
496 | LONG_L " $4, %[v2]\n\t" | |
497 | "bne $4, %[expect2], %l[cmpfail]\n\t" | |
498 | RSEQ_INJECT_ASM(5) | |
499 | #ifdef RSEQ_COMPARE_TWICE | |
500 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) | |
501 | LONG_L " $4, %[v]\n\t" | |
502 | "bne $4, %[expect], %l[error2]\n\t" | |
503 | LONG_L " $4, %[v2]\n\t" | |
504 | "bne $4, %[expect2], %l[error3]\n\t" | |
505 | #endif | |
506 | /* final store */ | |
507 | LONG_S " %[newv], %[v]\n\t" | |
508 | "2:\n\t" | |
509 | RSEQ_INJECT_ASM(6) | |
510 | "b 5f\n\t" | |
511 | RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) | |
512 | "5:\n\t" | |
513 | : /* gcc asm goto does not allow outputs */ | |
514 | : [cpu_id] "r" (cpu), | |
9698c399 | 515 | [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
00fa0940 | 516 | [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), |
784b0012 MD |
517 | /* cmp2 input */ |
518 | [v2] "m" (*v2), | |
519 | [expect2] "r" (expect2), | |
520 | /* final store input */ | |
521 | [v] "m" (*v), | |
522 | [expect] "r" (expect), | |
523 | [newv] "r" (newv) | |
524 | RSEQ_INJECT_INPUT | |
525 | : "$4", "memory" | |
526 | RSEQ_INJECT_CLOBBER | |
527 | : abort, cmpfail | |
528 | #ifdef RSEQ_COMPARE_TWICE | |
529 | , error1, error2, error3 | |
530 | #endif | |
531 | ); | |
784b0012 MD |
532 | return 0; |
533 | abort: | |
784b0012 MD |
534 | RSEQ_INJECT_FAILED |
535 | return -1; | |
536 | cmpfail: | |
784b0012 MD |
537 | return 1; |
538 | #ifdef RSEQ_COMPARE_TWICE | |
539 | error1: | |
540 | rseq_bug("cpu_id comparison failed"); | |
541 | error2: | |
542 | rseq_bug("1st expected value comparison failed"); | |
543 | error3: | |
544 | rseq_bug("2nd expected value comparison failed"); | |
545 | #endif | |
546 | } | |
547 | ||
548 | static inline __attribute__((always_inline)) | |
549 | int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, | |
550 | void *dst, void *src, size_t len, | |
551 | intptr_t newv, int cpu) | |
552 | { | |
553 | uintptr_t rseq_scratch[3]; | |
554 | ||
555 | RSEQ_INJECT_C(9) | |
556 | ||
784b0012 | 557 | __asm__ __volatile__ goto ( |
dd01d0fb | 558 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
90d9876e MD |
559 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
560 | #ifdef RSEQ_COMPARE_TWICE | |
561 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
562 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
563 | #endif | |
784b0012 MD |
564 | LONG_S " %[src], %[rseq_scratch0]\n\t" |
565 | LONG_S " %[dst], %[rseq_scratch1]\n\t" | |
566 | LONG_S " %[len], %[rseq_scratch2]\n\t" | |
567 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
568 | RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) | |
569 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) | |
570 | RSEQ_INJECT_ASM(3) | |
571 | LONG_L " $4, %[v]\n\t" | |
572 | "bne $4, %[expect], 5f\n\t" | |
573 | RSEQ_INJECT_ASM(4) | |
574 | #ifdef RSEQ_COMPARE_TWICE | |
575 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) | |
576 | LONG_L " $4, %[v]\n\t" | |
577 | "bne $4, %[expect], 7f\n\t" | |
578 | #endif | |
579 | /* try memcpy */ | |
580 | "beqz %[len], 333f\n\t" \ | |
581 | "222:\n\t" \ | |
582 | "lb $4, 0(%[src])\n\t" \ | |
583 | "sb $4, 0(%[dst])\n\t" \ | |
584 | LONG_ADDI " %[src], 1\n\t" \ | |
585 | LONG_ADDI " %[dst], 1\n\t" \ | |
586 | LONG_ADDI " %[len], -1\n\t" \ | |
587 | "bnez %[len], 222b\n\t" \ | |
588 | "333:\n\t" \ | |
589 | RSEQ_INJECT_ASM(5) | |
590 | /* final store */ | |
591 | LONG_S " %[newv], %[v]\n\t" | |
592 | "2:\n\t" | |
593 | RSEQ_INJECT_ASM(6) | |
594 | /* teardown */ | |
595 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
596 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
597 | LONG_L " %[src], %[rseq_scratch0]\n\t" | |
598 | "b 8f\n\t" | |
599 | RSEQ_ASM_DEFINE_ABORT(3, 4, | |
600 | /* teardown */ | |
601 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
602 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
603 | LONG_L " %[src], %[rseq_scratch0]\n\t", | |
604 | abort, 1b, 2b, 4f) | |
605 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
606 | /* teardown */ | |
607 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
608 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
609 | LONG_L " %[src], %[rseq_scratch0]\n\t", | |
610 | cmpfail) | |
611 | #ifdef RSEQ_COMPARE_TWICE | |
612 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
613 | /* teardown */ | |
614 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
615 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
616 | LONG_L " %[src], %[rseq_scratch0]\n\t", | |
617 | error1) | |
618 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
619 | /* teardown */ | |
620 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
621 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
622 | LONG_L " %[src], %[rseq_scratch0]\n\t", | |
623 | error2) | |
624 | #endif | |
625 | "8:\n\t" | |
626 | : /* gcc asm goto does not allow outputs */ | |
627 | : [cpu_id] "r" (cpu), | |
9698c399 | 628 | [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
00fa0940 | 629 | [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), |
784b0012 MD |
630 | /* final store input */ |
631 | [v] "m" (*v), | |
632 | [expect] "r" (expect), | |
633 | [newv] "r" (newv), | |
634 | /* try memcpy input */ | |
635 | [dst] "r" (dst), | |
636 | [src] "r" (src), | |
637 | [len] "r" (len), | |
638 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
639 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
640 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
641 | RSEQ_INJECT_INPUT | |
642 | : "$4", "memory" | |
643 | RSEQ_INJECT_CLOBBER | |
644 | : abort, cmpfail | |
645 | #ifdef RSEQ_COMPARE_TWICE | |
646 | , error1, error2 | |
647 | #endif | |
648 | ); | |
784b0012 MD |
649 | return 0; |
650 | abort: | |
784b0012 MD |
651 | RSEQ_INJECT_FAILED |
652 | return -1; | |
653 | cmpfail: | |
784b0012 MD |
654 | return 1; |
655 | #ifdef RSEQ_COMPARE_TWICE | |
656 | error1: | |
784b0012 MD |
657 | rseq_bug("cpu_id comparison failed"); |
658 | error2: | |
784b0012 MD |
659 | rseq_bug("expected value comparison failed"); |
660 | #endif | |
661 | } | |
662 | ||
663 | static inline __attribute__((always_inline)) | |
664 | int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, | |
665 | void *dst, void *src, size_t len, | |
666 | intptr_t newv, int cpu) | |
667 | { | |
668 | uintptr_t rseq_scratch[3]; | |
669 | ||
670 | RSEQ_INJECT_C(9) | |
671 | ||
784b0012 | 672 | __asm__ __volatile__ goto ( |
dd01d0fb | 673 | RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ |
90d9876e MD |
674 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) |
675 | #ifdef RSEQ_COMPARE_TWICE | |
676 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) | |
677 | RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) | |
678 | #endif | |
784b0012 MD |
679 | LONG_S " %[src], %[rseq_scratch0]\n\t" |
680 | LONG_S " %[dst], %[rseq_scratch1]\n\t" | |
681 | LONG_S " %[len], %[rseq_scratch2]\n\t" | |
682 | /* Start rseq by storing table entry pointer into rseq_cs. */ | |
683 | RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) | |
684 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) | |
685 | RSEQ_INJECT_ASM(3) | |
686 | LONG_L " $4, %[v]\n\t" | |
687 | "bne $4, %[expect], 5f\n\t" | |
688 | RSEQ_INJECT_ASM(4) | |
689 | #ifdef RSEQ_COMPARE_TWICE | |
690 | RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) | |
691 | LONG_L " $4, %[v]\n\t" | |
692 | "bne $4, %[expect], 7f\n\t" | |
693 | #endif | |
694 | /* try memcpy */ | |
695 | "beqz %[len], 333f\n\t" \ | |
696 | "222:\n\t" \ | |
697 | "lb $4, 0(%[src])\n\t" \ | |
698 | "sb $4, 0(%[dst])\n\t" \ | |
699 | LONG_ADDI " %[src], 1\n\t" \ | |
700 | LONG_ADDI " %[dst], 1\n\t" \ | |
701 | LONG_ADDI " %[len], -1\n\t" \ | |
702 | "bnez %[len], 222b\n\t" \ | |
703 | "333:\n\t" \ | |
704 | RSEQ_INJECT_ASM(5) | |
705 | "sync\n\t" /* full sync provides store-release */ | |
706 | /* final store */ | |
707 | LONG_S " %[newv], %[v]\n\t" | |
708 | "2:\n\t" | |
709 | RSEQ_INJECT_ASM(6) | |
710 | /* teardown */ | |
711 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
712 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
713 | LONG_L " %[src], %[rseq_scratch0]\n\t" | |
714 | "b 8f\n\t" | |
715 | RSEQ_ASM_DEFINE_ABORT(3, 4, | |
716 | /* teardown */ | |
717 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
718 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
719 | LONG_L " %[src], %[rseq_scratch0]\n\t", | |
720 | abort, 1b, 2b, 4f) | |
721 | RSEQ_ASM_DEFINE_CMPFAIL(5, | |
722 | /* teardown */ | |
723 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
724 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
725 | LONG_L " %[src], %[rseq_scratch0]\n\t", | |
726 | cmpfail) | |
727 | #ifdef RSEQ_COMPARE_TWICE | |
728 | RSEQ_ASM_DEFINE_CMPFAIL(6, | |
729 | /* teardown */ | |
730 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
731 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
732 | LONG_L " %[src], %[rseq_scratch0]\n\t", | |
733 | error1) | |
734 | RSEQ_ASM_DEFINE_CMPFAIL(7, | |
735 | /* teardown */ | |
736 | LONG_L " %[len], %[rseq_scratch2]\n\t" | |
737 | LONG_L " %[dst], %[rseq_scratch1]\n\t" | |
738 | LONG_L " %[src], %[rseq_scratch0]\n\t", | |
739 | error2) | |
740 | #endif | |
741 | "8:\n\t" | |
742 | : /* gcc asm goto does not allow outputs */ | |
743 | : [cpu_id] "r" (cpu), | |
9698c399 | 744 | [current_cpu_id] "m" (rseq_get_abi()->cpu_id), |
00fa0940 | 745 | [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), |
784b0012 MD |
746 | /* final store input */ |
747 | [v] "m" (*v), | |
748 | [expect] "r" (expect), | |
749 | [newv] "r" (newv), | |
750 | /* try memcpy input */ | |
751 | [dst] "r" (dst), | |
752 | [src] "r" (src), | |
753 | [len] "r" (len), | |
754 | [rseq_scratch0] "m" (rseq_scratch[0]), | |
755 | [rseq_scratch1] "m" (rseq_scratch[1]), | |
756 | [rseq_scratch2] "m" (rseq_scratch[2]) | |
757 | RSEQ_INJECT_INPUT | |
758 | : "$4", "memory" | |
759 | RSEQ_INJECT_CLOBBER | |
760 | : abort, cmpfail | |
761 | #ifdef RSEQ_COMPARE_TWICE | |
762 | , error1, error2 | |
763 | #endif | |
764 | ); | |
784b0012 MD |
765 | return 0; |
766 | abort: | |
784b0012 MD |
767 | RSEQ_INJECT_FAILED |
768 | return -1; | |
769 | cmpfail: | |
784b0012 MD |
770 | return 1; |
771 | #ifdef RSEQ_COMPARE_TWICE | |
772 | error1: | |
784b0012 MD |
773 | rseq_bug("cpu_id comparison failed"); |
774 | error2: | |
784b0012 MD |
775 | rseq_bug("expected value comparison failed"); |
776 | #endif | |
777 | } | |
778 | ||
779 | #endif /* !RSEQ_SKIP_FASTPATH */ |