1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3 /* SPDX-FileCopyrightText: 2016-2018 Boqun Feng <boqun.feng@gmail.com> */
9 #include "rseq-bits-template.h"
11 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
14 static inline __attribute__((always_inline
))
15 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev
)(intptr_t *v
, intptr_t expect
, intptr_t newv
, int cpu
)
19 __asm__ __volatile__
goto (
20 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
21 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
22 #ifdef RSEQ_COMPARE_TWICE
23 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
24 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
26 /* Start rseq by storing table entry pointer into rseq_cs. */
27 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
29 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
31 /* cmp @v equal to @expect */
32 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
34 #ifdef RSEQ_COMPARE_TWICE
36 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
37 /* cmp @v equal to @expect */
38 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
41 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 2)
43 RSEQ_ASM_DEFINE_ABORT(4, abort
)
44 : /* gcc asm goto does not allow outputs */
46 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
47 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
49 [expect
] "r" (expect
),
52 : "memory", "cc", "r17"
55 #ifdef RSEQ_COMPARE_TWICE
59 rseq_after_asm_goto();
62 rseq_after_asm_goto();
66 rseq_after_asm_goto();
68 #ifdef RSEQ_COMPARE_TWICE
70 rseq_after_asm_goto();
71 rseq_bug("cpu_id comparison failed");
73 rseq_after_asm_goto();
74 rseq_bug("expected value comparison failed");
78 static inline __attribute__((always_inline
))
79 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load
)(intptr_t *v
, intptr_t expectnot
,
80 long voffp
, intptr_t *load
, int cpu
)
84 __asm__ __volatile__
goto (
85 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
86 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
87 #ifdef RSEQ_COMPARE_TWICE
88 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
89 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
91 /* Start rseq by storing table entry pointer into rseq_cs. */
92 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
94 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
96 /* cmp @v not equal to @expectnot */
97 RSEQ_ASM_OP_CMPNE(v
, expectnot
, %l
[cmpfail
])
99 #ifdef RSEQ_COMPARE_TWICE
101 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
102 /* cmp @v not equal to @expectnot */
103 RSEQ_ASM_OP_CMPNE(v
, expectnot
, %l
[error2
])
105 /* load the value of @v */
106 RSEQ_ASM_OP_R_LOAD(v
)
107 /* store it in @load */
108 RSEQ_ASM_OP_R_STORE(load
)
109 /* dereference voffp(v) */
110 RSEQ_ASM_OP_R_LOADX(voffp
)
111 /* final store the value at voffp(v) */
112 RSEQ_ASM_OP_R_FINAL_STORE(v
, 2)
114 RSEQ_ASM_DEFINE_ABORT(4, abort
)
115 : /* gcc asm goto does not allow outputs */
116 : [cpu_id
] "r" (cpu
),
117 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
118 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
119 /* final store input */
121 [expectnot
] "r" (expectnot
),
125 : "memory", "cc", "r17"
128 #ifdef RSEQ_COMPARE_TWICE
132 rseq_after_asm_goto();
135 rseq_after_asm_goto();
139 rseq_after_asm_goto();
141 #ifdef RSEQ_COMPARE_TWICE
143 rseq_after_asm_goto();
144 rseq_bug("cpu_id comparison failed");
146 rseq_after_asm_goto();
147 rseq_bug("expected value comparison failed");
151 static inline __attribute__((always_inline
))
152 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv
)(intptr_t *v
, intptr_t count
, int cpu
)
156 __asm__ __volatile__
goto (
157 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
158 #ifdef RSEQ_COMPARE_TWICE
159 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
161 /* Start rseq by storing table entry pointer into rseq_cs. */
162 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
164 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
166 #ifdef RSEQ_COMPARE_TWICE
168 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
170 /* load the value of @v */
171 RSEQ_ASM_OP_R_LOAD(v
)
172 /* add @count to it */
173 RSEQ_ASM_OP_R_ADD(count
)
175 RSEQ_ASM_OP_R_FINAL_STORE(v
, 2)
177 RSEQ_ASM_DEFINE_ABORT(4, abort
)
178 : /* gcc asm goto does not allow outputs */
179 : [cpu_id
] "r" (cpu
),
180 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
181 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
182 /* final store input */
186 : "memory", "cc", "r17"
189 #ifdef RSEQ_COMPARE_TWICE
193 rseq_after_asm_goto();
196 rseq_after_asm_goto();
199 #ifdef RSEQ_COMPARE_TWICE
201 rseq_after_asm_goto();
202 rseq_bug("cpu_id comparison failed");
206 static inline __attribute__((always_inline
))
207 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev
)(intptr_t *v
, intptr_t expect
,
208 intptr_t *v2
, intptr_t expect2
,
209 intptr_t newv
, int cpu
)
213 __asm__ __volatile__
goto (
214 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
216 #ifdef RSEQ_COMPARE_TWICE
217 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
218 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
219 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error3
])
221 /* Start rseq by storing table entry pointer into rseq_cs. */
222 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
224 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
226 /* cmp @v equal to @expect */
227 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
229 /* cmp @v2 equal to @expct2 */
230 RSEQ_ASM_OP_CMPEQ(v2
, expect2
, %l
[cmpfail
])
232 #ifdef RSEQ_COMPARE_TWICE
234 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
235 /* cmp @v equal to @expect */
236 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
237 /* cmp @v2 equal to @expct2 */
238 RSEQ_ASM_OP_CMPEQ(v2
, expect2
, %l
[error3
])
241 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 2)
243 RSEQ_ASM_DEFINE_ABORT(4, abort
)
244 : /* gcc asm goto does not allow outputs */
245 : [cpu_id
] "r" (cpu
),
246 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
247 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
250 [expect2
] "r" (expect2
),
251 /* final store input */
253 [expect
] "r" (expect
),
256 : "memory", "cc", "r17"
259 #ifdef RSEQ_COMPARE_TWICE
260 , error1
, error2
, error3
263 rseq_after_asm_goto();
266 rseq_after_asm_goto();
270 rseq_after_asm_goto();
272 #ifdef RSEQ_COMPARE_TWICE
274 rseq_after_asm_goto();
275 rseq_bug("cpu_id comparison failed");
277 rseq_after_asm_goto();
278 rseq_bug("1st expected value comparison failed");
280 rseq_after_asm_goto();
281 rseq_bug("2nd expected value comparison failed");
285 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
286 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
288 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
289 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
291 static inline __attribute__((always_inline
))
292 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev
)(intptr_t *v
, intptr_t expect
,
293 intptr_t *v2
, intptr_t newv2
,
294 intptr_t newv
, int cpu
)
298 __asm__ __volatile__
goto (
299 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
300 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
301 #ifdef RSEQ_COMPARE_TWICE
302 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
303 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
305 /* Start rseq by storing table entry pointer into rseq_cs. */
306 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
308 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
310 /* cmp @v equal to @expect */
311 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
313 #ifdef RSEQ_COMPARE_TWICE
315 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
316 /* cmp @v equal to @expect */
317 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
320 RSEQ_ASM_OP_STORE(newv2
, v2
)
322 #ifdef RSEQ_TEMPLATE_MO_RELEASE
327 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 2)
329 RSEQ_ASM_DEFINE_ABORT(4, abort
)
330 : /* gcc asm goto does not allow outputs */
331 : [cpu_id
] "r" (cpu
),
332 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
333 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
334 /* try store input */
337 /* final store input */
339 [expect
] "r" (expect
),
342 : "memory", "cc", "r17"
345 #ifdef RSEQ_COMPARE_TWICE
349 rseq_after_asm_goto();
352 rseq_after_asm_goto();
356 rseq_after_asm_goto();
358 #ifdef RSEQ_COMPARE_TWICE
360 rseq_after_asm_goto();
361 rseq_bug("cpu_id comparison failed");
363 rseq_after_asm_goto();
364 rseq_bug("expected value comparison failed");
368 static inline __attribute__((always_inline
))
369 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev
)(intptr_t *v
, intptr_t expect
,
370 void *dst
, void *src
, size_t len
,
371 intptr_t newv
, int cpu
)
375 __asm__ __volatile__
goto (
376 RSEQ_ASM_DEFINE_TABLE(3, 1f
, 2f
, 4f
) /* start, commit, abort */
377 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[cmpfail
])
378 #ifdef RSEQ_COMPARE_TWICE
379 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error1
])
380 RSEQ_ASM_DEFINE_EXIT_POINT(1f
, %l
[error2
])
382 /* setup for mempcy */
383 "mr %%r19, %[len]\n\t"
384 "mr %%r20, %[src]\n\t"
385 "mr %%r21, %[dst]\n\t"
386 /* Start rseq by storing table entry pointer into rseq_cs. */
387 RSEQ_ASM_STORE_RSEQ_CS(1, 3b
, rseq_cs
)
389 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
391 /* cmp @v equal to @expect */
392 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[cmpfail
])
394 #ifdef RSEQ_COMPARE_TWICE
396 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, %l
[error1
])
397 /* cmp @v equal to @expect */
398 RSEQ_ASM_OP_CMPEQ(v
, expect
, %l
[error2
])
401 RSEQ_ASM_OP_R_MEMCPY()
403 #ifdef RSEQ_TEMPLATE_MO_RELEASE
408 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 2)
411 RSEQ_ASM_DEFINE_ABORT(4, abort
)
412 : /* gcc asm goto does not allow outputs */
413 : [cpu_id
] "r" (cpu
),
414 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
415 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
416 /* final store input */
418 [expect
] "r" (expect
),
420 /* try memcpy input */
425 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
428 #ifdef RSEQ_COMPARE_TWICE
432 rseq_after_asm_goto();
435 rseq_after_asm_goto();
439 rseq_after_asm_goto();
441 #ifdef RSEQ_COMPARE_TWICE
443 rseq_after_asm_goto();
444 rseq_bug("cpu_id comparison failed");
446 rseq_after_asm_goto();
447 rseq_bug("expected value comparison failed");
451 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
452 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
454 #include "rseq-bits-reset.h"