1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
4 #include "rseq-bits-template.h"
6 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
7 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
9 static inline __attribute__((always_inline
))
10 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev
)(intptr_t *v
, intptr_t expect
, intptr_t newv
, int cpu
)
14 __asm__ __volatile__
goto(RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
15 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[cmpfail]")
16 #ifdef RSEQ_COMPARE_TWICE
17 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error1]")
18 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error2]")
20 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
21 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
23 RSEQ_ASM_OP_CMPEQ(v
, expect
, "%l[cmpfail]")
25 #ifdef RSEQ_COMPARE_TWICE
26 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, "%l[error1]")
27 RSEQ_ASM_OP_CMPEQ(v
, expect
, "%l[error2]")
29 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
31 RSEQ_ASM_DEFINE_ABORT(4, abort
)
32 : /* gcc asm goto does not allow outputs */
34 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
35 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
37 [expect
] "r" (expect
),
40 : "memory", RSEQ_ASM_TMP_REG_1
43 #ifdef RSEQ_COMPARE_TWICE
54 #ifdef RSEQ_COMPARE_TWICE
56 rseq_bug("cpu_id comparison failed");
58 rseq_bug("expected value comparison failed");
62 static inline __attribute__((always_inline
))
63 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load
)(intptr_t *v
, intptr_t expectnot
,
64 off_t voffp
, intptr_t *load
, int cpu
)
68 __asm__ __volatile__
goto(RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
69 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[cmpfail]")
70 #ifdef RSEQ_COMPARE_TWICE
71 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error1]")
72 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error2]")
74 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
75 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
77 RSEQ_ASM_OP_CMPNE(v
, expectnot
, "%l[cmpfail]")
79 #ifdef RSEQ_COMPARE_TWICE
80 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, "%l[error1]")
81 RSEQ_ASM_OP_CMPNE(v
, expectnot
, "%l[error2]")
84 RSEQ_ASM_OP_R_STORE(load
)
85 RSEQ_ASM_OP_R_LOAD_OFF(voffp
)
86 RSEQ_ASM_OP_R_FINAL_STORE(v
, 3)
88 RSEQ_ASM_DEFINE_ABORT(4, abort
)
89 : /* gcc asm goto does not allow outputs */
91 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
92 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
94 [expectnot
] "r" (expectnot
),
98 : "memory", RSEQ_ASM_TMP_REG_1
101 #ifdef RSEQ_COMPARE_TWICE
111 #ifdef RSEQ_COMPARE_TWICE
113 rseq_bug("cpu_id comparison failed");
115 rseq_bug("expected value comparison failed");
119 static inline __attribute__((always_inline
))
120 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv
)(intptr_t *v
, intptr_t count
, int cpu
)
124 __asm__ __volatile__
goto(RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
125 #ifdef RSEQ_COMPARE_TWICE
126 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error1]")
128 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
129 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
131 #ifdef RSEQ_COMPARE_TWICE
132 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, "%l[error1]")
134 RSEQ_ASM_OP_R_LOAD(v
)
135 RSEQ_ASM_OP_R_ADD(count
)
136 RSEQ_ASM_OP_R_FINAL_STORE(v
, 3)
138 RSEQ_ASM_DEFINE_ABORT(4, abort
)
139 : /* gcc asm goto does not allow outputs */
140 : [cpu_id
] "r" (cpu
),
141 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
142 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
146 : "memory", RSEQ_ASM_TMP_REG_1
149 #ifdef RSEQ_COMPARE_TWICE
157 #ifdef RSEQ_COMPARE_TWICE
159 rseq_bug("cpu_id comparison failed");
163 static inline __attribute__((always_inline
))
164 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev
)(intptr_t *v
, intptr_t expect
,
165 intptr_t *v2
, intptr_t expect2
,
166 intptr_t newv
, int cpu
)
170 __asm__ __volatile__
goto(RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
171 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[cmpfail]")
172 #ifdef RSEQ_COMPARE_TWICE
173 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error1]")
174 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error2]")
175 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error3]")
177 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
178 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
180 RSEQ_ASM_OP_CMPEQ(v
, expect
, "%l[cmpfail]")
182 RSEQ_ASM_OP_CMPEQ(v2
, expect2
, "%l[cmpfail]")
184 #ifdef RSEQ_COMPARE_TWICE
185 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, "%l[error1]")
186 RSEQ_ASM_OP_CMPEQ(v
, expect
, "%l[error2]")
187 RSEQ_ASM_OP_CMPEQ(v2
, expect2
, "%l[error3]")
189 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
191 RSEQ_ASM_DEFINE_ABORT(4, abort
)
192 : /* gcc asm goto does not allow outputs */
193 : [cpu_id
] "r" (cpu
),
194 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
195 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
197 [expect
] "r" (expect
),
199 [expect2
] "r" (expect2
),
202 : "memory", RSEQ_ASM_TMP_REG_1
205 #ifdef RSEQ_COMPARE_TWICE
206 , error1
, error2
, error3
216 #ifdef RSEQ_COMPARE_TWICE
218 rseq_bug("cpu_id comparison failed");
220 rseq_bug("expected value comparison failed");
222 rseq_bug("2nd expected value comparison failed");
226 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
232 static inline __attribute__((always_inline
))
233 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv
)(intptr_t *ptr
, off_t off
, intptr_t inc
, int cpu
)
237 __asm__ __volatile__
goto(RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
238 #ifdef RSEQ_COMPARE_TWICE
239 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error1]")
241 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
242 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
244 #ifdef RSEQ_COMPARE_TWICE
245 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, "%l[error1]")
247 RSEQ_ASM_OP_R_DEREF_ADDV(ptr
, off
, 3)
249 RSEQ_ASM_DEFINE_ABORT(4, abort
)
250 : /* gcc asm goto does not allow outputs */
251 : [cpu_id
] "r" (cpu
),
252 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
253 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
258 : "memory", RSEQ_ASM_TMP_REG_1
261 #ifdef RSEQ_COMPARE_TWICE
269 #ifdef RSEQ_COMPARE_TWICE
271 rseq_bug("cpu_id comparison failed");
275 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
276 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
278 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
279 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
281 static inline __attribute__((always_inline
))
282 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev
)(intptr_t *v
, intptr_t expect
,
283 intptr_t *v2
, intptr_t newv2
,
284 intptr_t newv
, int cpu
)
288 __asm__ __volatile__
goto(RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
289 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[cmpfail]")
290 #ifdef RSEQ_COMPARE_TWICE
291 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error1]")
292 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error2]")
294 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
295 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
297 RSEQ_ASM_OP_CMPEQ(v
, expect
, "%l[cmpfail]")
299 #ifdef RSEQ_COMPARE_TWICE
300 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, "%l[error1]")
301 RSEQ_ASM_OP_CMPEQ(v
, expect
, "%l[error2]")
303 RSEQ_ASM_OP_STORE(newv2
, v2
)
305 #ifdef RSEQ_TEMPLATE_MO_RELEASE
306 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv
, v
, 3)
308 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
311 RSEQ_ASM_DEFINE_ABORT(4, abort
)
312 : /* gcc asm goto does not allow outputs */
313 : [cpu_id
] "r" (cpu
),
314 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
315 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
316 [expect
] "r" (expect
),
322 : "memory", RSEQ_ASM_TMP_REG_1
325 #ifdef RSEQ_COMPARE_TWICE
336 #ifdef RSEQ_COMPARE_TWICE
338 rseq_bug("cpu_id comparison failed");
340 rseq_bug("expected value comparison failed");
344 static inline __attribute__((always_inline
))
345 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev
)(intptr_t *v
, intptr_t expect
,
346 void *dst
, void *src
, size_t len
,
347 intptr_t newv
, int cpu
)
350 __asm__ __volatile__
goto(RSEQ_ASM_DEFINE_TABLE(1, 2f
, 3f
, 4f
)
351 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[cmpfail]")
352 #ifdef RSEQ_COMPARE_TWICE
353 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error1]")
354 RSEQ_ASM_DEFINE_EXIT_POINT(2f
, "%l[error2]")
356 RSEQ_ASM_STORE_RSEQ_CS(2, 1b
, rseq_cs
)
357 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, 4f
)
359 RSEQ_ASM_OP_CMPEQ(v
, expect
, "%l[cmpfail]")
361 #ifdef RSEQ_COMPARE_TWICE
362 RSEQ_ASM_CMP_CPU_ID(cpu_id
, current_cpu_id
, "%l[error1]")
363 RSEQ_ASM_OP_CMPEQ(v
, expect
, "%l[error2]")
365 RSEQ_ASM_OP_R_BAD_MEMCPY(dst
, src
, len
)
367 #ifdef RSEQ_TEMPLATE_MO_RELEASE
368 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv
, v
, 3)
370 RSEQ_ASM_OP_FINAL_STORE(newv
, v
, 3)
373 RSEQ_ASM_DEFINE_ABORT(4, abort
)
374 : /* gcc asm goto does not allow outputs */
375 : [cpu_id
] "r" (cpu
),
376 [current_cpu_id
] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD
),
377 [rseq_cs
] "m" (rseq_get_abi()->rseq_cs
.arch
.ptr
),
378 [expect
] "r" (expect
),
385 : "memory", RSEQ_ASM_TMP_REG_1
, RSEQ_ASM_TMP_REG_2
,
386 RSEQ_ASM_TMP_REG_3
, RSEQ_ASM_TMP_REG_4
389 #ifdef RSEQ_COMPARE_TWICE
400 #ifdef RSEQ_COMPARE_TWICE
402 rseq_bug("cpu_id comparison failed");
404 rseq_bug("expected value comparison failed");
408 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
409 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
411 #include "rseq-bits-reset.h"