projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Clarify logic of RSEQ_ASM*_CMP* macros
[librseq.git] / include / rseq / rseq-riscv-bits.h
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
3
4 #include "rseq-bits-template.h"
5
6 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
7 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
8
9 static inline __attribute__((always_inline))
10 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
11 {
12 RSEQ_INJECT_C(9)
13
14 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
15 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
16 #ifdef RSEQ_COMPARE_TWICE
17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
18 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
19 #endif
20 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
21 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
22 RSEQ_INJECT_ASM(3)
23 RSEQ_ASM_OP_CBNE(v, expect, "%l[cmpfail]")
24 RSEQ_INJECT_ASM(4)
25 #ifdef RSEQ_COMPARE_TWICE
26 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
27 RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]")
28 #endif
29 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
30 RSEQ_INJECT_ASM(5)
31 RSEQ_ASM_DEFINE_ABORT(4, abort)
32 : /* gcc asm goto does not allow outputs */
33 : [cpu_id] "r" (cpu),
34 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
35 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
36 [v] "m" (*v),
37 [expect] "r" (expect),
38 [newv] "r" (newv)
39 RSEQ_INJECT_INPUT
40 : "memory", RSEQ_ASM_TMP_REG_1
41 RSEQ_INJECT_CLOBBER
42 : abort, cmpfail
43 #ifdef RSEQ_COMPARE_TWICE
44 , error1, error2
45 #endif
46 );
47
48 return 0;
49 abort:
50 RSEQ_INJECT_FAILED
51 return -1;
52 cmpfail:
53 return 1;
54 #ifdef RSEQ_COMPARE_TWICE
55 error1:
56 rseq_bug("cpu_id comparison failed");
57 error2:
58 rseq_bug("expected value comparison failed");
59 #endif
60 }
61
62 static inline __attribute__((always_inline))
63 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
64 off_t voffp, intptr_t *load, int cpu)
65 {
66 RSEQ_INJECT_C(9)
67
68 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
69 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
70 #ifdef RSEQ_COMPARE_TWICE
71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
72 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
73 #endif
74 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
75 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
76 RSEQ_INJECT_ASM(3)
77 RSEQ_ASM_OP_CBEQ(v, expectnot, "%l[cmpfail]")
78 RSEQ_INJECT_ASM(4)
79 #ifdef RSEQ_COMPARE_TWICE
80 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
81 RSEQ_ASM_OP_CBEQ(v, expectnot, "%l[error2]")
82 #endif
83 RSEQ_ASM_OP_R_LOAD(v)
84 RSEQ_ASM_OP_R_STORE(load)
85 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
86 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
87 RSEQ_INJECT_ASM(5)
88 RSEQ_ASM_DEFINE_ABORT(4, abort)
89 : /* gcc asm goto does not allow outputs */
90 : [cpu_id] "r" (cpu),
91 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
92 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
93 [v] "m" (*v),
94 [expectnot] "r" (expectnot),
95 [load] "m" (*load),
96 [voffp] "r" (voffp)
97 RSEQ_INJECT_INPUT
98 : "memory", RSEQ_ASM_TMP_REG_1
99 RSEQ_INJECT_CLOBBER
100 : abort, cmpfail
101 #ifdef RSEQ_COMPARE_TWICE
102 , error1, error2
103 #endif
104 );
105 return 0;
106 abort:
107 RSEQ_INJECT_FAILED
108 return -1;
109 cmpfail:
110 return 1;
111 #ifdef RSEQ_COMPARE_TWICE
112 error1:
113 rseq_bug("cpu_id comparison failed");
114 error2:
115 rseq_bug("expected value comparison failed");
116 #endif
117 }
118
119 static inline __attribute__((always_inline))
120 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
121 {
122 RSEQ_INJECT_C(9)
123
124 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
125 #ifdef RSEQ_COMPARE_TWICE
126 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
127 #endif
128 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
129 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
130 RSEQ_INJECT_ASM(3)
131 #ifdef RSEQ_COMPARE_TWICE
132 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
133 #endif
134 RSEQ_ASM_OP_R_LOAD(v)
135 RSEQ_ASM_OP_R_ADD(count)
136 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
137 RSEQ_INJECT_ASM(4)
138 RSEQ_ASM_DEFINE_ABORT(4, abort)
139 : /* gcc asm goto does not allow outputs */
140 : [cpu_id] "r" (cpu),
141 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
142 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
143 [v] "m" (*v),
144 [count] "r" (count)
145 RSEQ_INJECT_INPUT
146 : "memory", RSEQ_ASM_TMP_REG_1
147 RSEQ_INJECT_CLOBBER
148 : abort
149 #ifdef RSEQ_COMPARE_TWICE
150 , error1
151 #endif
152 );
153 return 0;
154 abort:
155 RSEQ_INJECT_FAILED
156 return -1;
157 #ifdef RSEQ_COMPARE_TWICE
158 error1:
159 rseq_bug("cpu_id comparison failed");
160 #endif
161 }
162
163 static inline __attribute__((always_inline))
164 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
165 intptr_t *v2, intptr_t expect2,
166 intptr_t newv, int cpu)
167 {
168 RSEQ_INJECT_C(9)
169
170 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
171 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
172 #ifdef RSEQ_COMPARE_TWICE
173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
175 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
176 #endif
177 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
178 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
179 RSEQ_INJECT_ASM(3)
180 RSEQ_ASM_OP_CBNE(v, expect, "%l[cmpfail]")
181 RSEQ_INJECT_ASM(4)
182 RSEQ_ASM_OP_CBNE(v2, expect2, "%l[cmpfail]")
183 RSEQ_INJECT_ASM(5)
184 #ifdef RSEQ_COMPARE_TWICE
185 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
186 RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]")
187 RSEQ_ASM_OP_CBNE(v2, expect2, "%l[error3]")
188 #endif
189 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
190 RSEQ_INJECT_ASM(6)
191 RSEQ_ASM_DEFINE_ABORT(4, abort)
192 : /* gcc asm goto does not allow outputs */
193 : [cpu_id] "r" (cpu),
194 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
195 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
196 [v] "m" (*v),
197 [expect] "r" (expect),
198 [v2] "m" (*v2),
199 [expect2] "r" (expect2),
200 [newv] "r" (newv)
201 RSEQ_INJECT_INPUT
202 : "memory", RSEQ_ASM_TMP_REG_1
203 RSEQ_INJECT_CLOBBER
204 : abort, cmpfail
205 #ifdef RSEQ_COMPARE_TWICE
206 , error1, error2, error3
207 #endif
208 );
209
210 return 0;
211 abort:
212 RSEQ_INJECT_FAILED
213 return -1;
214 cmpfail:
215 return 1;
216 #ifdef RSEQ_COMPARE_TWICE
217 error1:
218 rseq_bug("cpu_id comparison failed");
219 error2:
220 rseq_bug("expected value comparison failed");
221 error3:
222 rseq_bug("2nd expected value comparison failed");
223 #endif
224 }
225
226 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
227
228 /*
229 * pval = *(ptr+off)
230 * *pval += inc;
231 */
232 static inline __attribute__((always_inline))
233 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
234 {
235 RSEQ_INJECT_C(9)
236
237 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
238 #ifdef RSEQ_COMPARE_TWICE
239 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
240 #endif
241 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
242 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
243 RSEQ_INJECT_ASM(3)
244 #ifdef RSEQ_COMPARE_TWICE
245 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
246 #endif
247 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
248 RSEQ_INJECT_ASM(4)
249 RSEQ_ASM_DEFINE_ABORT(4, abort)
250 : /* gcc asm goto does not allow outputs */
251 : [cpu_id] "r" (cpu),
252 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
253 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
254 [ptr] "r" (ptr),
255 [off] "er" (off),
256 [inc] "er" (inc)
257 RSEQ_INJECT_INPUT
258 : "memory", RSEQ_ASM_TMP_REG_1
259 RSEQ_INJECT_CLOBBER
260 : abort
261 #ifdef RSEQ_COMPARE_TWICE
262 , error1
263 #endif
264 );
265 return 0;
266 abort:
267 RSEQ_INJECT_FAILED
268 return -1;
269 #ifdef RSEQ_COMPARE_TWICE
270 error1:
271 rseq_bug("cpu_id comparison failed");
272 #endif
273 }
274
275 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
276 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
277
278 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
279 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
280
281 static inline __attribute__((always_inline))
282 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
283 intptr_t *v2, intptr_t newv2,
284 intptr_t newv, int cpu)
285 {
286 RSEQ_INJECT_C(9)
287
288 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
289 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
290 #ifdef RSEQ_COMPARE_TWICE
291 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
292 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
293 #endif
294 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
295 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
296 RSEQ_INJECT_ASM(3)
297 RSEQ_ASM_OP_CBNE(v, expect, "%l[cmpfail]")
298 RSEQ_INJECT_ASM(4)
299 #ifdef RSEQ_COMPARE_TWICE
300 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
301 RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]")
302 #endif
303 RSEQ_ASM_OP_STORE(newv2, v2)
304 RSEQ_INJECT_ASM(5)
305 #ifdef RSEQ_TEMPLATE_MO_RELEASE
306 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
307 #else
308 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
309 #endif
310 RSEQ_INJECT_ASM(6)
311 RSEQ_ASM_DEFINE_ABORT(4, abort)
312 : /* gcc asm goto does not allow outputs */
313 : [cpu_id] "r" (cpu),
314 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
315 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
316 [expect] "r" (expect),
317 [v] "m" (*v),
318 [newv] "r" (newv),
319 [v2] "m" (*v2),
320 [newv2] "r" (newv2)
321 RSEQ_INJECT_INPUT
322 : "memory", RSEQ_ASM_TMP_REG_1
323 RSEQ_INJECT_CLOBBER
324 : abort, cmpfail
325 #ifdef RSEQ_COMPARE_TWICE
326 , error1, error2
327 #endif
328 );
329
330 return 0;
331 abort:
332 RSEQ_INJECT_FAILED
333 return -1;
334 cmpfail:
335 return 1;
336 #ifdef RSEQ_COMPARE_TWICE
337 error1:
338 rseq_bug("cpu_id comparison failed");
339 error2:
340 rseq_bug("expected value comparison failed");
341 #endif
342 }
343
344 static inline __attribute__((always_inline))
345 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
346 void *dst, void *src, size_t len,
347 intptr_t newv, int cpu)
348 {
349 RSEQ_INJECT_C(9)
350 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
351 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
352 #ifdef RSEQ_COMPARE_TWICE
353 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
354 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
355 #endif
356 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
357 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
358 RSEQ_INJECT_ASM(3)
359 RSEQ_ASM_OP_CBNE(v, expect, "%l[cmpfail]")
360 RSEQ_INJECT_ASM(4)
361 #ifdef RSEQ_COMPARE_TWICE
362 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
363 RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]")
364 #endif
365 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
366 RSEQ_INJECT_ASM(5)
367 #ifdef RSEQ_TEMPLATE_MO_RELEASE
368 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
369 #else
370 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
371 #endif
372 RSEQ_INJECT_ASM(6)
373 RSEQ_ASM_DEFINE_ABORT(4, abort)
374 : /* gcc asm goto does not allow outputs */
375 : [cpu_id] "r" (cpu),
376 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
377 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
378 [expect] "r" (expect),
379 [v] "m" (*v),
380 [newv] "r" (newv),
381 [dst] "r" (dst),
382 [src] "r" (src),
383 [len] "r" (len)
384 RSEQ_INJECT_INPUT
385 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
386 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
387 RSEQ_INJECT_CLOBBER
388 : abort, cmpfail
389 #ifdef RSEQ_COMPARE_TWICE
390 , error1, error2
391 #endif
392 );
393
394 return 0;
395 abort:
396 RSEQ_INJECT_FAILED
397 return -1;
398 cmpfail:
399 return 1;
400 #ifdef RSEQ_COMPARE_TWICE
401 error1:
402 rseq_bug("cpu_id comparison failed");
403 error2:
404 rseq_bug("expected value comparison failed");
405 #endif
406 }
407
408 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
409 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
410
411 #include "rseq-bits-reset.h"
Librseq
This page took 0.039398 seconds and 5 git commands to generate.