projects / librseq.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Add rseq critical section pseudocode documentation
[librseq.git] / include / rseq / rseq-x86-bits.h
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3 /*
4 * rseq-x86-bits.h
5 */
6
7 #include "rseq-bits-template.h"
8
9 /*
10 * Refer to rseq-pseudocode.h for pseudo-code of the rseq critical
11 * section helpers.
12 */
13 #include "rseq-pseudocode.h"
14
15 #ifdef __x86_64__
16
17 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
18 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
19
20 static inline __attribute__((always_inline))
21 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
22 {
23 RSEQ_INJECT_C(9)
24
25 __asm__ __volatile__ goto (
26 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
27 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
28 #ifdef RSEQ_COMPARE_TWICE
29 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
30 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
31 #endif
32 /* Start rseq by storing table entry pointer into rseq_cs. */
33 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
34 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
35 RSEQ_INJECT_ASM(3)
36 "cmpq %[v], %[expect]\n\t"
37 "jnz %l[cmpfail]\n\t"
38 RSEQ_INJECT_ASM(4)
39 #ifdef RSEQ_COMPARE_TWICE
40 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
41 "cmpq %[v], %[expect]\n\t"
42 "jnz %l[error2]\n\t"
43 #endif
44 /* final store */
45 "movq %[newv], %[v]\n\t"
46 "2:\n\t"
47 RSEQ_INJECT_ASM(5)
48 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
49 : /* gcc asm goto does not allow outputs */
50 : [cpu_id] "r" (cpu),
51 [rseq_offset] "r" (rseq_offset),
52 [v] "m" (*v),
53 [expect] "r" (expect),
54 [newv] "r" (newv)
55 : "memory", "cc", "rax"
56 RSEQ_INJECT_CLOBBER
57 : abort, cmpfail
58 #ifdef RSEQ_COMPARE_TWICE
59 , error1, error2
60 #endif
61 );
62 rseq_after_asm_goto();
63 return 0;
64 abort:
65 rseq_after_asm_goto();
66 RSEQ_INJECT_FAILED
67 return -1;
68 cmpfail:
69 rseq_after_asm_goto();
70 return 1;
71 #ifdef RSEQ_COMPARE_TWICE
72 error1:
73 rseq_after_asm_goto();
74 rseq_bug("cpu_id comparison failed");
75 error2:
76 rseq_after_asm_goto();
77 rseq_bug("expected value comparison failed");
78 #endif
79 }
80
81 /*
82 * Compare @v against @expectnot. When it does _not_ match, load @v
83 * into @load, and store the content of *@v + voffp into @v.
84 */
85 static inline __attribute__((always_inline))
86 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
87 long voffp, intptr_t *load, int cpu)
88 {
89 RSEQ_INJECT_C(9)
90
91 __asm__ __volatile__ goto (
92 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
93 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
94 #ifdef RSEQ_COMPARE_TWICE
95 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
96 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
97 #endif
98 /* Start rseq by storing table entry pointer into rseq_cs. */
99 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
100 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
101 RSEQ_INJECT_ASM(3)
102 "movq %[v], %%rbx\n\t"
103 "cmpq %%rbx, %[expectnot]\n\t"
104 "je %l[cmpfail]\n\t"
105 RSEQ_INJECT_ASM(4)
106 #ifdef RSEQ_COMPARE_TWICE
107 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
108 "movq %[v], %%rbx\n\t"
109 "cmpq %%rbx, %[expectnot]\n\t"
110 "je %l[error2]\n\t"
111 #endif
112 "movq %%rbx, %[load]\n\t"
113 "addq %[voffp], %%rbx\n\t"
114 "movq (%%rbx), %%rbx\n\t"
115 /* final store */
116 "movq %%rbx, %[v]\n\t"
117 "2:\n\t"
118 RSEQ_INJECT_ASM(5)
119 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
120 : /* gcc asm goto does not allow outputs */
121 : [cpu_id] "r" (cpu),
122 [rseq_offset] "r" (rseq_offset),
123 /* final store input */
124 [v] "m" (*v),
125 [expectnot] "r" (expectnot),
126 [voffp] "er" (voffp),
127 [load] "m" (*load)
128 : "memory", "cc", "rax", "rbx"
129 RSEQ_INJECT_CLOBBER
130 : abort, cmpfail
131 #ifdef RSEQ_COMPARE_TWICE
132 , error1, error2
133 #endif
134 );
135 rseq_after_asm_goto();
136 return 0;
137 abort:
138 rseq_after_asm_goto();
139 RSEQ_INJECT_FAILED
140 return -1;
141 cmpfail:
142 rseq_after_asm_goto();
143 return 1;
144 #ifdef RSEQ_COMPARE_TWICE
145 error1:
146 rseq_after_asm_goto();
147 rseq_bug("cpu_id comparison failed");
148 error2:
149 rseq_after_asm_goto();
150 rseq_bug("expected value comparison failed");
151 #endif
152 }
153
154 static inline __attribute__((always_inline))
155 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
156 {
157 RSEQ_INJECT_C(9)
158
159 __asm__ __volatile__ goto (
160 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
161 #ifdef RSEQ_COMPARE_TWICE
162 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
163 #endif
164 /* Start rseq by storing table entry pointer into rseq_cs. */
165 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
166 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
167 RSEQ_INJECT_ASM(3)
168 #ifdef RSEQ_COMPARE_TWICE
169 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
170 #endif
171 /* final store */
172 "addq %[count], %[v]\n\t"
173 "2:\n\t"
174 RSEQ_INJECT_ASM(4)
175 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
176 : /* gcc asm goto does not allow outputs */
177 : [cpu_id] "r" (cpu),
178 [rseq_offset] "r" (rseq_offset),
179 /* final store input */
180 [v] "m" (*v),
181 [count] "er" (count)
182 : "memory", "cc", "rax"
183 RSEQ_INJECT_CLOBBER
184 : abort
185 #ifdef RSEQ_COMPARE_TWICE
186 , error1
187 #endif
188 );
189 rseq_after_asm_goto();
190 return 0;
191 abort:
192 rseq_after_asm_goto();
193 RSEQ_INJECT_FAILED
194 return -1;
195 #ifdef RSEQ_COMPARE_TWICE
196 error1:
197 rseq_after_asm_goto();
198 rseq_bug("cpu_id comparison failed");
199 #endif
200 }
201
202 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
203
204 /*
205 * pval = *(ptr+off)
206 * *pval += inc;
207 */
208 static inline __attribute__((always_inline))
209 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu)
210 {
211 RSEQ_INJECT_C(9)
212
213 __asm__ __volatile__ goto (
214 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
215 #ifdef RSEQ_COMPARE_TWICE
216 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
217 #endif
218 /* Start rseq by storing table entry pointer into rseq_cs. */
219 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
220 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
221 RSEQ_INJECT_ASM(3)
222 #ifdef RSEQ_COMPARE_TWICE
223 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
224 #endif
225 /* get p+v */
226 "movq %[ptr], %%rbx\n\t"
227 "addq %[off], %%rbx\n\t"
228 /* get pv */
229 "movq (%%rbx), %%rcx\n\t"
230 /* *pv += inc */
231 "addq %[inc], (%%rcx)\n\t"
232 "2:\n\t"
233 RSEQ_INJECT_ASM(4)
234 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
235 : /* gcc asm goto does not allow outputs */
236 : [cpu_id] "r" (cpu),
237 [rseq_offset] "r" (rseq_offset),
238 /* final store input */
239 [ptr] "m" (*ptr),
240 [off] "er" (off),
241 [inc] "er" (inc)
242 : "memory", "cc", "rax", "rbx", "rcx"
243 RSEQ_INJECT_CLOBBER
244 : abort
245 #ifdef RSEQ_COMPARE_TWICE
246 , error1
247 #endif
248 );
249 return 0;
250 abort:
251 RSEQ_INJECT_FAILED
252 return -1;
253 #ifdef RSEQ_COMPARE_TWICE
254 error1:
255 rseq_bug("cpu_id comparison failed");
256 #endif
257 }
258
259 static inline __attribute__((always_inline))
260 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
261 intptr_t *v2, intptr_t expect2,
262 intptr_t newv, int cpu)
263 {
264 RSEQ_INJECT_C(9)
265
266 __asm__ __volatile__ goto (
267 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
268 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
269 #ifdef RSEQ_COMPARE_TWICE
270 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
271 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
272 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
273 #endif
274 /* Start rseq by storing table entry pointer into rseq_cs. */
275 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
276 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
277 RSEQ_INJECT_ASM(3)
278 "cmpq %[v], %[expect]\n\t"
279 "jnz %l[cmpfail]\n\t"
280 RSEQ_INJECT_ASM(4)
281 "cmpq %[v2], %[expect2]\n\t"
282 "jnz %l[cmpfail]\n\t"
283 RSEQ_INJECT_ASM(5)
284 #ifdef RSEQ_COMPARE_TWICE
285 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
286 "cmpq %[v], %[expect]\n\t"
287 "jnz %l[error2]\n\t"
288 "cmpq %[v2], %[expect2]\n\t"
289 "jnz %l[error3]\n\t"
290 #endif
291 /* final store */
292 "movq %[newv], %[v]\n\t"
293 "2:\n\t"
294 RSEQ_INJECT_ASM(6)
295 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
296 : /* gcc asm goto does not allow outputs */
297 : [cpu_id] "r" (cpu),
298 [rseq_offset] "r" (rseq_offset),
299 /* cmp2 input */
300 [v2] "m" (*v2),
301 [expect2] "r" (expect2),
302 /* final store input */
303 [v] "m" (*v),
304 [expect] "r" (expect),
305 [newv] "r" (newv)
306 : "memory", "cc", "rax"
307 RSEQ_INJECT_CLOBBER
308 : abort, cmpfail
309 #ifdef RSEQ_COMPARE_TWICE
310 , error1, error2, error3
311 #endif
312 );
313 rseq_after_asm_goto();
314 return 0;
315 abort:
316 rseq_after_asm_goto();
317 RSEQ_INJECT_FAILED
318 return -1;
319 cmpfail:
320 rseq_after_asm_goto();
321 return 1;
322 #ifdef RSEQ_COMPARE_TWICE
323 error1:
324 rseq_after_asm_goto();
325 rseq_bug("cpu_id comparison failed");
326 error2:
327 rseq_after_asm_goto();
328 rseq_bug("1st expected value comparison failed");
329 error3:
330 rseq_after_asm_goto();
331 rseq_bug("2nd expected value comparison failed");
332 #endif
333 }
334
335 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
336 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
337
338 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
339 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
340
341 static inline __attribute__((always_inline))
342 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
343 intptr_t *v2, intptr_t newv2,
344 intptr_t newv, int cpu)
345 {
346 RSEQ_INJECT_C(9)
347
348 __asm__ __volatile__ goto (
349 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
350 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
351 #ifdef RSEQ_COMPARE_TWICE
352 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
353 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
354 #endif
355 /* Start rseq by storing table entry pointer into rseq_cs. */
356 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
357 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
358 RSEQ_INJECT_ASM(3)
359 "cmpq %[v], %[expect]\n\t"
360 "jnz %l[cmpfail]\n\t"
361 RSEQ_INJECT_ASM(4)
362 #ifdef RSEQ_COMPARE_TWICE
363 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
364 "cmpq %[v], %[expect]\n\t"
365 "jnz %l[error2]\n\t"
366 #endif
367 /* try store */
368 "movq %[newv2], %[v2]\n\t"
369 RSEQ_INJECT_ASM(5)
370 /* final store */
371 "movq %[newv], %[v]\n\t"
372 "2:\n\t"
373 RSEQ_INJECT_ASM(6)
374 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
375 : /* gcc asm goto does not allow outputs */
376 : [cpu_id] "r" (cpu),
377 [rseq_offset] "r" (rseq_offset),
378 /* try store input */
379 [v2] "m" (*v2),
380 [newv2] "r" (newv2),
381 /* final store input */
382 [v] "m" (*v),
383 [expect] "r" (expect),
384 [newv] "r" (newv)
385 : "memory", "cc", "rax"
386 RSEQ_INJECT_CLOBBER
387 : abort, cmpfail
388 #ifdef RSEQ_COMPARE_TWICE
389 , error1, error2
390 #endif
391 );
392 rseq_after_asm_goto();
393 return 0;
394 abort:
395 rseq_after_asm_goto();
396 RSEQ_INJECT_FAILED
397 return -1;
398 cmpfail:
399 rseq_after_asm_goto();
400 return 1;
401 #ifdef RSEQ_COMPARE_TWICE
402 error1:
403 rseq_after_asm_goto();
404 rseq_bug("cpu_id comparison failed");
405 error2:
406 rseq_after_asm_goto();
407 rseq_bug("expected value comparison failed");
408 #endif
409 }
410
411 static inline __attribute__((always_inline))
412 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
413 void *dst, void *src, size_t len,
414 intptr_t newv, int cpu)
415 {
416 uint64_t rseq_scratch[3];
417
418 RSEQ_INJECT_C(9)
419
420 __asm__ __volatile__ goto (
421 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
422 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
423 #ifdef RSEQ_COMPARE_TWICE
424 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
425 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
426 #endif
427 "movq %[src], %[rseq_scratch0]\n\t"
428 "movq %[dst], %[rseq_scratch1]\n\t"
429 "movq %[len], %[rseq_scratch2]\n\t"
430 /* Start rseq by storing table entry pointer into rseq_cs. */
431 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
432 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
433 RSEQ_INJECT_ASM(3)
434 "cmpq %[v], %[expect]\n\t"
435 "jnz 5f\n\t"
436 RSEQ_INJECT_ASM(4)
437 #ifdef RSEQ_COMPARE_TWICE
438 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)
439 "cmpq %[v], %[expect]\n\t"
440 "jnz 7f\n\t"
441 #endif
442 /* try memcpy */
443 "test %[len], %[len]\n\t" \
444 "jz 333f\n\t" \
445 "222:\n\t" \
446 "movb (%[src]), %%al\n\t" \
447 "movb %%al, (%[dst])\n\t" \
448 "inc %[src]\n\t" \
449 "inc %[dst]\n\t" \
450 "dec %[len]\n\t" \
451 "jnz 222b\n\t" \
452 "333:\n\t" \
453 RSEQ_INJECT_ASM(5)
454 /* final store */
455 "movq %[newv], %[v]\n\t"
456 "2:\n\t"
457 RSEQ_INJECT_ASM(6)
458 /* teardown */
459 "movq %[rseq_scratch2], %[len]\n\t"
460 "movq %[rseq_scratch1], %[dst]\n\t"
461 "movq %[rseq_scratch0], %[src]\n\t"
462 RSEQ_ASM_DEFINE_ABORT(4,
463 "movq %[rseq_scratch2], %[len]\n\t"
464 "movq %[rseq_scratch1], %[dst]\n\t"
465 "movq %[rseq_scratch0], %[src]\n\t",
466 abort)
467 RSEQ_ASM_DEFINE_CMPFAIL(5,
468 "movq %[rseq_scratch2], %[len]\n\t"
469 "movq %[rseq_scratch1], %[dst]\n\t"
470 "movq %[rseq_scratch0], %[src]\n\t",
471 cmpfail)
472 #ifdef RSEQ_COMPARE_TWICE
473 RSEQ_ASM_DEFINE_CMPFAIL(6,
474 "movq %[rseq_scratch2], %[len]\n\t"
475 "movq %[rseq_scratch1], %[dst]\n\t"
476 "movq %[rseq_scratch0], %[src]\n\t",
477 error1)
478 RSEQ_ASM_DEFINE_CMPFAIL(7,
479 "movq %[rseq_scratch2], %[len]\n\t"
480 "movq %[rseq_scratch1], %[dst]\n\t"
481 "movq %[rseq_scratch0], %[src]\n\t",
482 error2)
483 #endif
484 : /* gcc asm goto does not allow outputs */
485 : [cpu_id] "r" (cpu),
486 [rseq_offset] "r" (rseq_offset),
487 /* final store input */
488 [v] "m" (*v),
489 [expect] "r" (expect),
490 [newv] "r" (newv),
491 /* try memcpy input */
492 [dst] "r" (dst),
493 [src] "r" (src),
494 [len] "r" (len),
495 [rseq_scratch0] "m" (rseq_scratch[0]),
496 [rseq_scratch1] "m" (rseq_scratch[1]),
497 [rseq_scratch2] "m" (rseq_scratch[2])
498 : "memory", "cc", "rax"
499 RSEQ_INJECT_CLOBBER
500 : abort, cmpfail
501 #ifdef RSEQ_COMPARE_TWICE
502 , error1, error2
503 #endif
504 );
505 rseq_after_asm_goto();
506 return 0;
507 abort:
508 rseq_after_asm_goto();
509 RSEQ_INJECT_FAILED
510 return -1;
511 cmpfail:
512 rseq_after_asm_goto();
513 return 1;
514 #ifdef RSEQ_COMPARE_TWICE
515 error1:
516 rseq_after_asm_goto();
517 rseq_bug("cpu_id comparison failed");
518 error2:
519 rseq_after_asm_goto();
520 rseq_bug("expected value comparison failed");
521 #endif
522 }
523
524 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
525 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
526
527 #elif defined(__i386__)
528
529 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
530 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
531
532 static inline __attribute__((always_inline))
533 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
534 {
535 RSEQ_INJECT_C(9)
536
537 __asm__ __volatile__ goto (
538 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
539 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
540 #ifdef RSEQ_COMPARE_TWICE
541 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
542 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
543 #endif
544 /* Start rseq by storing table entry pointer into rseq_cs. */
545 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
546 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
547 RSEQ_INJECT_ASM(3)
548 "cmpl %[v], %[expect]\n\t"
549 "jnz %l[cmpfail]\n\t"
550 RSEQ_INJECT_ASM(4)
551 #ifdef RSEQ_COMPARE_TWICE
552 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
553 "cmpl %[v], %[expect]\n\t"
554 "jnz %l[error2]\n\t"
555 #endif
556 /* final store */
557 "movl %[newv], %[v]\n\t"
558 "2:\n\t"
559 RSEQ_INJECT_ASM(5)
560 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
561 : /* gcc asm goto does not allow outputs */
562 : [cpu_id] "r" (cpu),
563 [rseq_offset] "r" (rseq_offset),
564 [v] "m" (*v),
565 [expect] "r" (expect),
566 [newv] "r" (newv)
567 : "memory", "cc", "eax"
568 RSEQ_INJECT_CLOBBER
569 : abort, cmpfail
570 #ifdef RSEQ_COMPARE_TWICE
571 , error1, error2
572 #endif
573 );
574 rseq_after_asm_goto();
575 return 0;
576 abort:
577 rseq_after_asm_goto();
578 RSEQ_INJECT_FAILED
579 return -1;
580 cmpfail:
581 rseq_after_asm_goto();
582 return 1;
583 #ifdef RSEQ_COMPARE_TWICE
584 error1:
585 rseq_after_asm_goto();
586 rseq_bug("cpu_id comparison failed");
587 error2:
588 rseq_after_asm_goto();
589 rseq_bug("expected value comparison failed");
590 #endif
591 }
592
593 /*
594 * Compare @v against @expectnot. When it does _not_ match, load @v
595 * into @load, and store the content of *@v + voffp into @v.
596 */
597 static inline __attribute__((always_inline))
598 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
599 long voffp, intptr_t *load, int cpu)
600 {
601 RSEQ_INJECT_C(9)
602
603 __asm__ __volatile__ goto (
604 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
605 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
606 #ifdef RSEQ_COMPARE_TWICE
607 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
608 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
609 #endif
610 /* Start rseq by storing table entry pointer into rseq_cs. */
611 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
612 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
613 RSEQ_INJECT_ASM(3)
614 "movl %[v], %%ebx\n\t"
615 "cmpl %%ebx, %[expectnot]\n\t"
616 "je %l[cmpfail]\n\t"
617 RSEQ_INJECT_ASM(4)
618 #ifdef RSEQ_COMPARE_TWICE
619 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
620 "movl %[v], %%ebx\n\t"
621 "cmpl %%ebx, %[expectnot]\n\t"
622 "je %l[error2]\n\t"
623 #endif
624 "movl %%ebx, %[load]\n\t"
625 "addl %[voffp], %%ebx\n\t"
626 "movl (%%ebx), %%ebx\n\t"
627 /* final store */
628 "movl %%ebx, %[v]\n\t"
629 "2:\n\t"
630 RSEQ_INJECT_ASM(5)
631 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
632 : /* gcc asm goto does not allow outputs */
633 : [cpu_id] "r" (cpu),
634 [rseq_offset] "r" (rseq_offset),
635 /* final store input */
636 [v] "m" (*v),
637 [expectnot] "r" (expectnot),
638 [voffp] "ir" (voffp),
639 [load] "m" (*load)
640 : "memory", "cc", "eax", "ebx"
641 RSEQ_INJECT_CLOBBER
642 : abort, cmpfail
643 #ifdef RSEQ_COMPARE_TWICE
644 , error1, error2
645 #endif
646 );
647 rseq_after_asm_goto();
648 return 0;
649 abort:
650 rseq_after_asm_goto();
651 RSEQ_INJECT_FAILED
652 return -1;
653 cmpfail:
654 rseq_after_asm_goto();
655 return 1;
656 #ifdef RSEQ_COMPARE_TWICE
657 error1:
658 rseq_after_asm_goto();
659 rseq_bug("cpu_id comparison failed");
660 error2:
661 rseq_after_asm_goto();
662 rseq_bug("expected value comparison failed");
663 #endif
664 }
665
666 static inline __attribute__((always_inline))
667 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
668 {
669 RSEQ_INJECT_C(9)
670
671 __asm__ __volatile__ goto (
672 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
673 #ifdef RSEQ_COMPARE_TWICE
674 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
675 #endif
676 /* Start rseq by storing table entry pointer into rseq_cs. */
677 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
678 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
679 RSEQ_INJECT_ASM(3)
680 #ifdef RSEQ_COMPARE_TWICE
681 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
682 #endif
683 /* final store */
684 "addl %[count], %[v]\n\t"
685 "2:\n\t"
686 RSEQ_INJECT_ASM(4)
687 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
688 : /* gcc asm goto does not allow outputs */
689 : [cpu_id] "r" (cpu),
690 [rseq_offset] "r" (rseq_offset),
691 /* final store input */
692 [v] "m" (*v),
693 [count] "ir" (count)
694 : "memory", "cc", "eax"
695 RSEQ_INJECT_CLOBBER
696 : abort
697 #ifdef RSEQ_COMPARE_TWICE
698 , error1
699 #endif
700 );
701 rseq_after_asm_goto();
702 return 0;
703 abort:
704 rseq_after_asm_goto();
705 RSEQ_INJECT_FAILED
706 return -1;
707 #ifdef RSEQ_COMPARE_TWICE
708 error1:
709 rseq_after_asm_goto();
710 rseq_bug("cpu_id comparison failed");
711 #endif
712 }
713
714 static inline __attribute__((always_inline))
715 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
716 intptr_t *v2, intptr_t expect2,
717 intptr_t newv, int cpu)
718 {
719 RSEQ_INJECT_C(9)
720
721 __asm__ __volatile__ goto (
722 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
723 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
724 #ifdef RSEQ_COMPARE_TWICE
725 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
726 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
727 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
728 #endif
729 /* Start rseq by storing table entry pointer into rseq_cs. */
730 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
731 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
732 RSEQ_INJECT_ASM(3)
733 "cmpl %[v], %[expect]\n\t"
734 "jnz %l[cmpfail]\n\t"
735 RSEQ_INJECT_ASM(4)
736 "cmpl %[expect2], %[v2]\n\t"
737 "jnz %l[cmpfail]\n\t"
738 RSEQ_INJECT_ASM(5)
739 #ifdef RSEQ_COMPARE_TWICE
740 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
741 "cmpl %[v], %[expect]\n\t"
742 "jnz %l[error2]\n\t"
743 "cmpl %[expect2], %[v2]\n\t"
744 "jnz %l[error3]\n\t"
745 #endif
746 "movl %[newv], %%eax\n\t"
747 /* final store */
748 "movl %%eax, %[v]\n\t"
749 "2:\n\t"
750 RSEQ_INJECT_ASM(6)
751 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
752 : /* gcc asm goto does not allow outputs */
753 : [cpu_id] "r" (cpu),
754 [rseq_offset] "r" (rseq_offset),
755 /* cmp2 input */
756 [v2] "m" (*v2),
757 [expect2] "r" (expect2),
758 /* final store input */
759 [v] "m" (*v),
760 [expect] "r" (expect),
761 [newv] "m" (newv)
762 : "memory", "cc", "eax"
763 RSEQ_INJECT_CLOBBER
764 : abort, cmpfail
765 #ifdef RSEQ_COMPARE_TWICE
766 , error1, error2, error3
767 #endif
768 );
769 rseq_after_asm_goto();
770 return 0;
771 abort:
772 rseq_after_asm_goto();
773 RSEQ_INJECT_FAILED
774 return -1;
775 cmpfail:
776 rseq_after_asm_goto();
777 return 1;
778 #ifdef RSEQ_COMPARE_TWICE
779 error1:
780 rseq_after_asm_goto();
781 rseq_bug("cpu_id comparison failed");
782 error2:
783 rseq_after_asm_goto();
784 rseq_bug("1st expected value comparison failed");
785 error3:
786 rseq_after_asm_goto();
787 rseq_bug("2nd expected value comparison failed");
788 #endif
789 }
790
791 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
792 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
793
794 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
795 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
796
797 static inline __attribute__((always_inline))
798 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
799 intptr_t *v2, intptr_t newv2,
800 intptr_t newv, int cpu)
801 {
802 RSEQ_INJECT_C(9)
803
804 __asm__ __volatile__ goto (
805 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
806 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
807 #ifdef RSEQ_COMPARE_TWICE
808 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
809 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
810 #endif
811 /* Start rseq by storing table entry pointer into rseq_cs. */
812 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
813 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
814 RSEQ_INJECT_ASM(3)
815 "movl %[expect], %%eax\n\t"
816 "cmpl %[v], %%eax\n\t"
817 "jnz %l[cmpfail]\n\t"
818 RSEQ_INJECT_ASM(4)
819 #ifdef RSEQ_COMPARE_TWICE
820 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
821 "movl %[expect], %%eax\n\t"
822 "cmpl %[v], %%eax\n\t"
823 "jnz %l[error2]\n\t"
824 #endif
825 /* try store */
826 "movl %[newv2], %[v2]\n\t"
827 RSEQ_INJECT_ASM(5)
828 #ifdef RSEQ_TEMPLATE_MO_RELEASE
829 "lock; addl $0,-128(%%esp)\n\t"
830 #endif
831 /* final store */
832 "movl %[newv], %[v]\n\t"
833 "2:\n\t"
834 RSEQ_INJECT_ASM(6)
835 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
836 : /* gcc asm goto does not allow outputs */
837 : [cpu_id] "r" (cpu),
838 [rseq_offset] "r" (rseq_offset),
839 /* try store input */
840 [v2] "m" (*v2),
841 [newv2] "r" (newv2),
842 /* final store input */
843 [v] "m" (*v),
844 [expect] "m" (expect),
845 [newv] "r" (newv)
846 : "memory", "cc", "eax"
847 RSEQ_INJECT_CLOBBER
848 : abort, cmpfail
849 #ifdef RSEQ_COMPARE_TWICE
850 , error1, error2
851 #endif
852 );
853 rseq_after_asm_goto();
854 return 0;
855 abort:
856 rseq_after_asm_goto();
857 RSEQ_INJECT_FAILED
858 return -1;
859 cmpfail:
860 rseq_after_asm_goto();
861 return 1;
862 #ifdef RSEQ_COMPARE_TWICE
863 error1:
864 rseq_after_asm_goto();
865 rseq_bug("cpu_id comparison failed");
866 error2:
867 rseq_after_asm_goto();
868 rseq_bug("expected value comparison failed");
869 #endif
870
871 }
872
873 /* TODO: implement a faster memcpy. */
874 static inline __attribute__((always_inline))
875 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
876 void *dst, void *src, size_t len,
877 intptr_t newv, int cpu)
878 {
879 uint32_t rseq_scratch[3];
880
881 RSEQ_INJECT_C(9)
882
883 __asm__ __volatile__ goto (
884 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
885 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
886 #ifdef RSEQ_COMPARE_TWICE
887 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
888 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
889 #endif
890 "movl %[src], %[rseq_scratch0]\n\t"
891 "movl %[dst], %[rseq_scratch1]\n\t"
892 "movl %[len], %[rseq_scratch2]\n\t"
893 /* Start rseq by storing table entry pointer into rseq_cs. */
894 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
895 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
896 RSEQ_INJECT_ASM(3)
897 "movl %[expect], %%eax\n\t"
898 "cmpl %%eax, %[v]\n\t"
899 "jnz 5f\n\t"
900 RSEQ_INJECT_ASM(4)
901 #ifdef RSEQ_COMPARE_TWICE
902 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)
903 "movl %[expect], %%eax\n\t"
904 "cmpl %%eax, %[v]\n\t"
905 "jnz 7f\n\t"
906 #endif
907 /* try memcpy */
908 "test %[len], %[len]\n\t" \
909 "jz 333f\n\t" \
910 "222:\n\t" \
911 "movb (%[src]), %%al\n\t" \
912 "movb %%al, (%[dst])\n\t" \
913 "inc %[src]\n\t" \
914 "inc %[dst]\n\t" \
915 "dec %[len]\n\t" \
916 "jnz 222b\n\t" \
917 "333:\n\t" \
918 RSEQ_INJECT_ASM(5)
919 #ifdef RSEQ_TEMPLATE_MO_RELEASE
920 "lock; addl $0,-128(%%esp)\n\t"
921 #endif
922 "movl %[newv], %%eax\n\t"
923 /* final store */
924 "movl %%eax, %[v]\n\t"
925 "2:\n\t"
926 RSEQ_INJECT_ASM(6)
927 /* teardown */
928 "movl %[rseq_scratch2], %[len]\n\t"
929 "movl %[rseq_scratch1], %[dst]\n\t"
930 "movl %[rseq_scratch0], %[src]\n\t"
931 RSEQ_ASM_DEFINE_ABORT(4,
932 "movl %[rseq_scratch2], %[len]\n\t"
933 "movl %[rseq_scratch1], %[dst]\n\t"
934 "movl %[rseq_scratch0], %[src]\n\t",
935 abort)
936 RSEQ_ASM_DEFINE_CMPFAIL(5,
937 "movl %[rseq_scratch2], %[len]\n\t"
938 "movl %[rseq_scratch1], %[dst]\n\t"
939 "movl %[rseq_scratch0], %[src]\n\t",
940 cmpfail)
941 #ifdef RSEQ_COMPARE_TWICE
942 RSEQ_ASM_DEFINE_CMPFAIL(6,
943 "movl %[rseq_scratch2], %[len]\n\t"
944 "movl %[rseq_scratch1], %[dst]\n\t"
945 "movl %[rseq_scratch0], %[src]\n\t",
946 error1)
947 RSEQ_ASM_DEFINE_CMPFAIL(7,
948 "movl %[rseq_scratch2], %[len]\n\t"
949 "movl %[rseq_scratch1], %[dst]\n\t"
950 "movl %[rseq_scratch0], %[src]\n\t",
951 error2)
952 #endif
953 : /* gcc asm goto does not allow outputs */
954 : [cpu_id] "r" (cpu),
955 [rseq_offset] "r" (rseq_offset),
956 /* final store input */
957 [v] "m" (*v),
958 [expect] "m" (expect),
959 [newv] "m" (newv),
960 /* try memcpy input */
961 [dst] "r" (dst),
962 [src] "r" (src),
963 [len] "r" (len),
964 [rseq_scratch0] "m" (rseq_scratch[0]),
965 [rseq_scratch1] "m" (rseq_scratch[1]),
966 [rseq_scratch2] "m" (rseq_scratch[2])
967 : "memory", "cc", "eax"
968 RSEQ_INJECT_CLOBBER
969 : abort, cmpfail
970 #ifdef RSEQ_COMPARE_TWICE
971 , error1, error2
972 #endif
973 );
974 rseq_after_asm_goto();
975 return 0;
976 abort:
977 rseq_after_asm_goto();
978 RSEQ_INJECT_FAILED
979 return -1;
980 cmpfail:
981 rseq_after_asm_goto();
982 return 1;
983 #ifdef RSEQ_COMPARE_TWICE
984 error1:
985 rseq_after_asm_goto();
986 rseq_bug("cpu_id comparison failed");
987 error2:
988 rseq_after_asm_goto();
989 rseq_bug("expected value comparison failed");
990 #endif
991 }
992
993 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
994 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
995
996 #endif
997
998 #include "rseq-bits-reset.h"
Librseq
This page took 0.049152 seconds and 4 git commands to generate.