Commit | Line | Data |
---|---|---|
90702366 | 1 | /* SPDX-License-Identifier: MIT */ |
f2d7b530 MJ |
2 | /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ |
3 | ||
784b0012 MD |
4 | /* |
5 | * rseq-x86.h | |
784b0012 MD |
6 | */ |
7 | ||
809f5ee3 MD |
8 | #ifndef RSEQ_H |
9 | #error "Never use <rseq-x86.h> directly; include <rseq.h> instead." | |
10 | #endif | |
11 | ||
784b0012 MD |
12 | #include <stdint.h> |
13 | ||
a5fe9e95 MD |
14 | /* |
15 | * RSEQ_SIG is used with the following reserved undefined instructions, which | |
16 | * trap in user-space: | |
17 | * | |
18 | * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi | |
19 | * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi | |
20 | */ | |
784b0012 MD |
21 | #define RSEQ_SIG 0x53053053 |
22 | ||
809f5ee3 MD |
23 | /* |
24 | * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input | |
25 | * operands, we cannot use "m" input operands, and rather pass the __rseq_abi | |
26 | * address through a "r" input operand. | |
27 | */ | |
28 | ||
29 | /* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */ | |
05ebd3f9 MD |
30 | #define RSEQ_CPU_ID_OFFSET 4 |
31 | #define RSEQ_CS_OFFSET 8 | |
809f5ee3 | 32 | #define RSEQ_MM_CID_OFFSET 24 |
05ebd3f9 | 33 | |
784b0012 MD |
34 | #ifdef __x86_64__ |
35 | ||
7dc4bc29 MD |
36 | #define RSEQ_ASM_TP_SEGMENT %%fs |
37 | ||
784b0012 MD |
38 | #define rseq_smp_mb() \ |
39 | __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") | |
40 | #define rseq_smp_rmb() rseq_barrier() | |
41 | #define rseq_smp_wmb() rseq_barrier() | |
42 | ||
43 | #define rseq_smp_load_acquire(p) \ | |
44 | __extension__ ({ \ | |
3664a718 | 45 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
784b0012 MD |
46 | rseq_barrier(); \ |
47 | ____p1; \ | |
48 | }) | |
49 | ||
50 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
51 | ||
52 | #define rseq_smp_store_release(p, v) \ | |
53 | do { \ | |
54 | rseq_barrier(); \ | |
809f5ee3 | 55 | RSEQ_WRITE_ONCE(*(p), v); \ |
784b0012 MD |
56 | } while (0) |
57 | ||
784b0012 MD |
58 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ |
59 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 60 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
61 | ".balign 32\n\t" \ |
62 | __rseq_str(label) ":\n\t" \ | |
63 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
64 | ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
65 | ".popsection\n\t" \ |
66 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
67 | ".quad " __rseq_str(label) "b\n\t" \ | |
784b0012 MD |
68 | ".popsection\n\t" |
69 | ||
dd01d0fb | 70 | |
784b0012 MD |
71 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
72 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
73 | (post_commit_ip - start_ip), abort_ip) | |
74 | ||
90d9876e MD |
75 | /* |
76 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
77 | * of the critical section where a critical section can either branch to or |
78 | * reach through the normal course of its execution. The abort IP and the | |
79 | * post-commit IP are already part of the __rseq_cs section and should not be | |
80 | * explicitly defined as additional exit points. Knowing all exit points is | |
81 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
82 | */ |
83 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
84 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
85 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ | |
86 | ".popsection\n\t" | |
87 | ||
784b0012 MD |
88 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
89 | RSEQ_INJECT_ASM(1) \ | |
90 | "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ | |
05ebd3f9 | 91 | "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
92 | __rseq_str(label) ":\n\t" |
93 | ||
769ec9a5 | 94 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ |
784b0012 | 95 | RSEQ_INJECT_ASM(2) \ |
05ebd3f9 | 96 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
97 | "jnz " __rseq_str(label) "\n\t" |
98 | ||
99 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
100 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
101 | /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ |
102 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
103 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
104 | __rseq_str(label) ":\n\t" \ | |
105 | teardown \ | |
106 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
107 | ".popsection\n\t" | |
108 | ||
109 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
110 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
111 | __rseq_str(label) ":\n\t" \ | |
112 | teardown \ | |
113 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ | |
114 | ".popsection\n\t" | |
115 | ||
f1c6b55b | 116 | #elif defined(__i386__) |
784b0012 | 117 | |
53831176 MD |
118 | #define RSEQ_ASM_TP_SEGMENT %%gs |
119 | ||
784b0012 MD |
120 | #define rseq_smp_mb() \ |
121 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
122 | #define rseq_smp_rmb() \ | |
123 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
124 | #define rseq_smp_wmb() \ | |
125 | __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") | |
126 | ||
127 | #define rseq_smp_load_acquire(p) \ | |
128 | __extension__ ({ \ | |
3664a718 | 129 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
784b0012 MD |
130 | rseq_smp_mb(); \ |
131 | ____p1; \ | |
132 | }) | |
133 | ||
134 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
135 | ||
136 | #define rseq_smp_store_release(p, v) \ | |
137 | do { \ | |
138 | rseq_smp_mb(); \ | |
809f5ee3 | 139 | RSEQ_WRITE_ONCE(*p, v); \ |
784b0012 MD |
140 | } while (0) |
141 | ||
784b0012 MD |
142 | /* |
143 | * Use eax as scratch register and take memory operands as input to | |
144 | * lessen register pressure. Especially needed when compiling in O0. | |
145 | */ | |
146 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
147 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 148 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
149 | ".balign 32\n\t" \ |
150 | __rseq_str(label) ":\n\t" \ | |
151 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
152 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ | |
dd01d0fb MD |
153 | ".popsection\n\t" \ |
154 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
155 | ".long " __rseq_str(label) "b, 0x0\n\t" \ | |
784b0012 MD |
156 | ".popsection\n\t" |
157 | ||
158 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ | |
159 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
160 | (post_commit_ip - start_ip), abort_ip) | |
161 | ||
90d9876e MD |
162 | /* |
163 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
164 | * of the critical section where a critical section can either branch to or |
165 | * reach through the normal course of its execution. The abort IP and the | |
166 | * post-commit IP are already part of the __rseq_cs section and should not be | |
167 | * explicitly defined as additional exit points. Knowing all exit points is | |
168 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
169 | */ |
170 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
171 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
172 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ | |
173 | ".popsection\n\t" | |
174 | ||
784b0012 MD |
175 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
176 | RSEQ_INJECT_ASM(1) \ | |
05ebd3f9 | 177 | "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
178 | __rseq_str(label) ":\n\t" |
179 | ||
769ec9a5 | 180 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ |
784b0012 | 181 | RSEQ_INJECT_ASM(2) \ |
05ebd3f9 | 182 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
183 | "jnz " __rseq_str(label) "\n\t" |
184 | ||
185 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
186 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
187 | /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ |
188 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
189 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
190 | __rseq_str(label) ":\n\t" \ | |
191 | teardown \ | |
192 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
193 | ".popsection\n\t" | |
194 | ||
195 | #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ | |
196 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
197 | __rseq_str(label) ":\n\t" \ | |
198 | teardown \ | |
199 | "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ | |
200 | ".popsection\n\t" | |
201 | ||
784b0012 | 202 | #endif |
784b0012 | 203 | |
809f5ee3 | 204 | /* Per-cpu-id indexing. */ |
784b0012 | 205 | |
809f5ee3 MD |
206 | #define RSEQ_TEMPLATE_CPU_ID |
207 | #define RSEQ_TEMPLATE_MO_RELAXED | |
208 | #include "rseq-x86-bits.h" | |
209 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
784b0012 | 210 | |
809f5ee3 MD |
211 | #define RSEQ_TEMPLATE_MO_RELEASE |
212 | #include "rseq-x86-bits.h" | |
213 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
214 | #undef RSEQ_TEMPLATE_CPU_ID | |
784b0012 | 215 | |
809f5ee3 | 216 | /* Per-mm-cid indexing. */ |
784b0012 | 217 | |
809f5ee3 MD |
218 | #define RSEQ_TEMPLATE_MM_CID |
219 | #define RSEQ_TEMPLATE_MO_RELAXED | |
220 | #include "rseq-x86-bits.h" | |
221 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
784b0012 | 222 | |
809f5ee3 MD |
223 | #define RSEQ_TEMPLATE_MO_RELEASE |
224 | #include "rseq-x86-bits.h" | |
225 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
226 | #undef RSEQ_TEMPLATE_MM_CID | |
de28c254 | 227 | |
809f5ee3 | 228 | /* APIs which are not based on cpu ids. */ |
de28c254 | 229 | |
809f5ee3 MD |
230 | #define RSEQ_TEMPLATE_CPU_ID_NONE |
231 | #define RSEQ_TEMPLATE_MO_RELAXED | |
232 | #include "rseq-x86-bits.h" | |
233 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
234 | #undef RSEQ_TEMPLATE_CPU_ID_NONE |