Commit | Line | Data |
---|---|---|
90702366 | 1 | /* SPDX-License-Identifier: MIT */ |
f2d7b530 MJ |
2 | /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ |
3 | ||
784b0012 MD |
4 | /* |
5 | * rseq-x86.h | |
784b0012 MD |
6 | */ |
7 | ||
809f5ee3 MD |
8 | #ifndef RSEQ_H |
9 | #error "Never use <rseq-x86.h> directly; include <rseq.h> instead." | |
10 | #endif | |
11 | ||
784b0012 MD |
12 | #include <stdint.h> |
13 | ||
a5fe9e95 MD |
14 | /* |
15 | * RSEQ_SIG is used with the following reserved undefined instructions, which | |
16 | * trap in user-space: | |
17 | * | |
18 | * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi | |
19 | * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi | |
20 | */ | |
784b0012 MD |
21 | #define RSEQ_SIG 0x53053053 |
22 | ||
809f5ee3 MD |
23 | /* |
24 | * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input | |
25 | * operands, we cannot use "m" input operands, and rather pass the __rseq_abi | |
26 | * address through a "r" input operand. | |
27 | */ | |
28 | ||
29 | /* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */ | |
d9c11a15 MD |
30 | #define RSEQ_ASM_CPU_ID_OFFSET 4 |
31 | #define RSEQ_ASM_CS_OFFSET 8 | |
32 | #define RSEQ_ASM_MM_CID_OFFSET 24 | |
05ebd3f9 | 33 | |
784b0012 MD |
34 | #define rseq_smp_mb() \ |
35 | __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") | |
36 | #define rseq_smp_rmb() rseq_barrier() | |
37 | #define rseq_smp_wmb() rseq_barrier() | |
38 | ||
39 | #define rseq_smp_load_acquire(p) \ | |
40 | __extension__ ({ \ | |
3664a718 | 41 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
784b0012 MD |
42 | rseq_barrier(); \ |
43 | ____p1; \ | |
44 | }) | |
45 | ||
46 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() | |
47 | ||
48 | #define rseq_smp_store_release(p, v) \ | |
49 | do { \ | |
50 | rseq_barrier(); \ | |
809f5ee3 | 51 | RSEQ_WRITE_ONCE(*(p), v); \ |
784b0012 MD |
52 | } while (0) |
53 | ||
94f43fc9 MD |
54 | #ifdef __x86_64__ |
55 | ||
56 | #define RSEQ_ASM_TP_SEGMENT %%fs | |
57 | ||
784b0012 MD |
58 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ |
59 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 60 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
61 | ".balign 32\n\t" \ |
62 | __rseq_str(label) ":\n\t" \ | |
63 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
64 | ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ | |
dd01d0fb MD |
65 | ".popsection\n\t" \ |
66 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
67 | ".quad " __rseq_str(label) "b\n\t" \ | |
784b0012 MD |
68 | ".popsection\n\t" |
69 | ||
dd01d0fb | 70 | |
784b0012 MD |
71 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
72 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
73 | (post_commit_ip - start_ip), abort_ip) | |
74 | ||
90d9876e MD |
75 | /* |
76 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
77 | * of the critical section where a critical section can either branch to or |
78 | * reach through the normal course of its execution. The abort IP and the | |
79 | * post-commit IP are already part of the __rseq_cs section and should not be | |
80 | * explicitly defined as additional exit points. Knowing all exit points is | |
81 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
82 | */ |
83 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
84 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
85 | ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ | |
86 | ".popsection\n\t" | |
87 | ||
784b0012 MD |
88 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
89 | RSEQ_INJECT_ASM(1) \ | |
90 | "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ | |
05ebd3f9 | 91 | "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
92 | __rseq_str(label) ":\n\t" |
93 | ||
769ec9a5 | 94 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ |
784b0012 | 95 | RSEQ_INJECT_ASM(2) \ |
05ebd3f9 | 96 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
97 | "jnz " __rseq_str(label) "\n\t" |
98 | ||
99 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
100 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
101 | /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ |
102 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
103 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
104 | __rseq_str(label) ":\n\t" \ | |
105 | teardown \ | |
106 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
107 | ".popsection\n\t" | |
108 | ||
559d824f | 109 | #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \ |
784b0012 MD |
110 | ".pushsection __rseq_failure, \"ax\"\n\t" \ |
111 | __rseq_str(label) ":\n\t" \ | |
112 | teardown \ | |
559d824f | 113 | "jmp %l[" __rseq_str(target_label) "]\n\t" \ |
784b0012 MD |
114 | ".popsection\n\t" |
115 | ||
f1c6b55b | 116 | #elif defined(__i386__) |
784b0012 | 117 | |
53831176 MD |
118 | #define RSEQ_ASM_TP_SEGMENT %%gs |
119 | ||
784b0012 MD |
120 | /* |
121 | * Use eax as scratch register and take memory operands as input to | |
122 | * lessen register pressure. Especially needed when compiling in O0. | |
123 | */ | |
124 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ | |
125 | start_ip, post_commit_offset, abort_ip) \ | |
dd01d0fb | 126 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 MD |
127 | ".balign 32\n\t" \ |
128 | __rseq_str(label) ":\n\t" \ | |
129 | ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
130 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ | |
dd01d0fb MD |
131 | ".popsection\n\t" \ |
132 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
133 | ".long " __rseq_str(label) "b, 0x0\n\t" \ | |
784b0012 MD |
134 | ".popsection\n\t" |
135 | ||
136 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ | |
137 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
138 | (post_commit_ip - start_ip), abort_ip) | |
139 | ||
90d9876e MD |
140 | /* |
141 | * Exit points of a rseq critical section consist of all instructions outside | |
fd622cad MD |
142 | * of the critical section where a critical section can either branch to or |
143 | * reach through the normal course of its execution. The abort IP and the | |
144 | * post-commit IP are already part of the __rseq_cs section and should not be | |
145 | * explicitly defined as additional exit points. Knowing all exit points is | |
146 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
147 | */ |
148 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
149 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
150 | ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ | |
151 | ".popsection\n\t" | |
152 | ||
784b0012 MD |
153 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
154 | RSEQ_INJECT_ASM(1) \ | |
05ebd3f9 | 155 | "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ |
784b0012 MD |
156 | __rseq_str(label) ":\n\t" |
157 | ||
769ec9a5 | 158 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ |
784b0012 | 159 | RSEQ_INJECT_ASM(2) \ |
05ebd3f9 | 160 | "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ |
784b0012 MD |
161 | "jnz " __rseq_str(label) "\n\t" |
162 | ||
163 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ | |
164 | ".pushsection __rseq_failure, \"ax\"\n\t" \ | |
a5fe9e95 MD |
165 | /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ |
166 | ".byte 0x0f, 0xb9, 0x3d\n\t" \ | |
784b0012 MD |
167 | ".long " __rseq_str(RSEQ_SIG) "\n\t" \ |
168 | __rseq_str(label) ":\n\t" \ | |
169 | teardown \ | |
170 | "jmp %l[" __rseq_str(abort_label) "]\n\t" \ | |
171 | ".popsection\n\t" | |
172 | ||
559d824f | 173 | #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \ |
784b0012 MD |
174 | ".pushsection __rseq_failure, \"ax\"\n\t" \ |
175 | __rseq_str(label) ":\n\t" \ | |
176 | teardown \ | |
559d824f | 177 | "jmp %l[" __rseq_str(target_label) "]\n\t" \ |
784b0012 MD |
178 | ".popsection\n\t" |
179 | ||
784b0012 | 180 | #endif |
784b0012 | 181 | |
809f5ee3 | 182 | /* Per-cpu-id indexing. */ |
784b0012 | 183 | |
abf9e855 | 184 | #define RSEQ_TEMPLATE_INDEX_CPU_ID |
809f5ee3 MD |
185 | #define RSEQ_TEMPLATE_MO_RELAXED |
186 | #include "rseq-x86-bits.h" | |
187 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
784b0012 | 188 | |
809f5ee3 MD |
189 | #define RSEQ_TEMPLATE_MO_RELEASE |
190 | #include "rseq-x86-bits.h" | |
191 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
abf9e855 | 192 | #undef RSEQ_TEMPLATE_INDEX_CPU_ID |
784b0012 | 193 | |
809f5ee3 | 194 | /* Per-mm-cid indexing. */ |
784b0012 | 195 | |
abf9e855 | 196 | #define RSEQ_TEMPLATE_INDEX_MM_CID |
809f5ee3 MD |
197 | #define RSEQ_TEMPLATE_MO_RELAXED |
198 | #include "rseq-x86-bits.h" | |
199 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
784b0012 | 200 | |
809f5ee3 MD |
201 | #define RSEQ_TEMPLATE_MO_RELEASE |
202 | #include "rseq-x86-bits.h" | |
203 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
abf9e855 | 204 | #undef RSEQ_TEMPLATE_INDEX_MM_CID |
de28c254 | 205 | |
abf9e855 | 206 | /* APIs which are not indexed. */ |
de28c254 | 207 | |
abf9e855 | 208 | #define RSEQ_TEMPLATE_INDEX_NONE |
809f5ee3 MD |
209 | #define RSEQ_TEMPLATE_MO_RELAXED |
210 | #include "rseq-x86-bits.h" | |
211 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
abf9e855 | 212 | #undef RSEQ_TEMPLATE_INDEX_NONE |