Commit | Line | Data |
---|---|---|
90702366 | 1 | /* SPDX-License-Identifier: MIT */ |
1bb8dd7b | 2 | /* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ |
f2d7b530 | 3 | |
784b0012 MD |
4 | /* |
5 | * rseq-arm.h | |
784b0012 MD |
6 | */ |
7 | ||
1bb8dd7b MD |
8 | /* |
9 | * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those | |
10 | * are not part of the public API. | |
11 | */ | |
12 | ||
40903c88 | 13 | /* |
4b5bf705 MD |
14 | * - ARM little endian |
15 | * | |
40903c88 MD |
16 | * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand |
17 | * value 0x5de3. This traps if user-space reaches this instruction by mistake, | |
18 | * and the uncommon operand ensures the kernel does not move the instruction | |
19 | * pointer to attacker-controlled code on rseq abort. | |
20 | * | |
21 | * The instruction pattern in the A32 instruction set is: | |
22 | * | |
23 | * e7f5def3 udf #24035 ; 0x5de3 | |
24 | * | |
25 | * This translates to the following instruction pattern in the T16 instruction | |
26 | * set: | |
27 | * | |
28 | * little endian: | |
29 | * def3 udf #243 ; 0xf3 | |
30 | * e7f5 b.n <7f5> | |
31 | * | |
4b5bf705 | 32 | * - ARMv6+ big endian (BE8): |
40903c88 MD |
33 | * |
34 | * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian | |
4b5bf705 MD |
35 | * code and big-endian data. The data value of the signature needs to have its |
36 | * byte order reversed to generate the trap instruction: | |
37 | * | |
38 | * Data: 0xf3def5e7 | |
39 | * | |
40 | * Translates to this A32 instruction pattern: | |
41 | * | |
42 | * e7f5def3 udf #24035 ; 0x5de3 | |
43 | * | |
44 | * Translates to this T16 instruction pattern: | |
45 | * | |
46 | * def3 udf #243 ; 0xf3 | |
47 | * e7f5 b.n <7f5> | |
48 | * | |
49 | * - Prior to ARMv6 big endian (BE32): | |
50 | * | |
51 | * Prior to ARMv6, -mbig-endian generates big-endian code and data | |
52 | * (which match), so the endianness of the data representation of the | |
53 | * signature should not be reversed. However, the choice between BE32 | |
54 | * and BE8 is done by the linker, so we cannot know whether code and | |
55 | * data endianness will be mixed before the linker is invoked. So rather | |
56 | * than try to play tricks with the linker, the rseq signature is simply | |
57 | * data (not a trap instruction) prior to ARMv6 on big endian. This is | |
58 | * why the signature is expressed as data (.word) rather than as | |
59 | * instruction (.inst) in assembler. | |
40903c88 MD |
60 | */ |
61 | ||
4b5bf705 MD |
62 | #ifdef __ARMEB__ |
63 | #define RSEQ_SIG 0xf3def5e7 /* udf #24035 ; 0x5de3 (ARMv6+) */ | |
64 | #else | |
65 | #define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */ | |
40903c88 | 66 | #endif |
784b0012 | 67 | |
1bb8dd7b MD |
68 | /* |
69 | * Refer to the Linux kernel memory model (LKMM) for documentation of | |
70 | * the memory barriers. | |
71 | */ | |
72 | ||
73 | /* CPU memory barrier. */ | |
784b0012 | 74 | #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") |
1bb8dd7b | 75 | /* CPU read memory barrier */ |
784b0012 | 76 | #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") |
1bb8dd7b | 77 | /* CPU write memory barrier */ |
784b0012 MD |
78 | #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") |
79 | ||
1bb8dd7b | 80 | /* Acquire: One-way permeable barrier. */ |
784b0012 MD |
81 | #define rseq_smp_load_acquire(p) \ |
82 | __extension__ ({ \ | |
3664a718 | 83 | rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ |
784b0012 MD |
84 | rseq_smp_mb(); \ |
85 | ____p1; \ | |
86 | }) | |
87 | ||
1bb8dd7b | 88 | /* Acquire barrier after control dependency. */ |
784b0012 MD |
89 | #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() |
90 | ||
1bb8dd7b | 91 | /* Release: One-way permeable barrier. */ |
784b0012 MD |
92 | #define rseq_smp_store_release(p, v) \ |
93 | do { \ | |
94 | rseq_smp_mb(); \ | |
826417f6 | 95 | RSEQ_WRITE_ONCE(*(p), v); \ |
784b0012 MD |
96 | } while (0) |
97 | ||
1bb8dd7b | 98 | /* Only used in RSEQ_ASM_DEFINE_TABLE. */ |
dd01d0fb | 99 | #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ |
784b0012 | 100 | post_commit_offset, abort_ip) \ |
dd01d0fb | 101 | ".pushsection __rseq_cs, \"aw\"\n\t" \ |
784b0012 | 102 | ".balign 32\n\t" \ |
1bb8dd7b | 103 | __rseq_str(label) ":\n\t" \ |
784b0012 MD |
104 | ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ |
105 | ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ | |
dd01d0fb MD |
106 | ".popsection\n\t" \ |
107 | ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ | |
108 | ".word " __rseq_str(label) "b, 0x0\n\t" \ | |
784b0012 MD |
109 | ".popsection\n\t" |
110 | ||
1bb8dd7b MD |
111 | /* |
112 | * Define an rseq critical section structure of version 0 with no flags. | |
113 | * | |
114 | * @label: | |
115 | * Local label for the beginning of the critical section descriptor | |
116 | * structure. | |
117 | * @start_ip: | |
118 | * Pointer to the first instruction of the sequence of consecutive assembly | |
119 | * instructions. | |
120 | * @post_commit_ip: | |
121 | * Pointer to the instruction after the last instruction of the sequence of | |
122 | * consecutive assembly instructions. | |
123 | * @abort_ip: | |
124 | * Pointer to the instruction where to move the execution flow in case of | |
125 | * abort of the sequence of consecutive assembly instructions. | |
126 | */ | |
dd01d0fb MD |
127 | #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ |
128 | __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ | |
d15481d2 | 129 | (post_commit_ip) - (start_ip), abort_ip) |
784b0012 | 130 | |
90d9876e | 131 | /* |
1bb8dd7b MD |
132 | * Define the @exit_ip pointer as an exit point for the sequence of consecutive |
133 | * assembly instructions at @start_ip. | |
134 | * | |
135 | * @start_ip: | |
136 | * Pointer to the first instruction of the sequence of consecutive assembly | |
137 | * instructions. | |
138 | * @exit_ip: | |
139 | * Pointer to an exit point instruction. | |
140 | * | |
90d9876e | 141 | * Exit points of a rseq critical section consist of all instructions outside |
fd622cad MD |
142 | * of the critical section where a critical section can either branch to or |
143 | * reach through the normal course of its execution. The abort IP and the | |
144 | * post-commit IP are already part of the __rseq_cs section and should not be | |
145 | * explicitly defined as additional exit points. Knowing all exit points is | |
146 | * useful to assist debuggers stepping over the critical section. | |
90d9876e MD |
147 | */ |
148 | #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ | |
149 | ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ | |
150 | ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ | |
151 | ".popsection\n\t" | |
152 | ||
1bb8dd7b MD |
153 | /* |
154 | * Store the address of the critical section descriptor structure at | |
155 | * @cs_label into the @rseq_cs pointer and emit the label @label, which | |
156 | * is the beginning of the sequence of consecutive assembly instructions. | |
157 | * | |
158 | * @label: | |
159 | * Local label to the beginning of the sequence of consecutive assembly | |
160 | * instructions. | |
161 | * @cs_label: | |
162 | * Source local label to the critical section descriptor structure. | |
163 | * @rseq_cs: | |
164 | * Destination pointer where to store the address of the critical | |
165 | * section descriptor structure. | |
166 | */ | |
784b0012 MD |
167 | #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ |
168 | RSEQ_INJECT_ASM(1) \ | |
169 | "adr r0, " __rseq_str(cs_label) "\n\t" \ | |
170 | "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \ | |
171 | __rseq_str(label) ":\n\t" | |
172 | ||
1bb8dd7b | 173 | /* Only used in RSEQ_ASM_DEFINE_ABORT. */ |
1bb8c5e3 MD |
174 | #define __RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \ |
175 | table_label, version, flags, \ | |
784b0012 MD |
176 | start_ip, post_commit_offset, abort_ip) \ |
177 | ".balign 32\n\t" \ | |
178 | __rseq_str(table_label) ":\n\t" \ | |
179 | ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ | |
180 | ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ | |
4b5bf705 | 181 | ".word " __rseq_str(RSEQ_SIG) "\n\t" \ |
784b0012 MD |
182 | __rseq_str(label) ":\n\t" \ |
183 | teardown \ | |
184 | "b %l[" __rseq_str(abort_label) "]\n\t" | |
185 | ||
1bb8dd7b MD |
186 | /* |
187 | * Define a critical section abort handler. | |
188 | * | |
1bb8dd7b MD |
189 | * @label: |
190 | * Local label to the abort handler. | |
191 | * @teardown: | |
192 | * Sequence of instructions to run on abort. | |
193 | * @abort_label: | |
194 | * C label to jump to at the end of the sequence. | |
1bb8c5e3 MD |
195 | * @table_label: |
196 | * Local label to the critical section descriptor copy placed near | |
197 | * the program counter. This is done for performance reasons because | |
198 | * computing this address is faster than accessing the program data. | |
1bb8dd7b MD |
199 | * |
200 | * The purpose of @start_ip, @post_commit_ip, and @abort_ip are | |
201 | * documented in RSEQ_ASM_DEFINE_TABLE. | |
202 | */ | |
1bb8c5e3 MD |
203 | #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \ |
204 | table_label, start_ip, post_commit_ip, abort_ip) \ | |
205 | __RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \ | |
206 | table_label, 0x0, 0x0, start_ip, \ | |
784b0012 MD |
207 | (post_commit_ip - start_ip), abort_ip) |
208 | ||
1bb8dd7b MD |
209 | /* |
210 | * Define a critical section teardown handler. | |
211 | * | |
212 | * @label: | |
213 | * Local label to the teardown handler. | |
214 | * @teardown: | |
215 | * Sequence of instructions to run on teardown. | |
216 | * @target_label: | |
217 | * C label to jump to at the end of the sequence. | |
218 | */ | |
559d824f | 219 | #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \ |
784b0012 MD |
220 | __rseq_str(label) ":\n\t" \ |
221 | teardown \ | |
559d824f | 222 | "b %l[" __rseq_str(target_label) "]\n\t" |
784b0012 | 223 | |
1bb8dd7b MD |
224 | /* Jump to local label @label when @cpu_id != @current_cpu_id. */ |
225 | #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ | |
226 | RSEQ_INJECT_ASM(2) \ | |
227 | "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \ | |
228 | "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \ | |
229 | "bne " __rseq_str(label) "\n\t" | |
230 | ||
a66de583 | 231 | /* Per-cpu-id indexing. */ |
784b0012 | 232 | |
abf9e855 | 233 | #define RSEQ_TEMPLATE_INDEX_CPU_ID |
a66de583 MD |
234 | #define RSEQ_TEMPLATE_MO_RELAXED |
235 | #include "rseq-arm-bits.h" | |
236 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
784b0012 | 237 | |
a66de583 MD |
238 | #define RSEQ_TEMPLATE_MO_RELEASE |
239 | #include "rseq-arm-bits.h" | |
240 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
abf9e855 | 241 | #undef RSEQ_TEMPLATE_INDEX_CPU_ID |
784b0012 | 242 | |
a66de583 | 243 | /* Per-mm-cid indexing. */ |
784b0012 | 244 | |
abf9e855 | 245 | #define RSEQ_TEMPLATE_INDEX_MM_CID |
a66de583 MD |
246 | #define RSEQ_TEMPLATE_MO_RELAXED |
247 | #include "rseq-arm-bits.h" | |
248 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
784b0012 | 249 | |
a66de583 MD |
250 | #define RSEQ_TEMPLATE_MO_RELEASE |
251 | #include "rseq-arm-bits.h" | |
252 | #undef RSEQ_TEMPLATE_MO_RELEASE | |
abf9e855 | 253 | #undef RSEQ_TEMPLATE_INDEX_MM_CID |
784b0012 | 254 | |
abf9e855 | 255 | /* APIs which are not indexed. */ |
784b0012 | 256 | |
abf9e855 | 257 | #define RSEQ_TEMPLATE_INDEX_NONE |
a66de583 MD |
258 | #define RSEQ_TEMPLATE_MO_RELAXED |
259 | #include "rseq-arm-bits.h" | |
260 | #undef RSEQ_TEMPLATE_MO_RELAXED | |
abf9e855 | 261 | #undef RSEQ_TEMPLATE_INDEX_NONE |