Commit | Line | Data |
---|---|---|
57319d80 QR |
1 | /* |
2 | * mpx.c - Memory Protection eXtensions | |
3 | * | |
4 | * Copyright (c) 2014, Intel Corporation. | |
5 | * Qiaowei Ren <qiaowei.ren@intel.com> | |
6 | * Dave Hansen <dave.hansen@intel.com> | |
7 | */ | |
8 | #include <linux/kernel.h> | |
fcc7ffd6 | 9 | #include <linux/slab.h> |
57319d80 QR |
10 | #include <linux/syscalls.h> |
11 | #include <linux/sched/sysctl.h> | |
12 | ||
fe3d197f | 13 | #include <asm/insn.h> |
57319d80 | 14 | #include <asm/mman.h> |
1de4fa14 | 15 | #include <asm/mmu_context.h> |
57319d80 | 16 | #include <asm/mpx.h> |
fe3d197f | 17 | #include <asm/processor.h> |
78f7f1e5 | 18 | #include <asm/fpu/internal.h> |
57319d80 QR |
19 | |
20 | static const char *mpx_mapping_name(struct vm_area_struct *vma) | |
21 | { | |
22 | return "[mpx]"; | |
23 | } | |
24 | ||
25 | static struct vm_operations_struct mpx_vma_ops = { | |
26 | .name = mpx_mapping_name, | |
27 | }; | |
28 | ||
1de4fa14 DH |
29 | static int is_mpx_vma(struct vm_area_struct *vma) |
30 | { | |
31 | return (vma->vm_ops == &mpx_vma_ops); | |
32 | } | |
33 | ||
57319d80 QR |
34 | /* |
35 | * This is really a simplified "vm_mmap". it only handles MPX | |
36 | * bounds tables (the bounds directory is user-allocated). | |
37 | * | |
38 | * Later on, we use the vma->vm_ops to uniquely identify these | |
39 | * VMAs. | |
40 | */ | |
41 | static unsigned long mpx_mmap(unsigned long len) | |
42 | { | |
43 | unsigned long ret; | |
44 | unsigned long addr, pgoff; | |
45 | struct mm_struct *mm = current->mm; | |
46 | vm_flags_t vm_flags; | |
47 | struct vm_area_struct *vma; | |
48 | ||
49 | /* Only bounds table and bounds directory can be allocated here */ | |
50 | if (len != MPX_BD_SIZE_BYTES && len != MPX_BT_SIZE_BYTES) | |
51 | return -EINVAL; | |
52 | ||
53 | down_write(&mm->mmap_sem); | |
54 | ||
55 | /* Too many mappings? */ | |
56 | if (mm->map_count > sysctl_max_map_count) { | |
57 | ret = -ENOMEM; | |
58 | goto out; | |
59 | } | |
60 | ||
61 | /* Obtain the address to map to. we verify (or select) it and ensure | |
62 | * that it represents a valid section of the address space. | |
63 | */ | |
64 | addr = get_unmapped_area(NULL, 0, len, 0, MAP_ANONYMOUS | MAP_PRIVATE); | |
65 | if (addr & ~PAGE_MASK) { | |
66 | ret = addr; | |
67 | goto out; | |
68 | } | |
69 | ||
70 | vm_flags = VM_READ | VM_WRITE | VM_MPX | | |
71 | mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; | |
72 | ||
73 | /* Set pgoff according to addr for anon_vma */ | |
74 | pgoff = addr >> PAGE_SHIFT; | |
75 | ||
76 | ret = mmap_region(NULL, addr, len, vm_flags, pgoff); | |
77 | if (IS_ERR_VALUE(ret)) | |
78 | goto out; | |
79 | ||
80 | vma = find_vma(mm, ret); | |
81 | if (!vma) { | |
82 | ret = -ENOMEM; | |
83 | goto out; | |
84 | } | |
85 | vma->vm_ops = &mpx_vma_ops; | |
86 | ||
87 | if (vm_flags & VM_LOCKED) { | |
88 | up_write(&mm->mmap_sem); | |
89 | mm_populate(ret, len); | |
90 | return ret; | |
91 | } | |
92 | ||
93 | out: | |
94 | up_write(&mm->mmap_sem); | |
95 | return ret; | |
96 | } | |
fcc7ffd6 DH |
97 | |
98 | enum reg_type { | |
99 | REG_TYPE_RM = 0, | |
100 | REG_TYPE_INDEX, | |
101 | REG_TYPE_BASE, | |
102 | }; | |
103 | ||
68c009c4 DH |
104 | static int get_reg_offset(struct insn *insn, struct pt_regs *regs, |
105 | enum reg_type type) | |
fcc7ffd6 DH |
106 | { |
107 | int regno = 0; | |
108 | ||
109 | static const int regoff[] = { | |
110 | offsetof(struct pt_regs, ax), | |
111 | offsetof(struct pt_regs, cx), | |
112 | offsetof(struct pt_regs, dx), | |
113 | offsetof(struct pt_regs, bx), | |
114 | offsetof(struct pt_regs, sp), | |
115 | offsetof(struct pt_regs, bp), | |
116 | offsetof(struct pt_regs, si), | |
117 | offsetof(struct pt_regs, di), | |
118 | #ifdef CONFIG_X86_64 | |
119 | offsetof(struct pt_regs, r8), | |
120 | offsetof(struct pt_regs, r9), | |
121 | offsetof(struct pt_regs, r10), | |
122 | offsetof(struct pt_regs, r11), | |
123 | offsetof(struct pt_regs, r12), | |
124 | offsetof(struct pt_regs, r13), | |
125 | offsetof(struct pt_regs, r14), | |
126 | offsetof(struct pt_regs, r15), | |
127 | #endif | |
128 | }; | |
129 | int nr_registers = ARRAY_SIZE(regoff); | |
130 | /* | |
131 | * Don't possibly decode a 32-bit instructions as | |
132 | * reading a 64-bit-only register. | |
133 | */ | |
134 | if (IS_ENABLED(CONFIG_X86_64) && !insn->x86_64) | |
135 | nr_registers -= 8; | |
136 | ||
137 | switch (type) { | |
138 | case REG_TYPE_RM: | |
139 | regno = X86_MODRM_RM(insn->modrm.value); | |
140 | if (X86_REX_B(insn->rex_prefix.value) == 1) | |
141 | regno += 8; | |
142 | break; | |
143 | ||
144 | case REG_TYPE_INDEX: | |
145 | regno = X86_SIB_INDEX(insn->sib.value); | |
146 | if (X86_REX_X(insn->rex_prefix.value) == 1) | |
147 | regno += 8; | |
148 | break; | |
149 | ||
150 | case REG_TYPE_BASE: | |
151 | regno = X86_SIB_BASE(insn->sib.value); | |
152 | if (X86_REX_B(insn->rex_prefix.value) == 1) | |
153 | regno += 8; | |
154 | break; | |
155 | ||
156 | default: | |
157 | pr_err("invalid register type"); | |
158 | BUG(); | |
159 | break; | |
160 | } | |
161 | ||
162 | if (regno > nr_registers) { | |
163 | WARN_ONCE(1, "decoded an instruction with an invalid register"); | |
164 | return -EINVAL; | |
165 | } | |
166 | return regoff[regno]; | |
167 | } | |
168 | ||
169 | /* | |
170 | * return the address being referenced be instruction | |
171 | * for rm=3 returning the content of the rm reg | |
172 | * for rm!=3 calculates the address using SIB and Disp | |
173 | */ | |
174 | static void __user *mpx_get_addr_ref(struct insn *insn, struct pt_regs *regs) | |
175 | { | |
68c009c4 DH |
176 | unsigned long addr, base, indx; |
177 | int addr_offset, base_offset, indx_offset; | |
fcc7ffd6 DH |
178 | insn_byte_t sib; |
179 | ||
180 | insn_get_modrm(insn); | |
181 | insn_get_sib(insn); | |
182 | sib = insn->sib.value; | |
183 | ||
184 | if (X86_MODRM_MOD(insn->modrm.value) == 3) { | |
185 | addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM); | |
186 | if (addr_offset < 0) | |
187 | goto out_err; | |
188 | addr = regs_get_register(regs, addr_offset); | |
189 | } else { | |
190 | if (insn->sib.nbytes) { | |
191 | base_offset = get_reg_offset(insn, regs, REG_TYPE_BASE); | |
192 | if (base_offset < 0) | |
193 | goto out_err; | |
194 | ||
195 | indx_offset = get_reg_offset(insn, regs, REG_TYPE_INDEX); | |
196 | if (indx_offset < 0) | |
197 | goto out_err; | |
198 | ||
199 | base = regs_get_register(regs, base_offset); | |
200 | indx = regs_get_register(regs, indx_offset); | |
201 | addr = base + indx * (1 << X86_SIB_SCALE(sib)); | |
202 | } else { | |
203 | addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM); | |
204 | if (addr_offset < 0) | |
205 | goto out_err; | |
206 | addr = regs_get_register(regs, addr_offset); | |
207 | } | |
208 | addr += insn->displacement.value; | |
209 | } | |
210 | return (void __user *)addr; | |
211 | out_err: | |
212 | return (void __user *)-1; | |
213 | } | |
214 | ||
215 | static int mpx_insn_decode(struct insn *insn, | |
216 | struct pt_regs *regs) | |
217 | { | |
218 | unsigned char buf[MAX_INSN_SIZE]; | |
219 | int x86_64 = !test_thread_flag(TIF_IA32); | |
220 | int not_copied; | |
221 | int nr_copied; | |
222 | ||
223 | not_copied = copy_from_user(buf, (void __user *)regs->ip, sizeof(buf)); | |
224 | nr_copied = sizeof(buf) - not_copied; | |
225 | /* | |
226 | * The decoder _should_ fail nicely if we pass it a short buffer. | |
227 | * But, let's not depend on that implementation detail. If we | |
228 | * did not get anything, just error out now. | |
229 | */ | |
230 | if (!nr_copied) | |
231 | return -EFAULT; | |
232 | insn_init(insn, buf, nr_copied, x86_64); | |
233 | insn_get_length(insn); | |
234 | /* | |
235 | * copy_from_user() tries to get as many bytes as we could see in | |
236 | * the largest possible instruction. If the instruction we are | |
237 | * after is shorter than that _and_ we attempt to copy from | |
238 | * something unreadable, we might get a short read. This is OK | |
239 | * as long as the read did not stop in the middle of the | |
240 | * instruction. Check to see if we got a partial instruction. | |
241 | */ | |
242 | if (nr_copied < insn->length) | |
243 | return -EFAULT; | |
244 | ||
245 | insn_get_opcode(insn); | |
246 | /* | |
247 | * We only _really_ need to decode bndcl/bndcn/bndcu | |
248 | * Error out on anything else. | |
249 | */ | |
250 | if (insn->opcode.bytes[0] != 0x0f) | |
251 | goto bad_opcode; | |
252 | if ((insn->opcode.bytes[1] != 0x1a) && | |
253 | (insn->opcode.bytes[1] != 0x1b)) | |
254 | goto bad_opcode; | |
255 | ||
256 | return 0; | |
257 | bad_opcode: | |
258 | return -EINVAL; | |
259 | } | |
260 | ||
261 | /* | |
262 | * If a bounds overflow occurs then a #BR is generated. This | |
263 | * function decodes MPX instructions to get violation address | |
264 | * and set this address into extended struct siginfo. | |
265 | * | |
266 | * Note that this is not a super precise way of doing this. | |
267 | * Userspace could have, by the time we get here, written | |
268 | * anything it wants in to the instructions. We can not | |
269 | * trust anything about it. They might not be valid | |
270 | * instructions or might encode invalid registers, etc... | |
271 | * | |
272 | * The caller is expected to kfree() the returned siginfo_t. | |
273 | */ | |
46a6e0cf | 274 | siginfo_t *mpx_generate_siginfo(struct pt_regs *regs) |
fcc7ffd6 | 275 | { |
a84eeaa9 | 276 | const struct bndreg *bndregs, *bndreg; |
fe3d197f | 277 | siginfo_t *info = NULL; |
fcc7ffd6 DH |
278 | struct insn insn; |
279 | uint8_t bndregno; | |
280 | int err; | |
fcc7ffd6 DH |
281 | |
282 | err = mpx_insn_decode(&insn, regs); | |
283 | if (err) | |
284 | goto err_out; | |
285 | ||
286 | /* | |
287 | * We know at this point that we are only dealing with | |
288 | * MPX instructions. | |
289 | */ | |
290 | insn_get_modrm(&insn); | |
291 | bndregno = X86_MODRM_REG(insn.modrm.value); | |
292 | if (bndregno > 3) { | |
293 | err = -EINVAL; | |
294 | goto err_out; | |
295 | } | |
a84eeaa9 DH |
296 | /* get bndregs field from current task's xsave area */ |
297 | bndregs = get_xsave_field_ptr(XSTATE_BNDREGS); | |
fe3d197f DH |
298 | if (!bndregs) { |
299 | err = -EINVAL; | |
300 | goto err_out; | |
301 | } | |
302 | /* now go select the individual register in the set of 4 */ | |
303 | bndreg = &bndregs[bndregno]; | |
304 | ||
fcc7ffd6 DH |
305 | info = kzalloc(sizeof(*info), GFP_KERNEL); |
306 | if (!info) { | |
307 | err = -ENOMEM; | |
308 | goto err_out; | |
309 | } | |
310 | /* | |
311 | * The registers are always 64-bit, but the upper 32 | |
312 | * bits are ignored in 32-bit mode. Also, note that the | |
313 | * upper bounds are architecturally represented in 1's | |
314 | * complement form. | |
315 | * | |
316 | * The 'unsigned long' cast is because the compiler | |
317 | * complains when casting from integers to different-size | |
318 | * pointers. | |
319 | */ | |
fe3d197f DH |
320 | info->si_lower = (void __user *)(unsigned long)bndreg->lower_bound; |
321 | info->si_upper = (void __user *)(unsigned long)~bndreg->upper_bound; | |
fcc7ffd6 DH |
322 | info->si_addr_lsb = 0; |
323 | info->si_signo = SIGSEGV; | |
324 | info->si_errno = 0; | |
325 | info->si_code = SEGV_BNDERR; | |
326 | info->si_addr = mpx_get_addr_ref(&insn, regs); | |
327 | /* | |
328 | * We were not able to extract an address from the instruction, | |
329 | * probably because there was something invalid in it. | |
330 | */ | |
331 | if (info->si_addr == (void *)-1) { | |
332 | err = -EINVAL; | |
333 | goto err_out; | |
334 | } | |
335 | return info; | |
336 | err_out: | |
fe3d197f DH |
337 | /* info might be NULL, but kfree() handles that */ |
338 | kfree(info); | |
fcc7ffd6 DH |
339 | return ERR_PTR(err); |
340 | } | |
fe3d197f | 341 | |
46a6e0cf | 342 | static __user void *mpx_get_bounds_dir(void) |
fe3d197f | 343 | { |
a84eeaa9 | 344 | const struct bndcsr *bndcsr; |
fe3d197f DH |
345 | |
346 | if (!cpu_feature_enabled(X86_FEATURE_MPX)) | |
347 | return MPX_INVALID_BOUNDS_DIR; | |
348 | ||
814564a0 DH |
349 | /* |
350 | * 32-bit binaries on 64-bit kernels are currently | |
351 | * unsupported. | |
352 | */ | |
353 | if (IS_ENABLED(CONFIG_X86_64) && test_thread_flag(TIF_IA32)) | |
354 | return MPX_INVALID_BOUNDS_DIR; | |
fe3d197f DH |
355 | /* |
356 | * The bounds directory pointer is stored in a register | |
357 | * only accessible if we first do an xsave. | |
358 | */ | |
a84eeaa9 | 359 | bndcsr = get_xsave_field_ptr(XSTATE_BNDCSR); |
fe3d197f DH |
360 | if (!bndcsr) |
361 | return MPX_INVALID_BOUNDS_DIR; | |
362 | ||
363 | /* | |
364 | * Make sure the register looks valid by checking the | |
365 | * enable bit. | |
366 | */ | |
367 | if (!(bndcsr->bndcfgu & MPX_BNDCFG_ENABLE_FLAG)) | |
368 | return MPX_INVALID_BOUNDS_DIR; | |
369 | ||
370 | /* | |
371 | * Lastly, mask off the low bits used for configuration | |
372 | * flags, and return the address of the bounds table. | |
373 | */ | |
374 | return (void __user *)(unsigned long) | |
375 | (bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK); | |
376 | } | |
377 | ||
46a6e0cf | 378 | int mpx_enable_management(void) |
fe3d197f DH |
379 | { |
380 | void __user *bd_base = MPX_INVALID_BOUNDS_DIR; | |
46a6e0cf | 381 | struct mm_struct *mm = current->mm; |
fe3d197f DH |
382 | int ret = 0; |
383 | ||
384 | /* | |
385 | * runtime in the userspace will be responsible for allocation of | |
386 | * the bounds directory. Then, it will save the base of the bounds | |
387 | * directory into XSAVE/XRSTOR Save Area and enable MPX through | |
388 | * XRSTOR instruction. | |
389 | * | |
a84eeaa9 DH |
390 | * The copy_xregs_to_kernel() beneath get_xsave_field_ptr() is |
391 | * expected to be relatively expensive. Storing the bounds | |
392 | * directory here means that we do not have to do xsave in the | |
393 | * unmap path; we can just use mm->bd_addr instead. | |
fe3d197f | 394 | */ |
46a6e0cf | 395 | bd_base = mpx_get_bounds_dir(); |
fe3d197f DH |
396 | down_write(&mm->mmap_sem); |
397 | mm->bd_addr = bd_base; | |
398 | if (mm->bd_addr == MPX_INVALID_BOUNDS_DIR) | |
399 | ret = -ENXIO; | |
400 | ||
401 | up_write(&mm->mmap_sem); | |
402 | return ret; | |
403 | } | |
404 | ||
46a6e0cf | 405 | int mpx_disable_management(void) |
fe3d197f DH |
406 | { |
407 | struct mm_struct *mm = current->mm; | |
408 | ||
409 | if (!cpu_feature_enabled(X86_FEATURE_MPX)) | |
410 | return -ENXIO; | |
411 | ||
412 | down_write(&mm->mmap_sem); | |
413 | mm->bd_addr = MPX_INVALID_BOUNDS_DIR; | |
414 | up_write(&mm->mmap_sem); | |
415 | return 0; | |
416 | } | |
417 | ||
418 | /* | |
419 | * With 32-bit mode, MPX_BT_SIZE_BYTES is 4MB, and the size of each | |
420 | * bounds table is 16KB. With 64-bit mode, MPX_BT_SIZE_BYTES is 2GB, | |
421 | * and the size of each bounds table is 4MB. | |
422 | */ | |
423 | static int allocate_bt(long __user *bd_entry) | |
424 | { | |
425 | unsigned long expected_old_val = 0; | |
426 | unsigned long actual_old_val = 0; | |
427 | unsigned long bt_addr; | |
428 | int ret = 0; | |
429 | ||
430 | /* | |
431 | * Carve the virtual space out of userspace for the new | |
432 | * bounds table: | |
433 | */ | |
434 | bt_addr = mpx_mmap(MPX_BT_SIZE_BYTES); | |
435 | if (IS_ERR((void *)bt_addr)) | |
436 | return PTR_ERR((void *)bt_addr); | |
437 | /* | |
438 | * Set the valid flag (kinda like _PAGE_PRESENT in a pte) | |
439 | */ | |
440 | bt_addr = bt_addr | MPX_BD_ENTRY_VALID_FLAG; | |
441 | ||
442 | /* | |
443 | * Go poke the address of the new bounds table in to the | |
444 | * bounds directory entry out in userspace memory. Note: | |
445 | * we may race with another CPU instantiating the same table. | |
446 | * In that case the cmpxchg will see an unexpected | |
447 | * 'actual_old_val'. | |
448 | * | |
449 | * This can fault, but that's OK because we do not hold | |
450 | * mmap_sem at this point, unlike some of the other part | |
451 | * of the MPX code that have to pagefault_disable(). | |
452 | */ | |
453 | ret = user_atomic_cmpxchg_inatomic(&actual_old_val, bd_entry, | |
454 | expected_old_val, bt_addr); | |
455 | if (ret) | |
456 | goto out_unmap; | |
457 | ||
458 | /* | |
459 | * The user_atomic_cmpxchg_inatomic() will only return nonzero | |
460 | * for faults, *not* if the cmpxchg itself fails. Now we must | |
461 | * verify that the cmpxchg itself completed successfully. | |
462 | */ | |
463 | /* | |
464 | * We expected an empty 'expected_old_val', but instead found | |
465 | * an apparently valid entry. Assume we raced with another | |
466 | * thread to instantiate this table and desclare succecss. | |
467 | */ | |
468 | if (actual_old_val & MPX_BD_ENTRY_VALID_FLAG) { | |
469 | ret = 0; | |
470 | goto out_unmap; | |
471 | } | |
472 | /* | |
473 | * We found a non-empty bd_entry but it did not have the | |
474 | * VALID_FLAG set. Return an error which will result in | |
475 | * a SEGV since this probably means that somebody scribbled | |
476 | * some invalid data in to a bounds table. | |
477 | */ | |
478 | if (expected_old_val != actual_old_val) { | |
479 | ret = -EINVAL; | |
480 | goto out_unmap; | |
481 | } | |
482 | return 0; | |
483 | out_unmap: | |
484 | vm_munmap(bt_addr & MPX_BT_ADDR_MASK, MPX_BT_SIZE_BYTES); | |
485 | return ret; | |
486 | } | |
487 | ||
488 | /* | |
489 | * When a BNDSTX instruction attempts to save bounds to a bounds | |
490 | * table, it will first attempt to look up the table in the | |
491 | * first-level bounds directory. If it does not find a table in | |
492 | * the directory, a #BR is generated and we get here in order to | |
493 | * allocate a new table. | |
494 | * | |
495 | * With 32-bit mode, the size of BD is 4MB, and the size of each | |
496 | * bound table is 16KB. With 64-bit mode, the size of BD is 2GB, | |
497 | * and the size of each bound table is 4MB. | |
498 | */ | |
46a6e0cf | 499 | static int do_mpx_bt_fault(void) |
fe3d197f DH |
500 | { |
501 | unsigned long bd_entry, bd_base; | |
a84eeaa9 | 502 | const struct bndcsr *bndcsr; |
fe3d197f | 503 | |
a84eeaa9 | 504 | bndcsr = get_xsave_field_ptr(XSTATE_BNDCSR); |
fe3d197f DH |
505 | if (!bndcsr) |
506 | return -EINVAL; | |
507 | /* | |
508 | * Mask off the preserve and enable bits | |
509 | */ | |
510 | bd_base = bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK; | |
511 | /* | |
512 | * The hardware provides the address of the missing or invalid | |
513 | * entry via BNDSTATUS, so we don't have to go look it up. | |
514 | */ | |
515 | bd_entry = bndcsr->bndstatus & MPX_BNDSTA_ADDR_MASK; | |
516 | /* | |
517 | * Make sure the directory entry is within where we think | |
518 | * the directory is. | |
519 | */ | |
520 | if ((bd_entry < bd_base) || | |
521 | (bd_entry >= bd_base + MPX_BD_SIZE_BYTES)) | |
522 | return -EINVAL; | |
523 | ||
524 | return allocate_bt((long __user *)bd_entry); | |
525 | } | |
526 | ||
46a6e0cf | 527 | int mpx_handle_bd_fault(void) |
fe3d197f DH |
528 | { |
529 | /* | |
530 | * Userspace never asked us to manage the bounds tables, | |
531 | * so refuse to help. | |
532 | */ | |
533 | if (!kernel_managing_mpx_tables(current->mm)) | |
534 | return -EINVAL; | |
535 | ||
46a6e0cf | 536 | if (do_mpx_bt_fault()) { |
fe3d197f DH |
537 | force_sig(SIGSEGV, current); |
538 | /* | |
539 | * The force_sig() is essentially "handling" this | |
540 | * exception, so we do not pass up the error | |
541 | * from do_mpx_bt_fault(). | |
542 | */ | |
543 | } | |
544 | return 0; | |
545 | } | |
1de4fa14 DH |
546 | |
547 | /* | |
548 | * A thin wrapper around get_user_pages(). Returns 0 if the | |
549 | * fault was resolved or -errno if not. | |
550 | */ | |
551 | static int mpx_resolve_fault(long __user *addr, int write) | |
552 | { | |
553 | long gup_ret; | |
554 | int nr_pages = 1; | |
555 | int force = 0; | |
556 | ||
557 | gup_ret = get_user_pages(current, current->mm, (unsigned long)addr, | |
558 | nr_pages, write, force, NULL, NULL); | |
559 | /* | |
560 | * get_user_pages() returns number of pages gotten. | |
561 | * 0 means we failed to fault in and get anything, | |
562 | * probably because 'addr' is bad. | |
563 | */ | |
564 | if (!gup_ret) | |
565 | return -EFAULT; | |
566 | /* Other error, return it */ | |
567 | if (gup_ret < 0) | |
568 | return gup_ret; | |
569 | /* must have gup'd a page and gup_ret>0, success */ | |
570 | return 0; | |
571 | } | |
572 | ||
573 | /* | |
574 | * Get the base of bounds tables pointed by specific bounds | |
575 | * directory entry. | |
576 | */ | |
577 | static int get_bt_addr(struct mm_struct *mm, | |
578 | long __user *bd_entry, unsigned long *bt_addr) | |
579 | { | |
580 | int ret; | |
581 | int valid_bit; | |
582 | ||
583 | if (!access_ok(VERIFY_READ, (bd_entry), sizeof(*bd_entry))) | |
584 | return -EFAULT; | |
585 | ||
586 | while (1) { | |
587 | int need_write = 0; | |
588 | ||
589 | pagefault_disable(); | |
590 | ret = get_user(*bt_addr, bd_entry); | |
591 | pagefault_enable(); | |
592 | if (!ret) | |
593 | break; | |
594 | if (ret == -EFAULT) | |
595 | ret = mpx_resolve_fault(bd_entry, need_write); | |
596 | /* | |
597 | * If we could not resolve the fault, consider it | |
598 | * userspace's fault and error out. | |
599 | */ | |
600 | if (ret) | |
601 | return ret; | |
602 | } | |
603 | ||
604 | valid_bit = *bt_addr & MPX_BD_ENTRY_VALID_FLAG; | |
605 | *bt_addr &= MPX_BT_ADDR_MASK; | |
606 | ||
607 | /* | |
608 | * When the kernel is managing bounds tables, a bounds directory | |
609 | * entry will either have a valid address (plus the valid bit) | |
610 | * *OR* be completely empty. If we see a !valid entry *and* some | |
611 | * data in the address field, we know something is wrong. This | |
612 | * -EINVAL return will cause a SIGSEGV. | |
613 | */ | |
614 | if (!valid_bit && *bt_addr) | |
615 | return -EINVAL; | |
616 | /* | |
617 | * Do we have an completely zeroed bt entry? That is OK. It | |
618 | * just means there was no bounds table for this memory. Make | |
619 | * sure to distinguish this from -EINVAL, which will cause | |
620 | * a SEGV. | |
621 | */ | |
622 | if (!valid_bit) | |
623 | return -ENOENT; | |
624 | ||
625 | return 0; | |
626 | } | |
627 | ||
628 | /* | |
629 | * Free the backing physical pages of bounds table 'bt_addr'. | |
630 | * Assume start...end is within that bounds table. | |
631 | */ | |
632 | static int zap_bt_entries(struct mm_struct *mm, | |
633 | unsigned long bt_addr, | |
634 | unsigned long start, unsigned long end) | |
635 | { | |
636 | struct vm_area_struct *vma; | |
637 | unsigned long addr, len; | |
638 | ||
639 | /* | |
640 | * Find the first overlapping vma. If vma->vm_start > start, there | |
641 | * will be a hole in the bounds table. This -EINVAL return will | |
642 | * cause a SIGSEGV. | |
643 | */ | |
644 | vma = find_vma(mm, start); | |
645 | if (!vma || vma->vm_start > start) | |
646 | return -EINVAL; | |
647 | ||
648 | /* | |
649 | * A NUMA policy on a VM_MPX VMA could cause this bouds table to | |
650 | * be split. So we need to look across the entire 'start -> end' | |
651 | * range of this bounds table, find all of the VM_MPX VMAs, and | |
652 | * zap only those. | |
653 | */ | |
654 | addr = start; | |
655 | while (vma && vma->vm_start < end) { | |
656 | /* | |
657 | * We followed a bounds directory entry down | |
658 | * here. If we find a non-MPX VMA, that's bad, | |
659 | * so stop immediately and return an error. This | |
660 | * probably results in a SIGSEGV. | |
661 | */ | |
662 | if (!is_mpx_vma(vma)) | |
663 | return -EINVAL; | |
664 | ||
665 | len = min(vma->vm_end, end) - addr; | |
666 | zap_page_range(vma, addr, len, NULL); | |
667 | ||
668 | vma = vma->vm_next; | |
669 | addr = vma->vm_start; | |
670 | } | |
671 | ||
672 | return 0; | |
673 | } | |
674 | ||
675 | static int unmap_single_bt(struct mm_struct *mm, | |
676 | long __user *bd_entry, unsigned long bt_addr) | |
677 | { | |
678 | unsigned long expected_old_val = bt_addr | MPX_BD_ENTRY_VALID_FLAG; | |
679 | unsigned long actual_old_val = 0; | |
680 | int ret; | |
681 | ||
682 | while (1) { | |
683 | int need_write = 1; | |
684 | ||
685 | pagefault_disable(); | |
686 | ret = user_atomic_cmpxchg_inatomic(&actual_old_val, bd_entry, | |
687 | expected_old_val, 0); | |
688 | pagefault_enable(); | |
689 | if (!ret) | |
690 | break; | |
691 | if (ret == -EFAULT) | |
692 | ret = mpx_resolve_fault(bd_entry, need_write); | |
693 | /* | |
694 | * If we could not resolve the fault, consider it | |
695 | * userspace's fault and error out. | |
696 | */ | |
697 | if (ret) | |
698 | return ret; | |
699 | } | |
700 | /* | |
701 | * The cmpxchg was performed, check the results. | |
702 | */ | |
703 | if (actual_old_val != expected_old_val) { | |
704 | /* | |
705 | * Someone else raced with us to unmap the table. | |
706 | * There was no bounds table pointed to by the | |
707 | * directory, so declare success. Somebody freed | |
708 | * it. | |
709 | */ | |
710 | if (!actual_old_val) | |
711 | return 0; | |
712 | /* | |
713 | * Something messed with the bounds directory | |
714 | * entry. We hold mmap_sem for read or write | |
715 | * here, so it could not be a _new_ bounds table | |
716 | * that someone just allocated. Something is | |
717 | * wrong, so pass up the error and SIGSEGV. | |
718 | */ | |
719 | return -EINVAL; | |
720 | } | |
721 | ||
722 | /* | |
723 | * Note, we are likely being called under do_munmap() already. To | |
724 | * avoid recursion, do_munmap() will check whether it comes | |
725 | * from one bounds table through VM_MPX flag. | |
726 | */ | |
727 | return do_munmap(mm, bt_addr, MPX_BT_SIZE_BYTES); | |
728 | } | |
729 | ||
730 | /* | |
731 | * If the bounds table pointed by bounds directory 'bd_entry' is | |
732 | * not shared, unmap this whole bounds table. Otherwise, only free | |
733 | * those backing physical pages of bounds table entries covered | |
734 | * in this virtual address region start...end. | |
735 | */ | |
736 | static int unmap_shared_bt(struct mm_struct *mm, | |
737 | long __user *bd_entry, unsigned long start, | |
738 | unsigned long end, bool prev_shared, bool next_shared) | |
739 | { | |
740 | unsigned long bt_addr; | |
741 | int ret; | |
742 | ||
743 | ret = get_bt_addr(mm, bd_entry, &bt_addr); | |
744 | /* | |
745 | * We could see an "error" ret for not-present bounds | |
746 | * tables (not really an error), or actual errors, but | |
747 | * stop unmapping either way. | |
748 | */ | |
749 | if (ret) | |
750 | return ret; | |
751 | ||
752 | if (prev_shared && next_shared) | |
753 | ret = zap_bt_entries(mm, bt_addr, | |
754 | bt_addr+MPX_GET_BT_ENTRY_OFFSET(start), | |
755 | bt_addr+MPX_GET_BT_ENTRY_OFFSET(end)); | |
756 | else if (prev_shared) | |
757 | ret = zap_bt_entries(mm, bt_addr, | |
758 | bt_addr+MPX_GET_BT_ENTRY_OFFSET(start), | |
759 | bt_addr+MPX_BT_SIZE_BYTES); | |
760 | else if (next_shared) | |
761 | ret = zap_bt_entries(mm, bt_addr, bt_addr, | |
762 | bt_addr+MPX_GET_BT_ENTRY_OFFSET(end)); | |
763 | else | |
764 | ret = unmap_single_bt(mm, bd_entry, bt_addr); | |
765 | ||
766 | return ret; | |
767 | } | |
768 | ||
769 | /* | |
770 | * A virtual address region being munmap()ed might share bounds table | |
771 | * with adjacent VMAs. We only need to free the backing physical | |
772 | * memory of these shared bounds tables entries covered in this virtual | |
773 | * address region. | |
774 | */ | |
775 | static int unmap_edge_bts(struct mm_struct *mm, | |
776 | unsigned long start, unsigned long end) | |
777 | { | |
778 | int ret; | |
779 | long __user *bde_start, *bde_end; | |
780 | struct vm_area_struct *prev, *next; | |
781 | bool prev_shared = false, next_shared = false; | |
782 | ||
783 | bde_start = mm->bd_addr + MPX_GET_BD_ENTRY_OFFSET(start); | |
784 | bde_end = mm->bd_addr + MPX_GET_BD_ENTRY_OFFSET(end-1); | |
785 | ||
786 | /* | |
787 | * Check whether bde_start and bde_end are shared with adjacent | |
788 | * VMAs. | |
789 | * | |
790 | * We already unliked the VMAs from the mm's rbtree so 'start' | |
791 | * is guaranteed to be in a hole. This gets us the first VMA | |
792 | * before the hole in to 'prev' and the next VMA after the hole | |
793 | * in to 'next'. | |
794 | */ | |
795 | next = find_vma_prev(mm, start, &prev); | |
796 | if (prev && (mm->bd_addr + MPX_GET_BD_ENTRY_OFFSET(prev->vm_end-1)) | |
797 | == bde_start) | |
798 | prev_shared = true; | |
799 | if (next && (mm->bd_addr + MPX_GET_BD_ENTRY_OFFSET(next->vm_start)) | |
800 | == bde_end) | |
801 | next_shared = true; | |
802 | ||
803 | /* | |
804 | * This virtual address region being munmap()ed is only | |
805 | * covered by one bounds table. | |
806 | * | |
807 | * In this case, if this table is also shared with adjacent | |
808 | * VMAs, only part of the backing physical memory of the bounds | |
809 | * table need be freeed. Otherwise the whole bounds table need | |
810 | * be unmapped. | |
811 | */ | |
812 | if (bde_start == bde_end) { | |
813 | return unmap_shared_bt(mm, bde_start, start, end, | |
814 | prev_shared, next_shared); | |
815 | } | |
816 | ||
817 | /* | |
818 | * If more than one bounds tables are covered in this virtual | |
819 | * address region being munmap()ed, we need to separately check | |
820 | * whether bde_start and bde_end are shared with adjacent VMAs. | |
821 | */ | |
822 | ret = unmap_shared_bt(mm, bde_start, start, end, prev_shared, false); | |
823 | if (ret) | |
824 | return ret; | |
825 | ret = unmap_shared_bt(mm, bde_end, start, end, false, next_shared); | |
826 | if (ret) | |
827 | return ret; | |
828 | ||
829 | return 0; | |
830 | } | |
831 | ||
832 | static int mpx_unmap_tables(struct mm_struct *mm, | |
833 | unsigned long start, unsigned long end) | |
834 | { | |
835 | int ret; | |
836 | long __user *bd_entry, *bde_start, *bde_end; | |
837 | unsigned long bt_addr; | |
838 | ||
839 | /* | |
840 | * "Edge" bounds tables are those which are being used by the region | |
841 | * (start -> end), but that may be shared with adjacent areas. If they | |
842 | * turn out to be completely unshared, they will be freed. If they are | |
843 | * shared, we will free the backing store (like an MADV_DONTNEED) for | |
844 | * areas used by this region. | |
845 | */ | |
846 | ret = unmap_edge_bts(mm, start, end); | |
847 | switch (ret) { | |
848 | /* non-present tables are OK */ | |
849 | case 0: | |
850 | case -ENOENT: | |
851 | /* Success, or no tables to unmap */ | |
852 | break; | |
853 | case -EINVAL: | |
854 | case -EFAULT: | |
855 | default: | |
856 | return ret; | |
857 | } | |
858 | ||
859 | /* | |
860 | * Only unmap the bounds table that are | |
861 | * 1. fully covered | |
862 | * 2. not at the edges of the mapping, even if full aligned | |
863 | */ | |
864 | bde_start = mm->bd_addr + MPX_GET_BD_ENTRY_OFFSET(start); | |
865 | bde_end = mm->bd_addr + MPX_GET_BD_ENTRY_OFFSET(end-1); | |
866 | for (bd_entry = bde_start + 1; bd_entry < bde_end; bd_entry++) { | |
867 | ret = get_bt_addr(mm, bd_entry, &bt_addr); | |
868 | switch (ret) { | |
869 | case 0: | |
870 | break; | |
871 | case -ENOENT: | |
872 | /* No table here, try the next one */ | |
873 | continue; | |
874 | case -EINVAL: | |
875 | case -EFAULT: | |
876 | default: | |
877 | /* | |
878 | * Note: we are being strict here. | |
879 | * Any time we run in to an issue | |
880 | * unmapping tables, we stop and | |
881 | * SIGSEGV. | |
882 | */ | |
883 | return ret; | |
884 | } | |
885 | ||
886 | ret = unmap_single_bt(mm, bd_entry, bt_addr); | |
887 | if (ret) | |
888 | return ret; | |
889 | } | |
890 | ||
891 | return 0; | |
892 | } | |
893 | ||
894 | /* | |
895 | * Free unused bounds tables covered in a virtual address region being | |
896 | * munmap()ed. Assume end > start. | |
897 | * | |
898 | * This function will be called by do_munmap(), and the VMAs covering | |
899 | * the virtual address region start...end have already been split if | |
900 | * necessary, and the 'vma' is the first vma in this range (start -> end). | |
901 | */ | |
902 | void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma, | |
903 | unsigned long start, unsigned long end) | |
904 | { | |
905 | int ret; | |
906 | ||
907 | /* | |
908 | * Refuse to do anything unless userspace has asked | |
909 | * the kernel to help manage the bounds tables, | |
910 | */ | |
911 | if (!kernel_managing_mpx_tables(current->mm)) | |
912 | return; | |
913 | /* | |
914 | * This will look across the entire 'start -> end' range, | |
915 | * and find all of the non-VM_MPX VMAs. | |
916 | * | |
917 | * To avoid recursion, if a VM_MPX vma is found in the range | |
918 | * (start->end), we will not continue follow-up work. This | |
919 | * recursion represents having bounds tables for bounds tables, | |
920 | * which should not occur normally. Being strict about it here | |
921 | * helps ensure that we do not have an exploitable stack overflow. | |
922 | */ | |
923 | do { | |
924 | if (vma->vm_flags & VM_MPX) | |
925 | return; | |
926 | vma = vma->vm_next; | |
927 | } while (vma && vma->vm_start < end); | |
928 | ||
929 | ret = mpx_unmap_tables(mm, start, end); | |
930 | if (ret) | |
931 | force_sig(SIGSEGV, current); | |
932 | } |