Commit | Line | Data |
---|---|---|
69786cdb AR |
1 | /* |
2 | * This file contains some kasan initialization code. | |
3 | * | |
4 | * Copyright (c) 2015 Samsung Electronics Co., Ltd. | |
5 | * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | */ | |
12 | ||
13 | #include <linux/bootmem.h> | |
14 | #include <linux/init.h> | |
15 | #include <linux/kasan.h> | |
16 | #include <linux/kernel.h> | |
17 | #include <linux/memblock.h> | |
18 | #include <linux/pfn.h> | |
19 | ||
20 | #include <asm/page.h> | |
21 | #include <asm/pgalloc.h> | |
22 | ||
23 | /* | |
24 | * This page serves two purposes: | |
25 | * - It used as early shadow memory. The entire shadow region populated | |
26 | * with this page, before we will be able to setup normal shadow memory. | |
27 | * - Latter it reused it as zero shadow to cover large ranges of memory | |
28 | * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...). | |
29 | */ | |
30 | unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss; | |
31 | ||
32 | #if CONFIG_PGTABLE_LEVELS > 3 | |
33 | pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss; | |
34 | #endif | |
35 | #if CONFIG_PGTABLE_LEVELS > 2 | |
36 | pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss; | |
37 | #endif | |
38 | pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss; | |
39 | ||
40 | static __init void *early_alloc(size_t size, int node) | |
41 | { | |
42 | return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), | |
43 | BOOTMEM_ALLOC_ACCESSIBLE, node); | |
44 | } | |
45 | ||
46 | static void __init zero_pte_populate(pmd_t *pmd, unsigned long addr, | |
47 | unsigned long end) | |
48 | { | |
49 | pte_t *pte = pte_offset_kernel(pmd, addr); | |
50 | pte_t zero_pte; | |
51 | ||
52 | zero_pte = pfn_pte(PFN_DOWN(__pa(kasan_zero_page)), PAGE_KERNEL); | |
53 | zero_pte = pte_wrprotect(zero_pte); | |
54 | ||
55 | while (addr + PAGE_SIZE <= end) { | |
56 | set_pte_at(&init_mm, addr, pte, zero_pte); | |
57 | addr += PAGE_SIZE; | |
58 | pte = pte_offset_kernel(pmd, addr); | |
59 | } | |
60 | } | |
61 | ||
62 | static void __init zero_pmd_populate(pud_t *pud, unsigned long addr, | |
63 | unsigned long end) | |
64 | { | |
65 | pmd_t *pmd = pmd_offset(pud, addr); | |
66 | unsigned long next; | |
67 | ||
68 | do { | |
69 | next = pmd_addr_end(addr, end); | |
70 | ||
71 | if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { | |
72 | pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); | |
73 | continue; | |
74 | } | |
75 | ||
76 | if (pmd_none(*pmd)) { | |
77 | pmd_populate_kernel(&init_mm, pmd, | |
78 | early_alloc(PAGE_SIZE, NUMA_NO_NODE)); | |
79 | } | |
80 | zero_pte_populate(pmd, addr, next); | |
81 | } while (pmd++, addr = next, addr != end); | |
82 | } | |
83 | ||
84 | static void __init zero_pud_populate(pgd_t *pgd, unsigned long addr, | |
85 | unsigned long end) | |
86 | { | |
87 | pud_t *pud = pud_offset(pgd, addr); | |
88 | unsigned long next; | |
89 | ||
90 | do { | |
91 | next = pud_addr_end(addr, end); | |
92 | if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) { | |
93 | pmd_t *pmd; | |
94 | ||
95 | pud_populate(&init_mm, pud, kasan_zero_pmd); | |
96 | pmd = pmd_offset(pud, addr); | |
97 | pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); | |
98 | continue; | |
99 | } | |
100 | ||
101 | if (pud_none(*pud)) { | |
102 | pud_populate(&init_mm, pud, | |
103 | early_alloc(PAGE_SIZE, NUMA_NO_NODE)); | |
104 | } | |
105 | zero_pmd_populate(pud, addr, next); | |
106 | } while (pud++, addr = next, addr != end); | |
107 | } | |
108 | ||
109 | /** | |
110 | * kasan_populate_zero_shadow - populate shadow memory region with | |
111 | * kasan_zero_page | |
112 | * @shadow_start - start of the memory range to populate | |
113 | * @shadow_end - end of the memory range to populate | |
114 | */ | |
115 | void __init kasan_populate_zero_shadow(const void *shadow_start, | |
116 | const void *shadow_end) | |
117 | { | |
118 | unsigned long addr = (unsigned long)shadow_start; | |
119 | unsigned long end = (unsigned long)shadow_end; | |
120 | pgd_t *pgd = pgd_offset_k(addr); | |
121 | unsigned long next; | |
122 | ||
123 | do { | |
124 | next = pgd_addr_end(addr, end); | |
125 | ||
126 | if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { | |
127 | pud_t *pud; | |
128 | pmd_t *pmd; | |
129 | ||
130 | /* | |
131 | * kasan_zero_pud should be populated with pmds | |
132 | * at this moment. | |
133 | * [pud,pmd]_populate*() below needed only for | |
134 | * 3,2 - level page tables where we don't have | |
135 | * puds,pmds, so pgd_populate(), pud_populate() | |
136 | * is noops. | |
137 | */ | |
138 | pgd_populate(&init_mm, pgd, kasan_zero_pud); | |
139 | pud = pud_offset(pgd, addr); | |
140 | pud_populate(&init_mm, pud, kasan_zero_pmd); | |
141 | pmd = pmd_offset(pud, addr); | |
142 | pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); | |
143 | continue; | |
144 | } | |
145 | ||
146 | if (pgd_none(*pgd)) { | |
147 | pgd_populate(&init_mm, pgd, | |
148 | early_alloc(PAGE_SIZE, NUMA_NO_NODE)); | |
149 | } | |
150 | zero_pud_populate(pgd, addr, next); | |
151 | } while (pgd++, addr = next, addr != end); | |
152 | } |