Commit | Line | Data |
---|---|---|
084bd298 SC |
1 | /* |
2 | * arch/arm64/mm/hugetlbpage.c | |
3 | * | |
4 | * Copyright (C) 2013 Linaro Ltd. | |
5 | * | |
6 | * Based on arch/x86/mm/hugetlbpage.c. | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
084bd298 SC |
16 | */ |
17 | ||
18 | #include <linux/init.h> | |
19 | #include <linux/fs.h> | |
20 | #include <linux/mm.h> | |
21 | #include <linux/hugetlb.h> | |
22 | #include <linux/pagemap.h> | |
23 | #include <linux/err.h> | |
24 | #include <linux/sysctl.h> | |
25 | #include <asm/mman.h> | |
26 | #include <asm/tlb.h> | |
27 | #include <asm/tlbflush.h> | |
28 | #include <asm/pgalloc.h> | |
29 | ||
084bd298 SC |
30 | int pmd_huge(pmd_t pmd) |
31 | { | |
fd28f5d4 | 32 | return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); |
084bd298 SC |
33 | } |
34 | ||
35 | int pud_huge(pud_t pud) | |
36 | { | |
4797ec2d | 37 | #ifndef __PAGETABLE_PMD_FOLDED |
fd28f5d4 | 38 | return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT); |
4797ec2d MS |
39 | #else |
40 | return 0; | |
41 | #endif | |
084bd298 SC |
42 | } |
43 | ||
66b3923a DW |
44 | static int find_num_contig(struct mm_struct *mm, unsigned long addr, |
45 | pte_t *ptep, pte_t pte, size_t *pgsize) | |
46 | { | |
47 | pgd_t *pgd = pgd_offset(mm, addr); | |
48 | pud_t *pud; | |
49 | pmd_t *pmd; | |
50 | ||
51 | *pgsize = PAGE_SIZE; | |
52 | if (!pte_cont(pte)) | |
53 | return 1; | |
54 | if (!pgd_present(*pgd)) { | |
55 | VM_BUG_ON(!pgd_present(*pgd)); | |
56 | return 1; | |
57 | } | |
58 | pud = pud_offset(pgd, addr); | |
59 | if (!pud_present(*pud)) { | |
60 | VM_BUG_ON(!pud_present(*pud)); | |
61 | return 1; | |
62 | } | |
63 | pmd = pmd_offset(pud, addr); | |
64 | if (!pmd_present(*pmd)) { | |
65 | VM_BUG_ON(!pmd_present(*pmd)); | |
66 | return 1; | |
67 | } | |
68 | if ((pte_t *)pmd == ptep) { | |
69 | *pgsize = PMD_SIZE; | |
70 | return CONT_PMDS; | |
71 | } | |
72 | return CONT_PTES; | |
73 | } | |
74 | ||
75 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, | |
76 | pte_t *ptep, pte_t pte) | |
77 | { | |
78 | size_t pgsize; | |
79 | int i; | |
80 | int ncontig = find_num_contig(mm, addr, ptep, pte, &pgsize); | |
81 | unsigned long pfn; | |
82 | pgprot_t hugeprot; | |
83 | ||
84 | if (ncontig == 1) { | |
85 | set_pte_at(mm, addr, ptep, pte); | |
86 | return; | |
87 | } | |
88 | ||
89 | pfn = pte_pfn(pte); | |
90 | hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); | |
91 | for (i = 0; i < ncontig; i++) { | |
92 | pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, | |
93 | pte_val(pfn_pte(pfn, hugeprot))); | |
94 | set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); | |
95 | ptep++; | |
96 | pfn += pgsize >> PAGE_SHIFT; | |
97 | addr += pgsize; | |
98 | } | |
99 | } | |
100 | ||
101 | pte_t *huge_pte_alloc(struct mm_struct *mm, | |
102 | unsigned long addr, unsigned long sz) | |
103 | { | |
104 | pgd_t *pgd; | |
105 | pud_t *pud; | |
106 | pte_t *pte = NULL; | |
107 | ||
108 | pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz); | |
109 | pgd = pgd_offset(mm, addr); | |
110 | pud = pud_alloc(mm, pgd, addr); | |
111 | if (!pud) | |
112 | return NULL; | |
113 | ||
114 | if (sz == PUD_SIZE) { | |
115 | pte = (pte_t *)pud; | |
116 | } else if (sz == (PAGE_SIZE * CONT_PTES)) { | |
117 | pmd_t *pmd = pmd_alloc(mm, pud, addr); | |
118 | ||
119 | WARN_ON(addr & (sz - 1)); | |
120 | /* | |
121 | * Note that if this code were ever ported to the | |
122 | * 32-bit arm platform then it will cause trouble in | |
123 | * the case where CONFIG_HIGHPTE is set, since there | |
124 | * will be no pte_unmap() to correspond with this | |
125 | * pte_alloc_map(). | |
126 | */ | |
127 | pte = pte_alloc_map(mm, NULL, pmd, addr); | |
128 | } else if (sz == PMD_SIZE) { | |
129 | if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && | |
130 | pud_none(*pud)) | |
131 | pte = huge_pmd_share(mm, addr, pud); | |
132 | else | |
133 | pte = (pte_t *)pmd_alloc(mm, pud, addr); | |
134 | } else if (sz == (PMD_SIZE * CONT_PMDS)) { | |
135 | pmd_t *pmd; | |
136 | ||
137 | pmd = pmd_alloc(mm, pud, addr); | |
138 | WARN_ON(addr & (sz - 1)); | |
139 | return (pte_t *)pmd; | |
140 | } | |
141 | ||
142 | pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr, | |
143 | sz, pte, pte_val(*pte)); | |
144 | return pte; | |
145 | } | |
146 | ||
147 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | |
148 | { | |
149 | pgd_t *pgd; | |
150 | pud_t *pud; | |
151 | pmd_t *pmd = NULL; | |
152 | pte_t *pte = NULL; | |
153 | ||
154 | pgd = pgd_offset(mm, addr); | |
155 | pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd); | |
156 | if (!pgd_present(*pgd)) | |
157 | return NULL; | |
158 | pud = pud_offset(pgd, addr); | |
159 | if (!pud_present(*pud)) | |
160 | return NULL; | |
161 | ||
162 | if (pud_huge(*pud)) | |
163 | return (pte_t *)pud; | |
164 | pmd = pmd_offset(pud, addr); | |
165 | if (!pmd_present(*pmd)) | |
166 | return NULL; | |
167 | ||
168 | if (pte_cont(pmd_pte(*pmd))) { | |
169 | pmd = pmd_offset( | |
170 | pud, (addr & CONT_PMD_MASK)); | |
171 | return (pte_t *)pmd; | |
172 | } | |
173 | if (pmd_huge(*pmd)) | |
174 | return (pte_t *)pmd; | |
175 | pte = pte_offset_kernel(pmd, addr); | |
176 | if (pte_present(*pte) && pte_cont(*pte)) { | |
177 | pte = pte_offset_kernel( | |
178 | pmd, (addr & CONT_PTE_MASK)); | |
179 | return pte; | |
180 | } | |
181 | return NULL; | |
182 | } | |
183 | ||
184 | pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, | |
185 | struct page *page, int writable) | |
186 | { | |
187 | size_t pagesize = huge_page_size(hstate_vma(vma)); | |
188 | ||
189 | if (pagesize == CONT_PTE_SIZE) { | |
190 | entry = pte_mkcont(entry); | |
191 | } else if (pagesize == CONT_PMD_SIZE) { | |
192 | entry = pmd_pte(pmd_mkcont(pte_pmd(entry))); | |
193 | } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) { | |
194 | pr_warn("%s: unrecognized huge page size 0x%lx\n", | |
195 | __func__, pagesize); | |
196 | } | |
197 | return entry; | |
198 | } | |
199 | ||
200 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, | |
201 | unsigned long addr, pte_t *ptep) | |
202 | { | |
203 | pte_t pte; | |
204 | ||
205 | if (pte_cont(*ptep)) { | |
206 | int ncontig, i; | |
207 | size_t pgsize; | |
208 | pte_t *cpte; | |
209 | bool is_dirty = false; | |
210 | ||
211 | cpte = huge_pte_offset(mm, addr); | |
212 | ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); | |
213 | /* save the 1st pte to return */ | |
214 | pte = ptep_get_and_clear(mm, addr, cpte); | |
215 | for (i = 1; i < ncontig; ++i) { | |
216 | /* | |
217 | * If HW_AFDBM is enabled, then the HW could | |
218 | * turn on the dirty bit for any of the page | |
219 | * in the set, so check them all. | |
220 | */ | |
221 | ++cpte; | |
222 | if (pte_dirty(ptep_get_and_clear(mm, addr, cpte))) | |
223 | is_dirty = true; | |
224 | } | |
225 | if (is_dirty) | |
226 | return pte_mkdirty(pte); | |
227 | else | |
228 | return pte; | |
229 | } else { | |
230 | return ptep_get_and_clear(mm, addr, ptep); | |
231 | } | |
232 | } | |
233 | ||
234 | int huge_ptep_set_access_flags(struct vm_area_struct *vma, | |
235 | unsigned long addr, pte_t *ptep, | |
236 | pte_t pte, int dirty) | |
237 | { | |
238 | pte_t *cpte; | |
239 | ||
240 | if (pte_cont(pte)) { | |
241 | int ncontig, i, changed = 0; | |
242 | size_t pgsize = 0; | |
243 | unsigned long pfn = pte_pfn(pte); | |
244 | /* Select all bits except the pfn */ | |
245 | pgprot_t hugeprot = | |
246 | __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ | |
247 | pte_val(pte)); | |
248 | ||
249 | cpte = huge_pte_offset(vma->vm_mm, addr); | |
250 | pfn = pte_pfn(*cpte); | |
251 | ncontig = find_num_contig(vma->vm_mm, addr, cpte, | |
252 | *cpte, &pgsize); | |
253 | for (i = 0; i < ncontig; ++i, ++cpte) { | |
254 | changed = ptep_set_access_flags(vma, addr, cpte, | |
255 | pfn_pte(pfn, | |
256 | hugeprot), | |
257 | dirty); | |
258 | pfn += pgsize >> PAGE_SHIFT; | |
259 | } | |
260 | return changed; | |
261 | } else { | |
262 | return ptep_set_access_flags(vma, addr, ptep, pte, dirty); | |
263 | } | |
264 | } | |
265 | ||
266 | void huge_ptep_set_wrprotect(struct mm_struct *mm, | |
267 | unsigned long addr, pte_t *ptep) | |
268 | { | |
269 | if (pte_cont(*ptep)) { | |
270 | int ncontig, i; | |
271 | pte_t *cpte; | |
272 | size_t pgsize = 0; | |
273 | ||
274 | cpte = huge_pte_offset(mm, addr); | |
275 | ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize); | |
276 | for (i = 0; i < ncontig; ++i, ++cpte) | |
277 | ptep_set_wrprotect(mm, addr, cpte); | |
278 | } else { | |
279 | ptep_set_wrprotect(mm, addr, ptep); | |
280 | } | |
281 | } | |
282 | ||
283 | void huge_ptep_clear_flush(struct vm_area_struct *vma, | |
284 | unsigned long addr, pte_t *ptep) | |
285 | { | |
286 | if (pte_cont(*ptep)) { | |
287 | int ncontig, i; | |
288 | pte_t *cpte; | |
289 | size_t pgsize = 0; | |
290 | ||
291 | cpte = huge_pte_offset(vma->vm_mm, addr); | |
292 | ncontig = find_num_contig(vma->vm_mm, addr, cpte, | |
293 | *cpte, &pgsize); | |
294 | for (i = 0; i < ncontig; ++i, ++cpte) | |
295 | ptep_clear_flush(vma, addr, cpte); | |
296 | } else { | |
297 | ptep_clear_flush(vma, addr, ptep); | |
298 | } | |
299 | } | |
300 | ||
084bd298 SC |
301 | static __init int setup_hugepagesz(char *opt) |
302 | { | |
303 | unsigned long ps = memparse(opt, &opt); | |
66b3923a | 304 | |
084bd298 SC |
305 | if (ps == PMD_SIZE) { |
306 | hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); | |
307 | } else if (ps == PUD_SIZE) { | |
308 | hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); | |
66b3923a DW |
309 | } else if (ps == (PAGE_SIZE * CONT_PTES)) { |
310 | hugetlb_add_hstate(CONT_PTE_SHIFT); | |
311 | } else if (ps == (PMD_SIZE * CONT_PMDS)) { | |
312 | hugetlb_add_hstate((PMD_SHIFT + CONT_PMD_SHIFT) - PAGE_SHIFT); | |
084bd298 | 313 | } else { |
66b3923a | 314 | pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); |
084bd298 SC |
315 | return 0; |
316 | } | |
317 | return 1; | |
318 | } | |
319 | __setup("hugepagesz=", setup_hugepagesz); | |
66b3923a DW |
320 | |
321 | #ifdef CONFIG_ARM64_64K_PAGES | |
322 | static __init int add_default_hugepagesz(void) | |
323 | { | |
324 | if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) | |
325 | hugetlb_add_hstate(CONT_PMD_SHIFT); | |
326 | return 0; | |
327 | } | |
328 | arch_initcall(add_default_hugepagesz); | |
329 | #endif |