Commit | Line | Data |
---|---|---|
f048aace BH |
1 | /* |
2 | * This file contains the routines for TLB flushing. | |
3 | * On machines where the MMU does not use a hash table to store virtual to | |
4 | * physical translations (ie, SW loaded TLBs or Book3E compilant processors, | |
5 | * this does -not- include 603 however which shares the implementation with | |
6 | * hash based processors) | |
7 | * | |
8 | * -- BenH | |
9 | * | |
25d21ad6 BH |
10 | * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org> |
11 | * IBM Corp. | |
f048aace BH |
12 | * |
13 | * Derived from arch/ppc/mm/init.c: | |
14 | * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) | |
15 | * | |
16 | * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) | |
17 | * and Cort Dougan (PReP) (cort@cs.nmt.edu) | |
18 | * Copyright (C) 1996 Paul Mackerras | |
19 | * | |
20 | * Derived from "arch/i386/mm/init.c" | |
21 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds | |
22 | * | |
23 | * This program is free software; you can redistribute it and/or | |
24 | * modify it under the terms of the GNU General Public License | |
25 | * as published by the Free Software Foundation; either version | |
26 | * 2 of the License, or (at your option) any later version. | |
27 | * | |
28 | */ | |
29 | ||
30 | #include <linux/kernel.h> | |
31 | #include <linux/mm.h> | |
32 | #include <linux/init.h> | |
33 | #include <linux/highmem.h> | |
34 | #include <linux/pagemap.h> | |
35 | #include <linux/preempt.h> | |
36 | #include <linux/spinlock.h> | |
95f72d1e | 37 | #include <linux/memblock.h> |
91b191c7 | 38 | #include <linux/of_fdt.h> |
41151e77 | 39 | #include <linux/hugetlb.h> |
f048aace BH |
40 | |
41 | #include <asm/tlbflush.h> | |
42 | #include <asm/tlb.h> | |
25d21ad6 | 43 | #include <asm/code-patching.h> |
41151e77 | 44 | #include <asm/hugetlb.h> |
f048aace BH |
45 | |
46 | #include "mmu_decl.h" | |
47 | ||
41151e77 BB |
48 | /* |
49 | * This struct lists the sw-supported page sizes. The hardawre MMU may support | |
50 | * other sizes not listed here. The .ind field is only used on MMUs that have | |
51 | * indirect page table entries. | |
52 | */ | |
53 | #ifdef CONFIG_PPC_BOOK3E_MMU | |
54 | #ifdef CONFIG_FSL_BOOKE | |
55 | struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { | |
56 | [MMU_PAGE_4K] = { | |
57 | .shift = 12, | |
58 | .enc = BOOK3E_PAGESZ_4K, | |
59 | }, | |
60 | [MMU_PAGE_4M] = { | |
61 | .shift = 22, | |
62 | .enc = BOOK3E_PAGESZ_4M, | |
63 | }, | |
64 | [MMU_PAGE_16M] = { | |
65 | .shift = 24, | |
66 | .enc = BOOK3E_PAGESZ_16M, | |
67 | }, | |
68 | [MMU_PAGE_64M] = { | |
69 | .shift = 26, | |
70 | .enc = BOOK3E_PAGESZ_64M, | |
71 | }, | |
72 | [MMU_PAGE_256M] = { | |
73 | .shift = 28, | |
74 | .enc = BOOK3E_PAGESZ_256M, | |
75 | }, | |
76 | [MMU_PAGE_1G] = { | |
77 | .shift = 30, | |
78 | .enc = BOOK3E_PAGESZ_1GB, | |
79 | }, | |
80 | }; | |
81 | #else | |
25d21ad6 BH |
82 | struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { |
83 | [MMU_PAGE_4K] = { | |
84 | .shift = 12, | |
f2b26c92 | 85 | .ind = 20, |
25d21ad6 BH |
86 | .enc = BOOK3E_PAGESZ_4K, |
87 | }, | |
88 | [MMU_PAGE_16K] = { | |
89 | .shift = 14, | |
90 | .enc = BOOK3E_PAGESZ_16K, | |
91 | }, | |
92 | [MMU_PAGE_64K] = { | |
93 | .shift = 16, | |
f2b26c92 | 94 | .ind = 28, |
25d21ad6 BH |
95 | .enc = BOOK3E_PAGESZ_64K, |
96 | }, | |
97 | [MMU_PAGE_1M] = { | |
98 | .shift = 20, | |
99 | .enc = BOOK3E_PAGESZ_1M, | |
100 | }, | |
101 | [MMU_PAGE_16M] = { | |
102 | .shift = 24, | |
f2b26c92 | 103 | .ind = 36, |
25d21ad6 BH |
104 | .enc = BOOK3E_PAGESZ_16M, |
105 | }, | |
106 | [MMU_PAGE_256M] = { | |
107 | .shift = 28, | |
108 | .enc = BOOK3E_PAGESZ_256M, | |
109 | }, | |
110 | [MMU_PAGE_1G] = { | |
111 | .shift = 30, | |
112 | .enc = BOOK3E_PAGESZ_1GB, | |
113 | }, | |
114 | }; | |
41151e77 BB |
115 | #endif /* CONFIG_FSL_BOOKE */ |
116 | ||
25d21ad6 BH |
117 | static inline int mmu_get_tsize(int psize) |
118 | { | |
119 | return mmu_psize_defs[psize].enc; | |
120 | } | |
121 | #else | |
122 | static inline int mmu_get_tsize(int psize) | |
123 | { | |
124 | /* This isn't used on !Book3E for now */ | |
125 | return 0; | |
126 | } | |
41151e77 | 127 | #endif /* CONFIG_PPC_BOOK3E_MMU */ |
25d21ad6 BH |
128 | |
129 | /* The variables below are currently only used on 64-bit Book3E | |
130 | * though this will probably be made common with other nohash | |
131 | * implementations at some point | |
132 | */ | |
133 | #ifdef CONFIG_PPC64 | |
134 | ||
135 | int mmu_linear_psize; /* Page size used for the linear mapping */ | |
136 | int mmu_pte_psize; /* Page size used for PTE pages */ | |
32a74949 | 137 | int mmu_vmemmap_psize; /* Page size used for the virtual mem map */ |
25d21ad6 BH |
138 | int book3e_htw_enabled; /* Is HW tablewalk enabled ? */ |
139 | unsigned long linear_map_top; /* Top of linear mapping */ | |
140 | ||
141 | #endif /* CONFIG_PPC64 */ | |
142 | ||
3160b097 BB |
143 | #ifdef CONFIG_PPC_FSL_BOOK3E |
144 | /* next_tlbcam_idx is used to round-robin tlbcam entry assignment */ | |
145 | DEFINE_PER_CPU(int, next_tlbcam_idx); | |
146 | EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx); | |
147 | #endif | |
148 | ||
f048aace BH |
149 | /* |
150 | * Base TLB flushing operations: | |
151 | * | |
152 | * - flush_tlb_mm(mm) flushes the specified mm context TLB's | |
153 | * - flush_tlb_page(vma, vmaddr) flushes one page | |
154 | * - flush_tlb_range(vma, start, end) flushes a range of pages | |
155 | * - flush_tlb_kernel_range(start, end) flushes kernel pages | |
156 | * | |
157 | * - local_* variants of page and mm only apply to the current | |
158 | * processor | |
159 | */ | |
160 | ||
161 | /* | |
162 | * These are the base non-SMP variants of page and mm flushing | |
163 | */ | |
164 | void local_flush_tlb_mm(struct mm_struct *mm) | |
165 | { | |
166 | unsigned int pid; | |
167 | ||
168 | preempt_disable(); | |
169 | pid = mm->context.id; | |
170 | if (pid != MMU_NO_CONTEXT) | |
171 | _tlbil_pid(pid); | |
172 | preempt_enable(); | |
173 | } | |
174 | EXPORT_SYMBOL(local_flush_tlb_mm); | |
175 | ||
d4e167da BH |
176 | void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, |
177 | int tsize, int ind) | |
f048aace BH |
178 | { |
179 | unsigned int pid; | |
180 | ||
181 | preempt_disable(); | |
d4e167da | 182 | pid = mm ? mm->context.id : 0; |
f048aace | 183 | if (pid != MMU_NO_CONTEXT) |
d4e167da | 184 | _tlbil_va(vmaddr, pid, tsize, ind); |
f048aace BH |
185 | preempt_enable(); |
186 | } | |
f048aace | 187 | |
d4e167da BH |
188 | void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) |
189 | { | |
190 | __local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr, | |
25d21ad6 | 191 | mmu_get_tsize(mmu_virtual_psize), 0); |
d4e167da BH |
192 | } |
193 | EXPORT_SYMBOL(local_flush_tlb_page); | |
f048aace BH |
194 | |
195 | /* | |
196 | * And here are the SMP non-local implementations | |
197 | */ | |
198 | #ifdef CONFIG_SMP | |
199 | ||
3eb93c55 | 200 | static DEFINE_RAW_SPINLOCK(tlbivax_lock); |
f048aace | 201 | |
fcce8109 BH |
202 | static int mm_is_core_local(struct mm_struct *mm) |
203 | { | |
204 | return cpumask_subset(mm_cpumask(mm), | |
205 | topology_thread_cpumask(smp_processor_id())); | |
206 | } | |
207 | ||
f048aace BH |
208 | struct tlb_flush_param { |
209 | unsigned long addr; | |
210 | unsigned int pid; | |
d4e167da BH |
211 | unsigned int tsize; |
212 | unsigned int ind; | |
f048aace BH |
213 | }; |
214 | ||
215 | static void do_flush_tlb_mm_ipi(void *param) | |
216 | { | |
217 | struct tlb_flush_param *p = param; | |
218 | ||
219 | _tlbil_pid(p ? p->pid : 0); | |
220 | } | |
221 | ||
222 | static void do_flush_tlb_page_ipi(void *param) | |
223 | { | |
224 | struct tlb_flush_param *p = param; | |
225 | ||
d4e167da | 226 | _tlbil_va(p->addr, p->pid, p->tsize, p->ind); |
f048aace BH |
227 | } |
228 | ||
229 | ||
230 | /* Note on invalidations and PID: | |
231 | * | |
232 | * We snapshot the PID with preempt disabled. At this point, it can still | |
233 | * change either because: | |
234 | * - our context is being stolen (PID -> NO_CONTEXT) on another CPU | |
235 | * - we are invaliating some target that isn't currently running here | |
236 | * and is concurrently acquiring a new PID on another CPU | |
237 | * - some other CPU is re-acquiring a lost PID for this mm | |
238 | * etc... | |
239 | * | |
240 | * However, this shouldn't be a problem as we only guarantee | |
241 | * invalidation of TLB entries present prior to this call, so we | |
242 | * don't care about the PID changing, and invalidating a stale PID | |
243 | * is generally harmless. | |
244 | */ | |
245 | ||
246 | void flush_tlb_mm(struct mm_struct *mm) | |
247 | { | |
f048aace BH |
248 | unsigned int pid; |
249 | ||
250 | preempt_disable(); | |
251 | pid = mm->context.id; | |
252 | if (unlikely(pid == MMU_NO_CONTEXT)) | |
253 | goto no_context; | |
fcce8109 | 254 | if (!mm_is_core_local(mm)) { |
f048aace | 255 | struct tlb_flush_param p = { .pid = pid }; |
56aa4129 RR |
256 | /* Ignores smp_processor_id() even if set. */ |
257 | smp_call_function_many(mm_cpumask(mm), | |
258 | do_flush_tlb_mm_ipi, &p, 1); | |
f048aace BH |
259 | } |
260 | _tlbil_pid(pid); | |
261 | no_context: | |
262 | preempt_enable(); | |
263 | } | |
264 | EXPORT_SYMBOL(flush_tlb_mm); | |
265 | ||
d4e167da BH |
266 | void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, |
267 | int tsize, int ind) | |
f048aace | 268 | { |
56aa4129 | 269 | struct cpumask *cpu_mask; |
f048aace BH |
270 | unsigned int pid; |
271 | ||
272 | preempt_disable(); | |
d4e167da | 273 | pid = mm ? mm->context.id : 0; |
f048aace BH |
274 | if (unlikely(pid == MMU_NO_CONTEXT)) |
275 | goto bail; | |
d4e167da | 276 | cpu_mask = mm_cpumask(mm); |
fcce8109 | 277 | if (!mm_is_core_local(mm)) { |
f048aace BH |
278 | /* If broadcast tlbivax is supported, use it */ |
279 | if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) { | |
280 | int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL); | |
281 | if (lock) | |
3eb93c55 | 282 | raw_spin_lock(&tlbivax_lock); |
d4e167da | 283 | _tlbivax_bcast(vmaddr, pid, tsize, ind); |
f048aace | 284 | if (lock) |
3eb93c55 | 285 | raw_spin_unlock(&tlbivax_lock); |
f048aace BH |
286 | goto bail; |
287 | } else { | |
d4e167da BH |
288 | struct tlb_flush_param p = { |
289 | .pid = pid, | |
290 | .addr = vmaddr, | |
291 | .tsize = tsize, | |
292 | .ind = ind, | |
293 | }; | |
56aa4129 RR |
294 | /* Ignores smp_processor_id() even if set in cpu_mask */ |
295 | smp_call_function_many(cpu_mask, | |
f048aace BH |
296 | do_flush_tlb_page_ipi, &p, 1); |
297 | } | |
298 | } | |
d4e167da | 299 | _tlbil_va(vmaddr, pid, tsize, ind); |
f048aace BH |
300 | bail: |
301 | preempt_enable(); | |
302 | } | |
d4e167da BH |
303 | |
304 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) | |
305 | { | |
41151e77 BB |
306 | #ifdef CONFIG_HUGETLB_PAGE |
307 | if (is_vm_hugetlb_page(vma)) | |
308 | flush_hugetlb_page(vma, vmaddr); | |
309 | #endif | |
310 | ||
d4e167da | 311 | __flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr, |
25d21ad6 | 312 | mmu_get_tsize(mmu_virtual_psize), 0); |
d4e167da | 313 | } |
f048aace BH |
314 | EXPORT_SYMBOL(flush_tlb_page); |
315 | ||
316 | #endif /* CONFIG_SMP */ | |
317 | ||
91b191c7 DK |
318 | #ifdef CONFIG_PPC_47x |
319 | void __init early_init_mmu_47x(void) | |
320 | { | |
321 | #ifdef CONFIG_SMP | |
322 | unsigned long root = of_get_flat_dt_root(); | |
323 | if (of_get_flat_dt_prop(root, "cooperative-partition", NULL)) | |
324 | mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST); | |
325 | #endif /* CONFIG_SMP */ | |
326 | } | |
327 | #endif /* CONFIG_PPC_47x */ | |
328 | ||
f048aace BH |
329 | /* |
330 | * Flush kernel TLB entries in the given range | |
331 | */ | |
332 | void flush_tlb_kernel_range(unsigned long start, unsigned long end) | |
333 | { | |
334 | #ifdef CONFIG_SMP | |
335 | preempt_disable(); | |
336 | smp_call_function(do_flush_tlb_mm_ipi, NULL, 1); | |
337 | _tlbil_pid(0); | |
338 | preempt_enable(); | |
d6a09e0c | 339 | #else |
f048aace | 340 | _tlbil_pid(0); |
d6a09e0c | 341 | #endif |
f048aace BH |
342 | } |
343 | EXPORT_SYMBOL(flush_tlb_kernel_range); | |
344 | ||
345 | /* | |
346 | * Currently, for range flushing, we just do a full mm flush. This should | |
347 | * be optimized based on a threshold on the size of the range, since | |
348 | * some implementation can stack multiple tlbivax before a tlbsync but | |
349 | * for now, we keep it that way | |
350 | */ | |
351 | void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, | |
352 | unsigned long end) | |
353 | ||
354 | { | |
355 | flush_tlb_mm(vma->vm_mm); | |
356 | } | |
357 | EXPORT_SYMBOL(flush_tlb_range); | |
c7cc58a1 BH |
358 | |
359 | void tlb_flush(struct mmu_gather *tlb) | |
360 | { | |
361 | flush_tlb_mm(tlb->mm); | |
c7cc58a1 | 362 | } |
25d21ad6 BH |
363 | |
364 | /* | |
365 | * Below are functions specific to the 64-bit variant of Book3E though that | |
366 | * may change in the future | |
367 | */ | |
368 | ||
369 | #ifdef CONFIG_PPC64 | |
370 | ||
371 | /* | |
372 | * Handling of virtual linear page tables or indirect TLB entries | |
373 | * flushing when PTE pages are freed | |
374 | */ | |
375 | void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address) | |
376 | { | |
377 | int tsize = mmu_psize_defs[mmu_pte_psize].enc; | |
378 | ||
379 | if (book3e_htw_enabled) { | |
380 | unsigned long start = address & PMD_MASK; | |
381 | unsigned long end = address + PMD_SIZE; | |
382 | unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift; | |
383 | ||
384 | /* This isn't the most optimal, ideally we would factor out the | |
385 | * while preempt & CPU mask mucking around, or even the IPI but | |
386 | * it will do for now | |
387 | */ | |
388 | while (start < end) { | |
389 | __flush_tlb_page(tlb->mm, start, tsize, 1); | |
390 | start += size; | |
391 | } | |
392 | } else { | |
393 | unsigned long rmask = 0xf000000000000000ul; | |
394 | unsigned long rid = (address & rmask) | 0x1000000000000000ul; | |
395 | unsigned long vpte = address & ~rmask; | |
396 | ||
397 | #ifdef CONFIG_PPC_64K_PAGES | |
398 | vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful; | |
399 | #else | |
400 | vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful; | |
401 | #endif | |
402 | vpte |= rid; | |
403 | __flush_tlb_page(tlb->mm, vpte, tsize, 0); | |
404 | } | |
405 | } | |
406 | ||
f2b26c92 BH |
407 | static void setup_page_sizes(void) |
408 | { | |
988cf86d KG |
409 | unsigned int tlb0cfg; |
410 | unsigned int tlb0ps; | |
411 | unsigned int eptcfg; | |
f2b26c92 BH |
412 | int i, psize; |
413 | ||
988cf86d KG |
414 | #ifdef CONFIG_PPC_FSL_BOOK3E |
415 | unsigned int mmucfg = mfspr(SPRN_MMUCFG); | |
416 | ||
417 | if (((mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) && | |
418 | (mmu_has_feature(MMU_FTR_TYPE_FSL_E))) { | |
419 | unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG); | |
420 | unsigned int min_pg, max_pg; | |
421 | ||
422 | min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT; | |
423 | max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT; | |
424 | ||
425 | for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { | |
426 | struct mmu_psize_def *def; | |
427 | unsigned int shift; | |
428 | ||
429 | def = &mmu_psize_defs[psize]; | |
430 | shift = def->shift; | |
431 | ||
432 | if (shift == 0) | |
433 | continue; | |
434 | ||
435 | /* adjust to be in terms of 4^shift Kb */ | |
436 | shift = (shift - 10) >> 1; | |
437 | ||
438 | if ((shift >= min_pg) && (shift <= max_pg)) | |
439 | def->flags |= MMU_PAGE_SIZE_DIRECT; | |
440 | } | |
441 | ||
442 | goto no_indirect; | |
443 | } | |
444 | #endif | |
445 | ||
446 | tlb0cfg = mfspr(SPRN_TLB0CFG); | |
447 | tlb0ps = mfspr(SPRN_TLB0PS); | |
448 | eptcfg = mfspr(SPRN_EPTCFG); | |
449 | ||
f2b26c92 BH |
450 | /* Look for supported direct sizes */ |
451 | for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { | |
452 | struct mmu_psize_def *def = &mmu_psize_defs[psize]; | |
453 | ||
454 | if (tlb0ps & (1U << (def->shift - 10))) | |
455 | def->flags |= MMU_PAGE_SIZE_DIRECT; | |
456 | } | |
457 | ||
458 | /* Indirect page sizes supported ? */ | |
459 | if ((tlb0cfg & TLBnCFG_IND) == 0) | |
460 | goto no_indirect; | |
461 | ||
462 | /* Now, we only deal with one IND page size for each | |
463 | * direct size. Hopefully all implementations today are | |
464 | * unambiguous, but we might want to be careful in the | |
465 | * future. | |
466 | */ | |
467 | for (i = 0; i < 3; i++) { | |
468 | unsigned int ps, sps; | |
469 | ||
470 | sps = eptcfg & 0x1f; | |
471 | eptcfg >>= 5; | |
472 | ps = eptcfg & 0x1f; | |
473 | eptcfg >>= 5; | |
474 | if (!ps || !sps) | |
475 | continue; | |
476 | for (psize = 0; psize < MMU_PAGE_COUNT; psize++) { | |
477 | struct mmu_psize_def *def = &mmu_psize_defs[psize]; | |
478 | ||
479 | if (ps == (def->shift - 10)) | |
480 | def->flags |= MMU_PAGE_SIZE_INDIRECT; | |
481 | if (sps == (def->shift - 10)) | |
482 | def->ind = ps + 10; | |
483 | } | |
484 | } | |
485 | no_indirect: | |
486 | ||
487 | /* Cleanup array and print summary */ | |
488 | pr_info("MMU: Supported page sizes\n"); | |
489 | for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { | |
490 | struct mmu_psize_def *def = &mmu_psize_defs[psize]; | |
491 | const char *__page_type_names[] = { | |
492 | "unsupported", | |
493 | "direct", | |
494 | "indirect", | |
495 | "direct & indirect" | |
496 | }; | |
497 | if (def->flags == 0) { | |
498 | def->shift = 0; | |
499 | continue; | |
500 | } | |
501 | pr_info(" %8ld KB as %s\n", 1ul << (def->shift - 10), | |
502 | __page_type_names[def->flags & 0x3]); | |
503 | } | |
504 | } | |
505 | ||
f67f4ef5 | 506 | static void __patch_exception(int exc, unsigned long addr) |
25d21ad6 BH |
507 | { |
508 | extern unsigned int interrupt_base_book3e; | |
f67f4ef5 SW |
509 | unsigned int *ibase = &interrupt_base_book3e; |
510 | ||
511 | /* Our exceptions vectors start with a NOP and -then- a branch | |
512 | * to deal with single stepping from userspace which stops on | |
513 | * the second instruction. Thus we need to patch the second | |
514 | * instruction of the exception, not the first one | |
515 | */ | |
516 | ||
517 | patch_branch(ibase + (exc / 4) + 1, addr, 0); | |
518 | } | |
25d21ad6 | 519 | |
f67f4ef5 SW |
520 | #define patch_exception(exc, name) do { \ |
521 | extern unsigned int name; \ | |
522 | __patch_exception((exc), (unsigned long)&name); \ | |
523 | } while (0) | |
f2b26c92 | 524 | |
f67f4ef5 SW |
525 | static void setup_mmu_htw(void) |
526 | { | |
f2b26c92 BH |
527 | /* Check if HW tablewalk is present, and if yes, enable it by: |
528 | * | |
529 | * - patching the TLB miss handlers to branch to the | |
530 | * one dedicates to it | |
531 | * | |
532 | * - setting the global book3e_htw_enabled | |
533 | */ | |
534 | unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG); | |
535 | ||
536 | if ((tlb0cfg & TLBnCFG_IND) && | |
537 | (tlb0cfg & TLBnCFG_PT)) { | |
f67f4ef5 SW |
538 | patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e); |
539 | patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e); | |
f2b26c92 BH |
540 | book3e_htw_enabled = 1; |
541 | } | |
32d206eb KG |
542 | pr_info("MMU: Book3E HW tablewalk %s\n", |
543 | book3e_htw_enabled ? "enabled" : "not supported"); | |
f2b26c92 BH |
544 | } |
545 | ||
546 | /* | |
547 | * Early initialization of the MMU TLB code | |
548 | */ | |
549 | static void __early_init_mmu(int boot_cpu) | |
550 | { | |
25d21ad6 BH |
551 | unsigned int mas4; |
552 | ||
553 | /* XXX This will have to be decided at runtime, but right | |
32a74949 BH |
554 | * now our boot and TLB miss code hard wires it. Ideally |
555 | * we should find out a suitable page size and patch the | |
556 | * TLB miss code (either that or use the PACA to store | |
557 | * the value we want) | |
25d21ad6 BH |
558 | */ |
559 | mmu_linear_psize = MMU_PAGE_1G; | |
560 | ||
32a74949 BH |
561 | /* XXX This should be decided at runtime based on supported |
562 | * page sizes in the TLB, but for now let's assume 16M is | |
563 | * always there and a good fit (which it probably is) | |
564 | */ | |
565 | mmu_vmemmap_psize = MMU_PAGE_16M; | |
25d21ad6 | 566 | |
25d21ad6 BH |
567 | /* XXX This code only checks for TLB 0 capabilities and doesn't |
568 | * check what page size combos are supported by the HW. It | |
569 | * also doesn't handle the case where a separate array holds | |
570 | * the IND entries from the array loaded by the PT. | |
571 | */ | |
572 | if (boot_cpu) { | |
f2b26c92 BH |
573 | /* Look for supported page sizes */ |
574 | setup_page_sizes(); | |
25d21ad6 | 575 | |
f2b26c92 BH |
576 | /* Look for HW tablewalk support */ |
577 | setup_mmu_htw(); | |
25d21ad6 BH |
578 | } |
579 | ||
580 | /* Set MAS4 based on page table setting */ | |
581 | ||
582 | mas4 = 0x4 << MAS4_WIMGED_SHIFT; | |
583 | if (book3e_htw_enabled) { | |
584 | mas4 |= mas4 | MAS4_INDD; | |
585 | #ifdef CONFIG_PPC_64K_PAGES | |
586 | mas4 |= BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT; | |
587 | mmu_pte_psize = MMU_PAGE_256M; | |
588 | #else | |
589 | mas4 |= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT; | |
590 | mmu_pte_psize = MMU_PAGE_1M; | |
591 | #endif | |
592 | } else { | |
593 | #ifdef CONFIG_PPC_64K_PAGES | |
594 | mas4 |= BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT; | |
595 | #else | |
596 | mas4 |= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT; | |
597 | #endif | |
598 | mmu_pte_psize = mmu_virtual_psize; | |
599 | } | |
600 | mtspr(SPRN_MAS4, mas4); | |
601 | ||
602 | /* Set the global containing the top of the linear mapping | |
603 | * for use by the TLB miss code | |
604 | */ | |
95f72d1e | 605 | linear_map_top = memblock_end_of_DRAM(); |
25d21ad6 | 606 | |
55fd766b KG |
607 | #ifdef CONFIG_PPC_FSL_BOOK3E |
608 | if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { | |
609 | unsigned int num_cams; | |
610 | ||
611 | /* use a quarter of the TLBCAM for bolted linear map */ | |
612 | num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4; | |
613 | linear_map_top = map_mem_in_cams(linear_map_top, num_cams); | |
614 | ||
615 | /* limit memory so we dont have linear faults */ | |
616 | memblock_enforce_memory_limit(linear_map_top); | |
617 | memblock_analyze(); | |
f67f4ef5 SW |
618 | |
619 | patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e); | |
620 | patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e); | |
55fd766b KG |
621 | } |
622 | #endif | |
623 | ||
25d21ad6 BH |
624 | /* A sync won't hurt us after mucking around with |
625 | * the MMU configuration | |
626 | */ | |
627 | mb(); | |
e63075a3 BH |
628 | |
629 | memblock_set_current_limit(linear_map_top); | |
25d21ad6 BH |
630 | } |
631 | ||
632 | void __init early_init_mmu(void) | |
633 | { | |
634 | __early_init_mmu(1); | |
635 | } | |
636 | ||
637 | void __cpuinit early_init_mmu_secondary(void) | |
638 | { | |
639 | __early_init_mmu(0); | |
640 | } | |
641 | ||
cd3db0c4 BH |
642 | void setup_initial_memory_limit(phys_addr_t first_memblock_base, |
643 | phys_addr_t first_memblock_size) | |
644 | { | |
645 | /* On Embedded 64-bit, we adjust the RMA size to match | |
646 | * the bolted TLB entry. We know for now that only 1G | |
647 | * entries are supported though that may eventually | |
648 | * change. We crop it to the size of the first MEMBLOCK to | |
649 | * avoid going over total available memory just in case... | |
650 | */ | |
651 | ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000); | |
652 | ||
653 | /* Finally limit subsequent allocations */ | |
4a89261b | 654 | memblock_set_current_limit(first_memblock_base + ppc64_rma_size); |
cd3db0c4 | 655 | } |
91b191c7 DK |
656 | #else /* ! CONFIG_PPC64 */ |
657 | void __init early_init_mmu(void) | |
658 | { | |
659 | #ifdef CONFIG_PPC_47x | |
660 | early_init_mmu_47x(); | |
661 | #endif | |
662 | } | |
25d21ad6 | 663 | #endif /* CONFIG_PPC64 */ |