2 * linux/arch/arm/mm/mm-armv.c
4 * Copyright (C) 1998-2005 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * Page table sludge for ARM v3 and v4 processor architectures.
12 #include <linux/config.h>
13 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/bootmem.h>
17 #include <linux/highmem.h>
18 #include <linux/nodemask.h>
20 #include <asm/pgalloc.h>
23 #include <asm/setup.h>
24 #include <asm/tlbflush.h>
26 #include <asm/mach/map.h>
28 #define CPOLICY_UNCACHED 0
29 #define CPOLICY_BUFFERED 1
30 #define CPOLICY_WRITETHROUGH 2
31 #define CPOLICY_WRITEBACK 3
32 #define CPOLICY_WRITEALLOC 4
34 static unsigned int cachepolicy __initdata
= CPOLICY_WRITEBACK
;
35 static unsigned int ecc_mask __initdata
= 0;
36 pgprot_t pgprot_kernel
;
38 EXPORT_SYMBOL(pgprot_kernel
);
43 const char policy
[16];
49 static struct cachepolicy cache_policies
[] __initdata
= {
53 .pmd
= PMD_SECT_UNCACHED
,
58 .pmd
= PMD_SECT_BUFFERED
,
59 .pte
= PTE_BUFFERABLE
,
61 .policy
= "writethrough",
66 .policy
= "writeback",
69 .pte
= PTE_BUFFERABLE
|PTE_CACHEABLE
,
71 .policy
= "writealloc",
74 .pte
= PTE_BUFFERABLE
|PTE_CACHEABLE
,
79 * These are useful for identifing cache coherency
80 * problems by allowing the cache or the cache and
81 * writebuffer to be turned off. (Note: the write
82 * buffer should not be on and the cache off).
84 static void __init
early_cachepolicy(char **p
)
88 for (i
= 0; i
< ARRAY_SIZE(cache_policies
); i
++) {
89 int len
= strlen(cache_policies
[i
].policy
);
91 if (memcmp(*p
, cache_policies
[i
].policy
, len
) == 0) {
93 cr_alignment
&= ~cache_policies
[i
].cr_mask
;
94 cr_no_alignment
&= ~cache_policies
[i
].cr_mask
;
99 if (i
== ARRAY_SIZE(cache_policies
))
100 printk(KERN_ERR
"ERROR: unknown or unsupported cache policy\n");
102 set_cr(cr_alignment
);
105 static void __init
early_nocache(char **__unused
)
107 char *p
= "buffered";
108 printk(KERN_WARNING
"nocache is deprecated; use cachepolicy=%s\n", p
);
109 early_cachepolicy(&p
);
112 static void __init
early_nowrite(char **__unused
)
114 char *p
= "uncached";
115 printk(KERN_WARNING
"nowb is deprecated; use cachepolicy=%s\n", p
);
116 early_cachepolicy(&p
);
119 static void __init
early_ecc(char **p
)
121 if (memcmp(*p
, "on", 2) == 0) {
122 ecc_mask
= PMD_PROTECTION
;
124 } else if (memcmp(*p
, "off", 3) == 0) {
130 __early_param("nocache", early_nocache
);
131 __early_param("nowb", early_nowrite
);
132 __early_param("cachepolicy=", early_cachepolicy
);
133 __early_param("ecc=", early_ecc
);
135 static int __init
noalign_setup(char *__unused
)
137 cr_alignment
&= ~CR_A
;
138 cr_no_alignment
&= ~CR_A
;
139 set_cr(cr_alignment
);
143 __setup("noalign", noalign_setup
);
145 #define FIRST_KERNEL_PGD_NR (FIRST_USER_PGD_NR + USER_PTRS_PER_PGD)
147 static inline pmd_t
*pmd_off(pgd_t
*pgd
, unsigned long virt
)
149 return pmd_offset(pgd
, virt
);
152 static inline pmd_t
*pmd_off_k(unsigned long virt
)
154 return pmd_off(pgd_offset_k(virt
), virt
);
158 * need to get a 16k page for level 1
160 pgd_t
*get_pgd_slow(struct mm_struct
*mm
)
162 pgd_t
*new_pgd
, *init_pgd
;
163 pmd_t
*new_pmd
, *init_pmd
;
164 pte_t
*new_pte
, *init_pte
;
166 new_pgd
= (pgd_t
*)__get_free_pages(GFP_KERNEL
, 2);
170 memzero(new_pgd
, FIRST_KERNEL_PGD_NR
* sizeof(pgd_t
));
173 * Copy over the kernel and IO PGD entries
175 init_pgd
= pgd_offset_k(0);
176 memcpy(new_pgd
+ FIRST_KERNEL_PGD_NR
, init_pgd
+ FIRST_KERNEL_PGD_NR
,
177 (PTRS_PER_PGD
- FIRST_KERNEL_PGD_NR
) * sizeof(pgd_t
));
179 clean_dcache_area(new_pgd
, PTRS_PER_PGD
* sizeof(pgd_t
));
181 if (!vectors_high()) {
183 * This lock is here just to satisfy pmd_alloc and pte_lock
185 spin_lock(&mm
->page_table_lock
);
188 * On ARM, first page must always be allocated since it
189 * contains the machine vectors.
191 new_pmd
= pmd_alloc(mm
, new_pgd
, 0);
195 new_pte
= pte_alloc_map(mm
, new_pmd
, 0);
199 init_pmd
= pmd_offset(init_pgd
, 0);
200 init_pte
= pte_offset_map_nested(init_pmd
, 0);
201 set_pte(new_pte
, *init_pte
);
202 pte_unmap_nested(init_pte
);
205 spin_unlock(&mm
->page_table_lock
);
211 spin_unlock(&mm
->page_table_lock
);
213 free_pages((unsigned long)new_pgd
, 2);
217 spin_unlock(&mm
->page_table_lock
);
218 free_pages((unsigned long)new_pgd
, 2);
225 void free_pgd_slow(pgd_t
*pgd
)
233 /* pgd is always present and good */
234 pmd
= pmd_off(pgd
, 0);
243 pte
= pmd_page(*pmd
);
245 dec_page_state(nr_page_table_pages
);
249 free_pages((unsigned long) pgd
, 2);
253 * Create a SECTION PGD between VIRT and PHYS in domain
254 * DOMAIN with protection PROT. This operates on half-
255 * pgdir entry increments.
258 alloc_init_section(unsigned long virt
, unsigned long phys
, int prot
)
260 pmd_t
*pmdp
= pmd_off_k(virt
);
262 if (virt
& (1 << 20))
265 *pmdp
= __pmd(phys
| prot
);
266 flush_pmd_entry(pmdp
);
270 * Create a SUPER SECTION PGD between VIRT and PHYS with protection PROT
273 alloc_init_supersection(unsigned long virt
, unsigned long phys
, int prot
)
277 for (i
= 0; i
< 16; i
+= 1) {
278 alloc_init_section(virt
, phys
, prot
| PMD_SECT_SUPER
);
280 virt
+= (PGDIR_SIZE
/ 2);
285 * Add a PAGE mapping between VIRT and PHYS in domain
286 * DOMAIN with protection PROT. Note that due to the
287 * way we map the PTEs, we must allocate two PTE_SIZE'd
288 * blocks - one for the Linux pte table, and one for
289 * the hardware pte table.
292 alloc_init_page(unsigned long virt
, unsigned long phys
, unsigned int prot_l1
, pgprot_t prot
)
294 pmd_t
*pmdp
= pmd_off_k(virt
);
297 if (pmd_none(*pmdp
)) {
298 ptep
= alloc_bootmem_low_pages(2 * PTRS_PER_PTE
*
301 __pmd_populate(pmdp
, __pa(ptep
) | prot_l1
);
303 ptep
= pte_offset_kernel(pmdp
, virt
);
305 set_pte(ptep
, pfn_pte(phys
>> PAGE_SHIFT
, prot
));
309 unsigned int prot_pte
;
310 unsigned int prot_l1
;
311 unsigned int prot_sect
;
315 static struct mem_types mem_types
[] __initdata
= {
317 .prot_pte
= L_PTE_PRESENT
| L_PTE_YOUNG
| L_PTE_DIRTY
|
319 .prot_l1
= PMD_TYPE_TABLE
,
320 .prot_sect
= PMD_TYPE_SECT
| PMD_SECT_UNCACHED
|
325 .prot_sect
= PMD_TYPE_SECT
,
326 .domain
= DOMAIN_KERNEL
,
329 .prot_sect
= PMD_TYPE_SECT
| PMD_SECT_MINICACHE
,
330 .domain
= DOMAIN_KERNEL
,
333 .prot_pte
= L_PTE_PRESENT
| L_PTE_YOUNG
| L_PTE_DIRTY
|
335 .prot_l1
= PMD_TYPE_TABLE
,
336 .domain
= DOMAIN_USER
,
338 [MT_HIGH_VECTORS
] = {
339 .prot_pte
= L_PTE_PRESENT
| L_PTE_YOUNG
| L_PTE_DIRTY
|
340 L_PTE_USER
| L_PTE_EXEC
,
341 .prot_l1
= PMD_TYPE_TABLE
,
342 .domain
= DOMAIN_USER
,
345 .prot_sect
= PMD_TYPE_SECT
| PMD_SECT_AP_WRITE
,
346 .domain
= DOMAIN_KERNEL
,
349 .prot_sect
= PMD_TYPE_SECT
,
350 .domain
= DOMAIN_KERNEL
,
352 [MT_IXP2000_DEVICE
] = { /* IXP2400 requires XCB=101 for on-chip I/O */
353 .prot_pte
= L_PTE_PRESENT
| L_PTE_YOUNG
| L_PTE_DIRTY
|
355 .prot_l1
= PMD_TYPE_TABLE
,
356 .prot_sect
= PMD_TYPE_SECT
| PMD_SECT_UNCACHED
|
357 PMD_SECT_AP_WRITE
| PMD_SECT_BUFFERABLE
|
364 * Adjust the PMD section entries according to the CPU in use.
366 void __init
build_mem_type_table(void)
368 struct cachepolicy
*cp
;
369 unsigned int cr
= get_cr();
370 unsigned int user_pgprot
;
371 int cpu_arch
= cpu_architecture();
374 #if defined(CONFIG_CPU_DCACHE_DISABLE)
375 if (cachepolicy
> CPOLICY_BUFFERED
)
376 cachepolicy
= CPOLICY_BUFFERED
;
377 #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
378 if (cachepolicy
> CPOLICY_WRITETHROUGH
)
379 cachepolicy
= CPOLICY_WRITETHROUGH
;
381 if (cpu_arch
< CPU_ARCH_ARMv5
) {
382 if (cachepolicy
>= CPOLICY_WRITEALLOC
)
383 cachepolicy
= CPOLICY_WRITEBACK
;
387 if (cpu_arch
<= CPU_ARCH_ARMv5TEJ
) {
388 for (i
= 0; i
< ARRAY_SIZE(mem_types
); i
++) {
389 if (mem_types
[i
].prot_l1
)
390 mem_types
[i
].prot_l1
|= PMD_BIT4
;
391 if (mem_types
[i
].prot_sect
)
392 mem_types
[i
].prot_sect
|= PMD_BIT4
;
396 cp
= &cache_policies
[cachepolicy
];
397 user_pgprot
= cp
->pte
;
400 * ARMv6 and above have extended page tables.
402 if (cpu_arch
>= CPU_ARCH_ARMv6
&& (cr
& CR_XP
)) {
404 * bit 4 becomes XN which we must clear for the
405 * kernel memory mapping.
407 mem_types
[MT_MEMORY
].prot_sect
&= ~PMD_BIT4
;
408 mem_types
[MT_ROM
].prot_sect
&= ~PMD_BIT4
;
410 * Mark cache clean areas and XIP ROM read only
411 * from SVC mode and no access from userspace.
413 mem_types
[MT_ROM
].prot_sect
|= PMD_SECT_APX
|PMD_SECT_AP_WRITE
;
414 mem_types
[MT_MINICLEAN
].prot_sect
|= PMD_SECT_APX
|PMD_SECT_AP_WRITE
;
415 mem_types
[MT_CACHECLEAN
].prot_sect
|= PMD_SECT_APX
|PMD_SECT_AP_WRITE
;
418 * Mark the device area as "shared device"
420 mem_types
[MT_DEVICE
].prot_pte
|= L_PTE_BUFFERABLE
;
421 mem_types
[MT_DEVICE
].prot_sect
|= PMD_SECT_BUFFERED
;
424 * User pages need to be mapped with the ASID
427 user_pgprot
|= L_PTE_ASID
;
430 if (cpu_arch
>= CPU_ARCH_ARMv5
) {
431 mem_types
[MT_LOW_VECTORS
].prot_pte
|= cp
->pte
& PTE_CACHEABLE
;
432 mem_types
[MT_HIGH_VECTORS
].prot_pte
|= cp
->pte
& PTE_CACHEABLE
;
434 mem_types
[MT_LOW_VECTORS
].prot_pte
|= cp
->pte
;
435 mem_types
[MT_HIGH_VECTORS
].prot_pte
|= cp
->pte
;
436 mem_types
[MT_MINICLEAN
].prot_sect
&= ~PMD_SECT_TEX(1);
439 mem_types
[MT_LOW_VECTORS
].prot_l1
|= ecc_mask
;
440 mem_types
[MT_HIGH_VECTORS
].prot_l1
|= ecc_mask
;
441 mem_types
[MT_MEMORY
].prot_sect
|= ecc_mask
| cp
->pmd
;
442 mem_types
[MT_ROM
].prot_sect
|= cp
->pmd
;
444 for (i
= 0; i
< 16; i
++) {
445 unsigned long v
= pgprot_val(protection_map
[i
]);
446 v
= (v
& ~(PTE_BUFFERABLE
|PTE_CACHEABLE
)) | user_pgprot
;
447 protection_map
[i
] = __pgprot(v
);
450 pgprot_kernel
= __pgprot(L_PTE_PRESENT
| L_PTE_YOUNG
|
451 L_PTE_DIRTY
| L_PTE_WRITE
|
452 L_PTE_EXEC
| cp
->pte
);
456 mem_types
[MT_CACHECLEAN
].prot_sect
|= PMD_SECT_WT
;
460 mem_types
[MT_CACHECLEAN
].prot_sect
|= PMD_SECT_WB
;
463 printk("Memory policy: ECC %sabled, Data cache %s\n",
464 ecc_mask
? "en" : "dis", cp
->policy
);
467 #define vectors_base() (vectors_high() ? 0xffff0000 : 0)
470 * Create the page directory entries and any necessary
471 * page tables for the mapping specified by `md'. We
472 * are able to cope here with varying sizes and address
473 * offsets, and we take full advantage of sections and
476 void __init
create_mapping(struct map_desc
*md
)
478 unsigned long virt
, length
;
479 int prot_sect
, prot_l1
, domain
;
481 unsigned long off
= (u32
)__pfn_to_phys(md
->pfn
);
483 if (md
->virtual != vectors_base() && md
->virtual < TASK_SIZE
) {
484 printk(KERN_WARNING
"BUG: not creating mapping for "
485 "0x%016llx at 0x%08lx in user region\n",
486 __pfn_to_phys((u64
)md
->pfn
), md
->virtual);
490 if ((md
->type
== MT_DEVICE
|| md
->type
== MT_ROM
) &&
491 md
->virtual >= PAGE_OFFSET
&& md
->virtual < VMALLOC_END
) {
492 printk(KERN_WARNING
"BUG: mapping for 0x%016llx at 0x%08lx "
493 "overlaps vmalloc space\n",
494 __pfn_to_phys((u64
)md
->pfn
), md
->virtual);
497 domain
= mem_types
[md
->type
].domain
;
498 prot_pte
= __pgprot(mem_types
[md
->type
].prot_pte
);
499 prot_l1
= mem_types
[md
->type
].prot_l1
| PMD_DOMAIN(domain
);
500 prot_sect
= mem_types
[md
->type
].prot_sect
| PMD_DOMAIN(domain
);
503 * Catch 36-bit addresses
505 if(md
->pfn
>= 0x100000) {
507 printk(KERN_ERR
"MM: invalid domain in supersection "
508 "mapping for 0x%016llx at 0x%08lx\n",
509 __pfn_to_phys((u64
)md
->pfn
), md
->virtual);
512 if((md
->virtual | md
->length
| __pfn_to_phys(md
->pfn
))
513 & ~SUPERSECTION_MASK
) {
514 printk(KERN_ERR
"MM: cannot create mapping for "
515 "0x%016llx at 0x%08lx invalid alignment\n",
516 __pfn_to_phys((u64
)md
->pfn
), md
->virtual);
521 * Shift bits [35:32] of address into bits [23:20] of PMD
524 off
|= (((md
->pfn
>> (32 - PAGE_SHIFT
)) & 0xF) << 20);
531 if (mem_types
[md
->type
].prot_l1
== 0 &&
532 (virt
& 0xfffff || (virt
+ off
) & 0xfffff || (virt
+ length
) & 0xfffff)) {
533 printk(KERN_WARNING
"BUG: map for 0x%08lx at 0x%08lx can not "
534 "be mapped using pages, ignoring.\n",
535 __pfn_to_phys(md
->pfn
), md
->virtual);
539 while ((virt
& 0xfffff || (virt
+ off
) & 0xfffff) && length
>= PAGE_SIZE
) {
540 alloc_init_page(virt
, virt
+ off
, prot_l1
, prot_pte
);
546 /* N.B. ARMv6 supersections are only defined to work with domain 0.
547 * Since domain assignments can in fact be arbitrary, the
548 * 'domain == 0' check below is required to insure that ARMv6
549 * supersections are only allocated for domain 0 regardless
550 * of the actual domain assignments in use.
552 if (cpu_architecture() >= CPU_ARCH_ARMv6
&& domain
== 0) {
554 * Align to supersection boundary if !high pages.
555 * High pages have already been checked for proper
556 * alignment above and they will fail the SUPSERSECTION_MASK
557 * check because of the way the address is encoded into
560 if (md
->pfn
<= 0x100000) {
561 while ((virt
& ~SUPERSECTION_MASK
||
562 (virt
+ off
) & ~SUPERSECTION_MASK
) &&
563 length
>= (PGDIR_SIZE
/ 2)) {
564 alloc_init_section(virt
, virt
+ off
, prot_sect
);
566 virt
+= (PGDIR_SIZE
/ 2);
567 length
-= (PGDIR_SIZE
/ 2);
571 while (length
>= SUPERSECTION_SIZE
) {
572 alloc_init_supersection(virt
, virt
+ off
, prot_sect
);
574 virt
+= SUPERSECTION_SIZE
;
575 length
-= SUPERSECTION_SIZE
;
580 * A section mapping covers half a "pgdir" entry.
582 while (length
>= (PGDIR_SIZE
/ 2)) {
583 alloc_init_section(virt
, virt
+ off
, prot_sect
);
585 virt
+= (PGDIR_SIZE
/ 2);
586 length
-= (PGDIR_SIZE
/ 2);
589 while (length
>= PAGE_SIZE
) {
590 alloc_init_page(virt
, virt
+ off
, prot_l1
, prot_pte
);
598 * In order to soft-boot, we need to insert a 1:1 mapping in place of
599 * the user-mode pages. This will then ensure that we have predictable
600 * results when turning the mmu off
602 void setup_mm_for_reboot(char mode
)
604 unsigned long base_pmdval
;
608 if (current
->mm
&& current
->mm
->pgd
)
609 pgd
= current
->mm
->pgd
;
613 base_pmdval
= PMD_SECT_AP_WRITE
| PMD_SECT_AP_READ
| PMD_TYPE_SECT
;
614 if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ
)
615 base_pmdval
|= PMD_BIT4
;
617 for (i
= 0; i
< FIRST_USER_PGD_NR
+ USER_PTRS_PER_PGD
; i
++, pgd
++) {
618 unsigned long pmdval
= (i
<< PGDIR_SHIFT
) | base_pmdval
;
621 pmd
= pmd_off(pgd
, i
<< PGDIR_SHIFT
);
622 pmd
[0] = __pmd(pmdval
);
623 pmd
[1] = __pmd(pmdval
+ (1 << (PGDIR_SHIFT
- 1)));
624 flush_pmd_entry(pmd
);
629 * Create the architecture specific mappings
631 void __init
iotable_init(struct map_desc
*io_desc
, int nr
)
635 for (i
= 0; i
< nr
; i
++)
636 create_mapping(io_desc
+ i
);