2 * linux/arch/arm/mm/init.c
4 * Copyright (C) 1995-2005 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 #include <linux/kernel.h>
11 #include <linux/errno.h>
12 #include <linux/ptrace.h>
13 #include <linux/swap.h>
14 #include <linux/init.h>
15 #include <linux/bootmem.h>
16 #include <linux/mman.h>
17 #include <linux/nodemask.h>
18 #include <linux/initrd.h>
20 #include <asm/mach-types.h>
21 #include <asm/setup.h>
22 #include <asm/sizes.h>
25 #include <asm/mach/arch.h>
26 #include <asm/mach/map.h>
30 DEFINE_PER_CPU(struct mmu_gather
, mmu_gathers
);
32 extern pgd_t swapper_pg_dir
[PTRS_PER_PGD
];
33 extern void _stext
, _text
, _etext
, __data_start
, _end
, __init_begin
, __init_end
;
34 extern unsigned long phys_initrd_start
;
35 extern unsigned long phys_initrd_size
;
38 * The sole use of this is to pass memory configuration
39 * data from paging_init to mem_init.
41 static struct meminfo meminfo __initdata
= { 0, };
44 * empty_zero_page is a special page that is used for
45 * zero-initialized data and COW.
47 struct page
*empty_zero_page
;
50 * The pmd table for the upper-most set of pages.
56 int free
= 0, total
= 0, reserved
= 0;
57 int shared
= 0, cached
= 0, slab
= 0, node
;
59 printk("Mem-info:\n");
61 printk("Free swap: %6ldkB\n", nr_swap_pages
<<(PAGE_SHIFT
-10));
63 for_each_online_node(node
) {
64 struct page
*page
, *end
;
66 page
= NODE_MEM_MAP(node
);
67 end
= page
+ NODE_DATA(node
)->node_spanned_pages
;
71 if (PageReserved(page
))
73 else if (PageSwapCache(page
))
75 else if (PageSlab(page
))
77 else if (!page_count(page
))
80 shared
+= page_count(page
) - 1;
85 printk("%d pages of RAM\n", total
);
86 printk("%d free pages\n", free
);
87 printk("%d reserved pages\n", reserved
);
88 printk("%d slab pages\n", slab
);
89 printk("%d pages shared\n", shared
);
90 printk("%d pages swap cached\n", cached
);
93 #define for_each_nodebank(iter,mi,no) \
94 for (iter = 0; iter < mi->nr_banks; iter++) \
95 if (mi->bank[iter].node == no)
98 * FIXME: We really want to avoid allocating the bootmap bitmap
99 * over the top of the initrd. Hopefully, this is located towards
100 * the start of a bank, so if we allocate the bootmap bitmap at
101 * the end, we won't clash.
103 static unsigned int __init
104 find_bootmap_pfn(int node
, struct meminfo
*mi
, unsigned int bootmap_pages
)
106 unsigned int start_pfn
, bank
, bootmap_pfn
;
108 start_pfn
= PAGE_ALIGN(__pa(&_end
)) >> PAGE_SHIFT
;
111 for_each_nodebank(bank
, mi
, node
) {
112 unsigned int start
, end
;
114 start
= mi
->bank
[bank
].start
>> PAGE_SHIFT
;
115 end
= (mi
->bank
[bank
].size
+
116 mi
->bank
[bank
].start
) >> PAGE_SHIFT
;
121 if (start
< start_pfn
)
127 if (end
- start
>= bootmap_pages
) {
133 if (bootmap_pfn
== 0)
139 static int __init
check_initrd(struct meminfo
*mi
)
141 int initrd_node
= -2;
142 #ifdef CONFIG_BLK_DEV_INITRD
143 unsigned long end
= phys_initrd_start
+ phys_initrd_size
;
146 * Make sure that the initrd is within a valid area of
149 if (phys_initrd_size
) {
154 for (i
= 0; i
< mi
->nr_banks
; i
++) {
155 unsigned long bank_end
;
157 bank_end
= mi
->bank
[i
].start
+ mi
->bank
[i
].size
;
159 if (mi
->bank
[i
].start
<= phys_initrd_start
&&
161 initrd_node
= mi
->bank
[i
].node
;
165 if (initrd_node
== -1) {
166 printk(KERN_ERR
"initrd (0x%08lx - 0x%08lx) extends beyond "
167 "physical memory - disabling initrd\n",
168 phys_initrd_start
, end
);
169 phys_initrd_start
= phys_initrd_size
= 0;
177 * Reserve the various regions of node 0
179 static __init
void reserve_node_zero(pg_data_t
*pgdat
)
181 unsigned long res_size
= 0;
184 * Register the kernel text and data with bootmem.
185 * Note that this can only be in node 0.
187 #ifdef CONFIG_XIP_KERNEL
188 reserve_bootmem_node(pgdat
, __pa(&__data_start
), &_end
- &__data_start
);
190 reserve_bootmem_node(pgdat
, __pa(&_stext
), &_end
- &_stext
);
194 * Reserve the page tables. These are already in use,
195 * and can only be in node 0.
197 reserve_bootmem_node(pgdat
, __pa(swapper_pg_dir
),
198 PTRS_PER_PGD
* sizeof(pgd_t
));
201 * Hmm... This should go elsewhere, but we really really need to
202 * stop things allocating the low memory; ideally we need a better
203 * implementation of GFP_DMA which does not assume that DMA-able
204 * memory starts at zero.
206 if (machine_is_integrator() || machine_is_cintegrator())
207 res_size
= __pa(swapper_pg_dir
) - PHYS_OFFSET
;
210 * These should likewise go elsewhere. They pre-reserve the
211 * screen memory region at the start of main system memory.
213 if (machine_is_edb7211())
214 res_size
= 0x00020000;
215 if (machine_is_p720t())
216 res_size
= 0x00014000;
220 * Because of the SA1111 DMA bug, we want to preserve our
221 * precious DMA-able memory...
223 res_size
= __pa(swapper_pg_dir
) - PHYS_OFFSET
;
226 reserve_bootmem_node(pgdat
, PHYS_OFFSET
, res_size
);
229 static unsigned long __init
230 bootmem_init_node(int node
, int initrd_node
, struct meminfo
*mi
)
232 unsigned long zone_size
[MAX_NR_ZONES
], zhole_size
[MAX_NR_ZONES
];
233 unsigned long start_pfn
, end_pfn
, boot_pfn
;
234 unsigned int boot_pages
;
242 * Calculate the pfn range, and map the memory banks for this node.
244 for_each_nodebank(i
, mi
, node
) {
245 unsigned long start
, end
;
248 start
= mi
->bank
[i
].start
>> PAGE_SHIFT
;
249 end
= (mi
->bank
[i
].start
+ mi
->bank
[i
].size
) >> PAGE_SHIFT
;
251 if (start_pfn
> start
)
256 map
.pfn
= __phys_to_pfn(mi
->bank
[i
].start
);
257 map
.virtual = __phys_to_virt(mi
->bank
[i
].start
);
258 map
.length
= mi
->bank
[i
].size
;
259 map
.type
= MT_MEMORY
;
261 create_mapping(&map
);
265 * If there is no memory in this node, ignore it.
271 * Allocate the bootmem bitmap page.
273 boot_pages
= bootmem_bootmap_pages(end_pfn
- start_pfn
);
274 boot_pfn
= find_bootmap_pfn(node
, mi
, boot_pages
);
277 * Initialise the bootmem allocator for this node, handing the
278 * memory banks over to bootmem.
280 node_set_online(node
);
281 pgdat
= NODE_DATA(node
);
282 init_bootmem_node(pgdat
, boot_pfn
, start_pfn
, end_pfn
);
284 for_each_nodebank(i
, mi
, node
)
285 free_bootmem_node(pgdat
, mi
->bank
[i
].start
, mi
->bank
[i
].size
);
288 * Reserve the bootmem bitmap for this node.
290 reserve_bootmem_node(pgdat
, boot_pfn
<< PAGE_SHIFT
,
291 boot_pages
<< PAGE_SHIFT
);
293 #ifdef CONFIG_BLK_DEV_INITRD
295 * If the initrd is in this node, reserve its memory.
297 if (node
== initrd_node
) {
298 reserve_bootmem_node(pgdat
, phys_initrd_start
,
300 initrd_start
= __phys_to_virt(phys_initrd_start
);
301 initrd_end
= initrd_start
+ phys_initrd_size
;
306 * Finally, reserve any node zero regions.
309 reserve_node_zero(pgdat
);
312 * initialise the zones within this node.
314 memset(zone_size
, 0, sizeof(zone_size
));
315 memset(zhole_size
, 0, sizeof(zhole_size
));
318 * The size of this node has already been determined. If we need
319 * to do anything fancy with the allocation of this memory to the
320 * zones, now is the time to do it.
322 zone_size
[0] = end_pfn
- start_pfn
;
325 * For each bank in this node, calculate the size of the holes.
326 * holes = node_size - sum(bank_sizes_in_node)
328 zhole_size
[0] = zone_size
[0];
329 for_each_nodebank(i
, mi
, node
)
330 zhole_size
[0] -= mi
->bank
[i
].size
>> PAGE_SHIFT
;
333 * Adjust the sizes according to any special requirements for
336 arch_adjust_zones(node
, zone_size
, zhole_size
);
338 free_area_init_node(node
, pgdat
, zone_size
, start_pfn
, zhole_size
);
343 static void __init
bootmem_init(struct meminfo
*mi
)
345 unsigned long addr
, memend_pfn
= 0;
346 int node
, initrd_node
, i
;
349 * Invalidate the node number for empty or invalid memory banks
351 for (i
= 0; i
< mi
->nr_banks
; i
++)
352 if (mi
->bank
[i
].size
== 0 || mi
->bank
[i
].node
>= MAX_NUMNODES
)
353 mi
->bank
[i
].node
= -1;
355 memcpy(&meminfo
, mi
, sizeof(meminfo
));
358 * Clear out all the mappings below the kernel image.
360 for (addr
= 0; addr
< MODULE_START
; addr
+= PGDIR_SIZE
)
361 pmd_clear(pmd_off_k(addr
));
362 #ifdef CONFIG_XIP_KERNEL
363 /* The XIP kernel is mapped in the module area -- skip over it */
364 addr
= ((unsigned long)&_etext
+ PGDIR_SIZE
- 1) & PGDIR_MASK
;
366 for ( ; addr
< PAGE_OFFSET
; addr
+= PGDIR_SIZE
)
367 pmd_clear(pmd_off_k(addr
));
370 * Clear out all the kernel space mappings, except for the first
371 * memory bank, up to the end of the vmalloc region.
373 for (addr
= __phys_to_virt(mi
->bank
[0].start
+ mi
->bank
[0].size
);
374 addr
< VMALLOC_END
; addr
+= PGDIR_SIZE
)
375 pmd_clear(pmd_off_k(addr
));
378 * Locate which node contains the ramdisk image, if any.
380 initrd_node
= check_initrd(mi
);
383 * Run through each node initialising the bootmem allocator.
385 for_each_node(node
) {
386 unsigned long end_pfn
;
388 end_pfn
= bootmem_init_node(node
, initrd_node
, mi
);
391 * Remember the highest memory PFN.
393 if (end_pfn
> memend_pfn
)
394 memend_pfn
= end_pfn
;
397 high_memory
= __va(memend_pfn
<< PAGE_SHIFT
);
400 * This doesn't seem to be used by the Linux memory manager any
401 * more, but is used by ll_rw_block. If we can get rid of it, we
402 * also get rid of some of the stuff above as well.
404 * Note: max_low_pfn and max_pfn reflect the number of _pages_ in
405 * the system, not the maximum PFN.
407 max_pfn
= max_low_pfn
= memend_pfn
- PHYS_PFN_OFFSET
;
411 * Set up device the mappings. Since we clear out the page tables for all
412 * mappings above VMALLOC_END, we will remove any debug device mappings.
413 * This means you have to be careful how you debug this function, or any
414 * called function. This means you can't use any function or debugging
415 * method which may touch any device, otherwise the kernel _will_ crash.
417 static void __init
devicemaps_init(struct machine_desc
*mdesc
)
424 * Allocate the vector page early.
426 vectors
= alloc_bootmem_low_pages(PAGE_SIZE
);
429 for (addr
= VMALLOC_END
; addr
; addr
+= PGDIR_SIZE
)
430 pmd_clear(pmd_off_k(addr
));
433 * Map the kernel if it is XIP.
434 * It is always first in the modulearea.
436 #ifdef CONFIG_XIP_KERNEL
437 map
.pfn
= __phys_to_pfn(CONFIG_XIP_PHYS_ADDR
& SECTION_MASK
);
438 map
.virtual = MODULE_START
;
439 map
.length
= ((unsigned long)&_etext
- map
.virtual + ~SECTION_MASK
) & SECTION_MASK
;
441 create_mapping(&map
);
445 * Map the cache flushing regions.
448 map
.pfn
= __phys_to_pfn(FLUSH_BASE_PHYS
);
449 map
.virtual = FLUSH_BASE
;
451 map
.type
= MT_CACHECLEAN
;
452 create_mapping(&map
);
454 #ifdef FLUSH_BASE_MINICACHE
455 map
.pfn
= __phys_to_pfn(FLUSH_BASE_PHYS
+ SZ_1M
);
456 map
.virtual = FLUSH_BASE_MINICACHE
;
458 map
.type
= MT_MINICLEAN
;
459 create_mapping(&map
);
463 * Create a mapping for the machine vectors at the high-vectors
464 * location (0xffff0000). If we aren't using high-vectors, also
465 * create a mapping at the low-vectors virtual address.
467 map
.pfn
= __phys_to_pfn(virt_to_phys(vectors
));
468 map
.virtual = 0xffff0000;
469 map
.length
= PAGE_SIZE
;
470 map
.type
= MT_HIGH_VECTORS
;
471 create_mapping(&map
);
473 if (!vectors_high()) {
475 map
.type
= MT_LOW_VECTORS
;
476 create_mapping(&map
);
480 * Ask the machine support to map in the statically mapped devices.
486 * Finally flush the caches and tlb to ensure that we're in a
487 * consistent state wrt the writebuffer. This also ensures that
488 * any write-allocated cache lines in the vector page are written
489 * back. After this point, we can start to touch devices again.
491 local_flush_tlb_all();
496 * paging_init() sets up the page tables, initialises the zone memory
497 * maps, and sets up the zero page, bad page and bad page tables.
499 void __init
paging_init(struct meminfo
*mi
, struct machine_desc
*mdesc
)
503 build_mem_type_table();
505 devicemaps_init(mdesc
);
507 top_pmd
= pmd_off_k(0xffff0000);
510 * allocate the zero page. Note that we count on this going ok.
512 zero_page
= alloc_bootmem_low_pages(PAGE_SIZE
);
513 memzero(zero_page
, PAGE_SIZE
);
514 empty_zero_page
= virt_to_page(zero_page
);
515 flush_dcache_page(empty_zero_page
);
518 static inline void free_area(unsigned long addr
, unsigned long end
, char *s
)
520 unsigned int size
= (end
- addr
) >> 10;
522 for (; addr
< end
; addr
+= PAGE_SIZE
) {
523 struct page
*page
= virt_to_page(addr
);
524 ClearPageReserved(page
);
525 init_page_count(page
);
531 printk(KERN_INFO
"Freeing %s memory: %dK\n", s
, size
);
535 free_memmap(int node
, unsigned long start_pfn
, unsigned long end_pfn
)
537 struct page
*start_pg
, *end_pg
;
538 unsigned long pg
, pgend
;
541 * Convert start_pfn/end_pfn to a struct page pointer.
543 start_pg
= pfn_to_page(start_pfn
);
544 end_pg
= pfn_to_page(end_pfn
);
547 * Convert to physical addresses, and
548 * round start upwards and end downwards.
550 pg
= PAGE_ALIGN(__pa(start_pg
));
551 pgend
= __pa(end_pg
) & PAGE_MASK
;
554 * If there are free pages between these,
555 * free the section of the memmap array.
558 free_bootmem_node(NODE_DATA(node
), pg
, pgend
- pg
);
562 * The mem_map array can get very big. Free the unused area of the memory map.
564 static void __init
free_unused_memmap_node(int node
, struct meminfo
*mi
)
566 unsigned long bank_start
, prev_bank_end
= 0;
570 * [FIXME] This relies on each bank being in address order. This
571 * may not be the case, especially if the user has provided the
572 * information on the command line.
574 for_each_nodebank(i
, mi
, node
) {
575 bank_start
= mi
->bank
[i
].start
>> PAGE_SHIFT
;
576 if (bank_start
< prev_bank_end
) {
577 printk(KERN_ERR
"MEM: unordered memory banks. "
578 "Not freeing memmap.\n");
583 * If we had a previous bank, and there is a space
584 * between the current bank and the previous, free it.
586 if (prev_bank_end
&& prev_bank_end
!= bank_start
)
587 free_memmap(node
, prev_bank_end
, bank_start
);
589 prev_bank_end
= (mi
->bank
[i
].start
+
590 mi
->bank
[i
].size
) >> PAGE_SHIFT
;
595 * mem_init() marks the free areas in the mem_map and tells us how much
596 * memory is free. This is done after various parts of the system have
597 * claimed their memory after the kernel image.
599 void __init
mem_init(void)
601 unsigned int codepages
, datapages
, initpages
;
604 codepages
= &_etext
- &_text
;
605 datapages
= &_end
- &__data_start
;
606 initpages
= &__init_end
- &__init_begin
;
608 #ifndef CONFIG_DISCONTIGMEM
609 max_mapnr
= virt_to_page(high_memory
) - mem_map
;
612 /* this will put all unused low memory onto the freelists */
613 for_each_online_node(node
) {
614 pg_data_t
*pgdat
= NODE_DATA(node
);
616 free_unused_memmap_node(node
, &meminfo
);
618 if (pgdat
->node_spanned_pages
!= 0)
619 totalram_pages
+= free_all_bootmem_node(pgdat
);
623 /* now that our DMA memory is actually so designated, we can free it */
624 free_area(PAGE_OFFSET
, (unsigned long)swapper_pg_dir
, NULL
);
628 * Since our memory may not be contiguous, calculate the
629 * real number of pages we have in this system
631 printk(KERN_INFO
"Memory:");
634 for (i
= 0; i
< meminfo
.nr_banks
; i
++) {
635 num_physpages
+= meminfo
.bank
[i
].size
>> PAGE_SHIFT
;
636 printk(" %ldMB", meminfo
.bank
[i
].size
>> 20);
639 printk(" = %luMB total\n", num_physpages
>> (20 - PAGE_SHIFT
));
640 printk(KERN_NOTICE
"Memory: %luKB available (%dK code, "
641 "%dK data, %dK init)\n",
642 (unsigned long) nr_free_pages() << (PAGE_SHIFT
-10),
643 codepages
>> 10, datapages
>> 10, initpages
>> 10);
645 if (PAGE_SIZE
>= 16384 && num_physpages
<= 128) {
646 extern int sysctl_overcommit_memory
;
648 * On a machine this small we won't get
649 * anywhere without overcommit, so turn
652 sysctl_overcommit_memory
= OVERCOMMIT_ALWAYS
;
656 void free_initmem(void)
658 if (!machine_is_integrator() && !machine_is_cintegrator()) {
659 free_area((unsigned long)(&__init_begin
),
660 (unsigned long)(&__init_end
),
665 #ifdef CONFIG_BLK_DEV_INITRD
667 static int keep_initrd
;
669 void free_initrd_mem(unsigned long start
, unsigned long end
)
672 free_area(start
, end
, "initrd");
675 static int __init
keepinitrd_setup(char *__unused
)
681 __setup("keepinitrd", keepinitrd_setup
);