2 * linux/arch/x86_64/mm/init.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
18 #include <linux/swap.h>
19 #include <linux/smp.h>
20 #include <linux/init.h>
21 #include <linux/pagemap.h>
22 #include <linux/bootmem.h>
23 #include <linux/proc_fs.h>
24 #include <linux/pci.h>
25 #include <linux/pfn.h>
26 #include <linux/poison.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/module.h>
29 #include <linux/memory_hotplug.h>
30 #include <linux/nmi.h>
32 #include <asm/processor.h>
33 #include <asm/system.h>
34 #include <asm/uaccess.h>
35 #include <asm/pgtable.h>
36 #include <asm/pgalloc.h>
38 #include <asm/fixmap.h>
42 #include <asm/mmu_context.h>
43 #include <asm/proto.h>
45 #include <asm/sections.h>
46 #include <asm/kdebug.h>
50 # define Dprintk(x...)
53 const struct dma_mapping_ops
*dma_ops
;
54 EXPORT_SYMBOL(dma_ops
);
56 static unsigned long dma_reserve __initdata
;
58 DEFINE_PER_CPU(struct mmu_gather
, mmu_gathers
);
61 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
62 * physical space so we can cache the place of the first one and move
63 * around without checking the pgd every time.
68 long i
, total
= 0, reserved
= 0;
69 long shared
= 0, cached
= 0;
73 printk(KERN_INFO
"Mem-info:\n");
75 printk(KERN_INFO
"Free swap: %6ldkB\n",
76 nr_swap_pages
<< (PAGE_SHIFT
-10));
78 for_each_online_pgdat(pgdat
) {
79 for (i
= 0; i
< pgdat
->node_spanned_pages
; ++i
) {
81 * This loop can take a while with 256 GB and
82 * 4k pages so defer the NMI watchdog:
84 if (unlikely(i
% MAX_ORDER_NR_PAGES
== 0))
87 if (!pfn_valid(pgdat
->node_start_pfn
+ i
))
90 page
= pfn_to_page(pgdat
->node_start_pfn
+ i
);
92 if (PageReserved(page
))
94 else if (PageSwapCache(page
))
96 else if (page_count(page
))
97 shared
+= page_count(page
) - 1;
100 printk(KERN_INFO
"%lu pages of RAM\n", total
);
101 printk(KERN_INFO
"%lu reserved pages\n", reserved
);
102 printk(KERN_INFO
"%lu pages shared\n", shared
);
103 printk(KERN_INFO
"%lu pages swap cached\n", cached
);
108 static __init
void *spp_getpage(void)
113 ptr
= (void *) get_zeroed_page(GFP_ATOMIC
);
115 ptr
= alloc_bootmem_pages(PAGE_SIZE
);
117 if (!ptr
|| ((unsigned long)ptr
& ~PAGE_MASK
)) {
118 panic("set_pte_phys: cannot allocate page data %s\n",
119 after_bootmem
? "after bootmem" : "");
122 Dprintk("spp_getpage %p\n", ptr
);
128 set_pte_phys(unsigned long vaddr
, unsigned long phys
, pgprot_t prot
)
135 Dprintk("set_pte_phys %lx to %lx\n", vaddr
, phys
);
137 pgd
= pgd_offset_k(vaddr
);
138 if (pgd_none(*pgd
)) {
139 printk("PGD FIXMAP MISSING, it should be setup in head.S!\n");
142 pud
= pud_offset(pgd
, vaddr
);
143 if (pud_none(*pud
)) {
144 pmd
= (pmd_t
*) spp_getpage();
145 set_pud(pud
, __pud(__pa(pmd
) | _KERNPG_TABLE
| _PAGE_USER
));
146 if (pmd
!= pmd_offset(pud
, 0)) {
147 printk("PAGETABLE BUG #01! %p <-> %p\n",
148 pmd
, pmd_offset(pud
, 0));
152 pmd
= pmd_offset(pud
, vaddr
);
153 if (pmd_none(*pmd
)) {
154 pte
= (pte_t
*) spp_getpage();
155 set_pmd(pmd
, __pmd(__pa(pte
) | _KERNPG_TABLE
| _PAGE_USER
));
156 if (pte
!= pte_offset_kernel(pmd
, 0)) {
157 printk("PAGETABLE BUG #02!\n");
161 new_pte
= pfn_pte(phys
>> PAGE_SHIFT
, prot
);
163 pte
= pte_offset_kernel(pmd
, vaddr
);
164 if (!pte_none(*pte
) &&
165 pte_val(*pte
) != (pte_val(new_pte
) & __supported_pte_mask
))
167 set_pte(pte
, new_pte
);
170 * It's enough to flush this one mapping.
171 * (PGE mappings get flushed as well)
173 __flush_tlb_one(vaddr
);
176 /* NOTE: this is meant to be run only at boot */
178 __set_fixmap(enum fixed_addresses idx
, unsigned long phys
, pgprot_t prot
)
180 unsigned long address
= __fix_to_virt(idx
);
182 if (idx
>= __end_of_fixed_addresses
) {
183 printk("Invalid __set_fixmap\n");
186 set_pte_phys(address
, phys
, prot
);
189 static unsigned long __initdata table_start
;
190 static unsigned long __meminitdata table_end
;
192 static __meminit
void *alloc_low_page(unsigned long *phys
)
194 unsigned long pfn
= table_end
++;
198 adr
= (void *)get_zeroed_page(GFP_ATOMIC
);
205 panic("alloc_low_page: ran out of memory");
207 adr
= early_ioremap(pfn
* PAGE_SIZE
, PAGE_SIZE
);
208 memset(adr
, 0, PAGE_SIZE
);
209 *phys
= pfn
* PAGE_SIZE
;
213 static __meminit
void unmap_low_page(void *adr
)
218 early_iounmap(adr
, PAGE_SIZE
);
221 /* Must run before zap_low_mappings */
222 __meminit
void *early_ioremap(unsigned long addr
, unsigned long size
)
224 pmd_t
*pmd
, *last_pmd
;
228 pmds
= ((addr
& ~PMD_MASK
) + size
+ ~PMD_MASK
) / PMD_SIZE
;
229 vaddr
= __START_KERNEL_map
;
230 pmd
= level2_kernel_pgt
;
231 last_pmd
= level2_kernel_pgt
+ PTRS_PER_PMD
- 1;
233 for (; pmd
<= last_pmd
; pmd
++, vaddr
+= PMD_SIZE
) {
234 for (i
= 0; i
< pmds
; i
++) {
235 if (pmd_present(pmd
[i
]))
236 goto continue_outer_loop
;
238 vaddr
+= addr
& ~PMD_MASK
;
241 for (i
= 0; i
< pmds
; i
++, addr
+= PMD_SIZE
)
242 set_pmd(pmd
+i
, __pmd(addr
| __PAGE_KERNEL_LARGE_EXEC
));
245 return (void *)vaddr
;
249 printk("early_ioremap(0x%lx, %lu) failed\n", addr
, size
);
255 * To avoid virtual aliases later:
257 __meminit
void early_iounmap(void *addr
, unsigned long size
)
263 vaddr
= (unsigned long)addr
;
264 pmds
= ((vaddr
& ~PMD_MASK
) + size
+ ~PMD_MASK
) / PMD_SIZE
;
265 pmd
= level2_kernel_pgt
+ pmd_index(vaddr
);
267 for (i
= 0; i
< pmds
; i
++)
273 static void __meminit
274 phys_pmd_init(pmd_t
*pmd_page
, unsigned long address
, unsigned long end
)
276 int i
= pmd_index(address
);
278 for (; i
< PTRS_PER_PMD
; i
++, address
+= PMD_SIZE
) {
280 pmd_t
*pmd
= pmd_page
+ pmd_index(address
);
282 if (address
>= end
) {
283 if (!after_bootmem
) {
284 for (; i
< PTRS_PER_PMD
; i
++, pmd
++)
285 set_pmd(pmd
, __pmd(0));
293 entry
= __PAGE_KERNEL_LARGE
|_PAGE_GLOBAL
|address
;
294 entry
&= __supported_pte_mask
;
295 set_pmd(pmd
, __pmd(entry
));
299 static void __meminit
300 phys_pmd_update(pud_t
*pud
, unsigned long address
, unsigned long end
)
302 pmd_t
*pmd
= pmd_offset(pud
, 0);
303 spin_lock(&init_mm
.page_table_lock
);
304 phys_pmd_init(pmd
, address
, end
);
305 spin_unlock(&init_mm
.page_table_lock
);
309 static void __meminit
310 phys_pud_init(pud_t
*pud_page
, unsigned long addr
, unsigned long end
)
312 int i
= pud_index(addr
);
314 for (; i
< PTRS_PER_PUD
; i
++, addr
= (addr
& PUD_MASK
) + PUD_SIZE
) {
315 unsigned long pmd_phys
;
316 pud_t
*pud
= pud_page
+ pud_index(addr
);
322 if (!after_bootmem
&&
323 !e820_any_mapped(addr
, addr
+PUD_SIZE
, 0)) {
324 set_pud(pud
, __pud(0));
329 phys_pmd_update(pud
, addr
, end
);
333 pmd
= alloc_low_page(&pmd_phys
);
335 spin_lock(&init_mm
.page_table_lock
);
336 set_pud(pud
, __pud(pmd_phys
| _KERNPG_TABLE
));
337 phys_pmd_init(pmd
, addr
, end
);
338 spin_unlock(&init_mm
.page_table_lock
);
345 static void __init
find_early_table_space(unsigned long end
)
347 unsigned long puds
, pmds
, tables
, start
;
349 puds
= (end
+ PUD_SIZE
- 1) >> PUD_SHIFT
;
350 pmds
= (end
+ PMD_SIZE
- 1) >> PMD_SHIFT
;
351 tables
= round_up(puds
* sizeof(pud_t
), PAGE_SIZE
) +
352 round_up(pmds
* sizeof(pmd_t
), PAGE_SIZE
);
355 * RED-PEN putting page tables only on node 0 could
356 * cause a hotspot and fill up ZONE_DMA. The page tables
357 * need roughly 0.5KB per GB.
360 table_start
= find_e820_area(start
, end
, tables
);
361 if (table_start
== -1UL)
362 panic("Cannot find space for the kernel page tables");
364 table_start
>>= PAGE_SHIFT
;
365 table_end
= table_start
;
367 early_printk("kernel direct mapping tables up to %lx @ %lx-%lx\n",
368 end
, table_start
<< PAGE_SHIFT
,
369 (table_start
<< PAGE_SHIFT
) + tables
);
373 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
374 * This runs before bootmem is initialized and gets pages directly from
375 * the physical memory. To access them they are temporarily mapped.
377 void __init_refok
init_memory_mapping(unsigned long start
, unsigned long end
)
381 Dprintk("init_memory_mapping\n");
384 * Find space for the kernel direct mapping tables.
386 * Later we should allocate these tables in the local node of the
387 * memory mapped. Unfortunately this is done currently before the
388 * nodes are discovered.
391 find_early_table_space(end
);
393 start
= (unsigned long)__va(start
);
394 end
= (unsigned long)__va(end
);
396 for (; start
< end
; start
= next
) {
397 pgd_t
*pgd
= pgd_offset_k(start
);
398 unsigned long pud_phys
;
402 pud
= pud_offset(pgd
, start
& PGDIR_MASK
);
404 pud
= alloc_low_page(&pud_phys
);
406 next
= start
+ PGDIR_SIZE
;
409 phys_pud_init(pud
, __pa(start
), __pa(next
));
411 set_pgd(pgd_offset_k(start
), mk_kernel_pgd(pud_phys
));
416 mmu_cr4_features
= read_cr4();
419 reserve_early(table_start
<< PAGE_SHIFT
, table_end
<< PAGE_SHIFT
);
423 void __init
paging_init(void)
425 unsigned long max_zone_pfns
[MAX_NR_ZONES
];
427 memset(max_zone_pfns
, 0, sizeof(max_zone_pfns
));
428 max_zone_pfns
[ZONE_DMA
] = MAX_DMA_PFN
;
429 max_zone_pfns
[ZONE_DMA32
] = MAX_DMA32_PFN
;
430 max_zone_pfns
[ZONE_NORMAL
] = end_pfn
;
432 memory_present(0, 0, end_pfn
);
434 free_area_init_nodes(max_zone_pfns
);
439 * Unmap a kernel mapping if it exists. This is useful to avoid
440 * prefetches from the CPU leading to inconsistent cache lines.
441 * address and size must be aligned to 2MB boundaries.
442 * Does nothing when the mapping doesn't exist.
444 void __init
clear_kernel_mapping(unsigned long address
, unsigned long size
)
446 unsigned long end
= address
+ size
;
448 BUG_ON(address
& ~LARGE_PAGE_MASK
);
449 BUG_ON(size
& ~LARGE_PAGE_MASK
);
451 for (; address
< end
; address
+= LARGE_PAGE_SIZE
) {
452 pgd_t
*pgd
= pgd_offset_k(address
);
459 pud
= pud_offset(pgd
, address
);
463 pmd
= pmd_offset(pud
, address
);
464 if (!pmd
|| pmd_none(*pmd
))
467 if (!(pmd_val(*pmd
) & _PAGE_PSE
)) {
469 * Could handle this, but it should not happen
472 printk(KERN_ERR
"clear_kernel_mapping: "
473 "mapping has been split. will leak memory\n");
476 set_pmd(pmd
, __pmd(0));
482 * Memory hotplug specific functions
484 void online_page(struct page
*page
)
486 ClearPageReserved(page
);
487 init_page_count(page
);
493 #ifdef CONFIG_MEMORY_HOTPLUG
495 * Memory is added always to NORMAL zone. This means you will never get
496 * additional DMA/DMA32 memory.
498 int arch_add_memory(int nid
, u64 start
, u64 size
)
500 struct pglist_data
*pgdat
= NODE_DATA(nid
);
501 struct zone
*zone
= pgdat
->node_zones
+ ZONE_NORMAL
;
502 unsigned long start_pfn
= start
>> PAGE_SHIFT
;
503 unsigned long nr_pages
= size
>> PAGE_SHIFT
;
506 init_memory_mapping(start
, start
+ size
-1);
508 ret
= __add_pages(zone
, start_pfn
, nr_pages
);
510 printk("%s: Problem encountered in __add_pages!\n", __func__
);
514 EXPORT_SYMBOL_GPL(arch_add_memory
);
516 #if !defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA)
517 int memory_add_physaddr_to_nid(u64 start
)
521 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid
);
524 #endif /* CONFIG_MEMORY_HOTPLUG */
526 static struct kcore_list kcore_mem
, kcore_vmalloc
, kcore_kernel
,
527 kcore_modules
, kcore_vsyscall
;
529 void __init
mem_init(void)
531 long codesize
, reservedpages
, datasize
, initsize
;
535 /* clear_bss() already clear the empty_zero_page */
537 /* temporary debugging - double check it's true: */
541 for (i
= 0; i
< 1024; i
++)
542 WARN_ON_ONCE(empty_zero_page
[i
]);
547 /* this will put all low memory onto the freelists */
549 totalram_pages
= numa_free_all_bootmem();
551 totalram_pages
= free_all_bootmem();
553 reservedpages
= end_pfn
- totalram_pages
-
554 absent_pages_in_range(0, end_pfn
);
557 codesize
= (unsigned long) &_etext
- (unsigned long) &_text
;
558 datasize
= (unsigned long) &_edata
- (unsigned long) &_etext
;
559 initsize
= (unsigned long) &__init_end
- (unsigned long) &__init_begin
;
561 /* Register memory areas for /proc/kcore */
562 kclist_add(&kcore_mem
, __va(0), max_low_pfn
<< PAGE_SHIFT
);
563 kclist_add(&kcore_vmalloc
, (void *)VMALLOC_START
,
564 VMALLOC_END
-VMALLOC_START
);
565 kclist_add(&kcore_kernel
, &_stext
, _end
- _stext
);
566 kclist_add(&kcore_modules
, (void *)MODULES_VADDR
, MODULES_LEN
);
567 kclist_add(&kcore_vsyscall
, (void *)VSYSCALL_START
,
568 VSYSCALL_END
- VSYSCALL_START
);
570 printk("Memory: %luk/%luk available (%ldk kernel code, "
571 "%ldk reserved, %ldk data, %ldk init)\n",
572 (unsigned long) nr_free_pages() << (PAGE_SHIFT
-10),
573 end_pfn
<< (PAGE_SHIFT
-10),
575 reservedpages
<< (PAGE_SHIFT
-10),
580 void free_init_pages(char *what
, unsigned long begin
, unsigned long end
)
588 * If debugging page accesses then do not free this memory but
589 * mark them not present - any buggy init-section access will
590 * create a kernel page fault:
592 #ifdef CONFIG_DEBUG_PAGEALLOC
593 printk(KERN_INFO
"debug: unmapping init memory %08lx..%08lx\n",
594 begin
, PAGE_ALIGN(end
));
595 set_memory_np(begin
, (end
- begin
) >> PAGE_SHIFT
);
597 printk(KERN_INFO
"Freeing %s: %luk freed\n", what
, (end
- begin
) >> 10);
599 for (addr
= begin
; addr
< end
; addr
+= PAGE_SIZE
) {
600 ClearPageReserved(virt_to_page(addr
));
601 init_page_count(virt_to_page(addr
));
602 memset((void *)(addr
& ~(PAGE_SIZE
-1)),
603 POISON_FREE_INITMEM
, PAGE_SIZE
);
610 void free_initmem(void)
612 free_init_pages("unused kernel memory",
613 (unsigned long)(&__init_begin
),
614 (unsigned long)(&__init_end
));
617 #ifdef CONFIG_DEBUG_RODATA
618 const int rodata_test_data
= 0xC3;
619 EXPORT_SYMBOL_GPL(rodata_test_data
);
621 void mark_rodata_ro(void)
623 unsigned long start
= (unsigned long)_stext
, end
;
625 #ifdef CONFIG_HOTPLUG_CPU
626 /* It must still be possible to apply SMP alternatives. */
627 if (num_possible_cpus() > 1)
628 start
= (unsigned long)_etext
;
631 #ifdef CONFIG_KPROBES
632 start
= (unsigned long)__start_rodata
;
635 end
= (unsigned long)__end_rodata
;
636 start
= (start
+ PAGE_SIZE
- 1) & PAGE_MASK
;
641 set_memory_ro(start
, (end
- start
) >> PAGE_SHIFT
);
643 printk(KERN_INFO
"Write protecting the kernel read-only data: %luk\n",
644 (end
- start
) >> 10);
648 #ifdef CONFIG_CPA_DEBUG
649 printk("Testing CPA: undo %lx-%lx\n", start
, end
);
650 set_memory_rw(start
, (end
-start
) >> PAGE_SHIFT
);
652 printk("Testing CPA: again\n");
653 set_memory_ro(start
, (end
-start
) >> PAGE_SHIFT
);
658 #ifdef CONFIG_BLK_DEV_INITRD
659 void free_initrd_mem(unsigned long start
, unsigned long end
)
661 free_init_pages("initrd memory", start
, end
);
665 void __init
reserve_bootmem_generic(unsigned long phys
, unsigned len
)
668 int nid
= phys_to_nid(phys
);
670 unsigned long pfn
= phys
>> PAGE_SHIFT
;
672 if (pfn
>= end_pfn
) {
674 * This can happen with kdump kernels when accessing
677 if (pfn
< end_pfn_map
)
680 printk(KERN_ERR
"reserve_bootmem: illegal reserve %lx %u\n",
685 /* Should check here against the e820 map to avoid double free */
687 reserve_bootmem_node(NODE_DATA(nid
), phys
, len
);
689 reserve_bootmem(phys
, len
);
691 if (phys
+len
<= MAX_DMA_PFN
*PAGE_SIZE
) {
692 dma_reserve
+= len
/ PAGE_SIZE
;
693 set_dma_reserve(dma_reserve
);
697 int kern_addr_valid(unsigned long addr
)
699 unsigned long above
= ((long)addr
) >> __VIRTUAL_MASK_SHIFT
;
705 if (above
!= 0 && above
!= -1UL)
708 pgd
= pgd_offset_k(addr
);
712 pud
= pud_offset(pgd
, addr
);
716 pmd
= pmd_offset(pud
, addr
);
721 return pfn_valid(pmd_pfn(*pmd
));
723 pte
= pte_offset_kernel(pmd
, addr
);
727 return pfn_valid(pte_pfn(*pte
));
731 * A pseudo VMA to allow ptrace access for the vsyscall page. This only
732 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
733 * not need special handling anymore:
735 static struct vm_area_struct gate_vma
= {
736 .vm_start
= VSYSCALL_START
,
737 .vm_end
= VSYSCALL_START
+ (VSYSCALL_MAPPED_PAGES
* PAGE_SIZE
),
738 .vm_page_prot
= PAGE_READONLY_EXEC
,
739 .vm_flags
= VM_READ
| VM_EXEC
742 struct vm_area_struct
*get_gate_vma(struct task_struct
*tsk
)
744 #ifdef CONFIG_IA32_EMULATION
745 if (test_tsk_thread_flag(tsk
, TIF_IA32
))
751 int in_gate_area(struct task_struct
*task
, unsigned long addr
)
753 struct vm_area_struct
*vma
= get_gate_vma(task
);
758 return (addr
>= vma
->vm_start
) && (addr
< vma
->vm_end
);
762 * Use this when you have no reliable task/vma, typically from interrupt
763 * context. It is less reliable than using the task's vma and may give
766 int in_gate_area_no_task(unsigned long addr
)
768 return (addr
>= VSYSCALL_START
) && (addr
< VSYSCALL_END
);
771 const char *arch_vma_name(struct vm_area_struct
*vma
)
773 if (vma
->vm_mm
&& vma
->vm_start
== (long)vma
->vm_mm
->context
.vdso
)
775 if (vma
== &gate_vma
)
780 #ifdef CONFIG_SPARSEMEM_VMEMMAP
782 * Initialise the sparsemem vmemmap using huge-pages at the PMD level.
785 vmemmap_populate(struct page
*start_page
, unsigned long size
, int node
)
787 unsigned long addr
= (unsigned long)start_page
;
788 unsigned long end
= (unsigned long)(start_page
+ size
);
794 for (; addr
< end
; addr
= next
) {
795 next
= pmd_addr_end(addr
, end
);
797 pgd
= vmemmap_pgd_populate(addr
, node
);
801 pud
= vmemmap_pud_populate(pgd
, addr
, node
);
805 pmd
= pmd_offset(pud
, addr
);
806 if (pmd_none(*pmd
)) {
810 p
= vmemmap_alloc_block(PMD_SIZE
, node
);
814 entry
= pfn_pte(__pa(p
) >> PAGE_SHIFT
,
816 set_pmd(pmd
, __pmd(pte_val(entry
)));
818 printk(KERN_DEBUG
" [%lx-%lx] PMD ->%p on node %d\n",
819 addr
, addr
+ PMD_SIZE
- 1, p
, node
);
821 vmemmap_verify((pte_t
*)pmd
, node
, addr
, next
);