2 * Copyright IBM Corp. 2006
3 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
6 #include <linux/bootmem.h>
9 #include <linux/module.h>
10 #include <linux/list.h>
11 #include <linux/hugetlb.h>
12 #include <linux/slab.h>
13 #include <asm/pgalloc.h>
14 #include <asm/pgtable.h>
15 #include <asm/setup.h>
16 #include <asm/tlbflush.h>
17 #include <asm/sections.h>
19 static DEFINE_MUTEX(vmem_mutex
);
21 struct memory_segment
{
22 struct list_head list
;
27 static LIST_HEAD(mem_segs
);
29 static void __ref
*vmem_alloc_pages(unsigned int order
)
31 if (slab_is_available())
32 return (void *)__get_free_pages(GFP_KERNEL
, order
);
33 return alloc_bootmem_pages((1 << order
) * PAGE_SIZE
);
36 static inline pud_t
*vmem_pud_alloc(void)
41 pud
= vmem_alloc_pages(2);
44 clear_table((unsigned long *) pud
, _REGION3_ENTRY_EMPTY
, PAGE_SIZE
* 4);
49 static inline pmd_t
*vmem_pmd_alloc(void)
54 pmd
= vmem_alloc_pages(2);
57 clear_table((unsigned long *) pmd
, _SEGMENT_ENTRY_EMPTY
, PAGE_SIZE
* 4);
62 static pte_t __ref
*vmem_pte_alloc(unsigned long address
)
66 if (slab_is_available())
67 pte
= (pte_t
*) page_table_alloc(&init_mm
, address
);
69 pte
= alloc_bootmem(PTRS_PER_PTE
* sizeof(pte_t
));
72 clear_table((unsigned long *) pte
, _PAGE_INVALID
,
73 PTRS_PER_PTE
* sizeof(pte_t
));
78 * Add a physical memory range to the 1:1 mapping.
80 static int vmem_add_mem(unsigned long start
, unsigned long size
, int ro
)
82 unsigned long end
= start
+ size
;
83 unsigned long address
= start
;
90 while (address
< end
) {
91 pg_dir
= pgd_offset_k(address
);
92 if (pgd_none(*pg_dir
)) {
93 pu_dir
= vmem_pud_alloc();
96 pgd_populate(&init_mm
, pg_dir
, pu_dir
);
98 pu_dir
= pud_offset(pg_dir
, address
);
99 #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC)
100 if (MACHINE_HAS_EDAT2
&& pud_none(*pu_dir
) && address
&&
101 !(address
& ~PUD_MASK
) && (address
+ PUD_SIZE
<= end
)) {
102 pud_val(*pu_dir
) = __pa(address
) |
103 _REGION_ENTRY_TYPE_R3
| _REGION3_ENTRY_LARGE
|
104 (ro
? _REGION_ENTRY_PROTECT
: 0);
109 if (pud_none(*pu_dir
)) {
110 pm_dir
= vmem_pmd_alloc();
113 pud_populate(&init_mm
, pu_dir
, pm_dir
);
115 pm_dir
= pmd_offset(pu_dir
, address
);
116 #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC)
117 if (MACHINE_HAS_EDAT1
&& pmd_none(*pm_dir
) && address
&&
118 !(address
& ~PMD_MASK
) && (address
+ PMD_SIZE
<= end
)) {
119 pmd_val(*pm_dir
) = __pa(address
) |
120 _SEGMENT_ENTRY
| _SEGMENT_ENTRY_LARGE
|
121 _SEGMENT_ENTRY_YOUNG
|
122 (ro
? _SEGMENT_ENTRY_PROTECT
: 0);
127 if (pmd_none(*pm_dir
)) {
128 pt_dir
= vmem_pte_alloc(address
);
131 pmd_populate(&init_mm
, pm_dir
, pt_dir
);
134 pt_dir
= pte_offset_kernel(pm_dir
, address
);
135 pte_val(*pt_dir
) = __pa(address
) |
136 pgprot_val(ro
? PAGE_KERNEL_RO
: PAGE_KERNEL
);
137 address
+= PAGE_SIZE
;
145 * Remove a physical memory range from the 1:1 mapping.
146 * Currently only invalidates page table entries.
148 static void vmem_remove_range(unsigned long start
, unsigned long size
)
150 unsigned long end
= start
+ size
;
151 unsigned long address
= start
;
158 pte_val(pte
) = _PAGE_INVALID
;
159 while (address
< end
) {
160 pg_dir
= pgd_offset_k(address
);
161 if (pgd_none(*pg_dir
)) {
162 address
+= PGDIR_SIZE
;
165 pu_dir
= pud_offset(pg_dir
, address
);
166 if (pud_none(*pu_dir
)) {
170 if (pud_large(*pu_dir
)) {
175 pm_dir
= pmd_offset(pu_dir
, address
);
176 if (pmd_none(*pm_dir
)) {
180 if (pmd_large(*pm_dir
)) {
185 pt_dir
= pte_offset_kernel(pm_dir
, address
);
187 address
+= PAGE_SIZE
;
189 flush_tlb_kernel_range(start
, end
);
193 * Add a backed mem_map array to the virtual mem_map array.
195 int __meminit
vmemmap_populate(unsigned long start
, unsigned long end
, int node
)
197 unsigned long address
= start
;
204 for (address
= start
; address
< end
;) {
205 pg_dir
= pgd_offset_k(address
);
206 if (pgd_none(*pg_dir
)) {
207 pu_dir
= vmem_pud_alloc();
210 pgd_populate(&init_mm
, pg_dir
, pu_dir
);
213 pu_dir
= pud_offset(pg_dir
, address
);
214 if (pud_none(*pu_dir
)) {
215 pm_dir
= vmem_pmd_alloc();
218 pud_populate(&init_mm
, pu_dir
, pm_dir
);
221 pm_dir
= pmd_offset(pu_dir
, address
);
222 if (pmd_none(*pm_dir
)) {
224 /* Use 1MB frames for vmemmap if available. We always
225 * use large frames even if they are only partially
227 * Otherwise we would have also page tables since
228 * vmemmap_populate gets called for each section
230 if (MACHINE_HAS_EDAT1
) {
233 new_page
= vmemmap_alloc_block(PMD_SIZE
, node
);
236 pmd_val(*pm_dir
) = __pa(new_page
) |
237 _SEGMENT_ENTRY
| _SEGMENT_ENTRY_LARGE
|
239 address
= (address
+ PMD_SIZE
) & PMD_MASK
;
243 pt_dir
= vmem_pte_alloc(address
);
246 pmd_populate(&init_mm
, pm_dir
, pt_dir
);
247 } else if (pmd_large(*pm_dir
)) {
248 address
= (address
+ PMD_SIZE
) & PMD_MASK
;
252 pt_dir
= pte_offset_kernel(pm_dir
, address
);
253 if (pte_none(*pt_dir
)) {
254 unsigned long new_page
;
256 new_page
=__pa(vmem_alloc_pages(0));
260 __pa(new_page
) | pgprot_val(PAGE_KERNEL
);
262 address
+= PAGE_SIZE
;
264 memset((void *)start
, 0, end
- start
);
270 void vmemmap_free(unsigned long start
, unsigned long end
)
275 * Add memory segment to the segment list if it doesn't overlap with
276 * an already present segment.
278 static int insert_memory_segment(struct memory_segment
*seg
)
280 struct memory_segment
*tmp
;
282 if (seg
->start
+ seg
->size
> VMEM_MAX_PHYS
||
283 seg
->start
+ seg
->size
< seg
->start
)
286 list_for_each_entry(tmp
, &mem_segs
, list
) {
287 if (seg
->start
>= tmp
->start
+ tmp
->size
)
289 if (seg
->start
+ seg
->size
<= tmp
->start
)
293 list_add(&seg
->list
, &mem_segs
);
298 * Remove memory segment from the segment list.
300 static void remove_memory_segment(struct memory_segment
*seg
)
302 list_del(&seg
->list
);
305 static void __remove_shared_memory(struct memory_segment
*seg
)
307 remove_memory_segment(seg
);
308 vmem_remove_range(seg
->start
, seg
->size
);
311 int vmem_remove_mapping(unsigned long start
, unsigned long size
)
313 struct memory_segment
*seg
;
316 mutex_lock(&vmem_mutex
);
319 list_for_each_entry(seg
, &mem_segs
, list
) {
320 if (seg
->start
== start
&& seg
->size
== size
)
324 if (seg
->start
!= start
|| seg
->size
!= size
)
328 __remove_shared_memory(seg
);
331 mutex_unlock(&vmem_mutex
);
335 int vmem_add_mapping(unsigned long start
, unsigned long size
)
337 struct memory_segment
*seg
;
340 mutex_lock(&vmem_mutex
);
342 seg
= kzalloc(sizeof(*seg
), GFP_KERNEL
);
348 ret
= insert_memory_segment(seg
);
352 ret
= vmem_add_mem(start
, size
, 0);
358 __remove_shared_memory(seg
);
362 mutex_unlock(&vmem_mutex
);
367 * map whole physical memory to virtual memory (identity mapping)
368 * we reserve enough space in the vmalloc area for vmemmap to hotplug
369 * additional memory segments.
371 void __init
vmem_map_init(void)
373 unsigned long ro_start
, ro_end
;
374 unsigned long start
, end
;
377 ro_start
= PFN_ALIGN((unsigned long)&_stext
);
378 ro_end
= (unsigned long)&_eshared
& PAGE_MASK
;
379 for (i
= 0; i
< MEMORY_CHUNKS
; i
++) {
380 if (!memory_chunk
[i
].size
)
382 start
= memory_chunk
[i
].addr
;
383 end
= memory_chunk
[i
].addr
+ memory_chunk
[i
].size
;
384 if (start
>= ro_end
|| end
<= ro_start
)
385 vmem_add_mem(start
, end
- start
, 0);
386 else if (start
>= ro_start
&& end
<= ro_end
)
387 vmem_add_mem(start
, end
- start
, 1);
388 else if (start
>= ro_start
) {
389 vmem_add_mem(start
, ro_end
- start
, 1);
390 vmem_add_mem(ro_end
, end
- ro_end
, 0);
391 } else if (end
< ro_end
) {
392 vmem_add_mem(start
, ro_start
- start
, 0);
393 vmem_add_mem(ro_start
, end
- ro_start
, 1);
395 vmem_add_mem(start
, ro_start
- start
, 0);
396 vmem_add_mem(ro_start
, ro_end
- ro_start
, 1);
397 vmem_add_mem(ro_end
, end
- ro_end
, 0);
403 * Convert memory chunk array to a memory segment list so there is a single
404 * list that contains both r/w memory and shared memory segments.
406 static int __init
vmem_convert_memory_chunk(void)
408 struct memory_segment
*seg
;
411 mutex_lock(&vmem_mutex
);
412 for (i
= 0; i
< MEMORY_CHUNKS
; i
++) {
413 if (!memory_chunk
[i
].size
)
415 seg
= kzalloc(sizeof(*seg
), GFP_KERNEL
);
417 panic("Out of memory...\n");
418 seg
->start
= memory_chunk
[i
].addr
;
419 seg
->size
= memory_chunk
[i
].size
;
420 insert_memory_segment(seg
);
422 mutex_unlock(&vmem_mutex
);
426 core_initcall(vmem_convert_memory_chunk
);