Commit | Line | Data |
---|---|---|
f4eb07c1 HC |
1 | /* |
2 | * arch/s390/mm/vmem.c | |
3 | * | |
4 | * Copyright IBM Corp. 2006 | |
5 | * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> | |
6 | */ | |
7 | ||
8 | #include <linux/bootmem.h> | |
9 | #include <linux/pfn.h> | |
10 | #include <linux/mm.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/list.h> | |
13 | #include <asm/pgalloc.h> | |
14 | #include <asm/pgtable.h> | |
15 | #include <asm/setup.h> | |
16 | #include <asm/tlbflush.h> | |
17 | ||
f4eb07c1 HC |
18 | static DEFINE_MUTEX(vmem_mutex); |
19 | ||
20 | struct memory_segment { | |
21 | struct list_head list; | |
22 | unsigned long start; | |
23 | unsigned long size; | |
24 | }; | |
25 | ||
26 | static LIST_HEAD(mem_segs); | |
27 | ||
e62133b4 HC |
28 | void __meminit memmap_init(unsigned long size, int nid, unsigned long zone, |
29 | unsigned long start_pfn) | |
f4eb07c1 HC |
30 | { |
31 | struct page *start, *end; | |
32 | struct page *map_start, *map_end; | |
33 | int i; | |
34 | ||
35 | start = pfn_to_page(start_pfn); | |
36 | end = start + size; | |
37 | ||
38 | for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) { | |
39 | unsigned long cstart, cend; | |
40 | ||
41 | cstart = PFN_DOWN(memory_chunk[i].addr); | |
42 | cend = cstart + PFN_DOWN(memory_chunk[i].size); | |
43 | ||
44 | map_start = mem_map + cstart; | |
45 | map_end = mem_map + cend; | |
46 | ||
47 | if (map_start < start) | |
48 | map_start = start; | |
49 | if (map_end > end) | |
50 | map_end = end; | |
51 | ||
52 | map_start -= ((unsigned long) map_start & (PAGE_SIZE - 1)) | |
53 | / sizeof(struct page); | |
54 | map_end += ((PFN_ALIGN((unsigned long) map_end) | |
55 | - (unsigned long) map_end) | |
56 | / sizeof(struct page)); | |
57 | ||
58 | if (map_start < map_end) | |
59 | memmap_init_zone((unsigned long)(map_end - map_start), | |
a2f3aa02 DH |
60 | nid, zone, page_to_pfn(map_start), |
61 | MEMMAP_EARLY); | |
f4eb07c1 HC |
62 | } |
63 | } | |
64 | ||
2bc89b5e | 65 | static void __ref *vmem_alloc_pages(unsigned int order) |
f4eb07c1 HC |
66 | { |
67 | if (slab_is_available()) | |
68 | return (void *)__get_free_pages(GFP_KERNEL, order); | |
69 | return alloc_bootmem_pages((1 << order) * PAGE_SIZE); | |
70 | } | |
71 | ||
190a1d72 MS |
72 | #define vmem_pud_alloc() ({ BUG(); ((pud_t *) NULL); }) |
73 | ||
f4eb07c1 HC |
74 | static inline pmd_t *vmem_pmd_alloc(void) |
75 | { | |
3610cce8 | 76 | pmd_t *pmd = NULL; |
f4eb07c1 | 77 | |
3610cce8 MS |
78 | #ifdef CONFIG_64BIT |
79 | pmd = vmem_alloc_pages(2); | |
f4eb07c1 HC |
80 | if (!pmd) |
81 | return NULL; | |
3610cce8 MS |
82 | clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE*4); |
83 | #endif | |
f4eb07c1 HC |
84 | return pmd; |
85 | } | |
86 | ||
146e4b3c | 87 | static pte_t __init_refok *vmem_pte_alloc(void) |
f4eb07c1 | 88 | { |
146e4b3c | 89 | pte_t *pte; |
f4eb07c1 | 90 | |
146e4b3c MS |
91 | if (slab_is_available()) |
92 | pte = (pte_t *) page_table_alloc(&init_mm); | |
93 | else | |
94 | pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); | |
f4eb07c1 HC |
95 | if (!pte) |
96 | return NULL; | |
146e4b3c MS |
97 | clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, |
98 | PTRS_PER_PTE * sizeof(pte_t)); | |
f4eb07c1 HC |
99 | return pte; |
100 | } | |
101 | ||
102 | /* | |
103 | * Add a physical memory range to the 1:1 mapping. | |
104 | */ | |
105 | static int vmem_add_range(unsigned long start, unsigned long size) | |
106 | { | |
107 | unsigned long address; | |
108 | pgd_t *pg_dir; | |
190a1d72 | 109 | pud_t *pu_dir; |
f4eb07c1 HC |
110 | pmd_t *pm_dir; |
111 | pte_t *pt_dir; | |
112 | pte_t pte; | |
113 | int ret = -ENOMEM; | |
114 | ||
115 | for (address = start; address < start + size; address += PAGE_SIZE) { | |
116 | pg_dir = pgd_offset_k(address); | |
117 | if (pgd_none(*pg_dir)) { | |
190a1d72 MS |
118 | pu_dir = vmem_pud_alloc(); |
119 | if (!pu_dir) | |
120 | goto out; | |
121 | pgd_populate_kernel(&init_mm, pg_dir, pu_dir); | |
122 | } | |
123 | ||
124 | pu_dir = pud_offset(pg_dir, address); | |
125 | if (pud_none(*pu_dir)) { | |
f4eb07c1 HC |
126 | pm_dir = vmem_pmd_alloc(); |
127 | if (!pm_dir) | |
128 | goto out; | |
190a1d72 | 129 | pud_populate_kernel(&init_mm, pu_dir, pm_dir); |
f4eb07c1 HC |
130 | } |
131 | ||
190a1d72 | 132 | pm_dir = pmd_offset(pu_dir, address); |
f4eb07c1 HC |
133 | if (pmd_none(*pm_dir)) { |
134 | pt_dir = vmem_pte_alloc(); | |
135 | if (!pt_dir) | |
136 | goto out; | |
137 | pmd_populate_kernel(&init_mm, pm_dir, pt_dir); | |
138 | } | |
139 | ||
140 | pt_dir = pte_offset_kernel(pm_dir, address); | |
141 | pte = pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL); | |
c1821c2e | 142 | *pt_dir = pte; |
f4eb07c1 HC |
143 | } |
144 | ret = 0; | |
145 | out: | |
146 | flush_tlb_kernel_range(start, start + size); | |
147 | return ret; | |
148 | } | |
149 | ||
150 | /* | |
151 | * Remove a physical memory range from the 1:1 mapping. | |
152 | * Currently only invalidates page table entries. | |
153 | */ | |
154 | static void vmem_remove_range(unsigned long start, unsigned long size) | |
155 | { | |
156 | unsigned long address; | |
157 | pgd_t *pg_dir; | |
190a1d72 | 158 | pud_t *pu_dir; |
f4eb07c1 HC |
159 | pmd_t *pm_dir; |
160 | pte_t *pt_dir; | |
161 | pte_t pte; | |
162 | ||
163 | pte_val(pte) = _PAGE_TYPE_EMPTY; | |
164 | for (address = start; address < start + size; address += PAGE_SIZE) { | |
165 | pg_dir = pgd_offset_k(address); | |
190a1d72 MS |
166 | pu_dir = pud_offset(pg_dir, address); |
167 | if (pud_none(*pu_dir)) | |
f4eb07c1 | 168 | continue; |
190a1d72 | 169 | pm_dir = pmd_offset(pu_dir, address); |
f4eb07c1 HC |
170 | if (pmd_none(*pm_dir)) |
171 | continue; | |
172 | pt_dir = pte_offset_kernel(pm_dir, address); | |
c1821c2e | 173 | *pt_dir = pte; |
f4eb07c1 HC |
174 | } |
175 | flush_tlb_kernel_range(start, start + size); | |
176 | } | |
177 | ||
178 | /* | |
179 | * Add a backed mem_map array to the virtual mem_map array. | |
180 | */ | |
181 | static int vmem_add_mem_map(unsigned long start, unsigned long size) | |
182 | { | |
183 | unsigned long address, start_addr, end_addr; | |
184 | struct page *map_start, *map_end; | |
185 | pgd_t *pg_dir; | |
190a1d72 | 186 | pud_t *pu_dir; |
f4eb07c1 HC |
187 | pmd_t *pm_dir; |
188 | pte_t *pt_dir; | |
189 | pte_t pte; | |
190 | int ret = -ENOMEM; | |
191 | ||
5fd9c6e2 CB |
192 | map_start = VMEM_MAP + PFN_DOWN(start); |
193 | map_end = VMEM_MAP + PFN_DOWN(start + size); | |
f4eb07c1 HC |
194 | |
195 | start_addr = (unsigned long) map_start & PAGE_MASK; | |
196 | end_addr = PFN_ALIGN((unsigned long) map_end); | |
197 | ||
198 | for (address = start_addr; address < end_addr; address += PAGE_SIZE) { | |
199 | pg_dir = pgd_offset_k(address); | |
200 | if (pgd_none(*pg_dir)) { | |
190a1d72 MS |
201 | pu_dir = vmem_pud_alloc(); |
202 | if (!pu_dir) | |
203 | goto out; | |
204 | pgd_populate_kernel(&init_mm, pg_dir, pu_dir); | |
205 | } | |
206 | ||
207 | pu_dir = pud_offset(pg_dir, address); | |
208 | if (pud_none(*pu_dir)) { | |
f4eb07c1 HC |
209 | pm_dir = vmem_pmd_alloc(); |
210 | if (!pm_dir) | |
211 | goto out; | |
190a1d72 | 212 | pud_populate_kernel(&init_mm, pu_dir, pm_dir); |
f4eb07c1 HC |
213 | } |
214 | ||
190a1d72 | 215 | pm_dir = pmd_offset(pu_dir, address); |
f4eb07c1 HC |
216 | if (pmd_none(*pm_dir)) { |
217 | pt_dir = vmem_pte_alloc(); | |
218 | if (!pt_dir) | |
219 | goto out; | |
220 | pmd_populate_kernel(&init_mm, pm_dir, pt_dir); | |
221 | } | |
222 | ||
223 | pt_dir = pte_offset_kernel(pm_dir, address); | |
224 | if (pte_none(*pt_dir)) { | |
225 | unsigned long new_page; | |
226 | ||
227 | new_page =__pa(vmem_alloc_pages(0)); | |
228 | if (!new_page) | |
229 | goto out; | |
230 | pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL); | |
c1821c2e | 231 | *pt_dir = pte; |
f4eb07c1 HC |
232 | } |
233 | } | |
234 | ret = 0; | |
235 | out: | |
236 | flush_tlb_kernel_range(start_addr, end_addr); | |
237 | return ret; | |
238 | } | |
239 | ||
240 | static int vmem_add_mem(unsigned long start, unsigned long size) | |
241 | { | |
242 | int ret; | |
243 | ||
a2fd64d6 | 244 | ret = vmem_add_mem_map(start, size); |
f4eb07c1 HC |
245 | if (ret) |
246 | return ret; | |
a2fd64d6 | 247 | return vmem_add_range(start, size); |
f4eb07c1 HC |
248 | } |
249 | ||
250 | /* | |
251 | * Add memory segment to the segment list if it doesn't overlap with | |
252 | * an already present segment. | |
253 | */ | |
254 | static int insert_memory_segment(struct memory_segment *seg) | |
255 | { | |
256 | struct memory_segment *tmp; | |
257 | ||
0189103c | 258 | if (seg->start + seg->size >= VMEM_MAX_PHYS || |
f4eb07c1 HC |
259 | seg->start + seg->size < seg->start) |
260 | return -ERANGE; | |
261 | ||
262 | list_for_each_entry(tmp, &mem_segs, list) { | |
263 | if (seg->start >= tmp->start + tmp->size) | |
264 | continue; | |
265 | if (seg->start + seg->size <= tmp->start) | |
266 | continue; | |
267 | return -ENOSPC; | |
268 | } | |
269 | list_add(&seg->list, &mem_segs); | |
270 | return 0; | |
271 | } | |
272 | ||
273 | /* | |
274 | * Remove memory segment from the segment list. | |
275 | */ | |
276 | static void remove_memory_segment(struct memory_segment *seg) | |
277 | { | |
278 | list_del(&seg->list); | |
279 | } | |
280 | ||
281 | static void __remove_shared_memory(struct memory_segment *seg) | |
282 | { | |
283 | remove_memory_segment(seg); | |
284 | vmem_remove_range(seg->start, seg->size); | |
285 | } | |
286 | ||
287 | int remove_shared_memory(unsigned long start, unsigned long size) | |
288 | { | |
289 | struct memory_segment *seg; | |
290 | int ret; | |
291 | ||
292 | mutex_lock(&vmem_mutex); | |
293 | ||
294 | ret = -ENOENT; | |
295 | list_for_each_entry(seg, &mem_segs, list) { | |
296 | if (seg->start == start && seg->size == size) | |
297 | break; | |
298 | } | |
299 | ||
300 | if (seg->start != start || seg->size != size) | |
301 | goto out; | |
302 | ||
303 | ret = 0; | |
304 | __remove_shared_memory(seg); | |
305 | kfree(seg); | |
306 | out: | |
307 | mutex_unlock(&vmem_mutex); | |
308 | return ret; | |
309 | } | |
310 | ||
311 | int add_shared_memory(unsigned long start, unsigned long size) | |
312 | { | |
313 | struct memory_segment *seg; | |
314 | struct page *page; | |
315 | unsigned long pfn, num_pfn, end_pfn; | |
316 | int ret; | |
317 | ||
318 | mutex_lock(&vmem_mutex); | |
319 | ret = -ENOMEM; | |
320 | seg = kzalloc(sizeof(*seg), GFP_KERNEL); | |
321 | if (!seg) | |
322 | goto out; | |
323 | seg->start = start; | |
324 | seg->size = size; | |
325 | ||
326 | ret = insert_memory_segment(seg); | |
327 | if (ret) | |
328 | goto out_free; | |
329 | ||
330 | ret = vmem_add_mem(start, size); | |
331 | if (ret) | |
332 | goto out_remove; | |
333 | ||
334 | pfn = PFN_DOWN(start); | |
335 | num_pfn = PFN_DOWN(size); | |
336 | end_pfn = pfn + num_pfn; | |
337 | ||
338 | page = pfn_to_page(pfn); | |
339 | memset(page, 0, num_pfn * sizeof(struct page)); | |
340 | ||
341 | for (; pfn < end_pfn; pfn++) { | |
342 | page = pfn_to_page(pfn); | |
343 | init_page_count(page); | |
344 | reset_page_mapcount(page); | |
345 | SetPageReserved(page); | |
346 | INIT_LIST_HEAD(&page->lru); | |
347 | } | |
348 | goto out; | |
349 | ||
350 | out_remove: | |
351 | __remove_shared_memory(seg); | |
352 | out_free: | |
353 | kfree(seg); | |
354 | out: | |
355 | mutex_unlock(&vmem_mutex); | |
356 | return ret; | |
357 | } | |
358 | ||
359 | /* | |
360 | * map whole physical memory to virtual memory (identity mapping) | |
5fd9c6e2 CB |
361 | * we reserve enough space in the vmalloc area for vmemmap to hotplug |
362 | * additional memory segments. | |
f4eb07c1 HC |
363 | */ |
364 | void __init vmem_map_init(void) | |
365 | { | |
f4eb07c1 HC |
366 | int i; |
367 | ||
146e4b3c MS |
368 | INIT_LIST_HEAD(&init_mm.context.crst_list); |
369 | INIT_LIST_HEAD(&init_mm.context.pgtable_list); | |
370 | init_mm.context.noexec = 0; | |
5fd9c6e2 | 371 | NODE_DATA(0)->node_mem_map = VMEM_MAP; |
f4eb07c1 HC |
372 | for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) |
373 | vmem_add_mem(memory_chunk[i].addr, memory_chunk[i].size); | |
374 | } | |
375 | ||
376 | /* | |
377 | * Convert memory chunk array to a memory segment list so there is a single | |
378 | * list that contains both r/w memory and shared memory segments. | |
379 | */ | |
380 | static int __init vmem_convert_memory_chunk(void) | |
381 | { | |
382 | struct memory_segment *seg; | |
383 | int i; | |
384 | ||
385 | mutex_lock(&vmem_mutex); | |
9f4b0ba8 | 386 | for (i = 0; i < MEMORY_CHUNKS; i++) { |
f4eb07c1 HC |
387 | if (!memory_chunk[i].size) |
388 | continue; | |
389 | seg = kzalloc(sizeof(*seg), GFP_KERNEL); | |
390 | if (!seg) | |
391 | panic("Out of memory...\n"); | |
392 | seg->start = memory_chunk[i].addr; | |
393 | seg->size = memory_chunk[i].size; | |
394 | insert_memory_segment(seg); | |
395 | } | |
396 | mutex_unlock(&vmem_mutex); | |
397 | return 0; | |
398 | } | |
399 | ||
400 | core_initcall(vmem_convert_memory_chunk); |