Commit | Line | Data |
---|---|---|
e3cfe529 | 1 | /* |
1da177e4 LT |
2 | * Generic VM initialization for x86-64 NUMA setups. |
3 | * Copyright 2002,2003 Andi Kleen, SuSE Labs. | |
e3cfe529 | 4 | */ |
1da177e4 LT |
5 | #include <linux/kernel.h> |
6 | #include <linux/mm.h> | |
7 | #include <linux/string.h> | |
8 | #include <linux/init.h> | |
9 | #include <linux/bootmem.h> | |
72d7c3b3 | 10 | #include <linux/memblock.h> |
1da177e4 LT |
11 | #include <linux/mmzone.h> |
12 | #include <linux/ctype.h> | |
13 | #include <linux/module.h> | |
14 | #include <linux/nodemask.h> | |
3cc87e3f | 15 | #include <linux/sched.h> |
d8fc3afc | 16 | #include <linux/acpi.h> |
1da177e4 LT |
17 | |
18 | #include <asm/e820.h> | |
19 | #include <asm/proto.h> | |
20 | #include <asm/dma.h> | |
1da177e4 | 21 | #include <asm/acpi.h> |
23ac4ae8 | 22 | #include <asm/amd_nb.h> |
1da177e4 | 23 | |
b8ef9172 | 24 | #include "numa_internal.h" |
97e7b78d | 25 | |
6c231b7b | 26 | struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; |
e3cfe529 TG |
27 | EXPORT_SYMBOL(node_data); |
28 | ||
92d4a437 | 29 | nodemask_t numa_nodes_parsed __initdata; |
ec8cf29b | 30 | |
97e7b78d | 31 | static struct numa_meminfo numa_meminfo __initdata; |
ac7136b6 TH |
32 | static int numa_distance_cnt; |
33 | static u8 *numa_distance; | |
34 | ||
e3cfe529 | 35 | static void * __init early_node_mem(int nodeid, unsigned long start, |
24a5da73 YL |
36 | unsigned long end, unsigned long size, |
37 | unsigned long align) | |
a8062231 | 38 | { |
cef625ee | 39 | unsigned long mem; |
e3cfe529 | 40 | |
cef625ee YL |
41 | /* |
42 | * put it on high as possible | |
43 | * something will go with NODE_DATA | |
44 | */ | |
45 | if (start < (MAX_DMA_PFN<<PAGE_SHIFT)) | |
46 | start = MAX_DMA_PFN<<PAGE_SHIFT; | |
47 | if (start < (MAX_DMA32_PFN<<PAGE_SHIFT) && | |
48 | end > (MAX_DMA32_PFN<<PAGE_SHIFT)) | |
49 | start = MAX_DMA32_PFN<<PAGE_SHIFT; | |
72d7c3b3 YL |
50 | mem = memblock_x86_find_in_range_node(nodeid, start, end, size, align); |
51 | if (mem != MEMBLOCK_ERROR) | |
a8062231 | 52 | return __va(mem); |
9347e0b0 | 53 | |
cef625ee YL |
54 | /* extend the search scope */ |
55 | end = max_pfn_mapped << PAGE_SHIFT; | |
419db274 YL |
56 | start = MAX_DMA_PFN << PAGE_SHIFT; |
57 | mem = memblock_find_in_range(start, end, size, align); | |
72d7c3b3 | 58 | if (mem != MEMBLOCK_ERROR) |
a8062231 | 59 | return __va(mem); |
9347e0b0 | 60 | |
1842f90c | 61 | printk(KERN_ERR "Cannot find %lu bytes in node %d\n", |
e3cfe529 | 62 | size, nodeid); |
1842f90c YL |
63 | |
64 | return NULL; | |
a8062231 AK |
65 | } |
66 | ||
d9c515ea TH |
67 | static int __init numa_add_memblk_to(int nid, u64 start, u64 end, |
68 | struct numa_meminfo *mi) | |
ef396ec9 | 69 | { |
56e827fb TH |
70 | /* ignore zero length blks */ |
71 | if (start == end) | |
72 | return 0; | |
97e7b78d | 73 | |
56e827fb TH |
74 | /* whine about and ignore invalid blks */ |
75 | if (start > end || nid < 0 || nid >= MAX_NUMNODES) { | |
76 | pr_warning("NUMA: Warning: invalid memblk node %d (%Lx-%Lx)\n", | |
77 | nid, start, end); | |
78 | return 0; | |
ef396ec9 | 79 | } |
ef396ec9 | 80 | |
56e827fb TH |
81 | if (mi->nr_blks >= NR_NODE_MEMBLKS) { |
82 | pr_err("NUMA: too many memblk ranges\n"); | |
ef396ec9 TH |
83 | return -EINVAL; |
84 | } | |
85 | ||
97e7b78d TH |
86 | mi->blk[mi->nr_blks].start = start; |
87 | mi->blk[mi->nr_blks].end = end; | |
88 | mi->blk[mi->nr_blks].nid = nid; | |
89 | mi->nr_blks++; | |
ef396ec9 TH |
90 | return 0; |
91 | } | |
92 | ||
90e6b677 TH |
93 | /** |
94 | * numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo | |
95 | * @idx: Index of memblk to remove | |
96 | * @mi: numa_meminfo to remove memblk from | |
97 | * | |
98 | * Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and | |
99 | * decrementing @mi->nr_blks. | |
100 | */ | |
b8ef9172 | 101 | void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi) |
2e756be4 TH |
102 | { |
103 | mi->nr_blks--; | |
104 | memmove(&mi->blk[idx], &mi->blk[idx + 1], | |
105 | (mi->nr_blks - idx) * sizeof(mi->blk[0])); | |
106 | } | |
107 | ||
90e6b677 TH |
108 | /** |
109 | * numa_add_memblk - Add one numa_memblk to numa_meminfo | |
110 | * @nid: NUMA node ID of the new memblk | |
111 | * @start: Start address of the new memblk | |
112 | * @end: End address of the new memblk | |
113 | * | |
114 | * Add a new memblk to the default numa_meminfo. | |
115 | * | |
116 | * RETURNS: | |
117 | * 0 on success, -errno on failure. | |
118 | */ | |
d9c515ea TH |
119 | int __init numa_add_memblk(int nid, u64 start, u64 end) |
120 | { | |
121 | return numa_add_memblk_to(nid, start, end, &numa_meminfo); | |
122 | } | |
123 | ||
1da177e4 | 124 | /* Initialize bootmem allocator for a node */ |
7c43769a YL |
125 | void __init |
126 | setup_node_bootmem(int nodeid, unsigned long start, unsigned long end) | |
e3cfe529 | 127 | { |
08677214 | 128 | unsigned long start_pfn, last_pfn, nodedata_phys; |
7c43769a | 129 | const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE); |
1a27fc0a | 130 | int nid; |
1da177e4 | 131 | |
4c31e92b YL |
132 | if (!end) |
133 | return; | |
134 | ||
7c43769a YL |
135 | /* |
136 | * Don't confuse VM with a node that doesn't have the | |
137 | * minimum amount of memory: | |
138 | */ | |
139 | if (end && (end - start) < NODE_MIN_SIZE) | |
140 | return; | |
141 | ||
be3e89ee | 142 | start = roundup(start, ZONE_ALIGN); |
1da177e4 | 143 | |
08677214 | 144 | printk(KERN_INFO "Initmem setup node %d %016lx-%016lx\n", nodeid, |
e3cfe529 | 145 | start, end); |
1da177e4 LT |
146 | |
147 | start_pfn = start >> PAGE_SHIFT; | |
886533a3 | 148 | last_pfn = end >> PAGE_SHIFT; |
1da177e4 | 149 | |
24a5da73 YL |
150 | node_data[nodeid] = early_node_mem(nodeid, start, end, pgdat_size, |
151 | SMP_CACHE_BYTES); | |
a8062231 AK |
152 | if (node_data[nodeid] == NULL) |
153 | return; | |
154 | nodedata_phys = __pa(node_data[nodeid]); | |
a9ce6bc1 | 155 | memblock_x86_reserve_range(nodedata_phys, nodedata_phys + pgdat_size, "NODE_DATA"); |
6118f76f YL |
156 | printk(KERN_INFO " NODE_DATA [%016lx - %016lx]\n", nodedata_phys, |
157 | nodedata_phys + pgdat_size - 1); | |
05293608 | 158 | nid = early_pfn_to_nid(nodedata_phys >> PAGE_SHIFT); |
1842f90c YL |
159 | if (nid != nodeid) |
160 | printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nodeid, nid); | |
1da177e4 | 161 | |
1da177e4 | 162 | memset(NODE_DATA(nodeid), 0, sizeof(pg_data_t)); |
08677214 | 163 | NODE_DATA(nodeid)->node_id = nodeid; |
1da177e4 | 164 | NODE_DATA(nodeid)->node_start_pfn = start_pfn; |
886533a3 | 165 | NODE_DATA(nodeid)->node_spanned_pages = last_pfn - start_pfn; |
1da177e4 | 166 | |
1da177e4 | 167 | node_set_online(nodeid); |
e3cfe529 | 168 | } |
1da177e4 | 169 | |
90e6b677 TH |
170 | /** |
171 | * numa_cleanup_meminfo - Cleanup a numa_meminfo | |
172 | * @mi: numa_meminfo to clean up | |
173 | * | |
174 | * Sanitize @mi by merging and removing unncessary memblks. Also check for | |
175 | * conflicts and clear unused memblks. | |
176 | * | |
177 | * RETURNS: | |
178 | * 0 on success, -errno on failure. | |
179 | */ | |
b8ef9172 | 180 | int __init numa_cleanup_meminfo(struct numa_meminfo *mi) |
fd0435d8 | 181 | { |
56e827fb TH |
182 | const u64 low = 0; |
183 | const u64 high = (u64)max_pfn << PAGE_SHIFT; | |
2e756be4 | 184 | int i, j, k; |
ef396ec9 | 185 | |
2e756be4 | 186 | for (i = 0; i < mi->nr_blks; i++) { |
97e7b78d | 187 | struct numa_memblk *bi = &mi->blk[i]; |
ef396ec9 | 188 | |
56e827fb TH |
189 | /* make sure all blocks are inside the limits */ |
190 | bi->start = max(bi->start, low); | |
191 | bi->end = min(bi->end, high); | |
192 | ||
193 | /* and there's no empty block */ | |
194 | if (bi->start == bi->end) { | |
195 | numa_remove_memblk_from(i--, mi); | |
196 | continue; | |
197 | } | |
198 | ||
2e756be4 | 199 | for (j = i + 1; j < mi->nr_blks; j++) { |
97e7b78d | 200 | struct numa_memblk *bj = &mi->blk[j]; |
ef396ec9 TH |
201 | unsigned long start, end; |
202 | ||
56e827fb TH |
203 | /* |
204 | * See whether there are overlapping blocks. Whine | |
205 | * about but allow overlaps of the same nid. They | |
206 | * will be merged below. | |
207 | */ | |
208 | if (bi->end > bj->start && bi->start < bj->end) { | |
209 | if (bi->nid != bj->nid) { | |
210 | pr_err("NUMA: node %d (%Lx-%Lx) overlaps with node %d (%Lx-%Lx)\n", | |
211 | bi->nid, bi->start, bi->end, | |
212 | bj->nid, bj->start, bj->end); | |
213 | return -EINVAL; | |
214 | } | |
215 | pr_warning("NUMA: Warning: node %d (%Lx-%Lx) overlaps with itself (%Lx-%Lx)\n", | |
216 | bi->nid, bi->start, bi->end, | |
217 | bj->start, bj->end); | |
218 | } | |
219 | ||
2e756be4 TH |
220 | /* |
221 | * Join together blocks on the same node, holes | |
222 | * between which don't overlap with memory on other | |
223 | * nodes. | |
224 | */ | |
97e7b78d | 225 | if (bi->nid != bj->nid) |
ef396ec9 | 226 | continue; |
56e827fb TH |
227 | start = max(min(bi->start, bj->start), low); |
228 | end = min(max(bi->end, bj->end), high); | |
2e756be4 | 229 | for (k = 0; k < mi->nr_blks; k++) { |
97e7b78d TH |
230 | struct numa_memblk *bk = &mi->blk[k]; |
231 | ||
232 | if (bi->nid == bk->nid) | |
ef396ec9 | 233 | continue; |
97e7b78d | 234 | if (start < bk->end && end > bk->start) |
ef396ec9 TH |
235 | break; |
236 | } | |
97e7b78d | 237 | if (k < mi->nr_blks) |
ef396ec9 | 238 | continue; |
ef396ec9 | 239 | printk(KERN_INFO "NUMA: Node %d [%Lx,%Lx) + [%Lx,%Lx) -> [%lx,%lx)\n", |
97e7b78d | 240 | bi->nid, bi->start, bi->end, bj->start, bj->end, |
ef396ec9 | 241 | start, end); |
97e7b78d TH |
242 | bi->start = start; |
243 | bi->end = end; | |
2e756be4 | 244 | numa_remove_memblk_from(j--, mi); |
ef396ec9 TH |
245 | } |
246 | } | |
247 | ||
56e827fb TH |
248 | for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) { |
249 | mi->blk[i].start = mi->blk[i].end = 0; | |
250 | mi->blk[i].nid = NUMA_NO_NODE; | |
251 | } | |
252 | ||
f9c60251 TH |
253 | return 0; |
254 | } | |
255 | ||
4697bdcc TH |
256 | /* |
257 | * Set nodes, which have memory in @mi, in *@nodemask. | |
258 | */ | |
259 | static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask, | |
260 | const struct numa_meminfo *mi) | |
261 | { | |
262 | int i; | |
263 | ||
264 | for (i = 0; i < ARRAY_SIZE(mi->blk); i++) | |
265 | if (mi->blk[i].start != mi->blk[i].end && | |
266 | mi->blk[i].nid != NUMA_NO_NODE) | |
267 | node_set(mi->blk[i].nid, *nodemask); | |
268 | } | |
269 | ||
90e6b677 TH |
270 | /** |
271 | * numa_reset_distance - Reset NUMA distance table | |
272 | * | |
273 | * The current table is freed. The next numa_set_distance() call will | |
274 | * create a new one. | |
ac7136b6 | 275 | */ |
b8ef9172 | 276 | void __init numa_reset_distance(void) |
ac7136b6 | 277 | { |
ce003330 | 278 | size_t size = numa_distance_cnt * numa_distance_cnt * sizeof(numa_distance[0]); |
ac7136b6 | 279 | |
eb8c1e2c | 280 | /* numa_distance could be 1LU marking allocation failure, test cnt */ |
ce003330 | 281 | if (numa_distance_cnt) |
2ca230ba YL |
282 | memblock_x86_free_range(__pa(numa_distance), |
283 | __pa(numa_distance) + size); | |
ce003330 | 284 | numa_distance_cnt = 0; |
eb8c1e2c | 285 | numa_distance = NULL; /* enable table creation */ |
ac7136b6 TH |
286 | } |
287 | ||
2bf50555 YL |
288 | static int __init numa_alloc_distance(void) |
289 | { | |
290 | nodemask_t nodes_parsed; | |
291 | size_t size; | |
292 | int i, j, cnt = 0; | |
293 | u64 phys; | |
294 | ||
295 | /* size the new table and allocate it */ | |
296 | nodes_parsed = numa_nodes_parsed; | |
297 | numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo); | |
298 | ||
299 | for_each_node_mask(i, nodes_parsed) | |
300 | cnt = i; | |
1f565a89 DR |
301 | cnt++; |
302 | size = cnt * cnt * sizeof(numa_distance[0]); | |
2bf50555 YL |
303 | |
304 | phys = memblock_find_in_range(0, (u64)max_pfn_mapped << PAGE_SHIFT, | |
305 | size, PAGE_SIZE); | |
306 | if (phys == MEMBLOCK_ERROR) { | |
307 | pr_warning("NUMA: Warning: can't allocate distance table!\n"); | |
308 | /* don't retry until explicitly reset */ | |
309 | numa_distance = (void *)1LU; | |
310 | return -ENOMEM; | |
311 | } | |
312 | memblock_x86_reserve_range(phys, phys + size, "NUMA DIST"); | |
313 | ||
314 | numa_distance = __va(phys); | |
315 | numa_distance_cnt = cnt; | |
316 | ||
317 | /* fill with the default distances */ | |
318 | for (i = 0; i < cnt; i++) | |
319 | for (j = 0; j < cnt; j++) | |
320 | numa_distance[i * cnt + j] = i == j ? | |
321 | LOCAL_DISTANCE : REMOTE_DISTANCE; | |
322 | printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt); | |
323 | ||
324 | return 0; | |
325 | } | |
326 | ||
90e6b677 TH |
327 | /** |
328 | * numa_set_distance - Set NUMA distance from one NUMA to another | |
329 | * @from: the 'from' node to set distance | |
330 | * @to: the 'to' node to set distance | |
331 | * @distance: NUMA distance | |
332 | * | |
333 | * Set the distance from node @from to @to to @distance. If distance table | |
0d2eb44f | 334 | * doesn't exist, one which is large enough to accommodate all the currently |
90e6b677 | 335 | * known nodes will be created. |
eb8c1e2c TH |
336 | * |
337 | * If such table cannot be allocated, a warning is printed and further | |
338 | * calls are ignored until the distance table is reset with | |
339 | * numa_reset_distance(). | |
340 | * | |
341 | * If @from or @to is higher than the highest known node at the time of | |
342 | * table creation or @distance doesn't make sense, the call is ignored. | |
343 | * This is to allow simplification of specific NUMA config implementations. | |
ac7136b6 TH |
344 | */ |
345 | void __init numa_set_distance(int from, int to, int distance) | |
346 | { | |
2bf50555 YL |
347 | if (!numa_distance && numa_alloc_distance() < 0) |
348 | return; | |
ac7136b6 TH |
349 | |
350 | if (from >= numa_distance_cnt || to >= numa_distance_cnt) { | |
351 | printk_once(KERN_DEBUG "NUMA: Debug: distance out of bound, from=%d to=%d distance=%d\n", | |
352 | from, to, distance); | |
353 | return; | |
354 | } | |
355 | ||
356 | if ((u8)distance != distance || | |
357 | (from == to && distance != LOCAL_DISTANCE)) { | |
358 | pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n", | |
359 | from, to, distance); | |
360 | return; | |
361 | } | |
362 | ||
363 | numa_distance[from * numa_distance_cnt + to] = distance; | |
364 | } | |
365 | ||
366 | int __node_distance(int from, int to) | |
367 | { | |
ac7136b6 TH |
368 | if (from >= numa_distance_cnt || to >= numa_distance_cnt) |
369 | return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE; | |
370 | return numa_distance[from * numa_distance_cnt + to]; | |
371 | } | |
372 | EXPORT_SYMBOL(__node_distance); | |
373 | ||
f9c60251 TH |
374 | /* |
375 | * Sanity check to catch more bad NUMA configurations (they are amazingly | |
376 | * common). Make sure the nodes cover all memory. | |
377 | */ | |
91556237 | 378 | static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi) |
f9c60251 TH |
379 | { |
380 | unsigned long numaram, e820ram; | |
381 | int i; | |
382 | ||
383 | numaram = 0; | |
91556237 TH |
384 | for (i = 0; i < mi->nr_blks; i++) { |
385 | unsigned long s = mi->blk[i].start >> PAGE_SHIFT; | |
386 | unsigned long e = mi->blk[i].end >> PAGE_SHIFT; | |
f9c60251 | 387 | numaram += e - s; |
91556237 | 388 | numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e); |
f9c60251 TH |
389 | if ((long)numaram < 0) |
390 | numaram = 0; | |
391 | } | |
392 | ||
393 | e820ram = max_pfn - (memblock_x86_hole_size(0, | |
394 | max_pfn << PAGE_SHIFT) >> PAGE_SHIFT); | |
395 | /* We seem to lose 3 pages somewhere. Allow 1M of slack. */ | |
396 | if ((long)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) { | |
397 | printk(KERN_ERR "NUMA: nodes only cover %luMB of your %luMB e820 RAM. Not used.\n", | |
398 | (numaram << PAGE_SHIFT) >> 20, | |
399 | (e820ram << PAGE_SHIFT) >> 20); | |
91556237 | 400 | return false; |
f9c60251 | 401 | } |
91556237 | 402 | return true; |
f9c60251 TH |
403 | } |
404 | ||
405 | static int __init numa_register_memblks(struct numa_meminfo *mi) | |
406 | { | |
69efcc6d | 407 | int i, nid; |
f9c60251 TH |
408 | |
409 | /* Account for nodes with cpus and no memory */ | |
4697bdcc TH |
410 | node_possible_map = numa_nodes_parsed; |
411 | numa_nodemask_from_meminfo(&node_possible_map, mi); | |
f9c60251 TH |
412 | if (WARN_ON(nodes_empty(node_possible_map))) |
413 | return -EINVAL; | |
414 | ||
97e7b78d TH |
415 | for (i = 0; i < mi->nr_blks; i++) |
416 | memblock_x86_register_active_regions(mi->blk[i].nid, | |
417 | mi->blk[i].start >> PAGE_SHIFT, | |
418 | mi->blk[i].end >> PAGE_SHIFT); | |
fd0435d8 TH |
419 | |
420 | /* for out of order entries */ | |
421 | sort_node_map(); | |
91556237 | 422 | if (!numa_meminfo_cover_memory(mi)) |
fd0435d8 TH |
423 | return -EINVAL; |
424 | ||
69efcc6d YL |
425 | /* Finally register nodes. */ |
426 | for_each_node_mask(nid, node_possible_map) { | |
427 | u64 start = (u64)max_pfn << PAGE_SHIFT; | |
428 | u64 end = 0; | |
91556237 | 429 | |
69efcc6d YL |
430 | for (i = 0; i < mi->nr_blks; i++) { |
431 | if (nid != mi->blk[i].nid) | |
91556237 | 432 | continue; |
69efcc6d YL |
433 | start = min(mi->blk[i].start, start); |
434 | end = max(mi->blk[i].end, end); | |
91556237 | 435 | } |
69efcc6d YL |
436 | |
437 | if (start < end) | |
438 | setup_node_bootmem(nid, start, end); | |
91556237 | 439 | } |
fd0435d8 | 440 | |
ef396ec9 TH |
441 | return 0; |
442 | } | |
443 | ||
c09cedf4 DR |
444 | /** |
445 | * dummy_numma_init - Fallback dummy NUMA init | |
446 | * | |
447 | * Used if there's no underlying NUMA architecture, NUMA initialization | |
448 | * fails, or NUMA is disabled on the command line. | |
449 | * | |
450 | * Must online at least one node and add memory blocks that cover all | |
451 | * allowed memory. This function must not fail. | |
452 | */ | |
6d496f9f | 453 | static int __init dummy_numa_init(void) |
ffe77a46 | 454 | { |
1da177e4 LT |
455 | printk(KERN_INFO "%s\n", |
456 | numa_off ? "NUMA turned off" : "No NUMA configuration found"); | |
e3cfe529 | 457 | printk(KERN_INFO "Faking a node at %016lx-%016lx\n", |
86ef4dbf | 458 | 0LU, max_pfn << PAGE_SHIFT); |
ffe77a46 | 459 | |
92d4a437 | 460 | node_set(0, numa_nodes_parsed); |
43a662f0 | 461 | numa_add_memblk(0, 0, (u64)max_pfn << PAGE_SHIFT); |
ec8cf29b TH |
462 | |
463 | return 0; | |
464 | } | |
465 | ||
c09cedf4 | 466 | static int __init numa_init(int (*init_func)(void)) |
ffe77a46 | 467 | { |
c09cedf4 DR |
468 | int i; |
469 | int ret; | |
ffe77a46 | 470 | |
c09cedf4 DR |
471 | for (i = 0; i < MAX_LOCAL_APIC; i++) |
472 | set_apicid_to_node(i, NUMA_NO_NODE); | |
ffe77a46 | 473 | |
c09cedf4 DR |
474 | nodes_clear(numa_nodes_parsed); |
475 | nodes_clear(node_possible_map); | |
476 | nodes_clear(node_online_map); | |
477 | memset(&numa_meminfo, 0, sizeof(numa_meminfo)); | |
478 | remove_all_active_ranges(); | |
479 | numa_reset_distance(); | |
ffe77a46 | 480 | |
c09cedf4 DR |
481 | ret = init_func(); |
482 | if (ret < 0) | |
483 | return ret; | |
484 | ret = numa_cleanup_meminfo(&numa_meminfo); | |
485 | if (ret < 0) | |
486 | return ret; | |
ffe77a46 | 487 | |
c09cedf4 | 488 | numa_emulation(&numa_meminfo, numa_distance_cnt); |
206e4208 | 489 | |
c09cedf4 DR |
490 | ret = numa_register_memblks(&numa_meminfo); |
491 | if (ret < 0) | |
492 | return ret; | |
fbe99959 | 493 | |
c09cedf4 DR |
494 | for (i = 0; i < nr_cpu_ids; i++) { |
495 | int nid = early_cpu_to_node(i); | |
fbe99959 | 496 | |
c09cedf4 | 497 | if (nid == NUMA_NO_NODE) |
43a662f0 | 498 | continue; |
c09cedf4 DR |
499 | if (!node_online(nid)) |
500 | numa_clear_node(i); | |
501 | } | |
502 | numa_init_array(); | |
503 | return 0; | |
504 | } | |
43a662f0 | 505 | |
c09cedf4 DR |
506 | void __init initmem_init(void) |
507 | { | |
508 | int ret; | |
fd0435d8 | 509 | |
c09cedf4 DR |
510 | if (!numa_off) { |
511 | #ifdef CONFIG_ACPI_NUMA | |
512 | ret = numa_init(x86_acpi_numa_init); | |
513 | if (!ret) | |
514 | return; | |
515 | #endif | |
516 | #ifdef CONFIG_AMD_NUMA | |
517 | ret = numa_init(amd_numa_init); | |
518 | if (!ret) | |
519 | return; | |
520 | #endif | |
ffe77a46 | 521 | } |
c09cedf4 DR |
522 | |
523 | numa_init(dummy_numa_init); | |
69d81fcd AK |
524 | } |
525 | ||
e3cfe529 TG |
526 | unsigned long __init numa_free_all_bootmem(void) |
527 | { | |
1da177e4 | 528 | unsigned long pages = 0; |
e3cfe529 TG |
529 | int i; |
530 | ||
531 | for_each_online_node(i) | |
1da177e4 | 532 | pages += free_all_bootmem_node(NODE_DATA(i)); |
e3cfe529 | 533 | |
08677214 | 534 | pages += free_all_memory_core_early(MAX_NUMNODES); |
08677214 | 535 | |
1da177e4 | 536 | return pages; |
e3cfe529 | 537 | } |
1da177e4 | 538 | |
bbc9e2f4 | 539 | int __cpuinit numa_cpu_node(int cpu) |
d9c2d5ac | 540 | { |
bbc9e2f4 | 541 | int apicid = early_per_cpu(x86_cpu_to_apicid, cpu); |
d9c2d5ac | 542 | |
bbc9e2f4 TH |
543 | if (apicid != BAD_APICID) |
544 | return __apicid_to_node[apicid]; | |
545 | return NUMA_NO_NODE; | |
d9c2d5ac | 546 | } |