Commit | Line | Data |
---|---|---|
e3cfe529 | 1 | /* |
1da177e4 LT |
2 | * Generic VM initialization for x86-64 NUMA setups. |
3 | * Copyright 2002,2003 Andi Kleen, SuSE Labs. | |
e3cfe529 | 4 | */ |
1da177e4 LT |
5 | #include <linux/kernel.h> |
6 | #include <linux/mm.h> | |
7 | #include <linux/string.h> | |
8 | #include <linux/init.h> | |
9 | #include <linux/bootmem.h> | |
72d7c3b3 | 10 | #include <linux/memblock.h> |
1da177e4 LT |
11 | #include <linux/mmzone.h> |
12 | #include <linux/ctype.h> | |
13 | #include <linux/module.h> | |
14 | #include <linux/nodemask.h> | |
3cc87e3f | 15 | #include <linux/sched.h> |
d8fc3afc | 16 | #include <linux/acpi.h> |
1da177e4 LT |
17 | |
18 | #include <asm/e820.h> | |
19 | #include <asm/proto.h> | |
20 | #include <asm/dma.h> | |
1da177e4 | 21 | #include <asm/acpi.h> |
23ac4ae8 | 22 | #include <asm/amd_nb.h> |
1da177e4 | 23 | |
b8ef9172 | 24 | #include "numa_internal.h" |
97e7b78d | 25 | |
6c231b7b | 26 | struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; |
e3cfe529 TG |
27 | EXPORT_SYMBOL(node_data); |
28 | ||
92d4a437 | 29 | nodemask_t numa_nodes_parsed __initdata; |
ec8cf29b | 30 | |
9688678a TH |
31 | static struct numa_meminfo numa_meminfo |
32 | #ifndef CONFIG_MEMORY_HOTPLUG | |
33 | __initdata | |
34 | #endif | |
35 | ; | |
36 | ||
ac7136b6 TH |
37 | static int numa_distance_cnt; |
38 | static u8 *numa_distance; | |
39 | ||
e3cfe529 | 40 | static void * __init early_node_mem(int nodeid, unsigned long start, |
24a5da73 YL |
41 | unsigned long end, unsigned long size, |
42 | unsigned long align) | |
a8062231 | 43 | { |
cef625ee | 44 | unsigned long mem; |
e3cfe529 | 45 | |
cef625ee YL |
46 | /* |
47 | * put it on high as possible | |
48 | * something will go with NODE_DATA | |
49 | */ | |
50 | if (start < (MAX_DMA_PFN<<PAGE_SHIFT)) | |
51 | start = MAX_DMA_PFN<<PAGE_SHIFT; | |
52 | if (start < (MAX_DMA32_PFN<<PAGE_SHIFT) && | |
53 | end > (MAX_DMA32_PFN<<PAGE_SHIFT)) | |
54 | start = MAX_DMA32_PFN<<PAGE_SHIFT; | |
72d7c3b3 YL |
55 | mem = memblock_x86_find_in_range_node(nodeid, start, end, size, align); |
56 | if (mem != MEMBLOCK_ERROR) | |
a8062231 | 57 | return __va(mem); |
9347e0b0 | 58 | |
cef625ee YL |
59 | /* extend the search scope */ |
60 | end = max_pfn_mapped << PAGE_SHIFT; | |
419db274 YL |
61 | start = MAX_DMA_PFN << PAGE_SHIFT; |
62 | mem = memblock_find_in_range(start, end, size, align); | |
72d7c3b3 | 63 | if (mem != MEMBLOCK_ERROR) |
a8062231 | 64 | return __va(mem); |
9347e0b0 | 65 | |
1842f90c | 66 | printk(KERN_ERR "Cannot find %lu bytes in node %d\n", |
e3cfe529 | 67 | size, nodeid); |
1842f90c YL |
68 | |
69 | return NULL; | |
a8062231 AK |
70 | } |
71 | ||
d9c515ea TH |
72 | static int __init numa_add_memblk_to(int nid, u64 start, u64 end, |
73 | struct numa_meminfo *mi) | |
ef396ec9 | 74 | { |
56e827fb TH |
75 | /* ignore zero length blks */ |
76 | if (start == end) | |
77 | return 0; | |
97e7b78d | 78 | |
56e827fb TH |
79 | /* whine about and ignore invalid blks */ |
80 | if (start > end || nid < 0 || nid >= MAX_NUMNODES) { | |
81 | pr_warning("NUMA: Warning: invalid memblk node %d (%Lx-%Lx)\n", | |
82 | nid, start, end); | |
83 | return 0; | |
ef396ec9 | 84 | } |
ef396ec9 | 85 | |
56e827fb TH |
86 | if (mi->nr_blks >= NR_NODE_MEMBLKS) { |
87 | pr_err("NUMA: too many memblk ranges\n"); | |
ef396ec9 TH |
88 | return -EINVAL; |
89 | } | |
90 | ||
97e7b78d TH |
91 | mi->blk[mi->nr_blks].start = start; |
92 | mi->blk[mi->nr_blks].end = end; | |
93 | mi->blk[mi->nr_blks].nid = nid; | |
94 | mi->nr_blks++; | |
ef396ec9 TH |
95 | return 0; |
96 | } | |
97 | ||
90e6b677 TH |
98 | /** |
99 | * numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo | |
100 | * @idx: Index of memblk to remove | |
101 | * @mi: numa_meminfo to remove memblk from | |
102 | * | |
103 | * Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and | |
104 | * decrementing @mi->nr_blks. | |
105 | */ | |
b8ef9172 | 106 | void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi) |
2e756be4 TH |
107 | { |
108 | mi->nr_blks--; | |
109 | memmove(&mi->blk[idx], &mi->blk[idx + 1], | |
110 | (mi->nr_blks - idx) * sizeof(mi->blk[0])); | |
111 | } | |
112 | ||
90e6b677 TH |
113 | /** |
114 | * numa_add_memblk - Add one numa_memblk to numa_meminfo | |
115 | * @nid: NUMA node ID of the new memblk | |
116 | * @start: Start address of the new memblk | |
117 | * @end: End address of the new memblk | |
118 | * | |
119 | * Add a new memblk to the default numa_meminfo. | |
120 | * | |
121 | * RETURNS: | |
122 | * 0 on success, -errno on failure. | |
123 | */ | |
d9c515ea TH |
124 | int __init numa_add_memblk(int nid, u64 start, u64 end) |
125 | { | |
126 | return numa_add_memblk_to(nid, start, end, &numa_meminfo); | |
127 | } | |
128 | ||
1da177e4 | 129 | /* Initialize bootmem allocator for a node */ |
7c43769a YL |
130 | void __init |
131 | setup_node_bootmem(int nodeid, unsigned long start, unsigned long end) | |
e3cfe529 | 132 | { |
08677214 | 133 | unsigned long start_pfn, last_pfn, nodedata_phys; |
7c43769a | 134 | const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE); |
1a27fc0a | 135 | int nid; |
1da177e4 | 136 | |
4c31e92b YL |
137 | if (!end) |
138 | return; | |
139 | ||
7c43769a YL |
140 | /* |
141 | * Don't confuse VM with a node that doesn't have the | |
142 | * minimum amount of memory: | |
143 | */ | |
144 | if (end && (end - start) < NODE_MIN_SIZE) | |
145 | return; | |
146 | ||
be3e89ee | 147 | start = roundup(start, ZONE_ALIGN); |
1da177e4 | 148 | |
08677214 | 149 | printk(KERN_INFO "Initmem setup node %d %016lx-%016lx\n", nodeid, |
e3cfe529 | 150 | start, end); |
1da177e4 LT |
151 | |
152 | start_pfn = start >> PAGE_SHIFT; | |
886533a3 | 153 | last_pfn = end >> PAGE_SHIFT; |
1da177e4 | 154 | |
24a5da73 YL |
155 | node_data[nodeid] = early_node_mem(nodeid, start, end, pgdat_size, |
156 | SMP_CACHE_BYTES); | |
a8062231 AK |
157 | if (node_data[nodeid] == NULL) |
158 | return; | |
159 | nodedata_phys = __pa(node_data[nodeid]); | |
a9ce6bc1 | 160 | memblock_x86_reserve_range(nodedata_phys, nodedata_phys + pgdat_size, "NODE_DATA"); |
6118f76f YL |
161 | printk(KERN_INFO " NODE_DATA [%016lx - %016lx]\n", nodedata_phys, |
162 | nodedata_phys + pgdat_size - 1); | |
05293608 | 163 | nid = early_pfn_to_nid(nodedata_phys >> PAGE_SHIFT); |
1842f90c YL |
164 | if (nid != nodeid) |
165 | printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nodeid, nid); | |
1da177e4 | 166 | |
1da177e4 | 167 | memset(NODE_DATA(nodeid), 0, sizeof(pg_data_t)); |
08677214 | 168 | NODE_DATA(nodeid)->node_id = nodeid; |
1da177e4 | 169 | NODE_DATA(nodeid)->node_start_pfn = start_pfn; |
886533a3 | 170 | NODE_DATA(nodeid)->node_spanned_pages = last_pfn - start_pfn; |
1da177e4 | 171 | |
1da177e4 | 172 | node_set_online(nodeid); |
e3cfe529 | 173 | } |
1da177e4 | 174 | |
90e6b677 TH |
175 | /** |
176 | * numa_cleanup_meminfo - Cleanup a numa_meminfo | |
177 | * @mi: numa_meminfo to clean up | |
178 | * | |
179 | * Sanitize @mi by merging and removing unncessary memblks. Also check for | |
180 | * conflicts and clear unused memblks. | |
181 | * | |
182 | * RETURNS: | |
183 | * 0 on success, -errno on failure. | |
184 | */ | |
b8ef9172 | 185 | int __init numa_cleanup_meminfo(struct numa_meminfo *mi) |
fd0435d8 | 186 | { |
56e827fb TH |
187 | const u64 low = 0; |
188 | const u64 high = (u64)max_pfn << PAGE_SHIFT; | |
2e756be4 | 189 | int i, j, k; |
ef396ec9 | 190 | |
2e756be4 | 191 | for (i = 0; i < mi->nr_blks; i++) { |
97e7b78d | 192 | struct numa_memblk *bi = &mi->blk[i]; |
ef396ec9 | 193 | |
56e827fb TH |
194 | /* make sure all blocks are inside the limits */ |
195 | bi->start = max(bi->start, low); | |
196 | bi->end = min(bi->end, high); | |
197 | ||
198 | /* and there's no empty block */ | |
2be19102 | 199 | if (bi->start >= bi->end) { |
56e827fb TH |
200 | numa_remove_memblk_from(i--, mi); |
201 | continue; | |
202 | } | |
203 | ||
2e756be4 | 204 | for (j = i + 1; j < mi->nr_blks; j++) { |
97e7b78d | 205 | struct numa_memblk *bj = &mi->blk[j]; |
ef396ec9 TH |
206 | unsigned long start, end; |
207 | ||
56e827fb TH |
208 | /* |
209 | * See whether there are overlapping blocks. Whine | |
210 | * about but allow overlaps of the same nid. They | |
211 | * will be merged below. | |
212 | */ | |
213 | if (bi->end > bj->start && bi->start < bj->end) { | |
214 | if (bi->nid != bj->nid) { | |
215 | pr_err("NUMA: node %d (%Lx-%Lx) overlaps with node %d (%Lx-%Lx)\n", | |
216 | bi->nid, bi->start, bi->end, | |
217 | bj->nid, bj->start, bj->end); | |
218 | return -EINVAL; | |
219 | } | |
220 | pr_warning("NUMA: Warning: node %d (%Lx-%Lx) overlaps with itself (%Lx-%Lx)\n", | |
221 | bi->nid, bi->start, bi->end, | |
222 | bj->start, bj->end); | |
223 | } | |
224 | ||
2e756be4 TH |
225 | /* |
226 | * Join together blocks on the same node, holes | |
227 | * between which don't overlap with memory on other | |
228 | * nodes. | |
229 | */ | |
97e7b78d | 230 | if (bi->nid != bj->nid) |
ef396ec9 | 231 | continue; |
56e827fb TH |
232 | start = max(min(bi->start, bj->start), low); |
233 | end = min(max(bi->end, bj->end), high); | |
2e756be4 | 234 | for (k = 0; k < mi->nr_blks; k++) { |
97e7b78d TH |
235 | struct numa_memblk *bk = &mi->blk[k]; |
236 | ||
237 | if (bi->nid == bk->nid) | |
ef396ec9 | 238 | continue; |
97e7b78d | 239 | if (start < bk->end && end > bk->start) |
ef396ec9 TH |
240 | break; |
241 | } | |
97e7b78d | 242 | if (k < mi->nr_blks) |
ef396ec9 | 243 | continue; |
ef396ec9 | 244 | printk(KERN_INFO "NUMA: Node %d [%Lx,%Lx) + [%Lx,%Lx) -> [%lx,%lx)\n", |
97e7b78d | 245 | bi->nid, bi->start, bi->end, bj->start, bj->end, |
ef396ec9 | 246 | start, end); |
97e7b78d TH |
247 | bi->start = start; |
248 | bi->end = end; | |
2e756be4 | 249 | numa_remove_memblk_from(j--, mi); |
ef396ec9 TH |
250 | } |
251 | } | |
252 | ||
56e827fb TH |
253 | for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) { |
254 | mi->blk[i].start = mi->blk[i].end = 0; | |
255 | mi->blk[i].nid = NUMA_NO_NODE; | |
256 | } | |
257 | ||
f9c60251 TH |
258 | return 0; |
259 | } | |
260 | ||
4697bdcc TH |
261 | /* |
262 | * Set nodes, which have memory in @mi, in *@nodemask. | |
263 | */ | |
264 | static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask, | |
265 | const struct numa_meminfo *mi) | |
266 | { | |
267 | int i; | |
268 | ||
269 | for (i = 0; i < ARRAY_SIZE(mi->blk); i++) | |
270 | if (mi->blk[i].start != mi->blk[i].end && | |
271 | mi->blk[i].nid != NUMA_NO_NODE) | |
272 | node_set(mi->blk[i].nid, *nodemask); | |
273 | } | |
274 | ||
90e6b677 TH |
275 | /** |
276 | * numa_reset_distance - Reset NUMA distance table | |
277 | * | |
278 | * The current table is freed. The next numa_set_distance() call will | |
279 | * create a new one. | |
ac7136b6 | 280 | */ |
b8ef9172 | 281 | void __init numa_reset_distance(void) |
ac7136b6 | 282 | { |
ce003330 | 283 | size_t size = numa_distance_cnt * numa_distance_cnt * sizeof(numa_distance[0]); |
ac7136b6 | 284 | |
eb8c1e2c | 285 | /* numa_distance could be 1LU marking allocation failure, test cnt */ |
ce003330 | 286 | if (numa_distance_cnt) |
2ca230ba YL |
287 | memblock_x86_free_range(__pa(numa_distance), |
288 | __pa(numa_distance) + size); | |
ce003330 | 289 | numa_distance_cnt = 0; |
eb8c1e2c | 290 | numa_distance = NULL; /* enable table creation */ |
ac7136b6 TH |
291 | } |
292 | ||
2bf50555 YL |
293 | static int __init numa_alloc_distance(void) |
294 | { | |
295 | nodemask_t nodes_parsed; | |
296 | size_t size; | |
297 | int i, j, cnt = 0; | |
298 | u64 phys; | |
299 | ||
300 | /* size the new table and allocate it */ | |
301 | nodes_parsed = numa_nodes_parsed; | |
302 | numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo); | |
303 | ||
304 | for_each_node_mask(i, nodes_parsed) | |
305 | cnt = i; | |
1f565a89 DR |
306 | cnt++; |
307 | size = cnt * cnt * sizeof(numa_distance[0]); | |
2bf50555 YL |
308 | |
309 | phys = memblock_find_in_range(0, (u64)max_pfn_mapped << PAGE_SHIFT, | |
310 | size, PAGE_SIZE); | |
311 | if (phys == MEMBLOCK_ERROR) { | |
312 | pr_warning("NUMA: Warning: can't allocate distance table!\n"); | |
313 | /* don't retry until explicitly reset */ | |
314 | numa_distance = (void *)1LU; | |
315 | return -ENOMEM; | |
316 | } | |
317 | memblock_x86_reserve_range(phys, phys + size, "NUMA DIST"); | |
318 | ||
319 | numa_distance = __va(phys); | |
320 | numa_distance_cnt = cnt; | |
321 | ||
322 | /* fill with the default distances */ | |
323 | for (i = 0; i < cnt; i++) | |
324 | for (j = 0; j < cnt; j++) | |
325 | numa_distance[i * cnt + j] = i == j ? | |
326 | LOCAL_DISTANCE : REMOTE_DISTANCE; | |
327 | printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt); | |
328 | ||
329 | return 0; | |
330 | } | |
331 | ||
90e6b677 TH |
332 | /** |
333 | * numa_set_distance - Set NUMA distance from one NUMA to another | |
334 | * @from: the 'from' node to set distance | |
335 | * @to: the 'to' node to set distance | |
336 | * @distance: NUMA distance | |
337 | * | |
338 | * Set the distance from node @from to @to to @distance. If distance table | |
0d2eb44f | 339 | * doesn't exist, one which is large enough to accommodate all the currently |
90e6b677 | 340 | * known nodes will be created. |
eb8c1e2c TH |
341 | * |
342 | * If such table cannot be allocated, a warning is printed and further | |
343 | * calls are ignored until the distance table is reset with | |
344 | * numa_reset_distance(). | |
345 | * | |
346 | * If @from or @to is higher than the highest known node at the time of | |
347 | * table creation or @distance doesn't make sense, the call is ignored. | |
348 | * This is to allow simplification of specific NUMA config implementations. | |
ac7136b6 TH |
349 | */ |
350 | void __init numa_set_distance(int from, int to, int distance) | |
351 | { | |
2bf50555 YL |
352 | if (!numa_distance && numa_alloc_distance() < 0) |
353 | return; | |
ac7136b6 TH |
354 | |
355 | if (from >= numa_distance_cnt || to >= numa_distance_cnt) { | |
356 | printk_once(KERN_DEBUG "NUMA: Debug: distance out of bound, from=%d to=%d distance=%d\n", | |
357 | from, to, distance); | |
358 | return; | |
359 | } | |
360 | ||
361 | if ((u8)distance != distance || | |
362 | (from == to && distance != LOCAL_DISTANCE)) { | |
363 | pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n", | |
364 | from, to, distance); | |
365 | return; | |
366 | } | |
367 | ||
368 | numa_distance[from * numa_distance_cnt + to] = distance; | |
369 | } | |
370 | ||
371 | int __node_distance(int from, int to) | |
372 | { | |
ac7136b6 TH |
373 | if (from >= numa_distance_cnt || to >= numa_distance_cnt) |
374 | return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE; | |
375 | return numa_distance[from * numa_distance_cnt + to]; | |
376 | } | |
377 | EXPORT_SYMBOL(__node_distance); | |
378 | ||
f9c60251 TH |
379 | /* |
380 | * Sanity check to catch more bad NUMA configurations (they are amazingly | |
381 | * common). Make sure the nodes cover all memory. | |
382 | */ | |
91556237 | 383 | static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi) |
f9c60251 TH |
384 | { |
385 | unsigned long numaram, e820ram; | |
386 | int i; | |
387 | ||
388 | numaram = 0; | |
91556237 TH |
389 | for (i = 0; i < mi->nr_blks; i++) { |
390 | unsigned long s = mi->blk[i].start >> PAGE_SHIFT; | |
391 | unsigned long e = mi->blk[i].end >> PAGE_SHIFT; | |
f9c60251 | 392 | numaram += e - s; |
91556237 | 393 | numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e); |
f9c60251 TH |
394 | if ((long)numaram < 0) |
395 | numaram = 0; | |
396 | } | |
397 | ||
398 | e820ram = max_pfn - (memblock_x86_hole_size(0, | |
399 | max_pfn << PAGE_SHIFT) >> PAGE_SHIFT); | |
400 | /* We seem to lose 3 pages somewhere. Allow 1M of slack. */ | |
401 | if ((long)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) { | |
402 | printk(KERN_ERR "NUMA: nodes only cover %luMB of your %luMB e820 RAM. Not used.\n", | |
403 | (numaram << PAGE_SHIFT) >> 20, | |
404 | (e820ram << PAGE_SHIFT) >> 20); | |
91556237 | 405 | return false; |
f9c60251 | 406 | } |
91556237 | 407 | return true; |
f9c60251 TH |
408 | } |
409 | ||
410 | static int __init numa_register_memblks(struct numa_meminfo *mi) | |
411 | { | |
69efcc6d | 412 | int i, nid; |
f9c60251 TH |
413 | |
414 | /* Account for nodes with cpus and no memory */ | |
4697bdcc TH |
415 | node_possible_map = numa_nodes_parsed; |
416 | numa_nodemask_from_meminfo(&node_possible_map, mi); | |
f9c60251 TH |
417 | if (WARN_ON(nodes_empty(node_possible_map))) |
418 | return -EINVAL; | |
419 | ||
97e7b78d TH |
420 | for (i = 0; i < mi->nr_blks; i++) |
421 | memblock_x86_register_active_regions(mi->blk[i].nid, | |
422 | mi->blk[i].start >> PAGE_SHIFT, | |
423 | mi->blk[i].end >> PAGE_SHIFT); | |
fd0435d8 TH |
424 | |
425 | /* for out of order entries */ | |
426 | sort_node_map(); | |
91556237 | 427 | if (!numa_meminfo_cover_memory(mi)) |
fd0435d8 TH |
428 | return -EINVAL; |
429 | ||
69efcc6d YL |
430 | /* Finally register nodes. */ |
431 | for_each_node_mask(nid, node_possible_map) { | |
432 | u64 start = (u64)max_pfn << PAGE_SHIFT; | |
433 | u64 end = 0; | |
91556237 | 434 | |
69efcc6d YL |
435 | for (i = 0; i < mi->nr_blks; i++) { |
436 | if (nid != mi->blk[i].nid) | |
91556237 | 437 | continue; |
69efcc6d YL |
438 | start = min(mi->blk[i].start, start); |
439 | end = max(mi->blk[i].end, end); | |
91556237 | 440 | } |
69efcc6d YL |
441 | |
442 | if (start < end) | |
443 | setup_node_bootmem(nid, start, end); | |
91556237 | 444 | } |
fd0435d8 | 445 | |
ef396ec9 TH |
446 | return 0; |
447 | } | |
448 | ||
c09cedf4 DR |
449 | /** |
450 | * dummy_numma_init - Fallback dummy NUMA init | |
451 | * | |
452 | * Used if there's no underlying NUMA architecture, NUMA initialization | |
453 | * fails, or NUMA is disabled on the command line. | |
454 | * | |
455 | * Must online at least one node and add memory blocks that cover all | |
456 | * allowed memory. This function must not fail. | |
457 | */ | |
6d496f9f | 458 | static int __init dummy_numa_init(void) |
ffe77a46 | 459 | { |
1da177e4 LT |
460 | printk(KERN_INFO "%s\n", |
461 | numa_off ? "NUMA turned off" : "No NUMA configuration found"); | |
e3cfe529 | 462 | printk(KERN_INFO "Faking a node at %016lx-%016lx\n", |
86ef4dbf | 463 | 0LU, max_pfn << PAGE_SHIFT); |
ffe77a46 | 464 | |
92d4a437 | 465 | node_set(0, numa_nodes_parsed); |
43a662f0 | 466 | numa_add_memblk(0, 0, (u64)max_pfn << PAGE_SHIFT); |
ec8cf29b TH |
467 | |
468 | return 0; | |
469 | } | |
470 | ||
c09cedf4 | 471 | static int __init numa_init(int (*init_func)(void)) |
ffe77a46 | 472 | { |
c09cedf4 DR |
473 | int i; |
474 | int ret; | |
ffe77a46 | 475 | |
c09cedf4 DR |
476 | for (i = 0; i < MAX_LOCAL_APIC; i++) |
477 | set_apicid_to_node(i, NUMA_NO_NODE); | |
ffe77a46 | 478 | |
c09cedf4 DR |
479 | nodes_clear(numa_nodes_parsed); |
480 | nodes_clear(node_possible_map); | |
481 | nodes_clear(node_online_map); | |
482 | memset(&numa_meminfo, 0, sizeof(numa_meminfo)); | |
483 | remove_all_active_ranges(); | |
484 | numa_reset_distance(); | |
ffe77a46 | 485 | |
c09cedf4 DR |
486 | ret = init_func(); |
487 | if (ret < 0) | |
488 | return ret; | |
489 | ret = numa_cleanup_meminfo(&numa_meminfo); | |
490 | if (ret < 0) | |
491 | return ret; | |
ffe77a46 | 492 | |
c09cedf4 | 493 | numa_emulation(&numa_meminfo, numa_distance_cnt); |
206e4208 | 494 | |
c09cedf4 DR |
495 | ret = numa_register_memblks(&numa_meminfo); |
496 | if (ret < 0) | |
497 | return ret; | |
fbe99959 | 498 | |
c09cedf4 DR |
499 | for (i = 0; i < nr_cpu_ids; i++) { |
500 | int nid = early_cpu_to_node(i); | |
fbe99959 | 501 | |
c09cedf4 | 502 | if (nid == NUMA_NO_NODE) |
43a662f0 | 503 | continue; |
c09cedf4 DR |
504 | if (!node_online(nid)) |
505 | numa_clear_node(i); | |
506 | } | |
507 | numa_init_array(); | |
508 | return 0; | |
509 | } | |
43a662f0 | 510 | |
c09cedf4 DR |
511 | void __init initmem_init(void) |
512 | { | |
c09cedf4 DR |
513 | if (!numa_off) { |
514 | #ifdef CONFIG_ACPI_NUMA | |
711b8c87 | 515 | if (!numa_init(x86_acpi_numa_init)) |
c09cedf4 DR |
516 | return; |
517 | #endif | |
518 | #ifdef CONFIG_AMD_NUMA | |
711b8c87 | 519 | if (!numa_init(amd_numa_init)) |
c09cedf4 DR |
520 | return; |
521 | #endif | |
ffe77a46 | 522 | } |
c09cedf4 DR |
523 | |
524 | numa_init(dummy_numa_init); | |
69d81fcd AK |
525 | } |
526 | ||
e3cfe529 TG |
527 | unsigned long __init numa_free_all_bootmem(void) |
528 | { | |
1da177e4 | 529 | unsigned long pages = 0; |
e3cfe529 TG |
530 | int i; |
531 | ||
532 | for_each_online_node(i) | |
1da177e4 | 533 | pages += free_all_bootmem_node(NODE_DATA(i)); |
e3cfe529 | 534 | |
08677214 | 535 | pages += free_all_memory_core_early(MAX_NUMNODES); |
08677214 | 536 | |
1da177e4 | 537 | return pages; |
e3cfe529 | 538 | } |
1da177e4 | 539 | |
bbc9e2f4 | 540 | int __cpuinit numa_cpu_node(int cpu) |
d9c2d5ac | 541 | { |
bbc9e2f4 | 542 | int apicid = early_per_cpu(x86_cpu_to_apicid, cpu); |
d9c2d5ac | 543 | |
bbc9e2f4 TH |
544 | if (apicid != BAD_APICID) |
545 | return __apicid_to_node[apicid]; | |
546 | return NUMA_NO_NODE; | |
d9c2d5ac | 547 | } |
9688678a TH |
548 | |
549 | #ifdef CONFIG_MEMORY_HOTPLUG | |
550 | int memory_add_physaddr_to_nid(u64 start) | |
551 | { | |
552 | struct numa_meminfo *mi = &numa_meminfo; | |
553 | int nid = mi->blk[0].nid; | |
554 | int i; | |
555 | ||
556 | for (i = 0; i < mi->nr_blks; i++) | |
557 | if (mi->blk[i].start <= start && mi->blk[i].end > start) | |
558 | nid = mi->blk[i].nid; | |
559 | return nid; | |
560 | } | |
561 | EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); | |
562 | #endif |