2 * ACPI 3.0 based NUMA setup
3 * Copyright 2004 Andi Kleen, SuSE Labs.
5 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
7 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
8 * Assumes all memory regions belonging to a single proximity domain
9 * are in one chunk. Holes between them will be included in the node.
12 #include <linux/kernel.h>
13 #include <linux/acpi.h>
14 #include <linux/mmzone.h>
15 #include <linux/bitmap.h>
16 #include <linux/module.h>
17 #include <linux/topology.h>
18 #include <linux/bootmem.h>
20 #include <asm/proto.h>
24 #include <asm/uv/uv.h>
26 int acpi_numa __initdata
;
28 static struct acpi_table_slit
*acpi_slit
;
30 static nodemask_t nodes_parsed __initdata
;
31 static nodemask_t cpu_nodes_parsed __initdata
;
32 static struct bootnode nodes
[MAX_NUMNODES
] __initdata
;
33 static struct bootnode nodes_add
[MAX_NUMNODES
];
35 static int num_node_memblks __initdata
;
36 static struct bootnode node_memblk_range
[NR_NODE_MEMBLKS
] __initdata
;
37 static int memblk_nodeid
[NR_NODE_MEMBLKS
] __initdata
;
39 static __init
int setup_node(int pxm
)
41 return acpi_map_pxm_to_node(pxm
);
44 static __init
int conflicting_memblks(unsigned long start
, unsigned long end
)
47 for (i
= 0; i
< num_node_memblks
; i
++) {
48 struct bootnode
*nd
= &node_memblk_range
[i
];
49 if (nd
->start
== nd
->end
)
51 if (nd
->end
> start
&& nd
->start
< end
)
52 return memblk_nodeid
[i
];
53 if (nd
->end
== end
&& nd
->start
== start
)
54 return memblk_nodeid
[i
];
59 static __init
void cutoff_node(int i
, unsigned long start
, unsigned long end
)
61 struct bootnode
*nd
= &nodes
[i
];
63 if (nd
->start
< start
) {
65 if (nd
->end
< nd
->start
)
70 if (nd
->start
> nd
->end
)
75 static __init
void bad_srat(void)
78 printk(KERN_ERR
"SRAT: SRAT not used.\n");
80 for (i
= 0; i
< MAX_LOCAL_APIC
; i
++)
81 apicid_to_node
[i
] = NUMA_NO_NODE
;
82 for (i
= 0; i
< MAX_NUMNODES
; i
++) {
83 nodes
[i
].start
= nodes
[i
].end
= 0;
84 nodes_add
[i
].start
= nodes_add
[i
].end
= 0;
86 remove_all_active_ranges();
89 static __init
inline int srat_disabled(void)
91 return numa_off
|| acpi_numa
< 0;
94 /* Callback for SLIT parsing */
95 void __init
acpi_numa_slit_init(struct acpi_table_slit
*slit
)
100 length
= slit
->header
.length
;
101 phys
= find_e820_area(0, max_pfn_mapped
<<PAGE_SHIFT
, length
,
105 panic(" Can not save slit!\n");
107 acpi_slit
= __va(phys
);
108 memcpy(acpi_slit
, slit
, length
);
109 reserve_early(phys
, phys
+ length
, "ACPI SLIT");
112 /* Callback for Proximity Domain -> x2APIC mapping */
114 acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity
*pa
)
121 if (pa
->header
.length
< sizeof(struct acpi_srat_x2apic_cpu_affinity
)) {
125 if ((pa
->flags
& ACPI_SRAT_CPU_ENABLED
) == 0)
127 pxm
= pa
->proximity_domain
;
128 node
= setup_node(pxm
);
130 printk(KERN_ERR
"SRAT: Too many proximity domains %x\n", pxm
);
135 apic_id
= pa
->apic_id
;
136 apicid_to_node
[apic_id
] = node
;
137 node_set(node
, cpu_nodes_parsed
);
139 printk(KERN_INFO
"SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
143 /* Callback for Proximity Domain -> LAPIC mapping */
145 acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity
*pa
)
152 if (pa
->header
.length
!= sizeof(struct acpi_srat_cpu_affinity
)) {
156 if ((pa
->flags
& ACPI_SRAT_CPU_ENABLED
) == 0)
158 pxm
= pa
->proximity_domain_lo
;
159 node
= setup_node(pxm
);
161 printk(KERN_ERR
"SRAT: Too many proximity domains %x\n", pxm
);
166 if (get_uv_system_type() >= UV_X2APIC
)
167 apic_id
= (pa
->apic_id
<< 8) | pa
->local_sapic_eid
;
169 apic_id
= pa
->apic_id
;
170 apicid_to_node
[apic_id
] = node
;
171 node_set(node
, cpu_nodes_parsed
);
173 printk(KERN_INFO
"SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
177 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
178 static inline int save_add_info(void) {return 1;}
180 static inline int save_add_info(void) {return 0;}
184 * This code supports one contiguous hot add area per node
187 update_nodes_add(int node
, unsigned long start
, unsigned long end
)
189 unsigned long s_pfn
= start
>> PAGE_SHIFT
;
190 unsigned long e_pfn
= end
>> PAGE_SHIFT
;
192 struct bootnode
*nd
= &nodes_add
[node
];
194 /* I had some trouble with strange memory hotadd regions breaking
195 the boot. Be very strict here and reject anything unexpected.
196 If you want working memory hotadd write correct SRATs.
198 The node size check is a basic sanity check to guard against
200 if ((signed long)(end
- start
) < NODE_MIN_SIZE
) {
201 printk(KERN_ERR
"SRAT: Hotplug area too small\n");
205 /* This check might be a bit too strict, but I'm keeping it for now. */
206 if (absent_pages_in_range(s_pfn
, e_pfn
) != e_pfn
- s_pfn
) {
208 "SRAT: Hotplug area %lu -> %lu has existing memory\n",
215 if (nd
->start
== nd
->end
) {
220 if (nd
->start
== end
) {
224 if (nd
->end
== start
) {
229 printk(KERN_ERR
"SRAT: Hotplug zone not continuous. Partly ignored\n");
233 node_set(node
, cpu_nodes_parsed
);
234 printk(KERN_INFO
"SRAT: hot plug zone found %Lx - %Lx\n",
239 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
241 acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity
*ma
)
243 struct bootnode
*nd
, oldnode
;
244 unsigned long start
, end
;
250 if (ma
->header
.length
!= sizeof(struct acpi_srat_mem_affinity
)) {
254 if ((ma
->flags
& ACPI_SRAT_MEM_ENABLED
) == 0)
257 if ((ma
->flags
& ACPI_SRAT_MEM_HOT_PLUGGABLE
) && !save_add_info())
259 start
= ma
->base_address
;
260 end
= start
+ ma
->length
;
261 pxm
= ma
->proximity_domain
;
262 node
= setup_node(pxm
);
264 printk(KERN_ERR
"SRAT: Too many proximity domains.\n");
268 i
= conflicting_memblks(start
, end
);
271 "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
272 pxm
, start
, end
, nodes
[i
].start
, nodes
[i
].end
);
275 "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
276 pxm
, start
, end
, node_to_pxm(i
),
277 nodes
[i
].start
, nodes
[i
].end
);
283 if (!node_test_and_set(node
, nodes_parsed
)) {
287 if (start
< nd
->start
)
293 printk(KERN_INFO
"SRAT: Node %u PXM %u %lx-%lx\n", node
, pxm
,
296 if (ma
->flags
& ACPI_SRAT_MEM_HOT_PLUGGABLE
) {
297 update_nodes_add(node
, start
, end
);
298 /* restore nodes[node] */
300 if ((nd
->start
| nd
->end
) == 0)
301 node_clear(node
, nodes_parsed
);
304 node_memblk_range
[num_node_memblks
].start
= start
;
305 node_memblk_range
[num_node_memblks
].end
= end
;
306 memblk_nodeid
[num_node_memblks
] = node
;
310 /* Sanity check to catch more bad SRATs (they are amazingly common).
311 Make sure the PXMs cover all memory. */
312 static int __init
nodes_cover_memory(const struct bootnode
*nodes
)
315 unsigned long pxmram
, e820ram
;
318 for_each_node_mask(i
, nodes_parsed
) {
319 unsigned long s
= nodes
[i
].start
>> PAGE_SHIFT
;
320 unsigned long e
= nodes
[i
].end
>> PAGE_SHIFT
;
322 pxmram
-= __absent_pages_in_range(i
, s
, e
);
323 if ((long)pxmram
< 0)
327 e820ram
= max_pfn
- (e820_hole_size(0, max_pfn
<<PAGE_SHIFT
)>>PAGE_SHIFT
);
328 /* We seem to lose 3 pages somewhere. Allow 1M of slack. */
329 if ((long)(e820ram
- pxmram
) >= (1<<(20 - PAGE_SHIFT
))) {
331 "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
332 (pxmram
<< PAGE_SHIFT
) >> 20,
333 (e820ram
<< PAGE_SHIFT
) >> 20);
339 void __init
acpi_numa_arch_fixup(void) {}
341 int __init
acpi_get_nodes(struct bootnode
*physnodes
)
346 for_each_node_mask(i
, nodes_parsed
) {
347 physnodes
[ret
].start
= nodes
[i
].start
;
348 physnodes
[ret
].end
= nodes
[i
].end
;
354 /* Use the information discovered above to actually set up the nodes. */
355 int __init
acpi_scan_nodes(unsigned long start
, unsigned long end
)
362 /* First clean up the node list */
363 for (i
= 0; i
< MAX_NUMNODES
; i
++)
364 cutoff_node(i
, start
, end
);
367 * Join together blocks on the same node, holes between
368 * which don't overlap with memory on other nodes.
370 for (i
= 0; i
< num_node_memblks
; ++i
) {
373 for (j
= i
+ 1; j
< num_node_memblks
; ++j
) {
374 unsigned long start
, end
;
376 if (memblk_nodeid
[i
] != memblk_nodeid
[j
])
378 start
= min(node_memblk_range
[i
].end
,
379 node_memblk_range
[j
].end
);
380 end
= max(node_memblk_range
[i
].start
,
381 node_memblk_range
[j
].start
);
382 for (k
= 0; k
< num_node_memblks
; ++k
) {
383 if (memblk_nodeid
[i
] == memblk_nodeid
[k
])
385 if (start
< node_memblk_range
[k
].end
&&
386 end
> node_memblk_range
[k
].start
)
389 if (k
< num_node_memblks
)
391 start
= min(node_memblk_range
[i
].start
,
392 node_memblk_range
[j
].start
);
393 end
= max(node_memblk_range
[i
].end
,
394 node_memblk_range
[j
].end
);
395 printk(KERN_INFO
"SRAT: Node %d "
396 "[%Lx,%Lx) + [%Lx,%Lx) -> [%lx,%lx)\n",
398 node_memblk_range
[i
].start
,
399 node_memblk_range
[i
].end
,
400 node_memblk_range
[j
].start
,
401 node_memblk_range
[j
].end
,
403 node_memblk_range
[i
].start
= start
;
404 node_memblk_range
[i
].end
= end
;
405 k
= --num_node_memblks
- j
;
406 memmove(memblk_nodeid
+ j
, memblk_nodeid
+ j
+1,
407 k
* sizeof(*memblk_nodeid
));
408 memmove(node_memblk_range
+ j
, node_memblk_range
+ j
+1,
409 k
* sizeof(*node_memblk_range
));
414 memnode_shift
= compute_hash_shift(node_memblk_range
, num_node_memblks
,
416 if (memnode_shift
< 0) {
418 "SRAT: No NUMA node hash function found. Contact maintainer\n");
423 for_each_node_mask(i
, nodes_parsed
)
424 e820_register_active_regions(i
, nodes
[i
].start
>> PAGE_SHIFT
,
425 nodes
[i
].end
>> PAGE_SHIFT
);
426 /* for out of order entries in SRAT */
428 if (!nodes_cover_memory(nodes
)) {
433 /* Account for nodes with cpus and no memory */
434 nodes_or(node_possible_map
, nodes_parsed
, cpu_nodes_parsed
);
436 /* Finally register nodes */
437 for_each_node_mask(i
, node_possible_map
)
438 setup_node_bootmem(i
, nodes
[i
].start
, nodes
[i
].end
);
439 /* Try again in case setup_node_bootmem missed one due
440 to missing bootmem */
441 for_each_node_mask(i
, node_possible_map
)
443 setup_node_bootmem(i
, nodes
[i
].start
, nodes
[i
].end
);
445 for (i
= 0; i
< nr_cpu_ids
; i
++) {
446 int node
= early_cpu_to_node(i
);
448 if (node
== NUMA_NO_NODE
)
450 if (!node_online(node
))
457 #ifdef CONFIG_NUMA_EMU
458 static int fake_node_to_pxm_map
[MAX_NUMNODES
] __initdata
= {
459 [0 ... MAX_NUMNODES
-1] = PXM_INVAL
461 static s16 fake_apicid_to_node
[MAX_LOCAL_APIC
] __initdata
= {
462 [0 ... MAX_LOCAL_APIC
-1] = NUMA_NO_NODE
464 static int __init
find_node_by_addr(unsigned long addr
)
466 int ret
= NUMA_NO_NODE
;
469 for_each_node_mask(i
, nodes_parsed
) {
471 * Find the real node that this emulated node appears on. For
472 * the sake of simplicity, we only use a real node's starting
473 * address to determine which emulated node it appears on.
475 if (addr
>= nodes
[i
].start
&& addr
< nodes
[i
].end
) {
484 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
485 * mappings that respect the real ACPI topology but reflect our emulated
486 * environment. For each emulated node, we find which real node it appears on
487 * and create PXM to NID mappings for those fake nodes which mirror that
488 * locality. SLIT will now represent the correct distances between emulated
489 * nodes as a result of the real topology.
491 void __init
acpi_fake_nodes(const struct bootnode
*fake_nodes
, int num_nodes
)
495 printk(KERN_INFO
"Faking PXM affinity for fake nodes on real "
497 for (i
= 0; i
< num_nodes
; i
++) {
500 nid
= find_node_by_addr(fake_nodes
[i
].start
);
501 if (nid
== NUMA_NO_NODE
)
503 pxm
= node_to_pxm(nid
);
504 if (pxm
== PXM_INVAL
)
506 fake_node_to_pxm_map
[i
] = pxm
;
508 * For each apicid_to_node mapping that exists for this real
509 * node, it must now point to the fake node ID.
511 for (j
= 0; j
< MAX_LOCAL_APIC
; j
++)
512 if (apicid_to_node
[j
] == nid
&&
513 fake_apicid_to_node
[j
] == NUMA_NO_NODE
)
514 fake_apicid_to_node
[j
] = i
;
516 for (i
= 0; i
< num_nodes
; i
++)
517 __acpi_map_pxm_to_node(fake_node_to_pxm_map
[i
], i
);
518 memcpy(apicid_to_node
, fake_apicid_to_node
, sizeof(apicid_to_node
));
520 nodes_clear(nodes_parsed
);
521 for (i
= 0; i
< num_nodes
; i
++)
522 if (fake_nodes
[i
].start
!= fake_nodes
[i
].end
)
523 node_set(i
, nodes_parsed
);
526 static int null_slit_node_compare(int a
, int b
)
528 return node_to_pxm(a
) == node_to_pxm(b
);
531 static int null_slit_node_compare(int a
, int b
)
535 #endif /* CONFIG_NUMA_EMU */
537 int __node_distance(int a
, int b
)
542 return null_slit_node_compare(a
, b
) ? LOCAL_DISTANCE
:
544 index
= acpi_slit
->locality_count
* node_to_pxm(a
);
545 return acpi_slit
->entry
[index
+ node_to_pxm(b
)];
548 EXPORT_SYMBOL(__node_distance
);
550 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
551 int memory_add_physaddr_to_nid(u64 start
)
556 if (nodes_add
[i
].start
<= start
&& nodes_add
[i
].end
> start
)
561 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid
);