2 * Memory subsystem support
4 * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
5 * Dave Hansen <haveblue@us.ibm.com>
7 * This file provides the necessary infrastructure to represent
8 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
9 * All arch-independent code that assumes MEMORY_HOTPLUG requires
10 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/topology.h>
16 #include <linux/capability.h>
17 #include <linux/device.h>
18 #include <linux/memory.h>
19 #include <linux/memory_hotplug.h>
21 #include <linux/mutex.h>
22 #include <linux/stat.h>
23 #include <linux/slab.h>
25 #include <linux/atomic.h>
26 #include <asm/uaccess.h>
28 static DEFINE_MUTEX(mem_sysfs_mutex
);
30 #define MEMORY_CLASS_NAME "memory"
32 #define to_memory_block(dev) container_of(dev, struct memory_block, dev)
34 static int sections_per_block
;
36 static inline int base_memory_block_id(int section_nr
)
38 return section_nr
/ sections_per_block
;
41 static int memory_subsys_online(struct device
*dev
);
42 static int memory_subsys_offline(struct device
*dev
);
44 static struct bus_type memory_subsys
= {
45 .name
= MEMORY_CLASS_NAME
,
46 .dev_name
= MEMORY_CLASS_NAME
,
47 .online
= memory_subsys_online
,
48 .offline
= memory_subsys_offline
,
51 static BLOCKING_NOTIFIER_HEAD(memory_chain
);
53 int register_memory_notifier(struct notifier_block
*nb
)
55 return blocking_notifier_chain_register(&memory_chain
, nb
);
57 EXPORT_SYMBOL(register_memory_notifier
);
59 void unregister_memory_notifier(struct notifier_block
*nb
)
61 blocking_notifier_chain_unregister(&memory_chain
, nb
);
63 EXPORT_SYMBOL(unregister_memory_notifier
);
65 static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain
);
67 int register_memory_isolate_notifier(struct notifier_block
*nb
)
69 return atomic_notifier_chain_register(&memory_isolate_chain
, nb
);
71 EXPORT_SYMBOL(register_memory_isolate_notifier
);
73 void unregister_memory_isolate_notifier(struct notifier_block
*nb
)
75 atomic_notifier_chain_unregister(&memory_isolate_chain
, nb
);
77 EXPORT_SYMBOL(unregister_memory_isolate_notifier
);
79 static void memory_block_release(struct device
*dev
)
81 struct memory_block
*mem
= to_memory_block(dev
);
86 unsigned long __weak
memory_block_size_bytes(void)
88 return MIN_MEMORY_BLOCK_SIZE
;
91 static unsigned long get_memory_block_size(void)
93 unsigned long block_sz
;
95 block_sz
= memory_block_size_bytes();
97 /* Validate blk_sz is a power of 2 and not less than section size */
98 if ((block_sz
& (block_sz
- 1)) || (block_sz
< MIN_MEMORY_BLOCK_SIZE
)) {
100 block_sz
= MIN_MEMORY_BLOCK_SIZE
;
107 * use this as the physical section index that this memsection
111 static ssize_t
show_mem_start_phys_index(struct device
*dev
,
112 struct device_attribute
*attr
, char *buf
)
114 struct memory_block
*mem
= to_memory_block(dev
);
115 unsigned long phys_index
;
117 phys_index
= mem
->start_section_nr
/ sections_per_block
;
118 return sprintf(buf
, "%08lx\n", phys_index
);
122 * Show whether the section of memory is likely to be hot-removable
124 static ssize_t
show_mem_removable(struct device
*dev
,
125 struct device_attribute
*attr
, char *buf
)
127 unsigned long i
, pfn
;
129 struct memory_block
*mem
= to_memory_block(dev
);
131 for (i
= 0; i
< sections_per_block
; i
++) {
132 if (!present_section_nr(mem
->start_section_nr
+ i
))
134 pfn
= section_nr_to_pfn(mem
->start_section_nr
+ i
);
135 ret
&= is_mem_section_removable(pfn
, PAGES_PER_SECTION
);
138 return sprintf(buf
, "%d\n", ret
);
142 * online, offline, going offline, etc.
144 static ssize_t
show_mem_state(struct device
*dev
,
145 struct device_attribute
*attr
, char *buf
)
147 struct memory_block
*mem
= to_memory_block(dev
);
151 * We can probably put these states in a nice little array
152 * so that they're not open-coded
154 switch (mem
->state
) {
156 len
= sprintf(buf
, "online\n");
159 len
= sprintf(buf
, "offline\n");
161 case MEM_GOING_OFFLINE
:
162 len
= sprintf(buf
, "going-offline\n");
165 len
= sprintf(buf
, "ERROR-UNKNOWN-%ld\n",
174 int memory_notify(unsigned long val
, void *v
)
176 return blocking_notifier_call_chain(&memory_chain
, val
, v
);
179 int memory_isolate_notify(unsigned long val
, void *v
)
181 return atomic_notifier_call_chain(&memory_isolate_chain
, val
, v
);
185 * The probe routines leave the pages reserved, just as the bootmem code does.
186 * Make sure they're still that way.
188 static bool pages_correctly_reserved(unsigned long start_pfn
)
192 unsigned long pfn
= start_pfn
;
195 * memmap between sections is not contiguous except with
196 * SPARSEMEM_VMEMMAP. We lookup the page once per section
197 * and assume memmap is contiguous within each section
199 for (i
= 0; i
< sections_per_block
; i
++, pfn
+= PAGES_PER_SECTION
) {
200 if (WARN_ON_ONCE(!pfn_valid(pfn
)))
202 page
= pfn_to_page(pfn
);
204 for (j
= 0; j
< PAGES_PER_SECTION
; j
++) {
205 if (PageReserved(page
+ j
))
208 printk(KERN_WARNING
"section number %ld page number %d "
209 "not reserved, was it already online?\n",
210 pfn_to_section_nr(pfn
), j
);
220 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
221 * OK to have direct references to sparsemem variables in here.
224 memory_block_action(unsigned long phys_index
, unsigned long action
, int online_type
)
226 unsigned long start_pfn
;
227 unsigned long nr_pages
= PAGES_PER_SECTION
* sections_per_block
;
228 struct page
*first_page
;
231 first_page
= pfn_to_page(phys_index
<< PFN_SECTION_SHIFT
);
232 start_pfn
= page_to_pfn(first_page
);
236 if (!pages_correctly_reserved(start_pfn
))
239 ret
= online_pages(start_pfn
, nr_pages
, online_type
);
242 ret
= offline_pages(start_pfn
, nr_pages
);
245 WARN(1, KERN_WARNING
"%s(%ld, %ld) unknown action: "
246 "%ld\n", __func__
, phys_index
, action
, action
);
253 static int memory_block_change_state(struct memory_block
*mem
,
254 unsigned long to_state
, unsigned long from_state_req
)
258 if (mem
->state
!= from_state_req
)
261 if (to_state
== MEM_OFFLINE
)
262 mem
->state
= MEM_GOING_OFFLINE
;
264 ret
= memory_block_action(mem
->start_section_nr
, to_state
,
267 mem
->state
= ret
? from_state_req
: to_state
;
272 /* The device lock serializes operations on memory_subsys_[online|offline] */
273 static int memory_subsys_online(struct device
*dev
)
275 struct memory_block
*mem
= to_memory_block(dev
);
278 if (mem
->state
== MEM_ONLINE
)
282 * If we are called from store_mem_state(), online_type will be
283 * set >= 0 Otherwise we were called from the device online
284 * attribute and need to set the online_type.
286 if (mem
->online_type
< 0)
287 mem
->online_type
= ONLINE_KEEP
;
289 ret
= memory_block_change_state(mem
, MEM_ONLINE
, MEM_OFFLINE
);
291 /* clear online_type */
292 mem
->online_type
= -1;
297 static int memory_subsys_offline(struct device
*dev
)
299 struct memory_block
*mem
= to_memory_block(dev
);
301 if (mem
->state
== MEM_OFFLINE
)
304 return memory_block_change_state(mem
, MEM_OFFLINE
, MEM_ONLINE
);
308 store_mem_state(struct device
*dev
,
309 struct device_attribute
*attr
, const char *buf
, size_t count
)
311 struct memory_block
*mem
= to_memory_block(dev
);
312 int ret
, online_type
;
314 ret
= lock_device_hotplug_sysfs();
318 if (!strncmp(buf
, "online_kernel", min_t(int, count
, 13)))
319 online_type
= ONLINE_KERNEL
;
320 else if (!strncmp(buf
, "online_movable", min_t(int, count
, 14)))
321 online_type
= ONLINE_MOVABLE
;
322 else if (!strncmp(buf
, "online", min_t(int, count
, 6)))
323 online_type
= ONLINE_KEEP
;
324 else if (!strncmp(buf
, "offline", min_t(int, count
, 7)))
331 switch (online_type
) {
336 * mem->online_type is not protected so there can be a
337 * race here. However, when racing online, the first
338 * will succeed and the second will just return as the
339 * block will already be online. The online type
340 * could be either one, but that is expected.
342 mem
->online_type
= online_type
;
343 ret
= device_online(&mem
->dev
);
346 ret
= device_offline(&mem
->dev
);
349 ret
= -EINVAL
; /* should never happen */
353 unlock_device_hotplug();
361 * phys_device is a bad name for this. What I really want
362 * is a way to differentiate between memory ranges that
363 * are part of physical devices that constitute
364 * a complete removable unit or fru.
365 * i.e. do these ranges belong to the same physical device,
366 * s.t. if I offline all of these sections I can then
367 * remove the physical device?
369 static ssize_t
show_phys_device(struct device
*dev
,
370 struct device_attribute
*attr
, char *buf
)
372 struct memory_block
*mem
= to_memory_block(dev
);
373 return sprintf(buf
, "%d\n", mem
->phys_device
);
376 static DEVICE_ATTR(phys_index
, 0444, show_mem_start_phys_index
, NULL
);
377 static DEVICE_ATTR(state
, 0644, show_mem_state
, store_mem_state
);
378 static DEVICE_ATTR(phys_device
, 0444, show_phys_device
, NULL
);
379 static DEVICE_ATTR(removable
, 0444, show_mem_removable
, NULL
);
382 * Block size attribute stuff
385 print_block_size(struct device
*dev
, struct device_attribute
*attr
,
388 return sprintf(buf
, "%lx\n", get_memory_block_size());
391 static DEVICE_ATTR(block_size_bytes
, 0444, print_block_size
, NULL
);
394 * Some architectures will have custom drivers to do this, and
395 * will not need to do it from userspace. The fake hot-add code
396 * as well as ppc64 will do all of their discovery in userspace
397 * and will require this interface.
399 #ifdef CONFIG_ARCH_MEMORY_PROBE
401 memory_probe_store(struct device
*dev
, struct device_attribute
*attr
,
402 const char *buf
, size_t count
)
407 unsigned long pages_per_block
= PAGES_PER_SECTION
* sections_per_block
;
409 phys_addr
= simple_strtoull(buf
, NULL
, 0);
411 if (phys_addr
& ((pages_per_block
<< PAGE_SHIFT
) - 1))
414 for (i
= 0; i
< sections_per_block
; i
++) {
415 nid
= memory_add_physaddr_to_nid(phys_addr
);
416 ret
= add_memory(nid
, phys_addr
,
417 PAGES_PER_SECTION
<< PAGE_SHIFT
);
421 phys_addr
+= MIN_MEMORY_BLOCK_SIZE
;
429 static DEVICE_ATTR(probe
, S_IWUSR
, NULL
, memory_probe_store
);
432 #ifdef CONFIG_MEMORY_FAILURE
434 * Support for offlining pages of memory
437 /* Soft offline a page */
439 store_soft_offline_page(struct device
*dev
,
440 struct device_attribute
*attr
,
441 const char *buf
, size_t count
)
445 if (!capable(CAP_SYS_ADMIN
))
447 if (kstrtoull(buf
, 0, &pfn
) < 0)
452 ret
= soft_offline_page(pfn_to_page(pfn
), 0);
453 return ret
== 0 ? count
: ret
;
456 /* Forcibly offline a page, including killing processes. */
458 store_hard_offline_page(struct device
*dev
,
459 struct device_attribute
*attr
,
460 const char *buf
, size_t count
)
464 if (!capable(CAP_SYS_ADMIN
))
466 if (kstrtoull(buf
, 0, &pfn
) < 0)
469 ret
= memory_failure(pfn
, 0, 0);
470 return ret
? ret
: count
;
473 static DEVICE_ATTR(soft_offline_page
, S_IWUSR
, NULL
, store_soft_offline_page
);
474 static DEVICE_ATTR(hard_offline_page
, S_IWUSR
, NULL
, store_hard_offline_page
);
478 * Note that phys_device is optional. It is here to allow for
479 * differentiation between which *physical* devices each
480 * section belongs to...
482 int __weak
arch_get_memory_phys_device(unsigned long start_pfn
)
488 * A reference for the returned object is held and the reference for the
489 * hinted object is released.
491 struct memory_block
*find_memory_block_hinted(struct mem_section
*section
,
492 struct memory_block
*hint
)
494 int block_id
= base_memory_block_id(__section_nr(section
));
495 struct device
*hintdev
= hint
? &hint
->dev
: NULL
;
498 dev
= subsys_find_device_by_id(&memory_subsys
, block_id
, hintdev
);
500 put_device(&hint
->dev
);
503 return to_memory_block(dev
);
507 * For now, we have a linear search to go find the appropriate
508 * memory_block corresponding to a particular phys_index. If
509 * this gets to be a real problem, we can always use a radix
510 * tree or something here.
512 * This could be made generic for all device subsystems.
514 struct memory_block
*find_memory_block(struct mem_section
*section
)
516 return find_memory_block_hinted(section
, NULL
);
519 static struct attribute
*memory_memblk_attrs
[] = {
520 &dev_attr_phys_index
.attr
,
521 &dev_attr_state
.attr
,
522 &dev_attr_phys_device
.attr
,
523 &dev_attr_removable
.attr
,
527 static struct attribute_group memory_memblk_attr_group
= {
528 .attrs
= memory_memblk_attrs
,
531 static const struct attribute_group
*memory_memblk_attr_groups
[] = {
532 &memory_memblk_attr_group
,
537 * register_memory - Setup a sysfs device for a memory block
540 int register_memory(struct memory_block
*memory
)
542 memory
->dev
.bus
= &memory_subsys
;
543 memory
->dev
.id
= memory
->start_section_nr
/ sections_per_block
;
544 memory
->dev
.release
= memory_block_release
;
545 memory
->dev
.groups
= memory_memblk_attr_groups
;
546 memory
->dev
.offline
= memory
->state
== MEM_OFFLINE
;
548 return device_register(&memory
->dev
);
551 static int init_memory_block(struct memory_block
**memory
,
552 struct mem_section
*section
, unsigned long state
)
554 struct memory_block
*mem
;
555 unsigned long start_pfn
;
559 mem
= kzalloc(sizeof(*mem
), GFP_KERNEL
);
563 scn_nr
= __section_nr(section
);
564 mem
->start_section_nr
=
565 base_memory_block_id(scn_nr
) * sections_per_block
;
566 mem
->end_section_nr
= mem
->start_section_nr
+ sections_per_block
- 1;
568 mem
->section_count
++;
569 start_pfn
= section_nr_to_pfn(mem
->start_section_nr
);
570 mem
->phys_device
= arch_get_memory_phys_device(start_pfn
);
572 ret
= register_memory(mem
);
578 static int add_memory_block(int base_section_nr
)
580 struct memory_block
*mem
;
581 int i
, ret
, section_count
= 0, section_nr
;
583 for (i
= base_section_nr
;
584 (i
< base_section_nr
+ sections_per_block
) && i
< NR_MEM_SECTIONS
;
586 if (!present_section_nr(i
))
588 if (section_count
== 0)
593 if (section_count
== 0)
595 ret
= init_memory_block(&mem
, __nr_to_section(section_nr
), MEM_ONLINE
);
598 mem
->section_count
= section_count
;
604 * need an interface for the VM to add new memory regions,
605 * but without onlining it.
607 int register_new_memory(int nid
, struct mem_section
*section
)
610 struct memory_block
*mem
;
612 mutex_lock(&mem_sysfs_mutex
);
614 mem
= find_memory_block(section
);
616 mem
->section_count
++;
617 put_device(&mem
->dev
);
619 ret
= init_memory_block(&mem
, section
, MEM_OFFLINE
);
624 if (mem
->section_count
== sections_per_block
)
625 ret
= register_mem_sect_under_node(mem
, nid
);
627 mutex_unlock(&mem_sysfs_mutex
);
631 #ifdef CONFIG_MEMORY_HOTREMOVE
633 unregister_memory(struct memory_block
*memory
)
635 BUG_ON(memory
->dev
.bus
!= &memory_subsys
);
637 /* drop the ref. we got in remove_memory_block() */
638 put_device(&memory
->dev
);
639 device_unregister(&memory
->dev
);
642 static int remove_memory_block(unsigned long node_id
,
643 struct mem_section
*section
, int phys_device
)
645 struct memory_block
*mem
;
647 mutex_lock(&mem_sysfs_mutex
);
648 mem
= find_memory_block(section
);
649 unregister_mem_sect_under_nodes(mem
, __section_nr(section
));
651 mem
->section_count
--;
652 if (mem
->section_count
== 0)
653 unregister_memory(mem
);
655 put_device(&mem
->dev
);
657 mutex_unlock(&mem_sysfs_mutex
);
661 int unregister_memory_section(struct mem_section
*section
)
663 if (!present_section(section
))
666 return remove_memory_block(0, section
, 0);
668 #endif /* CONFIG_MEMORY_HOTREMOVE */
670 /* return true if the memory block is offlined, otherwise, return false */
671 bool is_memblock_offlined(struct memory_block
*mem
)
673 return mem
->state
== MEM_OFFLINE
;
676 static struct attribute
*memory_root_attrs
[] = {
677 #ifdef CONFIG_ARCH_MEMORY_PROBE
678 &dev_attr_probe
.attr
,
681 #ifdef CONFIG_MEMORY_FAILURE
682 &dev_attr_soft_offline_page
.attr
,
683 &dev_attr_hard_offline_page
.attr
,
686 &dev_attr_block_size_bytes
.attr
,
690 static struct attribute_group memory_root_attr_group
= {
691 .attrs
= memory_root_attrs
,
694 static const struct attribute_group
*memory_root_attr_groups
[] = {
695 &memory_root_attr_group
,
700 * Initialize the sysfs support for memory devices...
702 int __init
memory_dev_init(void)
707 unsigned long block_sz
;
709 ret
= subsys_system_register(&memory_subsys
, memory_root_attr_groups
);
713 block_sz
= get_memory_block_size();
714 sections_per_block
= block_sz
/ MIN_MEMORY_BLOCK_SIZE
;
717 * Create entries for memory sections that were found
718 * during boot and have been initialized
720 mutex_lock(&mem_sysfs_mutex
);
721 for (i
= 0; i
< NR_MEM_SECTIONS
; i
+= sections_per_block
) {
722 err
= add_memory_block(i
);
726 mutex_unlock(&mem_sysfs_mutex
);
730 printk(KERN_ERR
"%s() failed: %d\n", __func__
, ret
);