#define STABLE_FLAG 0x200 /* is listed from the stable tree */
/* The stable and unstable tree heads */
-static struct rb_root root_unstable_tree[MAX_NUMNODES];
-static struct rb_root root_stable_tree[MAX_NUMNODES];
+static struct rb_root one_stable_tree[1] = { RB_ROOT };
+static struct rb_root one_unstable_tree[1] = { RB_ROOT };
+static struct rb_root *root_stable_tree = one_stable_tree;
+static struct rb_root *root_unstable_tree = one_unstable_tree;
/* Recently migrated nodes of stable tree, pending proper placement */
static LIST_HEAD(migrate_nodes);
#ifdef CONFIG_NUMA
/* Zeroed when merging across nodes is not allowed */
static unsigned int ksm_merge_across_nodes = 1;
+static int ksm_nr_node_ids = 1;
#else
#define ksm_merge_across_nodes 1U
+#define ksm_nr_node_ids 1
#endif
#define KSM_RUN_STOP 0
static struct mm_slot *get_mm_slot(struct mm_struct *mm)
{
- struct hlist_node *node;
struct mm_slot *slot;
- hash_for_each_possible(mm_slots_hash, slot, node, link, (unsigned long)mm)
+ hash_for_each_possible(mm_slots_hash, slot, link, (unsigned long)mm)
if (slot->mm == mm)
return slot;
do {
cond_resched();
- page = follow_page(vma, addr, FOLL_GET);
+ page = follow_page(vma, addr, FOLL_GET | FOLL_MIGRATION);
if (IS_ERR_OR_NULL(page))
break;
if (PageKsm(page))
static void remove_node_from_stable_tree(struct stable_node *stable_node)
{
struct rmap_item *rmap_item;
- struct hlist_node *hlist;
- hlist_for_each_entry(rmap_item, hlist, &stable_node->hlist, hlist) {
+ hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
if (rmap_item->hlist.next)
ksm_pages_sharing--;
else
list_del(&stable_node->list);
else
rb_erase(&stable_node->node,
- &root_stable_tree[NUMA(stable_node->nid)]);
+ root_stable_tree + NUMA(stable_node->nid));
free_stable_node(stable_node);
}
BUG_ON(age > 1);
if (!age)
rb_erase(&rmap_item->node,
- &root_unstable_tree[NUMA(rmap_item->nid)]);
+ root_unstable_tree + NUMA(rmap_item->nid));
ksm_pages_unshared--;
rmap_item->address &= PAGE_MASK;
}
int nid;
int err = 0;
- for (nid = 0; nid < nr_node_ids; nid++) {
+ for (nid = 0; nid < ksm_nr_node_ids; nid++) {
while (root_stable_tree[nid].rb_node) {
stable_node = rb_entry(root_stable_tree[nid].rb_node,
struct stable_node, node);
static struct page *stable_tree_search(struct page *page)
{
int nid;
+ struct rb_root *root;
struct rb_node **new;
struct rb_node *parent;
struct stable_node *stable_node;
}
nid = get_kpfn_nid(page_to_pfn(page));
+ root = root_stable_tree + nid;
again:
- new = &root_stable_tree[nid].rb_node;
+ new = &root->rb_node;
parent = NULL;
while (*new) {
list_del(&page_node->list);
DO_NUMA(page_node->nid = nid);
rb_link_node(&page_node->node, parent, new);
- rb_insert_color(&page_node->node, &root_stable_tree[nid]);
+ rb_insert_color(&page_node->node, root);
get_page(page);
return page;
if (page_node) {
list_del(&page_node->list);
DO_NUMA(page_node->nid = nid);
- rb_replace_node(&stable_node->node,
- &page_node->node, &root_stable_tree[nid]);
+ rb_replace_node(&stable_node->node, &page_node->node, root);
get_page(page);
} else {
- rb_erase(&stable_node->node, &root_stable_tree[nid]);
+ rb_erase(&stable_node->node, root);
page = NULL;
}
stable_node->head = &migrate_nodes;
{
int nid;
unsigned long kpfn;
+ struct rb_root *root;
struct rb_node **new;
struct rb_node *parent = NULL;
struct stable_node *stable_node;
kpfn = page_to_pfn(kpage);
nid = get_kpfn_nid(kpfn);
- new = &root_stable_tree[nid].rb_node;
+ root = root_stable_tree + nid;
+ new = &root->rb_node;
while (*new) {
struct page *tree_page;
set_page_stable_node(kpage, stable_node);
DO_NUMA(stable_node->nid = nid);
rb_link_node(&stable_node->node, parent, new);
- rb_insert_color(&stable_node->node, &root_stable_tree[nid]);
+ rb_insert_color(&stable_node->node, root);
return stable_node;
}
int nid;
nid = get_kpfn_nid(page_to_pfn(page));
- root = &root_unstable_tree[nid];
+ root = root_unstable_tree + nid;
new = &root->rb_node;
while (*new) {
if (stable_node->head != &migrate_nodes &&
get_kpfn_nid(stable_node->kpfn) != NUMA(stable_node->nid)) {
rb_erase(&stable_node->node,
- &root_stable_tree[NUMA(stable_node->nid)]);
+ root_stable_tree + NUMA(stable_node->nid));
stable_node->head = &migrate_nodes;
list_add(&stable_node->list, stable_node->head);
}
}
}
- for (nid = 0; nid < nr_node_ids; nid++)
+ for (nid = 0; nid < ksm_nr_node_ids; nid++)
root_unstable_tree[nid] = RB_ROOT;
spin_lock(&ksm_mmlist_lock);
{
struct stable_node *stable_node;
struct rmap_item *rmap_item;
- struct hlist_node *hlist;
unsigned int mapcount = page_mapcount(page);
int referenced = 0;
int search_new_forks = 0;
if (!stable_node)
return 0;
again:
- hlist_for_each_entry(rmap_item, hlist, &stable_node->hlist, hlist) {
+ hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
struct anon_vma *anon_vma = rmap_item->anon_vma;
struct anon_vma_chain *vmac;
struct vm_area_struct *vma;
int try_to_unmap_ksm(struct page *page, enum ttu_flags flags)
{
struct stable_node *stable_node;
- struct hlist_node *hlist;
struct rmap_item *rmap_item;
int ret = SWAP_AGAIN;
int search_new_forks = 0;
if (!stable_node)
return SWAP_FAIL;
again:
- hlist_for_each_entry(rmap_item, hlist, &stable_node->hlist, hlist) {
+ hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
struct anon_vma *anon_vma = rmap_item->anon_vma;
struct anon_vma_chain *vmac;
struct vm_area_struct *vma;
struct vm_area_struct *, unsigned long, void *), void *arg)
{
struct stable_node *stable_node;
- struct hlist_node *hlist;
struct rmap_item *rmap_item;
int ret = SWAP_AGAIN;
int search_new_forks = 0;
if (!stable_node)
return ret;
again:
- hlist_for_each_entry(rmap_item, hlist, &stable_node->hlist, hlist) {
+ hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
struct anon_vma *anon_vma = rmap_item->anon_vma;
struct anon_vma_chain *vmac;
struct vm_area_struct *vma;
struct rb_node *node;
int nid;
- for (nid = 0; nid < nr_node_ids; nid++) {
- node = rb_first(&root_stable_tree[nid]);
+ for (nid = 0; nid < ksm_nr_node_ids; nid++) {
+ node = rb_first(root_stable_tree + nid);
while (node) {
stable_node = rb_entry(node, struct stable_node, node);
if (stable_node->kpfn >= start_pfn &&
* which is why we keep kpfn instead of page*
*/
remove_node_from_stable_tree(stable_node);
- node = rb_first(&root_stable_tree[nid]);
+ node = rb_first(root_stable_tree + nid);
} else
node = rb_next(node);
cond_resched();
if (ksm_merge_across_nodes != knob) {
if (ksm_pages_shared || remove_all_stable_nodes())
err = -EBUSY;
- else
+ else if (root_stable_tree == one_stable_tree) {
+ struct rb_root *buf;
+ /*
+ * This is the first time that we switch away from the
+ * default of merging across nodes: must now allocate
+ * a buffer to hold as many roots as may be needed.
+ * Allocate stable and unstable together:
+ * MAXSMP NODES_SHIFT 10 will use 16kB.
+ */
+ buf = kcalloc(nr_node_ids + nr_node_ids,
+ sizeof(*buf), GFP_KERNEL | __GFP_ZERO);
+ /* Let us assume that RB_ROOT is NULL is zero */
+ if (!buf)
+ err = -ENOMEM;
+ else {
+ root_stable_tree = buf;
+ root_unstable_tree = buf + nr_node_ids;
+ /* Stable tree is empty but not the unstable */
+ root_unstable_tree[0] = one_unstable_tree[0];
+ }
+ }
+ if (!err) {
ksm_merge_across_nodes = knob;
+ ksm_nr_node_ids = knob ? 1 : nr_node_ids;
+ }
}
mutex_unlock(&ksm_thread_mutex);
{
struct task_struct *ksm_thread;
int err;
- int nid;
err = ksm_slab_init();
if (err)
goto out;
- for (nid = 0; nid < nr_node_ids; nid++)
- root_stable_tree[nid] = RB_ROOT;
-
ksm_thread = kthread_run(ksm_scan_thread, NULL, "ksmd");
if (IS_ERR(ksm_thread)) {
printk(KERN_ERR "ksm: creating kthread failed\n");
projects / deliverable / linux.git / blobdiff