thp: implement split_huge_pmd()

[deliverable/linux.git] / mm / mempolicy.c

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 99d4c1d0b8583dc453ef992582074ef015f1fb49..5f7f9dace3546cf08256dd2674c54e25604448c6 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -493,7 +493,7 @@ static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
        pte_t *pte;
        spinlock_t *ptl;
 
-       split_huge_page_pmd(vma, addr, pmd);
+       split_huge_pmd(vma, pmd, addr);
        if (pmd_trans_unstable(pmd))
                return 0;
 
@@ -608,9 +608,6 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end,
 
        qp->prev = vma;
 
-       if (vma->vm_flags & VM_PFNMAP)
-               return 1;
-
        if (flags & MPOL_MF_LAZY) {
                /* Similar to task_numa_work, skip inaccessible VMAs */
                if (vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))
@@ -722,8 +719,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
                pgoff = vma->vm_pgoff +
                        ((vmstart - vma->vm_start) >> PAGE_SHIFT);
                prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
-                                 vma->anon_vma, vma->vm_file, pgoff,
-                                 new_pol);
+                                vma->anon_vma, vma->vm_file, pgoff,
+                                new_pol, vma->vm_userfaultfd_ctx);
                if (prev) {
                        vma = prev;
                        next = vma->vm_next;
@@ -945,7 +942,7 @@ static struct page *new_node_page(struct page *page, unsigned long node, int **x
                return alloc_huge_page_node(page_hstate(compound_head(page)),
                                        node);
        else
-               return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE |
+               return __alloc_pages_node(node, GFP_HIGHUSER_MOVABLE |
                                                    __GFP_THISNODE, 0);
 }
 
@@ -2001,7 +1998,7 @@ retry_cpuset:
                nmask = policy_nodemask(gfp, pol);
                if (!nmask || node_isset(hpage_node, *nmask)) {
                        mpol_cond_put(pol);
-                       page = alloc_pages_exact_node(hpage_node,
+                       page = __alloc_pages_node(hpage_node,
                                                gfp | __GFP_THISNODE, order);
                        goto out;
                }
@@ -2145,12 +2142,14 @@ bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
  *
  * Remember policies even when nobody has shared memory mapped.
  * The policies are kept in Red-Black tree linked from the inode.
- * They are protected by the sp->lock spinlock, which should be held
+ * They are protected by the sp->lock rwlock, which should be held
  * for any accesses to the tree.
  */
 
-/* lookup first element intersecting start-end */
-/* Caller holds sp->lock */
+/*
+ * lookup first element intersecting start-end.  Caller holds sp->lock for
+ * reading or for writing
+ */
 static struct sp_node *
 sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
 {
@@ -2181,8 +2180,10 @@ sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
        return rb_entry(n, struct sp_node, nd);
 }
 
-/* Insert a new shared policy into the list. */
-/* Caller holds sp->lock */
+/*
+ * Insert a new shared policy into the list.  Caller holds sp->lock for
+ * writing.
+ */
 static void sp_insert(struct shared_policy *sp, struct sp_node *new)
 {
        struct rb_node **p = &sp->root.rb_node;
@@ -2214,13 +2215,13 @@ mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
 
        if (!sp->root.rb_node)
                return NULL;
-       spin_lock(&sp->lock);
+       read_lock(&sp->lock);
        sn = sp_lookup(sp, idx, idx+1);
        if (sn) {
                mpol_get(sn->policy);
                pol = sn->policy;
        }
-       spin_unlock(&sp->lock);
+       read_unlock(&sp->lock);
        return pol;
 }
 
@@ -2363,7 +2364,7 @@ static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
        int ret = 0;
 
 restart:
-       spin_lock(&sp->lock);
+       write_lock(&sp->lock);
        n = sp_lookup(sp, start, end);
        /* Take care of old policies in the same range. */
        while (n && n->start < end) {
@@ -2396,7 +2397,7 @@ restart:
        }
        if (new)
                sp_insert(sp, new);
-       spin_unlock(&sp->lock);
+       write_unlock(&sp->lock);
        ret = 0;
 
 err_out:
@@ -2408,7 +2409,7 @@ err_out:
        return ret;
 
 alloc_new:
-       spin_unlock(&sp->lock);
+       write_unlock(&sp->lock);
        ret = -ENOMEM;
        n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
        if (!n_new)
@@ -2434,7 +2435,7 @@ void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
        int ret;
 
        sp->root = RB_ROOT;             /* empty tree == default mempolicy */
-       spin_lock_init(&sp->lock);
+       rwlock_init(&sp->lock);
 
        if (mpol) {
                struct vm_area_struct pvma;
@@ -2500,14 +2501,14 @@ void mpol_free_shared_policy(struct shared_policy *p)
 
        if (!p->root.rb_node)
                return;
-       spin_lock(&p->lock);
+       write_lock(&p->lock);
        next = rb_first(&p->root);
        while (next) {
                n = rb_entry(next, struct sp_node, nd);
                next = rb_next(&n->nd);
                sp_delete(p, n);
        }
-       spin_unlock(&p->lock);
+       write_unlock(&p->lock);
 }
 
 #ifdef CONFIG_NUMA_BALANCING