mm, compaction: more robust check for scanners meeting

[deliverable/linux.git] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 3ea6f908a5e01376da6be0d691e5d0c356bb8f4a..71a4822c832b9b63a570a6aba244c1e0ec199e6a 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -106,7 +106,7 @@ static struct khugepaged_scan khugepaged_scan = {
 };
 
 
-static int set_recommended_min_free_kbytes(void)
+static void set_recommended_min_free_kbytes(void)
 {
        struct zone *zone;
        int nr_zones = 0;
@@ -141,7 +141,6 @@ static int set_recommended_min_free_kbytes(void)
                min_free_kbytes = recommended_min;
        }
        setup_per_zone_wmarks();
-       return 0;
 }
 
 static int start_stop_khugepaged(void)
@@ -790,7 +789,7 @@ static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp)
 }
 
 /* Caller must hold page table lock. */
-bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
+static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
                struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
                struct page *zero_page)
 {
@@ -869,7 +868,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                                            flags);
 }
 
-static int insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
+static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
                pmd_t *pmd, unsigned long pfn, pgprot_t prot, bool write)
 {
        struct mm_struct *mm = vma->vm_mm;
@@ -887,7 +886,6 @@ static int insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
                update_mmu_cache_pmd(vma, addr, pmd);
        }
        spin_unlock(ptl);
-       return VM_FAULT_NOPAGE;
 }
 
 int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
@@ -909,7 +907,8 @@ int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
                return VM_FAULT_SIGBUS;
        if (track_pfn_insert(vma, &pgprot, pfn))
                return VM_FAULT_SIGBUS;
-       return insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write);
+       insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write);
+       return VM_FAULT_NOPAGE;
 }
 
 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
@@ -1456,50 +1455,41 @@ out:
 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                 pmd_t *pmd, unsigned long addr)
 {
+       pmd_t orig_pmd;
        spinlock_t *ptl;
-       int ret = 0;
 
-       if (__pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
-               pgtable_t pgtable;
-               pmd_t orig_pmd;
-               /*
-                * For architectures like ppc64 we look at deposited pgtable
-                * when calling pmdp_huge_get_and_clear. So do the
-                * pgtable_trans_huge_withdraw after finishing pmdp related
-                * operations.
-                */
-               orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
-                                                       tlb->fullmm);
-               tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
-               if (vma_is_dax(vma)) {
-                       if (is_huge_zero_pmd(orig_pmd)) {
-                               pgtable = NULL;
-                       } else {
-                               spin_unlock(ptl);
-                               return 1;
-                       }
-               } else {
-                       pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
-               }
-               if (is_huge_zero_pmd(orig_pmd)) {
-                       atomic_long_dec(&tlb->mm->nr_ptes);
-                       spin_unlock(ptl);
+       if (__pmd_trans_huge_lock(pmd, vma, &ptl) != 1)
+               return 0;
+       /*
+        * For architectures like ppc64 we look at deposited pgtable
+        * when calling pmdp_huge_get_and_clear. So do the
+        * pgtable_trans_huge_withdraw after finishing pmdp related
+        * operations.
+        */
+       orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
+                       tlb->fullmm);
+       tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
+       if (vma_is_dax(vma)) {
+               spin_unlock(ptl);
+               if (is_huge_zero_pmd(orig_pmd))
                        put_huge_zero_page();
-               } else {
-                       struct page *page = pmd_page(orig_pmd);
-                       page_remove_rmap(page);
-                       VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
-                       add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
-                       VM_BUG_ON_PAGE(!PageHead(page), page);
-                       atomic_long_dec(&tlb->mm->nr_ptes);
-                       spin_unlock(ptl);
-                       tlb_remove_page(tlb, page);
-               }
-               if (pgtable)
-                       pte_free(tlb->mm, pgtable);
-               ret = 1;
+       } else if (is_huge_zero_pmd(orig_pmd)) {
+               pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
+               atomic_long_dec(&tlb->mm->nr_ptes);
+               spin_unlock(ptl);
+               put_huge_zero_page();
+       } else {
+               struct page *page = pmd_page(orig_pmd);
+               page_remove_rmap(page);
+               VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
+               add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
+               VM_BUG_ON_PAGE(!PageHead(page), page);
+               pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
+               atomic_long_dec(&tlb->mm->nr_ptes);
+               spin_unlock(ptl);
+               tlb_remove_page(tlb, page);
        }
-       return ret;
+       return 1;
 }
 
 int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
@@ -2978,7 +2968,9 @@ again:
        if (unlikely(!pmd_trans_huge(*pmd)))
                goto unlock;
        if (vma_is_dax(vma)) {
-               pmdp_huge_clear_flush(vma, haddr, pmd);
+               pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
+               if (is_huge_zero_pmd(_pmd))
+                       put_huge_zero_page();
        } else if (is_huge_zero_pmd(*pmd)) {
                __split_huge_zero_page_pmd(vma, haddr, pmd);
        } else {