memcg: turn memcg_kmem_skip_account into a bit field

[deliverable/linux.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 9fd722769927f9e5bb5e03fb6516db7a5f7c8f42..989cb032eaf512913b46c2ee2aa8e7014649aa7d 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -582,7 +582,7 @@ retry_cpuset:
 
        for_each_zone_zonelist_nodemask(zone, z, zonelist,
                                                MAX_NR_ZONES - 1, nodemask) {
-               if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) {
+               if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
                        page = dequeue_huge_page_node(h, zone_to_nid(zone));
                        if (page) {
                                if (avoid_reserve)
@@ -2638,8 +2638,9 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 
        tlb_start_vma(tlb, vma);
        mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
+       address = start;
 again:
-       for (address = start; address < end; address += sz) {
+       for (; address < end; address += sz) {
                ptep = huge_pte_offset(mm, address);
                if (!ptep)
                        continue;
@@ -2686,6 +2687,7 @@ again:
                page_remove_rmap(page);
                force_flush = !__tlb_remove_page(tlb, page);
                if (force_flush) {
+                       address += sz;
                        spin_unlock(ptl);
                        break;
                }
@@ -2724,9 +2726,9 @@ void __unmap_hugepage_range_final(struct mmu_gather *tlb,
         * on its way out.  We're lucky that the flag has such an appropriate
         * name, and can in fact be safely cleared here. We could clear it
         * before the __unmap_hugepage_range above, but all that's necessary
-        * is to clear it before releasing the i_mmap_mutex. This works
+        * is to clear it before releasing the i_mmap_rwsem. This works
         * because in the context this is called, the VMA is about to be
-        * destroyed and the i_mmap_mutex is held.
+        * destroyed and the i_mmap_rwsem is held.
         */
        vma->vm_flags &= ~VM_MAYSHARE;
 }
@@ -2772,7 +2774,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
         * this mapping should be shared between all the VMAs,
         * __unmap_hugepage_range() is called as the lock is already held
         */
-       mutex_lock(&mapping->i_mmap_mutex);
+       i_mmap_lock_write(mapping);
        vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
                /* Do not unmap the current VMA */
                if (iter_vma == vma)
@@ -2789,7 +2791,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
                        unmap_hugepage_range(iter_vma, address,
                                             address + huge_page_size(h), page);
        }
-       mutex_unlock(&mapping->i_mmap_mutex);
+       i_mmap_unlock_write(mapping);
 }
 
 /*
@@ -3346,7 +3348,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
        flush_cache_range(vma, address, end);
 
        mmu_notifier_invalidate_range_start(mm, start, end);
-       mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
+       i_mmap_lock_write(vma->vm_file->f_mapping);
        for (; address < end; address += huge_page_size(h)) {
                spinlock_t *ptl;
                ptep = huge_pte_offset(mm, address);
@@ -3368,13 +3370,13 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
                spin_unlock(ptl);
        }
        /*
-        * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
+        * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
         * may have cleared our pud entry and done put_page on the page table:
-        * once we release i_mmap_mutex, another task can do the final put_page
+        * once we release i_mmap_rwsem, another task can do the final put_page
         * and that page table be reused and filled with junk.
         */
        flush_tlb_range(vma, start, end);
-       mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
+       i_mmap_unlock_write(vma->vm_file->f_mapping);
        mmu_notifier_invalidate_range_end(mm, start, end);
 
        return pages << h->order;
@@ -3523,7 +3525,7 @@ static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
  * and returns the corresponding pte. While this is not necessary for the
  * !shared pmd case because we can allocate the pmd later as well, it makes the
  * code much cleaner. pmd allocation is essential for the shared case because
- * pud has to be populated inside the same i_mmap_mutex section - otherwise
+ * pud has to be populated inside the same i_mmap_rwsem section - otherwise
  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
  * bad pmd for sharing.
  */
@@ -3542,7 +3544,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
        if (!vma_shareable(vma, addr))
                return (pte_t *)pmd_alloc(mm, pud, addr);
 
-       mutex_lock(&mapping->i_mmap_mutex);
+       i_mmap_lock_write(mapping);
        vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
                if (svma == vma)
                        continue;
@@ -3570,7 +3572,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
        spin_unlock(ptl);
 out:
        pte = (pte_t *)pmd_alloc(mm, pud, addr);
-       mutex_unlock(&mapping->i_mmap_mutex);
+       i_mmap_unlock_write(mapping);
        return pte;
 }