Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[deliverable/linux.git] / mm / compaction.c

diff --git a/mm/compaction.c b/mm/compaction.c
index b6ede459c1bbf5a70ebeadf0803ecc5784fa82cb..8c0d9459b54a02042dd2caf9489566188d9ea908 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -16,6 +16,7 @@
 #include <linux/sysfs.h>
 #include <linux/balloon_compaction.h>
 #include <linux/page-isolation.h>
+#include <linux/kasan.h>
 #include "internal.h"
 
 #ifdef CONFIG_COMPACTION
@@ -72,6 +73,7 @@ static void map_pages(struct list_head *list)
        list_for_each_entry(page, list, lru) {
                arch_alloc_page(page, 0);
                kernel_map_pages(page, 1, 1);
+               kasan_alloc_pages(page, 0);
        }
 }
 
@@ -124,6 +126,77 @@ static struct page *pageblock_pfn_to_page(unsigned long start_pfn,
 }
 
 #ifdef CONFIG_COMPACTION
+
+/* Do not skip compaction more than 64 times */
+#define COMPACT_MAX_DEFER_SHIFT 6
+
+/*
+ * Compaction is deferred when compaction fails to result in a page
+ * allocation success. 1 << compact_defer_limit compactions are skipped up
+ * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
+ */
+void defer_compaction(struct zone *zone, int order)
+{
+       zone->compact_considered = 0;
+       zone->compact_defer_shift++;
+
+       if (order < zone->compact_order_failed)
+               zone->compact_order_failed = order;
+
+       if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
+               zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
+
+       trace_mm_compaction_defer_compaction(zone, order);
+}
+
+/* Returns true if compaction should be skipped this time */
+bool compaction_deferred(struct zone *zone, int order)
+{
+       unsigned long defer_limit = 1UL << zone->compact_defer_shift;
+
+       if (order < zone->compact_order_failed)
+               return false;
+
+       /* Avoid possible overflow */
+       if (++zone->compact_considered > defer_limit)
+               zone->compact_considered = defer_limit;
+
+       if (zone->compact_considered >= defer_limit)
+               return false;
+
+       trace_mm_compaction_deferred(zone, order);
+
+       return true;
+}
+
+/*
+ * Update defer tracking counters after successful compaction of given order,
+ * which means an allocation either succeeded (alloc_success == true) or is
+ * expected to succeed.
+ */
+void compaction_defer_reset(struct zone *zone, int order,
+               bool alloc_success)
+{
+       if (alloc_success) {
+               zone->compact_considered = 0;
+               zone->compact_defer_shift = 0;
+       }
+       if (order >= zone->compact_order_failed)
+               zone->compact_order_failed = order + 1;
+
+       trace_mm_compaction_defer_reset(zone, order);
+}
+
+/* Returns true if restarting compaction after many failures */
+bool compaction_restarting(struct zone *zone, int order)
+{
+       if (order < zone->compact_order_failed)
+               return false;
+
+       return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
+               zone->compact_considered >= 1UL << zone->compact_defer_shift;
+}
+
 /* Returns true if the pageblock should be scanned for pages to isolate. */
 static inline bool isolation_suitable(struct compact_control *cc,
                                        struct page *page)
@@ -419,6 +492,13 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
 
                /* If a page was split, advance to the end of it */
                if (isolated) {
+                       cc->nr_freepages += isolated;
+                       if (!strict &&
+                               cc->nr_migratepages <= cc->nr_freepages) {
+                               blockpfn += isolated;
+                               break;
+                       }
+
                        blockpfn += isolated - 1;
                        cursor += isolated - 1;
                        continue;
@@ -828,7 +908,6 @@ static void isolate_freepages(struct compact_control *cc)
        unsigned long isolate_start_pfn; /* exact pfn we start at */
        unsigned long block_end_pfn;    /* end of current pageblock */
        unsigned long low_pfn;       /* lowest pfn scanner is able to scan */
-       int nr_freepages = cc->nr_freepages;
        struct list_head *freelist = &cc->freepages;
 
        /*
@@ -853,11 +932,11 @@ static void isolate_freepages(struct compact_control *cc)
         * pages on cc->migratepages. We stop searching if the migrate
         * and free page scanners meet or enough free pages are isolated.
         */
-       for (; block_start_pfn >= low_pfn && cc->nr_migratepages > nr_freepages;
+       for (; block_start_pfn >= low_pfn &&
+                       cc->nr_migratepages > cc->nr_freepages;
                                block_end_pfn = block_start_pfn,
                                block_start_pfn -= pageblock_nr_pages,
                                isolate_start_pfn = block_start_pfn) {
-               unsigned long isolated;
 
                /*
                 * This can iterate a massively long zone without finding any
@@ -882,9 +961,8 @@ static void isolate_freepages(struct compact_control *cc)
                        continue;
 
                /* Found a block suitable for isolating free pages from. */
-               isolated = isolate_freepages_block(cc, &isolate_start_pfn,
+               isolate_freepages_block(cc, &isolate_start_pfn,
                                        block_end_pfn, freelist, false);
-               nr_freepages += isolated;
 
                /*
                 * Remember where the free scanner should restart next time,
@@ -916,8 +994,6 @@ static void isolate_freepages(struct compact_control *cc)
         */
        if (block_start_pfn < low_pfn)
                cc->free_pfn = cc->migrate_pfn;
-
-       cc->nr_freepages = nr_freepages;
 }
 
 /*
@@ -1029,8 +1105,10 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
                low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn,
                                                                isolate_mode);
 
-               if (!low_pfn || cc->contended)
+               if (!low_pfn || cc->contended) {
+                       acct_isolated(zone, cc);
                        return ISOLATE_ABORT;
+               }
 
                /*
                 * Either we isolated something and proceed with migration. Or
@@ -1102,7 +1180,7 @@ static int __compact_finished(struct zone *zone, struct compact_control *cc,
                        return COMPACT_PARTIAL;
 
                /* Job done if allocation would set block type */
-               if (cc->order >= pageblock_order && area->nr_free)
+               if (order >= pageblock_order && area->nr_free)
                        return COMPACT_PARTIAL;
        }