mm, compaction: more robust check for scanners meeting

[deliverable/linux.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 2bd3d12ae106cb0463aecdece7e99a5e7a1a9a8b..084184e706c63184124bcf874cfe6702e0343950 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1306,6 +1306,17 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
        kasan_slab_free(s, x);
 }
 
+static void setup_object(struct kmem_cache *s, struct page *page,
+                               void *object)
+{
+       setup_object_debug(s, page, object);
+       if (unlikely(s->ctor)) {
+               kasan_unpoison_object_data(s, object);
+               s->ctor(object);
+               kasan_poison_object_data(s, object);
+       }
+}
+
 /*
  * Slab allocation and freeing
  */
@@ -1336,6 +1347,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
        struct page *page;
        struct kmem_cache_order_objects oo = s->oo;
        gfp_t alloc_gfp;
+       void *start, *p;
+       int idx, order;
 
        flags &= gfp_allowed_mask;
 
@@ -1349,6 +1362,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
         * so we fall-back to the minimum order allocation.
         */
        alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
+       if ((alloc_gfp & __GFP_WAIT) && oo_order(oo) > oo_order(s->min))
+               alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_WAIT;
 
        page = alloc_slab_page(s, alloc_gfp, node, oo);
        if (unlikely(!page)) {
@@ -1359,13 +1374,13 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                 * Try a lower order alloc if possible
                 */
                page = alloc_slab_page(s, alloc_gfp, node, oo);
-
-               if (page)
-                       stat(s, ORDER_FALLBACK);
+               if (unlikely(!page))
+                       goto out;
+               stat(s, ORDER_FALLBACK);
        }
 
-       if (kmemcheck_enabled && page
-               && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
+       if (kmemcheck_enabled &&
+           !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
                int pages = 1 << oo_order(oo);
 
                kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
@@ -1380,51 +1395,9 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                        kmemcheck_mark_unallocated_pages(page, pages);
        }
 
-       if (flags & __GFP_WAIT)
-               local_irq_disable();
-       if (!page)
-               return NULL;
-
        page->objects = oo_objects(oo);
-       mod_zone_page_state(page_zone(page),
-               (s->flags & SLAB_RECLAIM_ACCOUNT) ?
-               NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
-               1 << oo_order(oo));
-
-       return page;
-}
-
-static void setup_object(struct kmem_cache *s, struct page *page,
-                               void *object)
-{
-       setup_object_debug(s, page, object);
-       if (unlikely(s->ctor)) {
-               kasan_unpoison_object_data(s, object);
-               s->ctor(object);
-               kasan_poison_object_data(s, object);
-       }
-}
-
-static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
-{
-       struct page *page;
-       void *start;
-       void *p;
-       int order;
-       int idx;
-
-       if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
-               pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
-               BUG();
-       }
-
-       page = allocate_slab(s,
-               flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
-       if (!page)
-               goto out;
 
        order = compound_order(page);
-       inc_slabs_node(s, page_to_nid(page), page->objects);
        page->slab_cache = s;
        __SetPageSlab(page);
        if (page_is_pfmemalloc(page))
@@ -1448,10 +1421,34 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
        page->freelist = start;
        page->inuse = page->objects;
        page->frozen = 1;
+
 out:
+       if (flags & __GFP_WAIT)
+               local_irq_disable();
+       if (!page)
+               return NULL;
+
+       mod_zone_page_state(page_zone(page),
+               (s->flags & SLAB_RECLAIM_ACCOUNT) ?
+               NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
+               1 << oo_order(oo));
+
+       inc_slabs_node(s, page_to_nid(page), page->objects);
+
        return page;
 }
 
+static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
+{
+       if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
+               pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
+               BUG();
+       }
+
+       return allocate_slab(s,
+               flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
+}
+
 static void __free_slab(struct kmem_cache *s, struct page *page)
 {
        int order = compound_order(page);
@@ -2757,10 +2754,6 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
        struct page *page;
        int i;
 
-       /* Debugging fallback to generic bulk */
-       if (kmem_cache_debug(s))
-               return __kmem_cache_free_bulk(s, size, p);
-
        local_irq_disable();
        c = this_cpu_ptr(s->cpu_slab);
 
@@ -2768,8 +2761,13 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
                void *object = p[i];
 
                BUG_ON(!object);
+               /* kmem cache debug support */
+               s = cache_from_obj(s, object);
+               if (unlikely(!s))
+                       goto exit;
+               slab_free_hook(s, object);
+
                page = virt_to_head_page(object);
-               BUG_ON(s != page->slab_cache); /* Check if valid slab page */
 
                if (c->page == page) {
                        /* Fastpath: local CPU free */
@@ -2784,6 +2782,7 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
                        c = this_cpu_ptr(s->cpu_slab);
                }
        }
+exit:
        c->tid = next_tid(c->tid);
        local_irq_enable();
 }
@@ -2796,10 +2795,6 @@ bool kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
        struct kmem_cache_cpu *c;
        int i;
 
-       /* Debugging fallback to generic bulk */
-       if (kmem_cache_debug(s))
-               return __kmem_cache_alloc_bulk(s, flags, size, p);
-
        /*
         * Drain objects in the per cpu slab, while disabling local
         * IRQs, which protects against PREEMPT and interrupts
@@ -2828,8 +2823,20 @@ bool kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                        continue; /* goto for-loop */
                }
 
+               /* kmem_cache debug support */
+               s = slab_pre_alloc_hook(s, flags);
+               if (unlikely(!s)) {
+                       __kmem_cache_free_bulk(s, i, p);
+                       c->tid = next_tid(c->tid);
+                       local_irq_enable();
+                       return false;
+               }
+
                c->freelist = get_freepointer(s, object);
                p[i] = object;
+
+               /* kmem_cache debug support */
+               slab_post_alloc_hook(s, flags, object);
        }
        c->tid = next_tid(c->tid);
        local_irq_enable();
@@ -5278,7 +5285,7 @@ static int sysfs_slab_add(struct kmem_cache *s)
        s->kobj.kset = cache_kset(s);
        err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
        if (err)
-               goto out_put_kobj;
+               goto out;
 
        err = sysfs_create_group(&s->kobj, &slab_attr_group);
        if (err)
@@ -5305,8 +5312,6 @@ out:
        return err;
 out_del_kobj:
        kobject_del(&s->kobj);
-out_put_kobj:
-       kobject_put(&s->kobj);
        goto out;
 }