slub: Enable sysfs support for !CONFIG_SLUB_DEBUG

[deliverable/linux.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 064bda294af271e2480769965b114e296e1030a7..be4d66231c6f8d2648dfc2cfce8b6b252aa7543f 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -198,7 +198,7 @@ struct track {
 
 enum track_item { TRACK_ALLOC, TRACK_FREE };
 
-#ifdef CONFIG_SLUB_DEBUG
+#ifdef CONFIG_SYSFS
 static int sysfs_slab_add(struct kmem_cache *);
 static int sysfs_slab_alias(struct kmem_cache *, const char *);
 static void sysfs_slab_remove(struct kmem_cache *);
@@ -490,7 +490,7 @@ static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...)
        dump_stack();
 }
 
-static void init_object(struct kmem_cache *s, void *object, int active)
+static void init_object(struct kmem_cache *s, void *object, u8 val)
 {
        u8 *p = object;
 
@@ -500,9 +500,7 @@ static void init_object(struct kmem_cache *s, void *object, int active)
        }
 
        if (s->flags & SLAB_RED_ZONE)
-               memset(p + s->objsize,
-                       active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE,
-                       s->inuse - s->objsize);
+               memset(p + s->objsize, val, s->inuse - s->objsize);
 }
 
 static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes)
@@ -637,17 +635,14 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page)
 }
 
 static int check_object(struct kmem_cache *s, struct page *page,
-                                       void *object, int active)
+                                       void *object, u8 val)
 {
        u8 *p = object;
        u8 *endobject = object + s->objsize;
 
        if (s->flags & SLAB_RED_ZONE) {
-               unsigned int red =
-                       active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE;
-
                if (!check_bytes_and_report(s, page, object, "Redzone",
-                       endobject, red, s->inuse - s->objsize))
+                       endobject, val, s->inuse - s->objsize))
                        return 0;
        } else {
                if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
@@ -657,7 +652,7 @@ static int check_object(struct kmem_cache *s, struct page *page,
        }
 
        if (s->flags & SLAB_POISON) {
-               if (!active && (s->flags & __OBJECT_POISON) &&
+               if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
                        (!check_bytes_and_report(s, page, p, "Poison", p,
                                        POISON_FREE, s->objsize - 1) ||
                         !check_bytes_and_report(s, page, p, "Poison",
@@ -669,7 +664,7 @@ static int check_object(struct kmem_cache *s, struct page *page,
                check_pad_bytes(s, page, p);
        }
 
-       if (!s->offset && active)
+       if (!s->offset && val == SLUB_RED_ACTIVE)
                /*
                 * Object and freepointer overlap. Cannot check
                 * freepointer while object is allocated.
@@ -887,7 +882,7 @@ static void setup_object_debug(struct kmem_cache *s, struct page *page,
        if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
                return;
 
-       init_object(s, object, 0);
+       init_object(s, object, SLUB_RED_INACTIVE);
        init_tracking(s, object);
 }
 
@@ -907,14 +902,14 @@ static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *pa
                goto bad;
        }
 
-       if (!check_object(s, page, object, 0))
+       if (!check_object(s, page, object, SLUB_RED_INACTIVE))
                goto bad;
 
        /* Success perform special debug activities for allocs */
        if (s->flags & SLAB_STORE_USER)
                set_track(s, object, TRACK_ALLOC, addr);
        trace(s, page, object, 1);
-       init_object(s, object, 1);
+       init_object(s, object, SLUB_RED_ACTIVE);
        return 1;
 
 bad:
@@ -947,7 +942,7 @@ static noinline int free_debug_processing(struct kmem_cache *s,
                goto fail;
        }
 
-       if (!check_object(s, page, object, 1))
+       if (!check_object(s, page, object, SLUB_RED_ACTIVE))
                return 0;
 
        if (unlikely(s != page->slab)) {
@@ -971,7 +966,7 @@ static noinline int free_debug_processing(struct kmem_cache *s,
        if (s->flags & SLAB_STORE_USER)
                set_track(s, object, TRACK_FREE, addr);
        trace(s, page, object, 0);
-       init_object(s, object, 0);
+       init_object(s, object, SLUB_RED_INACTIVE);
        return 1;
 
 fail:
@@ -1075,7 +1070,7 @@ static inline int free_debug_processing(struct kmem_cache *s,
 static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
                        { return 1; }
 static inline int check_object(struct kmem_cache *s, struct page *page,
-                       void *object, int active) { return 1; }
+                       void *object, u8 val) { return 1; }
 static inline void add_full(struct kmem_cache_node *n, struct page *page) {}
 static inline unsigned long kmem_cache_flags(unsigned long objsize,
        unsigned long flags, const char *name,
@@ -1107,7 +1102,7 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x) {}
 static inline void slab_free_hook_irq(struct kmem_cache *s,
                void *object) {}
 
-#endif
+#endif /* CONFIG_SLUB_DEBUG */
 
 /*
  * Slab allocation and freeing
@@ -1235,7 +1230,7 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
                slab_pad_check(s, page);
                for_each_object(p, s, page_address(page),
                                                page->objects)
-                       check_object(s, page, p, 0);
+                       check_object(s, page, p, SLUB_RED_INACTIVE);
        }
 
        kmemcheck_free_shadow(page, compound_order(page));
@@ -1315,13 +1310,19 @@ static void add_partial(struct kmem_cache_node *n,
        spin_unlock(&n->list_lock);
 }
 
+static inline void __remove_partial(struct kmem_cache_node *n,
+                                       struct page *page)
+{
+       list_del(&page->lru);
+       n->nr_partial--;
+}
+
 static void remove_partial(struct kmem_cache *s, struct page *page)
 {
        struct kmem_cache_node *n = get_node(s, page_to_nid(page));
 
        spin_lock(&n->list_lock);
-       list_del(&page->lru);
-       n->nr_partial--;
+       __remove_partial(n, page);
        spin_unlock(&n->list_lock);
 }
 
@@ -1334,8 +1335,7 @@ static inline int lock_and_freeze_slab(struct kmem_cache_node *n,
                                                        struct page *page)
 {
        if (slab_trylock(page)) {
-               list_del(&page->lru);
-               n->nr_partial--;
+               __remove_partial(n, page);
                __SetPageSlubFrozen(page);
                return 1;
        }
@@ -1446,6 +1446,7 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
  * On exit the slab lock will have been dropped.
  */
 static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
+       __releases(bitlock)
 {
        struct kmem_cache_node *n = get_node(s, page_to_nid(page));
 
@@ -1488,6 +1489,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
  * Remove the cpu slab
  */
 static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
+       __releases(bitlock)
 {
        struct page *page = c->page;
        int tail = 1;
@@ -1716,7 +1718,7 @@ debug:
 
        c->page->inuse++;
        c->page->freelist = get_freepointer(s, object);
-       c->node = -1;
+       c->node = NUMA_NO_NODE;
        goto unlock_out;
 }
 
@@ -1790,7 +1792,6 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
-#endif
 
 #ifdef CONFIG_TRACING
 void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
@@ -1801,6 +1802,7 @@ void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
 #endif
+#endif
 
 /*
  * Slow patch handling. This may still be called frequently since objects
@@ -1893,7 +1895,7 @@ static __always_inline void slab_free(struct kmem_cache *s,
 
        slab_free_hook_irq(s, x);
 
-       if (likely(page == c->page && c->node >= 0)) {
+       if (likely(page == c->page && c->node != NUMA_NO_NODE)) {
                set_freepointer(s, object, c->freelist);
                c->freelist = object;
                stat(s, FREE_FASTPATH);
@@ -2143,7 +2145,7 @@ static void early_kmem_cache_node_alloc(int node)
        page->inuse++;
        kmem_cache_node->node[node] = n;
 #ifdef CONFIG_SLUB_DEBUG
-       init_object(kmem_cache_node, n, 1);
+       init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
        init_tracking(kmem_cache_node, n);
 #endif
        init_kmem_cache_node(n, kmem_cache_node);
@@ -2433,9 +2435,8 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page,
 #ifdef CONFIG_SLUB_DEBUG
        void *addr = page_address(page);
        void *p;
-       long *map = kzalloc(BITS_TO_LONGS(page->objects) * sizeof(long),
-                           GFP_ATOMIC);
-
+       unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
+                                    sizeof(long), GFP_ATOMIC);
        if (!map)
                return;
        slab_err(s, page, "%s", text);
@@ -2467,9 +2468,8 @@ static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
        spin_lock_irqsave(&n->list_lock, flags);
        list_for_each_entry_safe(page, h, &n->partial, lru) {
                if (!page->inuse) {
-                       list_del(&page->lru);
+                       __remove_partial(n, page);
                        discard_slab(s, page);
-                       n->nr_partial--;
                } else {
                        list_slab_objects(s, page,
                                "Objects remaining on kmem_cache_close()");
@@ -2673,6 +2673,7 @@ void *__kmalloc(size_t size, gfp_t flags)
 }
 EXPORT_SYMBOL(__kmalloc);
 
+#ifdef CONFIG_NUMA
 static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
        struct page *page;
@@ -2687,7 +2688,6 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
        return ptr;
 }
 
-#ifdef CONFIG_NUMA
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
        struct kmem_cache *s;
@@ -2827,8 +2827,7 @@ int kmem_cache_shrink(struct kmem_cache *s)
                                 * may have freed the last object and be
                                 * waiting to release the slab.
                                 */
-                               list_del(&page->lru);
-                               n->nr_partial--;
+                               __remove_partial(n, page);
                                slab_unlock(page);
                                discard_slab(s, page);
                        } else {
@@ -3343,6 +3342,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
        return ret;
 }
 
+#ifdef CONFIG_NUMA
 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
                                        int node, unsigned long caller)
 {
@@ -3371,8 +3371,9 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
 
        return ret;
 }
+#endif
 
-#ifdef CONFIG_SLUB_DEBUG
+#ifdef CONFIG_SYSFS
 static int count_inuse(struct page *page)
 {
        return page->inuse;
@@ -3382,7 +3383,9 @@ static int count_total(struct page *page)
 {
        return page->objects;
 }
+#endif
 
+#ifdef CONFIG_SLUB_DEBUG
 static int validate_slab(struct kmem_cache *s, struct page *page,
                                                unsigned long *map)
 {
@@ -3473,6 +3476,7 @@ static long validate_slab_cache(struct kmem_cache *s)
        kfree(map);
        return count;
 }
+#endif
 
 #ifdef SLUB_RESILIENCY_TEST
 static void resiliency_test(void)
@@ -3531,9 +3535,12 @@ static void resiliency_test(void)
        validate_slab_cache(kmalloc_caches[9]);
 }
 #else
+#ifdef CONFIG_SYSFS
 static void resiliency_test(void) {};
 #endif
+#endif
 
+#ifdef CONFIG_DEBUG
 /*
  * Generate lists of code addresses where slabcache objects are allocated
  * and freed.
@@ -3662,7 +3669,7 @@ static int add_location(struct loc_track *t, struct kmem_cache *s,
 
 static void process_slab(struct loc_track *t, struct kmem_cache *s,
                struct page *page, enum track_item alloc,
-               long *map)
+               unsigned long *map)
 {
        void *addr = page_address(page);
        void *p;
@@ -3762,7 +3769,9 @@ static int list_locations(struct kmem_cache *s, char *buf,
                len += sprintf(buf, "No data\n");
        return len;
 }
+#endif
 
+#ifdef CONFIG_SYSFS
 enum slab_stat_type {
        SL_ALL,                 /* All slabs */
        SL_PARTIAL,             /* Only partially allocated slabs */
@@ -3815,6 +3824,8 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
                }
        }
 
+       down_read(&slub_lock);
+#ifdef CONFIG_SLUB_DEBUG
        if (flags & SO_ALL) {
                for_each_node_state(node, N_NORMAL_MEMORY) {
                        struct kmem_cache_node *n = get_node(s, node);
@@ -3831,7 +3842,9 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
                        nodes[node] += x;
                }
 
-       } else if (flags & SO_PARTIAL) {
+       } else
+#endif
+       if (flags & SO_PARTIAL) {
                for_each_node_state(node, N_NORMAL_MEMORY) {
                        struct kmem_cache_node *n = get_node(s, node);
 
@@ -3856,6 +3869,7 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
        return x + sprintf(buf + x, "\n");
 }
 
+#ifdef CONFIG_SLUB_DEBUG
 static int any_slab_objects(struct kmem_cache *s)
 {
        int node;
@@ -3871,6 +3885,7 @@ static int any_slab_objects(struct kmem_cache *s)
        }
        return 0;
 }
+#endif
 
 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
 #define to_slab(n) container_of(n, struct kmem_cache, kobj);
@@ -3972,11 +3987,13 @@ static ssize_t aliases_show(struct kmem_cache *s, char *buf)
 }
 SLAB_ATTR_RO(aliases);
 
+#ifdef CONFIG_SLUB_DEBUG
 static ssize_t slabs_show(struct kmem_cache *s, char *buf)
 {
        return show_slab_objects(s, buf, SO_ALL);
 }
 SLAB_ATTR_RO(slabs);
+#endif
 
 static ssize_t partial_show(struct kmem_cache *s, char *buf)
 {
@@ -4002,6 +4019,7 @@ static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
 }
 SLAB_ATTR_RO(objects_partial);
 
+#ifdef CONFIG_SLUB_DEBUG
 static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
 {
        return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
@@ -4054,6 +4072,7 @@ static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
 }
 SLAB_ATTR(failslab);
 #endif
+#endif
 
 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
 {
@@ -4090,6 +4109,7 @@ static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
 }
 SLAB_ATTR_RO(destroy_by_rcu);
 
+#ifdef CONFIG_SLUB_DEBUG
 static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
 {
        return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
@@ -4165,6 +4185,7 @@ static ssize_t validate_store(struct kmem_cache *s,
        return ret;
 }
 SLAB_ATTR(validate);
+#endif
 
 static ssize_t shrink_show(struct kmem_cache *s, char *buf)
 {
@@ -4185,6 +4206,7 @@ static ssize_t shrink_store(struct kmem_cache *s,
 }
 SLAB_ATTR(shrink);
 
+#ifdef CONFIG_SLUB_DEBUG
 static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
 {
        if (!(s->flags & SLAB_STORE_USER))
@@ -4200,6 +4222,7 @@ static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
        return list_locations(s, buf, TRACK_FREE);
 }
 SLAB_ATTR_RO(free_calls);
+#endif
 
 #ifdef CONFIG_NUMA
 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
@@ -4306,25 +4329,33 @@ static struct attribute *slab_attrs[] = {
        &min_partial_attr.attr,
        &objects_attr.attr,
        &objects_partial_attr.attr,
+#ifdef CONFIG_SLUB_DEBUG
        &total_objects_attr.attr,
        &slabs_attr.attr,
+#endif
        &partial_attr.attr,
        &cpu_slabs_attr.attr,
        &ctor_attr.attr,
        &aliases_attr.attr,
        &align_attr.attr,
+#ifdef CONFIG_SLUB_DEBUG
        &sanity_checks_attr.attr,
        &trace_attr.attr,
+#endif
        &hwcache_align_attr.attr,
        &reclaim_account_attr.attr,
        &destroy_by_rcu_attr.attr,
+#ifdef CONFIG_SLUB_DEBUG
        &red_zone_attr.attr,
        &poison_attr.attr,
        &store_user_attr.attr,
        &validate_attr.attr,
+#endif
        &shrink_attr.attr,
+#ifdef CONFIG_SLUB_DEBUG
        &alloc_calls_attr.attr,
        &free_calls_attr.attr,
+#endif
 #ifdef CONFIG_ZONE_DMA
        &cache_dma_attr.attr,
 #endif
@@ -4607,7 +4638,7 @@ static int __init slab_sysfs_init(void)
 }
 
 __initcall(slab_sysfs_init);
-#endif
+#endif /* CONFIG_SYSFS */
 
 /*
  * The /proc/slabinfo ABI