X-Git-Url: http://drtracing.org/?a=blobdiff_plain;f=mm%2Fzsmalloc.c;h=cd2a81e8b4295ebcd5b001c9ba4db55c43410435;hb=394e31d2ceb4b9eae25bd9ed8ea8cb19a40ff181;hp=04a4f063b4fd70c5ff960686d026dcb705805283;hpb=48b4800a1c6af2cdda344ea4e2c843dcc1f6afc9;p=deliverable%2Flinux.git diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 04a4f063b4fd..cd2a81e8b429 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -20,6 +20,7 @@ * page->freelist(index): links together all component pages of a zspage * For the huge page, this is always 0, so we use this field * to store handle. + * page->units: first object offset in a subpage of zspage * * Usage of struct page flags: * PG_private: identifies the first component page @@ -50,7 +51,7 @@ #include #include #include -#include +#include #include #define ZSPAGE_MAGIC 0x58 @@ -137,9 +138,6 @@ */ #define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> CLASS_BITS) -/* - * We do not maintain any list for completely empty or full pages - */ enum fullness_group { ZS_EMPTY, ZS_ALMOST_EMPTY, @@ -467,11 +465,6 @@ static struct zpool_driver zs_zpool_driver = { MODULE_ALIAS("zpool-zsmalloc"); #endif /* CONFIG_ZPOOL */ -static unsigned int get_maxobj_per_zspage(int size, int pages_per_zspage) -{ - return pages_per_zspage * PAGE_SIZE / size; -} - /* per-cpu VM mapping areas for zspage accesses that cross page boundaries */ static DEFINE_PER_CPU(struct mapping_area, zs_map_area); @@ -635,8 +628,7 @@ static int zs_stats_size_show(struct seq_file *s, void *v) freeable = zs_can_compact(class); spin_unlock(&class->lock); - objs_per_zspage = get_maxobj_per_zspage(class->size, - class->pages_per_zspage); + objs_per_zspage = class->objs_per_zspage; pages_used = obj_allocated / objs_per_zspage * class->pages_per_zspage; @@ -945,8 +937,8 @@ static void unpin_tag(unsigned long handle) static void reset_page(struct page *page) { __ClearPageMovable(page); - clear_bit(PG_private, &page->flags); - clear_bit(PG_private_2, &page->flags); + ClearPagePrivate(page); + ClearPagePrivate2(page); set_page_private(page, 0); page_mapcount_reset(page); ClearPageHugeObject(page); @@ -1007,14 +999,14 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class, next = get_next_page(page); reset_page(page); unlock_page(page); + dec_zone_page_state(page, NR_ZSPAGES); put_page(page); page = next; } while (page != NULL); cache_free_zspage(pool, zspage); - zs_stat_dec(class, OBJ_ALLOCATED, get_maxobj_per_zspage( - class->size, class->pages_per_zspage)); + zs_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage); atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); } @@ -1052,7 +1044,7 @@ static void init_zspage(struct size_class *class, struct zspage *zspage) link = (struct link_free *)vaddr + off / sizeof(*link); while ((off += class->size) < PAGE_SIZE) { - link->next = freeobj++ << OBJ_ALLOCATED_TAG; + link->next = freeobj++ << OBJ_TAG_BITS; link += class->size / sizeof(*link); } @@ -1063,13 +1055,13 @@ static void init_zspage(struct size_class *class, struct zspage *zspage) */ next_page = get_next_page(page); if (next_page) { - link->next = freeobj++ << OBJ_ALLOCATED_TAG; + link->next = freeobj++ << OBJ_TAG_BITS; } else { /* - * Reset OBJ_ALLOCATED_TAG bit to last link to tell + * Reset OBJ_TAG_BITS bit to last link to tell * whether it's allocated object or not. */ - link->next = -1 << OBJ_ALLOCATED_TAG; + link->next = -1 << OBJ_TAG_BITS; } kunmap_atomic(vaddr); page = next_page; @@ -1137,11 +1129,15 @@ static struct zspage *alloc_zspage(struct zs_pool *pool, page = alloc_page(gfp); if (!page) { - while (--i >= 0) + while (--i >= 0) { + dec_zone_page_state(pages[i], NR_ZSPAGES); __free_page(pages[i]); + } cache_free_zspage(pool, zspage); return NULL; } + + inc_zone_page_state(page, NR_ZSPAGES); pages[i] = page; } @@ -1345,7 +1341,7 @@ static void zs_unregister_cpu_notifier(void) cpu_notifier_register_done(); } -static void init_zs_size_classes(void) +static void __init init_zs_size_classes(void) { int nr; @@ -1356,16 +1352,14 @@ static void init_zs_size_classes(void) zs_size_classes = nr; } -static bool can_merge(struct size_class *prev, int size, int pages_per_zspage) +static bool can_merge(struct size_class *prev, int pages_per_zspage, + int objs_per_zspage) { - if (prev->pages_per_zspage != pages_per_zspage) - return false; - - if (get_maxobj_per_zspage(prev->size, prev->pages_per_zspage) - != get_maxobj_per_zspage(size, pages_per_zspage)) - return false; + if (prev->pages_per_zspage == pages_per_zspage && + prev->objs_per_zspage == objs_per_zspage) + return true; - return true; + return false; } static bool zspage_full(struct size_class *class, struct zspage *zspage) @@ -1514,7 +1508,7 @@ static unsigned long obj_malloc(struct size_class *class, vaddr = kmap_atomic(m_page); link = (struct link_free *)vaddr + m_offset / sizeof(*link); - set_freeobj(zspage, link->next >> OBJ_ALLOCATED_TAG); + set_freeobj(zspage, link->next >> OBJ_TAG_BITS); if (likely(!PageHugeObject(m_page))) /* record handle in the header of allocated chunk */ link->handle = handle; @@ -1536,6 +1530,7 @@ static unsigned long obj_malloc(struct size_class *class, * zs_malloc - Allocate block of given size from pool. * @pool: pool to allocate from * @size: size of block to allocate + * @gfp: gfp flags when allocating object * * On success, handle to the allocated object is returned, * otherwise 0. @@ -1587,8 +1582,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp) record_obj(handle, obj); atomic_long_add(class->pages_per_zspage, &pool->pages_allocated); - zs_stat_inc(class, OBJ_ALLOCATED, get_maxobj_per_zspage( - class->size, class->pages_per_zspage)); + zs_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage); /* We completely set up zspage so mark them as movable */ SetZsPageMovable(pool, zspage); @@ -1616,7 +1610,7 @@ static void obj_free(struct size_class *class, unsigned long obj) /* Insert this object in containing zspage's freelist */ link = (struct link_free *)(vaddr + f_offset); - link->next = get_freeobj(zspage) << OBJ_ALLOCATED_TAG; + link->next = get_freeobj(zspage) << OBJ_TAG_BITS; kunmap_atomic(vaddr); set_freeobj(zspage, f_objidx); mod_zspage_inuse(zspage, -1); @@ -1736,10 +1730,11 @@ static void zs_object_copy(struct size_class *class, unsigned long dst, * return handle. */ static unsigned long find_alloced_obj(struct size_class *class, - struct page *page, int index) + struct page *page, int *obj_idx) { unsigned long head; int offset = 0; + int index = *obj_idx; unsigned long handle = 0; void *addr = kmap_atomic(page); @@ -1760,6 +1755,9 @@ static unsigned long find_alloced_obj(struct size_class *class, } kunmap_atomic(addr); + + *obj_idx = index; + return handle; } @@ -1771,7 +1769,7 @@ struct zs_compact_control { struct page *d_page; /* Starting object index within @s_page which used for live object * in the subpage. */ - int index; + int obj_idx; }; static int migrate_zspage(struct zs_pool *pool, struct size_class *class, @@ -1781,16 +1779,16 @@ static int migrate_zspage(struct zs_pool *pool, struct size_class *class, unsigned long handle; struct page *s_page = cc->s_page; struct page *d_page = cc->d_page; - unsigned long index = cc->index; + int obj_idx = cc->obj_idx; int ret = 0; while (1) { - handle = find_alloced_obj(class, s_page, index); + handle = find_alloced_obj(class, s_page, &obj_idx); if (!handle) { s_page = get_next_page(s_page); if (!s_page) break; - index = 0; + obj_idx = 0; continue; } @@ -1804,7 +1802,7 @@ static int migrate_zspage(struct zs_pool *pool, struct size_class *class, used_obj = handle_to_obj(handle); free_obj = obj_malloc(class, get_zspage(d_page), handle); zs_object_copy(class, free_obj, used_obj); - index++; + obj_idx++; /* * record_obj updates handle's value to free_obj and it will * invalidate lock bit(ie, HANDLE_PIN_BIT) of handle, which @@ -1819,7 +1817,7 @@ static int migrate_zspage(struct zs_pool *pool, struct size_class *class, /* Remember last position in this iteration */ cc->s_page = s_page; - cc->index = index; + cc->obj_idx = obj_idx; return ret; } @@ -2102,7 +2100,7 @@ int zs_page_migrate(struct address_space *mapping, struct page *newpage, put_page(page); page = newpage; - ret = 0; + ret = MIGRATEPAGE_SUCCESS; unpin_objects: for (addr = s_addr + offset; addr < s_addr + pos; addr += class->size) { @@ -2256,8 +2254,7 @@ static unsigned long zs_can_compact(struct size_class *class) return 0; obj_wasted = obj_allocated - obj_used; - obj_wasted /= get_maxobj_per_zspage(class->size, - class->pages_per_zspage); + obj_wasted /= class->objs_per_zspage; return obj_wasted * class->pages_per_zspage; } @@ -2274,7 +2271,7 @@ static void __zs_compact(struct zs_pool *pool, struct size_class *class) if (!zs_can_compact(class)) break; - cc.index = 0; + cc.obj_idx = 0; cc.s_page = get_first_page(src_zspage); while ((dst_zspage = isolate_zspage(class, false))) { @@ -2393,7 +2390,7 @@ static int zs_register_shrinker(struct zs_pool *pool) /** * zs_create_pool - Creates an allocation pool to work from. - * @flags: allocation flags used to allocate pool metadata + * @name: pool name to be created * * This function must be called before anything when using * the zsmalloc allocator. @@ -2433,6 +2430,7 @@ struct zs_pool *zs_create_pool(const char *name) for (i = zs_size_classes - 1; i >= 0; i--) { int size; int pages_per_zspage; + int objs_per_zspage; struct size_class *class; int fullness = 0; @@ -2440,6 +2438,7 @@ struct zs_pool *zs_create_pool(const char *name) if (size > ZS_MAX_ALLOC_SIZE) size = ZS_MAX_ALLOC_SIZE; pages_per_zspage = get_pages_per_zspage(size); + objs_per_zspage = pages_per_zspage * PAGE_SIZE / size; /* * size_class is used for normal zsmalloc operation such @@ -2451,7 +2450,7 @@ struct zs_pool *zs_create_pool(const char *name) * previous size_class if possible. */ if (prev_class) { - if (can_merge(prev_class, size, pages_per_zspage)) { + if (can_merge(prev_class, pages_per_zspage, objs_per_zspage)) { pool->size_class[i] = prev_class; continue; } @@ -2464,8 +2463,7 @@ struct zs_pool *zs_create_pool(const char *name) class->size = size; class->index = i; class->pages_per_zspage = pages_per_zspage; - class->objs_per_zspage = class->pages_per_zspage * - PAGE_SIZE / class->size; + class->objs_per_zspage = objs_per_zspage; spin_lock_init(&class->lock); pool->size_class[i] = class; for (fullness = ZS_EMPTY; fullness < NR_ZS_FULLNESS;