zram: optimize memory operations with clear_page()/copy_page()

[deliverable/linux.git] / drivers / staging / zram / zram_drv.c

diff --git a/drivers/staging/zram/zram_drv.c b/drivers/staging/zram/zram_drv.c
index e34e3fe0ae2e61f12f7c167ba74588d663dee09a..4b30fe5816f0f80f608681406986b142aa4d8985 100644 (file)
--- a/drivers/staging/zram/zram_drv.c
+++ b/drivers/staging/zram/zram_drv.c
@@ -37,7 +37,7 @@
 
 /* Globals */
 static int zram_major;
-struct zram *zram_devices;
+static struct zram *zram_devices;
 
 /* Module params (documentation at end) */
 static unsigned int num_devices = 1;
@@ -128,23 +128,26 @@ static void zram_free_page(struct zram *zram, size_t index)
        meta->table[index].size = 0;
 }
 
+static inline int is_partial_io(struct bio_vec *bvec)
+{
+       return bvec->bv_len != PAGE_SIZE;
+}
+
 static void handle_zero_page(struct bio_vec *bvec)
 {
        struct page *page = bvec->bv_page;
        void *user_mem;
 
        user_mem = kmap_atomic(page);
-       memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
+       if (is_partial_io(bvec))
+               memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
+       else
+               clear_page(user_mem);
        kunmap_atomic(user_mem);
 
        flush_dcache_page(page);
 }
 
-static inline int is_partial_io(struct bio_vec *bvec)
-{
-       return bvec->bv_len != PAGE_SIZE;
-}
-
 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
 {
        int ret = LZO_E_OK;
@@ -154,13 +157,13 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
        unsigned long handle = meta->table[index].handle;
 
        if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
-               memset(mem, 0, PAGE_SIZE);
+               clear_page(mem);
                return 0;
        }
 
        cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
        if (meta->table[index].size == PAGE_SIZE)
-               memcpy(mem, cmem, PAGE_SIZE);
+               copy_page(mem, cmem);
        else
                ret = lzo1x_decompress_safe(cmem, meta->table[index].size,
                                                mem, &clen);
@@ -272,8 +275,6 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
 
        if (page_zero_filled(uncmem)) {
                kunmap_atomic(user_mem);
-               if (is_partial_io(bvec))
-                       kfree(uncmem);
                zram->stats.pages_zero++;
                zram_set_flag(meta, index, ZRAM_ZERO);
                ret = 0;
@@ -304,18 +305,20 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
 
        handle = zs_malloc(meta->mem_pool, clen);
        if (!handle) {
-               pr_info("Error allocating memory for compressed "
-                       "page: %u, size=%zu\n", index, clen);
+               pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
+                       index, clen);
                ret = -ENOMEM;
                goto out;
        }
        cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
 
-       if ((clen == PAGE_SIZE) && !is_partial_io(bvec))
+       if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
                src = kmap_atomic(page);
-       memcpy(cmem, src, clen);
-       if ((clen == PAGE_SIZE) && !is_partial_io(bvec))
+               copy_page(cmem, src);
                kunmap_atomic(src);
+       } else {
+               memcpy(cmem, src, clen);
+       }
 
        zs_unmap_object(meta->mem_pool, handle);
 
@@ -422,13 +425,20 @@ out:
  */
 static inline int valid_io_request(struct zram *zram, struct bio *bio)
 {
-       if (unlikely(
-               (bio->bi_sector >= (zram->disksize >> SECTOR_SHIFT)) ||
-               (bio->bi_sector & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)) ||
-               (bio->bi_size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))) {
+       u64 start, end, bound;
+       
+       /* unaligned request */
+       if (unlikely(bio->bi_sector & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
+               return 0;
+       if (unlikely(bio->bi_size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
+               return 0;
 
+       start = bio->bi_sector;
+       end = start + (bio->bi_size >> SECTOR_SHIFT);
+       bound = zram->disksize >> SECTOR_SHIFT;
+       /* out of range range */
+       if (unlikely(start >= bound || end >= bound || start > end))
                return 0;
-       }
 
        /* I/O request is valid */
        return 1;
@@ -582,7 +592,9 @@ static void zram_slot_free_notify(struct block_device *bdev,
        struct zram *zram;
 
        zram = bdev->bd_disk->private_data;
+       down_write(&zram->lock);
        zram_free_page(zram, index);
+       up_write(&zram->lock);
        zram_stat64_inc(zram, &zram->stats.notify_free);
 }
 
@@ -593,7 +605,7 @@ static const struct block_device_operations zram_devops = {
 
 static int create_device(struct zram *zram, int device_id)
 {
-       int ret = 0;
+       int ret = -ENOMEM;
 
        init_rwsem(&zram->lock);
        init_rwsem(&zram->init_lock);
@@ -603,7 +615,6 @@ static int create_device(struct zram *zram, int device_id)
        if (!zram->queue) {
                pr_err("Error allocating disk queue for device %d\n",
                        device_id);
-               ret = -ENOMEM;
                goto out;
        }
 
@@ -613,11 +624,9 @@ static int create_device(struct zram *zram, int device_id)
         /* gendisk structure */
        zram->disk = alloc_disk(1);
        if (!zram->disk) {
-               blk_cleanup_queue(zram->queue);
                pr_warn("Error allocating disk structure for device %d\n",
                        device_id);
-               ret = -ENOMEM;
-               goto out;
+               goto out_free_queue;
        }
 
        zram->disk->major = zram_major;
@@ -646,11 +655,17 @@ static int create_device(struct zram *zram, int device_id)
                                &zram_disk_attr_group);
        if (ret < 0) {
                pr_warn("Error creating sysfs group");
-               goto out;
+               goto out_free_disk;
        }
 
        zram->init_done = 0;
+       return 0;
 
+out_free_disk:
+       del_gendisk(zram->disk);
+       put_disk(zram->disk);
+out_free_queue:
+       blk_cleanup_queue(zram->queue);
 out:
        return ret;
 }
@@ -669,11 +684,6 @@ static void destroy_device(struct zram *zram)
                blk_cleanup_queue(zram->queue);
 }
 
-unsigned int zram_get_num_devices(void)
-{
-       return num_devices;
-}
-
 static int __init zram_init(void)
 {
        int ret, dev_id;
@@ -727,8 +737,10 @@ static void __exit zram_exit(void)
        for (i = 0; i < num_devices; i++) {
                zram = &zram_devices[i];
 
+               get_disk(zram->disk);
                destroy_device(zram);
                zram_reset_device(zram);
+               put_disk(zram->disk);
        }
 
        unregister_blkdev(zram_major, "zram");