2 * Compressed RAM block device
4 * Copyright (C) 2008, 2009, 2010 Nitin Gupta
6 * This code is released using a dual license strategy: BSD/GPL
7 * You can choose the licence that better fits your requirements.
9 * Released under the terms of 3-clause BSD License
10 * Released under the terms of GNU General Public License Version 2.0
12 * Project home: http://compcache.googlecode.com
15 #define KMSG_COMPONENT "zram"
16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
18 #include <linux/module.h>
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/bitops.h>
22 #include <linux/blkdev.h>
23 #include <linux/buffer_head.h>
24 #include <linux/device.h>
25 #include <linux/genhd.h>
26 #include <linux/highmem.h>
27 #include <linux/slab.h>
28 #include <linux/lzo.h>
29 #include <linux/string.h>
30 #include <linux/vmalloc.h>
35 static int zram_major
;
36 static struct zram
*devices
;
38 /* Module params (documentation at end) */
39 static unsigned int num_devices
;
41 static int zram_test_flag(struct zram
*zram
, u32 index
,
42 enum zram_pageflags flag
)
44 return zram
->table
[index
].flags
& BIT(flag
);
47 static void zram_set_flag(struct zram
*zram
, u32 index
,
48 enum zram_pageflags flag
)
50 zram
->table
[index
].flags
|= BIT(flag
);
53 static void zram_clear_flag(struct zram
*zram
, u32 index
,
54 enum zram_pageflags flag
)
56 zram
->table
[index
].flags
&= ~BIT(flag
);
59 static int page_zero_filled(void *ptr
)
64 page
= (unsigned long *)ptr
;
66 for (pos
= 0; pos
!= PAGE_SIZE
/ sizeof(*page
); pos
++) {
74 static void zram_set_disksize(struct zram
*zram
, size_t totalram_bytes
)
76 if (!zram
->disksize
) {
78 "disk size not provided. You can use disksize_kb module "
79 "param to specify size.\nUsing default: (%u%% of RAM).\n",
80 default_disksize_perc_ram
82 zram
->disksize
= default_disksize_perc_ram
*
83 (totalram_bytes
/ 100);
86 if (zram
->disksize
> 2 * (totalram_bytes
)) {
88 "There is little point creating a zram of greater than "
89 "twice the size of memory since we expect a 2:1 compression "
90 "ratio. Note that zram uses about 0.1%% of the size of "
91 "the disk when not in use so a huge zram is "
93 "\tMemory Size: %zu kB\n"
94 "\tSize you selected: %zu kB\n"
95 "Continuing anyway ...\n",
96 totalram_bytes
>> 10, zram
->disksize
100 zram
->disksize
&= PAGE_MASK
;
103 static void zram_ioctl_get_stats(struct zram
*zram
,
104 struct zram_ioctl_stats
*s
)
106 s
->disksize
= zram
->disksize
;
108 #if defined(CONFIG_ZRAM_STATS)
110 struct zram_stats
*rs
= &zram
->stats
;
111 size_t succ_writes
, mem_used
;
112 unsigned int good_compress_perc
= 0, no_compress_perc
= 0;
114 mem_used
= xv_get_total_size_bytes(zram
->mem_pool
)
115 + (rs
->pages_expand
<< PAGE_SHIFT
);
116 succ_writes
= zram_stat64_read(zram
, &rs
->num_writes
) -
117 zram_stat64_read(zram
, &rs
->failed_writes
);
119 if (succ_writes
&& rs
->pages_stored
) {
120 good_compress_perc
= rs
->good_compress
* 100
122 no_compress_perc
= rs
->pages_expand
* 100
126 s
->num_reads
= zram_stat64_read(zram
, &rs
->num_reads
);
127 s
->num_writes
= zram_stat64_read(zram
, &rs
->num_writes
);
128 s
->failed_reads
= zram_stat64_read(zram
, &rs
->failed_reads
);
129 s
->failed_writes
= zram_stat64_read(zram
, &rs
->failed_writes
);
130 s
->invalid_io
= zram_stat64_read(zram
, &rs
->invalid_io
);
131 s
->notify_free
= zram_stat64_read(zram
, &rs
->notify_free
);
132 s
->pages_zero
= rs
->pages_zero
;
134 s
->good_compress_pct
= good_compress_perc
;
135 s
->pages_expand_pct
= no_compress_perc
;
137 s
->pages_stored
= rs
->pages_stored
;
138 s
->pages_used
= mem_used
>> PAGE_SHIFT
;
139 s
->orig_data_size
= rs
->pages_stored
<< PAGE_SHIFT
;
140 s
->compr_data_size
= rs
->compr_size
;
141 s
->mem_used_total
= mem_used
;
143 #endif /* CONFIG_ZRAM_STATS */
146 static void zram_free_page(struct zram
*zram
, size_t index
)
151 struct page
*page
= zram
->table
[index
].page
;
152 u32 offset
= zram
->table
[index
].offset
;
154 if (unlikely(!page
)) {
156 * No memory is allocated for zero filled pages.
157 * Simply clear zero page flag.
159 if (zram_test_flag(zram
, index
, ZRAM_ZERO
)) {
160 zram_clear_flag(zram
, index
, ZRAM_ZERO
);
161 zram_stat_dec(&zram
->stats
.pages_zero
);
166 if (unlikely(zram_test_flag(zram
, index
, ZRAM_UNCOMPRESSED
))) {
169 zram_clear_flag(zram
, index
, ZRAM_UNCOMPRESSED
);
170 zram_stat_dec(&zram
->stats
.pages_expand
);
174 obj
= kmap_atomic(page
, KM_USER0
) + offset
;
175 clen
= xv_get_object_size(obj
) - sizeof(struct zobj_header
);
176 kunmap_atomic(obj
, KM_USER0
);
178 xv_free(zram
->mem_pool
, page
, offset
);
179 if (clen
<= PAGE_SIZE
/ 2)
180 zram_stat_dec(&zram
->stats
.good_compress
);
183 zram
->stats
.compr_size
-= clen
;
184 zram_stat_dec(&zram
->stats
.pages_stored
);
186 zram
->table
[index
].page
= NULL
;
187 zram
->table
[index
].offset
= 0;
190 static void handle_zero_page(struct page
*page
)
194 user_mem
= kmap_atomic(page
, KM_USER0
);
195 memset(user_mem
, 0, PAGE_SIZE
);
196 kunmap_atomic(user_mem
, KM_USER0
);
198 flush_dcache_page(page
);
201 static void handle_uncompressed_page(struct zram
*zram
,
202 struct page
*page
, u32 index
)
204 unsigned char *user_mem
, *cmem
;
206 user_mem
= kmap_atomic(page
, KM_USER0
);
207 cmem
= kmap_atomic(zram
->table
[index
].page
, KM_USER1
) +
208 zram
->table
[index
].offset
;
210 memcpy(user_mem
, cmem
, PAGE_SIZE
);
211 kunmap_atomic(user_mem
, KM_USER0
);
212 kunmap_atomic(cmem
, KM_USER1
);
214 flush_dcache_page(page
);
217 static int zram_read(struct zram
*zram
, struct bio
*bio
)
222 struct bio_vec
*bvec
;
224 zram_stat64_inc(zram
, &zram
->stats
.num_reads
);
226 index
= bio
->bi_sector
>> SECTORS_PER_PAGE_SHIFT
;
227 bio_for_each_segment(bvec
, bio
, i
) {
231 struct zobj_header
*zheader
;
232 unsigned char *user_mem
, *cmem
;
234 page
= bvec
->bv_page
;
236 if (zram_test_flag(zram
, index
, ZRAM_ZERO
)) {
237 handle_zero_page(page
);
241 /* Requested page is not present in compressed area */
242 if (unlikely(!zram
->table
[index
].page
)) {
243 pr_debug("Read before write: sector=%lu, size=%u",
244 (ulong
)(bio
->bi_sector
), bio
->bi_size
);
249 /* Page is stored uncompressed since it's incompressible */
250 if (unlikely(zram_test_flag(zram
, index
, ZRAM_UNCOMPRESSED
))) {
251 handle_uncompressed_page(zram
, page
, index
);
255 user_mem
= kmap_atomic(page
, KM_USER0
);
258 cmem
= kmap_atomic(zram
->table
[index
].page
, KM_USER1
) +
259 zram
->table
[index
].offset
;
261 ret
= lzo1x_decompress_safe(
262 cmem
+ sizeof(*zheader
),
263 xv_get_object_size(cmem
) - sizeof(*zheader
),
266 kunmap_atomic(user_mem
, KM_USER0
);
267 kunmap_atomic(cmem
, KM_USER1
);
269 /* Should NEVER happen. Return bio error if it does. */
270 if (unlikely(ret
!= LZO_E_OK
)) {
271 pr_err("Decompression failed! err=%d, page=%u\n",
273 zram_stat64_inc(zram
, &zram
->stats
.failed_reads
);
277 flush_dcache_page(page
);
281 set_bit(BIO_UPTODATE
, &bio
->bi_flags
);
290 static int zram_write(struct zram
*zram
, struct bio
*bio
)
294 struct bio_vec
*bvec
;
296 zram_stat64_inc(zram
, &zram
->stats
.num_writes
);
298 index
= bio
->bi_sector
>> SECTORS_PER_PAGE_SHIFT
;
300 bio_for_each_segment(bvec
, bio
, i
) {
304 struct zobj_header
*zheader
;
305 struct page
*page
, *page_store
;
306 unsigned char *user_mem
, *cmem
, *src
;
308 page
= bvec
->bv_page
;
309 src
= zram
->compress_buffer
;
312 * System overwrites unused sectors. Free memory associated
313 * with this sector now.
315 if (zram
->table
[index
].page
||
316 zram_test_flag(zram
, index
, ZRAM_ZERO
))
317 zram_free_page(zram
, index
);
319 mutex_lock(&zram
->lock
);
321 user_mem
= kmap_atomic(page
, KM_USER0
);
322 if (page_zero_filled(user_mem
)) {
323 kunmap_atomic(user_mem
, KM_USER0
);
324 mutex_unlock(&zram
->lock
);
325 zram_stat_inc(&zram
->stats
.pages_zero
);
326 zram_set_flag(zram
, index
, ZRAM_ZERO
);
330 ret
= lzo1x_1_compress(user_mem
, PAGE_SIZE
, src
, &clen
,
331 zram
->compress_workmem
);
333 kunmap_atomic(user_mem
, KM_USER0
);
335 if (unlikely(ret
!= LZO_E_OK
)) {
336 mutex_unlock(&zram
->lock
);
337 pr_err("Compression failed! err=%d\n", ret
);
338 zram_stat64_inc(zram
, &zram
->stats
.failed_writes
);
343 * Page is incompressible. Store it as-is (uncompressed)
344 * since we do not want to return too many disk write
345 * errors which has side effect of hanging the system.
347 if (unlikely(clen
> max_zpage_size
)) {
349 page_store
= alloc_page(GFP_NOIO
| __GFP_HIGHMEM
);
350 if (unlikely(!page_store
)) {
351 mutex_unlock(&zram
->lock
);
352 pr_info("Error allocating memory for "
353 "incompressible page: %u\n", index
);
354 zram_stat64_inc(zram
,
355 &zram
->stats
.failed_writes
);
360 zram_set_flag(zram
, index
, ZRAM_UNCOMPRESSED
);
361 zram_stat_inc(&zram
->stats
.pages_expand
);
362 zram
->table
[index
].page
= page_store
;
363 src
= kmap_atomic(page
, KM_USER0
);
367 if (xv_malloc(zram
->mem_pool
, clen
+ sizeof(*zheader
),
368 &zram
->table
[index
].page
, &offset
,
369 GFP_NOIO
| __GFP_HIGHMEM
)) {
370 mutex_unlock(&zram
->lock
);
371 pr_info("Error allocating memory for compressed "
372 "page: %u, size=%zu\n", index
, clen
);
373 zram_stat64_inc(zram
, &zram
->stats
.failed_writes
);
378 zram
->table
[index
].offset
= offset
;
380 cmem
= kmap_atomic(zram
->table
[index
].page
, KM_USER1
) +
381 zram
->table
[index
].offset
;
384 /* Back-reference needed for memory defragmentation */
385 if (!zram_test_flag(zram
, index
, ZRAM_UNCOMPRESSED
)) {
386 zheader
= (struct zobj_header
*)cmem
;
387 zheader
->table_idx
= index
;
388 cmem
+= sizeof(*zheader
);
392 memcpy(cmem
, src
, clen
);
394 kunmap_atomic(cmem
, KM_USER1
);
395 if (unlikely(zram_test_flag(zram
, index
, ZRAM_UNCOMPRESSED
)))
396 kunmap_atomic(src
, KM_USER0
);
399 zram
->stats
.compr_size
+= clen
;
400 zram_stat_inc(&zram
->stats
.pages_stored
);
401 if (clen
<= PAGE_SIZE
/ 2)
402 zram_stat_inc(&zram
->stats
.good_compress
);
404 mutex_unlock(&zram
->lock
);
408 set_bit(BIO_UPTODATE
, &bio
->bi_flags
);
418 * Check if request is within bounds and page aligned.
420 static inline int valid_io_request(struct zram
*zram
, struct bio
*bio
)
423 (bio
->bi_sector
>= (zram
->disksize
>> SECTOR_SHIFT
)) ||
424 (bio
->bi_sector
& (SECTORS_PER_PAGE
- 1)) ||
425 (bio
->bi_size
& (PAGE_SIZE
- 1)))) {
430 /* I/O request is valid */
435 * Handler function for all zram I/O requests.
437 static int zram_make_request(struct request_queue
*queue
, struct bio
*bio
)
440 struct zram
*zram
= queue
->queuedata
;
442 if (unlikely(!zram
->init_done
)) {
447 if (!valid_io_request(zram
, bio
)) {
448 zram_stat64_inc(zram
, &zram
->stats
.invalid_io
);
453 switch (bio_data_dir(bio
)) {
455 ret
= zram_read(zram
, bio
);
459 ret
= zram_write(zram
, bio
);
466 static void reset_device(struct zram
*zram
)
470 /* Do not accept any new I/O request */
473 /* Free various per-device buffers */
474 kfree(zram
->compress_workmem
);
475 free_pages((unsigned long)zram
->compress_buffer
, 1);
477 zram
->compress_workmem
= NULL
;
478 zram
->compress_buffer
= NULL
;
480 /* Free all pages that are still in this zram device */
481 for (index
= 0; index
< zram
->disksize
>> PAGE_SHIFT
; index
++) {
485 page
= zram
->table
[index
].page
;
486 offset
= zram
->table
[index
].offset
;
491 if (unlikely(zram_test_flag(zram
, index
, ZRAM_UNCOMPRESSED
)))
494 xv_free(zram
->mem_pool
, page
, offset
);
500 xv_destroy_pool(zram
->mem_pool
);
501 zram
->mem_pool
= NULL
;
504 memset(&zram
->stats
, 0, sizeof(zram
->stats
));
509 static int zram_ioctl_init_device(struct zram
*zram
)
514 if (zram
->init_done
) {
515 pr_info("Device already initialized!\n");
519 zram_set_disksize(zram
, totalram_pages
<< PAGE_SHIFT
);
521 zram
->compress_workmem
= kzalloc(LZO1X_MEM_COMPRESS
, GFP_KERNEL
);
522 if (!zram
->compress_workmem
) {
523 pr_err("Error allocating compressor working memory!\n");
528 zram
->compress_buffer
= (void *)__get_free_pages(__GFP_ZERO
, 1);
529 if (!zram
->compress_buffer
) {
530 pr_err("Error allocating compressor buffer space\n");
535 num_pages
= zram
->disksize
>> PAGE_SHIFT
;
536 zram
->table
= vmalloc(num_pages
* sizeof(*zram
->table
));
538 pr_err("Error allocating zram address table\n");
539 /* To prevent accessing table entries during cleanup */
544 memset(zram
->table
, 0, num_pages
* sizeof(*zram
->table
));
546 set_capacity(zram
->disk
, zram
->disksize
>> SECTOR_SHIFT
);
548 /* zram devices sort of resembles non-rotational disks */
549 queue_flag_set_unlocked(QUEUE_FLAG_NONROT
, zram
->disk
->queue
);
551 zram
->mem_pool
= xv_create_pool();
552 if (!zram
->mem_pool
) {
553 pr_err("Error creating memory pool\n");
560 pr_debug("Initialization done!\n");
566 pr_err("Initialization failed: err=%d\n", ret
);
570 static int zram_ioctl_reset_device(struct zram
*zram
)
578 static int zram_ioctl(struct block_device
*bdev
, fmode_t mode
,
579 unsigned int cmd
, unsigned long arg
)
584 struct zram
*zram
= bdev
->bd_disk
->private_data
;
587 case ZRAMIO_SET_DISKSIZE_KB
:
588 if (zram
->init_done
) {
592 if (copy_from_user(&disksize_kb
, (void *)arg
,
597 zram
->disksize
= disksize_kb
<< 10;
598 pr_info("Disk size set to %zu kB\n", disksize_kb
);
601 case ZRAMIO_GET_STATS
:
603 struct zram_ioctl_stats
*stats
;
604 if (!zram
->init_done
) {
608 stats
= kzalloc(sizeof(*stats
), GFP_KERNEL
);
613 zram_ioctl_get_stats(zram
, stats
);
614 if (copy_to_user((void *)arg
, stats
, sizeof(*stats
))) {
623 ret
= zram_ioctl_init_device(zram
);
627 /* Do not reset an active device! */
628 if (bdev
->bd_holders
) {
633 /* Make sure all pending I/O is finished */
637 ret
= zram_ioctl_reset_device(zram
);
641 pr_info("Invalid ioctl %u\n", cmd
);
649 void zram_slot_free_notify(struct block_device
*bdev
, unsigned long index
)
653 zram
= bdev
->bd_disk
->private_data
;
654 zram_free_page(zram
, index
);
655 zram_stat64_inc(zram
, &zram
->stats
.notify_free
);
658 static const struct block_device_operations zram_devops
= {
660 .swap_slot_free_notify
= zram_slot_free_notify
,
664 static int create_device(struct zram
*zram
, int device_id
)
668 mutex_init(&zram
->lock
);
669 spin_lock_init(&zram
->stat64_lock
);
671 zram
->queue
= blk_alloc_queue(GFP_KERNEL
);
673 pr_err("Error allocating disk queue for device %d\n",
679 blk_queue_make_request(zram
->queue
, zram_make_request
);
680 zram
->queue
->queuedata
= zram
;
682 /* gendisk structure */
683 zram
->disk
= alloc_disk(1);
685 blk_cleanup_queue(zram
->queue
);
686 pr_warning("Error allocating disk structure for device %d\n",
692 zram
->disk
->major
= zram_major
;
693 zram
->disk
->first_minor
= device_id
;
694 zram
->disk
->fops
= &zram_devops
;
695 zram
->disk
->queue
= zram
->queue
;
696 zram
->disk
->private_data
= zram
;
697 snprintf(zram
->disk
->disk_name
, 16, "zram%d", device_id
);
699 /* Actual capacity set using ZRAMIO_SET_DISKSIZE_KB ioctl */
700 set_capacity(zram
->disk
, 0);
703 * To ensure that we always get PAGE_SIZE aligned
704 * and n*PAGE_SIZED sized I/O requests.
706 blk_queue_physical_block_size(zram
->disk
->queue
, PAGE_SIZE
);
707 blk_queue_logical_block_size(zram
->disk
->queue
, PAGE_SIZE
);
708 blk_queue_io_min(zram
->disk
->queue
, PAGE_SIZE
);
709 blk_queue_io_opt(zram
->disk
->queue
, PAGE_SIZE
);
711 add_disk(zram
->disk
);
719 static void destroy_device(struct zram
*zram
)
722 del_gendisk(zram
->disk
);
723 put_disk(zram
->disk
);
727 blk_cleanup_queue(zram
->queue
);
730 static int __init
zram_init(void)
734 if (num_devices
> max_num_devices
) {
735 pr_warning("Invalid value for num_devices: %u\n",
741 zram_major
= register_blkdev(0, "zram");
742 if (zram_major
<= 0) {
743 pr_warning("Unable to get major number\n");
749 pr_info("num_devices not specified. Using default: 1\n");
753 /* Allocate the device array and initialize each one */
754 pr_info("Creating %u devices ...\n", num_devices
);
755 devices
= kzalloc(num_devices
* sizeof(struct zram
), GFP_KERNEL
);
761 for (dev_id
= 0; dev_id
< num_devices
; dev_id
++) {
762 ret
= create_device(&devices
[dev_id
], dev_id
);
771 destroy_device(&devices
[--dev_id
]);
774 unregister_blkdev(zram_major
, "zram");
779 static void __exit
zram_exit(void)
784 for (i
= 0; i
< num_devices
; i
++) {
787 destroy_device(zram
);
792 unregister_blkdev(zram_major
, "zram");
795 pr_debug("Cleanup done!\n");
798 module_param(num_devices
, uint
, 0);
799 MODULE_PARM_DESC(num_devices
, "Number of zram devices");
801 module_init(zram_init
);
802 module_exit(zram_exit
);
804 MODULE_LICENSE("Dual BSD/GPL");
805 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
806 MODULE_DESCRIPTION("Compressed RAM Block Device");