Commit | Line | Data |
---|---|---|
306b0c95 | 1 | /* |
f1e3cfff | 2 | * Compressed RAM block device |
306b0c95 | 3 | * |
1130ebba | 4 | * Copyright (C) 2008, 2009, 2010 Nitin Gupta |
306b0c95 NG |
5 | * |
6 | * This code is released using a dual license strategy: BSD/GPL | |
7 | * You can choose the licence that better fits your requirements. | |
8 | * | |
9 | * Released under the terms of 3-clause BSD License | |
10 | * Released under the terms of GNU General Public License Version 2.0 | |
11 | * | |
12 | * Project home: http://compcache.googlecode.com | |
13 | */ | |
14 | ||
f1e3cfff | 15 | #define KMSG_COMPONENT "zram" |
306b0c95 NG |
16 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt |
17 | ||
18 | #include <linux/module.h> | |
19 | #include <linux/kernel.h> | |
8946a086 | 20 | #include <linux/bio.h> |
306b0c95 NG |
21 | #include <linux/bitops.h> |
22 | #include <linux/blkdev.h> | |
23 | #include <linux/buffer_head.h> | |
24 | #include <linux/device.h> | |
25 | #include <linux/genhd.h> | |
26 | #include <linux/highmem.h> | |
5a0e3ad6 | 27 | #include <linux/slab.h> |
306b0c95 | 28 | #include <linux/lzo.h> |
306b0c95 | 29 | #include <linux/string.h> |
306b0c95 | 30 | #include <linux/vmalloc.h> |
306b0c95 | 31 | |
16a4bfb9 | 32 | #include "zram_drv.h" |
306b0c95 NG |
33 | |
34 | /* Globals */ | |
f1e3cfff NG |
35 | static int zram_major; |
36 | static struct zram *devices; | |
306b0c95 | 37 | |
306b0c95 NG |
38 | /* Module params (documentation at end) */ |
39 | static unsigned int num_devices; | |
40 | ||
f1e3cfff NG |
41 | static int zram_test_flag(struct zram *zram, u32 index, |
42 | enum zram_pageflags flag) | |
306b0c95 | 43 | { |
f1e3cfff | 44 | return zram->table[index].flags & BIT(flag); |
306b0c95 NG |
45 | } |
46 | ||
f1e3cfff NG |
47 | static void zram_set_flag(struct zram *zram, u32 index, |
48 | enum zram_pageflags flag) | |
306b0c95 | 49 | { |
f1e3cfff | 50 | zram->table[index].flags |= BIT(flag); |
306b0c95 NG |
51 | } |
52 | ||
f1e3cfff NG |
53 | static void zram_clear_flag(struct zram *zram, u32 index, |
54 | enum zram_pageflags flag) | |
306b0c95 | 55 | { |
f1e3cfff | 56 | zram->table[index].flags &= ~BIT(flag); |
306b0c95 NG |
57 | } |
58 | ||
59 | static int page_zero_filled(void *ptr) | |
60 | { | |
61 | unsigned int pos; | |
62 | unsigned long *page; | |
63 | ||
64 | page = (unsigned long *)ptr; | |
65 | ||
66 | for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) { | |
67 | if (page[pos]) | |
68 | return 0; | |
69 | } | |
70 | ||
71 | return 1; | |
72 | } | |
73 | ||
f1e3cfff | 74 | static void zram_set_disksize(struct zram *zram, size_t totalram_bytes) |
306b0c95 | 75 | { |
f1e3cfff | 76 | if (!zram->disksize) { |
306b0c95 NG |
77 | pr_info( |
78 | "disk size not provided. You can use disksize_kb module " | |
79 | "param to specify size.\nUsing default: (%u%% of RAM).\n", | |
80 | default_disksize_perc_ram | |
81 | ); | |
f1e3cfff | 82 | zram->disksize = default_disksize_perc_ram * |
306b0c95 NG |
83 | (totalram_bytes / 100); |
84 | } | |
85 | ||
f1e3cfff | 86 | if (zram->disksize > 2 * (totalram_bytes)) { |
306b0c95 | 87 | pr_info( |
f1e3cfff | 88 | "There is little point creating a zram of greater than " |
306b0c95 | 89 | "twice the size of memory since we expect a 2:1 compression " |
f1e3cfff NG |
90 | "ratio. Note that zram uses about 0.1%% of the size of " |
91 | "the disk when not in use so a huge zram is " | |
306b0c95 NG |
92 | "wasteful.\n" |
93 | "\tMemory Size: %zu kB\n" | |
94 | "\tSize you selected: %zu kB\n" | |
95 | "Continuing anyway ...\n", | |
f1e3cfff | 96 | totalram_bytes >> 10, zram->disksize |
306b0c95 NG |
97 | ); |
98 | } | |
99 | ||
f1e3cfff | 100 | zram->disksize &= PAGE_MASK; |
306b0c95 NG |
101 | } |
102 | ||
f1e3cfff NG |
103 | static void zram_ioctl_get_stats(struct zram *zram, |
104 | struct zram_ioctl_stats *s) | |
306b0c95 | 105 | { |
f1e3cfff | 106 | s->disksize = zram->disksize; |
306b0c95 | 107 | |
f1e3cfff | 108 | #if defined(CONFIG_ZRAM_STATS) |
306b0c95 | 109 | { |
f1e3cfff | 110 | struct zram_stats *rs = &zram->stats; |
306b0c95 NG |
111 | size_t succ_writes, mem_used; |
112 | unsigned int good_compress_perc = 0, no_compress_perc = 0; | |
113 | ||
f1e3cfff | 114 | mem_used = xv_get_total_size_bytes(zram->mem_pool) |
306b0c95 | 115 | + (rs->pages_expand << PAGE_SHIFT); |
f1e3cfff NG |
116 | succ_writes = zram_stat64_read(zram, &rs->num_writes) - |
117 | zram_stat64_read(zram, &rs->failed_writes); | |
306b0c95 NG |
118 | |
119 | if (succ_writes && rs->pages_stored) { | |
120 | good_compress_perc = rs->good_compress * 100 | |
121 | / rs->pages_stored; | |
122 | no_compress_perc = rs->pages_expand * 100 | |
123 | / rs->pages_stored; | |
124 | } | |
125 | ||
f1e3cfff NG |
126 | s->num_reads = zram_stat64_read(zram, &rs->num_reads); |
127 | s->num_writes = zram_stat64_read(zram, &rs->num_writes); | |
128 | s->failed_reads = zram_stat64_read(zram, &rs->failed_reads); | |
129 | s->failed_writes = zram_stat64_read(zram, &rs->failed_writes); | |
130 | s->invalid_io = zram_stat64_read(zram, &rs->invalid_io); | |
131 | s->notify_free = zram_stat64_read(zram, &rs->notify_free); | |
306b0c95 NG |
132 | s->pages_zero = rs->pages_zero; |
133 | ||
134 | s->good_compress_pct = good_compress_perc; | |
135 | s->pages_expand_pct = no_compress_perc; | |
136 | ||
137 | s->pages_stored = rs->pages_stored; | |
138 | s->pages_used = mem_used >> PAGE_SHIFT; | |
139 | s->orig_data_size = rs->pages_stored << PAGE_SHIFT; | |
140 | s->compr_data_size = rs->compr_size; | |
141 | s->mem_used_total = mem_used; | |
306b0c95 | 142 | } |
f1e3cfff | 143 | #endif /* CONFIG_ZRAM_STATS */ |
306b0c95 NG |
144 | } |
145 | ||
f1e3cfff | 146 | static void zram_free_page(struct zram *zram, size_t index) |
306b0c95 NG |
147 | { |
148 | u32 clen; | |
149 | void *obj; | |
150 | ||
f1e3cfff NG |
151 | struct page *page = zram->table[index].page; |
152 | u32 offset = zram->table[index].offset; | |
306b0c95 NG |
153 | |
154 | if (unlikely(!page)) { | |
2e882281 NG |
155 | /* |
156 | * No memory is allocated for zero filled pages. | |
157 | * Simply clear zero page flag. | |
158 | */ | |
f1e3cfff NG |
159 | if (zram_test_flag(zram, index, ZRAM_ZERO)) { |
160 | zram_clear_flag(zram, index, ZRAM_ZERO); | |
161 | zram_stat_dec(&zram->stats.pages_zero); | |
306b0c95 NG |
162 | } |
163 | return; | |
164 | } | |
165 | ||
f1e3cfff | 166 | if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { |
306b0c95 NG |
167 | clen = PAGE_SIZE; |
168 | __free_page(page); | |
f1e3cfff NG |
169 | zram_clear_flag(zram, index, ZRAM_UNCOMPRESSED); |
170 | zram_stat_dec(&zram->stats.pages_expand); | |
306b0c95 NG |
171 | goto out; |
172 | } | |
173 | ||
174 | obj = kmap_atomic(page, KM_USER0) + offset; | |
175 | clen = xv_get_object_size(obj) - sizeof(struct zobj_header); | |
176 | kunmap_atomic(obj, KM_USER0); | |
177 | ||
f1e3cfff | 178 | xv_free(zram->mem_pool, page, offset); |
306b0c95 | 179 | if (clen <= PAGE_SIZE / 2) |
f1e3cfff | 180 | zram_stat_dec(&zram->stats.good_compress); |
306b0c95 NG |
181 | |
182 | out: | |
f1e3cfff NG |
183 | zram->stats.compr_size -= clen; |
184 | zram_stat_dec(&zram->stats.pages_stored); | |
306b0c95 | 185 | |
f1e3cfff NG |
186 | zram->table[index].page = NULL; |
187 | zram->table[index].offset = 0; | |
306b0c95 NG |
188 | } |
189 | ||
a1dd52af | 190 | static void handle_zero_page(struct page *page) |
306b0c95 NG |
191 | { |
192 | void *user_mem; | |
306b0c95 NG |
193 | |
194 | user_mem = kmap_atomic(page, KM_USER0); | |
195 | memset(user_mem, 0, PAGE_SIZE); | |
196 | kunmap_atomic(user_mem, KM_USER0); | |
197 | ||
30fb8a71 | 198 | flush_dcache_page(page); |
306b0c95 NG |
199 | } |
200 | ||
f1e3cfff | 201 | static void handle_uncompressed_page(struct zram *zram, |
a1dd52af | 202 | struct page *page, u32 index) |
306b0c95 | 203 | { |
306b0c95 NG |
204 | unsigned char *user_mem, *cmem; |
205 | ||
306b0c95 | 206 | user_mem = kmap_atomic(page, KM_USER0); |
f1e3cfff NG |
207 | cmem = kmap_atomic(zram->table[index].page, KM_USER1) + |
208 | zram->table[index].offset; | |
306b0c95 NG |
209 | |
210 | memcpy(user_mem, cmem, PAGE_SIZE); | |
211 | kunmap_atomic(user_mem, KM_USER0); | |
212 | kunmap_atomic(cmem, KM_USER1); | |
213 | ||
30fb8a71 | 214 | flush_dcache_page(page); |
306b0c95 NG |
215 | } |
216 | ||
f1e3cfff | 217 | static int zram_read(struct zram *zram, struct bio *bio) |
306b0c95 | 218 | { |
a1dd52af NG |
219 | |
220 | int i; | |
306b0c95 | 221 | u32 index; |
a1dd52af | 222 | struct bio_vec *bvec; |
306b0c95 | 223 | |
f1e3cfff | 224 | zram_stat64_inc(zram, &zram->stats.num_reads); |
306b0c95 | 225 | |
306b0c95 | 226 | index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT; |
a1dd52af NG |
227 | bio_for_each_segment(bvec, bio, i) { |
228 | int ret; | |
229 | size_t clen; | |
230 | struct page *page; | |
231 | struct zobj_header *zheader; | |
232 | unsigned char *user_mem, *cmem; | |
306b0c95 | 233 | |
a1dd52af | 234 | page = bvec->bv_page; |
306b0c95 | 235 | |
f1e3cfff | 236 | if (zram_test_flag(zram, index, ZRAM_ZERO)) { |
a1dd52af NG |
237 | handle_zero_page(page); |
238 | continue; | |
239 | } | |
306b0c95 | 240 | |
a1dd52af | 241 | /* Requested page is not present in compressed area */ |
f1e3cfff | 242 | if (unlikely(!zram->table[index].page)) { |
a1dd52af NG |
243 | pr_debug("Read before write: sector=%lu, size=%u", |
244 | (ulong)(bio->bi_sector), bio->bi_size); | |
245 | /* Do nothing */ | |
246 | continue; | |
247 | } | |
306b0c95 | 248 | |
a1dd52af | 249 | /* Page is stored uncompressed since it's incompressible */ |
f1e3cfff NG |
250 | if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { |
251 | handle_uncompressed_page(zram, page, index); | |
a1dd52af NG |
252 | continue; |
253 | } | |
306b0c95 | 254 | |
a1dd52af NG |
255 | user_mem = kmap_atomic(page, KM_USER0); |
256 | clen = PAGE_SIZE; | |
306b0c95 | 257 | |
f1e3cfff NG |
258 | cmem = kmap_atomic(zram->table[index].page, KM_USER1) + |
259 | zram->table[index].offset; | |
306b0c95 | 260 | |
a1dd52af NG |
261 | ret = lzo1x_decompress_safe( |
262 | cmem + sizeof(*zheader), | |
263 | xv_get_object_size(cmem) - sizeof(*zheader), | |
264 | user_mem, &clen); | |
306b0c95 | 265 | |
a1dd52af NG |
266 | kunmap_atomic(user_mem, KM_USER0); |
267 | kunmap_atomic(cmem, KM_USER1); | |
306b0c95 | 268 | |
a1dd52af NG |
269 | /* Should NEVER happen. Return bio error if it does. */ |
270 | if (unlikely(ret != LZO_E_OK)) { | |
271 | pr_err("Decompression failed! err=%d, page=%u\n", | |
272 | ret, index); | |
f1e3cfff | 273 | zram_stat64_inc(zram, &zram->stats.failed_reads); |
a1dd52af NG |
274 | goto out; |
275 | } | |
276 | ||
277 | flush_dcache_page(page); | |
278 | index++; | |
279 | } | |
306b0c95 NG |
280 | |
281 | set_bit(BIO_UPTODATE, &bio->bi_flags); | |
282 | bio_endio(bio, 0); | |
283 | return 0; | |
284 | ||
285 | out: | |
286 | bio_io_error(bio); | |
287 | return 0; | |
288 | } | |
289 | ||
f1e3cfff | 290 | static int zram_write(struct zram *zram, struct bio *bio) |
306b0c95 | 291 | { |
a1dd52af NG |
292 | int i; |
293 | u32 index; | |
294 | struct bio_vec *bvec; | |
306b0c95 | 295 | |
f1e3cfff | 296 | zram_stat64_inc(zram, &zram->stats.num_writes); |
306b0c95 | 297 | |
306b0c95 NG |
298 | index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT; |
299 | ||
a1dd52af NG |
300 | bio_for_each_segment(bvec, bio, i) { |
301 | int ret; | |
302 | u32 offset; | |
303 | size_t clen; | |
304 | struct zobj_header *zheader; | |
305 | struct page *page, *page_store; | |
306 | unsigned char *user_mem, *cmem, *src; | |
306b0c95 | 307 | |
a1dd52af | 308 | page = bvec->bv_page; |
f1e3cfff | 309 | src = zram->compress_buffer; |
306b0c95 | 310 | |
a1dd52af NG |
311 | /* |
312 | * System overwrites unused sectors. Free memory associated | |
313 | * with this sector now. | |
314 | */ | |
f1e3cfff NG |
315 | if (zram->table[index].page || |
316 | zram_test_flag(zram, index, ZRAM_ZERO)) | |
317 | zram_free_page(zram, index); | |
306b0c95 | 318 | |
f1e3cfff | 319 | mutex_lock(&zram->lock); |
306b0c95 | 320 | |
a1dd52af NG |
321 | user_mem = kmap_atomic(page, KM_USER0); |
322 | if (page_zero_filled(user_mem)) { | |
323 | kunmap_atomic(user_mem, KM_USER0); | |
f1e3cfff NG |
324 | mutex_unlock(&zram->lock); |
325 | zram_stat_inc(&zram->stats.pages_zero); | |
326 | zram_set_flag(zram, index, ZRAM_ZERO); | |
a1dd52af NG |
327 | continue; |
328 | } | |
306b0c95 | 329 | |
a1dd52af | 330 | ret = lzo1x_1_compress(user_mem, PAGE_SIZE, src, &clen, |
f1e3cfff | 331 | zram->compress_workmem); |
306b0c95 | 332 | |
a1dd52af | 333 | kunmap_atomic(user_mem, KM_USER0); |
306b0c95 | 334 | |
a1dd52af | 335 | if (unlikely(ret != LZO_E_OK)) { |
f1e3cfff | 336 | mutex_unlock(&zram->lock); |
a1dd52af | 337 | pr_err("Compression failed! err=%d\n", ret); |
f1e3cfff | 338 | zram_stat64_inc(zram, &zram->stats.failed_writes); |
306b0c95 NG |
339 | goto out; |
340 | } | |
341 | ||
a1dd52af NG |
342 | /* |
343 | * Page is incompressible. Store it as-is (uncompressed) | |
f1e3cfff | 344 | * since we do not want to return too many disk write |
a1dd52af NG |
345 | * errors which has side effect of hanging the system. |
346 | */ | |
347 | if (unlikely(clen > max_zpage_size)) { | |
348 | clen = PAGE_SIZE; | |
349 | page_store = alloc_page(GFP_NOIO | __GFP_HIGHMEM); | |
350 | if (unlikely(!page_store)) { | |
f1e3cfff | 351 | mutex_unlock(&zram->lock); |
a1dd52af NG |
352 | pr_info("Error allocating memory for " |
353 | "incompressible page: %u\n", index); | |
f1e3cfff NG |
354 | zram_stat64_inc(zram, |
355 | &zram->stats.failed_writes); | |
a1dd52af NG |
356 | goto out; |
357 | } | |
358 | ||
359 | offset = 0; | |
f1e3cfff NG |
360 | zram_set_flag(zram, index, ZRAM_UNCOMPRESSED); |
361 | zram_stat_inc(&zram->stats.pages_expand); | |
362 | zram->table[index].page = page_store; | |
a1dd52af NG |
363 | src = kmap_atomic(page, KM_USER0); |
364 | goto memstore; | |
365 | } | |
306b0c95 | 366 | |
f1e3cfff NG |
367 | if (xv_malloc(zram->mem_pool, clen + sizeof(*zheader), |
368 | &zram->table[index].page, &offset, | |
a1dd52af | 369 | GFP_NOIO | __GFP_HIGHMEM)) { |
f1e3cfff | 370 | mutex_unlock(&zram->lock); |
a1dd52af NG |
371 | pr_info("Error allocating memory for compressed " |
372 | "page: %u, size=%zu\n", index, clen); | |
f1e3cfff | 373 | zram_stat64_inc(zram, &zram->stats.failed_writes); |
a1dd52af NG |
374 | goto out; |
375 | } | |
306b0c95 NG |
376 | |
377 | memstore: | |
f1e3cfff | 378 | zram->table[index].offset = offset; |
306b0c95 | 379 | |
f1e3cfff NG |
380 | cmem = kmap_atomic(zram->table[index].page, KM_USER1) + |
381 | zram->table[index].offset; | |
306b0c95 NG |
382 | |
383 | #if 0 | |
a1dd52af | 384 | /* Back-reference needed for memory defragmentation */ |
f1e3cfff | 385 | if (!zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)) { |
a1dd52af NG |
386 | zheader = (struct zobj_header *)cmem; |
387 | zheader->table_idx = index; | |
388 | cmem += sizeof(*zheader); | |
389 | } | |
306b0c95 NG |
390 | #endif |
391 | ||
a1dd52af | 392 | memcpy(cmem, src, clen); |
306b0c95 | 393 | |
a1dd52af | 394 | kunmap_atomic(cmem, KM_USER1); |
f1e3cfff | 395 | if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) |
a1dd52af | 396 | kunmap_atomic(src, KM_USER0); |
306b0c95 | 397 | |
a1dd52af | 398 | /* Update stats */ |
f1e3cfff NG |
399 | zram->stats.compr_size += clen; |
400 | zram_stat_inc(&zram->stats.pages_stored); | |
a1dd52af | 401 | if (clen <= PAGE_SIZE / 2) |
f1e3cfff | 402 | zram_stat_inc(&zram->stats.good_compress); |
306b0c95 | 403 | |
f1e3cfff | 404 | mutex_unlock(&zram->lock); |
a1dd52af NG |
405 | index++; |
406 | } | |
306b0c95 NG |
407 | |
408 | set_bit(BIO_UPTODATE, &bio->bi_flags); | |
409 | bio_endio(bio, 0); | |
410 | return 0; | |
411 | ||
412 | out: | |
306b0c95 NG |
413 | bio_io_error(bio); |
414 | return 0; | |
415 | } | |
416 | ||
306b0c95 NG |
417 | /* |
418 | * Check if request is within bounds and page aligned. | |
419 | */ | |
f1e3cfff | 420 | static inline int valid_io_request(struct zram *zram, struct bio *bio) |
306b0c95 NG |
421 | { |
422 | if (unlikely( | |
f1e3cfff | 423 | (bio->bi_sector >= (zram->disksize >> SECTOR_SHIFT)) || |
306b0c95 | 424 | (bio->bi_sector & (SECTORS_PER_PAGE - 1)) || |
a1dd52af | 425 | (bio->bi_size & (PAGE_SIZE - 1)))) { |
306b0c95 NG |
426 | |
427 | return 0; | |
428 | } | |
429 | ||
a1dd52af | 430 | /* I/O request is valid */ |
306b0c95 NG |
431 | return 1; |
432 | } | |
433 | ||
434 | /* | |
f1e3cfff | 435 | * Handler function for all zram I/O requests. |
306b0c95 | 436 | */ |
f1e3cfff | 437 | static int zram_make_request(struct request_queue *queue, struct bio *bio) |
306b0c95 NG |
438 | { |
439 | int ret = 0; | |
f1e3cfff | 440 | struct zram *zram = queue->queuedata; |
306b0c95 | 441 | |
f1e3cfff | 442 | if (unlikely(!zram->init_done)) { |
306b0c95 NG |
443 | bio_io_error(bio); |
444 | return 0; | |
445 | } | |
446 | ||
f1e3cfff NG |
447 | if (!valid_io_request(zram, bio)) { |
448 | zram_stat64_inc(zram, &zram->stats.invalid_io); | |
306b0c95 NG |
449 | bio_io_error(bio); |
450 | return 0; | |
451 | } | |
452 | ||
453 | switch (bio_data_dir(bio)) { | |
454 | case READ: | |
f1e3cfff | 455 | ret = zram_read(zram, bio); |
306b0c95 NG |
456 | break; |
457 | ||
458 | case WRITE: | |
f1e3cfff | 459 | ret = zram_write(zram, bio); |
306b0c95 NG |
460 | break; |
461 | } | |
462 | ||
463 | return ret; | |
464 | } | |
465 | ||
f1e3cfff | 466 | static void reset_device(struct zram *zram) |
306b0c95 | 467 | { |
97a06382 | 468 | size_t index; |
306b0c95 | 469 | |
7eef7533 | 470 | /* Do not accept any new I/O request */ |
f1e3cfff | 471 | zram->init_done = 0; |
7eef7533 | 472 | |
306b0c95 | 473 | /* Free various per-device buffers */ |
f1e3cfff NG |
474 | kfree(zram->compress_workmem); |
475 | free_pages((unsigned long)zram->compress_buffer, 1); | |
306b0c95 | 476 | |
f1e3cfff NG |
477 | zram->compress_workmem = NULL; |
478 | zram->compress_buffer = NULL; | |
306b0c95 | 479 | |
f1e3cfff NG |
480 | /* Free all pages that are still in this zram device */ |
481 | for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) { | |
306b0c95 NG |
482 | struct page *page; |
483 | u16 offset; | |
484 | ||
f1e3cfff NG |
485 | page = zram->table[index].page; |
486 | offset = zram->table[index].offset; | |
306b0c95 NG |
487 | |
488 | if (!page) | |
489 | continue; | |
490 | ||
f1e3cfff | 491 | if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) |
306b0c95 NG |
492 | __free_page(page); |
493 | else | |
f1e3cfff | 494 | xv_free(zram->mem_pool, page, offset); |
306b0c95 NG |
495 | } |
496 | ||
f1e3cfff NG |
497 | vfree(zram->table); |
498 | zram->table = NULL; | |
306b0c95 | 499 | |
f1e3cfff NG |
500 | xv_destroy_pool(zram->mem_pool); |
501 | zram->mem_pool = NULL; | |
306b0c95 | 502 | |
306b0c95 | 503 | /* Reset stats */ |
f1e3cfff | 504 | memset(&zram->stats, 0, sizeof(zram->stats)); |
306b0c95 | 505 | |
f1e3cfff | 506 | zram->disksize = 0; |
306b0c95 NG |
507 | } |
508 | ||
f1e3cfff | 509 | static int zram_ioctl_init_device(struct zram *zram) |
306b0c95 NG |
510 | { |
511 | int ret; | |
512 | size_t num_pages; | |
306b0c95 | 513 | |
f1e3cfff | 514 | if (zram->init_done) { |
306b0c95 NG |
515 | pr_info("Device already initialized!\n"); |
516 | return -EBUSY; | |
517 | } | |
518 | ||
f1e3cfff | 519 | zram_set_disksize(zram, totalram_pages << PAGE_SHIFT); |
306b0c95 | 520 | |
f1e3cfff NG |
521 | zram->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL); |
522 | if (!zram->compress_workmem) { | |
306b0c95 NG |
523 | pr_err("Error allocating compressor working memory!\n"); |
524 | ret = -ENOMEM; | |
525 | goto fail; | |
526 | } | |
527 | ||
f1e3cfff NG |
528 | zram->compress_buffer = (void *)__get_free_pages(__GFP_ZERO, 1); |
529 | if (!zram->compress_buffer) { | |
306b0c95 NG |
530 | pr_err("Error allocating compressor buffer space\n"); |
531 | ret = -ENOMEM; | |
532 | goto fail; | |
533 | } | |
534 | ||
f1e3cfff NG |
535 | num_pages = zram->disksize >> PAGE_SHIFT; |
536 | zram->table = vmalloc(num_pages * sizeof(*zram->table)); | |
537 | if (!zram->table) { | |
538 | pr_err("Error allocating zram address table\n"); | |
306b0c95 | 539 | /* To prevent accessing table entries during cleanup */ |
f1e3cfff | 540 | zram->disksize = 0; |
306b0c95 NG |
541 | ret = -ENOMEM; |
542 | goto fail; | |
543 | } | |
f1e3cfff | 544 | memset(zram->table, 0, num_pages * sizeof(*zram->table)); |
306b0c95 | 545 | |
f1e3cfff | 546 | set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); |
306b0c95 | 547 | |
f1e3cfff NG |
548 | /* zram devices sort of resembles non-rotational disks */ |
549 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue); | |
306b0c95 | 550 | |
f1e3cfff NG |
551 | zram->mem_pool = xv_create_pool(); |
552 | if (!zram->mem_pool) { | |
306b0c95 NG |
553 | pr_err("Error creating memory pool\n"); |
554 | ret = -ENOMEM; | |
555 | goto fail; | |
556 | } | |
557 | ||
f1e3cfff | 558 | zram->init_done = 1; |
306b0c95 NG |
559 | |
560 | pr_debug("Initialization done!\n"); | |
561 | return 0; | |
562 | ||
563 | fail: | |
f1e3cfff | 564 | reset_device(zram); |
306b0c95 NG |
565 | |
566 | pr_err("Initialization failed: err=%d\n", ret); | |
567 | return ret; | |
568 | } | |
569 | ||
f1e3cfff | 570 | static int zram_ioctl_reset_device(struct zram *zram) |
306b0c95 | 571 | { |
f1e3cfff NG |
572 | if (zram->init_done) |
573 | reset_device(zram); | |
306b0c95 NG |
574 | |
575 | return 0; | |
576 | } | |
577 | ||
f1e3cfff | 578 | static int zram_ioctl(struct block_device *bdev, fmode_t mode, |
306b0c95 NG |
579 | unsigned int cmd, unsigned long arg) |
580 | { | |
581 | int ret = 0; | |
97a06382 | 582 | size_t disksize_kb; |
306b0c95 | 583 | |
f1e3cfff | 584 | struct zram *zram = bdev->bd_disk->private_data; |
306b0c95 NG |
585 | |
586 | switch (cmd) { | |
f1e3cfff NG |
587 | case ZRAMIO_SET_DISKSIZE_KB: |
588 | if (zram->init_done) { | |
306b0c95 NG |
589 | ret = -EBUSY; |
590 | goto out; | |
591 | } | |
592 | if (copy_from_user(&disksize_kb, (void *)arg, | |
593 | _IOC_SIZE(cmd))) { | |
594 | ret = -EFAULT; | |
595 | goto out; | |
596 | } | |
f1e3cfff | 597 | zram->disksize = disksize_kb << 10; |
306b0c95 NG |
598 | pr_info("Disk size set to %zu kB\n", disksize_kb); |
599 | break; | |
600 | ||
f1e3cfff | 601 | case ZRAMIO_GET_STATS: |
306b0c95 | 602 | { |
f1e3cfff NG |
603 | struct zram_ioctl_stats *stats; |
604 | if (!zram->init_done) { | |
306b0c95 NG |
605 | ret = -ENOTTY; |
606 | goto out; | |
607 | } | |
608 | stats = kzalloc(sizeof(*stats), GFP_KERNEL); | |
609 | if (!stats) { | |
610 | ret = -ENOMEM; | |
611 | goto out; | |
612 | } | |
f1e3cfff | 613 | zram_ioctl_get_stats(zram, stats); |
306b0c95 NG |
614 | if (copy_to_user((void *)arg, stats, sizeof(*stats))) { |
615 | kfree(stats); | |
616 | ret = -EFAULT; | |
617 | goto out; | |
618 | } | |
619 | kfree(stats); | |
620 | break; | |
621 | } | |
f1e3cfff NG |
622 | case ZRAMIO_INIT: |
623 | ret = zram_ioctl_init_device(zram); | |
306b0c95 NG |
624 | break; |
625 | ||
f1e3cfff | 626 | case ZRAMIO_RESET: |
306b0c95 NG |
627 | /* Do not reset an active device! */ |
628 | if (bdev->bd_holders) { | |
629 | ret = -EBUSY; | |
630 | goto out; | |
631 | } | |
7eef7533 NG |
632 | |
633 | /* Make sure all pending I/O is finished */ | |
634 | if (bdev) | |
635 | fsync_bdev(bdev); | |
636 | ||
f1e3cfff | 637 | ret = zram_ioctl_reset_device(zram); |
306b0c95 NG |
638 | break; |
639 | ||
640 | default: | |
641 | pr_info("Invalid ioctl %u\n", cmd); | |
642 | ret = -ENOTTY; | |
643 | } | |
644 | ||
645 | out: | |
646 | return ret; | |
647 | } | |
648 | ||
f1e3cfff | 649 | void zram_slot_free_notify(struct block_device *bdev, unsigned long index) |
107c161b | 650 | { |
f1e3cfff | 651 | struct zram *zram; |
107c161b | 652 | |
f1e3cfff NG |
653 | zram = bdev->bd_disk->private_data; |
654 | zram_free_page(zram, index); | |
655 | zram_stat64_inc(zram, &zram->stats.notify_free); | |
107c161b NG |
656 | } |
657 | ||
f1e3cfff NG |
658 | static const struct block_device_operations zram_devops = { |
659 | .ioctl = zram_ioctl, | |
660 | .swap_slot_free_notify = zram_slot_free_notify, | |
107c161b | 661 | .owner = THIS_MODULE |
306b0c95 NG |
662 | }; |
663 | ||
f1e3cfff | 664 | static int create_device(struct zram *zram, int device_id) |
306b0c95 | 665 | { |
de1a21a0 NG |
666 | int ret = 0; |
667 | ||
f1e3cfff NG |
668 | mutex_init(&zram->lock); |
669 | spin_lock_init(&zram->stat64_lock); | |
306b0c95 | 670 | |
f1e3cfff NG |
671 | zram->queue = blk_alloc_queue(GFP_KERNEL); |
672 | if (!zram->queue) { | |
306b0c95 NG |
673 | pr_err("Error allocating disk queue for device %d\n", |
674 | device_id); | |
de1a21a0 NG |
675 | ret = -ENOMEM; |
676 | goto out; | |
306b0c95 NG |
677 | } |
678 | ||
f1e3cfff NG |
679 | blk_queue_make_request(zram->queue, zram_make_request); |
680 | zram->queue->queuedata = zram; | |
306b0c95 NG |
681 | |
682 | /* gendisk structure */ | |
f1e3cfff NG |
683 | zram->disk = alloc_disk(1); |
684 | if (!zram->disk) { | |
685 | blk_cleanup_queue(zram->queue); | |
306b0c95 NG |
686 | pr_warning("Error allocating disk structure for device %d\n", |
687 | device_id); | |
de1a21a0 NG |
688 | ret = -ENOMEM; |
689 | goto out; | |
306b0c95 NG |
690 | } |
691 | ||
f1e3cfff NG |
692 | zram->disk->major = zram_major; |
693 | zram->disk->first_minor = device_id; | |
694 | zram->disk->fops = &zram_devops; | |
695 | zram->disk->queue = zram->queue; | |
696 | zram->disk->private_data = zram; | |
697 | snprintf(zram->disk->disk_name, 16, "zram%d", device_id); | |
306b0c95 | 698 | |
f1e3cfff NG |
699 | /* Actual capacity set using ZRAMIO_SET_DISKSIZE_KB ioctl */ |
700 | set_capacity(zram->disk, 0); | |
5d83d5a0 | 701 | |
a1dd52af NG |
702 | /* |
703 | * To ensure that we always get PAGE_SIZE aligned | |
704 | * and n*PAGE_SIZED sized I/O requests. | |
705 | */ | |
f1e3cfff NG |
706 | blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE); |
707 | blk_queue_logical_block_size(zram->disk->queue, PAGE_SIZE); | |
708 | blk_queue_io_min(zram->disk->queue, PAGE_SIZE); | |
709 | blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); | |
5d83d5a0 | 710 | |
f1e3cfff | 711 | add_disk(zram->disk); |
306b0c95 | 712 | |
f1e3cfff | 713 | zram->init_done = 0; |
de1a21a0 NG |
714 | |
715 | out: | |
716 | return ret; | |
306b0c95 NG |
717 | } |
718 | ||
f1e3cfff | 719 | static void destroy_device(struct zram *zram) |
306b0c95 | 720 | { |
f1e3cfff NG |
721 | if (zram->disk) { |
722 | del_gendisk(zram->disk); | |
723 | put_disk(zram->disk); | |
306b0c95 NG |
724 | } |
725 | ||
f1e3cfff NG |
726 | if (zram->queue) |
727 | blk_cleanup_queue(zram->queue); | |
306b0c95 NG |
728 | } |
729 | ||
f1e3cfff | 730 | static int __init zram_init(void) |
306b0c95 | 731 | { |
de1a21a0 | 732 | int ret, dev_id; |
306b0c95 NG |
733 | |
734 | if (num_devices > max_num_devices) { | |
735 | pr_warning("Invalid value for num_devices: %u\n", | |
736 | num_devices); | |
de1a21a0 NG |
737 | ret = -EINVAL; |
738 | goto out; | |
306b0c95 NG |
739 | } |
740 | ||
f1e3cfff NG |
741 | zram_major = register_blkdev(0, "zram"); |
742 | if (zram_major <= 0) { | |
306b0c95 | 743 | pr_warning("Unable to get major number\n"); |
de1a21a0 NG |
744 | ret = -EBUSY; |
745 | goto out; | |
306b0c95 NG |
746 | } |
747 | ||
748 | if (!num_devices) { | |
749 | pr_info("num_devices not specified. Using default: 1\n"); | |
750 | num_devices = 1; | |
751 | } | |
752 | ||
753 | /* Allocate the device array and initialize each one */ | |
754 | pr_info("Creating %u devices ...\n", num_devices); | |
f1e3cfff | 755 | devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL); |
de1a21a0 NG |
756 | if (!devices) { |
757 | ret = -ENOMEM; | |
758 | goto unregister; | |
759 | } | |
306b0c95 | 760 | |
de1a21a0 NG |
761 | for (dev_id = 0; dev_id < num_devices; dev_id++) { |
762 | ret = create_device(&devices[dev_id], dev_id); | |
763 | if (ret) | |
3bf040c7 | 764 | goto free_devices; |
de1a21a0 NG |
765 | } |
766 | ||
306b0c95 | 767 | return 0; |
de1a21a0 | 768 | |
3bf040c7 | 769 | free_devices: |
de1a21a0 NG |
770 | while (dev_id) |
771 | destroy_device(&devices[--dev_id]); | |
273ad8dc | 772 | kfree(devices); |
de1a21a0 | 773 | unregister: |
f1e3cfff | 774 | unregister_blkdev(zram_major, "zram"); |
de1a21a0 | 775 | out: |
306b0c95 NG |
776 | return ret; |
777 | } | |
778 | ||
f1e3cfff | 779 | static void __exit zram_exit(void) |
306b0c95 NG |
780 | { |
781 | int i; | |
f1e3cfff | 782 | struct zram *zram; |
306b0c95 NG |
783 | |
784 | for (i = 0; i < num_devices; i++) { | |
f1e3cfff | 785 | zram = &devices[i]; |
306b0c95 | 786 | |
f1e3cfff NG |
787 | destroy_device(zram); |
788 | if (zram->init_done) | |
789 | reset_device(zram); | |
306b0c95 NG |
790 | } |
791 | ||
f1e3cfff | 792 | unregister_blkdev(zram_major, "zram"); |
306b0c95 NG |
793 | |
794 | kfree(devices); | |
795 | pr_debug("Cleanup done!\n"); | |
796 | } | |
797 | ||
798 | module_param(num_devices, uint, 0); | |
f1e3cfff | 799 | MODULE_PARM_DESC(num_devices, "Number of zram devices"); |
306b0c95 | 800 | |
f1e3cfff NG |
801 | module_init(zram_init); |
802 | module_exit(zram_exit); | |
306b0c95 NG |
803 | |
804 | MODULE_LICENSE("Dual BSD/GPL"); | |
805 | MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>"); | |
f1e3cfff | 806 | MODULE_DESCRIPTION("Compressed RAM Block Device"); |