2 * linux/kernel/power/swap.c
4 * This file provides functions for reading the suspend image from
5 * and writing it to a swap partition.
7 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
8 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
9 * Copyright (C) 2010 Bojan Smojver <bojan@rexursive.com>
11 * This file is released under the GPLv2.
15 #include <linux/module.h>
16 #include <linux/file.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/genhd.h>
20 #include <linux/device.h>
21 #include <linux/bio.h>
22 #include <linux/blkdev.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
26 #include <linux/slab.h>
27 #include <linux/lzo.h>
28 #include <linux/vmalloc.h>
29 #include <linux/cpumask.h>
30 #include <linux/atomic.h>
31 #include <linux/kthread.h>
32 #include <linux/crc32.h>
36 #define HIBERNATE_SIG "S1SUSPEND"
39 * The swap map is a data structure used for keeping track of each page
40 * written to a swap partition. It consists of many swap_map_page
41 * structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
42 * These structures are stored on the swap and linked together with the
43 * help of the .next_swap member.
45 * The swap map is created during suspend. The swap map pages are
46 * allocated and populated one at a time, so we only need one memory
47 * page to set up the entire structure.
49 * During resume we pick up all swap_map_page structures into a list.
52 #define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1)
54 struct swap_map_page
{
55 sector_t entries
[MAP_PAGE_ENTRIES
];
59 struct swap_map_page_list
{
60 struct swap_map_page
*map
;
61 struct swap_map_page_list
*next
;
65 * The swap_map_handle structure is used for handling swap in
69 struct swap_map_handle
{
70 struct swap_map_page
*cur
;
71 struct swap_map_page_list
*maps
;
73 sector_t first_sector
;
75 unsigned long nr_free_pages
, written
;
79 struct swsusp_header
{
80 char reserved
[PAGE_SIZE
- 20 - sizeof(sector_t
) - sizeof(int) -
84 unsigned int flags
; /* Flags to pass to the "boot" kernel */
87 } __attribute__((packed
));
89 static struct swsusp_header
*swsusp_header
;
92 * The following functions are used for tracing the allocated
93 * swap pages, so that they can be freed in case of an error.
96 struct swsusp_extent
{
102 static struct rb_root swsusp_extents
= RB_ROOT
;
104 static int swsusp_extents_insert(unsigned long swap_offset
)
106 struct rb_node
**new = &(swsusp_extents
.rb_node
);
107 struct rb_node
*parent
= NULL
;
108 struct swsusp_extent
*ext
;
110 /* Figure out where to put the new node */
112 ext
= container_of(*new, struct swsusp_extent
, node
);
114 if (swap_offset
< ext
->start
) {
116 if (swap_offset
== ext
->start
- 1) {
120 new = &((*new)->rb_left
);
121 } else if (swap_offset
> ext
->end
) {
123 if (swap_offset
== ext
->end
+ 1) {
127 new = &((*new)->rb_right
);
129 /* It already is in the tree */
133 /* Add the new node and rebalance the tree. */
134 ext
= kzalloc(sizeof(struct swsusp_extent
), GFP_KERNEL
);
138 ext
->start
= swap_offset
;
139 ext
->end
= swap_offset
;
140 rb_link_node(&ext
->node
, parent
, new);
141 rb_insert_color(&ext
->node
, &swsusp_extents
);
146 * alloc_swapdev_block - allocate a swap page and register that it has
147 * been allocated, so that it can be freed in case of an error.
150 sector_t
alloc_swapdev_block(int swap
)
152 unsigned long offset
;
154 offset
= swp_offset(get_swap_page_of_type(swap
));
156 if (swsusp_extents_insert(offset
))
157 swap_free(swp_entry(swap
, offset
));
159 return swapdev_block(swap
, offset
);
165 * free_all_swap_pages - free swap pages allocated for saving image data.
166 * It also frees the extents used to register which swap entries had been
170 void free_all_swap_pages(int swap
)
172 struct rb_node
*node
;
174 while ((node
= swsusp_extents
.rb_node
)) {
175 struct swsusp_extent
*ext
;
176 unsigned long offset
;
178 ext
= container_of(node
, struct swsusp_extent
, node
);
179 rb_erase(node
, &swsusp_extents
);
180 for (offset
= ext
->start
; offset
<= ext
->end
; offset
++)
181 swap_free(swp_entry(swap
, offset
));
187 int swsusp_swap_in_use(void)
189 return (swsusp_extents
.rb_node
!= NULL
);
196 static unsigned short root_swap
= 0xffff;
197 struct block_device
*hib_resume_bdev
;
203 static int mark_swapfiles(struct swap_map_handle
*handle
, unsigned int flags
)
207 hib_bio_read_page(swsusp_resume_block
, swsusp_header
, NULL
);
208 if (!memcmp("SWAP-SPACE",swsusp_header
->sig
, 10) ||
209 !memcmp("SWAPSPACE2",swsusp_header
->sig
, 10)) {
210 memcpy(swsusp_header
->orig_sig
,swsusp_header
->sig
, 10);
211 memcpy(swsusp_header
->sig
, HIBERNATE_SIG
, 10);
212 swsusp_header
->image
= handle
->first_sector
;
213 swsusp_header
->flags
= flags
;
214 if (flags
& SF_CRC32_MODE
)
215 swsusp_header
->crc32
= handle
->crc32
;
216 error
= hib_bio_write_page(swsusp_resume_block
,
217 swsusp_header
, NULL
);
219 printk(KERN_ERR
"PM: Swap header not found!\n");
226 * swsusp_swap_check - check if the resume device is a swap device
227 * and get its index (if so)
229 * This is called before saving image
231 static int swsusp_swap_check(void)
235 res
= swap_type_of(swsusp_resume_device
, swsusp_resume_block
,
241 res
= blkdev_get(hib_resume_bdev
, FMODE_WRITE
, NULL
);
245 res
= set_blocksize(hib_resume_bdev
, PAGE_SIZE
);
247 blkdev_put(hib_resume_bdev
, FMODE_WRITE
);
253 * write_page - Write one page to given swap location.
254 * @buf: Address we're writing.
255 * @offset: Offset of the swap page we're writing to.
256 * @bio_chain: Link the next write BIO here
259 static int write_page(void *buf
, sector_t offset
, struct bio
**bio_chain
)
268 src
= (void *)__get_free_page(__GFP_WAIT
| __GFP_HIGH
);
272 ret
= hib_wait_on_bio_chain(bio_chain
); /* Free pages */
275 src
= (void *)__get_free_page(__GFP_WAIT
| __GFP_HIGH
);
280 bio_chain
= NULL
; /* Go synchronous */
287 return hib_bio_write_page(offset
, src
, bio_chain
);
290 static void release_swap_writer(struct swap_map_handle
*handle
)
293 free_page((unsigned long)handle
->cur
);
297 static int get_swap_writer(struct swap_map_handle
*handle
)
301 ret
= swsusp_swap_check();
304 printk(KERN_ERR
"PM: Cannot find swap device, try "
308 handle
->cur
= (struct swap_map_page
*)get_zeroed_page(GFP_KERNEL
);
313 handle
->cur_swap
= alloc_swapdev_block(root_swap
);
314 if (!handle
->cur_swap
) {
319 handle
->nr_free_pages
= nr_free_pages() >> 1;
321 handle
->first_sector
= handle
->cur_swap
;
324 release_swap_writer(handle
);
326 swsusp_close(FMODE_WRITE
);
330 static int swap_write_page(struct swap_map_handle
*handle
, void *buf
,
331 struct bio
**bio_chain
)
338 offset
= alloc_swapdev_block(root_swap
);
339 error
= write_page(buf
, offset
, bio_chain
);
342 handle
->cur
->entries
[handle
->k
++] = offset
;
343 if (handle
->k
>= MAP_PAGE_ENTRIES
) {
344 offset
= alloc_swapdev_block(root_swap
);
347 handle
->cur
->next_swap
= offset
;
348 error
= write_page(handle
->cur
, handle
->cur_swap
, bio_chain
);
351 clear_page(handle
->cur
);
352 handle
->cur_swap
= offset
;
355 if (bio_chain
&& ++handle
->written
> handle
->nr_free_pages
) {
356 error
= hib_wait_on_bio_chain(bio_chain
);
365 static int flush_swap_writer(struct swap_map_handle
*handle
)
367 if (handle
->cur
&& handle
->cur_swap
)
368 return write_page(handle
->cur
, handle
->cur_swap
, NULL
);
373 static int swap_writer_finish(struct swap_map_handle
*handle
,
374 unsigned int flags
, int error
)
377 flush_swap_writer(handle
);
378 printk(KERN_INFO
"PM: S");
379 error
= mark_swapfiles(handle
, flags
);
384 free_all_swap_pages(root_swap
);
385 release_swap_writer(handle
);
386 swsusp_close(FMODE_WRITE
);
391 /* We need to remember how much compressed data we need to read. */
392 #define LZO_HEADER sizeof(size_t)
394 /* Number of pages/bytes we'll compress at one time. */
395 #define LZO_UNC_PAGES 32
396 #define LZO_UNC_SIZE (LZO_UNC_PAGES * PAGE_SIZE)
398 /* Number of pages/bytes we need for compressed data (worst case). */
399 #define LZO_CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \
400 LZO_HEADER, PAGE_SIZE)
401 #define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE)
403 /* Maximum number of threads for compression/decompression. */
404 #define LZO_THREADS 3
406 /* Maximum number of pages for read buffering. */
407 #define LZO_READ_PAGES (MAP_PAGE_ENTRIES * 8)
411 * save_image - save the suspend image data
414 static int save_image(struct swap_map_handle
*handle
,
415 struct snapshot_handle
*snapshot
,
416 unsigned int nr_to_write
)
423 struct timeval start
;
426 printk(KERN_INFO
"PM: Saving image data pages (%u pages) ... ",
428 m
= nr_to_write
/ 100;
433 do_gettimeofday(&start
);
435 ret
= snapshot_read_next(snapshot
);
438 ret
= swap_write_page(handle
, data_of(*snapshot
), &bio
);
442 printk(KERN_CONT
"\b\b\b\b%3d%%", nr_pages
/ m
);
445 err2
= hib_wait_on_bio_chain(&bio
);
446 do_gettimeofday(&stop
);
450 printk(KERN_CONT
"\b\b\b\bdone\n");
452 printk(KERN_CONT
"\n");
453 swsusp_show_speed(&start
, &stop
, nr_to_write
, "Wrote");
458 * Structure used for CRC32.
461 struct task_struct
*thr
; /* thread */
462 atomic_t ready
; /* ready to start flag */
463 atomic_t stop
; /* ready to stop flag */
464 unsigned run_threads
; /* nr current threads */
465 wait_queue_head_t go
; /* start crc update */
466 wait_queue_head_t done
; /* crc update done */
467 u32
*crc32
; /* points to handle's crc32 */
468 size_t *unc_len
[LZO_THREADS
]; /* uncompressed lengths */
469 unsigned char *unc
[LZO_THREADS
]; /* uncompressed data */
473 * CRC32 update function that runs in its own thread.
475 static int crc32_threadfn(void *data
)
477 struct crc_data
*d
= data
;
481 wait_event(d
->go
, atomic_read(&d
->ready
) ||
482 kthread_should_stop());
483 if (kthread_should_stop()) {
485 atomic_set(&d
->stop
, 1);
489 atomic_set(&d
->ready
, 0);
491 for (i
= 0; i
< d
->run_threads
; i
++)
492 *d
->crc32
= crc32_le(*d
->crc32
,
493 d
->unc
[i
], *d
->unc_len
[i
]);
494 atomic_set(&d
->stop
, 1);
500 * Structure used for LZO data compression.
503 struct task_struct
*thr
; /* thread */
504 atomic_t ready
; /* ready to start flag */
505 atomic_t stop
; /* ready to stop flag */
506 int ret
; /* return code */
507 wait_queue_head_t go
; /* start compression */
508 wait_queue_head_t done
; /* compression done */
509 size_t unc_len
; /* uncompressed length */
510 size_t cmp_len
; /* compressed length */
511 unsigned char unc
[LZO_UNC_SIZE
]; /* uncompressed buffer */
512 unsigned char cmp
[LZO_CMP_SIZE
]; /* compressed buffer */
513 unsigned char wrk
[LZO1X_1_MEM_COMPRESS
]; /* compression workspace */
517 * Compression function that runs in its own thread.
519 static int lzo_compress_threadfn(void *data
)
521 struct cmp_data
*d
= data
;
524 wait_event(d
->go
, atomic_read(&d
->ready
) ||
525 kthread_should_stop());
526 if (kthread_should_stop()) {
529 atomic_set(&d
->stop
, 1);
533 atomic_set(&d
->ready
, 0);
535 d
->ret
= lzo1x_1_compress(d
->unc
, d
->unc_len
,
536 d
->cmp
+ LZO_HEADER
, &d
->cmp_len
,
538 atomic_set(&d
->stop
, 1);
545 * save_image_lzo - Save the suspend image data compressed with LZO.
546 * @handle: Swap mam handle to use for saving the image.
547 * @snapshot: Image to read data from.
548 * @nr_to_write: Number of pages to save.
550 static int save_image_lzo(struct swap_map_handle
*handle
,
551 struct snapshot_handle
*snapshot
,
552 unsigned int nr_to_write
)
559 struct timeval start
;
562 unsigned thr
, run_threads
, nr_threads
;
563 unsigned char *page
= NULL
;
564 struct cmp_data
*data
= NULL
;
565 struct crc_data
*crc
= NULL
;
568 * We'll limit the number of threads for compression to limit memory
571 nr_threads
= num_online_cpus() - 1;
572 nr_threads
= clamp_val(nr_threads
, 1, LZO_THREADS
);
574 page
= (void *)__get_free_page(__GFP_WAIT
| __GFP_HIGH
);
576 printk(KERN_ERR
"PM: Failed to allocate LZO page\n");
581 data
= vmalloc(sizeof(*data
) * nr_threads
);
583 printk(KERN_ERR
"PM: Failed to allocate LZO data\n");
587 for (thr
= 0; thr
< nr_threads
; thr
++)
588 memset(&data
[thr
], 0, offsetof(struct cmp_data
, go
));
590 crc
= kmalloc(sizeof(*crc
), GFP_KERNEL
);
592 printk(KERN_ERR
"PM: Failed to allocate crc\n");
596 memset(crc
, 0, offsetof(struct crc_data
, go
));
599 * Start the compression threads.
601 for (thr
= 0; thr
< nr_threads
; thr
++) {
602 init_waitqueue_head(&data
[thr
].go
);
603 init_waitqueue_head(&data
[thr
].done
);
605 data
[thr
].thr
= kthread_run(lzo_compress_threadfn
,
607 "image_compress/%u", thr
);
608 if (IS_ERR(data
[thr
].thr
)) {
609 data
[thr
].thr
= NULL
;
611 "PM: Cannot start compression threads\n");
618 * Adjust number of free pages after all allocations have been done.
619 * We don't want to run out of pages when writing.
621 handle
->nr_free_pages
= nr_free_pages() >> 1;
624 * Start the CRC32 thread.
626 init_waitqueue_head(&crc
->go
);
627 init_waitqueue_head(&crc
->done
);
630 crc
->crc32
= &handle
->crc32
;
631 for (thr
= 0; thr
< nr_threads
; thr
++) {
632 crc
->unc
[thr
] = data
[thr
].unc
;
633 crc
->unc_len
[thr
] = &data
[thr
].unc_len
;
636 crc
->thr
= kthread_run(crc32_threadfn
, crc
, "image_crc32");
637 if (IS_ERR(crc
->thr
)) {
639 printk(KERN_ERR
"PM: Cannot start CRC32 thread\n");
645 "PM: Using %u thread(s) for compression.\n"
646 "PM: Compressing and saving image data (%u pages) ... ",
647 nr_threads
, nr_to_write
);
648 m
= nr_to_write
/ 100;
653 do_gettimeofday(&start
);
655 for (thr
= 0; thr
< nr_threads
; thr
++) {
656 for (off
= 0; off
< LZO_UNC_SIZE
; off
+= PAGE_SIZE
) {
657 ret
= snapshot_read_next(snapshot
);
664 memcpy(data
[thr
].unc
+ off
,
665 data_of(*snapshot
), PAGE_SIZE
);
668 printk(KERN_CONT
"\b\b\b\b%3d%%",
675 data
[thr
].unc_len
= off
;
677 atomic_set(&data
[thr
].ready
, 1);
678 wake_up(&data
[thr
].go
);
684 crc
->run_threads
= thr
;
685 atomic_set(&crc
->ready
, 1);
688 for (run_threads
= thr
, thr
= 0; thr
< run_threads
; thr
++) {
689 wait_event(data
[thr
].done
,
690 atomic_read(&data
[thr
].stop
));
691 atomic_set(&data
[thr
].stop
, 0);
696 printk(KERN_ERR
"PM: LZO compression failed\n");
700 if (unlikely(!data
[thr
].cmp_len
||
702 lzo1x_worst_compress(data
[thr
].unc_len
))) {
704 "PM: Invalid LZO compressed length\n");
709 *(size_t *)data
[thr
].cmp
= data
[thr
].cmp_len
;
712 * Given we are writing one page at a time to disk, we
713 * copy that much from the buffer, although the last
714 * bit will likely be smaller than full page. This is
715 * OK - we saved the length of the compressed data, so
716 * any garbage at the end will be discarded when we
720 off
< LZO_HEADER
+ data
[thr
].cmp_len
;
722 memcpy(page
, data
[thr
].cmp
+ off
, PAGE_SIZE
);
724 ret
= swap_write_page(handle
, page
, &bio
);
730 wait_event(crc
->done
, atomic_read(&crc
->stop
));
731 atomic_set(&crc
->stop
, 0);
735 err2
= hib_wait_on_bio_chain(&bio
);
736 do_gettimeofday(&stop
);
740 printk(KERN_CONT
"\b\b\b\bdone\n");
742 printk(KERN_CONT
"\n");
744 swsusp_show_speed(&start
, &stop
, nr_to_write
, "Wrote");
748 kthread_stop(crc
->thr
);
752 for (thr
= 0; thr
< nr_threads
; thr
++)
754 kthread_stop(data
[thr
].thr
);
757 if (page
) free_page((unsigned long)page
);
763 * enough_swap - Make sure we have enough swap to save the image.
765 * Returns TRUE or FALSE after checking the total amount of swap
766 * space avaiable from the resume partition.
769 static int enough_swap(unsigned int nr_pages
, unsigned int flags
)
771 unsigned int free_swap
= count_swap_pages(root_swap
, 1);
772 unsigned int required
;
774 pr_debug("PM: Free swap pages: %u\n", free_swap
);
776 required
= PAGES_FOR_IO
+ ((flags
& SF_NOCOMPRESS_MODE
) ?
777 nr_pages
: (nr_pages
* LZO_CMP_PAGES
) / LZO_UNC_PAGES
+ 1);
778 return free_swap
> required
;
782 * swsusp_write - Write entire image and metadata.
783 * @flags: flags to pass to the "boot" kernel in the image header
785 * It is important _NOT_ to umount filesystems at this point. We want
786 * them synced (in case something goes wrong) but we DO not want to mark
787 * filesystem clean: it is not. (And it does not matter, if we resume
788 * correctly, we'll mark system clean, anyway.)
791 int swsusp_write(unsigned int flags
)
793 struct swap_map_handle handle
;
794 struct snapshot_handle snapshot
;
795 struct swsusp_info
*header
;
799 pages
= snapshot_get_image_size();
800 error
= get_swap_writer(&handle
);
802 printk(KERN_ERR
"PM: Cannot get swap writer\n");
805 if (!enough_swap(pages
, flags
)) {
806 printk(KERN_ERR
"PM: Not enough free swap\n");
810 memset(&snapshot
, 0, sizeof(struct snapshot_handle
));
811 error
= snapshot_read_next(&snapshot
);
812 if (error
< PAGE_SIZE
) {
818 header
= (struct swsusp_info
*)data_of(snapshot
);
819 error
= swap_write_page(&handle
, header
, NULL
);
821 error
= (flags
& SF_NOCOMPRESS_MODE
) ?
822 save_image(&handle
, &snapshot
, pages
- 1) :
823 save_image_lzo(&handle
, &snapshot
, pages
- 1);
826 error
= swap_writer_finish(&handle
, flags
, error
);
831 * The following functions allow us to read data using a swap map
832 * in a file-alike way
835 static void release_swap_reader(struct swap_map_handle
*handle
)
837 struct swap_map_page_list
*tmp
;
839 while (handle
->maps
) {
840 if (handle
->maps
->map
)
841 free_page((unsigned long)handle
->maps
->map
);
843 handle
->maps
= handle
->maps
->next
;
849 static int get_swap_reader(struct swap_map_handle
*handle
,
850 unsigned int *flags_p
)
853 struct swap_map_page_list
*tmp
, *last
;
856 *flags_p
= swsusp_header
->flags
;
858 if (!swsusp_header
->image
) /* how can this happen? */
862 last
= handle
->maps
= NULL
;
863 offset
= swsusp_header
->image
;
865 tmp
= kmalloc(sizeof(*handle
->maps
), GFP_KERNEL
);
867 release_swap_reader(handle
);
870 memset(tmp
, 0, sizeof(*tmp
));
877 tmp
->map
= (struct swap_map_page
*)
878 __get_free_page(__GFP_WAIT
| __GFP_HIGH
);
880 release_swap_reader(handle
);
884 error
= hib_bio_read_page(offset
, tmp
->map
, NULL
);
886 release_swap_reader(handle
);
889 offset
= tmp
->map
->next_swap
;
892 handle
->cur
= handle
->maps
->map
;
896 static int swap_read_page(struct swap_map_handle
*handle
, void *buf
,
897 struct bio
**bio_chain
)
901 struct swap_map_page_list
*tmp
;
905 offset
= handle
->cur
->entries
[handle
->k
];
908 error
= hib_bio_read_page(offset
, buf
, bio_chain
);
911 if (++handle
->k
>= MAP_PAGE_ENTRIES
) {
913 free_page((unsigned long)handle
->maps
->map
);
915 handle
->maps
= handle
->maps
->next
;
918 release_swap_reader(handle
);
920 handle
->cur
= handle
->maps
->map
;
925 static int swap_reader_finish(struct swap_map_handle
*handle
)
927 release_swap_reader(handle
);
933 * load_image - load the image using the swap map handle
934 * @handle and the snapshot handle @snapshot
935 * (assume there are @nr_pages pages to load)
938 static int load_image(struct swap_map_handle
*handle
,
939 struct snapshot_handle
*snapshot
,
940 unsigned int nr_to_read
)
944 struct timeval start
;
950 printk(KERN_INFO
"PM: Loading image data pages (%u pages) ... ",
952 m
= nr_to_read
/ 100;
957 do_gettimeofday(&start
);
959 ret
= snapshot_write_next(snapshot
);
962 ret
= swap_read_page(handle
, data_of(*snapshot
), &bio
);
965 if (snapshot
->sync_read
)
966 ret
= hib_wait_on_bio_chain(&bio
);
970 printk("\b\b\b\b%3d%%", nr_pages
/ m
);
973 err2
= hib_wait_on_bio_chain(&bio
);
974 do_gettimeofday(&stop
);
978 printk("\b\b\b\bdone\n");
979 snapshot_write_finalize(snapshot
);
980 if (!snapshot_image_loaded(snapshot
))
984 swsusp_show_speed(&start
, &stop
, nr_to_read
, "Read");
989 * Structure used for LZO data decompression.
992 struct task_struct
*thr
; /* thread */
993 atomic_t ready
; /* ready to start flag */
994 atomic_t stop
; /* ready to stop flag */
995 int ret
; /* return code */
996 wait_queue_head_t go
; /* start decompression */
997 wait_queue_head_t done
; /* decompression done */
998 size_t unc_len
; /* uncompressed length */
999 size_t cmp_len
; /* compressed length */
1000 unsigned char unc
[LZO_UNC_SIZE
]; /* uncompressed buffer */
1001 unsigned char cmp
[LZO_CMP_SIZE
]; /* compressed buffer */
1005 * Deompression function that runs in its own thread.
1007 static int lzo_decompress_threadfn(void *data
)
1009 struct dec_data
*d
= data
;
1012 wait_event(d
->go
, atomic_read(&d
->ready
) ||
1013 kthread_should_stop());
1014 if (kthread_should_stop()) {
1017 atomic_set(&d
->stop
, 1);
1021 atomic_set(&d
->ready
, 0);
1023 d
->unc_len
= LZO_UNC_SIZE
;
1024 d
->ret
= lzo1x_decompress_safe(d
->cmp
+ LZO_HEADER
, d
->cmp_len
,
1025 d
->unc
, &d
->unc_len
);
1026 atomic_set(&d
->stop
, 1);
1033 * load_image_lzo - Load compressed image data and decompress them with LZO.
1034 * @handle: Swap map handle to use for loading data.
1035 * @snapshot: Image to copy uncompressed data into.
1036 * @nr_to_read: Number of pages to load.
1038 static int load_image_lzo(struct swap_map_handle
*handle
,
1039 struct snapshot_handle
*snapshot
,
1040 unsigned int nr_to_read
)
1046 struct timeval start
;
1047 struct timeval stop
;
1050 unsigned i
, thr
, run_threads
, nr_threads
;
1051 unsigned ring
= 0, pg
= 0, ring_size
= 0,
1052 have
= 0, want
, need
, asked
= 0;
1053 unsigned long read_pages
;
1054 unsigned char **page
= NULL
;
1055 struct dec_data
*data
= NULL
;
1056 struct crc_data
*crc
= NULL
;
1059 * We'll limit the number of threads for decompression to limit memory
1062 nr_threads
= num_online_cpus() - 1;
1063 nr_threads
= clamp_val(nr_threads
, 1, LZO_THREADS
);
1065 page
= vmalloc(sizeof(*page
) * LZO_READ_PAGES
);
1067 printk(KERN_ERR
"PM: Failed to allocate LZO page\n");
1072 data
= vmalloc(sizeof(*data
) * nr_threads
);
1074 printk(KERN_ERR
"PM: Failed to allocate LZO data\n");
1078 for (thr
= 0; thr
< nr_threads
; thr
++)
1079 memset(&data
[thr
], 0, offsetof(struct dec_data
, go
));
1081 crc
= kmalloc(sizeof(*crc
), GFP_KERNEL
);
1083 printk(KERN_ERR
"PM: Failed to allocate crc\n");
1087 memset(crc
, 0, offsetof(struct crc_data
, go
));
1090 * Start the decompression threads.
1092 for (thr
= 0; thr
< nr_threads
; thr
++) {
1093 init_waitqueue_head(&data
[thr
].go
);
1094 init_waitqueue_head(&data
[thr
].done
);
1096 data
[thr
].thr
= kthread_run(lzo_decompress_threadfn
,
1098 "image_decompress/%u", thr
);
1099 if (IS_ERR(data
[thr
].thr
)) {
1100 data
[thr
].thr
= NULL
;
1102 "PM: Cannot start decompression threads\n");
1109 * Start the CRC32 thread.
1111 init_waitqueue_head(&crc
->go
);
1112 init_waitqueue_head(&crc
->done
);
1115 crc
->crc32
= &handle
->crc32
;
1116 for (thr
= 0; thr
< nr_threads
; thr
++) {
1117 crc
->unc
[thr
] = data
[thr
].unc
;
1118 crc
->unc_len
[thr
] = &data
[thr
].unc_len
;
1121 crc
->thr
= kthread_run(crc32_threadfn
, crc
, "image_crc32");
1122 if (IS_ERR(crc
->thr
)) {
1124 printk(KERN_ERR
"PM: Cannot start CRC32 thread\n");
1130 * Adjust number of pages for read buffering, in case we are short.
1132 read_pages
= (nr_free_pages() - snapshot_get_image_size()) >> 1;
1133 read_pages
= clamp_val(read_pages
, LZO_CMP_PAGES
, LZO_READ_PAGES
);
1135 for (i
= 0; i
< read_pages
; i
++) {
1136 page
[i
] = (void *)__get_free_page(i
< LZO_CMP_PAGES
?
1137 __GFP_WAIT
| __GFP_HIGH
:
1140 if (i
< LZO_CMP_PAGES
) {
1143 "PM: Failed to allocate LZO pages\n");
1151 want
= ring_size
= i
;
1154 "PM: Using %u thread(s) for decompression.\n"
1155 "PM: Loading and decompressing image data (%u pages) ... ",
1156 nr_threads
, nr_to_read
);
1157 m
= nr_to_read
/ 100;
1162 do_gettimeofday(&start
);
1164 ret
= snapshot_write_next(snapshot
);
1169 for (i
= 0; !eof
&& i
< want
; i
++) {
1170 ret
= swap_read_page(handle
, page
[ring
], &bio
);
1173 * On real read error, finish. On end of data,
1174 * set EOF flag and just exit the read loop.
1177 handle
->cur
->entries
[handle
->k
]) {
1184 if (++ring
>= ring_size
)
1191 * We are out of data, wait for some more.
1197 ret
= hib_wait_on_bio_chain(&bio
);
1206 if (crc
->run_threads
) {
1207 wait_event(crc
->done
, atomic_read(&crc
->stop
));
1208 atomic_set(&crc
->stop
, 0);
1209 crc
->run_threads
= 0;
1212 for (thr
= 0; have
&& thr
< nr_threads
; thr
++) {
1213 data
[thr
].cmp_len
= *(size_t *)page
[pg
];
1214 if (unlikely(!data
[thr
].cmp_len
||
1216 lzo1x_worst_compress(LZO_UNC_SIZE
))) {
1218 "PM: Invalid LZO compressed length\n");
1223 need
= DIV_ROUND_UP(data
[thr
].cmp_len
+ LZO_HEADER
,
1234 off
< LZO_HEADER
+ data
[thr
].cmp_len
;
1236 memcpy(data
[thr
].cmp
+ off
,
1237 page
[pg
], PAGE_SIZE
);
1240 if (++pg
>= ring_size
)
1244 atomic_set(&data
[thr
].ready
, 1);
1245 wake_up(&data
[thr
].go
);
1249 * Wait for more data while we are decompressing.
1251 if (have
< LZO_CMP_PAGES
&& asked
) {
1252 ret
= hib_wait_on_bio_chain(&bio
);
1261 for (run_threads
= thr
, thr
= 0; thr
< run_threads
; thr
++) {
1262 wait_event(data
[thr
].done
,
1263 atomic_read(&data
[thr
].stop
));
1264 atomic_set(&data
[thr
].stop
, 0);
1266 ret
= data
[thr
].ret
;
1270 "PM: LZO decompression failed\n");
1274 if (unlikely(!data
[thr
].unc_len
||
1275 data
[thr
].unc_len
> LZO_UNC_SIZE
||
1276 data
[thr
].unc_len
& (PAGE_SIZE
- 1))) {
1278 "PM: Invalid LZO uncompressed length\n");
1284 off
< data
[thr
].unc_len
; off
+= PAGE_SIZE
) {
1285 memcpy(data_of(*snapshot
),
1286 data
[thr
].unc
+ off
, PAGE_SIZE
);
1288 if (!(nr_pages
% m
))
1289 printk("\b\b\b\b%3d%%", nr_pages
/ m
);
1292 ret
= snapshot_write_next(snapshot
);
1294 crc
->run_threads
= thr
+ 1;
1295 atomic_set(&crc
->ready
, 1);
1302 crc
->run_threads
= thr
;
1303 atomic_set(&crc
->ready
, 1);
1308 if (crc
->run_threads
) {
1309 wait_event(crc
->done
, atomic_read(&crc
->stop
));
1310 atomic_set(&crc
->stop
, 0);
1312 do_gettimeofday(&stop
);
1314 printk("\b\b\b\bdone\n");
1315 snapshot_write_finalize(snapshot
);
1316 if (!snapshot_image_loaded(snapshot
))
1319 if (swsusp_header
->flags
& SF_CRC32_MODE
) {
1320 if(handle
->crc32
!= swsusp_header
->crc32
) {
1322 "PM: Invalid image CRC32!\n");
1329 swsusp_show_speed(&start
, &stop
, nr_to_read
, "Read");
1331 for (i
= 0; i
< ring_size
; i
++)
1332 free_page((unsigned long)page
[i
]);
1335 kthread_stop(crc
->thr
);
1339 for (thr
= 0; thr
< nr_threads
; thr
++)
1341 kthread_stop(data
[thr
].thr
);
1344 if (page
) vfree(page
);
1350 * swsusp_read - read the hibernation image.
1351 * @flags_p: flags passed by the "frozen" kernel in the image header should
1352 * be written into this memory location
1355 int swsusp_read(unsigned int *flags_p
)
1358 struct swap_map_handle handle
;
1359 struct snapshot_handle snapshot
;
1360 struct swsusp_info
*header
;
1362 memset(&snapshot
, 0, sizeof(struct snapshot_handle
));
1363 error
= snapshot_write_next(&snapshot
);
1364 if (error
< PAGE_SIZE
)
1365 return error
< 0 ? error
: -EFAULT
;
1366 header
= (struct swsusp_info
*)data_of(snapshot
);
1367 error
= get_swap_reader(&handle
, flags_p
);
1371 error
= swap_read_page(&handle
, header
, NULL
);
1373 error
= (*flags_p
& SF_NOCOMPRESS_MODE
) ?
1374 load_image(&handle
, &snapshot
, header
->pages
- 1) :
1375 load_image_lzo(&handle
, &snapshot
, header
->pages
- 1);
1377 swap_reader_finish(&handle
);
1380 pr_debug("PM: Image successfully loaded\n");
1382 pr_debug("PM: Error %d resuming\n", error
);
1387 * swsusp_check - Check for swsusp signature in the resume device
1390 int swsusp_check(void)
1394 hib_resume_bdev
= blkdev_get_by_dev(swsusp_resume_device
,
1396 if (!IS_ERR(hib_resume_bdev
)) {
1397 set_blocksize(hib_resume_bdev
, PAGE_SIZE
);
1398 clear_page(swsusp_header
);
1399 error
= hib_bio_read_page(swsusp_resume_block
,
1400 swsusp_header
, NULL
);
1404 if (!memcmp(HIBERNATE_SIG
, swsusp_header
->sig
, 10)) {
1405 memcpy(swsusp_header
->sig
, swsusp_header
->orig_sig
, 10);
1406 /* Reset swap signature now */
1407 error
= hib_bio_write_page(swsusp_resume_block
,
1408 swsusp_header
, NULL
);
1415 blkdev_put(hib_resume_bdev
, FMODE_READ
);
1417 pr_debug("PM: Image signature found, resuming\n");
1419 error
= PTR_ERR(hib_resume_bdev
);
1423 pr_debug("PM: Image not found (code %d)\n", error
);
1429 * swsusp_close - close swap device.
1432 void swsusp_close(fmode_t mode
)
1434 if (IS_ERR(hib_resume_bdev
)) {
1435 pr_debug("PM: Image device not initialised\n");
1439 blkdev_put(hib_resume_bdev
, mode
);
1442 static int swsusp_header_init(void)
1444 swsusp_header
= (struct swsusp_header
*) __get_free_page(GFP_KERNEL
);
1446 panic("Could not allocate memory for swsusp_header\n");
1450 core_initcall(swsusp_header_init
);
This page took 0.061019 seconds and 5 git commands to generate.