2 * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2006-2008 Red Hat GmbH
5 * This file is released under the GPL.
8 #include "dm-exception-store.h"
11 #include <linux/pagemap.h>
12 #include <linux/vmalloc.h>
13 #include <linux/export.h>
14 #include <linux/slab.h>
15 #include <linux/dm-io.h>
18 #define DM_MSG_PREFIX "persistent snapshot"
19 #define DM_CHUNK_SIZE_DEFAULT_SECTORS 32 /* 16KB */
21 /*-----------------------------------------------------------------
22 * Persistent snapshots, by persistent we mean that the snapshot
23 * will survive a reboot.
24 *---------------------------------------------------------------*/
27 * We need to store a record of which parts of the origin have
28 * been copied to the snapshot device. The snapshot code
29 * requires that we copy exception chunks to chunk aligned areas
30 * of the COW store. It makes sense therefore, to store the
31 * metadata in chunk size blocks.
33 * There is no backward or forward compatibility implemented,
34 * snapshots with different disk versions than the kernel will
35 * not be usable. It is expected that "lvcreate" will blank out
36 * the start of a fresh COW device before calling the snapshot
39 * The first chunk of the COW device just contains the header.
40 * After this there is a chunk filled with exception metadata,
41 * followed by as many exception chunks as can fit in the
44 * All on disk structures are in little-endian format. The end
45 * of the exceptions info is indicated by an exception with a
46 * new_chunk of 0, which is invalid since it would point to the
51 * Magic for persistent snapshots: "SnAp" - Feeble isn't it.
53 #define SNAP_MAGIC 0x70416e53
56 * The on-disk version of the metadata.
58 #define SNAPSHOT_DISK_VERSION 1
60 #define NUM_SNAPSHOT_HDR_CHUNKS 1
66 * Is this snapshot valid. There is no way of recovering
67 * an invalid snapshot.
72 * Simple, incrementing version. no backward
81 struct disk_exception
{
86 struct core_exception
{
91 struct commit_callback
{
92 void (*callback
)(void *, int success
);
97 * The top level structure for a persistent exception store.
100 struct dm_exception_store
*store
;
103 uint32_t exceptions_per_area
;
106 * Now that we have an asynchronous kcopyd there is no
107 * need for large chunk sizes, so it wont hurt to have a
108 * whole chunks worth of metadata in memory at once.
113 * An area of zeros used to clear the next area.
118 * An area used for header. The header can be written
119 * concurrently with metadata (when invalidating the snapshot),
120 * so it needs a separate buffer.
125 * Used to keep track of which metadata area the data in
128 chunk_t current_area
;
131 * The next free chunk for an exception.
133 * When creating exceptions, all the chunks here and above are
134 * free. It holds the next chunk to be allocated. On rare
135 * occasions (e.g. after a system crash) holes can be left in
136 * the exception store because chunks can be committed out of
139 * When merging exceptions, it does not necessarily mean all the
140 * chunks here and above are free. It holds the value it would
141 * have held if all chunks had been committed in order of
142 * allocation. Consequently the value may occasionally be
143 * slightly too low, but since it's only used for 'status' and
144 * it can never reach its minimum value too early this doesn't
151 * The index of next free exception in the current
154 uint32_t current_committed
;
156 atomic_t pending_count
;
157 uint32_t callback_count
;
158 struct commit_callback
*callbacks
;
159 struct dm_io_client
*io_client
;
161 struct workqueue_struct
*metadata_wq
;
164 static int alloc_area(struct pstore
*ps
)
169 len
= ps
->store
->chunk_size
<< SECTOR_SHIFT
;
172 * Allocate the chunk_size block of memory that will hold
173 * a single metadata area.
175 ps
->area
= vmalloc(len
);
179 ps
->zero_area
= vzalloc(len
);
183 ps
->header_area
= vmalloc(len
);
184 if (!ps
->header_area
)
185 goto err_header_area
;
190 vfree(ps
->zero_area
);
199 static void free_area(struct pstore
*ps
)
206 vfree(ps
->zero_area
);
207 ps
->zero_area
= NULL
;
210 vfree(ps
->header_area
);
211 ps
->header_area
= NULL
;
215 struct dm_io_region
*where
;
216 struct dm_io_request
*io_req
;
217 struct work_struct work
;
221 static void do_metadata(struct work_struct
*work
)
223 struct mdata_req
*req
= container_of(work
, struct mdata_req
, work
);
225 req
->result
= dm_io(req
->io_req
, 1, req
->where
, NULL
);
229 * Read or write a chunk aligned and sized block of data from a device.
231 static int chunk_io(struct pstore
*ps
, void *area
, chunk_t chunk
, int rw
,
234 struct dm_io_region where
= {
235 .bdev
= dm_snap_cow(ps
->store
->snap
)->bdev
,
236 .sector
= ps
->store
->chunk_size
* chunk
,
237 .count
= ps
->store
->chunk_size
,
239 struct dm_io_request io_req
= {
241 .mem
.type
= DM_IO_VMA
,
243 .client
= ps
->io_client
,
246 struct mdata_req req
;
249 return dm_io(&io_req
, 1, &where
, NULL
);
252 req
.io_req
= &io_req
;
255 * Issue the synchronous I/O from a different thread
256 * to avoid generic_make_request recursion.
258 INIT_WORK_ONSTACK(&req
.work
, do_metadata
);
259 queue_work(ps
->metadata_wq
, &req
.work
);
260 flush_workqueue(ps
->metadata_wq
);
261 destroy_work_on_stack(&req
.work
);
267 * Convert a metadata area index to a chunk index.
269 static chunk_t
area_location(struct pstore
*ps
, chunk_t area
)
271 return NUM_SNAPSHOT_HDR_CHUNKS
+ ((ps
->exceptions_per_area
+ 1) * area
);
274 static void skip_metadata(struct pstore
*ps
)
276 uint32_t stride
= ps
->exceptions_per_area
+ 1;
277 chunk_t next_free
= ps
->next_free
;
278 if (sector_div(next_free
, stride
) == NUM_SNAPSHOT_HDR_CHUNKS
)
283 * Read or write a metadata area. Remembering to skip the first
284 * chunk which holds the header.
286 static int area_io(struct pstore
*ps
, int rw
)
291 chunk
= area_location(ps
, ps
->current_area
);
293 r
= chunk_io(ps
, ps
->area
, chunk
, rw
, 0);
300 static void zero_memory_area(struct pstore
*ps
)
302 memset(ps
->area
, 0, ps
->store
->chunk_size
<< SECTOR_SHIFT
);
305 static int zero_disk_area(struct pstore
*ps
, chunk_t area
)
307 return chunk_io(ps
, ps
->zero_area
, area_location(ps
, area
), WRITE
, 0);
310 static int read_header(struct pstore
*ps
, int *new_snapshot
)
313 struct disk_header
*dh
;
315 int chunk_size_supplied
= 1;
319 * Use default chunk size (or logical_block_size, if larger)
322 if (!ps
->store
->chunk_size
) {
323 ps
->store
->chunk_size
= max(DM_CHUNK_SIZE_DEFAULT_SECTORS
,
324 bdev_logical_block_size(dm_snap_cow(ps
->store
->snap
)->
326 ps
->store
->chunk_mask
= ps
->store
->chunk_size
- 1;
327 ps
->store
->chunk_shift
= ffs(ps
->store
->chunk_size
) - 1;
328 chunk_size_supplied
= 0;
331 ps
->io_client
= dm_io_client_create();
332 if (IS_ERR(ps
->io_client
))
333 return PTR_ERR(ps
->io_client
);
339 r
= chunk_io(ps
, ps
->header_area
, 0, READ
, 1);
343 dh
= ps
->header_area
;
345 if (le32_to_cpu(dh
->magic
) == 0) {
350 if (le32_to_cpu(dh
->magic
) != SNAP_MAGIC
) {
351 DMWARN("Invalid or corrupt snapshot");
357 ps
->valid
= le32_to_cpu(dh
->valid
);
358 ps
->version
= le32_to_cpu(dh
->version
);
359 chunk_size
= le32_to_cpu(dh
->chunk_size
);
361 if (ps
->store
->chunk_size
== chunk_size
)
364 if (chunk_size_supplied
)
365 DMWARN("chunk size %u in device metadata overrides "
366 "table chunk size of %u.",
367 chunk_size
, ps
->store
->chunk_size
);
369 /* We had a bogus chunk_size. Fix stuff up. */
372 r
= dm_exception_store_set_chunk_size(ps
->store
, chunk_size
,
375 DMERR("invalid on-disk chunk size %u: %s.",
376 chunk_size
, chunk_err
);
388 static int write_header(struct pstore
*ps
)
390 struct disk_header
*dh
;
392 memset(ps
->header_area
, 0, ps
->store
->chunk_size
<< SECTOR_SHIFT
);
394 dh
= ps
->header_area
;
395 dh
->magic
= cpu_to_le32(SNAP_MAGIC
);
396 dh
->valid
= cpu_to_le32(ps
->valid
);
397 dh
->version
= cpu_to_le32(ps
->version
);
398 dh
->chunk_size
= cpu_to_le32(ps
->store
->chunk_size
);
400 return chunk_io(ps
, ps
->header_area
, 0, WRITE
, 1);
404 * Access functions for the disk exceptions, these do the endian conversions.
406 static struct disk_exception
*get_exception(struct pstore
*ps
, void *ps_area
,
409 BUG_ON(index
>= ps
->exceptions_per_area
);
411 return ((struct disk_exception
*) ps_area
) + index
;
414 static void read_exception(struct pstore
*ps
, void *ps_area
,
415 uint32_t index
, struct core_exception
*result
)
417 struct disk_exception
*de
= get_exception(ps
, ps_area
, index
);
420 result
->old_chunk
= le64_to_cpu(de
->old_chunk
);
421 result
->new_chunk
= le64_to_cpu(de
->new_chunk
);
424 static void write_exception(struct pstore
*ps
,
425 uint32_t index
, struct core_exception
*e
)
427 struct disk_exception
*de
= get_exception(ps
, ps
->area
, index
);
430 de
->old_chunk
= cpu_to_le64(e
->old_chunk
);
431 de
->new_chunk
= cpu_to_le64(e
->new_chunk
);
434 static void clear_exception(struct pstore
*ps
, uint32_t index
)
436 struct disk_exception
*de
= get_exception(ps
, ps
->area
, index
);
444 * Registers the exceptions that are present in the current area.
445 * 'full' is filled in to indicate if the area has been
448 static int insert_exceptions(struct pstore
*ps
, void *ps_area
,
449 int (*callback
)(void *callback_context
,
450 chunk_t old
, chunk_t
new),
451 void *callback_context
,
456 struct core_exception e
;
458 /* presume the area is full */
461 for (i
= 0; i
< ps
->exceptions_per_area
; i
++) {
462 read_exception(ps
, ps_area
, i
, &e
);
465 * If the new_chunk is pointing at the start of
466 * the COW device, where the first metadata area
467 * is we know that we've hit the end of the
468 * exceptions. Therefore the area is not full.
470 if (e
.new_chunk
== 0LL) {
471 ps
->current_committed
= i
;
477 * Keep track of the start of the free chunks.
479 if (ps
->next_free
<= e
.new_chunk
)
480 ps
->next_free
= e
.new_chunk
+ 1;
483 * Otherwise we add the exception to the snapshot.
485 r
= callback(callback_context
, e
.old_chunk
, e
.new_chunk
);
493 static int read_exceptions(struct pstore
*ps
,
494 int (*callback
)(void *callback_context
, chunk_t old
,
496 void *callback_context
)
499 struct dm_bufio_client
*client
;
501 client
= dm_bufio_client_create(dm_snap_cow(ps
->store
->snap
)->bdev
,
502 ps
->store
->chunk_size
<< SECTOR_SHIFT
,
506 return PTR_ERR(client
);
509 * Keeping reading chunks and inserting exceptions until
510 * we find a partially full area.
512 for (ps
->current_area
= 0; full
; ps
->current_area
++) {
513 struct dm_buffer
*bp
;
515 chunk_t chunk
= area_location(ps
, ps
->current_area
);
517 area
= dm_bufio_read(client
, chunk
, &bp
);
518 if (unlikely(IS_ERR(area
))) {
520 goto ret_destroy_bufio
;
523 r
= insert_exceptions(ps
, area
, callback
, callback_context
,
526 dm_bufio_release(bp
);
528 dm_bufio_forget(client
, chunk
);
531 goto ret_destroy_bufio
;
541 dm_bufio_client_destroy(client
);
546 static struct pstore
*get_info(struct dm_exception_store
*store
)
548 return (struct pstore
*) store
->context
;
551 static void persistent_usage(struct dm_exception_store
*store
,
552 sector_t
*total_sectors
,
553 sector_t
*sectors_allocated
,
554 sector_t
*metadata_sectors
)
556 struct pstore
*ps
= get_info(store
);
558 *sectors_allocated
= ps
->next_free
* store
->chunk_size
;
559 *total_sectors
= get_dev_size(dm_snap_cow(store
->snap
)->bdev
);
562 * First chunk is the fixed header.
563 * Then there are (ps->current_area + 1) metadata chunks, each one
564 * separated from the next by ps->exceptions_per_area data chunks.
566 *metadata_sectors
= (ps
->current_area
+ 1 + NUM_SNAPSHOT_HDR_CHUNKS
) *
570 static void persistent_dtr(struct dm_exception_store
*store
)
572 struct pstore
*ps
= get_info(store
);
574 destroy_workqueue(ps
->metadata_wq
);
576 /* Created in read_header */
578 dm_io_client_destroy(ps
->io_client
);
581 /* Allocated in persistent_read_metadata */
583 vfree(ps
->callbacks
);
588 static int persistent_read_metadata(struct dm_exception_store
*store
,
589 int (*callback
)(void *callback_context
,
590 chunk_t old
, chunk_t
new),
591 void *callback_context
)
593 int r
, uninitialized_var(new_snapshot
);
594 struct pstore
*ps
= get_info(store
);
597 * Read the snapshot header.
599 r
= read_header(ps
, &new_snapshot
);
604 * Now we know correct chunk_size, complete the initialisation.
606 ps
->exceptions_per_area
= (ps
->store
->chunk_size
<< SECTOR_SHIFT
) /
607 sizeof(struct disk_exception
);
608 ps
->callbacks
= dm_vcalloc(ps
->exceptions_per_area
,
609 sizeof(*ps
->callbacks
));
614 * Do we need to setup a new snapshot ?
617 r
= write_header(ps
);
619 DMWARN("write_header failed");
623 ps
->current_area
= 0;
624 zero_memory_area(ps
);
625 r
= zero_disk_area(ps
, 0);
627 DMWARN("zero_disk_area(0) failed");
633 if (ps
->version
!= SNAPSHOT_DISK_VERSION
) {
634 DMWARN("unable to handle snapshot disk version %d",
640 * Metadata are valid, but snapshot is invalidated
648 r
= read_exceptions(ps
, callback
, callback_context
);
653 static int persistent_prepare_exception(struct dm_exception_store
*store
,
654 struct dm_exception
*e
)
656 struct pstore
*ps
= get_info(store
);
657 sector_t size
= get_dev_size(dm_snap_cow(store
->snap
)->bdev
);
659 /* Is there enough room ? */
660 if (size
< ((ps
->next_free
+ 1) * store
->chunk_size
))
663 e
->new_chunk
= ps
->next_free
;
666 * Move onto the next free pending, making sure to take
667 * into account the location of the metadata chunks.
672 atomic_inc(&ps
->pending_count
);
676 static void persistent_commit_exception(struct dm_exception_store
*store
,
677 struct dm_exception
*e
,
678 void (*callback
) (void *, int success
),
679 void *callback_context
)
682 struct pstore
*ps
= get_info(store
);
683 struct core_exception ce
;
684 struct commit_callback
*cb
;
686 ce
.old_chunk
= e
->old_chunk
;
687 ce
.new_chunk
= e
->new_chunk
;
688 write_exception(ps
, ps
->current_committed
++, &ce
);
691 * Add the callback to the back of the array. This code
692 * is the only place where the callback array is
693 * manipulated, and we know that it will never be called
694 * multiple times concurrently.
696 cb
= ps
->callbacks
+ ps
->callback_count
++;
697 cb
->callback
= callback
;
698 cb
->context
= callback_context
;
701 * If there are exceptions in flight and we have not yet
702 * filled this metadata area there's nothing more to do.
704 if (!atomic_dec_and_test(&ps
->pending_count
) &&
705 (ps
->current_committed
!= ps
->exceptions_per_area
))
709 * If we completely filled the current area, then wipe the next one.
711 if ((ps
->current_committed
== ps
->exceptions_per_area
) &&
712 zero_disk_area(ps
, ps
->current_area
+ 1))
716 * Commit exceptions to disk.
718 if (ps
->valid
&& area_io(ps
, WRITE_FLUSH_FUA
))
722 * Advance to the next area if this one is full.
724 if (ps
->current_committed
== ps
->exceptions_per_area
) {
725 ps
->current_committed
= 0;
727 zero_memory_area(ps
);
730 for (i
= 0; i
< ps
->callback_count
; i
++) {
731 cb
= ps
->callbacks
+ i
;
732 cb
->callback(cb
->context
, ps
->valid
);
735 ps
->callback_count
= 0;
738 static int persistent_prepare_merge(struct dm_exception_store
*store
,
739 chunk_t
*last_old_chunk
,
740 chunk_t
*last_new_chunk
)
742 struct pstore
*ps
= get_info(store
);
743 struct core_exception ce
;
748 * When current area is empty, move back to preceding area.
750 if (!ps
->current_committed
) {
754 if (!ps
->current_area
)
758 r
= area_io(ps
, READ
);
761 ps
->current_committed
= ps
->exceptions_per_area
;
764 read_exception(ps
, ps
->area
, ps
->current_committed
- 1, &ce
);
765 *last_old_chunk
= ce
.old_chunk
;
766 *last_new_chunk
= ce
.new_chunk
;
769 * Find number of consecutive chunks within the current area,
772 for (nr_consecutive
= 1; nr_consecutive
< ps
->current_committed
;
774 read_exception(ps
, ps
->area
,
775 ps
->current_committed
- 1 - nr_consecutive
, &ce
);
776 if (ce
.old_chunk
!= *last_old_chunk
- nr_consecutive
||
777 ce
.new_chunk
!= *last_new_chunk
- nr_consecutive
)
781 return nr_consecutive
;
784 static int persistent_commit_merge(struct dm_exception_store
*store
,
788 struct pstore
*ps
= get_info(store
);
790 BUG_ON(nr_merged
> ps
->current_committed
);
792 for (i
= 0; i
< nr_merged
; i
++)
793 clear_exception(ps
, ps
->current_committed
- 1 - i
);
795 r
= area_io(ps
, WRITE_FLUSH_FUA
);
799 ps
->current_committed
-= nr_merged
;
802 * At this stage, only persistent_usage() uses ps->next_free, so
803 * we make no attempt to keep ps->next_free strictly accurate
804 * as exceptions may have been committed out-of-order originally.
805 * Once a snapshot has become merging, we set it to the value it
806 * would have held had all the exceptions been committed in order.
808 * ps->current_area does not get reduced by prepare_merge() until
809 * after commit_merge() has removed the nr_merged previous exceptions.
811 ps
->next_free
= area_location(ps
, ps
->current_area
) +
812 ps
->current_committed
+ 1;
817 static void persistent_drop_snapshot(struct dm_exception_store
*store
)
819 struct pstore
*ps
= get_info(store
);
822 if (write_header(ps
))
823 DMWARN("write header failed");
826 static int persistent_ctr(struct dm_exception_store
*store
,
827 unsigned argc
, char **argv
)
831 /* allocate the pstore */
832 ps
= kzalloc(sizeof(*ps
), GFP_KERNEL
);
838 ps
->version
= SNAPSHOT_DISK_VERSION
;
840 ps
->zero_area
= NULL
;
841 ps
->header_area
= NULL
;
842 ps
->next_free
= NUM_SNAPSHOT_HDR_CHUNKS
+ 1; /* header and 1st area */
843 ps
->current_committed
= 0;
845 ps
->callback_count
= 0;
846 atomic_set(&ps
->pending_count
, 0);
847 ps
->callbacks
= NULL
;
849 ps
->metadata_wq
= alloc_workqueue("ksnaphd", WQ_MEM_RECLAIM
, 0);
850 if (!ps
->metadata_wq
) {
852 DMERR("couldn't start header metadata update thread");
861 static unsigned persistent_status(struct dm_exception_store
*store
,
862 status_type_t status
, char *result
,
868 case STATUSTYPE_INFO
:
870 case STATUSTYPE_TABLE
:
871 DMEMIT(" P %llu", (unsigned long long)store
->chunk_size
);
877 static struct dm_exception_store_type _persistent_type
= {
878 .name
= "persistent",
879 .module
= THIS_MODULE
,
880 .ctr
= persistent_ctr
,
881 .dtr
= persistent_dtr
,
882 .read_metadata
= persistent_read_metadata
,
883 .prepare_exception
= persistent_prepare_exception
,
884 .commit_exception
= persistent_commit_exception
,
885 .prepare_merge
= persistent_prepare_merge
,
886 .commit_merge
= persistent_commit_merge
,
887 .drop_snapshot
= persistent_drop_snapshot
,
888 .usage
= persistent_usage
,
889 .status
= persistent_status
,
892 static struct dm_exception_store_type _persistent_compat_type
= {
894 .module
= THIS_MODULE
,
895 .ctr
= persistent_ctr
,
896 .dtr
= persistent_dtr
,
897 .read_metadata
= persistent_read_metadata
,
898 .prepare_exception
= persistent_prepare_exception
,
899 .commit_exception
= persistent_commit_exception
,
900 .prepare_merge
= persistent_prepare_merge
,
901 .commit_merge
= persistent_commit_merge
,
902 .drop_snapshot
= persistent_drop_snapshot
,
903 .usage
= persistent_usage
,
904 .status
= persistent_status
,
907 int dm_persistent_snapshot_init(void)
911 r
= dm_exception_store_type_register(&_persistent_type
);
913 DMERR("Unable to register persistent exception store type");
917 r
= dm_exception_store_type_register(&_persistent_compat_type
);
919 DMERR("Unable to register old-style persistent exception "
921 dm_exception_store_type_unregister(&_persistent_type
);
928 void dm_persistent_snapshot_exit(void)
930 dm_exception_store_type_unregister(&_persistent_type
);
931 dm_exception_store_type_unregister(&_persistent_compat_type
);