2 * Copyright (C) 2012 Red Hat, Inc.
4 * This file is released under the GPL.
7 #include "dm-cache-metadata.h"
9 #include "persistent-data/dm-array.h"
10 #include "persistent-data/dm-bitset.h"
11 #include "persistent-data/dm-space-map.h"
12 #include "persistent-data/dm-space-map-disk.h"
13 #include "persistent-data/dm-transaction-manager.h"
15 #include <linux/device-mapper.h>
17 /*----------------------------------------------------------------*/
19 #define DM_MSG_PREFIX "cache metadata"
21 #define CACHE_SUPERBLOCK_MAGIC 06142003
22 #define CACHE_SUPERBLOCK_LOCATION 0
25 * defines a range of metadata versions that this module can handle.
27 #define MIN_CACHE_VERSION 1
28 #define MAX_CACHE_VERSION 1
30 #define CACHE_METADATA_CACHE_SIZE 64
33 * 3 for btree insert +
34 * 2 for btree lookup used within space map
36 #define CACHE_MAX_CONCURRENT_LOCKS 5
37 #define SPACE_MAP_ROOT_SIZE 128
39 enum superblock_flag_bits
{
40 /* for spotting crashes that would invalidate the dirty bitset */
42 /* metadata must be checked using the tools */
47 * Each mapping from cache block -> origin block carries a set of flags.
51 * A valid mapping. Because we're using an array we clear this
52 * flag for an non existant mapping.
57 * The data on the cache is different from that on the origin.
62 struct cache_disk_superblock
{
71 __u8 policy_name
[CACHE_POLICY_NAME_SIZE
];
72 __le32 policy_hint_size
;
74 __u8 metadata_space_map_root
[SPACE_MAP_ROOT_SIZE
];
79 __le64 discard_block_size
;
80 __le64 discard_nr_blocks
;
82 __le32 data_block_size
;
83 __le32 metadata_block_size
;
87 __le32 compat_ro_flags
;
88 __le32 incompat_flags
;
95 __le32 policy_version
[CACHE_POLICY_VERSION_SIZE
];
98 struct dm_cache_metadata
{
100 struct list_head list
;
102 struct block_device
*bdev
;
103 struct dm_block_manager
*bm
;
104 struct dm_space_map
*metadata_sm
;
105 struct dm_transaction_manager
*tm
;
107 struct dm_array_info info
;
108 struct dm_array_info hint_info
;
109 struct dm_disk_bitset discard_info
;
111 struct rw_semaphore root_lock
;
114 dm_block_t hint_root
;
115 dm_block_t discard_root
;
117 sector_t discard_block_size
;
118 dm_dblock_t discard_nr_blocks
;
120 sector_t data_block_size
;
121 dm_cblock_t cache_blocks
;
123 bool clean_when_opened
:1;
125 char policy_name
[CACHE_POLICY_NAME_SIZE
];
126 unsigned policy_version
[CACHE_POLICY_VERSION_SIZE
];
127 size_t policy_hint_size
;
128 struct dm_cache_statistics stats
;
131 * Reading the space map root can fail, so we read it into this
132 * buffer before the superblock is locked and updated.
134 __u8 metadata_space_map_root
[SPACE_MAP_ROOT_SIZE
];
137 * Set if a transaction has to be aborted but the attempt to roll
138 * back to the previous (good) transaction failed. The only
139 * metadata operation permissible in this state is the closing of
145 /*-------------------------------------------------------------------
146 * superblock validator
147 *-----------------------------------------------------------------*/
149 #define SUPERBLOCK_CSUM_XOR 9031977
151 static void sb_prepare_for_write(struct dm_block_validator
*v
,
153 size_t sb_block_size
)
155 struct cache_disk_superblock
*disk_super
= dm_block_data(b
);
157 disk_super
->blocknr
= cpu_to_le64(dm_block_location(b
));
158 disk_super
->csum
= cpu_to_le32(dm_bm_checksum(&disk_super
->flags
,
159 sb_block_size
- sizeof(__le32
),
160 SUPERBLOCK_CSUM_XOR
));
163 static int check_metadata_version(struct cache_disk_superblock
*disk_super
)
165 uint32_t metadata_version
= le32_to_cpu(disk_super
->version
);
166 if (metadata_version
< MIN_CACHE_VERSION
|| metadata_version
> MAX_CACHE_VERSION
) {
167 DMERR("Cache metadata version %u found, but only versions between %u and %u supported.",
168 metadata_version
, MIN_CACHE_VERSION
, MAX_CACHE_VERSION
);
175 static int sb_check(struct dm_block_validator
*v
,
177 size_t sb_block_size
)
179 struct cache_disk_superblock
*disk_super
= dm_block_data(b
);
182 if (dm_block_location(b
) != le64_to_cpu(disk_super
->blocknr
)) {
183 DMERR("sb_check failed: blocknr %llu: wanted %llu",
184 le64_to_cpu(disk_super
->blocknr
),
185 (unsigned long long)dm_block_location(b
));
189 if (le64_to_cpu(disk_super
->magic
) != CACHE_SUPERBLOCK_MAGIC
) {
190 DMERR("sb_check failed: magic %llu: wanted %llu",
191 le64_to_cpu(disk_super
->magic
),
192 (unsigned long long)CACHE_SUPERBLOCK_MAGIC
);
196 csum_le
= cpu_to_le32(dm_bm_checksum(&disk_super
->flags
,
197 sb_block_size
- sizeof(__le32
),
198 SUPERBLOCK_CSUM_XOR
));
199 if (csum_le
!= disk_super
->csum
) {
200 DMERR("sb_check failed: csum %u: wanted %u",
201 le32_to_cpu(csum_le
), le32_to_cpu(disk_super
->csum
));
205 return check_metadata_version(disk_super
);
208 static struct dm_block_validator sb_validator
= {
209 .name
= "superblock",
210 .prepare_for_write
= sb_prepare_for_write
,
214 /*----------------------------------------------------------------*/
216 static int superblock_read_lock(struct dm_cache_metadata
*cmd
,
217 struct dm_block
**sblock
)
219 return dm_bm_read_lock(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
220 &sb_validator
, sblock
);
223 static int superblock_lock_zero(struct dm_cache_metadata
*cmd
,
224 struct dm_block
**sblock
)
226 return dm_bm_write_lock_zero(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
227 &sb_validator
, sblock
);
230 static int superblock_lock(struct dm_cache_metadata
*cmd
,
231 struct dm_block
**sblock
)
233 return dm_bm_write_lock(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
234 &sb_validator
, sblock
);
237 /*----------------------------------------------------------------*/
239 static int __superblock_all_zeroes(struct dm_block_manager
*bm
, bool *result
)
244 __le64
*data_le
, zero
= cpu_to_le64(0);
245 unsigned sb_block_size
= dm_bm_block_size(bm
) / sizeof(__le64
);
248 * We can't use a validator here - it may be all zeroes.
250 r
= dm_bm_read_lock(bm
, CACHE_SUPERBLOCK_LOCATION
, NULL
, &b
);
254 data_le
= dm_block_data(b
);
256 for (i
= 0; i
< sb_block_size
; i
++) {
257 if (data_le
[i
] != zero
) {
263 return dm_bm_unlock(b
);
266 static void __setup_mapping_info(struct dm_cache_metadata
*cmd
)
268 struct dm_btree_value_type vt
;
271 vt
.size
= sizeof(__le64
);
275 dm_array_info_init(&cmd
->info
, cmd
->tm
, &vt
);
277 if (cmd
->policy_hint_size
) {
278 vt
.size
= sizeof(__le32
);
279 dm_array_info_init(&cmd
->hint_info
, cmd
->tm
, &vt
);
283 static int __save_sm_root(struct dm_cache_metadata
*cmd
)
288 r
= dm_sm_root_size(cmd
->metadata_sm
, &metadata_len
);
292 return dm_sm_copy_root(cmd
->metadata_sm
, &cmd
->metadata_space_map_root
,
296 static void __copy_sm_root(struct dm_cache_metadata
*cmd
,
297 struct cache_disk_superblock
*disk_super
)
299 memcpy(&disk_super
->metadata_space_map_root
,
300 &cmd
->metadata_space_map_root
,
301 sizeof(cmd
->metadata_space_map_root
));
304 static int __write_initial_superblock(struct dm_cache_metadata
*cmd
)
307 struct dm_block
*sblock
;
308 struct cache_disk_superblock
*disk_super
;
309 sector_t bdev_size
= i_size_read(cmd
->bdev
->bd_inode
) >> SECTOR_SHIFT
;
311 /* FIXME: see if we can lose the max sectors limit */
312 if (bdev_size
> DM_CACHE_METADATA_MAX_SECTORS
)
313 bdev_size
= DM_CACHE_METADATA_MAX_SECTORS
;
315 r
= dm_tm_pre_commit(cmd
->tm
);
320 * dm_sm_copy_root() can fail. So we need to do it before we start
321 * updating the superblock.
323 r
= __save_sm_root(cmd
);
327 r
= superblock_lock_zero(cmd
, &sblock
);
331 disk_super
= dm_block_data(sblock
);
332 disk_super
->flags
= 0;
333 memset(disk_super
->uuid
, 0, sizeof(disk_super
->uuid
));
334 disk_super
->magic
= cpu_to_le64(CACHE_SUPERBLOCK_MAGIC
);
335 disk_super
->version
= cpu_to_le32(MAX_CACHE_VERSION
);
336 memset(disk_super
->policy_name
, 0, sizeof(disk_super
->policy_name
));
337 memset(disk_super
->policy_version
, 0, sizeof(disk_super
->policy_version
));
338 disk_super
->policy_hint_size
= 0;
340 __copy_sm_root(cmd
, disk_super
);
342 disk_super
->mapping_root
= cpu_to_le64(cmd
->root
);
343 disk_super
->hint_root
= cpu_to_le64(cmd
->hint_root
);
344 disk_super
->discard_root
= cpu_to_le64(cmd
->discard_root
);
345 disk_super
->discard_block_size
= cpu_to_le64(cmd
->discard_block_size
);
346 disk_super
->discard_nr_blocks
= cpu_to_le64(from_dblock(cmd
->discard_nr_blocks
));
347 disk_super
->metadata_block_size
= cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE
);
348 disk_super
->data_block_size
= cpu_to_le32(cmd
->data_block_size
);
349 disk_super
->cache_blocks
= cpu_to_le32(0);
351 disk_super
->read_hits
= cpu_to_le32(0);
352 disk_super
->read_misses
= cpu_to_le32(0);
353 disk_super
->write_hits
= cpu_to_le32(0);
354 disk_super
->write_misses
= cpu_to_le32(0);
356 return dm_tm_commit(cmd
->tm
, sblock
);
359 static int __format_metadata(struct dm_cache_metadata
*cmd
)
363 r
= dm_tm_create_with_sm(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
364 &cmd
->tm
, &cmd
->metadata_sm
);
366 DMERR("tm_create_with_sm failed");
370 __setup_mapping_info(cmd
);
372 r
= dm_array_empty(&cmd
->info
, &cmd
->root
);
376 dm_disk_bitset_init(cmd
->tm
, &cmd
->discard_info
);
378 r
= dm_bitset_empty(&cmd
->discard_info
, &cmd
->discard_root
);
382 cmd
->discard_block_size
= 0;
383 cmd
->discard_nr_blocks
= 0;
385 r
= __write_initial_superblock(cmd
);
389 cmd
->clean_when_opened
= true;
393 dm_tm_destroy(cmd
->tm
);
394 dm_sm_destroy(cmd
->metadata_sm
);
399 static int __check_incompat_features(struct cache_disk_superblock
*disk_super
,
400 struct dm_cache_metadata
*cmd
)
404 features
= le32_to_cpu(disk_super
->incompat_flags
) & ~DM_CACHE_FEATURE_INCOMPAT_SUPP
;
406 DMERR("could not access metadata due to unsupported optional features (%lx).",
407 (unsigned long)features
);
412 * Check for read-only metadata to skip the following RDWR checks.
414 if (get_disk_ro(cmd
->bdev
->bd_disk
))
417 features
= le32_to_cpu(disk_super
->compat_ro_flags
) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP
;
419 DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
420 (unsigned long)features
);
427 static int __open_metadata(struct dm_cache_metadata
*cmd
)
430 struct dm_block
*sblock
;
431 struct cache_disk_superblock
*disk_super
;
432 unsigned long sb_flags
;
434 r
= superblock_read_lock(cmd
, &sblock
);
436 DMERR("couldn't read lock superblock");
440 disk_super
= dm_block_data(sblock
);
442 /* Verify the data block size hasn't changed */
443 if (le32_to_cpu(disk_super
->data_block_size
) != cmd
->data_block_size
) {
444 DMERR("changing the data block size (from %u to %llu) is not supported",
445 le32_to_cpu(disk_super
->data_block_size
),
446 (unsigned long long)cmd
->data_block_size
);
451 r
= __check_incompat_features(disk_super
, cmd
);
455 r
= dm_tm_open_with_sm(cmd
->bm
, CACHE_SUPERBLOCK_LOCATION
,
456 disk_super
->metadata_space_map_root
,
457 sizeof(disk_super
->metadata_space_map_root
),
458 &cmd
->tm
, &cmd
->metadata_sm
);
460 DMERR("tm_open_with_sm failed");
464 __setup_mapping_info(cmd
);
465 dm_disk_bitset_init(cmd
->tm
, &cmd
->discard_info
);
466 sb_flags
= le32_to_cpu(disk_super
->flags
);
467 cmd
->clean_when_opened
= test_bit(CLEAN_SHUTDOWN
, &sb_flags
);
468 return dm_bm_unlock(sblock
);
471 dm_bm_unlock(sblock
);
475 static int __open_or_format_metadata(struct dm_cache_metadata
*cmd
,
479 bool unformatted
= false;
481 r
= __superblock_all_zeroes(cmd
->bm
, &unformatted
);
486 return format_device
? __format_metadata(cmd
) : -EPERM
;
488 return __open_metadata(cmd
);
491 static int __create_persistent_data_objects(struct dm_cache_metadata
*cmd
,
492 bool may_format_device
)
495 cmd
->bm
= dm_block_manager_create(cmd
->bdev
, DM_CACHE_METADATA_BLOCK_SIZE
<< SECTOR_SHIFT
,
496 CACHE_METADATA_CACHE_SIZE
,
497 CACHE_MAX_CONCURRENT_LOCKS
);
498 if (IS_ERR(cmd
->bm
)) {
499 DMERR("could not create block manager");
500 return PTR_ERR(cmd
->bm
);
503 r
= __open_or_format_metadata(cmd
, may_format_device
);
505 dm_block_manager_destroy(cmd
->bm
);
510 static void __destroy_persistent_data_objects(struct dm_cache_metadata
*cmd
)
512 dm_sm_destroy(cmd
->metadata_sm
);
513 dm_tm_destroy(cmd
->tm
);
514 dm_block_manager_destroy(cmd
->bm
);
517 typedef unsigned long (*flags_mutator
)(unsigned long);
519 static void update_flags(struct cache_disk_superblock
*disk_super
,
520 flags_mutator mutator
)
522 uint32_t sb_flags
= mutator(le32_to_cpu(disk_super
->flags
));
523 disk_super
->flags
= cpu_to_le32(sb_flags
);
526 static unsigned long set_clean_shutdown(unsigned long flags
)
528 set_bit(CLEAN_SHUTDOWN
, &flags
);
532 static unsigned long clear_clean_shutdown(unsigned long flags
)
534 clear_bit(CLEAN_SHUTDOWN
, &flags
);
538 static void read_superblock_fields(struct dm_cache_metadata
*cmd
,
539 struct cache_disk_superblock
*disk_super
)
541 cmd
->flags
= le32_to_cpu(disk_super
->flags
);
542 cmd
->root
= le64_to_cpu(disk_super
->mapping_root
);
543 cmd
->hint_root
= le64_to_cpu(disk_super
->hint_root
);
544 cmd
->discard_root
= le64_to_cpu(disk_super
->discard_root
);
545 cmd
->discard_block_size
= le64_to_cpu(disk_super
->discard_block_size
);
546 cmd
->discard_nr_blocks
= to_dblock(le64_to_cpu(disk_super
->discard_nr_blocks
));
547 cmd
->data_block_size
= le32_to_cpu(disk_super
->data_block_size
);
548 cmd
->cache_blocks
= to_cblock(le32_to_cpu(disk_super
->cache_blocks
));
549 strncpy(cmd
->policy_name
, disk_super
->policy_name
, sizeof(cmd
->policy_name
));
550 cmd
->policy_version
[0] = le32_to_cpu(disk_super
->policy_version
[0]);
551 cmd
->policy_version
[1] = le32_to_cpu(disk_super
->policy_version
[1]);
552 cmd
->policy_version
[2] = le32_to_cpu(disk_super
->policy_version
[2]);
553 cmd
->policy_hint_size
= le32_to_cpu(disk_super
->policy_hint_size
);
555 cmd
->stats
.read_hits
= le32_to_cpu(disk_super
->read_hits
);
556 cmd
->stats
.read_misses
= le32_to_cpu(disk_super
->read_misses
);
557 cmd
->stats
.write_hits
= le32_to_cpu(disk_super
->write_hits
);
558 cmd
->stats
.write_misses
= le32_to_cpu(disk_super
->write_misses
);
560 cmd
->changed
= false;
564 * The mutator updates the superblock flags.
566 static int __begin_transaction_flags(struct dm_cache_metadata
*cmd
,
567 flags_mutator mutator
)
570 struct cache_disk_superblock
*disk_super
;
571 struct dm_block
*sblock
;
573 r
= superblock_lock(cmd
, &sblock
);
577 disk_super
= dm_block_data(sblock
);
578 update_flags(disk_super
, mutator
);
579 read_superblock_fields(cmd
, disk_super
);
580 dm_bm_unlock(sblock
);
582 return dm_bm_flush(cmd
->bm
);
585 static int __begin_transaction(struct dm_cache_metadata
*cmd
)
588 struct cache_disk_superblock
*disk_super
;
589 struct dm_block
*sblock
;
592 * We re-read the superblock every time. Shouldn't need to do this
595 r
= superblock_read_lock(cmd
, &sblock
);
599 disk_super
= dm_block_data(sblock
);
600 read_superblock_fields(cmd
, disk_super
);
601 dm_bm_unlock(sblock
);
606 static int __commit_transaction(struct dm_cache_metadata
*cmd
,
607 flags_mutator mutator
)
610 struct cache_disk_superblock
*disk_super
;
611 struct dm_block
*sblock
;
614 * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
616 BUILD_BUG_ON(sizeof(struct cache_disk_superblock
) > 512);
618 r
= dm_bitset_flush(&cmd
->discard_info
, cmd
->discard_root
,
623 r
= dm_tm_pre_commit(cmd
->tm
);
627 r
= __save_sm_root(cmd
);
631 r
= superblock_lock(cmd
, &sblock
);
635 disk_super
= dm_block_data(sblock
);
637 disk_super
->flags
= cpu_to_le32(cmd
->flags
);
639 update_flags(disk_super
, mutator
);
641 disk_super
->mapping_root
= cpu_to_le64(cmd
->root
);
642 disk_super
->hint_root
= cpu_to_le64(cmd
->hint_root
);
643 disk_super
->discard_root
= cpu_to_le64(cmd
->discard_root
);
644 disk_super
->discard_block_size
= cpu_to_le64(cmd
->discard_block_size
);
645 disk_super
->discard_nr_blocks
= cpu_to_le64(from_dblock(cmd
->discard_nr_blocks
));
646 disk_super
->cache_blocks
= cpu_to_le32(from_cblock(cmd
->cache_blocks
));
647 strncpy(disk_super
->policy_name
, cmd
->policy_name
, sizeof(disk_super
->policy_name
));
648 disk_super
->policy_version
[0] = cpu_to_le32(cmd
->policy_version
[0]);
649 disk_super
->policy_version
[1] = cpu_to_le32(cmd
->policy_version
[1]);
650 disk_super
->policy_version
[2] = cpu_to_le32(cmd
->policy_version
[2]);
652 disk_super
->read_hits
= cpu_to_le32(cmd
->stats
.read_hits
);
653 disk_super
->read_misses
= cpu_to_le32(cmd
->stats
.read_misses
);
654 disk_super
->write_hits
= cpu_to_le32(cmd
->stats
.write_hits
);
655 disk_super
->write_misses
= cpu_to_le32(cmd
->stats
.write_misses
);
656 __copy_sm_root(cmd
, disk_super
);
658 return dm_tm_commit(cmd
->tm
, sblock
);
661 /*----------------------------------------------------------------*/
664 * The mappings are held in a dm-array that has 64-bit values stored in
665 * little-endian format. The index is the cblock, the high 48bits of the
666 * value are the oblock and the low 16 bit the flags.
668 #define FLAGS_MASK ((1 << 16) - 1)
670 static __le64
pack_value(dm_oblock_t block
, unsigned flags
)
672 uint64_t value
= from_oblock(block
);
674 value
= value
| (flags
& FLAGS_MASK
);
675 return cpu_to_le64(value
);
678 static void unpack_value(__le64 value_le
, dm_oblock_t
*block
, unsigned *flags
)
680 uint64_t value
= le64_to_cpu(value_le
);
681 uint64_t b
= value
>> 16;
682 *block
= to_oblock(b
);
683 *flags
= value
& FLAGS_MASK
;
686 /*----------------------------------------------------------------*/
688 static struct dm_cache_metadata
*metadata_open(struct block_device
*bdev
,
689 sector_t data_block_size
,
690 bool may_format_device
,
691 size_t policy_hint_size
)
694 struct dm_cache_metadata
*cmd
;
696 cmd
= kzalloc(sizeof(*cmd
), GFP_KERNEL
);
698 DMERR("could not allocate metadata struct");
699 return ERR_PTR(-ENOMEM
);
702 atomic_set(&cmd
->ref_count
, 1);
703 init_rwsem(&cmd
->root_lock
);
705 cmd
->data_block_size
= data_block_size
;
706 cmd
->cache_blocks
= 0;
707 cmd
->policy_hint_size
= policy_hint_size
;
709 cmd
->fail_io
= false;
711 r
= __create_persistent_data_objects(cmd
, may_format_device
);
717 r
= __begin_transaction_flags(cmd
, clear_clean_shutdown
);
719 dm_cache_metadata_close(cmd
);
727 * We keep a little list of ref counted metadata objects to prevent two
728 * different target instances creating separate bufio instances. This is
729 * an issue if a table is reloaded before the suspend.
731 static DEFINE_MUTEX(table_lock
);
732 static LIST_HEAD(table
);
734 static struct dm_cache_metadata
*lookup(struct block_device
*bdev
)
736 struct dm_cache_metadata
*cmd
;
738 list_for_each_entry(cmd
, &table
, list
)
739 if (cmd
->bdev
== bdev
) {
740 atomic_inc(&cmd
->ref_count
);
747 static struct dm_cache_metadata
*lookup_or_open(struct block_device
*bdev
,
748 sector_t data_block_size
,
749 bool may_format_device
,
750 size_t policy_hint_size
)
752 struct dm_cache_metadata
*cmd
, *cmd2
;
754 mutex_lock(&table_lock
);
756 mutex_unlock(&table_lock
);
761 cmd
= metadata_open(bdev
, data_block_size
, may_format_device
, policy_hint_size
);
763 mutex_lock(&table_lock
);
766 mutex_unlock(&table_lock
);
767 __destroy_persistent_data_objects(cmd
);
771 list_add(&cmd
->list
, &table
);
772 mutex_unlock(&table_lock
);
778 static bool same_params(struct dm_cache_metadata
*cmd
, sector_t data_block_size
)
780 if (cmd
->data_block_size
!= data_block_size
) {
781 DMERR("data_block_size (%llu) different from that in metadata (%llu)\n",
782 (unsigned long long) data_block_size
,
783 (unsigned long long) cmd
->data_block_size
);
790 struct dm_cache_metadata
*dm_cache_metadata_open(struct block_device
*bdev
,
791 sector_t data_block_size
,
792 bool may_format_device
,
793 size_t policy_hint_size
)
795 struct dm_cache_metadata
*cmd
= lookup_or_open(bdev
, data_block_size
,
796 may_format_device
, policy_hint_size
);
798 if (!IS_ERR(cmd
) && !same_params(cmd
, data_block_size
)) {
799 dm_cache_metadata_close(cmd
);
800 return ERR_PTR(-EINVAL
);
806 void dm_cache_metadata_close(struct dm_cache_metadata
*cmd
)
808 if (atomic_dec_and_test(&cmd
->ref_count
)) {
809 mutex_lock(&table_lock
);
810 list_del(&cmd
->list
);
811 mutex_unlock(&table_lock
);
814 __destroy_persistent_data_objects(cmd
);
820 * Checks that the given cache block is either unmapped or clean.
822 static int block_unmapped_or_clean(struct dm_cache_metadata
*cmd
, dm_cblock_t b
,
830 r
= dm_array_get_value(&cmd
->info
, cmd
->root
, from_cblock(b
), &value
);
832 DMERR("block_unmapped_or_clean failed");
836 unpack_value(value
, &ob
, &flags
);
837 *result
= !((flags
& M_VALID
) && (flags
& M_DIRTY
));
842 static int blocks_are_unmapped_or_clean(struct dm_cache_metadata
*cmd
,
843 dm_cblock_t begin
, dm_cblock_t end
,
849 while (begin
!= end
) {
850 r
= block_unmapped_or_clean(cmd
, begin
, result
);
855 DMERR("cache block %llu is dirty",
856 (unsigned long long) from_cblock(begin
));
860 begin
= to_cblock(from_cblock(begin
) + 1);
866 #define WRITE_LOCK(cmd) \
867 if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) \
869 down_write(&cmd->root_lock)
871 #define WRITE_LOCK_VOID(cmd) \
872 if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) \
874 down_write(&cmd->root_lock)
876 #define WRITE_UNLOCK(cmd) \
877 up_write(&cmd->root_lock)
879 int dm_cache_resize(struct dm_cache_metadata
*cmd
, dm_cblock_t new_cache_size
)
883 __le64 null_mapping
= pack_value(0, 0);
886 __dm_bless_for_disk(&null_mapping
);
888 if (from_cblock(new_cache_size
) < from_cblock(cmd
->cache_blocks
)) {
889 r
= blocks_are_unmapped_or_clean(cmd
, new_cache_size
, cmd
->cache_blocks
, &clean
);
891 __dm_unbless_for_disk(&null_mapping
);
896 DMERR("unable to shrink cache due to dirty blocks");
898 __dm_unbless_for_disk(&null_mapping
);
903 r
= dm_array_resize(&cmd
->info
, cmd
->root
, from_cblock(cmd
->cache_blocks
),
904 from_cblock(new_cache_size
),
905 &null_mapping
, &cmd
->root
);
907 cmd
->cache_blocks
= new_cache_size
;
916 int dm_cache_discard_bitset_resize(struct dm_cache_metadata
*cmd
,
917 sector_t discard_block_size
,
918 dm_dblock_t new_nr_entries
)
923 r
= dm_bitset_resize(&cmd
->discard_info
,
925 from_dblock(cmd
->discard_nr_blocks
),
926 from_dblock(new_nr_entries
),
927 false, &cmd
->discard_root
);
929 cmd
->discard_block_size
= discard_block_size
;
930 cmd
->discard_nr_blocks
= new_nr_entries
;
939 static int __set_discard(struct dm_cache_metadata
*cmd
, dm_dblock_t b
)
941 return dm_bitset_set_bit(&cmd
->discard_info
, cmd
->discard_root
,
942 from_dblock(b
), &cmd
->discard_root
);
945 static int __clear_discard(struct dm_cache_metadata
*cmd
, dm_dblock_t b
)
947 return dm_bitset_clear_bit(&cmd
->discard_info
, cmd
->discard_root
,
948 from_dblock(b
), &cmd
->discard_root
);
951 static int __is_discarded(struct dm_cache_metadata
*cmd
, dm_dblock_t b
,
954 return dm_bitset_test_bit(&cmd
->discard_info
, cmd
->discard_root
,
955 from_dblock(b
), &cmd
->discard_root
,
959 static int __discard(struct dm_cache_metadata
*cmd
,
960 dm_dblock_t dblock
, bool discard
)
964 r
= (discard
? __set_discard
: __clear_discard
)(cmd
, dblock
);
972 int dm_cache_set_discard(struct dm_cache_metadata
*cmd
,
973 dm_dblock_t dblock
, bool discard
)
978 r
= __discard(cmd
, dblock
, discard
);
984 static int __load_discards(struct dm_cache_metadata
*cmd
,
985 load_discard_fn fn
, void *context
)
991 for (b
= 0; b
< from_dblock(cmd
->discard_nr_blocks
); b
++) {
992 dm_dblock_t dblock
= to_dblock(b
);
994 if (cmd
->clean_when_opened
) {
995 r
= __is_discarded(cmd
, dblock
, &discard
);
1001 r
= fn(context
, cmd
->discard_block_size
, dblock
, discard
);
1009 int dm_cache_load_discards(struct dm_cache_metadata
*cmd
,
1010 load_discard_fn fn
, void *context
)
1014 down_read(&cmd
->root_lock
);
1015 r
= __load_discards(cmd
, fn
, context
);
1016 up_read(&cmd
->root_lock
);
1021 dm_cblock_t
dm_cache_size(struct dm_cache_metadata
*cmd
)
1025 down_read(&cmd
->root_lock
);
1026 r
= cmd
->cache_blocks
;
1027 up_read(&cmd
->root_lock
);
1032 static int __remove(struct dm_cache_metadata
*cmd
, dm_cblock_t cblock
)
1035 __le64 value
= pack_value(0, 0);
1037 __dm_bless_for_disk(&value
);
1038 r
= dm_array_set_value(&cmd
->info
, cmd
->root
, from_cblock(cblock
),
1039 &value
, &cmd
->root
);
1043 cmd
->changed
= true;
1047 int dm_cache_remove_mapping(struct dm_cache_metadata
*cmd
, dm_cblock_t cblock
)
1052 r
= __remove(cmd
, cblock
);
1058 static int __insert(struct dm_cache_metadata
*cmd
,
1059 dm_cblock_t cblock
, dm_oblock_t oblock
)
1062 __le64 value
= pack_value(oblock
, M_VALID
);
1063 __dm_bless_for_disk(&value
);
1065 r
= dm_array_set_value(&cmd
->info
, cmd
->root
, from_cblock(cblock
),
1066 &value
, &cmd
->root
);
1070 cmd
->changed
= true;
1074 int dm_cache_insert_mapping(struct dm_cache_metadata
*cmd
,
1075 dm_cblock_t cblock
, dm_oblock_t oblock
)
1080 r
= __insert(cmd
, cblock
, oblock
);
1090 struct dm_cache_metadata
*cmd
;
1091 bool respect_dirty_flags
;
1095 static bool policy_unchanged(struct dm_cache_metadata
*cmd
,
1096 struct dm_cache_policy
*policy
)
1098 const char *policy_name
= dm_cache_policy_get_name(policy
);
1099 const unsigned *policy_version
= dm_cache_policy_get_version(policy
);
1100 size_t policy_hint_size
= dm_cache_policy_get_hint_size(policy
);
1103 * Ensure policy names match.
1105 if (strncmp(cmd
->policy_name
, policy_name
, sizeof(cmd
->policy_name
)))
1109 * Ensure policy major versions match.
1111 if (cmd
->policy_version
[0] != policy_version
[0])
1115 * Ensure policy hint sizes match.
1117 if (cmd
->policy_hint_size
!= policy_hint_size
)
1123 static bool hints_array_initialized(struct dm_cache_metadata
*cmd
)
1125 return cmd
->hint_root
&& cmd
->policy_hint_size
;
1128 static bool hints_array_available(struct dm_cache_metadata
*cmd
,
1129 struct dm_cache_policy
*policy
)
1131 return cmd
->clean_when_opened
&& policy_unchanged(cmd
, policy
) &&
1132 hints_array_initialized(cmd
);
1135 static int __load_mapping(void *context
, uint64_t cblock
, void *leaf
)
1140 __le32 hint_value
= 0;
1143 struct thunk
*thunk
= context
;
1144 struct dm_cache_metadata
*cmd
= thunk
->cmd
;
1146 memcpy(&value
, leaf
, sizeof(value
));
1147 unpack_value(value
, &oblock
, &flags
);
1149 if (flags
& M_VALID
) {
1150 if (thunk
->hints_valid
) {
1151 r
= dm_array_get_value(&cmd
->hint_info
, cmd
->hint_root
,
1152 cblock
, &hint_value
);
1153 if (r
&& r
!= -ENODATA
)
1157 dirty
= thunk
->respect_dirty_flags
? (flags
& M_DIRTY
) : true;
1158 r
= thunk
->fn(thunk
->context
, oblock
, to_cblock(cblock
),
1159 dirty
, le32_to_cpu(hint_value
), thunk
->hints_valid
);
1165 static int __load_mappings(struct dm_cache_metadata
*cmd
,
1166 struct dm_cache_policy
*policy
,
1167 load_mapping_fn fn
, void *context
)
1172 thunk
.context
= context
;
1175 thunk
.respect_dirty_flags
= cmd
->clean_when_opened
;
1176 thunk
.hints_valid
= hints_array_available(cmd
, policy
);
1178 return dm_array_walk(&cmd
->info
, cmd
->root
, __load_mapping
, &thunk
);
1181 int dm_cache_load_mappings(struct dm_cache_metadata
*cmd
,
1182 struct dm_cache_policy
*policy
,
1183 load_mapping_fn fn
, void *context
)
1187 down_read(&cmd
->root_lock
);
1188 r
= __load_mappings(cmd
, policy
, fn
, context
);
1189 up_read(&cmd
->root_lock
);
1194 static int __dump_mapping(void *context
, uint64_t cblock
, void *leaf
)
1201 memcpy(&value
, leaf
, sizeof(value
));
1202 unpack_value(value
, &oblock
, &flags
);
1207 static int __dump_mappings(struct dm_cache_metadata
*cmd
)
1209 return dm_array_walk(&cmd
->info
, cmd
->root
, __dump_mapping
, NULL
);
1212 void dm_cache_dump(struct dm_cache_metadata
*cmd
)
1214 down_read(&cmd
->root_lock
);
1215 __dump_mappings(cmd
);
1216 up_read(&cmd
->root_lock
);
1219 int dm_cache_changed_this_transaction(struct dm_cache_metadata
*cmd
)
1223 down_read(&cmd
->root_lock
);
1225 up_read(&cmd
->root_lock
);
1230 static int __dirty(struct dm_cache_metadata
*cmd
, dm_cblock_t cblock
, bool dirty
)
1237 r
= dm_array_get_value(&cmd
->info
, cmd
->root
, from_cblock(cblock
), &value
);
1241 unpack_value(value
, &oblock
, &flags
);
1243 if (((flags
& M_DIRTY
) && dirty
) || (!(flags
& M_DIRTY
) && !dirty
))
1244 /* nothing to be done */
1247 value
= pack_value(oblock
, (flags
& ~M_DIRTY
) | (dirty
? M_DIRTY
: 0));
1248 __dm_bless_for_disk(&value
);
1250 r
= dm_array_set_value(&cmd
->info
, cmd
->root
, from_cblock(cblock
),
1251 &value
, &cmd
->root
);
1255 cmd
->changed
= true;
1260 int dm_cache_set_dirty(struct dm_cache_metadata
*cmd
,
1261 dm_cblock_t cblock
, bool dirty
)
1266 r
= __dirty(cmd
, cblock
, dirty
);
1272 void dm_cache_metadata_get_stats(struct dm_cache_metadata
*cmd
,
1273 struct dm_cache_statistics
*stats
)
1275 down_read(&cmd
->root_lock
);
1276 *stats
= cmd
->stats
;
1277 up_read(&cmd
->root_lock
);
1280 void dm_cache_metadata_set_stats(struct dm_cache_metadata
*cmd
,
1281 struct dm_cache_statistics
*stats
)
1283 WRITE_LOCK_VOID(cmd
);
1284 cmd
->stats
= *stats
;
1288 int dm_cache_commit(struct dm_cache_metadata
*cmd
, bool clean_shutdown
)
1291 flags_mutator mutator
= (clean_shutdown
? set_clean_shutdown
:
1292 clear_clean_shutdown
);
1295 r
= __commit_transaction(cmd
, mutator
);
1299 r
= __begin_transaction(cmd
);
1306 int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata
*cmd
,
1311 down_read(&cmd
->root_lock
);
1312 r
= dm_sm_get_nr_free(cmd
->metadata_sm
, result
);
1313 up_read(&cmd
->root_lock
);
1318 int dm_cache_get_metadata_dev_size(struct dm_cache_metadata
*cmd
,
1323 down_read(&cmd
->root_lock
);
1324 r
= dm_sm_get_nr_blocks(cmd
->metadata_sm
, result
);
1325 up_read(&cmd
->root_lock
);
1330 /*----------------------------------------------------------------*/
1332 static int begin_hints(struct dm_cache_metadata
*cmd
, struct dm_cache_policy
*policy
)
1337 const char *policy_name
= dm_cache_policy_get_name(policy
);
1338 const unsigned *policy_version
= dm_cache_policy_get_version(policy
);
1340 if (!policy_name
[0] ||
1341 (strlen(policy_name
) > sizeof(cmd
->policy_name
) - 1))
1344 if (!policy_unchanged(cmd
, policy
)) {
1345 strncpy(cmd
->policy_name
, policy_name
, sizeof(cmd
->policy_name
));
1346 memcpy(cmd
->policy_version
, policy_version
, sizeof(cmd
->policy_version
));
1348 hint_size
= dm_cache_policy_get_hint_size(policy
);
1350 return 0; /* short-circuit hints initialization */
1351 cmd
->policy_hint_size
= hint_size
;
1353 if (cmd
->hint_root
) {
1354 r
= dm_array_del(&cmd
->hint_info
, cmd
->hint_root
);
1359 r
= dm_array_empty(&cmd
->hint_info
, &cmd
->hint_root
);
1363 value
= cpu_to_le32(0);
1364 __dm_bless_for_disk(&value
);
1365 r
= dm_array_resize(&cmd
->hint_info
, cmd
->hint_root
, 0,
1366 from_cblock(cmd
->cache_blocks
),
1367 &value
, &cmd
->hint_root
);
1375 static int save_hint(void *context
, dm_cblock_t cblock
, dm_oblock_t oblock
, uint32_t hint
)
1377 struct dm_cache_metadata
*cmd
= context
;
1378 __le32 value
= cpu_to_le32(hint
);
1381 __dm_bless_for_disk(&value
);
1383 r
= dm_array_set_value(&cmd
->hint_info
, cmd
->hint_root
,
1384 from_cblock(cblock
), &value
, &cmd
->hint_root
);
1385 cmd
->changed
= true;
1390 static int write_hints(struct dm_cache_metadata
*cmd
, struct dm_cache_policy
*policy
)
1394 r
= begin_hints(cmd
, policy
);
1396 DMERR("begin_hints failed");
1400 return policy_walk_mappings(policy
, save_hint
, cmd
);
1403 int dm_cache_write_hints(struct dm_cache_metadata
*cmd
, struct dm_cache_policy
*policy
)
1408 r
= write_hints(cmd
, policy
);
1414 int dm_cache_metadata_all_clean(struct dm_cache_metadata
*cmd
, bool *result
)
1416 return blocks_are_unmapped_or_clean(cmd
, 0, cmd
->cache_blocks
, result
);
1419 void dm_cache_metadata_set_read_only(struct dm_cache_metadata
*cmd
)
1421 WRITE_LOCK_VOID(cmd
);
1422 dm_bm_set_read_only(cmd
->bm
);
1426 void dm_cache_metadata_set_read_write(struct dm_cache_metadata
*cmd
)
1428 WRITE_LOCK_VOID(cmd
);
1429 dm_bm_set_read_write(cmd
->bm
);
1433 int dm_cache_metadata_set_needs_check(struct dm_cache_metadata
*cmd
)
1436 struct dm_block
*sblock
;
1437 struct cache_disk_superblock
*disk_super
;
1440 * We ignore fail_io for this function.
1442 down_write(&cmd
->root_lock
);
1443 set_bit(NEEDS_CHECK
, &cmd
->flags
);
1445 r
= superblock_lock(cmd
, &sblock
);
1447 DMERR("couldn't read superblock");
1451 disk_super
= dm_block_data(sblock
);
1452 disk_super
->flags
= cpu_to_le32(cmd
->flags
);
1454 dm_bm_unlock(sblock
);
1457 up_write(&cmd
->root_lock
);
1461 bool dm_cache_metadata_needs_check(struct dm_cache_metadata
*cmd
)
1465 down_read(&cmd
->root_lock
);
1466 needs_check
= !!test_bit(NEEDS_CHECK
, &cmd
->flags
);
1467 up_read(&cmd
->root_lock
);
1472 int dm_cache_metadata_abort(struct dm_cache_metadata
*cmd
)
1477 __destroy_persistent_data_objects(cmd
);
1478 r
= __create_persistent_data_objects(cmd
, false);
1480 cmd
->fail_io
= true;