2 * Copyright (C) 2012 Red Hat, Inc.
4 * Author: Mikulas Patocka <mpatocka@redhat.com>
6 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
8 * This file is released under the GPLv2.
10 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
11 * default prefetch value. Data are read in "prefetch_cluster" chunks from the
12 * hash device. Setting this greatly improves performance when data and hash
13 * are on the same disk on different partitions on devices with poor random
19 #include <linux/module.h>
20 #include <linux/device-mapper.h>
21 #include <linux/reboot.h>
22 #include <crypto/hash.h>
24 #define DM_MSG_PREFIX "verity"
26 #define DM_VERITY_ENV_LENGTH 42
27 #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR"
29 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
31 #define DM_VERITY_MAX_LEVELS 63
32 #define DM_VERITY_MAX_CORRUPTED_ERRS 100
34 #define DM_VERITY_OPT_LOGGING "ignore_corruption"
35 #define DM_VERITY_OPT_RESTART "restart_on_corruption"
37 static unsigned dm_verity_prefetch_cluster
= DM_VERITY_DEFAULT_PREFETCH_SIZE
;
39 module_param_named(prefetch_cluster
, dm_verity_prefetch_cluster
, uint
, S_IRUGO
| S_IWUSR
);
43 DM_VERITY_MODE_LOGGING
,
44 DM_VERITY_MODE_RESTART
47 enum verity_block_type
{
48 DM_VERITY_BLOCK_TYPE_DATA
,
49 DM_VERITY_BLOCK_TYPE_METADATA
53 struct dm_dev
*data_dev
;
54 struct dm_dev
*hash_dev
;
56 struct dm_bufio_client
*bufio
;
58 struct crypto_shash
*tfm
;
59 u8
*root_digest
; /* digest of the root block */
60 u8
*salt
; /* salt: its size is salt_size */
62 sector_t data_start
; /* data offset in 512-byte sectors */
63 sector_t hash_start
; /* hash start in blocks */
64 sector_t data_blocks
; /* the number of data blocks */
65 sector_t hash_blocks
; /* the number of hash blocks */
66 unsigned char data_dev_block_bits
; /* log2(data blocksize) */
67 unsigned char hash_dev_block_bits
; /* log2(hash blocksize) */
68 unsigned char hash_per_block_bits
; /* log2(hashes in hash block) */
69 unsigned char levels
; /* the number of tree levels */
70 unsigned char version
;
71 unsigned digest_size
; /* digest size for the current hash algorithm */
72 unsigned shash_descsize
;/* the size of temporary space for crypto */
73 int hash_failed
; /* set to 1 if hash of any block failed */
74 enum verity_mode mode
; /* mode for handling verification errors */
75 unsigned corrupted_errs
;/* Number of errors for corrupted blocks */
77 struct workqueue_struct
*verify_wq
;
79 /* starting blocks for each tree level. 0 is the lowest level. */
80 sector_t hash_level_block
[DM_VERITY_MAX_LEVELS
];
86 /* original values of bio->bi_end_io and bio->bi_private */
87 bio_end_io_t
*orig_bi_end_io
;
88 void *orig_bi_private
;
93 struct bvec_iter iter
;
95 struct work_struct work
;
98 * Three variably-size fields follow this struct:
100 * u8 hash_desc[v->shash_descsize];
101 * u8 real_digest[v->digest_size];
102 * u8 want_digest[v->digest_size];
104 * To access them use: io_hash_desc(), io_real_digest() and io_want_digest().
108 struct dm_verity_prefetch_work
{
109 struct work_struct work
;
115 static struct shash_desc
*io_hash_desc(struct dm_verity
*v
, struct dm_verity_io
*io
)
117 return (struct shash_desc
*)(io
+ 1);
120 static u8
*io_real_digest(struct dm_verity
*v
, struct dm_verity_io
*io
)
122 return (u8
*)(io
+ 1) + v
->shash_descsize
;
125 static u8
*io_want_digest(struct dm_verity
*v
, struct dm_verity_io
*io
)
127 return (u8
*)(io
+ 1) + v
->shash_descsize
+ v
->digest_size
;
131 * Auxiliary structure appended to each dm-bufio buffer. If the value
132 * hash_verified is nonzero, hash of the block has been verified.
134 * The variable hash_verified is set to 0 when allocating the buffer, then
135 * it can be changed to 1 and it is never reset to 0 again.
137 * There is no lock around this value, a race condition can at worst cause
138 * that multiple processes verify the hash of the same buffer simultaneously
139 * and write 1 to hash_verified simultaneously.
140 * This condition is harmless, so we don't need locking.
147 * Initialize struct buffer_aux for a freshly created buffer.
149 static void dm_bufio_alloc_callback(struct dm_buffer
*buf
)
151 struct buffer_aux
*aux
= dm_bufio_get_aux_data(buf
);
153 aux
->hash_verified
= 0;
157 * Translate input sector number to the sector number on the target device.
159 static sector_t
verity_map_sector(struct dm_verity
*v
, sector_t bi_sector
)
161 return v
->data_start
+ dm_target_offset(v
->ti
, bi_sector
);
165 * Return hash position of a specified block at a specified tree level
166 * (0 is the lowest level).
167 * The lowest "hash_per_block_bits"-bits of the result denote hash position
168 * inside a hash block. The remaining bits denote location of the hash block.
170 static sector_t
verity_position_at_level(struct dm_verity
*v
, sector_t block
,
173 return block
>> (level
* v
->hash_per_block_bits
);
176 static void verity_hash_at_level(struct dm_verity
*v
, sector_t block
, int level
,
177 sector_t
*hash_block
, unsigned *offset
)
179 sector_t position
= verity_position_at_level(v
, block
, level
);
182 *hash_block
= v
->hash_level_block
[level
] + (position
>> v
->hash_per_block_bits
);
187 idx
= position
& ((1 << v
->hash_per_block_bits
) - 1);
189 *offset
= idx
* v
->digest_size
;
191 *offset
= idx
<< (v
->hash_dev_block_bits
- v
->hash_per_block_bits
);
195 * Handle verification errors.
197 static int verity_handle_err(struct dm_verity
*v
, enum verity_block_type type
,
198 unsigned long long block
)
200 char verity_env
[DM_VERITY_ENV_LENGTH
];
201 char *envp
[] = { verity_env
, NULL
};
202 const char *type_str
= "";
203 struct mapped_device
*md
= dm_table_get_md(v
->ti
->table
);
205 /* Corruption should be visible in device status in all modes */
208 if (v
->corrupted_errs
>= DM_VERITY_MAX_CORRUPTED_ERRS
)
214 case DM_VERITY_BLOCK_TYPE_DATA
:
217 case DM_VERITY_BLOCK_TYPE_METADATA
:
218 type_str
= "metadata";
224 DMERR("%s: %s block %llu is corrupted", v
->data_dev
->name
, type_str
,
227 if (v
->corrupted_errs
== DM_VERITY_MAX_CORRUPTED_ERRS
)
228 DMERR("%s: reached maximum errors", v
->data_dev
->name
);
230 snprintf(verity_env
, DM_VERITY_ENV_LENGTH
, "%s=%d,%llu",
231 DM_VERITY_ENV_VAR_NAME
, type
, block
);
233 kobject_uevent_env(&disk_to_dev(dm_disk(md
))->kobj
, KOBJ_CHANGE
, envp
);
236 if (v
->mode
== DM_VERITY_MODE_LOGGING
)
239 if (v
->mode
== DM_VERITY_MODE_RESTART
)
240 kernel_restart("dm-verity device corrupted");
246 * Verify hash of a metadata block pertaining to the specified data block
247 * ("block" argument) at a specified level ("level" argument).
249 * On successful return, io_want_digest(v, io) contains the hash value for
250 * a lower tree level or for the data block (if we're at the lowest leve).
252 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
253 * If "skip_unverified" is false, unverified buffer is hashed and verified
254 * against current value of io_want_digest(v, io).
256 static int verity_verify_level(struct dm_verity_io
*io
, sector_t block
,
257 int level
, bool skip_unverified
)
259 struct dm_verity
*v
= io
->v
;
260 struct dm_buffer
*buf
;
261 struct buffer_aux
*aux
;
267 verity_hash_at_level(v
, block
, level
, &hash_block
, &offset
);
269 data
= dm_bufio_read(v
->bufio
, hash_block
, &buf
);
270 if (unlikely(IS_ERR(data
)))
271 return PTR_ERR(data
);
273 aux
= dm_bufio_get_aux_data(buf
);
275 if (!aux
->hash_verified
) {
276 struct shash_desc
*desc
;
279 if (skip_unverified
) {
284 desc
= io_hash_desc(v
, io
);
286 desc
->flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
287 r
= crypto_shash_init(desc
);
289 DMERR("crypto_shash_init failed: %d", r
);
293 if (likely(v
->version
>= 1)) {
294 r
= crypto_shash_update(desc
, v
->salt
, v
->salt_size
);
296 DMERR("crypto_shash_update failed: %d", r
);
301 r
= crypto_shash_update(desc
, data
, 1 << v
->hash_dev_block_bits
);
303 DMERR("crypto_shash_update failed: %d", r
);
308 r
= crypto_shash_update(desc
, v
->salt
, v
->salt_size
);
310 DMERR("crypto_shash_update failed: %d", r
);
315 result
= io_real_digest(v
, io
);
316 r
= crypto_shash_final(desc
, result
);
318 DMERR("crypto_shash_final failed: %d", r
);
321 if (unlikely(memcmp(result
, io_want_digest(v
, io
), v
->digest_size
))) {
322 if (verity_handle_err(v
, DM_VERITY_BLOCK_TYPE_METADATA
,
328 aux
->hash_verified
= 1;
333 memcpy(io_want_digest(v
, io
), data
, v
->digest_size
);
335 dm_bufio_release(buf
);
339 dm_bufio_release(buf
);
345 * Verify one "dm_verity_io" structure.
347 static int verity_verify_io(struct dm_verity_io
*io
)
349 struct dm_verity
*v
= io
->v
;
350 struct bio
*bio
= dm_bio_from_per_bio_data(io
,
351 v
->ti
->per_bio_data_size
);
355 for (b
= 0; b
< io
->n_blocks
; b
++) {
356 struct shash_desc
*desc
;
361 if (likely(v
->levels
)) {
363 * First, we try to get the requested hash for
364 * the current block. If the hash block itself is
365 * verified, zero is returned. If it isn't, this
366 * function returns 0 and we fall back to whole
367 * chain verification.
369 int r
= verity_verify_level(io
, io
->block
+ b
, 0, true);
371 goto test_block_hash
;
376 memcpy(io_want_digest(v
, io
), v
->root_digest
, v
->digest_size
);
378 for (i
= v
->levels
- 1; i
>= 0; i
--) {
379 int r
= verity_verify_level(io
, io
->block
+ b
, i
, false);
385 desc
= io_hash_desc(v
, io
);
387 desc
->flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
388 r
= crypto_shash_init(desc
);
390 DMERR("crypto_shash_init failed: %d", r
);
394 if (likely(v
->version
>= 1)) {
395 r
= crypto_shash_update(desc
, v
->salt
, v
->salt_size
);
397 DMERR("crypto_shash_update failed: %d", r
);
401 todo
= 1 << v
->data_dev_block_bits
;
405 struct bio_vec bv
= bio_iter_iovec(bio
, io
->iter
);
407 page
= kmap_atomic(bv
.bv_page
);
409 if (likely(len
>= todo
))
411 r
= crypto_shash_update(desc
, page
+ bv
.bv_offset
, len
);
415 DMERR("crypto_shash_update failed: %d", r
);
419 bio_advance_iter(bio
, &io
->iter
, len
);
424 r
= crypto_shash_update(desc
, v
->salt
, v
->salt_size
);
426 DMERR("crypto_shash_update failed: %d", r
);
431 result
= io_real_digest(v
, io
);
432 r
= crypto_shash_final(desc
, result
);
434 DMERR("crypto_shash_final failed: %d", r
);
437 if (unlikely(memcmp(result
, io_want_digest(v
, io
), v
->digest_size
))) {
438 if (verity_handle_err(v
, DM_VERITY_BLOCK_TYPE_DATA
,
448 * End one "io" structure with a given error.
450 static void verity_finish_io(struct dm_verity_io
*io
, int error
)
452 struct dm_verity
*v
= io
->v
;
453 struct bio
*bio
= dm_bio_from_per_bio_data(io
, v
->ti
->per_bio_data_size
);
455 bio
->bi_end_io
= io
->orig_bi_end_io
;
456 bio
->bi_private
= io
->orig_bi_private
;
458 bio_endio(bio
, error
);
461 static void verity_work(struct work_struct
*w
)
463 struct dm_verity_io
*io
= container_of(w
, struct dm_verity_io
, work
);
465 verity_finish_io(io
, verity_verify_io(io
));
468 static void verity_end_io(struct bio
*bio
, int error
)
470 struct dm_verity_io
*io
= bio
->bi_private
;
473 verity_finish_io(io
, error
);
477 INIT_WORK(&io
->work
, verity_work
);
478 queue_work(io
->v
->verify_wq
, &io
->work
);
482 * Prefetch buffers for the specified io.
483 * The root buffer is not prefetched, it is assumed that it will be cached
486 static void verity_prefetch_io(struct work_struct
*work
)
488 struct dm_verity_prefetch_work
*pw
=
489 container_of(work
, struct dm_verity_prefetch_work
, work
);
490 struct dm_verity
*v
= pw
->v
;
493 for (i
= v
->levels
- 2; i
>= 0; i
--) {
494 sector_t hash_block_start
;
495 sector_t hash_block_end
;
496 verity_hash_at_level(v
, pw
->block
, i
, &hash_block_start
, NULL
);
497 verity_hash_at_level(v
, pw
->block
+ pw
->n_blocks
- 1, i
, &hash_block_end
, NULL
);
499 unsigned cluster
= ACCESS_ONCE(dm_verity_prefetch_cluster
);
501 cluster
>>= v
->data_dev_block_bits
;
502 if (unlikely(!cluster
))
503 goto no_prefetch_cluster
;
505 if (unlikely(cluster
& (cluster
- 1)))
506 cluster
= 1 << __fls(cluster
);
508 hash_block_start
&= ~(sector_t
)(cluster
- 1);
509 hash_block_end
|= cluster
- 1;
510 if (unlikely(hash_block_end
>= v
->hash_blocks
))
511 hash_block_end
= v
->hash_blocks
- 1;
514 dm_bufio_prefetch(v
->bufio
, hash_block_start
,
515 hash_block_end
- hash_block_start
+ 1);
521 static void verity_submit_prefetch(struct dm_verity
*v
, struct dm_verity_io
*io
)
523 struct dm_verity_prefetch_work
*pw
;
525 pw
= kmalloc(sizeof(struct dm_verity_prefetch_work
),
526 GFP_NOIO
| __GFP_NORETRY
| __GFP_NOMEMALLOC
| __GFP_NOWARN
);
531 INIT_WORK(&pw
->work
, verity_prefetch_io
);
533 pw
->block
= io
->block
;
534 pw
->n_blocks
= io
->n_blocks
;
535 queue_work(v
->verify_wq
, &pw
->work
);
539 * Bio map function. It allocates dm_verity_io structure and bio vector and
540 * fills them. Then it issues prefetches and the I/O.
542 static int verity_map(struct dm_target
*ti
, struct bio
*bio
)
544 struct dm_verity
*v
= ti
->private;
545 struct dm_verity_io
*io
;
547 bio
->bi_bdev
= v
->data_dev
->bdev
;
548 bio
->bi_iter
.bi_sector
= verity_map_sector(v
, bio
->bi_iter
.bi_sector
);
550 if (((unsigned)bio
->bi_iter
.bi_sector
| bio_sectors(bio
)) &
551 ((1 << (v
->data_dev_block_bits
- SECTOR_SHIFT
)) - 1)) {
552 DMERR_LIMIT("unaligned io");
556 if (bio_end_sector(bio
) >>
557 (v
->data_dev_block_bits
- SECTOR_SHIFT
) > v
->data_blocks
) {
558 DMERR_LIMIT("io out of range");
562 if (bio_data_dir(bio
) == WRITE
)
565 io
= dm_per_bio_data(bio
, ti
->per_bio_data_size
);
567 io
->orig_bi_end_io
= bio
->bi_end_io
;
568 io
->orig_bi_private
= bio
->bi_private
;
569 io
->block
= bio
->bi_iter
.bi_sector
>> (v
->data_dev_block_bits
- SECTOR_SHIFT
);
570 io
->n_blocks
= bio
->bi_iter
.bi_size
>> v
->data_dev_block_bits
;
572 bio
->bi_end_io
= verity_end_io
;
573 bio
->bi_private
= io
;
574 io
->iter
= bio
->bi_iter
;
576 verity_submit_prefetch(v
, io
);
578 generic_make_request(bio
);
580 return DM_MAPIO_SUBMITTED
;
584 * Status: V (valid) or C (corruption found)
586 static void verity_status(struct dm_target
*ti
, status_type_t type
,
587 unsigned status_flags
, char *result
, unsigned maxlen
)
589 struct dm_verity
*v
= ti
->private;
594 case STATUSTYPE_INFO
:
595 DMEMIT("%c", v
->hash_failed
? 'C' : 'V');
597 case STATUSTYPE_TABLE
:
598 DMEMIT("%u %s %s %u %u %llu %llu %s ",
602 1 << v
->data_dev_block_bits
,
603 1 << v
->hash_dev_block_bits
,
604 (unsigned long long)v
->data_blocks
,
605 (unsigned long long)v
->hash_start
,
608 for (x
= 0; x
< v
->digest_size
; x
++)
609 DMEMIT("%02x", v
->root_digest
[x
]);
614 for (x
= 0; x
< v
->salt_size
; x
++)
615 DMEMIT("%02x", v
->salt
[x
]);
616 if (v
->mode
!= DM_VERITY_MODE_EIO
) {
619 case DM_VERITY_MODE_LOGGING
:
620 DMEMIT(DM_VERITY_OPT_LOGGING
);
622 case DM_VERITY_MODE_RESTART
:
623 DMEMIT(DM_VERITY_OPT_RESTART
);
633 static int verity_ioctl(struct dm_target
*ti
, unsigned cmd
,
636 struct dm_verity
*v
= ti
->private;
640 ti
->len
!= i_size_read(v
->data_dev
->bdev
->bd_inode
) >> SECTOR_SHIFT
)
641 r
= scsi_verify_blk_ioctl(NULL
, cmd
);
643 return r
? : __blkdev_driver_ioctl(v
->data_dev
->bdev
, v
->data_dev
->mode
,
647 static int verity_merge(struct dm_target
*ti
, struct bvec_merge_data
*bvm
,
648 struct bio_vec
*biovec
, int max_size
)
650 struct dm_verity
*v
= ti
->private;
651 struct request_queue
*q
= bdev_get_queue(v
->data_dev
->bdev
);
653 if (!q
->merge_bvec_fn
)
656 bvm
->bi_bdev
= v
->data_dev
->bdev
;
657 bvm
->bi_sector
= verity_map_sector(v
, bvm
->bi_sector
);
659 return min(max_size
, q
->merge_bvec_fn(q
, bvm
, biovec
));
662 static int verity_iterate_devices(struct dm_target
*ti
,
663 iterate_devices_callout_fn fn
, void *data
)
665 struct dm_verity
*v
= ti
->private;
667 return fn(ti
, v
->data_dev
, v
->data_start
, ti
->len
, data
);
670 static void verity_io_hints(struct dm_target
*ti
, struct queue_limits
*limits
)
672 struct dm_verity
*v
= ti
->private;
674 if (limits
->logical_block_size
< 1 << v
->data_dev_block_bits
)
675 limits
->logical_block_size
= 1 << v
->data_dev_block_bits
;
677 if (limits
->physical_block_size
< 1 << v
->data_dev_block_bits
)
678 limits
->physical_block_size
= 1 << v
->data_dev_block_bits
;
680 blk_limits_io_min(limits
, limits
->logical_block_size
);
683 static void verity_dtr(struct dm_target
*ti
)
685 struct dm_verity
*v
= ti
->private;
688 destroy_workqueue(v
->verify_wq
);
691 dm_bufio_client_destroy(v
->bufio
);
694 kfree(v
->root_digest
);
697 crypto_free_shash(v
->tfm
);
702 dm_put_device(ti
, v
->hash_dev
);
705 dm_put_device(ti
, v
->data_dev
);
712 * <version> The current format is version 1.
713 * Vsn 0 is compatible with original Chromium OS releases.
718 * <the number of data blocks>
722 * <salt> Hex string or "-" if no salt.
724 static int verity_ctr(struct dm_target
*ti
, unsigned argc
, char **argv
)
727 struct dm_arg_set as
;
728 const char *opt_string
;
729 unsigned int num
, opt_params
;
730 unsigned long long num_ll
;
733 sector_t hash_position
;
736 static struct dm_arg _args
[] = {
737 {0, 1, "Invalid number of feature args"},
740 v
= kzalloc(sizeof(struct dm_verity
), GFP_KERNEL
);
742 ti
->error
= "Cannot allocate verity structure";
748 if ((dm_table_get_mode(ti
->table
) & ~FMODE_READ
)) {
749 ti
->error
= "Device must be readonly";
755 ti
->error
= "Not enough arguments";
760 if (sscanf(argv
[0], "%u%c", &num
, &dummy
) != 1 ||
762 ti
->error
= "Invalid version";
768 r
= dm_get_device(ti
, argv
[1], FMODE_READ
, &v
->data_dev
);
770 ti
->error
= "Data device lookup failed";
774 r
= dm_get_device(ti
, argv
[2], FMODE_READ
, &v
->hash_dev
);
776 ti
->error
= "Data device lookup failed";
780 if (sscanf(argv
[3], "%u%c", &num
, &dummy
) != 1 ||
781 !num
|| (num
& (num
- 1)) ||
782 num
< bdev_logical_block_size(v
->data_dev
->bdev
) ||
784 ti
->error
= "Invalid data device block size";
788 v
->data_dev_block_bits
= __ffs(num
);
790 if (sscanf(argv
[4], "%u%c", &num
, &dummy
) != 1 ||
791 !num
|| (num
& (num
- 1)) ||
792 num
< bdev_logical_block_size(v
->hash_dev
->bdev
) ||
794 ti
->error
= "Invalid hash device block size";
798 v
->hash_dev_block_bits
= __ffs(num
);
800 if (sscanf(argv
[5], "%llu%c", &num_ll
, &dummy
) != 1 ||
801 (sector_t
)(num_ll
<< (v
->data_dev_block_bits
- SECTOR_SHIFT
))
802 >> (v
->data_dev_block_bits
- SECTOR_SHIFT
) != num_ll
) {
803 ti
->error
= "Invalid data blocks";
807 v
->data_blocks
= num_ll
;
809 if (ti
->len
> (v
->data_blocks
<< (v
->data_dev_block_bits
- SECTOR_SHIFT
))) {
810 ti
->error
= "Data device is too small";
815 if (sscanf(argv
[6], "%llu%c", &num_ll
, &dummy
) != 1 ||
816 (sector_t
)(num_ll
<< (v
->hash_dev_block_bits
- SECTOR_SHIFT
))
817 >> (v
->hash_dev_block_bits
- SECTOR_SHIFT
) != num_ll
) {
818 ti
->error
= "Invalid hash start";
822 v
->hash_start
= num_ll
;
824 v
->alg_name
= kstrdup(argv
[7], GFP_KERNEL
);
826 ti
->error
= "Cannot allocate algorithm name";
831 v
->tfm
= crypto_alloc_shash(v
->alg_name
, 0, 0);
832 if (IS_ERR(v
->tfm
)) {
833 ti
->error
= "Cannot initialize hash function";
838 v
->digest_size
= crypto_shash_digestsize(v
->tfm
);
839 if ((1 << v
->hash_dev_block_bits
) < v
->digest_size
* 2) {
840 ti
->error
= "Digest size too big";
845 sizeof(struct shash_desc
) + crypto_shash_descsize(v
->tfm
);
847 v
->root_digest
= kmalloc(v
->digest_size
, GFP_KERNEL
);
848 if (!v
->root_digest
) {
849 ti
->error
= "Cannot allocate root digest";
853 if (strlen(argv
[8]) != v
->digest_size
* 2 ||
854 hex2bin(v
->root_digest
, argv
[8], v
->digest_size
)) {
855 ti
->error
= "Invalid root digest";
860 if (strcmp(argv
[9], "-")) {
861 v
->salt_size
= strlen(argv
[9]) / 2;
862 v
->salt
= kmalloc(v
->salt_size
, GFP_KERNEL
);
864 ti
->error
= "Cannot allocate salt";
868 if (strlen(argv
[9]) != v
->salt_size
* 2 ||
869 hex2bin(v
->salt
, argv
[9], v
->salt_size
)) {
870 ti
->error
= "Invalid salt";
879 /* Optional parameters */
884 r
= dm_read_arg_group(_args
, &as
, &opt_params
, &ti
->error
);
890 opt_string
= dm_shift_arg(&as
);
892 ti
->error
= "Not enough feature arguments";
897 if (!strcasecmp(opt_string
, DM_VERITY_OPT_LOGGING
))
898 v
->mode
= DM_VERITY_MODE_LOGGING
;
899 else if (!strcasecmp(opt_string
, DM_VERITY_OPT_RESTART
))
900 v
->mode
= DM_VERITY_MODE_RESTART
;
902 ti
->error
= "Invalid feature arguments";
909 v
->hash_per_block_bits
=
910 __fls((1 << v
->hash_dev_block_bits
) / v
->digest_size
);
914 while (v
->hash_per_block_bits
* v
->levels
< 64 &&
915 (unsigned long long)(v
->data_blocks
- 1) >>
916 (v
->hash_per_block_bits
* v
->levels
))
919 if (v
->levels
> DM_VERITY_MAX_LEVELS
) {
920 ti
->error
= "Too many tree levels";
925 hash_position
= v
->hash_start
;
926 for (i
= v
->levels
- 1; i
>= 0; i
--) {
928 v
->hash_level_block
[i
] = hash_position
;
929 s
= (v
->data_blocks
+ ((sector_t
)1 << ((i
+ 1) * v
->hash_per_block_bits
)) - 1)
930 >> ((i
+ 1) * v
->hash_per_block_bits
);
931 if (hash_position
+ s
< hash_position
) {
932 ti
->error
= "Hash device offset overflow";
938 v
->hash_blocks
= hash_position
;
940 v
->bufio
= dm_bufio_client_create(v
->hash_dev
->bdev
,
941 1 << v
->hash_dev_block_bits
, 1, sizeof(struct buffer_aux
),
942 dm_bufio_alloc_callback
, NULL
);
943 if (IS_ERR(v
->bufio
)) {
944 ti
->error
= "Cannot initialize dm-bufio";
945 r
= PTR_ERR(v
->bufio
);
950 if (dm_bufio_get_device_size(v
->bufio
) < v
->hash_blocks
) {
951 ti
->error
= "Hash device is too small";
956 ti
->per_bio_data_size
= roundup(sizeof(struct dm_verity_io
) + v
->shash_descsize
+ v
->digest_size
* 2, __alignof__(struct dm_verity_io
));
958 /* WQ_UNBOUND greatly improves performance when running on ramdisk */
959 v
->verify_wq
= alloc_workqueue("kverityd", WQ_CPU_INTENSIVE
| WQ_MEM_RECLAIM
| WQ_UNBOUND
, num_online_cpus());
961 ti
->error
= "Cannot allocate workqueue";
974 static struct target_type verity_target
= {
976 .version
= {1, 2, 0},
977 .module
= THIS_MODULE
,
981 .status
= verity_status
,
982 .ioctl
= verity_ioctl
,
983 .merge
= verity_merge
,
984 .iterate_devices
= verity_iterate_devices
,
985 .io_hints
= verity_io_hints
,
988 static int __init
dm_verity_init(void)
992 r
= dm_register_target(&verity_target
);
994 DMERR("register failed %d", r
);
999 static void __exit
dm_verity_exit(void)
1001 dm_unregister_target(&verity_target
);
1004 module_init(dm_verity_init
);
1005 module_exit(dm_verity_exit
);
1007 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
1008 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
1009 MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
1010 MODULE_DESCRIPTION(DM_NAME
" target for transparent disk integrity checking");
1011 MODULE_LICENSE("GPL");