2 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
4 * Copyright (C) 2006-2007 Red Hat, Inc. All rights reserved.
6 * This file is released under the GPL.
10 #include <linux/module.h>
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/bio.h>
14 #include <linux/blkdev.h>
15 #include <linux/mempool.h>
16 #include <linux/slab.h>
17 #include <linux/crypto.h>
18 #include <linux/workqueue.h>
19 #include <linux/backing-dev.h>
20 #include <asm/atomic.h>
21 #include <linux/scatterlist.h>
23 #include <asm/unaligned.h>
27 #define DM_MSG_PREFIX "crypt"
28 #define MESG_STR(x) x, sizeof(x)
31 * context holding the current state of a multi-part conversion
33 struct convert_context
{
36 unsigned int offset_in
;
37 unsigned int offset_out
;
44 * per bio private data
47 struct dm_target
*target
;
49 struct work_struct work
;
51 struct convert_context ctx
;
58 struct dm_crypt_request
{
59 struct scatterlist sg_in
;
60 struct scatterlist sg_out
;
65 struct crypt_iv_operations
{
66 int (*ctr
)(struct crypt_config
*cc
, struct dm_target
*ti
,
68 void (*dtr
)(struct crypt_config
*cc
);
69 const char *(*status
)(struct crypt_config
*cc
);
70 int (*generator
)(struct crypt_config
*cc
, u8
*iv
, sector_t sector
);
74 * Crypt: maps a linear range of a block device
75 * and encrypts / decrypts at the same time.
77 enum flags
{ DM_CRYPT_SUSPENDED
, DM_CRYPT_KEY_VALID
};
83 * pool for per bio private data and
84 * for encryption buffer pages
90 struct workqueue_struct
*io_queue
;
91 struct workqueue_struct
*crypt_queue
;
95 struct crypt_iv_operations
*iv_gen_ops
;
98 struct crypto_cipher
*essiv_tfm
;
102 unsigned int iv_size
;
104 char cipher
[CRYPTO_MAX_ALG_NAME
];
105 char chainmode
[CRYPTO_MAX_ALG_NAME
];
106 struct crypto_blkcipher
*tfm
;
108 unsigned int key_size
;
113 #define MIN_POOL_PAGES 32
114 #define MIN_BIO_PAGES 8
116 static struct kmem_cache
*_crypt_io_pool
;
118 static void clone_init(struct dm_crypt_io
*, struct bio
*);
119 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
);
122 * Different IV generation algorithms:
124 * plain: the initial vector is the 32-bit little-endian version of the sector
125 * number, padded with zeros if necessary.
127 * essiv: "encrypted sector|salt initial vector", the sector number is
128 * encrypted with the bulk cipher using a salt as key. The salt
129 * should be derived from the bulk cipher's key via hashing.
131 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
132 * (needed for LRW-32-AES and possible other narrow block modes)
134 * null: the initial vector is always zero. Provides compatibility with
135 * obsolete loop_fish2 devices. Do not use for new devices.
137 * plumb: unimplemented, see:
138 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
141 static int crypt_iv_plain_gen(struct crypt_config
*cc
, u8
*iv
, sector_t sector
)
143 memset(iv
, 0, cc
->iv_size
);
144 *(u32
*)iv
= cpu_to_le32(sector
& 0xffffffff);
149 static int crypt_iv_essiv_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
152 struct crypto_cipher
*essiv_tfm
;
153 struct crypto_hash
*hash_tfm
;
154 struct hash_desc desc
;
155 struct scatterlist sg
;
156 unsigned int saltsize
;
161 ti
->error
= "Digest algorithm missing for ESSIV mode";
165 /* Hash the cipher key with the given hash algorithm */
166 hash_tfm
= crypto_alloc_hash(opts
, 0, CRYPTO_ALG_ASYNC
);
167 if (IS_ERR(hash_tfm
)) {
168 ti
->error
= "Error initializing ESSIV hash";
169 return PTR_ERR(hash_tfm
);
172 saltsize
= crypto_hash_digestsize(hash_tfm
);
173 salt
= kmalloc(saltsize
, GFP_KERNEL
);
175 ti
->error
= "Error kmallocing salt storage in ESSIV";
176 crypto_free_hash(hash_tfm
);
180 sg_init_one(&sg
, cc
->key
, cc
->key_size
);
182 desc
.flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
183 err
= crypto_hash_digest(&desc
, &sg
, cc
->key_size
, salt
);
184 crypto_free_hash(hash_tfm
);
187 ti
->error
= "Error calculating hash in ESSIV";
192 /* Setup the essiv_tfm with the given salt */
193 essiv_tfm
= crypto_alloc_cipher(cc
->cipher
, 0, CRYPTO_ALG_ASYNC
);
194 if (IS_ERR(essiv_tfm
)) {
195 ti
->error
= "Error allocating crypto tfm for ESSIV";
197 return PTR_ERR(essiv_tfm
);
199 if (crypto_cipher_blocksize(essiv_tfm
) !=
200 crypto_blkcipher_ivsize(cc
->tfm
)) {
201 ti
->error
= "Block size of ESSIV cipher does "
202 "not match IV size of block cipher";
203 crypto_free_cipher(essiv_tfm
);
207 err
= crypto_cipher_setkey(essiv_tfm
, salt
, saltsize
);
209 ti
->error
= "Failed to set key for ESSIV cipher";
210 crypto_free_cipher(essiv_tfm
);
216 cc
->iv_gen_private
.essiv_tfm
= essiv_tfm
;
220 static void crypt_iv_essiv_dtr(struct crypt_config
*cc
)
222 crypto_free_cipher(cc
->iv_gen_private
.essiv_tfm
);
223 cc
->iv_gen_private
.essiv_tfm
= NULL
;
226 static int crypt_iv_essiv_gen(struct crypt_config
*cc
, u8
*iv
, sector_t sector
)
228 memset(iv
, 0, cc
->iv_size
);
229 *(u64
*)iv
= cpu_to_le64(sector
);
230 crypto_cipher_encrypt_one(cc
->iv_gen_private
.essiv_tfm
, iv
, iv
);
234 static int crypt_iv_benbi_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
237 unsigned int bs
= crypto_blkcipher_blocksize(cc
->tfm
);
240 /* we need to calculate how far we must shift the sector count
241 * to get the cipher block count, we use this shift in _gen */
243 if (1 << log
!= bs
) {
244 ti
->error
= "cypher blocksize is not a power of 2";
249 ti
->error
= "cypher blocksize is > 512";
253 cc
->iv_gen_private
.benbi_shift
= 9 - log
;
258 static void crypt_iv_benbi_dtr(struct crypt_config
*cc
)
262 static int crypt_iv_benbi_gen(struct crypt_config
*cc
, u8
*iv
, sector_t sector
)
266 memset(iv
, 0, cc
->iv_size
- sizeof(u64
)); /* rest is cleared below */
268 val
= cpu_to_be64(((u64
)sector
<< cc
->iv_gen_private
.benbi_shift
) + 1);
269 put_unaligned(val
, (__be64
*)(iv
+ cc
->iv_size
- sizeof(u64
)));
274 static int crypt_iv_null_gen(struct crypt_config
*cc
, u8
*iv
, sector_t sector
)
276 memset(iv
, 0, cc
->iv_size
);
281 static struct crypt_iv_operations crypt_iv_plain_ops
= {
282 .generator
= crypt_iv_plain_gen
285 static struct crypt_iv_operations crypt_iv_essiv_ops
= {
286 .ctr
= crypt_iv_essiv_ctr
,
287 .dtr
= crypt_iv_essiv_dtr
,
288 .generator
= crypt_iv_essiv_gen
291 static struct crypt_iv_operations crypt_iv_benbi_ops
= {
292 .ctr
= crypt_iv_benbi_ctr
,
293 .dtr
= crypt_iv_benbi_dtr
,
294 .generator
= crypt_iv_benbi_gen
297 static struct crypt_iv_operations crypt_iv_null_ops
= {
298 .generator
= crypt_iv_null_gen
302 crypt_convert_scatterlist(struct crypt_config
*cc
, struct scatterlist
*out
,
303 struct scatterlist
*in
, unsigned int length
,
304 int write
, sector_t sector
)
306 u8 iv
[cc
->iv_size
] __attribute__ ((aligned(__alignof__(u64
))));
307 struct blkcipher_desc desc
= {
310 .flags
= CRYPTO_TFM_REQ_MAY_SLEEP
,
314 if (cc
->iv_gen_ops
) {
315 r
= cc
->iv_gen_ops
->generator(cc
, iv
, sector
);
320 r
= crypto_blkcipher_encrypt_iv(&desc
, out
, in
, length
);
322 r
= crypto_blkcipher_decrypt_iv(&desc
, out
, in
, length
);
325 r
= crypto_blkcipher_encrypt(&desc
, out
, in
, length
);
327 r
= crypto_blkcipher_decrypt(&desc
, out
, in
, length
);
333 static void crypt_convert_init(struct crypt_config
*cc
,
334 struct convert_context
*ctx
,
335 struct bio
*bio_out
, struct bio
*bio_in
,
338 ctx
->bio_in
= bio_in
;
339 ctx
->bio_out
= bio_out
;
342 ctx
->idx_in
= bio_in
? bio_in
->bi_idx
: 0;
343 ctx
->idx_out
= bio_out
? bio_out
->bi_idx
: 0;
344 ctx
->sector
= sector
+ cc
->iv_offset
;
347 static int crypt_convert_block(struct crypt_config
*cc
,
348 struct convert_context
*ctx
)
350 struct bio_vec
*bv_in
= bio_iovec_idx(ctx
->bio_in
, ctx
->idx_in
);
351 struct bio_vec
*bv_out
= bio_iovec_idx(ctx
->bio_out
, ctx
->idx_out
);
352 struct dm_crypt_request dmreq
;
354 sg_init_table(&dmreq
.sg_in
, 1);
355 sg_set_page(&dmreq
.sg_in
, bv_in
->bv_page
, 1 << SECTOR_SHIFT
,
356 bv_in
->bv_offset
+ ctx
->offset_in
);
358 sg_init_table(&dmreq
.sg_out
, 1);
359 sg_set_page(&dmreq
.sg_out
, bv_out
->bv_page
, 1 << SECTOR_SHIFT
,
360 bv_out
->bv_offset
+ ctx
->offset_out
);
362 ctx
->offset_in
+= 1 << SECTOR_SHIFT
;
363 if (ctx
->offset_in
>= bv_in
->bv_len
) {
368 ctx
->offset_out
+= 1 << SECTOR_SHIFT
;
369 if (ctx
->offset_out
>= bv_out
->bv_len
) {
374 return crypt_convert_scatterlist(cc
, &dmreq
.sg_out
, &dmreq
.sg_in
,
376 bio_data_dir(ctx
->bio_in
) == WRITE
,
381 * Encrypt / decrypt data from one bio to another one (can be the same one)
383 static int crypt_convert(struct crypt_config
*cc
,
384 struct convert_context
*ctx
)
388 while(ctx
->idx_in
< ctx
->bio_in
->bi_vcnt
&&
389 ctx
->idx_out
< ctx
->bio_out
->bi_vcnt
) {
390 r
= crypt_convert_block(cc
, ctx
);
400 static void dm_crypt_bio_destructor(struct bio
*bio
)
402 struct dm_crypt_io
*io
= bio
->bi_private
;
403 struct crypt_config
*cc
= io
->target
->private;
405 bio_free(bio
, cc
->bs
);
409 * Generate a new unfragmented bio with the given size
410 * This should never violate the device limitations
411 * May return a smaller bio when running out of pages
413 static struct bio
*crypt_alloc_buffer(struct dm_crypt_io
*io
, unsigned size
)
415 struct crypt_config
*cc
= io
->target
->private;
417 unsigned int nr_iovecs
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
418 gfp_t gfp_mask
= GFP_NOIO
| __GFP_HIGHMEM
;
422 clone
= bio_alloc_bioset(GFP_NOIO
, nr_iovecs
, cc
->bs
);
426 clone_init(io
, clone
);
428 for (i
= 0; i
< nr_iovecs
; i
++) {
429 page
= mempool_alloc(cc
->page_pool
, gfp_mask
);
434 * if additional pages cannot be allocated without waiting,
435 * return a partially allocated bio, the caller will then try
436 * to allocate additional bios while submitting this partial bio
438 if (i
== (MIN_BIO_PAGES
- 1))
439 gfp_mask
= (gfp_mask
| __GFP_NOWARN
) & ~__GFP_WAIT
;
441 len
= (size
> PAGE_SIZE
) ? PAGE_SIZE
: size
;
443 if (!bio_add_page(clone
, page
, len
, 0)) {
444 mempool_free(page
, cc
->page_pool
);
451 if (!clone
->bi_size
) {
459 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
)
464 for (i
= 0; i
< clone
->bi_vcnt
; i
++) {
465 bv
= bio_iovec_idx(clone
, i
);
466 BUG_ON(!bv
->bv_page
);
467 mempool_free(bv
->bv_page
, cc
->page_pool
);
473 * One of the bios was finished. Check for completion of
474 * the whole request and correctly clean up the buffer.
476 static void crypt_dec_pending(struct dm_crypt_io
*io
)
478 struct crypt_config
*cc
= io
->target
->private;
480 if (!atomic_dec_and_test(&io
->pending
))
483 bio_endio(io
->base_bio
, io
->error
);
484 mempool_free(io
, cc
->io_pool
);
488 * kcryptd/kcryptd_io:
490 * Needed because it would be very unwise to do decryption in an
493 * kcryptd performs the actual encryption or decryption.
495 * kcryptd_io performs the IO submission.
497 * They must be separated as otherwise the final stages could be
498 * starved by new requests which can block in the first stages due
499 * to memory allocation.
501 static void crypt_endio(struct bio
*clone
, int error
)
503 struct dm_crypt_io
*io
= clone
->bi_private
;
504 struct crypt_config
*cc
= io
->target
->private;
505 unsigned rw
= bio_data_dir(clone
);
507 if (unlikely(!bio_flagged(clone
, BIO_UPTODATE
) && !error
))
511 * free the processed pages
514 crypt_free_buffer_pages(cc
, clone
);
518 if (rw
== READ
&& !error
) {
519 kcryptd_queue_crypt(io
);
526 crypt_dec_pending(io
);
529 static void clone_init(struct dm_crypt_io
*io
, struct bio
*clone
)
531 struct crypt_config
*cc
= io
->target
->private;
533 clone
->bi_private
= io
;
534 clone
->bi_end_io
= crypt_endio
;
535 clone
->bi_bdev
= cc
->dev
->bdev
;
536 clone
->bi_rw
= io
->base_bio
->bi_rw
;
537 clone
->bi_destructor
= dm_crypt_bio_destructor
;
540 static void kcryptd_io_read(struct dm_crypt_io
*io
)
542 struct crypt_config
*cc
= io
->target
->private;
543 struct bio
*base_bio
= io
->base_bio
;
546 atomic_inc(&io
->pending
);
549 * The block layer might modify the bvec array, so always
550 * copy the required bvecs because we need the original
551 * one in order to decrypt the whole bio data *afterwards*.
553 clone
= bio_alloc_bioset(GFP_NOIO
, bio_segments(base_bio
), cc
->bs
);
554 if (unlikely(!clone
)) {
556 crypt_dec_pending(io
);
560 clone_init(io
, clone
);
562 clone
->bi_vcnt
= bio_segments(base_bio
);
563 clone
->bi_size
= base_bio
->bi_size
;
564 clone
->bi_sector
= cc
->start
+ io
->sector
;
565 memcpy(clone
->bi_io_vec
, bio_iovec(base_bio
),
566 sizeof(struct bio_vec
) * clone
->bi_vcnt
);
568 generic_make_request(clone
);
571 static void kcryptd_io_write(struct dm_crypt_io
*io
)
575 static void kcryptd_io(struct work_struct
*work
)
577 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
579 if (bio_data_dir(io
->base_bio
) == READ
)
582 kcryptd_io_write(io
);
585 static void kcryptd_queue_io(struct dm_crypt_io
*io
)
587 struct crypt_config
*cc
= io
->target
->private;
589 INIT_WORK(&io
->work
, kcryptd_io
);
590 queue_work(cc
->io_queue
, &io
->work
);
593 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io
*io
, int error
)
595 struct bio
*clone
= io
->ctx
.bio_out
;
596 struct crypt_config
*cc
= io
->target
->private;
598 if (unlikely(error
< 0)) {
599 crypt_free_buffer_pages(cc
, clone
);
605 /* crypt_convert should have filled the clone bio */
606 BUG_ON(io
->ctx
.idx_out
< clone
->bi_vcnt
);
608 clone
->bi_sector
= cc
->start
+ io
->sector
;
609 io
->sector
+= bio_sectors(clone
);
611 atomic_inc(&io
->pending
);
612 generic_make_request(clone
);
615 static void kcryptd_crypt_write_convert_loop(struct dm_crypt_io
*io
)
617 struct crypt_config
*cc
= io
->target
->private;
619 unsigned remaining
= io
->base_bio
->bi_size
;
623 * The allocated buffers can be smaller than the whole bio,
624 * so repeat the whole process until all the data can be handled.
627 clone
= crypt_alloc_buffer(io
, remaining
);
628 if (unlikely(!clone
)) {
633 io
->ctx
.bio_out
= clone
;
636 remaining
-= clone
->bi_size
;
638 r
= crypt_convert(cc
, &io
->ctx
);
640 kcryptd_crypt_write_io_submit(io
, r
);
644 /* out of memory -> run queues */
645 if (unlikely(remaining
))
646 congestion_wait(WRITE
, HZ
/100);
650 static void kcryptd_crypt_write_convert(struct dm_crypt_io
*io
)
652 struct crypt_config
*cc
= io
->target
->private;
655 * Prevent io from disappearing until this function completes.
657 atomic_inc(&io
->pending
);
659 crypt_convert_init(cc
, &io
->ctx
, NULL
, io
->base_bio
, io
->sector
);
660 kcryptd_crypt_write_convert_loop(io
);
662 crypt_dec_pending(io
);
665 static void kcryptd_crypt_read_done(struct dm_crypt_io
*io
, int error
)
667 if (unlikely(error
< 0))
670 crypt_dec_pending(io
);
673 static void kcryptd_crypt_read_convert(struct dm_crypt_io
*io
)
675 struct crypt_config
*cc
= io
->target
->private;
678 crypt_convert_init(cc
, &io
->ctx
, io
->base_bio
, io
->base_bio
,
681 r
= crypt_convert(cc
, &io
->ctx
);
683 kcryptd_crypt_read_done(io
, r
);
686 static void kcryptd_crypt(struct work_struct
*work
)
688 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
690 if (bio_data_dir(io
->base_bio
) == READ
)
691 kcryptd_crypt_read_convert(io
);
693 kcryptd_crypt_write_convert(io
);
696 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
)
698 struct crypt_config
*cc
= io
->target
->private;
700 INIT_WORK(&io
->work
, kcryptd_crypt
);
701 queue_work(cc
->crypt_queue
, &io
->work
);
705 * Decode key from its hex representation
707 static int crypt_decode_key(u8
*key
, char *hex
, unsigned int size
)
715 for (i
= 0; i
< size
; i
++) {
719 key
[i
] = (u8
)simple_strtoul(buffer
, &endp
, 16);
721 if (endp
!= &buffer
[2])
732 * Encode key into its hex representation
734 static void crypt_encode_key(char *hex
, u8
*key
, unsigned int size
)
738 for (i
= 0; i
< size
; i
++) {
739 sprintf(hex
, "%02x", *key
);
745 static int crypt_set_key(struct crypt_config
*cc
, char *key
)
747 unsigned key_size
= strlen(key
) >> 1;
749 if (cc
->key_size
&& cc
->key_size
!= key_size
)
752 cc
->key_size
= key_size
; /* initial settings */
754 if ((!key_size
&& strcmp(key
, "-")) ||
755 (key_size
&& crypt_decode_key(cc
->key
, key
, key_size
) < 0))
758 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
763 static int crypt_wipe_key(struct crypt_config
*cc
)
765 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
766 memset(&cc
->key
, 0, cc
->key_size
* sizeof(u8
));
771 * Construct an encryption mapping:
772 * <cipher> <key> <iv_offset> <dev_path> <start>
774 static int crypt_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
776 struct crypt_config
*cc
;
777 struct crypto_blkcipher
*tfm
;
783 unsigned int key_size
;
784 unsigned long long tmpll
;
787 ti
->error
= "Not enough arguments";
792 cipher
= strsep(&tmp
, "-");
793 chainmode
= strsep(&tmp
, "-");
794 ivopts
= strsep(&tmp
, "-");
795 ivmode
= strsep(&ivopts
, ":");
798 DMWARN("Unexpected additional cipher options");
800 key_size
= strlen(argv
[1]) >> 1;
802 cc
= kzalloc(sizeof(*cc
) + key_size
* sizeof(u8
), GFP_KERNEL
);
805 "Cannot allocate transparent encryption context";
809 if (crypt_set_key(cc
, argv
[1])) {
810 ti
->error
= "Error decoding key";
814 /* Compatiblity mode for old dm-crypt cipher strings */
815 if (!chainmode
|| (strcmp(chainmode
, "plain") == 0 && !ivmode
)) {
820 if (strcmp(chainmode
, "ecb") && !ivmode
) {
821 ti
->error
= "This chaining mode requires an IV mechanism";
825 if (snprintf(cc
->cipher
, CRYPTO_MAX_ALG_NAME
, "%s(%s)",
826 chainmode
, cipher
) >= CRYPTO_MAX_ALG_NAME
) {
827 ti
->error
= "Chain mode + cipher name is too long";
831 tfm
= crypto_alloc_blkcipher(cc
->cipher
, 0, CRYPTO_ALG_ASYNC
);
833 ti
->error
= "Error allocating crypto tfm";
837 strcpy(cc
->cipher
, cipher
);
838 strcpy(cc
->chainmode
, chainmode
);
842 * Choose ivmode. Valid modes: "plain", "essiv:<esshash>", "benbi".
843 * See comments at iv code
847 cc
->iv_gen_ops
= NULL
;
848 else if (strcmp(ivmode
, "plain") == 0)
849 cc
->iv_gen_ops
= &crypt_iv_plain_ops
;
850 else if (strcmp(ivmode
, "essiv") == 0)
851 cc
->iv_gen_ops
= &crypt_iv_essiv_ops
;
852 else if (strcmp(ivmode
, "benbi") == 0)
853 cc
->iv_gen_ops
= &crypt_iv_benbi_ops
;
854 else if (strcmp(ivmode
, "null") == 0)
855 cc
->iv_gen_ops
= &crypt_iv_null_ops
;
857 ti
->error
= "Invalid IV mode";
861 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->ctr
&&
862 cc
->iv_gen_ops
->ctr(cc
, ti
, ivopts
) < 0)
865 cc
->iv_size
= crypto_blkcipher_ivsize(tfm
);
867 /* at least a 64 bit sector number should fit in our buffer */
868 cc
->iv_size
= max(cc
->iv_size
,
869 (unsigned int)(sizeof(u64
) / sizeof(u8
)));
871 if (cc
->iv_gen_ops
) {
872 DMWARN("Selected cipher does not support IVs");
873 if (cc
->iv_gen_ops
->dtr
)
874 cc
->iv_gen_ops
->dtr(cc
);
875 cc
->iv_gen_ops
= NULL
;
879 cc
->io_pool
= mempool_create_slab_pool(MIN_IOS
, _crypt_io_pool
);
881 ti
->error
= "Cannot allocate crypt io mempool";
885 cc
->page_pool
= mempool_create_page_pool(MIN_POOL_PAGES
, 0);
886 if (!cc
->page_pool
) {
887 ti
->error
= "Cannot allocate page mempool";
891 cc
->bs
= bioset_create(MIN_IOS
, MIN_IOS
);
893 ti
->error
= "Cannot allocate crypt bioset";
897 if (crypto_blkcipher_setkey(tfm
, cc
->key
, key_size
) < 0) {
898 ti
->error
= "Error setting key";
902 if (sscanf(argv
[2], "%llu", &tmpll
) != 1) {
903 ti
->error
= "Invalid iv_offset sector";
906 cc
->iv_offset
= tmpll
;
908 if (sscanf(argv
[4], "%llu", &tmpll
) != 1) {
909 ti
->error
= "Invalid device sector";
914 if (dm_get_device(ti
, argv
[3], cc
->start
, ti
->len
,
915 dm_table_get_mode(ti
->table
), &cc
->dev
)) {
916 ti
->error
= "Device lookup failed";
920 if (ivmode
&& cc
->iv_gen_ops
) {
923 cc
->iv_mode
= kmalloc(strlen(ivmode
) + 1, GFP_KERNEL
);
925 ti
->error
= "Error kmallocing iv_mode string";
926 goto bad_ivmode_string
;
928 strcpy(cc
->iv_mode
, ivmode
);
932 cc
->io_queue
= create_singlethread_workqueue("kcryptd_io");
934 ti
->error
= "Couldn't create kcryptd io queue";
938 cc
->crypt_queue
= create_singlethread_workqueue("kcryptd");
939 if (!cc
->crypt_queue
) {
940 ti
->error
= "Couldn't create kcryptd queue";
941 goto bad_crypt_queue
;
948 destroy_workqueue(cc
->io_queue
);
952 dm_put_device(ti
, cc
->dev
);
956 mempool_destroy(cc
->page_pool
);
958 mempool_destroy(cc
->io_pool
);
960 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
961 cc
->iv_gen_ops
->dtr(cc
);
963 crypto_free_blkcipher(tfm
);
965 /* Must zero key material before freeing */
966 memset(cc
, 0, sizeof(*cc
) + cc
->key_size
* sizeof(u8
));
971 static void crypt_dtr(struct dm_target
*ti
)
973 struct crypt_config
*cc
= (struct crypt_config
*) ti
->private;
975 destroy_workqueue(cc
->io_queue
);
976 destroy_workqueue(cc
->crypt_queue
);
979 mempool_destroy(cc
->page_pool
);
980 mempool_destroy(cc
->io_pool
);
983 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
984 cc
->iv_gen_ops
->dtr(cc
);
985 crypto_free_blkcipher(cc
->tfm
);
986 dm_put_device(ti
, cc
->dev
);
988 /* Must zero key material before freeing */
989 memset(cc
, 0, sizeof(*cc
) + cc
->key_size
* sizeof(u8
));
993 static int crypt_map(struct dm_target
*ti
, struct bio
*bio
,
994 union map_info
*map_context
)
996 struct crypt_config
*cc
= ti
->private;
997 struct dm_crypt_io
*io
;
999 io
= mempool_alloc(cc
->io_pool
, GFP_NOIO
);
1002 io
->sector
= bio
->bi_sector
- ti
->begin
;
1004 atomic_set(&io
->pending
, 0);
1006 if (bio_data_dir(io
->base_bio
) == READ
)
1007 kcryptd_queue_io(io
);
1009 kcryptd_queue_crypt(io
);
1011 return DM_MAPIO_SUBMITTED
;
1014 static int crypt_status(struct dm_target
*ti
, status_type_t type
,
1015 char *result
, unsigned int maxlen
)
1017 struct crypt_config
*cc
= (struct crypt_config
*) ti
->private;
1018 unsigned int sz
= 0;
1021 case STATUSTYPE_INFO
:
1025 case STATUSTYPE_TABLE
:
1027 DMEMIT("%s-%s-%s ", cc
->cipher
, cc
->chainmode
,
1030 DMEMIT("%s-%s ", cc
->cipher
, cc
->chainmode
);
1032 if (cc
->key_size
> 0) {
1033 if ((maxlen
- sz
) < ((cc
->key_size
<< 1) + 1))
1036 crypt_encode_key(result
+ sz
, cc
->key
, cc
->key_size
);
1037 sz
+= cc
->key_size
<< 1;
1044 DMEMIT(" %llu %s %llu", (unsigned long long)cc
->iv_offset
,
1045 cc
->dev
->name
, (unsigned long long)cc
->start
);
1051 static void crypt_postsuspend(struct dm_target
*ti
)
1053 struct crypt_config
*cc
= ti
->private;
1055 set_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1058 static int crypt_preresume(struct dm_target
*ti
)
1060 struct crypt_config
*cc
= ti
->private;
1062 if (!test_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
)) {
1063 DMERR("aborting resume - crypt key is not set.");
1070 static void crypt_resume(struct dm_target
*ti
)
1072 struct crypt_config
*cc
= ti
->private;
1074 clear_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
1077 /* Message interface
1081 static int crypt_message(struct dm_target
*ti
, unsigned argc
, char **argv
)
1083 struct crypt_config
*cc
= ti
->private;
1088 if (!strnicmp(argv
[0], MESG_STR("key"))) {
1089 if (!test_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
)) {
1090 DMWARN("not suspended during key manipulation.");
1093 if (argc
== 3 && !strnicmp(argv
[1], MESG_STR("set")))
1094 return crypt_set_key(cc
, argv
[2]);
1095 if (argc
== 2 && !strnicmp(argv
[1], MESG_STR("wipe")))
1096 return crypt_wipe_key(cc
);
1100 DMWARN("unrecognised message received.");
1104 static struct target_type crypt_target
= {
1106 .version
= {1, 5, 0},
1107 .module
= THIS_MODULE
,
1111 .status
= crypt_status
,
1112 .postsuspend
= crypt_postsuspend
,
1113 .preresume
= crypt_preresume
,
1114 .resume
= crypt_resume
,
1115 .message
= crypt_message
,
1118 static int __init
dm_crypt_init(void)
1122 _crypt_io_pool
= KMEM_CACHE(dm_crypt_io
, 0);
1123 if (!_crypt_io_pool
)
1126 r
= dm_register_target(&crypt_target
);
1128 DMERR("register failed %d", r
);
1129 kmem_cache_destroy(_crypt_io_pool
);
1135 static void __exit
dm_crypt_exit(void)
1137 int r
= dm_unregister_target(&crypt_target
);
1140 DMERR("unregister failed %d", r
);
1142 kmem_cache_destroy(_crypt_io_pool
);
1145 module_init(dm_crypt_init
);
1146 module_exit(dm_crypt_exit
);
1148 MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
1149 MODULE_DESCRIPTION(DM_NAME
" target for transparent encryption / decryption");
1150 MODULE_LICENSE("GPL");