2 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
12 #include <asm/hwcap.h>
13 #include <crypto/aes.h>
14 #include <crypto/ablk_helper.h>
15 #include <crypto/algapi.h>
16 #include <linux/module.h>
17 #include <linux/cpufeature.h>
18 #include <crypto/xts.h>
20 #include "aes-ce-setkey.h"
22 #ifdef USE_V8_CRYPTO_EXTENSIONS
25 #define aes_setkey ce_aes_setkey
26 #define aes_expandkey ce_aes_expandkey
27 #define aes_ecb_encrypt ce_aes_ecb_encrypt
28 #define aes_ecb_decrypt ce_aes_ecb_decrypt
29 #define aes_cbc_encrypt ce_aes_cbc_encrypt
30 #define aes_cbc_decrypt ce_aes_cbc_decrypt
31 #define aes_ctr_encrypt ce_aes_ctr_encrypt
32 #define aes_xts_encrypt ce_aes_xts_encrypt
33 #define aes_xts_decrypt ce_aes_xts_decrypt
34 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
38 #define aes_setkey crypto_aes_set_key
39 #define aes_expandkey crypto_aes_expand_key
40 #define aes_ecb_encrypt neon_aes_ecb_encrypt
41 #define aes_ecb_decrypt neon_aes_ecb_decrypt
42 #define aes_cbc_encrypt neon_aes_cbc_encrypt
43 #define aes_cbc_decrypt neon_aes_cbc_decrypt
44 #define aes_ctr_encrypt neon_aes_ctr_encrypt
45 #define aes_xts_encrypt neon_aes_xts_encrypt
46 #define aes_xts_decrypt neon_aes_xts_decrypt
47 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
48 MODULE_ALIAS_CRYPTO("ecb(aes)");
49 MODULE_ALIAS_CRYPTO("cbc(aes)");
50 MODULE_ALIAS_CRYPTO("ctr(aes)");
51 MODULE_ALIAS_CRYPTO("xts(aes)");
54 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
55 MODULE_LICENSE("GPL v2");
57 /* defined in aes-modes.S */
58 asmlinkage
void aes_ecb_encrypt(u8 out
[], u8
const in
[], u8
const rk
[],
59 int rounds
, int blocks
, int first
);
60 asmlinkage
void aes_ecb_decrypt(u8 out
[], u8
const in
[], u8
const rk
[],
61 int rounds
, int blocks
, int first
);
63 asmlinkage
void aes_cbc_encrypt(u8 out
[], u8
const in
[], u8
const rk
[],
64 int rounds
, int blocks
, u8 iv
[], int first
);
65 asmlinkage
void aes_cbc_decrypt(u8 out
[], u8
const in
[], u8
const rk
[],
66 int rounds
, int blocks
, u8 iv
[], int first
);
68 asmlinkage
void aes_ctr_encrypt(u8 out
[], u8
const in
[], u8
const rk
[],
69 int rounds
, int blocks
, u8 ctr
[], int first
);
71 asmlinkage
void aes_xts_encrypt(u8 out
[], u8
const in
[], u8
const rk1
[],
72 int rounds
, int blocks
, u8
const rk2
[], u8 iv
[],
74 asmlinkage
void aes_xts_decrypt(u8 out
[], u8
const in
[], u8
const rk1
[],
75 int rounds
, int blocks
, u8
const rk2
[], u8 iv
[],
78 struct crypto_aes_xts_ctx
{
79 struct crypto_aes_ctx key1
;
80 struct crypto_aes_ctx
__aligned(8) key2
;
83 static int xts_set_key(struct crypto_tfm
*tfm
, const u8
*in_key
,
86 struct crypto_aes_xts_ctx
*ctx
= crypto_tfm_ctx(tfm
);
89 ret
= xts_check_key(tfm
, in_key
, key_len
);
93 ret
= aes_expandkey(&ctx
->key1
, in_key
, key_len
/ 2);
95 ret
= aes_expandkey(&ctx
->key2
, &in_key
[key_len
/ 2],
100 tfm
->crt_flags
|= CRYPTO_TFM_RES_BAD_KEY_LEN
;
104 static int ecb_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
105 struct scatterlist
*src
, unsigned int nbytes
)
107 struct crypto_aes_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
108 int err
, first
, rounds
= 6 + ctx
->key_length
/ 4;
109 struct blkcipher_walk walk
;
112 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
113 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
114 err
= blkcipher_walk_virt(desc
, &walk
);
117 for (first
= 1; (blocks
= (walk
.nbytes
/ AES_BLOCK_SIZE
)); first
= 0) {
118 aes_ecb_encrypt(walk
.dst
.virt
.addr
, walk
.src
.virt
.addr
,
119 (u8
*)ctx
->key_enc
, rounds
, blocks
, first
);
120 err
= blkcipher_walk_done(desc
, &walk
, walk
.nbytes
% AES_BLOCK_SIZE
);
126 static int ecb_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
127 struct scatterlist
*src
, unsigned int nbytes
)
129 struct crypto_aes_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
130 int err
, first
, rounds
= 6 + ctx
->key_length
/ 4;
131 struct blkcipher_walk walk
;
134 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
135 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
136 err
= blkcipher_walk_virt(desc
, &walk
);
139 for (first
= 1; (blocks
= (walk
.nbytes
/ AES_BLOCK_SIZE
)); first
= 0) {
140 aes_ecb_decrypt(walk
.dst
.virt
.addr
, walk
.src
.virt
.addr
,
141 (u8
*)ctx
->key_dec
, rounds
, blocks
, first
);
142 err
= blkcipher_walk_done(desc
, &walk
, walk
.nbytes
% AES_BLOCK_SIZE
);
148 static int cbc_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
149 struct scatterlist
*src
, unsigned int nbytes
)
151 struct crypto_aes_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
152 int err
, first
, rounds
= 6 + ctx
->key_length
/ 4;
153 struct blkcipher_walk walk
;
156 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
157 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
158 err
= blkcipher_walk_virt(desc
, &walk
);
161 for (first
= 1; (blocks
= (walk
.nbytes
/ AES_BLOCK_SIZE
)); first
= 0) {
162 aes_cbc_encrypt(walk
.dst
.virt
.addr
, walk
.src
.virt
.addr
,
163 (u8
*)ctx
->key_enc
, rounds
, blocks
, walk
.iv
,
165 err
= blkcipher_walk_done(desc
, &walk
, walk
.nbytes
% AES_BLOCK_SIZE
);
171 static int cbc_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
172 struct scatterlist
*src
, unsigned int nbytes
)
174 struct crypto_aes_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
175 int err
, first
, rounds
= 6 + ctx
->key_length
/ 4;
176 struct blkcipher_walk walk
;
179 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
180 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
181 err
= blkcipher_walk_virt(desc
, &walk
);
184 for (first
= 1; (blocks
= (walk
.nbytes
/ AES_BLOCK_SIZE
)); first
= 0) {
185 aes_cbc_decrypt(walk
.dst
.virt
.addr
, walk
.src
.virt
.addr
,
186 (u8
*)ctx
->key_dec
, rounds
, blocks
, walk
.iv
,
188 err
= blkcipher_walk_done(desc
, &walk
, walk
.nbytes
% AES_BLOCK_SIZE
);
194 static int ctr_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
195 struct scatterlist
*src
, unsigned int nbytes
)
197 struct crypto_aes_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
198 int err
, first
, rounds
= 6 + ctx
->key_length
/ 4;
199 struct blkcipher_walk walk
;
202 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
203 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
204 err
= blkcipher_walk_virt_block(desc
, &walk
, AES_BLOCK_SIZE
);
208 while ((blocks
= (walk
.nbytes
/ AES_BLOCK_SIZE
))) {
209 aes_ctr_encrypt(walk
.dst
.virt
.addr
, walk
.src
.virt
.addr
,
210 (u8
*)ctx
->key_enc
, rounds
, blocks
, walk
.iv
,
213 nbytes
-= blocks
* AES_BLOCK_SIZE
;
214 if (nbytes
&& nbytes
== walk
.nbytes
% AES_BLOCK_SIZE
)
216 err
= blkcipher_walk_done(desc
, &walk
,
217 walk
.nbytes
% AES_BLOCK_SIZE
);
220 u8
*tdst
= walk
.dst
.virt
.addr
+ blocks
* AES_BLOCK_SIZE
;
221 u8
*tsrc
= walk
.src
.virt
.addr
+ blocks
* AES_BLOCK_SIZE
;
222 u8
__aligned(8) tail
[AES_BLOCK_SIZE
];
225 * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
226 * to tell aes_ctr_encrypt() to only read half a block.
228 blocks
= (nbytes
<= 8) ? -1 : 1;
230 aes_ctr_encrypt(tail
, tsrc
, (u8
*)ctx
->key_enc
, rounds
,
231 blocks
, walk
.iv
, first
);
232 memcpy(tdst
, tail
, nbytes
);
233 err
= blkcipher_walk_done(desc
, &walk
, 0);
240 static int xts_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
241 struct scatterlist
*src
, unsigned int nbytes
)
243 struct crypto_aes_xts_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
244 int err
, first
, rounds
= 6 + ctx
->key1
.key_length
/ 4;
245 struct blkcipher_walk walk
;
248 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
249 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
250 err
= blkcipher_walk_virt(desc
, &walk
);
253 for (first
= 1; (blocks
= (walk
.nbytes
/ AES_BLOCK_SIZE
)); first
= 0) {
254 aes_xts_encrypt(walk
.dst
.virt
.addr
, walk
.src
.virt
.addr
,
255 (u8
*)ctx
->key1
.key_enc
, rounds
, blocks
,
256 (u8
*)ctx
->key2
.key_enc
, walk
.iv
, first
);
257 err
= blkcipher_walk_done(desc
, &walk
, walk
.nbytes
% AES_BLOCK_SIZE
);
264 static int xts_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
265 struct scatterlist
*src
, unsigned int nbytes
)
267 struct crypto_aes_xts_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
268 int err
, first
, rounds
= 6 + ctx
->key1
.key_length
/ 4;
269 struct blkcipher_walk walk
;
272 desc
->flags
&= ~CRYPTO_TFM_REQ_MAY_SLEEP
;
273 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
274 err
= blkcipher_walk_virt(desc
, &walk
);
277 for (first
= 1; (blocks
= (walk
.nbytes
/ AES_BLOCK_SIZE
)); first
= 0) {
278 aes_xts_decrypt(walk
.dst
.virt
.addr
, walk
.src
.virt
.addr
,
279 (u8
*)ctx
->key1
.key_dec
, rounds
, blocks
,
280 (u8
*)ctx
->key2
.key_enc
, walk
.iv
, first
);
281 err
= blkcipher_walk_done(desc
, &walk
, walk
.nbytes
% AES_BLOCK_SIZE
);
288 static struct crypto_alg aes_algs
[] = { {
289 .cra_name
= "__ecb-aes-" MODE
,
290 .cra_driver_name
= "__driver-ecb-aes-" MODE
,
292 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
|
294 .cra_blocksize
= AES_BLOCK_SIZE
,
295 .cra_ctxsize
= sizeof(struct crypto_aes_ctx
),
297 .cra_type
= &crypto_blkcipher_type
,
298 .cra_module
= THIS_MODULE
,
300 .min_keysize
= AES_MIN_KEY_SIZE
,
301 .max_keysize
= AES_MAX_KEY_SIZE
,
303 .setkey
= aes_setkey
,
304 .encrypt
= ecb_encrypt
,
305 .decrypt
= ecb_decrypt
,
308 .cra_name
= "__cbc-aes-" MODE
,
309 .cra_driver_name
= "__driver-cbc-aes-" MODE
,
311 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
|
313 .cra_blocksize
= AES_BLOCK_SIZE
,
314 .cra_ctxsize
= sizeof(struct crypto_aes_ctx
),
316 .cra_type
= &crypto_blkcipher_type
,
317 .cra_module
= THIS_MODULE
,
319 .min_keysize
= AES_MIN_KEY_SIZE
,
320 .max_keysize
= AES_MAX_KEY_SIZE
,
321 .ivsize
= AES_BLOCK_SIZE
,
322 .setkey
= aes_setkey
,
323 .encrypt
= cbc_encrypt
,
324 .decrypt
= cbc_decrypt
,
327 .cra_name
= "__ctr-aes-" MODE
,
328 .cra_driver_name
= "__driver-ctr-aes-" MODE
,
330 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
|
333 .cra_ctxsize
= sizeof(struct crypto_aes_ctx
),
335 .cra_type
= &crypto_blkcipher_type
,
336 .cra_module
= THIS_MODULE
,
338 .min_keysize
= AES_MIN_KEY_SIZE
,
339 .max_keysize
= AES_MAX_KEY_SIZE
,
340 .ivsize
= AES_BLOCK_SIZE
,
341 .setkey
= aes_setkey
,
342 .encrypt
= ctr_encrypt
,
343 .decrypt
= ctr_encrypt
,
346 .cra_name
= "__xts-aes-" MODE
,
347 .cra_driver_name
= "__driver-xts-aes-" MODE
,
349 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
|
351 .cra_blocksize
= AES_BLOCK_SIZE
,
352 .cra_ctxsize
= sizeof(struct crypto_aes_xts_ctx
),
354 .cra_type
= &crypto_blkcipher_type
,
355 .cra_module
= THIS_MODULE
,
357 .min_keysize
= 2 * AES_MIN_KEY_SIZE
,
358 .max_keysize
= 2 * AES_MAX_KEY_SIZE
,
359 .ivsize
= AES_BLOCK_SIZE
,
360 .setkey
= xts_set_key
,
361 .encrypt
= xts_encrypt
,
362 .decrypt
= xts_decrypt
,
365 .cra_name
= "ecb(aes)",
366 .cra_driver_name
= "ecb-aes-" MODE
,
367 .cra_priority
= PRIO
,
368 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|CRYPTO_ALG_ASYNC
,
369 .cra_blocksize
= AES_BLOCK_SIZE
,
370 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
372 .cra_type
= &crypto_ablkcipher_type
,
373 .cra_module
= THIS_MODULE
,
374 .cra_init
= ablk_init
,
375 .cra_exit
= ablk_exit
,
377 .min_keysize
= AES_MIN_KEY_SIZE
,
378 .max_keysize
= AES_MAX_KEY_SIZE
,
380 .setkey
= ablk_set_key
,
381 .encrypt
= ablk_encrypt
,
382 .decrypt
= ablk_decrypt
,
385 .cra_name
= "cbc(aes)",
386 .cra_driver_name
= "cbc-aes-" MODE
,
387 .cra_priority
= PRIO
,
388 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|CRYPTO_ALG_ASYNC
,
389 .cra_blocksize
= AES_BLOCK_SIZE
,
390 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
392 .cra_type
= &crypto_ablkcipher_type
,
393 .cra_module
= THIS_MODULE
,
394 .cra_init
= ablk_init
,
395 .cra_exit
= ablk_exit
,
397 .min_keysize
= AES_MIN_KEY_SIZE
,
398 .max_keysize
= AES_MAX_KEY_SIZE
,
399 .ivsize
= AES_BLOCK_SIZE
,
400 .setkey
= ablk_set_key
,
401 .encrypt
= ablk_encrypt
,
402 .decrypt
= ablk_decrypt
,
405 .cra_name
= "ctr(aes)",
406 .cra_driver_name
= "ctr-aes-" MODE
,
407 .cra_priority
= PRIO
,
408 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|CRYPTO_ALG_ASYNC
,
410 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
412 .cra_type
= &crypto_ablkcipher_type
,
413 .cra_module
= THIS_MODULE
,
414 .cra_init
= ablk_init
,
415 .cra_exit
= ablk_exit
,
417 .min_keysize
= AES_MIN_KEY_SIZE
,
418 .max_keysize
= AES_MAX_KEY_SIZE
,
419 .ivsize
= AES_BLOCK_SIZE
,
420 .setkey
= ablk_set_key
,
421 .encrypt
= ablk_encrypt
,
422 .decrypt
= ablk_decrypt
,
425 .cra_name
= "xts(aes)",
426 .cra_driver_name
= "xts-aes-" MODE
,
427 .cra_priority
= PRIO
,
428 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|CRYPTO_ALG_ASYNC
,
429 .cra_blocksize
= AES_BLOCK_SIZE
,
430 .cra_ctxsize
= sizeof(struct async_helper_ctx
),
432 .cra_type
= &crypto_ablkcipher_type
,
433 .cra_module
= THIS_MODULE
,
434 .cra_init
= ablk_init
,
435 .cra_exit
= ablk_exit
,
437 .min_keysize
= 2 * AES_MIN_KEY_SIZE
,
438 .max_keysize
= 2 * AES_MAX_KEY_SIZE
,
439 .ivsize
= AES_BLOCK_SIZE
,
440 .setkey
= ablk_set_key
,
441 .encrypt
= ablk_encrypt
,
442 .decrypt
= ablk_decrypt
,
446 static int __init
aes_init(void)
448 return crypto_register_algs(aes_algs
, ARRAY_SIZE(aes_algs
));
451 static void __exit
aes_exit(void)
453 crypto_unregister_algs(aes_algs
, ARRAY_SIZE(aes_algs
));
456 #ifdef USE_V8_CRYPTO_EXTENSIONS
457 module_cpu_feature_match(AES
, aes_init
);
459 module_init(aes_init
);
461 module_exit(aes_exit
);