projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
KVM: x86: mmu: remove argument to kvm_init_shadow_mmu and kvm_init_shadow_ept_mmu
[deliverable/linux.git] / arch / x86 / crypto / aesni-intel_glue.c
1 /*
2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 */
21
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/crypto.h>
25 #include <linux/module.h>
26 #include <linux/err.h>
27 #include <crypto/algapi.h>
28 #include <crypto/aes.h>
29 #include <crypto/cryptd.h>
30 #include <crypto/ctr.h>
31 #include <crypto/b128ops.h>
32 #include <crypto/lrw.h>
33 #include <crypto/xts.h>
34 #include <asm/cpu_device_id.h>
35 #include <asm/i387.h>
36 #include <asm/crypto/aes.h>
37 #include <crypto/ablk_helper.h>
38 #include <crypto/scatterwalk.h>
39 #include <crypto/internal/aead.h>
40 #include <linux/workqueue.h>
41 #include <linux/spinlock.h>
42 #ifdef CONFIG_X86_64
43 #include <asm/crypto/glue_helper.h>
44 #endif
45
46 /* This data is stored at the end of the crypto_tfm struct.
47 * It's a type of per "session" data storage location.
48 * This needs to be 16 byte aligned.
49 */
50 struct aesni_rfc4106_gcm_ctx {
51 u8 hash_subkey[16];
52 struct crypto_aes_ctx aes_key_expanded;
53 u8 nonce[4];
54 struct cryptd_aead *cryptd_tfm;
55 };
56
57 struct aesni_gcm_set_hash_subkey_result {
58 int err;
59 struct completion completion;
60 };
61
62 struct aesni_hash_subkey_req_data {
63 u8 iv[16];
64 struct aesni_gcm_set_hash_subkey_result result;
65 struct scatterlist sg;
66 };
67
68 #define AESNI_ALIGN (16)
69 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
70 #define RFC4106_HASH_SUBKEY_SIZE 16
71
72 struct aesni_lrw_ctx {
73 struct lrw_table_ctx lrw_table;
74 u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
75 };
76
77 struct aesni_xts_ctx {
78 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
79 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
80 };
81
82 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
83 unsigned int key_len);
84 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
85 const u8 *in);
86 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
87 const u8 *in);
88 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
89 const u8 *in, unsigned int len);
90 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
91 const u8 *in, unsigned int len);
92 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
93 const u8 *in, unsigned int len, u8 *iv);
94 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
95 const u8 *in, unsigned int len, u8 *iv);
96
97 int crypto_fpu_init(void);
98 void crypto_fpu_exit(void);
99
100 #define AVX_GEN2_OPTSIZE 640
101 #define AVX_GEN4_OPTSIZE 4096
102
103 #ifdef CONFIG_X86_64
104
105 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
106 const u8 *in, unsigned int len, u8 *iv);
107 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
108 const u8 *in, unsigned int len, u8 *iv);
109
110 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
111 const u8 *in, bool enc, u8 *iv);
112
113 /* asmlinkage void aesni_gcm_enc()
114 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
115 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
116 * const u8 *in, Plaintext input
117 * unsigned long plaintext_len, Length of data in bytes for encryption.
118 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
119 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
120 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
121 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
122 * const u8 *aad, Additional Authentication Data (AAD)
123 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
124 * is going to be 8 or 12 bytes
125 * u8 *auth_tag, Authenticated Tag output.
126 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
127 * Valid values are 16 (most likely), 12 or 8.
128 */
129 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
130 const u8 *in, unsigned long plaintext_len, u8 *iv,
131 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
132 u8 *auth_tag, unsigned long auth_tag_len);
133
134 /* asmlinkage void aesni_gcm_dec()
135 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
136 * u8 *out, Plaintext output. Decrypt in-place is allowed.
137 * const u8 *in, Ciphertext input
138 * unsigned long ciphertext_len, Length of data in bytes for decryption.
139 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
140 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
141 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
142 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
143 * const u8 *aad, Additional Authentication Data (AAD)
144 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
145 * to be 8 or 12 bytes
146 * u8 *auth_tag, Authenticated Tag output.
147 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
148 * Valid values are 16 (most likely), 12 or 8.
149 */
150 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
151 const u8 *in, unsigned long ciphertext_len, u8 *iv,
152 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
153 u8 *auth_tag, unsigned long auth_tag_len);
154
155
156 #ifdef CONFIG_AS_AVX
157 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
158 void *keys, u8 *out, unsigned int num_bytes);
159 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
160 void *keys, u8 *out, unsigned int num_bytes);
161 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
162 void *keys, u8 *out, unsigned int num_bytes);
163 /*
164 * asmlinkage void aesni_gcm_precomp_avx_gen2()
165 * gcm_data *my_ctx_data, context data
166 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
167 */
168 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);
169
170 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
171 const u8 *in, unsigned long plaintext_len, u8 *iv,
172 const u8 *aad, unsigned long aad_len,
173 u8 *auth_tag, unsigned long auth_tag_len);
174
175 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
176 const u8 *in, unsigned long ciphertext_len, u8 *iv,
177 const u8 *aad, unsigned long aad_len,
178 u8 *auth_tag, unsigned long auth_tag_len);
179
180 static void aesni_gcm_enc_avx(void *ctx, u8 *out,
181 const u8 *in, unsigned long plaintext_len, u8 *iv,
182 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
183 u8 *auth_tag, unsigned long auth_tag_len)
184 {
185 if (plaintext_len < AVX_GEN2_OPTSIZE) {
186 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
187 aad_len, auth_tag, auth_tag_len);
188 } else {
189 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
190 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
191 aad_len, auth_tag, auth_tag_len);
192 }
193 }
194
195 static void aesni_gcm_dec_avx(void *ctx, u8 *out,
196 const u8 *in, unsigned long ciphertext_len, u8 *iv,
197 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
198 u8 *auth_tag, unsigned long auth_tag_len)
199 {
200 if (ciphertext_len < AVX_GEN2_OPTSIZE) {
201 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
202 aad_len, auth_tag, auth_tag_len);
203 } else {
204 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
205 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
206 aad_len, auth_tag, auth_tag_len);
207 }
208 }
209 #endif
210
211 #ifdef CONFIG_AS_AVX2
212 /*
213 * asmlinkage void aesni_gcm_precomp_avx_gen4()
214 * gcm_data *my_ctx_data, context data
215 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
216 */
217 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);
218
219 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
220 const u8 *in, unsigned long plaintext_len, u8 *iv,
221 const u8 *aad, unsigned long aad_len,
222 u8 *auth_tag, unsigned long auth_tag_len);
223
224 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
225 const u8 *in, unsigned long ciphertext_len, u8 *iv,
226 const u8 *aad, unsigned long aad_len,
227 u8 *auth_tag, unsigned long auth_tag_len);
228
229 static void aesni_gcm_enc_avx2(void *ctx, u8 *out,
230 const u8 *in, unsigned long plaintext_len, u8 *iv,
231 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
232 u8 *auth_tag, unsigned long auth_tag_len)
233 {
234 if (plaintext_len < AVX_GEN2_OPTSIZE) {
235 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
236 aad_len, auth_tag, auth_tag_len);
237 } else if (plaintext_len < AVX_GEN4_OPTSIZE) {
238 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
239 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
240 aad_len, auth_tag, auth_tag_len);
241 } else {
242 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
243 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
244 aad_len, auth_tag, auth_tag_len);
245 }
246 }
247
248 static void aesni_gcm_dec_avx2(void *ctx, u8 *out,
249 const u8 *in, unsigned long ciphertext_len, u8 *iv,
250 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
251 u8 *auth_tag, unsigned long auth_tag_len)
252 {
253 if (ciphertext_len < AVX_GEN2_OPTSIZE) {
254 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
255 aad, aad_len, auth_tag, auth_tag_len);
256 } else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
257 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
258 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
259 aad_len, auth_tag, auth_tag_len);
260 } else {
261 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey);
262 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
263 aad_len, auth_tag, auth_tag_len);
264 }
265 }
266 #endif
267
268 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out,
269 const u8 *in, unsigned long plaintext_len, u8 *iv,
270 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
271 u8 *auth_tag, unsigned long auth_tag_len);
272
273 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out,
274 const u8 *in, unsigned long ciphertext_len, u8 *iv,
275 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
276 u8 *auth_tag, unsigned long auth_tag_len);
277
278 static inline struct
279 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
280 {
281 return
282 (struct aesni_rfc4106_gcm_ctx *)
283 PTR_ALIGN((u8 *)
284 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
285 }
286 #endif
287
288 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
289 {
290 unsigned long addr = (unsigned long)raw_ctx;
291 unsigned long align = AESNI_ALIGN;
292
293 if (align <= crypto_tfm_ctx_alignment())
294 align = 1;
295 return (struct crypto_aes_ctx *)ALIGN(addr, align);
296 }
297
298 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
299 const u8 *in_key, unsigned int key_len)
300 {
301 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
302 u32 *flags = &tfm->crt_flags;
303 int err;
304
305 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
306 key_len != AES_KEYSIZE_256) {
307 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
308 return -EINVAL;
309 }
310
311 if (!irq_fpu_usable())
312 err = crypto_aes_expand_key(ctx, in_key, key_len);
313 else {
314 kernel_fpu_begin();
315 err = aesni_set_key(ctx, in_key, key_len);
316 kernel_fpu_end();
317 }
318
319 return err;
320 }
321
322 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
323 unsigned int key_len)
324 {
325 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
326 }
327
328 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
329 {
330 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
331
332 if (!irq_fpu_usable())
333 crypto_aes_encrypt_x86(ctx, dst, src);
334 else {
335 kernel_fpu_begin();
336 aesni_enc(ctx, dst, src);
337 kernel_fpu_end();
338 }
339 }
340
341 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
342 {
343 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
344
345 if (!irq_fpu_usable())
346 crypto_aes_decrypt_x86(ctx, dst, src);
347 else {
348 kernel_fpu_begin();
349 aesni_dec(ctx, dst, src);
350 kernel_fpu_end();
351 }
352 }
353
354 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
355 {
356 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
357
358 aesni_enc(ctx, dst, src);
359 }
360
361 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
362 {
363 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
364
365 aesni_dec(ctx, dst, src);
366 }
367
368 static int ecb_encrypt(struct blkcipher_desc *desc,
369 struct scatterlist *dst, struct scatterlist *src,
370 unsigned int nbytes)
371 {
372 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
373 struct blkcipher_walk walk;
374 int err;
375
376 blkcipher_walk_init(&walk, dst, src, nbytes);
377 err = blkcipher_walk_virt(desc, &walk);
378 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
379
380 kernel_fpu_begin();
381 while ((nbytes = walk.nbytes)) {
382 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
383 nbytes & AES_BLOCK_MASK);
384 nbytes &= AES_BLOCK_SIZE - 1;
385 err = blkcipher_walk_done(desc, &walk, nbytes);
386 }
387 kernel_fpu_end();
388
389 return err;
390 }
391
392 static int ecb_decrypt(struct blkcipher_desc *desc,
393 struct scatterlist *dst, struct scatterlist *src,
394 unsigned int nbytes)
395 {
396 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
397 struct blkcipher_walk walk;
398 int err;
399
400 blkcipher_walk_init(&walk, dst, src, nbytes);
401 err = blkcipher_walk_virt(desc, &walk);
402 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
403
404 kernel_fpu_begin();
405 while ((nbytes = walk.nbytes)) {
406 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
407 nbytes & AES_BLOCK_MASK);
408 nbytes &= AES_BLOCK_SIZE - 1;
409 err = blkcipher_walk_done(desc, &walk, nbytes);
410 }
411 kernel_fpu_end();
412
413 return err;
414 }
415
416 static int cbc_encrypt(struct blkcipher_desc *desc,
417 struct scatterlist *dst, struct scatterlist *src,
418 unsigned int nbytes)
419 {
420 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
421 struct blkcipher_walk walk;
422 int err;
423
424 blkcipher_walk_init(&walk, dst, src, nbytes);
425 err = blkcipher_walk_virt(desc, &walk);
426 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
427
428 kernel_fpu_begin();
429 while ((nbytes = walk.nbytes)) {
430 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
431 nbytes & AES_BLOCK_MASK, walk.iv);
432 nbytes &= AES_BLOCK_SIZE - 1;
433 err = blkcipher_walk_done(desc, &walk, nbytes);
434 }
435 kernel_fpu_end();
436
437 return err;
438 }
439
440 static int cbc_decrypt(struct blkcipher_desc *desc,
441 struct scatterlist *dst, struct scatterlist *src,
442 unsigned int nbytes)
443 {
444 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
445 struct blkcipher_walk walk;
446 int err;
447
448 blkcipher_walk_init(&walk, dst, src, nbytes);
449 err = blkcipher_walk_virt(desc, &walk);
450 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
451
452 kernel_fpu_begin();
453 while ((nbytes = walk.nbytes)) {
454 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
455 nbytes & AES_BLOCK_MASK, walk.iv);
456 nbytes &= AES_BLOCK_SIZE - 1;
457 err = blkcipher_walk_done(desc, &walk, nbytes);
458 }
459 kernel_fpu_end();
460
461 return err;
462 }
463
464 #ifdef CONFIG_X86_64
465 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
466 struct blkcipher_walk *walk)
467 {
468 u8 *ctrblk = walk->iv;
469 u8 keystream[AES_BLOCK_SIZE];
470 u8 *src = walk->src.virt.addr;
471 u8 *dst = walk->dst.virt.addr;
472 unsigned int nbytes = walk->nbytes;
473
474 aesni_enc(ctx, keystream, ctrblk);
475 crypto_xor(keystream, src, nbytes);
476 memcpy(dst, keystream, nbytes);
477 crypto_inc(ctrblk, AES_BLOCK_SIZE);
478 }
479
480 #ifdef CONFIG_AS_AVX
481 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
482 const u8 *in, unsigned int len, u8 *iv)
483 {
484 /*
485 * based on key length, override with the by8 version
486 * of ctr mode encryption/decryption for improved performance
487 * aes_set_key_common() ensures that key length is one of
488 * {128,192,256}
489 */
490 if (ctx->key_length == AES_KEYSIZE_128)
491 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
492 else if (ctx->key_length == AES_KEYSIZE_192)
493 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
494 else
495 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
496 }
497 #endif
498
499 static int ctr_crypt(struct blkcipher_desc *desc,
500 struct scatterlist *dst, struct scatterlist *src,
501 unsigned int nbytes)
502 {
503 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
504 struct blkcipher_walk walk;
505 int err;
506
507 blkcipher_walk_init(&walk, dst, src, nbytes);
508 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
509 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
510
511 kernel_fpu_begin();
512 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
513 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
514 nbytes & AES_BLOCK_MASK, walk.iv);
515 nbytes &= AES_BLOCK_SIZE - 1;
516 err = blkcipher_walk_done(desc, &walk, nbytes);
517 }
518 if (walk.nbytes) {
519 ctr_crypt_final(ctx, &walk);
520 err = blkcipher_walk_done(desc, &walk, 0);
521 }
522 kernel_fpu_end();
523
524 return err;
525 }
526 #endif
527
528 static int ablk_ecb_init(struct crypto_tfm *tfm)
529 {
530 return ablk_init_common(tfm, "__driver-ecb-aes-aesni");
531 }
532
533 static int ablk_cbc_init(struct crypto_tfm *tfm)
534 {
535 return ablk_init_common(tfm, "__driver-cbc-aes-aesni");
536 }
537
538 #ifdef CONFIG_X86_64
539 static int ablk_ctr_init(struct crypto_tfm *tfm)
540 {
541 return ablk_init_common(tfm, "__driver-ctr-aes-aesni");
542 }
543
544 #endif
545
546 #if IS_ENABLED(CONFIG_CRYPTO_PCBC)
547 static int ablk_pcbc_init(struct crypto_tfm *tfm)
548 {
549 return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))");
550 }
551 #endif
552
553 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
554 {
555 aesni_ecb_enc(ctx, blks, blks, nbytes);
556 }
557
558 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
559 {
560 aesni_ecb_dec(ctx, blks, blks, nbytes);
561 }
562
563 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
564 unsigned int keylen)
565 {
566 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
567 int err;
568
569 err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,
570 keylen - AES_BLOCK_SIZE);
571 if (err)
572 return err;
573
574 return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);
575 }
576
577 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm)
578 {
579 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
580
581 lrw_free_table(&ctx->lrw_table);
582 }
583
584 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
585 struct scatterlist *src, unsigned int nbytes)
586 {
587 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
588 be128 buf[8];
589 struct lrw_crypt_req req = {
590 .tbuf = buf,
591 .tbuflen = sizeof(buf),
592
593 .table_ctx = &ctx->lrw_table,
594 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
595 .crypt_fn = lrw_xts_encrypt_callback,
596 };
597 int ret;
598
599 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
600
601 kernel_fpu_begin();
602 ret = lrw_crypt(desc, dst, src, nbytes, &req);
603 kernel_fpu_end();
604
605 return ret;
606 }
607
608 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
609 struct scatterlist *src, unsigned int nbytes)
610 {
611 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
612 be128 buf[8];
613 struct lrw_crypt_req req = {
614 .tbuf = buf,
615 .tbuflen = sizeof(buf),
616
617 .table_ctx = &ctx->lrw_table,
618 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
619 .crypt_fn = lrw_xts_decrypt_callback,
620 };
621 int ret;
622
623 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
624
625 kernel_fpu_begin();
626 ret = lrw_crypt(desc, dst, src, nbytes, &req);
627 kernel_fpu_end();
628
629 return ret;
630 }
631
632 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
633 unsigned int keylen)
634 {
635 struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);
636 u32 *flags = &tfm->crt_flags;
637 int err;
638
639 /* key consists of keys of equal size concatenated, therefore
640 * the length must be even
641 */
642 if (keylen % 2) {
643 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
644 return -EINVAL;
645 }
646
647 /* first half of xts-key is for crypt */
648 err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);
649 if (err)
650 return err;
651
652 /* second half of xts-key is for tweak */
653 return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,
654 keylen / 2);
655 }
656
657
658 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
659 {
660 aesni_enc(ctx, out, in);
661 }
662
663 #ifdef CONFIG_X86_64
664
665 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
666 {
667 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
668 }
669
670 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
671 {
672 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
673 }
674
675 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
676 {
677 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
678 }
679
680 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
681 {
682 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
683 }
684
685 static const struct common_glue_ctx aesni_enc_xts = {
686 .num_funcs = 2,
687 .fpu_blocks_limit = 1,
688
689 .funcs = { {
690 .num_blocks = 8,
691 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
692 }, {
693 .num_blocks = 1,
694 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
695 } }
696 };
697
698 static const struct common_glue_ctx aesni_dec_xts = {
699 .num_funcs = 2,
700 .fpu_blocks_limit = 1,
701
702 .funcs = { {
703 .num_blocks = 8,
704 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
705 }, {
706 .num_blocks = 1,
707 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
708 } }
709 };
710
711 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
712 struct scatterlist *src, unsigned int nbytes)
713 {
714 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
715
716 return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes,
717 XTS_TWEAK_CAST(aesni_xts_tweak),
718 aes_ctx(ctx->raw_tweak_ctx),
719 aes_ctx(ctx->raw_crypt_ctx));
720 }
721
722 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
723 struct scatterlist *src, unsigned int nbytes)
724 {
725 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
726
727 return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes,
728 XTS_TWEAK_CAST(aesni_xts_tweak),
729 aes_ctx(ctx->raw_tweak_ctx),
730 aes_ctx(ctx->raw_crypt_ctx));
731 }
732
733 #else
734
735 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
736 struct scatterlist *src, unsigned int nbytes)
737 {
738 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
739 be128 buf[8];
740 struct xts_crypt_req req = {
741 .tbuf = buf,
742 .tbuflen = sizeof(buf),
743
744 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
745 .tweak_fn = aesni_xts_tweak,
746 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
747 .crypt_fn = lrw_xts_encrypt_callback,
748 };
749 int ret;
750
751 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
752
753 kernel_fpu_begin();
754 ret = xts_crypt(desc, dst, src, nbytes, &req);
755 kernel_fpu_end();
756
757 return ret;
758 }
759
760 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
761 struct scatterlist *src, unsigned int nbytes)
762 {
763 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
764 be128 buf[8];
765 struct xts_crypt_req req = {
766 .tbuf = buf,
767 .tbuflen = sizeof(buf),
768
769 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
770 .tweak_fn = aesni_xts_tweak,
771 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
772 .crypt_fn = lrw_xts_decrypt_callback,
773 };
774 int ret;
775
776 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
777
778 kernel_fpu_begin();
779 ret = xts_crypt(desc, dst, src, nbytes, &req);
780 kernel_fpu_end();
781
782 return ret;
783 }
784
785 #endif
786
787 #ifdef CONFIG_X86_64
788 static int rfc4106_init(struct crypto_tfm *tfm)
789 {
790 struct cryptd_aead *cryptd_tfm;
791 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
792 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
793 struct crypto_aead *cryptd_child;
794 struct aesni_rfc4106_gcm_ctx *child_ctx;
795 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
796 if (IS_ERR(cryptd_tfm))
797 return PTR_ERR(cryptd_tfm);
798
799 cryptd_child = cryptd_aead_child(cryptd_tfm);
800 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
801 memcpy(child_ctx, ctx, sizeof(*ctx));
802 ctx->cryptd_tfm = cryptd_tfm;
803 tfm->crt_aead.reqsize = sizeof(struct aead_request)
804 + crypto_aead_reqsize(&cryptd_tfm->base);
805 return 0;
806 }
807
808 static void rfc4106_exit(struct crypto_tfm *tfm)
809 {
810 struct aesni_rfc4106_gcm_ctx *ctx =
811 (struct aesni_rfc4106_gcm_ctx *)
812 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
813 if (!IS_ERR(ctx->cryptd_tfm))
814 cryptd_free_aead(ctx->cryptd_tfm);
815 return;
816 }
817
818 static void
819 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
820 {
821 struct aesni_gcm_set_hash_subkey_result *result = req->data;
822
823 if (err == -EINPROGRESS)
824 return;
825 result->err = err;
826 complete(&result->completion);
827 }
828
829 static int
830 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
831 {
832 struct crypto_ablkcipher *ctr_tfm;
833 struct ablkcipher_request *req;
834 int ret = -EINVAL;
835 struct aesni_hash_subkey_req_data *req_data;
836
837 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
838 if (IS_ERR(ctr_tfm))
839 return PTR_ERR(ctr_tfm);
840
841 crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
842
843 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
844 if (ret)
845 goto out_free_ablkcipher;
846
847 ret = -ENOMEM;
848 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
849 if (!req)
850 goto out_free_ablkcipher;
851
852 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
853 if (!req_data)
854 goto out_free_request;
855
856 memset(req_data->iv, 0, sizeof(req_data->iv));
857
858 /* Clear the data in the hash sub key container to zero.*/
859 /* We want to cipher all zeros to create the hash sub key. */
860 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
861
862 init_completion(&req_data->result.completion);
863 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
864 ablkcipher_request_set_tfm(req, ctr_tfm);
865 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
866 CRYPTO_TFM_REQ_MAY_BACKLOG,
867 rfc4106_set_hash_subkey_done,
868 &req_data->result);
869
870 ablkcipher_request_set_crypt(req, &req_data->sg,
871 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
872
873 ret = crypto_ablkcipher_encrypt(req);
874 if (ret == -EINPROGRESS || ret == -EBUSY) {
875 ret = wait_for_completion_interruptible
876 (&req_data->result.completion);
877 if (!ret)
878 ret = req_data->result.err;
879 }
880 kfree(req_data);
881 out_free_request:
882 ablkcipher_request_free(req);
883 out_free_ablkcipher:
884 crypto_free_ablkcipher(ctr_tfm);
885 return ret;
886 }
887
888 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
889 unsigned int key_len)
890 {
891 int ret = 0;
892 struct crypto_tfm *tfm = crypto_aead_tfm(parent);
893 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
894 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
895 struct aesni_rfc4106_gcm_ctx *child_ctx =
896 aesni_rfc4106_gcm_ctx_get(cryptd_child);
897 u8 *new_key_align, *new_key_mem = NULL;
898
899 if (key_len < 4) {
900 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
901 return -EINVAL;
902 }
903 /*Account for 4 byte nonce at the end.*/
904 key_len -= 4;
905 if (key_len != AES_KEYSIZE_128) {
906 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
907 return -EINVAL;
908 }
909
910 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
911 /*This must be on a 16 byte boundary!*/
912 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
913 return -EINVAL;
914
915 if ((unsigned long)key % AESNI_ALIGN) {
916 /*key is not aligned: use an auxuliar aligned pointer*/
917 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
918 if (!new_key_mem)
919 return -ENOMEM;
920
921 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
922 memcpy(new_key_align, key, key_len);
923 key = new_key_align;
924 }
925
926 if (!irq_fpu_usable())
927 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
928 key, key_len);
929 else {
930 kernel_fpu_begin();
931 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
932 kernel_fpu_end();
933 }
934 /*This must be on a 16 byte boundary!*/
935 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
936 ret = -EINVAL;
937 goto exit;
938 }
939 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
940 memcpy(child_ctx, ctx, sizeof(*ctx));
941 exit:
942 kfree(new_key_mem);
943 return ret;
944 }
945
946 /* This is the Integrity Check Value (aka the authentication tag length and can
947 * be 8, 12 or 16 bytes long. */
948 static int rfc4106_set_authsize(struct crypto_aead *parent,
949 unsigned int authsize)
950 {
951 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
952 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
953
954 switch (authsize) {
955 case 8:
956 case 12:
957 case 16:
958 break;
959 default:
960 return -EINVAL;
961 }
962 crypto_aead_crt(parent)->authsize = authsize;
963 crypto_aead_crt(cryptd_child)->authsize = authsize;
964 return 0;
965 }
966
967 static int rfc4106_encrypt(struct aead_request *req)
968 {
969 int ret;
970 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
971 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
972
973 if (!irq_fpu_usable()) {
974 struct aead_request *cryptd_req =
975 (struct aead_request *) aead_request_ctx(req);
976 memcpy(cryptd_req, req, sizeof(*req));
977 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
978 return crypto_aead_encrypt(cryptd_req);
979 } else {
980 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
981 kernel_fpu_begin();
982 ret = cryptd_child->base.crt_aead.encrypt(req);
983 kernel_fpu_end();
984 return ret;
985 }
986 }
987
988 static int rfc4106_decrypt(struct aead_request *req)
989 {
990 int ret;
991 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
992 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
993
994 if (!irq_fpu_usable()) {
995 struct aead_request *cryptd_req =
996 (struct aead_request *) aead_request_ctx(req);
997 memcpy(cryptd_req, req, sizeof(*req));
998 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
999 return crypto_aead_decrypt(cryptd_req);
1000 } else {
1001 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1002 kernel_fpu_begin();
1003 ret = cryptd_child->base.crt_aead.decrypt(req);
1004 kernel_fpu_end();
1005 return ret;
1006 }
1007 }
1008
1009 static int __driver_rfc4106_encrypt(struct aead_request *req)
1010 {
1011 u8 one_entry_in_sg = 0;
1012 u8 *src, *dst, *assoc;
1013 __be32 counter = cpu_to_be32(1);
1014 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1015 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1016 void *aes_ctx = &(ctx->aes_key_expanded);
1017 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1018 u8 iv_tab[16+AESNI_ALIGN];
1019 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1020 struct scatter_walk src_sg_walk;
1021 struct scatter_walk assoc_sg_walk;
1022 struct scatter_walk dst_sg_walk;
1023 unsigned int i;
1024
1025 /* Assuming we are supporting rfc4106 64-bit extended */
1026 /* sequence numbers We need to have the AAD length equal */
1027 /* to 8 or 12 bytes */
1028 if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1029 return -EINVAL;
1030 /* IV below built */
1031 for (i = 0; i < 4; i++)
1032 *(iv+i) = ctx->nonce[i];
1033 for (i = 0; i < 8; i++)
1034 *(iv+4+i) = req->iv[i];
1035 *((__be32 *)(iv+12)) = counter;
1036
1037 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1038 one_entry_in_sg = 1;
1039 scatterwalk_start(&src_sg_walk, req->src);
1040 scatterwalk_start(&assoc_sg_walk, req->assoc);
1041 src = scatterwalk_map(&src_sg_walk);
1042 assoc = scatterwalk_map(&assoc_sg_walk);
1043 dst = src;
1044 if (unlikely(req->src != req->dst)) {
1045 scatterwalk_start(&dst_sg_walk, req->dst);
1046 dst = scatterwalk_map(&dst_sg_walk);
1047 }
1048
1049 } else {
1050 /* Allocate memory for src, dst, assoc */
1051 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1052 GFP_ATOMIC);
1053 if (unlikely(!src))
1054 return -ENOMEM;
1055 assoc = (src + req->cryptlen + auth_tag_len);
1056 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1057 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1058 req->assoclen, 0);
1059 dst = src;
1060 }
1061
1062 aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1063 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1064 + ((unsigned long)req->cryptlen), auth_tag_len);
1065
1066 /* The authTag (aka the Integrity Check Value) needs to be written
1067 * back to the packet. */
1068 if (one_entry_in_sg) {
1069 if (unlikely(req->src != req->dst)) {
1070 scatterwalk_unmap(dst);
1071 scatterwalk_done(&dst_sg_walk, 0, 0);
1072 }
1073 scatterwalk_unmap(src);
1074 scatterwalk_unmap(assoc);
1075 scatterwalk_done(&src_sg_walk, 0, 0);
1076 scatterwalk_done(&assoc_sg_walk, 0, 0);
1077 } else {
1078 scatterwalk_map_and_copy(dst, req->dst, 0,
1079 req->cryptlen + auth_tag_len, 1);
1080 kfree(src);
1081 }
1082 return 0;
1083 }
1084
1085 static int __driver_rfc4106_decrypt(struct aead_request *req)
1086 {
1087 u8 one_entry_in_sg = 0;
1088 u8 *src, *dst, *assoc;
1089 unsigned long tempCipherLen = 0;
1090 __be32 counter = cpu_to_be32(1);
1091 int retval = 0;
1092 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1093 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1094 void *aes_ctx = &(ctx->aes_key_expanded);
1095 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1096 u8 iv_and_authTag[32+AESNI_ALIGN];
1097 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1098 u8 *authTag = iv + 16;
1099 struct scatter_walk src_sg_walk;
1100 struct scatter_walk assoc_sg_walk;
1101 struct scatter_walk dst_sg_walk;
1102 unsigned int i;
1103
1104 if (unlikely((req->cryptlen < auth_tag_len) ||
1105 (req->assoclen != 8 && req->assoclen != 12)))
1106 return -EINVAL;
1107 /* Assuming we are supporting rfc4106 64-bit extended */
1108 /* sequence numbers We need to have the AAD length */
1109 /* equal to 8 or 12 bytes */
1110
1111 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1112 /* IV below built */
1113 for (i = 0; i < 4; i++)
1114 *(iv+i) = ctx->nonce[i];
1115 for (i = 0; i < 8; i++)
1116 *(iv+4+i) = req->iv[i];
1117 *((__be32 *)(iv+12)) = counter;
1118
1119 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1120 one_entry_in_sg = 1;
1121 scatterwalk_start(&src_sg_walk, req->src);
1122 scatterwalk_start(&assoc_sg_walk, req->assoc);
1123 src = scatterwalk_map(&src_sg_walk);
1124 assoc = scatterwalk_map(&assoc_sg_walk);
1125 dst = src;
1126 if (unlikely(req->src != req->dst)) {
1127 scatterwalk_start(&dst_sg_walk, req->dst);
1128 dst = scatterwalk_map(&dst_sg_walk);
1129 }
1130
1131 } else {
1132 /* Allocate memory for src, dst, assoc */
1133 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1134 if (!src)
1135 return -ENOMEM;
1136 assoc = (src + req->cryptlen + auth_tag_len);
1137 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1138 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1139 req->assoclen, 0);
1140 dst = src;
1141 }
1142
1143 aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
1144 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1145 authTag, auth_tag_len);
1146
1147 /* Compare generated tag with passed in tag. */
1148 retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
1149 -EBADMSG : 0;
1150
1151 if (one_entry_in_sg) {
1152 if (unlikely(req->src != req->dst)) {
1153 scatterwalk_unmap(dst);
1154 scatterwalk_done(&dst_sg_walk, 0, 0);
1155 }
1156 scatterwalk_unmap(src);
1157 scatterwalk_unmap(assoc);
1158 scatterwalk_done(&src_sg_walk, 0, 0);
1159 scatterwalk_done(&assoc_sg_walk, 0, 0);
1160 } else {
1161 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1162 kfree(src);
1163 }
1164 return retval;
1165 }
1166 #endif
1167
1168 static struct crypto_alg aesni_algs[] = { {
1169 .cra_name = "aes",
1170 .cra_driver_name = "aes-aesni",
1171 .cra_priority = 300,
1172 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
1173 .cra_blocksize = AES_BLOCK_SIZE,
1174 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
1175 AESNI_ALIGN - 1,
1176 .cra_alignmask = 0,
1177 .cra_module = THIS_MODULE,
1178 .cra_u = {
1179 .cipher = {
1180 .cia_min_keysize = AES_MIN_KEY_SIZE,
1181 .cia_max_keysize = AES_MAX_KEY_SIZE,
1182 .cia_setkey = aes_set_key,
1183 .cia_encrypt = aes_encrypt,
1184 .cia_decrypt = aes_decrypt
1185 }
1186 }
1187 }, {
1188 .cra_name = "__aes-aesni",
1189 .cra_driver_name = "__driver-aes-aesni",
1190 .cra_priority = 0,
1191 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
1192 .cra_blocksize = AES_BLOCK_SIZE,
1193 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
1194 AESNI_ALIGN - 1,
1195 .cra_alignmask = 0,
1196 .cra_module = THIS_MODULE,
1197 .cra_u = {
1198 .cipher = {
1199 .cia_min_keysize = AES_MIN_KEY_SIZE,
1200 .cia_max_keysize = AES_MAX_KEY_SIZE,
1201 .cia_setkey = aes_set_key,
1202 .cia_encrypt = __aes_encrypt,
1203 .cia_decrypt = __aes_decrypt
1204 }
1205 }
1206 }, {
1207 .cra_name = "__ecb-aes-aesni",
1208 .cra_driver_name = "__driver-ecb-aes-aesni",
1209 .cra_priority = 0,
1210 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1211 .cra_blocksize = AES_BLOCK_SIZE,
1212 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
1213 AESNI_ALIGN - 1,
1214 .cra_alignmask = 0,
1215 .cra_type = &crypto_blkcipher_type,
1216 .cra_module = THIS_MODULE,
1217 .cra_u = {
1218 .blkcipher = {
1219 .min_keysize = AES_MIN_KEY_SIZE,
1220 .max_keysize = AES_MAX_KEY_SIZE,
1221 .setkey = aes_set_key,
1222 .encrypt = ecb_encrypt,
1223 .decrypt = ecb_decrypt,
1224 },
1225 },
1226 }, {
1227 .cra_name = "__cbc-aes-aesni",
1228 .cra_driver_name = "__driver-cbc-aes-aesni",
1229 .cra_priority = 0,
1230 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1231 .cra_blocksize = AES_BLOCK_SIZE,
1232 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
1233 AESNI_ALIGN - 1,
1234 .cra_alignmask = 0,
1235 .cra_type = &crypto_blkcipher_type,
1236 .cra_module = THIS_MODULE,
1237 .cra_u = {
1238 .blkcipher = {
1239 .min_keysize = AES_MIN_KEY_SIZE,
1240 .max_keysize = AES_MAX_KEY_SIZE,
1241 .setkey = aes_set_key,
1242 .encrypt = cbc_encrypt,
1243 .decrypt = cbc_decrypt,
1244 },
1245 },
1246 }, {
1247 .cra_name = "ecb(aes)",
1248 .cra_driver_name = "ecb-aes-aesni",
1249 .cra_priority = 400,
1250 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1251 .cra_blocksize = AES_BLOCK_SIZE,
1252 .cra_ctxsize = sizeof(struct async_helper_ctx),
1253 .cra_alignmask = 0,
1254 .cra_type = &crypto_ablkcipher_type,
1255 .cra_module = THIS_MODULE,
1256 .cra_init = ablk_ecb_init,
1257 .cra_exit = ablk_exit,
1258 .cra_u = {
1259 .ablkcipher = {
1260 .min_keysize = AES_MIN_KEY_SIZE,
1261 .max_keysize = AES_MAX_KEY_SIZE,
1262 .setkey = ablk_set_key,
1263 .encrypt = ablk_encrypt,
1264 .decrypt = ablk_decrypt,
1265 },
1266 },
1267 }, {
1268 .cra_name = "cbc(aes)",
1269 .cra_driver_name = "cbc-aes-aesni",
1270 .cra_priority = 400,
1271 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1272 .cra_blocksize = AES_BLOCK_SIZE,
1273 .cra_ctxsize = sizeof(struct async_helper_ctx),
1274 .cra_alignmask = 0,
1275 .cra_type = &crypto_ablkcipher_type,
1276 .cra_module = THIS_MODULE,
1277 .cra_init = ablk_cbc_init,
1278 .cra_exit = ablk_exit,
1279 .cra_u = {
1280 .ablkcipher = {
1281 .min_keysize = AES_MIN_KEY_SIZE,
1282 .max_keysize = AES_MAX_KEY_SIZE,
1283 .ivsize = AES_BLOCK_SIZE,
1284 .setkey = ablk_set_key,
1285 .encrypt = ablk_encrypt,
1286 .decrypt = ablk_decrypt,
1287 },
1288 },
1289 #ifdef CONFIG_X86_64
1290 }, {
1291 .cra_name = "__ctr-aes-aesni",
1292 .cra_driver_name = "__driver-ctr-aes-aesni",
1293 .cra_priority = 0,
1294 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1295 .cra_blocksize = 1,
1296 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
1297 AESNI_ALIGN - 1,
1298 .cra_alignmask = 0,
1299 .cra_type = &crypto_blkcipher_type,
1300 .cra_module = THIS_MODULE,
1301 .cra_u = {
1302 .blkcipher = {
1303 .min_keysize = AES_MIN_KEY_SIZE,
1304 .max_keysize = AES_MAX_KEY_SIZE,
1305 .ivsize = AES_BLOCK_SIZE,
1306 .setkey = aes_set_key,
1307 .encrypt = ctr_crypt,
1308 .decrypt = ctr_crypt,
1309 },
1310 },
1311 }, {
1312 .cra_name = "ctr(aes)",
1313 .cra_driver_name = "ctr-aes-aesni",
1314 .cra_priority = 400,
1315 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1316 .cra_blocksize = 1,
1317 .cra_ctxsize = sizeof(struct async_helper_ctx),
1318 .cra_alignmask = 0,
1319 .cra_type = &crypto_ablkcipher_type,
1320 .cra_module = THIS_MODULE,
1321 .cra_init = ablk_ctr_init,
1322 .cra_exit = ablk_exit,
1323 .cra_u = {
1324 .ablkcipher = {
1325 .min_keysize = AES_MIN_KEY_SIZE,
1326 .max_keysize = AES_MAX_KEY_SIZE,
1327 .ivsize = AES_BLOCK_SIZE,
1328 .setkey = ablk_set_key,
1329 .encrypt = ablk_encrypt,
1330 .decrypt = ablk_encrypt,
1331 .geniv = "chainiv",
1332 },
1333 },
1334 }, {
1335 .cra_name = "__gcm-aes-aesni",
1336 .cra_driver_name = "__driver-gcm-aes-aesni",
1337 .cra_priority = 0,
1338 .cra_flags = CRYPTO_ALG_TYPE_AEAD,
1339 .cra_blocksize = 1,
1340 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) +
1341 AESNI_ALIGN,
1342 .cra_alignmask = 0,
1343 .cra_type = &crypto_aead_type,
1344 .cra_module = THIS_MODULE,
1345 .cra_u = {
1346 .aead = {
1347 .encrypt = __driver_rfc4106_encrypt,
1348 .decrypt = __driver_rfc4106_decrypt,
1349 },
1350 },
1351 }, {
1352 .cra_name = "rfc4106(gcm(aes))",
1353 .cra_driver_name = "rfc4106-gcm-aesni",
1354 .cra_priority = 400,
1355 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1356 .cra_blocksize = 1,
1357 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) +
1358 AESNI_ALIGN,
1359 .cra_alignmask = 0,
1360 .cra_type = &crypto_nivaead_type,
1361 .cra_module = THIS_MODULE,
1362 .cra_init = rfc4106_init,
1363 .cra_exit = rfc4106_exit,
1364 .cra_u = {
1365 .aead = {
1366 .setkey = rfc4106_set_key,
1367 .setauthsize = rfc4106_set_authsize,
1368 .encrypt = rfc4106_encrypt,
1369 .decrypt = rfc4106_decrypt,
1370 .geniv = "seqiv",
1371 .ivsize = 8,
1372 .maxauthsize = 16,
1373 },
1374 },
1375 #endif
1376 #if IS_ENABLED(CONFIG_CRYPTO_PCBC)
1377 }, {
1378 .cra_name = "pcbc(aes)",
1379 .cra_driver_name = "pcbc-aes-aesni",
1380 .cra_priority = 400,
1381 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1382 .cra_blocksize = AES_BLOCK_SIZE,
1383 .cra_ctxsize = sizeof(struct async_helper_ctx),
1384 .cra_alignmask = 0,
1385 .cra_type = &crypto_ablkcipher_type,
1386 .cra_module = THIS_MODULE,
1387 .cra_init = ablk_pcbc_init,
1388 .cra_exit = ablk_exit,
1389 .cra_u = {
1390 .ablkcipher = {
1391 .min_keysize = AES_MIN_KEY_SIZE,
1392 .max_keysize = AES_MAX_KEY_SIZE,
1393 .ivsize = AES_BLOCK_SIZE,
1394 .setkey = ablk_set_key,
1395 .encrypt = ablk_encrypt,
1396 .decrypt = ablk_decrypt,
1397 },
1398 },
1399 #endif
1400 }, {
1401 .cra_name = "__lrw-aes-aesni",
1402 .cra_driver_name = "__driver-lrw-aes-aesni",
1403 .cra_priority = 0,
1404 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1405 .cra_blocksize = AES_BLOCK_SIZE,
1406 .cra_ctxsize = sizeof(struct aesni_lrw_ctx),
1407 .cra_alignmask = 0,
1408 .cra_type = &crypto_blkcipher_type,
1409 .cra_module = THIS_MODULE,
1410 .cra_exit = lrw_aesni_exit_tfm,
1411 .cra_u = {
1412 .blkcipher = {
1413 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1414 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1415 .ivsize = AES_BLOCK_SIZE,
1416 .setkey = lrw_aesni_setkey,
1417 .encrypt = lrw_encrypt,
1418 .decrypt = lrw_decrypt,
1419 },
1420 },
1421 }, {
1422 .cra_name = "__xts-aes-aesni",
1423 .cra_driver_name = "__driver-xts-aes-aesni",
1424 .cra_priority = 0,
1425 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1426 .cra_blocksize = AES_BLOCK_SIZE,
1427 .cra_ctxsize = sizeof(struct aesni_xts_ctx),
1428 .cra_alignmask = 0,
1429 .cra_type = &crypto_blkcipher_type,
1430 .cra_module = THIS_MODULE,
1431 .cra_u = {
1432 .blkcipher = {
1433 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1434 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1435 .ivsize = AES_BLOCK_SIZE,
1436 .setkey = xts_aesni_setkey,
1437 .encrypt = xts_encrypt,
1438 .decrypt = xts_decrypt,
1439 },
1440 },
1441 }, {
1442 .cra_name = "lrw(aes)",
1443 .cra_driver_name = "lrw-aes-aesni",
1444 .cra_priority = 400,
1445 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1446 .cra_blocksize = AES_BLOCK_SIZE,
1447 .cra_ctxsize = sizeof(struct async_helper_ctx),
1448 .cra_alignmask = 0,
1449 .cra_type = &crypto_ablkcipher_type,
1450 .cra_module = THIS_MODULE,
1451 .cra_init = ablk_init,
1452 .cra_exit = ablk_exit,
1453 .cra_u = {
1454 .ablkcipher = {
1455 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1456 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1457 .ivsize = AES_BLOCK_SIZE,
1458 .setkey = ablk_set_key,
1459 .encrypt = ablk_encrypt,
1460 .decrypt = ablk_decrypt,
1461 },
1462 },
1463 }, {
1464 .cra_name = "xts(aes)",
1465 .cra_driver_name = "xts-aes-aesni",
1466 .cra_priority = 400,
1467 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1468 .cra_blocksize = AES_BLOCK_SIZE,
1469 .cra_ctxsize = sizeof(struct async_helper_ctx),
1470 .cra_alignmask = 0,
1471 .cra_type = &crypto_ablkcipher_type,
1472 .cra_module = THIS_MODULE,
1473 .cra_init = ablk_init,
1474 .cra_exit = ablk_exit,
1475 .cra_u = {
1476 .ablkcipher = {
1477 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1478 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1479 .ivsize = AES_BLOCK_SIZE,
1480 .setkey = ablk_set_key,
1481 .encrypt = ablk_encrypt,
1482 .decrypt = ablk_decrypt,
1483 },
1484 },
1485 } };
1486
1487
1488 static const struct x86_cpu_id aesni_cpu_id[] = {
1489 X86_FEATURE_MATCH(X86_FEATURE_AES),
1490 {}
1491 };
1492 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1493
1494 static int __init aesni_init(void)
1495 {
1496 int err;
1497
1498 if (!x86_match_cpu(aesni_cpu_id))
1499 return -ENODEV;
1500 #ifdef CONFIG_X86_64
1501 #ifdef CONFIG_AS_AVX2
1502 if (boot_cpu_has(X86_FEATURE_AVX2)) {
1503 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1504 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
1505 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
1506 } else
1507 #endif
1508 #ifdef CONFIG_AS_AVX
1509 if (boot_cpu_has(X86_FEATURE_AVX)) {
1510 pr_info("AVX version of gcm_enc/dec engaged.\n");
1511 aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
1512 aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
1513 } else
1514 #endif
1515 {
1516 pr_info("SSE version of gcm_enc/dec engaged.\n");
1517 aesni_gcm_enc_tfm = aesni_gcm_enc;
1518 aesni_gcm_dec_tfm = aesni_gcm_dec;
1519 }
1520 aesni_ctr_enc_tfm = aesni_ctr_enc;
1521 #ifdef CONFIG_AS_AVX
1522 if (cpu_has_avx) {
1523 /* optimize performance of ctr mode encryption transform */
1524 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1525 pr_info("AES CTR mode by8 optimization enabled\n");
1526 }
1527 #endif
1528 #endif
1529
1530 err = crypto_fpu_init();
1531 if (err)
1532 return err;
1533
1534 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1535 }
1536
1537 static void __exit aesni_exit(void)
1538 {
1539 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1540
1541 crypto_fpu_exit();
1542 }
1543
1544 module_init(aesni_init);
1545 module_exit(aesni_exit);
1546
1547 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1548 MODULE_LICENSE("GPL");
1549 MODULE_ALIAS_CRYPTO("aes");
Linux kernel - Deliverables
This page took 0.102718 seconds and 5 git commands to generate.