Commit | Line | Data |
---|---|---|
107778b5 JG |
1 | /* |
2 | * Glue Code for AVX assembler version of Twofish Cipher | |
3 | * | |
4 | * Copyright (C) 2012 Johannes Goetzfried | |
5 | * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> | |
6 | * | |
18be4527 JK |
7 | * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> |
8 | * | |
107778b5 JG |
9 | * This program is free software; you can redistribute it and/or modify |
10 | * it under the terms of the GNU General Public License as published by | |
11 | * the Free Software Foundation; either version 2 of the License, or | |
12 | * (at your option) any later version. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, | |
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
17 | * GNU General Public License for more details. | |
18 | * | |
19 | * You should have received a copy of the GNU General Public License | |
20 | * along with this program; if not, write to the Free Software | |
21 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 | |
22 | * USA | |
23 | * | |
24 | */ | |
25 | ||
26 | #include <linux/module.h> | |
27 | #include <linux/hardirq.h> | |
28 | #include <linux/types.h> | |
29 | #include <linux/crypto.h> | |
30 | #include <linux/err.h> | |
801201aa | 31 | #include <crypto/ablk_helper.h> |
107778b5 JG |
32 | #include <crypto/algapi.h> |
33 | #include <crypto/twofish.h> | |
34 | #include <crypto/cryptd.h> | |
35 | #include <crypto/b128ops.h> | |
36 | #include <crypto/ctr.h> | |
37 | #include <crypto/lrw.h> | |
38 | #include <crypto/xts.h> | |
df6b35f4 | 39 | #include <asm/fpu/api.h> |
107778b5 | 40 | #include <asm/xcr.h> |
a137fb6b | 41 | #include <asm/fpu/xsave.h> |
a7378d4e | 42 | #include <asm/crypto/twofish.h> |
a7378d4e | 43 | #include <asm/crypto/glue_helper.h> |
107778b5 JG |
44 | #include <crypto/scatterwalk.h> |
45 | #include <linux/workqueue.h> | |
46 | #include <linux/spinlock.h> | |
47 | ||
107778b5 JG |
48 | #define TWOFISH_PARALLEL_BLOCKS 8 |
49 | ||
107778b5 | 50 | /* 8-way parallel cipher functions */ |
8435a3c3 JK |
51 | asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst, |
52 | const u8 *src); | |
53 | asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst, | |
107778b5 JG |
54 | const u8 *src); |
55 | ||
8435a3c3 JK |
56 | asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst, |
57 | const u8 *src); | |
58 | asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst, | |
59 | const u8 *src, le128 *iv); | |
107778b5 | 60 | |
18be4527 JK |
61 | asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst, |
62 | const u8 *src, le128 *iv); | |
63 | asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst, | |
64 | const u8 *src, le128 *iv); | |
65 | ||
8435a3c3 | 66 | static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, |
107778b5 JG |
67 | const u8 *src) |
68 | { | |
8435a3c3 | 69 | __twofish_enc_blk_3way(ctx, dst, src, false); |
107778b5 JG |
70 | } |
71 | ||
99f42f93 | 72 | static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) |
18be4527 JK |
73 | { |
74 | glue_xts_crypt_128bit_one(ctx, dst, src, iv, | |
75 | GLUE_FUNC_CAST(twofish_enc_blk)); | |
76 | } | |
77 | ||
99f42f93 | 78 | static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) |
18be4527 JK |
79 | { |
80 | glue_xts_crypt_128bit_one(ctx, dst, src, iv, | |
81 | GLUE_FUNC_CAST(twofish_dec_blk)); | |
82 | } | |
83 | ||
107778b5 | 84 | |
a7378d4e JK |
85 | static const struct common_glue_ctx twofish_enc = { |
86 | .num_funcs = 3, | |
87 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
88 | ||
89 | .funcs = { { | |
90 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
8435a3c3 | 91 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) } |
a7378d4e JK |
92 | }, { |
93 | .num_blocks = 3, | |
94 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } | |
95 | }, { | |
96 | .num_blocks = 1, | |
97 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } | |
98 | } } | |
99 | }; | |
100 | ||
101 | static const struct common_glue_ctx twofish_ctr = { | |
102 | .num_funcs = 3, | |
103 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
104 | ||
105 | .funcs = { { | |
106 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
8435a3c3 | 107 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) } |
a7378d4e JK |
108 | }, { |
109 | .num_blocks = 3, | |
110 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) } | |
111 | }, { | |
112 | .num_blocks = 1, | |
113 | .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) } | |
114 | } } | |
115 | }; | |
116 | ||
18be4527 JK |
117 | static const struct common_glue_ctx twofish_enc_xts = { |
118 | .num_funcs = 2, | |
119 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
120 | ||
121 | .funcs = { { | |
122 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
123 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) } | |
124 | }, { | |
125 | .num_blocks = 1, | |
126 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) } | |
127 | } } | |
128 | }; | |
129 | ||
a7378d4e JK |
130 | static const struct common_glue_ctx twofish_dec = { |
131 | .num_funcs = 3, | |
132 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
133 | ||
134 | .funcs = { { | |
135 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
8435a3c3 | 136 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) } |
a7378d4e JK |
137 | }, { |
138 | .num_blocks = 3, | |
139 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } | |
140 | }, { | |
141 | .num_blocks = 1, | |
142 | .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } | |
143 | } } | |
144 | }; | |
145 | ||
146 | static const struct common_glue_ctx twofish_dec_cbc = { | |
147 | .num_funcs = 3, | |
148 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
149 | ||
150 | .funcs = { { | |
151 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
8435a3c3 | 152 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) } |
a7378d4e JK |
153 | }, { |
154 | .num_blocks = 3, | |
155 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } | |
156 | }, { | |
157 | .num_blocks = 1, | |
158 | .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } | |
159 | } } | |
160 | }; | |
161 | ||
18be4527 JK |
162 | static const struct common_glue_ctx twofish_dec_xts = { |
163 | .num_funcs = 2, | |
164 | .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, | |
165 | ||
166 | .funcs = { { | |
167 | .num_blocks = TWOFISH_PARALLEL_BLOCKS, | |
168 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) } | |
169 | }, { | |
170 | .num_blocks = 1, | |
171 | .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) } | |
172 | } } | |
173 | }; | |
174 | ||
107778b5 JG |
175 | static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, |
176 | struct scatterlist *src, unsigned int nbytes) | |
177 | { | |
a7378d4e | 178 | return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes); |
107778b5 JG |
179 | } |
180 | ||
181 | static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | |
182 | struct scatterlist *src, unsigned int nbytes) | |
183 | { | |
a7378d4e | 184 | return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes); |
107778b5 JG |
185 | } |
186 | ||
187 | static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | |
188 | struct scatterlist *src, unsigned int nbytes) | |
189 | { | |
a7378d4e JK |
190 | return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc, |
191 | dst, src, nbytes); | |
107778b5 JG |
192 | } |
193 | ||
194 | static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | |
195 | struct scatterlist *src, unsigned int nbytes) | |
196 | { | |
a7378d4e JK |
197 | return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src, |
198 | nbytes); | |
107778b5 JG |
199 | } |
200 | ||
a7378d4e JK |
201 | static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, |
202 | struct scatterlist *src, unsigned int nbytes) | |
107778b5 | 203 | { |
a7378d4e | 204 | return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes); |
107778b5 JG |
205 | } |
206 | ||
a7378d4e | 207 | static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes) |
107778b5 | 208 | { |
a7378d4e JK |
209 | return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL, |
210 | fpu_enabled, nbytes); | |
107778b5 JG |
211 | } |
212 | ||
a7378d4e | 213 | static inline void twofish_fpu_end(bool fpu_enabled) |
107778b5 | 214 | { |
a7378d4e | 215 | glue_fpu_end(fpu_enabled); |
107778b5 JG |
216 | } |
217 | ||
218 | struct crypt_priv { | |
219 | struct twofish_ctx *ctx; | |
220 | bool fpu_enabled; | |
221 | }; | |
222 | ||
223 | static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) | |
224 | { | |
225 | const unsigned int bsize = TF_BLOCK_SIZE; | |
226 | struct crypt_priv *ctx = priv; | |
227 | int i; | |
228 | ||
229 | ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); | |
230 | ||
231 | if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { | |
8435a3c3 | 232 | twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst); |
107778b5 JG |
233 | return; |
234 | } | |
235 | ||
236 | for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) | |
237 | twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst); | |
238 | ||
239 | nbytes %= bsize * 3; | |
240 | ||
241 | for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) | |
242 | twofish_enc_blk(ctx->ctx, srcdst, srcdst); | |
243 | } | |
244 | ||
245 | static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) | |
246 | { | |
247 | const unsigned int bsize = TF_BLOCK_SIZE; | |
248 | struct crypt_priv *ctx = priv; | |
249 | int i; | |
250 | ||
251 | ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); | |
252 | ||
253 | if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { | |
8435a3c3 | 254 | twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst); |
107778b5 JG |
255 | return; |
256 | } | |
257 | ||
258 | for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) | |
259 | twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst); | |
260 | ||
261 | nbytes %= bsize * 3; | |
262 | ||
263 | for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) | |
264 | twofish_dec_blk(ctx->ctx, srcdst, srcdst); | |
265 | } | |
266 | ||
107778b5 JG |
267 | static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, |
268 | struct scatterlist *src, unsigned int nbytes) | |
269 | { | |
270 | struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | |
271 | be128 buf[TWOFISH_PARALLEL_BLOCKS]; | |
272 | struct crypt_priv crypt_ctx = { | |
273 | .ctx = &ctx->twofish_ctx, | |
274 | .fpu_enabled = false, | |
275 | }; | |
276 | struct lrw_crypt_req req = { | |
277 | .tbuf = buf, | |
278 | .tbuflen = sizeof(buf), | |
279 | ||
280 | .table_ctx = &ctx->lrw_table, | |
281 | .crypt_ctx = &crypt_ctx, | |
282 | .crypt_fn = encrypt_callback, | |
283 | }; | |
284 | int ret; | |
285 | ||
286 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | |
287 | ret = lrw_crypt(desc, dst, src, nbytes, &req); | |
288 | twofish_fpu_end(crypt_ctx.fpu_enabled); | |
289 | ||
290 | return ret; | |
291 | } | |
292 | ||
293 | static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | |
294 | struct scatterlist *src, unsigned int nbytes) | |
295 | { | |
296 | struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | |
297 | be128 buf[TWOFISH_PARALLEL_BLOCKS]; | |
298 | struct crypt_priv crypt_ctx = { | |
299 | .ctx = &ctx->twofish_ctx, | |
300 | .fpu_enabled = false, | |
301 | }; | |
302 | struct lrw_crypt_req req = { | |
303 | .tbuf = buf, | |
304 | .tbuflen = sizeof(buf), | |
305 | ||
306 | .table_ctx = &ctx->lrw_table, | |
307 | .crypt_ctx = &crypt_ctx, | |
308 | .crypt_fn = decrypt_callback, | |
309 | }; | |
310 | int ret; | |
311 | ||
312 | desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; | |
313 | ret = lrw_crypt(desc, dst, src, nbytes, &req); | |
314 | twofish_fpu_end(crypt_ctx.fpu_enabled); | |
315 | ||
316 | return ret; | |
317 | } | |
318 | ||
107778b5 JG |
319 | static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, |
320 | struct scatterlist *src, unsigned int nbytes) | |
321 | { | |
322 | struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | |
107778b5 | 323 | |
18be4527 JK |
324 | return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes, |
325 | XTS_TWEAK_CAST(twofish_enc_blk), | |
326 | &ctx->tweak_ctx, &ctx->crypt_ctx); | |
107778b5 JG |
327 | } |
328 | ||
329 | static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, | |
330 | struct scatterlist *src, unsigned int nbytes) | |
331 | { | |
332 | struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); | |
107778b5 | 333 | |
18be4527 JK |
334 | return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes, |
335 | XTS_TWEAK_CAST(twofish_enc_blk), | |
336 | &ctx->tweak_ctx, &ctx->crypt_ctx); | |
107778b5 JG |
337 | } |
338 | ||
107778b5 JG |
339 | static struct crypto_alg twofish_algs[10] = { { |
340 | .cra_name = "__ecb-twofish-avx", | |
341 | .cra_driver_name = "__driver-ecb-twofish-avx", | |
342 | .cra_priority = 0, | |
4dda66f6 SM |
343 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | |
344 | CRYPTO_ALG_INTERNAL, | |
107778b5 JG |
345 | .cra_blocksize = TF_BLOCK_SIZE, |
346 | .cra_ctxsize = sizeof(struct twofish_ctx), | |
347 | .cra_alignmask = 0, | |
348 | .cra_type = &crypto_blkcipher_type, | |
349 | .cra_module = THIS_MODULE, | |
107778b5 JG |
350 | .cra_u = { |
351 | .blkcipher = { | |
352 | .min_keysize = TF_MIN_KEY_SIZE, | |
353 | .max_keysize = TF_MAX_KEY_SIZE, | |
354 | .setkey = twofish_setkey, | |
355 | .encrypt = ecb_encrypt, | |
356 | .decrypt = ecb_decrypt, | |
357 | }, | |
358 | }, | |
359 | }, { | |
360 | .cra_name = "__cbc-twofish-avx", | |
361 | .cra_driver_name = "__driver-cbc-twofish-avx", | |
362 | .cra_priority = 0, | |
4dda66f6 SM |
363 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | |
364 | CRYPTO_ALG_INTERNAL, | |
107778b5 JG |
365 | .cra_blocksize = TF_BLOCK_SIZE, |
366 | .cra_ctxsize = sizeof(struct twofish_ctx), | |
367 | .cra_alignmask = 0, | |
368 | .cra_type = &crypto_blkcipher_type, | |
369 | .cra_module = THIS_MODULE, | |
107778b5 JG |
370 | .cra_u = { |
371 | .blkcipher = { | |
372 | .min_keysize = TF_MIN_KEY_SIZE, | |
373 | .max_keysize = TF_MAX_KEY_SIZE, | |
374 | .setkey = twofish_setkey, | |
375 | .encrypt = cbc_encrypt, | |
376 | .decrypt = cbc_decrypt, | |
377 | }, | |
378 | }, | |
379 | }, { | |
380 | .cra_name = "__ctr-twofish-avx", | |
381 | .cra_driver_name = "__driver-ctr-twofish-avx", | |
382 | .cra_priority = 0, | |
4dda66f6 SM |
383 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | |
384 | CRYPTO_ALG_INTERNAL, | |
107778b5 JG |
385 | .cra_blocksize = 1, |
386 | .cra_ctxsize = sizeof(struct twofish_ctx), | |
387 | .cra_alignmask = 0, | |
388 | .cra_type = &crypto_blkcipher_type, | |
389 | .cra_module = THIS_MODULE, | |
107778b5 JG |
390 | .cra_u = { |
391 | .blkcipher = { | |
392 | .min_keysize = TF_MIN_KEY_SIZE, | |
393 | .max_keysize = TF_MAX_KEY_SIZE, | |
394 | .ivsize = TF_BLOCK_SIZE, | |
395 | .setkey = twofish_setkey, | |
396 | .encrypt = ctr_crypt, | |
397 | .decrypt = ctr_crypt, | |
398 | }, | |
399 | }, | |
400 | }, { | |
401 | .cra_name = "__lrw-twofish-avx", | |
402 | .cra_driver_name = "__driver-lrw-twofish-avx", | |
403 | .cra_priority = 0, | |
4dda66f6 SM |
404 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | |
405 | CRYPTO_ALG_INTERNAL, | |
107778b5 JG |
406 | .cra_blocksize = TF_BLOCK_SIZE, |
407 | .cra_ctxsize = sizeof(struct twofish_lrw_ctx), | |
408 | .cra_alignmask = 0, | |
409 | .cra_type = &crypto_blkcipher_type, | |
410 | .cra_module = THIS_MODULE, | |
a7378d4e | 411 | .cra_exit = lrw_twofish_exit_tfm, |
107778b5 JG |
412 | .cra_u = { |
413 | .blkcipher = { | |
414 | .min_keysize = TF_MIN_KEY_SIZE + | |
415 | TF_BLOCK_SIZE, | |
416 | .max_keysize = TF_MAX_KEY_SIZE + | |
417 | TF_BLOCK_SIZE, | |
418 | .ivsize = TF_BLOCK_SIZE, | |
419 | .setkey = lrw_twofish_setkey, | |
420 | .encrypt = lrw_encrypt, | |
421 | .decrypt = lrw_decrypt, | |
422 | }, | |
423 | }, | |
424 | }, { | |
425 | .cra_name = "__xts-twofish-avx", | |
426 | .cra_driver_name = "__driver-xts-twofish-avx", | |
427 | .cra_priority = 0, | |
4dda66f6 SM |
428 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | |
429 | CRYPTO_ALG_INTERNAL, | |
107778b5 JG |
430 | .cra_blocksize = TF_BLOCK_SIZE, |
431 | .cra_ctxsize = sizeof(struct twofish_xts_ctx), | |
432 | .cra_alignmask = 0, | |
433 | .cra_type = &crypto_blkcipher_type, | |
434 | .cra_module = THIS_MODULE, | |
107778b5 JG |
435 | .cra_u = { |
436 | .blkcipher = { | |
437 | .min_keysize = TF_MIN_KEY_SIZE * 2, | |
438 | .max_keysize = TF_MAX_KEY_SIZE * 2, | |
439 | .ivsize = TF_BLOCK_SIZE, | |
440 | .setkey = xts_twofish_setkey, | |
441 | .encrypt = xts_encrypt, | |
442 | .decrypt = xts_decrypt, | |
443 | }, | |
444 | }, | |
445 | }, { | |
446 | .cra_name = "ecb(twofish)", | |
447 | .cra_driver_name = "ecb-twofish-avx", | |
448 | .cra_priority = 400, | |
449 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | |
450 | .cra_blocksize = TF_BLOCK_SIZE, | |
30a04008 | 451 | .cra_ctxsize = sizeof(struct async_helper_ctx), |
107778b5 JG |
452 | .cra_alignmask = 0, |
453 | .cra_type = &crypto_ablkcipher_type, | |
454 | .cra_module = THIS_MODULE, | |
107778b5 JG |
455 | .cra_init = ablk_init, |
456 | .cra_exit = ablk_exit, | |
457 | .cra_u = { | |
458 | .ablkcipher = { | |
459 | .min_keysize = TF_MIN_KEY_SIZE, | |
460 | .max_keysize = TF_MAX_KEY_SIZE, | |
461 | .setkey = ablk_set_key, | |
462 | .encrypt = ablk_encrypt, | |
463 | .decrypt = ablk_decrypt, | |
464 | }, | |
465 | }, | |
466 | }, { | |
467 | .cra_name = "cbc(twofish)", | |
468 | .cra_driver_name = "cbc-twofish-avx", | |
469 | .cra_priority = 400, | |
470 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | |
471 | .cra_blocksize = TF_BLOCK_SIZE, | |
30a04008 | 472 | .cra_ctxsize = sizeof(struct async_helper_ctx), |
107778b5 JG |
473 | .cra_alignmask = 0, |
474 | .cra_type = &crypto_ablkcipher_type, | |
475 | .cra_module = THIS_MODULE, | |
107778b5 JG |
476 | .cra_init = ablk_init, |
477 | .cra_exit = ablk_exit, | |
478 | .cra_u = { | |
479 | .ablkcipher = { | |
480 | .min_keysize = TF_MIN_KEY_SIZE, | |
481 | .max_keysize = TF_MAX_KEY_SIZE, | |
482 | .ivsize = TF_BLOCK_SIZE, | |
483 | .setkey = ablk_set_key, | |
484 | .encrypt = __ablk_encrypt, | |
485 | .decrypt = ablk_decrypt, | |
486 | }, | |
487 | }, | |
488 | }, { | |
489 | .cra_name = "ctr(twofish)", | |
490 | .cra_driver_name = "ctr-twofish-avx", | |
491 | .cra_priority = 400, | |
492 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | |
493 | .cra_blocksize = 1, | |
30a04008 | 494 | .cra_ctxsize = sizeof(struct async_helper_ctx), |
107778b5 JG |
495 | .cra_alignmask = 0, |
496 | .cra_type = &crypto_ablkcipher_type, | |
497 | .cra_module = THIS_MODULE, | |
107778b5 JG |
498 | .cra_init = ablk_init, |
499 | .cra_exit = ablk_exit, | |
500 | .cra_u = { | |
501 | .ablkcipher = { | |
502 | .min_keysize = TF_MIN_KEY_SIZE, | |
503 | .max_keysize = TF_MAX_KEY_SIZE, | |
504 | .ivsize = TF_BLOCK_SIZE, | |
505 | .setkey = ablk_set_key, | |
506 | .encrypt = ablk_encrypt, | |
507 | .decrypt = ablk_encrypt, | |
508 | .geniv = "chainiv", | |
509 | }, | |
510 | }, | |
511 | }, { | |
512 | .cra_name = "lrw(twofish)", | |
513 | .cra_driver_name = "lrw-twofish-avx", | |
514 | .cra_priority = 400, | |
515 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | |
516 | .cra_blocksize = TF_BLOCK_SIZE, | |
30a04008 | 517 | .cra_ctxsize = sizeof(struct async_helper_ctx), |
107778b5 JG |
518 | .cra_alignmask = 0, |
519 | .cra_type = &crypto_ablkcipher_type, | |
520 | .cra_module = THIS_MODULE, | |
107778b5 JG |
521 | .cra_init = ablk_init, |
522 | .cra_exit = ablk_exit, | |
523 | .cra_u = { | |
524 | .ablkcipher = { | |
525 | .min_keysize = TF_MIN_KEY_SIZE + | |
526 | TF_BLOCK_SIZE, | |
527 | .max_keysize = TF_MAX_KEY_SIZE + | |
528 | TF_BLOCK_SIZE, | |
529 | .ivsize = TF_BLOCK_SIZE, | |
530 | .setkey = ablk_set_key, | |
531 | .encrypt = ablk_encrypt, | |
532 | .decrypt = ablk_decrypt, | |
533 | }, | |
534 | }, | |
535 | }, { | |
536 | .cra_name = "xts(twofish)", | |
537 | .cra_driver_name = "xts-twofish-avx", | |
538 | .cra_priority = 400, | |
539 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | |
540 | .cra_blocksize = TF_BLOCK_SIZE, | |
30a04008 | 541 | .cra_ctxsize = sizeof(struct async_helper_ctx), |
107778b5 JG |
542 | .cra_alignmask = 0, |
543 | .cra_type = &crypto_ablkcipher_type, | |
544 | .cra_module = THIS_MODULE, | |
107778b5 JG |
545 | .cra_init = ablk_init, |
546 | .cra_exit = ablk_exit, | |
547 | .cra_u = { | |
548 | .ablkcipher = { | |
549 | .min_keysize = TF_MIN_KEY_SIZE * 2, | |
550 | .max_keysize = TF_MAX_KEY_SIZE * 2, | |
551 | .ivsize = TF_BLOCK_SIZE, | |
552 | .setkey = ablk_set_key, | |
553 | .encrypt = ablk_encrypt, | |
554 | .decrypt = ablk_decrypt, | |
555 | }, | |
556 | }, | |
557 | } }; | |
558 | ||
559 | static int __init twofish_init(void) | |
560 | { | |
561 | u64 xcr0; | |
562 | ||
563 | if (!cpu_has_avx || !cpu_has_osxsave) { | |
564 | printk(KERN_INFO "AVX instructions are not detected.\n"); | |
565 | return -ENODEV; | |
566 | } | |
567 | ||
568 | xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); | |
569 | if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { | |
570 | printk(KERN_INFO "AVX detected but unusable.\n"); | |
571 | return -ENODEV; | |
572 | } | |
573 | ||
574 | return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs)); | |
575 | } | |
576 | ||
577 | static void __exit twofish_exit(void) | |
578 | { | |
579 | crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs)); | |
580 | } | |
581 | ||
582 | module_init(twofish_init); | |
583 | module_exit(twofish_exit); | |
584 | ||
585 | MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); | |
586 | MODULE_LICENSE("GPL"); | |
5d26a105 | 587 | MODULE_ALIAS_CRYPTO("twofish"); |