arch/x86/crypto/serpent_avx_glue.c

   1 /*
   2  * Glue Code for AVX assembler versions of Serpent Cipher
   3  *
   4  * Copyright (C) 2012 Johannes Goetzfried
   5  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
   6  *
   7  * Copyright © 2011-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  22  * USA
  23  *
  24  */
  25
  26 #include <linux/module.h>
  27 #include <linux/hardirq.h>
  28 #include <linux/types.h>
  29 #include <linux/crypto.h>
  30 #include <linux/err.h>
  31 #include <crypto/ablk_helper.h>
  32 #include <crypto/algapi.h>
  33 #include <crypto/serpent.h>
  34 #include <crypto/cryptd.h>
  35 #include <crypto/b128ops.h>
  36 #include <crypto/ctr.h>
  37 #include <crypto/lrw.h>
  38 #include <crypto/xts.h>
  39 #include <asm/xcr.h>
  40 #include <asm/fpu/xstate.h>
  41 #include <asm/crypto/serpent-avx.h>
  42 #include <asm/crypto/glue_helper.h>
  43
  44 /* 8-way parallel cipher functions */
  45 asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  46                                          const u8 *src);
  47 EXPORT_SYMBOL_GPL(serpent_ecb_enc_8way_avx);
  48
  49 asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  50                                          const u8 *src);
  51 EXPORT_SYMBOL_GPL(serpent_ecb_dec_8way_avx);
  52
  53 asmlinkage void serpent_cbc_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  54                                          const u8 *src);
  55 EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx);
  56
  57 asmlinkage void serpent_ctr_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  58                                      const u8 *src, le128 *iv);
  59 EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx);
  60
  61 asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  62                                          const u8 *src, le128 *iv);
  63 EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx);
  64
  65 asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
  66                                          const u8 *src, le128 *iv);
  67 EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx);
  68
  69 void __serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  70 {
  71         be128 ctrblk;
  72
  73         le128_to_be128(&ctrblk, iv);
  74         le128_inc(iv);
  75
  76         __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
  77         u128_xor(dst, src, (u128 *)&ctrblk);
  78 }
  79 EXPORT_SYMBOL_GPL(__serpent_crypt_ctr);
  80
  81 void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  82 {
  83         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  84                                   GLUE_FUNC_CAST(__serpent_encrypt));
  85 }
  86 EXPORT_SYMBOL_GPL(serpent_xts_enc);
  87
  88 void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
  89 {
  90         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
  91                                   GLUE_FUNC_CAST(__serpent_decrypt));
  92 }
  93 EXPORT_SYMBOL_GPL(serpent_xts_dec);
  94
  95
  96 static const struct common_glue_ctx serpent_enc = {
  97         .num_funcs = 2,
  98         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
  99
 100         .funcs = { {
 101                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 102                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
 103         }, {
 104                 .num_blocks = 1,
 105                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
 106         } }
 107 };
 108
 109 static const struct common_glue_ctx serpent_ctr = {
 110         .num_funcs = 2,
 111         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 112
 113         .funcs = { {
 114                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 115                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
 116         }, {
 117                 .num_blocks = 1,
 118                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
 119         } }
 120 };
 121
 122 static const struct common_glue_ctx serpent_enc_xts = {
 123         .num_funcs = 2,
 124         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 125
 126         .funcs = { {
 127                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 128                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
 129         }, {
 130                 .num_blocks = 1,
 131                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
 132         } }
 133 };
 134
 135 static const struct common_glue_ctx serpent_dec = {
 136         .num_funcs = 2,
 137         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 138
 139         .funcs = { {
 140                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 141                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
 142         }, {
 143                 .num_blocks = 1,
 144                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
 145         } }
 146 };
 147
 148 static const struct common_glue_ctx serpent_dec_cbc = {
 149         .num_funcs = 2,
 150         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 151
 152         .funcs = { {
 153                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 154                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
 155         }, {
 156                 .num_blocks = 1,
 157                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
 158         } }
 159 };
 160
 161 static const struct common_glue_ctx serpent_dec_xts = {
 162         .num_funcs = 2,
 163         .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
 164
 165         .funcs = { {
 166                 .num_blocks = SERPENT_PARALLEL_BLOCKS,
 167                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
 168         }, {
 169                 .num_blocks = 1,
 170                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
 171         } }
 172 };
 173
 174 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 175                        struct scatterlist *src, unsigned int nbytes)
 176 {
 177         return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
 178 }
 179
 180 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 181                        struct scatterlist *src, unsigned int nbytes)
 182 {
 183         return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
 184 }
 185
 186 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 187                        struct scatterlist *src, unsigned int nbytes)
 188 {
 189         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
 190                                      dst, src, nbytes);
 191 }
 192
 193 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 194                        struct scatterlist *src, unsigned int nbytes)
 195 {
 196         return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
 197                                        nbytes);
 198 }
 199
 200 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 201                      struct scatterlist *src, unsigned int nbytes)
 202 {
 203         return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
 204 }
 205
 206 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
 207 {
 208         return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
 209                               NULL, fpu_enabled, nbytes);
 210 }
 211
 212 static inline void serpent_fpu_end(bool fpu_enabled)
 213 {
 214         glue_fpu_end(fpu_enabled);
 215 }
 216
 217 struct crypt_priv {
 218         struct serpent_ctx *ctx;
 219         bool fpu_enabled;
 220 };
 221
 222 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 223 {
 224         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 225         struct crypt_priv *ctx = priv;
 226         int i;
 227
 228         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
 229
 230         if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
 231                 serpent_ecb_enc_8way_avx(ctx->ctx, srcdst, srcdst);
 232                 return;
 233         }
 234
 235         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 236                 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
 237 }
 238
 239 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 240 {
 241         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 242         struct crypt_priv *ctx = priv;
 243         int i;
 244
 245         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
 246
 247         if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
 248                 serpent_ecb_dec_8way_avx(ctx->ctx, srcdst, srcdst);
 249                 return;
 250         }
 251
 252         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 253                 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
 254 }
 255
 256 int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
 257                        unsigned int keylen)
 258 {
 259         struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 260         int err;
 261
 262         err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
 263                                                         SERPENT_BLOCK_SIZE);
 264         if (err)
 265                 return err;
 266
 267         return lrw_init_table(&ctx->lrw_table, key + keylen -
 268                                                 SERPENT_BLOCK_SIZE);
 269 }
 270 EXPORT_SYMBOL_GPL(lrw_serpent_setkey);
 271
 272 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 273                        struct scatterlist *src, unsigned int nbytes)
 274 {
 275         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 276         be128 buf[SERPENT_PARALLEL_BLOCKS];
 277         struct crypt_priv crypt_ctx = {
 278                 .ctx = &ctx->serpent_ctx,
 279                 .fpu_enabled = false,
 280         };
 281         struct lrw_crypt_req req = {
 282                 .tbuf = buf,
 283                 .tbuflen = sizeof(buf),
 284
 285                 .table_ctx = &ctx->lrw_table,
 286                 .crypt_ctx = &crypt_ctx,
 287                 .crypt_fn = encrypt_callback,
 288         };
 289         int ret;
 290
 291         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 292         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 293         serpent_fpu_end(crypt_ctx.fpu_enabled);
 294
 295         return ret;
 296 }
 297
 298 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 299                        struct scatterlist *src, unsigned int nbytes)
 300 {
 301         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 302         be128 buf[SERPENT_PARALLEL_BLOCKS];
 303         struct crypt_priv crypt_ctx = {
 304                 .ctx = &ctx->serpent_ctx,
 305                 .fpu_enabled = false,
 306         };
 307         struct lrw_crypt_req req = {
 308                 .tbuf = buf,
 309                 .tbuflen = sizeof(buf),
 310
 311                 .table_ctx = &ctx->lrw_table,
 312                 .crypt_ctx = &crypt_ctx,
 313                 .crypt_fn = decrypt_callback,
 314         };
 315         int ret;
 316
 317         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 318         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 319         serpent_fpu_end(crypt_ctx.fpu_enabled);
 320
 321         return ret;
 322 }
 323
 324 void lrw_serpent_exit_tfm(struct crypto_tfm *tfm)
 325 {
 326         struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 327
 328         lrw_free_table(&ctx->lrw_table);
 329 }
 330 EXPORT_SYMBOL_GPL(lrw_serpent_exit_tfm);
 331
 332 int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
 333                        unsigned int keylen)
 334 {
 335         struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 336         u32 *flags = &tfm->crt_flags;
 337         int err;
 338
 339         /* key consists of keys of equal size concatenated, therefore
 340          * the length must be even
 341          */
 342         if (keylen % 2) {
 343                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 344                 return -EINVAL;
 345         }
 346
 347         /* first half of xts-key is for crypt */
 348         err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
 349         if (err)
 350                 return err;
 351
 352         /* second half of xts-key is for tweak */
 353         return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
 354 }
 355 EXPORT_SYMBOL_GPL(xts_serpent_setkey);
 356
 357 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 358                        struct scatterlist *src, unsigned int nbytes)
 359 {
 360         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 361
 362         return glue_xts_crypt_128bit(&serpent_enc_xts, desc, dst, src, nbytes,
 363                                      XTS_TWEAK_CAST(__serpent_encrypt),
 364                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 365 }
 366
 367 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 368                        struct scatterlist *src, unsigned int nbytes)
 369 {
 370         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 371
 372         return glue_xts_crypt_128bit(&serpent_dec_xts, desc, dst, src, nbytes,
 373                                      XTS_TWEAK_CAST(__serpent_encrypt),
 374                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 375 }
 376
 377 static struct crypto_alg serpent_algs[10] = { {
 378         .cra_name               = "__ecb-serpent-avx",
 379         .cra_driver_name        = "__driver-ecb-serpent-avx",
 380         .cra_priority           = 0,
 381         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 382                                   CRYPTO_ALG_INTERNAL,
 383         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 384         .cra_ctxsize            = sizeof(struct serpent_ctx),
 385         .cra_alignmask          = 0,
 386         .cra_type               = &crypto_blkcipher_type,
 387         .cra_module             = THIS_MODULE,
 388         .cra_u = {
 389                 .blkcipher = {
 390                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 391                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 392                         .setkey         = serpent_setkey,
 393                         .encrypt        = ecb_encrypt,
 394                         .decrypt        = ecb_decrypt,
 395                 },
 396         },
 397 }, {
 398         .cra_name               = "__cbc-serpent-avx",
 399         .cra_driver_name        = "__driver-cbc-serpent-avx",
 400         .cra_priority           = 0,
 401         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 402                                   CRYPTO_ALG_INTERNAL,
 403         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 404         .cra_ctxsize            = sizeof(struct serpent_ctx),
 405         .cra_alignmask          = 0,
 406         .cra_type               = &crypto_blkcipher_type,
 407         .cra_module             = THIS_MODULE,
 408         .cra_u = {
 409                 .blkcipher = {
 410                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 411                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 412                         .setkey         = serpent_setkey,
 413                         .encrypt        = cbc_encrypt,
 414                         .decrypt        = cbc_decrypt,
 415                 },
 416         },
 417 }, {
 418         .cra_name               = "__ctr-serpent-avx",
 419         .cra_driver_name        = "__driver-ctr-serpent-avx",
 420         .cra_priority           = 0,
 421         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 422                                   CRYPTO_ALG_INTERNAL,
 423         .cra_blocksize          = 1,
 424         .cra_ctxsize            = sizeof(struct serpent_ctx),
 425         .cra_alignmask          = 0,
 426         .cra_type               = &crypto_blkcipher_type,
 427         .cra_module             = THIS_MODULE,
 428         .cra_u = {
 429                 .blkcipher = {
 430                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 431                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 432                         .ivsize         = SERPENT_BLOCK_SIZE,
 433                         .setkey         = serpent_setkey,
 434                         .encrypt        = ctr_crypt,
 435                         .decrypt        = ctr_crypt,
 436                 },
 437         },
 438 }, {
 439         .cra_name               = "__lrw-serpent-avx",
 440         .cra_driver_name        = "__driver-lrw-serpent-avx",
 441         .cra_priority           = 0,
 442         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 443                                   CRYPTO_ALG_INTERNAL,
 444         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 445         .cra_ctxsize            = sizeof(struct serpent_lrw_ctx),
 446         .cra_alignmask          = 0,
 447         .cra_type               = &crypto_blkcipher_type,
 448         .cra_module             = THIS_MODULE,
 449         .cra_exit               = lrw_serpent_exit_tfm,
 450         .cra_u = {
 451                 .blkcipher = {
 452                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
 453                                           SERPENT_BLOCK_SIZE,
 454                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
 455                                           SERPENT_BLOCK_SIZE,
 456                         .ivsize         = SERPENT_BLOCK_SIZE,
 457                         .setkey         = lrw_serpent_setkey,
 458                         .encrypt        = lrw_encrypt,
 459                         .decrypt        = lrw_decrypt,
 460                 },
 461         },
 462 }, {
 463         .cra_name               = "__xts-serpent-avx",
 464         .cra_driver_name        = "__driver-xts-serpent-avx",
 465         .cra_priority           = 0,
 466         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 467                                   CRYPTO_ALG_INTERNAL,
 468         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 469         .cra_ctxsize            = sizeof(struct serpent_xts_ctx),
 470         .cra_alignmask          = 0,
 471         .cra_type               = &crypto_blkcipher_type,
 472         .cra_module             = THIS_MODULE,
 473         .cra_u = {
 474                 .blkcipher = {
 475                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
 476                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
 477                         .ivsize         = SERPENT_BLOCK_SIZE,
 478                         .setkey         = xts_serpent_setkey,
 479                         .encrypt        = xts_encrypt,
 480                         .decrypt        = xts_decrypt,
 481                 },
 482         },
 483 }, {
 484         .cra_name               = "ecb(serpent)",
 485         .cra_driver_name        = "ecb-serpent-avx",
 486         .cra_priority           = 500,
 487         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 488         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 489         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 490         .cra_alignmask          = 0,
 491         .cra_type               = &crypto_ablkcipher_type,
 492         .cra_module             = THIS_MODULE,
 493         .cra_init               = ablk_init,
 494         .cra_exit               = ablk_exit,
 495         .cra_u = {
 496                 .ablkcipher = {
 497                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 498                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 499                         .setkey         = ablk_set_key,
 500                         .encrypt        = ablk_encrypt,
 501                         .decrypt        = ablk_decrypt,
 502                 },
 503         },
 504 }, {
 505         .cra_name               = "cbc(serpent)",
 506         .cra_driver_name        = "cbc-serpent-avx",
 507         .cra_priority           = 500,
 508         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 509         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 510         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 511         .cra_alignmask          = 0,
 512         .cra_type               = &crypto_ablkcipher_type,
 513         .cra_module             = THIS_MODULE,
 514         .cra_init               = ablk_init,
 515         .cra_exit               = ablk_exit,
 516         .cra_u = {
 517                 .ablkcipher = {
 518                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 519                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 520                         .ivsize         = SERPENT_BLOCK_SIZE,
 521                         .setkey         = ablk_set_key,
 522                         .encrypt        = __ablk_encrypt,
 523                         .decrypt        = ablk_decrypt,
 524                 },
 525         },
 526 }, {
 527         .cra_name               = "ctr(serpent)",
 528         .cra_driver_name        = "ctr-serpent-avx",
 529         .cra_priority           = 500,
 530         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 531         .cra_blocksize          = 1,
 532         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 533         .cra_alignmask          = 0,
 534         .cra_type               = &crypto_ablkcipher_type,
 535         .cra_module             = THIS_MODULE,
 536         .cra_init               = ablk_init,
 537         .cra_exit               = ablk_exit,
 538         .cra_u = {
 539                 .ablkcipher = {
 540                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 541                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 542                         .ivsize         = SERPENT_BLOCK_SIZE,
 543                         .setkey         = ablk_set_key,
 544                         .encrypt        = ablk_encrypt,
 545                         .decrypt        = ablk_encrypt,
 546                         .geniv          = "chainiv",
 547                 },
 548         },
 549 }, {
 550         .cra_name               = "lrw(serpent)",
 551         .cra_driver_name        = "lrw-serpent-avx",
 552         .cra_priority           = 500,
 553         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 554         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 555         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 556         .cra_alignmask          = 0,
 557         .cra_type               = &crypto_ablkcipher_type,
 558         .cra_module             = THIS_MODULE,
 559         .cra_init               = ablk_init,
 560         .cra_exit               = ablk_exit,
 561         .cra_u = {
 562                 .ablkcipher = {
 563                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
 564                                           SERPENT_BLOCK_SIZE,
 565                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
 566                                           SERPENT_BLOCK_SIZE,
 567                         .ivsize         = SERPENT_BLOCK_SIZE,
 568                         .setkey         = ablk_set_key,
 569                         .encrypt        = ablk_encrypt,
 570                         .decrypt        = ablk_decrypt,
 571                 },
 572         },
 573 }, {
 574         .cra_name               = "xts(serpent)",
 575         .cra_driver_name        = "xts-serpent-avx",
 576         .cra_priority           = 500,
 577         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 578         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 579         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 580         .cra_alignmask          = 0,
 581         .cra_type               = &crypto_ablkcipher_type,
 582         .cra_module             = THIS_MODULE,
 583         .cra_init               = ablk_init,
 584         .cra_exit               = ablk_exit,
 585         .cra_u = {
 586                 .ablkcipher = {
 587                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
 588                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
 589                         .ivsize         = SERPENT_BLOCK_SIZE,
 590                         .setkey         = ablk_set_key,
 591                         .encrypt        = ablk_encrypt,
 592                         .decrypt        = ablk_decrypt,
 593                 },
 594         },
 595 } };
 596
 597 static int __init serpent_init(void)
 598 {
 599         u64 xcr0;
 600
 601         if (!cpu_has_avx || !cpu_has_osxsave) {
 602                 printk(KERN_INFO "AVX instructions are not detected.\n");
 603                 return -ENODEV;
 604         }
 605
 606         xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
 607         if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
 608                 printk(KERN_INFO "AVX detected but unusable.\n");
 609                 return -ENODEV;
 610         }
 611
 612         return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
 613 }
 614
 615 static void __exit serpent_exit(void)
 616 {
 617         crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
 618 }
 619
 620 module_init(serpent_init);
 621 module_exit(serpent_exit);
 622
 623 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
 624 MODULE_LICENSE("GPL");
 625 MODULE_ALIAS_CRYPTO("serpent");