[deliverable/linux.git] / arch / x86 / crypto / twofish_avx_glue.c

/*
 * Glue Code for AVX assembler version of Twofish Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/hardirq.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/ablk_helper.h>
#include <crypto/algapi.h>
#include <crypto/twofish.h>
#include <crypto/cryptd.h>
#include <crypto/b128ops.h>
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/fpu/api.h>
#include <asm/xcr.h>
#include <asm/fpu/xstate.h>
#include <asm/crypto/twofish.h>
#include <asm/crypto/glue_helper.h>
#include <crypto/scatterwalk.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>

#define TWOFISH_PARALLEL_BLOCKS 8

/* 8-way parallel cipher functions */
asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src);
asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src);

asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src);
asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
				 const u8 *src, le128 *iv);

asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src, le128 *iv);
asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
				     const u8 *src, le128 *iv);

static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
					const u8 *src)
{
	__twofish_enc_blk_3way(ctx, dst, src, false);
}

static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
				  GLUE_FUNC_CAST(twofish_enc_blk));
}

static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
				  GLUE_FUNC_CAST(twofish_dec_blk));
}


static const struct common_glue_ctx twofish_enc = {
	.num_funcs = 3,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
	}, {
		.num_blocks = 3,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
	} }
};

static const struct common_glue_ctx twofish_ctr = {
	.num_funcs = 3,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
	}, {
		.num_blocks = 3,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
	} }
};

static const struct common_glue_ctx twofish_enc_xts = {
	.num_funcs = 2,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
	} }
};

static const struct common_glue_ctx twofish_dec = {
	.num_funcs = 3,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
	}, {
		.num_blocks = 3,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
	} }
};

static const struct common_glue_ctx twofish_dec_cbc = {
	.num_funcs = 3,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
	}, {
		.num_blocks = 3,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
	} }
};

static const struct common_glue_ctx twofish_dec_xts = {
	.num_funcs = 2,
	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
	} }
};

static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
}

static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
}

static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
				       dst, src, nbytes);
}

static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
				       nbytes);
}

static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		     struct scatterlist *src, unsigned int nbytes)
{
	return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
}

static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
{
	return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
			      fpu_enabled, nbytes);
}

static inline void twofish_fpu_end(bool fpu_enabled)
{
	glue_fpu_end(fpu_enabled);
}

struct crypt_priv {
	struct twofish_ctx *ctx;
	bool fpu_enabled;
};

static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = TF_BLOCK_SIZE;
	struct crypt_priv *ctx = priv;
	int i;

	ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);

	if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
		twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
		twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);

	nbytes %= bsize * 3;

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		twofish_enc_blk(ctx->ctx, srcdst, srcdst);
}

static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = TF_BLOCK_SIZE;
	struct crypt_priv *ctx = priv;
	int i;

	ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);

	if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
		twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
		twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);

	nbytes %= bsize * 3;

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		twofish_dec_blk(ctx->ctx, srcdst, srcdst);
}

static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[TWOFISH_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->twofish_ctx,
		.fpu_enabled = false,
	};
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = encrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	twofish_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[TWOFISH_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->twofish_ctx,
		.fpu_enabled = false,
	};
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = decrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	twofish_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
				     XTS_TWEAK_CAST(twofish_enc_blk),
				     &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
				     XTS_TWEAK_CAST(twofish_enc_blk),
				     &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct crypto_alg twofish_algs[10] = { {
	.cra_name		= "__ecb-twofish-avx",
	.cra_driver_name	= "__driver-ecb-twofish-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct twofish_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.setkey		= twofish_setkey,
			.encrypt	= ecb_encrypt,
			.decrypt	= ecb_decrypt,
		},
	},
}, {
	.cra_name		= "__cbc-twofish-avx",
	.cra_driver_name	= "__driver-cbc-twofish-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct twofish_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.setkey		= twofish_setkey,
			.encrypt	= cbc_encrypt,
			.decrypt	= cbc_decrypt,
		},
	},
}, {
	.cra_name		= "__ctr-twofish-avx",
	.cra_driver_name	= "__driver-ctr-twofish-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct twofish_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= twofish_setkey,
			.encrypt	= ctr_crypt,
			.decrypt	= ctr_crypt,
		},
	},
}, {
	.cra_name		= "__lrw-twofish-avx",
	.cra_driver_name	= "__driver-lrw-twofish-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct twofish_lrw_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_exit		= lrw_twofish_exit_tfm,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE +
					  TF_BLOCK_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE +
					  TF_BLOCK_SIZE,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= lrw_twofish_setkey,
			.encrypt	= lrw_encrypt,
			.decrypt	= lrw_decrypt,
		},
	},
}, {
	.cra_name		= "__xts-twofish-avx",
	.cra_driver_name	= "__driver-xts-twofish-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct twofish_xts_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE * 2,
			.max_keysize	= TF_MAX_KEY_SIZE * 2,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= xts_twofish_setkey,
			.encrypt	= xts_encrypt,
			.decrypt	= xts_decrypt,
		},
	},
}, {
	.cra_name		= "ecb(twofish)",
	.cra_driver_name	= "ecb-twofish-avx",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "cbc(twofish)",
	.cra_driver_name	= "cbc-twofish-avx",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= __ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "ctr(twofish)",
	.cra_driver_name	= "ctr-twofish-avx",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_encrypt,
			.geniv		= "chainiv",
		},
	},
}, {
	.cra_name		= "lrw(twofish)",
	.cra_driver_name	= "lrw-twofish-avx",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE +
					  TF_BLOCK_SIZE,
			.max_keysize	= TF_MAX_KEY_SIZE +
					  TF_BLOCK_SIZE,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "xts(twofish)",
	.cra_driver_name	= "xts-twofish-avx",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= TF_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= TF_MIN_KEY_SIZE * 2,
			.max_keysize	= TF_MAX_KEY_SIZE * 2,
			.ivsize		= TF_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
} };

static int __init twofish_init(void)
{
	u64 xcr0;

	if (!cpu_has_avx || !cpu_has_osxsave) {
		printk(KERN_INFO "AVX instructions are not detected.\n");
		return -ENODEV;
	}

	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
		printk(KERN_INFO "AVX detected but unusable.\n");
		return -ENODEV;
	}

	return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
}

static void __exit twofish_exit(void)
{
	crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
}

module_init(twofish_init);
module_exit(twofish_exit);

MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("twofish");
Commit	Line	Data
107778b5 JG	1	/*
	2	* Glue Code for AVX assembler version of Twofish Cipher
	3	*
	4	* Copyright (C) 2012 Johannes Goetzfried
	5	* <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
	6	*
18be4527 JK	7	* Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
18be4527 JK	8	*
107778b5 JG	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License as published by
	11	* the Free Software Foundation; either version 2 of the License, or
	12	* (at your option) any later version.
	13	*
	14	* This program is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	* GNU General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU General Public License
	20	* along with this program; if not, write to the Free Software
	21	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	22	* USA
	23	*
	24	*/
	25
	26	#include <linux/module.h>
	27	#include <linux/hardirq.h>
	28	#include <linux/types.h>
	29	#include <linux/crypto.h>
	30	#include <linux/err.h>
801201aa	31	#include <crypto/ablk_helper.h>
107778b5 JG	32	#include <crypto/algapi.h>
	33	#include <crypto/twofish.h>
	34	#include <crypto/cryptd.h>
	35	#include <crypto/b128ops.h>
	36	#include <crypto/ctr.h>
	37	#include <crypto/lrw.h>
	38	#include <crypto/xts.h>
df6b35f4	39	#include <asm/fpu/api.h>
107778b5	40	#include <asm/xcr.h>
669ebabb	41	#include <asm/fpu/xstate.h>
a7378d4e	42	#include <asm/crypto/twofish.h>
a7378d4e	43	#include <asm/crypto/glue_helper.h>
107778b5 JG	44	#include <crypto/scatterwalk.h>
	45	#include <linux/workqueue.h>
	46	#include <linux/spinlock.h>
	47
107778b5 JG	48	#define TWOFISH_PARALLEL_BLOCKS 8
107778b5 JG	49
107778b5	50	/* 8-way parallel cipher functions */
8435a3c3 JK	51	asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx ctx, u8 dst,
	52	const u8 *src);
	53	asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx ctx, u8 dst,
107778b5 JG	54	const u8 *src);
107778b5 JG	55
8435a3c3 JK	56	asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx ctx, u8 dst,
	57	const u8 *src);
	58	asmlinkage void twofish_ctr_8way(struct twofish_ctx ctx, u8 dst,
	59	const u8 src, le128 iv);
107778b5	60
18be4527 JK	61	asmlinkage void twofish_xts_enc_8way(struct twofish_ctx ctx, u8 dst,
	62	const u8 src, le128 iv);
	63	asmlinkage void twofish_xts_dec_8way(struct twofish_ctx ctx, u8 dst,
	64	const u8 src, le128 iv);
	65
8435a3c3	66	static inline void twofish_enc_blk_3way(struct twofish_ctx ctx, u8 dst,
107778b5 JG	67	const u8 *src)
107778b5 JG	68	{
8435a3c3	69	__twofish_enc_blk_3way(ctx, dst, src, false);
107778b5 JG	70	}
107778b5 JG	71
99f42f93	72	static void twofish_xts_enc(void ctx, u128 dst, const u128 src, le128 iv)
18be4527 JK	73	{
	74	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
	75	GLUE_FUNC_CAST(twofish_enc_blk));
	76	}
	77
99f42f93	78	static void twofish_xts_dec(void ctx, u128 dst, const u128 src, le128 iv)
18be4527 JK	79	{
	80	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
	81	GLUE_FUNC_CAST(twofish_dec_blk));
	82	}
	83
107778b5	84
a7378d4e JK	85	static const struct common_glue_ctx twofish_enc = {
	86	.num_funcs = 3,
	87	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	88
	89	.funcs = { {
	90	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
8435a3c3	91	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
a7378d4e JK	92	}, {
	93	.num_blocks = 3,
	94	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
	95	}, {
	96	.num_blocks = 1,
	97	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
	98	} }
	99	};
	100
	101	static const struct common_glue_ctx twofish_ctr = {
	102	.num_funcs = 3,
	103	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	104
	105	.funcs = { {
	106	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
8435a3c3	107	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
a7378d4e JK	108	}, {
	109	.num_blocks = 3,
	110	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
	111	}, {
	112	.num_blocks = 1,
	113	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
	114	} }
	115	};
	116
18be4527 JK	117	static const struct common_glue_ctx twofish_enc_xts = {
	118	.num_funcs = 2,
	119	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	120
	121	.funcs = { {
	122	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
	123	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
	124	}, {
	125	.num_blocks = 1,
	126	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
	127	} }
	128	};
	129
a7378d4e JK	130	static const struct common_glue_ctx twofish_dec = {
	131	.num_funcs = 3,
	132	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	133
	134	.funcs = { {
	135	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
8435a3c3	136	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
a7378d4e JK	137	}, {
	138	.num_blocks = 3,
	139	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
	140	}, {
	141	.num_blocks = 1,
	142	.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
	143	} }
	144	};
	145
	146	static const struct common_glue_ctx twofish_dec_cbc = {
	147	.num_funcs = 3,
	148	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	149
	150	.funcs = { {
	151	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
8435a3c3	152	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
a7378d4e JK	153	}, {
	154	.num_blocks = 3,
	155	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
	156	}, {
	157	.num_blocks = 1,
	158	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
	159	} }
	160	};
	161
18be4527 JK	162	static const struct common_glue_ctx twofish_dec_xts = {
	163	.num_funcs = 2,
	164	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
	165
	166	.funcs = { {
	167	.num_blocks = TWOFISH_PARALLEL_BLOCKS,
	168	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
	169	}, {
	170	.num_blocks = 1,
	171	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
	172	} }
	173	};
	174
107778b5 JG	175	static int ecb_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	176	struct scatterlist *src, unsigned int nbytes)
	177	{
a7378d4e	178	return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
107778b5 JG	179	}
	180
	181	static int ecb_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	182	struct scatterlist *src, unsigned int nbytes)
	183	{
a7378d4e	184	return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
107778b5 JG	185	}
	186
	187	static int cbc_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	188	struct scatterlist *src, unsigned int nbytes)
	189	{
a7378d4e JK	190	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
a7378d4e JK	191	dst, src, nbytes);
107778b5 JG	192	}
	193
	194	static int cbc_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	195	struct scatterlist *src, unsigned int nbytes)
	196	{
a7378d4e JK	197	return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
a7378d4e JK	198	nbytes);
107778b5 JG	199	}
107778b5 JG	200
a7378d4e JK	201	static int ctr_crypt(struct blkcipher_desc desc, struct scatterlist dst,
a7378d4e JK	202	struct scatterlist *src, unsigned int nbytes)
107778b5	203	{
a7378d4e	204	return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
107778b5 JG	205	}
107778b5 JG	206
a7378d4e	207	static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
107778b5	208	{
a7378d4e JK	209	return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
a7378d4e JK	210	fpu_enabled, nbytes);
107778b5 JG	211	}
107778b5 JG	212
a7378d4e	213	static inline void twofish_fpu_end(bool fpu_enabled)
107778b5	214	{
a7378d4e	215	glue_fpu_end(fpu_enabled);
107778b5 JG	216	}
	217
	218	struct crypt_priv {
	219	struct twofish_ctx *ctx;
	220	bool fpu_enabled;
	221	};
	222
	223	static void encrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	224	{
	225	const unsigned int bsize = TF_BLOCK_SIZE;
	226	struct crypt_priv *ctx = priv;
	227	int i;
	228
	229	ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
	230
	231	if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
8435a3c3	232	twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
107778b5 JG	233	return;
	234	}
	235
	236	for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
	237	twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
	238
	239	nbytes %= bsize * 3;
	240
	241	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	242	twofish_enc_blk(ctx->ctx, srcdst, srcdst);
	243	}
	244
	245	static void decrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	246	{
	247	const unsigned int bsize = TF_BLOCK_SIZE;
	248	struct crypt_priv *ctx = priv;
	249	int i;
	250
	251	ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
	252
	253	if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
8435a3c3	254	twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
107778b5 JG	255	return;
	256	}
	257
	258	for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
	259	twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
	260
	261	nbytes %= bsize * 3;
	262
	263	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	264	twofish_dec_blk(ctx->ctx, srcdst, srcdst);
	265	}
	266
107778b5 JG	267	static int lrw_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	268	struct scatterlist *src, unsigned int nbytes)
	269	{
	270	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	271	be128 buf[TWOFISH_PARALLEL_BLOCKS];
	272	struct crypt_priv crypt_ctx = {
	273	.ctx = &ctx->twofish_ctx,
	274	.fpu_enabled = false,
	275	};
	276	struct lrw_crypt_req req = {
	277	.tbuf = buf,
	278	.tbuflen = sizeof(buf),
	279
	280	.table_ctx = &ctx->lrw_table,
	281	.crypt_ctx = &crypt_ctx,
	282	.crypt_fn = encrypt_callback,
	283	};
	284	int ret;
	285
	286	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	287	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	288	twofish_fpu_end(crypt_ctx.fpu_enabled);
	289
	290	return ret;
	291	}
	292
	293	static int lrw_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	294	struct scatterlist *src, unsigned int nbytes)
	295	{
	296	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	297	be128 buf[TWOFISH_PARALLEL_BLOCKS];
	298	struct crypt_priv crypt_ctx = {
	299	.ctx = &ctx->twofish_ctx,
	300	.fpu_enabled = false,
	301	};
	302	struct lrw_crypt_req req = {
	303	.tbuf = buf,
	304	.tbuflen = sizeof(buf),
	305
	306	.table_ctx = &ctx->lrw_table,
	307	.crypt_ctx = &crypt_ctx,
	308	.crypt_fn = decrypt_callback,
	309	};
	310	int ret;
	311
	312	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	313	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	314	twofish_fpu_end(crypt_ctx.fpu_enabled);
	315
	316	return ret;
	317	}
	318
107778b5 JG	319	static int xts_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	320	struct scatterlist *src, unsigned int nbytes)
	321	{
	322	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
107778b5	323
18be4527 JK	324	return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
	325	XTS_TWEAK_CAST(twofish_enc_blk),
	326	&ctx->tweak_ctx, &ctx->crypt_ctx);
107778b5 JG	327	}
	328
	329	static int xts_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	330	struct scatterlist *src, unsigned int nbytes)
	331	{
	332	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
107778b5	333
18be4527 JK	334	return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
	335	XTS_TWEAK_CAST(twofish_enc_blk),
	336	&ctx->tweak_ctx, &ctx->crypt_ctx);
107778b5 JG	337	}
107778b5 JG	338
107778b5 JG	339	static struct crypto_alg twofish_algs[10] = { {
	340	.cra_name = "__ecb-twofish-avx",
	341	.cra_driver_name = "__driver-ecb-twofish-avx",
	342	.cra_priority = 0,
4dda66f6 SM	343	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
4dda66f6 SM	344	CRYPTO_ALG_INTERNAL,
107778b5 JG	345	.cra_blocksize = TF_BLOCK_SIZE,
	346	.cra_ctxsize = sizeof(struct twofish_ctx),
	347	.cra_alignmask = 0,
	348	.cra_type = &crypto_blkcipher_type,
	349	.cra_module = THIS_MODULE,
107778b5 JG	350	.cra_u = {
	351	.blkcipher = {
	352	.min_keysize = TF_MIN_KEY_SIZE,
	353	.max_keysize = TF_MAX_KEY_SIZE,
	354	.setkey = twofish_setkey,
	355	.encrypt = ecb_encrypt,
	356	.decrypt = ecb_decrypt,
	357	},
	358	},
	359	}, {
	360	.cra_name = "__cbc-twofish-avx",
	361	.cra_driver_name = "__driver-cbc-twofish-avx",
	362	.cra_priority = 0,
4dda66f6 SM	363	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
4dda66f6 SM	364	CRYPTO_ALG_INTERNAL,
107778b5 JG	365	.cra_blocksize = TF_BLOCK_SIZE,
	366	.cra_ctxsize = sizeof(struct twofish_ctx),
	367	.cra_alignmask = 0,
	368	.cra_type = &crypto_blkcipher_type,
	369	.cra_module = THIS_MODULE,
107778b5 JG	370	.cra_u = {
	371	.blkcipher = {
	372	.min_keysize = TF_MIN_KEY_SIZE,
	373	.max_keysize = TF_MAX_KEY_SIZE,
	374	.setkey = twofish_setkey,
	375	.encrypt = cbc_encrypt,
	376	.decrypt = cbc_decrypt,
	377	},
	378	},
	379	}, {
	380	.cra_name = "__ctr-twofish-avx",
	381	.cra_driver_name = "__driver-ctr-twofish-avx",
	382	.cra_priority = 0,
4dda66f6 SM	383	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
4dda66f6 SM	384	CRYPTO_ALG_INTERNAL,
107778b5 JG	385	.cra_blocksize = 1,
	386	.cra_ctxsize = sizeof(struct twofish_ctx),
	387	.cra_alignmask = 0,
	388	.cra_type = &crypto_blkcipher_type,
	389	.cra_module = THIS_MODULE,
107778b5 JG	390	.cra_u = {
	391	.blkcipher = {
	392	.min_keysize = TF_MIN_KEY_SIZE,
	393	.max_keysize = TF_MAX_KEY_SIZE,
	394	.ivsize = TF_BLOCK_SIZE,
	395	.setkey = twofish_setkey,
	396	.encrypt = ctr_crypt,
	397	.decrypt = ctr_crypt,
	398	},
	399	},
	400	}, {
	401	.cra_name = "__lrw-twofish-avx",
	402	.cra_driver_name = "__driver-lrw-twofish-avx",
	403	.cra_priority = 0,
4dda66f6 SM	404	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
4dda66f6 SM	405	CRYPTO_ALG_INTERNAL,
107778b5 JG	406	.cra_blocksize = TF_BLOCK_SIZE,
	407	.cra_ctxsize = sizeof(struct twofish_lrw_ctx),
	408	.cra_alignmask = 0,
	409	.cra_type = &crypto_blkcipher_type,
	410	.cra_module = THIS_MODULE,
a7378d4e	411	.cra_exit = lrw_twofish_exit_tfm,
107778b5 JG	412	.cra_u = {
	413	.blkcipher = {
	414	.min_keysize = TF_MIN_KEY_SIZE +
	415	TF_BLOCK_SIZE,
	416	.max_keysize = TF_MAX_KEY_SIZE +
	417	TF_BLOCK_SIZE,
	418	.ivsize = TF_BLOCK_SIZE,
	419	.setkey = lrw_twofish_setkey,
	420	.encrypt = lrw_encrypt,
	421	.decrypt = lrw_decrypt,
	422	},
	423	},
	424	}, {
	425	.cra_name = "__xts-twofish-avx",
	426	.cra_driver_name = "__driver-xts-twofish-avx",
	427	.cra_priority = 0,
4dda66f6 SM	428	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
4dda66f6 SM	429	CRYPTO_ALG_INTERNAL,
107778b5 JG	430	.cra_blocksize = TF_BLOCK_SIZE,
	431	.cra_ctxsize = sizeof(struct twofish_xts_ctx),
	432	.cra_alignmask = 0,
	433	.cra_type = &crypto_blkcipher_type,
	434	.cra_module = THIS_MODULE,
107778b5 JG	435	.cra_u = {
	436	.blkcipher = {
	437	.min_keysize = TF_MIN_KEY_SIZE * 2,
	438	.max_keysize = TF_MAX_KEY_SIZE * 2,
	439	.ivsize = TF_BLOCK_SIZE,
	440	.setkey = xts_twofish_setkey,
	441	.encrypt = xts_encrypt,
	442	.decrypt = xts_decrypt,
	443	},
	444	},
	445	}, {
	446	.cra_name = "ecb(twofish)",
	447	.cra_driver_name = "ecb-twofish-avx",
	448	.cra_priority = 400,
	449	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	450	.cra_blocksize = TF_BLOCK_SIZE,
30a04008	451	.cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5 JG	452	.cra_alignmask = 0,
	453	.cra_type = &crypto_ablkcipher_type,
	454	.cra_module = THIS_MODULE,
107778b5 JG	455	.cra_init = ablk_init,
	456	.cra_exit = ablk_exit,
	457	.cra_u = {
	458	.ablkcipher = {
	459	.min_keysize = TF_MIN_KEY_SIZE,
	460	.max_keysize = TF_MAX_KEY_SIZE,
	461	.setkey = ablk_set_key,
	462	.encrypt = ablk_encrypt,
	463	.decrypt = ablk_decrypt,
	464	},
	465	},
	466	}, {
	467	.cra_name = "cbc(twofish)",
	468	.cra_driver_name = "cbc-twofish-avx",
	469	.cra_priority = 400,
	470	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	471	.cra_blocksize = TF_BLOCK_SIZE,
30a04008	472	.cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5 JG	473	.cra_alignmask = 0,
	474	.cra_type = &crypto_ablkcipher_type,
	475	.cra_module = THIS_MODULE,
107778b5 JG	476	.cra_init = ablk_init,
	477	.cra_exit = ablk_exit,
	478	.cra_u = {
	479	.ablkcipher = {
	480	.min_keysize = TF_MIN_KEY_SIZE,
	481	.max_keysize = TF_MAX_KEY_SIZE,
	482	.ivsize = TF_BLOCK_SIZE,
	483	.setkey = ablk_set_key,
	484	.encrypt = __ablk_encrypt,
	485	.decrypt = ablk_decrypt,
	486	},
	487	},
	488	}, {
	489	.cra_name = "ctr(twofish)",
	490	.cra_driver_name = "ctr-twofish-avx",
	491	.cra_priority = 400,
	492	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	493	.cra_blocksize = 1,
30a04008	494	.cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5 JG	495	.cra_alignmask = 0,
	496	.cra_type = &crypto_ablkcipher_type,
	497	.cra_module = THIS_MODULE,
107778b5 JG	498	.cra_init = ablk_init,
	499	.cra_exit = ablk_exit,
	500	.cra_u = {
	501	.ablkcipher = {
	502	.min_keysize = TF_MIN_KEY_SIZE,
	503	.max_keysize = TF_MAX_KEY_SIZE,
	504	.ivsize = TF_BLOCK_SIZE,
	505	.setkey = ablk_set_key,
	506	.encrypt = ablk_encrypt,
	507	.decrypt = ablk_encrypt,
	508	.geniv = "chainiv",
	509	},
	510	},
	511	}, {
	512	.cra_name = "lrw(twofish)",
	513	.cra_driver_name = "lrw-twofish-avx",
	514	.cra_priority = 400,
	515	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	516	.cra_blocksize = TF_BLOCK_SIZE,
30a04008	517	.cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5 JG	518	.cra_alignmask = 0,
	519	.cra_type = &crypto_ablkcipher_type,
	520	.cra_module = THIS_MODULE,
107778b5 JG	521	.cra_init = ablk_init,
	522	.cra_exit = ablk_exit,
	523	.cra_u = {
	524	.ablkcipher = {
	525	.min_keysize = TF_MIN_KEY_SIZE +
	526	TF_BLOCK_SIZE,
	527	.max_keysize = TF_MAX_KEY_SIZE +
	528	TF_BLOCK_SIZE,
	529	.ivsize = TF_BLOCK_SIZE,
	530	.setkey = ablk_set_key,
	531	.encrypt = ablk_encrypt,
	532	.decrypt = ablk_decrypt,
	533	},
	534	},
	535	}, {
	536	.cra_name = "xts(twofish)",
	537	.cra_driver_name = "xts-twofish-avx",
	538	.cra_priority = 400,
	539	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	540	.cra_blocksize = TF_BLOCK_SIZE,
30a04008	541	.cra_ctxsize = sizeof(struct async_helper_ctx),
107778b5 JG	542	.cra_alignmask = 0,
	543	.cra_type = &crypto_ablkcipher_type,
	544	.cra_module = THIS_MODULE,
107778b5 JG	545	.cra_init = ablk_init,
	546	.cra_exit = ablk_exit,
	547	.cra_u = {
	548	.ablkcipher = {
	549	.min_keysize = TF_MIN_KEY_SIZE * 2,
	550	.max_keysize = TF_MAX_KEY_SIZE * 2,
	551	.ivsize = TF_BLOCK_SIZE,
	552	.setkey = ablk_set_key,
	553	.encrypt = ablk_encrypt,
	554	.decrypt = ablk_decrypt,
	555	},
	556	},
	557	} };
	558
	559	static int __init twofish_init(void)
	560	{
	561	u64 xcr0;
	562
	563	if (!cpu_has_avx \|\| !cpu_has_osxsave) {
	564	printk(KERN_INFO "AVX instructions are not detected.\n");
	565	return -ENODEV;
	566	}
	567
	568	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
	569	if ((xcr0 & (XSTATE_SSE \| XSTATE_YMM)) != (XSTATE_SSE \| XSTATE_YMM)) {
	570	printk(KERN_INFO "AVX detected but unusable.\n");
	571	return -ENODEV;
	572	}
	573
	574	return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
	575	}
	576
	577	static void __exit twofish_exit(void)
	578	{
	579	crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
	580	}
	581
	582	module_init(twofish_init);
	583	module_exit(twofish_exit);
	584
	585	MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
	586	MODULE_LICENSE("GPL");
5d26a105	587	MODULE_ALIAS_CRYPTO("twofish");