[deliverable/linux.git] / arch / x86 / crypto / cast6_avx_glue.c

/*
 * Glue Code for the AVX assembler implemention of the Cast6 Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/hardirq.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/ablk_helper.h>
#include <crypto/algapi.h>
#include <crypto/cast6.h>
#include <crypto/cryptd.h>
#include <crypto/b128ops.h>
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/xcr.h>
#include <asm/fpu/xstate.h>
#include <asm/crypto/glue_helper.h>

#define CAST6_PARALLEL_BLOCKS 8

asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src);
asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src);

asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src);
asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src,
			       le128 *iv);

asmlinkage void cast6_xts_enc_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src, le128 *iv);
asmlinkage void cast6_xts_dec_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src, le128 *iv);

static void cast6_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
				  GLUE_FUNC_CAST(__cast6_encrypt));
}

static void cast6_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
				  GLUE_FUNC_CAST(__cast6_decrypt));
}

static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	be128 ctrblk;

	le128_to_be128(&ctrblk, iv);
	le128_inc(iv);

	__cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
	u128_xor(dst, src, (u128 *)&ctrblk);
}

static const struct common_glue_ctx cast6_enc = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
	} }
};

static const struct common_glue_ctx cast6_ctr = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
	} }
};

static const struct common_glue_ctx cast6_enc_xts = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) }
	} }
};

static const struct common_glue_ctx cast6_dec = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
	} }
};

static const struct common_glue_ctx cast6_dec_cbc = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
	} }
};

static const struct common_glue_ctx cast6_dec_xts = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) }
	} }
};

static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
}

static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
}

static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
				       dst, src, nbytes);
}

static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
				       nbytes);
}

static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		     struct scatterlist *src, unsigned int nbytes)
{
	return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
}

static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
{
	return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
			      NULL, fpu_enabled, nbytes);
}

static inline void cast6_fpu_end(bool fpu_enabled)
{
	glue_fpu_end(fpu_enabled);
}

struct crypt_priv {
	struct cast6_ctx *ctx;
	bool fpu_enabled;
};

static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = CAST6_BLOCK_SIZE;
	struct crypt_priv *ctx = priv;
	int i;

	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);

	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
		cast6_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		__cast6_encrypt(ctx->ctx, srcdst, srcdst);
}

static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = CAST6_BLOCK_SIZE;
	struct crypt_priv *ctx = priv;
	int i;

	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);

	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
		cast6_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		__cast6_decrypt(ctx->ctx, srcdst, srcdst);
}

struct cast6_lrw_ctx {
	struct lrw_table_ctx lrw_table;
	struct cast6_ctx cast6_ctx;
};

static int lrw_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	int err;

	err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
			     &tfm->crt_flags);
	if (err)
		return err;

	return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
}

static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[CAST6_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->cast6_ctx,
		.fpu_enabled = false,
	};
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = encrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	cast6_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[CAST6_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->cast6_ctx,
		.fpu_enabled = false,
	};
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = decrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	cast6_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static void lrw_exit_tfm(struct crypto_tfm *tfm)
{
	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);

	lrw_free_table(&ctx->lrw_table);
}

struct cast6_xts_ctx {
	struct cast6_ctx tweak_ctx;
	struct cast6_ctx crypt_ctx;
};

static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
	u32 *flags = &tfm->crt_flags;
	int err;

	/* key consists of keys of equal size concatenated, therefore
	 * the length must be even
	 */
	if (keylen % 2) {
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		return -EINVAL;
	}

	/* first half of xts-key is for crypt */
	err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
	if (err)
		return err;

	/* second half of xts-key is for tweak */
	return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
			      flags);
}

static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	return glue_xts_crypt_128bit(&cast6_enc_xts, desc, dst, src, nbytes,
				     XTS_TWEAK_CAST(__cast6_encrypt),
				     &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	return glue_xts_crypt_128bit(&cast6_dec_xts, desc, dst, src, nbytes,
				     XTS_TWEAK_CAST(__cast6_encrypt),
				     &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct crypto_alg cast6_algs[10] = { {
	.cra_name		= "__ecb-cast6-avx",
	.cra_driver_name	= "__driver-ecb-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct cast6_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.setkey		= cast6_setkey,
			.encrypt	= ecb_encrypt,
			.decrypt	= ecb_decrypt,
		},
	},
}, {
	.cra_name		= "__cbc-cast6-avx",
	.cra_driver_name	= "__driver-cbc-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct cast6_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.setkey		= cast6_setkey,
			.encrypt	= cbc_encrypt,
			.decrypt	= cbc_decrypt,
		},
	},
}, {
	.cra_name		= "__ctr-cast6-avx",
	.cra_driver_name	= "__driver-ctr-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct cast6_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= cast6_setkey,
			.encrypt	= ctr_crypt,
			.decrypt	= ctr_crypt,
		},
	},
}, {
	.cra_name		= "__lrw-cast6-avx",
	.cra_driver_name	= "__driver-lrw-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct cast6_lrw_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_exit		= lrw_exit_tfm,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE +
					  CAST6_BLOCK_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE +
					  CAST6_BLOCK_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= lrw_cast6_setkey,
			.encrypt	= lrw_encrypt,
			.decrypt	= lrw_decrypt,
		},
	},
}, {
	.cra_name		= "__xts-cast6-avx",
	.cra_driver_name	= "__driver-xts-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct cast6_xts_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE * 2,
			.max_keysize	= CAST6_MAX_KEY_SIZE * 2,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= xts_cast6_setkey,
			.encrypt	= xts_encrypt,
			.decrypt	= xts_decrypt,
		},
	},
}, {
	.cra_name		= "ecb(cast6)",
	.cra_driver_name	= "ecb-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "cbc(cast6)",
	.cra_driver_name	= "cbc-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= __ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "ctr(cast6)",
	.cra_driver_name	= "ctr-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_encrypt,
			.geniv		= "chainiv",
		},
	},
}, {
	.cra_name		= "lrw(cast6)",
	.cra_driver_name	= "lrw-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE +
					  CAST6_BLOCK_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE +
					  CAST6_BLOCK_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "xts(cast6)",
	.cra_driver_name	= "xts-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE * 2,
			.max_keysize	= CAST6_MAX_KEY_SIZE * 2,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
} };

static int __init cast6_init(void)
{
	u64 xcr0;

	if (!cpu_has_avx || !cpu_has_osxsave) {
		pr_info("AVX instructions are not detected.\n");
		return -ENODEV;
	}

	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
		pr_info("AVX detected but unusable.\n");
		return -ENODEV;
	}

	return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
}

static void __exit cast6_exit(void)
{
	crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
}

module_init(cast6_init);
module_exit(cast6_exit);

MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("cast6");
Commit	Line	Data
4ea1277d JG	1	/*
	2	* Glue Code for the AVX assembler implemention of the Cast6 Cipher
	3	*
	4	* Copyright (C) 2012 Johannes Goetzfried
	5	* <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
	6	*
70177286 JK	7	* Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
70177286 JK	8	*
4ea1277d JG	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License as published by
	11	* the Free Software Foundation; either version 2 of the License, or
	12	* (at your option) any later version.
	13	*
	14	* This program is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	* GNU General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU General Public License
	20	* along with this program; if not, write to the Free Software
	21	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	22	* USA
	23	*
	24	*/
	25
	26	#include <linux/module.h>
	27	#include <linux/hardirq.h>
	28	#include <linux/types.h>
	29	#include <linux/crypto.h>
	30	#include <linux/err.h>
801201aa	31	#include <crypto/ablk_helper.h>
4ea1277d JG	32	#include <crypto/algapi.h>
	33	#include <crypto/cast6.h>
	34	#include <crypto/cryptd.h>
	35	#include <crypto/b128ops.h>
	36	#include <crypto/ctr.h>
	37	#include <crypto/lrw.h>
	38	#include <crypto/xts.h>
	39	#include <asm/xcr.h>
669ebabb	40	#include <asm/fpu/xstate.h>
4ea1277d JG	41	#include <asm/crypto/glue_helper.h>
	42
	43	#define CAST6_PARALLEL_BLOCKS 8
	44
cba1cce0 JK	45	asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx ctx, u8 dst,
	46	const u8 *src);
	47	asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx ctx, u8 dst,
4ea1277d JG	48	const u8 *src);
4ea1277d JG	49
cba1cce0 JK	50	asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx ctx, u8 dst,
	51	const u8 *src);
	52	asmlinkage void cast6_ctr_8way(struct cast6_ctx ctx, u8 dst, const u8 *src,
	53	le128 *iv);
4ea1277d	54
70177286 JK	55	asmlinkage void cast6_xts_enc_8way(struct cast6_ctx ctx, u8 dst,
	56	const u8 src, le128 iv);
	57	asmlinkage void cast6_xts_dec_8way(struct cast6_ctx ctx, u8 dst,
	58	const u8 src, le128 iv);
	59
	60	static void cast6_xts_enc(void ctx, u128 dst, const u128 src, le128 iv)
	61	{
	62	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
	63	GLUE_FUNC_CAST(__cast6_encrypt));
	64	}
	65
	66	static void cast6_xts_dec(void ctx, u128 dst, const u128 src, le128 iv)
	67	{
	68	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
	69	GLUE_FUNC_CAST(__cast6_decrypt));
	70	}
	71
58990986	72	static void cast6_crypt_ctr(void ctx, u128 dst, const u128 src, le128 iv)
4ea1277d JG	73	{
	74	be128 ctrblk;
	75
58990986 JK	76	le128_to_be128(&ctrblk, iv);
58990986 JK	77	le128_inc(iv);
4ea1277d JG	78
	79	__cast6_encrypt(ctx, (u8 )&ctrblk, (u8 )&ctrblk);
	80	u128_xor(dst, src, (u128 *)&ctrblk);
	81	}
	82
4ea1277d JG	83	static const struct common_glue_ctx cast6_enc = {
	84	.num_funcs = 2,
	85	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	86
	87	.funcs = { {
	88	.num_blocks = CAST6_PARALLEL_BLOCKS,
cba1cce0	89	.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
4ea1277d JG	90	}, {
	91	.num_blocks = 1,
	92	.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
	93	} }
	94	};
	95
	96	static const struct common_glue_ctx cast6_ctr = {
	97	.num_funcs = 2,
	98	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	99
	100	.funcs = { {
	101	.num_blocks = CAST6_PARALLEL_BLOCKS,
cba1cce0	102	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
4ea1277d JG	103	}, {
	104	.num_blocks = 1,
	105	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
	106	} }
	107	};
	108
70177286 JK	109	static const struct common_glue_ctx cast6_enc_xts = {
	110	.num_funcs = 2,
	111	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	112
	113	.funcs = { {
	114	.num_blocks = CAST6_PARALLEL_BLOCKS,
	115	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) }
	116	}, {
	117	.num_blocks = 1,
	118	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) }
	119	} }
	120	};
	121
4ea1277d JG	122	static const struct common_glue_ctx cast6_dec = {
	123	.num_funcs = 2,
	124	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	125
	126	.funcs = { {
	127	.num_blocks = CAST6_PARALLEL_BLOCKS,
cba1cce0	128	.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
4ea1277d JG	129	}, {
	130	.num_blocks = 1,
	131	.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
	132	} }
	133	};
	134
	135	static const struct common_glue_ctx cast6_dec_cbc = {
	136	.num_funcs = 2,
	137	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	138
	139	.funcs = { {
	140	.num_blocks = CAST6_PARALLEL_BLOCKS,
cba1cce0	141	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
4ea1277d JG	142	}, {
	143	.num_blocks = 1,
	144	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
	145	} }
	146	};
	147
70177286 JK	148	static const struct common_glue_ctx cast6_dec_xts = {
	149	.num_funcs = 2,
	150	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	151
	152	.funcs = { {
	153	.num_blocks = CAST6_PARALLEL_BLOCKS,
	154	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) }
	155	}, {
	156	.num_blocks = 1,
	157	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) }
	158	} }
	159	};
	160
4ea1277d JG	161	static int ecb_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	162	struct scatterlist *src, unsigned int nbytes)
	163	{
	164	return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
	165	}
	166
	167	static int ecb_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	168	struct scatterlist *src, unsigned int nbytes)
	169	{
	170	return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
	171	}
	172
	173	static int cbc_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	174	struct scatterlist *src, unsigned int nbytes)
	175	{
	176	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
	177	dst, src, nbytes);
	178	}
	179
	180	static int cbc_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	181	struct scatterlist *src, unsigned int nbytes)
	182	{
	183	return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
	184	nbytes);
	185	}
	186
	187	static int ctr_crypt(struct blkcipher_desc desc, struct scatterlist dst,
	188	struct scatterlist *src, unsigned int nbytes)
	189	{
	190	return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
	191	}
	192
	193	static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
	194	{
	195	return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
	196	NULL, fpu_enabled, nbytes);
	197	}
	198
	199	static inline void cast6_fpu_end(bool fpu_enabled)
	200	{
	201	glue_fpu_end(fpu_enabled);
	202	}
	203
	204	struct crypt_priv {
	205	struct cast6_ctx *ctx;
	206	bool fpu_enabled;
	207	};
	208
	209	static void encrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	210	{
	211	const unsigned int bsize = CAST6_BLOCK_SIZE;
	212	struct crypt_priv *ctx = priv;
	213	int i;
	214
	215	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
	216
	217	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
cba1cce0	218	cast6_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
4ea1277d JG	219	return;
	220	}
	221
	222	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	223	__cast6_encrypt(ctx->ctx, srcdst, srcdst);
	224	}
	225
	226	static void decrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	227	{
	228	const unsigned int bsize = CAST6_BLOCK_SIZE;
	229	struct crypt_priv *ctx = priv;
	230	int i;
	231
	232	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
	233
	234	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
cba1cce0	235	cast6_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
4ea1277d JG	236	return;
	237	}
	238
	239	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	240	__cast6_decrypt(ctx->ctx, srcdst, srcdst);
	241	}
	242
	243	struct cast6_lrw_ctx {
	244	struct lrw_table_ctx lrw_table;
	245	struct cast6_ctx cast6_ctx;
	246	};
	247
	248	static int lrw_cast6_setkey(struct crypto_tfm tfm, const u8 key,
	249	unsigned int keylen)
	250	{
	251	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	252	int err;
	253
	254	err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
	255	&tfm->crt_flags);
	256	if (err)
	257	return err;
	258
	259	return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
	260	}
	261
	262	static int lrw_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	263	struct scatterlist *src, unsigned int nbytes)
	264	{
	265	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	266	be128 buf[CAST6_PARALLEL_BLOCKS];
	267	struct crypt_priv crypt_ctx = {
	268	.ctx = &ctx->cast6_ctx,
	269	.fpu_enabled = false,
	270	};
	271	struct lrw_crypt_req req = {
	272	.tbuf = buf,
	273	.tbuflen = sizeof(buf),
	274
	275	.table_ctx = &ctx->lrw_table,
	276	.crypt_ctx = &crypt_ctx,
	277	.crypt_fn = encrypt_callback,
	278	};
	279	int ret;
	280
	281	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	282	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	283	cast6_fpu_end(crypt_ctx.fpu_enabled);
	284
	285	return ret;
	286	}
	287
	288	static int lrw_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	289	struct scatterlist *src, unsigned int nbytes)
	290	{
	291	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	292	be128 buf[CAST6_PARALLEL_BLOCKS];
	293	struct crypt_priv crypt_ctx = {
	294	.ctx = &ctx->cast6_ctx,
	295	.fpu_enabled = false,
	296	};
	297	struct lrw_crypt_req req = {
	298	.tbuf = buf,
	299	.tbuflen = sizeof(buf),
300
301	.table_ctx = &ctx->lrw_table,
302	.crypt_ctx = &crypt_ctx,
303	.crypt_fn = decrypt_callback,
304	};
305	int ret;
306
307	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
308	ret = lrw_crypt(desc, dst, src, nbytes, &req);
309	cast6_fpu_end(crypt_ctx.fpu_enabled);
310
311	return ret;
312	}
313
314	static void lrw_exit_tfm(struct crypto_tfm *tfm)
315	{
316	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
317
318	lrw_free_table(&ctx->lrw_table);
319	}
320
321	struct cast6_xts_ctx {
322	struct cast6_ctx tweak_ctx;
323	struct cast6_ctx crypt_ctx;
324	};
325
326	static int xts_cast6_setkey(struct crypto_tfm tfm, const u8 key,
327	unsigned int keylen)
328	{
329	struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
330	u32 *flags = &tfm->crt_flags;
331	int err;
332
333	/* key consists of keys of equal size concatenated, therefore
334	* the length must be even
335	*/
336	if (keylen % 2) {
337	*flags \|= CRYPTO_TFM_RES_BAD_KEY_LEN;
338	return -EINVAL;
339	}
340
341	/* first half of xts-key is for crypt */
342	err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
343	if (err)
344	return err;
345
346	/* second half of xts-key is for tweak */
347	return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
348	flags);
349	}
350
351	static int xts_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
352	struct scatterlist *src, unsigned int nbytes)
353	{
354	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
4ea1277d	355
70177286 JK	356	return glue_xts_crypt_128bit(&cast6_enc_xts, desc, dst, src, nbytes,
	357	XTS_TWEAK_CAST(__cast6_encrypt),
	358	&ctx->tweak_ctx, &ctx->crypt_ctx);
4ea1277d JG	359	}
	360
	361	static int xts_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	362	struct scatterlist *src, unsigned int nbytes)
	363	{
	364	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
4ea1277d	365
70177286 JK	366	return glue_xts_crypt_128bit(&cast6_dec_xts, desc, dst, src, nbytes,
	367	XTS_TWEAK_CAST(__cast6_encrypt),
	368	&ctx->tweak_ctx, &ctx->crypt_ctx);
4ea1277d JG	369	}
	370
	371	static struct crypto_alg cast6_algs[10] = { {
	372	.cra_name = "__ecb-cast6-avx",
	373	.cra_driver_name = "__driver-ecb-cast6-avx",
	374	.cra_priority = 0,
e69b8a46 SM	375	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	376	CRYPTO_ALG_INTERNAL,
4ea1277d JG	377	.cra_blocksize = CAST6_BLOCK_SIZE,
	378	.cra_ctxsize = sizeof(struct cast6_ctx),
	379	.cra_alignmask = 0,
	380	.cra_type = &crypto_blkcipher_type,
	381	.cra_module = THIS_MODULE,
	382	.cra_u = {
	383	.blkcipher = {
	384	.min_keysize = CAST6_MIN_KEY_SIZE,
	385	.max_keysize = CAST6_MAX_KEY_SIZE,
	386	.setkey = cast6_setkey,
	387	.encrypt = ecb_encrypt,
	388	.decrypt = ecb_decrypt,
	389	},
	390	},
	391	}, {
	392	.cra_name = "__cbc-cast6-avx",
	393	.cra_driver_name = "__driver-cbc-cast6-avx",
	394	.cra_priority = 0,
e69b8a46 SM	395	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	396	CRYPTO_ALG_INTERNAL,
4ea1277d JG	397	.cra_blocksize = CAST6_BLOCK_SIZE,
	398	.cra_ctxsize = sizeof(struct cast6_ctx),
	399	.cra_alignmask = 0,
	400	.cra_type = &crypto_blkcipher_type,
	401	.cra_module = THIS_MODULE,
	402	.cra_u = {
	403	.blkcipher = {
	404	.min_keysize = CAST6_MIN_KEY_SIZE,
	405	.max_keysize = CAST6_MAX_KEY_SIZE,
	406	.setkey = cast6_setkey,
	407	.encrypt = cbc_encrypt,
	408	.decrypt = cbc_decrypt,
	409	},
	410	},
	411	}, {
	412	.cra_name = "__ctr-cast6-avx",
	413	.cra_driver_name = "__driver-ctr-cast6-avx",
	414	.cra_priority = 0,
e69b8a46 SM	415	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	416	CRYPTO_ALG_INTERNAL,
4ea1277d JG	417	.cra_blocksize = 1,
	418	.cra_ctxsize = sizeof(struct cast6_ctx),
	419	.cra_alignmask = 0,
	420	.cra_type = &crypto_blkcipher_type,
	421	.cra_module = THIS_MODULE,
	422	.cra_u = {
	423	.blkcipher = {
	424	.min_keysize = CAST6_MIN_KEY_SIZE,
	425	.max_keysize = CAST6_MAX_KEY_SIZE,
	426	.ivsize = CAST6_BLOCK_SIZE,
	427	.setkey = cast6_setkey,
	428	.encrypt = ctr_crypt,
	429	.decrypt = ctr_crypt,
	430	},
	431	},
	432	}, {
	433	.cra_name = "__lrw-cast6-avx",
	434	.cra_driver_name = "__driver-lrw-cast6-avx",
	435	.cra_priority = 0,
e69b8a46 SM	436	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	437	CRYPTO_ALG_INTERNAL,
4ea1277d JG	438	.cra_blocksize = CAST6_BLOCK_SIZE,
	439	.cra_ctxsize = sizeof(struct cast6_lrw_ctx),
	440	.cra_alignmask = 0,
	441	.cra_type = &crypto_blkcipher_type,
	442	.cra_module = THIS_MODULE,
	443	.cra_exit = lrw_exit_tfm,
	444	.cra_u = {
	445	.blkcipher = {
	446	.min_keysize = CAST6_MIN_KEY_SIZE +
	447	CAST6_BLOCK_SIZE,
	448	.max_keysize = CAST6_MAX_KEY_SIZE +
	449	CAST6_BLOCK_SIZE,
	450	.ivsize = CAST6_BLOCK_SIZE,
	451	.setkey = lrw_cast6_setkey,
	452	.encrypt = lrw_encrypt,
	453	.decrypt = lrw_decrypt,
	454	},
	455	},
	456	}, {
	457	.cra_name = "__xts-cast6-avx",
	458	.cra_driver_name = "__driver-xts-cast6-avx",
	459	.cra_priority = 0,
e69b8a46 SM	460	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	461	CRYPTO_ALG_INTERNAL,
4ea1277d JG	462	.cra_blocksize = CAST6_BLOCK_SIZE,
	463	.cra_ctxsize = sizeof(struct cast6_xts_ctx),
	464	.cra_alignmask = 0,
	465	.cra_type = &crypto_blkcipher_type,
	466	.cra_module = THIS_MODULE,
	467	.cra_u = {
	468	.blkcipher = {
	469	.min_keysize = CAST6_MIN_KEY_SIZE * 2,
	470	.max_keysize = CAST6_MAX_KEY_SIZE * 2,
	471	.ivsize = CAST6_BLOCK_SIZE,
	472	.setkey = xts_cast6_setkey,
	473	.encrypt = xts_encrypt,
	474	.decrypt = xts_decrypt,
	475	},
	476	},
	477	}, {
	478	.cra_name = "ecb(cast6)",
	479	.cra_driver_name = "ecb-cast6-avx",
	480	.cra_priority = 200,
	481	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	482	.cra_blocksize = CAST6_BLOCK_SIZE,
	483	.cra_ctxsize = sizeof(struct async_helper_ctx),
	484	.cra_alignmask = 0,
	485	.cra_type = &crypto_ablkcipher_type,
	486	.cra_module = THIS_MODULE,
	487	.cra_init = ablk_init,
	488	.cra_exit = ablk_exit,
	489	.cra_u = {
	490	.ablkcipher = {
	491	.min_keysize = CAST6_MIN_KEY_SIZE,
	492	.max_keysize = CAST6_MAX_KEY_SIZE,
	493	.setkey = ablk_set_key,
	494	.encrypt = ablk_encrypt,
	495	.decrypt = ablk_decrypt,
	496	},
	497	},
	498	}, {
	499	.cra_name = "cbc(cast6)",
	500	.cra_driver_name = "cbc-cast6-avx",
	501	.cra_priority = 200,
	502	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	503	.cra_blocksize = CAST6_BLOCK_SIZE,
	504	.cra_ctxsize = sizeof(struct async_helper_ctx),
	505	.cra_alignmask = 0,
	506	.cra_type = &crypto_ablkcipher_type,
	507	.cra_module = THIS_MODULE,
	508	.cra_init = ablk_init,
	509	.cra_exit = ablk_exit,
	510	.cra_u = {
	511	.ablkcipher = {
	512	.min_keysize = CAST6_MIN_KEY_SIZE,
	513	.max_keysize = CAST6_MAX_KEY_SIZE,
	514	.ivsize = CAST6_BLOCK_SIZE,
	515	.setkey = ablk_set_key,
	516	.encrypt = __ablk_encrypt,
	517	.decrypt = ablk_decrypt,
	518	},
	519	},
	520	}, {
	521	.cra_name = "ctr(cast6)",
	522	.cra_driver_name = "ctr-cast6-avx",
	523	.cra_priority = 200,
	524	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	525	.cra_blocksize = 1,
526	.cra_ctxsize = sizeof(struct async_helper_ctx),
527	.cra_alignmask = 0,
528	.cra_type = &crypto_ablkcipher_type,
529	.cra_module = THIS_MODULE,
530	.cra_init = ablk_init,
531	.cra_exit = ablk_exit,
532	.cra_u = {
533	.ablkcipher = {
534	.min_keysize = CAST6_MIN_KEY_SIZE,
535	.max_keysize = CAST6_MAX_KEY_SIZE,
536	.ivsize = CAST6_BLOCK_SIZE,
537	.setkey = ablk_set_key,
538	.encrypt = ablk_encrypt,
539	.decrypt = ablk_encrypt,
540	.geniv = "chainiv",
541	},
542	},
543	}, {
544	.cra_name = "lrw(cast6)",
545	.cra_driver_name = "lrw-cast6-avx",
546	.cra_priority = 200,
547	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
548	.cra_blocksize = CAST6_BLOCK_SIZE,
549	.cra_ctxsize = sizeof(struct async_helper_ctx),
550	.cra_alignmask = 0,
551	.cra_type = &crypto_ablkcipher_type,
552	.cra_module = THIS_MODULE,
553	.cra_init = ablk_init,
554	.cra_exit = ablk_exit,
555	.cra_u = {
556	.ablkcipher = {
557	.min_keysize = CAST6_MIN_KEY_SIZE +
558	CAST6_BLOCK_SIZE,
559	.max_keysize = CAST6_MAX_KEY_SIZE +
560	CAST6_BLOCK_SIZE,
561	.ivsize = CAST6_BLOCK_SIZE,
562	.setkey = ablk_set_key,
563	.encrypt = ablk_encrypt,
564	.decrypt = ablk_decrypt,
565	},
566	},
567	}, {
568	.cra_name = "xts(cast6)",
569	.cra_driver_name = "xts-cast6-avx",
570	.cra_priority = 200,
571	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
572	.cra_blocksize = CAST6_BLOCK_SIZE,
573	.cra_ctxsize = sizeof(struct async_helper_ctx),
574	.cra_alignmask = 0,
575	.cra_type = &crypto_ablkcipher_type,
576	.cra_module = THIS_MODULE,
577	.cra_init = ablk_init,
578	.cra_exit = ablk_exit,
579	.cra_u = {
580	.ablkcipher = {
581	.min_keysize = CAST6_MIN_KEY_SIZE * 2,
582	.max_keysize = CAST6_MAX_KEY_SIZE * 2,
583	.ivsize = CAST6_BLOCK_SIZE,
584	.setkey = ablk_set_key,
585	.encrypt = ablk_encrypt,
586	.decrypt = ablk_decrypt,
587	},
588	},
589	} };
590
591	static int __init cast6_init(void)
592	{
593	u64 xcr0;
594
595	if (!cpu_has_avx \|\| !cpu_has_osxsave) {
596	pr_info("AVX instructions are not detected.\n");
597	return -ENODEV;
598	}
599
600	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
601	if ((xcr0 & (XSTATE_SSE \| XSTATE_YMM)) != (XSTATE_SSE \| XSTATE_YMM)) {
602	pr_info("AVX detected but unusable.\n");
603	return -ENODEV;
604	}
605
606	return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
607	}
608
609	static void __exit cast6_exit(void)
610	{
611	crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
612	}
613
614	module_init(cast6_init);
615	module_exit(cast6_exit);
616
617	MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
618	MODULE_LICENSE("GPL");
5d26a105	619	MODULE_ALIAS_CRYPTO("cast6");