[deliverable/linux.git] / arch / x86 / crypto / cast6_avx_glue.c

/*
 * Glue Code for the AVX assembler implemention of the Cast6 Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/hardirq.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/ablk_helper.h>
#include <crypto/algapi.h>
#include <crypto/cast6.h>
#include <crypto/cryptd.h>
#include <crypto/b128ops.h>
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/fpu/api.h>
#include <asm/crypto/glue_helper.h>

#define CAST6_PARALLEL_BLOCKS 8

asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src);
asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src);

asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src);
asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src,
			       le128 *iv);

asmlinkage void cast6_xts_enc_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src, le128 *iv);
asmlinkage void cast6_xts_dec_8way(struct cast6_ctx *ctx, u8 *dst,
				   const u8 *src, le128 *iv);

static void cast6_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
				  GLUE_FUNC_CAST(__cast6_encrypt));
}

static void cast6_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
				  GLUE_FUNC_CAST(__cast6_decrypt));
}

static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	be128 ctrblk;

	le128_to_be128(&ctrblk, iv);
	le128_inc(iv);

	__cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
	u128_xor(dst, src, (u128 *)&ctrblk);
}

static const struct common_glue_ctx cast6_enc = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
	} }
};

static const struct common_glue_ctx cast6_ctr = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
	} }
};

static const struct common_glue_ctx cast6_enc_xts = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) }
	} }
};

static const struct common_glue_ctx cast6_dec = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
	} }
};

static const struct common_glue_ctx cast6_dec_cbc = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
	} }
};

static const struct common_glue_ctx cast6_dec_xts = {
	.num_funcs = 2,
	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = CAST6_PARALLEL_BLOCKS,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) }
	}, {
		.num_blocks = 1,
		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) }
	} }
};

static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
}

static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
}

static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
				       dst, src, nbytes);
}

static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
				       nbytes);
}

static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		     struct scatterlist *src, unsigned int nbytes)
{
	return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
}

static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
{
	return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
			      NULL, fpu_enabled, nbytes);
}

static inline void cast6_fpu_end(bool fpu_enabled)
{
	glue_fpu_end(fpu_enabled);
}

struct crypt_priv {
	struct cast6_ctx *ctx;
	bool fpu_enabled;
};

static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = CAST6_BLOCK_SIZE;
	struct crypt_priv *ctx = priv;
	int i;

	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);

	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
		cast6_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		__cast6_encrypt(ctx->ctx, srcdst, srcdst);
}

static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = CAST6_BLOCK_SIZE;
	struct crypt_priv *ctx = priv;
	int i;

	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);

	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
		cast6_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		__cast6_decrypt(ctx->ctx, srcdst, srcdst);
}

struct cast6_lrw_ctx {
	struct lrw_table_ctx lrw_table;
	struct cast6_ctx cast6_ctx;
};

static int lrw_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	int err;

	err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
			     &tfm->crt_flags);
	if (err)
		return err;

	return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
}

static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[CAST6_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->cast6_ctx,
		.fpu_enabled = false,
	};
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = encrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	cast6_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[CAST6_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->cast6_ctx,
		.fpu_enabled = false,
	};
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = decrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	cast6_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static void lrw_exit_tfm(struct crypto_tfm *tfm)
{
	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);

	lrw_free_table(&ctx->lrw_table);
}

struct cast6_xts_ctx {
	struct cast6_ctx tweak_ctx;
	struct cast6_ctx crypt_ctx;
};

static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
	u32 *flags = &tfm->crt_flags;
	int err;

	err = xts_check_key(tfm, key, keylen);
	if (err)
		return err;

	/* first half of xts-key is for crypt */
	err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
	if (err)
		return err;

	/* second half of xts-key is for tweak */
	return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
			      flags);
}

static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	return glue_xts_crypt_128bit(&cast6_enc_xts, desc, dst, src, nbytes,
				     XTS_TWEAK_CAST(__cast6_encrypt),
				     &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);

	return glue_xts_crypt_128bit(&cast6_dec_xts, desc, dst, src, nbytes,
				     XTS_TWEAK_CAST(__cast6_encrypt),
				     &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct crypto_alg cast6_algs[10] = { {
	.cra_name		= "__ecb-cast6-avx",
	.cra_driver_name	= "__driver-ecb-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct cast6_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.setkey		= cast6_setkey,
			.encrypt	= ecb_encrypt,
			.decrypt	= ecb_decrypt,
		},
	},
}, {
	.cra_name		= "__cbc-cast6-avx",
	.cra_driver_name	= "__driver-cbc-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct cast6_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.setkey		= cast6_setkey,
			.encrypt	= cbc_encrypt,
			.decrypt	= cbc_decrypt,
		},
	},
}, {
	.cra_name		= "__ctr-cast6-avx",
	.cra_driver_name	= "__driver-ctr-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct cast6_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= cast6_setkey,
			.encrypt	= ctr_crypt,
			.decrypt	= ctr_crypt,
		},
	},
}, {
	.cra_name		= "__lrw-cast6-avx",
	.cra_driver_name	= "__driver-lrw-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct cast6_lrw_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_exit		= lrw_exit_tfm,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE +
					  CAST6_BLOCK_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE +
					  CAST6_BLOCK_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= lrw_cast6_setkey,
			.encrypt	= lrw_encrypt,
			.decrypt	= lrw_decrypt,
		},
	},
}, {
	.cra_name		= "__xts-cast6-avx",
	.cra_driver_name	= "__driver-xts-cast6-avx",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
				  CRYPTO_ALG_INTERNAL,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct cast6_xts_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE * 2,
			.max_keysize	= CAST6_MAX_KEY_SIZE * 2,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= xts_cast6_setkey,
			.encrypt	= xts_encrypt,
			.decrypt	= xts_decrypt,
		},
	},
}, {
	.cra_name		= "ecb(cast6)",
	.cra_driver_name	= "ecb-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "cbc(cast6)",
	.cra_driver_name	= "cbc-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= __ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "ctr(cast6)",
	.cra_driver_name	= "ctr-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_encrypt,
			.geniv		= "chainiv",
		},
	},
}, {
	.cra_name		= "lrw(cast6)",
	.cra_driver_name	= "lrw-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE +
					  CAST6_BLOCK_SIZE,
			.max_keysize	= CAST6_MAX_KEY_SIZE +
					  CAST6_BLOCK_SIZE,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "xts(cast6)",
	.cra_driver_name	= "xts-cast6-avx",
	.cra_priority		= 200,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= CAST6_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= CAST6_MIN_KEY_SIZE * 2,
			.max_keysize	= CAST6_MAX_KEY_SIZE * 2,
			.ivsize		= CAST6_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
} };

static int __init cast6_init(void)
{
	const char *feature_name;

	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
				&feature_name)) {
		pr_info("CPU feature '%s' is not supported.\n", feature_name);
		return -ENODEV;
	}

	return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
}

static void __exit cast6_exit(void)
{
	crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
}

module_init(cast6_init);
module_exit(cast6_exit);

MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("cast6");
Commit	Line	Data
4ea1277d JG	1	/*
	2	* Glue Code for the AVX assembler implemention of the Cast6 Cipher
	3	*
	4	* Copyright (C) 2012 Johannes Goetzfried
	5	* <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
	6	*
70177286 JK	7	* Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
70177286 JK	8	*
4ea1277d JG	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License as published by
	11	* the Free Software Foundation; either version 2 of the License, or
	12	* (at your option) any later version.
	13	*
	14	* This program is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	* GNU General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU General Public License
	20	* along with this program; if not, write to the Free Software
	21	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	22	* USA
	23	*
	24	*/
	25
	26	#include <linux/module.h>
	27	#include <linux/hardirq.h>
	28	#include <linux/types.h>
	29	#include <linux/crypto.h>
	30	#include <linux/err.h>
801201aa	31	#include <crypto/ablk_helper.h>
4ea1277d JG	32	#include <crypto/algapi.h>
	33	#include <crypto/cast6.h>
	34	#include <crypto/cryptd.h>
	35	#include <crypto/b128ops.h>
	36	#include <crypto/ctr.h>
	37	#include <crypto/lrw.h>
	38	#include <crypto/xts.h>
1debf7db	39	#include <asm/fpu/api.h>
4ea1277d JG	40	#include <asm/crypto/glue_helper.h>
	41
	42	#define CAST6_PARALLEL_BLOCKS 8
	43
cba1cce0 JK	44	asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx ctx, u8 dst,
	45	const u8 *src);
	46	asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx ctx, u8 dst,
4ea1277d JG	47	const u8 *src);
4ea1277d JG	48
cba1cce0 JK	49	asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx ctx, u8 dst,
	50	const u8 *src);
	51	asmlinkage void cast6_ctr_8way(struct cast6_ctx ctx, u8 dst, const u8 *src,
	52	le128 *iv);
4ea1277d	53
70177286 JK	54	asmlinkage void cast6_xts_enc_8way(struct cast6_ctx ctx, u8 dst,
	55	const u8 src, le128 iv);
	56	asmlinkage void cast6_xts_dec_8way(struct cast6_ctx ctx, u8 dst,
	57	const u8 src, le128 iv);
	58
	59	static void cast6_xts_enc(void ctx, u128 dst, const u128 src, le128 iv)
	60	{
	61	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
	62	GLUE_FUNC_CAST(__cast6_encrypt));
	63	}
	64
	65	static void cast6_xts_dec(void ctx, u128 dst, const u128 src, le128 iv)
	66	{
	67	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
	68	GLUE_FUNC_CAST(__cast6_decrypt));
	69	}
	70
58990986	71	static void cast6_crypt_ctr(void ctx, u128 dst, const u128 src, le128 iv)
4ea1277d JG	72	{
	73	be128 ctrblk;
	74
58990986 JK	75	le128_to_be128(&ctrblk, iv);
58990986 JK	76	le128_inc(iv);
4ea1277d JG	77
	78	__cast6_encrypt(ctx, (u8 )&ctrblk, (u8 )&ctrblk);
	79	u128_xor(dst, src, (u128 *)&ctrblk);
	80	}
	81
4ea1277d JG	82	static const struct common_glue_ctx cast6_enc = {
	83	.num_funcs = 2,
	84	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	85
	86	.funcs = { {
	87	.num_blocks = CAST6_PARALLEL_BLOCKS,
cba1cce0	88	.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
4ea1277d JG	89	}, {
	90	.num_blocks = 1,
	91	.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
	92	} }
	93	};
	94
	95	static const struct common_glue_ctx cast6_ctr = {
	96	.num_funcs = 2,
	97	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	98
	99	.funcs = { {
	100	.num_blocks = CAST6_PARALLEL_BLOCKS,
cba1cce0	101	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
4ea1277d JG	102	}, {
	103	.num_blocks = 1,
	104	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
	105	} }
	106	};
	107
70177286 JK	108	static const struct common_glue_ctx cast6_enc_xts = {
	109	.num_funcs = 2,
	110	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	111
	112	.funcs = { {
	113	.num_blocks = CAST6_PARALLEL_BLOCKS,
	114	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) }
	115	}, {
	116	.num_blocks = 1,
	117	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) }
	118	} }
	119	};
	120
4ea1277d JG	121	static const struct common_glue_ctx cast6_dec = {
	122	.num_funcs = 2,
	123	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	124
	125	.funcs = { {
	126	.num_blocks = CAST6_PARALLEL_BLOCKS,
cba1cce0	127	.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
4ea1277d JG	128	}, {
	129	.num_blocks = 1,
	130	.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
	131	} }
	132	};
	133
	134	static const struct common_glue_ctx cast6_dec_cbc = {
	135	.num_funcs = 2,
	136	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	137
	138	.funcs = { {
	139	.num_blocks = CAST6_PARALLEL_BLOCKS,
cba1cce0	140	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
4ea1277d JG	141	}, {
	142	.num_blocks = 1,
	143	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
	144	} }
	145	};
	146
70177286 JK	147	static const struct common_glue_ctx cast6_dec_xts = {
	148	.num_funcs = 2,
	149	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
	150
	151	.funcs = { {
	152	.num_blocks = CAST6_PARALLEL_BLOCKS,
	153	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) }
	154	}, {
	155	.num_blocks = 1,
	156	.fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) }
	157	} }
	158	};
	159
4ea1277d JG	160	static int ecb_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	161	struct scatterlist *src, unsigned int nbytes)
	162	{
	163	return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
	164	}
	165
	166	static int ecb_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	167	struct scatterlist *src, unsigned int nbytes)
	168	{
	169	return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
	170	}
	171
	172	static int cbc_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	173	struct scatterlist *src, unsigned int nbytes)
	174	{
	175	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
	176	dst, src, nbytes);
	177	}
	178
	179	static int cbc_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	180	struct scatterlist *src, unsigned int nbytes)
	181	{
	182	return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
	183	nbytes);
	184	}
	185
	186	static int ctr_crypt(struct blkcipher_desc desc, struct scatterlist dst,
	187	struct scatterlist *src, unsigned int nbytes)
	188	{
	189	return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
	190	}
	191
	192	static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
	193	{
	194	return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
	195	NULL, fpu_enabled, nbytes);
	196	}
	197
	198	static inline void cast6_fpu_end(bool fpu_enabled)
	199	{
	200	glue_fpu_end(fpu_enabled);
	201	}
	202
	203	struct crypt_priv {
	204	struct cast6_ctx *ctx;
	205	bool fpu_enabled;
	206	};
	207
	208	static void encrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	209	{
	210	const unsigned int bsize = CAST6_BLOCK_SIZE;
	211	struct crypt_priv *ctx = priv;
	212	int i;
	213
	214	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
	215
	216	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
cba1cce0	217	cast6_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
4ea1277d JG	218	return;
	219	}
	220
	221	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	222	__cast6_encrypt(ctx->ctx, srcdst, srcdst);
	223	}
	224
	225	static void decrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	226	{
	227	const unsigned int bsize = CAST6_BLOCK_SIZE;
	228	struct crypt_priv *ctx = priv;
	229	int i;
	230
	231	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
	232
	233	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
cba1cce0	234	cast6_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
4ea1277d JG	235	return;
	236	}
	237
	238	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	239	__cast6_decrypt(ctx->ctx, srcdst, srcdst);
	240	}
	241
	242	struct cast6_lrw_ctx {
	243	struct lrw_table_ctx lrw_table;
	244	struct cast6_ctx cast6_ctx;
	245	};
	246
	247	static int lrw_cast6_setkey(struct crypto_tfm tfm, const u8 key,
	248	unsigned int keylen)
	249	{
	250	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	251	int err;
	252
	253	err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
	254	&tfm->crt_flags);
	255	if (err)
	256	return err;
	257
	258	return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
	259	}
	260
	261	static int lrw_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	262	struct scatterlist *src, unsigned int nbytes)
	263	{
	264	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	265	be128 buf[CAST6_PARALLEL_BLOCKS];
	266	struct crypt_priv crypt_ctx = {
	267	.ctx = &ctx->cast6_ctx,
	268	.fpu_enabled = false,
	269	};
	270	struct lrw_crypt_req req = {
	271	.tbuf = buf,
	272	.tbuflen = sizeof(buf),
	273
	274	.table_ctx = &ctx->lrw_table,
	275	.crypt_ctx = &crypt_ctx,
	276	.crypt_fn = encrypt_callback,
	277	};
	278	int ret;
	279
	280	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	281	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	282	cast6_fpu_end(crypt_ctx.fpu_enabled);
	283
	284	return ret;
	285	}
	286
	287	static int lrw_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	288	struct scatterlist *src, unsigned int nbytes)
	289	{
	290	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	291	be128 buf[CAST6_PARALLEL_BLOCKS];
	292	struct crypt_priv crypt_ctx = {
	293	.ctx = &ctx->cast6_ctx,
	294	.fpu_enabled = false,
	295	};
	296	struct lrw_crypt_req req = {
	297	.tbuf = buf,
	298	.tbuflen = sizeof(buf),
299
300	.table_ctx = &ctx->lrw_table,
301	.crypt_ctx = &crypt_ctx,
302	.crypt_fn = decrypt_callback,
303	};
304	int ret;
305
306	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
307	ret = lrw_crypt(desc, dst, src, nbytes, &req);
308	cast6_fpu_end(crypt_ctx.fpu_enabled);
309
310	return ret;
311	}
312
313	static void lrw_exit_tfm(struct crypto_tfm *tfm)
314	{
315	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
316
317	lrw_free_table(&ctx->lrw_table);
318	}
319
320	struct cast6_xts_ctx {
321	struct cast6_ctx tweak_ctx;
322	struct cast6_ctx crypt_ctx;
323	};
324
325	static int xts_cast6_setkey(struct crypto_tfm tfm, const u8 key,
326	unsigned int keylen)
327	{
328	struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
329	u32 *flags = &tfm->crt_flags;
330	int err;
331
28856a9e SM	332	err = xts_check_key(tfm, key, keylen);
	333	if (err)
	334	return err;
4ea1277d JG	335
	336	/* first half of xts-key is for crypt */
	337	err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
	338	if (err)
	339	return err;
	340
	341	/* second half of xts-key is for tweak */
	342	return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
	343	flags);
	344	}
	345
	346	static int xts_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	347	struct scatterlist *src, unsigned int nbytes)
	348	{
	349	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
4ea1277d	350
70177286 JK	351	return glue_xts_crypt_128bit(&cast6_enc_xts, desc, dst, src, nbytes,
	352	XTS_TWEAK_CAST(__cast6_encrypt),
	353	&ctx->tweak_ctx, &ctx->crypt_ctx);
4ea1277d JG	354	}
	355
	356	static int xts_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	357	struct scatterlist *src, unsigned int nbytes)
	358	{
	359	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
4ea1277d	360
70177286 JK	361	return glue_xts_crypt_128bit(&cast6_dec_xts, desc, dst, src, nbytes,
	362	XTS_TWEAK_CAST(__cast6_encrypt),
	363	&ctx->tweak_ctx, &ctx->crypt_ctx);
4ea1277d JG	364	}
	365
	366	static struct crypto_alg cast6_algs[10] = { {
	367	.cra_name = "__ecb-cast6-avx",
	368	.cra_driver_name = "__driver-ecb-cast6-avx",
	369	.cra_priority = 0,
e69b8a46 SM	370	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	371	CRYPTO_ALG_INTERNAL,
4ea1277d JG	372	.cra_blocksize = CAST6_BLOCK_SIZE,
	373	.cra_ctxsize = sizeof(struct cast6_ctx),
	374	.cra_alignmask = 0,
	375	.cra_type = &crypto_blkcipher_type,
	376	.cra_module = THIS_MODULE,
	377	.cra_u = {
	378	.blkcipher = {
	379	.min_keysize = CAST6_MIN_KEY_SIZE,
	380	.max_keysize = CAST6_MAX_KEY_SIZE,
	381	.setkey = cast6_setkey,
	382	.encrypt = ecb_encrypt,
	383	.decrypt = ecb_decrypt,
	384	},
	385	},
	386	}, {
	387	.cra_name = "__cbc-cast6-avx",
	388	.cra_driver_name = "__driver-cbc-cast6-avx",
	389	.cra_priority = 0,
e69b8a46 SM	390	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	391	CRYPTO_ALG_INTERNAL,
4ea1277d JG	392	.cra_blocksize = CAST6_BLOCK_SIZE,
	393	.cra_ctxsize = sizeof(struct cast6_ctx),
	394	.cra_alignmask = 0,
	395	.cra_type = &crypto_blkcipher_type,
	396	.cra_module = THIS_MODULE,
	397	.cra_u = {
	398	.blkcipher = {
	399	.min_keysize = CAST6_MIN_KEY_SIZE,
	400	.max_keysize = CAST6_MAX_KEY_SIZE,
	401	.setkey = cast6_setkey,
	402	.encrypt = cbc_encrypt,
	403	.decrypt = cbc_decrypt,
	404	},
	405	},
	406	}, {
	407	.cra_name = "__ctr-cast6-avx",
	408	.cra_driver_name = "__driver-ctr-cast6-avx",
	409	.cra_priority = 0,
e69b8a46 SM	410	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	411	CRYPTO_ALG_INTERNAL,
4ea1277d JG	412	.cra_blocksize = 1,
	413	.cra_ctxsize = sizeof(struct cast6_ctx),
	414	.cra_alignmask = 0,
	415	.cra_type = &crypto_blkcipher_type,
	416	.cra_module = THIS_MODULE,
	417	.cra_u = {
	418	.blkcipher = {
	419	.min_keysize = CAST6_MIN_KEY_SIZE,
	420	.max_keysize = CAST6_MAX_KEY_SIZE,
	421	.ivsize = CAST6_BLOCK_SIZE,
	422	.setkey = cast6_setkey,
	423	.encrypt = ctr_crypt,
	424	.decrypt = ctr_crypt,
	425	},
	426	},
	427	}, {
	428	.cra_name = "__lrw-cast6-avx",
	429	.cra_driver_name = "__driver-lrw-cast6-avx",
	430	.cra_priority = 0,
e69b8a46 SM	431	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	432	CRYPTO_ALG_INTERNAL,
4ea1277d JG	433	.cra_blocksize = CAST6_BLOCK_SIZE,
	434	.cra_ctxsize = sizeof(struct cast6_lrw_ctx),
	435	.cra_alignmask = 0,
	436	.cra_type = &crypto_blkcipher_type,
	437	.cra_module = THIS_MODULE,
	438	.cra_exit = lrw_exit_tfm,
	439	.cra_u = {
	440	.blkcipher = {
	441	.min_keysize = CAST6_MIN_KEY_SIZE +
	442	CAST6_BLOCK_SIZE,
	443	.max_keysize = CAST6_MAX_KEY_SIZE +
	444	CAST6_BLOCK_SIZE,
	445	.ivsize = CAST6_BLOCK_SIZE,
	446	.setkey = lrw_cast6_setkey,
	447	.encrypt = lrw_encrypt,
	448	.decrypt = lrw_decrypt,
	449	},
	450	},
	451	}, {
	452	.cra_name = "__xts-cast6-avx",
	453	.cra_driver_name = "__driver-xts-cast6-avx",
	454	.cra_priority = 0,
e69b8a46 SM	455	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER \|
e69b8a46 SM	456	CRYPTO_ALG_INTERNAL,
4ea1277d JG	457	.cra_blocksize = CAST6_BLOCK_SIZE,
	458	.cra_ctxsize = sizeof(struct cast6_xts_ctx),
	459	.cra_alignmask = 0,
	460	.cra_type = &crypto_blkcipher_type,
	461	.cra_module = THIS_MODULE,
	462	.cra_u = {
	463	.blkcipher = {
	464	.min_keysize = CAST6_MIN_KEY_SIZE * 2,
	465	.max_keysize = CAST6_MAX_KEY_SIZE * 2,
	466	.ivsize = CAST6_BLOCK_SIZE,
	467	.setkey = xts_cast6_setkey,
	468	.encrypt = xts_encrypt,
	469	.decrypt = xts_decrypt,
	470	},
	471	},
	472	}, {
	473	.cra_name = "ecb(cast6)",
	474	.cra_driver_name = "ecb-cast6-avx",
	475	.cra_priority = 200,
	476	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	477	.cra_blocksize = CAST6_BLOCK_SIZE,
	478	.cra_ctxsize = sizeof(struct async_helper_ctx),
	479	.cra_alignmask = 0,
	480	.cra_type = &crypto_ablkcipher_type,
	481	.cra_module = THIS_MODULE,
	482	.cra_init = ablk_init,
	483	.cra_exit = ablk_exit,
	484	.cra_u = {
	485	.ablkcipher = {
	486	.min_keysize = CAST6_MIN_KEY_SIZE,
	487	.max_keysize = CAST6_MAX_KEY_SIZE,
	488	.setkey = ablk_set_key,
	489	.encrypt = ablk_encrypt,
	490	.decrypt = ablk_decrypt,
	491	},
	492	},
	493	}, {
	494	.cra_name = "cbc(cast6)",
	495	.cra_driver_name = "cbc-cast6-avx",
	496	.cra_priority = 200,
	497	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	498	.cra_blocksize = CAST6_BLOCK_SIZE,
	499	.cra_ctxsize = sizeof(struct async_helper_ctx),
	500	.cra_alignmask = 0,
	501	.cra_type = &crypto_ablkcipher_type,
	502	.cra_module = THIS_MODULE,
	503	.cra_init = ablk_init,
	504	.cra_exit = ablk_exit,
	505	.cra_u = {
	506	.ablkcipher = {
	507	.min_keysize = CAST6_MIN_KEY_SIZE,
	508	.max_keysize = CAST6_MAX_KEY_SIZE,
	509	.ivsize = CAST6_BLOCK_SIZE,
	510	.setkey = ablk_set_key,
	511	.encrypt = __ablk_encrypt,
	512	.decrypt = ablk_decrypt,
	513	},
	514	},
	515	}, {
	516	.cra_name = "ctr(cast6)",
	517	.cra_driver_name = "ctr-cast6-avx",
	518	.cra_priority = 200,
	519	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	520	.cra_blocksize = 1,
521	.cra_ctxsize = sizeof(struct async_helper_ctx),
522	.cra_alignmask = 0,
523	.cra_type = &crypto_ablkcipher_type,
524	.cra_module = THIS_MODULE,
525	.cra_init = ablk_init,
526	.cra_exit = ablk_exit,
527	.cra_u = {
528	.ablkcipher = {
529	.min_keysize = CAST6_MIN_KEY_SIZE,
530	.max_keysize = CAST6_MAX_KEY_SIZE,
531	.ivsize = CAST6_BLOCK_SIZE,
532	.setkey = ablk_set_key,
533	.encrypt = ablk_encrypt,
534	.decrypt = ablk_encrypt,
535	.geniv = "chainiv",
536	},
537	},
538	}, {
539	.cra_name = "lrw(cast6)",
540	.cra_driver_name = "lrw-cast6-avx",
541	.cra_priority = 200,
542	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
543	.cra_blocksize = CAST6_BLOCK_SIZE,
544	.cra_ctxsize = sizeof(struct async_helper_ctx),
545	.cra_alignmask = 0,
546	.cra_type = &crypto_ablkcipher_type,
547	.cra_module = THIS_MODULE,
548	.cra_init = ablk_init,
549	.cra_exit = ablk_exit,
550	.cra_u = {
551	.ablkcipher = {
552	.min_keysize = CAST6_MIN_KEY_SIZE +
553	CAST6_BLOCK_SIZE,
554	.max_keysize = CAST6_MAX_KEY_SIZE +
555	CAST6_BLOCK_SIZE,
556	.ivsize = CAST6_BLOCK_SIZE,
557	.setkey = ablk_set_key,
558	.encrypt = ablk_encrypt,
559	.decrypt = ablk_decrypt,
560	},
561	},
562	}, {
563	.cra_name = "xts(cast6)",
564	.cra_driver_name = "xts-cast6-avx",
565	.cra_priority = 200,
566	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
567	.cra_blocksize = CAST6_BLOCK_SIZE,
568	.cra_ctxsize = sizeof(struct async_helper_ctx),
569	.cra_alignmask = 0,
570	.cra_type = &crypto_ablkcipher_type,
571	.cra_module = THIS_MODULE,
572	.cra_init = ablk_init,
573	.cra_exit = ablk_exit,
574	.cra_u = {
575	.ablkcipher = {
576	.min_keysize = CAST6_MIN_KEY_SIZE * 2,
577	.max_keysize = CAST6_MAX_KEY_SIZE * 2,
578	.ivsize = CAST6_BLOCK_SIZE,
579	.setkey = ablk_set_key,
580	.encrypt = ablk_encrypt,
581	.decrypt = ablk_decrypt,
582	},
583	},
584	} };
585
586	static int __init cast6_init(void)
587	{
1debf7db	588	const char *feature_name;
4ea1277d	589
d91cab78 DH	590	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE \| XFEATURE_MASK_YMM,
d91cab78 DH	591	&feature_name)) {
1debf7db	592	pr_info("CPU feature '%s' is not supported.\n", feature_name);
4ea1277d JG	593	return -ENODEV;
	594	}
	595
	596	return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
	597	}
	598
	599	static void __exit cast6_exit(void)
	600	{
	601	crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
	602	}
	603
	604	module_init(cast6_init);
	605	module_exit(cast6_exit);
	606
	607	MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
	608	MODULE_LICENSE("GPL");
5d26a105	609	MODULE_ALIAS_CRYPTO("cast6");