[deliverable/linux.git] / arch / x86 / crypto / serpent_sse2_glue.c

/*
 * Glue Code for SSE2 assembler versions of Serpent Cipher
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * Glue code based on aesni-intel_glue.c by:
 *  Copyright (C) 2008, Intel Corp.
 *    Author: Huang Ying <ying.huang@intel.com>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/hardirq.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/ablk_helper.h>
#include <crypto/algapi.h>
#include <crypto/serpent.h>
#include <crypto/cryptd.h>
#include <crypto/b128ops.h>
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/crypto/serpent-sse2.h>
#include <asm/crypto/glue_helper.h>

static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
{
	u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
	unsigned int j;

	for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
		ivs[j] = src[j];

	serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);

	for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
		u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
}

static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
	be128 ctrblk;

	le128_to_be128(&ctrblk, iv);
	le128_inc(iv);

	__serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
	u128_xor(dst, src, (u128 *)&ctrblk);
}

static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
				   le128 *iv)
{
	be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
	unsigned int i;

	for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
		if (dst != src)
			dst[i] = src[i];

		le128_to_be128(&ctrblks[i], iv);
		le128_inc(iv);
	}

	serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
}

static const struct common_glue_ctx serpent_enc = {
	.num_funcs = 2,
	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = SERPENT_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
	} }
};

static const struct common_glue_ctx serpent_ctr = {
	.num_funcs = 2,
	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = SERPENT_PARALLEL_BLOCKS,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
	} }
};

static const struct common_glue_ctx serpent_dec = {
	.num_funcs = 2,
	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = SERPENT_PARALLEL_BLOCKS,
		.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
	}, {
		.num_blocks = 1,
		.fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
	} }
};

static const struct common_glue_ctx serpent_dec_cbc = {
	.num_funcs = 2,
	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

	.funcs = { {
		.num_blocks = SERPENT_PARALLEL_BLOCKS,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
	}, {
		.num_blocks = 1,
		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
	} }
};

static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
}

static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
}

static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
				     dst, src, nbytes);
}

static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
				       nbytes);
}

static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		     struct scatterlist *src, unsigned int nbytes)
{
	return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
}

static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
{
	return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
			      NULL, fpu_enabled, nbytes);
}

static inline void serpent_fpu_end(bool fpu_enabled)
{
	glue_fpu_end(fpu_enabled);
}

struct crypt_priv {
	struct serpent_ctx *ctx;
	bool fpu_enabled;
};

static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = SERPENT_BLOCK_SIZE;
	struct crypt_priv *ctx = priv;
	int i;

	ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);

	if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
		serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		__serpent_encrypt(ctx->ctx, srcdst, srcdst);
}

static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
{
	const unsigned int bsize = SERPENT_BLOCK_SIZE;
	struct crypt_priv *ctx = priv;
	int i;

	ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);

	if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
		serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
		return;
	}

	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
		__serpent_decrypt(ctx->ctx, srcdst, srcdst);
}

struct serpent_lrw_ctx {
	struct lrw_table_ctx lrw_table;
	struct serpent_ctx serpent_ctx;
};

static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	int err;

	err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
							SERPENT_BLOCK_SIZE);
	if (err)
		return err;

	return lrw_init_table(&ctx->lrw_table, key + keylen -
						SERPENT_BLOCK_SIZE);
}

static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[SERPENT_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->serpent_ctx,
		.fpu_enabled = false,
	};
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = encrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	serpent_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[SERPENT_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->serpent_ctx,
		.fpu_enabled = false,
	};
	struct lrw_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.table_ctx = &ctx->lrw_table,
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = decrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	serpent_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static void lrw_exit_tfm(struct crypto_tfm *tfm)
{
	struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);

	lrw_free_table(&ctx->lrw_table);
}

struct serpent_xts_ctx {
	struct serpent_ctx tweak_ctx;
	struct serpent_ctx crypt_ctx;
};

static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
			      unsigned int keylen)
{
	struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
	u32 *flags = &tfm->crt_flags;
	int err;

	/* key consists of keys of equal size concatenated, therefore
	 * the length must be even
	 */
	if (keylen % 2) {
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		return -EINVAL;
	}

	/* first half of xts-key is for crypt */
	err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
	if (err)
		return err;

	/* second half of xts-key is for tweak */
	return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
}

static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[SERPENT_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->crypt_ctx,
		.fpu_enabled = false,
	};
	struct xts_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.tweak_ctx = &ctx->tweak_ctx,
		.tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = encrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = xts_crypt(desc, dst, src, nbytes, &req);
	serpent_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
		       struct scatterlist *src, unsigned int nbytes)
{
	struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	be128 buf[SERPENT_PARALLEL_BLOCKS];
	struct crypt_priv crypt_ctx = {
		.ctx = &ctx->crypt_ctx,
		.fpu_enabled = false,
	};
	struct xts_crypt_req req = {
		.tbuf = buf,
		.tbuflen = sizeof(buf),

		.tweak_ctx = &ctx->tweak_ctx,
		.tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
		.crypt_ctx = &crypt_ctx,
		.crypt_fn = decrypt_callback,
	};
	int ret;

	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	ret = xts_crypt(desc, dst, src, nbytes, &req);
	serpent_fpu_end(crypt_ctx.fpu_enabled);

	return ret;
}

static struct crypto_alg serpent_algs[10] = { {
	.cra_name		= "__ecb-serpent-sse2",
	.cra_driver_name	= "__driver-ecb-serpent-sse2",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= SERPENT_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct serpent_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE,
			.max_keysize	= SERPENT_MAX_KEY_SIZE,
			.setkey		= serpent_setkey,
			.encrypt	= ecb_encrypt,
			.decrypt	= ecb_decrypt,
		},
	},
}, {
	.cra_name		= "__cbc-serpent-sse2",
	.cra_driver_name	= "__driver-cbc-serpent-sse2",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= SERPENT_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct serpent_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE,
			.max_keysize	= SERPENT_MAX_KEY_SIZE,
			.setkey		= serpent_setkey,
			.encrypt	= cbc_encrypt,
			.decrypt	= cbc_decrypt,
		},
	},
}, {
	.cra_name		= "__ctr-serpent-sse2",
	.cra_driver_name	= "__driver-ctr-serpent-sse2",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct serpent_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE,
			.max_keysize	= SERPENT_MAX_KEY_SIZE,
			.ivsize		= SERPENT_BLOCK_SIZE,
			.setkey		= serpent_setkey,
			.encrypt	= ctr_crypt,
			.decrypt	= ctr_crypt,
		},
	},
}, {
	.cra_name		= "__lrw-serpent-sse2",
	.cra_driver_name	= "__driver-lrw-serpent-sse2",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= SERPENT_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct serpent_lrw_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_exit		= lrw_exit_tfm,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE +
					  SERPENT_BLOCK_SIZE,
			.max_keysize	= SERPENT_MAX_KEY_SIZE +
					  SERPENT_BLOCK_SIZE,
			.ivsize		= SERPENT_BLOCK_SIZE,
			.setkey		= lrw_serpent_setkey,
			.encrypt	= lrw_encrypt,
			.decrypt	= lrw_decrypt,
		},
	},
}, {
	.cra_name		= "__xts-serpent-sse2",
	.cra_driver_name	= "__driver-xts-serpent-sse2",
	.cra_priority		= 0,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= SERPENT_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct serpent_xts_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE * 2,
			.max_keysize	= SERPENT_MAX_KEY_SIZE * 2,
			.ivsize		= SERPENT_BLOCK_SIZE,
			.setkey		= xts_serpent_setkey,
			.encrypt	= xts_encrypt,
			.decrypt	= xts_decrypt,
		},
	},
}, {
	.cra_name		= "ecb(serpent)",
	.cra_driver_name	= "ecb-serpent-sse2",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= SERPENT_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE,
			.max_keysize	= SERPENT_MAX_KEY_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "cbc(serpent)",
	.cra_driver_name	= "cbc-serpent-sse2",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= SERPENT_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE,
			.max_keysize	= SERPENT_MAX_KEY_SIZE,
			.ivsize		= SERPENT_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= __ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "ctr(serpent)",
	.cra_driver_name	= "ctr-serpent-sse2",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= 1,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE,
			.max_keysize	= SERPENT_MAX_KEY_SIZE,
			.ivsize		= SERPENT_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_encrypt,
			.geniv		= "chainiv",
		},
	},
}, {
	.cra_name		= "lrw(serpent)",
	.cra_driver_name	= "lrw-serpent-sse2",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= SERPENT_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE +
					  SERPENT_BLOCK_SIZE,
			.max_keysize	= SERPENT_MAX_KEY_SIZE +
					  SERPENT_BLOCK_SIZE,
			.ivsize		= SERPENT_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
}, {
	.cra_name		= "xts(serpent)",
	.cra_driver_name	= "xts-serpent-sse2",
	.cra_priority		= 400,
	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
	.cra_blocksize		= SERPENT_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct async_helper_ctx),
	.cra_alignmask		= 0,
	.cra_type		= &crypto_ablkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_init		= ablk_init,
	.cra_exit		= ablk_exit,
	.cra_u = {
		.ablkcipher = {
			.min_keysize	= SERPENT_MIN_KEY_SIZE * 2,
			.max_keysize	= SERPENT_MAX_KEY_SIZE * 2,
			.ivsize		= SERPENT_BLOCK_SIZE,
			.setkey		= ablk_set_key,
			.encrypt	= ablk_encrypt,
			.decrypt	= ablk_decrypt,
		},
	},
} };

static int __init serpent_sse2_init(void)
{
	if (!cpu_has_xmm2) {
		printk(KERN_INFO "SSE2 instructions are not detected.\n");
		return -ENODEV;
	}

	return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
}

static void __exit serpent_sse2_exit(void)
{
	crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
}

module_init(serpent_sse2_init);
module_exit(serpent_sse2_exit);

MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("serpent");
Commit	Line	Data
937c30d7 JK	1	/*
	2	* Glue Code for SSE2 assembler versions of Serpent Cipher
	3	*
	4	* Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
	5	*
	6	* Glue code based on aesni-intel_glue.c by:
	7	* Copyright (C) 2008, Intel Corp.
	8	* Author: Huang Ying <ying.huang@intel.com>
	9	*
	10	* CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
	11	* Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
	12	* CTR part based on code (crypto/ctr.c) by:
	13	* (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
	14	*
	15	* This program is free software; you can redistribute it and/or modify
	16	* it under the terms of the GNU General Public License as published by
	17	* the Free Software Foundation; either version 2 of the License, or
	18	* (at your option) any later version.
	19	*
	20	* This program is distributed in the hope that it will be useful,
	21	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	22	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	23	* GNU General Public License for more details.
	24	*
	25	* You should have received a copy of the GNU General Public License
	26	* along with this program; if not, write to the Free Software
	27	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	28	* USA
	29	*
	30	*/
	31
	32	#include <linux/module.h>
	33	#include <linux/hardirq.h>
	34	#include <linux/types.h>
	35	#include <linux/crypto.h>
	36	#include <linux/err.h>
801201aa	37	#include <crypto/ablk_helper.h>
937c30d7 JK	38	#include <crypto/algapi.h>
	39	#include <crypto/serpent.h>
	40	#include <crypto/cryptd.h>
	41	#include <crypto/b128ops.h>
	42	#include <crypto/ctr.h>
18482053	43	#include <crypto/lrw.h>
5962f8b6	44	#include <crypto/xts.h>
d4af0e9d	45	#include <asm/crypto/serpent-sse2.h>
596d8750	46	#include <asm/crypto/glue_helper.h>
937c30d7	47
e81792fb JK	48	static void serpent_decrypt_cbc_xway(void ctx, u128 dst, const u128 *src)
	49	{
	50	u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
	51	unsigned int j;
	52
	53	for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
	54	ivs[j] = src[j];
	55
	56	serpent_dec_blk_xway(ctx, (u8 )dst, (u8 )src);
	57
	58	for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
	59	u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
	60	}
	61
58990986	62	static void serpent_crypt_ctr(void ctx, u128 dst, const u128 src, le128 iv)
e81792fb JK	63	{
	64	be128 ctrblk;
	65
58990986 JK	66	le128_to_be128(&ctrblk, iv);
58990986 JK	67	le128_inc(iv);
e81792fb JK	68
	69	__serpent_encrypt(ctx, (u8 )&ctrblk, (u8 )&ctrblk);
	70	u128_xor(dst, src, (u128 *)&ctrblk);
	71	}
	72
	73	static void serpent_crypt_ctr_xway(void ctx, u128 dst, const u128 *src,
58990986	74	le128 *iv)
e81792fb JK	75	{
	76	be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
	77	unsigned int i;
	78
	79	for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
	80	if (dst != src)
	81	dst[i] = src[i];
	82
58990986 JK	83	le128_to_be128(&ctrblks[i], iv);
58990986 JK	84	le128_inc(iv);
e81792fb JK	85	}
	86
	87	serpent_enc_blk_xway_xor(ctx, (u8 )dst, (u8 )ctrblks);
	88	}
	89
	90	static const struct common_glue_ctx serpent_enc = {
	91	.num_funcs = 2,
	92	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
	93
	94	.funcs = { {
	95	.num_blocks = SERPENT_PARALLEL_BLOCKS,
	96	.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
	97	}, {
	98	.num_blocks = 1,
	99	.fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
	100	} }
	101	};
	102
	103	static const struct common_glue_ctx serpent_ctr = {
	104	.num_funcs = 2,
	105	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
	106
	107	.funcs = { {
	108	.num_blocks = SERPENT_PARALLEL_BLOCKS,
	109	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
	110	}, {
	111	.num_blocks = 1,
	112	.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
	113	} }
	114	};
	115
	116	static const struct common_glue_ctx serpent_dec = {
	117	.num_funcs = 2,
	118	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
	119
	120	.funcs = { {
	121	.num_blocks = SERPENT_PARALLEL_BLOCKS,
	122	.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
	123	}, {
	124	.num_blocks = 1,
	125	.fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
	126	} }
	127	};
	128
	129	static const struct common_glue_ctx serpent_dec_cbc = {
	130	.num_funcs = 2,
	131	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
	132
	133	.funcs = { {
	134	.num_blocks = SERPENT_PARALLEL_BLOCKS,
	135	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
	136	}, {
	137	.num_blocks = 1,
	138	.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
	139	} }
	140	};
	141
	142	static int ecb_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	143	struct scatterlist *src, unsigned int nbytes)
	144	{
	145	return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
	146	}
	147
	148	static int ecb_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
149	struct scatterlist *src, unsigned int nbytes)
150	{
151	return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
152	}
153
154	static int cbc_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
155	struct scatterlist *src, unsigned int nbytes)
156	{
157	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
158	dst, src, nbytes);
159	}
160
161	static int cbc_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
162	struct scatterlist *src, unsigned int nbytes)
163	{
164	return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
165	nbytes);
166	}
167
168	static int ctr_crypt(struct blkcipher_desc desc, struct scatterlist dst,
169	struct scatterlist *src, unsigned int nbytes)
170	{
171	return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
172	}
173
174	static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
175	{
176	return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
177	NULL, fpu_enabled, nbytes);
178	}
179
180	static inline void serpent_fpu_end(bool fpu_enabled)
181	{
182	glue_fpu_end(fpu_enabled);
183	}
184
18482053 JK	185	struct crypt_priv {
	186	struct serpent_ctx *ctx;
	187	bool fpu_enabled;
	188	};
	189
	190	static void encrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	191	{
	192	const unsigned int bsize = SERPENT_BLOCK_SIZE;
	193	struct crypt_priv *ctx = priv;
	194	int i;
	195
	196	ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
	197
	198	if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
	199	serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
	200	return;
	201	}
	202
	203	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	204	__serpent_encrypt(ctx->ctx, srcdst, srcdst);
	205	}
	206
	207	static void decrypt_callback(void priv, u8 srcdst, unsigned int nbytes)
	208	{
	209	const unsigned int bsize = SERPENT_BLOCK_SIZE;
	210	struct crypt_priv *ctx = priv;
	211	int i;
	212
	213	ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
	214
	215	if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
	216	serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
	217	return;
	218	}
	219
	220	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
	221	__serpent_decrypt(ctx->ctx, srcdst, srcdst);
	222	}
	223
	224	struct serpent_lrw_ctx {
	225	struct lrw_table_ctx lrw_table;
	226	struct serpent_ctx serpent_ctx;
	227	};
	228
	229	static int lrw_serpent_setkey(struct crypto_tfm tfm, const u8 key,
	230	unsigned int keylen)
	231	{
	232	struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	233	int err;
	234
	235	err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
	236	SERPENT_BLOCK_SIZE);
	237	if (err)
	238	return err;
	239
	240	return lrw_init_table(&ctx->lrw_table, key + keylen -
	241	SERPENT_BLOCK_SIZE);
	242	}
	243
	244	static int lrw_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	245	struct scatterlist *src, unsigned int nbytes)
	246	{
	247	struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	248	be128 buf[SERPENT_PARALLEL_BLOCKS];
249	struct crypt_priv crypt_ctx = {
250	.ctx = &ctx->serpent_ctx,
251	.fpu_enabled = false,
252	};
253	struct lrw_crypt_req req = {
254	.tbuf = buf,
255	.tbuflen = sizeof(buf),
256
257	.table_ctx = &ctx->lrw_table,
258	.crypt_ctx = &crypt_ctx,
259	.crypt_fn = encrypt_callback,
260	};
261	int ret;
262
d3564338	263	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
18482053 JK	264	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	265	serpent_fpu_end(crypt_ctx.fpu_enabled);
	266
	267	return ret;
	268	}
	269
	270	static int lrw_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	271	struct scatterlist *src, unsigned int nbytes)
	272	{
	273	struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	274	be128 buf[SERPENT_PARALLEL_BLOCKS];
	275	struct crypt_priv crypt_ctx = {
	276	.ctx = &ctx->serpent_ctx,
	277	.fpu_enabled = false,
	278	};
	279	struct lrw_crypt_req req = {
	280	.tbuf = buf,
	281	.tbuflen = sizeof(buf),
	282
	283	.table_ctx = &ctx->lrw_table,
	284	.crypt_ctx = &crypt_ctx,
	285	.crypt_fn = decrypt_callback,
	286	};
	287	int ret;
	288
d3564338	289	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
18482053 JK	290	ret = lrw_crypt(desc, dst, src, nbytes, &req);
	291	serpent_fpu_end(crypt_ctx.fpu_enabled);
	292
	293	return ret;
	294	}
	295
	296	static void lrw_exit_tfm(struct crypto_tfm *tfm)
	297	{
	298	struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
	299
	300	lrw_free_table(&ctx->lrw_table);
	301	}
	302
5962f8b6 JK	303	struct serpent_xts_ctx {
	304	struct serpent_ctx tweak_ctx;
	305	struct serpent_ctx crypt_ctx;
	306	};
	307
	308	static int xts_serpent_setkey(struct crypto_tfm tfm, const u8 key,
	309	unsigned int keylen)
	310	{
	311	struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
	312	u32 *flags = &tfm->crt_flags;
	313	int err;
	314
	315	/* key consists of keys of equal size concatenated, therefore
	316	* the length must be even
	317	*/
	318	if (keylen % 2) {
	319	*flags \|= CRYPTO_TFM_RES_BAD_KEY_LEN;
	320	return -EINVAL;
	321	}
	322
	323	/* first half of xts-key is for crypt */
	324	err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
	325	if (err)
	326	return err;
	327
	328	/* second half of xts-key is for tweak */
	329	return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
	330	}
	331
	332	static int xts_encrypt(struct blkcipher_desc desc, struct scatterlist dst,
	333	struct scatterlist *src, unsigned int nbytes)
	334	{
	335	struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	336	be128 buf[SERPENT_PARALLEL_BLOCKS];
	337	struct crypt_priv crypt_ctx = {
	338	.ctx = &ctx->crypt_ctx,
	339	.fpu_enabled = false,
	340	};
	341	struct xts_crypt_req req = {
	342	.tbuf = buf,
	343	.tbuflen = sizeof(buf),
	344
	345	.tweak_ctx = &ctx->tweak_ctx,
	346	.tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
	347	.crypt_ctx = &crypt_ctx,
	348	.crypt_fn = encrypt_callback,
	349	};
	350	int ret;
	351
d3564338	352	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
5962f8b6 JK	353	ret = xts_crypt(desc, dst, src, nbytes, &req);
	354	serpent_fpu_end(crypt_ctx.fpu_enabled);
	355
	356	return ret;
	357	}
	358
	359	static int xts_decrypt(struct blkcipher_desc desc, struct scatterlist dst,
	360	struct scatterlist *src, unsigned int nbytes)
	361	{
	362	struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	363	be128 buf[SERPENT_PARALLEL_BLOCKS];
	364	struct crypt_priv crypt_ctx = {
	365	.ctx = &ctx->crypt_ctx,
	366	.fpu_enabled = false,
	367	};
	368	struct xts_crypt_req req = {
	369	.tbuf = buf,
	370	.tbuflen = sizeof(buf),
	371
	372	.tweak_ctx = &ctx->tweak_ctx,
	373	.tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
	374	.crypt_ctx = &crypt_ctx,
	375	.crypt_fn = decrypt_callback,
	376	};
	377	int ret;
	378
d3564338	379	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
5962f8b6 JK	380	ret = xts_crypt(desc, dst, src, nbytes, &req);
	381	serpent_fpu_end(crypt_ctx.fpu_enabled);
	382
	383	return ret;
	384	}
	385
35474c3b JK	386	static struct crypto_alg serpent_algs[10] = { {
	387	.cra_name = "__ecb-serpent-sse2",
	388	.cra_driver_name = "__driver-ecb-serpent-sse2",
	389	.cra_priority = 0,
	390	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	391	.cra_blocksize = SERPENT_BLOCK_SIZE,
	392	.cra_ctxsize = sizeof(struct serpent_ctx),
	393	.cra_alignmask = 0,
	394	.cra_type = &crypto_blkcipher_type,
	395	.cra_module = THIS_MODULE,
35474c3b JK	396	.cra_u = {
	397	.blkcipher = {
	398	.min_keysize = SERPENT_MIN_KEY_SIZE,
	399	.max_keysize = SERPENT_MAX_KEY_SIZE,
	400	.setkey = serpent_setkey,
	401	.encrypt = ecb_encrypt,
	402	.decrypt = ecb_decrypt,
	403	},
	404	},
	405	}, {
	406	.cra_name = "__cbc-serpent-sse2",
	407	.cra_driver_name = "__driver-cbc-serpent-sse2",
	408	.cra_priority = 0,
	409	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	410	.cra_blocksize = SERPENT_BLOCK_SIZE,
	411	.cra_ctxsize = sizeof(struct serpent_ctx),
	412	.cra_alignmask = 0,
	413	.cra_type = &crypto_blkcipher_type,
	414	.cra_module = THIS_MODULE,
35474c3b JK	415	.cra_u = {
	416	.blkcipher = {
	417	.min_keysize = SERPENT_MIN_KEY_SIZE,
	418	.max_keysize = SERPENT_MAX_KEY_SIZE,
	419	.setkey = serpent_setkey,
	420	.encrypt = cbc_encrypt,
	421	.decrypt = cbc_decrypt,
	422	},
	423	},
	424	}, {
	425	.cra_name = "__ctr-serpent-sse2",
	426	.cra_driver_name = "__driver-ctr-serpent-sse2",
	427	.cra_priority = 0,
	428	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	429	.cra_blocksize = 1,
	430	.cra_ctxsize = sizeof(struct serpent_ctx),
	431	.cra_alignmask = 0,
	432	.cra_type = &crypto_blkcipher_type,
	433	.cra_module = THIS_MODULE,
35474c3b JK	434	.cra_u = {
	435	.blkcipher = {
	436	.min_keysize = SERPENT_MIN_KEY_SIZE,
	437	.max_keysize = SERPENT_MAX_KEY_SIZE,
	438	.ivsize = SERPENT_BLOCK_SIZE,
	439	.setkey = serpent_setkey,
	440	.encrypt = ctr_crypt,
	441	.decrypt = ctr_crypt,
	442	},
	443	},
	444	}, {
	445	.cra_name = "__lrw-serpent-sse2",
	446	.cra_driver_name = "__driver-lrw-serpent-sse2",
	447	.cra_priority = 0,
	448	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	449	.cra_blocksize = SERPENT_BLOCK_SIZE,
	450	.cra_ctxsize = sizeof(struct serpent_lrw_ctx),
	451	.cra_alignmask = 0,
	452	.cra_type = &crypto_blkcipher_type,
	453	.cra_module = THIS_MODULE,
35474c3b JK	454	.cra_exit = lrw_exit_tfm,
	455	.cra_u = {
	456	.blkcipher = {
	457	.min_keysize = SERPENT_MIN_KEY_SIZE +
	458	SERPENT_BLOCK_SIZE,
	459	.max_keysize = SERPENT_MAX_KEY_SIZE +
	460	SERPENT_BLOCK_SIZE,
	461	.ivsize = SERPENT_BLOCK_SIZE,
	462	.setkey = lrw_serpent_setkey,
	463	.encrypt = lrw_encrypt,
	464	.decrypt = lrw_decrypt,
	465	},
	466	},
	467	}, {
	468	.cra_name = "__xts-serpent-sse2",
	469	.cra_driver_name = "__driver-xts-serpent-sse2",
	470	.cra_priority = 0,
	471	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	472	.cra_blocksize = SERPENT_BLOCK_SIZE,
	473	.cra_ctxsize = sizeof(struct serpent_xts_ctx),
	474	.cra_alignmask = 0,
	475	.cra_type = &crypto_blkcipher_type,
	476	.cra_module = THIS_MODULE,
35474c3b JK	477	.cra_u = {
	478	.blkcipher = {
	479	.min_keysize = SERPENT_MIN_KEY_SIZE * 2,
	480	.max_keysize = SERPENT_MAX_KEY_SIZE * 2,
	481	.ivsize = SERPENT_BLOCK_SIZE,
	482	.setkey = xts_serpent_setkey,
	483	.encrypt = xts_encrypt,
	484	.decrypt = xts_decrypt,
	485	},
	486	},
	487	}, {
937c30d7 JK	488	.cra_name = "ecb(serpent)",
	489	.cra_driver_name = "ecb-serpent-sse2",
	490	.cra_priority = 400,
	491	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	492	.cra_blocksize = SERPENT_BLOCK_SIZE,
ffaf9156	493	.cra_ctxsize = sizeof(struct async_helper_ctx),
937c30d7 JK	494	.cra_alignmask = 0,
	495	.cra_type = &crypto_ablkcipher_type,
	496	.cra_module = THIS_MODULE,
435d3e51	497	.cra_init = ablk_init,
937c30d7 JK	498	.cra_exit = ablk_exit,
	499	.cra_u = {
	500	.ablkcipher = {
	501	.min_keysize = SERPENT_MIN_KEY_SIZE,
	502	.max_keysize = SERPENT_MAX_KEY_SIZE,
	503	.setkey = ablk_set_key,
	504	.encrypt = ablk_encrypt,
	505	.decrypt = ablk_decrypt,
	506	},
	507	},
35474c3b	508	}, {
937c30d7 JK	509	.cra_name = "cbc(serpent)",
	510	.cra_driver_name = "cbc-serpent-sse2",
	511	.cra_priority = 400,
	512	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	513	.cra_blocksize = SERPENT_BLOCK_SIZE,
ffaf9156	514	.cra_ctxsize = sizeof(struct async_helper_ctx),
937c30d7 JK	515	.cra_alignmask = 0,
	516	.cra_type = &crypto_ablkcipher_type,
	517	.cra_module = THIS_MODULE,
435d3e51	518	.cra_init = ablk_init,
937c30d7 JK	519	.cra_exit = ablk_exit,
	520	.cra_u = {
	521	.ablkcipher = {
	522	.min_keysize = SERPENT_MIN_KEY_SIZE,
	523	.max_keysize = SERPENT_MAX_KEY_SIZE,
	524	.ivsize = SERPENT_BLOCK_SIZE,
	525	.setkey = ablk_set_key,
	526	.encrypt = __ablk_encrypt,
	527	.decrypt = ablk_decrypt,
	528	},
	529	},
35474c3b	530	}, {
937c30d7 JK	531	.cra_name = "ctr(serpent)",
	532	.cra_driver_name = "ctr-serpent-sse2",
	533	.cra_priority = 400,
	534	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	535	.cra_blocksize = 1,
ffaf9156	536	.cra_ctxsize = sizeof(struct async_helper_ctx),
937c30d7 JK	537	.cra_alignmask = 0,
	538	.cra_type = &crypto_ablkcipher_type,
	539	.cra_module = THIS_MODULE,
435d3e51	540	.cra_init = ablk_init,
937c30d7 JK	541	.cra_exit = ablk_exit,
	542	.cra_u = {
	543	.ablkcipher = {
	544	.min_keysize = SERPENT_MIN_KEY_SIZE,
	545	.max_keysize = SERPENT_MAX_KEY_SIZE,
	546	.ivsize = SERPENT_BLOCK_SIZE,
	547	.setkey = ablk_set_key,
	548	.encrypt = ablk_encrypt,
	549	.decrypt = ablk_encrypt,
	550	.geniv = "chainiv",
	551	},
	552	},
35474c3b	553	}, {
18482053 JK	554	.cra_name = "lrw(serpent)",
	555	.cra_driver_name = "lrw-serpent-sse2",
	556	.cra_priority = 400,
	557	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	558	.cra_blocksize = SERPENT_BLOCK_SIZE,
ffaf9156	559	.cra_ctxsize = sizeof(struct async_helper_ctx),
18482053 JK	560	.cra_alignmask = 0,
	561	.cra_type = &crypto_ablkcipher_type,
	562	.cra_module = THIS_MODULE,
435d3e51	563	.cra_init = ablk_init,
18482053 JK	564	.cra_exit = ablk_exit,
	565	.cra_u = {
	566	.ablkcipher = {
	567	.min_keysize = SERPENT_MIN_KEY_SIZE +
	568	SERPENT_BLOCK_SIZE,
	569	.max_keysize = SERPENT_MAX_KEY_SIZE +
	570	SERPENT_BLOCK_SIZE,
	571	.ivsize = SERPENT_BLOCK_SIZE,
	572	.setkey = ablk_set_key,
	573	.encrypt = ablk_encrypt,
	574	.decrypt = ablk_decrypt,
	575	},
	576	},
35474c3b	577	}, {
5962f8b6 JK	578	.cra_name = "xts(serpent)",
	579	.cra_driver_name = "xts-serpent-sse2",
	580	.cra_priority = 400,
	581	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \| CRYPTO_ALG_ASYNC,
	582	.cra_blocksize = SERPENT_BLOCK_SIZE,
ffaf9156	583	.cra_ctxsize = sizeof(struct async_helper_ctx),
5962f8b6 JK	584	.cra_alignmask = 0,
	585	.cra_type = &crypto_ablkcipher_type,
	586	.cra_module = THIS_MODULE,
435d3e51	587	.cra_init = ablk_init,
5962f8b6 JK	588	.cra_exit = ablk_exit,
	589	.cra_u = {
	590	.ablkcipher = {
	591	.min_keysize = SERPENT_MIN_KEY_SIZE * 2,
	592	.max_keysize = SERPENT_MAX_KEY_SIZE * 2,
	593	.ivsize = SERPENT_BLOCK_SIZE,
	594	.setkey = ablk_set_key,
	595	.encrypt = ablk_encrypt,
	596	.decrypt = ablk_decrypt,
	597	},
	598	},
35474c3b	599	} };
5962f8b6	600
937c30d7 JK	601	static int __init serpent_sse2_init(void)
937c30d7 JK	602	{
937c30d7 JK	603	if (!cpu_has_xmm2) {
	604	printk(KERN_INFO "SSE2 instructions are not detected.\n");
	605	return -ENODEV;
	606	}
	607
35474c3b	608	return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
937c30d7 JK	609	}
	610
	611	static void __exit serpent_sse2_exit(void)
	612	{
35474c3b	613	crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
937c30d7 JK	614	}
	615
	616	module_init(serpent_sse2_init);
	617	module_exit(serpent_sse2_exit);
	618
	619	MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
	620	MODULE_LICENSE("GPL");
5d26a105	621	MODULE_ALIAS_CRYPTO("serpent");