[deliverable/linux.git] / drivers / crypto / padlock-aes.c

/* 
 * Cryptographic API.
 *
 * Support for VIA PadLock hardware crypto engine.
 *
 * Copyright (c) 2004  Michal Ludvig <michal@logix.cz>
 *
 */

#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <asm/byteorder.h>
#include "padlock.h"

/* Control word. */
struct cword {
	unsigned int __attribute__ ((__packed__))
		rounds:4,
		algo:3,
		keygen:1,
		interm:1,
		encdec:1,
		ksize:2;
} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));

/* Whenever making any changes to the following
 * structure *make sure* you keep E, d_data
 * and cword aligned on 16 Bytes boundaries and
 * the Hardware can access 16 * 16 bytes of E and d_data
 * (only the first 15 * 16 bytes matter but the HW reads
 * more).
 */
struct aes_ctx {
	u32 E[AES_MAX_KEYLENGTH_U32]
		__attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
	u32 d_data[AES_MAX_KEYLENGTH_U32]
		__attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
	struct {
		struct cword encrypt;
		struct cword decrypt;
	} cword;
	u32 *D;
};

/* Tells whether the ACE is capable to generate
   the extended key for a given key_len. */
static inline int
aes_hw_extkey_available(uint8_t key_len)
{
	/* TODO: We should check the actual CPU model/stepping
	         as it's possible that the capability will be
	         added in the next CPU revisions. */
	if (key_len == 16)
		return 1;
	return 0;
}

static inline struct aes_ctx *aes_ctx_common(void *ctx)
{
	unsigned long addr = (unsigned long)ctx;
	unsigned long align = PADLOCK_ALIGNMENT;

	if (align <= crypto_tfm_ctx_alignment())
		align = 1;
	return (struct aes_ctx *)ALIGN(addr, align);
}

static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
{
	return aes_ctx_common(crypto_tfm_ctx(tfm));
}

static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
{
	return aes_ctx_common(crypto_blkcipher_ctx(tfm));
}

static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
		       unsigned int key_len)
{
	struct aes_ctx *ctx = aes_ctx(tfm);
	const __le32 *key = (const __le32 *)in_key;
	u32 *flags = &tfm->crt_flags;
	struct crypto_aes_ctx gen_aes;

	if (key_len % 8) {
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		return -EINVAL;
	}

	/*
	 * If the hardware is capable of generating the extended key
	 * itself we must supply the plain key for both encryption
	 * and decryption.
	 */
	ctx->D = ctx->E;

	ctx->E[0] = le32_to_cpu(key[0]);
	ctx->E[1] = le32_to_cpu(key[1]);
	ctx->E[2] = le32_to_cpu(key[2]);
	ctx->E[3] = le32_to_cpu(key[3]);

	/* Prepare control words. */
	memset(&ctx->cword, 0, sizeof(ctx->cword));

	ctx->cword.decrypt.encdec = 1;
	ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
	ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
	ctx->cword.encrypt.ksize = (key_len - 16) / 8;
	ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;

	/* Don't generate extended keys if the hardware can do it. */
	if (aes_hw_extkey_available(key_len))
		return 0;

	ctx->D = ctx->d_data;
	ctx->cword.encrypt.keygen = 1;
	ctx->cword.decrypt.keygen = 1;

	if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) {
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		return -EINVAL;
	}

	memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
	memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
	return 0;
}

/* ====== Encryption/decryption routines ====== */

/* These are the real call to PadLock. */
static inline void padlock_reset_key(void)
{
	asm volatile ("pushfl; popfl");
}

static inline void padlock_xcrypt(const u8 *input, u8 *output, void *key,
				  void *control_word)
{
	asm volatile (".byte 0xf3,0x0f,0xa7,0xc8"	/* rep xcryptecb */
		      : "+S"(input), "+D"(output)
		      : "d"(control_word), "b"(key), "c"(1));
}

static void aes_crypt_copy(const u8 *in, u8 *out, u32 *key, struct cword *cword)
{
	u8 buf[AES_BLOCK_SIZE * 2 + PADLOCK_ALIGNMENT - 1];
	u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);

	memcpy(tmp, in, AES_BLOCK_SIZE);
	padlock_xcrypt(tmp, out, key, cword);
}

static inline void aes_crypt(const u8 *in, u8 *out, u32 *key,
			     struct cword *cword)
{
	/* padlock_xcrypt requires at least two blocks of data. */
	if (unlikely(!(((unsigned long)in ^ (PAGE_SIZE - AES_BLOCK_SIZE)) &
		       (PAGE_SIZE - 1)))) {
		aes_crypt_copy(in, out, key, cword);
		return;
	}

	padlock_xcrypt(in, out, key, cword);
}

static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
				      void *control_word, u32 count)
{
	if (count == 1) {
		aes_crypt(input, output, key, control_word);
		return;
	}

	asm volatile ("test $1, %%cl;"
		      "je 1f;"
		      "lea -1(%%ecx), %%eax;"
		      "mov $1, %%ecx;"
		      ".byte 0xf3,0x0f,0xa7,0xc8;"	/* rep xcryptecb */
		      "mov %%eax, %%ecx;"
		      "1:"
		      ".byte 0xf3,0x0f,0xa7,0xc8"	/* rep xcryptecb */
		      : "+S"(input), "+D"(output)
		      : "d"(control_word), "b"(key), "c"(count)
		      : "ax");
}

static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
				     u8 *iv, void *control_word, u32 count)
{
	/* rep xcryptcbc */
	asm volatile (".byte 0xf3,0x0f,0xa7,0xd0"
		      : "+S" (input), "+D" (output), "+a" (iv)
		      : "d" (control_word), "b" (key), "c" (count));
	return iv;
}

static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
	struct aes_ctx *ctx = aes_ctx(tfm);
	padlock_reset_key();
	aes_crypt(in, out, ctx->E, &ctx->cword.encrypt);
}

static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
	struct aes_ctx *ctx = aes_ctx(tfm);
	padlock_reset_key();
	aes_crypt(in, out, ctx->D, &ctx->cword.decrypt);
}

static struct crypto_alg aes_alg = {
	.cra_name		=	"aes",
	.cra_driver_name	=	"aes-padlock",
	.cra_priority		=	PADLOCK_CRA_PRIORITY,
	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
	.cra_blocksize		=	AES_BLOCK_SIZE,
	.cra_ctxsize		=	sizeof(struct aes_ctx),
	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
	.cra_module		=	THIS_MODULE,
	.cra_list		=	LIST_HEAD_INIT(aes_alg.cra_list),
	.cra_u			=	{
		.cipher = {
			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
			.cia_setkey	   	= 	aes_set_key,
			.cia_encrypt	 	=	aes_encrypt,
			.cia_decrypt	  	=	aes_decrypt,
		}
	}
};

static int ecb_aes_encrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;

	padlock_reset_key();

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	while ((nbytes = walk.nbytes)) {
		padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
				   ctx->E, &ctx->cword.encrypt,
				   nbytes / AES_BLOCK_SIZE);
		nbytes &= AES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	return err;
}

static int ecb_aes_decrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;

	padlock_reset_key();

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	while ((nbytes = walk.nbytes)) {
		padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
				   ctx->D, &ctx->cword.decrypt,
				   nbytes / AES_BLOCK_SIZE);
		nbytes &= AES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	return err;
}

static struct crypto_alg ecb_aes_alg = {
	.cra_name		=	"ecb(aes)",
	.cra_driver_name	=	"ecb-aes-padlock",
	.cra_priority		=	PADLOCK_COMPOSITE_PRIORITY,
	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		=	AES_BLOCK_SIZE,
	.cra_ctxsize		=	sizeof(struct aes_ctx),
	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
	.cra_type		=	&crypto_blkcipher_type,
	.cra_module		=	THIS_MODULE,
	.cra_list		=	LIST_HEAD_INIT(ecb_aes_alg.cra_list),
	.cra_u			=	{
		.blkcipher = {
			.min_keysize		=	AES_MIN_KEY_SIZE,
			.max_keysize		=	AES_MAX_KEY_SIZE,
			.setkey	   		= 	aes_set_key,
			.encrypt		=	ecb_aes_encrypt,
			.decrypt		=	ecb_aes_decrypt,
		}
	}
};

static int cbc_aes_encrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;

	padlock_reset_key();

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	while ((nbytes = walk.nbytes)) {
		u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
					    walk.dst.virt.addr, ctx->E,
					    walk.iv, &ctx->cword.encrypt,
					    nbytes / AES_BLOCK_SIZE);
		memcpy(walk.iv, iv, AES_BLOCK_SIZE);
		nbytes &= AES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	return err;
}

static int cbc_aes_decrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;

	padlock_reset_key();

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	while ((nbytes = walk.nbytes)) {
		padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
				   ctx->D, walk.iv, &ctx->cword.decrypt,
				   nbytes / AES_BLOCK_SIZE);
		nbytes &= AES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	return err;
}

static struct crypto_alg cbc_aes_alg = {
	.cra_name		=	"cbc(aes)",
	.cra_driver_name	=	"cbc-aes-padlock",
	.cra_priority		=	PADLOCK_COMPOSITE_PRIORITY,
	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		=	AES_BLOCK_SIZE,
	.cra_ctxsize		=	sizeof(struct aes_ctx),
	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
	.cra_type		=	&crypto_blkcipher_type,
	.cra_module		=	THIS_MODULE,
	.cra_list		=	LIST_HEAD_INIT(cbc_aes_alg.cra_list),
	.cra_u			=	{
		.blkcipher = {
			.min_keysize		=	AES_MIN_KEY_SIZE,
			.max_keysize		=	AES_MAX_KEY_SIZE,
			.ivsize			=	AES_BLOCK_SIZE,
			.setkey	   		= 	aes_set_key,
			.encrypt		=	cbc_aes_encrypt,
			.decrypt		=	cbc_aes_decrypt,
		}
	}
};

static int __init padlock_init(void)
{
	int ret;

	if (!cpu_has_xcrypt) {
		printk(KERN_ERR PFX "VIA PadLock not detected.\n");
		return -ENODEV;
	}

	if (!cpu_has_xcrypt_enabled) {
		printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
		return -ENODEV;
	}

	if ((ret = crypto_register_alg(&aes_alg)))
		goto aes_err;

	if ((ret = crypto_register_alg(&ecb_aes_alg)))
		goto ecb_aes_err;

	if ((ret = crypto_register_alg(&cbc_aes_alg)))
		goto cbc_aes_err;

	printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");

out:
	return ret;

cbc_aes_err:
	crypto_unregister_alg(&ecb_aes_alg);
ecb_aes_err:
	crypto_unregister_alg(&aes_alg);
aes_err:
	printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
	goto out;
}

static void __exit padlock_fini(void)
{
	crypto_unregister_alg(&cbc_aes_alg);
	crypto_unregister_alg(&ecb_aes_alg);
	crypto_unregister_alg(&aes_alg);
}

module_init(padlock_init);
module_exit(padlock_fini);

MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Michal Ludvig");

MODULE_ALIAS("aes");
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Cryptographic API.
	3	*
	4	* Support for VIA PadLock hardware crypto engine.
	5	*
	6	* Copyright (c) 2004 Michal Ludvig <michal@logix.cz>
	7	*
1da177e4 LT	8	*/
1da177e4 LT	9
28ce728a	10	#include <crypto/algapi.h>
89e12654	11	#include <crypto/aes.h>
1da177e4 LT	12	#include <linux/module.h>
	13	#include <linux/init.h>
	14	#include <linux/types.h>
	15	#include <linux/errno.h>
1da177e4	16	#include <linux/interrupt.h>
6789b2dc	17	#include <linux/kernel.h>
1da177e4 LT	18	#include <asm/byteorder.h>
	19	#include "padlock.h"
	20
ccc17c34 ML	21	/* Control word. */
	22	struct cword {
	23	unsigned int __attribute__ ((__packed__))
	24	rounds:4,
	25	algo:3,
	26	keygen:1,
	27	interm:1,
	28	encdec:1,
	29	ksize:2;
	30	} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
	31
cc08632f ML	32	/* Whenever making any changes to the following
cc08632f ML	33	* structure make sure you keep E, d_data
7dc748e4 SS	34	* and cword aligned on 16 Bytes boundaries and
	35	* the Hardware can access 16 * 16 bytes of E and d_data
	36	* (only the first 15 * 16 bytes matter but the HW reads
	37	* more).
	38	*/
1da177e4	39	struct aes_ctx {
7dc748e4 SS	40	u32 E[AES_MAX_KEYLENGTH_U32]
	41	__attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
	42	u32 d_data[AES_MAX_KEYLENGTH_U32]
	43	__attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
6789b2dc HX	44	struct {
	45	struct cword encrypt;
	46	struct cword decrypt;
	47	} cword;
82062c72	48	u32 *D;
1da177e4 LT	49	};
1da177e4 LT	50
1da177e4 LT	51	/* Tells whether the ACE is capable to generate
	52	the extended key for a given key_len. */
	53	static inline int
	54	aes_hw_extkey_available(uint8_t key_len)
	55	{
	56	/* TODO: We should check the actual CPU model/stepping
	57	as it's possible that the capability will be
	58	added in the next CPU revisions. */
	59	if (key_len == 16)
	60	return 1;
	61	return 0;
	62	}
	63
28ce728a	64	static inline struct aes_ctx aes_ctx_common(void ctx)
6789b2dc	65	{
28ce728a	66	unsigned long addr = (unsigned long)ctx;
f10b7897 HX	67	unsigned long align = PADLOCK_ALIGNMENT;
	68
	69	if (align <= crypto_tfm_ctx_alignment())
	70	align = 1;
6c2bb98b	71	return (struct aes_ctx *)ALIGN(addr, align);
6789b2dc HX	72	}
6789b2dc HX	73
28ce728a HX	74	static inline struct aes_ctx aes_ctx(struct crypto_tfm tfm)
	75	{
	76	return aes_ctx_common(crypto_tfm_ctx(tfm));
	77	}
	78
	79	static inline struct aes_ctx blk_aes_ctx(struct crypto_blkcipher tfm)
	80	{
	81	return aes_ctx_common(crypto_blkcipher_ctx(tfm));
	82	}
	83
6c2bb98b	84	static int aes_set_key(struct crypto_tfm tfm, const u8 in_key,
560c06ae	85	unsigned int key_len)
1da177e4	86	{
6c2bb98b	87	struct aes_ctx *ctx = aes_ctx(tfm);
06ace7a9	88	const __le32 key = (const __le32 )in_key;
560c06ae	89	u32 *flags = &tfm->crt_flags;
7dc748e4	90	struct crypto_aes_ctx gen_aes;
1da177e4	91
560c06ae	92	if (key_len % 8) {
1da177e4 LT	93	*flags \|= CRYPTO_TFM_RES_BAD_KEY_LEN;
	94	return -EINVAL;
	95	}
	96
6789b2dc HX	97	/*
	98	* If the hardware is capable of generating the extended key
	99	* itself we must supply the plain key for both encryption
	100	* and decryption.
	101	*/
82062c72	102	ctx->D = ctx->E;
1da177e4	103
7dc748e4 SS	104	ctx->E[0] = le32_to_cpu(key[0]);
	105	ctx->E[1] = le32_to_cpu(key[1]);
	106	ctx->E[2] = le32_to_cpu(key[2]);
	107	ctx->E[3] = le32_to_cpu(key[3]);
1da177e4	108
6789b2dc HX	109	/* Prepare control words. */
	110	memset(&ctx->cword, 0, sizeof(ctx->cword));
	111
	112	ctx->cword.decrypt.encdec = 1;
	113	ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
	114	ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
	115	ctx->cword.encrypt.ksize = (key_len - 16) / 8;
	116	ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
	117
1da177e4 LT	118	/* Don't generate extended keys if the hardware can do it. */
	119	if (aes_hw_extkey_available(key_len))
	120	return 0;
	121
6789b2dc HX	122	ctx->D = ctx->d_data;
	123	ctx->cword.encrypt.keygen = 1;
	124	ctx->cword.decrypt.keygen = 1;
	125
7dc748e4 SS	126	if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) {
	127	*flags \|= CRYPTO_TFM_RES_BAD_KEY_LEN;
	128	return -EINVAL;
1da177e4 LT	129	}
1da177e4 LT	130
7dc748e4 SS	131	memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
7dc748e4 SS	132	memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
1da177e4 LT	133	return 0;
	134	}
	135
	136	/* ====== Encryption/decryption routines ====== */
	137
28e8c3ad	138	/* These are the real call to PadLock. */
866cd902 HX	139	static inline void padlock_reset_key(void)
	140	{
	141	asm volatile ("pushfl; popfl");
	142	}
	143
d4a7dd8e HX	144	static inline void padlock_xcrypt(const u8 input, u8 output, void *key,
	145	void *control_word)
	146	{
	147	asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
	148	: "+S"(input), "+D"(output)
	149	: "d"(control_word), "b"(key), "c"(1));
	150	}
	151
	152	static void aes_crypt_copy(const u8 in, u8 out, u32 key, struct cword cword)
	153	{
490fe3f0 HX	154	u8 buf[AES_BLOCK_SIZE * 2 + PADLOCK_ALIGNMENT - 1];
490fe3f0 HX	155	u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
d4a7dd8e HX	156
	157	memcpy(tmp, in, AES_BLOCK_SIZE);
	158	padlock_xcrypt(tmp, out, key, cword);
	159	}
	160
	161	static inline void aes_crypt(const u8 in, u8 out, u32 *key,
	162	struct cword *cword)
	163	{
d4a7dd8e HX	164	/* padlock_xcrypt requires at least two blocks of data. */
	165	if (unlikely(!(((unsigned long)in ^ (PAGE_SIZE - AES_BLOCK_SIZE)) &
	166	(PAGE_SIZE - 1)))) {
	167	aes_crypt_copy(in, out, key, cword);
	168	return;
	169	}
	170
	171	padlock_xcrypt(in, out, key, cword);
	172	}
	173
6789b2dc HX	174	static inline void padlock_xcrypt_ecb(const u8 input, u8 output, void *key,
6789b2dc HX	175	void *control_word, u32 count)
1da177e4	176	{
d4a7dd8e HX	177	if (count == 1) {
	178	aes_crypt(input, output, key, control_word);
	179	return;
	180	}
	181
d4a7dd8e HX	182	asm volatile ("test $1, %%cl;"
	183	"je 1f;"
	184	"lea -1(%%ecx), %%eax;"
	185	"mov $1, %%ecx;"
	186	".byte 0xf3,0x0f,0xa7,0xc8;" /* rep xcryptecb */
	187	"mov %%eax, %%ecx;"
	188	"1:"
	189	".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
1da177e4	190	: "+S"(input), "+D"(output)
d4a7dd8e HX	191	: "d"(control_word), "b"(key), "c"(count)
d4a7dd8e HX	192	: "ax");
1da177e4 LT	193	}
1da177e4 LT	194
476df259 HX	195	static inline u8 padlock_xcrypt_cbc(const u8 input, u8 output, void key,
476df259 HX	196	u8 iv, void control_word, u32 count)
28e8c3ad	197	{
28e8c3ad HX	198	/* rep xcryptcbc */
	199	asm volatile (".byte 0xf3,0x0f,0xa7,0xd0"
	200	: "+S" (input), "+D" (output), "+a" (iv)
	201	: "d" (control_word), "b" (key), "c" (count));
476df259	202	return iv;
28e8c3ad HX	203	}
28e8c3ad HX	204
6c2bb98b	205	static void aes_encrypt(struct crypto_tfm tfm, u8 out, const u8 *in)
1da177e4	206	{
6c2bb98b	207	struct aes_ctx *ctx = aes_ctx(tfm);
866cd902	208	padlock_reset_key();
d4a7dd8e	209	aes_crypt(in, out, ctx->E, &ctx->cword.encrypt);
1da177e4 LT	210	}
1da177e4 LT	211
6c2bb98b	212	static void aes_decrypt(struct crypto_tfm tfm, u8 out, const u8 *in)
1da177e4	213	{
6c2bb98b	214	struct aes_ctx *ctx = aes_ctx(tfm);
866cd902	215	padlock_reset_key();
d4a7dd8e	216	aes_crypt(in, out, ctx->D, &ctx->cword.decrypt);
1da177e4 LT	217	}
	218
	219	static struct crypto_alg aes_alg = {
	220	.cra_name = "aes",
c8a19c91	221	.cra_driver_name = "aes-padlock",
ccc17c34	222	.cra_priority = PADLOCK_CRA_PRIORITY,
1da177e4 LT	223	.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
1da177e4 LT	224	.cra_blocksize = AES_BLOCK_SIZE,
fbdae9f3	225	.cra_ctxsize = sizeof(struct aes_ctx),
6789b2dc	226	.cra_alignmask = PADLOCK_ALIGNMENT - 1,
1da177e4 LT	227	.cra_module = THIS_MODULE,
	228	.cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
	229	.cra_u = {
	230	.cipher = {
	231	.cia_min_keysize = AES_MIN_KEY_SIZE,
	232	.cia_max_keysize = AES_MAX_KEY_SIZE,
	233	.cia_setkey = aes_set_key,
	234	.cia_encrypt = aes_encrypt,
28e8c3ad	235	.cia_decrypt = aes_decrypt,
1da177e4 LT	236	}
	237	}
	238	};
	239
28ce728a HX	240	static int ecb_aes_encrypt(struct blkcipher_desc *desc,
	241	struct scatterlist dst, struct scatterlist src,
	242	unsigned int nbytes)
	243	{
	244	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	245	struct blkcipher_walk walk;
	246	int err;
	247
866cd902 HX	248	padlock_reset_key();
866cd902 HX	249
28ce728a HX	250	blkcipher_walk_init(&walk, dst, src, nbytes);
	251	err = blkcipher_walk_virt(desc, &walk);
	252
	253	while ((nbytes = walk.nbytes)) {
	254	padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
	255	ctx->E, &ctx->cword.encrypt,
	256	nbytes / AES_BLOCK_SIZE);
	257	nbytes &= AES_BLOCK_SIZE - 1;
	258	err = blkcipher_walk_done(desc, &walk, nbytes);
	259	}
	260
	261	return err;
	262	}
	263
	264	static int ecb_aes_decrypt(struct blkcipher_desc *desc,
	265	struct scatterlist dst, struct scatterlist src,
	266	unsigned int nbytes)
	267	{
	268	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	269	struct blkcipher_walk walk;
	270	int err;
	271
866cd902 HX	272	padlock_reset_key();
866cd902 HX	273
28ce728a HX	274	blkcipher_walk_init(&walk, dst, src, nbytes);
	275	err = blkcipher_walk_virt(desc, &walk);
	276
	277	while ((nbytes = walk.nbytes)) {
	278	padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
	279	ctx->D, &ctx->cword.decrypt,
	280	nbytes / AES_BLOCK_SIZE);
	281	nbytes &= AES_BLOCK_SIZE - 1;
	282	err = blkcipher_walk_done(desc, &walk, nbytes);
	283	}
	284
	285	return err;
	286	}
	287
	288	static struct crypto_alg ecb_aes_alg = {
	289	.cra_name = "ecb(aes)",
	290	.cra_driver_name = "ecb-aes-padlock",
	291	.cra_priority = PADLOCK_COMPOSITE_PRIORITY,
	292	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	293	.cra_blocksize = AES_BLOCK_SIZE,
	294	.cra_ctxsize = sizeof(struct aes_ctx),
	295	.cra_alignmask = PADLOCK_ALIGNMENT - 1,
	296	.cra_type = &crypto_blkcipher_type,
	297	.cra_module = THIS_MODULE,
	298	.cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list),
	299	.cra_u = {
	300	.blkcipher = {
	301	.min_keysize = AES_MIN_KEY_SIZE,
	302	.max_keysize = AES_MAX_KEY_SIZE,
	303	.setkey = aes_set_key,
	304	.encrypt = ecb_aes_encrypt,
	305	.decrypt = ecb_aes_decrypt,
	306	}
	307	}
	308	};
	309
	310	static int cbc_aes_encrypt(struct blkcipher_desc *desc,
	311	struct scatterlist dst, struct scatterlist src,
	312	unsigned int nbytes)
	313	{
	314	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	315	struct blkcipher_walk walk;
	316	int err;
	317
866cd902 HX	318	padlock_reset_key();
866cd902 HX	319
28ce728a HX	320	blkcipher_walk_init(&walk, dst, src, nbytes);
	321	err = blkcipher_walk_virt(desc, &walk);
	322
	323	while ((nbytes = walk.nbytes)) {
	324	u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
	325	walk.dst.virt.addr, ctx->E,
	326	walk.iv, &ctx->cword.encrypt,
	327	nbytes / AES_BLOCK_SIZE);
	328	memcpy(walk.iv, iv, AES_BLOCK_SIZE);
	329	nbytes &= AES_BLOCK_SIZE - 1;
	330	err = blkcipher_walk_done(desc, &walk, nbytes);
	331	}
	332
	333	return err;
	334	}
	335
	336	static int cbc_aes_decrypt(struct blkcipher_desc *desc,
	337	struct scatterlist dst, struct scatterlist src,
	338	unsigned int nbytes)
	339	{
	340	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	341	struct blkcipher_walk walk;
	342	int err;
	343
866cd902 HX	344	padlock_reset_key();
866cd902 HX	345
28ce728a HX	346	blkcipher_walk_init(&walk, dst, src, nbytes);
	347	err = blkcipher_walk_virt(desc, &walk);
	348
	349	while ((nbytes = walk.nbytes)) {
	350	padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
	351	ctx->D, walk.iv, &ctx->cword.decrypt,
	352	nbytes / AES_BLOCK_SIZE);
	353	nbytes &= AES_BLOCK_SIZE - 1;
	354	err = blkcipher_walk_done(desc, &walk, nbytes);
	355	}
	356
	357	return err;
	358	}
	359
	360	static struct crypto_alg cbc_aes_alg = {
	361	.cra_name = "cbc(aes)",
	362	.cra_driver_name = "cbc-aes-padlock",
	363	.cra_priority = PADLOCK_COMPOSITE_PRIORITY,
	364	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	365	.cra_blocksize = AES_BLOCK_SIZE,
	366	.cra_ctxsize = sizeof(struct aes_ctx),
	367	.cra_alignmask = PADLOCK_ALIGNMENT - 1,
	368	.cra_type = &crypto_blkcipher_type,
	369	.cra_module = THIS_MODULE,
	370	.cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list),
	371	.cra_u = {
	372	.blkcipher = {
	373	.min_keysize = AES_MIN_KEY_SIZE,
	374	.max_keysize = AES_MAX_KEY_SIZE,
	375	.ivsize = AES_BLOCK_SIZE,
	376	.setkey = aes_set_key,
	377	.encrypt = cbc_aes_encrypt,
	378	.decrypt = cbc_aes_decrypt,
	379	}
	380	}
	381	};
	382
1191f0a4	383	static int __init padlock_init(void)
1da177e4	384	{
1191f0a4 ML	385	int ret;
	386
	387	if (!cpu_has_xcrypt) {
	388	printk(KERN_ERR PFX "VIA PadLock not detected.\n");
	389	return -ENODEV;
	390	}
	391
	392	if (!cpu_has_xcrypt_enabled) {
	393	printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
	394	return -ENODEV;
	395	}
1da177e4	396
28ce728a HX	397	if ((ret = crypto_register_alg(&aes_alg)))
	398	goto aes_err;
	399
	400	if ((ret = crypto_register_alg(&ecb_aes_alg)))
	401	goto ecb_aes_err;
	402
	403	if ((ret = crypto_register_alg(&cbc_aes_alg)))
	404	goto cbc_aes_err;
1191f0a4 ML	405
	406	printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
	407
28ce728a	408	out:
1191f0a4	409	return ret;
28ce728a HX	410
	411	cbc_aes_err:
	412	crypto_unregister_alg(&ecb_aes_alg);
	413	ecb_aes_err:
	414	crypto_unregister_alg(&aes_alg);
	415	aes_err:
	416	printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
	417	goto out;
1da177e4 LT	418	}
1da177e4 LT	419
1191f0a4	420	static void __exit padlock_fini(void)
1da177e4	421	{
28ce728a HX	422	crypto_unregister_alg(&cbc_aes_alg);
28ce728a HX	423	crypto_unregister_alg(&ecb_aes_alg);
1da177e4 LT	424	crypto_unregister_alg(&aes_alg);
1da177e4 LT	425	}
1191f0a4 ML	426
	427	module_init(padlock_init);
	428	module_exit(padlock_fini);
	429
	430	MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
	431	MODULE_LICENSE("GPL");
	432	MODULE_AUTHOR("Michal Ludvig");
	433
f8246af0	434	MODULE_ALIAS("aes");