[deliverable/linux.git] / arch / x86 / crypto / glue_helper.c

/*
 * Shared glue code for 128bit block ciphers
 *
 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <crypto/b128ops.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>
#include <crypto/scatterwalk.h>

static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
				   struct blkcipher_desc *desc,
				   struct blkcipher_walk *walk)
{
	void *ctx = crypto_blkcipher_ctx(desc->tfm);
	const unsigned int bsize = 128 / 8;
	unsigned int nbytes, i, func_bytes;
	bool fpu_enabled = false;
	int err;

	err = blkcipher_walk_virt(desc, walk);

	while ((nbytes = walk->nbytes)) {
		u8 *wsrc = walk->src.virt.addr;
		u8 *wdst = walk->dst.virt.addr;

		fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
					     desc, fpu_enabled, nbytes);

		for (i = 0; i < gctx->num_funcs; i++) {
			func_bytes = bsize * gctx->funcs[i].num_blocks;

			/* Process multi-block batch */
			if (nbytes >= func_bytes) {
				do {
					gctx->funcs[i].fn_u.ecb(ctx, wdst,
								wsrc);

					wsrc += func_bytes;
					wdst += func_bytes;
					nbytes -= func_bytes;
				} while (nbytes >= func_bytes);

				if (nbytes < bsize)
					goto done;
			}
		}

done:
		err = blkcipher_walk_done(desc, walk, nbytes);
	}

	glue_fpu_end(fpu_enabled);
	return err;
}

int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
			  struct blkcipher_desc *desc, struct scatterlist *dst,
			  struct scatterlist *src, unsigned int nbytes)
{
	struct blkcipher_walk walk;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	return __glue_ecb_crypt_128bit(gctx, desc, &walk);
}
EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit);

static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
					      struct blkcipher_desc *desc,
					      struct blkcipher_walk *walk)
{
	void *ctx = crypto_blkcipher_ctx(desc->tfm);
	const unsigned int bsize = 128 / 8;
	unsigned int nbytes = walk->nbytes;
	u128 *src = (u128 *)walk->src.virt.addr;
	u128 *dst = (u128 *)walk->dst.virt.addr;
	u128 *iv = (u128 *)walk->iv;

	do {
		u128_xor(dst, src, iv);
		fn(ctx, (u8 *)dst, (u8 *)dst);
		iv = dst;

		src += 1;
		dst += 1;
		nbytes -= bsize;
	} while (nbytes >= bsize);

	*(u128 *)walk->iv = *iv;
	return nbytes;
}

int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
			    struct blkcipher_desc *desc,
			    struct scatterlist *dst,
			    struct scatterlist *src, unsigned int nbytes)
{
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	while ((nbytes = walk.nbytes)) {
		nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk);
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	return err;
}
EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit);

static unsigned int
__glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
			  struct blkcipher_desc *desc,
			  struct blkcipher_walk *walk)
{
	void *ctx = crypto_blkcipher_ctx(desc->tfm);
	const unsigned int bsize = 128 / 8;
	unsigned int nbytes = walk->nbytes;
	u128 *src = (u128 *)walk->src.virt.addr;
	u128 *dst = (u128 *)walk->dst.virt.addr;
	u128 last_iv;
	unsigned int num_blocks, func_bytes;
	unsigned int i;

	/* Start of the last block. */
	src += nbytes / bsize - 1;
	dst += nbytes / bsize - 1;

	last_iv = *src;

	for (i = 0; i < gctx->num_funcs; i++) {
		num_blocks = gctx->funcs[i].num_blocks;
		func_bytes = bsize * num_blocks;

		/* Process multi-block batch */
		if (nbytes >= func_bytes) {
			do {
				nbytes -= func_bytes - bsize;
				src -= num_blocks - 1;
				dst -= num_blocks - 1;

				gctx->funcs[i].fn_u.cbc(ctx, dst, src);

				nbytes -= bsize;
				if (nbytes < bsize)
					goto done;

				u128_xor(dst, dst, src - 1);
				src -= 1;
				dst -= 1;
			} while (nbytes >= func_bytes);

			if (nbytes < bsize)
				goto done;
		}
	}

done:
	u128_xor(dst, dst, (u128 *)walk->iv);
	*(u128 *)walk->iv = last_iv;

	return nbytes;
}

int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
			    struct blkcipher_desc *desc,
			    struct scatterlist *dst,
			    struct scatterlist *src, unsigned int nbytes)
{
	const unsigned int bsize = 128 / 8;
	bool fpu_enabled = false;
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	while ((nbytes = walk.nbytes)) {
		fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
					     desc, fpu_enabled, nbytes);
		nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	glue_fpu_end(fpu_enabled);
	return err;
}
EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);

static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
					struct blkcipher_desc *desc,
					struct blkcipher_walk *walk)
{
	void *ctx = crypto_blkcipher_ctx(desc->tfm);
	u8 *src = (u8 *)walk->src.virt.addr;
	u8 *dst = (u8 *)walk->dst.virt.addr;
	unsigned int nbytes = walk->nbytes;
	le128 ctrblk;
	u128 tmp;

	be128_to_le128(&ctrblk, (be128 *)walk->iv);

	memcpy(&tmp, src, nbytes);
	fn_ctr(ctx, &tmp, &tmp, &ctrblk);
	memcpy(dst, &tmp, nbytes);

	le128_to_be128((be128 *)walk->iv, &ctrblk);
}

static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
					    struct blkcipher_desc *desc,
					    struct blkcipher_walk *walk)
{
	const unsigned int bsize = 128 / 8;
	void *ctx = crypto_blkcipher_ctx(desc->tfm);
	unsigned int nbytes = walk->nbytes;
	u128 *src = (u128 *)walk->src.virt.addr;
	u128 *dst = (u128 *)walk->dst.virt.addr;
	le128 ctrblk;
	unsigned int num_blocks, func_bytes;
	unsigned int i;

	be128_to_le128(&ctrblk, (be128 *)walk->iv);

	/* Process multi-block batch */
	for (i = 0; i < gctx->num_funcs; i++) {
		num_blocks = gctx->funcs[i].num_blocks;
		func_bytes = bsize * num_blocks;

		if (nbytes >= func_bytes) {
			do {
				gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk);

				src += num_blocks;
				dst += num_blocks;
				nbytes -= func_bytes;
			} while (nbytes >= func_bytes);

			if (nbytes < bsize)
				goto done;
		}
	}

done:
	le128_to_be128((be128 *)walk->iv, &ctrblk);
	return nbytes;
}

int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
			  struct blkcipher_desc *desc, struct scatterlist *dst,
			  struct scatterlist *src, unsigned int nbytes)
{
	const unsigned int bsize = 128 / 8;
	bool fpu_enabled = false;
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt_block(desc, &walk, bsize);

	while ((nbytes = walk.nbytes) >= bsize) {
		fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
					     desc, fpu_enabled, nbytes);
		nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	glue_fpu_end(fpu_enabled);

	if (walk.nbytes) {
		glue_ctr_crypt_final_128bit(
			gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
		err = blkcipher_walk_done(desc, &walk, 0);
	}

	return err;
}
EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit);

static unsigned int __glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
					    void *ctx,
					    struct blkcipher_desc *desc,
					    struct blkcipher_walk *walk)
{
	const unsigned int bsize = 128 / 8;
	unsigned int nbytes = walk->nbytes;
	u128 *src = (u128 *)walk->src.virt.addr;
	u128 *dst = (u128 *)walk->dst.virt.addr;
	unsigned int num_blocks, func_bytes;
	unsigned int i;

	/* Process multi-block batch */
	for (i = 0; i < gctx->num_funcs; i++) {
		num_blocks = gctx->funcs[i].num_blocks;
		func_bytes = bsize * num_blocks;

		if (nbytes >= func_bytes) {
			do {
				gctx->funcs[i].fn_u.xts(ctx, dst, src,
							(le128 *)walk->iv);

				src += num_blocks;
				dst += num_blocks;
				nbytes -= func_bytes;
			} while (nbytes >= func_bytes);

			if (nbytes < bsize)
				goto done;
		}
	}

done:
	return nbytes;
}

/* for implementations implementing faster XTS IV generator */
int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
			  struct blkcipher_desc *desc, struct scatterlist *dst,
			  struct scatterlist *src, unsigned int nbytes,
			  void (*tweak_fn)(void *ctx, u8 *dst, const u8 *src),
			  void *tweak_ctx, void *crypt_ctx)
{
	const unsigned int bsize = 128 / 8;
	bool fpu_enabled = false;
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);

	err = blkcipher_walk_virt(desc, &walk);
	nbytes = walk.nbytes;
	if (!nbytes)
		return err;

	/* set minimum length to bsize, for tweak_fn */
	fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
				     desc, fpu_enabled,
				     nbytes < bsize ? bsize : nbytes);

	/* calculate first value of T */
	tweak_fn(tweak_ctx, walk.iv, walk.iv);

	while (nbytes) {
		nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk);

		err = blkcipher_walk_done(desc, &walk, nbytes);
		nbytes = walk.nbytes;
	}

	glue_fpu_end(fpu_enabled);

	return err;
}
EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);

void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
			       common_glue_func_t fn)
{
	le128 ivblk = *iv;

	/* generate next IV */
	le128_gf128mul_x_ble(iv, &ivblk);

	/* CC <- T xor C */
	u128_xor(dst, src, (u128 *)&ivblk);

	/* PP <- D(Key2,CC) */
	fn(ctx, (u8 *)dst, (u8 *)dst);

	/* P <- T xor PP */
	u128_xor(dst, dst, (u128 *)&ivblk);
}
EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);

MODULE_LICENSE("GPL");
Commit	Line	Data
596d8750 JK	1	/*
	2	* Shared glue code for 128bit block ciphers
	3	*
a05248ed	4	* Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
596d8750 JK	5	*
	6	* CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
	7	* Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
	8	* CTR part based on code (crypto/ctr.c) by:
	9	* (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of the GNU General Public License as published by
	13	* the Free Software Foundation; either version 2 of the License, or
	14	* (at your option) any later version.
	15	*
	16	* This program is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	19	* GNU General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with this program; if not, write to the Free Software
	23	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	24	* USA
	25	*
	26	*/
	27
	28	#include <linux/module.h>
	29	#include <crypto/b128ops.h>
	30	#include <crypto/lrw.h>
	31	#include <crypto/xts.h>
	32	#include <asm/crypto/glue_helper.h>
	33	#include <crypto/scatterwalk.h>
	34
	35	static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
	36	struct blkcipher_desc *desc,
	37	struct blkcipher_walk *walk)
	38	{
	39	void *ctx = crypto_blkcipher_ctx(desc->tfm);
	40	const unsigned int bsize = 128 / 8;
	41	unsigned int nbytes, i, func_bytes;
	42	bool fpu_enabled = false;
	43	int err;
	44
	45	err = blkcipher_walk_virt(desc, walk);
	46
	47	while ((nbytes = walk->nbytes)) {
	48	u8 *wsrc = walk->src.virt.addr;
	49	u8 *wdst = walk->dst.virt.addr;
	50
	51	fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
	52	desc, fpu_enabled, nbytes);
	53
	54	for (i = 0; i < gctx->num_funcs; i++) {
	55	func_bytes = bsize * gctx->funcs[i].num_blocks;
	56
	57	/* Process multi-block batch */
	58	if (nbytes >= func_bytes) {
	59	do {
	60	gctx->funcs[i].fn_u.ecb(ctx, wdst,
	61	wsrc);
	62
	63	wsrc += func_bytes;
	64	wdst += func_bytes;
	65	nbytes -= func_bytes;
	66	} while (nbytes >= func_bytes);
	67
	68	if (nbytes < bsize)
69	goto done;
70	}
71	}
72
73	done:
74	err = blkcipher_walk_done(desc, walk, nbytes);
75	}
76
77	glue_fpu_end(fpu_enabled);
78	return err;
79	}
80
81	int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
82	struct blkcipher_desc desc, struct scatterlist dst,
83	struct scatterlist *src, unsigned int nbytes)
84	{
85	struct blkcipher_walk walk;
86
87	blkcipher_walk_init(&walk, dst, src, nbytes);
88	return __glue_ecb_crypt_128bit(gctx, desc, &walk);
89	}
90	EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit);
91
92	static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
93	struct blkcipher_desc *desc,
94	struct blkcipher_walk *walk)
95	{
96	void *ctx = crypto_blkcipher_ctx(desc->tfm);
97	const unsigned int bsize = 128 / 8;
98	unsigned int nbytes = walk->nbytes;
99	u128 src = (u128 )walk->src.virt.addr;
100	u128 dst = (u128 )walk->dst.virt.addr;
101	u128 iv = (u128 )walk->iv;
102
103	do {
104	u128_xor(dst, src, iv);
105	fn(ctx, (u8 )dst, (u8 )dst);
106	iv = dst;
107
108	src += 1;
109	dst += 1;
110	nbytes -= bsize;
111	} while (nbytes >= bsize);
112
c9f97a27	113	(u128 )walk->iv = *iv;
596d8750 JK	114	return nbytes;
	115	}
	116
	117	int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
	118	struct blkcipher_desc *desc,
	119	struct scatterlist *dst,
	120	struct scatterlist *src, unsigned int nbytes)
	121	{
	122	struct blkcipher_walk walk;
	123	int err;
	124
	125	blkcipher_walk_init(&walk, dst, src, nbytes);
	126	err = blkcipher_walk_virt(desc, &walk);
	127
	128	while ((nbytes = walk.nbytes)) {
	129	nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk);
	130	err = blkcipher_walk_done(desc, &walk, nbytes);
	131	}
	132
	133	return err;
	134	}
	135	EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit);
	136
	137	static unsigned int
	138	__glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
	139	struct blkcipher_desc *desc,
	140	struct blkcipher_walk *walk)
	141	{
	142	void *ctx = crypto_blkcipher_ctx(desc->tfm);
	143	const unsigned int bsize = 128 / 8;
	144	unsigned int nbytes = walk->nbytes;
	145	u128 src = (u128 )walk->src.virt.addr;
	146	u128 dst = (u128 )walk->dst.virt.addr;
	147	u128 last_iv;
	148	unsigned int num_blocks, func_bytes;
	149	unsigned int i;
	150
	151	/* Start of the last block. */
	152	src += nbytes / bsize - 1;
	153	dst += nbytes / bsize - 1;
	154
	155	last_iv = *src;
	156
	157	for (i = 0; i < gctx->num_funcs; i++) {
	158	num_blocks = gctx->funcs[i].num_blocks;
	159	func_bytes = bsize * num_blocks;
	160
	161	/* Process multi-block batch */
	162	if (nbytes >= func_bytes) {
	163	do {
	164	nbytes -= func_bytes - bsize;
	165	src -= num_blocks - 1;
	166	dst -= num_blocks - 1;
	167
	168	gctx->funcs[i].fn_u.cbc(ctx, dst, src);
	169
	170	nbytes -= bsize;
	171	if (nbytes < bsize)
	172	goto done;
	173
	174	u128_xor(dst, dst, src - 1);
	175	src -= 1;
	176	dst -= 1;
	177	} while (nbytes >= func_bytes);
178
179	if (nbytes < bsize)
180	goto done;
181	}
182	}
183
184	done:
185	u128_xor(dst, dst, (u128 *)walk->iv);
186	(u128 )walk->iv = last_iv;
187
188	return nbytes;
189	}
190
191	int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
192	struct blkcipher_desc *desc,
193	struct scatterlist *dst,
194	struct scatterlist *src, unsigned int nbytes)
195	{
196	const unsigned int bsize = 128 / 8;
197	bool fpu_enabled = false;
198	struct blkcipher_walk walk;
199	int err;
200
201	blkcipher_walk_init(&walk, dst, src, nbytes);
202	err = blkcipher_walk_virt(desc, &walk);
203
204	while ((nbytes = walk.nbytes)) {
205	fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
206	desc, fpu_enabled, nbytes);
207	nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
208	err = blkcipher_walk_done(desc, &walk, nbytes);
209	}
210
211	glue_fpu_end(fpu_enabled);
212	return err;
213	}
214	EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);
215
216	static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
217	struct blkcipher_desc *desc,
218	struct blkcipher_walk *walk)
219	{
220	void *ctx = crypto_blkcipher_ctx(desc->tfm);
221	u8 src = (u8 )walk->src.virt.addr;
222	u8 dst = (u8 )walk->dst.virt.addr;
223	unsigned int nbytes = walk->nbytes;
58990986	224	le128 ctrblk;
596d8750 JK	225	u128 tmp;
596d8750 JK	226
58990986	227	be128_to_le128(&ctrblk, (be128 *)walk->iv);
596d8750 JK	228
	229	memcpy(&tmp, src, nbytes);
	230	fn_ctr(ctx, &tmp, &tmp, &ctrblk);
	231	memcpy(dst, &tmp, nbytes);
	232
58990986	233	le128_to_be128((be128 *)walk->iv, &ctrblk);
596d8750	234	}
596d8750 JK	235
	236	static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
	237	struct blkcipher_desc *desc,
	238	struct blkcipher_walk *walk)
	239	{
	240	const unsigned int bsize = 128 / 8;
	241	void *ctx = crypto_blkcipher_ctx(desc->tfm);
	242	unsigned int nbytes = walk->nbytes;
	243	u128 src = (u128 )walk->src.virt.addr;
	244	u128 dst = (u128 )walk->dst.virt.addr;
58990986	245	le128 ctrblk;
596d8750 JK	246	unsigned int num_blocks, func_bytes;
	247	unsigned int i;
	248
58990986	249	be128_to_le128(&ctrblk, (be128 *)walk->iv);
596d8750 JK	250
	251	/* Process multi-block batch */
	252	for (i = 0; i < gctx->num_funcs; i++) {
	253	num_blocks = gctx->funcs[i].num_blocks;
	254	func_bytes = bsize * num_blocks;
	255
	256	if (nbytes >= func_bytes) {
	257	do {
	258	gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk);
	259
	260	src += num_blocks;
	261	dst += num_blocks;
	262	nbytes -= func_bytes;
	263	} while (nbytes >= func_bytes);
	264
	265	if (nbytes < bsize)
	266	goto done;
	267	}
	268	}
	269
	270	done:
58990986	271	le128_to_be128((be128 *)walk->iv, &ctrblk);
596d8750 JK	272	return nbytes;
	273	}
	274
	275	int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
	276	struct blkcipher_desc desc, struct scatterlist dst,
	277	struct scatterlist *src, unsigned int nbytes)
	278	{
	279	const unsigned int bsize = 128 / 8;
	280	bool fpu_enabled = false;
	281	struct blkcipher_walk walk;
	282	int err;
	283
	284	blkcipher_walk_init(&walk, dst, src, nbytes);
	285	err = blkcipher_walk_virt_block(desc, &walk, bsize);
	286
	287	while ((nbytes = walk.nbytes) >= bsize) {
	288	fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
	289	desc, fpu_enabled, nbytes);
	290	nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
	291	err = blkcipher_walk_done(desc, &walk, nbytes);
	292	}
	293
	294	glue_fpu_end(fpu_enabled);
	295
	296	if (walk.nbytes) {
	297	glue_ctr_crypt_final_128bit(
	298	gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
	299	err = blkcipher_walk_done(desc, &walk, 0);
	300	}
	301
	302	return err;
	303	}
	304	EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit);
	305
a05248ed JK	306	static unsigned int __glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
	307	void *ctx,
	308	struct blkcipher_desc *desc,
	309	struct blkcipher_walk *walk)
	310	{
	311	const unsigned int bsize = 128 / 8;
	312	unsigned int nbytes = walk->nbytes;
	313	u128 src = (u128 )walk->src.virt.addr;
	314	u128 dst = (u128 )walk->dst.virt.addr;
	315	unsigned int num_blocks, func_bytes;
	316	unsigned int i;
	317
	318	/* Process multi-block batch */
	319	for (i = 0; i < gctx->num_funcs; i++) {
	320	num_blocks = gctx->funcs[i].num_blocks;
	321	func_bytes = bsize * num_blocks;
	322
	323	if (nbytes >= func_bytes) {
	324	do {
	325	gctx->funcs[i].fn_u.xts(ctx, dst, src,
	326	(le128 *)walk->iv);
	327
	328	src += num_blocks;
	329	dst += num_blocks;
	330	nbytes -= func_bytes;
	331	} while (nbytes >= func_bytes);
	332
	333	if (nbytes < bsize)
	334	goto done;
	335	}
	336	}
	337
	338	done:
	339	return nbytes;
	340	}
	341
	342	/* for implementations implementing faster XTS IV generator */
	343	int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
	344	struct blkcipher_desc desc, struct scatterlist dst,
	345	struct scatterlist *src, unsigned int nbytes,
	346	void (tweak_fn)(void ctx, u8 dst, const u8 src),
	347	void tweak_ctx, void crypt_ctx)
	348	{
	349	const unsigned int bsize = 128 / 8;
	350	bool fpu_enabled = false;
	351	struct blkcipher_walk walk;
	352	int err;
	353
	354	blkcipher_walk_init(&walk, dst, src, nbytes);
	355
	356	err = blkcipher_walk_virt(desc, &walk);
	357	nbytes = walk.nbytes;
	358	if (!nbytes)
	359	return err;
	360
	361	/* set minimum length to bsize, for tweak_fn */
	362	fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
	363	desc, fpu_enabled,
	364	nbytes < bsize ? bsize : nbytes);
	365
	366	/* calculate first value of T */
	367	tweak_fn(tweak_ctx, walk.iv, walk.iv);
	368
	369	while (nbytes) {
370	nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk);
371
372	err = blkcipher_walk_done(desc, &walk, nbytes);
373	nbytes = walk.nbytes;
374	}
375
376	glue_fpu_end(fpu_enabled);
377
378	return err;
379	}
380	EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);
381
382	void glue_xts_crypt_128bit_one(void ctx, u128 dst, const u128 src, le128 iv,
383	common_glue_func_t fn)
384	{
385	le128 ivblk = *iv;
386
387	/* generate next IV */
388	le128_gf128mul_x_ble(iv, &ivblk);
389
390	/* CC <- T xor C */
391	u128_xor(dst, src, (u128 *)&ivblk);
392
393	/* PP <- D(Key2,CC) */
394	fn(ctx, (u8 )dst, (u8 )dst);
395
396	/* P <- T xor PP */
397	u128_xor(dst, dst, (u128 *)&ivblk);
398	}
399	EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);
400
596d8750	401	MODULE_LICENSE("GPL");