Commit | Line | Data |
---|---|---|
cc333cd6 MC |
1 | /** |
2 | * GHASH routines supporting VMX instructions on the Power 8 | |
3 | * | |
4 | * Copyright (C) 2015 International Business Machines Inc. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License as published by | |
8 | * the Free Software Foundation; version 2 only. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program; if not, write to the Free Software | |
17 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
18 | * | |
19 | * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> | |
20 | */ | |
21 | ||
22 | #include <linux/types.h> | |
23 | #include <linux/err.h> | |
24 | #include <linux/crypto.h> | |
25 | #include <linux/delay.h> | |
26 | #include <linux/hardirq.h> | |
27 | #include <asm/switch_to.h> | |
28 | #include <crypto/aes.h> | |
29 | #include <crypto/scatterwalk.h> | |
30 | #include <crypto/internal/hash.h> | |
31 | #include <crypto/b128ops.h> | |
32 | ||
33 | #define IN_INTERRUPT in_interrupt() | |
34 | ||
35 | #define GHASH_BLOCK_SIZE (16) | |
36 | #define GHASH_DIGEST_SIZE (16) | |
37 | #define GHASH_KEY_LEN (16) | |
38 | ||
39 | void gcm_init_p8(u128 htable[16], const u64 Xi[2]); | |
40 | void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); | |
41 | void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], | |
4beb1060 | 42 | const u8 *in, size_t len); |
cc333cd6 MC |
43 | |
44 | struct p8_ghash_ctx { | |
4beb1060 HX |
45 | u128 htable[16]; |
46 | struct crypto_shash *fallback; | |
cc333cd6 MC |
47 | }; |
48 | ||
49 | struct p8_ghash_desc_ctx { | |
4beb1060 HX |
50 | u64 shash[2]; |
51 | u8 buffer[GHASH_DIGEST_SIZE]; | |
52 | int bytes; | |
53 | struct shash_desc fallback_desc; | |
cc333cd6 MC |
54 | }; |
55 | ||
56 | static int p8_ghash_init_tfm(struct crypto_tfm *tfm) | |
57 | { | |
4beb1060 HX |
58 | const char *alg; |
59 | struct crypto_shash *fallback; | |
60 | struct crypto_shash *shash_tfm = __crypto_shash_cast(tfm); | |
61 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); | |
62 | ||
63 | if (!(alg = crypto_tfm_alg_name(tfm))) { | |
64 | printk(KERN_ERR "Failed to get algorithm name.\n"); | |
65 | return -ENOENT; | |
66 | } | |
67 | ||
68 | fallback = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); | |
69 | if (IS_ERR(fallback)) { | |
70 | printk(KERN_ERR | |
71 | "Failed to allocate transformation for '%s': %ld\n", | |
72 | alg, PTR_ERR(fallback)); | |
73 | return PTR_ERR(fallback); | |
74 | } | |
75 | printk(KERN_INFO "Using '%s' as fallback implementation.\n", | |
76 | crypto_tfm_alg_driver_name(crypto_shash_tfm(fallback))); | |
77 | ||
78 | crypto_shash_set_flags(fallback, | |
79 | crypto_shash_get_flags((struct crypto_shash | |
80 | *) tfm)); | |
81 | ctx->fallback = fallback; | |
82 | ||
83 | shash_tfm->descsize = sizeof(struct p8_ghash_desc_ctx) | |
84 | + crypto_shash_descsize(fallback); | |
85 | ||
86 | return 0; | |
cc333cd6 MC |
87 | } |
88 | ||
89 | static void p8_ghash_exit_tfm(struct crypto_tfm *tfm) | |
90 | { | |
4beb1060 | 91 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); |
cc333cd6 | 92 | |
4beb1060 HX |
93 | if (ctx->fallback) { |
94 | crypto_free_shash(ctx->fallback); | |
95 | ctx->fallback = NULL; | |
96 | } | |
cc333cd6 MC |
97 | } |
98 | ||
99 | static int p8_ghash_init(struct shash_desc *desc) | |
100 | { | |
4beb1060 HX |
101 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); |
102 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); | |
103 | ||
104 | dctx->bytes = 0; | |
105 | memset(dctx->shash, 0, GHASH_DIGEST_SIZE); | |
106 | dctx->fallback_desc.tfm = ctx->fallback; | |
107 | dctx->fallback_desc.flags = desc->flags; | |
108 | return crypto_shash_init(&dctx->fallback_desc); | |
cc333cd6 MC |
109 | } |
110 | ||
111 | static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, | |
4beb1060 | 112 | unsigned int keylen) |
cc333cd6 | 113 | { |
4beb1060 | 114 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm)); |
cc333cd6 | 115 | |
4beb1060 HX |
116 | if (keylen != GHASH_KEY_LEN) |
117 | return -EINVAL; | |
cc333cd6 | 118 | |
44d21c3f | 119 | preempt_disable(); |
4beb1060 HX |
120 | pagefault_disable(); |
121 | enable_kernel_altivec(); | |
122 | enable_kernel_fp(); | |
123 | gcm_init_p8(ctx->htable, (const u64 *) key); | |
124 | pagefault_enable(); | |
44d21c3f | 125 | preempt_enable(); |
4beb1060 | 126 | return crypto_shash_setkey(ctx->fallback, key, keylen); |
cc333cd6 MC |
127 | } |
128 | ||
129 | static int p8_ghash_update(struct shash_desc *desc, | |
4beb1060 | 130 | const u8 *src, unsigned int srclen) |
cc333cd6 | 131 | { |
4beb1060 HX |
132 | unsigned int len; |
133 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); | |
134 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); | |
135 | ||
136 | if (IN_INTERRUPT) { | |
137 | return crypto_shash_update(&dctx->fallback_desc, src, | |
138 | srclen); | |
139 | } else { | |
140 | if (dctx->bytes) { | |
141 | if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { | |
142 | memcpy(dctx->buffer + dctx->bytes, src, | |
143 | srclen); | |
144 | dctx->bytes += srclen; | |
145 | return 0; | |
146 | } | |
147 | memcpy(dctx->buffer + dctx->bytes, src, | |
148 | GHASH_DIGEST_SIZE - dctx->bytes); | |
44d21c3f | 149 | preempt_disable(); |
4beb1060 HX |
150 | pagefault_disable(); |
151 | enable_kernel_altivec(); | |
152 | enable_kernel_fp(); | |
153 | gcm_ghash_p8(dctx->shash, ctx->htable, | |
154 | dctx->buffer, GHASH_DIGEST_SIZE); | |
155 | pagefault_enable(); | |
44d21c3f | 156 | preempt_enable(); |
4beb1060 HX |
157 | src += GHASH_DIGEST_SIZE - dctx->bytes; |
158 | srclen -= GHASH_DIGEST_SIZE - dctx->bytes; | |
159 | dctx->bytes = 0; | |
160 | } | |
161 | len = srclen & ~(GHASH_DIGEST_SIZE - 1); | |
162 | if (len) { | |
44d21c3f | 163 | preempt_disable(); |
4beb1060 HX |
164 | pagefault_disable(); |
165 | enable_kernel_altivec(); | |
166 | enable_kernel_fp(); | |
167 | gcm_ghash_p8(dctx->shash, ctx->htable, src, len); | |
168 | pagefault_enable(); | |
44d21c3f | 169 | preempt_enable(); |
4beb1060 HX |
170 | src += len; |
171 | srclen -= len; | |
172 | } | |
173 | if (srclen) { | |
174 | memcpy(dctx->buffer, src, srclen); | |
175 | dctx->bytes = srclen; | |
176 | } | |
177 | return 0; | |
178 | } | |
cc333cd6 MC |
179 | } |
180 | ||
181 | static int p8_ghash_final(struct shash_desc *desc, u8 *out) | |
182 | { | |
4beb1060 HX |
183 | int i; |
184 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); | |
185 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); | |
186 | ||
187 | if (IN_INTERRUPT) { | |
188 | return crypto_shash_final(&dctx->fallback_desc, out); | |
189 | } else { | |
190 | if (dctx->bytes) { | |
191 | for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) | |
192 | dctx->buffer[i] = 0; | |
44d21c3f | 193 | preempt_disable(); |
4beb1060 HX |
194 | pagefault_disable(); |
195 | enable_kernel_altivec(); | |
196 | enable_kernel_fp(); | |
197 | gcm_ghash_p8(dctx->shash, ctx->htable, | |
198 | dctx->buffer, GHASH_DIGEST_SIZE); | |
199 | pagefault_enable(); | |
44d21c3f | 200 | preempt_enable(); |
4beb1060 HX |
201 | dctx->bytes = 0; |
202 | } | |
203 | memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); | |
204 | return 0; | |
205 | } | |
cc333cd6 MC |
206 | } |
207 | ||
208 | struct shash_alg p8_ghash_alg = { | |
4beb1060 HX |
209 | .digestsize = GHASH_DIGEST_SIZE, |
210 | .init = p8_ghash_init, | |
211 | .update = p8_ghash_update, | |
212 | .final = p8_ghash_final, | |
213 | .setkey = p8_ghash_setkey, | |
214 | .descsize = sizeof(struct p8_ghash_desc_ctx), | |
215 | .base = { | |
216 | .cra_name = "ghash", | |
217 | .cra_driver_name = "p8_ghash", | |
218 | .cra_priority = 1000, | |
219 | .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_NEED_FALLBACK, | |
220 | .cra_blocksize = GHASH_BLOCK_SIZE, | |
221 | .cra_ctxsize = sizeof(struct p8_ghash_ctx), | |
222 | .cra_module = THIS_MODULE, | |
223 | .cra_init = p8_ghash_init_tfm, | |
224 | .cra_exit = p8_ghash_exit_tfm, | |
225 | }, | |
cc333cd6 | 226 | }; |