Commit | Line | Data |
---|---|---|
f1e866b1 AB |
1 | /* |
2 | * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. | |
3 | * | |
4 | * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License version 2 as published | |
8 | * by the Free Software Foundation. | |
9 | */ | |
10 | ||
11 | #include <asm/hwcap.h> | |
12 | #include <asm/neon.h> | |
13 | #include <asm/simd.h> | |
14 | #include <asm/unaligned.h> | |
15 | #include <crypto/cryptd.h> | |
16 | #include <crypto/internal/hash.h> | |
17 | #include <crypto/gf128mul.h> | |
18 | #include <linux/crypto.h> | |
19 | #include <linux/module.h> | |
20 | ||
21 | MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions"); | |
22 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); | |
23 | MODULE_LICENSE("GPL v2"); | |
24 | ||
25 | #define GHASH_BLOCK_SIZE 16 | |
26 | #define GHASH_DIGEST_SIZE 16 | |
27 | ||
28 | struct ghash_key { | |
29 | u64 a; | |
30 | u64 b; | |
31 | }; | |
32 | ||
33 | struct ghash_desc_ctx { | |
34 | u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; | |
35 | u8 buf[GHASH_BLOCK_SIZE]; | |
36 | u32 count; | |
37 | }; | |
38 | ||
39 | struct ghash_async_ctx { | |
40 | struct cryptd_ahash *cryptd_tfm; | |
41 | }; | |
42 | ||
43 | asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src, | |
44 | struct ghash_key const *k, const char *head); | |
45 | ||
46 | static int ghash_init(struct shash_desc *desc) | |
47 | { | |
48 | struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); | |
49 | ||
50 | *ctx = (struct ghash_desc_ctx){}; | |
51 | return 0; | |
52 | } | |
53 | ||
54 | static int ghash_update(struct shash_desc *desc, const u8 *src, | |
55 | unsigned int len) | |
56 | { | |
57 | struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); | |
58 | unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; | |
59 | ||
60 | ctx->count += len; | |
61 | ||
62 | if ((partial + len) >= GHASH_BLOCK_SIZE) { | |
63 | struct ghash_key *key = crypto_shash_ctx(desc->tfm); | |
64 | int blocks; | |
65 | ||
66 | if (partial) { | |
67 | int p = GHASH_BLOCK_SIZE - partial; | |
68 | ||
69 | memcpy(ctx->buf + partial, src, p); | |
70 | src += p; | |
71 | len -= p; | |
72 | } | |
73 | ||
74 | blocks = len / GHASH_BLOCK_SIZE; | |
75 | len %= GHASH_BLOCK_SIZE; | |
76 | ||
77 | kernel_neon_begin(); | |
78 | pmull_ghash_update(blocks, ctx->digest, src, key, | |
79 | partial ? ctx->buf : NULL); | |
80 | kernel_neon_end(); | |
81 | src += blocks * GHASH_BLOCK_SIZE; | |
82 | partial = 0; | |
83 | } | |
84 | if (len) | |
85 | memcpy(ctx->buf + partial, src, len); | |
86 | return 0; | |
87 | } | |
88 | ||
89 | static int ghash_final(struct shash_desc *desc, u8 *dst) | |
90 | { | |
91 | struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); | |
92 | unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; | |
93 | ||
94 | if (partial) { | |
95 | struct ghash_key *key = crypto_shash_ctx(desc->tfm); | |
96 | ||
97 | memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); | |
98 | kernel_neon_begin(); | |
99 | pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL); | |
100 | kernel_neon_end(); | |
101 | } | |
102 | put_unaligned_be64(ctx->digest[1], dst); | |
103 | put_unaligned_be64(ctx->digest[0], dst + 8); | |
104 | ||
105 | *ctx = (struct ghash_desc_ctx){}; | |
106 | return 0; | |
107 | } | |
108 | ||
109 | static int ghash_setkey(struct crypto_shash *tfm, | |
110 | const u8 *inkey, unsigned int keylen) | |
111 | { | |
112 | struct ghash_key *key = crypto_shash_ctx(tfm); | |
113 | u64 a, b; | |
114 | ||
115 | if (keylen != GHASH_BLOCK_SIZE) { | |
116 | crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); | |
117 | return -EINVAL; | |
118 | } | |
119 | ||
120 | /* perform multiplication by 'x' in GF(2^128) */ | |
121 | b = get_unaligned_be64(inkey); | |
122 | a = get_unaligned_be64(inkey + 8); | |
123 | ||
124 | key->a = (a << 1) | (b >> 63); | |
125 | key->b = (b << 1) | (a >> 63); | |
126 | ||
127 | if (b >> 63) | |
128 | key->b ^= 0xc200000000000000UL; | |
129 | ||
130 | return 0; | |
131 | } | |
132 | ||
133 | static struct shash_alg ghash_alg = { | |
134 | .digestsize = GHASH_DIGEST_SIZE, | |
135 | .init = ghash_init, | |
136 | .update = ghash_update, | |
137 | .final = ghash_final, | |
138 | .setkey = ghash_setkey, | |
139 | .descsize = sizeof(struct ghash_desc_ctx), | |
140 | .base = { | |
71f89917 | 141 | .cra_name = "__ghash", |
f1e866b1 AB |
142 | .cra_driver_name = "__driver-ghash-ce", |
143 | .cra_priority = 0, | |
4b3f4e37 | 144 | .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_INTERNAL, |
f1e866b1 AB |
145 | .cra_blocksize = GHASH_BLOCK_SIZE, |
146 | .cra_ctxsize = sizeof(struct ghash_key), | |
147 | .cra_module = THIS_MODULE, | |
148 | }, | |
149 | }; | |
150 | ||
151 | static int ghash_async_init(struct ahash_request *req) | |
152 | { | |
153 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
154 | struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); | |
155 | struct ahash_request *cryptd_req = ahash_request_ctx(req); | |
156 | struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; | |
820573eb HX |
157 | struct shash_desc *desc = cryptd_shash_desc(cryptd_req); |
158 | struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); | |
f1e866b1 | 159 | |
820573eb HX |
160 | desc->tfm = child; |
161 | desc->flags = req->base.flags; | |
162 | return crypto_shash_init(desc); | |
f1e866b1 AB |
163 | } |
164 | ||
165 | static int ghash_async_update(struct ahash_request *req) | |
166 | { | |
167 | struct ahash_request *cryptd_req = ahash_request_ctx(req); | |
820573eb HX |
168 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
169 | struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); | |
170 | struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; | |
f1e866b1 | 171 | |
820573eb HX |
172 | if (!may_use_simd() || |
173 | (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { | |
f1e866b1 AB |
174 | memcpy(cryptd_req, req, sizeof(*req)); |
175 | ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); | |
176 | return crypto_ahash_update(cryptd_req); | |
177 | } else { | |
178 | struct shash_desc *desc = cryptd_shash_desc(cryptd_req); | |
179 | return shash_ahash_update(req, desc); | |
180 | } | |
181 | } | |
182 | ||
183 | static int ghash_async_final(struct ahash_request *req) | |
184 | { | |
185 | struct ahash_request *cryptd_req = ahash_request_ctx(req); | |
820573eb HX |
186 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
187 | struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); | |
188 | struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; | |
f1e866b1 | 189 | |
820573eb HX |
190 | if (!may_use_simd() || |
191 | (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { | |
f1e866b1 AB |
192 | memcpy(cryptd_req, req, sizeof(*req)); |
193 | ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); | |
194 | return crypto_ahash_final(cryptd_req); | |
195 | } else { | |
196 | struct shash_desc *desc = cryptd_shash_desc(cryptd_req); | |
197 | return crypto_shash_final(desc, req->result); | |
198 | } | |
199 | } | |
200 | ||
201 | static int ghash_async_digest(struct ahash_request *req) | |
202 | { | |
203 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
204 | struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); | |
205 | struct ahash_request *cryptd_req = ahash_request_ctx(req); | |
206 | struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; | |
207 | ||
820573eb HX |
208 | if (!may_use_simd() || |
209 | (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { | |
f1e866b1 AB |
210 | memcpy(cryptd_req, req, sizeof(*req)); |
211 | ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); | |
212 | return crypto_ahash_digest(cryptd_req); | |
213 | } else { | |
214 | struct shash_desc *desc = cryptd_shash_desc(cryptd_req); | |
215 | struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); | |
216 | ||
217 | desc->tfm = child; | |
218 | desc->flags = req->base.flags; | |
219 | return shash_ahash_digest(req, desc); | |
220 | } | |
221 | } | |
222 | ||
ed4767d6 AB |
223 | static int ghash_async_import(struct ahash_request *req, const void *in) |
224 | { | |
225 | struct ahash_request *cryptd_req = ahash_request_ctx(req); | |
226 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
227 | struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); | |
228 | struct shash_desc *desc = cryptd_shash_desc(cryptd_req); | |
229 | ||
230 | desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm); | |
231 | desc->flags = req->base.flags; | |
232 | ||
233 | return crypto_shash_import(desc, in); | |
234 | } | |
235 | ||
236 | static int ghash_async_export(struct ahash_request *req, void *out) | |
237 | { | |
238 | struct ahash_request *cryptd_req = ahash_request_ctx(req); | |
239 | struct shash_desc *desc = cryptd_shash_desc(cryptd_req); | |
240 | ||
241 | return crypto_shash_export(desc, out); | |
242 | } | |
243 | ||
f1e866b1 AB |
244 | static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, |
245 | unsigned int keylen) | |
246 | { | |
247 | struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); | |
248 | struct crypto_ahash *child = &ctx->cryptd_tfm->base; | |
249 | int err; | |
250 | ||
251 | crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); | |
252 | crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) | |
253 | & CRYPTO_TFM_REQ_MASK); | |
254 | err = crypto_ahash_setkey(child, key, keylen); | |
255 | crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child) | |
256 | & CRYPTO_TFM_RES_MASK); | |
257 | ||
258 | return err; | |
259 | } | |
260 | ||
261 | static int ghash_async_init_tfm(struct crypto_tfm *tfm) | |
262 | { | |
263 | struct cryptd_ahash *cryptd_tfm; | |
264 | struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); | |
265 | ||
4b3f4e37 SM |
266 | cryptd_tfm = cryptd_alloc_ahash("__driver-ghash-ce", |
267 | CRYPTO_ALG_INTERNAL, | |
268 | CRYPTO_ALG_INTERNAL); | |
f1e866b1 AB |
269 | if (IS_ERR(cryptd_tfm)) |
270 | return PTR_ERR(cryptd_tfm); | |
271 | ctx->cryptd_tfm = cryptd_tfm; | |
272 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), | |
273 | sizeof(struct ahash_request) + | |
274 | crypto_ahash_reqsize(&cryptd_tfm->base)); | |
275 | ||
276 | return 0; | |
277 | } | |
278 | ||
279 | static void ghash_async_exit_tfm(struct crypto_tfm *tfm) | |
280 | { | |
281 | struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); | |
282 | ||
283 | cryptd_free_ahash(ctx->cryptd_tfm); | |
284 | } | |
285 | ||
286 | static struct ahash_alg ghash_async_alg = { | |
287 | .init = ghash_async_init, | |
288 | .update = ghash_async_update, | |
289 | .final = ghash_async_final, | |
290 | .setkey = ghash_async_setkey, | |
291 | .digest = ghash_async_digest, | |
ed4767d6 AB |
292 | .import = ghash_async_import, |
293 | .export = ghash_async_export, | |
f1e866b1 | 294 | .halg.digestsize = GHASH_DIGEST_SIZE, |
ed4767d6 | 295 | .halg.statesize = sizeof(struct ghash_desc_ctx), |
f1e866b1 AB |
296 | .halg.base = { |
297 | .cra_name = "ghash", | |
298 | .cra_driver_name = "ghash-ce", | |
299 | .cra_priority = 300, | |
300 | .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, | |
301 | .cra_blocksize = GHASH_BLOCK_SIZE, | |
302 | .cra_type = &crypto_ahash_type, | |
303 | .cra_ctxsize = sizeof(struct ghash_async_ctx), | |
304 | .cra_module = THIS_MODULE, | |
305 | .cra_init = ghash_async_init_tfm, | |
306 | .cra_exit = ghash_async_exit_tfm, | |
307 | }, | |
308 | }; | |
309 | ||
310 | static int __init ghash_ce_mod_init(void) | |
311 | { | |
312 | int err; | |
313 | ||
314 | if (!(elf_hwcap2 & HWCAP2_PMULL)) | |
315 | return -ENODEV; | |
316 | ||
317 | err = crypto_register_shash(&ghash_alg); | |
318 | if (err) | |
319 | return err; | |
320 | err = crypto_register_ahash(&ghash_async_alg); | |
321 | if (err) | |
322 | goto err_shash; | |
323 | ||
324 | return 0; | |
325 | ||
326 | err_shash: | |
327 | crypto_unregister_shash(&ghash_alg); | |
328 | return err; | |
329 | } | |
330 | ||
331 | static void __exit ghash_ce_mod_exit(void) | |
332 | { | |
333 | crypto_unregister_ahash(&ghash_async_alg); | |
334 | crypto_unregister_shash(&ghash_alg); | |
335 | } | |
336 | ||
337 | module_init(ghash_ce_mod_init); | |
338 | module_exit(ghash_ce_mod_exit); |