Commit | Line | Data |
---|---|---|
124b53d0 HX |
1 | /* |
2 | * Software async crypto daemon. | |
3 | * | |
4 | * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License as published by the Free | |
8 | * Software Foundation; either version 2 of the License, or (at your option) | |
9 | * any later version. | |
10 | * | |
11 | */ | |
12 | ||
13 | #include <crypto/algapi.h> | |
18e33e6d | 14 | #include <crypto/internal/hash.h> |
1cac2cbc | 15 | #include <crypto/cryptd.h> |
254eff77 | 16 | #include <crypto/crypto_wq.h> |
124b53d0 HX |
17 | #include <linux/err.h> |
18 | #include <linux/init.h> | |
19 | #include <linux/kernel.h> | |
124b53d0 HX |
20 | #include <linux/list.h> |
21 | #include <linux/module.h> | |
124b53d0 HX |
22 | #include <linux/scatterlist.h> |
23 | #include <linux/sched.h> | |
24 | #include <linux/slab.h> | |
124b53d0 | 25 | |
254eff77 | 26 | #define CRYPTD_MAX_CPU_QLEN 100 |
124b53d0 | 27 | |
254eff77 | 28 | struct cryptd_cpu_queue { |
124b53d0 | 29 | struct crypto_queue queue; |
254eff77 HY |
30 | struct work_struct work; |
31 | }; | |
32 | ||
33 | struct cryptd_queue { | |
34 | struct cryptd_cpu_queue *cpu_queue; | |
124b53d0 HX |
35 | }; |
36 | ||
37 | struct cryptd_instance_ctx { | |
38 | struct crypto_spawn spawn; | |
254eff77 | 39 | struct cryptd_queue *queue; |
124b53d0 HX |
40 | }; |
41 | ||
46309d89 HX |
42 | struct hashd_instance_ctx { |
43 | struct crypto_shash_spawn spawn; | |
44 | struct cryptd_queue *queue; | |
45 | }; | |
46 | ||
124b53d0 HX |
47 | struct cryptd_blkcipher_ctx { |
48 | struct crypto_blkcipher *child; | |
49 | }; | |
50 | ||
51 | struct cryptd_blkcipher_request_ctx { | |
52 | crypto_completion_t complete; | |
53 | }; | |
54 | ||
b8a28251 | 55 | struct cryptd_hash_ctx { |
46309d89 | 56 | struct crypto_shash *child; |
b8a28251 LH |
57 | }; |
58 | ||
59 | struct cryptd_hash_request_ctx { | |
60 | crypto_completion_t complete; | |
46309d89 | 61 | struct shash_desc desc; |
b8a28251 | 62 | }; |
124b53d0 | 63 | |
254eff77 HY |
64 | static void cryptd_queue_worker(struct work_struct *work); |
65 | ||
66 | static int cryptd_init_queue(struct cryptd_queue *queue, | |
67 | unsigned int max_cpu_qlen) | |
68 | { | |
69 | int cpu; | |
70 | struct cryptd_cpu_queue *cpu_queue; | |
71 | ||
72 | queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); | |
73 | if (!queue->cpu_queue) | |
74 | return -ENOMEM; | |
75 | for_each_possible_cpu(cpu) { | |
76 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
77 | crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); | |
78 | INIT_WORK(&cpu_queue->work, cryptd_queue_worker); | |
79 | } | |
80 | return 0; | |
81 | } | |
82 | ||
83 | static void cryptd_fini_queue(struct cryptd_queue *queue) | |
84 | { | |
85 | int cpu; | |
86 | struct cryptd_cpu_queue *cpu_queue; | |
87 | ||
88 | for_each_possible_cpu(cpu) { | |
89 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
90 | BUG_ON(cpu_queue->queue.qlen); | |
91 | } | |
92 | free_percpu(queue->cpu_queue); | |
93 | } | |
94 | ||
95 | static int cryptd_enqueue_request(struct cryptd_queue *queue, | |
96 | struct crypto_async_request *request) | |
97 | { | |
98 | int cpu, err; | |
99 | struct cryptd_cpu_queue *cpu_queue; | |
100 | ||
101 | cpu = get_cpu(); | |
102 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
103 | err = crypto_enqueue_request(&cpu_queue->queue, request); | |
104 | queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); | |
105 | put_cpu(); | |
106 | ||
107 | return err; | |
108 | } | |
109 | ||
110 | /* Called in workqueue context, do one real cryption work (via | |
111 | * req->complete) and reschedule itself if there are more work to | |
112 | * do. */ | |
113 | static void cryptd_queue_worker(struct work_struct *work) | |
114 | { | |
115 | struct cryptd_cpu_queue *cpu_queue; | |
116 | struct crypto_async_request *req, *backlog; | |
117 | ||
118 | cpu_queue = container_of(work, struct cryptd_cpu_queue, work); | |
119 | /* Only handle one request at a time to avoid hogging crypto | |
120 | * workqueue. preempt_disable/enable is used to prevent | |
121 | * being preempted by cryptd_enqueue_request() */ | |
122 | preempt_disable(); | |
123 | backlog = crypto_get_backlog(&cpu_queue->queue); | |
124 | req = crypto_dequeue_request(&cpu_queue->queue); | |
125 | preempt_enable(); | |
126 | ||
127 | if (!req) | |
128 | return; | |
129 | ||
130 | if (backlog) | |
131 | backlog->complete(backlog, -EINPROGRESS); | |
132 | req->complete(req, 0); | |
133 | ||
134 | if (cpu_queue->queue.qlen) | |
135 | queue_work(kcrypto_wq, &cpu_queue->work); | |
136 | } | |
137 | ||
138 | static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) | |
124b53d0 HX |
139 | { |
140 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
141 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
254eff77 | 142 | return ictx->queue; |
124b53d0 HX |
143 | } |
144 | ||
145 | static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent, | |
146 | const u8 *key, unsigned int keylen) | |
147 | { | |
148 | struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent); | |
149 | struct crypto_blkcipher *child = ctx->child; | |
150 | int err; | |
151 | ||
152 | crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); | |
153 | crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) & | |
154 | CRYPTO_TFM_REQ_MASK); | |
155 | err = crypto_blkcipher_setkey(child, key, keylen); | |
156 | crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) & | |
157 | CRYPTO_TFM_RES_MASK); | |
158 | return err; | |
159 | } | |
160 | ||
161 | static void cryptd_blkcipher_crypt(struct ablkcipher_request *req, | |
162 | struct crypto_blkcipher *child, | |
163 | int err, | |
164 | int (*crypt)(struct blkcipher_desc *desc, | |
165 | struct scatterlist *dst, | |
166 | struct scatterlist *src, | |
167 | unsigned int len)) | |
168 | { | |
169 | struct cryptd_blkcipher_request_ctx *rctx; | |
170 | struct blkcipher_desc desc; | |
171 | ||
172 | rctx = ablkcipher_request_ctx(req); | |
173 | ||
93aa7f8a HX |
174 | if (unlikely(err == -EINPROGRESS)) |
175 | goto out; | |
124b53d0 HX |
176 | |
177 | desc.tfm = child; | |
178 | desc.info = req->info; | |
179 | desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
180 | ||
181 | err = crypt(&desc, req->dst, req->src, req->nbytes); | |
182 | ||
183 | req->base.complete = rctx->complete; | |
184 | ||
93aa7f8a | 185 | out: |
124b53d0 | 186 | local_bh_disable(); |
93aa7f8a | 187 | rctx->complete(&req->base, err); |
124b53d0 HX |
188 | local_bh_enable(); |
189 | } | |
190 | ||
191 | static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err) | |
192 | { | |
193 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); | |
194 | struct crypto_blkcipher *child = ctx->child; | |
195 | ||
196 | cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, | |
197 | crypto_blkcipher_crt(child)->encrypt); | |
198 | } | |
199 | ||
200 | static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err) | |
201 | { | |
202 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); | |
203 | struct crypto_blkcipher *child = ctx->child; | |
204 | ||
205 | cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, | |
206 | crypto_blkcipher_crt(child)->decrypt); | |
207 | } | |
208 | ||
209 | static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req, | |
210 | crypto_completion_t complete) | |
211 | { | |
212 | struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req); | |
213 | struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); | |
254eff77 | 214 | struct cryptd_queue *queue; |
124b53d0 | 215 | |
254eff77 | 216 | queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm)); |
124b53d0 HX |
217 | rctx->complete = req->base.complete; |
218 | req->base.complete = complete; | |
219 | ||
254eff77 | 220 | return cryptd_enqueue_request(queue, &req->base); |
124b53d0 HX |
221 | } |
222 | ||
223 | static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req) | |
224 | { | |
225 | return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt); | |
226 | } | |
227 | ||
228 | static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req) | |
229 | { | |
230 | return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt); | |
231 | } | |
232 | ||
233 | static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm) | |
234 | { | |
235 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
236 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
237 | struct crypto_spawn *spawn = &ictx->spawn; | |
238 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); | |
239 | struct crypto_blkcipher *cipher; | |
240 | ||
241 | cipher = crypto_spawn_blkcipher(spawn); | |
242 | if (IS_ERR(cipher)) | |
243 | return PTR_ERR(cipher); | |
244 | ||
245 | ctx->child = cipher; | |
246 | tfm->crt_ablkcipher.reqsize = | |
247 | sizeof(struct cryptd_blkcipher_request_ctx); | |
248 | return 0; | |
249 | } | |
250 | ||
251 | static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm) | |
252 | { | |
253 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); | |
124b53d0 HX |
254 | |
255 | crypto_free_blkcipher(ctx->child); | |
256 | } | |
257 | ||
0b535adf HX |
258 | static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, |
259 | unsigned int tail) | |
124b53d0 | 260 | { |
0b535adf | 261 | char *p; |
124b53d0 | 262 | struct crypto_instance *inst; |
124b53d0 HX |
263 | int err; |
264 | ||
0b535adf HX |
265 | p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); |
266 | if (!p) | |
267 | return ERR_PTR(-ENOMEM); | |
268 | ||
269 | inst = (void *)(p + head); | |
124b53d0 HX |
270 | |
271 | err = -ENAMETOOLONG; | |
272 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | |
273 | "cryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) | |
274 | goto out_free_inst; | |
275 | ||
124b53d0 HX |
276 | memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); |
277 | ||
278 | inst->alg.cra_priority = alg->cra_priority + 50; | |
279 | inst->alg.cra_blocksize = alg->cra_blocksize; | |
280 | inst->alg.cra_alignmask = alg->cra_alignmask; | |
281 | ||
282 | out: | |
0b535adf | 283 | return p; |
124b53d0 HX |
284 | |
285 | out_free_inst: | |
0b535adf HX |
286 | kfree(p); |
287 | p = ERR_PTR(err); | |
124b53d0 HX |
288 | goto out; |
289 | } | |
290 | ||
9cd899a3 HX |
291 | static int cryptd_create_blkcipher(struct crypto_template *tmpl, |
292 | struct rtattr **tb, | |
293 | struct cryptd_queue *queue) | |
124b53d0 | 294 | { |
46309d89 | 295 | struct cryptd_instance_ctx *ctx; |
124b53d0 HX |
296 | struct crypto_instance *inst; |
297 | struct crypto_alg *alg; | |
46309d89 | 298 | int err; |
124b53d0 HX |
299 | |
300 | alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER, | |
332f8840 | 301 | CRYPTO_ALG_TYPE_MASK); |
124b53d0 | 302 | if (IS_ERR(alg)) |
9cd899a3 | 303 | return PTR_ERR(alg); |
124b53d0 | 304 | |
0b535adf | 305 | inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); |
124b53d0 HX |
306 | if (IS_ERR(inst)) |
307 | goto out_put_alg; | |
308 | ||
46309d89 HX |
309 | ctx = crypto_instance_ctx(inst); |
310 | ctx->queue = queue; | |
311 | ||
312 | err = crypto_init_spawn(&ctx->spawn, alg, inst, | |
313 | CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); | |
314 | if (err) | |
315 | goto out_free_inst; | |
316 | ||
332f8840 | 317 | inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; |
124b53d0 HX |
318 | inst->alg.cra_type = &crypto_ablkcipher_type; |
319 | ||
320 | inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize; | |
321 | inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize; | |
322 | inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize; | |
323 | ||
927eead5 HX |
324 | inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv; |
325 | ||
124b53d0 HX |
326 | inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx); |
327 | ||
328 | inst->alg.cra_init = cryptd_blkcipher_init_tfm; | |
329 | inst->alg.cra_exit = cryptd_blkcipher_exit_tfm; | |
330 | ||
331 | inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey; | |
332 | inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue; | |
333 | inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue; | |
334 | ||
9cd899a3 HX |
335 | err = crypto_register_instance(tmpl, inst); |
336 | if (err) { | |
337 | crypto_drop_spawn(&ctx->spawn); | |
338 | out_free_inst: | |
339 | kfree(inst); | |
340 | } | |
341 | ||
124b53d0 HX |
342 | out_put_alg: |
343 | crypto_mod_put(alg); | |
9cd899a3 | 344 | return err; |
124b53d0 HX |
345 | } |
346 | ||
b8a28251 LH |
347 | static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) |
348 | { | |
349 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
46309d89 HX |
350 | struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); |
351 | struct crypto_shash_spawn *spawn = &ictx->spawn; | |
b8a28251 | 352 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); |
46309d89 | 353 | struct crypto_shash *hash; |
b8a28251 | 354 | |
46309d89 HX |
355 | hash = crypto_spawn_shash(spawn); |
356 | if (IS_ERR(hash)) | |
357 | return PTR_ERR(hash); | |
b8a28251 | 358 | |
46309d89 | 359 | ctx->child = hash; |
0d6669e2 HX |
360 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), |
361 | sizeof(struct cryptd_hash_request_ctx) + | |
362 | crypto_shash_descsize(hash)); | |
b8a28251 LH |
363 | return 0; |
364 | } | |
365 | ||
366 | static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) | |
367 | { | |
368 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
b8a28251 | 369 | |
46309d89 | 370 | crypto_free_shash(ctx->child); |
b8a28251 LH |
371 | } |
372 | ||
373 | static int cryptd_hash_setkey(struct crypto_ahash *parent, | |
374 | const u8 *key, unsigned int keylen) | |
375 | { | |
376 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); | |
46309d89 | 377 | struct crypto_shash *child = ctx->child; |
b8a28251 LH |
378 | int err; |
379 | ||
46309d89 HX |
380 | crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
381 | crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & | |
382 | CRYPTO_TFM_REQ_MASK); | |
383 | err = crypto_shash_setkey(child, key, keylen); | |
384 | crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) & | |
385 | CRYPTO_TFM_RES_MASK); | |
b8a28251 LH |
386 | return err; |
387 | } | |
388 | ||
389 | static int cryptd_hash_enqueue(struct ahash_request *req, | |
390 | crypto_completion_t complete) | |
391 | { | |
392 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
393 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
254eff77 HY |
394 | struct cryptd_queue *queue = |
395 | cryptd_get_queue(crypto_ahash_tfm(tfm)); | |
b8a28251 LH |
396 | |
397 | rctx->complete = req->base.complete; | |
398 | req->base.complete = complete; | |
399 | ||
254eff77 | 400 | return cryptd_enqueue_request(queue, &req->base); |
b8a28251 LH |
401 | } |
402 | ||
403 | static void cryptd_hash_init(struct crypto_async_request *req_async, int err) | |
404 | { | |
46309d89 HX |
405 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
406 | struct crypto_shash *child = ctx->child; | |
407 | struct ahash_request *req = ahash_request_cast(req_async); | |
408 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
409 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
410 | |
411 | if (unlikely(err == -EINPROGRESS)) | |
412 | goto out; | |
413 | ||
46309d89 HX |
414 | desc->tfm = child; |
415 | desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
b8a28251 | 416 | |
46309d89 | 417 | err = crypto_shash_init(desc); |
b8a28251 LH |
418 | |
419 | req->base.complete = rctx->complete; | |
420 | ||
421 | out: | |
422 | local_bh_disable(); | |
423 | rctx->complete(&req->base, err); | |
424 | local_bh_enable(); | |
425 | } | |
426 | ||
427 | static int cryptd_hash_init_enqueue(struct ahash_request *req) | |
428 | { | |
429 | return cryptd_hash_enqueue(req, cryptd_hash_init); | |
430 | } | |
431 | ||
432 | static void cryptd_hash_update(struct crypto_async_request *req_async, int err) | |
433 | { | |
46309d89 | 434 | struct ahash_request *req = ahash_request_cast(req_async); |
b8a28251 | 435 | struct cryptd_hash_request_ctx *rctx; |
b8a28251 LH |
436 | |
437 | rctx = ahash_request_ctx(req); | |
438 | ||
439 | if (unlikely(err == -EINPROGRESS)) | |
440 | goto out; | |
441 | ||
46309d89 | 442 | err = shash_ahash_update(req, &rctx->desc); |
b8a28251 LH |
443 | |
444 | req->base.complete = rctx->complete; | |
445 | ||
446 | out: | |
447 | local_bh_disable(); | |
448 | rctx->complete(&req->base, err); | |
449 | local_bh_enable(); | |
450 | } | |
451 | ||
452 | static int cryptd_hash_update_enqueue(struct ahash_request *req) | |
453 | { | |
454 | return cryptd_hash_enqueue(req, cryptd_hash_update); | |
455 | } | |
456 | ||
457 | static void cryptd_hash_final(struct crypto_async_request *req_async, int err) | |
458 | { | |
46309d89 HX |
459 | struct ahash_request *req = ahash_request_cast(req_async); |
460 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
b8a28251 LH |
461 | |
462 | if (unlikely(err == -EINPROGRESS)) | |
463 | goto out; | |
464 | ||
46309d89 | 465 | err = crypto_shash_final(&rctx->desc, req->result); |
b8a28251 LH |
466 | |
467 | req->base.complete = rctx->complete; | |
468 | ||
469 | out: | |
470 | local_bh_disable(); | |
471 | rctx->complete(&req->base, err); | |
472 | local_bh_enable(); | |
473 | } | |
474 | ||
475 | static int cryptd_hash_final_enqueue(struct ahash_request *req) | |
476 | { | |
477 | return cryptd_hash_enqueue(req, cryptd_hash_final); | |
478 | } | |
479 | ||
480 | static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) | |
481 | { | |
46309d89 HX |
482 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
483 | struct crypto_shash *child = ctx->child; | |
484 | struct ahash_request *req = ahash_request_cast(req_async); | |
485 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
486 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
487 | |
488 | if (unlikely(err == -EINPROGRESS)) | |
489 | goto out; | |
490 | ||
46309d89 HX |
491 | desc->tfm = child; |
492 | desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
b8a28251 | 493 | |
46309d89 | 494 | err = shash_ahash_digest(req, desc); |
b8a28251 LH |
495 | |
496 | req->base.complete = rctx->complete; | |
497 | ||
498 | out: | |
499 | local_bh_disable(); | |
500 | rctx->complete(&req->base, err); | |
501 | local_bh_enable(); | |
502 | } | |
503 | ||
504 | static int cryptd_hash_digest_enqueue(struct ahash_request *req) | |
505 | { | |
506 | return cryptd_hash_enqueue(req, cryptd_hash_digest); | |
507 | } | |
508 | ||
9cd899a3 HX |
509 | static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, |
510 | struct cryptd_queue *queue) | |
b8a28251 | 511 | { |
46309d89 | 512 | struct hashd_instance_ctx *ctx; |
0b535adf | 513 | struct ahash_instance *inst; |
46309d89 | 514 | struct shash_alg *salg; |
b8a28251 | 515 | struct crypto_alg *alg; |
46309d89 | 516 | int err; |
b8a28251 | 517 | |
46309d89 HX |
518 | salg = shash_attr_alg(tb[1], 0, 0); |
519 | if (IS_ERR(salg)) | |
9cd899a3 | 520 | return PTR_ERR(salg); |
b8a28251 | 521 | |
46309d89 | 522 | alg = &salg->base; |
0b535adf HX |
523 | inst = cryptd_alloc_instance(alg, ahash_instance_headroom(), |
524 | sizeof(*ctx)); | |
b8a28251 LH |
525 | if (IS_ERR(inst)) |
526 | goto out_put_alg; | |
527 | ||
0b535adf | 528 | ctx = ahash_instance_ctx(inst); |
46309d89 HX |
529 | ctx->queue = queue; |
530 | ||
0b535adf HX |
531 | err = crypto_init_shash_spawn(&ctx->spawn, salg, |
532 | ahash_crypto_instance(inst)); | |
46309d89 HX |
533 | if (err) |
534 | goto out_free_inst; | |
535 | ||
0b535adf | 536 | inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC; |
b8a28251 | 537 | |
0b535adf HX |
538 | inst->alg.halg.digestsize = salg->digestsize; |
539 | inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); | |
b8a28251 | 540 | |
0b535adf HX |
541 | inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; |
542 | inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; | |
b8a28251 | 543 | |
0b535adf HX |
544 | inst->alg.init = cryptd_hash_init_enqueue; |
545 | inst->alg.update = cryptd_hash_update_enqueue; | |
546 | inst->alg.final = cryptd_hash_final_enqueue; | |
547 | inst->alg.setkey = cryptd_hash_setkey; | |
548 | inst->alg.digest = cryptd_hash_digest_enqueue; | |
b8a28251 | 549 | |
0b535adf | 550 | err = ahash_register_instance(tmpl, inst); |
9cd899a3 HX |
551 | if (err) { |
552 | crypto_drop_shash(&ctx->spawn); | |
553 | out_free_inst: | |
554 | kfree(inst); | |
555 | } | |
556 | ||
b8a28251 LH |
557 | out_put_alg: |
558 | crypto_mod_put(alg); | |
9cd899a3 | 559 | return err; |
b8a28251 LH |
560 | } |
561 | ||
254eff77 | 562 | static struct cryptd_queue queue; |
124b53d0 | 563 | |
9cd899a3 | 564 | static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) |
124b53d0 HX |
565 | { |
566 | struct crypto_attr_type *algt; | |
567 | ||
568 | algt = crypto_get_attr_type(tb); | |
569 | if (IS_ERR(algt)) | |
9cd899a3 | 570 | return PTR_ERR(algt); |
124b53d0 HX |
571 | |
572 | switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { | |
573 | case CRYPTO_ALG_TYPE_BLKCIPHER: | |
9cd899a3 | 574 | return cryptd_create_blkcipher(tmpl, tb, &queue); |
b8a28251 | 575 | case CRYPTO_ALG_TYPE_DIGEST: |
9cd899a3 | 576 | return cryptd_create_hash(tmpl, tb, &queue); |
124b53d0 HX |
577 | } |
578 | ||
9cd899a3 | 579 | return -EINVAL; |
124b53d0 HX |
580 | } |
581 | ||
582 | static void cryptd_free(struct crypto_instance *inst) | |
583 | { | |
584 | struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst); | |
0b535adf HX |
585 | struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); |
586 | ||
587 | switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { | |
588 | case CRYPTO_ALG_TYPE_AHASH: | |
589 | crypto_drop_shash(&hctx->spawn); | |
590 | kfree(ahash_instance(inst)); | |
591 | return; | |
592 | } | |
124b53d0 HX |
593 | |
594 | crypto_drop_spawn(&ctx->spawn); | |
595 | kfree(inst); | |
596 | } | |
597 | ||
598 | static struct crypto_template cryptd_tmpl = { | |
599 | .name = "cryptd", | |
9cd899a3 | 600 | .create = cryptd_create, |
124b53d0 HX |
601 | .free = cryptd_free, |
602 | .module = THIS_MODULE, | |
603 | }; | |
604 | ||
1cac2cbc HY |
605 | struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, |
606 | u32 type, u32 mask) | |
607 | { | |
608 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
505fd21d | 609 | struct crypto_tfm *tfm; |
1cac2cbc HY |
610 | |
611 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
612 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
613 | return ERR_PTR(-EINVAL); | |
505fd21d HY |
614 | type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV); |
615 | type |= CRYPTO_ALG_TYPE_BLKCIPHER; | |
616 | mask &= ~CRYPTO_ALG_TYPE_MASK; | |
617 | mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK); | |
618 | tfm = crypto_alloc_base(cryptd_alg_name, type, mask); | |
1cac2cbc HY |
619 | if (IS_ERR(tfm)) |
620 | return ERR_CAST(tfm); | |
505fd21d HY |
621 | if (tfm->__crt_alg->cra_module != THIS_MODULE) { |
622 | crypto_free_tfm(tfm); | |
1cac2cbc HY |
623 | return ERR_PTR(-EINVAL); |
624 | } | |
625 | ||
505fd21d | 626 | return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm)); |
1cac2cbc HY |
627 | } |
628 | EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher); | |
629 | ||
630 | struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm) | |
631 | { | |
632 | struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); | |
633 | return ctx->child; | |
634 | } | |
635 | EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child); | |
636 | ||
637 | void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) | |
638 | { | |
639 | crypto_free_ablkcipher(&tfm->base); | |
640 | } | |
641 | EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); | |
642 | ||
124b53d0 HX |
643 | static int __init cryptd_init(void) |
644 | { | |
645 | int err; | |
646 | ||
254eff77 | 647 | err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN); |
124b53d0 HX |
648 | if (err) |
649 | return err; | |
650 | ||
651 | err = crypto_register_template(&cryptd_tmpl); | |
652 | if (err) | |
254eff77 | 653 | cryptd_fini_queue(&queue); |
124b53d0 HX |
654 | |
655 | return err; | |
656 | } | |
657 | ||
658 | static void __exit cryptd_exit(void) | |
659 | { | |
254eff77 | 660 | cryptd_fini_queue(&queue); |
124b53d0 HX |
661 | crypto_unregister_template(&cryptd_tmpl); |
662 | } | |
663 | ||
664 | module_init(cryptd_init); | |
665 | module_exit(cryptd_exit); | |
666 | ||
667 | MODULE_LICENSE("GPL"); | |
668 | MODULE_DESCRIPTION("Software async crypto daemon"); |