Commit | Line | Data |
---|---|---|
124b53d0 HX |
1 | /* |
2 | * Software async crypto daemon. | |
3 | * | |
4 | * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> | |
5 | * | |
298c926c AH |
6 | * Added AEAD support to cryptd. |
7 | * Authors: Tadeusz Struk (tadeusz.struk@intel.com) | |
8 | * Adrian Hoban <adrian.hoban@intel.com> | |
9 | * Gabriele Paoloni <gabriele.paoloni@intel.com> | |
10 | * Aidan O'Mahony (aidan.o.mahony@intel.com) | |
11 | * Copyright (c) 2010, Intel Corporation. | |
12 | * | |
124b53d0 HX |
13 | * This program is free software; you can redistribute it and/or modify it |
14 | * under the terms of the GNU General Public License as published by the Free | |
15 | * Software Foundation; either version 2 of the License, or (at your option) | |
16 | * any later version. | |
17 | * | |
18 | */ | |
19 | ||
20 | #include <crypto/algapi.h> | |
18e33e6d | 21 | #include <crypto/internal/hash.h> |
298c926c | 22 | #include <crypto/internal/aead.h> |
1cac2cbc | 23 | #include <crypto/cryptd.h> |
254eff77 | 24 | #include <crypto/crypto_wq.h> |
81760ea6 | 25 | #include <linux/atomic.h> |
124b53d0 HX |
26 | #include <linux/err.h> |
27 | #include <linux/init.h> | |
28 | #include <linux/kernel.h> | |
124b53d0 HX |
29 | #include <linux/list.h> |
30 | #include <linux/module.h> | |
124b53d0 HX |
31 | #include <linux/scatterlist.h> |
32 | #include <linux/sched.h> | |
33 | #include <linux/slab.h> | |
124b53d0 | 34 | |
81760ea6 | 35 | #define CRYPTD_MAX_CPU_QLEN 1000 |
124b53d0 | 36 | |
254eff77 | 37 | struct cryptd_cpu_queue { |
124b53d0 | 38 | struct crypto_queue queue; |
254eff77 HY |
39 | struct work_struct work; |
40 | }; | |
41 | ||
42 | struct cryptd_queue { | |
a29d8b8e | 43 | struct cryptd_cpu_queue __percpu *cpu_queue; |
124b53d0 HX |
44 | }; |
45 | ||
46 | struct cryptd_instance_ctx { | |
47 | struct crypto_spawn spawn; | |
254eff77 | 48 | struct cryptd_queue *queue; |
124b53d0 HX |
49 | }; |
50 | ||
46309d89 HX |
51 | struct hashd_instance_ctx { |
52 | struct crypto_shash_spawn spawn; | |
53 | struct cryptd_queue *queue; | |
54 | }; | |
55 | ||
298c926c AH |
56 | struct aead_instance_ctx { |
57 | struct crypto_aead_spawn aead_spawn; | |
58 | struct cryptd_queue *queue; | |
59 | }; | |
60 | ||
124b53d0 | 61 | struct cryptd_blkcipher_ctx { |
81760ea6 | 62 | atomic_t refcnt; |
124b53d0 HX |
63 | struct crypto_blkcipher *child; |
64 | }; | |
65 | ||
66 | struct cryptd_blkcipher_request_ctx { | |
67 | crypto_completion_t complete; | |
68 | }; | |
69 | ||
b8a28251 | 70 | struct cryptd_hash_ctx { |
81760ea6 | 71 | atomic_t refcnt; |
46309d89 | 72 | struct crypto_shash *child; |
b8a28251 LH |
73 | }; |
74 | ||
75 | struct cryptd_hash_request_ctx { | |
76 | crypto_completion_t complete; | |
46309d89 | 77 | struct shash_desc desc; |
b8a28251 | 78 | }; |
124b53d0 | 79 | |
298c926c | 80 | struct cryptd_aead_ctx { |
81760ea6 | 81 | atomic_t refcnt; |
298c926c AH |
82 | struct crypto_aead *child; |
83 | }; | |
84 | ||
85 | struct cryptd_aead_request_ctx { | |
86 | crypto_completion_t complete; | |
87 | }; | |
88 | ||
254eff77 HY |
89 | static void cryptd_queue_worker(struct work_struct *work); |
90 | ||
91 | static int cryptd_init_queue(struct cryptd_queue *queue, | |
92 | unsigned int max_cpu_qlen) | |
93 | { | |
94 | int cpu; | |
95 | struct cryptd_cpu_queue *cpu_queue; | |
96 | ||
97 | queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); | |
98 | if (!queue->cpu_queue) | |
99 | return -ENOMEM; | |
100 | for_each_possible_cpu(cpu) { | |
101 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
102 | crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); | |
103 | INIT_WORK(&cpu_queue->work, cryptd_queue_worker); | |
104 | } | |
105 | return 0; | |
106 | } | |
107 | ||
108 | static void cryptd_fini_queue(struct cryptd_queue *queue) | |
109 | { | |
110 | int cpu; | |
111 | struct cryptd_cpu_queue *cpu_queue; | |
112 | ||
113 | for_each_possible_cpu(cpu) { | |
114 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
115 | BUG_ON(cpu_queue->queue.qlen); | |
116 | } | |
117 | free_percpu(queue->cpu_queue); | |
118 | } | |
119 | ||
120 | static int cryptd_enqueue_request(struct cryptd_queue *queue, | |
121 | struct crypto_async_request *request) | |
122 | { | |
123 | int cpu, err; | |
124 | struct cryptd_cpu_queue *cpu_queue; | |
81760ea6 HX |
125 | struct crypto_tfm *tfm; |
126 | atomic_t *refcnt; | |
127 | bool may_backlog; | |
254eff77 HY |
128 | |
129 | cpu = get_cpu(); | |
0b44f486 | 130 | cpu_queue = this_cpu_ptr(queue->cpu_queue); |
254eff77 | 131 | err = crypto_enqueue_request(&cpu_queue->queue, request); |
81760ea6 HX |
132 | |
133 | refcnt = crypto_tfm_ctx(request->tfm); | |
134 | may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; | |
135 | ||
136 | if (err == -EBUSY && !may_backlog) | |
137 | goto out_put_cpu; | |
138 | ||
254eff77 | 139 | queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); |
81760ea6 HX |
140 | |
141 | if (!atomic_read(refcnt)) | |
142 | goto out_put_cpu; | |
143 | ||
144 | tfm = request->tfm; | |
145 | atomic_inc(refcnt); | |
146 | ||
147 | out_put_cpu: | |
254eff77 HY |
148 | put_cpu(); |
149 | ||
150 | return err; | |
151 | } | |
152 | ||
153 | /* Called in workqueue context, do one real cryption work (via | |
154 | * req->complete) and reschedule itself if there are more work to | |
155 | * do. */ | |
156 | static void cryptd_queue_worker(struct work_struct *work) | |
157 | { | |
158 | struct cryptd_cpu_queue *cpu_queue; | |
159 | struct crypto_async_request *req, *backlog; | |
160 | ||
161 | cpu_queue = container_of(work, struct cryptd_cpu_queue, work); | |
9efade1b JK |
162 | /* |
163 | * Only handle one request at a time to avoid hogging crypto workqueue. | |
164 | * preempt_disable/enable is used to prevent being preempted by | |
165 | * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent | |
166 | * cryptd_enqueue_request() being accessed from software interrupts. | |
167 | */ | |
168 | local_bh_disable(); | |
254eff77 HY |
169 | preempt_disable(); |
170 | backlog = crypto_get_backlog(&cpu_queue->queue); | |
171 | req = crypto_dequeue_request(&cpu_queue->queue); | |
172 | preempt_enable(); | |
9efade1b | 173 | local_bh_enable(); |
254eff77 HY |
174 | |
175 | if (!req) | |
176 | return; | |
177 | ||
178 | if (backlog) | |
179 | backlog->complete(backlog, -EINPROGRESS); | |
180 | req->complete(req, 0); | |
181 | ||
182 | if (cpu_queue->queue.qlen) | |
183 | queue_work(kcrypto_wq, &cpu_queue->work); | |
184 | } | |
185 | ||
186 | static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) | |
124b53d0 HX |
187 | { |
188 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
189 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
254eff77 | 190 | return ictx->queue; |
124b53d0 HX |
191 | } |
192 | ||
466a7b9e SM |
193 | static inline void cryptd_check_internal(struct rtattr **tb, u32 *type, |
194 | u32 *mask) | |
195 | { | |
196 | struct crypto_attr_type *algt; | |
197 | ||
198 | algt = crypto_get_attr_type(tb); | |
199 | if (IS_ERR(algt)) | |
200 | return; | |
f6da3205 | 201 | |
5e4b8c1f HX |
202 | *type |= algt->type & CRYPTO_ALG_INTERNAL; |
203 | *mask |= algt->mask & CRYPTO_ALG_INTERNAL; | |
466a7b9e SM |
204 | } |
205 | ||
124b53d0 HX |
206 | static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent, |
207 | const u8 *key, unsigned int keylen) | |
208 | { | |
209 | struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent); | |
210 | struct crypto_blkcipher *child = ctx->child; | |
211 | int err; | |
212 | ||
213 | crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); | |
214 | crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) & | |
215 | CRYPTO_TFM_REQ_MASK); | |
216 | err = crypto_blkcipher_setkey(child, key, keylen); | |
217 | crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) & | |
218 | CRYPTO_TFM_RES_MASK); | |
219 | return err; | |
220 | } | |
221 | ||
222 | static void cryptd_blkcipher_crypt(struct ablkcipher_request *req, | |
223 | struct crypto_blkcipher *child, | |
224 | int err, | |
225 | int (*crypt)(struct blkcipher_desc *desc, | |
226 | struct scatterlist *dst, | |
227 | struct scatterlist *src, | |
228 | unsigned int len)) | |
229 | { | |
230 | struct cryptd_blkcipher_request_ctx *rctx; | |
81760ea6 HX |
231 | struct cryptd_blkcipher_ctx *ctx; |
232 | struct crypto_ablkcipher *tfm; | |
124b53d0 | 233 | struct blkcipher_desc desc; |
81760ea6 | 234 | int refcnt; |
124b53d0 HX |
235 | |
236 | rctx = ablkcipher_request_ctx(req); | |
237 | ||
93aa7f8a HX |
238 | if (unlikely(err == -EINPROGRESS)) |
239 | goto out; | |
124b53d0 HX |
240 | |
241 | desc.tfm = child; | |
242 | desc.info = req->info; | |
243 | desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
244 | ||
245 | err = crypt(&desc, req->dst, req->src, req->nbytes); | |
246 | ||
247 | req->base.complete = rctx->complete; | |
248 | ||
93aa7f8a | 249 | out: |
81760ea6 HX |
250 | tfm = crypto_ablkcipher_reqtfm(req); |
251 | ctx = crypto_ablkcipher_ctx(tfm); | |
252 | refcnt = atomic_read(&ctx->refcnt); | |
253 | ||
124b53d0 | 254 | local_bh_disable(); |
93aa7f8a | 255 | rctx->complete(&req->base, err); |
124b53d0 | 256 | local_bh_enable(); |
81760ea6 HX |
257 | |
258 | if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt)) | |
259 | crypto_free_ablkcipher(tfm); | |
124b53d0 HX |
260 | } |
261 | ||
262 | static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err) | |
263 | { | |
264 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); | |
265 | struct crypto_blkcipher *child = ctx->child; | |
266 | ||
267 | cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, | |
268 | crypto_blkcipher_crt(child)->encrypt); | |
269 | } | |
270 | ||
271 | static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err) | |
272 | { | |
273 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); | |
274 | struct crypto_blkcipher *child = ctx->child; | |
275 | ||
276 | cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, | |
277 | crypto_blkcipher_crt(child)->decrypt); | |
278 | } | |
279 | ||
280 | static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req, | |
3e3dc25f | 281 | crypto_completion_t compl) |
124b53d0 HX |
282 | { |
283 | struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req); | |
284 | struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); | |
254eff77 | 285 | struct cryptd_queue *queue; |
124b53d0 | 286 | |
254eff77 | 287 | queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm)); |
124b53d0 | 288 | rctx->complete = req->base.complete; |
3e3dc25f | 289 | req->base.complete = compl; |
124b53d0 | 290 | |
254eff77 | 291 | return cryptd_enqueue_request(queue, &req->base); |
124b53d0 HX |
292 | } |
293 | ||
294 | static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req) | |
295 | { | |
296 | return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt); | |
297 | } | |
298 | ||
299 | static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req) | |
300 | { | |
301 | return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt); | |
302 | } | |
303 | ||
304 | static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm) | |
305 | { | |
306 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
307 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
308 | struct crypto_spawn *spawn = &ictx->spawn; | |
309 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); | |
310 | struct crypto_blkcipher *cipher; | |
311 | ||
312 | cipher = crypto_spawn_blkcipher(spawn); | |
313 | if (IS_ERR(cipher)) | |
314 | return PTR_ERR(cipher); | |
315 | ||
316 | ctx->child = cipher; | |
317 | tfm->crt_ablkcipher.reqsize = | |
318 | sizeof(struct cryptd_blkcipher_request_ctx); | |
319 | return 0; | |
320 | } | |
321 | ||
322 | static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm) | |
323 | { | |
324 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); | |
124b53d0 HX |
325 | |
326 | crypto_free_blkcipher(ctx->child); | |
327 | } | |
328 | ||
9b8c456e HX |
329 | static int cryptd_init_instance(struct crypto_instance *inst, |
330 | struct crypto_alg *alg) | |
331 | { | |
332 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | |
333 | "cryptd(%s)", | |
334 | alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) | |
335 | return -ENAMETOOLONG; | |
336 | ||
337 | memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); | |
338 | ||
339 | inst->alg.cra_priority = alg->cra_priority + 50; | |
340 | inst->alg.cra_blocksize = alg->cra_blocksize; | |
341 | inst->alg.cra_alignmask = alg->cra_alignmask; | |
342 | ||
343 | return 0; | |
344 | } | |
345 | ||
0b535adf HX |
346 | static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, |
347 | unsigned int tail) | |
124b53d0 | 348 | { |
0b535adf | 349 | char *p; |
124b53d0 | 350 | struct crypto_instance *inst; |
124b53d0 HX |
351 | int err; |
352 | ||
0b535adf HX |
353 | p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); |
354 | if (!p) | |
355 | return ERR_PTR(-ENOMEM); | |
356 | ||
357 | inst = (void *)(p + head); | |
124b53d0 | 358 | |
9b8c456e HX |
359 | err = cryptd_init_instance(inst, alg); |
360 | if (err) | |
124b53d0 HX |
361 | goto out_free_inst; |
362 | ||
124b53d0 | 363 | out: |
0b535adf | 364 | return p; |
124b53d0 HX |
365 | |
366 | out_free_inst: | |
0b535adf HX |
367 | kfree(p); |
368 | p = ERR_PTR(err); | |
124b53d0 HX |
369 | goto out; |
370 | } | |
371 | ||
9cd899a3 HX |
372 | static int cryptd_create_blkcipher(struct crypto_template *tmpl, |
373 | struct rtattr **tb, | |
374 | struct cryptd_queue *queue) | |
124b53d0 | 375 | { |
46309d89 | 376 | struct cryptd_instance_ctx *ctx; |
124b53d0 HX |
377 | struct crypto_instance *inst; |
378 | struct crypto_alg *alg; | |
466a7b9e SM |
379 | u32 type = CRYPTO_ALG_TYPE_BLKCIPHER; |
380 | u32 mask = CRYPTO_ALG_TYPE_MASK; | |
46309d89 | 381 | int err; |
124b53d0 | 382 | |
466a7b9e SM |
383 | cryptd_check_internal(tb, &type, &mask); |
384 | ||
385 | alg = crypto_get_attr_alg(tb, type, mask); | |
124b53d0 | 386 | if (IS_ERR(alg)) |
9cd899a3 | 387 | return PTR_ERR(alg); |
124b53d0 | 388 | |
0b535adf | 389 | inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); |
05ed8758 | 390 | err = PTR_ERR(inst); |
124b53d0 HX |
391 | if (IS_ERR(inst)) |
392 | goto out_put_alg; | |
393 | ||
46309d89 HX |
394 | ctx = crypto_instance_ctx(inst); |
395 | ctx->queue = queue; | |
396 | ||
397 | err = crypto_init_spawn(&ctx->spawn, alg, inst, | |
398 | CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); | |
399 | if (err) | |
400 | goto out_free_inst; | |
401 | ||
466a7b9e SM |
402 | type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; |
403 | if (alg->cra_flags & CRYPTO_ALG_INTERNAL) | |
404 | type |= CRYPTO_ALG_INTERNAL; | |
405 | inst->alg.cra_flags = type; | |
124b53d0 HX |
406 | inst->alg.cra_type = &crypto_ablkcipher_type; |
407 | ||
408 | inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize; | |
409 | inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize; | |
410 | inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize; | |
411 | ||
927eead5 HX |
412 | inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv; |
413 | ||
124b53d0 HX |
414 | inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx); |
415 | ||
416 | inst->alg.cra_init = cryptd_blkcipher_init_tfm; | |
417 | inst->alg.cra_exit = cryptd_blkcipher_exit_tfm; | |
418 | ||
419 | inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey; | |
420 | inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue; | |
421 | inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue; | |
422 | ||
9cd899a3 HX |
423 | err = crypto_register_instance(tmpl, inst); |
424 | if (err) { | |
425 | crypto_drop_spawn(&ctx->spawn); | |
426 | out_free_inst: | |
427 | kfree(inst); | |
428 | } | |
429 | ||
124b53d0 HX |
430 | out_put_alg: |
431 | crypto_mod_put(alg); | |
9cd899a3 | 432 | return err; |
124b53d0 HX |
433 | } |
434 | ||
b8a28251 LH |
435 | static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) |
436 | { | |
437 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
46309d89 HX |
438 | struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); |
439 | struct crypto_shash_spawn *spawn = &ictx->spawn; | |
b8a28251 | 440 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); |
46309d89 | 441 | struct crypto_shash *hash; |
b8a28251 | 442 | |
46309d89 HX |
443 | hash = crypto_spawn_shash(spawn); |
444 | if (IS_ERR(hash)) | |
445 | return PTR_ERR(hash); | |
b8a28251 | 446 | |
46309d89 | 447 | ctx->child = hash; |
0d6669e2 HX |
448 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), |
449 | sizeof(struct cryptd_hash_request_ctx) + | |
450 | crypto_shash_descsize(hash)); | |
b8a28251 LH |
451 | return 0; |
452 | } | |
453 | ||
454 | static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) | |
455 | { | |
456 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
b8a28251 | 457 | |
46309d89 | 458 | crypto_free_shash(ctx->child); |
b8a28251 LH |
459 | } |
460 | ||
461 | static int cryptd_hash_setkey(struct crypto_ahash *parent, | |
462 | const u8 *key, unsigned int keylen) | |
463 | { | |
464 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); | |
46309d89 | 465 | struct crypto_shash *child = ctx->child; |
b8a28251 LH |
466 | int err; |
467 | ||
46309d89 HX |
468 | crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
469 | crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & | |
470 | CRYPTO_TFM_REQ_MASK); | |
471 | err = crypto_shash_setkey(child, key, keylen); | |
472 | crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) & | |
473 | CRYPTO_TFM_RES_MASK); | |
b8a28251 LH |
474 | return err; |
475 | } | |
476 | ||
477 | static int cryptd_hash_enqueue(struct ahash_request *req, | |
3e3dc25f | 478 | crypto_completion_t compl) |
b8a28251 LH |
479 | { |
480 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
481 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
254eff77 HY |
482 | struct cryptd_queue *queue = |
483 | cryptd_get_queue(crypto_ahash_tfm(tfm)); | |
b8a28251 LH |
484 | |
485 | rctx->complete = req->base.complete; | |
3e3dc25f | 486 | req->base.complete = compl; |
b8a28251 | 487 | |
254eff77 | 488 | return cryptd_enqueue_request(queue, &req->base); |
b8a28251 LH |
489 | } |
490 | ||
81760ea6 HX |
491 | static void cryptd_hash_complete(struct ahash_request *req, int err) |
492 | { | |
493 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
494 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
495 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
496 | int refcnt = atomic_read(&ctx->refcnt); | |
497 | ||
498 | local_bh_disable(); | |
499 | rctx->complete(&req->base, err); | |
500 | local_bh_enable(); | |
501 | ||
502 | if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt)) | |
503 | crypto_free_ahash(tfm); | |
504 | } | |
505 | ||
b8a28251 LH |
506 | static void cryptd_hash_init(struct crypto_async_request *req_async, int err) |
507 | { | |
46309d89 HX |
508 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
509 | struct crypto_shash *child = ctx->child; | |
510 | struct ahash_request *req = ahash_request_cast(req_async); | |
511 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
512 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
513 | |
514 | if (unlikely(err == -EINPROGRESS)) | |
515 | goto out; | |
516 | ||
46309d89 HX |
517 | desc->tfm = child; |
518 | desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
b8a28251 | 519 | |
46309d89 | 520 | err = crypto_shash_init(desc); |
b8a28251 LH |
521 | |
522 | req->base.complete = rctx->complete; | |
523 | ||
524 | out: | |
81760ea6 | 525 | cryptd_hash_complete(req, err); |
b8a28251 LH |
526 | } |
527 | ||
528 | static int cryptd_hash_init_enqueue(struct ahash_request *req) | |
529 | { | |
530 | return cryptd_hash_enqueue(req, cryptd_hash_init); | |
531 | } | |
532 | ||
533 | static void cryptd_hash_update(struct crypto_async_request *req_async, int err) | |
534 | { | |
46309d89 | 535 | struct ahash_request *req = ahash_request_cast(req_async); |
b8a28251 | 536 | struct cryptd_hash_request_ctx *rctx; |
b8a28251 LH |
537 | |
538 | rctx = ahash_request_ctx(req); | |
539 | ||
540 | if (unlikely(err == -EINPROGRESS)) | |
541 | goto out; | |
542 | ||
46309d89 | 543 | err = shash_ahash_update(req, &rctx->desc); |
b8a28251 LH |
544 | |
545 | req->base.complete = rctx->complete; | |
546 | ||
547 | out: | |
81760ea6 | 548 | cryptd_hash_complete(req, err); |
b8a28251 LH |
549 | } |
550 | ||
551 | static int cryptd_hash_update_enqueue(struct ahash_request *req) | |
552 | { | |
553 | return cryptd_hash_enqueue(req, cryptd_hash_update); | |
554 | } | |
555 | ||
556 | static void cryptd_hash_final(struct crypto_async_request *req_async, int err) | |
557 | { | |
46309d89 HX |
558 | struct ahash_request *req = ahash_request_cast(req_async); |
559 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
b8a28251 LH |
560 | |
561 | if (unlikely(err == -EINPROGRESS)) | |
562 | goto out; | |
563 | ||
46309d89 | 564 | err = crypto_shash_final(&rctx->desc, req->result); |
b8a28251 LH |
565 | |
566 | req->base.complete = rctx->complete; | |
567 | ||
568 | out: | |
81760ea6 | 569 | cryptd_hash_complete(req, err); |
b8a28251 LH |
570 | } |
571 | ||
572 | static int cryptd_hash_final_enqueue(struct ahash_request *req) | |
573 | { | |
574 | return cryptd_hash_enqueue(req, cryptd_hash_final); | |
575 | } | |
576 | ||
6fba00d1 HX |
577 | static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) |
578 | { | |
579 | struct ahash_request *req = ahash_request_cast(req_async); | |
580 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
581 | ||
582 | if (unlikely(err == -EINPROGRESS)) | |
583 | goto out; | |
584 | ||
585 | err = shash_ahash_finup(req, &rctx->desc); | |
586 | ||
587 | req->base.complete = rctx->complete; | |
588 | ||
589 | out: | |
81760ea6 | 590 | cryptd_hash_complete(req, err); |
6fba00d1 HX |
591 | } |
592 | ||
593 | static int cryptd_hash_finup_enqueue(struct ahash_request *req) | |
594 | { | |
595 | return cryptd_hash_enqueue(req, cryptd_hash_finup); | |
596 | } | |
597 | ||
b8a28251 LH |
598 | static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) |
599 | { | |
46309d89 HX |
600 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
601 | struct crypto_shash *child = ctx->child; | |
602 | struct ahash_request *req = ahash_request_cast(req_async); | |
603 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
604 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
605 | |
606 | if (unlikely(err == -EINPROGRESS)) | |
607 | goto out; | |
608 | ||
46309d89 HX |
609 | desc->tfm = child; |
610 | desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
b8a28251 | 611 | |
46309d89 | 612 | err = shash_ahash_digest(req, desc); |
b8a28251 LH |
613 | |
614 | req->base.complete = rctx->complete; | |
615 | ||
616 | out: | |
81760ea6 | 617 | cryptd_hash_complete(req, err); |
b8a28251 LH |
618 | } |
619 | ||
620 | static int cryptd_hash_digest_enqueue(struct ahash_request *req) | |
621 | { | |
622 | return cryptd_hash_enqueue(req, cryptd_hash_digest); | |
623 | } | |
624 | ||
6fba00d1 HX |
625 | static int cryptd_hash_export(struct ahash_request *req, void *out) |
626 | { | |
627 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
628 | ||
629 | return crypto_shash_export(&rctx->desc, out); | |
630 | } | |
631 | ||
632 | static int cryptd_hash_import(struct ahash_request *req, const void *in) | |
633 | { | |
0bd22235 AB |
634 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
635 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
636 | struct shash_desc *desc = cryptd_shash_desc(req); | |
637 | ||
638 | desc->tfm = ctx->child; | |
639 | desc->flags = req->base.flags; | |
6fba00d1 | 640 | |
0bd22235 | 641 | return crypto_shash_import(desc, in); |
6fba00d1 HX |
642 | } |
643 | ||
9cd899a3 HX |
644 | static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, |
645 | struct cryptd_queue *queue) | |
b8a28251 | 646 | { |
46309d89 | 647 | struct hashd_instance_ctx *ctx; |
0b535adf | 648 | struct ahash_instance *inst; |
46309d89 | 649 | struct shash_alg *salg; |
b8a28251 | 650 | struct crypto_alg *alg; |
466a7b9e SM |
651 | u32 type = 0; |
652 | u32 mask = 0; | |
46309d89 | 653 | int err; |
b8a28251 | 654 | |
466a7b9e SM |
655 | cryptd_check_internal(tb, &type, &mask); |
656 | ||
657 | salg = shash_attr_alg(tb[1], type, mask); | |
46309d89 | 658 | if (IS_ERR(salg)) |
9cd899a3 | 659 | return PTR_ERR(salg); |
b8a28251 | 660 | |
46309d89 | 661 | alg = &salg->base; |
0b535adf HX |
662 | inst = cryptd_alloc_instance(alg, ahash_instance_headroom(), |
663 | sizeof(*ctx)); | |
05ed8758 | 664 | err = PTR_ERR(inst); |
b8a28251 LH |
665 | if (IS_ERR(inst)) |
666 | goto out_put_alg; | |
667 | ||
0b535adf | 668 | ctx = ahash_instance_ctx(inst); |
46309d89 HX |
669 | ctx->queue = queue; |
670 | ||
0b535adf HX |
671 | err = crypto_init_shash_spawn(&ctx->spawn, salg, |
672 | ahash_crypto_instance(inst)); | |
46309d89 HX |
673 | if (err) |
674 | goto out_free_inst; | |
675 | ||
466a7b9e SM |
676 | type = CRYPTO_ALG_ASYNC; |
677 | if (alg->cra_flags & CRYPTO_ALG_INTERNAL) | |
678 | type |= CRYPTO_ALG_INTERNAL; | |
679 | inst->alg.halg.base.cra_flags = type; | |
b8a28251 | 680 | |
0b535adf | 681 | inst->alg.halg.digestsize = salg->digestsize; |
1a078340 | 682 | inst->alg.halg.statesize = salg->statesize; |
0b535adf | 683 | inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); |
b8a28251 | 684 | |
0b535adf HX |
685 | inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; |
686 | inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; | |
b8a28251 | 687 | |
0b535adf HX |
688 | inst->alg.init = cryptd_hash_init_enqueue; |
689 | inst->alg.update = cryptd_hash_update_enqueue; | |
690 | inst->alg.final = cryptd_hash_final_enqueue; | |
6fba00d1 HX |
691 | inst->alg.finup = cryptd_hash_finup_enqueue; |
692 | inst->alg.export = cryptd_hash_export; | |
693 | inst->alg.import = cryptd_hash_import; | |
0b535adf HX |
694 | inst->alg.setkey = cryptd_hash_setkey; |
695 | inst->alg.digest = cryptd_hash_digest_enqueue; | |
b8a28251 | 696 | |
0b535adf | 697 | err = ahash_register_instance(tmpl, inst); |
9cd899a3 HX |
698 | if (err) { |
699 | crypto_drop_shash(&ctx->spawn); | |
700 | out_free_inst: | |
701 | kfree(inst); | |
702 | } | |
703 | ||
b8a28251 LH |
704 | out_put_alg: |
705 | crypto_mod_put(alg); | |
9cd899a3 | 706 | return err; |
b8a28251 LH |
707 | } |
708 | ||
92b9876b HX |
709 | static int cryptd_aead_setkey(struct crypto_aead *parent, |
710 | const u8 *key, unsigned int keylen) | |
711 | { | |
712 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
713 | struct crypto_aead *child = ctx->child; | |
714 | ||
715 | return crypto_aead_setkey(child, key, keylen); | |
716 | } | |
717 | ||
718 | static int cryptd_aead_setauthsize(struct crypto_aead *parent, | |
719 | unsigned int authsize) | |
720 | { | |
721 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
722 | struct crypto_aead *child = ctx->child; | |
723 | ||
724 | return crypto_aead_setauthsize(child, authsize); | |
725 | } | |
726 | ||
298c926c AH |
727 | static void cryptd_aead_crypt(struct aead_request *req, |
728 | struct crypto_aead *child, | |
729 | int err, | |
730 | int (*crypt)(struct aead_request *req)) | |
731 | { | |
732 | struct cryptd_aead_request_ctx *rctx; | |
81760ea6 | 733 | struct cryptd_aead_ctx *ctx; |
ec9f2006 | 734 | crypto_completion_t compl; |
81760ea6 HX |
735 | struct crypto_aead *tfm; |
736 | int refcnt; | |
ec9f2006 | 737 | |
298c926c | 738 | rctx = aead_request_ctx(req); |
ec9f2006 | 739 | compl = rctx->complete; |
298c926c | 740 | |
31bd44e7 HX |
741 | tfm = crypto_aead_reqtfm(req); |
742 | ||
298c926c AH |
743 | if (unlikely(err == -EINPROGRESS)) |
744 | goto out; | |
745 | aead_request_set_tfm(req, child); | |
746 | err = crypt( req ); | |
81760ea6 | 747 | |
298c926c | 748 | out: |
81760ea6 HX |
749 | ctx = crypto_aead_ctx(tfm); |
750 | refcnt = atomic_read(&ctx->refcnt); | |
751 | ||
298c926c | 752 | local_bh_disable(); |
ec9f2006 | 753 | compl(&req->base, err); |
298c926c | 754 | local_bh_enable(); |
81760ea6 HX |
755 | |
756 | if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt)) | |
757 | crypto_free_aead(tfm); | |
298c926c AH |
758 | } |
759 | ||
760 | static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) | |
761 | { | |
762 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
763 | struct crypto_aead *child = ctx->child; | |
764 | struct aead_request *req; | |
765 | ||
766 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 767 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt); |
298c926c AH |
768 | } |
769 | ||
770 | static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) | |
771 | { | |
772 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
773 | struct crypto_aead *child = ctx->child; | |
774 | struct aead_request *req; | |
775 | ||
776 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 777 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt); |
298c926c AH |
778 | } |
779 | ||
780 | static int cryptd_aead_enqueue(struct aead_request *req, | |
3e3dc25f | 781 | crypto_completion_t compl) |
298c926c AH |
782 | { |
783 | struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); | |
784 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | |
785 | struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); | |
786 | ||
787 | rctx->complete = req->base.complete; | |
3e3dc25f | 788 | req->base.complete = compl; |
298c926c AH |
789 | return cryptd_enqueue_request(queue, &req->base); |
790 | } | |
791 | ||
792 | static int cryptd_aead_encrypt_enqueue(struct aead_request *req) | |
793 | { | |
794 | return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); | |
795 | } | |
796 | ||
797 | static int cryptd_aead_decrypt_enqueue(struct aead_request *req) | |
798 | { | |
799 | return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); | |
800 | } | |
801 | ||
f614e546 | 802 | static int cryptd_aead_init_tfm(struct crypto_aead *tfm) |
298c926c | 803 | { |
f614e546 HX |
804 | struct aead_instance *inst = aead_alg_instance(tfm); |
805 | struct aead_instance_ctx *ictx = aead_instance_ctx(inst); | |
298c926c | 806 | struct crypto_aead_spawn *spawn = &ictx->aead_spawn; |
f614e546 | 807 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
808 | struct crypto_aead *cipher; |
809 | ||
810 | cipher = crypto_spawn_aead(spawn); | |
811 | if (IS_ERR(cipher)) | |
812 | return PTR_ERR(cipher); | |
813 | ||
298c926c | 814 | ctx->child = cipher; |
ec9f2006 HX |
815 | crypto_aead_set_reqsize( |
816 | tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx), | |
817 | crypto_aead_reqsize(cipher))); | |
298c926c AH |
818 | return 0; |
819 | } | |
820 | ||
f614e546 | 821 | static void cryptd_aead_exit_tfm(struct crypto_aead *tfm) |
298c926c | 822 | { |
f614e546 | 823 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
824 | crypto_free_aead(ctx->child); |
825 | } | |
826 | ||
827 | static int cryptd_create_aead(struct crypto_template *tmpl, | |
828 | struct rtattr **tb, | |
829 | struct cryptd_queue *queue) | |
830 | { | |
831 | struct aead_instance_ctx *ctx; | |
f614e546 HX |
832 | struct aead_instance *inst; |
833 | struct aead_alg *alg; | |
9b8c456e HX |
834 | const char *name; |
835 | u32 type = 0; | |
ec9f2006 | 836 | u32 mask = CRYPTO_ALG_ASYNC; |
298c926c AH |
837 | int err; |
838 | ||
466a7b9e SM |
839 | cryptd_check_internal(tb, &type, &mask); |
840 | ||
9b8c456e HX |
841 | name = crypto_attr_alg_name(tb[1]); |
842 | if (IS_ERR(name)) | |
843 | return PTR_ERR(name); | |
298c926c | 844 | |
9b8c456e HX |
845 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
846 | if (!inst) | |
847 | return -ENOMEM; | |
298c926c | 848 | |
f614e546 | 849 | ctx = aead_instance_ctx(inst); |
298c926c AH |
850 | ctx->queue = queue; |
851 | ||
f614e546 | 852 | crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst)); |
9b8c456e | 853 | err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask); |
298c926c AH |
854 | if (err) |
855 | goto out_free_inst; | |
856 | ||
f614e546 HX |
857 | alg = crypto_spawn_aead_alg(&ctx->aead_spawn); |
858 | err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base); | |
9b8c456e HX |
859 | if (err) |
860 | goto out_drop_aead; | |
861 | ||
f614e546 | 862 | inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC | |
5e4b8c1f | 863 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); |
f614e546 | 864 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx); |
298c926c | 865 | |
f614e546 HX |
866 | inst->alg.ivsize = crypto_aead_alg_ivsize(alg); |
867 | inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); | |
868 | ||
869 | inst->alg.init = cryptd_aead_init_tfm; | |
870 | inst->alg.exit = cryptd_aead_exit_tfm; | |
871 | inst->alg.setkey = cryptd_aead_setkey; | |
872 | inst->alg.setauthsize = cryptd_aead_setauthsize; | |
873 | inst->alg.encrypt = cryptd_aead_encrypt_enqueue; | |
874 | inst->alg.decrypt = cryptd_aead_decrypt_enqueue; | |
875 | ||
876 | err = aead_register_instance(tmpl, inst); | |
298c926c | 877 | if (err) { |
9b8c456e HX |
878 | out_drop_aead: |
879 | crypto_drop_aead(&ctx->aead_spawn); | |
298c926c AH |
880 | out_free_inst: |
881 | kfree(inst); | |
882 | } | |
298c926c AH |
883 | return err; |
884 | } | |
885 | ||
254eff77 | 886 | static struct cryptd_queue queue; |
124b53d0 | 887 | |
9cd899a3 | 888 | static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) |
124b53d0 HX |
889 | { |
890 | struct crypto_attr_type *algt; | |
891 | ||
892 | algt = crypto_get_attr_type(tb); | |
893 | if (IS_ERR(algt)) | |
9cd899a3 | 894 | return PTR_ERR(algt); |
124b53d0 HX |
895 | |
896 | switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { | |
897 | case CRYPTO_ALG_TYPE_BLKCIPHER: | |
9cd899a3 | 898 | return cryptd_create_blkcipher(tmpl, tb, &queue); |
b8a28251 | 899 | case CRYPTO_ALG_TYPE_DIGEST: |
9cd899a3 | 900 | return cryptd_create_hash(tmpl, tb, &queue); |
298c926c AH |
901 | case CRYPTO_ALG_TYPE_AEAD: |
902 | return cryptd_create_aead(tmpl, tb, &queue); | |
124b53d0 HX |
903 | } |
904 | ||
9cd899a3 | 905 | return -EINVAL; |
124b53d0 HX |
906 | } |
907 | ||
908 | static void cryptd_free(struct crypto_instance *inst) | |
909 | { | |
910 | struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst); | |
0b535adf | 911 | struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); |
298c926c | 912 | struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst); |
0b535adf HX |
913 | |
914 | switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { | |
915 | case CRYPTO_ALG_TYPE_AHASH: | |
916 | crypto_drop_shash(&hctx->spawn); | |
917 | kfree(ahash_instance(inst)); | |
918 | return; | |
298c926c | 919 | case CRYPTO_ALG_TYPE_AEAD: |
f614e546 HX |
920 | crypto_drop_aead(&aead_ctx->aead_spawn); |
921 | kfree(aead_instance(inst)); | |
298c926c AH |
922 | return; |
923 | default: | |
924 | crypto_drop_spawn(&ctx->spawn); | |
925 | kfree(inst); | |
0b535adf | 926 | } |
124b53d0 HX |
927 | } |
928 | ||
929 | static struct crypto_template cryptd_tmpl = { | |
930 | .name = "cryptd", | |
9cd899a3 | 931 | .create = cryptd_create, |
124b53d0 HX |
932 | .free = cryptd_free, |
933 | .module = THIS_MODULE, | |
934 | }; | |
935 | ||
1cac2cbc HY |
936 | struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, |
937 | u32 type, u32 mask) | |
938 | { | |
939 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 940 | struct cryptd_blkcipher_ctx *ctx; |
505fd21d | 941 | struct crypto_tfm *tfm; |
1cac2cbc HY |
942 | |
943 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
944 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
945 | return ERR_PTR(-EINVAL); | |
c012a79d | 946 | type = crypto_skcipher_type(type); |
505fd21d HY |
947 | mask &= ~CRYPTO_ALG_TYPE_MASK; |
948 | mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK); | |
949 | tfm = crypto_alloc_base(cryptd_alg_name, type, mask); | |
1cac2cbc HY |
950 | if (IS_ERR(tfm)) |
951 | return ERR_CAST(tfm); | |
505fd21d HY |
952 | if (tfm->__crt_alg->cra_module != THIS_MODULE) { |
953 | crypto_free_tfm(tfm); | |
1cac2cbc HY |
954 | return ERR_PTR(-EINVAL); |
955 | } | |
956 | ||
81760ea6 HX |
957 | ctx = crypto_tfm_ctx(tfm); |
958 | atomic_set(&ctx->refcnt, 1); | |
959 | ||
505fd21d | 960 | return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm)); |
1cac2cbc HY |
961 | } |
962 | EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher); | |
963 | ||
964 | struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm) | |
965 | { | |
966 | struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); | |
967 | return ctx->child; | |
968 | } | |
969 | EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child); | |
970 | ||
81760ea6 HX |
971 | bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm) |
972 | { | |
973 | struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); | |
974 | ||
975 | return atomic_read(&ctx->refcnt) - 1; | |
976 | } | |
977 | EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued); | |
978 | ||
1cac2cbc HY |
979 | void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) |
980 | { | |
81760ea6 HX |
981 | struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); |
982 | ||
983 | if (atomic_dec_and_test(&ctx->refcnt)) | |
984 | crypto_free_ablkcipher(&tfm->base); | |
1cac2cbc HY |
985 | } |
986 | EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); | |
987 | ||
ace13663 HY |
988 | struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, |
989 | u32 type, u32 mask) | |
990 | { | |
991 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 992 | struct cryptd_hash_ctx *ctx; |
ace13663 HY |
993 | struct crypto_ahash *tfm; |
994 | ||
995 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
996 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
997 | return ERR_PTR(-EINVAL); | |
998 | tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); | |
999 | if (IS_ERR(tfm)) | |
1000 | return ERR_CAST(tfm); | |
1001 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
1002 | crypto_free_ahash(tfm); | |
1003 | return ERR_PTR(-EINVAL); | |
1004 | } | |
1005 | ||
81760ea6 HX |
1006 | ctx = crypto_ahash_ctx(tfm); |
1007 | atomic_set(&ctx->refcnt, 1); | |
1008 | ||
ace13663 HY |
1009 | return __cryptd_ahash_cast(tfm); |
1010 | } | |
1011 | EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); | |
1012 | ||
1013 | struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) | |
1014 | { | |
1015 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
1016 | ||
1017 | return ctx->child; | |
1018 | } | |
1019 | EXPORT_SYMBOL_GPL(cryptd_ahash_child); | |
1020 | ||
0e1227d3 HY |
1021 | struct shash_desc *cryptd_shash_desc(struct ahash_request *req) |
1022 | { | |
1023 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
1024 | return &rctx->desc; | |
1025 | } | |
1026 | EXPORT_SYMBOL_GPL(cryptd_shash_desc); | |
1027 | ||
81760ea6 HX |
1028 | bool cryptd_ahash_queued(struct cryptd_ahash *tfm) |
1029 | { | |
1030 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
1031 | ||
1032 | return atomic_read(&ctx->refcnt) - 1; | |
1033 | } | |
1034 | EXPORT_SYMBOL_GPL(cryptd_ahash_queued); | |
1035 | ||
ace13663 HY |
1036 | void cryptd_free_ahash(struct cryptd_ahash *tfm) |
1037 | { | |
81760ea6 HX |
1038 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); |
1039 | ||
1040 | if (atomic_dec_and_test(&ctx->refcnt)) | |
1041 | crypto_free_ahash(&tfm->base); | |
ace13663 HY |
1042 | } |
1043 | EXPORT_SYMBOL_GPL(cryptd_free_ahash); | |
1044 | ||
298c926c AH |
1045 | struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, |
1046 | u32 type, u32 mask) | |
1047 | { | |
1048 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 1049 | struct cryptd_aead_ctx *ctx; |
298c926c AH |
1050 | struct crypto_aead *tfm; |
1051 | ||
1052 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
1053 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
1054 | return ERR_PTR(-EINVAL); | |
1055 | tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); | |
1056 | if (IS_ERR(tfm)) | |
1057 | return ERR_CAST(tfm); | |
1058 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
1059 | crypto_free_aead(tfm); | |
1060 | return ERR_PTR(-EINVAL); | |
1061 | } | |
81760ea6 HX |
1062 | |
1063 | ctx = crypto_aead_ctx(tfm); | |
1064 | atomic_set(&ctx->refcnt, 1); | |
1065 | ||
298c926c AH |
1066 | return __cryptd_aead_cast(tfm); |
1067 | } | |
1068 | EXPORT_SYMBOL_GPL(cryptd_alloc_aead); | |
1069 | ||
1070 | struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) | |
1071 | { | |
1072 | struct cryptd_aead_ctx *ctx; | |
1073 | ctx = crypto_aead_ctx(&tfm->base); | |
1074 | return ctx->child; | |
1075 | } | |
1076 | EXPORT_SYMBOL_GPL(cryptd_aead_child); | |
1077 | ||
81760ea6 HX |
1078 | bool cryptd_aead_queued(struct cryptd_aead *tfm) |
1079 | { | |
1080 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); | |
1081 | ||
1082 | return atomic_read(&ctx->refcnt) - 1; | |
1083 | } | |
1084 | EXPORT_SYMBOL_GPL(cryptd_aead_queued); | |
1085 | ||
298c926c AH |
1086 | void cryptd_free_aead(struct cryptd_aead *tfm) |
1087 | { | |
81760ea6 HX |
1088 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); |
1089 | ||
1090 | if (atomic_dec_and_test(&ctx->refcnt)) | |
1091 | crypto_free_aead(&tfm->base); | |
298c926c AH |
1092 | } |
1093 | EXPORT_SYMBOL_GPL(cryptd_free_aead); | |
1094 | ||
124b53d0 HX |
1095 | static int __init cryptd_init(void) |
1096 | { | |
1097 | int err; | |
1098 | ||
254eff77 | 1099 | err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN); |
124b53d0 HX |
1100 | if (err) |
1101 | return err; | |
1102 | ||
1103 | err = crypto_register_template(&cryptd_tmpl); | |
1104 | if (err) | |
254eff77 | 1105 | cryptd_fini_queue(&queue); |
124b53d0 HX |
1106 | |
1107 | return err; | |
1108 | } | |
1109 | ||
1110 | static void __exit cryptd_exit(void) | |
1111 | { | |
254eff77 | 1112 | cryptd_fini_queue(&queue); |
124b53d0 HX |
1113 | crypto_unregister_template(&cryptd_tmpl); |
1114 | } | |
1115 | ||
b2bac6ac | 1116 | subsys_initcall(cryptd_init); |
124b53d0 HX |
1117 | module_exit(cryptd_exit); |
1118 | ||
1119 | MODULE_LICENSE("GPL"); | |
1120 | MODULE_DESCRIPTION("Software async crypto daemon"); | |
4943ba16 | 1121 | MODULE_ALIAS_CRYPTO("cryptd"); |