Commit | Line | Data |
---|---|---|
735d37b5 BW |
1 | /* |
2 | * Handle async block request by crypto hardware engine. | |
3 | * | |
4 | * Copyright (C) 2016 Linaro, Inc. | |
5 | * | |
6 | * Author: Baolin Wang <baolin.wang@linaro.org> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify it | |
9 | * under the terms of the GNU General Public License as published by the Free | |
10 | * Software Foundation; either version 2 of the License, or (at your option) | |
11 | * any later version. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <linux/err.h> | |
16 | #include <linux/delay.h> | |
17 | #include "internal.h" | |
18 | ||
19 | #define CRYPTO_ENGINE_MAX_QLEN 10 | |
20 | ||
21 | void crypto_finalize_request(struct crypto_engine *engine, | |
22 | struct ablkcipher_request *req, int err); | |
23 | ||
24 | /** | |
25 | * crypto_pump_requests - dequeue one request from engine queue to process | |
26 | * @engine: the hardware engine | |
27 | * @in_kthread: true if we are in the context of the request pump thread | |
28 | * | |
29 | * This function checks if there is any request in the engine queue that | |
30 | * needs processing and if so call out to the driver to initialize hardware | |
31 | * and handle each request. | |
32 | */ | |
33 | static void crypto_pump_requests(struct crypto_engine *engine, | |
34 | bool in_kthread) | |
35 | { | |
36 | struct crypto_async_request *async_req, *backlog; | |
37 | struct ablkcipher_request *req; | |
38 | unsigned long flags; | |
39 | bool was_busy = false; | |
40 | int ret; | |
41 | ||
42 | spin_lock_irqsave(&engine->queue_lock, flags); | |
43 | ||
44 | /* Make sure we are not already running a request */ | |
45 | if (engine->cur_req) | |
46 | goto out; | |
47 | ||
48 | /* If another context is idling then defer */ | |
49 | if (engine->idling) { | |
50 | queue_kthread_work(&engine->kworker, &engine->pump_requests); | |
51 | goto out; | |
52 | } | |
53 | ||
54 | /* Check if the engine queue is idle */ | |
55 | if (!crypto_queue_len(&engine->queue) || !engine->running) { | |
56 | if (!engine->busy) | |
57 | goto out; | |
58 | ||
59 | /* Only do teardown in the thread */ | |
60 | if (!in_kthread) { | |
61 | queue_kthread_work(&engine->kworker, | |
62 | &engine->pump_requests); | |
63 | goto out; | |
64 | } | |
65 | ||
66 | engine->busy = false; | |
67 | engine->idling = true; | |
68 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
69 | ||
70 | if (engine->unprepare_crypt_hardware && | |
71 | engine->unprepare_crypt_hardware(engine)) | |
72 | pr_err("failed to unprepare crypt hardware\n"); | |
73 | ||
74 | spin_lock_irqsave(&engine->queue_lock, flags); | |
75 | engine->idling = false; | |
76 | goto out; | |
77 | } | |
78 | ||
79 | /* Get the fist request from the engine queue to handle */ | |
80 | backlog = crypto_get_backlog(&engine->queue); | |
81 | async_req = crypto_dequeue_request(&engine->queue); | |
82 | if (!async_req) | |
83 | goto out; | |
84 | ||
85 | req = ablkcipher_request_cast(async_req); | |
86 | ||
87 | engine->cur_req = req; | |
88 | if (backlog) | |
89 | backlog->complete(backlog, -EINPROGRESS); | |
90 | ||
91 | if (engine->busy) | |
92 | was_busy = true; | |
93 | else | |
94 | engine->busy = true; | |
95 | ||
96 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
97 | ||
98 | /* Until here we get the request need to be encrypted successfully */ | |
99 | if (!was_busy && engine->prepare_crypt_hardware) { | |
100 | ret = engine->prepare_crypt_hardware(engine); | |
101 | if (ret) { | |
102 | pr_err("failed to prepare crypt hardware\n"); | |
103 | goto req_err; | |
104 | } | |
105 | } | |
106 | ||
107 | if (engine->prepare_request) { | |
108 | ret = engine->prepare_request(engine, engine->cur_req); | |
109 | if (ret) { | |
110 | pr_err("failed to prepare request: %d\n", ret); | |
111 | goto req_err; | |
112 | } | |
113 | engine->cur_req_prepared = true; | |
114 | } | |
115 | ||
116 | ret = engine->crypt_one_request(engine, engine->cur_req); | |
117 | if (ret) { | |
118 | pr_err("failed to crypt one request from queue\n"); | |
119 | goto req_err; | |
120 | } | |
121 | return; | |
122 | ||
123 | req_err: | |
124 | crypto_finalize_request(engine, engine->cur_req, ret); | |
125 | return; | |
126 | ||
127 | out: | |
128 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
129 | } | |
130 | ||
131 | static void crypto_pump_work(struct kthread_work *work) | |
132 | { | |
133 | struct crypto_engine *engine = | |
134 | container_of(work, struct crypto_engine, pump_requests); | |
135 | ||
136 | crypto_pump_requests(engine, true); | |
137 | } | |
138 | ||
139 | /** | |
140 | * crypto_transfer_request - transfer the new request into the engine queue | |
141 | * @engine: the hardware engine | |
142 | * @req: the request need to be listed into the engine queue | |
143 | */ | |
144 | int crypto_transfer_request(struct crypto_engine *engine, | |
145 | struct ablkcipher_request *req, bool need_pump) | |
146 | { | |
147 | unsigned long flags; | |
148 | int ret; | |
149 | ||
150 | spin_lock_irqsave(&engine->queue_lock, flags); | |
151 | ||
152 | if (!engine->running) { | |
153 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
154 | return -ESHUTDOWN; | |
155 | } | |
156 | ||
157 | ret = ablkcipher_enqueue_request(&engine->queue, req); | |
158 | ||
159 | if (!engine->busy && need_pump) | |
160 | queue_kthread_work(&engine->kworker, &engine->pump_requests); | |
161 | ||
162 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
163 | return ret; | |
164 | } | |
165 | EXPORT_SYMBOL_GPL(crypto_transfer_request); | |
166 | ||
167 | /** | |
168 | * crypto_transfer_request_to_engine - transfer one request to list into the | |
169 | * engine queue | |
170 | * @engine: the hardware engine | |
171 | * @req: the request need to be listed into the engine queue | |
172 | */ | |
173 | int crypto_transfer_request_to_engine(struct crypto_engine *engine, | |
174 | struct ablkcipher_request *req) | |
175 | { | |
176 | return crypto_transfer_request(engine, req, true); | |
177 | } | |
178 | EXPORT_SYMBOL_GPL(crypto_transfer_request_to_engine); | |
179 | ||
180 | /** | |
181 | * crypto_finalize_request - finalize one request if the request is done | |
182 | * @engine: the hardware engine | |
183 | * @req: the request need to be finalized | |
184 | * @err: error number | |
185 | */ | |
186 | void crypto_finalize_request(struct crypto_engine *engine, | |
187 | struct ablkcipher_request *req, int err) | |
188 | { | |
189 | unsigned long flags; | |
190 | bool finalize_cur_req = false; | |
191 | int ret; | |
192 | ||
193 | spin_lock_irqsave(&engine->queue_lock, flags); | |
194 | if (engine->cur_req == req) | |
195 | finalize_cur_req = true; | |
196 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
197 | ||
198 | if (finalize_cur_req) { | |
199 | if (engine->cur_req_prepared && engine->unprepare_request) { | |
200 | ret = engine->unprepare_request(engine, req); | |
201 | if (ret) | |
202 | pr_err("failed to unprepare request\n"); | |
203 | } | |
204 | ||
205 | spin_lock_irqsave(&engine->queue_lock, flags); | |
206 | engine->cur_req = NULL; | |
207 | engine->cur_req_prepared = false; | |
208 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
209 | } | |
210 | ||
211 | req->base.complete(&req->base, err); | |
212 | ||
213 | queue_kthread_work(&engine->kworker, &engine->pump_requests); | |
214 | } | |
215 | EXPORT_SYMBOL_GPL(crypto_finalize_request); | |
216 | ||
217 | /** | |
218 | * crypto_engine_start - start the hardware engine | |
219 | * @engine: the hardware engine need to be started | |
220 | * | |
221 | * Return 0 on success, else on fail. | |
222 | */ | |
223 | int crypto_engine_start(struct crypto_engine *engine) | |
224 | { | |
225 | unsigned long flags; | |
226 | ||
227 | spin_lock_irqsave(&engine->queue_lock, flags); | |
228 | ||
229 | if (engine->running || engine->busy) { | |
230 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
231 | return -EBUSY; | |
232 | } | |
233 | ||
234 | engine->running = true; | |
235 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
236 | ||
237 | queue_kthread_work(&engine->kworker, &engine->pump_requests); | |
238 | ||
239 | return 0; | |
240 | } | |
241 | EXPORT_SYMBOL_GPL(crypto_engine_start); | |
242 | ||
243 | /** | |
244 | * crypto_engine_stop - stop the hardware engine | |
245 | * @engine: the hardware engine need to be stopped | |
246 | * | |
247 | * Return 0 on success, else on fail. | |
248 | */ | |
249 | int crypto_engine_stop(struct crypto_engine *engine) | |
250 | { | |
251 | unsigned long flags; | |
252 | unsigned limit = 500; | |
253 | int ret = 0; | |
254 | ||
255 | spin_lock_irqsave(&engine->queue_lock, flags); | |
256 | ||
257 | /* | |
258 | * If the engine queue is not empty or the engine is on busy state, | |
259 | * we need to wait for a while to pump the requests of engine queue. | |
260 | */ | |
261 | while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) { | |
262 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
263 | msleep(20); | |
264 | spin_lock_irqsave(&engine->queue_lock, flags); | |
265 | } | |
266 | ||
267 | if (crypto_queue_len(&engine->queue) || engine->busy) | |
268 | ret = -EBUSY; | |
269 | else | |
270 | engine->running = false; | |
271 | ||
272 | spin_unlock_irqrestore(&engine->queue_lock, flags); | |
273 | ||
274 | if (ret) | |
275 | pr_warn("could not stop engine\n"); | |
276 | ||
277 | return ret; | |
278 | } | |
279 | EXPORT_SYMBOL_GPL(crypto_engine_stop); | |
280 | ||
281 | /** | |
282 | * crypto_engine_alloc_init - allocate crypto hardware engine structure and | |
283 | * initialize it. | |
284 | * @dev: the device attached with one hardware engine | |
285 | * @rt: whether this queue is set to run as a realtime task | |
286 | * | |
287 | * This must be called from context that can sleep. | |
288 | * Return: the crypto engine structure on success, else NULL. | |
289 | */ | |
290 | struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) | |
291 | { | |
292 | struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; | |
293 | struct crypto_engine *engine; | |
294 | ||
295 | if (!dev) | |
296 | return NULL; | |
297 | ||
298 | engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL); | |
299 | if (!engine) | |
300 | return NULL; | |
301 | ||
302 | engine->rt = rt; | |
303 | engine->running = false; | |
304 | engine->busy = false; | |
305 | engine->idling = false; | |
306 | engine->cur_req_prepared = false; | |
307 | engine->priv_data = dev; | |
308 | snprintf(engine->name, sizeof(engine->name), | |
309 | "%s-engine", dev_name(dev)); | |
310 | ||
311 | crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN); | |
312 | spin_lock_init(&engine->queue_lock); | |
313 | ||
314 | init_kthread_worker(&engine->kworker); | |
315 | engine->kworker_task = kthread_run(kthread_worker_fn, | |
316 | &engine->kworker, "%s", | |
317 | engine->name); | |
318 | if (IS_ERR(engine->kworker_task)) { | |
319 | dev_err(dev, "failed to create crypto request pump task\n"); | |
320 | return NULL; | |
321 | } | |
322 | init_kthread_work(&engine->pump_requests, crypto_pump_work); | |
323 | ||
324 | if (engine->rt) { | |
325 | dev_info(dev, "will run requests pump with realtime priority\n"); | |
326 | sched_setscheduler(engine->kworker_task, SCHED_FIFO, ¶m); | |
327 | } | |
328 | ||
329 | return engine; | |
330 | } | |
331 | EXPORT_SYMBOL_GPL(crypto_engine_alloc_init); | |
332 | ||
333 | /** | |
334 | * crypto_engine_exit - free the resources of hardware engine when exit | |
335 | * @engine: the hardware engine need to be freed | |
336 | * | |
337 | * Return 0 for success. | |
338 | */ | |
339 | int crypto_engine_exit(struct crypto_engine *engine) | |
340 | { | |
341 | int ret; | |
342 | ||
343 | ret = crypto_engine_stop(engine); | |
344 | if (ret) | |
345 | return ret; | |
346 | ||
347 | flush_kthread_worker(&engine->kworker); | |
348 | kthread_stop(engine->kworker_task); | |
349 | ||
350 | return 0; | |
351 | } | |
352 | EXPORT_SYMBOL_GPL(crypto_engine_exit); | |
353 | ||
354 | MODULE_LICENSE("GPL"); | |
355 | MODULE_DESCRIPTION("Crypto hardware engine framework"); |