Commit | Line | Data |
---|---|---|
db509a45 BB |
1 | /* |
2 | * Provide TDMA helper functions used by cipher and hash algorithm | |
3 | * implementations. | |
4 | * | |
5 | * Author: Boris Brezillon <boris.brezillon@free-electrons.com> | |
6 | * Author: Arnaud Ebalard <arno@natisbad.org> | |
7 | * | |
8 | * This work is based on an initial version written by | |
9 | * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > | |
10 | * | |
11 | * This program is free software; you can redistribute it and/or modify it | |
12 | * under the terms of the GNU General Public License version 2 as published | |
13 | * by the Free Software Foundation. | |
14 | */ | |
15 | ||
16 | #include "cesa.h" | |
17 | ||
18 | bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter, | |
19 | struct mv_cesa_sg_dma_iter *sgiter, | |
20 | unsigned int len) | |
21 | { | |
22 | if (!sgiter->sg) | |
23 | return false; | |
24 | ||
25 | sgiter->op_offset += len; | |
26 | sgiter->offset += len; | |
27 | if (sgiter->offset == sg_dma_len(sgiter->sg)) { | |
28 | if (sg_is_last(sgiter->sg)) | |
29 | return false; | |
30 | sgiter->offset = 0; | |
31 | sgiter->sg = sg_next(sgiter->sg); | |
32 | } | |
33 | ||
34 | if (sgiter->op_offset == iter->op_len) | |
35 | return false; | |
36 | ||
37 | return true; | |
38 | } | |
39 | ||
53da740f | 40 | void mv_cesa_dma_step(struct mv_cesa_req *dreq) |
db509a45 | 41 | { |
53da740f | 42 | struct mv_cesa_engine *engine = dreq->engine; |
db509a45 | 43 | |
b1508561 | 44 | writel_relaxed(0, engine->regs + CESA_SA_CFG); |
db509a45 BB |
45 | |
46 | mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); | |
b1508561 RK |
47 | writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B | |
48 | CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN, | |
49 | engine->regs + CESA_TDMA_CONTROL); | |
50 | ||
51 | writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT | | |
52 | CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS, | |
53 | engine->regs + CESA_SA_CFG); | |
54 | writel_relaxed(dreq->chain.first->cur_dma, | |
55 | engine->regs + CESA_TDMA_NEXT_ADDR); | |
f6283088 RP |
56 | BUG_ON(readl(engine->regs + CESA_SA_CMD) & |
57 | CESA_SA_CMD_EN_CESA_SA_ACCL0); | |
db509a45 BB |
58 | writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); |
59 | } | |
60 | ||
53da740f | 61 | void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq) |
db509a45 BB |
62 | { |
63 | struct mv_cesa_tdma_desc *tdma; | |
64 | ||
65 | for (tdma = dreq->chain.first; tdma;) { | |
66 | struct mv_cesa_tdma_desc *old_tdma = tdma; | |
b99acf79 | 67 | u32 type = tdma->flags & CESA_TDMA_TYPE_MSK; |
db509a45 | 68 | |
b99acf79 | 69 | if (type == CESA_TDMA_OP) |
db509a45 BB |
70 | dma_pool_free(cesa_dev->dma->op_pool, tdma->op, |
71 | le32_to_cpu(tdma->src)); | |
bac8e805 RP |
72 | else if (type == CESA_TDMA_IV) |
73 | dma_pool_free(cesa_dev->dma->iv_pool, tdma->data, | |
74 | le32_to_cpu(tdma->dst)); | |
db509a45 BB |
75 | |
76 | tdma = tdma->next; | |
77 | dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma, | |
5d754137 | 78 | old_tdma->cur_dma); |
db509a45 BB |
79 | } |
80 | ||
81 | dreq->chain.first = NULL; | |
82 | dreq->chain.last = NULL; | |
83 | } | |
84 | ||
53da740f | 85 | void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, |
db509a45 BB |
86 | struct mv_cesa_engine *engine) |
87 | { | |
88 | struct mv_cesa_tdma_desc *tdma; | |
89 | ||
90 | for (tdma = dreq->chain.first; tdma; tdma = tdma->next) { | |
91 | if (tdma->flags & CESA_TDMA_DST_IN_SRAM) | |
92 | tdma->dst = cpu_to_le32(tdma->dst + engine->sram_dma); | |
93 | ||
94 | if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) | |
95 | tdma->src = cpu_to_le32(tdma->src + engine->sram_dma); | |
96 | ||
b99acf79 | 97 | if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP) |
db509a45 BB |
98 | mv_cesa_adjust_op(engine, tdma->op); |
99 | } | |
100 | } | |
101 | ||
85030c51 RP |
102 | void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, |
103 | struct mv_cesa_req *dreq) | |
104 | { | |
105 | if (engine->chain.first == NULL && engine->chain.last == NULL) { | |
106 | engine->chain.first = dreq->chain.first; | |
107 | engine->chain.last = dreq->chain.last; | |
108 | } else { | |
109 | struct mv_cesa_tdma_desc *last; | |
110 | ||
111 | last = engine->chain.last; | |
112 | last->next = dreq->chain.first; | |
113 | engine->chain.last = dreq->chain.last; | |
114 | ||
115 | if (!(last->flags & CESA_TDMA_BREAK_CHAIN)) | |
116 | last->next_dma = dreq->chain.first->cur_dma; | |
117 | } | |
118 | } | |
119 | ||
120 | int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) | |
121 | { | |
122 | struct crypto_async_request *req = NULL; | |
123 | struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL; | |
124 | dma_addr_t tdma_cur; | |
125 | int res = 0; | |
126 | ||
127 | tdma_cur = readl(engine->regs + CESA_TDMA_CUR); | |
128 | ||
129 | for (tdma = engine->chain.first; tdma; tdma = next) { | |
130 | spin_lock_bh(&engine->lock); | |
131 | next = tdma->next; | |
132 | spin_unlock_bh(&engine->lock); | |
133 | ||
134 | if (tdma->flags & CESA_TDMA_END_OF_REQ) { | |
135 | struct crypto_async_request *backlog = NULL; | |
136 | struct mv_cesa_ctx *ctx; | |
137 | u32 current_status; | |
138 | ||
139 | spin_lock_bh(&engine->lock); | |
140 | /* | |
141 | * if req is NULL, this means we're processing the | |
142 | * request in engine->req. | |
143 | */ | |
144 | if (!req) | |
145 | req = engine->req; | |
146 | else | |
147 | req = mv_cesa_dequeue_req_locked(engine, | |
148 | &backlog); | |
149 | ||
150 | /* Re-chaining to the next request */ | |
151 | engine->chain.first = tdma->next; | |
152 | tdma->next = NULL; | |
153 | ||
154 | /* If this is the last request, clear the chain */ | |
155 | if (engine->chain.first == NULL) | |
156 | engine->chain.last = NULL; | |
157 | spin_unlock_bh(&engine->lock); | |
158 | ||
159 | ctx = crypto_tfm_ctx(req->tfm); | |
160 | current_status = (tdma->cur_dma == tdma_cur) ? | |
161 | status : CESA_SA_INT_ACC0_IDMA_DONE; | |
162 | res = ctx->ops->process(req, current_status); | |
163 | ctx->ops->complete(req); | |
164 | ||
165 | if (res == 0) | |
166 | mv_cesa_engine_enqueue_complete_request(engine, | |
167 | req); | |
168 | ||
169 | if (backlog) | |
170 | backlog->complete(backlog, -EINPROGRESS); | |
171 | } | |
172 | ||
173 | if (res || tdma->cur_dma == tdma_cur) | |
174 | break; | |
175 | } | |
176 | ||
177 | /* Save the last request in error to engine->req, so that the core | |
178 | * knows which request was fautly */ | |
179 | if (res) { | |
180 | spin_lock_bh(&engine->lock); | |
181 | engine->req = req; | |
182 | spin_unlock_bh(&engine->lock); | |
183 | } | |
184 | ||
185 | return res; | |
186 | } | |
187 | ||
db509a45 BB |
188 | static struct mv_cesa_tdma_desc * |
189 | mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) | |
190 | { | |
191 | struct mv_cesa_tdma_desc *new_tdma = NULL; | |
192 | dma_addr_t dma_handle; | |
193 | ||
472d640b JL |
194 | new_tdma = dma_pool_zalloc(cesa_dev->dma->tdma_desc_pool, flags, |
195 | &dma_handle); | |
db509a45 BB |
196 | if (!new_tdma) |
197 | return ERR_PTR(-ENOMEM); | |
198 | ||
5d754137 | 199 | new_tdma->cur_dma = dma_handle; |
db509a45 | 200 | if (chain->last) { |
5d754137 | 201 | chain->last->next_dma = cpu_to_le32(dma_handle); |
db509a45 BB |
202 | chain->last->next = new_tdma; |
203 | } else { | |
204 | chain->first = new_tdma; | |
205 | } | |
206 | ||
207 | chain->last = new_tdma; | |
208 | ||
209 | return new_tdma; | |
210 | } | |
211 | ||
bac8e805 RP |
212 | int mv_cesa_dma_add_iv_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, |
213 | u32 size, u32 flags, gfp_t gfp_flags) | |
214 | { | |
215 | ||
216 | struct mv_cesa_tdma_desc *tdma; | |
217 | u8 *iv; | |
218 | dma_addr_t dma_handle; | |
219 | ||
220 | tdma = mv_cesa_dma_add_desc(chain, gfp_flags); | |
221 | if (IS_ERR(tdma)) | |
222 | return PTR_ERR(tdma); | |
223 | ||
aa6416ee | 224 | iv = dma_pool_alloc(cesa_dev->dma->iv_pool, gfp_flags, &dma_handle); |
bac8e805 RP |
225 | if (!iv) |
226 | return -ENOMEM; | |
227 | ||
228 | tdma->byte_cnt = cpu_to_le32(size | BIT(31)); | |
229 | tdma->src = src; | |
230 | tdma->dst = cpu_to_le32(dma_handle); | |
231 | tdma->data = iv; | |
232 | ||
233 | flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); | |
234 | tdma->flags = flags | CESA_TDMA_IV; | |
235 | return 0; | |
236 | } | |
237 | ||
db509a45 BB |
238 | struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, |
239 | const struct mv_cesa_op_ctx *op_templ, | |
240 | bool skip_ctx, | |
241 | gfp_t flags) | |
242 | { | |
243 | struct mv_cesa_tdma_desc *tdma; | |
244 | struct mv_cesa_op_ctx *op; | |
245 | dma_addr_t dma_handle; | |
6de59d45 | 246 | unsigned int size; |
db509a45 BB |
247 | |
248 | tdma = mv_cesa_dma_add_desc(chain, flags); | |
249 | if (IS_ERR(tdma)) | |
250 | return ERR_CAST(tdma); | |
251 | ||
252 | op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle); | |
253 | if (!op) | |
254 | return ERR_PTR(-ENOMEM); | |
255 | ||
256 | *op = *op_templ; | |
257 | ||
6de59d45 RK |
258 | size = skip_ctx ? sizeof(op->desc) : sizeof(*op); |
259 | ||
db509a45 BB |
260 | tdma = chain->last; |
261 | tdma->op = op; | |
6de59d45 | 262 | tdma->byte_cnt = cpu_to_le32(size | BIT(31)); |
ea1f662b | 263 | tdma->src = cpu_to_le32(dma_handle); |
36225b91 | 264 | tdma->dst = CESA_SA_CFG_SRAM_OFFSET; |
db509a45 BB |
265 | tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; |
266 | ||
267 | return op; | |
268 | } | |
269 | ||
270 | int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, | |
271 | dma_addr_t dst, dma_addr_t src, u32 size, | |
272 | u32 flags, gfp_t gfp_flags) | |
273 | { | |
274 | struct mv_cesa_tdma_desc *tdma; | |
275 | ||
276 | tdma = mv_cesa_dma_add_desc(chain, gfp_flags); | |
277 | if (IS_ERR(tdma)) | |
278 | return PTR_ERR(tdma); | |
279 | ||
6de59d45 | 280 | tdma->byte_cnt = cpu_to_le32(size | BIT(31)); |
db509a45 BB |
281 | tdma->src = src; |
282 | tdma->dst = dst; | |
283 | ||
284 | flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); | |
285 | tdma->flags = flags | CESA_TDMA_DATA; | |
286 | ||
287 | return 0; | |
288 | } | |
289 | ||
35622eae | 290 | int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags) |
db509a45 BB |
291 | { |
292 | struct mv_cesa_tdma_desc *tdma; | |
293 | ||
294 | tdma = mv_cesa_dma_add_desc(chain, flags); | |
295 | if (IS_ERR(tdma)) | |
296 | return PTR_ERR(tdma); | |
297 | ||
298 | return 0; | |
299 | } | |
300 | ||
35622eae | 301 | int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) |
db509a45 BB |
302 | { |
303 | struct mv_cesa_tdma_desc *tdma; | |
304 | ||
305 | tdma = mv_cesa_dma_add_desc(chain, flags); | |
306 | if (IS_ERR(tdma)) | |
307 | return PTR_ERR(tdma); | |
308 | ||
6de59d45 | 309 | tdma->byte_cnt = cpu_to_le32(BIT(31)); |
db509a45 BB |
310 | |
311 | return 0; | |
312 | } | |
313 | ||
314 | int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, | |
315 | struct mv_cesa_dma_iter *dma_iter, | |
316 | struct mv_cesa_sg_dma_iter *sgiter, | |
317 | gfp_t gfp_flags) | |
318 | { | |
319 | u32 flags = sgiter->dir == DMA_TO_DEVICE ? | |
320 | CESA_TDMA_DST_IN_SRAM : CESA_TDMA_SRC_IN_SRAM; | |
321 | unsigned int len; | |
322 | ||
323 | do { | |
324 | dma_addr_t dst, src; | |
325 | int ret; | |
326 | ||
327 | len = mv_cesa_req_dma_iter_transfer_len(dma_iter, sgiter); | |
328 | if (sgiter->dir == DMA_TO_DEVICE) { | |
329 | dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; | |
330 | src = sg_dma_address(sgiter->sg) + sgiter->offset; | |
331 | } else { | |
332 | dst = sg_dma_address(sgiter->sg) + sgiter->offset; | |
333 | src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; | |
334 | } | |
335 | ||
336 | ret = mv_cesa_dma_add_data_transfer(chain, dst, src, len, | |
337 | flags, gfp_flags); | |
338 | if (ret) | |
339 | return ret; | |
340 | ||
341 | } while (mv_cesa_req_dma_iter_next_transfer(dma_iter, sgiter, len)); | |
342 | ||
343 | return 0; | |
344 | } |