Commit | Line | Data |
---|---|---|
8e8ec596 KP |
1 | /* |
2 | * CAAM/SEC 4.x transport/backend driver | |
3 | * JobR backend functionality | |
4 | * | |
4bba1e9f | 5 | * Copyright 2008-2012 Freescale Semiconductor, Inc. |
8e8ec596 KP |
6 | */ |
7 | ||
5af50730 RH |
8 | #include <linux/of_irq.h> |
9 | ||
8e8ec596 KP |
10 | #include "compat.h" |
11 | #include "regs.h" | |
12 | #include "jr.h" | |
13 | #include "desc.h" | |
14 | #include "intern.h" | |
15 | ||
16 | /* Main per-ring interrupt handler */ | |
17 | static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) | |
18 | { | |
19 | struct device *dev = st_dev; | |
20 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); | |
21 | u32 irqstate; | |
22 | ||
23 | /* | |
24 | * Check the output ring for ready responses, kick | |
25 | * tasklet if jobs done. | |
26 | */ | |
27 | irqstate = rd_reg32(&jrp->rregs->jrintstatus); | |
28 | if (!irqstate) | |
29 | return IRQ_NONE; | |
30 | ||
31 | /* | |
32 | * If JobR error, we got more development work to do | |
33 | * Flag a bug now, but we really need to shut down and | |
34 | * restart the queue (and fix code). | |
35 | */ | |
36 | if (irqstate & JRINT_JR_ERROR) { | |
37 | dev_err(dev, "job ring error: irqstate: %08x\n", irqstate); | |
38 | BUG(); | |
39 | } | |
40 | ||
41 | /* mask valid interrupts */ | |
42 | setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); | |
43 | ||
44 | /* Have valid interrupt at this point, just ACK and trigger */ | |
45 | wr_reg32(&jrp->rregs->jrintstatus, irqstate); | |
46 | ||
47 | preempt_disable(); | |
a0ca6ca0 | 48 | tasklet_schedule(&jrp->irqtask); |
8e8ec596 KP |
49 | preempt_enable(); |
50 | ||
51 | return IRQ_HANDLED; | |
52 | } | |
53 | ||
54 | /* Deferred service handler, run as interrupt-fired tasklet */ | |
55 | static void caam_jr_dequeue(unsigned long devarg) | |
56 | { | |
57 | int hw_idx, sw_idx, i, head, tail; | |
58 | struct device *dev = (struct device *)devarg; | |
59 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); | |
60 | void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg); | |
61 | u32 *userdesc, userstatus; | |
62 | void *userarg; | |
8e8ec596 | 63 | |
a8ea07c2 | 64 | while (rd_reg32(&jrp->rregs->outring_used)) { |
8e8ec596 | 65 | |
a8ea07c2 | 66 | head = ACCESS_ONCE(jrp->head); |
8e8ec596 | 67 | |
ce026cb9 | 68 | spin_lock(&jrp->outlock); |
8e8ec596 | 69 | |
a8ea07c2 | 70 | sw_idx = tail = jrp->tail; |
8e8ec596 | 71 | hw_idx = jrp->out_ring_read_index; |
a8ea07c2 | 72 | |
8e8ec596 KP |
73 | for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) { |
74 | sw_idx = (tail + i) & (JOBR_DEPTH - 1); | |
75 | ||
76 | smp_read_barrier_depends(); | |
77 | ||
78 | if (jrp->outring[hw_idx].desc == | |
79 | jrp->entinfo[sw_idx].desc_addr_dma) | |
80 | break; /* found */ | |
81 | } | |
82 | /* we should never fail to find a matching descriptor */ | |
83 | BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0); | |
84 | ||
85 | /* Unmap just-run descriptor so we can post-process */ | |
86 | dma_unmap_single(dev, jrp->outring[hw_idx].desc, | |
87 | jrp->entinfo[sw_idx].desc_size, | |
88 | DMA_TO_DEVICE); | |
89 | ||
90 | /* mark completed, avoid matching on a recycled desc addr */ | |
91 | jrp->entinfo[sw_idx].desc_addr_dma = 0; | |
92 | ||
93 | /* Stash callback params for use outside of lock */ | |
94 | usercall = jrp->entinfo[sw_idx].callbk; | |
95 | userarg = jrp->entinfo[sw_idx].cbkarg; | |
96 | userdesc = jrp->entinfo[sw_idx].desc_addr_virt; | |
97 | userstatus = jrp->outring[hw_idx].jrstatus; | |
98 | ||
14a8e29c KP |
99 | /* set done */ |
100 | wr_reg32(&jrp->rregs->outring_rmvd, 1); | |
8e8ec596 KP |
101 | |
102 | jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) & | |
103 | (JOBR_DEPTH - 1); | |
104 | ||
105 | /* | |
106 | * if this job completed out-of-order, do not increment | |
107 | * the tail. Otherwise, increment tail by 1 plus the | |
108 | * number of subsequent jobs already completed out-of-order | |
109 | */ | |
110 | if (sw_idx == tail) { | |
111 | do { | |
112 | tail = (tail + 1) & (JOBR_DEPTH - 1); | |
113 | smp_read_barrier_depends(); | |
114 | } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 && | |
115 | jrp->entinfo[tail].desc_addr_dma == 0); | |
116 | ||
117 | jrp->tail = tail; | |
118 | } | |
119 | ||
ce026cb9 | 120 | spin_unlock(&jrp->outlock); |
8e8ec596 KP |
121 | |
122 | /* Finally, execute user's callback */ | |
123 | usercall(dev, userdesc, userstatus, userarg); | |
8e8ec596 KP |
124 | } |
125 | ||
8e8ec596 KP |
126 | /* reenable / unmask IRQs */ |
127 | clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); | |
128 | } | |
129 | ||
8e8ec596 KP |
130 | /** |
131 | * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK, | |
132 | * -EBUSY if the queue is full, -EIO if it cannot map the caller's | |
133 | * descriptor. | |
134 | * @dev: device of the job ring to be used. This device should have | |
135 | * been assigned prior by caam_jr_register(). | |
136 | * @desc: points to a job descriptor that execute our request. All | |
137 | * descriptors (and all referenced data) must be in a DMAable | |
138 | * region, and all data references must be physical addresses | |
139 | * accessible to CAAM (i.e. within a PAMU window granted | |
140 | * to it). | |
141 | * @cbk: pointer to a callback function to be invoked upon completion | |
142 | * of this request. This has the form: | |
143 | * callback(struct device *dev, u32 *desc, u32 stat, void *arg) | |
144 | * where: | |
145 | * @dev: contains the job ring device that processed this | |
146 | * response. | |
147 | * @desc: descriptor that initiated the request, same as | |
148 | * "desc" being argued to caam_jr_enqueue(). | |
149 | * @status: untranslated status received from CAAM. See the | |
150 | * reference manual for a detailed description of | |
151 | * error meaning, or see the JRSTA definitions in the | |
152 | * register header file | |
153 | * @areq: optional pointer to an argument passed with the | |
154 | * original request | |
155 | * @areq: optional pointer to a user argument for use at callback | |
156 | * time. | |
157 | **/ | |
158 | int caam_jr_enqueue(struct device *dev, u32 *desc, | |
159 | void (*cbk)(struct device *dev, u32 *desc, | |
160 | u32 status, void *areq), | |
161 | void *areq) | |
162 | { | |
163 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); | |
164 | struct caam_jrentry_info *head_entry; | |
8e8ec596 KP |
165 | int head, tail, desc_size; |
166 | dma_addr_t desc_dma; | |
167 | ||
168 | desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32); | |
169 | desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE); | |
170 | if (dma_mapping_error(dev, desc_dma)) { | |
171 | dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n"); | |
172 | return -EIO; | |
173 | } | |
174 | ||
ce026cb9 | 175 | spin_lock_bh(&jrp->inplock); |
8e8ec596 KP |
176 | |
177 | head = jrp->head; | |
178 | tail = ACCESS_ONCE(jrp->tail); | |
179 | ||
180 | if (!rd_reg32(&jrp->rregs->inpring_avail) || | |
181 | CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) { | |
ce026cb9 | 182 | spin_unlock_bh(&jrp->inplock); |
8e8ec596 KP |
183 | dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE); |
184 | return -EBUSY; | |
185 | } | |
186 | ||
187 | head_entry = &jrp->entinfo[head]; | |
188 | head_entry->desc_addr_virt = desc; | |
189 | head_entry->desc_size = desc_size; | |
190 | head_entry->callbk = (void *)cbk; | |
191 | head_entry->cbkarg = areq; | |
192 | head_entry->desc_addr_dma = desc_dma; | |
193 | ||
194 | jrp->inpring[jrp->inp_ring_write_index] = desc_dma; | |
195 | ||
196 | smp_wmb(); | |
197 | ||
198 | jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) & | |
199 | (JOBR_DEPTH - 1); | |
200 | jrp->head = (head + 1) & (JOBR_DEPTH - 1); | |
201 | ||
8e8ec596 KP |
202 | wr_reg32(&jrp->rregs->inpring_jobadd, 1); |
203 | ||
ce026cb9 | 204 | spin_unlock_bh(&jrp->inplock); |
8e8ec596 KP |
205 | |
206 | return 0; | |
207 | } | |
208 | EXPORT_SYMBOL(caam_jr_enqueue); | |
209 | ||
210 | static int caam_reset_hw_jr(struct device *dev) | |
211 | { | |
212 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); | |
213 | unsigned int timeout = 100000; | |
214 | ||
215 | /* | |
9620fd95 KP |
216 | * mask interrupts since we are going to poll |
217 | * for reset completion status | |
8e8ec596 | 218 | */ |
9620fd95 | 219 | setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); |
8e8ec596 KP |
220 | |
221 | /* initiate flush (required prior to reset) */ | |
222 | wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); | |
223 | while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) == | |
224 | JRINT_ERR_HALT_INPROGRESS) && --timeout) | |
225 | cpu_relax(); | |
226 | ||
227 | if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) != | |
228 | JRINT_ERR_HALT_COMPLETE || timeout == 0) { | |
229 | dev_err(dev, "failed to flush job ring %d\n", jrp->ridx); | |
230 | return -EIO; | |
231 | } | |
232 | ||
233 | /* initiate reset */ | |
234 | timeout = 100000; | |
235 | wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); | |
236 | while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout) | |
237 | cpu_relax(); | |
238 | ||
239 | if (timeout == 0) { | |
240 | dev_err(dev, "failed to reset job ring %d\n", jrp->ridx); | |
241 | return -EIO; | |
242 | } | |
243 | ||
9620fd95 KP |
244 | /* unmask interrupts */ |
245 | clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); | |
8e8ec596 KP |
246 | |
247 | return 0; | |
248 | } | |
249 | ||
250 | /* | |
251 | * Init JobR independent of platform property detection | |
252 | */ | |
253 | static int caam_jr_init(struct device *dev) | |
254 | { | |
255 | struct caam_drv_private_jr *jrp; | |
256 | dma_addr_t inpbusaddr, outbusaddr; | |
257 | int i, error; | |
258 | ||
259 | jrp = dev_get_drvdata(dev); | |
260 | ||
a0ca6ca0 | 261 | tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev); |
9620fd95 | 262 | |
a0ca6ca0 | 263 | /* Connect job ring interrupt handler. */ |
9620fd95 KP |
264 | error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED, |
265 | "caam-jobr", dev); | |
266 | if (error) { | |
267 | dev_err(dev, "can't connect JobR %d interrupt (%d)\n", | |
268 | jrp->ridx, jrp->irq); | |
269 | irq_dispose_mapping(jrp->irq); | |
270 | jrp->irq = 0; | |
271 | return -EINVAL; | |
272 | } | |
273 | ||
8e8ec596 KP |
274 | error = caam_reset_hw_jr(dev); |
275 | if (error) | |
276 | return error; | |
277 | ||
1af8ea86 BB |
278 | jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, |
279 | &inpbusaddr, GFP_KERNEL); | |
280 | ||
281 | jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) * | |
282 | JOBR_DEPTH, &outbusaddr, GFP_KERNEL); | |
8e8ec596 KP |
283 | |
284 | jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH, | |
285 | GFP_KERNEL); | |
286 | ||
287 | if ((jrp->inpring == NULL) || (jrp->outring == NULL) || | |
288 | (jrp->entinfo == NULL)) { | |
289 | dev_err(dev, "can't allocate job rings for %d\n", | |
290 | jrp->ridx); | |
291 | return -ENOMEM; | |
292 | } | |
293 | ||
294 | for (i = 0; i < JOBR_DEPTH; i++) | |
295 | jrp->entinfo[i].desc_addr_dma = !0; | |
296 | ||
297 | /* Setup rings */ | |
8e8ec596 KP |
298 | jrp->inp_ring_write_index = 0; |
299 | jrp->out_ring_read_index = 0; | |
300 | jrp->head = 0; | |
301 | jrp->tail = 0; | |
302 | ||
303 | wr_reg64(&jrp->rregs->inpring_base, inpbusaddr); | |
304 | wr_reg64(&jrp->rregs->outring_base, outbusaddr); | |
305 | wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH); | |
306 | wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH); | |
307 | ||
308 | jrp->ringsize = JOBR_DEPTH; | |
309 | ||
310 | spin_lock_init(&jrp->inplock); | |
311 | spin_lock_init(&jrp->outlock); | |
312 | ||
313 | /* Select interrupt coalescing parameters */ | |
314 | setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC | | |
315 | (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | | |
316 | (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); | |
317 | ||
8e8ec596 KP |
318 | return 0; |
319 | } | |
320 | ||
321 | /* | |
322 | * Shutdown JobR independent of platform property code | |
323 | */ | |
324 | int caam_jr_shutdown(struct device *dev) | |
325 | { | |
326 | struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); | |
327 | dma_addr_t inpbusaddr, outbusaddr; | |
a0ca6ca0 | 328 | int ret; |
8e8ec596 KP |
329 | |
330 | ret = caam_reset_hw_jr(dev); | |
331 | ||
a0ca6ca0 | 332 | tasklet_kill(&jrp->irqtask); |
8e8ec596 KP |
333 | |
334 | /* Release interrupt */ | |
335 | free_irq(jrp->irq, dev); | |
336 | ||
337 | /* Free rings */ | |
338 | inpbusaddr = rd_reg64(&jrp->rregs->inpring_base); | |
339 | outbusaddr = rd_reg64(&jrp->rregs->outring_base); | |
1af8ea86 BB |
340 | dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, |
341 | jrp->inpring, inpbusaddr); | |
342 | dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, | |
343 | jrp->outring, outbusaddr); | |
8e8ec596 | 344 | kfree(jrp->entinfo); |
3862de1f | 345 | of_device_unregister(jrp->jr_pdev); |
8e8ec596 KP |
346 | |
347 | return ret; | |
348 | } | |
349 | ||
350 | /* | |
351 | * Probe routine for each detected JobR subsystem. It assumes that | |
352 | * property detection was picked up externally. | |
353 | */ | |
354 | int caam_jr_probe(struct platform_device *pdev, struct device_node *np, | |
355 | int ring) | |
356 | { | |
357 | struct device *ctrldev, *jrdev; | |
358 | struct platform_device *jr_pdev; | |
359 | struct caam_drv_private *ctrlpriv; | |
360 | struct caam_drv_private_jr *jrpriv; | |
361 | u32 *jroffset; | |
362 | int error; | |
363 | ||
364 | ctrldev = &pdev->dev; | |
365 | ctrlpriv = dev_get_drvdata(ctrldev); | |
366 | ||
367 | jrpriv = kmalloc(sizeof(struct caam_drv_private_jr), | |
368 | GFP_KERNEL); | |
369 | if (jrpriv == NULL) { | |
370 | dev_err(ctrldev, "can't alloc private mem for job ring %d\n", | |
371 | ring); | |
372 | return -ENOMEM; | |
373 | } | |
374 | jrpriv->parentdev = ctrldev; /* point back to parent */ | |
375 | jrpriv->ridx = ring; /* save ring identity relative to detection */ | |
376 | ||
377 | /* | |
378 | * Derive a pointer to the detected JobRs regs | |
379 | * Driver has already iomapped the entire space, we just | |
380 | * need to add in the offset to this JobR. Don't know if I | |
381 | * like this long-term, but it'll run | |
382 | */ | |
383 | jroffset = (u32 *)of_get_property(np, "reg", NULL); | |
384 | jrpriv->rregs = (struct caam_job_ring __iomem *)((void *)ctrlpriv->ctrl | |
385 | + *jroffset); | |
386 | ||
387 | /* Build a local dev for each detected queue */ | |
388 | jr_pdev = of_platform_device_create(np, NULL, ctrldev); | |
389 | if (jr_pdev == NULL) { | |
390 | kfree(jrpriv); | |
391 | return -EINVAL; | |
392 | } | |
3862de1f VG |
393 | |
394 | jrpriv->jr_pdev = jr_pdev; | |
8e8ec596 KP |
395 | jrdev = &jr_pdev->dev; |
396 | dev_set_drvdata(jrdev, jrpriv); | |
397 | ctrlpriv->jrdev[ring] = jrdev; | |
398 | ||
e13af18a KP |
399 | if (sizeof(dma_addr_t) == sizeof(u64)) |
400 | if (of_device_is_compatible(np, "fsl,sec-v5.0-job-ring")) | |
401 | dma_set_mask(jrdev, DMA_BIT_MASK(40)); | |
402 | else | |
403 | dma_set_mask(jrdev, DMA_BIT_MASK(36)); | |
404 | else | |
405 | dma_set_mask(jrdev, DMA_BIT_MASK(32)); | |
406 | ||
8e8ec596 KP |
407 | /* Identify the interrupt */ |
408 | jrpriv->irq = of_irq_to_resource(np, 0, NULL); | |
409 | ||
410 | /* Now do the platform independent part */ | |
411 | error = caam_jr_init(jrdev); /* now turn on hardware */ | |
412 | if (error) { | |
3862de1f | 413 | of_device_unregister(jr_pdev); |
8e8ec596 KP |
414 | kfree(jrpriv); |
415 | return error; | |
416 | } | |
417 | ||
418 | return error; | |
419 | } |