2 * CAAM/SEC 4.x transport/backend driver
3 * JobR backend functionality
5 * Copyright 2008-2012 Freescale Semiconductor, Inc.
8 #include <linux/of_irq.h>
16 /* Main per-ring interrupt handler */
17 static irqreturn_t
caam_jr_interrupt(int irq
, void *st_dev
)
19 struct device
*dev
= st_dev
;
20 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
24 * Check the output ring for ready responses, kick
25 * tasklet if jobs done.
27 irqstate
= rd_reg32(&jrp
->rregs
->jrintstatus
);
32 * If JobR error, we got more development work to do
33 * Flag a bug now, but we really need to shut down and
34 * restart the queue (and fix code).
36 if (irqstate
& JRINT_JR_ERROR
) {
37 dev_err(dev
, "job ring error: irqstate: %08x\n", irqstate
);
41 /* mask valid interrupts */
42 setbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
44 /* Have valid interrupt at this point, just ACK and trigger */
45 wr_reg32(&jrp
->rregs
->jrintstatus
, irqstate
);
48 tasklet_schedule(&jrp
->irqtask
);
54 /* Deferred service handler, run as interrupt-fired tasklet */
55 static void caam_jr_dequeue(unsigned long devarg
)
57 int hw_idx
, sw_idx
, i
, head
, tail
;
58 struct device
*dev
= (struct device
*)devarg
;
59 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
60 void (*usercall
)(struct device
*dev
, u32
*desc
, u32 status
, void *arg
);
61 u32
*userdesc
, userstatus
;
64 while (rd_reg32(&jrp
->rregs
->outring_used
)) {
66 head
= ACCESS_ONCE(jrp
->head
);
68 spin_lock(&jrp
->outlock
);
70 sw_idx
= tail
= jrp
->tail
;
71 hw_idx
= jrp
->out_ring_read_index
;
73 for (i
= 0; CIRC_CNT(head
, tail
+ i
, JOBR_DEPTH
) >= 1; i
++) {
74 sw_idx
= (tail
+ i
) & (JOBR_DEPTH
- 1);
76 smp_read_barrier_depends();
78 if (jrp
->outring
[hw_idx
].desc
==
79 jrp
->entinfo
[sw_idx
].desc_addr_dma
)
82 /* we should never fail to find a matching descriptor */
83 BUG_ON(CIRC_CNT(head
, tail
+ i
, JOBR_DEPTH
) <= 0);
85 /* Unmap just-run descriptor so we can post-process */
86 dma_unmap_single(dev
, jrp
->outring
[hw_idx
].desc
,
87 jrp
->entinfo
[sw_idx
].desc_size
,
90 /* mark completed, avoid matching on a recycled desc addr */
91 jrp
->entinfo
[sw_idx
].desc_addr_dma
= 0;
93 /* Stash callback params for use outside of lock */
94 usercall
= jrp
->entinfo
[sw_idx
].callbk
;
95 userarg
= jrp
->entinfo
[sw_idx
].cbkarg
;
96 userdesc
= jrp
->entinfo
[sw_idx
].desc_addr_virt
;
97 userstatus
= jrp
->outring
[hw_idx
].jrstatus
;
100 wr_reg32(&jrp
->rregs
->outring_rmvd
, 1);
102 jrp
->out_ring_read_index
= (jrp
->out_ring_read_index
+ 1) &
106 * if this job completed out-of-order, do not increment
107 * the tail. Otherwise, increment tail by 1 plus the
108 * number of subsequent jobs already completed out-of-order
110 if (sw_idx
== tail
) {
112 tail
= (tail
+ 1) & (JOBR_DEPTH
- 1);
113 smp_read_barrier_depends();
114 } while (CIRC_CNT(head
, tail
, JOBR_DEPTH
) >= 1 &&
115 jrp
->entinfo
[tail
].desc_addr_dma
== 0);
120 spin_unlock(&jrp
->outlock
);
122 /* Finally, execute user's callback */
123 usercall(dev
, userdesc
, userstatus
, userarg
);
126 /* reenable / unmask IRQs */
127 clrbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
131 * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
132 * -EBUSY if the queue is full, -EIO if it cannot map the caller's
134 * @dev: device of the job ring to be used. This device should have
135 * been assigned prior by caam_jr_register().
136 * @desc: points to a job descriptor that execute our request. All
137 * descriptors (and all referenced data) must be in a DMAable
138 * region, and all data references must be physical addresses
139 * accessible to CAAM (i.e. within a PAMU window granted
141 * @cbk: pointer to a callback function to be invoked upon completion
142 * of this request. This has the form:
143 * callback(struct device *dev, u32 *desc, u32 stat, void *arg)
145 * @dev: contains the job ring device that processed this
147 * @desc: descriptor that initiated the request, same as
148 * "desc" being argued to caam_jr_enqueue().
149 * @status: untranslated status received from CAAM. See the
150 * reference manual for a detailed description of
151 * error meaning, or see the JRSTA definitions in the
152 * register header file
153 * @areq: optional pointer to an argument passed with the
155 * @areq: optional pointer to a user argument for use at callback
158 int caam_jr_enqueue(struct device
*dev
, u32
*desc
,
159 void (*cbk
)(struct device
*dev
, u32
*desc
,
160 u32 status
, void *areq
),
163 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
164 struct caam_jrentry_info
*head_entry
;
165 int head
, tail
, desc_size
;
168 desc_size
= (*desc
& HDR_JD_LENGTH_MASK
) * sizeof(u32
);
169 desc_dma
= dma_map_single(dev
, desc
, desc_size
, DMA_TO_DEVICE
);
170 if (dma_mapping_error(dev
, desc_dma
)) {
171 dev_err(dev
, "caam_jr_enqueue(): can't map jobdesc\n");
175 spin_lock_bh(&jrp
->inplock
);
178 tail
= ACCESS_ONCE(jrp
->tail
);
180 if (!rd_reg32(&jrp
->rregs
->inpring_avail
) ||
181 CIRC_SPACE(head
, tail
, JOBR_DEPTH
) <= 0) {
182 spin_unlock_bh(&jrp
->inplock
);
183 dma_unmap_single(dev
, desc_dma
, desc_size
, DMA_TO_DEVICE
);
187 head_entry
= &jrp
->entinfo
[head
];
188 head_entry
->desc_addr_virt
= desc
;
189 head_entry
->desc_size
= desc_size
;
190 head_entry
->callbk
= (void *)cbk
;
191 head_entry
->cbkarg
= areq
;
192 head_entry
->desc_addr_dma
= desc_dma
;
194 jrp
->inpring
[jrp
->inp_ring_write_index
] = desc_dma
;
198 jrp
->inp_ring_write_index
= (jrp
->inp_ring_write_index
+ 1) &
200 jrp
->head
= (head
+ 1) & (JOBR_DEPTH
- 1);
202 wr_reg32(&jrp
->rregs
->inpring_jobadd
, 1);
204 spin_unlock_bh(&jrp
->inplock
);
208 EXPORT_SYMBOL(caam_jr_enqueue
);
210 static int caam_reset_hw_jr(struct device
*dev
)
212 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
213 unsigned int timeout
= 100000;
216 * mask interrupts since we are going to poll
217 * for reset completion status
219 setbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
221 /* initiate flush (required prior to reset) */
222 wr_reg32(&jrp
->rregs
->jrcommand
, JRCR_RESET
);
223 while (((rd_reg32(&jrp
->rregs
->jrintstatus
) & JRINT_ERR_HALT_MASK
) ==
224 JRINT_ERR_HALT_INPROGRESS
) && --timeout
)
227 if ((rd_reg32(&jrp
->rregs
->jrintstatus
) & JRINT_ERR_HALT_MASK
) !=
228 JRINT_ERR_HALT_COMPLETE
|| timeout
== 0) {
229 dev_err(dev
, "failed to flush job ring %d\n", jrp
->ridx
);
235 wr_reg32(&jrp
->rregs
->jrcommand
, JRCR_RESET
);
236 while ((rd_reg32(&jrp
->rregs
->jrcommand
) & JRCR_RESET
) && --timeout
)
240 dev_err(dev
, "failed to reset job ring %d\n", jrp
->ridx
);
244 /* unmask interrupts */
245 clrbits32(&jrp
->rregs
->rconfig_lo
, JRCFG_IMSK
);
251 * Init JobR independent of platform property detection
253 static int caam_jr_init(struct device
*dev
)
255 struct caam_drv_private_jr
*jrp
;
256 dma_addr_t inpbusaddr
, outbusaddr
;
259 jrp
= dev_get_drvdata(dev
);
261 tasklet_init(&jrp
->irqtask
, caam_jr_dequeue
, (unsigned long)dev
);
263 /* Connect job ring interrupt handler. */
264 error
= request_irq(jrp
->irq
, caam_jr_interrupt
, IRQF_SHARED
,
267 dev_err(dev
, "can't connect JobR %d interrupt (%d)\n",
268 jrp
->ridx
, jrp
->irq
);
269 irq_dispose_mapping(jrp
->irq
);
274 error
= caam_reset_hw_jr(dev
);
278 jrp
->inpring
= dma_alloc_coherent(dev
, sizeof(dma_addr_t
) * JOBR_DEPTH
,
279 &inpbusaddr
, GFP_KERNEL
);
281 jrp
->outring
= dma_alloc_coherent(dev
, sizeof(struct jr_outentry
) *
282 JOBR_DEPTH
, &outbusaddr
, GFP_KERNEL
);
284 jrp
->entinfo
= kzalloc(sizeof(struct caam_jrentry_info
) * JOBR_DEPTH
,
287 if ((jrp
->inpring
== NULL
) || (jrp
->outring
== NULL
) ||
288 (jrp
->entinfo
== NULL
)) {
289 dev_err(dev
, "can't allocate job rings for %d\n",
294 for (i
= 0; i
< JOBR_DEPTH
; i
++)
295 jrp
->entinfo
[i
].desc_addr_dma
= !0;
298 jrp
->inp_ring_write_index
= 0;
299 jrp
->out_ring_read_index
= 0;
303 wr_reg64(&jrp
->rregs
->inpring_base
, inpbusaddr
);
304 wr_reg64(&jrp
->rregs
->outring_base
, outbusaddr
);
305 wr_reg32(&jrp
->rregs
->inpring_size
, JOBR_DEPTH
);
306 wr_reg32(&jrp
->rregs
->outring_size
, JOBR_DEPTH
);
308 jrp
->ringsize
= JOBR_DEPTH
;
310 spin_lock_init(&jrp
->inplock
);
311 spin_lock_init(&jrp
->outlock
);
313 /* Select interrupt coalescing parameters */
314 setbits32(&jrp
->rregs
->rconfig_lo
, JOBR_INTC
|
315 (JOBR_INTC_COUNT_THLD
<< JRCFG_ICDCT_SHIFT
) |
316 (JOBR_INTC_TIME_THLD
<< JRCFG_ICTT_SHIFT
));
322 * Shutdown JobR independent of platform property code
324 int caam_jr_shutdown(struct device
*dev
)
326 struct caam_drv_private_jr
*jrp
= dev_get_drvdata(dev
);
327 dma_addr_t inpbusaddr
, outbusaddr
;
330 ret
= caam_reset_hw_jr(dev
);
332 tasklet_kill(&jrp
->irqtask
);
334 /* Release interrupt */
335 free_irq(jrp
->irq
, dev
);
338 inpbusaddr
= rd_reg64(&jrp
->rregs
->inpring_base
);
339 outbusaddr
= rd_reg64(&jrp
->rregs
->outring_base
);
340 dma_free_coherent(dev
, sizeof(dma_addr_t
) * JOBR_DEPTH
,
341 jrp
->inpring
, inpbusaddr
);
342 dma_free_coherent(dev
, sizeof(struct jr_outentry
) * JOBR_DEPTH
,
343 jrp
->outring
, outbusaddr
);
345 of_device_unregister(jrp
->jr_pdev
);
351 * Probe routine for each detected JobR subsystem. It assumes that
352 * property detection was picked up externally.
354 int caam_jr_probe(struct platform_device
*pdev
, struct device_node
*np
,
357 struct device
*ctrldev
, *jrdev
;
358 struct platform_device
*jr_pdev
;
359 struct caam_drv_private
*ctrlpriv
;
360 struct caam_drv_private_jr
*jrpriv
;
364 ctrldev
= &pdev
->dev
;
365 ctrlpriv
= dev_get_drvdata(ctrldev
);
367 jrpriv
= kmalloc(sizeof(struct caam_drv_private_jr
),
369 if (jrpriv
== NULL
) {
370 dev_err(ctrldev
, "can't alloc private mem for job ring %d\n",
374 jrpriv
->parentdev
= ctrldev
; /* point back to parent */
375 jrpriv
->ridx
= ring
; /* save ring identity relative to detection */
378 * Derive a pointer to the detected JobRs regs
379 * Driver has already iomapped the entire space, we just
380 * need to add in the offset to this JobR. Don't know if I
381 * like this long-term, but it'll run
383 jroffset
= (u32
*)of_get_property(np
, "reg", NULL
);
384 jrpriv
->rregs
= (struct caam_job_ring __iomem
*)((void *)ctrlpriv
->ctrl
387 /* Build a local dev for each detected queue */
388 jr_pdev
= of_platform_device_create(np
, NULL
, ctrldev
);
389 if (jr_pdev
== NULL
) {
394 jrpriv
->jr_pdev
= jr_pdev
;
395 jrdev
= &jr_pdev
->dev
;
396 dev_set_drvdata(jrdev
, jrpriv
);
397 ctrlpriv
->jrdev
[ring
] = jrdev
;
399 if (sizeof(dma_addr_t
) == sizeof(u64
))
400 if (of_device_is_compatible(np
, "fsl,sec-v5.0-job-ring"))
401 dma_set_mask(jrdev
, DMA_BIT_MASK(40));
403 dma_set_mask(jrdev
, DMA_BIT_MASK(36));
405 dma_set_mask(jrdev
, DMA_BIT_MASK(32));
407 /* Identify the interrupt */
408 jrpriv
->irq
= of_irq_to_resource(np
, 0, NULL
);
410 /* Now do the platform independent part */
411 error
= caam_jr_init(jrdev
); /* now turn on hardware */
413 of_device_unregister(jr_pdev
);