2 * Copyright (c) 2013 Linaro Ltd.
3 * Copyright (c) 2013 Hisilicon Limited.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
9 #include <linux/sched.h>
10 #include <linux/device.h>
11 #include <linux/dmaengine.h>
12 #include <linux/init.h>
13 #include <linux/interrupt.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/of_device.h>
21 #include <linux/clk.h>
22 #include <linux/of_dma.h>
26 #define DRIVER_NAME "k3-dma"
27 #define DMA_MAX_SIZE 0x1ffc
33 #define INT_TC1_MASK 0x18
34 #define INT_ERR1_MASK 0x20
35 #define INT_ERR2_MASK 0x24
36 #define INT_TC1_RAW 0x600
37 #define INT_ERR1_RAW 0x608
38 #define INT_ERR2_RAW 0x610
41 #define CX_CUR_CNT 0x704
48 #define AXI_CFG_DEFAULT 0x201201
50 #define CX_LLI_CHAIN_EN 0x2
52 #define CX_CFG_MEM2PER (0x1 << 2)
53 #define CX_CFG_PER2MEM (0x2 << 2)
54 #define CX_CFG_SRCINCR (0x1 << 31)
55 #define CX_CFG_DSTINCR (0x1 << 30)
66 struct k3_dma_desc_sw
{
67 struct virt_dma_desc vd
;
68 dma_addr_t desc_hw_lli
;
71 struct k3_desc_hw desc_hw
[0];
78 struct virt_dma_chan vc
;
79 struct k3_dma_phy
*phy
;
80 struct list_head node
;
81 enum dma_transfer_direction dir
;
83 enum dma_status status
;
89 struct k3_dma_chan
*vchan
;
90 struct k3_dma_desc_sw
*ds_run
;
91 struct k3_dma_desc_sw
*ds_done
;
95 struct dma_device slave
;
97 struct tasklet_struct task
;
99 struct list_head chan_pending
;
100 struct k3_dma_phy
*phy
;
101 struct k3_dma_chan
*chans
;
108 #define to_k3_dma(dmadev) container_of(dmadev, struct k3_dma_dev, slave)
110 static struct k3_dma_chan
*to_k3_chan(struct dma_chan
*chan
)
112 return container_of(chan
, struct k3_dma_chan
, vc
.chan
);
115 static void k3_dma_pause_dma(struct k3_dma_phy
*phy
, bool on
)
120 val
= readl_relaxed(phy
->base
+ CX_CFG
);
122 writel_relaxed(val
, phy
->base
+ CX_CFG
);
124 val
= readl_relaxed(phy
->base
+ CX_CFG
);
126 writel_relaxed(val
, phy
->base
+ CX_CFG
);
130 static void k3_dma_terminate_chan(struct k3_dma_phy
*phy
, struct k3_dma_dev
*d
)
134 k3_dma_pause_dma(phy
, false);
136 val
= 0x1 << phy
->idx
;
137 writel_relaxed(val
, d
->base
+ INT_TC1_RAW
);
138 writel_relaxed(val
, d
->base
+ INT_ERR1_RAW
);
139 writel_relaxed(val
, d
->base
+ INT_ERR2_RAW
);
142 static void k3_dma_set_desc(struct k3_dma_phy
*phy
, struct k3_desc_hw
*hw
)
144 writel_relaxed(hw
->lli
, phy
->base
+ CX_LLI
);
145 writel_relaxed(hw
->count
, phy
->base
+ CX_CNT
);
146 writel_relaxed(hw
->saddr
, phy
->base
+ CX_SRC
);
147 writel_relaxed(hw
->daddr
, phy
->base
+ CX_DST
);
148 writel_relaxed(AXI_CFG_DEFAULT
, phy
->base
+ AXI_CFG
);
149 writel_relaxed(hw
->config
, phy
->base
+ CX_CFG
);
152 static u32
k3_dma_get_curr_cnt(struct k3_dma_dev
*d
, struct k3_dma_phy
*phy
)
156 cnt
= readl_relaxed(d
->base
+ CX_CUR_CNT
+ phy
->idx
* 0x10);
161 static u32
k3_dma_get_curr_lli(struct k3_dma_phy
*phy
)
163 return readl_relaxed(phy
->base
+ CX_LLI
);
166 static u32
k3_dma_get_chan_stat(struct k3_dma_dev
*d
)
168 return readl_relaxed(d
->base
+ CH_STAT
);
171 static void k3_dma_enable_dma(struct k3_dma_dev
*d
, bool on
)
174 /* set same priority */
175 writel_relaxed(0x0, d
->base
+ CH_PRI
);
178 writel_relaxed(0xffff, d
->base
+ INT_TC1_MASK
);
179 writel_relaxed(0xffff, d
->base
+ INT_ERR1_MASK
);
180 writel_relaxed(0xffff, d
->base
+ INT_ERR2_MASK
);
183 writel_relaxed(0x0, d
->base
+ INT_TC1_MASK
);
184 writel_relaxed(0x0, d
->base
+ INT_ERR1_MASK
);
185 writel_relaxed(0x0, d
->base
+ INT_ERR2_MASK
);
189 static irqreturn_t
k3_dma_int_handler(int irq
, void *dev_id
)
191 struct k3_dma_dev
*d
= (struct k3_dma_dev
*)dev_id
;
192 struct k3_dma_phy
*p
;
193 struct k3_dma_chan
*c
;
194 u32 stat
= readl_relaxed(d
->base
+ INT_STAT
);
195 u32 tc1
= readl_relaxed(d
->base
+ INT_TC1
);
196 u32 err1
= readl_relaxed(d
->base
+ INT_ERR1
);
197 u32 err2
= readl_relaxed(d
->base
+ INT_ERR2
);
203 if (likely(tc1
& BIT(i
))) {
209 spin_lock_irqsave(&c
->vc
.lock
, flags
);
210 vchan_cookie_complete(&p
->ds_run
->vd
);
211 p
->ds_done
= p
->ds_run
;
212 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
216 if (unlikely((err1
& BIT(i
)) || (err2
& BIT(i
))))
217 dev_warn(d
->slave
.dev
, "DMA ERR\n");
220 writel_relaxed(irq_chan
, d
->base
+ INT_TC1_RAW
);
221 writel_relaxed(err1
, d
->base
+ INT_ERR1_RAW
);
222 writel_relaxed(err2
, d
->base
+ INT_ERR2_RAW
);
225 tasklet_schedule(&d
->task
);
231 static int k3_dma_start_txd(struct k3_dma_chan
*c
)
233 struct k3_dma_dev
*d
= to_k3_dma(c
->vc
.chan
.device
);
234 struct virt_dma_desc
*vd
= vchan_next_desc(&c
->vc
);
239 if (BIT(c
->phy
->idx
) & k3_dma_get_chan_stat(d
))
243 struct k3_dma_desc_sw
*ds
=
244 container_of(vd
, struct k3_dma_desc_sw
, vd
);
246 * fetch and remove request from vc->desc_issued
247 * so vc->desc_issued only contains desc pending
249 list_del(&ds
->vd
.node
);
251 c
->phy
->ds_done
= NULL
;
253 k3_dma_set_desc(c
->phy
, &ds
->desc_hw
[0]);
256 c
->phy
->ds_done
= NULL
;
257 c
->phy
->ds_run
= NULL
;
261 static void k3_dma_tasklet(unsigned long arg
)
263 struct k3_dma_dev
*d
= (struct k3_dma_dev
*)arg
;
264 struct k3_dma_phy
*p
;
265 struct k3_dma_chan
*c
, *cn
;
266 unsigned pch
, pch_alloc
= 0;
268 /* check new dma request of running channel in vc->desc_issued */
269 list_for_each_entry_safe(c
, cn
, &d
->slave
.channels
, vc
.chan
.device_node
) {
270 spin_lock_irq(&c
->vc
.lock
);
272 if (p
&& p
->ds_done
) {
273 if (k3_dma_start_txd(c
)) {
274 /* No current txd associated with this channel */
275 dev_dbg(d
->slave
.dev
, "pchan %u: free\n", p
->idx
);
276 /* Mark this channel free */
281 spin_unlock_irq(&c
->vc
.lock
);
284 /* check new channel request in d->chan_pending */
285 spin_lock_irq(&d
->lock
);
286 for (pch
= 0; pch
< d
->dma_channels
; pch
++) {
289 if (p
->vchan
== NULL
&& !list_empty(&d
->chan_pending
)) {
290 c
= list_first_entry(&d
->chan_pending
,
291 struct k3_dma_chan
, node
);
292 /* remove from d->chan_pending */
293 list_del_init(&c
->node
);
294 pch_alloc
|= 1 << pch
;
295 /* Mark this channel allocated */
298 dev_dbg(d
->slave
.dev
, "pchan %u: alloc vchan %p\n", pch
, &c
->vc
);
301 spin_unlock_irq(&d
->lock
);
303 for (pch
= 0; pch
< d
->dma_channels
; pch
++) {
304 if (pch_alloc
& (1 << pch
)) {
308 spin_lock_irq(&c
->vc
.lock
);
310 spin_unlock_irq(&c
->vc
.lock
);
316 static void k3_dma_free_chan_resources(struct dma_chan
*chan
)
318 struct k3_dma_chan
*c
= to_k3_chan(chan
);
319 struct k3_dma_dev
*d
= to_k3_dma(chan
->device
);
322 spin_lock_irqsave(&d
->lock
, flags
);
323 list_del_init(&c
->node
);
324 spin_unlock_irqrestore(&d
->lock
, flags
);
326 vchan_free_chan_resources(&c
->vc
);
330 static enum dma_status
k3_dma_tx_status(struct dma_chan
*chan
,
331 dma_cookie_t cookie
, struct dma_tx_state
*state
)
333 struct k3_dma_chan
*c
= to_k3_chan(chan
);
334 struct k3_dma_dev
*d
= to_k3_dma(chan
->device
);
335 struct k3_dma_phy
*p
;
336 struct virt_dma_desc
*vd
;
341 ret
= dma_cookie_status(&c
->vc
.chan
, cookie
, state
);
342 if (ret
== DMA_COMPLETE
)
345 spin_lock_irqsave(&c
->vc
.lock
, flags
);
350 * If the cookie is on our issue queue, then the residue is
353 vd
= vchan_find_desc(&c
->vc
, cookie
);
355 bytes
= container_of(vd
, struct k3_dma_desc_sw
, vd
)->size
;
356 } else if ((!p
) || (!p
->ds_run
)) {
359 struct k3_dma_desc_sw
*ds
= p
->ds_run
;
360 u32 clli
= 0, index
= 0;
362 bytes
= k3_dma_get_curr_cnt(d
, p
);
363 clli
= k3_dma_get_curr_lli(p
);
364 index
= (clli
- ds
->desc_hw_lli
) / sizeof(struct k3_desc_hw
);
365 for (; index
< ds
->desc_num
; index
++) {
366 bytes
+= ds
->desc_hw
[index
].count
;
368 if (!ds
->desc_hw
[index
].lli
)
372 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
373 dma_set_residue(state
, bytes
);
377 static void k3_dma_issue_pending(struct dma_chan
*chan
)
379 struct k3_dma_chan
*c
= to_k3_chan(chan
);
380 struct k3_dma_dev
*d
= to_k3_dma(chan
->device
);
383 spin_lock_irqsave(&c
->vc
.lock
, flags
);
384 /* add request to vc->desc_issued */
385 if (vchan_issue_pending(&c
->vc
)) {
388 if (list_empty(&c
->node
)) {
389 /* if new channel, add chan_pending */
390 list_add_tail(&c
->node
, &d
->chan_pending
);
391 /* check in tasklet */
392 tasklet_schedule(&d
->task
);
393 dev_dbg(d
->slave
.dev
, "vchan %p: issued\n", &c
->vc
);
396 spin_unlock(&d
->lock
);
398 dev_dbg(d
->slave
.dev
, "vchan %p: nothing to issue\n", &c
->vc
);
399 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
402 static void k3_dma_fill_desc(struct k3_dma_desc_sw
*ds
, dma_addr_t dst
,
403 dma_addr_t src
, size_t len
, u32 num
, u32 ccfg
)
405 if ((num
+ 1) < ds
->desc_num
)
406 ds
->desc_hw
[num
].lli
= ds
->desc_hw_lli
+ (num
+ 1) *
407 sizeof(struct k3_desc_hw
);
408 ds
->desc_hw
[num
].lli
|= CX_LLI_CHAIN_EN
;
409 ds
->desc_hw
[num
].count
= len
;
410 ds
->desc_hw
[num
].saddr
= src
;
411 ds
->desc_hw
[num
].daddr
= dst
;
412 ds
->desc_hw
[num
].config
= ccfg
;
415 static struct dma_async_tx_descriptor
*k3_dma_prep_memcpy(
416 struct dma_chan
*chan
, dma_addr_t dst
, dma_addr_t src
,
417 size_t len
, unsigned long flags
)
419 struct k3_dma_chan
*c
= to_k3_chan(chan
);
420 struct k3_dma_desc_sw
*ds
;
427 num
= DIV_ROUND_UP(len
, DMA_MAX_SIZE
);
428 ds
= kzalloc(sizeof(*ds
) + num
* sizeof(ds
->desc_hw
[0]), GFP_ATOMIC
);
432 ds
->desc_hw_lli
= __virt_to_phys((unsigned long)&ds
->desc_hw
[0]);
438 /* default is memtomem, without calling device_config */
439 c
->ccfg
= CX_CFG_SRCINCR
| CX_CFG_DSTINCR
| CX_CFG_EN
;
440 c
->ccfg
|= (0xf << 20) | (0xf << 24); /* burst = 16 */
441 c
->ccfg
|= (0x3 << 12) | (0x3 << 16); /* width = 64 bit */
445 copy
= min_t(size_t, len
, DMA_MAX_SIZE
);
446 k3_dma_fill_desc(ds
, dst
, src
, copy
, num
++, c
->ccfg
);
448 if (c
->dir
== DMA_MEM_TO_DEV
) {
450 } else if (c
->dir
== DMA_DEV_TO_MEM
) {
459 ds
->desc_hw
[num
-1].lli
= 0; /* end of link */
460 return vchan_tx_prep(&c
->vc
, &ds
->vd
, flags
);
463 static struct dma_async_tx_descriptor
*k3_dma_prep_slave_sg(
464 struct dma_chan
*chan
, struct scatterlist
*sgl
, unsigned int sglen
,
465 enum dma_transfer_direction dir
, unsigned long flags
, void *context
)
467 struct k3_dma_chan
*c
= to_k3_chan(chan
);
468 struct k3_dma_desc_sw
*ds
;
469 size_t len
, avail
, total
= 0;
470 struct scatterlist
*sg
;
471 dma_addr_t addr
, src
= 0, dst
= 0;
477 for_each_sg(sgl
, sg
, sglen
, i
) {
478 avail
= sg_dma_len(sg
);
479 if (avail
> DMA_MAX_SIZE
)
480 num
+= DIV_ROUND_UP(avail
, DMA_MAX_SIZE
) - 1;
483 ds
= kzalloc(sizeof(*ds
) + num
* sizeof(ds
->desc_hw
[0]), GFP_ATOMIC
);
487 ds
->desc_hw_lli
= __virt_to_phys((unsigned long)&ds
->desc_hw
[0]);
491 for_each_sg(sgl
, sg
, sglen
, i
) {
492 addr
= sg_dma_address(sg
);
493 avail
= sg_dma_len(sg
);
497 len
= min_t(size_t, avail
, DMA_MAX_SIZE
);
499 if (dir
== DMA_MEM_TO_DEV
) {
502 } else if (dir
== DMA_DEV_TO_MEM
) {
507 k3_dma_fill_desc(ds
, dst
, src
, len
, num
++, c
->ccfg
);
514 ds
->desc_hw
[num
-1].lli
= 0; /* end of link */
516 return vchan_tx_prep(&c
->vc
, &ds
->vd
, flags
);
519 static int k3_dma_config(struct dma_chan
*chan
,
520 struct dma_slave_config
*cfg
)
522 struct k3_dma_chan
*c
= to_k3_chan(chan
);
523 u32 maxburst
= 0, val
= 0;
524 enum dma_slave_buswidth width
= DMA_SLAVE_BUSWIDTH_UNDEFINED
;
528 c
->dir
= cfg
->direction
;
529 if (c
->dir
== DMA_DEV_TO_MEM
) {
530 c
->ccfg
= CX_CFG_DSTINCR
;
531 c
->dev_addr
= cfg
->src_addr
;
532 maxburst
= cfg
->src_maxburst
;
533 width
= cfg
->src_addr_width
;
534 } else if (c
->dir
== DMA_MEM_TO_DEV
) {
535 c
->ccfg
= CX_CFG_SRCINCR
;
536 c
->dev_addr
= cfg
->dst_addr
;
537 maxburst
= cfg
->dst_maxburst
;
538 width
= cfg
->dst_addr_width
;
541 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
542 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
543 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
544 case DMA_SLAVE_BUSWIDTH_8_BYTES
:
551 c
->ccfg
|= (val
<< 12) | (val
<< 16);
553 if ((maxburst
== 0) || (maxburst
> 16))
557 c
->ccfg
|= (val
<< 20) | (val
<< 24);
558 c
->ccfg
|= CX_CFG_MEM2PER
| CX_CFG_EN
;
560 /* specific request line */
561 c
->ccfg
|= c
->vc
.chan
.chan_id
<< 4;
566 static int k3_dma_terminate_all(struct dma_chan
*chan
)
568 struct k3_dma_chan
*c
= to_k3_chan(chan
);
569 struct k3_dma_dev
*d
= to_k3_dma(chan
->device
);
570 struct k3_dma_phy
*p
= c
->phy
;
574 dev_dbg(d
->slave
.dev
, "vchan %p: terminate all\n", &c
->vc
);
576 /* Prevent this channel being scheduled */
578 list_del_init(&c
->node
);
579 spin_unlock(&d
->lock
);
581 /* Clear the tx descriptor lists */
582 spin_lock_irqsave(&c
->vc
.lock
, flags
);
583 vchan_get_all_descriptors(&c
->vc
, &head
);
585 /* vchan is assigned to a pchan - stop the channel */
586 k3_dma_terminate_chan(p
, d
);
589 p
->ds_run
= p
->ds_done
= NULL
;
591 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
592 vchan_dma_desc_free_list(&c
->vc
, &head
);
597 static int k3_dma_transfer_pause(struct dma_chan
*chan
)
599 struct k3_dma_chan
*c
= to_k3_chan(chan
);
600 struct k3_dma_dev
*d
= to_k3_dma(chan
->device
);
601 struct k3_dma_phy
*p
= c
->phy
;
603 dev_dbg(d
->slave
.dev
, "vchan %p: pause\n", &c
->vc
);
604 if (c
->status
== DMA_IN_PROGRESS
) {
605 c
->status
= DMA_PAUSED
;
607 k3_dma_pause_dma(p
, false);
610 list_del_init(&c
->node
);
611 spin_unlock(&d
->lock
);
618 static int k3_dma_transfer_resume(struct dma_chan
*chan
)
620 struct k3_dma_chan
*c
= to_k3_chan(chan
);
621 struct k3_dma_dev
*d
= to_k3_dma(chan
->device
);
622 struct k3_dma_phy
*p
= c
->phy
;
625 dev_dbg(d
->slave
.dev
, "vchan %p: resume\n", &c
->vc
);
626 spin_lock_irqsave(&c
->vc
.lock
, flags
);
627 if (c
->status
== DMA_PAUSED
) {
628 c
->status
= DMA_IN_PROGRESS
;
630 k3_dma_pause_dma(p
, true);
631 } else if (!list_empty(&c
->vc
.desc_issued
)) {
633 list_add_tail(&c
->node
, &d
->chan_pending
);
634 spin_unlock(&d
->lock
);
637 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
642 static void k3_dma_free_desc(struct virt_dma_desc
*vd
)
644 struct k3_dma_desc_sw
*ds
=
645 container_of(vd
, struct k3_dma_desc_sw
, vd
);
650 static const struct of_device_id k3_pdma_dt_ids
[] = {
651 { .compatible
= "hisilicon,k3-dma-1.0", },
654 MODULE_DEVICE_TABLE(of
, k3_pdma_dt_ids
);
656 static struct dma_chan
*k3_of_dma_simple_xlate(struct of_phandle_args
*dma_spec
,
657 struct of_dma
*ofdma
)
659 struct k3_dma_dev
*d
= ofdma
->of_dma_data
;
660 unsigned int request
= dma_spec
->args
[0];
662 if (request
> d
->dma_requests
)
665 return dma_get_slave_channel(&(d
->chans
[request
].vc
.chan
));
668 static int k3_dma_probe(struct platform_device
*op
)
670 struct k3_dma_dev
*d
;
671 const struct of_device_id
*of_id
;
672 struct resource
*iores
;
675 iores
= platform_get_resource(op
, IORESOURCE_MEM
, 0);
679 d
= devm_kzalloc(&op
->dev
, sizeof(*d
), GFP_KERNEL
);
683 d
->base
= devm_ioremap_resource(&op
->dev
, iores
);
685 return PTR_ERR(d
->base
);
687 of_id
= of_match_device(k3_pdma_dt_ids
, &op
->dev
);
689 of_property_read_u32((&op
->dev
)->of_node
,
690 "dma-channels", &d
->dma_channels
);
691 of_property_read_u32((&op
->dev
)->of_node
,
692 "dma-requests", &d
->dma_requests
);
695 d
->clk
= devm_clk_get(&op
->dev
, NULL
);
696 if (IS_ERR(d
->clk
)) {
697 dev_err(&op
->dev
, "no dma clk\n");
698 return PTR_ERR(d
->clk
);
701 irq
= platform_get_irq(op
, 0);
702 ret
= devm_request_irq(&op
->dev
, irq
,
703 k3_dma_int_handler
, 0, DRIVER_NAME
, d
);
709 /* init phy channel */
710 d
->phy
= devm_kzalloc(&op
->dev
,
711 d
->dma_channels
* sizeof(struct k3_dma_phy
), GFP_KERNEL
);
715 for (i
= 0; i
< d
->dma_channels
; i
++) {
716 struct k3_dma_phy
*p
= &d
->phy
[i
];
719 p
->base
= d
->base
+ i
* 0x40;
722 INIT_LIST_HEAD(&d
->slave
.channels
);
723 dma_cap_set(DMA_SLAVE
, d
->slave
.cap_mask
);
724 dma_cap_set(DMA_MEMCPY
, d
->slave
.cap_mask
);
725 d
->slave
.dev
= &op
->dev
;
726 d
->slave
.device_free_chan_resources
= k3_dma_free_chan_resources
;
727 d
->slave
.device_tx_status
= k3_dma_tx_status
;
728 d
->slave
.device_prep_dma_memcpy
= k3_dma_prep_memcpy
;
729 d
->slave
.device_prep_slave_sg
= k3_dma_prep_slave_sg
;
730 d
->slave
.device_issue_pending
= k3_dma_issue_pending
;
731 d
->slave
.device_config
= k3_dma_config
;
732 d
->slave
.device_pause
= k3_dma_transfer_pause
;
733 d
->slave
.device_resume
= k3_dma_transfer_resume
;
734 d
->slave
.device_terminate_all
= k3_dma_terminate_all
;
735 d
->slave
.copy_align
= DMAENGINE_ALIGN_8_BYTES
;
737 /* init virtual channel */
738 d
->chans
= devm_kzalloc(&op
->dev
,
739 d
->dma_requests
* sizeof(struct k3_dma_chan
), GFP_KERNEL
);
740 if (d
->chans
== NULL
)
743 for (i
= 0; i
< d
->dma_requests
; i
++) {
744 struct k3_dma_chan
*c
= &d
->chans
[i
];
746 c
->status
= DMA_IN_PROGRESS
;
747 INIT_LIST_HEAD(&c
->node
);
748 c
->vc
.desc_free
= k3_dma_free_desc
;
749 vchan_init(&c
->vc
, &d
->slave
);
752 /* Enable clock before accessing registers */
753 ret
= clk_prepare_enable(d
->clk
);
755 dev_err(&op
->dev
, "clk_prepare_enable failed: %d\n", ret
);
759 k3_dma_enable_dma(d
, true);
761 ret
= dma_async_device_register(&d
->slave
);
763 goto dma_async_register_fail
;
765 ret
= of_dma_controller_register((&op
->dev
)->of_node
,
766 k3_of_dma_simple_xlate
, d
);
768 goto of_dma_register_fail
;
770 spin_lock_init(&d
->lock
);
771 INIT_LIST_HEAD(&d
->chan_pending
);
772 tasklet_init(&d
->task
, k3_dma_tasklet
, (unsigned long)d
);
773 platform_set_drvdata(op
, d
);
774 dev_info(&op
->dev
, "initialized\n");
778 of_dma_register_fail
:
779 dma_async_device_unregister(&d
->slave
);
780 dma_async_register_fail
:
781 clk_disable_unprepare(d
->clk
);
785 static int k3_dma_remove(struct platform_device
*op
)
787 struct k3_dma_chan
*c
, *cn
;
788 struct k3_dma_dev
*d
= platform_get_drvdata(op
);
790 dma_async_device_unregister(&d
->slave
);
791 of_dma_controller_free((&op
->dev
)->of_node
);
793 devm_free_irq(&op
->dev
, d
->irq
, d
);
795 list_for_each_entry_safe(c
, cn
, &d
->slave
.channels
, vc
.chan
.device_node
) {
796 list_del(&c
->vc
.chan
.device_node
);
797 tasklet_kill(&c
->vc
.task
);
799 tasklet_kill(&d
->task
);
800 clk_disable_unprepare(d
->clk
);
804 #ifdef CONFIG_PM_SLEEP
805 static int k3_dma_suspend_dev(struct device
*dev
)
807 struct k3_dma_dev
*d
= dev_get_drvdata(dev
);
810 stat
= k3_dma_get_chan_stat(d
);
812 dev_warn(d
->slave
.dev
,
813 "chan %d is running fail to suspend\n", stat
);
816 k3_dma_enable_dma(d
, false);
817 clk_disable_unprepare(d
->clk
);
821 static int k3_dma_resume_dev(struct device
*dev
)
823 struct k3_dma_dev
*d
= dev_get_drvdata(dev
);
826 ret
= clk_prepare_enable(d
->clk
);
828 dev_err(d
->slave
.dev
, "clk_prepare_enable failed: %d\n", ret
);
831 k3_dma_enable_dma(d
, true);
836 static SIMPLE_DEV_PM_OPS(k3_dma_pmops
, k3_dma_suspend_dev
, k3_dma_resume_dev
);
838 static struct platform_driver k3_pdma_driver
= {
842 .of_match_table
= k3_pdma_dt_ids
,
844 .probe
= k3_dma_probe
,
845 .remove
= k3_dma_remove
,
848 module_platform_driver(k3_pdma_driver
);
850 MODULE_DESCRIPTION("Hisilicon k3 DMA Driver");
851 MODULE_ALIAS("platform:k3dma");
852 MODULE_LICENSE("GPL v2");