2 * OMAP DMAengine support
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 #include <linux/delay.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/err.h>
12 #include <linux/init.h>
13 #include <linux/interrupt.h>
14 #include <linux/list.h>
15 #include <linux/module.h>
16 #include <linux/omap-dma.h>
17 #include <linux/platform_device.h>
18 #include <linux/slab.h>
19 #include <linux/spinlock.h>
20 #include <linux/of_dma.h>
21 #include <linux/of_device.h>
25 #define OMAP_SDMA_REQUESTS 127
26 #define OMAP_SDMA_CHANNELS 32
29 struct dma_device ddev
;
32 const struct omap_dma_reg
*reg_map
;
33 struct omap_system_dma_plat_info
*plat
;
35 unsigned dma_requests
;
37 uint32_t irq_enable_mask
;
38 struct omap_chan
*lch_map
[OMAP_SDMA_CHANNELS
];
42 struct virt_dma_chan vc
;
43 void __iomem
*channel_base
;
44 const struct omap_dma_reg
*reg_map
;
47 struct dma_slave_config cfg
;
53 struct omap_desc
*desc
;
59 uint32_t en
; /* number of elements (24-bit) */
60 uint32_t fn
; /* number of frames (16-bit) */
64 struct virt_dma_desc vd
;
65 enum dma_transfer_direction dir
;
68 int16_t fi
; /* for OMAP_DMA_SYNC_PACKET */
69 uint8_t es
; /* CSDP_DATA_TYPE_xxx */
70 uint32_t ccr
; /* CCR value */
71 uint16_t clnk_ctrl
; /* CLNK_CTRL value */
72 uint16_t cicr
; /* CICR value */
73 uint32_t csdp
; /* CSDP value */
81 CCR_READ_PRIORITY
= BIT(6),
83 CCR_AUTO_INIT
= BIT(8), /* OMAP1 only */
84 CCR_REPEAT
= BIT(9), /* OMAP1 only */
85 CCR_OMAP31_DISABLE
= BIT(10), /* OMAP1 only */
86 CCR_SUSPEND_SENSITIVE
= BIT(8), /* OMAP2+ only */
87 CCR_RD_ACTIVE
= BIT(9), /* OMAP2+ only */
88 CCR_WR_ACTIVE
= BIT(10), /* OMAP2+ only */
89 CCR_SRC_AMODE_CONSTANT
= 0 << 12,
90 CCR_SRC_AMODE_POSTINC
= 1 << 12,
91 CCR_SRC_AMODE_SGLIDX
= 2 << 12,
92 CCR_SRC_AMODE_DBLIDX
= 3 << 12,
93 CCR_DST_AMODE_CONSTANT
= 0 << 14,
94 CCR_DST_AMODE_POSTINC
= 1 << 14,
95 CCR_DST_AMODE_SGLIDX
= 2 << 14,
96 CCR_DST_AMODE_DBLIDX
= 3 << 14,
97 CCR_CONSTANT_FILL
= BIT(16),
98 CCR_TRANSPARENT_COPY
= BIT(17),
100 CCR_SUPERVISOR
= BIT(22),
101 CCR_PREFETCH
= BIT(23),
102 CCR_TRIGGER_SRC
= BIT(24),
103 CCR_BUFFERING_DISABLE
= BIT(25),
104 CCR_WRITE_PRIORITY
= BIT(26),
105 CCR_SYNC_ELEMENT
= 0,
106 CCR_SYNC_FRAME
= CCR_FS
,
107 CCR_SYNC_BLOCK
= CCR_BS
,
108 CCR_SYNC_PACKET
= CCR_BS
| CCR_FS
,
110 CSDP_DATA_TYPE_8
= 0,
111 CSDP_DATA_TYPE_16
= 1,
112 CSDP_DATA_TYPE_32
= 2,
113 CSDP_SRC_PORT_EMIFF
= 0 << 2, /* OMAP1 only */
114 CSDP_SRC_PORT_EMIFS
= 1 << 2, /* OMAP1 only */
115 CSDP_SRC_PORT_OCP_T1
= 2 << 2, /* OMAP1 only */
116 CSDP_SRC_PORT_TIPB
= 3 << 2, /* OMAP1 only */
117 CSDP_SRC_PORT_OCP_T2
= 4 << 2, /* OMAP1 only */
118 CSDP_SRC_PORT_MPUI
= 5 << 2, /* OMAP1 only */
119 CSDP_SRC_PACKED
= BIT(6),
120 CSDP_SRC_BURST_1
= 0 << 7,
121 CSDP_SRC_BURST_16
= 1 << 7,
122 CSDP_SRC_BURST_32
= 2 << 7,
123 CSDP_SRC_BURST_64
= 3 << 7,
124 CSDP_DST_PORT_EMIFF
= 0 << 9, /* OMAP1 only */
125 CSDP_DST_PORT_EMIFS
= 1 << 9, /* OMAP1 only */
126 CSDP_DST_PORT_OCP_T1
= 2 << 9, /* OMAP1 only */
127 CSDP_DST_PORT_TIPB
= 3 << 9, /* OMAP1 only */
128 CSDP_DST_PORT_OCP_T2
= 4 << 9, /* OMAP1 only */
129 CSDP_DST_PORT_MPUI
= 5 << 9, /* OMAP1 only */
130 CSDP_DST_PACKED
= BIT(13),
131 CSDP_DST_BURST_1
= 0 << 14,
132 CSDP_DST_BURST_16
= 1 << 14,
133 CSDP_DST_BURST_32
= 2 << 14,
134 CSDP_DST_BURST_64
= 3 << 14,
136 CICR_TOUT_IE
= BIT(0), /* OMAP1 only */
137 CICR_DROP_IE
= BIT(1),
138 CICR_HALF_IE
= BIT(2),
139 CICR_FRAME_IE
= BIT(3),
140 CICR_LAST_IE
= BIT(4),
141 CICR_BLOCK_IE
= BIT(5),
142 CICR_PKT_IE
= BIT(7), /* OMAP2+ only */
143 CICR_TRANS_ERR_IE
= BIT(8), /* OMAP2+ only */
144 CICR_SUPERVISOR_ERR_IE
= BIT(10), /* OMAP2+ only */
145 CICR_MISALIGNED_ERR_IE
= BIT(11), /* OMAP2+ only */
146 CICR_DRAIN_IE
= BIT(12), /* OMAP2+ only */
147 CICR_SUPER_BLOCK_IE
= BIT(14), /* OMAP2+ only */
149 CLNK_CTRL_ENABLE_LNK
= BIT(15),
152 static const unsigned es_bytes
[] = {
153 [CSDP_DATA_TYPE_8
] = 1,
154 [CSDP_DATA_TYPE_16
] = 2,
155 [CSDP_DATA_TYPE_32
] = 4,
158 static struct of_dma_filter_info omap_dma_info
= {
159 .filter_fn
= omap_dma_filter_fn
,
162 static inline struct omap_dmadev
*to_omap_dma_dev(struct dma_device
*d
)
164 return container_of(d
, struct omap_dmadev
, ddev
);
167 static inline struct omap_chan
*to_omap_dma_chan(struct dma_chan
*c
)
169 return container_of(c
, struct omap_chan
, vc
.chan
);
172 static inline struct omap_desc
*to_omap_dma_desc(struct dma_async_tx_descriptor
*t
)
174 return container_of(t
, struct omap_desc
, vd
.tx
);
177 static void omap_dma_desc_free(struct virt_dma_desc
*vd
)
179 kfree(container_of(vd
, struct omap_desc
, vd
));
182 static void omap_dma_write(uint32_t val
, unsigned type
, void __iomem
*addr
)
185 case OMAP_DMA_REG_16BIT
:
186 writew_relaxed(val
, addr
);
188 case OMAP_DMA_REG_2X16BIT
:
189 writew_relaxed(val
, addr
);
190 writew_relaxed(val
>> 16, addr
+ 2);
192 case OMAP_DMA_REG_32BIT
:
193 writel_relaxed(val
, addr
);
200 static unsigned omap_dma_read(unsigned type
, void __iomem
*addr
)
205 case OMAP_DMA_REG_16BIT
:
206 val
= readw_relaxed(addr
);
208 case OMAP_DMA_REG_2X16BIT
:
209 val
= readw_relaxed(addr
);
210 val
|= readw_relaxed(addr
+ 2) << 16;
212 case OMAP_DMA_REG_32BIT
:
213 val
= readl_relaxed(addr
);
223 static void omap_dma_glbl_write(struct omap_dmadev
*od
, unsigned reg
, unsigned val
)
225 const struct omap_dma_reg
*r
= od
->reg_map
+ reg
;
229 omap_dma_write(val
, r
->type
, od
->base
+ r
->offset
);
232 static unsigned omap_dma_glbl_read(struct omap_dmadev
*od
, unsigned reg
)
234 const struct omap_dma_reg
*r
= od
->reg_map
+ reg
;
238 return omap_dma_read(r
->type
, od
->base
+ r
->offset
);
241 static void omap_dma_chan_write(struct omap_chan
*c
, unsigned reg
, unsigned val
)
243 const struct omap_dma_reg
*r
= c
->reg_map
+ reg
;
245 omap_dma_write(val
, r
->type
, c
->channel_base
+ r
->offset
);
248 static unsigned omap_dma_chan_read(struct omap_chan
*c
, unsigned reg
)
250 const struct omap_dma_reg
*r
= c
->reg_map
+ reg
;
252 return omap_dma_read(r
->type
, c
->channel_base
+ r
->offset
);
255 static void omap_dma_clear_csr(struct omap_chan
*c
)
258 omap_dma_chan_read(c
, CSR
);
260 omap_dma_chan_write(c
, CSR
, ~0);
263 static unsigned omap_dma_get_csr(struct omap_chan
*c
)
265 unsigned val
= omap_dma_chan_read(c
, CSR
);
268 omap_dma_chan_write(c
, CSR
, val
);
273 static void omap_dma_assign(struct omap_dmadev
*od
, struct omap_chan
*c
,
276 c
->channel_base
= od
->base
+ od
->plat
->channel_stride
* lch
;
278 od
->lch_map
[lch
] = c
;
281 static void omap_dma_start(struct omap_chan
*c
, struct omap_desc
*d
)
283 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
285 if (__dma_omap15xx(od
->plat
->dma_attr
))
286 omap_dma_chan_write(c
, CPC
, 0);
288 omap_dma_chan_write(c
, CDAC
, 0);
290 omap_dma_clear_csr(c
);
292 /* Enable interrupts */
293 omap_dma_chan_write(c
, CICR
, d
->cicr
);
296 omap_dma_chan_write(c
, CCR
, d
->ccr
| CCR_ENABLE
);
299 static void omap_dma_stop(struct omap_chan
*c
)
301 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
305 omap_dma_chan_write(c
, CICR
, 0);
307 omap_dma_clear_csr(c
);
309 val
= omap_dma_chan_read(c
, CCR
);
310 if (od
->plat
->errata
& DMA_ERRATA_i541
&& val
& CCR_TRIGGER_SRC
) {
314 sysconfig
= omap_dma_glbl_read(od
, OCP_SYSCONFIG
);
315 val
= sysconfig
& ~DMA_SYSCONFIG_MIDLEMODE_MASK
;
316 val
|= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE
);
317 omap_dma_glbl_write(od
, OCP_SYSCONFIG
, val
);
319 val
= omap_dma_chan_read(c
, CCR
);
321 omap_dma_chan_write(c
, CCR
, val
);
323 /* Wait for sDMA FIFO to drain */
325 val
= omap_dma_chan_read(c
, CCR
);
326 if (!(val
& (CCR_RD_ACTIVE
| CCR_WR_ACTIVE
)))
335 if (val
& (CCR_RD_ACTIVE
| CCR_WR_ACTIVE
))
336 dev_err(c
->vc
.chan
.device
->dev
,
337 "DMA drain did not complete on lch %d\n",
340 omap_dma_glbl_write(od
, OCP_SYSCONFIG
, sysconfig
);
343 omap_dma_chan_write(c
, CCR
, val
);
348 if (!__dma_omap15xx(od
->plat
->dma_attr
) && c
->cyclic
) {
349 val
= omap_dma_chan_read(c
, CLNK_CTRL
);
352 val
|= 1 << 14; /* set the STOP_LNK bit */
354 val
&= ~CLNK_CTRL_ENABLE_LNK
;
356 omap_dma_chan_write(c
, CLNK_CTRL
, val
);
360 static void omap_dma_start_sg(struct omap_chan
*c
, struct omap_desc
*d
,
363 struct omap_sg
*sg
= d
->sg
+ idx
;
364 unsigned cxsa
, cxei
, cxfi
;
366 if (d
->dir
== DMA_DEV_TO_MEM
|| d
->dir
== DMA_MEM_TO_MEM
) {
376 omap_dma_chan_write(c
, cxsa
, sg
->addr
);
377 omap_dma_chan_write(c
, cxei
, 0);
378 omap_dma_chan_write(c
, cxfi
, 0);
379 omap_dma_chan_write(c
, CEN
, sg
->en
);
380 omap_dma_chan_write(c
, CFN
, sg
->fn
);
382 omap_dma_start(c
, d
);
385 static void omap_dma_start_desc(struct omap_chan
*c
)
387 struct virt_dma_desc
*vd
= vchan_next_desc(&c
->vc
);
389 unsigned cxsa
, cxei
, cxfi
;
398 c
->desc
= d
= to_omap_dma_desc(&vd
->tx
);
402 * This provides the necessary barrier to ensure data held in
403 * DMA coherent memory is visible to the DMA engine prior to
404 * the transfer starting.
408 omap_dma_chan_write(c
, CCR
, d
->ccr
);
410 omap_dma_chan_write(c
, CCR2
, d
->ccr
>> 16);
412 if (d
->dir
== DMA_DEV_TO_MEM
|| d
->dir
== DMA_MEM_TO_MEM
) {
422 omap_dma_chan_write(c
, cxsa
, d
->dev_addr
);
423 omap_dma_chan_write(c
, cxei
, 0);
424 omap_dma_chan_write(c
, cxfi
, d
->fi
);
425 omap_dma_chan_write(c
, CSDP
, d
->csdp
);
426 omap_dma_chan_write(c
, CLNK_CTRL
, d
->clnk_ctrl
);
428 omap_dma_start_sg(c
, d
, 0);
431 static void omap_dma_callback(int ch
, u16 status
, void *data
)
433 struct omap_chan
*c
= data
;
437 spin_lock_irqsave(&c
->vc
.lock
, flags
);
441 if (++c
->sgidx
< d
->sglen
) {
442 omap_dma_start_sg(c
, d
, c
->sgidx
);
444 omap_dma_start_desc(c
);
445 vchan_cookie_complete(&d
->vd
);
448 vchan_cyclic_callback(&d
->vd
);
451 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
454 static irqreturn_t
omap_dma_irq(int irq
, void *devid
)
456 struct omap_dmadev
*od
= devid
;
457 unsigned status
, channel
;
459 spin_lock(&od
->irq_lock
);
461 status
= omap_dma_glbl_read(od
, IRQSTATUS_L1
);
462 status
&= od
->irq_enable_mask
;
464 spin_unlock(&od
->irq_lock
);
468 while ((channel
= ffs(status
)) != 0) {
476 c
= od
->lch_map
[channel
];
478 /* This should never happen */
479 dev_err(od
->ddev
.dev
, "invalid channel %u\n", channel
);
483 csr
= omap_dma_get_csr(c
);
484 omap_dma_glbl_write(od
, IRQSTATUS_L1
, mask
);
486 omap_dma_callback(channel
, csr
, c
);
489 spin_unlock(&od
->irq_lock
);
494 static int omap_dma_alloc_chan_resources(struct dma_chan
*chan
)
496 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
497 struct omap_chan
*c
= to_omap_dma_chan(chan
);
501 ret
= omap_request_dma(c
->dma_sig
, "DMA engine",
502 omap_dma_callback
, c
, &c
->dma_ch
);
504 ret
= omap_request_dma(c
->dma_sig
, "DMA engine", NULL
, NULL
,
508 dev_dbg(od
->ddev
.dev
, "allocating channel %u for %u\n",
509 c
->dma_ch
, c
->dma_sig
);
512 omap_dma_assign(od
, c
, c
->dma_ch
);
517 spin_lock_irq(&od
->irq_lock
);
518 val
= BIT(c
->dma_ch
);
519 omap_dma_glbl_write(od
, IRQSTATUS_L1
, val
);
520 od
->irq_enable_mask
|= val
;
521 omap_dma_glbl_write(od
, IRQENABLE_L1
, od
->irq_enable_mask
);
523 val
= omap_dma_glbl_read(od
, IRQENABLE_L0
);
524 val
&= ~BIT(c
->dma_ch
);
525 omap_dma_glbl_write(od
, IRQENABLE_L0
, val
);
526 spin_unlock_irq(&od
->irq_lock
);
531 if (__dma_omap16xx(od
->plat
->dma_attr
)) {
532 c
->ccr
= CCR_OMAP31_DISABLE
;
533 /* Duplicate what plat-omap/dma.c does */
534 c
->ccr
|= c
->dma_ch
+ 1;
536 c
->ccr
= c
->dma_sig
& 0x1f;
539 c
->ccr
= c
->dma_sig
& 0x1f;
540 c
->ccr
|= (c
->dma_sig
& ~0x1f) << 14;
542 if (od
->plat
->errata
& DMA_ERRATA_IFRAME_BUFFERING
)
543 c
->ccr
|= CCR_BUFFERING_DISABLE
;
548 static void omap_dma_free_chan_resources(struct dma_chan
*chan
)
550 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
551 struct omap_chan
*c
= to_omap_dma_chan(chan
);
554 spin_lock_irq(&od
->irq_lock
);
555 od
->irq_enable_mask
&= ~BIT(c
->dma_ch
);
556 omap_dma_glbl_write(od
, IRQENABLE_L1
, od
->irq_enable_mask
);
557 spin_unlock_irq(&od
->irq_lock
);
560 c
->channel_base
= NULL
;
561 od
->lch_map
[c
->dma_ch
] = NULL
;
562 vchan_free_chan_resources(&c
->vc
);
563 omap_free_dma(c
->dma_ch
);
565 dev_dbg(od
->ddev
.dev
, "freeing channel for %u\n", c
->dma_sig
);
569 static size_t omap_dma_sg_size(struct omap_sg
*sg
)
571 return sg
->en
* sg
->fn
;
574 static size_t omap_dma_desc_size(struct omap_desc
*d
)
579 for (size
= i
= 0; i
< d
->sglen
; i
++)
580 size
+= omap_dma_sg_size(&d
->sg
[i
]);
582 return size
* es_bytes
[d
->es
];
585 static size_t omap_dma_desc_size_pos(struct omap_desc
*d
, dma_addr_t addr
)
588 size_t size
, es_size
= es_bytes
[d
->es
];
590 for (size
= i
= 0; i
< d
->sglen
; i
++) {
591 size_t this_size
= omap_dma_sg_size(&d
->sg
[i
]) * es_size
;
595 else if (addr
>= d
->sg
[i
].addr
&&
596 addr
< d
->sg
[i
].addr
+ this_size
)
597 size
+= d
->sg
[i
].addr
+ this_size
- addr
;
603 * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
604 * read before the DMA controller finished disabling the channel.
606 static uint32_t omap_dma_chan_read_3_3(struct omap_chan
*c
, unsigned reg
)
608 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
611 val
= omap_dma_chan_read(c
, reg
);
612 if (val
== 0 && od
->plat
->errata
& DMA_ERRATA_3_3
)
613 val
= omap_dma_chan_read(c
, reg
);
618 static dma_addr_t
omap_dma_get_src_pos(struct omap_chan
*c
)
620 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
621 dma_addr_t addr
, cdac
;
623 if (__dma_omap15xx(od
->plat
->dma_attr
)) {
624 addr
= omap_dma_chan_read(c
, CPC
);
626 addr
= omap_dma_chan_read_3_3(c
, CSAC
);
627 cdac
= omap_dma_chan_read_3_3(c
, CDAC
);
630 * CDAC == 0 indicates that the DMA transfer on the channel has
631 * not been started (no data has been transferred so far).
632 * Return the programmed source start address in this case.
635 addr
= omap_dma_chan_read(c
, CSSA
);
639 addr
|= omap_dma_chan_read(c
, CSSA
) & 0xffff0000;
644 static dma_addr_t
omap_dma_get_dst_pos(struct omap_chan
*c
)
646 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
649 if (__dma_omap15xx(od
->plat
->dma_attr
)) {
650 addr
= omap_dma_chan_read(c
, CPC
);
652 addr
= omap_dma_chan_read_3_3(c
, CDAC
);
655 * CDAC == 0 indicates that the DMA transfer on the channel
656 * has not been started (no data has been transferred so
657 * far). Return the programmed destination start address in
661 addr
= omap_dma_chan_read(c
, CDSA
);
665 addr
|= omap_dma_chan_read(c
, CDSA
) & 0xffff0000;
670 static enum dma_status
omap_dma_tx_status(struct dma_chan
*chan
,
671 dma_cookie_t cookie
, struct dma_tx_state
*txstate
)
673 struct omap_chan
*c
= to_omap_dma_chan(chan
);
674 struct virt_dma_desc
*vd
;
679 ccr
= omap_dma_chan_read(c
, CCR
);
680 /* The channel is no longer active, handle the completion right away */
681 if (!(ccr
& CCR_ENABLE
))
682 omap_dma_callback(c
->dma_ch
, 0, c
);
684 ret
= dma_cookie_status(chan
, cookie
, txstate
);
685 if (ret
== DMA_COMPLETE
|| !txstate
)
688 spin_lock_irqsave(&c
->vc
.lock
, flags
);
689 vd
= vchan_find_desc(&c
->vc
, cookie
);
691 txstate
->residue
= omap_dma_desc_size(to_omap_dma_desc(&vd
->tx
));
692 } else if (c
->desc
&& c
->desc
->vd
.tx
.cookie
== cookie
) {
693 struct omap_desc
*d
= c
->desc
;
696 if (d
->dir
== DMA_MEM_TO_DEV
)
697 pos
= omap_dma_get_src_pos(c
);
698 else if (d
->dir
== DMA_DEV_TO_MEM
|| d
->dir
== DMA_MEM_TO_MEM
)
699 pos
= omap_dma_get_dst_pos(c
);
703 txstate
->residue
= omap_dma_desc_size_pos(d
, pos
);
705 txstate
->residue
= 0;
707 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
712 static void omap_dma_issue_pending(struct dma_chan
*chan
)
714 struct omap_chan
*c
= to_omap_dma_chan(chan
);
717 spin_lock_irqsave(&c
->vc
.lock
, flags
);
718 if (vchan_issue_pending(&c
->vc
) && !c
->desc
)
719 omap_dma_start_desc(c
);
720 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
723 static struct dma_async_tx_descriptor
*omap_dma_prep_slave_sg(
724 struct dma_chan
*chan
, struct scatterlist
*sgl
, unsigned sglen
,
725 enum dma_transfer_direction dir
, unsigned long tx_flags
, void *context
)
727 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
728 struct omap_chan
*c
= to_omap_dma_chan(chan
);
729 enum dma_slave_buswidth dev_width
;
730 struct scatterlist
*sgent
;
733 unsigned i
, es
, en
, frame_bytes
;
736 if (dir
== DMA_DEV_TO_MEM
) {
737 dev_addr
= c
->cfg
.src_addr
;
738 dev_width
= c
->cfg
.src_addr_width
;
739 burst
= c
->cfg
.src_maxburst
;
740 } else if (dir
== DMA_MEM_TO_DEV
) {
741 dev_addr
= c
->cfg
.dst_addr
;
742 dev_width
= c
->cfg
.dst_addr_width
;
743 burst
= c
->cfg
.dst_maxburst
;
745 dev_err(chan
->device
->dev
, "%s: bad direction?\n", __func__
);
749 /* Bus width translates to the element size (ES) */
751 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
752 es
= CSDP_DATA_TYPE_8
;
754 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
755 es
= CSDP_DATA_TYPE_16
;
757 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
758 es
= CSDP_DATA_TYPE_32
;
760 default: /* not reached */
764 /* Now allocate and setup the descriptor. */
765 d
= kzalloc(sizeof(*d
) + sglen
* sizeof(d
->sg
[0]), GFP_ATOMIC
);
770 d
->dev_addr
= dev_addr
;
773 d
->ccr
= c
->ccr
| CCR_SYNC_FRAME
;
774 if (dir
== DMA_DEV_TO_MEM
)
775 d
->ccr
|= CCR_DST_AMODE_POSTINC
| CCR_SRC_AMODE_CONSTANT
;
777 d
->ccr
|= CCR_DST_AMODE_CONSTANT
| CCR_SRC_AMODE_POSTINC
;
779 d
->cicr
= CICR_DROP_IE
| CICR_BLOCK_IE
;
783 d
->cicr
|= CICR_TOUT_IE
;
785 if (dir
== DMA_DEV_TO_MEM
)
786 d
->csdp
|= CSDP_DST_PORT_EMIFF
| CSDP_SRC_PORT_TIPB
;
788 d
->csdp
|= CSDP_DST_PORT_TIPB
| CSDP_SRC_PORT_EMIFF
;
790 if (dir
== DMA_DEV_TO_MEM
)
791 d
->ccr
|= CCR_TRIGGER_SRC
;
793 d
->cicr
|= CICR_MISALIGNED_ERR_IE
| CICR_TRANS_ERR_IE
;
795 if (od
->plat
->errata
& DMA_ERRATA_PARALLEL_CHANNELS
)
796 d
->clnk_ctrl
= c
->dma_ch
;
799 * Build our scatterlist entries: each contains the address,
800 * the number of elements (EN) in each frame, and the number of
801 * frames (FN). Number of bytes for this entry = ES * EN * FN.
803 * Burst size translates to number of elements with frame sync.
804 * Note: DMA engine defines burst to be the number of dev-width
808 frame_bytes
= es_bytes
[es
] * en
;
809 for_each_sg(sgl
, sgent
, sglen
, i
) {
810 d
->sg
[i
].addr
= sg_dma_address(sgent
);
812 d
->sg
[i
].fn
= sg_dma_len(sgent
) / frame_bytes
;
817 return vchan_tx_prep(&c
->vc
, &d
->vd
, tx_flags
);
820 static struct dma_async_tx_descriptor
*omap_dma_prep_dma_cyclic(
821 struct dma_chan
*chan
, dma_addr_t buf_addr
, size_t buf_len
,
822 size_t period_len
, enum dma_transfer_direction dir
, unsigned long flags
)
824 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
825 struct omap_chan
*c
= to_omap_dma_chan(chan
);
826 enum dma_slave_buswidth dev_width
;
832 if (dir
== DMA_DEV_TO_MEM
) {
833 dev_addr
= c
->cfg
.src_addr
;
834 dev_width
= c
->cfg
.src_addr_width
;
835 burst
= c
->cfg
.src_maxburst
;
836 } else if (dir
== DMA_MEM_TO_DEV
) {
837 dev_addr
= c
->cfg
.dst_addr
;
838 dev_width
= c
->cfg
.dst_addr_width
;
839 burst
= c
->cfg
.dst_maxburst
;
841 dev_err(chan
->device
->dev
, "%s: bad direction?\n", __func__
);
845 /* Bus width translates to the element size (ES) */
847 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
848 es
= CSDP_DATA_TYPE_8
;
850 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
851 es
= CSDP_DATA_TYPE_16
;
853 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
854 es
= CSDP_DATA_TYPE_32
;
856 default: /* not reached */
860 /* Now allocate and setup the descriptor. */
861 d
= kzalloc(sizeof(*d
) + sizeof(d
->sg
[0]), GFP_ATOMIC
);
866 d
->dev_addr
= dev_addr
;
869 d
->sg
[0].addr
= buf_addr
;
870 d
->sg
[0].en
= period_len
/ es_bytes
[es
];
871 d
->sg
[0].fn
= buf_len
/ period_len
;
875 if (dir
== DMA_DEV_TO_MEM
)
876 d
->ccr
|= CCR_DST_AMODE_POSTINC
| CCR_SRC_AMODE_CONSTANT
;
878 d
->ccr
|= CCR_DST_AMODE_CONSTANT
| CCR_SRC_AMODE_POSTINC
;
880 d
->cicr
= CICR_DROP_IE
;
881 if (flags
& DMA_PREP_INTERRUPT
)
882 d
->cicr
|= CICR_FRAME_IE
;
887 d
->cicr
|= CICR_TOUT_IE
;
889 if (dir
== DMA_DEV_TO_MEM
)
890 d
->csdp
|= CSDP_DST_PORT_EMIFF
| CSDP_SRC_PORT_MPUI
;
892 d
->csdp
|= CSDP_DST_PORT_MPUI
| CSDP_SRC_PORT_EMIFF
;
895 d
->ccr
|= CCR_SYNC_PACKET
;
897 d
->ccr
|= CCR_SYNC_ELEMENT
;
899 if (dir
== DMA_DEV_TO_MEM
) {
900 d
->ccr
|= CCR_TRIGGER_SRC
;
901 d
->csdp
|= CSDP_DST_PACKED
;
903 d
->csdp
|= CSDP_SRC_PACKED
;
906 d
->cicr
|= CICR_MISALIGNED_ERR_IE
| CICR_TRANS_ERR_IE
;
908 d
->csdp
|= CSDP_DST_BURST_64
| CSDP_SRC_BURST_64
;
911 if (__dma_omap15xx(od
->plat
->dma_attr
))
912 d
->ccr
|= CCR_AUTO_INIT
| CCR_REPEAT
;
914 d
->clnk_ctrl
= c
->dma_ch
| CLNK_CTRL_ENABLE_LNK
;
918 return vchan_tx_prep(&c
->vc
, &d
->vd
, flags
);
921 static struct dma_async_tx_descriptor
*omap_dma_prep_dma_memcpy(
922 struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
923 size_t len
, unsigned long tx_flags
)
925 struct omap_chan
*c
= to_omap_dma_chan(chan
);
929 d
= kzalloc(sizeof(*d
) + sizeof(d
->sg
[0]), GFP_ATOMIC
);
933 data_type
= __ffs((src
| dest
| len
));
934 if (data_type
> CSDP_DATA_TYPE_32
)
935 data_type
= CSDP_DATA_TYPE_32
;
937 d
->dir
= DMA_MEM_TO_MEM
;
941 d
->sg
[0].en
= len
/ BIT(data_type
);
943 d
->sg
[0].addr
= dest
;
946 d
->ccr
|= CCR_DST_AMODE_POSTINC
| CCR_SRC_AMODE_POSTINC
;
948 d
->cicr
= CICR_DROP_IE
;
949 if (tx_flags
& DMA_PREP_INTERRUPT
)
950 d
->cicr
|= CICR_FRAME_IE
;
955 d
->cicr
|= CICR_TOUT_IE
;
956 d
->csdp
|= CSDP_DST_PORT_EMIFF
| CSDP_SRC_PORT_EMIFF
;
958 d
->csdp
|= CSDP_DST_PACKED
| CSDP_SRC_PACKED
;
959 d
->cicr
|= CICR_MISALIGNED_ERR_IE
| CICR_TRANS_ERR_IE
;
960 d
->csdp
|= CSDP_DST_BURST_64
| CSDP_SRC_BURST_64
;
963 return vchan_tx_prep(&c
->vc
, &d
->vd
, tx_flags
);
966 static int omap_dma_slave_config(struct dma_chan
*chan
, struct dma_slave_config
*cfg
)
968 struct omap_chan
*c
= to_omap_dma_chan(chan
);
970 if (cfg
->src_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
||
971 cfg
->dst_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
)
974 memcpy(&c
->cfg
, cfg
, sizeof(c
->cfg
));
979 static int omap_dma_terminate_all(struct dma_chan
*chan
)
981 struct omap_chan
*c
= to_omap_dma_chan(chan
);
985 spin_lock_irqsave(&c
->vc
.lock
, flags
);
988 * Stop DMA activity: we assume the callback will not be called
989 * after omap_dma_stop() returns (even if it does, it will see
990 * c->desc is NULL and exit.)
993 omap_dma_desc_free(&c
->desc
->vd
);
995 /* Avoid stopping the dma twice */
1005 vchan_get_all_descriptors(&c
->vc
, &head
);
1006 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
1007 vchan_dma_desc_free_list(&c
->vc
, &head
);
1012 static int omap_dma_pause(struct dma_chan
*chan
)
1014 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1016 /* Pause/Resume only allowed with cyclic mode */
1028 static int omap_dma_resume(struct dma_chan
*chan
)
1030 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1032 /* Pause/Resume only allowed with cyclic mode */
1039 /* Restore channel link register */
1040 omap_dma_chan_write(c
, CLNK_CTRL
, c
->desc
->clnk_ctrl
);
1042 omap_dma_start(c
, c
->desc
);
1049 static int omap_dma_chan_init(struct omap_dmadev
*od
)
1051 struct omap_chan
*c
;
1053 c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
1057 c
->reg_map
= od
->reg_map
;
1058 c
->vc
.desc_free
= omap_dma_desc_free
;
1059 vchan_init(&c
->vc
, &od
->ddev
);
1064 static void omap_dma_free(struct omap_dmadev
*od
)
1066 while (!list_empty(&od
->ddev
.channels
)) {
1067 struct omap_chan
*c
= list_first_entry(&od
->ddev
.channels
,
1068 struct omap_chan
, vc
.chan
.device_node
);
1070 list_del(&c
->vc
.chan
.device_node
);
1071 tasklet_kill(&c
->vc
.task
);
1076 #define OMAP_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1077 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1078 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1080 static int omap_dma_probe(struct platform_device
*pdev
)
1082 struct omap_dmadev
*od
;
1083 struct resource
*res
;
1086 od
= devm_kzalloc(&pdev
->dev
, sizeof(*od
), GFP_KERNEL
);
1090 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1091 od
->base
= devm_ioremap_resource(&pdev
->dev
, res
);
1092 if (IS_ERR(od
->base
))
1093 return PTR_ERR(od
->base
);
1095 od
->plat
= omap_get_plat_info();
1097 return -EPROBE_DEFER
;
1099 od
->reg_map
= od
->plat
->reg_map
;
1101 dma_cap_set(DMA_SLAVE
, od
->ddev
.cap_mask
);
1102 dma_cap_set(DMA_CYCLIC
, od
->ddev
.cap_mask
);
1103 dma_cap_set(DMA_MEMCPY
, od
->ddev
.cap_mask
);
1104 od
->ddev
.device_alloc_chan_resources
= omap_dma_alloc_chan_resources
;
1105 od
->ddev
.device_free_chan_resources
= omap_dma_free_chan_resources
;
1106 od
->ddev
.device_tx_status
= omap_dma_tx_status
;
1107 od
->ddev
.device_issue_pending
= omap_dma_issue_pending
;
1108 od
->ddev
.device_prep_slave_sg
= omap_dma_prep_slave_sg
;
1109 od
->ddev
.device_prep_dma_cyclic
= omap_dma_prep_dma_cyclic
;
1110 od
->ddev
.device_prep_dma_memcpy
= omap_dma_prep_dma_memcpy
;
1111 od
->ddev
.device_config
= omap_dma_slave_config
;
1112 od
->ddev
.device_pause
= omap_dma_pause
;
1113 od
->ddev
.device_resume
= omap_dma_resume
;
1114 od
->ddev
.device_terminate_all
= omap_dma_terminate_all
;
1115 od
->ddev
.src_addr_widths
= OMAP_DMA_BUSWIDTHS
;
1116 od
->ddev
.dst_addr_widths
= OMAP_DMA_BUSWIDTHS
;
1117 od
->ddev
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
1118 od
->ddev
.residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
1119 od
->ddev
.dev
= &pdev
->dev
;
1120 INIT_LIST_HEAD(&od
->ddev
.channels
);
1121 spin_lock_init(&od
->lock
);
1122 spin_lock_init(&od
->irq_lock
);
1124 od
->dma_requests
= OMAP_SDMA_REQUESTS
;
1125 if (pdev
->dev
.of_node
&& of_property_read_u32(pdev
->dev
.of_node
,
1127 &od
->dma_requests
)) {
1128 dev_info(&pdev
->dev
,
1129 "Missing dma-requests property, using %u.\n",
1130 OMAP_SDMA_REQUESTS
);
1133 for (i
= 0; i
< OMAP_SDMA_CHANNELS
; i
++) {
1134 rc
= omap_dma_chan_init(od
);
1141 irq
= platform_get_irq(pdev
, 1);
1143 dev_info(&pdev
->dev
, "failed to get L1 IRQ: %d\n", irq
);
1146 /* Disable all interrupts */
1147 od
->irq_enable_mask
= 0;
1148 omap_dma_glbl_write(od
, IRQENABLE_L1
, 0);
1150 rc
= devm_request_irq(&pdev
->dev
, irq
, omap_dma_irq
,
1151 IRQF_SHARED
, "omap-dma-engine", od
);
1156 od
->ddev
.filter
.map
= od
->plat
->slave_map
;
1157 od
->ddev
.filter
.mapcnt
= od
->plat
->slavecnt
;
1158 od
->ddev
.filter
.fn
= omap_dma_filter_fn
;
1160 rc
= dma_async_device_register(&od
->ddev
);
1162 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1168 platform_set_drvdata(pdev
, od
);
1170 if (pdev
->dev
.of_node
) {
1171 omap_dma_info
.dma_cap
= od
->ddev
.cap_mask
;
1173 /* Device-tree DMA controller registration */
1174 rc
= of_dma_controller_register(pdev
->dev
.of_node
,
1175 of_dma_simple_xlate
, &omap_dma_info
);
1177 pr_warn("OMAP-DMA: failed to register DMA controller\n");
1178 dma_async_device_unregister(&od
->ddev
);
1183 dev_info(&pdev
->dev
, "OMAP DMA engine driver\n");
1188 static int omap_dma_remove(struct platform_device
*pdev
)
1190 struct omap_dmadev
*od
= platform_get_drvdata(pdev
);
1192 if (pdev
->dev
.of_node
)
1193 of_dma_controller_free(pdev
->dev
.of_node
);
1195 dma_async_device_unregister(&od
->ddev
);
1198 /* Disable all interrupts */
1199 omap_dma_glbl_write(od
, IRQENABLE_L0
, 0);
1207 static const struct of_device_id omap_dma_match
[] = {
1208 { .compatible
= "ti,omap2420-sdma", },
1209 { .compatible
= "ti,omap2430-sdma", },
1210 { .compatible
= "ti,omap3430-sdma", },
1211 { .compatible
= "ti,omap3630-sdma", },
1212 { .compatible
= "ti,omap4430-sdma", },
1215 MODULE_DEVICE_TABLE(of
, omap_dma_match
);
1217 static struct platform_driver omap_dma_driver
= {
1218 .probe
= omap_dma_probe
,
1219 .remove
= omap_dma_remove
,
1221 .name
= "omap-dma-engine",
1222 .of_match_table
= of_match_ptr(omap_dma_match
),
1226 bool omap_dma_filter_fn(struct dma_chan
*chan
, void *param
)
1228 if (chan
->device
->dev
->driver
== &omap_dma_driver
.driver
) {
1229 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
1230 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1231 unsigned req
= *(unsigned *)param
;
1233 if (req
<= od
->dma_requests
) {
1240 EXPORT_SYMBOL_GPL(omap_dma_filter_fn
);
1242 static int omap_dma_init(void)
1244 return platform_driver_register(&omap_dma_driver
);
1246 subsys_initcall(omap_dma_init
);
1248 static void __exit
omap_dma_exit(void)
1250 platform_driver_unregister(&omap_dma_driver
);
1252 module_exit(omap_dma_exit
);
1254 MODULE_AUTHOR("Russell King");
1255 MODULE_LICENSE("GPL");