2 * Copyright 2015 Robert Jarzmik <robert.jarzmik@free.fr>
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #include <linux/module.h>
11 #include <linux/init.h>
12 #include <linux/types.h>
13 #include <linux/interrupt.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/slab.h>
16 #include <linux/dmaengine.h>
17 #include <linux/platform_device.h>
18 #include <linux/device.h>
19 #include <linux/platform_data/mmp_dma.h>
20 #include <linux/dmapool.h>
21 #include <linux/of_device.h>
22 #include <linux/of_dma.h>
24 #include <linux/wait.h>
25 #include <linux/dma/pxa-dma.h>
27 #include "dmaengine.h"
30 #define DCSR(n) (0x0000 + ((n) << 2))
31 #define DALGN(n) 0x00a0
33 #define DDADR(n) (0x0200 + ((n) << 4))
34 #define DSADR(n) (0x0204 + ((n) << 4))
35 #define DTADR(n) (0x0208 + ((n) << 4))
36 #define DCMD(n) (0x020c + ((n) << 4))
38 #define PXA_DCSR_RUN BIT(31) /* Run Bit (read / write) */
39 #define PXA_DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */
40 #define PXA_DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (R/W) */
41 #define PXA_DCSR_REQPEND BIT(8) /* Request Pending (read-only) */
42 #define PXA_DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */
43 #define PXA_DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */
44 #define PXA_DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */
45 #define PXA_DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */
47 #define PXA_DCSR_EORIRQEN BIT(28) /* End of Receive IRQ Enable (R/W) */
48 #define PXA_DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */
49 #define PXA_DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */
50 #define PXA_DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */
51 #define PXA_DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */
52 #define PXA_DCSR_CMPST BIT(10) /* The Descriptor Compare Status */
53 #define PXA_DCSR_EORINTR BIT(9) /* The end of Receive */
55 #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */
56 #define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */
58 #define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */
59 #define DDADR_STOP BIT(0) /* Stop (read / write) */
61 #define PXA_DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */
62 #define PXA_DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */
63 #define PXA_DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */
64 #define PXA_DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */
65 #define PXA_DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */
66 #define PXA_DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */
67 #define PXA_DCMD_ENDIAN BIT(18) /* Device Endian-ness. */
68 #define PXA_DCMD_BURST8 (1 << 16) /* 8 byte burst */
69 #define PXA_DCMD_BURST16 (2 << 16) /* 16 byte burst */
70 #define PXA_DCMD_BURST32 (3 << 16) /* 32 byte burst */
71 #define PXA_DCMD_WIDTH1 (1 << 14) /* 1 byte width */
72 #define PXA_DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */
73 #define PXA_DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */
74 #define PXA_DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */
76 #define PDMA_ALIGNMENT 3
77 #define PDMA_MAX_DESC_BYTES (PXA_DCMD_LENGTH & ~((1 << PDMA_ALIGNMENT) - 1))
80 u32 ddadr
; /* Points to the next descriptor + flags */
81 u32 dsadr
; /* DSADR value for the current transfer */
82 u32 dtadr
; /* DTADR value for the current transfer */
83 u32 dcmd
; /* DCMD value for the current transfer */
87 struct virt_dma_desc vd
; /* Virtual descriptor */
88 int nb_desc
; /* Number of hw. descriptors */
89 size_t len
; /* Number of bytes xfered */
90 dma_addr_t first
; /* First descriptor's addr */
92 /* At least one descriptor has an src/dst address not multiple of 8 */
95 struct dma_pool
*desc_pool
; /* Channel's used allocator */
97 struct pxad_desc_hw
*hw_desc
[]; /* DMA coherent descriptors */
103 struct pxad_chan
*vchan
;
107 struct virt_dma_chan vc
; /* Virtual channel */
108 u32 drcmr
; /* Requestor of the channel */
109 enum pxad_chan_prio prio
; /* Required priority of phy */
111 * At least one desc_sw in submitted or issued transfers on this channel
112 * has one address such as: addr % 8 != 0. This implies the DALGN
113 * setting on the phy.
116 struct dma_slave_config cfg
; /* Runtime config */
118 /* protected by vc->lock */
119 struct pxad_phy
*phy
;
120 struct dma_pool
*desc_pool
; /* Descriptors pool */
121 dma_cookie_t bus_error
;
123 wait_queue_head_t wq_state
;
127 struct dma_device slave
;
131 struct pxad_phy
*phys
;
132 spinlock_t phy_lock
; /* Phy association */
133 #ifdef CONFIG_DEBUG_FS
134 struct dentry
*dbgfs_root
;
135 struct dentry
*dbgfs_state
;
136 struct dentry
**dbgfs_chan
;
140 #define tx_to_pxad_desc(tx) \
141 container_of(tx, struct pxad_desc_sw, async_tx)
142 #define to_pxad_chan(dchan) \
143 container_of(dchan, struct pxad_chan, vc.chan)
144 #define to_pxad_dev(dmadev) \
145 container_of(dmadev, struct pxad_device, slave)
146 #define to_pxad_sw_desc(_vd) \
147 container_of((_vd), struct pxad_desc_sw, vd)
149 #define _phy_readl_relaxed(phy, _reg) \
150 readl_relaxed((phy)->base + _reg((phy)->idx))
151 #define phy_readl_relaxed(phy, _reg) \
154 _v = readl_relaxed((phy)->base + _reg((phy)->idx)); \
155 dev_vdbg(&phy->vchan->vc.chan.dev->device, \
156 "%s(): readl(%s): 0x%08x\n", __func__, #_reg, \
160 #define phy_writel(phy, val, _reg) \
162 writel((val), (phy)->base + _reg((phy)->idx)); \
163 dev_vdbg(&phy->vchan->vc.chan.dev->device, \
164 "%s(): writel(0x%08x, %s)\n", \
165 __func__, (u32)(val), #_reg); \
167 #define phy_writel_relaxed(phy, val, _reg) \
169 writel_relaxed((val), (phy)->base + _reg((phy)->idx)); \
170 dev_vdbg(&phy->vchan->vc.chan.dev->device, \
171 "%s(): writel_relaxed(0x%08x, %s)\n", \
172 __func__, (u32)(val), #_reg); \
175 static unsigned int pxad_drcmr(unsigned int line
)
178 return 0x100 + line
* 4;
179 return 0x1000 + line
* 4;
185 #ifdef CONFIG_DEBUG_FS
186 #include <linux/debugfs.h>
187 #include <linux/uaccess.h>
188 #include <linux/seq_file.h>
190 static int dbg_show_requester_chan(struct seq_file
*s
, void *p
)
192 struct pxad_phy
*phy
= s
->private;
196 seq_printf(s
, "DMA channel %d requester :\n", phy
->idx
);
197 for (i
= 0; i
< 70; i
++) {
198 drcmr
= readl_relaxed(phy
->base
+ pxad_drcmr(i
));
199 if ((drcmr
& DRCMR_CHLNUM
) == phy
->idx
)
200 seq_printf(s
, "\tRequester %d (MAPVLD=%d)\n", i
,
201 !!(drcmr
& DRCMR_MAPVLD
));
206 static inline int dbg_burst_from_dcmd(u32 dcmd
)
208 int burst
= (dcmd
>> 16) & 0x3;
210 return burst
? 4 << burst
: 0;
213 static int is_phys_valid(unsigned long addr
)
215 return pfn_valid(__phys_to_pfn(addr
));
218 #define PXA_DCSR_STR(flag) (dcsr & PXA_DCSR_##flag ? #flag" " : "")
219 #define PXA_DCMD_STR(flag) (dcmd & PXA_DCMD_##flag ? #flag" " : "")
221 static int dbg_show_descriptors(struct seq_file
*s
, void *p
)
223 struct pxad_phy
*phy
= s
->private;
224 int i
, max_show
= 20, burst
, width
;
226 unsigned long phys_desc
, ddadr
;
227 struct pxad_desc_hw
*desc
;
229 phys_desc
= ddadr
= _phy_readl_relaxed(phy
, DDADR
);
231 seq_printf(s
, "DMA channel %d descriptors :\n", phy
->idx
);
232 seq_printf(s
, "[%03d] First descriptor unknown\n", 0);
233 for (i
= 1; i
< max_show
&& is_phys_valid(phys_desc
); i
++) {
234 desc
= phys_to_virt(phys_desc
);
236 burst
= dbg_burst_from_dcmd(dcmd
);
237 width
= (1 << ((dcmd
>> 14) & 0x3)) >> 1;
239 seq_printf(s
, "[%03d] Desc at %08lx(virt %p)\n",
241 seq_printf(s
, "\tDDADR = %08x\n", desc
->ddadr
);
242 seq_printf(s
, "\tDSADR = %08x\n", desc
->dsadr
);
243 seq_printf(s
, "\tDTADR = %08x\n", desc
->dtadr
);
244 seq_printf(s
, "\tDCMD = %08x (%s%s%s%s%s%s%sburst=%d width=%d len=%d)\n",
246 PXA_DCMD_STR(INCSRCADDR
), PXA_DCMD_STR(INCTRGADDR
),
247 PXA_DCMD_STR(FLOWSRC
), PXA_DCMD_STR(FLOWTRG
),
248 PXA_DCMD_STR(STARTIRQEN
), PXA_DCMD_STR(ENDIRQEN
),
249 PXA_DCMD_STR(ENDIAN
), burst
, width
,
250 dcmd
& PXA_DCMD_LENGTH
);
251 phys_desc
= desc
->ddadr
;
254 seq_printf(s
, "[%03d] Desc at %08lx ... max display reached\n",
257 seq_printf(s
, "[%03d] Desc at %08lx is %s\n",
258 i
, phys_desc
, phys_desc
== DDADR_STOP
?
259 "DDADR_STOP" : "invalid");
264 static int dbg_show_chan_state(struct seq_file
*s
, void *p
)
266 struct pxad_phy
*phy
= s
->private;
269 static const char * const str_prio
[] = {
270 "high", "normal", "low", "invalid"
273 dcsr
= _phy_readl_relaxed(phy
, DCSR
);
274 dcmd
= _phy_readl_relaxed(phy
, DCMD
);
275 burst
= dbg_burst_from_dcmd(dcmd
);
276 width
= (1 << ((dcmd
>> 14) & 0x3)) >> 1;
278 seq_printf(s
, "DMA channel %d\n", phy
->idx
);
279 seq_printf(s
, "\tPriority : %s\n",
280 str_prio
[(phy
->idx
& 0xf) / 4]);
281 seq_printf(s
, "\tUnaligned transfer bit: %s\n",
282 _phy_readl_relaxed(phy
, DALGN
) & BIT(phy
->idx
) ?
284 seq_printf(s
, "\tDCSR = %08x (%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n",
285 dcsr
, PXA_DCSR_STR(RUN
), PXA_DCSR_STR(NODESC
),
286 PXA_DCSR_STR(STOPIRQEN
), PXA_DCSR_STR(EORIRQEN
),
287 PXA_DCSR_STR(EORJMPEN
), PXA_DCSR_STR(EORSTOPEN
),
288 PXA_DCSR_STR(SETCMPST
), PXA_DCSR_STR(CLRCMPST
),
289 PXA_DCSR_STR(CMPST
), PXA_DCSR_STR(EORINTR
),
290 PXA_DCSR_STR(REQPEND
), PXA_DCSR_STR(STOPSTATE
),
291 PXA_DCSR_STR(ENDINTR
), PXA_DCSR_STR(STARTINTR
),
292 PXA_DCSR_STR(BUSERR
));
294 seq_printf(s
, "\tDCMD = %08x (%s%s%s%s%s%s%sburst=%d width=%d len=%d)\n",
296 PXA_DCMD_STR(INCSRCADDR
), PXA_DCMD_STR(INCTRGADDR
),
297 PXA_DCMD_STR(FLOWSRC
), PXA_DCMD_STR(FLOWTRG
),
298 PXA_DCMD_STR(STARTIRQEN
), PXA_DCMD_STR(ENDIRQEN
),
299 PXA_DCMD_STR(ENDIAN
), burst
, width
, dcmd
& PXA_DCMD_LENGTH
);
300 seq_printf(s
, "\tDSADR = %08x\n", _phy_readl_relaxed(phy
, DSADR
));
301 seq_printf(s
, "\tDTADR = %08x\n", _phy_readl_relaxed(phy
, DTADR
));
302 seq_printf(s
, "\tDDADR = %08x\n", _phy_readl_relaxed(phy
, DDADR
));
307 static int dbg_show_state(struct seq_file
*s
, void *p
)
309 struct pxad_device
*pdev
= s
->private;
311 /* basic device status */
312 seq_puts(s
, "DMA engine status\n");
313 seq_printf(s
, "\tChannel number: %d\n", pdev
->nr_chans
);
318 #define DBGFS_FUNC_DECL(name) \
319 static int dbg_open_##name(struct inode *inode, struct file *file) \
321 return single_open(file, dbg_show_##name, inode->i_private); \
323 static const struct file_operations dbg_fops_##name = { \
324 .open = dbg_open_##name, \
325 .llseek = seq_lseek, \
327 .release = single_release, \
330 DBGFS_FUNC_DECL(state
);
331 DBGFS_FUNC_DECL(chan_state
);
332 DBGFS_FUNC_DECL(descriptors
);
333 DBGFS_FUNC_DECL(requester_chan
);
335 static struct dentry
*pxad_dbg_alloc_chan(struct pxad_device
*pdev
,
336 int ch
, struct dentry
*chandir
)
339 struct dentry
*chan
, *chan_state
= NULL
, *chan_descr
= NULL
;
340 struct dentry
*chan_reqs
= NULL
;
343 scnprintf(chan_name
, sizeof(chan_name
), "%d", ch
);
344 chan
= debugfs_create_dir(chan_name
, chandir
);
345 dt
= (void *)&pdev
->phys
[ch
];
348 chan_state
= debugfs_create_file("state", 0400, chan
, dt
,
349 &dbg_fops_chan_state
);
351 chan_descr
= debugfs_create_file("descriptors", 0400, chan
, dt
,
352 &dbg_fops_descriptors
);
354 chan_reqs
= debugfs_create_file("requesters", 0400, chan
, dt
,
355 &dbg_fops_requester_chan
);
362 debugfs_remove_recursive(chan
);
366 static void pxad_init_debugfs(struct pxad_device
*pdev
)
369 struct dentry
*chandir
;
371 pdev
->dbgfs_root
= debugfs_create_dir(dev_name(pdev
->slave
.dev
), NULL
);
372 if (IS_ERR(pdev
->dbgfs_root
) || !pdev
->dbgfs_root
)
375 pdev
->dbgfs_state
= debugfs_create_file("state", 0400, pdev
->dbgfs_root
,
376 pdev
, &dbg_fops_state
);
377 if (!pdev
->dbgfs_state
)
381 kmalloc_array(pdev
->nr_chans
, sizeof(*pdev
->dbgfs_state
),
383 if (!pdev
->dbgfs_chan
)
386 chandir
= debugfs_create_dir("channels", pdev
->dbgfs_root
);
390 for (i
= 0; i
< pdev
->nr_chans
; i
++) {
391 pdev
->dbgfs_chan
[i
] = pxad_dbg_alloc_chan(pdev
, i
, chandir
);
392 if (!pdev
->dbgfs_chan
[i
])
399 kfree(pdev
->dbgfs_chan
);
402 debugfs_remove_recursive(pdev
->dbgfs_root
);
404 pr_err("pxad: debugfs is not available\n");
407 static void pxad_cleanup_debugfs(struct pxad_device
*pdev
)
409 debugfs_remove_recursive(pdev
->dbgfs_root
);
412 static inline void pxad_init_debugfs(struct pxad_device
*pdev
) {}
413 static inline void pxad_cleanup_debugfs(struct pxad_device
*pdev
) {}
417 * In the transition phase where legacy pxa handling is done at the same time as
418 * mmp_dma, the DMA physical channel split between the 2 DMA providers is done
419 * through legacy_reserved. Legacy code reserves DMA channels by settings
420 * corresponding bits in legacy_reserved.
422 static u32 legacy_reserved
;
423 static u32 legacy_unavailable
;
425 static struct pxad_phy
*lookup_phy(struct pxad_chan
*pchan
)
428 struct pxad_device
*pdev
= to_pxad_dev(pchan
->vc
.chan
.device
);
429 struct pxad_phy
*phy
, *found
= NULL
;
433 * dma channel priorities
434 * ch 0 - 3, 16 - 19 <--> (0)
435 * ch 4 - 7, 20 - 23 <--> (1)
436 * ch 8 - 11, 24 - 27 <--> (2)
437 * ch 12 - 15, 28 - 31 <--> (3)
440 spin_lock_irqsave(&pdev
->phy_lock
, flags
);
441 for (prio
= pchan
->prio
; prio
>= PXAD_PRIO_HIGHEST
; prio
--) {
442 for (i
= 0; i
< pdev
->nr_chans
; i
++) {
443 if (prio
!= (i
& 0xf) >> 2)
445 if ((i
< 32) && (legacy_reserved
& BIT(i
)))
447 phy
= &pdev
->phys
[i
];
452 legacy_unavailable
|= BIT(i
);
459 spin_unlock_irqrestore(&pdev
->phy_lock
, flags
);
460 dev_dbg(&pchan
->vc
.chan
.dev
->device
,
461 "%s(): phy=%p(%d)\n", __func__
, found
,
462 found
? found
->idx
: -1);
467 static void pxad_free_phy(struct pxad_chan
*chan
)
469 struct pxad_device
*pdev
= to_pxad_dev(chan
->vc
.chan
.device
);
474 dev_dbg(&chan
->vc
.chan
.dev
->device
,
475 "%s(): freeing\n", __func__
);
479 /* clear the channel mapping in DRCMR */
480 if (chan
->drcmr
<= pdev
->nr_requestors
) {
481 reg
= pxad_drcmr(chan
->drcmr
);
482 writel_relaxed(0, chan
->phy
->base
+ reg
);
485 spin_lock_irqsave(&pdev
->phy_lock
, flags
);
486 for (i
= 0; i
< 32; i
++)
487 if (chan
->phy
== &pdev
->phys
[i
])
488 legacy_unavailable
&= ~BIT(i
);
489 chan
->phy
->vchan
= NULL
;
491 spin_unlock_irqrestore(&pdev
->phy_lock
, flags
);
494 static bool is_chan_running(struct pxad_chan
*chan
)
497 struct pxad_phy
*phy
= chan
->phy
;
501 dcsr
= phy_readl_relaxed(phy
, DCSR
);
502 return dcsr
& PXA_DCSR_RUN
;
505 static bool is_running_chan_misaligned(struct pxad_chan
*chan
)
510 dalgn
= phy_readl_relaxed(chan
->phy
, DALGN
);
511 return dalgn
& (BIT(chan
->phy
->idx
));
514 static void phy_enable(struct pxad_phy
*phy
, bool misaligned
)
516 struct pxad_device
*pdev
;
522 dev_dbg(&phy
->vchan
->vc
.chan
.dev
->device
,
523 "%s(); phy=%p(%d) misaligned=%d\n", __func__
,
524 phy
, phy
->idx
, misaligned
);
526 pdev
= to_pxad_dev(phy
->vchan
->vc
.chan
.device
);
527 if (phy
->vchan
->drcmr
<= pdev
->nr_requestors
) {
528 reg
= pxad_drcmr(phy
->vchan
->drcmr
);
529 writel_relaxed(DRCMR_MAPVLD
| phy
->idx
, phy
->base
+ reg
);
532 dalgn
= phy_readl_relaxed(phy
, DALGN
);
534 dalgn
|= BIT(phy
->idx
);
536 dalgn
&= ~BIT(phy
->idx
);
537 phy_writel_relaxed(phy
, dalgn
, DALGN
);
539 phy_writel(phy
, PXA_DCSR_STOPIRQEN
| PXA_DCSR_ENDINTR
|
540 PXA_DCSR_BUSERR
| PXA_DCSR_RUN
, DCSR
);
543 static void phy_disable(struct pxad_phy
*phy
)
550 dcsr
= phy_readl_relaxed(phy
, DCSR
);
551 dev_dbg(&phy
->vchan
->vc
.chan
.dev
->device
,
552 "%s(): phy=%p(%d)\n", __func__
, phy
, phy
->idx
);
553 phy_writel(phy
, dcsr
& ~PXA_DCSR_RUN
& ~PXA_DCSR_STOPIRQEN
, DCSR
);
556 static void pxad_launch_chan(struct pxad_chan
*chan
,
557 struct pxad_desc_sw
*desc
)
559 dev_dbg(&chan
->vc
.chan
.dev
->device
,
560 "%s(): desc=%p\n", __func__
, desc
);
562 chan
->phy
= lookup_phy(chan
);
564 dev_dbg(&chan
->vc
.chan
.dev
->device
,
565 "%s(): no free dma channel\n", __func__
);
572 * Program the descriptor's address into the DMA controller,
573 * then start the DMA transaction
575 phy_writel(chan
->phy
, desc
->first
, DDADR
);
576 phy_enable(chan
->phy
, chan
->misaligned
);
577 wake_up(&chan
->wq_state
);
580 static void set_updater_desc(struct pxad_desc_sw
*sw_desc
,
583 struct pxad_desc_hw
*updater
=
584 sw_desc
->hw_desc
[sw_desc
->nb_desc
- 1];
585 dma_addr_t dma
= sw_desc
->hw_desc
[sw_desc
->nb_desc
- 2]->ddadr
;
587 updater
->ddadr
= DDADR_STOP
;
588 updater
->dsadr
= dma
;
589 updater
->dtadr
= dma
+ 8;
590 updater
->dcmd
= PXA_DCMD_WIDTH4
| PXA_DCMD_BURST32
|
591 (PXA_DCMD_LENGTH
& sizeof(u32
));
592 if (flags
& DMA_PREP_INTERRUPT
)
593 updater
->dcmd
|= PXA_DCMD_ENDIRQEN
;
595 sw_desc
->hw_desc
[sw_desc
->nb_desc
- 2]->ddadr
= sw_desc
->first
;
598 static bool is_desc_completed(struct virt_dma_desc
*vd
)
600 struct pxad_desc_sw
*sw_desc
= to_pxad_sw_desc(vd
);
601 struct pxad_desc_hw
*updater
=
602 sw_desc
->hw_desc
[sw_desc
->nb_desc
- 1];
604 return updater
->dtadr
!= (updater
->dsadr
+ 8);
607 static void pxad_desc_chain(struct virt_dma_desc
*vd1
,
608 struct virt_dma_desc
*vd2
)
610 struct pxad_desc_sw
*desc1
= to_pxad_sw_desc(vd1
);
611 struct pxad_desc_sw
*desc2
= to_pxad_sw_desc(vd2
);
612 dma_addr_t dma_to_chain
;
614 dma_to_chain
= desc2
->first
;
615 desc1
->hw_desc
[desc1
->nb_desc
- 1]->ddadr
= dma_to_chain
;
618 static bool pxad_try_hotchain(struct virt_dma_chan
*vc
,
619 struct virt_dma_desc
*vd
)
621 struct virt_dma_desc
*vd_last_issued
= NULL
;
622 struct pxad_chan
*chan
= to_pxad_chan(&vc
->chan
);
625 * Attempt to hot chain the tx if the phy is still running. This is
626 * considered successful only if either the channel is still running
627 * after the chaining, or if the chained transfer is completed after
628 * having been hot chained.
629 * A change of alignment is not allowed, and forbids hotchaining.
631 if (is_chan_running(chan
)) {
632 BUG_ON(list_empty(&vc
->desc_issued
));
634 if (!is_running_chan_misaligned(chan
) &&
635 to_pxad_sw_desc(vd
)->misaligned
)
638 vd_last_issued
= list_entry(vc
->desc_issued
.prev
,
639 struct virt_dma_desc
, node
);
640 pxad_desc_chain(vd_last_issued
, vd
);
641 if (is_chan_running(chan
) || is_desc_completed(vd_last_issued
))
648 static unsigned int clear_chan_irq(struct pxad_phy
*phy
)
651 u32 dint
= readl(phy
->base
+ DINT
);
653 if (!(dint
& BIT(phy
->idx
)))
657 dcsr
= phy_readl_relaxed(phy
, DCSR
);
658 phy_writel(phy
, dcsr
, DCSR
);
659 if ((dcsr
& PXA_DCSR_BUSERR
) && (phy
->vchan
))
660 dev_warn(&phy
->vchan
->vc
.chan
.dev
->device
,
661 "%s(chan=%p): PXA_DCSR_BUSERR\n",
662 __func__
, &phy
->vchan
);
664 return dcsr
& ~PXA_DCSR_RUN
;
667 static irqreturn_t
pxad_chan_handler(int irq
, void *dev_id
)
669 struct pxad_phy
*phy
= dev_id
;
670 struct pxad_chan
*chan
= phy
->vchan
;
671 struct virt_dma_desc
*vd
, *tmp
;
674 dma_cookie_t last_started
= 0;
678 dcsr
= clear_chan_irq(phy
);
679 if (dcsr
& PXA_DCSR_RUN
)
682 spin_lock_irqsave(&chan
->vc
.lock
, flags
);
683 list_for_each_entry_safe(vd
, tmp
, &chan
->vc
.desc_issued
, node
) {
684 dev_dbg(&chan
->vc
.chan
.dev
->device
,
685 "%s(): checking txd %p[%x]: completed=%d\n",
686 __func__
, vd
, vd
->tx
.cookie
, is_desc_completed(vd
));
687 last_started
= vd
->tx
.cookie
;
688 if (to_pxad_sw_desc(vd
)->cyclic
) {
689 vchan_cyclic_callback(vd
);
692 if (is_desc_completed(vd
)) {
694 vchan_cookie_complete(vd
);
700 if (dcsr
& PXA_DCSR_BUSERR
) {
701 chan
->bus_error
= last_started
;
705 if (!chan
->bus_error
&& dcsr
& PXA_DCSR_STOPSTATE
) {
706 dev_dbg(&chan
->vc
.chan
.dev
->device
,
707 "%s(): channel stopped, submitted_empty=%d issued_empty=%d",
709 list_empty(&chan
->vc
.desc_submitted
),
710 list_empty(&chan
->vc
.desc_issued
));
711 phy_writel_relaxed(phy
, dcsr
& ~PXA_DCSR_STOPIRQEN
, DCSR
);
713 if (list_empty(&chan
->vc
.desc_issued
)) {
715 !list_empty(&chan
->vc
.desc_submitted
);
717 vd
= list_first_entry(&chan
->vc
.desc_issued
,
718 struct virt_dma_desc
, node
);
719 pxad_launch_chan(chan
, to_pxad_sw_desc(vd
));
722 spin_unlock_irqrestore(&chan
->vc
.lock
, flags
);
723 wake_up(&chan
->wq_state
);
728 static irqreturn_t
pxad_int_handler(int irq
, void *dev_id
)
730 struct pxad_device
*pdev
= dev_id
;
731 struct pxad_phy
*phy
;
732 u32 dint
= readl(pdev
->base
+ DINT
);
733 int i
, ret
= IRQ_NONE
;
738 phy
= &pdev
->phys
[i
];
739 if ((i
< 32) && (legacy_reserved
& BIT(i
)))
741 if (pxad_chan_handler(irq
, phy
) == IRQ_HANDLED
)
748 static int pxad_alloc_chan_resources(struct dma_chan
*dchan
)
750 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
751 struct pxad_device
*pdev
= to_pxad_dev(chan
->vc
.chan
.device
);
756 chan
->desc_pool
= dma_pool_create(dma_chan_name(dchan
),
758 sizeof(struct pxad_desc_hw
),
759 __alignof__(struct pxad_desc_hw
),
761 if (!chan
->desc_pool
) {
762 dev_err(&chan
->vc
.chan
.dev
->device
,
763 "%s(): unable to allocate descriptor pool\n",
771 static void pxad_free_chan_resources(struct dma_chan
*dchan
)
773 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
775 vchan_free_chan_resources(&chan
->vc
);
776 dma_pool_destroy(chan
->desc_pool
);
777 chan
->desc_pool
= NULL
;
781 static void pxad_free_desc(struct virt_dma_desc
*vd
)
785 struct pxad_desc_sw
*sw_desc
= to_pxad_sw_desc(vd
);
787 BUG_ON(sw_desc
->nb_desc
== 0);
788 for (i
= sw_desc
->nb_desc
- 1; i
>= 0; i
--) {
790 dma
= sw_desc
->hw_desc
[i
- 1]->ddadr
;
792 dma
= sw_desc
->first
;
793 dma_pool_free(sw_desc
->desc_pool
,
794 sw_desc
->hw_desc
[i
], dma
);
796 sw_desc
->nb_desc
= 0;
800 static struct pxad_desc_sw
*
801 pxad_alloc_desc(struct pxad_chan
*chan
, unsigned int nb_hw_desc
)
803 struct pxad_desc_sw
*sw_desc
;
807 sw_desc
= kzalloc(sizeof(*sw_desc
) +
808 nb_hw_desc
* sizeof(struct pxad_desc_hw
*),
812 sw_desc
->desc_pool
= chan
->desc_pool
;
814 for (i
= 0; i
< nb_hw_desc
; i
++) {
815 sw_desc
->hw_desc
[i
] = dma_pool_alloc(sw_desc
->desc_pool
,
817 if (!sw_desc
->hw_desc
[i
]) {
818 dev_err(&chan
->vc
.chan
.dev
->device
,
819 "%s(): Couldn't allocate the %dth hw_desc from dma_pool %p\n",
820 __func__
, i
, sw_desc
->desc_pool
);
825 sw_desc
->first
= dma
;
827 sw_desc
->hw_desc
[i
- 1]->ddadr
= dma
;
833 pxad_free_desc(&sw_desc
->vd
);
837 static dma_cookie_t
pxad_tx_submit(struct dma_async_tx_descriptor
*tx
)
839 struct virt_dma_chan
*vc
= to_virt_chan(tx
->chan
);
840 struct pxad_chan
*chan
= to_pxad_chan(&vc
->chan
);
841 struct virt_dma_desc
*vd_chained
= NULL
,
842 *vd
= container_of(tx
, struct virt_dma_desc
, tx
);
846 set_updater_desc(to_pxad_sw_desc(vd
), tx
->flags
);
848 spin_lock_irqsave(&vc
->lock
, flags
);
849 cookie
= dma_cookie_assign(tx
);
851 if (list_empty(&vc
->desc_submitted
) && pxad_try_hotchain(vc
, vd
)) {
852 list_move_tail(&vd
->node
, &vc
->desc_issued
);
853 dev_dbg(&chan
->vc
.chan
.dev
->device
,
854 "%s(): txd %p[%x]: submitted (hot linked)\n",
855 __func__
, vd
, cookie
);
860 * Fallback to placing the tx in the submitted queue
862 if (!list_empty(&vc
->desc_submitted
)) {
863 vd_chained
= list_entry(vc
->desc_submitted
.prev
,
864 struct virt_dma_desc
, node
);
866 * Only chain the descriptors if no new misalignment is
867 * introduced. If a new misalignment is chained, let the channel
868 * stop, and be relaunched in misalign mode from the irq
871 if (chan
->misaligned
|| !to_pxad_sw_desc(vd
)->misaligned
)
872 pxad_desc_chain(vd_chained
, vd
);
876 dev_dbg(&chan
->vc
.chan
.dev
->device
,
877 "%s(): txd %p[%x]: submitted (%s linked)\n",
878 __func__
, vd
, cookie
, vd_chained
? "cold" : "not");
879 list_move_tail(&vd
->node
, &vc
->desc_submitted
);
880 chan
->misaligned
|= to_pxad_sw_desc(vd
)->misaligned
;
883 spin_unlock_irqrestore(&vc
->lock
, flags
);
887 static void pxad_issue_pending(struct dma_chan
*dchan
)
889 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
890 struct virt_dma_desc
*vd_first
;
893 spin_lock_irqsave(&chan
->vc
.lock
, flags
);
894 if (list_empty(&chan
->vc
.desc_submitted
))
897 vd_first
= list_first_entry(&chan
->vc
.desc_submitted
,
898 struct virt_dma_desc
, node
);
899 dev_dbg(&chan
->vc
.chan
.dev
->device
,
900 "%s(): txd %p[%x]", __func__
, vd_first
, vd_first
->tx
.cookie
);
902 vchan_issue_pending(&chan
->vc
);
903 if (!pxad_try_hotchain(&chan
->vc
, vd_first
))
904 pxad_launch_chan(chan
, to_pxad_sw_desc(vd_first
));
906 spin_unlock_irqrestore(&chan
->vc
.lock
, flags
);
909 static inline struct dma_async_tx_descriptor
*
910 pxad_tx_prep(struct virt_dma_chan
*vc
, struct virt_dma_desc
*vd
,
911 unsigned long tx_flags
)
913 struct dma_async_tx_descriptor
*tx
;
914 struct pxad_chan
*chan
= container_of(vc
, struct pxad_chan
, vc
);
916 INIT_LIST_HEAD(&vd
->node
);
917 tx
= vchan_tx_prep(vc
, vd
, tx_flags
);
918 tx
->tx_submit
= pxad_tx_submit
;
919 dev_dbg(&chan
->vc
.chan
.dev
->device
,
920 "%s(): vc=%p txd=%p[%x] flags=0x%lx\n", __func__
,
921 vc
, vd
, vd
->tx
.cookie
,
927 static void pxad_get_config(struct pxad_chan
*chan
,
928 enum dma_transfer_direction dir
,
929 u32
*dcmd
, u32
*dev_src
, u32
*dev_dst
)
931 u32 maxburst
= 0, dev_addr
= 0;
932 enum dma_slave_buswidth width
= DMA_SLAVE_BUSWIDTH_UNDEFINED
;
933 struct pxad_device
*pdev
= to_pxad_dev(chan
->vc
.chan
.device
);
936 if (dir
== DMA_DEV_TO_MEM
) {
937 maxburst
= chan
->cfg
.src_maxburst
;
938 width
= chan
->cfg
.src_addr_width
;
939 dev_addr
= chan
->cfg
.src_addr
;
941 *dcmd
|= PXA_DCMD_INCTRGADDR
;
942 if (chan
->drcmr
<= pdev
->nr_requestors
)
943 *dcmd
|= PXA_DCMD_FLOWSRC
;
945 if (dir
== DMA_MEM_TO_DEV
) {
946 maxburst
= chan
->cfg
.dst_maxburst
;
947 width
= chan
->cfg
.dst_addr_width
;
948 dev_addr
= chan
->cfg
.dst_addr
;
950 *dcmd
|= PXA_DCMD_INCSRCADDR
;
951 if (chan
->drcmr
<= pdev
->nr_requestors
)
952 *dcmd
|= PXA_DCMD_FLOWTRG
;
954 if (dir
== DMA_MEM_TO_MEM
)
955 *dcmd
|= PXA_DCMD_BURST32
| PXA_DCMD_INCTRGADDR
|
958 dev_dbg(&chan
->vc
.chan
.dev
->device
,
959 "%s(): dev_addr=0x%x maxburst=%d width=%d dir=%d\n",
960 __func__
, dev_addr
, maxburst
, width
, dir
);
962 if (width
== DMA_SLAVE_BUSWIDTH_1_BYTE
)
963 *dcmd
|= PXA_DCMD_WIDTH1
;
964 else if (width
== DMA_SLAVE_BUSWIDTH_2_BYTES
)
965 *dcmd
|= PXA_DCMD_WIDTH2
;
966 else if (width
== DMA_SLAVE_BUSWIDTH_4_BYTES
)
967 *dcmd
|= PXA_DCMD_WIDTH4
;
970 *dcmd
|= PXA_DCMD_BURST8
;
971 else if (maxburst
== 16)
972 *dcmd
|= PXA_DCMD_BURST16
;
973 else if (maxburst
== 32)
974 *dcmd
|= PXA_DCMD_BURST32
;
976 /* FIXME: drivers should be ported over to use the filter
977 * function. Once that's done, the following two lines can
980 if (chan
->cfg
.slave_id
)
981 chan
->drcmr
= chan
->cfg
.slave_id
;
984 static struct dma_async_tx_descriptor
*
985 pxad_prep_memcpy(struct dma_chan
*dchan
,
986 dma_addr_t dma_dst
, dma_addr_t dma_src
,
987 size_t len
, unsigned long flags
)
989 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
990 struct pxad_desc_sw
*sw_desc
;
991 struct pxad_desc_hw
*hw_desc
;
993 unsigned int i
, nb_desc
= 0;
999 dev_dbg(&chan
->vc
.chan
.dev
->device
,
1000 "%s(): dma_dst=0x%lx dma_src=0x%lx len=%zu flags=%lx\n",
1001 __func__
, (unsigned long)dma_dst
, (unsigned long)dma_src
,
1003 pxad_get_config(chan
, DMA_MEM_TO_MEM
, &dcmd
, NULL
, NULL
);
1005 nb_desc
= DIV_ROUND_UP(len
, PDMA_MAX_DESC_BYTES
);
1006 sw_desc
= pxad_alloc_desc(chan
, nb_desc
+ 1);
1011 if (!IS_ALIGNED(dma_src
, 1 << PDMA_ALIGNMENT
) ||
1012 !IS_ALIGNED(dma_dst
, 1 << PDMA_ALIGNMENT
))
1013 sw_desc
->misaligned
= true;
1017 hw_desc
= sw_desc
->hw_desc
[i
++];
1018 copy
= min_t(size_t, len
, PDMA_MAX_DESC_BYTES
);
1019 hw_desc
->dcmd
= dcmd
| (PXA_DCMD_LENGTH
& copy
);
1020 hw_desc
->dsadr
= dma_src
;
1021 hw_desc
->dtadr
= dma_dst
;
1026 set_updater_desc(sw_desc
, flags
);
1028 return pxad_tx_prep(&chan
->vc
, &sw_desc
->vd
, flags
);
1031 static struct dma_async_tx_descriptor
*
1032 pxad_prep_slave_sg(struct dma_chan
*dchan
, struct scatterlist
*sgl
,
1033 unsigned int sg_len
, enum dma_transfer_direction dir
,
1034 unsigned long flags
, void *context
)
1036 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
1037 struct pxad_desc_sw
*sw_desc
;
1039 struct scatterlist
*sg
;
1041 u32 dcmd
, dsadr
= 0, dtadr
= 0;
1042 unsigned int nb_desc
= 0, i
, j
= 0;
1044 if ((sgl
== NULL
) || (sg_len
== 0))
1047 pxad_get_config(chan
, dir
, &dcmd
, &dsadr
, &dtadr
);
1048 dev_dbg(&chan
->vc
.chan
.dev
->device
,
1049 "%s(): dir=%d flags=%lx\n", __func__
, dir
, flags
);
1051 for_each_sg(sgl
, sg
, sg_len
, i
)
1052 nb_desc
+= DIV_ROUND_UP(sg_dma_len(sg
), PDMA_MAX_DESC_BYTES
);
1053 sw_desc
= pxad_alloc_desc(chan
, nb_desc
+ 1);
1057 for_each_sg(sgl
, sg
, sg_len
, i
) {
1058 dma
= sg_dma_address(sg
);
1059 avail
= sg_dma_len(sg
);
1060 sw_desc
->len
+= avail
;
1063 len
= min_t(size_t, avail
, PDMA_MAX_DESC_BYTES
);
1065 sw_desc
->misaligned
= true;
1067 sw_desc
->hw_desc
[j
]->dcmd
=
1068 dcmd
| (PXA_DCMD_LENGTH
& len
);
1069 sw_desc
->hw_desc
[j
]->dsadr
= dsadr
? dsadr
: dma
;
1070 sw_desc
->hw_desc
[j
++]->dtadr
= dtadr
? dtadr
: dma
;
1076 set_updater_desc(sw_desc
, flags
);
1078 return pxad_tx_prep(&chan
->vc
, &sw_desc
->vd
, flags
);
1081 static struct dma_async_tx_descriptor
*
1082 pxad_prep_dma_cyclic(struct dma_chan
*dchan
,
1083 dma_addr_t buf_addr
, size_t len
, size_t period_len
,
1084 enum dma_transfer_direction dir
, unsigned long flags
)
1086 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
1087 struct pxad_desc_sw
*sw_desc
;
1088 struct pxad_desc_hw
**phw_desc
;
1090 u32 dcmd
, dsadr
= 0, dtadr
= 0;
1091 unsigned int nb_desc
= 0;
1093 if (!dchan
|| !len
|| !period_len
)
1095 if ((dir
!= DMA_DEV_TO_MEM
) && (dir
!= DMA_MEM_TO_DEV
)) {
1096 dev_err(&chan
->vc
.chan
.dev
->device
,
1097 "Unsupported direction for cyclic DMA\n");
1100 /* the buffer length must be a multiple of period_len */
1101 if (len
% period_len
!= 0 || period_len
> PDMA_MAX_DESC_BYTES
||
1102 !IS_ALIGNED(period_len
, 1 << PDMA_ALIGNMENT
))
1105 pxad_get_config(chan
, dir
, &dcmd
, &dsadr
, &dtadr
);
1106 dcmd
|= PXA_DCMD_ENDIRQEN
| (PXA_DCMD_LENGTH
& period_len
);
1107 dev_dbg(&chan
->vc
.chan
.dev
->device
,
1108 "%s(): buf_addr=0x%lx len=%zu period=%zu dir=%d flags=%lx\n",
1109 __func__
, (unsigned long)buf_addr
, len
, period_len
, dir
, flags
);
1111 nb_desc
= DIV_ROUND_UP(period_len
, PDMA_MAX_DESC_BYTES
);
1112 nb_desc
*= DIV_ROUND_UP(len
, period_len
);
1113 sw_desc
= pxad_alloc_desc(chan
, nb_desc
+ 1);
1116 sw_desc
->cyclic
= true;
1119 phw_desc
= sw_desc
->hw_desc
;
1122 phw_desc
[0]->dsadr
= dsadr
? dsadr
: dma
;
1123 phw_desc
[0]->dtadr
= dtadr
? dtadr
: dma
;
1124 phw_desc
[0]->dcmd
= dcmd
;
1129 set_updater_desc(sw_desc
, flags
);
1131 return pxad_tx_prep(&chan
->vc
, &sw_desc
->vd
, flags
);
1134 static int pxad_config(struct dma_chan
*dchan
,
1135 struct dma_slave_config
*cfg
)
1137 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
1146 static int pxad_terminate_all(struct dma_chan
*dchan
)
1148 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
1149 struct pxad_device
*pdev
= to_pxad_dev(chan
->vc
.chan
.device
);
1150 struct virt_dma_desc
*vd
= NULL
;
1151 unsigned long flags
;
1152 struct pxad_phy
*phy
;
1155 dev_dbg(&chan
->vc
.chan
.dev
->device
,
1156 "%s(): vchan %p: terminate all\n", __func__
, &chan
->vc
);
1158 spin_lock_irqsave(&chan
->vc
.lock
, flags
);
1159 vchan_get_all_descriptors(&chan
->vc
, &head
);
1161 list_for_each_entry(vd
, &head
, node
) {
1162 dev_dbg(&chan
->vc
.chan
.dev
->device
,
1163 "%s(): cancelling txd %p[%x] (completed=%d)", __func__
,
1164 vd
, vd
->tx
.cookie
, is_desc_completed(vd
));
1169 phy_disable(chan
->phy
);
1170 pxad_free_phy(chan
);
1172 spin_lock(&pdev
->phy_lock
);
1174 spin_unlock(&pdev
->phy_lock
);
1176 spin_unlock_irqrestore(&chan
->vc
.lock
, flags
);
1177 vchan_dma_desc_free_list(&chan
->vc
, &head
);
1182 static unsigned int pxad_residue(struct pxad_chan
*chan
,
1183 dma_cookie_t cookie
)
1185 struct virt_dma_desc
*vd
= NULL
;
1186 struct pxad_desc_sw
*sw_desc
= NULL
;
1187 struct pxad_desc_hw
*hw_desc
= NULL
;
1188 u32 curr
, start
, len
, end
, residue
= 0;
1189 unsigned long flags
;
1190 bool passed
= false;
1194 * If the channel does not have a phy pointer anymore, it has already
1195 * been completed. Therefore, its residue is 0.
1200 spin_lock_irqsave(&chan
->vc
.lock
, flags
);
1202 vd
= vchan_find_desc(&chan
->vc
, cookie
);
1206 sw_desc
= to_pxad_sw_desc(vd
);
1207 if (sw_desc
->hw_desc
[0]->dcmd
& PXA_DCMD_INCSRCADDR
)
1208 curr
= phy_readl_relaxed(chan
->phy
, DSADR
);
1210 curr
= phy_readl_relaxed(chan
->phy
, DTADR
);
1213 * curr has to be actually read before checking descriptor
1214 * completion, so that a curr inside a status updater
1215 * descriptor implies the following test returns true, and
1216 * preventing reordering of curr load and the test.
1219 if (is_desc_completed(vd
))
1222 for (i
= 0; i
< sw_desc
->nb_desc
- 1; i
++) {
1223 hw_desc
= sw_desc
->hw_desc
[i
];
1224 if (sw_desc
->hw_desc
[0]->dcmd
& PXA_DCMD_INCSRCADDR
)
1225 start
= hw_desc
->dsadr
;
1227 start
= hw_desc
->dtadr
;
1228 len
= hw_desc
->dcmd
& PXA_DCMD_LENGTH
;
1232 * 'passed' will be latched once we found the descriptor
1233 * which lies inside the boundaries of the curr
1234 * pointer. All descriptors that occur in the list
1235 * _after_ we found that partially handled descriptor
1236 * are still to be processed and are hence added to the
1237 * residual bytes counter.
1242 } else if (curr
>= start
&& curr
<= end
) {
1243 residue
+= end
- curr
;
1248 residue
= sw_desc
->len
;
1251 spin_unlock_irqrestore(&chan
->vc
.lock
, flags
);
1252 dev_dbg(&chan
->vc
.chan
.dev
->device
,
1253 "%s(): txd %p[%x] sw_desc=%p: %d\n",
1254 __func__
, vd
, cookie
, sw_desc
, residue
);
1258 static enum dma_status
pxad_tx_status(struct dma_chan
*dchan
,
1259 dma_cookie_t cookie
,
1260 struct dma_tx_state
*txstate
)
1262 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
1263 enum dma_status ret
;
1265 if (cookie
== chan
->bus_error
)
1268 ret
= dma_cookie_status(dchan
, cookie
, txstate
);
1269 if (likely(txstate
&& (ret
!= DMA_ERROR
)))
1270 dma_set_residue(txstate
, pxad_residue(chan
, cookie
));
1275 static void pxad_synchronize(struct dma_chan
*dchan
)
1277 struct pxad_chan
*chan
= to_pxad_chan(dchan
);
1279 wait_event(chan
->wq_state
, !is_chan_running(chan
));
1280 vchan_synchronize(&chan
->vc
);
1283 static void pxad_free_channels(struct dma_device
*dmadev
)
1285 struct pxad_chan
*c
, *cn
;
1287 list_for_each_entry_safe(c
, cn
, &dmadev
->channels
,
1288 vc
.chan
.device_node
) {
1289 list_del(&c
->vc
.chan
.device_node
);
1290 tasklet_kill(&c
->vc
.task
);
1294 static int pxad_remove(struct platform_device
*op
)
1296 struct pxad_device
*pdev
= platform_get_drvdata(op
);
1298 pxad_cleanup_debugfs(pdev
);
1299 pxad_free_channels(&pdev
->slave
);
1300 dma_async_device_unregister(&pdev
->slave
);
1304 static int pxad_init_phys(struct platform_device
*op
,
1305 struct pxad_device
*pdev
,
1306 unsigned int nb_phy_chans
)
1308 int irq0
, irq
, nr_irq
= 0, i
, ret
;
1309 struct pxad_phy
*phy
;
1311 irq0
= platform_get_irq(op
, 0);
1315 pdev
->phys
= devm_kcalloc(&op
->dev
, nb_phy_chans
,
1316 sizeof(pdev
->phys
[0]), GFP_KERNEL
);
1320 for (i
= 0; i
< nb_phy_chans
; i
++)
1321 if (platform_get_irq(op
, i
) > 0)
1324 for (i
= 0; i
< nb_phy_chans
; i
++) {
1325 phy
= &pdev
->phys
[i
];
1326 phy
->base
= pdev
->base
;
1328 irq
= platform_get_irq(op
, i
);
1329 if ((nr_irq
> 1) && (irq
> 0))
1330 ret
= devm_request_irq(&op
->dev
, irq
,
1332 IRQF_SHARED
, "pxa-dma", phy
);
1333 if ((nr_irq
== 1) && (i
== 0))
1334 ret
= devm_request_irq(&op
->dev
, irq0
,
1336 IRQF_SHARED
, "pxa-dma", pdev
);
1338 dev_err(pdev
->slave
.dev
,
1339 "%s(): can't request irq %d:%d\n", __func__
,
1348 static const struct of_device_id pxad_dt_ids
[] = {
1349 { .compatible
= "marvell,pdma-1.0", },
1352 MODULE_DEVICE_TABLE(of
, pxad_dt_ids
);
1354 static struct dma_chan
*pxad_dma_xlate(struct of_phandle_args
*dma_spec
,
1355 struct of_dma
*ofdma
)
1357 struct pxad_device
*d
= ofdma
->of_dma_data
;
1358 struct dma_chan
*chan
;
1360 chan
= dma_get_any_slave_channel(&d
->slave
);
1364 to_pxad_chan(chan
)->drcmr
= dma_spec
->args
[0];
1365 to_pxad_chan(chan
)->prio
= dma_spec
->args
[1];
1370 static int pxad_init_dmadev(struct platform_device
*op
,
1371 struct pxad_device
*pdev
,
1372 unsigned int nr_phy_chans
,
1373 unsigned int nr_requestors
)
1377 struct pxad_chan
*c
;
1379 pdev
->nr_chans
= nr_phy_chans
;
1380 pdev
->nr_requestors
= nr_requestors
;
1381 INIT_LIST_HEAD(&pdev
->slave
.channels
);
1382 pdev
->slave
.device_alloc_chan_resources
= pxad_alloc_chan_resources
;
1383 pdev
->slave
.device_free_chan_resources
= pxad_free_chan_resources
;
1384 pdev
->slave
.device_tx_status
= pxad_tx_status
;
1385 pdev
->slave
.device_issue_pending
= pxad_issue_pending
;
1386 pdev
->slave
.device_config
= pxad_config
;
1387 pdev
->slave
.device_synchronize
= pxad_synchronize
;
1388 pdev
->slave
.device_terminate_all
= pxad_terminate_all
;
1390 if (op
->dev
.coherent_dma_mask
)
1391 dma_set_mask(&op
->dev
, op
->dev
.coherent_dma_mask
);
1393 dma_set_mask(&op
->dev
, DMA_BIT_MASK(32));
1395 ret
= pxad_init_phys(op
, pdev
, nr_phy_chans
);
1399 for (i
= 0; i
< nr_phy_chans
; i
++) {
1400 c
= devm_kzalloc(&op
->dev
, sizeof(*c
), GFP_KERNEL
);
1403 c
->vc
.desc_free
= pxad_free_desc
;
1404 vchan_init(&c
->vc
, &pdev
->slave
);
1405 init_waitqueue_head(&c
->wq_state
);
1408 return dma_async_device_register(&pdev
->slave
);
1411 static int pxad_probe(struct platform_device
*op
)
1413 struct pxad_device
*pdev
;
1414 const struct of_device_id
*of_id
;
1415 struct mmp_dma_platdata
*pdata
= dev_get_platdata(&op
->dev
);
1416 struct resource
*iores
;
1417 int ret
, dma_channels
= 0, nb_requestors
= 0;
1418 const enum dma_slave_buswidth widths
=
1419 DMA_SLAVE_BUSWIDTH_1_BYTE
| DMA_SLAVE_BUSWIDTH_2_BYTES
|
1420 DMA_SLAVE_BUSWIDTH_4_BYTES
;
1422 pdev
= devm_kzalloc(&op
->dev
, sizeof(*pdev
), GFP_KERNEL
);
1426 spin_lock_init(&pdev
->phy_lock
);
1428 iores
= platform_get_resource(op
, IORESOURCE_MEM
, 0);
1429 pdev
->base
= devm_ioremap_resource(&op
->dev
, iores
);
1430 if (IS_ERR(pdev
->base
))
1431 return PTR_ERR(pdev
->base
);
1433 of_id
= of_match_device(pxad_dt_ids
, &op
->dev
);
1435 of_property_read_u32(op
->dev
.of_node
, "#dma-channels",
1437 ret
= of_property_read_u32(op
->dev
.of_node
, "#dma-requests",
1440 dev_warn(pdev
->slave
.dev
,
1441 "#dma-requests set to default 32 as missing in OF: %d",
1445 } else if (pdata
&& pdata
->dma_channels
) {
1446 dma_channels
= pdata
->dma_channels
;
1447 nb_requestors
= pdata
->nb_requestors
;
1449 dma_channels
= 32; /* default 32 channel */
1452 dma_cap_set(DMA_SLAVE
, pdev
->slave
.cap_mask
);
1453 dma_cap_set(DMA_MEMCPY
, pdev
->slave
.cap_mask
);
1454 dma_cap_set(DMA_CYCLIC
, pdev
->slave
.cap_mask
);
1455 dma_cap_set(DMA_PRIVATE
, pdev
->slave
.cap_mask
);
1456 pdev
->slave
.device_prep_dma_memcpy
= pxad_prep_memcpy
;
1457 pdev
->slave
.device_prep_slave_sg
= pxad_prep_slave_sg
;
1458 pdev
->slave
.device_prep_dma_cyclic
= pxad_prep_dma_cyclic
;
1460 pdev
->slave
.copy_align
= PDMA_ALIGNMENT
;
1461 pdev
->slave
.src_addr_widths
= widths
;
1462 pdev
->slave
.dst_addr_widths
= widths
;
1463 pdev
->slave
.directions
= BIT(DMA_MEM_TO_DEV
) | BIT(DMA_DEV_TO_MEM
);
1464 pdev
->slave
.residue_granularity
= DMA_RESIDUE_GRANULARITY_DESCRIPTOR
;
1465 pdev
->slave
.descriptor_reuse
= true;
1467 pdev
->slave
.dev
= &op
->dev
;
1468 ret
= pxad_init_dmadev(op
, pdev
, dma_channels
, nb_requestors
);
1470 dev_err(pdev
->slave
.dev
, "unable to register\n");
1474 if (op
->dev
.of_node
) {
1475 /* Device-tree DMA controller registration */
1476 ret
= of_dma_controller_register(op
->dev
.of_node
,
1477 pxad_dma_xlate
, pdev
);
1479 dev_err(pdev
->slave
.dev
,
1480 "of_dma_controller_register failed\n");
1485 platform_set_drvdata(op
, pdev
);
1486 pxad_init_debugfs(pdev
);
1487 dev_info(pdev
->slave
.dev
, "initialized %d channels on %d requestors\n",
1488 dma_channels
, nb_requestors
);
1492 static const struct platform_device_id pxad_id_table
[] = {
1497 static struct platform_driver pxad_driver
= {
1500 .of_match_table
= pxad_dt_ids
,
1502 .id_table
= pxad_id_table
,
1503 .probe
= pxad_probe
,
1504 .remove
= pxad_remove
,
1507 bool pxad_filter_fn(struct dma_chan
*chan
, void *param
)
1509 struct pxad_chan
*c
= to_pxad_chan(chan
);
1510 struct pxad_param
*p
= param
;
1512 if (chan
->device
->dev
->driver
!= &pxad_driver
.driver
)
1515 c
->drcmr
= p
->drcmr
;
1520 EXPORT_SYMBOL_GPL(pxad_filter_fn
);
1522 int pxad_toggle_reserved_channel(int legacy_channel
)
1524 if (legacy_unavailable
& (BIT(legacy_channel
)))
1526 legacy_reserved
^= BIT(legacy_channel
);
1529 EXPORT_SYMBOL_GPL(pxad_toggle_reserved_channel
);
1531 module_platform_driver(pxad_driver
);
1533 MODULE_DESCRIPTION("Marvell PXA Peripheral DMA Driver");
1534 MODULE_AUTHOR("Robert Jarzmik <robert.jarzmik@free.fr>");
1535 MODULE_LICENSE("GPL v2");