2 * Freescale MPC85xx, MPC83xx DMA Engine support
4 * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved.
7 * Zhang Wei <wei.zhang@freescale.com>, Jul 2007
8 * Ebony Zhu <ebony.zhu@freescale.com>, May 2007
11 * DMA engine driver for Freescale MPC8540 DMA controller, which is
12 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
13 * The support for MPC8349 DMA controller is also added.
15 * This driver instructs the DMA controller to issue the PCI Read Multiple
16 * command for PCI read operations, instead of using the default PCI Read Line
17 * command. Please be aware that this setting may result in read pre-fetching
20 * This is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/slab.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/dmapool.h>
36 #include <linux/of_platform.h>
38 #include "dmaengine.h"
41 #define chan_dbg(chan, fmt, arg...) \
42 dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
43 #define chan_err(chan, fmt, arg...) \
44 dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
46 static const char msg_ld_oom
[] = "No free memory for link descriptor";
52 static void set_sr(struct fsldma_chan
*chan
, u32 val
)
54 DMA_OUT(chan
, &chan
->regs
->sr
, val
, 32);
57 static u32
get_sr(struct fsldma_chan
*chan
)
59 return DMA_IN(chan
, &chan
->regs
->sr
, 32);
62 static void set_cdar(struct fsldma_chan
*chan
, dma_addr_t addr
)
64 DMA_OUT(chan
, &chan
->regs
->cdar
, addr
| FSL_DMA_SNEN
, 64);
67 static dma_addr_t
get_cdar(struct fsldma_chan
*chan
)
69 return DMA_IN(chan
, &chan
->regs
->cdar
, 64) & ~FSL_DMA_SNEN
;
72 static u32
get_bcr(struct fsldma_chan
*chan
)
74 return DMA_IN(chan
, &chan
->regs
->bcr
, 32);
81 static void set_desc_cnt(struct fsldma_chan
*chan
,
82 struct fsl_dma_ld_hw
*hw
, u32 count
)
84 hw
->count
= CPU_TO_DMA(chan
, count
, 32);
87 static u32
get_desc_cnt(struct fsldma_chan
*chan
, struct fsl_desc_sw
*desc
)
89 return DMA_TO_CPU(chan
, desc
->hw
.count
, 32);
92 static void set_desc_src(struct fsldma_chan
*chan
,
93 struct fsl_dma_ld_hw
*hw
, dma_addr_t src
)
97 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
98 ? ((u64
)FSL_DMA_SATR_SREADTYPE_SNOOP_READ
<< 32) : 0;
99 hw
->src_addr
= CPU_TO_DMA(chan
, snoop_bits
| src
, 64);
102 static dma_addr_t
get_desc_src(struct fsldma_chan
*chan
,
103 struct fsl_desc_sw
*desc
)
107 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
108 ? ((u64
)FSL_DMA_SATR_SREADTYPE_SNOOP_READ
<< 32) : 0;
109 return DMA_TO_CPU(chan
, desc
->hw
.src_addr
, 64) & ~snoop_bits
;
112 static void set_desc_dst(struct fsldma_chan
*chan
,
113 struct fsl_dma_ld_hw
*hw
, dma_addr_t dst
)
117 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
118 ? ((u64
)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE
<< 32) : 0;
119 hw
->dst_addr
= CPU_TO_DMA(chan
, snoop_bits
| dst
, 64);
122 static dma_addr_t
get_desc_dst(struct fsldma_chan
*chan
,
123 struct fsl_desc_sw
*desc
)
127 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
128 ? ((u64
)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE
<< 32) : 0;
129 return DMA_TO_CPU(chan
, desc
->hw
.dst_addr
, 64) & ~snoop_bits
;
132 static void set_desc_next(struct fsldma_chan
*chan
,
133 struct fsl_dma_ld_hw
*hw
, dma_addr_t next
)
137 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_83XX
)
139 hw
->next_ln_addr
= CPU_TO_DMA(chan
, snoop_bits
| next
, 64);
142 static void set_ld_eol(struct fsldma_chan
*chan
, struct fsl_desc_sw
*desc
)
146 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_83XX
)
149 desc
->hw
.next_ln_addr
= CPU_TO_DMA(chan
,
150 DMA_TO_CPU(chan
, desc
->hw
.next_ln_addr
, 64) | FSL_DMA_EOL
155 * DMA Engine Hardware Control Helpers
158 static void dma_init(struct fsldma_chan
*chan
)
160 /* Reset the channel */
161 DMA_OUT(chan
, &chan
->regs
->mr
, 0, 32);
163 switch (chan
->feature
& FSL_DMA_IP_MASK
) {
164 case FSL_DMA_IP_85XX
:
165 /* Set the channel to below modes:
166 * EIE - Error interrupt enable
167 * EOLNIE - End of links interrupt enable
168 * BWC - Bandwidth sharing among channels
170 DMA_OUT(chan
, &chan
->regs
->mr
, FSL_DMA_MR_BWC
171 | FSL_DMA_MR_EIE
| FSL_DMA_MR_EOLNIE
, 32);
173 case FSL_DMA_IP_83XX
:
174 /* Set the channel to below modes:
175 * EOTIE - End-of-transfer interrupt enable
176 * PRC_RM - PCI read multiple
178 DMA_OUT(chan
, &chan
->regs
->mr
, FSL_DMA_MR_EOTIE
179 | FSL_DMA_MR_PRC_RM
, 32);
184 static int dma_is_idle(struct fsldma_chan
*chan
)
186 u32 sr
= get_sr(chan
);
187 return (!(sr
& FSL_DMA_SR_CB
)) || (sr
& FSL_DMA_SR_CH
);
191 * Start the DMA controller
194 * - the CDAR register must point to the start descriptor
195 * - the MRn[CS] bit must be cleared
197 static void dma_start(struct fsldma_chan
*chan
)
201 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
203 if (chan
->feature
& FSL_DMA_CHAN_PAUSE_EXT
) {
204 DMA_OUT(chan
, &chan
->regs
->bcr
, 0, 32);
205 mode
|= FSL_DMA_MR_EMP_EN
;
207 mode
&= ~FSL_DMA_MR_EMP_EN
;
210 if (chan
->feature
& FSL_DMA_CHAN_START_EXT
) {
211 mode
|= FSL_DMA_MR_EMS_EN
;
213 mode
&= ~FSL_DMA_MR_EMS_EN
;
214 mode
|= FSL_DMA_MR_CS
;
217 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
220 static void dma_halt(struct fsldma_chan
*chan
)
225 /* read the mode register */
226 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
229 * The 85xx controller supports channel abort, which will stop
230 * the current transfer. On 83xx, this bit is the transfer error
231 * mask bit, which should not be changed.
233 if ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
) {
234 mode
|= FSL_DMA_MR_CA
;
235 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
237 mode
&= ~FSL_DMA_MR_CA
;
240 /* stop the DMA controller */
241 mode
&= ~(FSL_DMA_MR_CS
| FSL_DMA_MR_EMS_EN
);
242 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
244 /* wait for the DMA controller to become idle */
245 for (i
= 0; i
< 100; i
++) {
246 if (dma_is_idle(chan
))
252 if (!dma_is_idle(chan
))
253 chan_err(chan
, "DMA halt timeout!\n");
257 * fsl_chan_set_src_loop_size - Set source address hold transfer size
258 * @chan : Freescale DMA channel
259 * @size : Address loop size, 0 for disable loop
261 * The set source address hold transfer size. The source
262 * address hold or loop transfer size is when the DMA transfer
263 * data from source address (SA), if the loop size is 4, the DMA will
264 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
265 * SA + 1 ... and so on.
267 static void fsl_chan_set_src_loop_size(struct fsldma_chan
*chan
, int size
)
271 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
275 mode
&= ~FSL_DMA_MR_SAHE
;
281 mode
|= FSL_DMA_MR_SAHE
| (__ilog2(size
) << 14);
285 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
289 * fsl_chan_set_dst_loop_size - Set destination address hold transfer size
290 * @chan : Freescale DMA channel
291 * @size : Address loop size, 0 for disable loop
293 * The set destination address hold transfer size. The destination
294 * address hold or loop transfer size is when the DMA transfer
295 * data to destination address (TA), if the loop size is 4, the DMA will
296 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
297 * TA + 1 ... and so on.
299 static void fsl_chan_set_dst_loop_size(struct fsldma_chan
*chan
, int size
)
303 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
307 mode
&= ~FSL_DMA_MR_DAHE
;
313 mode
|= FSL_DMA_MR_DAHE
| (__ilog2(size
) << 16);
317 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
321 * fsl_chan_set_request_count - Set DMA Request Count for external control
322 * @chan : Freescale DMA channel
323 * @size : Number of bytes to transfer in a single request
325 * The Freescale DMA channel can be controlled by the external signal DREQ#.
326 * The DMA request count is how many bytes are allowed to transfer before
327 * pausing the channel, after which a new assertion of DREQ# resumes channel
330 * A size of 0 disables external pause control. The maximum size is 1024.
332 static void fsl_chan_set_request_count(struct fsldma_chan
*chan
, int size
)
338 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
339 mode
|= (__ilog2(size
) << 24) & 0x0f000000;
341 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
345 * fsl_chan_toggle_ext_pause - Toggle channel external pause status
346 * @chan : Freescale DMA channel
347 * @enable : 0 is disabled, 1 is enabled.
349 * The Freescale DMA channel can be controlled by the external signal DREQ#.
350 * The DMA Request Count feature should be used in addition to this feature
351 * to set the number of bytes to transfer before pausing the channel.
353 static void fsl_chan_toggle_ext_pause(struct fsldma_chan
*chan
, int enable
)
356 chan
->feature
|= FSL_DMA_CHAN_PAUSE_EXT
;
358 chan
->feature
&= ~FSL_DMA_CHAN_PAUSE_EXT
;
362 * fsl_chan_toggle_ext_start - Toggle channel external start status
363 * @chan : Freescale DMA channel
364 * @enable : 0 is disabled, 1 is enabled.
366 * If enable the external start, the channel can be started by an
367 * external DMA start pin. So the dma_start() does not start the
368 * transfer immediately. The DMA channel will wait for the
369 * control pin asserted.
371 static void fsl_chan_toggle_ext_start(struct fsldma_chan
*chan
, int enable
)
374 chan
->feature
|= FSL_DMA_CHAN_START_EXT
;
376 chan
->feature
&= ~FSL_DMA_CHAN_START_EXT
;
379 static void append_ld_queue(struct fsldma_chan
*chan
, struct fsl_desc_sw
*desc
)
381 struct fsl_desc_sw
*tail
= to_fsl_desc(chan
->ld_pending
.prev
);
383 if (list_empty(&chan
->ld_pending
))
387 * Add the hardware descriptor to the chain of hardware descriptors
388 * that already exists in memory.
390 * This will un-set the EOL bit of the existing transaction, and the
391 * last link in this transaction will become the EOL descriptor.
393 set_desc_next(chan
, &tail
->hw
, desc
->async_tx
.phys
);
396 * Add the software descriptor and all children to the list
397 * of pending transactions
400 list_splice_tail_init(&desc
->tx_list
, &chan
->ld_pending
);
403 static dma_cookie_t
fsl_dma_tx_submit(struct dma_async_tx_descriptor
*tx
)
405 struct fsldma_chan
*chan
= to_fsl_chan(tx
->chan
);
406 struct fsl_desc_sw
*desc
= tx_to_fsl_desc(tx
);
407 struct fsl_desc_sw
*child
;
411 spin_lock_irqsave(&chan
->desc_lock
, flags
);
414 * assign cookies to all of the software descriptors
415 * that make up this transaction
417 list_for_each_entry(child
, &desc
->tx_list
, node
) {
418 cookie
= dma_cookie_assign(&child
->async_tx
);
421 /* put this transaction onto the tail of the pending queue */
422 append_ld_queue(chan
, desc
);
424 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
430 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
431 * @chan : Freescale DMA channel
433 * Return - The descriptor allocated. NULL for failed.
435 static struct fsl_desc_sw
*fsl_dma_alloc_descriptor(struct fsldma_chan
*chan
)
437 struct fsl_desc_sw
*desc
;
440 desc
= dma_pool_alloc(chan
->desc_pool
, GFP_ATOMIC
, &pdesc
);
442 chan_dbg(chan
, "out of memory for link descriptor\n");
446 memset(desc
, 0, sizeof(*desc
));
447 INIT_LIST_HEAD(&desc
->tx_list
);
448 dma_async_tx_descriptor_init(&desc
->async_tx
, &chan
->common
);
449 desc
->async_tx
.tx_submit
= fsl_dma_tx_submit
;
450 desc
->async_tx
.phys
= pdesc
;
452 #ifdef FSL_DMA_LD_DEBUG
453 chan_dbg(chan
, "LD %p allocated\n", desc
);
460 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
461 * @chan : Freescale DMA channel
463 * This function will create a dma pool for descriptor allocation.
465 * Return - The number of descriptors allocated.
467 static int fsl_dma_alloc_chan_resources(struct dma_chan
*dchan
)
469 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
471 /* Has this channel already been allocated? */
476 * We need the descriptor to be aligned to 32bytes
477 * for meeting FSL DMA specification requirement.
479 chan
->desc_pool
= dma_pool_create(chan
->name
, chan
->dev
,
480 sizeof(struct fsl_desc_sw
),
481 __alignof__(struct fsl_desc_sw
), 0);
482 if (!chan
->desc_pool
) {
483 chan_err(chan
, "unable to allocate descriptor pool\n");
487 /* there is at least one descriptor free to be allocated */
492 * fsldma_free_desc_list - Free all descriptors in a queue
493 * @chan: Freescae DMA channel
494 * @list: the list to free
496 * LOCKING: must hold chan->desc_lock
498 static void fsldma_free_desc_list(struct fsldma_chan
*chan
,
499 struct list_head
*list
)
501 struct fsl_desc_sw
*desc
, *_desc
;
503 list_for_each_entry_safe(desc
, _desc
, list
, node
) {
504 list_del(&desc
->node
);
505 #ifdef FSL_DMA_LD_DEBUG
506 chan_dbg(chan
, "LD %p free\n", desc
);
508 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
512 static void fsldma_free_desc_list_reverse(struct fsldma_chan
*chan
,
513 struct list_head
*list
)
515 struct fsl_desc_sw
*desc
, *_desc
;
517 list_for_each_entry_safe_reverse(desc
, _desc
, list
, node
) {
518 list_del(&desc
->node
);
519 #ifdef FSL_DMA_LD_DEBUG
520 chan_dbg(chan
, "LD %p free\n", desc
);
522 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
527 * fsl_dma_free_chan_resources - Free all resources of the channel.
528 * @chan : Freescale DMA channel
530 static void fsl_dma_free_chan_resources(struct dma_chan
*dchan
)
532 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
535 chan_dbg(chan
, "free all channel resources\n");
536 spin_lock_irqsave(&chan
->desc_lock
, flags
);
537 fsldma_free_desc_list(chan
, &chan
->ld_pending
);
538 fsldma_free_desc_list(chan
, &chan
->ld_running
);
539 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
541 dma_pool_destroy(chan
->desc_pool
);
542 chan
->desc_pool
= NULL
;
545 static struct dma_async_tx_descriptor
*
546 fsl_dma_prep_interrupt(struct dma_chan
*dchan
, unsigned long flags
)
548 struct fsldma_chan
*chan
;
549 struct fsl_desc_sw
*new;
554 chan
= to_fsl_chan(dchan
);
556 new = fsl_dma_alloc_descriptor(chan
);
558 chan_err(chan
, "%s\n", msg_ld_oom
);
562 new->async_tx
.cookie
= -EBUSY
;
563 new->async_tx
.flags
= flags
;
565 /* Insert the link descriptor to the LD ring */
566 list_add_tail(&new->node
, &new->tx_list
);
568 /* Set End-of-link to the last link descriptor of new list */
569 set_ld_eol(chan
, new);
571 return &new->async_tx
;
574 static struct dma_async_tx_descriptor
*
575 fsl_dma_prep_memcpy(struct dma_chan
*dchan
,
576 dma_addr_t dma_dst
, dma_addr_t dma_src
,
577 size_t len
, unsigned long flags
)
579 struct fsldma_chan
*chan
;
580 struct fsl_desc_sw
*first
= NULL
, *prev
= NULL
, *new;
589 chan
= to_fsl_chan(dchan
);
593 /* Allocate the link descriptor from DMA pool */
594 new = fsl_dma_alloc_descriptor(chan
);
596 chan_err(chan
, "%s\n", msg_ld_oom
);
600 copy
= min(len
, (size_t)FSL_DMA_BCR_MAX_CNT
);
602 set_desc_cnt(chan
, &new->hw
, copy
);
603 set_desc_src(chan
, &new->hw
, dma_src
);
604 set_desc_dst(chan
, &new->hw
, dma_dst
);
609 set_desc_next(chan
, &prev
->hw
, new->async_tx
.phys
);
611 new->async_tx
.cookie
= 0;
612 async_tx_ack(&new->async_tx
);
619 /* Insert the link descriptor to the LD ring */
620 list_add_tail(&new->node
, &first
->tx_list
);
623 new->async_tx
.flags
= flags
; /* client is in control of this ack */
624 new->async_tx
.cookie
= -EBUSY
;
626 /* Set End-of-link to the last link descriptor of new list */
627 set_ld_eol(chan
, new);
629 return &first
->async_tx
;
635 fsldma_free_desc_list_reverse(chan
, &first
->tx_list
);
639 static struct dma_async_tx_descriptor
*fsl_dma_prep_sg(struct dma_chan
*dchan
,
640 struct scatterlist
*dst_sg
, unsigned int dst_nents
,
641 struct scatterlist
*src_sg
, unsigned int src_nents
,
644 struct fsl_desc_sw
*first
= NULL
, *prev
= NULL
, *new = NULL
;
645 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
646 size_t dst_avail
, src_avail
;
650 /* basic sanity checks */
651 if (dst_nents
== 0 || src_nents
== 0)
654 if (dst_sg
== NULL
|| src_sg
== NULL
)
658 * TODO: should we check that both scatterlists have the same
659 * TODO: number of bytes in total? Is that really an error?
662 /* get prepared for the loop */
663 dst_avail
= sg_dma_len(dst_sg
);
664 src_avail
= sg_dma_len(src_sg
);
666 /* run until we are out of scatterlist entries */
669 /* create the largest transaction possible */
670 len
= min_t(size_t, src_avail
, dst_avail
);
671 len
= min_t(size_t, len
, FSL_DMA_BCR_MAX_CNT
);
675 dst
= sg_dma_address(dst_sg
) + sg_dma_len(dst_sg
) - dst_avail
;
676 src
= sg_dma_address(src_sg
) + sg_dma_len(src_sg
) - src_avail
;
678 /* allocate and populate the descriptor */
679 new = fsl_dma_alloc_descriptor(chan
);
681 chan_err(chan
, "%s\n", msg_ld_oom
);
685 set_desc_cnt(chan
, &new->hw
, len
);
686 set_desc_src(chan
, &new->hw
, src
);
687 set_desc_dst(chan
, &new->hw
, dst
);
692 set_desc_next(chan
, &prev
->hw
, new->async_tx
.phys
);
694 new->async_tx
.cookie
= 0;
695 async_tx_ack(&new->async_tx
);
698 /* Insert the link descriptor to the LD ring */
699 list_add_tail(&new->node
, &first
->tx_list
);
701 /* update metadata */
706 /* fetch the next dst scatterlist entry */
707 if (dst_avail
== 0) {
709 /* no more entries: we're done */
713 /* fetch the next entry: if there are no more: done */
714 dst_sg
= sg_next(dst_sg
);
719 dst_avail
= sg_dma_len(dst_sg
);
722 /* fetch the next src scatterlist entry */
723 if (src_avail
== 0) {
725 /* no more entries: we're done */
729 /* fetch the next entry: if there are no more: done */
730 src_sg
= sg_next(src_sg
);
735 src_avail
= sg_dma_len(src_sg
);
739 new->async_tx
.flags
= flags
; /* client is in control of this ack */
740 new->async_tx
.cookie
= -EBUSY
;
742 /* Set End-of-link to the last link descriptor of new list */
743 set_ld_eol(chan
, new);
745 return &first
->async_tx
;
751 fsldma_free_desc_list_reverse(chan
, &first
->tx_list
);
756 * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
758 * @sgl: scatterlist to transfer to/from
759 * @sg_len: number of entries in @scatterlist
760 * @direction: DMA direction
761 * @flags: DMAEngine flags
763 * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
764 * DMA_SLAVE API, this gets the device-specific information from the
765 * chan->private variable.
767 static struct dma_async_tx_descriptor
*fsl_dma_prep_slave_sg(
768 struct dma_chan
*dchan
, struct scatterlist
*sgl
, unsigned int sg_len
,
769 enum dma_transfer_direction direction
, unsigned long flags
)
772 * This operation is not supported on the Freescale DMA controller
774 * However, we need to provide the function pointer to allow the
775 * device_control() method to work.
780 static int fsl_dma_device_control(struct dma_chan
*dchan
,
781 enum dma_ctrl_cmd cmd
, unsigned long arg
)
783 struct dma_slave_config
*config
;
784 struct fsldma_chan
*chan
;
791 chan
= to_fsl_chan(dchan
);
794 case DMA_TERMINATE_ALL
:
795 spin_lock_irqsave(&chan
->desc_lock
, flags
);
797 /* Halt the DMA engine */
800 /* Remove and free all of the descriptors in the LD queue */
801 fsldma_free_desc_list(chan
, &chan
->ld_pending
);
802 fsldma_free_desc_list(chan
, &chan
->ld_running
);
805 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
808 case DMA_SLAVE_CONFIG
:
809 config
= (struct dma_slave_config
*)arg
;
811 /* make sure the channel supports setting burst size */
812 if (!chan
->set_request_count
)
815 /* we set the controller burst size depending on direction */
816 if (config
->direction
== DMA_MEM_TO_DEV
)
817 size
= config
->dst_addr_width
* config
->dst_maxburst
;
819 size
= config
->src_addr_width
* config
->src_maxburst
;
821 chan
->set_request_count(chan
, size
);
824 case FSLDMA_EXTERNAL_START
:
826 /* make sure the channel supports external start */
827 if (!chan
->toggle_ext_start
)
830 chan
->toggle_ext_start(chan
, arg
);
841 * fsldma_cleanup_descriptor - cleanup and free a single link descriptor
842 * @chan: Freescale DMA channel
843 * @desc: descriptor to cleanup and free
845 * This function is used on a descriptor which has been executed by the DMA
846 * controller. It will run any callbacks, submit any dependencies, and then
847 * free the descriptor.
849 static void fsldma_cleanup_descriptor(struct fsldma_chan
*chan
,
850 struct fsl_desc_sw
*desc
)
852 struct dma_async_tx_descriptor
*txd
= &desc
->async_tx
;
853 struct device
*dev
= chan
->common
.device
->dev
;
854 dma_addr_t src
= get_desc_src(chan
, desc
);
855 dma_addr_t dst
= get_desc_dst(chan
, desc
);
856 u32 len
= get_desc_cnt(chan
, desc
);
858 /* Run the link descriptor callback function */
860 #ifdef FSL_DMA_LD_DEBUG
861 chan_dbg(chan
, "LD %p callback\n", desc
);
863 txd
->callback(txd
->callback_param
);
866 /* Run any dependencies */
867 dma_run_dependencies(txd
);
869 /* Unmap the dst buffer, if requested */
870 if (!(txd
->flags
& DMA_COMPL_SKIP_DEST_UNMAP
)) {
871 if (txd
->flags
& DMA_COMPL_DEST_UNMAP_SINGLE
)
872 dma_unmap_single(dev
, dst
, len
, DMA_FROM_DEVICE
);
874 dma_unmap_page(dev
, dst
, len
, DMA_FROM_DEVICE
);
877 /* Unmap the src buffer, if requested */
878 if (!(txd
->flags
& DMA_COMPL_SKIP_SRC_UNMAP
)) {
879 if (txd
->flags
& DMA_COMPL_SRC_UNMAP_SINGLE
)
880 dma_unmap_single(dev
, src
, len
, DMA_TO_DEVICE
);
882 dma_unmap_page(dev
, src
, len
, DMA_TO_DEVICE
);
885 #ifdef FSL_DMA_LD_DEBUG
886 chan_dbg(chan
, "LD %p free\n", desc
);
888 dma_pool_free(chan
->desc_pool
, desc
, txd
->phys
);
892 * fsl_chan_xfer_ld_queue - transfer any pending transactions
893 * @chan : Freescale DMA channel
895 * HARDWARE STATE: idle
896 * LOCKING: must hold chan->desc_lock
898 static void fsl_chan_xfer_ld_queue(struct fsldma_chan
*chan
)
900 struct fsl_desc_sw
*desc
;
903 * If the list of pending descriptors is empty, then we
904 * don't need to do any work at all
906 if (list_empty(&chan
->ld_pending
)) {
907 chan_dbg(chan
, "no pending LDs\n");
912 * The DMA controller is not idle, which means that the interrupt
913 * handler will start any queued transactions when it runs after
914 * this transaction finishes
917 chan_dbg(chan
, "DMA controller still busy\n");
922 * If there are some link descriptors which have not been
923 * transferred, we need to start the controller
927 * Move all elements from the queue of pending transactions
928 * onto the list of running transactions
930 chan_dbg(chan
, "idle, starting controller\n");
931 desc
= list_first_entry(&chan
->ld_pending
, struct fsl_desc_sw
, node
);
932 list_splice_tail_init(&chan
->ld_pending
, &chan
->ld_running
);
935 * The 85xx DMA controller doesn't clear the channel start bit
936 * automatically at the end of a transfer. Therefore we must clear
937 * it in software before starting the transfer.
939 if ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
) {
942 mode
= DMA_IN(chan
, &chan
->regs
->mr
, 32);
943 mode
&= ~FSL_DMA_MR_CS
;
944 DMA_OUT(chan
, &chan
->regs
->mr
, mode
, 32);
948 * Program the descriptor's address into the DMA controller,
949 * then start the DMA transaction
951 set_cdar(chan
, desc
->async_tx
.phys
);
959 * fsl_dma_memcpy_issue_pending - Issue the DMA start command
960 * @chan : Freescale DMA channel
962 static void fsl_dma_memcpy_issue_pending(struct dma_chan
*dchan
)
964 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
967 spin_lock_irqsave(&chan
->desc_lock
, flags
);
968 fsl_chan_xfer_ld_queue(chan
);
969 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
973 * fsl_tx_status - Determine the DMA status
974 * @chan : Freescale DMA channel
976 static enum dma_status
fsl_tx_status(struct dma_chan
*dchan
,
978 struct dma_tx_state
*txstate
)
980 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
981 dma_cookie_t last_complete
;
982 dma_cookie_t last_used
;
985 spin_lock_irqsave(&chan
->desc_lock
, flags
);
987 last_complete
= dchan
->completed_cookie
;
988 last_used
= dchan
->cookie
;
990 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
992 dma_set_tx_state(txstate
, last_complete
, last_used
, 0);
993 return dma_async_is_complete(cookie
, last_complete
, last_used
);
996 /*----------------------------------------------------------------------------*/
997 /* Interrupt Handling */
998 /*----------------------------------------------------------------------------*/
1000 static irqreturn_t
fsldma_chan_irq(int irq
, void *data
)
1002 struct fsldma_chan
*chan
= data
;
1005 /* save and clear the status register */
1006 stat
= get_sr(chan
);
1008 chan_dbg(chan
, "irq: stat = 0x%x\n", stat
);
1010 /* check that this was really our device */
1011 stat
&= ~(FSL_DMA_SR_CB
| FSL_DMA_SR_CH
);
1015 if (stat
& FSL_DMA_SR_TE
)
1016 chan_err(chan
, "Transfer Error!\n");
1020 * The DMA_INTERRUPT async_tx is a NULL transfer, which will
1021 * triger a PE interrupt.
1023 if (stat
& FSL_DMA_SR_PE
) {
1024 chan_dbg(chan
, "irq: Programming Error INT\n");
1025 stat
&= ~FSL_DMA_SR_PE
;
1026 if (get_bcr(chan
) != 0)
1027 chan_err(chan
, "Programming Error!\n");
1031 * For MPC8349, EOCDI event need to update cookie
1032 * and start the next transfer if it exist.
1034 if (stat
& FSL_DMA_SR_EOCDI
) {
1035 chan_dbg(chan
, "irq: End-of-Chain link INT\n");
1036 stat
&= ~FSL_DMA_SR_EOCDI
;
1040 * If it current transfer is the end-of-transfer,
1041 * we should clear the Channel Start bit for
1042 * prepare next transfer.
1044 if (stat
& FSL_DMA_SR_EOLNI
) {
1045 chan_dbg(chan
, "irq: End-of-link INT\n");
1046 stat
&= ~FSL_DMA_SR_EOLNI
;
1049 /* check that the DMA controller is really idle */
1050 if (!dma_is_idle(chan
))
1051 chan_err(chan
, "irq: controller not idle!\n");
1053 /* check that we handled all of the bits */
1055 chan_err(chan
, "irq: unhandled sr 0x%08x\n", stat
);
1058 * Schedule the tasklet to handle all cleanup of the current
1059 * transaction. It will start a new transaction if there is
1062 tasklet_schedule(&chan
->tasklet
);
1063 chan_dbg(chan
, "irq: Exit\n");
1067 static void dma_do_tasklet(unsigned long data
)
1069 struct fsldma_chan
*chan
= (struct fsldma_chan
*)data
;
1070 struct fsl_desc_sw
*desc
, *_desc
;
1071 LIST_HEAD(ld_cleanup
);
1072 unsigned long flags
;
1074 chan_dbg(chan
, "tasklet entry\n");
1076 spin_lock_irqsave(&chan
->desc_lock
, flags
);
1078 /* update the cookie if we have some descriptors to cleanup */
1079 if (!list_empty(&chan
->ld_running
)) {
1080 dma_cookie_t cookie
;
1082 desc
= to_fsl_desc(chan
->ld_running
.prev
);
1083 cookie
= desc
->async_tx
.cookie
;
1085 chan
->common
.completed_cookie
= cookie
;
1086 chan_dbg(chan
, "completed_cookie=%d\n", cookie
);
1090 * move the descriptors to a temporary list so we can drop the lock
1091 * during the entire cleanup operation
1093 list_splice_tail_init(&chan
->ld_running
, &ld_cleanup
);
1095 /* the hardware is now idle and ready for more */
1099 * Start any pending transactions automatically
1101 * In the ideal case, we keep the DMA controller busy while we go
1102 * ahead and free the descriptors below.
1104 fsl_chan_xfer_ld_queue(chan
);
1105 spin_unlock_irqrestore(&chan
->desc_lock
, flags
);
1107 /* Run the callback for each descriptor, in order */
1108 list_for_each_entry_safe(desc
, _desc
, &ld_cleanup
, node
) {
1110 /* Remove from the list of transactions */
1111 list_del(&desc
->node
);
1113 /* Run all cleanup for this descriptor */
1114 fsldma_cleanup_descriptor(chan
, desc
);
1117 chan_dbg(chan
, "tasklet exit\n");
1120 static irqreturn_t
fsldma_ctrl_irq(int irq
, void *data
)
1122 struct fsldma_device
*fdev
= data
;
1123 struct fsldma_chan
*chan
;
1124 unsigned int handled
= 0;
1128 gsr
= (fdev
->feature
& FSL_DMA_BIG_ENDIAN
) ? in_be32(fdev
->regs
)
1129 : in_le32(fdev
->regs
);
1131 dev_dbg(fdev
->dev
, "IRQ: gsr 0x%.8x\n", gsr
);
1133 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1134 chan
= fdev
->chan
[i
];
1139 dev_dbg(fdev
->dev
, "IRQ: chan %d\n", chan
->id
);
1140 fsldma_chan_irq(irq
, chan
);
1148 return IRQ_RETVAL(handled
);
1151 static void fsldma_free_irqs(struct fsldma_device
*fdev
)
1153 struct fsldma_chan
*chan
;
1156 if (fdev
->irq
!= NO_IRQ
) {
1157 dev_dbg(fdev
->dev
, "free per-controller IRQ\n");
1158 free_irq(fdev
->irq
, fdev
);
1162 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1163 chan
= fdev
->chan
[i
];
1164 if (chan
&& chan
->irq
!= NO_IRQ
) {
1165 chan_dbg(chan
, "free per-channel IRQ\n");
1166 free_irq(chan
->irq
, chan
);
1171 static int fsldma_request_irqs(struct fsldma_device
*fdev
)
1173 struct fsldma_chan
*chan
;
1177 /* if we have a per-controller IRQ, use that */
1178 if (fdev
->irq
!= NO_IRQ
) {
1179 dev_dbg(fdev
->dev
, "request per-controller IRQ\n");
1180 ret
= request_irq(fdev
->irq
, fsldma_ctrl_irq
, IRQF_SHARED
,
1181 "fsldma-controller", fdev
);
1185 /* no per-controller IRQ, use the per-channel IRQs */
1186 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1187 chan
= fdev
->chan
[i
];
1191 if (chan
->irq
== NO_IRQ
) {
1192 chan_err(chan
, "interrupts property missing in device tree\n");
1197 chan_dbg(chan
, "request per-channel IRQ\n");
1198 ret
= request_irq(chan
->irq
, fsldma_chan_irq
, IRQF_SHARED
,
1199 "fsldma-chan", chan
);
1201 chan_err(chan
, "unable to request per-channel IRQ\n");
1209 for (/* none */; i
>= 0; i
--) {
1210 chan
= fdev
->chan
[i
];
1214 if (chan
->irq
== NO_IRQ
)
1217 free_irq(chan
->irq
, chan
);
1223 /*----------------------------------------------------------------------------*/
1224 /* OpenFirmware Subsystem */
1225 /*----------------------------------------------------------------------------*/
1227 static int __devinit
fsl_dma_chan_probe(struct fsldma_device
*fdev
,
1228 struct device_node
*node
, u32 feature
, const char *compatible
)
1230 struct fsldma_chan
*chan
;
1231 struct resource res
;
1235 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
1237 dev_err(fdev
->dev
, "no free memory for DMA channels!\n");
1242 /* ioremap registers for use */
1243 chan
->regs
= of_iomap(node
, 0);
1245 dev_err(fdev
->dev
, "unable to ioremap registers\n");
1250 err
= of_address_to_resource(node
, 0, &res
);
1252 dev_err(fdev
->dev
, "unable to find 'reg' property\n");
1253 goto out_iounmap_regs
;
1256 chan
->feature
= feature
;
1258 fdev
->feature
= chan
->feature
;
1261 * If the DMA device's feature is different than the feature
1262 * of its channels, report the bug
1264 WARN_ON(fdev
->feature
!= chan
->feature
);
1266 chan
->dev
= fdev
->dev
;
1267 chan
->id
= ((res
.start
- 0x100) & 0xfff) >> 7;
1268 if (chan
->id
>= FSL_DMA_MAX_CHANS_PER_DEVICE
) {
1269 dev_err(fdev
->dev
, "too many channels for device\n");
1271 goto out_iounmap_regs
;
1274 fdev
->chan
[chan
->id
] = chan
;
1275 tasklet_init(&chan
->tasklet
, dma_do_tasklet
, (unsigned long)chan
);
1276 snprintf(chan
->name
, sizeof(chan
->name
), "chan%d", chan
->id
);
1278 /* Initialize the channel */
1281 /* Clear cdar registers */
1284 switch (chan
->feature
& FSL_DMA_IP_MASK
) {
1285 case FSL_DMA_IP_85XX
:
1286 chan
->toggle_ext_pause
= fsl_chan_toggle_ext_pause
;
1287 case FSL_DMA_IP_83XX
:
1288 chan
->toggle_ext_start
= fsl_chan_toggle_ext_start
;
1289 chan
->set_src_loop_size
= fsl_chan_set_src_loop_size
;
1290 chan
->set_dst_loop_size
= fsl_chan_set_dst_loop_size
;
1291 chan
->set_request_count
= fsl_chan_set_request_count
;
1294 spin_lock_init(&chan
->desc_lock
);
1295 INIT_LIST_HEAD(&chan
->ld_pending
);
1296 INIT_LIST_HEAD(&chan
->ld_running
);
1299 chan
->common
.device
= &fdev
->common
;
1301 /* find the IRQ line, if it exists in the device tree */
1302 chan
->irq
= irq_of_parse_and_map(node
, 0);
1304 /* Add the channel to DMA device channel list */
1305 list_add_tail(&chan
->common
.device_node
, &fdev
->common
.channels
);
1306 fdev
->common
.chancnt
++;
1308 dev_info(fdev
->dev
, "#%d (%s), irq %d\n", chan
->id
, compatible
,
1309 chan
->irq
!= NO_IRQ
? chan
->irq
: fdev
->irq
);
1314 iounmap(chan
->regs
);
1321 static void fsl_dma_chan_remove(struct fsldma_chan
*chan
)
1323 irq_dispose_mapping(chan
->irq
);
1324 list_del(&chan
->common
.device_node
);
1325 iounmap(chan
->regs
);
1329 static int __devinit
fsldma_of_probe(struct platform_device
*op
)
1331 struct fsldma_device
*fdev
;
1332 struct device_node
*child
;
1335 fdev
= kzalloc(sizeof(*fdev
), GFP_KERNEL
);
1337 dev_err(&op
->dev
, "No enough memory for 'priv'\n");
1342 fdev
->dev
= &op
->dev
;
1343 INIT_LIST_HEAD(&fdev
->common
.channels
);
1345 /* ioremap the registers for use */
1346 fdev
->regs
= of_iomap(op
->dev
.of_node
, 0);
1348 dev_err(&op
->dev
, "unable to ioremap registers\n");
1353 /* map the channel IRQ if it exists, but don't hookup the handler yet */
1354 fdev
->irq
= irq_of_parse_and_map(op
->dev
.of_node
, 0);
1356 dma_cap_set(DMA_MEMCPY
, fdev
->common
.cap_mask
);
1357 dma_cap_set(DMA_INTERRUPT
, fdev
->common
.cap_mask
);
1358 dma_cap_set(DMA_SG
, fdev
->common
.cap_mask
);
1359 dma_cap_set(DMA_SLAVE
, fdev
->common
.cap_mask
);
1360 fdev
->common
.device_alloc_chan_resources
= fsl_dma_alloc_chan_resources
;
1361 fdev
->common
.device_free_chan_resources
= fsl_dma_free_chan_resources
;
1362 fdev
->common
.device_prep_dma_interrupt
= fsl_dma_prep_interrupt
;
1363 fdev
->common
.device_prep_dma_memcpy
= fsl_dma_prep_memcpy
;
1364 fdev
->common
.device_prep_dma_sg
= fsl_dma_prep_sg
;
1365 fdev
->common
.device_tx_status
= fsl_tx_status
;
1366 fdev
->common
.device_issue_pending
= fsl_dma_memcpy_issue_pending
;
1367 fdev
->common
.device_prep_slave_sg
= fsl_dma_prep_slave_sg
;
1368 fdev
->common
.device_control
= fsl_dma_device_control
;
1369 fdev
->common
.dev
= &op
->dev
;
1371 dma_set_mask(&(op
->dev
), DMA_BIT_MASK(36));
1373 dev_set_drvdata(&op
->dev
, fdev
);
1376 * We cannot use of_platform_bus_probe() because there is no
1377 * of_platform_bus_remove(). Instead, we manually instantiate every DMA
1380 for_each_child_of_node(op
->dev
.of_node
, child
) {
1381 if (of_device_is_compatible(child
, "fsl,eloplus-dma-channel")) {
1382 fsl_dma_chan_probe(fdev
, child
,
1383 FSL_DMA_IP_85XX
| FSL_DMA_BIG_ENDIAN
,
1384 "fsl,eloplus-dma-channel");
1387 if (of_device_is_compatible(child
, "fsl,elo-dma-channel")) {
1388 fsl_dma_chan_probe(fdev
, child
,
1389 FSL_DMA_IP_83XX
| FSL_DMA_LITTLE_ENDIAN
,
1390 "fsl,elo-dma-channel");
1395 * Hookup the IRQ handler(s)
1397 * If we have a per-controller interrupt, we prefer that to the
1398 * per-channel interrupts to reduce the number of shared interrupt
1399 * handlers on the same IRQ line
1401 err
= fsldma_request_irqs(fdev
);
1403 dev_err(fdev
->dev
, "unable to request IRQs\n");
1407 dma_async_device_register(&fdev
->common
);
1411 irq_dispose_mapping(fdev
->irq
);
1417 static int fsldma_of_remove(struct platform_device
*op
)
1419 struct fsldma_device
*fdev
;
1422 fdev
= dev_get_drvdata(&op
->dev
);
1423 dma_async_device_unregister(&fdev
->common
);
1425 fsldma_free_irqs(fdev
);
1427 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1429 fsl_dma_chan_remove(fdev
->chan
[i
]);
1432 iounmap(fdev
->regs
);
1433 dev_set_drvdata(&op
->dev
, NULL
);
1439 static const struct of_device_id fsldma_of_ids
[] = {
1440 { .compatible
= "fsl,eloplus-dma", },
1441 { .compatible
= "fsl,elo-dma", },
1445 static struct platform_driver fsldma_of_driver
= {
1447 .name
= "fsl-elo-dma",
1448 .owner
= THIS_MODULE
,
1449 .of_match_table
= fsldma_of_ids
,
1451 .probe
= fsldma_of_probe
,
1452 .remove
= fsldma_of_remove
,
1455 /*----------------------------------------------------------------------------*/
1456 /* Module Init / Exit */
1457 /*----------------------------------------------------------------------------*/
1459 static __init
int fsldma_init(void)
1461 pr_info("Freescale Elo / Elo Plus DMA driver\n");
1462 return platform_driver_register(&fsldma_of_driver
);
1465 static void __exit
fsldma_exit(void)
1467 platform_driver_unregister(&fsldma_of_driver
);
1470 subsys_initcall(fsldma_init
);
1471 module_exit(fsldma_exit
);
1473 MODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver");
1474 MODULE_LICENSE("GPL");